diff --git a/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy b/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy index b2eaefd9bc498..6a681d219f439 100644 --- a/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy +++ b/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy @@ -5,8 +5,5 @@ Description: The attributes: audio_buf_size - audio buffer size - fn_cap - capture pcm device file name - fn_cntl - control device file name - fn_play - playback pcm device file name req_buf_size - ISO OUT endpoint request buffer size req_count - ISO OUT endpoint request count diff --git a/Documentation/ABI/testing/sysfs-bus-nvdimm b/Documentation/ABI/testing/sysfs-bus-nvdimm index 64eb8f4c6a41f..46dafd8482b9e 100644 --- a/Documentation/ABI/testing/sysfs-bus-nvdimm +++ b/Documentation/ABI/testing/sysfs-bus-nvdimm @@ -48,7 +48,8 @@ What: /sys/bus/nd/devices/nmemX/cxl/id Date: November 2022 KernelVersion: 6.2 Contact: Dave Jiang -Description: (RO) Show the id (serial) of the device. This is CXL specific. +Description: (RO) Show the id (serial) of the device, formatted as an + unsigned 64-bit decimal value. This is CXL specific. What: /sys/bus/nd/devices/nmemX/cxl/provider Date: November 2022 diff --git a/Documentation/ABI/testing/sysfs-devices-platform-trackpoint b/Documentation/ABI/testing/sysfs-devices-platform-trackpoint index df11901a6b3df..7954434b0434a 100644 --- a/Documentation/ABI/testing/sysfs-devices-platform-trackpoint +++ b/Documentation/ABI/testing/sysfs-devices-platform-trackpoint @@ -5,7 +5,7 @@ Contact: linux-input@vger.kernel.org Description: (RW) Trackpoint sensitivity. -What: /sys/devices/platform/i8042/.../intertia +What: /sys/devices/platform/i8042/.../inertia Date: Aug, 2005 KernelVersion: 2.6.14 Contact: linux-input@vger.kernel.org diff --git a/Documentation/ABI/testing/sysfs-driver-hid-ayaneo b/Documentation/ABI/testing/sysfs-driver-hid-ayaneo new file mode 100644 index 0000000000000..525b08277f75b --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-hid-ayaneo @@ -0,0 +1,75 @@ +What: /sys/bus/hid/drivers/hid-ayaneo//module_left +What: /sys/bus/hid/drivers/hid-ayaneo//module_right +Date: September 2026 +KernelVersion: 7.4 +Contact: Matías Martínez +Description: + Reports the type of the module currently inserted in the + left/right slot of the AYANEO 3 detachable controller, as + the raw identifier reported by the controller firmware in + hexadecimal (e.g. "0x04"). Bits 0-5 encode the module + type, bit 6 indicates the module is inserted rotated. + + Reading these attributes queries the controller and can + take up to a second. + +What: /sys/bus/hid/drivers/hid-ayaneo//eject +Date: September 2026 +KernelVersion: 7.4 +Contact: Matías Martínez +Description: + Write-only. Writing "left", "right" or "both" asks the + controller firmware to release the corresponding + module(s). The write blocks until the firmware confirms + the release handshake (typically a few seconds). The + module is physically released once controller power is + subsequently cut through the ayaneo-ec platform driver's + controller_power attribute. That final step is left to + userspace. + +What: /sys/bus/hid/drivers/hid-ayaneo//reset +Date: September 2026 +KernelVersion: 7.4 +Contact: Matías Martínez +Description: + Write-only. Writing "1" asks the controller firmware to + perform a quick reset of the controller configuration. + +What: /sys/bus/hid/drivers/hid-ayaneo//rumble_intensity +Date: September 2026 +KernelVersion: 7.4 +Contact: Matías Martínez +Description: + Controls the vibration strength of the AYANEO 3 controller + rumble motors. Reading returns the selected level. Supported + writes are the values listed by rumble_intensity_index. + Defaults to "medium", matching the firmware default. + +What: /sys/bus/hid/drivers/hid-ayaneo//rumble_intensity_index +Date: September 2026 +KernelVersion: 7.4 +Contact: Matías Martínez +Description: + Read-only. Space-separated list of the supported + rumble_intensity values: "off low medium high". + +What: /sys/class/leds/:rgb:joystick_rings/effect +Date: September 2026 +KernelVersion: 7.4 +Contact: Matías Martínez +Description: + Controls the firmware LED effect of the joystick rings. + + Values are "monocolor", "breathe" or "rainbow", as listed + by the effect_index attribute. "rainbow" is a + firmware-timed colour cycle whose hue and tempo are fixed + by the firmware: brightness scales its amplitude and the + multi_intensity values are ignored while it is selected. + +What: /sys/class/leds/:rgb:joystick_rings/effect_index +Date: September 2026 +KernelVersion: 7.4 +Contact: Matías Martínez +Description: + Read-only. Space-separated list of the supported effect + values: "monocolor breathe rainbow". diff --git a/Documentation/ABI/testing/sysfs-fs-erofs b/Documentation/ABI/testing/sysfs-fs-erofs index e4cf6fc6a1066..0b8b4354e40b8 100644 --- a/Documentation/ABI/testing/sysfs-fs-erofs +++ b/Documentation/ABI/testing/sysfs-fs-erofs @@ -5,7 +5,7 @@ Description: Shows all enabled kernel features. Supported features: compr_cfgs, big_pcluster, chunked_file, device_table, compr_head2, sb_chksum, ztailpacking, dedupe, fragments, - 48bit, metabox. + xattr_prefixes, 48bit, metabox. What: /sys/fs/erofs//sync_decompress Date: November 2021 diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index 4fdec63a47d4a..59010885618cc 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -112,7 +112,7 @@ Description: Writing a value to this file sets the update interval of the DAMON context in microseconds as the value. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/access_bp +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/access_bp Date: Feb 2025 Contact: SJ Park Description: Writing a value to this file sets the monitoring intervals @@ -120,7 +120,7 @@ Description: Writing a value to this file sets the monitoring intervals the given time interval (aggrs in same directory), in bp (1/10,000). Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/aggrs +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/aggrs Date: Feb 2025 Contact: SJ Park Description: Writing a value to this file sets the time interval to achieve @@ -128,14 +128,14 @@ Description: Writing a value to this file sets the time interval to achieve access events ratio (access_bp in same directory) within. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/min_sample_us +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/min_sample_us Date: Feb 2025 Contact: SJ Park Description: Writing a value to this file sets the minimum value of auto-tuned sampling interval in microseconds. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/max_sample_us +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/max_sample_us Date: Feb 2025 Contact: SJ Park Description: Writing a value to this file sets the maximum value of diff --git a/Documentation/ABI/testing/sysfs-platform-ayn-ec b/Documentation/ABI/testing/sysfs-platform-ayn-ec new file mode 100644 index 0000000000000..32cb6f7ca2fc5 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-platform-ayn-ec @@ -0,0 +1,59 @@ +What: /sys/class/hwmon/hwmon[0-9]/pwm1_enable +Date: July 2025 +KernelVersion: 6.17 +Contact: "Derek J. Clark" +Description: + This sets the PWM fan mode of operation. Valid values are [0-3]. + Values [0-2] conform with standard hwmon operating modes. Value 3 + enables user defined fan curve settings. + + Applies to AYN Loki and Tectoy Zeenix lines of handheld devices. + +What: /sys/class/hwmon/hwmon[0-9]/pwm1_auto_point[1-5]_pwm +Date: July 2025 +KernelVersion: 6.17 +Contact: "Derek J. Clark" +Description: + This sets the PWM fan duty cycle for the given index of the fan curve. + When the temperature reaches the corresponding pwm1_auto_point[1-5]_temp, + the EC will automatically increase the fan duty cycle to the given value. + + Values are [0-255] + + Applies to AYN Loki and Tectoy Zeenix lines of handheld devices. + +What: /sys/class/hwmon/hwmon[0-9]/pwm1_auto_point[1-5]_temp +Date: July 2025 +KernelVersion: 6.17 +Contact: "Derek J. Clark" +Description: + This sets the activation temperature for the given index of the fan curve. + When the temperature reaches the given value, the EC will automatically + increase the fan duty cycle to the corresponding pwm1_auto_point[1-5]_pwm + value. + + Values are [0-100] + + Applies to AYN Loki and Tectoy Zeenix lines of handheld devices. + +What: /sys/class/leds/ayn:rgb:joystick_rings/effect +Date: July 2025 +KernelVersion: 6.17 +Contact: "Derek J. Clark" +Description: + This controls the display effect of the RGB interface. + + Values are monocolor or breathe. + + Applies to AYN Loki and Tectoy Zeenix lines of handheld devices. + +What: /sys/class/leds/ayn:rgb:joystick_rings/effect_index +Date: July 2025 +KernelVersion: 6.17 +Contact: "Derek J. Clark" +Description: + This displays the available options for the effect attribute. + + Values are monocolor or breathe. + + Applies to AYN Loki and Tectoy Zeenix lines of handheld devices. diff --git a/Documentation/admin-guide/blockdev/zoned_loop.rst b/Documentation/admin-guide/blockdev/zoned_loop.rst index f4f1f3121bf95..95974425ca054 100644 --- a/Documentation/admin-guide/blockdev/zoned_loop.rst +++ b/Documentation/admin-guide/blockdev/zoned_loop.rst @@ -30,11 +30,10 @@ indicates the position of the write pointer of the zone. When resetting a sequential zone, its backing file size is truncated to zero. Conversely, for a zone finish operation, the backing file is truncated to the -zone size. With this, the maximum capacity of a zloop zoned block device created -can be larger configured to be larger than the storage space available on the -backing file system. Of course, for such configuration, writing more data than -the storage space available on the backing file system will result in write -errors. +zone capacity. With this, a zloop zoned block device can be configured with a +larger capacity than the storage space available on the backing file system. Of +course, for such configuration, writing more data than the storage space +available on the backing file system will result in write errors. The zoned loop block device driver implements a complete zone transition state machine. That is, zones can be empty, implicitly opened, explicitly opened, diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 7db63c0029221..7d2a44af52c93 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -47,7 +47,6 @@ Features: - pages are linked to per-memcg LRU exclusively, and there is no global LRU. - optionally, memory+swap usage can be accounted and limited. - hierarchical accounting - - soft limit - moving (recharging) account at moving a task is selectable. - usage threshold notifier - memory pressure notifier @@ -76,10 +75,9 @@ Brief summary of control files. memory.memsw.failcnt show the number of memory+Swap hits limits memory.max_usage_in_bytes show max memory usage recorded memory.memsw.max_usage_in_bytes show max memory+Swap usage recorded - memory.soft_limit_in_bytes set/show soft limit of memory usage - This knob is not available on CONFIG_PREEMPT_RT systems. - This knob is deprecated and shouldn't be - used. + memory.soft_limit_in_bytes This knob is deprecated and has no effect. + Writes are ignored and reads always + return the maximum value. memory.stat show various statistics memory.use_hierarchy set/show hierarchical account enabled This knob is deprecated and shouldn't be @@ -340,9 +338,6 @@ memory.kmem.usage_in_bytes, or in a separate counter when it makes sense. The main "kmem" counter is fed into the main counter, so kmem charges will also be visible from the user counter. -Currently no soft limit is implemented for kernel memory. It is future work -to trigger slab reclaim when those limits are reached. - 2.7.1 Current Kernel Memory resources accounted ----------------------------------------------- @@ -710,42 +705,10 @@ For compatibility reasons writing 1 to memory.use_hierarchy will always pass:: THIS IS DEPRECATED! -Soft limits allow for greater sharing of memory. The idea behind soft limits -is to allow control groups to use as much of the memory as needed, provided - -a. There is no memory contention -b. They do not exceed their hard limit - -When the system detects memory contention or low memory, control groups -are pushed back to their soft limits. If the soft limit of each control -group is very high, they are pushed back as much as possible to make -sure that one control group does not starve the others of memory. - -Please note that soft limits is a best-effort feature; it comes with -no guarantees, but it does its best to make sure that when memory is -heavily contended for, memory is allocated based on the soft limit -hints/setup. Currently soft limit based reclaim is set up such that -it gets invoked from balance_pgdat (kswapd). - -7.1 Interface -------------- - -Soft limits can be setup by using the following commands (in this example we -assume a soft limit of 256 MiB):: - - # echo 256M > memory.soft_limit_in_bytes - -If we want to change this to 1G, we can at any time use:: +Writing to memory.soft_limit_in_bytes has no effect and reading it will +always return the maximum value. - # echo 1G > memory.soft_limit_in_bytes - -.. note:: - Soft limits take effect over a long period of time, since they involve - reclaiming memory for balancing between memory cgroups - -.. note:: - It is recommended to set the soft limit always below the hard limit, - otherwise the hard limit will take precedence. +Use memory.low and memory.min in cgroup v2 instead. .. _cgroup-v1-memory-move-charges: diff --git a/Documentation/admin-guide/device-mapper/dm-ima.rst b/Documentation/admin-guide/device-mapper/dm-ima.rst index a4aa50a828e00..2a3b50ffbee4e 100644 --- a/Documentation/admin-guide/device-mapper/dm-ima.rst +++ b/Documentation/admin-guide/device-mapper/dm-ima.rst @@ -424,7 +424,8 @@ section above) has the following data format for 'integrity' target. target_attributes := "," "," "," "," "," [ ","] [ ","] "," - "," "," "," "," + "," "," "," "," + "," "," "," ";" target_name := "target_name=integrity" @@ -438,6 +439,7 @@ section above) has the following data format for 'integrity' target. block_size := "block_size=" recalculate := "recalculate=" allow_discards := "allow_discards=" + allow_discards_keyed := "allow_discards_keyed=" fix_padding := "fix_padding=" fix_hmac := "fix_hmac=" legacy_recalculate := "legacy_recalculate=" @@ -455,7 +457,8 @@ section above) has the following data format for 'integrity' target. dm_version=4.45.0; name=integrity1,uuid=,major=253,minor=1,minor_count=1,num_targets=1; target_index=0,target_begin=0,target_len=7856,target_name=integrity,target_version=1.10.0, - dev_name=253:0,start=0,tag_size=32,mode=J,recalculate=n,allow_discards=n,fix_padding=n, + dev_name=253:0,start=0,tag_size=32,mode=J,recalculate=n,allow_discards=n, + allow_discards_keyed=n,fix_padding=n, fix_hmac=n,legacy_recalculate=n,journal_sectors=88,interleave_sectors=32768,buffer_sectors=128; diff --git a/Documentation/admin-guide/device-mapper/dm-integrity.rst b/Documentation/admin-guide/device-mapper/dm-integrity.rst index c2e18ecc065c9..9c21301423c9e 100644 --- a/Documentation/admin-guide/device-mapper/dm-integrity.rst +++ b/Documentation/admin-guide/device-mapper/dm-integrity.rst @@ -190,6 +190,19 @@ allow_discards Allow block discard requests (a.k.a. TRIM) for the integrity device. Discards are only allowed to devices using internal hash. + A discarded block is marked with a constant filler tag that anyone + with raw write access to the backing device can forge without the + key. Use allow_discards_keyed instead on new volumes. + +allow_discards_keyed + Like allow_discards, but marks a discarded block with a keyed + checksum of the sector number, HMAC_key(salt || sector), instead of + the constant filler tag, so it can't be forged without the + integrity key. + + Not compatible with volumes that already have discarded blocks + marked the old way; only use on a freshly formatted volume. + fix_padding Use a smaller padding of the tag area that is more space-efficient. If this option is not present, large padding is diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index e8ff36982d97b..d0f1742883651 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4922,18 +4922,6 @@ Kernel parameters Set to non-zero if a chip is present that snoops speed changes. Disabled by default. - pata_legacy.ht6560a= [HW,LIBATA] - Format: - Set to 1, 2, or 3 for HT 6560A on the primary channel, - the secondary channel, or both channels respectively. - Disabled by default. - - pata_legacy.ht6560b= [HW,LIBATA] - Format: - Set to 1, 2, or 3 for HT 6560B on the primary channel, - the secondary channel, or both channels respectively. - Disabled by default. - pata_legacy.iordy_mask= [HW,LIBATA] Format: IORDY enable mask. Set individual bits to allow IORDY @@ -4946,18 +4934,6 @@ Kernel parameters with the sequence. By default IORDY is allowed across all channels. - pata_legacy.opti82c46x= [HW,LIBATA] - Format: - Set to 1, 2, or 3 for Opti 82c611A on the primary - channel, the secondary channel, or both channels - respectively. Disabled by default. - - pata_legacy.opti82c611a= [HW,LIBATA] - Format: - Set to 1, 2, or 3 for Opti 82c465MV on the primary - channel, the secondary channel, or both channels - respectively. Disabled by default. - pata_legacy.pio_mask= [HW,LIBATA] Format: PIO mode mask for autospeed devices. Set individual @@ -4981,19 +4957,6 @@ Kernel parameters the first port in the list above (0x1f0), and so on. By default all supported ports are probed. - pata_legacy.qdi= [HW,LIBATA] - Format: - Set to non-zero to probe QDI controllers. By default - set to 1 if CONFIG_PATA_QDI_MODULE, 0 otherwise. - - pata_legacy.winbond= [HW,LIBATA] - Format: - Set to non-zero to probe Winbond controllers. Use - the standard I/O port (0x130) if 1, otherwise the - value given is the I/O port to use (typically 0x1b0). - By default set to 1 if CONFIG_PATA_WINBOND_VLB_MODULE, - 0 otherwise. - pata_platform.pio_mask= [HW,LIBATA] Format: Supported PIO mode mask. Set individual bits to allow diff --git a/Documentation/arch/arm64/booting.rst b/Documentation/arch/arm64/booting.rst index 13ef311dace83..3fea4b14ef7c2 100644 --- a/Documentation/arch/arm64/booting.rst +++ b/Documentation/arch/arm64/booting.rst @@ -465,6 +465,7 @@ Before jumping into the kernel, the following conditions must be met: - HDFGWTR2_EL2.nPMICNTR_EL0 (bit 2) must be initialised to 0b1. - HDFGWTR2_EL2.nPMICFILTR_EL0 (bit 3) must be initialised to 0b1. - HDFGWTR2_EL2.nPMUACR_EL1 (bit 4) must be initialised to 0b1. + - HDFGWTR2_EL2.nPMZR_EL0 (bit 21) must be initialised to 0b1. For CPUs with SPE data source filtering (FEAT_SPE_FDS): diff --git a/Documentation/arch/arm64/memory-tagging-extension.rst b/Documentation/arch/arm64/memory-tagging-extension.rst index e6fe428f0e2a4..1d32fc5df1838 100644 --- a/Documentation/arch/arm64/memory-tagging-extension.rst +++ b/Documentation/arch/arm64/memory-tagging-extension.rst @@ -208,11 +208,10 @@ will use the corresponding aligned address. tracer's space cannot be accessed or does not have valid tags. - ``-EPERM`` - the specified process cannot be traced. - ``-EIO`` - the tracee's address range cannot be accessed (e.g. invalid - address) and no tags copied. ``iov_len`` not updated. + address) or does not have valid tags (not mapped with the ``PROT_MTE`` + flag) and no tags copied. ``iov_len`` not updated. - ``-EFAULT`` - fault on accessing the tracer's memory (``struct iovec`` or ``iov_base`` buffer) and no tags copied. ``iov_len`` not updated. -- ``-EOPNOTSUPP`` - the tracee's address does not have valid tags (never - mapped with the ``PROT_MTE`` flag). ``iov_len`` not updated. **Note**: There are no transient errors for the requests above, so user programs should not retry in case of a non-zero system call return. diff --git a/Documentation/arch/s390/pci.rst b/Documentation/arch/s390/pci.rst index 80f4ba1931599..565434626fb5c 100644 --- a/Documentation/arch/s390/pci.rst +++ b/Documentation/arch/s390/pci.rst @@ -67,7 +67,7 @@ Entries specific to zPCI functions and entries that hold zPCI information. A physical function that currently supports a virtual function cannot be powered off until all virtual functions are removed with: - echo 0 > /sys/bus/pci/devices/DDDD:BB:dd.f/sriov_numvf + echo 0 > /sys/bus/pci/devices/DDDD:BB:dd.f/sriov_numvfs * /sys/bus/pci/devices/DDDD:BB:dd.f/: diff --git a/Documentation/devicetree/bindings/media/nxp,imx8-isi.yaml b/Documentation/devicetree/bindings/media/nxp,imx8-isi.yaml index b59c4ce30b8b6..279c9cd3034ad 100644 --- a/Documentation/devicetree/bindings/media/nxp,imx8-isi.yaml +++ b/Documentation/devicetree/bindings/media/nxp,imx8-isi.yaml @@ -117,7 +117,9 @@ allOf: compatible: not: contains: - const: fsl,imx91-isi + enum: + - fsl,imx8ulp-isi + - fsl,imx91-isi then: required: - fsl,blk-ctrl diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml index 4bb511719f313..56758a85ead8d 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml @@ -67,7 +67,7 @@ $defs: Specify the alternative function to be configured for the specified pins. - enum: [ aoss_cti, atest_char, atest_usb20, atest_usb21, + enum: [ gpio, aoss_cti, atest_char, atest_usb20, atest_usb21, aud_intfc0_clk, aud_intfc0_data, aud_intfc0_ws, aud_intfc10_clk, aud_intfc10_data, aud_intfc10_ws, aud_intfc1_clk, aud_intfc1_data, aud_intfc1_ws, @@ -98,9 +98,10 @@ $defs: pcie3_clk_req_n, phase_flag, pll_bist_sync, pll_clk_aux, prng_rosc0, prng_rosc1, pwrbrk_i_n, qdss, qdss_cti, qspi, qup0_se0, qup0_se1, qup0_se2, qup0_se3, qup0_se4, qup0_se5, - qup1_se0, qup1_se1, qup1_se3, qup1_se2, qup1_se4, qup1_se5, + qup1_se0, qup1_se1, qup1_se2_01, qup1_se2_23, qup1_se3_01, + qup1_se3_23, qup1_se4, qup1_se5, qup1_se6, qup2_se0, qup2_se1, qup2_se2, qup2_se3, qup2_se4, - qup2_se5, qup2_se6, + qup2_se5, qup2_se6, qup3_se0_mira, qup3_se0_mirb, sailss_ospi, sdc4_clk, sdc4_cmd, sdc4_data, smb_alert, smb_alert_n, smb_clk, smb_dat, tb_trig_sdc4, tmess_prng0, tmess_prng1, tsc_timer, tsense_pwm, usb0_hs, diff --git a/Documentation/filesystems/fscrypt.rst b/Documentation/filesystems/fscrypt.rst index c0dd35f1af12b..e4882b73120e3 100644 --- a/Documentation/filesystems/fscrypt.rst +++ b/Documentation/filesystems/fscrypt.rst @@ -1238,6 +1238,10 @@ astute users may notice some differences in behavior: - DAX (Direct Access) is not supported on encrypted files. +- Encrypted files cannot be used directly as swap files. To swap to + an encrypted file, set up a loopback device on top of it. + Alternatively, encrypted swap can use a dm-crypt device. + - The maximum length of an encrypted symlink is 2 bytes shorter than the maximum length of an unencrypted symlink. For example, on an EXT4 filesystem with a 4K block size, unencrypted symlinks can be up diff --git a/Documentation/hwmon/cgbc-hwmon.rst b/Documentation/hwmon/cgbc-hwmon.rst index 3a5e6e6e8639e..c6d09232392ac 100644 --- a/Documentation/hwmon/cgbc-hwmon.rst +++ b/Documentation/hwmon/cgbc-hwmon.rst @@ -28,34 +28,44 @@ system. Name Description ============= ====================== temp1_input CPU temperature -temp2_input Box temperature +temp2_input Case temperature temp3_input Ambient temperature -temp4_input Board temperature -temp5_input Carrier temperature -temp6_input Chipset temperature -temp7_input Video temperature +temp4_input CPU Board temperature +temp5_input Carrier Board temperature +temp6_input System Chipset temperature +temp7_input Video Controller/Board temperature temp8_input Other temperature -temp9_input TOPDIM temperature -temp10_input BOTTOMDIM temperature -in0_input CPU voltage +temp9_input Top DIMM 0 temperature +temp10_input Bottom DIMM 0 temperature +temp11_input Alternate Board temperature +temp12_input Top DIMM 1 temperature +temp13_input Top DIMM 2 temperature +temp14_input Top DIMM 3 temperature +temp15_input Top DIMM 4 temperature +temp16_input Top DIMM 5 temperature +temp17_input Top DIMM 6 temperature +temp18_input Top DIMM 7 temperature +temp19_input Bottom DIMM 1 temperature +in0_input CPU Core voltage in1_input DC Runtime voltage in2_input DC Standby voltage in3_input CMOS Battery voltage -in4_input Battery voltage +in4_input Battery Supply voltage in5_input AC voltage in6_input Other voltage -in7_input 5V voltage +in7_input 5V Runtime voltage in8_input 5V Standby voltage -in9_input 3V3 voltage +in9_input 3V3 Runtime voltage in10_input 3V3 Standby voltage in11_input VCore A voltage in12_input VCore B voltage -in13_input 12V voltage -curr1_input DC current -curr2_input 5V current -curr3_input 12V current +in13_input 12V Runtime voltage +in14_input 12V Standby voltage +curr1_input DC Runtime current +curr2_input 5V Runtime current +curr3_input 12V Runtime current fan1_input CPU fan -fan2_input Box fan +fan2_input Case fan fan3_input Ambient fan fan4_input Chiptset fan fan5_input Video fan diff --git a/Documentation/hwmon/emc1403.rst b/Documentation/hwmon/emc1403.rst index 77060d5153238..ce258f0fccc60 100644 --- a/Documentation/hwmon/emc1403.rst +++ b/Documentation/hwmon/emc1403.rst @@ -71,10 +71,10 @@ and EMC14x8 support eight sensors (one internal, seven external). The chips implement three limits for each sensor: low (tempX_min), high (tempX_max) and critical (tempX_crit.) The chips also implement an -hysteresis mechanism which applies to all limits. The relative difference -is stored in a single register on the chip, which means that the relative -difference between the limit and its hysteresis is always the same for -all three limits. +hysteresis mechanism which applies to high and critical limits. The relative +difference is stored in a single register on the chip, which means that the +relative difference between the limit and its hysteresis is always the same +for high and critical limits. This implementation detail implies the following: diff --git a/Documentation/hwmon/gpd-fan.rst b/Documentation/hwmon/gpd-fan.rst index 29527a77fe882..b27657d330563 100644 --- a/Documentation/hwmon/gpd-fan.rst +++ b/Documentation/hwmon/gpd-fan.rst @@ -67,7 +67,7 @@ pwm1_enable at full speed. Write "1" to set to manual, write "2" to let the EC control decide fan speed. Read this attribute to see current status. - NB:In consideration of the safety of the device, when setting to manual mode, + NB: In consideration of the safety of the device, when setting to manual mode, the pwm speed will be set to the maximum value (255) by default. You can set a different value by writing pwm1 later. diff --git a/Documentation/hwmon/hwmon-kernel-api.rst b/Documentation/hwmon/hwmon-kernel-api.rst index 9fcde32a140df..c3eb433a78f61 100644 --- a/Documentation/hwmon/hwmon-kernel-api.rst +++ b/Documentation/hwmon/hwmon-kernel-api.rst @@ -42,6 +42,9 @@ register/unregister functions:: char *devm_hwmon_sanitize_name(struct device *dev, const char *name); + int hwmon_notify_event(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel); + void hwmon_lock(struct device *dev); void hwmon_unlock(struct device *dev); @@ -90,6 +93,18 @@ implemented in the driver, or debugfs functions, hwmon_lock() and hwmon_unlock() can be used to ensure that calls to those functions are serialized. Those functions also support guard() and scoped_guard() variants. +Drivers can call hwmon_notify_event() to notify userspace and the thermal +subsystem when a hardware monitoring event (such as an alarm or a fault +condition) occurs or clears. The parameters are the hwmon device, the sensor +type, the attribute identifier associated with the event (such as +hwmon_temp_max_alarm or hwmon_fan_fault), and the sensor channel number. +hwmon_notify_event() generates a sysfs event (calling sysfs_notify()) and a +udev event with the attribute name passed in the NAME environment property +(e.g., "NAME=temp1_max_alarm"). If the event is for a temperature sensor and +the sensor is attached to a thermal zone, it also notifies the thermal +subsystem to update the thermal zone. hwmon_notify_event() returns 0 on +success or a negative error code on failure. + Using devm_hwmon_device_register_with_info() -------------------------------------------- diff --git a/Documentation/leds/leds-st1202.rst b/Documentation/leds/leds-st1202.rst index 1a09fbfcedcff..a2353549469ee 100644 --- a/Documentation/leds/leds-st1202.rst +++ b/Documentation/leds/leds-st1202.rst @@ -17,7 +17,7 @@ To be compatible with the hardware pattern format, maximum 8 tuples of brightness (PWM) and duration must be written to hw_pattern. - Min pattern duration: 22 ms -- Max pattern duration: 5660 ms +- Max pattern duration: 5610 ms The format of the hardware pattern values should be: "brightness duration brightness duration ..." diff --git a/Documentation/netlink/specs/rt-link.yaml b/Documentation/netlink/specs/rt-link.yaml index 68c26a70bb649..c75731ecbe1af 100644 --- a/Documentation/netlink/specs/rt-link.yaml +++ b/Documentation/netlink/specs/rt-link.yaml @@ -898,6 +898,8 @@ attribute-sets: - name: txqlen type: u32 + checks: + max: 32767 - name: map type: binary diff --git a/Documentation/netlink/specs/rt-neigh.yaml b/Documentation/netlink/specs/rt-neigh.yaml index 0f46ef3135905..c8e55c98d5649 100644 --- a/Documentation/netlink/specs/rt-neigh.yaml +++ b/Documentation/netlink/specs/rt-neigh.yaml @@ -341,6 +341,9 @@ attribute-sets: - name: interval-probe-time-ms type: u64 + checks: + min: 1 + max: 86400000 operations: enum-model: directional diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index 208f46967ee59..f7af0286341c9 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -248,7 +248,7 @@ neigh/default/unres_qlen - INTEGER neigh/default/interval_probe_time_ms - INTEGER The probe interval for neighbor entries with NTF_MANAGED flag, - the min value is 1. + the min value is 1, and the max value is 86400000 (1 day). Default: 5000 @@ -874,6 +874,8 @@ tcp_rmem - vector of 3 INTEGERs: min, default, max case this value is ignored. Default: between 131072 and 32MB, depending on RAM size. + Each of the three values cannot be set below 4096. + tcp_sack - BOOLEAN Enable select acknowledgments (SACKS). diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 2771ea4cc14af..96ff66c91ceb2 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -229,7 +229,7 @@ optional. The following modified excerpt is from void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) { - exit_type = ei->type; + exit_type = ei->kind; } SEC(".struct_ops") diff --git a/Documentation/sunrpc/xdr/nfs4_1.x b/Documentation/sunrpc/xdr/nfs4_1.x index 5b45547b2ebc8..72d439b717356 100644 --- a/Documentation/sunrpc/xdr/nfs4_1.x +++ b/Documentation/sunrpc/xdr/nfs4_1.x @@ -45,19 +45,162 @@ pragma header nfs4; /* * Basic typedefs for RFC 1832 data type definitions */ -typedef hyper int64_t; -typedef unsigned int uint32_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef hyper int64_t; +typedef unsigned hyper uint64_t; + +const NFS4_VERIFIER_SIZE = 8; +const NFS4_FHSIZE = 128; + +enum nfsstat4 { + NFS4_OK = 0, /* everything is okay */ + NFS4ERR_PERM = 1, /* caller not privileged */ + NFS4ERR_NOENT = 2, /* no such file/directory */ + NFS4ERR_IO = 5, /* hard I/O error */ + NFS4ERR_NXIO = 6, /* no such device */ + NFS4ERR_ACCESS = 13, /* access denied */ + NFS4ERR_EXIST = 17, /* file already exists */ + NFS4ERR_XDEV = 18, /* different filesystems */ + + /* + * Please do not allocate value 19; it was used in NFSv3 + * and we do not want a value in NFSv3 to have a different + * meaning in NFSv4.x. + */ + + NFS4ERR_NOTDIR = 20, /* should be a directory */ + NFS4ERR_ISDIR = 21, /* should not be directory */ + NFS4ERR_INVAL = 22, /* invalid argument */ + NFS4ERR_FBIG = 27, /* file exceeds server max */ + NFS4ERR_NOSPC = 28, /* no space on filesystem */ + NFS4ERR_ROFS = 30, /* read-only filesystem */ + NFS4ERR_MLINK = 31, /* too many hard links */ + NFS4ERR_NAMETOOLONG = 63, /* name exceeds server max */ + NFS4ERR_NOTEMPTY = 66, /* directory not empty */ + NFS4ERR_DQUOT = 69, /* hard quota limit reached*/ + NFS4ERR_STALE = 70, /* file no longer exists */ + NFS4ERR_BADHANDLE = 10001,/* Illegal filehandle */ + NFS4ERR_BAD_COOKIE = 10003,/* READDIR cookie is stale */ + NFS4ERR_NOTSUPP = 10004,/* operation not supported */ + NFS4ERR_TOOSMALL = 10005,/* response limit exceeded */ + NFS4ERR_SERVERFAULT = 10006,/* undefined server error */ + NFS4ERR_BADTYPE = 10007,/* type invalid for CREATE */ + NFS4ERR_DELAY = 10008,/* file "busy" - retry */ + NFS4ERR_SAME = 10009,/* nverify says attrs same */ + NFS4ERR_DENIED = 10010,/* lock unavailable */ + NFS4ERR_EXPIRED = 10011,/* lock lease expired */ + NFS4ERR_LOCKED = 10012,/* I/O failed due to lock */ + NFS4ERR_GRACE = 10013,/* in grace period */ + NFS4ERR_FHEXPIRED = 10014,/* filehandle expired */ + NFS4ERR_SHARE_DENIED = 10015,/* share reserve denied */ + NFS4ERR_WRONGSEC = 10016,/* wrong security flavor */ + NFS4ERR_CLID_INUSE = 10017,/* clientid in use */ + + /* NFS4ERR_RESOURCE is not a valid error in NFSv4.1 */ + NFS4ERR_RESOURCE = 10018,/* resource exhaustion */ + + NFS4ERR_MOVED = 10019,/* filesystem relocated */ + NFS4ERR_NOFILEHANDLE = 10020,/* current FH is not set */ + NFS4ERR_MINOR_VERS_MISMATCH= 10021,/* minor vers not supp */ + NFS4ERR_STALE_CLIENTID = 10022,/* server has rebooted */ + NFS4ERR_STALE_STATEID = 10023,/* server has rebooted */ + NFS4ERR_OLD_STATEID = 10024,/* state is out of sync */ + NFS4ERR_BAD_STATEID = 10025,/* incorrect stateid */ + NFS4ERR_BAD_SEQID = 10026,/* request is out of seq. */ + NFS4ERR_NOT_SAME = 10027,/* verify - attrs not same */ + NFS4ERR_LOCK_RANGE = 10028,/* overlapping lock range */ + NFS4ERR_SYMLINK = 10029,/* should be file/directory*/ + NFS4ERR_RESTOREFH = 10030,/* no saved filehandle */ + NFS4ERR_LEASE_MOVED = 10031,/* some filesystem moved */ + NFS4ERR_ATTRNOTSUPP = 10032,/* recommended attr not sup*/ + NFS4ERR_NO_GRACE = 10033,/* reclaim outside of grace*/ + NFS4ERR_RECLAIM_BAD = 10034,/* reclaim error at server */ + NFS4ERR_RECLAIM_CONFLICT= 10035,/* conflict on reclaim */ + NFS4ERR_BADXDR = 10036,/* XDR decode failed */ + NFS4ERR_LOCKS_HELD = 10037,/* file locks held at CLOSE*/ + NFS4ERR_OPENMODE = 10038,/* conflict in OPEN and I/O*/ + NFS4ERR_BADOWNER = 10039,/* owner translation bad */ + NFS4ERR_BADCHAR = 10040,/* utf-8 char not supported*/ + NFS4ERR_BADNAME = 10041,/* name not supported */ + NFS4ERR_BAD_RANGE = 10042,/* lock range not supported*/ + NFS4ERR_LOCK_NOTSUPP = 10043,/* no atomic up/downgrade */ + NFS4ERR_OP_ILLEGAL = 10044,/* undefined operation */ + NFS4ERR_DEADLOCK = 10045,/* file locking deadlock */ + NFS4ERR_FILE_OPEN = 10046,/* open file blocks op. */ + NFS4ERR_ADMIN_REVOKED = 10047,/* lockowner state revoked */ + NFS4ERR_CB_PATH_DOWN = 10048,/* callback path down */ + + /* NFSv4.1 errors start here. */ + + NFS4ERR_BADIOMODE = 10049, + NFS4ERR_BADLAYOUT = 10050, + NFS4ERR_BAD_SESSION_DIGEST = 10051, + NFS4ERR_BADSESSION = 10052, + NFS4ERR_BADSLOT = 10053, + NFS4ERR_COMPLETE_ALREADY = 10054, + NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, + NFS4ERR_DELEG_ALREADY_WANTED = 10056, + NFS4ERR_BACK_CHAN_BUSY = 10057,/*backchan reqs outstanding*/ + NFS4ERR_LAYOUTTRYLATER = 10058, + NFS4ERR_LAYOUTUNAVAILABLE = 10059, + NFS4ERR_NOMATCHING_LAYOUT = 10060, + NFS4ERR_RECALLCONFLICT = 10061, + NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, + NFS4ERR_SEQ_MISORDERED = 10063,/* unexpected seq.ID in req*/ + NFS4ERR_SEQUENCE_POS = 10064,/* [CB_]SEQ. op not 1st op */ + NFS4ERR_REQ_TOO_BIG = 10065,/* request too big */ + NFS4ERR_REP_TOO_BIG = 10066,/* reply too big */ + NFS4ERR_REP_TOO_BIG_TO_CACHE =10067,/* rep. not all cached*/ + NFS4ERR_RETRY_UNCACHED_REP =10068,/* retry & rep. uncached*/ + NFS4ERR_UNSAFE_COMPOUND =10069,/* retry/recovery too hard */ + NFS4ERR_TOO_MANY_OPS = 10070,/*too many ops in [CB_]COMP*/ + NFS4ERR_OP_NOT_IN_SESSION =10071,/* op needs [CB_]SEQ. op */ + NFS4ERR_HASH_ALG_UNSUPP = 10072, /* hash alg. not supp. */ + /* Error 10073 is unused. */ + NFS4ERR_CLIENTID_BUSY = 10074,/* clientid has state */ + NFS4ERR_PNFS_IO_HOLE = 10075,/* IO to _SPARSE file hole */ + NFS4ERR_SEQ_FALSE_RETRY= 10076,/* Retry != original req. */ + NFS4ERR_BAD_HIGH_SLOT = 10077,/* req has bad highest_slot*/ + NFS4ERR_DEADSESSION = 10078,/*new req sent to dead sess*/ + NFS4ERR_ENCR_ALG_UNSUPP= 10079,/* encr alg. not supp. */ + NFS4ERR_PNFS_NO_LAYOUT = 10080,/* I/O without a layout */ + NFS4ERR_NOT_ONLY_OP = 10081,/* addl ops not allowed */ + NFS4ERR_WRONG_CRED = 10082,/* op done by wrong cred */ + NFS4ERR_WRONG_TYPE = 10083,/* op on wrong type object */ + NFS4ERR_DIRDELEG_UNAVAIL=10084,/* delegation not avail. */ + NFS4ERR_REJECT_DELEG = 10085,/* cb rejected delegation */ + NFS4ERR_RETURNCONFLICT = 10086,/* layout get before return*/ + NFS4ERR_DELEG_REVOKED = 10087, /* deleg./layout revoked */ + NFS4ERR_PARTNER_NOTSUPP = 10088, + NFS4ERR_PARTNER_NO_AUTH = 10089, + NFS4ERR_UNION_NOTSUPP = 10090, + NFS4ERR_OFFLOAD_DENIED = 10091, + NFS4ERR_WRONG_LFS = 10092, + NFS4ERR_BADLABEL = 10093, + NFS4ERR_OFFLOAD_NO_REQS = 10094, + NFS4ERR_NOXATTR = 10095, + NFS4ERR_XATTR2BIG = 10096 +}; /* * Basic data types */ +typedef opaque attrlist4<>; typedef uint32_t bitmap4<>; +typedef opaque verifier4[NFS4_VERIFIER_SIZE]; +typedef uint64_t nfs_cookie4; +typedef opaque nfs_fh4; typedef opaque utf8string<>; typedef utf8string utf8str_cis; typedef utf8string utf8str_cs; typedef utf8string utf8str_mixed; +typedef utf8str_cs component4; +typedef utf8str_cs linktext4; +typedef component4 pathname4<>; + /* * Timeval */ @@ -66,6 +209,21 @@ struct nfstime4 { uint32_t nseconds; }; +/* + * File attribute container + */ +struct fattr4 { + bitmap4 attrmask; + attrlist4 attr_vals; +}; + +/* + * Stateid + */ +struct stateid4 { + uint32_t seqid; + opaque other[12]; +}; /* * The following content was extracted from draft-ietf-nfsv4-delstid @@ -245,3 +403,82 @@ const FATTR4_ACL_TRUEFORM = 89; const FATTR4_ACL_TRUEFORM_SCOPE = 90; const FATTR4_POSIX_DEFAULT_ACL = 91; const FATTR4_POSIX_ACCESS_ACL = 92; + +/* + * Directory notification types. + */ +enum notify_type4 { + NOTIFY4_CHANGE_CHILD_ATTRS = 0, + NOTIFY4_CHANGE_DIR_ATTRS = 1, + NOTIFY4_REMOVE_ENTRY = 2, + NOTIFY4_ADD_ENTRY = 3, + NOTIFY4_RENAME_ENTRY = 4, + NOTIFY4_CHANGE_COOKIE_VERIFIER = 5 +}; + +/* Changed entry information. */ +struct notify_entry4 { + component4 ne_file; + fattr4 ne_attrs; +}; + +/* Previous entry information */ +struct prev_entry4 { + notify_entry4 pe_prev_entry; + /* what READDIR returned for this entry */ + nfs_cookie4 pe_prev_entry_cookie; +}; + +struct notify_remove4 { + notify_entry4 nrm_old_entry; + nfs_cookie4 nrm_old_entry_cookie; +}; + +struct notify_add4 { + /* + * Information on object + * possibly renamed over. + */ + notify_remove4 nad_old_entry<1>; + notify_entry4 nad_new_entry; + /* what READDIR would have returned for this entry */ + nfs_cookie4 nad_new_entry_cookie<1>; + prev_entry4 nad_prev_entry<1>; + bool nad_last_entry; +}; + +struct notify_attr4 { + notify_entry4 na_changed_entry; +}; + +struct notify_rename4 { + notify_remove4 nrn_old_entry; + notify_add4 nrn_new_entry; +}; + +struct notify_verifier4 { + verifier4 nv_old_cookieverf; + verifier4 nv_new_cookieverf; +}; + +/* + * Objects of type notify_<>4 and + * notify_device_<>4 are encoded in this. + */ +typedef opaque notifylist4<>; + +struct notify4 { + /* composed from notify_type4 or notify_deviceid_type4 */ + bitmap4 notify_mask; + notifylist4 notify_vals; +}; + +struct CB_NOTIFY4args { + stateid4 cna_stateid; + nfs_fh4 cna_fh; + notify4 cna_changes<>; +}; + +struct CB_NOTIFY4res { + nfsstat4 cnr_status; +}; diff --git a/Documentation/trace/hisi-ptt.rst b/Documentation/trace/hisi-ptt.rst index 6eef28ebb0c7b..f6a2655f99e57 100644 --- a/Documentation/trace/hisi-ptt.rst +++ b/Documentation/trace/hisi-ptt.rst @@ -285,20 +285,20 @@ according to the format described previously (take 8DW as an example): [...perf headers and other information] . ... HISI PTT data: size 4194304 bytes . 00000000: 00 00 00 00 Prefix - . 00000004: 01 00 00 60 Header DW0 - . 00000008: 0f 1e 00 01 Header DW1 - . 0000000c: 04 00 00 00 Header DW2 - . 00000010: 40 00 81 02 Header DW3 - . 00000014: 33 c0 04 00 Time + . 00000004: 60 00 00 01 Header DW0 + . 00000008: 01 00 1e 0f Header DW1 + . 0000000c: 00 00 00 04 Header DW2 + . 00000010: 02 81 00 40 Header DW3 + . 00000014: 00 04 c0 33 Time . 00000020: 00 00 00 00 Prefix - . 00000024: 01 00 00 60 Header DW0 - . 00000028: 0f 1e 00 01 Header DW1 - . 0000002c: 04 00 00 00 Header DW2 - . 00000030: 40 00 81 02 Header DW3 - . 00000034: 02 00 00 00 Time + . 00000024: 60 00 00 01 Header DW0 + . 00000028: 01 00 1e 0f Header DW1 + . 0000002c: 00 00 00 04 Header DW2 + . 00000030: 02 81 00 40 Header DW3 + . 00000034: 00 00 00 02 Time . 00000040: 00 00 00 00 Prefix - . 00000044: 01 00 00 60 Header DW0 - . 00000048: 0f 1e 00 01 Header DW1 - . 0000004c: 04 00 00 00 Header DW2 - . 00000050: 40 00 81 02 Header DW3 + . 00000044: 60 00 00 01 Header DW0 + . 00000048: 01 00 1e 0f Header DW1 + . 0000004c: 00 00 00 04 Header DW2 + . 00000050: 02 81 00 40 Header DW3 [...] diff --git a/Documentation/usb/gadget-testing.rst b/Documentation/usb/gadget-testing.rst index a6e8292f320a4..4921e5307d493 100644 --- a/Documentation/usb/gadget-testing.rst +++ b/Documentation/usb/gadget-testing.rst @@ -714,9 +714,6 @@ The uac1 function provides these attributes in its function directory: =============== ==================================== audio_buf_size audio buffer size - fn_cap capture pcm device file name - fn_cntl control device file name - fn_play playback pcm device file name req_buf_size ISO OUT endpoint request buffer size req_count ISO OUT endpoint request count =============== ==================================== diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-colorimetry.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-colorimetry.rst index 38a4136d7220e..0a7da2f168507 100644 --- a/Documentation/userspace-api/media/v4l/ext-ctrls-colorimetry.rst +++ b/Documentation/userspace-api/media/v4l/ext-ctrls-colorimetry.rst @@ -80,15 +80,25 @@ Colorimetry Control IDs - ``white_point_x`` - Specifies the normalized x chromaticity coordinate of the white point of the mastering display in increments of 0.00002. + When both ``white_point_x`` and ``white_point_y`` are zero, + the white point chromaticity is unknown. If either coordinate is + non-zero, both coordinates shall be within their valid ranges. * - __u16 - ``white_point_y`` - Specifies the normalized y chromaticity coordinate of the white point of the mastering display in increments of 0.00002. + When both ``white_point_x`` and ``white_point_y`` are zero, + the white point chromaticity is unknown. If either coordinate is + non-zero, both coordinates shall be within their valid ranges. * - __u32 - ``max_luminance`` - Specifies the nominal maximum display luminance of the mastering display in units of 0.0001 cd/m\ :sup:`2`. + A value of zero indicates that the nominal maximum display + luminance is unknown. * - __u32 - ``min_luminance`` - - specifies the nominal minimum display luminance of the mastering + - Specifies the nominal minimum display luminance of the mastering display in units of 0.0001 cd/m\ :sup:`2`. + A value of zero indicates that the nominal minimum display + luminance is unknown. diff --git a/MAINTAINERS b/MAINTAINERS index 8014b9f8253ed..e2f46fb5bfc76 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4459,6 +4459,20 @@ F: Documentation/devicetree/bindings/spi/axiado,ax3000-spi.yaml F: drivers/spi/spi-axiado.c F: drivers/spi/spi-axiado.h +AYN PLATFORM EC DRIVER +M: Derek J. Clark +L: platform-driver-x86@vger.kernel.org +S: Maintained +F: Documentation/ABI/testing/sysfs-platform-ayn-ec +F: drivers/platform/x86/ayn-ec.c + +AYANEO 3 CONTROLLER HID DRIVER +M: Matías Martínez +L: linux-input@vger.kernel.org +S: Maintained +F: Documentation/ABI/testing/sysfs-driver-hid-ayaneo +F: drivers/hid/hid-ayaneo.c + AYANEO PLATFORM EC DRIVER M: Antheas Kapenekakis L: platform-driver-x86@vger.kernel.org @@ -26241,6 +26255,7 @@ R: Andy Shevchenko L: linux-i2c@vger.kernel.org S: Supported F: drivers/i2c/busses/i2c-designware-* +F: include/linux/designware_i2c.h SYNOPSYS DESIGNWARE I2C DRIVER - AMDISP M: Nirujogi Pratap diff --git a/Makefile b/Makefile index ddcc6dca1ece8..6f8ed7329a207 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 7 PATCHLEVEL = 2 -SUBLEVEL = 0 +SUBLEVEL = 9 EXTRAVERSION = NAME = Baby Opossum Posse @@ -1083,6 +1083,16 @@ endif export CC_FLAGS_SCS endif +ifdef CONFIG_RUST_INLINE_HELPERS +# `rustc` normally emits traps for unreachable paths during code generation. +# With inline helpers, Clang performs code generation from the linked bitcode +# instead, so request the same behavior explicitly. Otherwise `objtool` may +# follow an impossible Rust path into the next function. +CC_FLAGS_RUST_INLINE_HELPERS := -mllvm -trap-unreachable \ + -mllvm -no-trap-after-noreturn +export CC_FLAGS_RUST_INLINE_HELPERS +endif + ifdef CONFIG_LTO_CLANG ifdef CONFIG_LTO_CLANG_FULL CC_FLAGS_LTO := -flto @@ -1118,7 +1128,8 @@ endif ifdef CONFIG_RUST # Always pass -Zsanitizer-cfi-normalize-integers as CONFIG_RUST selects # CONFIG_CFI_ICALL_NORMALIZE_INTEGERS. - RUSTC_FLAGS_CFI := -Zsanitizer=kcfi -Zsanitizer-cfi-normalize-integers + # Disable function merging as LLVM incorrectly merges functions with different KCFI types. + RUSTC_FLAGS_CFI := -Zsanitizer=kcfi -Zsanitizer-cfi-normalize-integers -Zmerge-functions=disabled KBUILD_RUSTFLAGS += $(RUSTC_FLAGS_CFI) export RUSTC_FLAGS_CFI endif @@ -1350,7 +1361,7 @@ PHONY += vmlinux_o vmlinux_o: vmlinux.a $(KBUILD_VMLINUX_LIBS) $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.vmlinux_o -vmlinux.o modules.builtin.modinfo modules.builtin: vmlinux_o +vmlinux.o: vmlinux_o @: PHONY += vmlinux @@ -1594,10 +1605,10 @@ tools/%: FORCE PHONY += kselftest kselftest: headers - $(Q)$(MAKE) -C $(srctree)/tools/testing/selftests run_tests + $(Q)unset sub_make_done; $(MAKE) -C $(srctree)/tools/testing/selftests run_tests kselftest-%: headers FORCE - $(Q)$(MAKE) -C $(srctree)/tools/testing/selftests $* + $(Q)unset sub_make_done; $(MAKE) -C $(srctree)/tools/testing/selftests $* PHONY += kselftest-merge kselftest-merge: diff --git a/arch/alpha/include/uapi/asm/fpu.h b/arch/alpha/include/uapi/asm/fpu.h index cea9eafa056fc..d28dc36786e27 100644 --- a/arch/alpha/include/uapi/asm/fpu.h +++ b/arch/alpha/include/uapi/asm/fpu.h @@ -101,7 +101,12 @@ ieee_swcr_to_fpcr(unsigned long sw) | IEEE_TRAP_ENABLE_OVF)) << 48; fp |= (~sw & (IEEE_TRAP_ENABLE_UNF | IEEE_TRAP_ENABLE_INE)) << 57; fp |= (sw & IEEE_MAP_UMZ ? FPCR_UNDZ | FPCR_UNFD : 0); - fp |= (~sw & IEEE_TRAP_ENABLE_DNO) << 41; + /* + * Disable denormal operand traps only when denormal inputs are to be + * flushed to zero. Otherwise they must keep trapping, so that /S + * instructions reach the kernel emulation handler. + */ + fp |= (sw & IEEE_MAP_DMZ ? FPCR_DNOD : 0); return fp; } @@ -116,7 +121,6 @@ ieee_fpcr_to_swcr(unsigned long fp) | IEEE_TRAP_ENABLE_OVF); sw |= (~fp >> 57) & (IEEE_TRAP_ENABLE_UNF | IEEE_TRAP_ENABLE_INE); sw |= (fp >> 47) & IEEE_MAP_UMZ; - sw |= (~fp >> 41) & IEEE_TRAP_ENABLE_DNO; return sw; } diff --git a/arch/alpha/kernel/pci-sysfs.c b/arch/alpha/kernel/pci-sysfs.c index 94dbc470cd6c0..7050f0f7fe3d7 100644 --- a/arch/alpha/kernel/pci-sysfs.c +++ b/arch/alpha/kernel/pci-sysfs.c @@ -224,17 +224,17 @@ int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, size_t size) switch(size) { case 1: - outb(port, val); + outb(val, port); return 1; case 2: if (port & 1) return -EINVAL; - outw(port, val); + outw(val, port); return 2; case 4: if (port & 3) return -EINVAL; - outl(port, val); + outl(val, port); return 4; } return -EINVAL; diff --git a/arch/alpha/kernel/sys_marvel.c b/arch/alpha/kernel/sys_marvel.c index 1f99b03effc2e..a37707e05e34e 100644 --- a/arch/alpha/kernel/sys_marvel.c +++ b/arch/alpha/kernel/sys_marvel.c @@ -263,6 +263,18 @@ init_io7_irqs(struct io7 *io7, */ printk(" Interrupts reported to CPU at PE %u\n", boot_cpuid); + /* Set up the lsi irqs. */ + for (i = 0; i < 128; ++i) { + irq_set_chip_and_handler(base + i, lsi_ops, handle_level_irq); + irq_set_status_flags(base + i, IRQ_LEVEL); + } + + /* Set up the msi irqs. */ + for (i = 128; i < (128 + 512); ++i) { + irq_set_chip_and_handler(base + i, msi_ops, handle_level_irq); + irq_set_status_flags(base + i, IRQ_LEVEL); + } + raw_spin_lock(&io7->irq_lock); /* set up the error irqs */ @@ -272,12 +284,6 @@ init_io7_irqs(struct io7 *io7, io7_redirect_irq(io7, &io7->csrs->STV_CTL.csr, boot_cpuid); io7_redirect_irq(io7, &io7->csrs->HEI_CTL.csr, boot_cpuid); - /* Set up the lsi irqs. */ - for (i = 0; i < 128; ++i) { - irq_set_chip_and_handler(base + i, lsi_ops, handle_level_irq); - irq_set_status_flags(i, IRQ_LEVEL); - } - /* Disable the implemented irqs in hardware. */ for (i = 0; i < 0x60; ++i) init_one_io7_lsi(io7, i, boot_cpuid); @@ -285,13 +291,6 @@ init_io7_irqs(struct io7 *io7, init_one_io7_lsi(io7, 0x74, boot_cpuid); init_one_io7_lsi(io7, 0x75, boot_cpuid); - - /* Set up the msi irqs. */ - for (i = 128; i < (128 + 512); ++i) { - irq_set_chip_and_handler(base + i, msi_ops, handle_level_irq); - irq_set_status_flags(i, IRQ_LEVEL); - } - for (i = 0; i < 16; ++i) init_one_io7_msi(io7, i, boot_cpuid); diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c index 7004397937cfd..7cd20e5f9ee03 100644 --- a/arch/alpha/kernel/traps.c +++ b/arch/alpha/kernel/traps.c @@ -166,12 +166,12 @@ static long dummy_emul(void) { return 0; } long (*alpha_fp_emul_imprecise)(struct pt_regs *regs, unsigned long writemask) = (void *)dummy_emul; EXPORT_SYMBOL_GPL(alpha_fp_emul_imprecise); -long (*alpha_fp_emul) (unsigned long pc) +long (*alpha_fp_emul) (unsigned long pc, unsigned long summary) = (void *)dummy_emul; EXPORT_SYMBOL_GPL(alpha_fp_emul); #else long alpha_fp_emul_imprecise(struct pt_regs *regs, unsigned long writemask); -long alpha_fp_emul (unsigned long pc); +long alpha_fp_emul (unsigned long pc, unsigned long summary); #endif asmlinkage void @@ -185,7 +185,7 @@ do_entArith(unsigned long summary, unsigned long write_mask, emulate the instruction. If the processor supports precise exceptions, we don't have to search. */ if (!amask(AMASK_PRECISE_TRAP)) - si_code = alpha_fp_emul(regs->pc - 4); + si_code = alpha_fp_emul(regs->pc - 4, summary); else si_code = alpha_fp_emul_imprecise(regs, write_mask); if (si_code == 0) diff --git a/arch/alpha/math-emu/math.c b/arch/alpha/math-emu/math.c index 68d420bfd3c0d..e3f2df3729e3a 100644 --- a/arch/alpha/math-emu/math.c +++ b/arch/alpha/math-emu/math.c @@ -52,13 +52,13 @@ MODULE_DESCRIPTION("FP Software completion module"); MODULE_LICENSE("GPL v2"); extern long (*alpha_fp_emul_imprecise)(struct pt_regs *, unsigned long); -extern long (*alpha_fp_emul) (unsigned long pc); +extern long (*alpha_fp_emul) (unsigned long pc, unsigned long summary); static long (*save_emul_imprecise)(struct pt_regs *, unsigned long); -static long (*save_emul) (unsigned long pc); +static long (*save_emul) (unsigned long pc, unsigned long summary); long do_alpha_fp_emul_imprecise(struct pt_regs *, unsigned long); -long do_alpha_fp_emul(unsigned long); +long do_alpha_fp_emul(unsigned long, unsigned long); static int alpha_fp_emul_init_module(void) { @@ -86,7 +86,22 @@ module_exit(alpha_fp_emul_cleanup_module); /* - * Emulate the floating point instruction at address PC. Returns -1 if the + * Exception bits of the exception summary register (EXC_SUM). Bit 0 is the + * software completion bit; bits 1 through 5 report the exceptions the + * hardware attributed to the trapping instruction, and lie at the same + * positions as the corresponding IEEE_TRAP_ENABLE_* bits. + */ +#define EXC_SUM_INV (1UL << 1) +#define EXC_SUM_DZE (1UL << 2) +#define EXC_SUM_OVF (1UL << 3) +#define EXC_SUM_UNF (1UL << 4) +#define EXC_SUM_INE (1UL << 5) +#define EXC_SUM_MASK (EXC_SUM_INV | EXC_SUM_DZE | EXC_SUM_OVF \ + | EXC_SUM_UNF | EXC_SUM_INE) + +/* + * Emulate the floating point instruction at address PC. SUMMARY is the + * exception summary register the trap was delivered with. Returns -1 if the * instruction to be emulated is illegal (such as with the opDEC trap), else * the SI_CODE for a SIGFPE signal, else 0 if everything's ok. * @@ -95,7 +110,7 @@ module_exit(alpha_fp_emul_cleanup_module); * stick the result of the operation into the appropriate register. */ long -alpha_fp_emul (unsigned long pc) +alpha_fp_emul (unsigned long pc, unsigned long summary) { FP_DECL_EX; FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); @@ -300,12 +315,56 @@ alpha_fp_emul (unsigned long pc) swcr |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); current_thread_info()->ieee_state |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + } - /* Update hardware control register. */ - fpcr &= (~FPCR_MASK | FPCR_DYN_MASK); - fpcr |= ieee_swcr_to_fpcr(swcr); - wrfpcr(fpcr); + /* + * EV6 records exception status bits in the FPCR before delivering the + * software completion trap, and swcr_update_status() above merged them + * into SWCR. Some can be wrong for the instruction we just emulated: + * a CVTTS of a value exactly representable as a subnormal sets FPCR_UNF + * even though the result is exact. Clear the exceptions the trap + * reported but that soft-fp did not raise. + */ + if (implver() == IMPLVER_EV6) { + unsigned long spurious = summary & EXC_SUM_MASK; + + if (spurious & (EXC_SUM_UNF | EXC_SUM_OVF)) { + /* + * EXC_SUM reports only the underflow or overflow, + * but the hardware sets INE alongside it in the FPCR. + */ + spurious |= EXC_SUM_INE; + } else if (!spurious) { + /* + * No exception reported, so this was a denormal + * operand trap, for which INE and UNF can be + * fabricated as well. + */ + spurious = EXC_SUM_INE | EXC_SUM_UNF; + } + /* + * Never clear an exception software has confirmed. Every + * instruction that genuinely raises one traps for software + * completion and is recorded in ieee_state above, so a bit + * found there -- including one just set from _fex -- belongs + * to this or an earlier instruction and must survive. + */ + spurious &= ~(current_thread_info()->ieee_state + >> IEEE_STATUS_TO_EXCSUM_SHIFT); + + swcr &= ~(spurious << IEEE_STATUS_TO_EXCSUM_SHIFT); + } + + /* + * Update hardware control register. This has to happen even when + * soft-fp raised nothing, to clear any fabricated bits. + */ + fpcr &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr |= ieee_swcr_to_fpcr(swcr); + wrfpcr(fpcr); + + if (_fex) { /* Do we generate a signal? */ _fex = _fex & swcr & IEEE_TRAP_ENABLE_MASK; si_code = 0; @@ -387,9 +446,16 @@ alpha_fp_emul_imprecise (struct pt_regs *regs, unsigned long write_mask) break; } if (!write_mask) { - /* Re-execute insns in the trap-shadow. */ + /* + * Re-execute insns in the trap-shadow. Pass no + * exception summary: it describes the trap, which + * was taken anywhere in the shadow, and so is not + * attribution for this instruction. Nothing is + * lost, since only EV6 -- which traps precisely and + * never comes this way -- needs it. + */ regs->pc = trigger_pc + 4; - si_code = alpha_fp_emul(trigger_pc); + si_code = alpha_fp_emul(trigger_pc, 0); goto egress; } trigger_pc -= 4; diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 9187240a02db5..bcb53bc9b931a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -98,7 +98,7 @@ config ARM select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE if ARM_LPAE select HAVE_ARM_SMCCC if CPU_V7 - select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32 + select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32 && !CPU_32v3 select HAVE_CONTEXT_TRACKING_USER select HAVE_C_RECORDMCOUNT select HAVE_BUILDTIME_MCOUNT_SORT @@ -133,6 +133,7 @@ config ARM select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_POSIX_CPU_TIMERS_TASK_WORK select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RSEQ diff --git a/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi b/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi index 51a6464aab9a3..cabf619c2e217 100644 --- a/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi +++ b/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi @@ -185,7 +185,7 @@ pmu { compatible = "arm,cortex-a8-pmu"; - interrupts = <3>; + interrupts = <66>; }; reserved-memory { diff --git a/arch/arm/boot/dts/marvell/armada-388-helios4.dts b/arch/arm/boot/dts/marvell/armada-388-helios4.dts index ec134e22bae3e..003b0708e9656 100644 --- a/arch/arm/boot/dts/marvell/armada-388-helios4.dts +++ b/arch/arm/boot/dts/marvell/armada-388-helios4.dts @@ -169,6 +169,7 @@ gpio-controller; #gpio-cells = <2>; reg = <0x20>; + vcc-supply = <®_3p3v>; pinctrl-names = "default"; pinctrl-0 = <&pca0_pins>; interrupt-parent = <&gpio0>; @@ -201,6 +202,10 @@ reg = <0x4c>; vcc-supply = <®_3p3v>; }; + + eeprom@53 { + vcc-supply = <®_3p3v>; + }; }; i2c@11100 { @@ -217,13 +222,17 @@ status = "okay"; #address-cells = <1>; #size-cells = <0>; + ahci-supply = <®_3p3v>; + phy-supply = <®_3p3v>; sata0: sata-port@0 { reg = <0>; + target-supply = <®_5p0v_hdd>; }; sata1: sata-port@1 { reg = <1>; + target-supply = <®_5p0v_hdd>; }; }; @@ -231,13 +240,17 @@ status = "okay"; #address-cells = <1>; #size-cells = <0>; + ahci-supply = <®_3p3v>; + phy-supply = <®_3p3v>; sata2: sata-port@0 { reg = <0>; + target-supply = <®_5p0v_hdd>; }; sata3: sata-port@1 { reg = <1>; + target-supply = <®_5p0v_hdd>; }; }; diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index cd4b34c96e35e..38feb30dfb5f8 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -929,10 +929,6 @@ static void hw_breakpoint_cfi_handler(struct pt_regs *regs) break; } } -#else -static void hw_breakpoint_cfi_handler(struct pt_regs *regs) -{ -} #endif /* @@ -964,9 +960,14 @@ static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, case ARM_ENTRY_SYNC_WATCHPOINT: watchpoint_handler(addr, fsr, regs); break; +#ifdef CONFIG_CFI case ARM_ENTRY_CFI_BREAKPOINT: - hw_breakpoint_cfi_handler(regs); + if (user_mode(regs)) + ret = 1; /* Don't handle userspace BKPT */ + else + hw_breakpoint_cfi_handler(regs); break; +#endif default: ret = 1; /* Unhandled fault. */ } diff --git a/arch/arm/mach-imx/avic.c b/arch/arm/mach-imx/avic.c index 3067c06b4b8eb..6873a50bbe2c0 100644 --- a/arch/arm/mach-imx/avic.c +++ b/arch/arm/mach-imx/avic.c @@ -173,6 +173,7 @@ static void __init mxc_init_irq(void __iomem *irqbase) np = of_find_compatible_node(NULL, NULL, "fsl,imx25-ccm"); mx25_ccm_base = of_iomap(np, 0); + of_node_put(np); if (mx25_ccm_base) { /* @@ -203,6 +204,7 @@ static void __init mxc_init_irq(void __iomem *irqbase) np = of_find_compatible_node(NULL, NULL, "fsl,avic"); domain = irq_domain_create_legacy(of_fwnode_handle(np), AVIC_NUM_IRQS, irq_base, 0, &irq_domain_simple_ops, NULL); + of_node_put(np); WARN_ON(!domain); for (i = 0; i < AVIC_NUM_IRQS / 32; i++, irq_base += 32) diff --git a/arch/arm/mach-imx/src.c b/arch/arm/mach-imx/src.c index 59a8e8cc44693..c3c80b4c3d53b 100644 --- a/arch/arm/mach-imx/src.c +++ b/arch/arm/mach-imx/src.c @@ -171,6 +171,7 @@ void __init imx_src_init(void) if (!np) return; src_base = of_iomap(np, 0); + of_node_put(np); WARN_ON(!src_base); /* @@ -195,6 +196,7 @@ void __init imx7_src_init(void) return; src_base = of_iomap(np, 0); + of_node_put(np); if (!src_base) return; @@ -203,6 +205,7 @@ void __init imx7_src_init(void) return; gpc_base = of_iomap(np, 0); + of_node_put(np); if (!gpc_base) return; } diff --git a/arch/arm/mach-lpc32xx/common.c b/arch/arm/mach-lpc32xx/common.c index 304ea61a07160..35ed3569c5a35 100644 --- a/arch/arm/mach-lpc32xx/common.c +++ b/arch/arm/mach-lpc32xx/common.c @@ -106,7 +106,7 @@ void __init lpc32xx_map_io(void) iotable_init(lpc32xx_io_desc, ARRAY_SIZE(lpc32xx_io_desc)); } -static int __init lpc32xx_check_uid(void) +void __init lpc32xx_check_uid(void) { u32 uid[4]; @@ -119,7 +119,4 @@ static int __init lpc32xx_check_uid(void) system_serial_low = uid[0]; system_serial_high = uid[1]; } - - return 1; } -arch_initcall(lpc32xx_check_uid); diff --git a/arch/arm/mach-lpc32xx/common.h b/arch/arm/mach-lpc32xx/common.h index 32f0ad2178077..06b20bea324e1 100644 --- a/arch/arm/mach-lpc32xx/common.h +++ b/arch/arm/mach-lpc32xx/common.h @@ -16,6 +16,8 @@ * Other arch specific structures and functions */ extern void __init lpc32xx_map_io(void); +extern void __init lpc32xx_check_uid(void); +extern void __init lpc32xx_pm_init(void); extern void __init lpc32xx_serial_init(void); /* diff --git a/arch/arm/mach-lpc32xx/phy3250.c b/arch/arm/mach-lpc32xx/phy3250.c index 66701bf432488..ddc6333ca55da 100644 --- a/arch/arm/mach-lpc32xx/phy3250.c +++ b/arch/arm/mach-lpc32xx/phy3250.c @@ -71,6 +71,8 @@ static const struct of_dev_auxdata lpc32xx_auxdata_lookup[] __initconst = { static void __init lpc3250_machine_init(void) { + lpc32xx_check_uid(); + lpc32xx_pm_init(); lpc32xx_serial_init(); of_platform_default_populate(NULL, lpc32xx_auxdata_lookup, NULL); diff --git a/arch/arm/mach-lpc32xx/pm.c b/arch/arm/mach-lpc32xx/pm.c index 2572bd89a5e8d..9b5c5e1462ed3 100644 --- a/arch/arm/mach-lpc32xx/pm.c +++ b/arch/arm/mach-lpc32xx/pm.c @@ -120,7 +120,7 @@ static const struct platform_suspend_ops lpc32xx_pm_ops = { #define EMC_DYN_MEM_CTRL_OFS 0x20 #define EMC_SRMMC (1 << 3) #define EMC_CTRL_REG io_p2v(LPC32XX_EMC_BASE + EMC_DYN_MEM_CTRL_OFS) -static int __init lpc32xx_pm_init(void) +void __init lpc32xx_pm_init(void) { /* * Setup SDRAM self-refresh clock to automatically disable o @@ -129,7 +129,4 @@ static int __init lpc32xx_pm_init(void) __raw_writel(__raw_readl(EMC_CTRL_REG) | EMC_SRMMC, EMC_CTRL_REG); suspend_set_ops(&lpc32xx_pm_ops); - - return 0; } -arch_initcall(lpc32xx_pm_init); diff --git a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c index 974107ff18b4e..03cd523dff874 100644 --- a/arch/arm/mach-omap2/omap_hwmod.c +++ b/arch/arm/mach-omap2/omap_hwmod.c @@ -2331,13 +2331,15 @@ static int __init _init(struct omap_hwmod *oh, void *data) if (r < 0) { WARN(1, "omap_hwmod: %s: doesn't have mpu register target base\n", oh->name); - return 0; + r = 0; + goto out_put_node; } r = _init_clocks(oh, np); if (r < 0) { WARN(1, "omap_hwmod: %s: couldn't init clocks\n", oh->name); - return -EINVAL; + r = -EINVAL; + goto out_put_node; } if (np) { @@ -2345,13 +2347,19 @@ static int __init _init(struct omap_hwmod *oh, void *data) parse_module_flags(oh, np); child = of_get_next_child(np, NULL); - if (child) + if (child) { parse_module_flags(oh, child); + of_node_put(child); + } } oh->_state = _HWMOD_STATE_INITIALIZED; - return 0; + r = 0; + +out_put_node: + of_node_put(bus); + return r; } /** @@ -3608,13 +3616,13 @@ int omap_hwmod_init_module(struct device *dev, #ifdef CONFIG_SERIAL_EARLYCON static void __init omap_hwmod_setup_earlycon_flags(void) { - struct device_node *np; + struct device_node *np, *chosen; struct omap_hwmod *oh; const char *uart; - np = of_find_node_by_path("/chosen"); - if (np) { - uart = of_get_property(np, "stdout-path", NULL); + chosen = of_find_node_by_path("/chosen"); + if (chosen) { + uart = of_get_property(chosen, "stdout-path", NULL); if (uart) { np = of_find_node_by_path(uart); if (np) { @@ -3629,8 +3637,10 @@ static void __init omap_hwmod_setup_earlycon_flags(void) if (oh) oh->flags |= DEBUG_OMAPUART_FLAGS; } + of_node_put(np); } } + of_node_put(chosen); } #endif diff --git a/arch/arm/mach-socfpga/Kconfig b/arch/arm/mach-socfpga/Kconfig index eb72c240c2486..528c5c1368c37 100644 --- a/arch/arm/mach-socfpga/Kconfig +++ b/arch/arm/mach-socfpga/Kconfig @@ -16,7 +16,7 @@ menuconfig ARCH_INTEL_SOCFPGA select ARM_ERRATA_775420 select PL310_ERRATA_588369 select PL310_ERRATA_727915 - select PL310_ERRATA_753970 if PL310 + select PL310_ERRATA_753970 select PL310_ERRATA_769419 select RESET_CONTROLLER diff --git a/arch/arm/mach-tegra/irq.c b/arch/arm/mach-tegra/irq.c index 4e1ee70b2a3f1..e5a611dce7e74 100644 --- a/arch/arm/mach-tegra/irq.c +++ b/arch/arm/mach-tegra/irq.c @@ -66,9 +66,9 @@ static const struct of_device_id tegra114_dt_gic_match[] __initconst = { static void __init tegra114_gic_cpu_pm_registration(void) { - struct device_node *dn; + struct device_node *dn __free(device_node) = + of_find_matching_node(NULL, tegra114_dt_gic_match); - dn = of_find_matching_node(NULL, tegra114_dt_gic_match); if (!dn) return; @@ -88,7 +88,10 @@ static const struct of_device_id tegra_ictlr_match[] __initconst = { void __init tegra_init_irq(void) { - if (WARN_ON(!of_find_matching_node(NULL, tegra_ictlr_match))) + struct device_node *dn __free(device_node) = + of_find_matching_node(NULL, tegra_ictlr_match); + + if (WARN_ON(!dn)) pr_warn("Outdated DT detected, suspend/resume will NOT work\n"); tegra114_gic_cpu_pm_registration(); diff --git a/arch/arm/mach-ux500/cpu-db8500.c b/arch/arm/mach-ux500/cpu-db8500.c index b1a70f203372b..0d7530fb6ad07 100644 --- a/arch/arm/mach-ux500/cpu-db8500.c +++ b/arch/arm/mach-ux500/cpu-db8500.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include @@ -81,7 +81,7 @@ static void __init ux500_init_irq(void) struct resource r; irqchip_init(); - prcmu_early_init(); + db8500_prcmu_early_init(); np = of_find_compatible_node(NULL, NULL, "stericsson,db8500-prcmu"); of_address_to_resource(np, 0, &r); of_node_put(np); @@ -101,7 +101,7 @@ static void ux500_restart(enum reboot_mode mode, const char *cmd) local_irq_disable(); local_fiq_disable(); - prcmu_system_reset(0); + db8500_prcmu_system_reset(0); } static const struct of_device_id u8500_local_bus_nodes[] = { diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index e62cc4be5adf6..0a09d4ff77185 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -181,7 +181,11 @@ __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig, pr_err("8<--- cut here ---\n"); pr_err("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n", tsk->comm, sig, addr, fsr); - show_pte(KERN_ERR, tsk->mm, addr); + if (likely(addr < TASK_SIZE)) { + mmap_write_lock(tsk->mm); + show_pte(KERN_ERR, tsk->mm, addr); + mmap_write_unlock(tsk->mm); + } show_regs(regs); } #endif @@ -633,10 +637,21 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs)) return; + if (likely(user_mode(regs))) + local_irq_enable(); + pr_alert("8<--- cut here ---\n"); pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n", inf->name, fsr, addr); - show_pte(KERN_ALERT, current->mm, addr); + if (likely(user_mode(regs))) { + if (addr < TASK_SIZE) { + mmap_write_lock(current->mm); + show_pte(KERN_ALERT, current->mm, addr); + mmap_write_unlock(current->mm); + } + } else { + show_pte(KERN_ALERT, current->mm, addr); + } arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr, fsr, 0); @@ -663,6 +678,9 @@ do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs)) return; + if (likely(user_mode(regs))) + local_irq_enable(); + pr_alert("8<--- cut here ---\n"); pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n", inf->name, ifsr, addr); diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919b..fc57d90d92c1d 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2116,6 +2116,8 @@ config ARM64_BTI_KERNEL depends on !CC_IS_GCC || GCC_VERSION >= 100100 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106671 depends on !CC_IS_GCC + # https://github.com/llvm/llvm-project/issues/215547 + depends on !CC_IS_CLANG || CLANG_VERSION < 210000 depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_ARGS) help Build the kernel with Branch Target Identification annotations diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi index 4bc6c1ef2cde4..f958bdbb0d333 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi @@ -230,8 +230,8 @@ interrupts = <7 5 IRQ_TYPE_EDGE_RISING>; /* PH5 */ vdd-supply = <®_dldo1>; vddio-supply = <®_dldo1>; - mount-matrix = "0", "1", "0", - "-1", "0", "0", + mount-matrix = "0", "-1", "0", + "1", "0", "0", "0", "0", "1"; }; }; diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dtsi b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dtsi old mode 100755 new mode 100644 diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk_emmc.dts b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk_emmc.dts old mode 100755 new mode 100644 diff --git a/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-an400.dts b/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-an400.dts index cab2ee9ea0d31..ca7536f772ff1 100644 --- a/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-an400.dts +++ b/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-an400.dts @@ -33,7 +33,7 @@ }; &uart_a { - clocks = <&xtal>, <&xtal>, <&xtal>; + clocks = <&xtal>, <&clkc_periphs CLKID_SYS_UART_A>, <&xtal>; clock-names = "xtal", "pclk", "baud"; status = "okay"; }; diff --git a/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-khadas-vim4.dts b/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-khadas-vim4.dts index c41525a34b721..77abfd555ec5b 100644 --- a/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-khadas-vim4.dts +++ b/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-khadas-vim4.dts @@ -71,7 +71,6 @@ vin-supply = <&vddao_3v3>; gpio = <&gpio GPIOD_11 GPIO_ACTIVE_LOW>; regulator-boot-on; - regulator-always-on; }; sdio_pwrseq: sdio-pwrseq { @@ -105,6 +104,66 @@ enable-active-high; }; + vdd_ddr: regulator-vddddr { + /* + * SY8003ADFC Regulator. + */ + compatible = "pwm-regulator"; + regulator-name = "VDDDDR"; + regulator-min-microvolt = <690000>; + regulator-max-microvolt = <890000>; + pwm-supply = <&dc_in>; + pwms = <&pwm_ao_gh 0 1500 0>; + pwm-dutycycle-range = <100 0>; + regulator-boot-on; + regulator-always-on; + }; + + vdd_ee: regulator-vddee { + /* + * MP8756GD Regulator. + */ + compatible = "pwm-regulator"; + regulator-name = "VDDEE"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <922000>; + pwm-supply = <&dc_in>; + pwms = <&pwm_ao_ab 0 1500 0>; + pwm-dutycycle-range = <100 0>; + regulator-boot-on; + regulator-always-on; + }; + + vdd_gpu: regulator-vddgpu { + /* + * SY8003ADFC Regulator. + */ + compatible = "pwm-regulator"; + regulator-name = "VDDGPU"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <922000>; + pwm-supply = <&dc_in>; + pwms = <&pwm_ao_ef 0 1500 0>; + pwm-dutycycle-range = <100 0>; + regulator-boot-on; + regulator-always-on; + }; + + vdd_npu: regulator-vddnpu { + /* + * SY8003ADFC Regulator. + */ + compatible = "pwm-regulator"; + regulator-name = "VDDNPU"; + regulator-min-microvolt = <733000>; + regulator-max-microvolt = <933000>; + pwm-supply = <&dc_in>; + pwms = <&pwm_ao_ef 1 1500 0>; + pwm-dutycycle-range = <100 0>; + regulator-boot-on; + regulator-always-on; + }; + vddao_1v8: regulator-vddao-1v8 { compatible = "regulator-fixed"; regulator-name = "VDDAO_1V8"; @@ -123,6 +182,36 @@ regulator-always-on; }; + vddcpu_a: regulator-vddcpu-a { + /* + * MP8756GD Regulator. + */ + compatible = "pwm-regulator"; + regulator-name = "VDDCPU_A"; + regulator-min-microvolt = <689000>; + regulator-max-microvolt = <1049000>; + pwm-supply = <&dc_in>; + pwms = <&pwm_ao_cd 1 1500 0>; + pwm-dutycycle-range = <100 0>; + regulator-boot-on; + regulator-always-on; + }; + + vddcpu_b: regulator-vddcpu-b { + /* + * MP8756GD Regulator. + */ + compatible = "pwm-regulator"; + regulator-name = "VDDCPU_B"; + regulator-min-microvolt = <689000>; + regulator-max-microvolt = <1049000>; + pwm-supply = <&dc_in>; + pwms = <&pwm_ao_ab 1 1500 0>; + pwm-dutycycle-range = <100 0>; + regulator-boot-on; + regulator-always-on; + }; + vddio_1v8: regulator-vddio-1v8 { compatible = "regulator-fixed"; regulator-name = "VDDIO_1V8"; @@ -173,9 +262,27 @@ pinctrl-names = "default"; }; +&pwm_ao_ab { + status = "okay"; + pinctrl-0 = <&pwm_ao_a_pins>, <&pwm_ao_b_pins>; + pinctrl-names = "default"; +}; + &pwm_ao_cd { status = "okay"; - pinctrl-0 = <&pwm_ao_c_d_pins>; + pinctrl-0 = <&pwm_ao_c_d_pins>, <&pwm_ao_d_pins>; + pinctrl-names = "default"; +}; + +&pwm_ao_ef { + status = "okay"; + pinctrl-0 = <&pwm_ao_e_pins>, <&pwm_ao_f_pins>; + pinctrl-names = "default"; +}; + +&pwm_ao_gh { + status = "okay"; + pinctrl-0 = <&pwm_ao_g_e_pins>; pinctrl-names = "default"; }; @@ -266,6 +373,6 @@ &uart_a { status = "okay"; - clocks = <&xtal>, <&xtal>, <&xtal>; + clocks = <&xtal>, <&clkc_periphs CLKID_SYS_UART_A>, <&xtal>; clock-names = "xtal", "pclk", "baud"; }; diff --git a/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi b/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi index cc371fcd18967..1711e24cce17b 100644 --- a/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi +++ b/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi @@ -458,9 +458,25 @@ }; }; - pwm_ao_g_pins: pwm-ao-g { + pwm_ao_g_d11_pins: pwm-ao-g-d11 { mux { - groups = "pwm_ao_g"; + groups = "pwm_ao_g_d11"; + function = "pwm_ao_g"; + bias-disable; + }; + }; + + pwm_ao_g_d7_pins: pwm-ao-g-d7 { + mux { + groups = "pwm_ao_g_d7"; + function = "pwm_ao_g"; + bias-disable; + }; + }; + + pwm_ao_g_e_pins: pwm-ao-g-e { + mux { + groups = "pwm_ao_g_e"; function = "pwm_ao_g"; bias-disable; }; @@ -474,9 +490,17 @@ }; }; - pwm_ao_h_pins: pwm-ao-h { + pwm_ao_h_d5_pins: pwm-ao-h-d5 { mux { - groups = "pwm_ao_h"; + groups = "pwm_ao_h_d5"; + function = "pwm_ao_h"; + bias-disable; + }; + }; + + pwm_ao_h_d10_pins: pwm-ao-h-d10 { + mux { + groups = "pwm_ao_h_d10"; function = "pwm_ao_h"; bias-disable; }; @@ -522,9 +546,17 @@ }; }; - pwm_vs_pins: pwm-vs { + pwm_vs_y_pins: pwm-vs-y { + mux { + groups = "pwm_vs_y"; + function = "pwm_vs"; + bias-disable; + }; + }; + + pwm_vs_h_pins: pwm-vs-h { mux { - groups = "pwm_vs"; + groups = "pwm_vs_h"; function = "pwm_vs"; bias-disable; }; diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts b/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts index 285c6ac1dd613..2baf210a2a40e 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts +++ b/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts @@ -431,6 +431,10 @@ }; }; +&mipi_pcie_analog_dphy { + status = "okay"; +}; + &pdm { pinctrl-0 = <&pdm_dclk_a14_pins>, <&pdm_din0_pins>, <&pdm_din1_pins>, <&pdm_din2_pins>, <&pdm_din3_pins>; diff --git a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi index f1f53fd98ae25..b7a7f4fae7dc2 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi @@ -481,7 +481,8 @@ "nand_ale", "nand_cle", "nand_wen_clk", - "nand_ren_wr"; + "nand_ren_wr", + "nand_rb0"; function = "nand"; input-enable; bias-pull-up; diff --git a/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi b/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi index 5e4233ccfde46..f473d81f67ffa 100644 --- a/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi @@ -124,10 +124,10 @@ audio_subsys: bus@59000000 { compatible = "fsl,imx8qm-asrc"; reg = <0x59000000 0x10000>; interrupts = ; - clocks = <&asrc0_lpcg IMX_LPCG_CLK_0>, - <&asrc0_lpcg IMX_LPCG_CLK_0>, - <&aud_pll_div0_lpcg IMX_LPCG_CLK_4>, - <&aud_pll_div1_lpcg IMX_LPCG_CLK_4>, + clocks = <&asrc0_lpcg IMX_LPCG_CLK_4>, + <&asrc0_lpcg IMX_LPCG_CLK_4>, + <&aud_pll_div0_lpcg IMX_LPCG_CLK_0>, + <&aud_pll_div1_lpcg IMX_LPCG_CLK_0>, <&acm IMX_ADMA_ACM_AUD_CLK0_SEL>, <&acm IMX_ADMA_ACM_AUD_CLK1_SEL>, <&clk_dummy>, diff --git a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts index 443e4fd5b9bfc..285bf79864ebd 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts @@ -775,6 +775,10 @@ status = "okay"; }; +&mu2 { + status = "okay"; +}; + &pwm1 { pinctrl-0 = <&pinctrl_pwm1>; pinctrl-names = "default"; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts b/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts index 5fb9714215bfe..f43330d1ff8b6 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts @@ -562,6 +562,8 @@ pinctrl_hdmi: hdmigrp { fsl,pins = < + MX8MP_IOMUXC_HDMI_DDC_SCL__HDMIMIX_HDMI_SCL 0x1c2 + MX8MP_IOMUXC_HDMI_DDC_SDA__HDMIMIX_HDMI_SDA 0x1c2 MX8MP_IOMUXC_HDMI_CEC__HDMIMIX_HDMI_CEC 0x10 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi index c79b1df339db1..f881912cde460 100644 --- a/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi +++ b/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi @@ -15,9 +15,9 @@ rtc1 = &bbnsm_rtc; }; - memory@40000000 { + memory@80000000 { device_type = "memory"; - reg = <0x0 0x40000000 0 0x80000000>; + reg = <0x0 0x80000000 0 0x40000000>; }; chosen { diff --git a/arch/arm64/boot/dts/freescale/imx94.dtsi b/arch/arm64/boot/dts/freescale/imx94.dtsi index a6cb5a6e848b3..1f9035e6cf159 100644 --- a/arch/arm64/boot/dts/freescale/imx94.dtsi +++ b/arch/arm64/boot/dts/freescale/imx94.dtsi @@ -1374,8 +1374,9 @@ <0 0x4c360000 0 0x10000>, <0 0x4c340000 0 0x4000>; reg-names = "dbi", "config", "atu", "app"; - ranges = <0x81000000 0x0 0x00000000 0x0 0x6ff00000 0 0x00100000>, - <0x82000000 0x0 0x10000000 0x9 0x10000000 0 0x80000000>; + ranges = <0x43000000 0x9 0x00000000 0x9 0x00000000 0x0 0xe0000000>, + <0x82000000 0x0 0xe0000000 0x9 0xe0000000 0x0 0x10000000>, + <0x81000000 0x0 0x00000000 0x9 0xf0000000 0x0 0x00100000>; #address-cells = <3>; #size-cells = <2>; device_type = "pci"; diff --git a/arch/arm64/boot/dts/freescale/imx943.dtsi b/arch/arm64/boot/dts/freescale/imx943.dtsi index ed030d4bc7bd9..cf5b3dbb47ff7 100644 --- a/arch/arm64/boot/dts/freescale/imx943.dtsi +++ b/arch/arm64/boot/dts/freescale/imx943.dtsi @@ -218,8 +218,9 @@ <0 0x4c3e0000 0 0x10000>, <0 0x4c3c0000 0 0x4000>; reg-names = "dbi", "config", "atu", "app"; - ranges = <0x81000000 0 0x00000000 0x8 0x8ff00000 0 0x00100000>, - <0x82000000 0 0x10000000 0xa 0x10000000 0 0x80000000>; + ranges = <0x43000000 0xa 0x00000000 0xa 0x00000000 0x0 0xe0000000>, + <0x82000000 0x0 0xe0000000 0xa 0xe0000000 0x0 0x10000000>, + <0x81000000 0x0 0x00000000 0xa 0xf0000000 0x0 0x00100000>; #address-cells = <3>; #size-cells = <2>; device_type = "pci"; diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi index 7d760470201fa..a6c5398a81e3c 100644 --- a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi +++ b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi @@ -24,6 +24,7 @@ serial1 = &lpuart1; serial2 = &lpuart6; serial3 = &lpuart3; + serial4 = &lpuart5; }; chosen { diff --git a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts index f4d73c8b1a6d3..bd00e7df87c67 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts +++ b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts @@ -290,7 +290,8 @@ &usb3 { status = "okay"; - phys = <&comphy2 0>; + phys = <&usb2_utmi_otg_phy>, <&comphy2 0>; + phy-names = "usb2-phy", "usb3-phy"; }; &mdio { diff --git a/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi b/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi index 9f5a0ec563e8c..8256279c56299 100644 --- a/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi @@ -265,8 +265,6 @@ }; &disp_dsi0 { - #address-cells = <1>; - #size-cells = <0>; status = "okay"; ports { @@ -1049,8 +1047,6 @@ ethernet@1 { compatible = "usb424,7850"; reg = <1>; - #address-cells = <1>; - #size-cells = <0>; mdio { #address-cells = <1>; diff --git a/arch/arm64/boot/dts/nvidia/tegra264.dtsi b/arch/arm64/boot/dts/nvidia/tegra264.dtsi index 2d2cb1a3d95cb..bcbf0a558d8db 100644 --- a/arch/arm64/boot/dts/nvidia/tegra264.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra264.dtsi @@ -3393,7 +3393,7 @@ cmdqv1: cmdqv@5200000 { compatible = "nvidia,tegra264-cmdqv"; reg = <0x00 0x5200000 0x0 0x830000>; - interrupts = ; + interrupts = ; status = "disabled"; }; @@ -3413,7 +3413,7 @@ cmdqv2: cmdqv@6200000 { compatible = "nvidia,tegra264-cmdqv"; reg = <0x00 0x6200000 0x0 0x830000>; - interrupts = ; + interrupts = ; status = "disabled"; }; @@ -3486,7 +3486,7 @@ cmdqv0: cmdqv@a200000 { compatible = "nvidia,tegra264-cmdqv"; reg = <0x00 0xa200000 0x0 0x830000>; - interrupts = ; + interrupts = ; status = "disabled"; }; @@ -3506,7 +3506,7 @@ cmdqv4: cmdqv@b200000 { compatible = "nvidia,tegra264-cmdqv"; reg = <0x00 0xb200000 0x0 0x830000>; - interrupts = ; + interrupts = ; status = "disabled"; }; @@ -3831,7 +3831,7 @@ cmdqv3: cmdqv@6200000 { compatible = "nvidia,tegra264-cmdqv"; reg = <0x00 0x6200000 0x0 0x830000>; - interrupts = ; + interrupts = ; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/qcom/agatti.dtsi b/arch/arm64/boot/dts/qcom/agatti.dtsi index f0b6ae9b81528..590bd2432d85b 100644 --- a/arch/arm64/boot/dts/qcom/agatti.dtsi +++ b/arch/arm64/boot/dts/qcom/agatti.dtsi @@ -2197,6 +2197,7 @@ "gcc_disp_gpll0_div_clk_src", "dsi0_phy_pll_out_byteclk", "dsi0_phy_pll_out_dsiclk"; + power-domains = <&rpmpd QCM2290_VDDCX>; #power-domain-cells = <1>; #clock-cells = <1>; #reset-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/eliza-mtp.dts b/arch/arm64/boot/dts/qcom/eliza-mtp.dts index 1374afd9d14e3..9d58ef20213a0 100644 --- a/arch/arm64/boot/dts/qcom/eliza-mtp.dts +++ b/arch/arm64/boot/dts/qcom/eliza-mtp.dts @@ -535,8 +535,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3g>; - vdda-pll-supply = <&vreg_l7k>; + vdda-phy-supply = <&vreg_l7k>; + vdda-pll-supply = <&vreg_l3g>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/eliza.dtsi b/arch/arm64/boot/dts/qcom/eliza.dtsi index 72b786fec195f..8fa69be150508 100644 --- a/arch/arm64/boot/dts/qcom/eliza.dtsi +++ b/arch/arm64/boot/dts/qcom/eliza.dtsi @@ -1232,8 +1232,6 @@ #size-cells = <2>; ranges; - status = "disabled"; - i2c0: i2c@a80000 { compatible = "qcom,geni-i2c"; reg = <0x0 0x00a80000 0x0 0x4000>; @@ -1969,7 +1967,7 @@ }; remoteproc_adsp: remoteproc@3000000 { - compatible = "qcom,eliza-adsp-pas"; + compatible = "qcom,eliza-adsp-pas", "qcom,sm8550-adsp-pas"; reg = <0x0 0x03000000 0x0 0x10000>; interrupts-extended = <&pdc 6 IRQ_TYPE_EDGE_RISING>, diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dts b/arch/arm64/boot/dts/qcom/glymur-crd.dts index c98dfb3941fa3..6125617de82a6 100644 --- a/arch/arm64/boot/dts/qcom/glymur-crd.dts +++ b/arch/arm64/boot/dts/qcom/glymur-crd.dts @@ -8,198 +8,9 @@ #include "glymur.dtsi" #include "glymur-crd.dtsi" -#include - / { model = "Qualcomm Technologies, Inc. Glymur CRD"; compatible = "qcom,glymur-crd", "qcom,glymur"; - - pmic-glink { - compatible = "qcom,glymur-pmic-glink", - "qcom,pmic-glink"; - #address-cells = <1>; - #size-cells = <0>; - - connector@0 { - compatible = "usb-c-connector"; - reg = <0>; - power-role = "dual"; - data-role = "dual"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - pmic_glink_hs_in: endpoint { - remote-endpoint = <&usb_0_dwc3_hs>; - }; - }; - - port@1 { - reg = <1>; - - pmic_glink_ss_in: endpoint { - remote-endpoint = <&usb_0_qmpphy_out>; - }; - }; - }; - }; - - connector@1 { - compatible = "usb-c-connector"; - reg = <1>; - power-role = "dual"; - data-role = "dual"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - pmic_glink_hs_in1: endpoint { - remote-endpoint = <&usb_1_dwc3_hs>; - }; - }; - - port@1 { - reg = <1>; - - pmic_glink_ss_in1: endpoint { - remote-endpoint = <&usb_1_qmpphy_out>; - }; - }; - }; - }; - }; - - vreg_edp_3p3: regulator-edp-3p3 { - compatible = "regulator-fixed"; - - regulator-name = "VREG_EDP_3P3"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - - gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>; - enable-active-high; - - pinctrl-0 = <&edp_reg_en>; - pinctrl-names = "default"; - - regulator-boot-on; - }; - - vreg_misc_3p3: regulator-misc-3p3 { - compatible = "regulator-fixed"; - - regulator-name = "VREG_MISC_3P3"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - - gpio = <&pmh0110_f_e0_gpios 6 GPIO_ACTIVE_HIGH>; - enable-active-high; - - pinctrl-0 = <&misc_3p3_reg_en>; - pinctrl-names = "default"; - - regulator-boot-on; - }; -}; - -&i2c0 { - clock-frequency = <400000>; - - status = "okay"; - - touchpad@2c { - compatible = "hid-over-i2c"; - reg = <0x2c>; - - hid-descr-addr = <0x20>; - interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>; - - vdd-supply = <&vreg_misc_3p3>; - vddl-supply = <&vreg_l15b_e0_1p8>; - - pinctrl-0 = <&tpad_default>; - pinctrl-names = "default"; - - wakeup-source; - }; - - keyboard@3a { - compatible = "hid-over-i2c"; - reg = <0x3a>; - - hid-descr-addr = <0x1>; - interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>; - - vdd-supply = <&vreg_misc_3p3>; - vddl-supply = <&vreg_l15b_e0_1p8>; - - pinctrl-0 = <&kybd_default>; - pinctrl-names = "default"; - - wakeup-source; - }; -}; - -&i2c8 { - clock-frequency = <400000>; - - status = "okay"; - - touchscreen@38 { - compatible = "hid-over-i2c"; - reg = <0x38>; - - hid-descr-addr = <0x1>; - interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>; - - vdd-supply = <&vreg_misc_3p3>; - vddl-supply = <&vreg_l15b_e0_1p8>; - - pinctrl-0 = <&ts0_default>; - pinctrl-names = "default"; - }; -}; - -&i2c5 { - clock-frequency = <400000>; - - status = "okay"; - - ptn3222_0: redriver@43 { - compatible = "nxp,ptn3222"; - reg = <0x43>; - - reset-gpios = <&tlmm 8 GPIO_ACTIVE_LOW>; - - vdd3v3-supply = <&vreg_l8b_e0_1p50>; - vdd1v8-supply = <&vreg_l15b_e0_1p8>; - - #phy-cells = <0>; - }; - - ptn3222_1: redriver@47 { - compatible = "nxp,ptn3222"; - reg = <0x47>; - - reset-gpios = <&tlmm 9 GPIO_ACTIVE_LOW>; - - vdd3v3-supply = <&vreg_l8b_e0_1p50>; - vdd1v8-supply = <&vreg_l15b_e0_1p8>; - - #phy-cells = <0>; - }; -}; - -&mdss { - status = "okay"; }; &mdss_dp0 { @@ -217,213 +28,3 @@ &mdss_dp1_out { link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; }; - -&mdss_dp3 { - /delete-property/ #sound-dai-cells; - - status = "okay"; - - aux-bus { - panel { - compatible = "samsung,atna60cl08", "samsung,atna33xc20"; - enable-gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>; - power-supply = <&vreg_edp_3p3>; - - pinctrl-0 = <&edp_bl_en>; - pinctrl-names = "default"; - - port { - edp_panel_in: endpoint { - remote-endpoint = <&mdss_dp3_out>; - }; - }; - }; - }; -}; - -&mdss_dp3_out { - data-lanes = <0 1 2 3>; - link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; - - remote-endpoint = <&edp_panel_in>; -}; - -&mdss_dp3_phy { - vdda-phy-supply = <&vreg_l2f_e1_0p83>; - vdda-pll-supply = <&vreg_l4f_e1_1p08>; - - status = "okay"; -}; - -&pmh0110_f_e0_gpios { - misc_3p3_reg_en: misc-3p3-reg-en-state { - pins = "gpio6"; - function = "normal"; - bias-disable; - input-disable; - output-enable; - drive-push-pull; - power-source = <1>; /* 1.8 V */ - qcom,drive-strength = ; - }; -}; - -&smb2370_j_e2_eusb2_repeater { - vdd18-supply = <&vreg_l15b_e0_1p8>; - vdd3-supply = <&vreg_l7b_e0_2p79>; -}; - -&smb2370_k_e2_eusb2_repeater { - vdd18-supply = <&vreg_l15b_e0_1p8>; - vdd3-supply = <&vreg_l7b_e0_2p79>; -}; - -&tlmm { - edp_bl_en: edp-bl-en-state { - pins = "gpio18"; - function = "gpio"; - drive-strength = <16>; - bias-disable; - }; - - edp_reg_en: edp-reg-en-state { - pins = "gpio70"; - function = "gpio"; - drive-strength = <16>; - bias-disable; - }; - - kybd_default: kybd-default-state { - pins = "gpio67"; - function = "gpio"; - bias-disable; - }; - - tpad_default: tpad-default-state { - pins = "gpio3"; - function = "gpio"; - bias-disable; - }; - - ts0_default: ts0-default-state { - int-n-pins { - pins = "gpio51"; - function = "gpio"; - bias-disable; - }; - - reset-n-pins { - pins = "gpio48"; - function = "gpio"; - drive-strength = <16>; - bias-disable; - }; - }; -}; - -&usb_0 { - status = "okay"; -}; - -&usb_0_dwc3_hs { - remote-endpoint = <&pmic_glink_hs_in>; -}; - -&usb_0_hsphy { - vdd-supply = <&vreg_l3f_e0_0p72>; - vdda12-supply = <&vreg_l4h_e0_1p2>; - - phys = <&smb2370_j_e2_eusb2_repeater>; - - status = "okay"; -}; - -&usb_0_qmpphy { - vdda-phy-supply = <&vreg_l4h_e0_1p2>; - vdda-pll-supply = <&vreg_l3f_e0_0p72>; - refgen-supply = <&vreg_l2f_e0_0p82>; - - status = "okay"; -}; - -&usb_0_qmpphy_out { - remote-endpoint = <&pmic_glink_ss_in>; -}; - -&usb_1 { - status = "okay"; -}; - -&usb_1_dwc3_hs { - remote-endpoint = <&pmic_glink_hs_in1>; -}; - -&usb_1_hsphy { - vdd-supply = <&vreg_l3f_e0_0p72>; - vdda12-supply = <&vreg_l4h_e0_1p2>; - - phys = <&smb2370_k_e2_eusb2_repeater>; - - status = "okay"; -}; - -&usb_1_qmpphy { - vdda-phy-supply = <&vreg_l4h_e0_1p2>; - vdda-pll-supply = <&vreg_l1h_e0_0p89>; - refgen-supply = <&vreg_l2f_e0_0p82>; - - status = "okay"; -}; - -&usb_1_qmpphy_out { - remote-endpoint = <&pmic_glink_ss_in1>; -}; - -&usb_hs { - status = "okay"; -}; - -&usb_hs_phy { - vdd-supply = <&vreg_l2h_e0_0p72>; - vdda12-supply = <&vreg_l4h_e0_1p2>; - - phys = <&ptn3222_1>; - - status = "okay"; -}; - -&usb_mp { - status = "okay"; -}; - -&usb_mp_hsphy0 { - vdd-supply = <&vreg_l2h_e0_0p72>; - vdda12-supply = <&vreg_l4h_e0_1p2>; - - phys = <&ptn3222_0>; - - status = "okay"; -}; - -&usb_mp_hsphy1 { - vdd-supply = <&vreg_l2h_e0_0p72>; - vdda12-supply = <&vreg_l4h_e0_1p2>; - - status = "okay"; -}; - -&usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l4h_e0_1p2>; - vdda-pll-supply = <&vreg_l2h_e0_0p72>; - refgen-supply = <&vreg_l4f_e1_1p08>; - - status = "okay"; -}; - -&usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l4h_e0_1p2>; - vdda-pll-supply = <&vreg_l2h_e0_0p72>; - refgen-supply = <&vreg_l4f_e1_1p08>; - - status = "okay"; -}; diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi index e784b538f42e1..79548f6f03f3f 100644 --- a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi +++ b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi @@ -11,6 +11,7 @@ #include "smb2370.dtsi" /* SPMI2: SID-9/10/11 */ #include +#include / { model = "Qualcomm Technologies, Inc. Glymur CRD"; @@ -22,6 +23,7 @@ i2c0 = &i2c0; i2c1 = &i2c4; i2c2 = &i2c5; + i2c3 = &i2c9; spi0 = &spi18; }; @@ -68,6 +70,101 @@ }; }; + pmic-glink { + compatible = "qcom,glymur-pmic-glink", + "qcom,pmic-glink"; + #address-cells = <1>; + #size-cells = <0>; + + connector@0 { + compatible = "usb-c-connector"; + reg = <0>; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + pmic_glink_hs_in: endpoint { + remote-endpoint = <&usb_0_dwc3_hs>; + }; + }; + + port@1 { + reg = <1>; + + pmic_glink_ss_in: endpoint { + remote-endpoint = <&usb_0_qmpphy_out>; + }; + }; + }; + }; + + connector@1 { + compatible = "usb-c-connector"; + reg = <1>; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + pmic_glink_hs_in1: endpoint { + remote-endpoint = <&usb_1_dwc3_hs>; + }; + }; + + port@1 { + reg = <1>; + + pmic_glink_ss_in1: endpoint { + remote-endpoint = <&usb_1_qmpphy_out>; + }; + }; + }; + }; + }; + + vreg_edp_3p3: regulator-edp-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_EDP_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&edp_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_misc_3p3: regulator-misc-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_MISC_3P3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmh0110_f_e0_gpios 6 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&misc_3p3_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + vreg_nvme: regulator-nvme { compatible = "regulator-fixed"; @@ -444,6 +541,152 @@ }; }; +&i2c9 { + clock-frequency = <400000>; + + status = "okay"; + + embedded-controller@76 { + compatible = "qcom,glymur-crd-ec", "qcom,hamoa-crd-ec"; + reg = <0x76>; + + interrupts-extended = <&tlmm 66 IRQ_TYPE_EDGE_FALLING>; + + pinctrl-0 = <&ec_int_n_default>; + pinctrl-names = "default"; + }; +}; + +&i2c0 { + clock-frequency = <400000>; + + status = "okay"; + + touchpad@2c { + compatible = "hid-over-i2c"; + reg = <0x2c>; + + hid-descr-addr = <0x20>; + interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>; + + vdd-supply = <&vreg_misc_3p3>; + vddl-supply = <&vreg_l15b_e0_1p8>; + + pinctrl-0 = <&tpad_default>; + pinctrl-names = "default"; + + wakeup-source; + }; + + keyboard@3a { + compatible = "hid-over-i2c"; + reg = <0x3a>; + + hid-descr-addr = <0x1>; + interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>; + + vdd-supply = <&vreg_misc_3p3>; + vddl-supply = <&vreg_l15b_e0_1p8>; + + pinctrl-0 = <&kybd_default>; + pinctrl-names = "default"; + + wakeup-source; + }; +}; + +&i2c5 { + clock-frequency = <400000>; + + status = "okay"; + + ptn3222_0: redriver@43 { + compatible = "nxp,ptn3222"; + reg = <0x43>; + + reset-gpios = <&tlmm 8 GPIO_ACTIVE_LOW>; + + vdd3v3-supply = <&vreg_l8b_e0_1p50>; + vdd1v8-supply = <&vreg_l15b_e0_1p8>; + + #phy-cells = <0>; + }; + + ptn3222_1: redriver@47 { + compatible = "nxp,ptn3222"; + reg = <0x47>; + + reset-gpios = <&tlmm 9 GPIO_ACTIVE_LOW>; + + vdd3v3-supply = <&vreg_l8b_e0_1p50>; + vdd1v8-supply = <&vreg_l15b_e0_1p8>; + + #phy-cells = <0>; + }; +}; + +&i2c8 { + clock-frequency = <400000>; + + status = "okay"; + + touchscreen@38 { + compatible = "focaltech,ft3d81", "focaltech,ft8112"; + reg = <0x38>; + + interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>; + + vcc33-supply = <&vreg_misc_3p3>; + vccio-supply = <&vreg_l15b_e0_1p8>; + + reset-gpios = <&tlmm 48 GPIO_ACTIVE_LOW>; + + pinctrl-0 = <&ts0_default>; + pinctrl-names = "default"; + }; +}; + +&mdss { + status = "okay"; +}; + +&mdss_dp3 { + /delete-property/ #sound-dai-cells; + + status = "okay"; + + aux-bus { + panel { + compatible = "samsung,atna60cl08", "samsung,atna33xc20"; + enable-gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>; + power-supply = <&vreg_edp_3p3>; + + pinctrl-0 = <&edp_bl_en>; + pinctrl-names = "default"; + + port { + edp_panel_in: endpoint { + remote-endpoint = <&mdss_dp3_out>; + }; + }; + }; + }; +}; + +&mdss_dp3_out { + data-lanes = <0 1 2 3>; + link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>; + + remote-endpoint = <&edp_panel_in>; +}; + +&mdss_dp3_phy { + vdda-phy-supply = <&vreg_l2f_e1_0p83>; + vdda-pll-supply = <&vreg_l4f_e1_1p08>; + + status = "okay"; +}; + &pcie3b { vddpe-3v3-supply = <&vreg_nvmesec>; @@ -537,14 +780,6 @@ wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>; }; -&pmh0101_gpios { - nvme_reg_en: nvme-reg-en-state { - pins = "gpio14"; - function = "normal"; - bias-disable; - }; -}; - &pmh0110_f_e1_gpios { nvme_sec_reg_en: nvme-reg-en-state { pins = "gpio14"; @@ -560,6 +795,25 @@ output-disable; bias-pull-up; }; + + nvme_reg_en: nvme-reg-en-state { + pins = "gpio14"; + function = "normal"; + bias-disable; + }; +}; + +&pmh0110_f_e0_gpios { + misc_3p3_reg_en: misc-3p3-reg-en-state { + pins = "gpio6"; + function = "normal"; + bias-disable; + input-disable; + output-enable; + drive-push-pull; + power-source = <1>; /* 1.8 V */ + qcom,drive-strength = ; + }; }; &pmk8850_rtc { @@ -585,17 +839,53 @@ status = "okay"; }; +&smb2370_j_e2_eusb2_repeater { + vdd18-supply = <&vreg_l15b_e0_1p8>; + vdd3-supply = <&vreg_l7b_e0_2p79>; +}; + +&smb2370_k_e2_eusb2_repeater { + vdd18-supply = <&vreg_l15b_e0_1p8>; + vdd3-supply = <&vreg_l7b_e0_2p79>; +}; + &tlmm { gpio-reserved-ranges = <4 4>, /* EC TZ Secure I3C */ <10 2>, /* OOB UART */ <44 4>; /* Security SPI (TPM) */ + edp_bl_en: edp-bl-en-state { + pins = "gpio18"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + + edp_reg_en: edp-reg-en-state { + pins = "gpio70"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + hall_int_n_default: hall-int-n-state { pins = "gpio92"; function = "gpio"; bias-disable; }; + ec_int_n_default: ec-int-n-state { + pins = "gpio66"; + function = "gpio"; + bias-disable; + }; + + kybd_default: kybd-default-state { + pins = "gpio67"; + function = "gpio"; + bias-disable; + }; + pcie4_default: pcie4-default-state { clkreq-n-pins { pins = "gpio147"; @@ -688,6 +978,27 @@ }; }; + tpad_default: tpad-default-state { + pins = "gpio3"; + function = "gpio"; + bias-disable; + }; + + ts0_default: ts0-default-state { + int-n-pins { + pins = "gpio51"; + function = "gpio"; + bias-disable; + }; + + reset-n-pins { + pins = "gpio48"; + function = "gpio"; + drive-strength = <16>; + bias-disable; + }; + }; + wcn_wlan_bt_en: wcn-wlan-bt-en-state { pins = "gpio116", "gpio117"; function = "gpio"; @@ -726,3 +1037,110 @@ vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>; }; }; + +&usb_0 { + status = "okay"; +}; + +&usb_0_dwc3_hs { + remote-endpoint = <&pmic_glink_hs_in>; +}; + +&usb_0_hsphy { + vdd-supply = <&vreg_l3f_e0_0p72>; + vdda12-supply = <&vreg_l4h_e0_1p2>; + + phys = <&smb2370_j_e2_eusb2_repeater>; + + status = "okay"; +}; + +&usb_0_qmpphy { + vdda-phy-supply = <&vreg_l3f_e0_0p72>; + vdda-pll-supply = <&vreg_l4h_e0_1p2>; + refgen-supply = <&vreg_l2f_e0_0p82>; + + status = "okay"; +}; + +&usb_0_qmpphy_out { + remote-endpoint = <&pmic_glink_ss_in>; +}; + +&usb_1 { + status = "okay"; +}; + +&usb_1_dwc3_hs { + remote-endpoint = <&pmic_glink_hs_in1>; +}; + +&usb_1_hsphy { + vdd-supply = <&vreg_l3f_e0_0p72>; + vdda12-supply = <&vreg_l4h_e0_1p2>; + + phys = <&smb2370_k_e2_eusb2_repeater>; + + status = "okay"; +}; + +&usb_1_qmpphy { + vdda-phy-supply = <&vreg_l1h_e0_0p89>; + vdda-pll-supply = <&vreg_l4h_e0_1p2>; + refgen-supply = <&vreg_l2f_e0_0p82>; + + status = "okay"; +}; + +&usb_1_qmpphy_out { + remote-endpoint = <&pmic_glink_ss_in1>; +}; + +&usb_hs { + status = "okay"; +}; + +&usb_hs_phy { + vdd-supply = <&vreg_l2h_e0_0p72>; + vdda12-supply = <&vreg_l4h_e0_1p2>; + + phys = <&ptn3222_1>; + + status = "okay"; +}; + +&usb_mp { + status = "okay"; +}; + +&usb_mp_hsphy0 { + vdd-supply = <&vreg_l2h_e0_0p72>; + vdda12-supply = <&vreg_l4h_e0_1p2>; + + phys = <&ptn3222_0>; + + status = "okay"; +}; + +&usb_mp_hsphy1 { + vdd-supply = <&vreg_l2h_e0_0p72>; + vdda12-supply = <&vreg_l4h_e0_1p2>; + + status = "okay"; +}; + +&usb_mp_qmpphy0 { + vdda-phy-supply = <&vreg_l2h_e0_0p72>; + vdda-pll-supply = <&vreg_l4h_e0_1p2>; + refgen-supply = <&vreg_l4f_e1_1p08>; + + status = "okay"; +}; + +&usb_mp_qmpphy1 { + vdda-phy-supply = <&vreg_l2h_e0_0p72>; + vdda-pll-supply = <&vreg_l4h_e0_1p2>; + refgen-supply = <&vreg_l4f_e1_1p08>; + + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/glymur.dtsi b/arch/arm64/boot/dts/qcom/glymur.dtsi index 129be417ac424..16f5185474811 100644 --- a/arch/arm64/boot/dts/qcom/glymur.dtsi +++ b/arch/arm64/boot/dts/qcom/glymur.dtsi @@ -602,13 +602,13 @@ no-map; }; - soccpdtb_mem: soccpdtb@892e0000 { - reg = <0x0 0x892e0000 0x0 0x20000>; + soccp_mem: soccp@88e00000 { + reg = <0x0 0x88e00000 0x0 0x400000>; no-map; }; - soccp_mem: soccp@89300000 { - reg = <0x0 0x89300000 0x0 0x400000>; + soccpdtb_mem: soccpdtb@89200000 { + reg = <0x0 0x89200000 0x0 0x20000>; no-map; }; @@ -786,8 +786,8 @@ <&usb_0_qmpphy QMP_USB43DP_USB3_PIPE_CLK>, <&usb_1_qmpphy QMP_USB43DP_USB3_PIPE_CLK>, <&usb_2_qmpphy QMP_USB43DP_USB3_PIPE_CLK>, - <&usb_mp_qmpphy0 QMP_USB43DP_USB3_PIPE_CLK>, - <&usb_mp_qmpphy1 QMP_USB43DP_USB3_PIPE_CLK>, + <&usb_mp_qmpphy0>, /* USB3 UNI PHY pipe 0 */ + <&usb_mp_qmpphy1>, /* USB3 UNI PHY pipe 1 */ <0>, /* USB4 PHY 0 pcie pipe */ <0>, /* USB4 PHY 0 Max pipe */ <0>, /* USB4 PHY 1 pcie pipe */ @@ -797,6 +797,7 @@ #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; + power-domains = <&rpmhpd RPMHPD_CX>; }; gpi_dma2: dma-controller@800000 { @@ -4835,9 +4836,13 @@ pdc: interrupt-controller@b220000 { compatible = "qcom,glymur-pdc", "qcom,pdc"; reg = <0x0 0x0b220000 0x0 0x10000>; - qcom,pdc-ranges = <0 745 51>, - <51 527 47>, - <98 609 32>, + qcom,pdc-ranges = <0 745 38>, + <40 785 11>, + <51 527 4>, + <57 533 10>, + <70 546 4>, + <75 551 18>, + <108 619 22>, <130 717 12>, <142 251 5>, <147 796 16>, @@ -5029,7 +5034,7 @@ #size-cells = <0>; }; - spmi_bus2: spmi@c48000 { + spmi_bus2: spmi@c448000 { reg = <0x0 0x0c448000 0x0 0x4000>, <0x0 0x0c8e0000 0x0 0x10000>, <0x0 0x0c44c000 0x0 0x8000>; diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi index 9c5e77df00547..bc70fe5db96f5 100644 --- a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi @@ -590,8 +590,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -611,8 +611,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -632,8 +632,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -670,15 +670,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts b/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts index bfb7cea56df96..2848250083891 100644 --- a/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts +++ b/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts @@ -1101,8 +1101,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j>; - vdda-pll-supply = <&vreg_l1j>; + vdda-phy-supply = <&vreg_l1j>; + vdda-pll-supply = <&vreg_l2j>; status = "okay"; }; @@ -1112,8 +1112,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j>; - vdda-pll-supply = <&vreg_l2d>; + vdda-phy-supply = <&vreg_l2d>; + vdda-pll-supply = <&vreg_l2j>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1138,8 +1138,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j>; - vdda-pll-supply = <&vreg_l2d>; + vdda-phy-supply = <&vreg_l2d>; + vdda-pll-supply = <&vreg_l2j>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1186,15 +1186,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e>; - vdda-pll-supply = <&vreg_l3c>; + vdda-phy-supply = <&vreg_l3c>; + vdda-pll-supply = <&vreg_l3e>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e>; - vdda-pll-supply = <&vreg_l3c>; + vdda-phy-supply = <&vreg_l3c>; + vdda-pll-supply = <&vreg_l3e>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi index 4ba751a65142b..fd86f4761eab9 100644 --- a/arch/arm64/boot/dts/qcom/hamoa.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi @@ -2868,7 +2868,7 @@ reg = <0 0x00fd3000 0 0x154>; #phy-cells = <0>; - clocks = <&tcsr TCSR_USB2_1_CLKREF_EN>; + clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "ref"; resets = <&gcc GCC_QUSB2PHY_PRIM_BCR>; @@ -3010,7 +3010,7 @@ reg = <0 0x00fde000 0 0x154>; #phy-cells = <0>; - clocks = <&tcsr TCSR_USB2_1_CLKREF_EN>; + clocks = <&tcsr TCSR_USB2_2_CLKREF_EN>; clock-names = "ref"; resets = <&gcc GCC_QUSB2PHY_TERT_BCR>; diff --git a/arch/arm64/boot/dts/qcom/ipq5018.dtsi b/arch/arm64/boot/dts/qcom/ipq5018.dtsi index 6f8004a22a1ff..52fc2d266b44b 100644 --- a/arch/arm64/boot/dts/qcom/ipq5018.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq5018.dtsi @@ -229,6 +229,9 @@ compatible = "ethernet-phy-id004d.d0c0"; reg = <7>; + clocks = <&gcc GCC_GEPHY_RX_CLK>, + <&gcc GCC_GEPHY_TX_CLK>; + clock-names = "rx", "tx"; resets = <&gcc GCC_GEPHY_MISC_ARES>; }; }; @@ -256,7 +259,7 @@ "sys"; #clock-cells = <1>; assigned-clocks = <&cmn_pll IPQ5018_CMN_PLL_CLK>; - assigned-clock-rates-u64 = /bits/ 64 <9600000000>; + assigned-clock-rates-u64 = /bits/ 64 <4800000000>; }; qfprom: qfprom@a0000 { diff --git a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts index f9b5b5718b904..2c1ae40dc7ec9 100644 --- a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts +++ b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts @@ -1343,8 +1343,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l1d_1p2>; - vdda-pll-supply = <&vreg_l4f_0p8>; + vdda-phy-supply = <&vreg_l4f_0p8>; + vdda-pll-supply = <&vreg_l1d_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts index 55d02219ef4e9..28779698221d2 100644 --- a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts +++ b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts @@ -838,8 +838,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l1d_1p2>; - vdda-pll-supply = <&vreg_l4f_0p8>; + vdda-phy-supply = <&vreg_l4f_0p8>; + vdda-pll-supply = <&vreg_l1d_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/kaanapali.dtsi b/arch/arm64/boot/dts/qcom/kaanapali.dtsi index 7aa9653bd456e..011fb00beb0ea 100644 --- a/arch/arm64/boot/dts/qcom/kaanapali.dtsi +++ b/arch/arm64/boot/dts/qcom/kaanapali.dtsi @@ -2310,8 +2310,8 @@ operating-points-v2 = <&pcie0_opp_table>; - iommu-map = <0 &apps_smmu 0x1400 0x1>, - <0x100 &apps_smmu 0x1401 0x1>; + iommu-map = <0 &apps_smmu 0x1400 0x0 0x1>, + <0x100 &apps_smmu 0x1401 0x0 0x1>; interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>, <0 0 0 2 &intc 0 0 GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>, diff --git a/arch/arm64/boot/dts/qcom/kodiak.dtsi b/arch/arm64/boot/dts/qcom/kodiak.dtsi index fa540d8c2615d..48639568f15e4 100644 --- a/arch/arm64/boot/dts/qcom/kodiak.dtsi +++ b/arch/arm64/boot/dts/qcom/kodiak.dtsi @@ -191,9 +191,12 @@ qcom,vmid = ; }; - adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap@9cb80000 { - reg = <0x0 0x9cb80000 0x0 0x800000>; - no-map; + adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap { + compatible = "shared-dma-pool"; + alloc-ranges = <0x0 0x80000000 0x0 0x80000000>; + reusable; + alignment = <0x0 0x400000>; + size = <0x0 0x800000>; }; }; @@ -2278,8 +2281,8 @@ "aggre0", "aggre1"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2424,8 +2427,8 @@ dma-coherent; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso b/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso index 44bd9b1a17652..22c975628ed79 100644 --- a/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso +++ b/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso @@ -204,15 +204,15 @@ }; &pcie0 { - iommu-map = <0x0 &pcie_smmu 0x0 0x1>, - <0x100 &pcie_smmu 0x1 0x1>, - <0x208 &pcie_smmu 0x2 0x1>, - <0x210 &pcie_smmu 0x3 0x1>, - <0x218 &pcie_smmu 0x4 0x1>, - <0x300 &pcie_smmu 0x5 0x1>, - <0x400 &pcie_smmu 0x6 0x1>, - <0x500 &pcie_smmu 0x7 0x1>, - <0x501 &pcie_smmu 0x8 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0 0x0 0x1>, + <0x100 &pcie_smmu 0x1 0x0 0x1>, + <0x208 &pcie_smmu 0x2 0x0 0x1>, + <0x210 &pcie_smmu 0x3 0x0 0x1>, + <0x218 &pcie_smmu 0x4 0x0 0x1>, + <0x300 &pcie_smmu 0x5 0x0 0x1>, + <0x400 &pcie_smmu 0x6 0x0 0x1>, + <0x500 &pcie_smmu 0x7 0x0 0x1>, + <0x501 &pcie_smmu 0x8 0x0 0x1>; }; &pcieport0 { diff --git a/arch/arm64/boot/dts/qcom/lemans-evk.dts b/arch/arm64/boot/dts/qcom/lemans-evk.dts index 34dfc8d22b6a5..b835fce8b336f 100644 --- a/arch/arm64/boot/dts/qcom/lemans-evk.dts +++ b/arch/arm64/boot/dts/qcom/lemans-evk.dts @@ -1008,8 +1008,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l1c>; - vdda-pll-supply = <&vreg_l7a>; + vdda-phy-supply = <&vreg_l7a>; + vdda-pll-supply = <&vreg_l1c>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi index 40f88498999bd..b2195fc030957 100644 --- a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi +++ b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi @@ -1103,8 +1103,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l1c>; - vdda-pll-supply = <&vreg_l7a>; + vdda-phy-supply = <&vreg_l7a>; + vdda-pll-supply = <&vreg_l1c>; status = "okay"; }; @@ -1126,8 +1126,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l1c>; - vdda-pll-supply = <&vreg_l7a>; + vdda-phy-supply = <&vreg_l7a>; + vdda-pll-supply = <&vreg_l1c>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/lemans.dtsi b/arch/arm64/boot/dts/qcom/lemans.dtsi index 353a6e6fd3acb..47d26dd832f7d 100644 --- a/arch/arm64/boot/dts/qcom/lemans.dtsi +++ b/arch/arm64/boot/dts/qcom/lemans.dtsi @@ -2760,8 +2760,8 @@ <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, - <0x100 &pcie_smmu 0x0001 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0000 0x0 0x1>, + <0x100 &pcie_smmu 0x0001 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>, <&gcc GCC_PCIE_0_LINK_DOWN_BCR>; @@ -2933,8 +2933,8 @@ <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &pcie_smmu 0x0080 0x1>, - <0x100 &pcie_smmu 0x0081 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0080 0x0 0x1>, + <0x100 &pcie_smmu 0x0081 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>, <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; diff --git a/arch/arm64/boot/dts/qcom/milos.dtsi b/arch/arm64/boot/dts/qcom/milos.dtsi index 8c92329889538..262fa88012562 100644 --- a/arch/arm64/boot/dts/qcom/milos.dtsi +++ b/arch/arm64/boot/dts/qcom/milos.dtsi @@ -1724,6 +1724,8 @@ iommus = <&apps_smmu 0x540 0>; + resets = <&gcc GCC_SDCC2_BCR>; + bus-width = <4>; qcom,dll-config = <0x0007442c>; diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso b/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso index e6beb4393430b..a6e57ec8a1e14 100644 --- a/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso +++ b/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso @@ -50,15 +50,15 @@ }; &pcie0 { - iommu-map = <0x0 &pcie_smmu 0x0 0x1>, - <0x100 &pcie_smmu 0x1 0x1>, - <0x208 &pcie_smmu 0x2 0x1>, - <0x210 &pcie_smmu 0x3 0x1>, - <0x218 &pcie_smmu 0x4 0x1>, - <0x300 &pcie_smmu 0x5 0x1>, - <0x400 &pcie_smmu 0x6 0x1>, - <0x500 &pcie_smmu 0x7 0x1>, - <0x501 &pcie_smmu 0x8 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0 0x0 0x1>, + <0x100 &pcie_smmu 0x1 0x0 0x1>, + <0x208 &pcie_smmu 0x2 0x0 0x1>, + <0x210 &pcie_smmu 0x3 0x0 0x1>, + <0x218 &pcie_smmu 0x4 0x0 0x1>, + <0x300 &pcie_smmu 0x5 0x0 0x1>, + <0x400 &pcie_smmu 0x6 0x0 0x1>, + <0x500 &pcie_smmu 0x7 0x0 0x1>, + <0x501 &pcie_smmu 0x8 0x0 0x1>; }; &pcieport0 { diff --git a/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi b/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi index 9b5ed55939b86..36af1ad2105d5 100644 --- a/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi +++ b/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi @@ -196,14 +196,14 @@ /* PCIe0 Gen4 x2 */ &pcie0 { - iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, - <0x100 &pcie_smmu 0x0001 0x1>, - <0x200 &pcie_smmu 0x0007 0x1>, - <0x208 &pcie_smmu 0x0002 0x1>, - <0x210 &pcie_smmu 0x0003 0x1>, - <0x218 &pcie_smmu 0x0004 0x1>, - <0x300 &pcie_smmu 0x0005 0x1>, - <0x400 &pcie_smmu 0x0006 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0000 0x0 0x1>, + <0x100 &pcie_smmu 0x0001 0x0 0x1>, + <0x200 &pcie_smmu 0x0007 0x0 0x1>, + <0x208 &pcie_smmu 0x0002 0x0 0x1>, + <0x210 &pcie_smmu 0x0003 0x0 0x1>, + <0x218 &pcie_smmu 0x0004 0x0 0x1>, + <0x300 &pcie_smmu 0x0005 0x0 0x1>, + <0x400 &pcie_smmu 0x0006 0x0 0x1>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/monaco.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi index a350a8ba48fa3..61ef690f5fdf7 100644 --- a/arch/arm64/boot/dts/qcom/monaco.dtsi +++ b/arch/arm64/boot/dts/qcom/monaco.dtsi @@ -2349,8 +2349,8 @@ interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, - <0x100 &pcie_smmu 0x0001 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0000 0x0 0x1>, + <0x100 &pcie_smmu 0x0001 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>, <&gcc GCC_PCIE_0_LINK_DOWN_BCR>; @@ -2526,8 +2526,8 @@ &config_noc SLAVE_PCIE_1 QCOM_ICC_TAG_ACTIVE_ONLY>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &pcie_smmu 0x0080 0x1>, - <0x100 &pcie_smmu 0x0081 0x1>; + iommu-map = <0x0 &pcie_smmu 0x0080 0x0 0x1>, + <0x100 &pcie_smmu 0x0081 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>, <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; @@ -3017,14 +3017,6 @@ #address-cells = <1>; #size-cells = <0>; - port@0 { - reg = <0>; - - swao_rep_out0: endpoint { - remote-endpoint = <&qdss_rep_in>; - }; - }; - port@1 { reg = <1>; @@ -3734,6 +3726,14 @@ #address-cells = <1>; #size-cells = <0>; + port@0 { + reg = <0>; + + swao_rep_out0: endpoint { + remote-endpoint = <&qdss_rep_in>; + }; + }; + port@1 { reg = <1>; diff --git a/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts b/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts index 18832a3b9a1c3..57f549f06f73c 100644 --- a/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts +++ b/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts @@ -455,14 +455,14 @@ bias-disable; }; - ts_int_default: ts-int-state { + ts_int_default: ts-int-default-state { pins = "gpio65"; function = "gpio"; drive-strength = <2>; bias-pull-down; }; - ts_int_sleep: ts-int-state { + ts_int_sleep: ts-int-sleep-state { pins = "gpio65"; function = "gpio"; drive-strength = <2>; diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts index fd3a2121465b0..ca22e2f9d20a2 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts +++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts @@ -39,7 +39,7 @@ haptics: drv2604@5a { compatible = "ti,drv2604"; reg = <0x5a>; - enable-gpio = <&tlmm 93 0x00>; + enable-gpios = <&tlmm 93 GPIO_ACTIVE_HIGH>; mode = ; library-sel = ; pinctrl-names = "default","sleep"; diff --git a/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi b/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi index 3650f2501886b..04d4741cdb5f0 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi @@ -229,10 +229,6 @@ }; }; -&blsp1_i2c5_sleep { - bias-disable; -}; - &blsp1_uart3 { status = "okay"; diff --git a/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts b/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts index 30222f6608da7..69528771eda25 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts +++ b/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts @@ -217,11 +217,6 @@ }; }; -&blsp1_i2c5_sleep { - /delete-property/ bias-pull-up; - bias-disable; -}; - &blsp1_uart3 { status = "okay"; diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi index 3477060116372..5038d0009c1a9 100644 --- a/arch/arm64/boot/dts/qcom/msm8998.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi @@ -1183,7 +1183,7 @@ pins = "gpio2", "gpio3"; function = "blsp_i2c1"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c2_default: blsp1-i2c2-default-state { @@ -1197,7 +1197,7 @@ pins = "gpio32", "gpio33"; function = "blsp_i2c2"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c3_default: blsp1-i2c3-default-state { @@ -1211,7 +1211,7 @@ pins = "gpio47", "gpio48"; function = "blsp_i2c3"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c4_default: blsp1-i2c4-default-state { @@ -1225,7 +1225,7 @@ pins = "gpio10", "gpio11"; function = "blsp_i2c4"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c5_default: blsp1-i2c5-default-state { @@ -1239,7 +1239,7 @@ pins = "gpio87", "gpio88"; function = "blsp_i2c5"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c6_default: blsp1-i2c6-default-state { @@ -1253,7 +1253,7 @@ pins = "gpio43", "gpio44"; function = "blsp_i2c6"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_spi_b_default: blsp1-spi-b-default-state { @@ -1318,7 +1318,7 @@ pins = "gpio55", "gpio56"; function = "blsp_i2c7"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c2_default: blsp2-i2c2-default-state { @@ -1332,7 +1332,7 @@ pins = "gpio6", "gpio7"; function = "blsp_i2c8"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c3_default: blsp2-i2c3-default-state { @@ -1346,7 +1346,7 @@ pins = "gpio51", "gpio52"; function = "blsp_i2c9"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c4_default: blsp2-i2c4-default-state { @@ -1360,7 +1360,7 @@ pins = "gpio67", "gpio68"; function = "blsp_i2c10"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c5_default: blsp2-i2c5-default-state { @@ -1374,7 +1374,7 @@ pins = "gpio60", "gpio61"; function = "blsp_i2c11"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c6_default: blsp2-i2c6-default-state { @@ -1388,7 +1388,7 @@ pins = "gpio83", "gpio84"; function = "blsp_i2c12"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_spi1_default: blsp2-spi1-default-state { diff --git a/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi index 394e65518ac50..50ea8a148ada0 100644 --- a/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi +++ b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi @@ -583,8 +583,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -604,8 +604,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -625,8 +625,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -663,15 +663,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts b/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts index 04cb9230d29fd..92a7d80b7fee6 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts @@ -1502,8 +1502,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b>; - vdda-pll-supply = <&vreg_l1b>; + vdda-phy-supply = <&vreg_l1b>; + vdda-pll-supply = <&vreg_l6b>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts index bdc02260f902b..58cce89a0c1ce 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts @@ -1141,8 +1141,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p912>; + vdda-phy-supply = <&vreg_l1b_0p912>; + vdda-pll-supply = <&vreg_l6b_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts b/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts index bf18c48520813..17804de03e177 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts @@ -839,8 +839,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p912>; + vdda-phy-supply = <&vreg_l1b_0p912>; + vdda-pll-supply = <&vreg_l6b_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts b/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts index 797f37596bf19..cdaacc09f56c1 100644 --- a/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts +++ b/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts @@ -976,8 +976,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b>; - vdda-pll-supply = <&vreg_l1b>; + vdda-phy-supply = <&vreg_l1b>; + vdda-pll-supply = <&vreg_l6b>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcs404.dtsi b/arch/arm64/boot/dts/qcom/qcs404.dtsi index 4328c1dda898c..736accfc34f90 100644 --- a/arch/arm64/boot/dts/qcom/qcs404.dtsi +++ b/arch/arm64/boot/dts/qcom/qcs404.dtsi @@ -677,11 +677,11 @@ assigned-clock-rates = <19200000>, <200000000>; interrupts = , - , - ; + , + ; interrupt-names = "pwr_event", - "hs_phy_irq", - "qusb2_phy"; + "qusb2_phy", + "hs_phy_irq"; status = "disabled"; @@ -716,11 +716,11 @@ assigned-clock-rates = <19200000>, <133333333>; interrupts = , - , - ; + , + ; interrupt-names = "pwr_event", - "hs_phy_irq", - "qusb2_phy"; + "qusb2_phy", + "hs_phy_irq"; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts b/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts index bb5a42b038f19..696fef50f577b 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts +++ b/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts @@ -546,15 +546,15 @@ pinctrl-names = "default"; /* Support for QPS615 PCIe switch */ - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>, - <0x208 &apps_smmu 0x1c84 0x1>, - <0x210 &apps_smmu 0x1c85 0x1>, - <0x218 &apps_smmu 0x1c86 0x1>, - <0x300 &apps_smmu 0x1c87 0x1>, - <0x400 &apps_smmu 0x1c88 0x1>, - <0x500 &apps_smmu 0x1c89 0x1>, - <0x501 &apps_smmu 0x1c90 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>, + <0x208 &apps_smmu 0x1c84 0x0 0x1>, + <0x210 &apps_smmu 0x1c85 0x0 0x1>, + <0x218 &apps_smmu 0x1c86 0x0 0x1>, + <0x300 &apps_smmu 0x1c87 0x0 0x1>, + <0x400 &apps_smmu 0x1c88 0x0 0x1>, + <0x500 &apps_smmu 0x1c89 0x0 0x1>, + <0x501 &apps_smmu 0x1c90 0x0 0x1>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso index 83908db335afa..14c64439ae1e6 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso +++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso @@ -54,15 +54,15 @@ pinctrl-0 = <&pcie0_reset_n>, <&pcie0_wake_n>, <&pcie0_clkreq_n>; pinctrl-names = "default"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>, - <0x208 &apps_smmu 0x1c04 0x1>, - <0x210 &apps_smmu 0x1c05 0x1>, - <0x218 &apps_smmu 0x1c06 0x1>, - <0x300 &apps_smmu 0x1c07 0x1>, - <0x400 &apps_smmu 0x1c08 0x1>, - <0x500 &apps_smmu 0x1c09 0x1>, - <0x501 &apps_smmu 0x1c10 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>, + <0x208 &apps_smmu 0x1c04 0x0 0x1>, + <0x210 &apps_smmu 0x1c05 0x0 0x1>, + <0x218 &apps_smmu 0x1c06 0x0 0x1>, + <0x300 &apps_smmu 0x1c07 0x0 0x1>, + <0x400 &apps_smmu 0x1c08 0x0 0x1>, + <0x500 &apps_smmu 0x1c09 0x0 0x1>, + <0x501 &apps_smmu 0x1c10 0x0 0x1>; status = "okay"; }; @@ -151,22 +151,22 @@ }; &pcie1 { - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>, - <0x208 &apps_smmu 0x1c84 0x1>, - <0x210 &apps_smmu 0x1c85 0x1>, - <0x218 &apps_smmu 0x1c86 0x1>, - <0x300 &apps_smmu 0x1c87 0x1>, - <0x408 &apps_smmu 0x1c90 0x1>, - <0x410 &apps_smmu 0x1c91 0x1>, - <0x418 &apps_smmu 0x1c92 0x1>, - <0x500 &apps_smmu 0x1c93 0x1>, - <0x600 &apps_smmu 0x1c94 0x1>, - <0x700 &apps_smmu 0x1c95 0x1>, - <0x701 &apps_smmu 0x1c96 0x1>, - <0x800 &apps_smmu 0x1c97 0x1>, - <0x900 &apps_smmu 0x1c98 0x1>, - <0x901 &apps_smmu 0x1c99 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>, + <0x208 &apps_smmu 0x1c84 0x0 0x1>, + <0x210 &apps_smmu 0x1c85 0x0 0x1>, + <0x218 &apps_smmu 0x1c86 0x0 0x1>, + <0x300 &apps_smmu 0x1c87 0x0 0x1>, + <0x408 &apps_smmu 0x1c90 0x0 0x1>, + <0x410 &apps_smmu 0x1c91 0x0 0x1>, + <0x418 &apps_smmu 0x1c92 0x0 0x1>, + <0x500 &apps_smmu 0x1c93 0x0 0x1>, + <0x600 &apps_smmu 0x1c94 0x0 0x1>, + <0x700 &apps_smmu 0x1c95 0x0 0x1>, + <0x701 &apps_smmu 0x1c96 0x0 0x1>, + <0x800 &apps_smmu 0x1c97 0x0 0x1>, + <0x900 &apps_smmu 0x1c98 0x0 0x1>, + <0x901 &apps_smmu 0x1c99 0x0 0x1>; }; &pcie1_switch0_dsp1 { diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts index 37a3b51323ce5..eeb422586896f 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts +++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts @@ -846,15 +846,15 @@ pinctrl-0 = <&pcie1_reset_n>, <&pcie1_wake_n>, <&pcie1_clkreq_n>; pinctrl-names = "default"; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>, - <0x208 &apps_smmu 0x1c84 0x1>, - <0x210 &apps_smmu 0x1c85 0x1>, - <0x218 &apps_smmu 0x1c86 0x1>, - <0x300 &apps_smmu 0x1c87 0x1>, - <0x400 &apps_smmu 0x1c88 0x1>, - <0x500 &apps_smmu 0x1c89 0x1>, - <0x501 &apps_smmu 0x1c90 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>, + <0x208 &apps_smmu 0x1c84 0x0 0x1>, + <0x210 &apps_smmu 0x1c85 0x0 0x1>, + <0x218 &apps_smmu 0x1c86 0x0 0x1>, + <0x300 &apps_smmu 0x1c87 0x0 0x1>, + <0x400 &apps_smmu 0x1c88 0x0 0x1>, + <0x500 &apps_smmu 0x1c89 0x0 0x1>, + <0x501 &apps_smmu 0x1c90 0x0 0x1>; status = "okay"; }; @@ -1419,8 +1419,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p912>; + vdda-phy-supply = <&vreg_l1b_0p912>; + vdda-pll-supply = <&vreg_l6b_1p2>; orientation-switch; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts index a5ad796cb65d0..1e773f9300f71 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts +++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts @@ -711,15 +711,15 @@ <&pcie1_wake_n>; pinctrl-names = "default"; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>, - <0x208 &apps_smmu 0x1c84 0x1>, - <0x210 &apps_smmu 0x1c85 0x1>, - <0x218 &apps_smmu 0x1c86 0x1>, - <0x300 &apps_smmu 0x1c87 0x1>, - <0x400 &apps_smmu 0x1c88 0x1>, - <0x500 &apps_smmu 0x1c89 0x1>, - <0x501 &apps_smmu 0x1c90 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>, + <0x208 &apps_smmu 0x1c84 0x0 0x1>, + <0x210 &apps_smmu 0x1c85 0x0 0x1>, + <0x218 &apps_smmu 0x1c86 0x0 0x1>, + <0x300 &apps_smmu 0x1c87 0x0 0x1>, + <0x400 &apps_smmu 0x1c88 0x0 0x1>, + <0x500 &apps_smmu 0x1c89 0x0 0x1>, + <0x501 &apps_smmu 0x1c90 0x0 0x1>; status = "okay"; }; @@ -1059,8 +1059,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p912>; + vdda-phy-supply = <&vreg_l1b_0p912>; + vdda-pll-supply = <&vreg_l6b_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts index f47efca42d48d..7336118ae7e34 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts +++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts @@ -1103,8 +1103,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p912>; + vdda-phy-supply = <&vreg_l1b_0p912>; + vdda-pll-supply = <&vreg_l6b_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi b/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi index e6ebb643203b6..ae301d78e0710 100644 --- a/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi +++ b/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi @@ -412,8 +412,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3f_0p88>; + vdda-phy-supply = <&vreg_l3f_0p88>; + vdda-pll-supply = <&vreg_l3e_1p2>; }; &xo_board { diff --git a/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts b/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts index 690b484352edf..d6af679abe1a3 100644 --- a/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts +++ b/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts @@ -1424,8 +1424,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p92>; + vdda-phy-supply = <&vreg_l18a_0p92>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &usb_1_qmpphy_out { @@ -1451,8 +1451,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p92>; + vdda-phy-supply = <&vreg_l18a_0p92>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &vamacro { diff --git a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts index 388d5ecee9497..3e22ee8b50924 100644 --- a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts +++ b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts @@ -428,8 +428,8 @@ &ufs_mem_phy { status = "okay"; - vdda-phy-supply = <&vreg_l8c_1p2>; - vdda-pll-supply = <&vreg_l5a_0p88>; + vdda-phy-supply = <&vreg_l5a_0p88>; + vdda-pll-supply = <&vreg_l8c_1p2>; }; &usb_1 { @@ -474,8 +474,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l8c_1p2>; - vdda-pll-supply = <&vdda_usb_ss_dp_core_1>; + vdda-phy-supply = <&vdda_usb_ss_dp_core_1>; + vdda-pll-supply = <&vreg_l8c_1p2>; }; &pcie0 { diff --git a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts index d28d691624279..7178ae087038c 100644 --- a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts +++ b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts @@ -604,8 +604,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l3a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l3a>; status = "okay"; }; @@ -628,8 +628,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l4c>; - vdda-pll-supply = <&vreg_l1c>; + vdda-phy-supply = <&vreg_l1c>; + vdda-pll-supply = <&vreg_l4c>; status = "okay"; }; @@ -671,15 +671,15 @@ }; &usb_2_qmpphy0 { - vdda-phy-supply = <&vreg_l3a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l3a>; status = "okay"; }; &usb_2_qmpphy1 { - vdda-phy-supply = <&vreg_l3a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l3a>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sa8540p-ride.dts b/arch/arm64/boot/dts/qcom/sa8540p-ride.dts index 44177e9b64b52..df40c44c447e7 100644 --- a/arch/arm64/boot/dts/qcom/sa8540p-ride.dts +++ b/arch/arm64/boot/dts/qcom/sa8540p-ride.dts @@ -478,8 +478,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l3a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l3a>; status = "okay"; }; @@ -493,8 +493,8 @@ }; &usb_2_qmpphy0 { - vdda-phy-supply = <&vreg_l3a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l3a>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts b/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts index 74778a5b19ba6..80428c0b3b4e6 100644 --- a/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts +++ b/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts @@ -551,8 +551,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p91>; + vdda-phy-supply = <&vreg_l1a_0p91>; + vdda-pll-supply = <&vreg_l3a_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sar2130p.dtsi b/arch/arm64/boot/dts/qcom/sar2130p.dtsi index 3c9529bb2f76f..99002ddaf6f63 100644 --- a/arch/arm64/boot/dts/qcom/sar2130p.dtsi +++ b/arch/arm64/boot/dts/qcom/sar2130p.dtsi @@ -1329,8 +1329,8 @@ &config_noc SLAVE_PCIE_0 QCOM_ICC_TAG_ALWAYS>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -1455,8 +1455,8 @@ &config_noc SLAVE_PCIE_1 QCOM_ICC_TAG_ALWAYS>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &apps_smmu 0x1e00 0x1>, - <0x100 &apps_smmu 0x1e01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1e00 0x0 0x1>, + <0x100 &apps_smmu 0x1e01 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>, <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; diff --git a/arch/arm64/boot/dts/qcom/sc7180-acer-aspire1.dts b/arch/arm64/boot/dts/qcom/sc7180-acer-aspire1.dts index 1514da6362698..d9c70897b6f1c 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-acer-aspire1.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-acer-aspire1.dts @@ -631,8 +631,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l4a_0p8>; + vdda-phy-supply = <&vreg_l4a_0p8>; + vdda-pll-supply = <&vreg_l3c_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts b/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts index b84da371581b9..3100fab18787a 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts @@ -563,8 +563,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l4a_0p8>; + vdda-phy-supply = <&vreg_l4a_0p8>; + vdda-pll-supply = <&vreg_l3c_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; diff --git a/arch/arm64/boot/dts/qcom/sc7180-idp.dts b/arch/arm64/boot/dts/qcom/sc7180-idp.dts index 0bce3eefca2e9..acfb9937651d6 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-idp.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-idp.dts @@ -461,8 +461,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l4a_0p8>; + vdda-phy-supply = <&vreg_l4a_0p8>; + vdda-pll-supply = <&vreg_l3c_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi index b398f69917f0e..0e0bc8543f714 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi @@ -99,6 +99,11 @@ stdout-path = "serial0:115200n8"; }; + firmware { + #address-cells = <2>; + #size-cells = <2>; + }; + /* FIXED REGULATORS - parents above children */ /* This is the top level supply and variable voltage */ @@ -993,8 +998,8 @@ ap_spi_fp: &spi10 { &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb_ss_dp_1p2>; - vdda-pll-supply = <&vdda_usb_ss_dp_core>; + vdda-phy-supply = <&vdda_usb_ss_dp_core>; + vdda-pll-supply = <&vdda_usb_ss_dp_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi index 5c5e4f1dd2217..58ea0532c0fbb 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi @@ -25,6 +25,11 @@ stdout-path = "serial0:115200n8"; }; + firmware { + #address-cells = <2>; + #size-cells = <2>; + }; + /* * FIXED REGULATORS * diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi index 8cac4ce9c8515..23ced0621c99e 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi @@ -542,8 +542,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p8>; + vdda-phy-supply = <&vreg_l1b_0p8>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &uart7 { diff --git a/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi b/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi index 9ae9af40f4d21..264029ff3f903 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi @@ -465,8 +465,8 @@ mos_bt_uart: &uart7 { }; &usb_1_qmpphy { - vdda-phy-supply = <&vdd_a_usbssdp_0_1p2>; - vdda-pll-supply = <&vdd_a_usbssdp_0_core>; + vdda-phy-supply = <&vdd_a_usbssdp_0_core>; + vdda-pll-supply = <&vdd_a_usbssdp_0_1p2>; }; &usb_2_hsphy { diff --git a/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts b/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts index d86a31ddede29..565e5a9979b1a 100644 --- a/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts +++ b/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts @@ -26,6 +26,7 @@ compatible = "pwm-backlight"; pwms = <&pmc8180c_lpg 4 1000000>; enable-gpios = <&pmc8180c_gpios 8 GPIO_ACTIVE_HIGH>; + power-supply = <&vled_bl_pw>; pinctrl-0 = <&bl_pwm_default>; pinctrl-names = "default"; @@ -157,14 +158,39 @@ }; }; - vph_pwr: vph-pwr-regulator { + vled_bl_pw: regulator-vled-bl-pw { compatible = "regulator-fixed"; - regulator-name = "vph_pwr"; - regulator-min-microvolt = <3700000>; - regulator-max-microvolt = <3700000>; + + regulator-name = "VLED_BL_PW"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmc8180_2_gpios 1 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&bl_pwr_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_lcm_3v3: regulator-edp-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_LCM_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 130 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&lcm_3v3_en>; + pinctrl-names = "default"; + + regulator-boot-on; }; - vreg_s4a_1p8: pm8150-s4-regulator { + vreg_s4a_1p8: regulator-pm8150-s4 { compatible = "regulator-fixed"; regulator-name = "vreg_s4a_1p8"; @@ -177,6 +203,13 @@ vin-supply = <&vph_pwr>; }; + vph_pwr: regulator-vph-pwr { + compatible = "regulator-fixed"; + regulator-name = "vph_pwr"; + regulator-min-microvolt = <3700000>; + regulator-max-microvolt = <3700000>; + }; + usbprim-sbu-mux { compatible = "pericom,pi3usb102", "gpio-sbu-mux"; @@ -438,6 +471,7 @@ aux-bus { panel { compatible = "edp-panel"; + power-supply = <&vreg_lcm_3v3>; no-hpd; backlight = <&backlight>; @@ -472,6 +506,13 @@ status = "okay"; }; +&pmc8180_2_gpios { + bl_pwr_en: bl-pwr-en-state { + pins = "gpio1"; + function = "normal"; + }; +}; + &pmc8180_pwrkey { status = "okay"; }; @@ -570,15 +611,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; status = "okay"; }; @@ -592,8 +633,8 @@ }; &usb_prim_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; orientation-switch; @@ -633,8 +674,8 @@ }; &usb_sec_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; orientation-switch; @@ -765,6 +806,12 @@ drive-strength = <2>; }; + lcm_3v3_en: lcm-3v3-en-state { + pins = "gpio130"; + function = "gpio"; + bias-disable; + }; + usbprim_sbu_default: usbprim-sbu-state { oe-n-pins { pins = "gpio152"; diff --git a/arch/arm64/boot/dts/qcom/sc8180x-primus.dts b/arch/arm64/boot/dts/qcom/sc8180x-primus.dts index aff398390eba7..c10a936b320a9 100644 --- a/arch/arm64/boot/dts/qcom/sc8180x-primus.dts +++ b/arch/arm64/boot/dts/qcom/sc8180x-primus.dts @@ -29,9 +29,10 @@ compatible = "pwm-backlight"; pwms = <&pmc8180c_lpg 4 1000000>; enable-gpios = <&pmc8180c_gpios 8 GPIO_ACTIVE_HIGH>; + power-supply = <&vled_bl_pw>; - pinctrl-names = "default"; pinctrl-0 = <&bl_pwm_default>; + pinctrl-names = "default"; }; chosen { @@ -167,7 +168,39 @@ }; }; - vreg_nvme_0p9: nvme-0p9-regulator { + vled_bl_pw: regulator-vled-bl-pw { + compatible = "regulator-fixed"; + + regulator-name = "VLED_BL_PW"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmc8180_2_gpios 1 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&bl_pwr_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_lcm_3v3: regulator-edp-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_LCM_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 130 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&lcm_3v3_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_nvme_0p9: regulator-nvme-0p9 { compatible = "regulator-fixed"; regulator-name = "vreg_nvme_0p9"; @@ -177,7 +210,7 @@ regulator-always-on; }; - vreg_nvme_3p3: nvme-3p3-regulator { + vreg_nvme_3p3: regulator-nvme-3p3 { compatible = "regulator-fixed"; regulator-name = "vreg_nvme_3p3"; @@ -190,7 +223,7 @@ regulator-always-on; }; - vdd_kb_tp_3v3: vdd-kb-tp-3v3-regulator { + vdd_kb_tp_3v3: regulator-vdd-kb-tp-3v3 { compatible = "regulator-fixed"; regulator-name = "vdd_kb_tp_3v3"; regulator-min-microvolt = <3300000>; @@ -205,7 +238,7 @@ pinctrl-0 = <&kb_tp_3v3_en_active_state>; }; - vph_pwr: vph-pwr-regulator { + vph_pwr: regulator-vph-pwr { compatible = "regulator-fixed"; regulator-name = "vph_pwr"; regulator-min-microvolt = <3700000>; @@ -540,6 +573,7 @@ aux-bus { panel { compatible = "edp-panel"; + power-supply = <&vreg_lcm_3v3>; backlight = <&backlight>; @@ -669,15 +703,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; status = "okay"; }; @@ -691,8 +725,8 @@ }; &usb_prim_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; orientation-switch; @@ -728,8 +762,8 @@ }; &usb_sec_qmpphy { - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5e_0p88>; + vdda-phy-supply = <&vreg_l5e_0p88>; + vdda-pll-supply = <&vreg_l3c_1p2>; orientation-switch; @@ -769,6 +803,12 @@ }; /* PINCTRL */ +&pmc8180_2_gpios { + bl_pwr_en: bl-pwr-en-state { + pins = "gpio1"; + function = "normal"; + }; +}; &pmc8180c_gpios { bl_pwm_default: bl-pwm-default-state { @@ -950,4 +990,10 @@ bias-pull-up; }; }; + + lcm_3v3_en: lcm-3v3-en-state { + pins = "gpio130"; + function = "gpio"; + bias-disable; + }; }; diff --git a/arch/arm64/boot/dts/qcom/sc8180x.dtsi b/arch/arm64/boot/dts/qcom/sc8180x.dtsi index 45391768e2458..5d3f0c4d7b464 100644 --- a/arch/arm64/boot/dts/qcom/sc8180x.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8180x.dtsi @@ -1767,8 +1767,8 @@ assigned-clocks = <&gcc GCC_PCIE_0_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1d80 0x1>, - <0x100 &apps_smmu 0x1d81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1d80 0x0 0x1>, + <0x100 &apps_smmu 0x1d81 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -1886,8 +1886,8 @@ assigned-clocks = <&gcc GCC_PCIE_3_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1e00 0x1>, - <0x100 &apps_smmu 0x1e01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1e00 0x0 0x1>, + <0x100 &apps_smmu 0x1e01 0x0 0x1>; resets = <&gcc GCC_PCIE_3_BCR>; reset-names = "pci"; @@ -2006,8 +2006,8 @@ assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; @@ -2126,8 +2126,8 @@ assigned-clocks = <&gcc GCC_PCIE_2_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1d00 0x1>, - <0x100 &apps_smmu 0x1d01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1d00 0x0 0x1>, + <0x100 &apps_smmu 0x1d01 0x0 0x1>; resets = <&gcc GCC_PCIE_2_BCR>; reset-names = "pci"; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts index dcdeefd287283..ee85d5e09520e 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts @@ -182,7 +182,7 @@ regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; - gpio = <&pmc8280_1_gpios 2 GPIO_ACTIVE_HIGH>; + gpio = <&pmc8280_1_gpios 1 GPIO_ACTIVE_HIGH>; enable-active-high; pinctrl-names = "default"; @@ -915,7 +915,7 @@ }; misc_3p3_reg_en: misc-3p3-reg-en-state { - pins = "gpio2"; + pins = "gpio1"; function = "normal"; }; }; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts index abd9c5a67b9ff..3ddd44e16e67f 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts @@ -1555,7 +1555,7 @@ mclk-pins { pins = "gpio17"; function = "cam_mclk"; - drive-strength = <16>; + drive-strength = <6>; bias-disable; }; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts index f2b4470d4407f..aa79704b5d552 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts @@ -243,7 +243,7 @@ usb0-sbu-mux { compatible = "pericom,pi3usb102", "gpio-sbu-mux"; - enable-gpios = <&tlmm 101 GPIO_ACTIVE_LOW>; + enable-gpios = <&tlmm 100 GPIO_ACTIVE_LOW>; select-gpios = <&tlmm 164 GPIO_ACTIVE_HIGH>; pinctrl-0 = <&usb0_sbu_default>; @@ -996,7 +996,7 @@ usb0_sbu_default: usb0-sbu-state { oe-n-pins { - pins = "gpio101"; + pins = "gpio100"; function = "gpio"; bias-disable; drive-strength = <16>; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts index 125af356e24b9..0cea30e76d5fb 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts @@ -763,20 +763,11 @@ }; &pmk8280_rtc { - nvmem-cells = <&rtc_offset>; - nvmem-cell-names = "offset"; + qcom,uefi-rtc-info; status = "okay"; }; -&pmk8280_sdam_6 { - status = "okay"; - - rtc_offset: rtc-offset@bc { - reg = <0xbc 0x4>; - }; -}; - &pmk8280_vadc { channel@144 { reg = ; @@ -1003,8 +994,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l4d>; - vdda-pll-supply = <&vreg_l9d>; + vdda-phy-supply = <&vreg_l9d>; + vdda-pll-supply = <&vreg_l4d>; orientation-switch; @@ -1040,8 +1031,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l3b>; - vdda-pll-supply = <&vreg_l4b>; + vdda-phy-supply = <&vreg_l4b>; + vdda-pll-supply = <&vreg_l3b>; orientation-switch; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index b0de9e262f299..d7d1279008fa6 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -692,11 +692,6 @@ no-map; }; - pil_gpu_mem: gpu-mem@8bf00000 { - reg = <0 0x8bf00000 0 0x2000>; - no-map; - }; - pil_adsp_mem: adsp-region@86c00000 { reg = <0 0x86c00000 0 0x2000000>; no-map; @@ -712,6 +707,11 @@ no-map; }; + pil_gpu_mem: gpu-mem@8bf00000 { + reg = <0 0x8bf00000 0 0x2000>; + no-map; + }; + pil_nsp1_mem: cdsp1-region@8c600000 { reg = <0 0x8c600000 0 0x1e00000>; no-map; diff --git a/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts b/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts index 2f55db0c8ce35..9ea3e5e76bf9e 100644 --- a/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts +++ b/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts @@ -130,7 +130,7 @@ led: led@1 { reg = <1>; - default-trigger = "backlight"; + linux,default-trigger = "backlight"; function = LED_FUNCTION_BACKLIGHT; led-sources = <0 1 2>; }; diff --git a/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi b/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi index 0f57b915186b7..b4854801a5f5e 100644 --- a/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi @@ -522,6 +522,11 @@ }; }; +&lpi_tlmm { + /* sensor gpios are protected */ + gpio-reserved-ranges = <0 8>, <12 6>; +}; + &mdss { status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts index 02416812b6a7f..846138149b2cd 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts @@ -1121,8 +1121,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l26a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p875>; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l26a_1p2>; }; &usb_2 { @@ -1147,8 +1147,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l26a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p875>; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l26a_1p2>; }; &ufs_mem_hc { diff --git a/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi index b2dc0327dff85..ebeebbde64053 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi @@ -571,8 +571,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l26a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p875>; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l26a_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi index e0c3566761bf8..c14b4b3d9a3c9 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi @@ -677,8 +677,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts index 091568642faa7..00ea30afc9f6d 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts @@ -729,8 +729,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; }; &usb_2 { @@ -761,8 +761,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb2_ss_1p2>; - vdda-pll-supply = <&vdda_usb2_ss_core>; + vdda-phy-supply = <&vdda_usb2_ss_core>; + vdda-pll-supply = <&vdda_usb2_ss_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts index 0d915ecb0444f..cd735417adddc 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts @@ -949,8 +949,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts b/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts index f21b914f22b51..ca503af9b832b 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts @@ -876,8 +876,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l26a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p875>; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l26a_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi index 54e58ca046934..710b296e49643 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi @@ -645,8 +645,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l26a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p875>; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l26a_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts index a44d6e776c82e..cce7348e583ac 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts @@ -668,8 +668,8 @@ }; &usb_1_qmpphy { - vdda-pll-supply = <&vreg_l1a_0p875>; - vdda-phy-supply = <&vreg_l26a_1p2>; + vdda-pll-supply = <&vreg_l26a_1p2>; + vdda-phy-supply = <&vreg_l1a_0p875>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi index 4ae8627d6dbc3..7c2a439dffdb1 100644 --- a/arch/arm64/boot/dts/qcom/sdm845.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi @@ -2372,22 +2372,22 @@ "slave_q2a", "tbu"; - iommu-map = <0x0 &apps_smmu 0x1c10 0x1>, - <0x100 &apps_smmu 0x1c11 0x1>, - <0x200 &apps_smmu 0x1c12 0x1>, - <0x300 &apps_smmu 0x1c13 0x1>, - <0x400 &apps_smmu 0x1c14 0x1>, - <0x500 &apps_smmu 0x1c15 0x1>, - <0x600 &apps_smmu 0x1c16 0x1>, - <0x700 &apps_smmu 0x1c17 0x1>, - <0x800 &apps_smmu 0x1c18 0x1>, - <0x900 &apps_smmu 0x1c19 0x1>, - <0xa00 &apps_smmu 0x1c1a 0x1>, - <0xb00 &apps_smmu 0x1c1b 0x1>, - <0xc00 &apps_smmu 0x1c1c 0x1>, - <0xd00 &apps_smmu 0x1c1d 0x1>, - <0xe00 &apps_smmu 0x1c1e 0x1>, - <0xf00 &apps_smmu 0x1c1f 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c10 0x0 0x1>, + <0x100 &apps_smmu 0x1c11 0x0 0x1>, + <0x200 &apps_smmu 0x1c12 0x0 0x1>, + <0x300 &apps_smmu 0x1c13 0x0 0x1>, + <0x400 &apps_smmu 0x1c14 0x0 0x1>, + <0x500 &apps_smmu 0x1c15 0x0 0x1>, + <0x600 &apps_smmu 0x1c16 0x0 0x1>, + <0x700 &apps_smmu 0x1c17 0x0 0x1>, + <0x800 &apps_smmu 0x1c18 0x0 0x1>, + <0x900 &apps_smmu 0x1c19 0x0 0x1>, + <0xa00 &apps_smmu 0x1c1a 0x0 0x1>, + <0xb00 &apps_smmu 0x1c1b 0x0 0x1>, + <0xc00 &apps_smmu 0x1c1c 0x0 0x1>, + <0xd00 &apps_smmu 0x1c1d 0x0 0x1>, + <0xe00 &apps_smmu 0x1c1e 0x0 0x1>, + <0xf00 &apps_smmu 0x1c1f 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2502,22 +2502,22 @@ assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>, - <0x200 &apps_smmu 0x1c02 0x1>, - <0x300 &apps_smmu 0x1c03 0x1>, - <0x400 &apps_smmu 0x1c04 0x1>, - <0x500 &apps_smmu 0x1c05 0x1>, - <0x600 &apps_smmu 0x1c06 0x1>, - <0x700 &apps_smmu 0x1c07 0x1>, - <0x800 &apps_smmu 0x1c08 0x1>, - <0x900 &apps_smmu 0x1c09 0x1>, - <0xa00 &apps_smmu 0x1c0a 0x1>, - <0xb00 &apps_smmu 0x1c0b 0x1>, - <0xc00 &apps_smmu 0x1c0c 0x1>, - <0xd00 &apps_smmu 0x1c0d 0x1>, - <0xe00 &apps_smmu 0x1c0e 0x1>, - <0xf00 &apps_smmu 0x1c0f 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>, + <0x200 &apps_smmu 0x1c02 0x0 0x1>, + <0x300 &apps_smmu 0x1c03 0x0 0x1>, + <0x400 &apps_smmu 0x1c04 0x0 0x1>, + <0x500 &apps_smmu 0x1c05 0x0 0x1>, + <0x600 &apps_smmu 0x1c06 0x0 0x1>, + <0x700 &apps_smmu 0x1c07 0x0 0x1>, + <0x800 &apps_smmu 0x1c08 0x0 0x1>, + <0x900 &apps_smmu 0x1c09 0x0 0x1>, + <0xa00 &apps_smmu 0x1c0a 0x0 0x1>, + <0xb00 &apps_smmu 0x1c0b 0x0 0x1>, + <0xc00 &apps_smmu 0x1c0c 0x0 0x1>, + <0xd00 &apps_smmu 0x1c0d 0x0 0x1>, + <0xe00 &apps_smmu 0x1c0e 0x0 0x1>, + <0xf00 &apps_smmu 0x1c0f 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; diff --git a/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts b/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts index f048653818702..a4339b562b604 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts @@ -872,8 +872,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; status = "okay"; }; @@ -898,8 +898,8 @@ }; &usb_2_qmpphy { - vdda-phy-supply = <&vdda_usb2_ss_1p2>; - vdda-pll-supply = <&vdda_usb2_ss_core>; + vdda-phy-supply = <&vdda_usb2_ss_core>; + vdda-pll-supply = <&vdda_usb2_ss_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts index 8cdaef5360d11..3c353d27d7e89 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts @@ -864,8 +864,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; }; &usb_1_qmpphy_out { @@ -957,8 +957,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb2_ss_1p2>; - vdda-pll-supply = <&vdda_usb2_ss_core>; + vdda-phy-supply = <&vdda_usb2_ss_core>; + vdda-pll-supply = <&vdda_usb2_ss_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts b/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts index a676d3ea01b99..b7a1e2317fc2f 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts @@ -609,8 +609,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb1_ss_1p2>; - vdda-pll-supply = <&vdda_usb1_ss_core>; + vdda-phy-supply = <&vdda_usb1_ss_core>; + vdda-pll-supply = <&vdda_usb1_ss_1p2>; }; &usb_2 { @@ -635,8 +635,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vdda_usb2_ss_1p2>; - vdda-pll-supply = <&vdda_usb2_ss_core>; + vdda-phy-supply = <&vdda_usb2_ss_core>; + vdda-pll-supply = <&vdda_usb2_ss_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts b/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts index 0f23eaef01f21..b290f0f4c0e10 100644 --- a/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts +++ b/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts @@ -151,7 +151,7 @@ interrupts-extended = <&tlmm 80 IRQ_TYPE_LEVEL_LOW>; - irq-gpios = <&tlmm 80 IRQ_TYPE_LEVEL_LOW>; + irq-gpios = <&tlmm 80 GPIO_ACTIVE_LOW>; reset-gpios = <&tlmm 71 GPIO_ACTIVE_HIGH>; AVDD28-supply = <&ts_vdd_supply>; VDDIO-supply = <&ts_vddio_supply>; diff --git a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts index b9d9439e230b4..139f2b401af50 100644 --- a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts +++ b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts @@ -198,7 +198,7 @@ status = "okay"; panel@0 { - compatible = "samsung,s6e8fco-m1906f9"; + compatible = "samsung,s6e8fc0-m1906f9"; reg = <0>; reset-gpios = <&tlmm 90 GPIO_ACTIVE_LOW>; diff --git a/arch/arm64/boot/dts/qcom/sm6125.dtsi b/arch/arm64/boot/dts/qcom/sm6125.dtsi index 6e84c226948c0..a3caf5c87ae76 100644 --- a/arch/arm64/boot/dts/qcom/sm6125.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6125.dtsi @@ -374,22 +374,23 @@ }; soc@0 { - #address-cells = <1>; - #size-cells = <1>; - ranges = <0x00 0x00 0x00 0xffffffff>; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0 0 0 0 0x10 0>; + dma-ranges = <0 0 0 0 0x10 0>; compatible = "simple-bus"; tcsr_mutex: hwlock@340000 { compatible = "qcom,tcsr-mutex"; - reg = <0x00340000 0x20000>; + reg = <0x0 0x00340000 0x0 0x20000>; #hwlock-cells = <1>; }; tlmm: pinctrl@500000 { compatible = "qcom,sm6125-tlmm"; - reg = <0x00500000 0x400000>, - <0x00900000 0x400000>, - <0x00d00000 0x400000>; + reg = <0x0 0x00500000 0x0 0x400000>, + <0x0 0x00900000 0x0 0x400000>, + <0x0 0x00d00000 0x0 0x400000>; reg-names = "west", "south", "east"; interrupts = ; gpio-controller; @@ -672,7 +673,7 @@ gcc: clock-controller@1400000 { compatible = "qcom,gcc-sm6125"; - reg = <0x01400000 0x1f0000>; + reg = <0x0 0x01400000 0x0 0x1f0000>; #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; @@ -682,7 +683,7 @@ hsusb_phy1: phy@1613000 { compatible = "qcom,msm8996-qusb2-phy"; - reg = <0x01613000 0x180>; + reg = <0x0 0x01613000 0x0 0x180>; #phy-cells = <0>; clocks = <&gcc GCC_AHB2PHY_USB_CLK>, @@ -695,18 +696,18 @@ rng: rng@1b53000 { compatible = "qcom,prng-ee"; - reg = <0x01b53000 0x1000>; + reg = <0x0 0x01b53000 0x0 0x1000>; clocks = <&gcc GCC_PRNG_AHB_CLK>; clock-names = "core"; }; spmi_bus: spmi@1c40000 { compatible = "qcom,spmi-pmic-arb"; - reg = <0x01c40000 0x1100>, - <0x01e00000 0x2000000>, - <0x03e00000 0x100000>, - <0x03f00000 0xa0000>, - <0x01c0a000 0x26000>; + reg = <0x0 0x01c40000 0x0 0x1100>, + <0x0 0x01e00000 0x0 0x2000000>, + <0x0 0x03e00000 0x0 0x100000>, + <0x0 0x03f00000 0x0 0xa0000>, + <0x0 0x01c0a000 0x0 0x26000>; reg-names = "core", "chnls", "obsrvr", "intr", "cnfg"; interrupt-names = "periph_irq"; interrupts = ; @@ -720,12 +721,13 @@ rpm_msg_ram: sram@45f0000 { compatible = "qcom,rpm-msg-ram"; - reg = <0x045f0000 0x7000>; + reg = <0x0 0x045f0000 0x0 0x7000>; }; sdhc_1: mmc@4744000 { compatible = "qcom,sm6125-sdhci", "qcom,sdhci-msm-v5"; - reg = <0x04744000 0x1000>, <0x04745000 0x1000>; + reg = <0x0 0x04744000 0x0 0x1000>, + <0x0 0x04745000 0x0 0x1000>; reg-names = "hc", "cqhci"; interrupts = , @@ -752,7 +754,7 @@ sdhc_2: mmc@4784000 { compatible = "qcom,sm6125-sdhci", "qcom,sdhci-msm-v5"; - reg = <0x04784000 0x1000>; + reg = <0x0 0x04784000 0x0 0x1000>; reg-names = "hc"; interrupts = , @@ -780,7 +782,8 @@ ufs_mem_hc: ufshc@4804000 { compatible = "qcom,sm6125-ufshc", "qcom,ufshc", "jedec,ufs-2.0"; - reg = <0x04804000 0x3000>, <0x04810000 0x8000>; + reg = <0x0 0x04804000 0x0 0x3000>, + <0x0 0x04810000 0x0 0x8000>; reg-names = "std", "ice"; interrupts = ; @@ -825,7 +828,7 @@ ufs_mem_phy: phy@4807000 { compatible = "qcom,sm6125-qmp-ufs-phy"; - reg = <0x04807000 0xdb8>; + reg = <0x0 0x04807000 0x0 0xdb8>; clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>, <&gcc GCC_UFS_PHY_PHY_AUX_CLK>, @@ -846,7 +849,7 @@ gpi_dma0: dma-controller@4a00000 { compatible = "qcom,sm6125-gpi-dma", "qcom,sdm845-gpi-dma"; - reg = <0x04a00000 0x60000>; + reg = <0x0 0x04a00000 0x0 0x60000>; interrupts = , , , @@ -864,19 +867,19 @@ qupv3_id_0: geniqup@4ac0000 { compatible = "qcom,geni-se-qup"; - reg = <0x04ac0000 0x2000>; + reg = <0x0 0x04ac0000 0x0 0x2000>; clocks = <&gcc GCC_QUPV3_WRAP_0_M_AHB_CLK>, <&gcc GCC_QUPV3_WRAP_0_S_AHB_CLK>; clock-names = "m-ahb", "s-ahb"; iommus = <&apps_smmu 0x123 0x0>; - #address-cells = <1>; - #size-cells = <1>; + #address-cells = <2>; + #size-cells = <2>; ranges; status = "disabled"; i2c0: i2c@4a80000 { compatible = "qcom,geni-i2c"; - reg = <0x04a80000 0x4000>; + reg = <0x0 0x04a80000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; clock-names = "se"; interrupts = ; @@ -893,7 +896,7 @@ spi0: spi@4a80000 { compatible = "qcom,geni-spi"; - reg = <0x04a80000 0x4000>; + reg = <0x0 0x04a80000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; clock-names = "se"; interrupts = ; @@ -910,7 +913,7 @@ i2c1: i2c@4a84000 { compatible = "qcom,geni-i2c"; - reg = <0x04a84000 0x4000>; + reg = <0x0 0x04a84000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; clock-names = "se"; interrupts = ; @@ -927,7 +930,7 @@ i2c2: i2c@4a88000 { compatible = "qcom,geni-i2c"; - reg = <0x04a88000 0x4000>; + reg = <0x0 0x04a88000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; clock-names = "se"; interrupts = ; @@ -944,7 +947,7 @@ spi2: spi@4a88000 { compatible = "qcom,geni-spi"; - reg = <0x04a88000 0x4000>; + reg = <0x0 0x04a88000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; clock-names = "se"; interrupts = ; @@ -961,7 +964,7 @@ i2c3: i2c@4a8c000 { compatible = "qcom,geni-i2c"; - reg = <0x04a8c000 0x4000>; + reg = <0x0 0x04a8c000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>; clock-names = "se"; interrupts = ; @@ -978,7 +981,7 @@ i2c4: i2c@4a90000 { compatible = "qcom,geni-i2c"; - reg = <0x04a90000 0x4000>; + reg = <0x0 0x04a90000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; clock-names = "se"; interrupts = ; @@ -995,7 +998,7 @@ uart4: serial@4a90000 { compatible = "qcom,geni-debug-uart"; - reg = <0x04a90000 0x4000>; + reg = <0x0 0x04a90000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; clock-names = "se"; interrupts = ; @@ -1007,7 +1010,7 @@ gpi_dma1: dma-controller@4c00000 { compatible = "qcom,sm6125-gpi-dma", "qcom,sdm845-gpi-dma"; - reg = <0x04c00000 0x60000>; + reg = <0x0 0x04c00000 0x0 0x60000>; interrupts = , , , @@ -1025,19 +1028,19 @@ qupv3_id_1: geniqup@4cc0000 { compatible = "qcom,geni-se-qup"; - reg = <0x04cc0000 0x2000>; + reg = <0x0 0x04cc0000 0x0 0x2000>; clocks = <&gcc GCC_QUPV3_WRAP_1_M_AHB_CLK>, <&gcc GCC_QUPV3_WRAP_1_S_AHB_CLK>; clock-names = "m-ahb", "s-ahb"; iommus = <&apps_smmu 0x143 0x0>; - #address-cells = <1>; - #size-cells = <1>; + #address-cells = <2>; + #size-cells = <2>; ranges; status = "disabled"; i2c5: i2c@4c80000 { compatible = "qcom,geni-i2c"; - reg = <0x04c80000 0x4000>; + reg = <0x0 0x04c80000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; clock-names = "se"; interrupts = ; @@ -1054,7 +1057,7 @@ spi5: spi@4c80000 { compatible = "qcom,geni-spi"; - reg = <0x04c80000 0x4000>; + reg = <0x0 0x04c80000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; clock-names = "se"; interrupts = ; @@ -1071,7 +1074,7 @@ i2c6: i2c@4c84000 { compatible = "qcom,geni-i2c"; - reg = <0x04c84000 0x4000>; + reg = <0x0 0x04c84000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; clock-names = "se"; interrupts = ; @@ -1088,7 +1091,7 @@ spi6: spi@4c84000 { compatible = "qcom,geni-spi"; - reg = <0x04c84000 0x4000>; + reg = <0x0 0x04c84000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; clock-names = "se"; interrupts = ; @@ -1105,7 +1108,7 @@ i2c7: i2c@4c88000 { compatible = "qcom,geni-i2c"; - reg = <0x04c88000 0x4000>; + reg = <0x0 0x04c88000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>; clock-names = "se"; interrupts = ; @@ -1122,7 +1125,7 @@ i2c8: i2c@4c8c000 { compatible = "qcom,geni-i2c"; - reg = <0x04c8c000 0x4000>; + reg = <0x0 0x04c8c000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; clock-names = "se"; interrupts = ; @@ -1139,7 +1142,7 @@ spi8: spi@4c8c000 { compatible = "qcom,geni-spi"; - reg = <0x04c8c000 0x4000>; + reg = <0x0 0x04c8c000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; clock-names = "se"; interrupts = ; @@ -1156,7 +1159,7 @@ i2c9: i2c@4c90000 { compatible = "qcom,geni-i2c"; - reg = <0x04c90000 0x4000>; + reg = <0x0 0x04c90000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; clock-names = "se"; interrupts = ; @@ -1173,7 +1176,7 @@ spi9: spi@4c90000 { compatible = "qcom,geni-spi"; - reg = <0x04c90000 0x4000>; + reg = <0x0 0x04c90000 0x0 0x4000>; clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; clock-names = "se"; interrupts = ; @@ -1191,9 +1194,9 @@ usb3: usb@4ef8800 { compatible = "qcom,sm6125-dwc3", "qcom,dwc3"; - reg = <0x04ef8800 0x400>; - #address-cells = <1>; - #size-cells = <1>; + reg = <0x0 0x04ef8800 0x0 0x400>; + #address-cells = <2>; + #size-cells = <2>; ranges; clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>, @@ -1228,7 +1231,7 @@ usb3_dwc3: usb@4e00000 { compatible = "snps,dwc3"; - reg = <0x04e00000 0xcd00>; + reg = <0x0 0x04e00000 0x0 0xcd00>; interrupts = ; iommus = <&apps_smmu 0x100 0x0>; phys = <&hsusb_phy1>; @@ -1244,12 +1247,12 @@ sram@4690000 { compatible = "qcom,rpm-stats"; - reg = <0x04690000 0x10000>; + reg = <0x0 0x04690000 0x0 0x10000>; }; mdss: display-subsystem@5e00000 { compatible = "qcom,sm6125-mdss"; - reg = <0x05e00000 0x1000>; + reg = <0x0 0x05e00000 0x0 0x1000>; reg-names = "mdss"; interrupts = ; @@ -1269,16 +1272,16 @@ iommus = <&apps_smmu 0x400 0x0>; - #address-cells = <1>; - #size-cells = <1>; + #address-cells = <2>; + #size-cells = <2>; ranges; status = "disabled"; mdss_mdp: display-controller@5e01000 { compatible = "qcom,sm6125-dpu"; - reg = <0x05e01000 0x83208>, - <0x05eb0000 0x3000>; + reg = <0x0 0x05e01000 0x0 0x83208>, + <0x0 0x05eb0000 0x0 0x3000>; reg-names = "mdp", "vbif"; interrupt-parent = <&mdss>; @@ -1348,7 +1351,7 @@ mdss_dsi0: dsi@5e94000 { compatible = "qcom,sm6125-dsi-ctrl", "qcom,mdss-dsi-ctrl"; - reg = <0x05e94000 0x400>; + reg = <0x0 0x05e94000 0x0 0x400>; reg-names = "dsi_ctrl"; interrupt-parent = <&mdss>; @@ -1417,9 +1420,9 @@ mdss_dsi0_phy: phy@5e94400 { compatible = "qcom,sm6125-dsi-phy-14nm"; - reg = <0x05e94400 0x100>, - <0x05e94500 0x300>, - <0x05e94800 0x188>; + reg = <0x0 0x05e94400 0x0 0x100>, + <0x0 0x05e94500 0x0 0x300>, + <0x0 0x05e94800 0x0 0x188>; reg-names = "dsi_phy", "dsi_phy_lane", "dsi_pll"; @@ -1441,7 +1444,7 @@ dispcc: clock-controller@5f00000 { compatible = "qcom,sm6125-dispcc"; - reg = <0x05f00000 0x20000>; + reg = <0x0 0x05f00000 0x0 0x20000>; clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>, <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>, @@ -1470,7 +1473,7 @@ apps_smmu: iommu@c600000 { compatible = "qcom,sm6125-smmu-500", "qcom,smmu-500", "arm,mmu-500"; - reg = <0x0c600000 0x80000>; + reg = <0x0 0x0c600000 0x0 0x80000>; interrupts = , , , @@ -1544,74 +1547,74 @@ apcs_glb: mailbox@f111000 { compatible = "qcom,sm6125-apcs-hmss-global", "qcom,msm8994-apcs-kpss-global"; - reg = <0x0f111000 0x1000>; + reg = <0x0 0x0f111000 0x0 0x1000>; #mbox-cells = <1>; }; timer@f120000 { compatible = "arm,armv7-timer-mem"; - #address-cells = <1>; + #address-cells = <2>; #size-cells = <1>; - ranges; - reg = <0x0f120000 0x1000>; + reg = <0x0 0x0f120000 0x0 0x1000>; + ranges = <0x0 0x0 0x0 0x0 0x20000000>; clock-frequency = <19200000>; frame@f121000 { frame-number = <0>; interrupts = , ; - reg = <0x0f121000 0x1000>, - <0x0f122000 0x1000>; + reg = <0x0 0x0f121000 0x1000>, + <0x0 0x0f122000 0x1000>; }; frame@f123000 { frame-number = <1>; interrupts = ; - reg = <0x0f123000 0x1000>; + reg = <0x0 0x0f123000 0x1000>; status = "disabled"; }; frame@f124000 { frame-number = <2>; interrupts = ; - reg = <0x0f124000 0x1000>; + reg = <0x0 0x0f124000 0x1000>; status = "disabled"; }; frame@f125000 { frame-number = <3>; interrupts = ; - reg = <0x0f125000 0x1000>; + reg = <0x0 0x0f125000 0x1000>; status = "disabled"; }; frame@f126000 { frame-number = <4>; interrupts = ; - reg = <0x0f126000 0x1000>; + reg = <0x0 0x0f126000 0x1000>; status = "disabled"; }; frame@f127000 { frame-number = <5>; interrupts = ; - reg = <0x0f127000 0x1000>; + reg = <0x0 0x0f127000 0x1000>; status = "disabled"; }; frame@f128000 { frame-number = <6>; interrupts = ; - reg = <0x0f128000 0x1000>; + reg = <0x0 0x0f128000 0x1000>; status = "disabled"; }; }; intc: interrupt-controller@f200000 { compatible = "arm,gic-v3"; - reg = <0x0f200000 0x20000>, - <0x0f300000 0x100000>; + reg = <0x0 0x0f200000 0x0 0x20000>, + <0x0 0x0f300000 0x0 0x100000>; #interrupt-cells = <3>; interrupt-controller; interrupts = ; diff --git a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts index 23f950067a08b..ce97e5bddecb1 100644 --- a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts +++ b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts @@ -48,7 +48,7 @@ stdout-path = "serial0:115200n8"; - framebuffer0: framebuffer@a000000 { + framebuffer0: framebuffer@a0000000 { compatible = "simple-framebuffer"; reg = <0 0xa0000000 0 (2340 * 1080 * 4)>; width = <1080>; @@ -1247,8 +1247,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l22a>; - vdda-pll-supply = <&vreg_l16a>; + vdda-phy-supply = <&vreg_l16a>; + vdda-pll-supply = <&vreg_l22a>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8150-hdk.dts b/arch/arm64/boot/dts/qcom/sm8150-hdk.dts index 8fb04ce72f91e..cdced5c848d3a 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-hdk.dts @@ -801,8 +801,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l18a_0p8>; + vdda-phy-supply = <&vreg_l18a_0p8>; + vdda-pll-supply = <&vreg_l3c_1p2>; orientation-switch; }; @@ -812,8 +812,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l5a_0p875>; + vdda-phy-supply = <&vreg_l5a_0p875>; + vdda-pll-supply = <&vreg_l3c_1p2>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts b/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts index 835ef929ff2df..f37e11bde067d 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts @@ -520,8 +520,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vdda_usb_ss_dp_core_1>; + vdda-phy-supply = <&vdda_usb_ss_dp_core_1>; + vdda-pll-supply = <&vreg_l3c_1p2>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8150-mtp.dts b/arch/arm64/boot/dts/qcom/sm8150-mtp.dts index 0f2d511624a8b..6a6564d8d7f7c 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-mtp.dts @@ -438,8 +438,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vdda_usb_ss_dp_core_1>; + vdda-phy-supply = <&vdda_usb_ss_dp_core_1>; + vdda-pll-supply = <&vreg_l3c_1p2>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi b/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi index 70fd6455518b9..6c2e2bf869d0d 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi @@ -869,6 +869,6 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l3c_1p2>; - vdda-pll-supply = <&vreg_l18a_0p8>; + vdda-phy-supply = <&vreg_l18a_0p8>; + vdda-pll-supply = <&vreg_l3c_1p2>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi index 0e101096209ab..109a76f4ca75f 100644 --- a/arch/arm64/boot/dts/qcom/sm8150.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi @@ -1893,8 +1893,8 @@ "bus_slave", "slave_q2a"; - iommu-map = <0x0 &apps_smmu 0x1d80 0x1>, - <0x100 &apps_smmu 0x1d81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1d80 0x0 0x1>, + <0x100 &apps_smmu 0x1d81 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2011,8 +2011,8 @@ assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1e00 0x1>, - <0x100 &apps_smmu 0x1e01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1e00 0x0 0x1>, + <0x100 &apps_smmu 0x1e01 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; diff --git a/arch/arm64/boot/dts/qcom/sm8250-hdk.dts b/arch/arm64/boot/dts/qcom/sm8250-hdk.dts index 3ea9d2b1a7d58..c1c673086df51 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-hdk.dts @@ -438,14 +438,14 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p92>; + vdda-phy-supply = <&vreg_l18a_0p92>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p92>; + vdda-phy-supply = <&vreg_l18a_0p92>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8250-mtp.dts b/arch/arm64/boot/dts/qcom/sm8250-mtp.dts index 51779b99176d0..14c8ca602110f 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-mtp.dts @@ -862,8 +862,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p9>; + vdda-phy-supply = <&vreg_l18a_0p9>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &usb_2 { @@ -885,8 +885,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p9>; + vdda-phy-supply = <&vreg_l18a_0p9>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi b/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi index d8289b2698f37..51c0caaf4ea12 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi @@ -715,8 +715,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p9>; + vdda-phy-supply = <&vreg_l18a_0p9>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi index 51b57c697a753..3d48467c52c99 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi @@ -28,7 +28,7 @@ /* required for bootloader to select correct board */ qcom,msm-id = ; /* SM8250 v2.1 */ - qcom,board-id = <0x10008 0>; + qcom,board-id = <0x2f 0>; aliases { serial0 = &uart6; diff --git a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-pipa.dts b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-pipa.dts index 078ba13f8762e..f11e4e244dffd 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-pipa.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-pipa.dts @@ -515,8 +515,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l9a_1p2>; - vdda-pll-supply = <&vreg_l18a_0p9>; + vdda-phy-supply = <&vreg_l18a_0p9>; + vdda-pll-supply = <&vreg_l9a_1p2>; }; &ufs_mem_hc { diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi index 7076720413ab2..ab461b5883c73 100644 --- a/arch/arm64/boot/dts/qcom/sm8250.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi @@ -2190,8 +2190,8 @@ "tbu", "ddrss_sf_tbu"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2317,8 +2317,8 @@ assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; @@ -2444,8 +2444,8 @@ assigned-clocks = <&gcc GCC_PCIE_2_AUX_CLK>; assigned-clock-rates = <19200000>; - iommu-map = <0x0 &apps_smmu 0x1d00 0x1>, - <0x100 &apps_smmu 0x1d01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1d00 0x0 0x1>, + <0x100 &apps_smmu 0x1d01 0x0 0x1>; resets = <&gcc GCC_PCIE_2_BCR>; reset-names = "pci"; @@ -4326,8 +4326,12 @@ interrupts = ; power-domains = <&videocc MVS0C_GDSC>, <&videocc MVS0_GDSC>, - <&rpmhpd RPMHPD_MX>; - power-domain-names = "venus", "vcodec0", "mx"; + <&rpmhpd RPMHPD_MX>, + <&rpmhpd RPMHPD_MMCX>; + power-domain-names = "venus", + "vcodec0", + "mx", + "mmcx"; operating-points-v2 = <&venus_opp_table>; clocks = <&gcc GCC_VIDEO_AXI0_CLK>, @@ -4351,24 +4355,28 @@ venus_opp_table: opp-table { compatible = "operating-points-v2"; - opp-720000000 { - opp-hz = /bits/ 64 <720000000>; - required-opps = <&rpmhpd_opp_low_svs>; + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_svs>, + <&rpmhpd_opp_low_svs>; }; - opp-1014000000 { - opp-hz = /bits/ 64 <1014000000>; - required-opps = <&rpmhpd_opp_svs>; + opp-338000000 { + opp-hz = /bits/ 64 <338000000>; + required-opps = <&rpmhpd_opp_svs>, + <&rpmhpd_opp_svs>; }; - opp-1098000000 { - opp-hz = /bits/ 64 <1098000000>; - required-opps = <&rpmhpd_opp_svs_l1>; + opp-366000000 { + opp-hz = /bits/ 64 <366000000>; + required-opps = <&rpmhpd_opp_svs_l1>, + <&rpmhpd_opp_svs_l1>; }; - opp-1332000000 { - opp-hz = /bits/ 64 <1332000000>; - required-opps = <&rpmhpd_opp_nom>; + opp-444000000 { + opp-hz = /bits/ 64 <444000000>; + required-opps = <&rpmhpd_opp_svs_l1>, + <&rpmhpd_opp_nom>; }; }; }; diff --git a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts index 5f975d0094658..b43dbb5921b18 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts @@ -860,8 +860,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p88>; + vdda-phy-supply = <&vreg_l1b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &usb_1_qmpphy_out { @@ -890,8 +890,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l5b_0p88>; + vdda-phy-supply = <&vreg_l5b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; /* PINCTRL - additions to nodes defined in sm8350.dtsi */ diff --git a/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts b/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts index 3bd5e57cbcdaa..57bb8c322e8b0 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts +++ b/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts @@ -358,8 +358,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p88>; + vdda-phy-supply = <&vreg_l1b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &usb_2 { @@ -377,6 +377,6 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l5b_0p88>; + vdda-phy-supply = <&vreg_l5b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8350-mtp.dts b/arch/arm64/boot/dts/qcom/sm8350-mtp.dts index 8bee57f3b25a1..859985e561aa5 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8350-mtp.dts @@ -354,8 +354,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p88>; + vdda-phy-supply = <&vreg_l1b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &usb_2 { @@ -373,8 +373,8 @@ &usb_2_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l5b_0p88>; + vdda-phy-supply = <&vreg_l5b_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &ipa { diff --git a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi index 7ae1eb0a7ccea..1f9d094fee8fc 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi @@ -917,6 +917,6 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&pm8350_l6>; - vdda-pll-supply = <&pm8350_l1>; + vdda-phy-supply = <&pm8350_l1>; + vdda-pll-supply = <&pm8350_l6>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi index c830953156ec6..20c3ab9465d2e 100644 --- a/arch/arm64/boot/dts/qcom/sm8350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi @@ -1575,8 +1575,8 @@ "aggre1", "aggre0"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -1684,8 +1684,8 @@ "ddrss_sf_tbu", "aggre1"; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; diff --git a/arch/arm64/boot/dts/qcom/sm8450-hdk.dts b/arch/arm64/boot/dts/qcom/sm8450-hdk.dts index b37998cd9a2c3..4ade28f40cae0 100644 --- a/arch/arm64/boot/dts/qcom/sm8450-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8450-hdk.dts @@ -1214,8 +1214,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p91>; + vdda-phy-supply = <&vreg_l1b_0p91>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &usb_1_qmpphy_out { diff --git a/arch/arm64/boot/dts/qcom/sm8450-qrd.dts b/arch/arm64/boot/dts/qcom/sm8450-qrd.dts index 56db5f79f59d2..4993059483e9a 100644 --- a/arch/arm64/boot/dts/qcom/sm8450-qrd.dts +++ b/arch/arm64/boot/dts/qcom/sm8450-qrd.dts @@ -527,8 +527,8 @@ &usb_1_qmpphy { status = "okay"; - vdda-phy-supply = <&vreg_l6b_1p2>; - vdda-pll-supply = <&vreg_l1b_0p91>; + vdda-phy-supply = <&vreg_l1b_0p91>; + vdda-pll-supply = <&vreg_l6b_1p2>; }; &usb_1_qmpphy_out { diff --git a/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara.dtsi b/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara.dtsi index 6bd315e109925..4fb5627d81264 100644 --- a/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara.dtsi @@ -793,7 +793,7 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&pm8350_l6>; - vdda-pll-supply = <&pm8350_l1>; + vdda-phy-supply = <&pm8350_l1>; + vdda-pll-supply = <&pm8350_l6>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi index 56cb6e959e4ee..18ab94de4ac9e 100644 --- a/arch/arm64/boot/dts/qcom/sm8450.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi @@ -2023,8 +2023,8 @@ "aggre0", "aggre1"; - iommu-map = <0x0 &apps_smmu 0x1c00 0x1>, - <0x100 &apps_smmu 0x1c01 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c00 0x0 0x1>, + <0x100 &apps_smmu 0x1c01 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2188,8 +2188,8 @@ "ddrss_sf_tbu", "aggre1"; - iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, - <0x100 &apps_smmu 0x1c81 0x1>; + iommu-map = <0x0 &apps_smmu 0x1c80 0x0 0x1>, + <0x100 &apps_smmu 0x1c81 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>; reset-names = "pci"; diff --git a/arch/arm64/boot/dts/qcom/sm8550-hdk.dts b/arch/arm64/boot/dts/qcom/sm8550-hdk.dts index ee13e6136a825..25ff5e39dd346 100644 --- a/arch/arm64/boot/dts/qcom/sm8550-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8550-hdk.dts @@ -1374,8 +1374,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3f_0p88>; + vdda-phy-supply = <&vreg_l3f_0p88>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8550-mtp.dts b/arch/arm64/boot/dts/qcom/sm8550-mtp.dts index 5769be83cfbd3..7c634572c6b71 100644 --- a/arch/arm64/boot/dts/qcom/sm8550-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8550-mtp.dts @@ -971,8 +971,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3f_0p91>; + vdda-phy-supply = <&vreg_l3f_0p91>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8550-qrd.dts b/arch/arm64/boot/dts/qcom/sm8550-qrd.dts index 2fb2e0be5e4c6..ceb5cd3c5bf3e 100644 --- a/arch/arm64/boot/dts/qcom/sm8550-qrd.dts +++ b/arch/arm64/boot/dts/qcom/sm8550-qrd.dts @@ -1274,8 +1274,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3f_0p88>; + vdda-phy-supply = <&vreg_l3f_0p88>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts b/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts index 0e6ed6fce6147..367f6eae5e652 100644 --- a/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts +++ b/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts @@ -766,8 +766,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&pm8550vs_2_l3>; - vdda-pll-supply = <&pm8550ve_l3>; + vdda-phy-supply = <&pm8550ve_l3>; + vdda-pll-supply = <&pm8550vs_2_l3>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi index 396201905ef25..69babd26c6789 100644 --- a/arch/arm64/boot/dts/qcom/sm8550.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi @@ -2382,8 +2382,8 @@ msi-map = <0x0 &gic_its 0x1400 0x1>, <0x100 &gic_its 0x1401 0x1>; - iommu-map = <0x0 &apps_smmu 0x1400 0x1>, - <0x100 &apps_smmu 0x1401 0x1>; + iommu-map = <0x0 &apps_smmu 0x1400 0x0 0x1>, + <0x100 &apps_smmu 0x1401 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2561,8 +2561,8 @@ msi-map = <0x0 &gic_its 0x1480 0x1>, <0x100 &gic_its 0x1481 0x1>; - iommu-map = <0x0 &apps_smmu 0x1480 0x1>, - <0x100 &apps_smmu 0x1481 0x1>; + iommu-map = <0x0 &apps_smmu 0x1480 0x0 0x1>, + <0x100 &apps_smmu 0x1481 0x0 0x1>; resets = <&gcc GCC_PCIE_1_BCR>, <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; diff --git a/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts b/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts index 0dc994f4e48d9..6ddf30ecabfef 100644 --- a/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts +++ b/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts @@ -1536,8 +1536,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3i_1p2>; - vdda-pll-supply = <&vreg_l3g_0p91>; + vdda-phy-supply = <&vreg_l3g_0p91>; + vdda-pll-supply = <&vreg_l3i_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8650-hdk.dts b/arch/arm64/boot/dts/qcom/sm8650-hdk.dts index eabc828c05b4c..865ee15be9e21 100644 --- a/arch/arm64/boot/dts/qcom/sm8650-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8650-hdk.dts @@ -1338,8 +1338,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3i_1p2>; - vdda-pll-supply = <&vreg_l3g_0p91>; + vdda-phy-supply = <&vreg_l3g_0p91>; + vdda-pll-supply = <&vreg_l3i_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8650-mtp.dts b/arch/arm64/boot/dts/qcom/sm8650-mtp.dts index dd6e33d2dc5d7..26045430d34ad 100644 --- a/arch/arm64/boot/dts/qcom/sm8650-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8650-mtp.dts @@ -886,8 +886,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3i_1p2>; - vdda-pll-supply = <&vreg_l3g_0p91>; + vdda-phy-supply = <&vreg_l3g_0p91>; + vdda-pll-supply = <&vreg_l3i_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8650-qrd.dts b/arch/arm64/boot/dts/qcom/sm8650-qrd.dts index a3982ae229290..c00e3ae208cd8 100644 --- a/arch/arm64/boot/dts/qcom/sm8650-qrd.dts +++ b/arch/arm64/boot/dts/qcom/sm8650-qrd.dts @@ -1357,8 +1357,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3i_1p2>; - vdda-pll-supply = <&vreg_l3g_0p91>; + vdda-phy-supply = <&vreg_l3g_0p91>; + vdda-pll-supply = <&vreg_l3i_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8650.dtsi b/arch/arm64/boot/dts/qcom/sm8650.dtsi index 65c4a2b46d215..b01993ba82630 100644 --- a/arch/arm64/boot/dts/qcom/sm8650.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8650.dtsi @@ -3626,8 +3626,8 @@ operating-points-v2 = <&pcie0_opp_table>; - iommu-map = <0 &apps_smmu 0x1400 0x1>, - <0x100 &apps_smmu 0x1401 0x1>; + iommu-map = <0 &apps_smmu 0x1400 0x0 0x1>, + <0x100 &apps_smmu 0x1401 0x0 0x1>; interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH 0>, <0 0 0 2 &intc 0 0 GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH 0>, @@ -3819,8 +3819,8 @@ operating-points-v2 = <&pcie1_opp_table>; - iommu-map = <0 &apps_smmu 0x1480 0x1>, - <0x100 &apps_smmu 0x1481 0x1>; + iommu-map = <0 &apps_smmu 0x1480 0x0 0x1>, + <0x100 &apps_smmu 0x1481 0x0 0x1>; interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH 0>, <0 0 0 2 &intc 0 0 GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH 0>, diff --git a/arch/arm64/boot/dts/qcom/sm8750-mtp.dts b/arch/arm64/boot/dts/qcom/sm8750-mtp.dts index 3837f6785320d..9b5fbc004df4e 100644 --- a/arch/arm64/boot/dts/qcom/sm8750-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8750-mtp.dts @@ -1404,8 +1404,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3g_1p2>; - vdda-pll-supply = <&vreg_l2d_0p88>; + vdda-phy-supply = <&vreg_l2d_0p88>; + vdda-pll-supply = <&vreg_l3g_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8750-qrd.dts b/arch/arm64/boot/dts/qcom/sm8750-qrd.dts index 801c46d556022..b56317cfa9fc7 100644 --- a/arch/arm64/boot/dts/qcom/sm8750-qrd.dts +++ b/arch/arm64/boot/dts/qcom/sm8750-qrd.dts @@ -1132,8 +1132,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3g_1p2>; - vdda-pll-supply = <&vreg_l2d_0p88>; + vdda-phy-supply = <&vreg_l2d_0p88>; + vdda-pll-supply = <&vreg_l3g_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi index fafed417c66fc..c15ad5de0aa87 100644 --- a/arch/arm64/boot/dts/qcom/sm8750.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi @@ -5280,8 +5280,8 @@ interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &apps_smmu 0x1400 0x1>, - <0x100 &apps_smmu 0x1401 0x1>; + iommu-map = <0x0 &apps_smmu 0x1400 0x0 0x1>, + <0x100 &apps_smmu 0x1401 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -5445,6 +5445,7 @@ phy-names = "ufsphy"; #reset-cells = <1>; + qcom,ice = <&ice>; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi b/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi index 294354c034c37..6469c6fa9f634 100644 --- a/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi +++ b/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi @@ -589,8 +589,8 @@ }; &usb_qmpphy_2 { - vdda-phy-supply = <&vreg_l11a>; - vdda-pll-supply = <&vreg_l5a>; + vdda-phy-supply = <&vreg_l5a>; + vdda-pll-supply = <&vreg_l11a>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/talos.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi index fb1bbc51bb8a4..23990890280bd 100644 --- a/arch/arm64/boot/dts/qcom/talos.dtsi +++ b/arch/arm64/boot/dts/qcom/talos.dtsi @@ -351,7 +351,7 @@ }; opp-1094400000 { - opp-hz = /bits/ 64 <109440000>; + opp-hz = /bits/ 64 <1094400000>; opp-peak-kBps = <(1017600 * 4) (940800 * 16)>; }; @@ -1349,8 +1349,8 @@ &config_noc SLAVE_PCIE_0 QCOM_ICC_TAG_ACTIVE_ONLY>; interconnect-names = "pcie-mem", "cpu-pcie"; - iommu-map = <0x0 &apps_smmu 0x400 0x1>, - <0x100 &apps_smmu 0x401 0x1>; + iommu-map = <0x0 &apps_smmu 0x400 0x0 0x1>, + <0x100 &apps_smmu 0x401 0x0 0x1>; resets = <&gcc GCC_PCIE_0_BCR>; reset-names = "pci"; @@ -2017,7 +2017,7 @@ }; }; - gmu: gmu@506a000 { + gmu: gmu@506d000 { compatible = "qcom,adreno-rgmu-612.0", "qcom,adreno-rgmu"; reg = <0x0 0x0506d000 0x0 0x2c000>; diff --git a/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi b/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi index 48c4ad648354e..0dc7c1b5a14a0 100644 --- a/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi @@ -1226,8 +1226,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1256,8 +1256,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1277,8 +1277,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1342,15 +1342,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi index 66d566808f583..59cbbfbce4905 100644 --- a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi @@ -1454,8 +1454,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p9>; + vdda-phy-supply = <&vreg_l1j_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1484,8 +1484,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1505,8 +1505,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1551,15 +1551,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1-crd.dtsi b/arch/arm64/boot/dts/qcom/x1-crd.dtsi index a9c5c523575ee..107fe2867e736 100644 --- a/arch/arm64/boot/dts/qcom/x1-crd.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-crd.dtsi @@ -1734,8 +1734,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1764,8 +1764,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1794,8 +1794,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1836,15 +1836,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi index db291730130c7..4cda708455d84 100644 --- a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi @@ -589,6 +589,7 @@ regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; regulator-initial-mode = ; + regulator-always-on; }; vreg_l13b_3p0: ldo13 { @@ -610,6 +611,7 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; + regulator-always-on; }; }; @@ -1568,8 +1570,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1598,8 +1600,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1637,8 +1639,8 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; @@ -1651,8 +1653,8 @@ }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi b/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi index 02708f23a8657..dba8cee43e294 100644 --- a/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi @@ -1466,8 +1466,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; @@ -1496,8 +1496,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; @@ -1533,8 +1533,8 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi b/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi index 7559557610ed4..ce7d958ca6879 100644 --- a/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi @@ -1268,8 +1268,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1298,8 +1298,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts b/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts index 2e38402e2c140..be1a8ac8c699e 100644 --- a/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts +++ b/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts @@ -1382,8 +1382,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1413,8 +1413,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1443,8 +1443,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1486,15 +1486,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi b/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi index 5d49df41be02a..0bee2a3d6e7a9 100644 --- a/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi +++ b/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi @@ -1601,8 +1601,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1631,8 +1631,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1652,8 +1652,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1717,15 +1717,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts b/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts index ce7b10ea89b6d..2be5a37c00ccc 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts @@ -1286,8 +1286,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p9>; + vdda-phy-supply = <&vreg_l1j_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1316,8 +1316,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1357,15 +1357,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts b/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts index beb1475d7fa0c..9eb2dfd99f20d 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts @@ -1564,8 +1564,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1594,8 +1594,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1624,8 +1624,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts b/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts index f95b1f9f439d3..e4a0b86847cdd 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts @@ -1407,8 +1407,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p9>; + vdda-phy-supply = <&vreg_l1j_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1428,8 +1428,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1497,16 +1497,16 @@ /* Right side USB-A (USB-SS 3) */ &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; /* Left side USB-A (USB-SS 4) */ &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p9>; + vdda-phy-supply = <&vreg_l3c_0p9>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi b/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi index 28342cb84ded7..13d0f09e0c2a1 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi +++ b/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi @@ -1491,8 +1491,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j>; - vdda-pll-supply = <&vreg_l1j>; + vdda-phy-supply = <&vreg_l1j>; + vdda-pll-supply = <&vreg_l2j>; status = "okay"; }; @@ -1521,8 +1521,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j>; - vdda-pll-supply = <&vreg_l2d>; + vdda-phy-supply = <&vreg_l2d>; + vdda-pll-supply = <&vreg_l2j>; status = "okay"; }; @@ -1565,15 +1565,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e>; - vdda-pll-supply = <&vreg_l3c>; + vdda-phy-supply = <&vreg_l3c>; + vdda-pll-supply = <&vreg_l3e>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e>; - vdda-pll-supply = <&vreg_l3c>; + vdda-phy-supply = <&vreg_l3c>; + vdda-pll-supply = <&vreg_l3e>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts b/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts index 8afbac349cc91..78380fec8839e 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts @@ -1416,8 +1416,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1446,8 +1446,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1476,8 +1476,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1519,15 +1519,15 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts b/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts index 500809772097e..913645bc3595d 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts +++ b/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts @@ -1561,8 +1561,8 @@ }; &usb_1_ss0_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l1j_0p8>; + vdda-phy-supply = <&vreg_l1j_0p8>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1591,8 +1591,8 @@ }; &usb_1_ss1_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; status = "okay"; }; @@ -1631,8 +1631,8 @@ }; &usb_1_ss2_qmpphy { - vdda-phy-supply = <&vreg_l2j_1p2>; - vdda-pll-supply = <&vreg_l2d_0p9>; + vdda-phy-supply = <&vreg_l2d_0p9>; + vdda-pll-supply = <&vreg_l2j_1p2>; /delete-property/ mode-switch; /delete-property/ orientation-switch; @@ -1685,8 +1685,8 @@ }; &usb_mp_qmpphy0 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; @@ -1701,8 +1701,8 @@ }; &usb_mp_qmpphy1 { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3c_0p8>; + vdda-phy-supply = <&vreg_l3c_0p8>; + vdda-pll-supply = <&vreg_l3e_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/renesas/r8a779f0.dtsi b/arch/arm64/boot/dts/renesas/r8a779f0.dtsi index cbb161c863ac7..a5118c2f27429 100644 --- a/arch/arm64/boot/dts/renesas/r8a779f0.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779f0.dtsi @@ -901,6 +901,7 @@ clocks = <&cpg CPG_MOD 1514>, <&ufs30_clk>; clock-names = "fck", "ref_clk"; freq-table-hz = <200000000 200000000>, <38400000 38400000>; + lanes-per-direction = <1>; power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; resets = <&cpg 1514>; status = "disabled"; diff --git a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi index b48da8534a3df..4de93fceb605a 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi @@ -931,7 +931,7 @@ reg = <0 0x13400000 0 0x10000>; ranges = <0x02000000 0 0x30000000 0 0x30000000 0 0x08000000>, <0x43000000 4 0x40000000 4 0x40000000 6 0x00000000>; - dma-ranges = <0x42000000 0 0x40000000 0 0x40000000 2 0x00000000>; + dma-ranges = <0x43000000 0 0x40000000 0 0x40000000 2 0x00000000>; bus-range = <0x0 0xff>; interrupts = , , @@ -1394,23 +1394,28 @@ mtl_tx_setup0: tx-queues-config { snps,tx-queues-to-use = <4>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; snps,priority = <0x1>; }; queue1 { + snps,weight = <0x12>; snps,dcb-algorithm; snps,priority = <0x2>; }; queue2 { + snps,weight = <0x14>; snps,dcb-algorithm; snps,priority = <0x4>; }; queue3 { + snps,weight = <0x18>; snps,dcb-algorithm; snps,priority = <0x8>; }; @@ -1494,23 +1499,28 @@ mtl_tx_setup1: tx-queues-config { snps,tx-queues-to-use = <4>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; snps,priority = <0x1>; }; queue1 { + snps,weight = <0x12>; snps,dcb-algorithm; snps,priority = <0x2>; }; queue2 { + snps,weight = <0x14>; snps,dcb-algorithm; snps,priority = <0x4>; }; queue3 { + snps,weight = <0x18>; snps,dcb-algorithm; snps,priority = <0x8>; }; diff --git a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi index d6c8c39df2a4d..6e1444b80a28f 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi @@ -1049,7 +1049,7 @@ reg = <0 0x13400000 0 0x10000>; ranges = <0x02000000 0 0x30000000 0 0x30000000 0 0x8000000>, <0x43000000 4 0x40000000 4 0x40000000 6 0x00000000>; - dma-ranges = <0x42000000 0 0x40000000 0 0x40000000 2 0x00000000>; + dma-ranges = <0x43000000 0 0x40000000 0 0x40000000 2 0x00000000>; bus-range = <0x0 0xff>; interrupts = , , @@ -1569,23 +1569,28 @@ mtl_tx_setup0: tx-queues-config { snps,tx-queues-to-use = <4>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; snps,priority = <0x1>; }; queue1 { + snps,weight = <0x12>; snps,dcb-algorithm; snps,priority = <0x2>; }; queue2 { + snps,weight = <0x14>; snps,dcb-algorithm; snps,priority = <0x4>; }; queue3 { + snps,weight = <0x18>; snps,dcb-algorithm; snps,priority = <0x8>; }; @@ -1670,23 +1675,28 @@ mtl_tx_setup1: tx-queues-config { snps,tx-queues-to-use = <4>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; snps,priority = <0x1>; }; queue1 { + snps,weight = <0x12>; snps,dcb-algorithm; snps,priority = <0x2>; }; queue2 { + snps,weight = <0x14>; snps,dcb-algorithm; snps,priority = <0x4>; }; queue3 { + snps,weight = <0x18>; snps,dcb-algorithm; snps,priority = <0x8>; }; diff --git a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi index 1e94366bb7eee..99d197f309d00 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi @@ -1559,23 +1559,28 @@ mtl_tx_setup0: tx-queues-config { snps,tx-queues-to-use = <4>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; snps,priority = <0x1>; }; queue1 { + snps,weight = <0x12>; snps,dcb-algorithm; snps,priority = <0x2>; }; queue2 { + snps,weight = <0x14>; snps,dcb-algorithm; snps,priority = <0x4>; }; queue3 { + snps,weight = <0x18>; snps,dcb-algorithm; snps,priority = <0x8>; }; @@ -1660,23 +1665,28 @@ mtl_tx_setup1: tx-queues-config { snps,tx-queues-to-use = <4>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; snps,priority = <0x1>; }; queue1 { + snps,weight = <0x12>; snps,dcb-algorithm; snps,priority = <0x2>; }; queue2 { + snps,weight = <0x14>; snps,dcb-algorithm; snps,priority = <0x4>; }; queue3 { + snps,weight = <0x18>; snps,dcb-algorithm; snps,priority = <0x8>; }; diff --git a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi index 40494159831d8..bac39390ead74 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi @@ -642,36 +642,45 @@ mtl_tx_setup0: tx-queues-config { snps,tx-queues-to-use = <8>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; }; queue1 { + snps,weight = <0x11>; snps,dcb-algorithm; }; queue2 { + snps,weight = <0x12>; snps,dcb-algorithm; }; queue3 { + snps,weight = <0x13>; snps,dcb-algorithm; }; queue4 { + snps,weight = <0x14>; snps,dcb-algorithm; }; queue5 { + snps,weight = <0x15>; snps,dcb-algorithm; }; queue6 { + snps,weight = <0x16>; snps,dcb-algorithm; }; queue7 { + snps,weight = <0x17>; snps,dcb-algorithm; }; }; @@ -788,36 +797,45 @@ mtl_tx_setup1: tx-queues-config { snps,tx-queues-to-use = <8>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; }; queue1 { + snps,weight = <0x11>; snps,dcb-algorithm; }; queue2 { + snps,weight = <0x12>; snps,dcb-algorithm; }; queue3 { + snps,weight = <0x13>; snps,dcb-algorithm; }; queue4 { + snps,weight = <0x14>; snps,dcb-algorithm; }; queue5 { + snps,weight = <0x15>; snps,dcb-algorithm; }; queue6 { + snps,weight = <0x16>; snps,dcb-algorithm; }; queue7 { + snps,weight = <0x17>; snps,dcb-algorithm; }; }; @@ -934,36 +952,45 @@ mtl_tx_setup2: tx-queues-config { snps,tx-queues-to-use = <8>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; }; queue1 { + snps,weight = <0x11>; snps,dcb-algorithm; }; queue2 { + snps,weight = <0x12>; snps,dcb-algorithm; }; queue3 { + snps,weight = <0x13>; snps,dcb-algorithm; }; queue4 { + snps,weight = <0x14>; snps,dcb-algorithm; }; queue5 { + snps,weight = <0x15>; snps,dcb-algorithm; }; queue6 { + snps,weight = <0x16>; snps,dcb-algorithm; }; queue7 { + snps,weight = <0x17>; snps,dcb-algorithm; }; }; diff --git a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts index e9ed2de128f6f..987e44d0bf957 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts +++ b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts @@ -256,8 +256,7 @@ /* * GMAC1 Pin Configuration: * - * SW2[8] ON - use pins P33_2-P33_7, P34_0-P34_5, P34_7 and - * P35_0-P35_2 for Ethernet port 3 + * SW2[8] ON - use pins P33_2-P33_7 and P34_0-P34_5 for Ethernet port 3 */ gmac1_pins: gmac1-pins { pinmux = , /* ETH3_TXCLK */ @@ -272,10 +271,6 @@ , /* ETH3_RXD2 */ , /* ETH3_RXD3 */ , /* ETH3_RXDV */ - , /* ETH3_TXER */ - , /* ETH3_RXER */ - , /* ETH3_CRS */ - , /* ETH3_COL */ , /* GMAC1_MDC */ , /* GMAC1_MDIO */ , /* ETH3_REFCLK */ @@ -286,8 +281,7 @@ * GMAC2 Pin Configuration: * * SW2[6] OFF - connect MDC/MDIO of Ethernet port 2 to GMAC2 - * SW2[7] ON - use pins P29_1-P29_7, P30_0-P30_4, and P31_2-P31_5 - * for Ethernet port 2 + * SW2[7] ON - use pins P29_1-P29_7 and P30_0-P30_4 for Ethernet port 2 */ gmac2_pins: gmac2-pins { pinmux = , /* ETH2_TXCLK */ @@ -302,10 +296,6 @@ , /* ETH2_RXD2 */ , /* ETH2_RXD3 */ , /* ETH2_RXDV */ - , /* ETH2_TXER */ - , /* ETH2_RXER */ - , /* ETH2_CRS */ - , /* ETH2_COL */ , /* GMAC2_MDC */ , /* GMAC2_MDIO */ , /* ETH2_REFCLK */ diff --git a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi index e8d4f76949ccb..03b976d93e105 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi @@ -643,36 +643,45 @@ mtl_tx_setup0: tx-queues-config { snps,tx-queues-to-use = <8>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; }; queue1 { + snps,weight = <0x11>; snps,dcb-algorithm; }; queue2 { + snps,weight = <0x12>; snps,dcb-algorithm; }; queue3 { + snps,weight = <0x13>; snps,dcb-algorithm; }; queue4 { + snps,weight = <0x14>; snps,dcb-algorithm; }; queue5 { + snps,weight = <0x15>; snps,dcb-algorithm; }; queue6 { + snps,weight = <0x16>; snps,dcb-algorithm; }; queue7 { + snps,weight = <0x17>; snps,dcb-algorithm; }; }; @@ -790,36 +799,45 @@ mtl_tx_setup1: tx-queues-config { snps,tx-queues-to-use = <8>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; }; queue1 { + snps,weight = <0x11>; snps,dcb-algorithm; }; queue2 { + snps,weight = <0x12>; snps,dcb-algorithm; }; queue3 { + snps,weight = <0x13>; snps,dcb-algorithm; }; queue4 { + snps,weight = <0x14>; snps,dcb-algorithm; }; queue5 { + snps,weight = <0x15>; snps,dcb-algorithm; }; queue6 { + snps,weight = <0x16>; snps,dcb-algorithm; }; queue7 { + snps,weight = <0x17>; snps,dcb-algorithm; }; }; @@ -937,36 +955,45 @@ mtl_tx_setup2: tx-queues-config { snps,tx-queues-to-use = <8>; + snps,tx-sched-wrr; queue0 { + snps,weight = <0x10>; snps,dcb-algorithm; }; queue1 { + snps,weight = <0x11>; snps,dcb-algorithm; }; queue2 { + snps,weight = <0x12>; snps,dcb-algorithm; }; queue3 { + snps,weight = <0x13>; snps,dcb-algorithm; }; queue4 { + snps,weight = <0x14>; snps,dcb-algorithm; }; queue5 { + snps,weight = <0x15>; snps,dcb-algorithm; }; queue6 { + snps,weight = <0x16>; snps,dcb-algorithm; }; queue7 { + snps,weight = <0x17>; snps,dcb-algorithm; }; }; diff --git a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts index ef6cc7497c2c4..a66502d8d82bd 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts +++ b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts @@ -339,9 +339,7 @@ /* * GMAC1 Pin Configuration: * - * DSW5[8] ON - use pins P00_0-P00_2, P33_2-P33_7, P34_0-P34_6 - * for Ethernet port 3 - * DSW12[1] OFF; DSW12[2] ON - use pin P00_3 for Ethernet port 3 + * DSW5[8] ON - use pins P33_2-P33_7 and P34_0-P34_6 for Ethernet port 3 */ gmac1_pins: gmac1-pins { pinmux = , /* ETH3_TXCLK */ @@ -356,10 +354,6 @@ , /* ETH3_RXD2 */ , /* ETH3_RXD3 */ , /* ETH3_RXDV */ - , /* ETH3_TXER */ - , /* ETH3_RXER */ - , /* ETH3_CRS */ - , /* ETH3_COL */ , /* GMAC1_MDC */ , /* GMAC1_MDIO */ , /* ETH3_REFCLK */ @@ -370,8 +364,7 @@ * GMAC2 Pin Configuration: * * DSW5[6] OFF - connect MDC/MDIO of Ethernet port 2 to GMAC2 - * DSW5[7] ON - use pins P29_1-P29_7, P30_0-P30_4, P30_7, - * P31_2, P31_4 and P31_5 are used for Ethernet port 2 + * DSW5[7] ON - use pins P29_1-P29_7 and P30_0-P30_4 for Ethernet port 2 * DSW13[7] OFF; DSW13[8] ON - use pin P13_7 for IRQ14 */ gmac2_pins: gmac2-pins { @@ -387,10 +380,6 @@ , /* ETH2_RXD2 */ , /* ETH2_RXD3 */ , /* ETH2_RXDV */ - , /* ETH2_TXER */ - , /* ETH2_RXER */ - , /* ETH2_CRS */ - , /* ETH2_COL */ , /* GMAC2_MDC */ , /* GMAC2_MDIO */ , /* ETH2_REFCLK */ diff --git a/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi b/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi index add917af5de78..f14f9eca7d342 100644 --- a/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi @@ -35,7 +35,7 @@ compatible = "mmc-pwrseq-emmc"; pinctrl-0 = <&emmc_reset>; pinctrl-names = "default"; - reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_LOW>; }; gpio-leds { diff --git a/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi b/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi index 192791993f059..02200de695d31 100644 --- a/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi @@ -33,7 +33,7 @@ compatible = "mmc-pwrseq-emmc"; pinctrl-0 = <&emmc_reset>; pinctrl-names = "default"; - reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_LOW>; }; gpio-leds { diff --git a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi index 973b4c5880e24..29794216592d8 100644 --- a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi @@ -26,7 +26,7 @@ compatible = "mmc-pwrseq-emmc"; pinctrl-0 = <&emmc_reset>; pinctrl-names = "default"; - reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_LOW>; }; leds { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi index 3f3cb0eb58096..5435fbc270954 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi @@ -505,8 +505,7 @@ ap_i2c_tp: &i2c5 { mvl_wifi: wifi@0,0 { compatible = "pci1b4b,2b42"; reg = <0x0000 0x0 0x0 0x0 0x0>; - interrupt-parent = <&gpio0>; - interrupts = <8 IRQ_TYPE_LEVEL_LOW>; + interrupts-extended = <&gpio0 8 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&wlan_host_wake_l>; wakeup-source; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts index 8d26bd9b75003..d46cdfe3f7849 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts @@ -734,12 +734,6 @@ }; }; - wifi { - wifi_host_wake_l: wifi-host-wake-l { - rockchip,pins = <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - wireless-bluetooth { bt_wake_pin: bt-wake-pin { rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; @@ -766,19 +760,7 @@ pinctrl-names = "default"; pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; sd-uhs-sdr104; - #address-cells = <1>; - #size-cells = <0>; status = "okay"; - - brcmf: wifi@1 { - compatible = "brcm,bcm4329-fmac"; - reg = <1>; - interrupt-parent = <&gpio4>; - interrupts = ; - interrupt-names = "host-wake"; - pinctrl-names = "default"; - pinctrl-0 = <&wifi_host_wake_l>; - }; }; &pwm0 { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc-plus.dts b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc-plus.dts index 4f2831097624e..f6d62ee51f1b9 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc-plus.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc-plus.dts @@ -136,6 +136,18 @@ <3 RK_PD7 1 &pcfg_pull_none>; }; +&i2s0_8ch_bus_bclk_off { + rockchip,pins = + <3 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>, + <3 RK_PD1 1 &pcfg_pull_none>, + <3 RK_PD2 1 &pcfg_pull_none>, + <3 RK_PD3 1 &pcfg_pull_none>, + <3 RK_PD4 1 &pcfg_pull_none>, + <3 RK_PD5 1 &pcfg_pull_none>, + <3 RK_PD6 1 &pcfg_pull_none>, + <3 RK_PD7 1 &pcfg_pull_none>; +}; + &i2s1 { pinctrl-names = "default"; pinctrl-0 = <&i2s_8ch_mclk_pin>, <&i2s1_2ch_bus>; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi index b6cf03a7ba66b..04cf285e6c2af 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi @@ -569,7 +569,7 @@ reg = <0x48>; status = "okay"; ti,x-plate-ohms = <660>; - ti,rt-thr = <3000>; + ti,max-rt = <3000>; ti,fuzzx = <32>; ti,fuzzy = <16>; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi index fc1fdbfd31622..376ad04e07869 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi @@ -1910,8 +1910,8 @@ edp0: edp@fdec0000 { compatible = "rockchip,rk3588-edp"; reg = <0x0 0xfdec0000 0x0 0x1000>; - clocks = <&cru CLK_EDP0_24M>, <&cru PCLK_EDP0>; - clock-names = "dp", "pclk"; + clocks = <&cru CLK_EDP0_24M>, <&cru PCLK_EDP0>, <&cru HCLK_VO1>; + clock-names = "dp", "pclk", "hclk"; interrupts = ; phys = <&hdptxphy0>; phy-names = "dp"; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi index a2640014ee042..b251bb129cdbf 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi @@ -285,8 +285,8 @@ edp1: edp@fded0000 { compatible = "rockchip,rk3588-edp"; reg = <0x0 0xfded0000 0x0 0x1000>; - clocks = <&cru CLK_EDP1_24M>, <&cru PCLK_EDP1>; - clock-names = "dp", "pclk"; + clocks = <&cru CLK_EDP1_24M>, <&cru PCLK_EDP1>, <&cru HCLK_VO1>; + clock-names = "dp", "pclk", "hclk"; interrupts = ; phys = <&hdptxphy1>; phy-names = "dp"; diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts b/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts index d534d662c40f1..99853880aaac6 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts +++ b/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts @@ -23,16 +23,19 @@ compatible = "simple-audio-card"; pinctrl-names = "default"; pinctrl-0 = <&hp_detect>; + simple-audio-card,aux-devs = <&headphones_amp>; simple-audio-card,name = "rockchip,es8388"; - simple-audio-card,bitclock-master = <&masterdai>; + simple-audio-card,bitclock-master = <&cpudai>; simple-audio-card,format = "i2s"; - simple-audio-card,frame-master = <&masterdai>; + simple-audio-card,frame-master = <&cpudai>; simple-audio-card,hp-det-gpios = <&gpio1 RK_PA6 GPIO_ACTIVE_LOW>; simple-audio-card,mclk-fs = <256>; simple-audio-card,pin-switches = "Headphones"; simple-audio-card,routing = - "Headphones", "LOUT1", - "Headphones", "ROUT1", + "Headphones", "Headphone Amp OUTL", + "Headphones", "Headphone Amp OUTR", + "Headphone Amp INL", "LOUT2", + "Headphone Amp INR", "ROUT2", "LINPUT1", "Microphone Jack", "RINPUT1", "Microphone Jack", "LINPUT2", "Onboard Microphone", @@ -47,11 +50,17 @@ system-clock-frequency = <12288000>; }; - simple-audio-card,cpu { + cpudai: simple-audio-card,cpu { sound-dai = <&i2s0_8ch>; }; }; + headphones_amp: audio-amplifier-headphones { + compatible = "simple-audio-amplifier"; + enable-gpios = <&gpio1 RK_PA4 GPIO_ACTIVE_HIGH>; + sound-name-prefix = "Headphone Amp"; + }; + chosen { stdout-path = "serial2:1500000n8"; }; @@ -327,12 +336,12 @@ es8388: audio-codec@11 { compatible = "everest,es8388", "everest,es8328"; reg = <0x11>; - clocks = <&cru I2S1_8CH_MCLKOUT>; + clocks = <&cru I2S0_8CH_MCLKOUT>; AVDD-supply = <&vcc_3v3_s0>; DVDD-supply = <&vcc_1v8_s0>; HPVDD-supply = <&vcc_3v3_s0>; PVDD-supply = <&vcc_3v3_s0>; - assigned-clocks = <&cru I2S1_8CH_MCLKOUT>; + assigned-clocks = <&cru I2S0_8CH_MCLKOUT>; assigned-clock-rates = <12288000>; #sound-dai-cells = <0>; }; diff --git a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi index 1b1d3970888b8..a9a83781d40f5 100644 --- a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi @@ -1401,7 +1401,7 @@ icssg0_mdio: mdio@32400 { compatible = "ti,davinci_mdio"; reg = <0x32400 0x100>; - clocks = <&k3_clks 62 3>; + clocks = <&k3_clks 81 0>; clock-names = "fck"; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h index 71493b760b839..3c008821219c2 100644 --- a/arch/arm64/include/asm/cpu.h +++ b/arch/arm64/include/asm/cpu.h @@ -78,5 +78,6 @@ void __init cpuinfo_store_boot_cpu(void); void __init init_cpu_features(struct cpuinfo_arm64 *info); void update_cpu_features(int cpu, struct cpuinfo_arm64 *info, struct cpuinfo_arm64 *boot); +bool gmid_el1_accessible(const struct cpuinfo_arm64 *info); #endif /* __ASM_CPU_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index a57870fa96db5..f6a7200700ccf 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -627,13 +627,6 @@ static inline bool id_aa64pfr1_mpamfrac(u64 pfr1) return val > 0; } -static inline bool id_aa64pfr1_mte(u64 pfr1) -{ - u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MTE_SHIFT); - - return val >= ID_AA64PFR1_EL1_MTE_MTE2; -} - void __init setup_boot_cpu_features(void); void __init setup_system_features(void); void __init setup_user_features(void); diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index aa8ec9df80243..87560d8b254e6 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -418,6 +418,7 @@ b.lt .Lskip_fgt2_\@ mov x0, xzr + mov x2, xzr mrs x1, id_aa64dfr0_el1 ubfx x1, x1, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4 cmp x1, #ID_AA64DFR0_EL1_PMUVer_V3P9 @@ -426,6 +427,11 @@ orr x0, x0, #HDFGRTR2_EL2_nPMICNTR_EL0 orr x0, x0, #HDFGRTR2_EL2_nPMICFILTR_EL0 orr x0, x0, #HDFGRTR2_EL2_nPMUACR_EL1 + orr x2, x2, #HDFGWTR2_EL2_nPMICNTR_EL0 + orr x2, x2, #HDFGWTR2_EL2_nPMICFILTR_EL0 + orr x2, x2, #HDFGWTR2_EL2_nPMUACR_EL1 + /* PMZR_EL0 is write-only, so it has no read trap to disable */ + orr x2, x2, #HDFGWTR2_EL2_nPMZR_EL0 .Lskip_pmuv3p9_\@: /* If SPE is implemented, */ __spe_vers_imp .Lskip_spefds_\@, ID_AA64DFR0_EL1_PMSVer_IMP, x1 @@ -436,10 +442,11 @@ cbz x1, .Lskip_spefds_\@ /* disable traps of PMSDSFR to EL2. */ orr x0, x0, #HDFGRTR2_EL2_nPMSDSFR_EL1 + orr x2, x2, #HDFGWTR2_EL2_nPMSDSFR_EL1 .Lskip_spefds_\@: msr_s SYS_HDFGRTR2_EL2, x0 - msr_s SYS_HDFGWTR2_EL2, x0 + msr_s SYS_HDFGWTR2_EL2, x2 msr_s SYS_HFGRTR2_EL2, xzr msr_s SYS_HFGWTR2_EL2, xzr msr_s SYS_HFGITR2_EL2, xzr diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 21c8e400107ca..31e67f6bd4a70 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -272,7 +272,8 @@ static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size, pgprot_t prot; ptval_t user_prot_val = pgprot_val(user_prot); - if (WARN_ON_ONCE(!(user_prot_val & PTE_USER))) + /* Reject PROT_NONE and exec-only */ + if (!(user_prot_val & PTE_USER)) return NULL; prot = __pgprot_modify(PAGE_KERNEL, PTE_ATTRINDX_MASK, diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index bae2c4f92ef5c..7f94635e69c9c 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -322,7 +322,7 @@ struct kvm_arch { * Stage 2 paging state for VMs with nested S2 using a virtual * VMID. */ - struct kvm_s2_mmu *nested_mmus; + struct kvm_s2_mmu **nested_mmus; size_t nested_mmus_size; int nested_mmus_next; @@ -411,9 +411,6 @@ struct kvm_arch { /* Masks for VNCR-backed and general EL2 sysregs */ struct kvm_sysreg_masks *sysreg_masks; - /* Count the number of VNCR_EL2 currently mapped */ - atomic_t vncr_map_count; - /* * For an untrusted host VM, 'pkvm.handle' is used to lookup * the associated pKVM instance in the hypervisor. diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 012d711034d17..586026e859030 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -66,7 +66,8 @@ static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) extern bool forward_smc_trap(struct kvm_vcpu *vcpu); extern bool forward_debug_exception(struct kvm_vcpu *vcpu); -extern void kvm_init_nested(struct kvm *kvm); +extern int kvm_init_nested(struct kvm *kvm); +extern void kvm_destroy_nested(struct kvm *kvm); extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu); extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu); extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu); @@ -291,6 +292,13 @@ static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) base = (val & GENMASK(36, 0)) << shift; + /* + * We only deal with at most 48bit VA/IPA, so 48 is where we + * sign-extend from. Should we support FEAT_L{VP}A* at some point, + * this will need to be revisited. + */ + base = (u64)sign_extend64(base, 48); + if (asid) *asid = FIELD_GET(TLBIR_ASID_MASK, val); @@ -298,6 +306,12 @@ static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) num = FIELD_GET(GENMASK(43, 39), val); *range = __TLBI_RANGE_PAGES(num, scale) << shift; + /* Cap the range to the correct half of the address space */ + if (!(base & BIT(48))) + *range = min(*range, (BIT(48) - base)); + else + *range = min(*range, ~base + 1); + return base; } @@ -388,6 +402,8 @@ struct s1_walk_result { bool failed; }; +#define S1_MMU_DISABLED (-127) + static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) { wr->fst = fst; @@ -396,6 +412,11 @@ static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) wr->failed = true; } +static inline bool s1_walk_translated(struct s1_walk_result *wr) +{ + return wr->level != S1_MMU_DISABLED; +} + int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, struct s1_walk_result *wr, u64 va); int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 74fedd9c5ff02..ce5fd7cfc4e43 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -62,8 +62,7 @@ static inline bool kvm_pkvm_ioctl_allowed(struct kvm *kvm, unsigned int ioctl) int r; r = kvm_get_cap_for_kvm_ioctl(ioctl, &ext); - - if (WARN_ON_ONCE(r < 0)) + if (r < 0) return false; return kvm_pkvm_ext_allowed(kvm, ext); diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h index b57b2bb009677..8cf4068ce1b56 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -77,7 +77,7 @@ __percpu_##name##_case_##sz(void *ptr, unsigned long val) \ " stxr" #sfx "\t%w[loop], %" #w "[tmp], %[ptr]\n" \ " cbnz %w[loop], 1b", \ /* LSE atomics */ \ - #op_lse "\t%" #w "[val], %" #w "[tmp], %[ptr]\n" \ + #op_lse #sfx "\t%" #w "[val], %" #w "[tmp], %[ptr]\n" \ __nops(3)) \ : [loop] "=&r" (loop), [tmp] "=&r" (tmp), \ [ptr] "+Q"(*(u##sz *)ptr) \ @@ -98,7 +98,7 @@ __percpu_##name##_return_case_##sz(void *ptr, unsigned long val) \ " stxr" #sfx "\t%w[loop], %" #w "[ret], %[ptr]\n" \ " cbnz %w[loop], 1b", \ /* LSE atomics */ \ - #op_lse "\t%" #w "[val], %" #w "[ret], %[ptr]\n" \ + #op_lse #sfx "\t%" #w "[val], %" #w "[ret], %[ptr]\n" \ #op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \ __nops(2)) \ : [loop] "=&r" (loop), [ret] "=&r" (ret), \ @@ -179,13 +179,13 @@ PERCPU_RET_OP(add, add, ldadd) _pcp_protect_return(__percpu_read_64, pcp) #define this_cpu_write_1(pcp, val) \ - _pcp_protect(__percpu_write_8, pcp, (unsigned long)val) + _pcp_protect(__percpu_write_8, pcp, (unsigned long)(val)) #define this_cpu_write_2(pcp, val) \ - _pcp_protect(__percpu_write_16, pcp, (unsigned long)val) + _pcp_protect(__percpu_write_16, pcp, (unsigned long)(val)) #define this_cpu_write_4(pcp, val) \ - _pcp_protect(__percpu_write_32, pcp, (unsigned long)val) + _pcp_protect(__percpu_write_32, pcp, (unsigned long)(val)) #define this_cpu_write_8(pcp, val) \ - _pcp_protect(__percpu_write_64, pcp, (unsigned long)val) + _pcp_protect(__percpu_write_64, pcp, (unsigned long)(val)) #define this_cpu_add_1(pcp, val) \ _pcp_protect(__percpu_add_case_8, pcp, val) @@ -206,13 +206,13 @@ PERCPU_RET_OP(add, add, ldadd) _pcp_protect_return(__percpu_add_return_case_64, pcp, val) #define this_cpu_and_1(pcp, val) \ - _pcp_protect(__percpu_andnot_case_8, pcp, ~val) + _pcp_protect(__percpu_andnot_case_8, pcp, ~(u8)(val)) #define this_cpu_and_2(pcp, val) \ - _pcp_protect(__percpu_andnot_case_16, pcp, ~val) + _pcp_protect(__percpu_andnot_case_16, pcp, ~(u16)(val)) #define this_cpu_and_4(pcp, val) \ - _pcp_protect(__percpu_andnot_case_32, pcp, ~val) + _pcp_protect(__percpu_andnot_case_32, pcp, ~(u32)(val)) #define this_cpu_and_8(pcp, val) \ - _pcp_protect(__percpu_andnot_case_64, pcp, ~val) + _pcp_protect(__percpu_andnot_case_64, pcp, ~(u64)(val)) #define this_cpu_or_1(pcp, val) \ _pcp_protect(__percpu_or_case_8, pcp, val) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index c2a627f393144..6dfbcacd9ba09 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -264,7 +264,7 @@ static inline void task_set_sve_vl_onexec(struct task_struct *task, #define SCTLR_USER_MASK \ (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA | SCTLR_ELx_ENDB | \ - SCTLR_EL1_TCF0_MASK) + SCTLR_EL1_TCF0_MASK | SCTLR_EL1_TCSO0_MASK) static inline void arch_thread_struct_whitelist(unsigned long *offset, unsigned long *size) diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h index 50a195eda8ed2..3aa8fa2fcec76 100644 --- a/arch/arm64/include/asm/ptdump.h +++ b/arch/arm64/include/asm/ptdump.h @@ -50,6 +50,8 @@ struct ptdump_pg_state { const struct addr_marker *marker; const struct mm_struct *mm; unsigned long start_address; + /* exclusive end, ULONG_MAX represents an end at 1 << 64 */ + unsigned long end_address; int level; ptval_t current_prot; bool check_wx; diff --git a/arch/arm64/include/asm/rsi_cmds.h b/arch/arm64/include/asm/rsi_cmds.h index 2c8763876dfb7..c1fab41f671ec 100644 --- a/arch/arm64/include/asm/rsi_cmds.h +++ b/arch/arm64/include/asm/rsi_cmds.h @@ -88,6 +88,14 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, return res.a0; } +#define RSI_ATTEST_CHALLENGE_MIN_SIZE 32 +#define RSI_ATTEST_CHALLENGE_MAX_SIZE 64 + +struct rsi_attestation_token_init_args { + unsigned long fid; + u8 challenge[RSI_ATTEST_CHALLENGE_MAX_SIZE]; +}; + /** * rsi_attestation_token_init - Initialise the operation to retrieve an * attestation token. @@ -109,18 +117,21 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, static inline long rsi_attestation_token_init(const u8 *challenge, unsigned long size) { - struct arm_smccc_1_2_regs regs = { 0 }; + union { + struct arm_smccc_1_2_regs regs; + struct rsi_attestation_token_init_args init; + } args = { 0 }; - /* The challenge must be at least 32bytes and at most 64bytes */ - if (!challenge || size < 32 || size > 64) + if (!challenge || size < RSI_ATTEST_CHALLENGE_MIN_SIZE || + size > RSI_ATTEST_CHALLENGE_MAX_SIZE) return -EINVAL; - regs.a0 = SMC_RSI_ATTESTATION_TOKEN_INIT; - memcpy(®s.a1, challenge, size); - arm_smccc_1_2_smc(®s, ®s); + args.init.fid = SMC_RSI_ATTESTATION_TOKEN_INIT; + memcpy(args.init.challenge, challenge, size); + arm_smccc_1_2_smc(&args.regs, &args.regs); - if (regs.a0 == RSI_SUCCESS) - return regs.a1; + if (args.regs.a0 == RSI_SUCCESS) + return args.regs.a1; return -EINVAL; } diff --git a/arch/arm64/kernel/compat_alignment.c b/arch/arm64/kernel/compat_alignment.c index b68e1d328d4cb..9b58d0aa38d22 100644 --- a/arch/arm64/kernel/compat_alignment.c +++ b/arch/arm64/kernel/compat_alignment.c @@ -114,8 +114,8 @@ do_alignment_ldrdstrd(unsigned long addr, u32 instr, struct pt_regs *regs) static int do_alignment_ldmstm(unsigned long addr, u32 instr, struct pt_regs *regs) { - unsigned int rd, rn, nr_regs, regbits; - unsigned long eaddr, newaddr; + unsigned int rd, rn, regbits; + unsigned long eaddr, newaddr, nr_regs; unsigned int val; /* count the number of registers in the mask to be transferred */ diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 1995e1198648e..be47908bb9d22 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -28,18 +28,21 @@ bool cpu_errata_set_target_impl(u64 num, void *impl_cpus) return true; } +static inline bool __is_midr_in_range(u32 midr, struct midr_range const *range) +{ + return midr_is_cpu_model_range(midr, range->model, + range->rv_min, range->rv_max); +} + static inline bool is_midr_in_range(struct midr_range const *range) { int i; if (!target_impl_cpu_num) - return midr_is_cpu_model_range(read_cpuid_id(), range->model, - range->rv_min, range->rv_max); + return __is_midr_in_range(read_cpuid_id(), range); for (i = 0; i < target_impl_cpu_num; i++) { - if (midr_is_cpu_model_range(target_impl_cpus[i].midr, - range->model, - range->rv_min, range->rv_max)) + if (__is_midr_in_range(target_impl_cpus[i].midr, range)) return true; } return false; @@ -59,7 +62,7 @@ __is_affected_midr_range(const struct arm64_cpu_capabilities *entry, u32 midr, u32 revidr) { const struct arm64_midr_revidr *fix; - if (!is_midr_in_range(&entry->midr_range)) + if (!__is_midr_in_range(midr, &entry->midr_range)) return false; midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK; @@ -82,7 +85,7 @@ is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope) for (i = 0; i < target_impl_cpu_num; i++) { if (__is_affected_midr_range(entry, target_impl_cpus[i].midr, - target_impl_cpus[i].midr)) + target_impl_cpus[i].revidr)) return true; } return false; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 9a22df0c5120f..103e70d683ca2 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1176,6 +1176,33 @@ static bool detect_ftr_has_mpam(void) return id_aa64pfr0_mpam(pfr0) || id_aa64pfr1_mpamfrac(pfr1); } +bool gmid_el1_accessible(const struct cpuinfo_arm64 *info) +{ + const struct arm64_ftr_bits *ftrp; + s64 mte, ovr; + u64 ftr_mask; + + /* No ID register reflects CONFIG_ARM64_MTE. */ + if (!IS_ENABLED(CONFIG_ARM64_MTE)) + return false; + + for (ftrp = ftr_id_aa64pfr1; ftrp->width; ftrp++) { + if (ftrp->shift == ID_AA64PFR1_EL1_MTE_SHIFT) + break; + } + + ftr_mask = arm64_ftr_mask(ftrp); + mte = arm64_ftr_value(ftrp, info->reg_id_aa64pfr1); + + /* The boot CPU runs before init_cpu_ftr_reg() strips unsafe overrides. */ + if ((id_aa64pfr1_override.mask & ftr_mask) == ftr_mask) { + ovr = arm64_ftr_value(ftrp, id_aa64pfr1_override.val); + mte = arm64_ftr_safe_value(ftrp, ovr, mte); + } + + return mte >= ID_AA64PFR1_EL1_MTE_MTE2; +} + void __init init_cpu_features(struct cpuinfo_arm64 *info) { /* Before we start using the tables, make sure it is sorted */ @@ -1228,7 +1255,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) init_cpu_ftr_reg(SYS_MPAMIDR_EL1, info->reg_mpamidr); } - if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) + if (gmid_el1_accessible(info)) init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid); } @@ -1490,11 +1517,9 @@ void update_cpu_features(int cpu, * they read/write depends on the GMID_EL1.BS field. Check that the * value is the same on all CPUs. */ - if (IS_ENABLED(CONFIG_ARM64_MTE) && - id_aa64pfr1_mte(info->reg_id_aa64pfr1)) { + if (gmid_el1_accessible(info)) taint |= check_update_ftr_reg(SYS_GMID_EL1, cpu, info->reg_gmid, boot->reg_gmid); - } /* * If we don't have AArch32 at all then skip the checks entirely diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index d50e2a9b066b3..45c63f3d75c53 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -502,7 +502,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) info->reg_id_aa64smfr0 = read_cpuid(ID_AA64SMFR0_EL1); info->reg_id_aa64fpfr0 = read_cpuid(ID_AA64FPFR0_EL1); - if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) + if (gmid_el1_accessible(info)) info->reg_gmid = read_cpuid(GMID_EL1); if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 30cd7f8043986..0ec90fd1754ed 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -184,6 +184,8 @@ void arch_efi_call_virt_setup(void) efi_virtmap_load(); } + __efi_fpsimd_begin(); + /* * Enable access to the valid TTBR0_EL1 and invoke the errata * workaround directly since there is no return from exception when @@ -191,8 +193,6 @@ void arch_efi_call_virt_setup(void) */ uaccess_ttbr0_enable(); post_ttbr_update_workaround(); - - __efi_fpsimd_begin(); } void arch_efi_call_virt_teardown(void) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index ceb4eb11232a6..bda824628a20e 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -52,16 +52,36 @@ static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *reg * After this function returns it is not safe to call regular kernel code, * instrumentable code, or any code which may trigger an exception. */ -static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, - irqentry_state_t state) +static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) { - local_irq_disable(); - irqentry_exit_to_kernel_mode_preempt(regs, state); local_daif_mask(); mte_check_tfsr_exit(); irqentry_exit_to_kernel_mode_after_preempt(regs, state); } +/* + * We are returning from the context which allows involuntary kernel preemption + */ +static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs, + irqentry_state_t state) +{ + irqentry_exit_to_kernel_mode_preempt(regs, state); + __arm64_exit_to_kernel_mode(regs, state); +} + +static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) +{ + if (!regs_irqs_disabled(regs)) { + local_irq_disable(); + arm64_exit_to_kernel_mode_preempt(regs, state); + return; + } + + __arm64_exit_to_kernel_mode(regs, state); +} + static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) { enter_from_user_mode(regs); @@ -509,7 +529,7 @@ static __always_inline void __el1_irq(struct pt_regs *regs, do_interrupt_handler(regs, handler); irq_exit_rcu(); - arm64_exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode_preempt(regs, state); } static void noinstr el1_interrupt(struct pt_regs *regs, void (*handler)(struct pt_regs *)) diff --git a/arch/arm64/kernel/hibernate-asm.S b/arch/arm64/kernel/hibernate-asm.S index 0e1d9c3c6a933..2baefe7a82d36 100644 --- a/arch/arm64/kernel/hibernate-asm.S +++ b/arch/arm64/kernel/hibernate-asm.S @@ -89,6 +89,8 @@ alternative_insn "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE isb cbz x24, 3f /* Do we need to re-initialise EL2? */ + mov x1, x24 + mov x0, #HVC_SET_VECTORS hvc #0 3: ret SYM_CODE_END(swsusp_arch_suspend_exit) diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 9717568518ba7..424291c547f02 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -348,8 +348,10 @@ int swsusp_arch_suspend(void) crash_prepare_suspend(); ret = swsusp_mte_save_tags(); - if (ret) + if (ret) { + local_daif_restore(flags); return ret; + } sleep_cpu = smp_processor_id(); ret = swsusp_save(); @@ -421,8 +423,8 @@ int __nocfi swsusp_arch_resume(void) * Create a second copy of just the linear map, and use this when * restoring. */ - rc = trans_pgd_create_copy(&trans_info, &tmp_pg_dir, PAGE_OFFSET, - PAGE_END); + rc = trans_pgd_create_copy(&trans_info, &tmp_pg_dir, + _PAGE_OFFSET(vabits_actual), PAGE_END); if (rc) return rc; @@ -465,9 +467,21 @@ int __nocfi swsusp_arch_resume(void) if (el2_reset_needed()) __hyp_set_vectors(el2_vectors); + /* + * It is necessary to mask all DAIF exceptions here as: + * + * - The copy of swsusp_arch_suspend_exit() in the hibernation + * text cannot handle taking any exceptions. + * + * - The suspended kernel masked all DAIF exceptions in + * swsusp_arch_resume(), and expects to be re-entered in the + * same state : with all DAIF exceptions masked. + */ + local_daif_save(); hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, resume_hdr.reenter_kernel, restore_pblist, resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); + unreachable(); return 0; } diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index c5693a32e49b0..8f9bc2327dc85 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -129,7 +129,8 @@ int machine_kexec_post_load(struct kimage *kimage) } /* Create a copy of the linear map */ - rc = trans_pgd_create_copy(&info, &trans_pgd, PAGE_OFFSET, PAGE_END); + rc = trans_pgd_create_copy(&info, &trans_pgd, + _PAGE_OFFSET(vabits_actual), PAGE_END); if (rc) return rc; kimage->arch.ttbr1 = __pa(trans_pgd); diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 1a9aad6ef22a0..31f5c6b0510e7 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -476,7 +476,7 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, * was never mapped with PROT_MTE. */ if (!(vma->vm_flags & VM_MTE)) { - err = -EOPNOTSUPP; + err = -EIO; put_page(page); break; } diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 7bb6553fec087..3bcf86154d95c 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -1023,6 +1023,11 @@ static int __init parse_spectre_bhb_param(char *str) } early_param("nospectre_bhb", parse_spectre_bhb_param); +static bool spectre_bhb_mitigations_off(void) +{ + return __nospectre_bhb || cpu_mitigations_off(); +} + void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) { bp_hardening_cb_t cpu_cb; @@ -1036,6 +1041,8 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) /* No point mitigating Spectre-BHB alone. */ } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { /* Do nothing */ + } else if (spectre_bhb_mitigations_off()) { + /* Mitigation disabled on the command line */ } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { state = SPECTRE_MITIGATED; set_bit(BHB_HW, &system_bhb_mitigations); @@ -1201,6 +1208,6 @@ void spectre_print_disabled_mitigations(void) if (spectre_v4_mitigations_off()) pr_info("spectre-v4 %s", spectre_disabled_suffix); - if (__nospectre_bhb || cpu_mitigations_off()) + if (spectre_bhb_mitigations_off()) pr_info("spectre-bhb %s", spectre_disabled_suffix); } diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 4d08598e2891d..4955166723b88 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -560,6 +560,42 @@ static int gpr_get(struct task_struct *target, return membuf_write(&to, uregs, sizeof(*uregs)); } +static void update_syscall_orig_x0_after_ptrace(struct task_struct *target) +{ + struct pt_regs *regs = task_pt_regs(target); + struct kernel_siginfo *info = target->last_siginfo; + + /* + * Skip the update for NO_SYSCALL (set either by the user or the + * tracer), as regs[0] holds the return value (see the comment in + * el0_svc_common()) and can be unwound using syscall_rollback(). + */ + if (regs->syscallno == NO_SYSCALL) + return; + + /* We should only be called when target is in a ptrace stop */ + if (WARN_ON_ONCE(!info)) + return; + + /* + * For compat tasks, orig_r0 is provided directly through GPR index + * 17. + */ + if (is_compat_thread(task_thread_info(target))) + return; + + /* + * Don't update orig_x0 for a syscall-exit-stop, as x0 now contains the + * return value of the system call. + */ + if ((info->si_code & ~0x80) == SIGTRAP && + target->ptrace_message == PTRACE_EVENTMSG_SYSCALL_EXIT) { + return; + } + + regs->orig_x0 = regs->regs[0]; +} + static int gpr_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) @@ -575,6 +611,14 @@ static int gpr_set(struct task_struct *target, const struct user_regset *regset, return -EINVAL; task_pt_regs(target)->user_regs = newregs; + + /* + * Keep orig_x0 authoritative so that seccomp (via + * syscall_get_arguments()), audit and the restart path all see the same + * first argument the syscall is dispatched with, even if it has been + * updated by a tracer. + */ + update_syscall_orig_x0_after_ptrace(target); return 0; } @@ -753,6 +797,12 @@ static int system_call_set(struct task_struct *target, return ret; task_pt_regs(target)->syscallno = syscallno; + + /* + * Re-sync orig_x0 in case the syscall number has been changed + * from NO_SYSCALL. + */ + update_syscall_orig_x0_after_ptrace(target); return ret; } @@ -801,7 +851,7 @@ static void sve_init_header_from_task(struct user_sve_header *header, if (active) header->size = SVE_PT_SIZE(vq, header->flags); else - header->size = sizeof(header); + header->size = sizeof(*header); header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl), SVE_PT_REGS_SVE); } @@ -837,7 +887,7 @@ static int sve_get_common(struct task_struct *target, * from the other mode to userspace. */ if (header.size == sizeof(header)) - return 0; + return to.left; switch ((header.flags & SVE_PT_REGS_MASK)) { case SVE_PT_REGS_FPSIMD: diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index cdcdd160e5b69..3ab90aa24efb9 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -1086,7 +1086,7 @@ static void ipi_teardown(int cpu) disable_percpu_irq(ipi_irq_base + i); } } else { - disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i))); + disable_irq_nosync(irq_desc_get_irq(get_ipi_desc(cpu, i))); } } } diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 9a6c72a186727..ad04ef4bb8211 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -236,8 +236,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); #endif - kvm_init_nested(kvm); - ret = kvm_share_hyp(kvm, kvm + 1); if (ret) return ret; @@ -252,6 +250,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (ret) goto err_free_cpumask; + ret = kvm_init_nested(kvm); + if (ret) + goto err_uninit_mmu; + if (is_protected_kvm_enabled()) { /* * If any failures occur after this is successful, make sure to @@ -280,6 +282,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) err_uninit_mmu: kvm_uninit_stage2_mmu(kvm); + kvm_destroy_nested(kvm); err_free_cpumask: free_cpumask_var(kvm->arch.supported_cpus); err_unshare_kvm: @@ -337,6 +340,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_unshare_hyp(kvm, kvm + 1); + kvm_destroy_nested(kvm); kvm_arm_teardown_hypercalls(kvm); } diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 640f2dc00a8ba..0926426b87989 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -11,8 +11,6 @@ #include #include -#define S1_MMU_DISABLED (-127) - static int get_ia_size(struct s1_walk_info *wi) { return 64 - wi->txsz; diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 3c82f392845d1..b32742d9dd73e 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -1432,7 +1432,7 @@ static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = { SR_FGT(OP_AT_S1E1A, HFGITR, ATS1E1A, 1), SR_FGT(OP_COSP_RCTX, HFGITR, COSPRCTX, 1), SR_FGT(OP_GCSPUSHX, HFGITR, nGCSEPP, 0), - SR_FGT(OP_GCSPOPX, HFGITR, nGCSEPP, 0), + SR_FGT(OP_GCSPOPCX, HFGITR, nGCSEPP, 0), SR_FGT(OP_GCSPUSHM, HFGITR, nGCSPUSHM_EL1, 0), SR_FGT(OP_BRB_IALL, HFGITR, nBRBIALL, 0), SR_FGT(OP_BRB_INJ, HFGITR, nBRBINJ, 0), diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 29935c7da1dec..cab27f7bd423a 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -52,6 +52,7 @@ int __pkvm_host_test_clear_young_guest(u64 gfn, u64 nr_pages, bool mkold, struct int __pkvm_host_mkyoung_guest(u64 gfn, struct pkvm_hyp_vcpu *vcpu); bool addr_is_memory(phys_addr_t phys); +bool addr_is_hyp_text(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot); int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id); int kvm_host_prepare_stage2(void *pgt_pool_base); diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 4e329e39a695a..7a99361306426 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -443,6 +443,14 @@ bool addr_is_memory(phys_addr_t phys) return !!find_mem_range(phys, &range); } +bool addr_is_hyp_text(phys_addr_t phys) +{ + phys_addr_t start = ALIGN_DOWN(__hyp_pa(__hyp_text_start), PAGE_SIZE); + phys_addr_t end = PAGE_ALIGN(__hyp_pa(__hyp_text_end)); + + return phys >= start && phys < end; +} + static bool is_in_mem_range(u64 addr, struct kvm_mem_range *range) { return range->start <= addr && addr < range->end; diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 24d6f164129ac..53532f0c41628 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -360,7 +360,7 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc if (test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &host_arch_flags)) hyp_vm->kvm.arch.midr_el1 = host_kvm->arch.midr_el1; - return; + goto out; } if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE)) @@ -379,13 +379,14 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC)) set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed_features); - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) { + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) set_bit(KVM_ARM_VCPU_SVE, allowed_features); - kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_GUEST_HAS_SVE); - } bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, allowed_features, KVM_VCPU_MAX_FEATURES); +out: + __assign_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &kvm->arch.flags, + kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_SVE)); } static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu) @@ -451,7 +452,7 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h unsigned int sve_max_vl; size_t sve_state_size; void *sve_state; - int ret = 0; + int ret; if (!vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE)) { vcpu_clear_flag(vcpu, VCPU_SVE_FINALIZED); @@ -460,25 +461,21 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h /* Limit guest vector length to the maximum supported by the host. */ sve_max_vl = min(READ_ONCE(host_vcpu->arch.sve_max_vl), kvm_host_sve_max_vl); - sve_state_size = sve_state_size_from_vl(sve_max_vl); sve_state = kern_hyp_va(READ_ONCE(host_vcpu->arch.sve_state)); - if (!sve_state || !sve_state_size) { - ret = -EINVAL; - goto err; - } + if (!sve_vl_valid(sve_max_vl) || !sve_state) + return -EINVAL; + + sve_state_size = sve_state_size_from_vl(sve_max_vl); ret = hyp_pin_shared_mem(sve_state, sve_state + sve_state_size); if (ret) - goto err; + return ret; vcpu->arch.sve_state = sve_state; vcpu->arch.sve_max_vl = sve_max_vl; return 0; -err: - clear_bit(KVM_ARM_VCPU_SVE, vcpu->kvm->arch.vcpu_features); - return ret; } static int vm_copy_id_regs(struct pkvm_hyp_vcpu *hyp_vcpu) diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 75b00c3233102..bb667cd7080b4 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -217,7 +217,7 @@ static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx, case PKVM_PAGE_OWNED: set_hyp_state(page, PKVM_PAGE_OWNED); /* hyp text is RO in the host stage-2 to be inspected on panic. */ - if (prot == PAGE_HYP_EXEC) { + if (addr_is_hyp_text(phys)) { set_host_state(page, PKVM_NOPAGE); return host_stage2_idmap_locked(phys, PAGE_SIZE, KVM_PGTABLE_PROT_R); } else { @@ -269,6 +269,16 @@ static int fix_host_ownership(void) return ret; } + /* The stacks sit in the private VA range, not the linear map. */ + for (i = 0; i < hyp_nr_cpus; i++) { + struct kvm_nvhe_init_params *params = per_cpu_ptr(&kvm_init_params, i); + u64 start = params->stack_hyp_va - NVHE_STACK_SIZE; + + ret = kvm_pgtable_walk(&pkvm_pgtable, start, NVHE_STACK_SIZE, &walker); + if (ret) + return ret; + } + return 0; } diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index bbe9cebd3d9d5..c09b1d411c584 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -427,8 +427,7 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) * If we have to check for any VNCR mapping being invalidated, * go back to the slow path for further processing. */ - if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) && - atomic_read(&vcpu->kvm->arch.vncr_map_count)) + if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) return false; __kvm_skip_instr(vcpu); diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index b11b8821c9fbc..dfa25bb6f25d4 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -185,7 +185,8 @@ static int kvm_smccc_set_filter(struct kvm *kvm, struct kvm_smccc_filter __user start = filter.base; end = start + filter.nr_functions - 1; - if (end < start || filter.action >= NR_SMCCC_FILTER_ACTIONS) + if (!filter.nr_functions || end < start || + filter.action >= NR_SMCCC_FILTER_ACTIONS) return -EINVAL; mutex_lock(&kvm->arch.config_lock); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 2d95203386baa..2ed0836cb929b 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -59,27 +59,36 @@ static phys_addr_t stage2_range_addr_end(phys_addr_t addr, phys_addr_t end) * long will also starve other vCPUs. We have to also make sure that the page * tables are not freed while we released the lock. */ -static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, +static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t start, phys_addr_t end, int (*fn)(struct kvm_pgtable *, u64, u64), bool resched) { struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + bool lock_dropped = false; + phys_addr_t addr = start; int ret; u64 next; do { struct kvm_pgtable *pgt = mmu->pgt; + /* + * We may be raced on PGT teardown when we release the + * kvm->mmu_lock. That's fine as the PGT is legitimately no + * longer present. + */ if (!pgt) - return -EINVAL; + return lock_dropped ? 0 : -EINVAL; next = stage2_range_addr_end(addr, end); ret = fn(pgt, addr, next - addr); if (ret) break; - if (resched && next != end) + if (resched && next != end) { cond_resched_rwlock_write(&kvm->mmu_lock); + lock_dropped = true; + } } while (addr = next, addr != end); return ret; diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index dfb96edbdc43c..982bbf3110e01 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -27,7 +27,7 @@ struct vncr_tlb { bool hpa_writable; /* -1 when not mapped on a CPU */ - int cpu; + atomic_t cpu; /* * true if the TLB is valid. Can only be changed with the @@ -44,11 +44,23 @@ struct vncr_tlb { */ #define S2_MMU_PER_VCPU 2 -void kvm_init_nested(struct kvm *kvm) +int kvm_init_nested(struct kvm *kvm) { - kvm->arch.nested_mmus = NULL; + kvm->arch.nested_mmus = kvmalloc_objs(struct kvm_s2_mmu *, + KVM_MAX_VCPUS * S2_MMU_PER_VCPU, + GFP_KERNEL_ACCOUNT); kvm->arch.nested_mmus_size = 0; - atomic_set(&kvm->arch.vncr_map_count, 0); + + return kvm->arch.nested_mmus ? 0 : -ENOMEM; +} + +void kvm_destroy_nested(struct kvm *kvm) +{ + for (int i = 0; i < kvm->arch.nested_mmus_size; i+= S2_MMU_PER_VCPU) + kvfree(kvm->arch.nested_mmus[i]); + + kvm->arch.nested_mmus_size = 0; + kvfree(kvm->arch.nested_mmus); } static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) @@ -69,8 +81,9 @@ static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) { struct kvm *kvm = vcpu->kvm; - struct kvm_s2_mmu *tmp; - int num_mmus, ret = 0; + int num_mmus; + + lockdep_assert_held(&kvm->arch.config_lock); if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) && !cpus_have_final_cap(ARM64_HAS_HCR_NV1)) @@ -83,51 +96,40 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) if (!vcpu->arch.ctxt.vncr_array) return -ENOMEM; - /* - * Let's treat memory allocation failures as benign: If we fail to - * allocate anything, return an error and keep the allocated array - * alive. Userspace may try to recover by initializing the vcpu - * again, and there is no reason to affect the whole VM for this. - */ num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; if (num_mmus > kvm->arch.nested_mmus_size) { - tmp = kvcalloc(num_mmus, sizeof(*tmp), GFP_KERNEL_ACCOUNT); - if (!tmp) - return -ENOMEM; - - write_lock(&kvm->mmu_lock); + struct kvm_s2_mmu *tmp; + int i, ret = 0; - if (kvm->arch.nested_mmus_size) { - memcpy(tmp, kvm->arch.nested_mmus, - size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size)); + tmp = kvcalloc(S2_MMU_PER_VCPU, sizeof(*tmp), GFP_KERNEL_ACCOUNT); + if (!tmp) + ret = -ENOMEM; - for (int i = 0; i < kvm->arch.nested_mmus_size; i++) - tmp[i].pgt->mmu = &tmp[i]; + for (i = 0; !ret && i < S2_MMU_PER_VCPU; i++) { + ret = init_nested_s2_mmu(kvm, &tmp[i]); + if (ret) + break; } - swap(kvm->arch.nested_mmus, tmp); - - write_unlock(&kvm->mmu_lock); - - kvfree(tmp); - } + if (ret) { + while (--i >= 0) + kvm_free_stage2_pgd(&tmp[i]); - for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++) - ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]); + kvfree(tmp); + free_page((unsigned long)vcpu->arch.ctxt.vncr_array); + vcpu->arch.ctxt.vncr_array = NULL; + return ret; + } - if (ret) { - for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) - kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); + guard(write_lock)(&kvm->mmu_lock); - free_page((unsigned long)vcpu->arch.ctxt.vncr_array); - vcpu->arch.ctxt.vncr_array = NULL; + for (i = 0; i < S2_MMU_PER_VCPU; i++) + kvm->arch.nested_mmus[i + kvm->arch.nested_mmus_size] = &tmp[i]; - return ret; + kvm->arch.nested_mmus_size += S2_MMU_PER_VCPU; } - kvm->arch.nested_mmus_size = num_mmus; - return 0; } @@ -506,7 +508,7 @@ int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, return ret; } -static unsigned int ttl_to_size(u8 ttl) +static unsigned int __ttl_to_size(u8 ttl) { int level = ttl & 3; int gran = (ttl >> 2) & 3; @@ -562,10 +564,22 @@ static unsigned int ttl_to_size(u8 ttl) return max_size; } -static u8 pgshift_level_to_ttl(u16 shift, u8 level) +static unsigned int ttl_to_size(u8 ttl) +{ + return __ttl_to_size(ttl) ?: SZ_1G; +} + +static u8 pgshift_level_to_ttl(u16 shift, s8 level) { u8 ttl; + /* + * If we don't have a proper level, fallback to the maximum + * size. + */ + if (level < 0) + return 0; + switch(shift) { case 12: ttl = TLBI_TTL_TG_4K; @@ -676,7 +690,11 @@ unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) ttl = get_guest_mapping_ttl(mmu, addr); } - max_size = ttl_to_size(ttl); + /* + * Don't use the default 1GB fallback, as we can adapt to the + * max mapping size we allow at S2. + */ + max_size = __ttl_to_size(ttl); if (!max_size) { /* Compute the maximum extent of the invalidation */ @@ -725,7 +743,7 @@ void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, write_lock(&kvm->mmu_lock); for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (!kvm_s2_mmu_valid(mmu)) continue; @@ -767,7 +785,7 @@ struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) * if S2 translation is disabled. */ for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (!kvm_s2_mmu_valid(mmu)) continue; @@ -806,7 +824,7 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) for (i = kvm->arch.nested_mmus_next; i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next); i++) { - s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; + s2_mmu = kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; if (atomic_read(&s2_mmu->refcnt) == 0) break; @@ -879,18 +897,41 @@ void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) } } +/* + * Unmapping an L1 VNCR can happen concurrently without the mmu lock being + * effective (vcpu_put() vs TLBI handling). The atomic_xchg below ensures + * that only one CPU sets it to -1 while getting a valid CPU number back. + */ +static int unmap_l1_vncr(struct vncr_tlb *vt) +{ + int cpu = atomic_xchg_relaxed(&vt->cpu, -1); + + if (cpu != -1) + clear_fixmap(vncr_fixmap(cpu)); + + return cpu; +} + static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu) { if (!host_data_test_flag(L1_VNCR_MAPPED)) return; - BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id()); BUG_ON(is_hyp_ctxt(vcpu)); - clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); - vcpu->arch.vncr_tlb->cpu = -1; + /* + * Unconditionally unmap the local VNCR if we have lost the race + * against a concurrent TLBI. Otherwise we could end-up running + * another vcpu with VNCR still mapped if the TLBI thread is + * preempted between the exchange and the clear_fixmap(). + * + * Note that we do not care about the TLBI nuking the fixmap behind + * the back of an running vcpu. This will only generate a fault and + * possibly a retranslation. + */ + if (unmap_l1_vncr(vcpu->arch.vncr_tlb) == -1) + clear_fixmap(vncr_fixmap(smp_processor_id())); host_data_clear_flag(L1_VNCR_MAPPED); - atomic_dec(&vcpu->kvm->arch.vncr_map_count); } void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) @@ -981,8 +1022,21 @@ u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime) static void invalidate_vncr(struct vncr_tlb *vt) { vt->valid = false; - if (vt->cpu != -1) - clear_fixmap(vncr_fixmap(vt->cpu)); + unmap_l1_vncr(vt); +} + +static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, + u64 scope_start, u64 scope_size) +{ + u64 tlb_size, tlb_start, tlb_end, scope_end; + + tlb_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); + + tlb_start = addr & ~(tlb_size - 1); + tlb_end = tlb_start + tlb_size - 1; + scope_end = scope_start + scope_size - 1; + + return !(tlb_end < scope_start || tlb_start > scope_end); } /* @@ -1007,19 +1061,15 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) return; - kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { - u64 ipa_start, ipa_end, ipa_size; - - ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, - vt->wr.level)); - ipa_start = vt->wr.pa & ~(ipa_size - 1); - ipa_end = ipa_start + ipa_size; - - if (ipa_end <= start || ipa_start >= end) - continue; - - invalidate_vncr(vt); - } + /* + * Note that invalidating the VNCR on the back of an MMU notifier + * doesn't require messing with the invalidation counter for a + * parallel walk. The notifier itself will have bumped the counter, + * making sure we rewalk. + */ + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) + if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) + invalidate_vncr(vt); } struct s1e2_tlbi_scope { @@ -1044,29 +1094,29 @@ static void invalidate_vncr_va(struct kvm *kvm, lockdep_assert_held_write(&kvm->mmu_lock); - kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { - u64 va_start, va_end, va_size; - - va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, - vt->wr.level)); - va_start = vt->gva & ~(va_size - 1); - va_end = va_start + va_size; + /* + * We might be performing a parallel S1 walk, so bump up the + * invalidation counter even in the absence of an actual VNCR TLB + * invalidation, as this could indicate that the guest has gone + * through a BBM sequence. + */ + kvm->mmu_invalidate_seq++; + smp_wmb(); + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { switch (scope->type) { case TLBI_ALL: break; case TLBI_VA: - if (va_end <= scope->va || - va_start >= (scope->va + scope->size)) + if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) continue; if (vt->wr.nG && vt->wr.asid != scope->asid) continue; break; case TLBI_VAA: - if (va_end <= scope->va || - va_start >= (scope->va + scope->size)) + if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) continue; break; @@ -1126,8 +1176,6 @@ static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, case OP_TLBI_VALE1OSNXS: scope->type = TLBI_VA; scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); - if (!scope->size) - scope->size = SZ_1G; scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); break; @@ -1154,8 +1202,6 @@ static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, case OP_TLBI_VAALE1OSNXS: scope->type = TLBI_VAA; scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); - if (!scope->size) - scope->size = SZ_1G; scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); break; case OP_TLBI_RVAE2: @@ -1213,6 +1259,17 @@ void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val) invalidate_vncr_va(vcpu->kvm, &scope); } +static void kvm_invalidate_vncr_ipa_all(struct kvm *kvm) +{ + struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; + + lockdep_assert_held_write(&kvm->mmu_lock); + + /* if the mmu lock was dropped, pgt teardown may have raced. */ + if (pgt) + kvm_invalidate_vncr_ipa(kvm, 0, BIT(pgt->ia_bits)); +} + void kvm_nested_s2_wp(struct kvm *kvm) { int i; @@ -1223,13 +1280,13 @@ void kvm_nested_s2_wp(struct kvm *kvm) return; for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (kvm_s2_mmu_valid(mmu)) kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); } - kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); + kvm_invalidate_vncr_ipa_all(kvm); } void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) @@ -1242,13 +1299,13 @@ void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) return; for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (kvm_s2_mmu_valid(mmu)) kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); } - kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); + kvm_invalidate_vncr_ipa_all(kvm); } void kvm_nested_s2_flush(struct kvm *kvm) @@ -1261,7 +1318,7 @@ void kvm_nested_s2_flush(struct kvm *kvm) return; for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (kvm_s2_mmu_valid(mmu)) kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu)); @@ -1270,17 +1327,12 @@ void kvm_nested_s2_flush(struct kvm *kvm) void kvm_arch_flush_shadow_all(struct kvm *kvm) { - int i; - - for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (!WARN_ON(atomic_read(&mmu->refcnt))) kvm_free_stage2_pgd(mmu); } - kvfree(kvm->arch.nested_mmus); - kvm->arch.nested_mmus = NULL; - kvm->arch.nested_mmus_size = 0; kvm_uninit_stage2_mmu(kvm); } @@ -1391,15 +1443,15 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) va = read_vncr_el2(vcpu); + mmu_seq = vcpu->kvm->mmu_invalidate_seq; + smp_rmb(); + ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va); if (ret) return ret; write_fault = kvm_is_write_fault(vcpu); - mmu_seq = vcpu->kvm->mmu_invalidate_seq; - smp_rmb(); - gfn = vt->wr.pa >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); if (!memslot) { @@ -1447,7 +1499,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) vt->hpa = pfn << PAGE_SHIFT; vt->hpa_writable = writable; vt->valid = true; - vt->cpu = -1; + atomic_set(&vt->cpu, -1); kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); @@ -1568,14 +1620,16 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) if (!vt->valid) return; + /* We cache the MMU state in the TLB. Check that it matches. */ + if (!!(vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_M) != s1_walk_translated(&vt->wr)) + return; + if (read_vncr_el2(vcpu) != vt->gva) return; if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid) return; - vt->cpu = smp_processor_id(); - if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) prot = PAGE_KERNEL; else if (vt->wr.pr) @@ -1590,9 +1644,9 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) * FIXME: WO doesn't work at all, need POE support in the kernel. */ if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { - __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot); + atomic_set(&vt->cpu, smp_processor_id()); + __set_fixmap(vncr_fixmap(atomic_read(&vt->cpu)), vt->hpa, prot); host_data_set_flag(L1_VNCR_MAPPED); - atomic_inc(&vcpu->kvm->arch.vncr_map_count); } } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5d5c579d45790..e566bbce7d229 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -889,6 +889,7 @@ static u64 *demux_wb_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) struct kvm_guest_debug_arch *dbg = &vcpu->arch.vcpu_debug_state; switch (rd->Op2) { + case 0b001: case 0b100: return &dbg->dbg_bvr[rd->CRm]; case 0b101: @@ -4057,6 +4058,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); u64 base, range; + int pa_bits; if (!kvm_supported_tlbi_ipas2_op(vcpu, sys_encoding)) return undef_access(vcpu, p, r); @@ -4068,6 +4070,16 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, */ base = decode_range_tlbi(p->regval, &range, NULL); + /* + * Ignore TLBIs that start out of PA_bits range, and cap the + * invalidation to the [base:bit(PA_bits)] interval. + */ + pa_bits = kvm_get_pa_bits(vcpu->kvm); + if (fls64(base) > pa_bits) + return true; + + range = min(range, BIT_ULL(pa_bits) - base); + kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr), &(union tlbi_info) { .range = { diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 907057881b26a..fdac6e24762da 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -176,6 +176,7 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) } kvm->arch.vgic.vgic_model = 0; + kvm->arch.vgic.in_kernel = false; goto out_unlock; } diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 36ab3e4929154..7dc5ef5a6e33a 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -1658,7 +1658,7 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, unsigned long val) { const struct vgic_its_abi *abi = vgic_its_get_abi(its); - u64 entry_size, table_type; + u64 old, entry_size, table_type; u64 reg, *regptr, clearbits = 0; /* When GITS_CTLR.Enable is 1, we ignore write accesses. */ @@ -1681,7 +1681,9 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, return; } - reg = update_64bit_reg(*regptr, addr & 7, len, val); + old = *regptr; + + reg = update_64bit_reg(old, addr & 7, len, val); reg &= ~GITS_BASER_RO_MASK; reg &= ~clearbits; @@ -1691,7 +1693,8 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, *regptr = reg; - if (!(reg & GITS_BASER_VALID)) { + /* The ITS driver rewrites an unchanged GITS_BASER on resume. */ + if (reg != old) { /* Take the its_lock to prevent a race with a save/restore */ mutex_lock(&its->its_lock); switch (table_type) { @@ -1702,6 +1705,8 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, vgic_its_free_collection_list(kvm, its); break; } + /* A concurrent injection may have cached a translation. */ + vgic_its_invalidate_cache(its); mutex_unlock(&its->its_lock); } } @@ -2019,18 +2024,22 @@ static int vgic_its_attr_regs_access(struct kvm_device *dev, return ret; } -static u32 compute_next_devid_offset(struct list_head *h, +static u32 compute_next_devid_offset(struct vgic_its *its, u64 baser, struct its_device *dev) { - struct its_device *next; - u32 next_offset; + struct its_device *next = dev; - if (list_is_last(&dev->dev_list, h)) - return 0; - next = list_next_entry(dev, dev_list); - next_offset = next->device_id - dev->device_id; + /* + * Point at the next device vgic_its_save_device_tables() saves. It + * sorts device_list first, so the subtraction cannot underflow. + */ + list_for_each_entry_continue(next, &its->device_list, dev_list) { + if (vgic_its_check_id(its, baser, next->device_id, NULL)) + return min_t(u32, next->device_id - dev->device_id, + VITS_DTE_MAX_DEVID_OFFSET); + } - return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET); + return 0; } static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite) @@ -2119,6 +2128,14 @@ static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev, u32 next_offset; u64 val; + /* + * MAPC with V=0 keeps the ITEs mapped but drops their collection, + * and with it the ICID. Save a zeroed entry, which the restore path + * reads back as invalid. + */ + if (!ite->collection) + return vgic_its_write_entry_lock(its, gpa, 0ULL, ite); + next_offset = compute_next_eventid_offset(&dev->itt_head, ite); val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) | ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) | @@ -2262,17 +2279,18 @@ static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev) * vgic_its_save_dte - Save a device table entry at a given GPA * * @its: ITS handle + * @baser: GITS_BASER the caller is saving against * @dev: ITS device * @ptr: GPA */ -static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev, - gpa_t ptr) +static int vgic_its_save_dte(struct vgic_its *its, u64 baser, + struct its_device *dev, gpa_t ptr) { u64 val, itt_addr_field; u32 next_offset; itt_addr_field = dev->itt_addr >> 8; - next_offset = compute_next_devid_offset(&its->device_list, dev); + next_offset = compute_next_devid_offset(its, baser, dev); val = (1ULL << KVM_ITS_DTE_VALID_SHIFT | ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) | (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) | @@ -2371,15 +2389,16 @@ static int vgic_its_save_device_tables(struct vgic_its *its) int ret; gpa_t eaddr; + /* Don't fail a save that userspace must be able to issue. */ if (!vgic_its_check_id(its, baser, dev->device_id, &eaddr)) - return -EINVAL; + continue; ret = vgic_its_save_itt(its, dev); if (ret) return ret; - ret = vgic_its_save_dte(its, dev, eaddr); + ret = vgic_its_save_dte(its, baser, dev, eaddr); if (ret) return ret; } diff --git a/arch/arm64/kvm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c index cafa3cb32bda6..7182f63fc9382 100644 --- a/arch/arm64/kvm/vgic/vgic-v2.c +++ b/arch/arm64/kvm/vgic/vgic-v2.c @@ -170,8 +170,9 @@ void vgic_v2_deactivate(struct kvm_vcpu *vcpu, u32 val) /* Make sure we're in the same context as LR handling */ local_irq_save(flags); + /* Guest-supplied INTID: out of range yields no irq, so ignore it */ irq = vgic_get_vcpu_irq(vcpu, val); - if (WARN_ON_ONCE(!irq)) + if (!irq) goto out; /* See the corresponding v3 code for the rationale */ diff --git a/arch/arm64/kvm/vgic/vgic-v3-nested.c b/arch/arm64/kvm/vgic/vgic-v3-nested.c index 5c69fa615823c..e3e84d7f5ad51 100644 --- a/arch/arm64/kvm/vgic/vgic-v3-nested.c +++ b/arch/arm64/kvm/vgic/vgic-v3-nested.c @@ -152,7 +152,7 @@ static void vgic_compute_mi_state(struct kvm_vcpu *vcpu, struct mi_state *mi_sta eisr |= BIT(i); if (!(lr & ICH_LR_STATE)) elrsr |= BIT(i); - pend |= (lr & ICH_LR_PENDING_BIT); + pend |= (lr & ICH_LR_STATE) == ICH_LR_PENDING_BIT; } mi_state->eisr = eisr; diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 7aa417440f6a2..0e1c121766ae1 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -617,9 +617,13 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) bool is_pending; bool stored; + irq = vgic_get_irq(kvm, index); + if (!irq) + continue; + vcpu = irq->target_vcpu; if (!vcpu) - continue; + goto put_irq; pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser); @@ -630,7 +634,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) if (ptr != last_ptr) { ret = kvm_read_guest_lock(kvm, ptr, &val, 1); if (ret) - goto out; + goto put_irq; last_ptr = ptr; } @@ -642,7 +646,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) vgic_v4_get_vlpi_state(irq, &is_pending); if (stored == is_pending) - continue; + goto put_irq; if (is_pending) val |= 1 << bit_nr; @@ -650,6 +654,8 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) val &= ~(1 << bit_nr); ret = vgic_write_guest_lock(kvm, ptr, &val, 1); +put_irq: + vgic_put_irq(kvm, irq); if (ret) goto out; } diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 448c917494f30..b33e1ca4a7815 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,4 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 + +# KCSAN uses udelay for introducing watchpoint delay; avoid recursion. +KCSAN_SANITIZE_delay.o := n + lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_page.o \ clear_page.o csum.o insn.o memchr.o memcpy.o \ diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 0b52557652be6..75c3e463df2ef 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -154,6 +155,9 @@ static void show_pte(unsigned long addr) pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n", mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, vabits_actual, mm_to_pgd_phys(mm)); + + guard(irqsave)(); + pgdp = pgd_offset(mm, addr); pgd = READ_ONCE(*pgdp); pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd)); @@ -167,25 +171,25 @@ static void show_pte(unsigned long addr) if (pgd_none(pgd) || pgd_bad(pgd)) break; - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_lockless(pgdp, pgd, addr); p4d = READ_ONCE(*p4dp); pr_cont(", p4d=%016llx", p4d_val(p4d)); if (p4d_none(p4d) || p4d_bad(p4d)) break; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_lockless(p4dp, p4d, addr); pud = READ_ONCE(*pudp); pr_cont(", pud=%016llx", pud_val(pud)); if (pud_none(pud) || pud_bad(pud)) break; - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_lockless(pudp, pud, addr); pmd = READ_ONCE(*pmdp); pr_cont(", pmd=%016llx", pmd_val(pmd)); if (pmd_none(pmd) || pmd_bad(pmd)) break; - ptep = pte_offset_map(pmdp, addr); + ptep = pte_offset_map(&pmd, addr); if (!ptep) break; diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index 5a76c59b5ada5..9d9bcb6738f4c 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -278,9 +278,19 @@ void note_page_pgd(struct ptdump_state *pt_st, unsigned long addr, pgd_t pgd) void note_page_flush(struct ptdump_state *pt_st) { + struct ptdump_pg_state *st = container_of(pt_st, struct ptdump_pg_state, ptdump); + unsigned long end = st->end_address; pte_t pte_zero = {0}; - note_page(pt_st, 0, -1, pte_val(pte_zero)); + /* + * Address spaces that end at 1 << 64 have end_address == ULONG_MAX, + * but note_page() expects the exclusive end. In this case adjust end + * to the wraparound value 0. + */ + if (end == ULONG_MAX) + end = 0; + + note_page(pt_st, end, -1, pte_val(pte_zero)); } void ptdump_walk(struct seq_file *s, struct ptdump_info *info) @@ -296,6 +306,7 @@ void ptdump_walk(struct seq_file *s, struct ptdump_info *info) .marker = info->markers, .mm = info->mm, .pg_level = &kernel_pg_levels[0], + .end_address = end, .level = -1, .ptdump = { .note_page_pte = note_page_pte, @@ -337,6 +348,7 @@ bool ptdump_check_wx(void) { -1, NULL}, }, .pg_level = &kernel_pg_levels[0], + .end_address = ~0UL, .level = -1, .check_wx = true, .ptdump = { diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index b0075ece4a6e7..8c0c9b0fda16f 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -600,6 +600,8 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) * 12 registers are on the stack */ emit(A64_SUB_I(1, A64_SP, A64_FP, 96), ctx); + /* The callback may use its own BPF stack, set up fp for it. */ + ctx->fp_used = true; } /* Stack must be multiples of 16B */ @@ -1082,23 +1084,27 @@ static void build_epilogue(struct jit_ctx *ctx, bool was_classic) * * Bit layout of `fixup` (32-bit): * - * +-----------+--------+-----------+-----------+----------+ - * | 31-27 | 26-22 | 21 | 20-16 | 15-0 | - * | | | | | | - * | FIXUP_REG | Unused | ARENA_ACC | ARENA_REG | OFFSET | - * +-----------+--------+-----------+-----------+----------+ + * +-----------+--------+-------------+-----------+-----------+----------+ + * | 31-27 | 26-23 | 22 | 21 | 20-16 | 15-0 | + * | | | | | | | + * | FIXUP_REG | Unused | ARENA_WRITE | ARENA_ACC | ARENA_REG | OFFSET | + * +-----------+--------+-------------+-----------+-----------+----------+ * * - OFFSET (16 bits): Offset used to compute address for Load/Store instruction. * - ARENA_REG (5 bits): Register that is used to calculate the address for load/store when * accessing the arena region. * - ARENA_ACCESS (1 bit): This bit is set when the faulting instruction accessed the arena region. + * - ARENA_WRITE (1 bit): This bit is set when the faulting instruction wrote to the arena region. + * It is independent of FIXUP_REG, since a read-modify-write both writes to + * memory and reads the old value into a register. * - FIXUP_REG (5 bits): Destination register for the load instruction (cleared on fault) or set to - * DONT_CLEAR if it is a store instruction. + * DONT_CLEAR if the instruction does not read into a register. */ #define BPF_FIXUP_OFFSET_MASK GENMASK(15, 0) #define BPF_FIXUP_ARENA_REG_MASK GENMASK(20, 16) #define BPF_ARENA_ACCESS BIT(21) +#define BPF_ARENA_WRITE BIT(22) #define BPF_FIXUP_REG_MASK GENMASK(31, 27) #define DONT_CLEAR 5 /* Unused ARM64 register from BPF's POV */ @@ -1109,7 +1115,7 @@ bool ex_handler_bpf(const struct exception_table_entry *ex, s16 off = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); int arena_reg = FIELD_GET(BPF_FIXUP_ARENA_REG_MASK, ex->fixup); bool is_arena = !!(ex->fixup & BPF_ARENA_ACCESS); - bool is_write = (dst_reg == DONT_CLEAR); + bool is_write = !!(ex->fixup & BPF_ARENA_WRITE); unsigned long addr; if (is_arena) { @@ -1132,7 +1138,7 @@ static int add_exception_handler(const struct bpf_insn *insn, { off_t ins_offset; s16 off = insn->off; - bool is_arena; + bool is_arena, is_write; int arena_reg; unsigned long pc; struct exception_table_entry *ex; @@ -1178,13 +1184,21 @@ static int add_exception_handler(const struct bpf_insn *insn, ex->insn = ins_offset; - if (BPF_CLASS(insn->code) != BPF_LDX) - dst_reg = DONT_CLEAR; + /* + * A load-acquire is of BPF_STX class, but reads from src_reg into + * dst_reg like a BPF_LDX does, hence it must not be treated as a store + * here. A read-modify-write carrying BPF_FETCH is reported as a write + * even though it does have a register to clear, see the callers. + */ + is_write = BPF_CLASS(insn->code) != BPF_LDX && + !bpf_atomic_is_load_acq(insn); ex->fixup = FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); if (is_arena) { ex->fixup |= BPF_ARENA_ACCESS; + if (is_write) + ex->fixup |= BPF_ARENA_WRITE; /* * insn->src_reg/dst_reg holds the address in the arena region with upper 32-bits * being zero because of a preceding addr_space_cast(r, 0x0, 0x1) instruction. @@ -1193,7 +1207,7 @@ static int add_exception_handler(const struct bpf_insn *insn, * memory access. Pass the reg holding the unmodified 32-bit address to * ex_handler_bpf. */ - if (BPF_CLASS(insn->code) == BPF_LDX) + if (BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn)) arena_reg = bpf2a64[insn->src_reg]; else arena_reg = bpf2a64[insn->dst_reg]; @@ -1871,7 +1885,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn break; } - ret = add_exception_handler(insn, ctx, dst); + ret = add_exception_handler(insn, ctx, DONT_CLEAR); if (ret) return ret; break; @@ -1938,7 +1952,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn break; } - ret = add_exception_handler(insn, ctx, dst); + ret = add_exception_handler(insn, ctx, DONT_CLEAR); if (ret) return ret; break; @@ -1961,7 +1975,16 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn return ret; if (BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { - ret = add_exception_handler(insn, ctx, dst); + /* + * A load-acquire reads into dst_reg, and a read-modify-write + * carrying BPF_FETCH reads the old value into src_reg, or into + * r0 for a BPF_CMPXCHG. Clear that register on fault, the + * remaining atomics have no destination register. + */ + int load_reg = bpf_atomic_load_reg(insn); + + ret = add_exception_handler(insn, ctx, load_reg < 0 ? + DONT_CLEAR : bpf2a64[load_reg]); if (ret) return ret; } @@ -2491,9 +2514,8 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes) } static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, - const struct btf_func_model *m, - const struct arg_aux *a, - bool for_call_origin) + const struct btf_func_model *m, const struct arg_aux *a, + bool for_call_origin, bool is_struct_ops) { int i; int reg; @@ -2513,7 +2535,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, bargs_off += 8; } - soff = 32; /* on stack arguments start from FP + 32 */ + /* + * On-stack arguments start above the frame(s) pushed by the trampoline + * prologue. Entered through the fentry call from a traced function, the + * prologue saves both the parent (FP/x9) and the traced function + * (FP/LR) frames, so the arguments start at FP + 32. A struct_ops + * callback is called indirectly and only the FP/LR frame is saved, so + * they start at FP + 16. + */ + soff = is_struct_ops ? 16 : 32; doff = (for_call_origin ? oargs_off : bargs_off); /* save on stack arguments */ @@ -2709,7 +2739,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, store_func_meta(ctx, func_meta, func_meta_off); /* save args for bpf */ - save_args(ctx, bargs_off, oargs_off, m, a, false); + save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops); /* save callee saved registers */ emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx); @@ -2758,7 +2788,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, if (flags & BPF_TRAMP_F_CALL_ORIG) { /* save args for original func */ - save_args(ctx, bargs_off, oargs_off, m, a, true); + save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops); /* call original func */ emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx); emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx); diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 7cb61aca3797f..03f5e1755a633 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -3736,7 +3736,9 @@ Sysreg ICC_CR0_EL1 3 1 12 0 1 Res0 63:39 Field 38 PID Field 37:32 IPPT -Res0 31:1 +Res0 31:3 +Field 2 LINK_IDLE +Field 1 LINK Field 0 EN EndSysreg diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index c68cdcc76d60e..3261f46f22442 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -93,11 +93,11 @@ csky_syscall_trace: ldw a2, (sp, LSAVE_A2) ldw a3, (sp, LSAVE_A3) #if defined(__CSKYABIV2__) - subi sp, 8 ldw r9, (sp, LSAVE_A4) + ldw r10, (sp, LSAVE_A5) + subi sp, 8 stw r9, (sp, 0x0) - ldw r9, (sp, LSAVE_A5) - stw r9, (sp, 0x4) + stw r10, (sp, 0x4) jsr syscallid /* Do system call */ addi sp, 8 #else diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index d8d2523250179..14d31e9a3d705 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -175,7 +175,7 @@ config LOONGARCH select HAVE_RELIABLE_STACKTRACE if UNWINDER_ORC select HAVE_RETHOOK select HAVE_RSEQ - select HAVE_RUST + select HAVE_RUST if !KASAN select HAVE_SAMPLE_FTRACE_DIRECT select HAVE_SAMPLE_FTRACE_DIRECT_MULTI select HAVE_SETUP_PER_CPU_AREA if NUMA diff --git a/arch/loongarch/include/asm/kvm_dmsintc.h b/arch/loongarch/include/asm/kvm_dmsintc.h index 5a71b9ccbe78e..7c0158764d844 100644 --- a/arch/loongarch/include/asm/kvm_dmsintc.h +++ b/arch/loongarch/include/asm/kvm_dmsintc.h @@ -20,6 +20,7 @@ struct dmsintc_state { }; int kvm_loongarch_register_dmsintc_device(void); +void kvm_loongarch_unregister_dmsintc_device(void); void dmsintc_inject_irq(struct kvm_vcpu *vcpu); int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data, int level); int dmsintc_deliver_msi_to_vcpu(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 vector, int level); diff --git a/arch/loongarch/include/asm/kvm_eiointc.h b/arch/loongarch/include/asm/kvm_eiointc.h index 8b7a2fa3f7f86..9633fbfc066d6 100644 --- a/arch/loongarch/include/asm/kvm_eiointc.h +++ b/arch/loongarch/include/asm/kvm_eiointc.h @@ -79,6 +79,7 @@ struct loongarch_eiointc { }; int kvm_loongarch_register_eiointc_device(void); +void kvm_loongarch_unregister_eiointc_device(void); void eiointc_set_irq(struct loongarch_eiointc *s, int irq, int level); #endif /* __ASM_KVM_EIOINTC_H */ diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 23cfbecebbd79..3538a199aac62 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -125,6 +125,7 @@ struct kvm_arch { unsigned int pte_shifts[MAX_PGTABLE_LEVELS]; unsigned int root_level; spinlock_t phyid_map_lock; + spinlock_t pv_setting_lock; struct kvm_phyid_map *phyid_map; /* Enabled PV features */ unsigned long pv_features; diff --git a/arch/loongarch/include/asm/kvm_ipi.h b/arch/loongarch/include/asm/kvm_ipi.h index 060163dfb4a38..d1d72d4bb8d15 100644 --- a/arch/loongarch/include/asm/kvm_ipi.h +++ b/arch/loongarch/include/asm/kvm_ipi.h @@ -41,5 +41,6 @@ struct ipi_state { #define IOCSR_ANY_SEND 0x158 int kvm_loongarch_register_ipi_device(void); +void kvm_loongarch_unregister_ipi_device(void); #endif diff --git a/arch/loongarch/include/asm/kvm_pch_pic.h b/arch/loongarch/include/asm/kvm_pch_pic.h index e74b3b742634d..887b0431fd206 100644 --- a/arch/loongarch/include/asm/kvm_pch_pic.h +++ b/arch/loongarch/include/asm/kvm_pch_pic.h @@ -70,6 +70,7 @@ struct loongarch_pch_pic { struct kvm_kernel_irq_routing_entry; int kvm_loongarch_register_pch_pic_device(void); +void kvm_loongarch_unregister_pch_pic_device(void); void pch_pic_set_irq(struct loongarch_pch_pic *s, int irq, int level); int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, int level); diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 223528c04d739..a2191044435d2 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -96,7 +96,7 @@ struct vm_area_struct; #ifdef CONFIG_64BIT #define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE)) -#define MODULES_END (MODULES_VADDR + SZ_256M) +#define MODULES_END (MODULES_VADDR + SZ_2G) /* 256MB for text, rest for data */ #ifdef CONFIG_KFENCE #define KFENCE_AREA_SIZE (((CONFIG_KFENCE_NUM_OBJECTS + 1) * 2 + 2) * PAGE_SIZE) @@ -125,6 +125,13 @@ struct vm_area_struct; #endif +/* Needed to limit get_free_mem_region() */ +#ifndef CONFIG_SPARSEMEM +#define DIRECT_MAP_PHYSMEM_END ((1ULL << (cpu_pabits + 1)) - 1) +#else +#define DIRECT_MAP_PHYSMEM_END min((1ULL << (cpu_pabits + 1)) - 1, (1ULL << MAX_PHYSMEM_BITS) - 1) +#endif + #define ptep_get(ptep) READ_ONCE(*(ptep)) #define pmdp_get(pmdp) READ_ONCE(*(pmdp)) diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c index 873e90990771b..cb454ee92b204 100644 --- a/arch/loongarch/kernel/acpi.c +++ b/arch/loongarch/kernel/acpi.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -202,7 +203,7 @@ static void __init acpi_process_madt(void) int pptt_enabled; static int acpi_nr_packages; -static int acpi_package_ids[MAX_PACKAGES]; +static int acpi_package_ids[MAX(MAX_PACKAGES, KVM_MAX_VCPUS)]; int __init parse_acpi_topology(void) { diff --git a/arch/loongarch/kernel/kprobes.c b/arch/loongarch/kernel/kprobes.c index 1985ed30dd16f..ddfefea174727 100644 --- a/arch/loongarch/kernel/kprobes.c +++ b/arch/loongarch/kernel/kprobes.c @@ -275,6 +275,9 @@ bool kprobe_singlestep_handler(struct pt_regs *regs) struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); unsigned long addr = instruction_pointer(regs); + if (!cur) + return false; + if (cur && (kcb->kprobe_status & (KPROBE_HIT_SS | KPROBE_REENTER)) && ((unsigned long)&cur->ainsn.insn[1] == addr)) { restore_local_irqflag(kcb, regs); diff --git a/arch/loongarch/kernel/rethook_trampoline.S b/arch/loongarch/kernel/rethook_trampoline.S index 2e009fbea53f2..1601894446845 100644 --- a/arch/loongarch/kernel/rethook_trampoline.S +++ b/arch/loongarch/kernel/rethook_trampoline.S @@ -24,7 +24,6 @@ cfi_st t6, PT_R18 cfi_st t7, PT_R19 cfi_st t8, PT_R20 - cfi_st u0, PT_R21 cfi_st fp, PT_R22 cfi_st s0, PT_R23 cfi_st s1, PT_R24 @@ -59,7 +58,6 @@ cfi_ld t6, PT_R18 cfi_ld t7, PT_R19 cfi_ld t8, PT_R20 - cfi_ld u0, PT_R21 cfi_ld fp, PT_R22 cfi_ld s0, PT_R23 cfi_ld s1, PT_R24 diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index 8572b63478bb9..482ba17bcd3d8 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -481,7 +481,6 @@ int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst) srcu_read_unlock(&vcpu->kvm->srcu, idx); if (!ret) { kvm_complete_mmio_read(vcpu, run); - update_pc(&vcpu->arch); vcpu->mmio_needed = 0; return EMULATE_DONE; } diff --git a/arch/loongarch/kvm/intc/dmsintc.c b/arch/loongarch/kvm/intc/dmsintc.c index de25735ce0394..f5d3cdf291c32 100644 --- a/arch/loongarch/kvm/intc/dmsintc.c +++ b/arch/loongarch/kvm/intc/dmsintc.c @@ -19,8 +19,7 @@ void dmsintc_inject_irq(struct kvm_vcpu *vcpu) for (i = 0; i < 4; i++) { old = atomic64_read(&(ds->vector_map[i])); - if (old) - vector[i] = atomic64_xchg(&(ds->vector_map[i]), 0); + vector[i] = old ? atomic64_xchg(&(ds->vector_map[i]), 0) : 0; } if (vector[0]) { @@ -180,3 +179,8 @@ int kvm_loongarch_register_dmsintc_device(void) { return kvm_register_device_ops(&kvm_dmsintc_dev_ops, KVM_DEV_TYPE_LOONGARCH_DMSINTC); } + +void kvm_loongarch_unregister_dmsintc_device(void) +{ + kvm_unregister_device_ops(KVM_DEV_TYPE_LOONGARCH_DMSINTC); +} diff --git a/arch/loongarch/kvm/intc/eiointc.c b/arch/loongarch/kvm/intc/eiointc.c index 2b14485d14a70..5fd9d4c9238e1 100644 --- a/arch/loongarch/kvm/intc/eiointc.c +++ b/arch/loongarch/kvm/intc/eiointc.c @@ -691,3 +691,8 @@ int kvm_loongarch_register_eiointc_device(void) { return kvm_register_device_ops(&kvm_eiointc_dev_ops, KVM_DEV_TYPE_LOONGARCH_EIOINTC); } + +void kvm_loongarch_unregister_eiointc_device(void) +{ + kvm_unregister_device_ops(KVM_DEV_TYPE_LOONGARCH_EIOINTC); +} diff --git a/arch/loongarch/kvm/intc/ipi.c b/arch/loongarch/kvm/intc/ipi.c index 4fa0897d7bdb0..1e613741d7b8e 100644 --- a/arch/loongarch/kvm/intc/ipi.c +++ b/arch/loongarch/kvm/intc/ipi.c @@ -466,3 +466,8 @@ int kvm_loongarch_register_ipi_device(void) { return kvm_register_device_ops(&kvm_ipi_dev_ops, KVM_DEV_TYPE_LOONGARCH_IPI); } + +void kvm_loongarch_unregister_ipi_device(void) +{ + kvm_unregister_device_ops(KVM_DEV_TYPE_LOONGARCH_IPI); +} diff --git a/arch/loongarch/kvm/intc/pch_pic.c b/arch/loongarch/kvm/intc/pch_pic.c index 175a630aceb41..40a4aa3000d62 100644 --- a/arch/loongarch/kvm/intc/pch_pic.c +++ b/arch/loongarch/kvm/intc/pch_pic.c @@ -78,6 +78,9 @@ int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, int return dmsintc_set_irq(kvm, msg_addr, e->msi.data, level); } + if (e->msi.data >= EIOINTC_IRQS) + return -EINVAL; + eiointc_set_irq(kvm->arch.eiointc, e->msi.data, level); return 0; @@ -500,3 +503,8 @@ int kvm_loongarch_register_pch_pic_device(void) { return kvm_register_device_ops(&kvm_pch_pic_dev_ops, KVM_DEV_TYPE_LOONGARCH_PCHPIC); } + +void kvm_loongarch_unregister_pch_pic_device(void) +{ + kvm_unregister_device_ops(KVM_DEV_TYPE_LOONGARCH_PCHPIC); +} diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c index 3e1005526f4bd..236523d2449d8 100644 --- a/arch/loongarch/kvm/main.c +++ b/arch/loongarch/kvm/main.c @@ -385,27 +385,52 @@ static int kvm_loongarch_env_init(void) /* Register LoongArch IPI interrupt controller interface. */ ret = kvm_loongarch_register_ipi_device(); if (ret) - return ret; + goto err_env; /* Register LoongArch EIOINTC interrupt controller interface. */ ret = kvm_loongarch_register_eiointc_device(); if (ret) - return ret; + goto err_ipi; /* Register LoongArch PCH-PIC interrupt controller interface. */ ret = kvm_loongarch_register_pch_pic_device(); if (ret) - return ret; + goto err_eiointc; /* Register LoongArch DMSINTC interrupt contrroller interface */ - if (cpu_has_msgint) + if (cpu_has_msgint) { ret = kvm_loongarch_register_dmsintc_device(); + if (ret) + goto err_pch_pic; + } + + return 0; + +err_pch_pic: + kvm_loongarch_unregister_pch_pic_device(); +err_eiointc: + kvm_loongarch_unregister_eiointc_device(); +err_ipi: + kvm_loongarch_unregister_ipi_device(); +err_env: + kvm_unregister_perf_callbacks(); + kfree(kvm_loongarch_ops); + kvm_loongarch_ops = NULL; + free_percpu(vmcs); + vmcs = NULL; return ret; } static void kvm_loongarch_env_exit(void) { + if (cpu_has_msgint) + kvm_loongarch_unregister_dmsintc_device(); + + kvm_loongarch_unregister_pch_pic_device(); + kvm_loongarch_unregister_eiointc_device(); + kvm_loongarch_unregister_ipi_device(); + if (vmcs) free_percpu(vmcs); @@ -428,7 +453,11 @@ static int kvm_loongarch_init(void) if (r) return r; - return kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); + r = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); + if (r) + kvm_loongarch_env_exit(); + + return r; } static void kvm_loongarch_exit(void) diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c index e104897aa5328..4ca7c6ea8f088 100644 --- a/arch/loongarch/kvm/mmu.c +++ b/arch/loongarch/kvm/mmu.c @@ -383,6 +383,16 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_memory_slot hva_t hva_start; size_t size, gpa_offset, hva_offset; + /* + * The generic code allocates a fresh, zeroed memslot for every change, + * so the arch flags computed below must be carried over when only the + * userspace flags change, e.g. when dirty logging is toggled. + */ + if (change == KVM_MR_FLAGS_ONLY) { + new->arch = old->arch; + return 0; + } + if ((change != KVM_MR_MOVE) && (change != KVM_MR_CREATE)) return 0; /* diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 20c207d80e31a..ae6a6e30ab826 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -300,10 +300,10 @@ static int kvm_pre_enter_guest(struct kvm_vcpu *vcpu) */ local_irq_disable(); kvm_deliver_exception(vcpu); + kvm_check_vpid(vcpu); /* Make sure the vcpu mode has been written */ smp_store_mb(vcpu->mode, IN_GUEST_MODE); kvm_deliver_intr(vcpu); - kvm_check_vpid(vcpu); /* * Called after function kvm_check_vpid() @@ -1165,10 +1165,14 @@ static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu, return -EINVAL; /* All vCPUs need set the same PV features */ + spin_lock(&kvm->arch.pv_setting_lock); if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED) - && ((kvm->arch.pv_features & valid) != val)) + && ((kvm->arch.pv_features & valid) != val)) { + spin_unlock(&kvm->arch.pv_setting_lock); return -EINVAL; + } kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED; + spin_unlock(&kvm->arch.pv_setting_lock); return 0; default: return -ENXIO; diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c index 1317c718f896a..86ba7d54e81f8 100644 --- a/arch/loongarch/kvm/vm.c +++ b/arch/loongarch/kvm/vm.c @@ -76,6 +76,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) return -ENOMEM; } spin_lock_init(&kvm->arch.phyid_map_lock); + spin_lock_init(&kvm->arch.pv_setting_lock); kvm_init_vmcs(kvm); kvm_vm_init_features(kvm); diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c index 3407030f3e7a2..4b46c5d30708d 100644 --- a/arch/loongarch/mm/init.c +++ b/arch/loongarch/mm/init.c @@ -237,15 +237,26 @@ pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; EXPORT_SYMBOL(invalid_pte_table); #if defined(CONFIG_EXECMEM) && defined(MODULES_VADDR) +#define MODULES_TEXT_START (MODULES_VADDR) +#define MODULES_TEXT_END (MODULES_VADDR + SZ_256M) +#define MODULES_DATA_START (MODULES_VADDR + SZ_256M) +#define MODULES_DATA_END (MODULES_END) + static struct execmem_info execmem_info __ro_after_init; struct execmem_info __init *execmem_arch_setup(void) { execmem_info = (struct execmem_info){ .ranges = { - [EXECMEM_DEFAULT] = { - .start = MODULES_VADDR, - .end = MODULES_END, + [EXECMEM_MODULE_TEXT] = { + .start = MODULES_TEXT_START, + .end = MODULES_TEXT_END, + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + [EXECMEM_MODULE_DATA] = { + .start = MODULES_DATA_START, + .end = MODULES_DATA_END, .pgprot = PAGE_KERNEL, .alignment = 1, }, diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 29c281bef28ef..45eee761df9ee 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -124,6 +124,9 @@ static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) * | tcc | * +-------------------------+ * | tcc_ptr | + * +-------------------------+ + * | arena | + * | (optional) | * +-------------------------+ <--BPF_REG_FP * | prog->aux->stack_depth | * | (optional) | @@ -145,7 +148,7 @@ static void build_prologue(struct jit_ctx *ctx) stack_adjust += sizeof(long) * 2; if (ctx->arena_vm_start) - stack_adjust += 8; + stack_adjust += sizeof(long); stack_adjust = round_up(stack_adjust, 16); stack_adjust += bpf_stack_adjust; @@ -194,13 +197,13 @@ static void build_prologue(struct jit_ctx *ctx) store_offset -= sizeof(long); emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); + prepare_bpf_tail_call_cnt(ctx, &store_offset); + if (ctx->arena_vm_start) { store_offset -= sizeof(long); emit_insn(ctx, std, REG_ARENA, LOONGARCH_GPR_SP, store_offset); } - prepare_bpf_tail_call_cnt(ctx, &store_offset); - emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); if (bpf_stack_adjust) @@ -241,21 +244,18 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) load_offset -= sizeof(long); emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); + /* Only restore the TCC state into REG_TCC from the higher slot */ + load_offset -= sizeof(long); + emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); + + /* Skip the unused local 'tcc_ptr' slot to align with arena */ + load_offset -= sizeof(long); + if (ctx->arena_vm_start) { load_offset -= sizeof(long); emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset); } - /* - * When push into the stack, follow the order of tcc then tcc_ptr. - * When pop from the stack, first pop tcc_ptr then followed by tcc. - */ - load_offset -= 2 * sizeof(long); - emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); - - load_offset += sizeof(long); - emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); - emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); if (!is_tail_call) { @@ -290,17 +290,13 @@ bool bpf_jit_supports_far_kfunc_call(void) static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) { - int off, tc_ninsn = 0; + int off, jmp_offset; int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); u8 a1 = LOONGARCH_GPR_A1; u8 a2 = LOONGARCH_GPR_A2; u8 t1 = LOONGARCH_GPR_T1; u8 t2 = LOONGARCH_GPR_T2; u8 t3 = LOONGARCH_GPR_T3; - const int idx0 = ctx->idx; - -#define cur_offset (ctx->idx - idx0) -#define jmp_offset (tc_ninsn - (cur_offset)) /* * a0: &ctx @@ -310,12 +306,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) * if (index >= array->map.max_entries) * goto out; */ - tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; emit_zext_32(ctx, a2, true); off = offsetof(struct bpf_array, map.max_entries); emit_insn(ctx, ldwu, t1, a1, off); /* bgeu $a2, $t1, jmp_offset */ + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) goto toofar; @@ -326,6 +322,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); emit_insn(ctx, ldd, t3, REG_TCC, 0); emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JSGE, t3, t2, jmp_offset) < 0) goto toofar; @@ -340,6 +337,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) off = offsetof(struct bpf_array, ptrs); emit_insn(ctx, ldd, t2, t2, off); /* beq $t2, $zero, jmp_offset */ + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) goto toofar; @@ -355,8 +353,6 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) toofar: pr_info_once("tail_call: jump too far\n"); return -1; -#undef cur_offset -#undef jmp_offset } static void emit_store_stack_imm64(struct jit_ctx *ctx, int reg, int stack_off, u64 imm64) @@ -738,7 +734,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext move_reg(ctx, t1, src); emit_zext_32(ctx, t1, true); move_imm(ctx, dst, (ctx->user_vm_start >> 32) << 32, false); - emit_insn(ctx, beq, t1, LOONGARCH_GPR_ZERO, 1); + emit_insn(ctx, beq, t1, LOONGARCH_GPR_ZERO, 2); emit_insn(ctx, or, t1, dst, t1); move_reg(ctx, dst, t1); break; diff --git a/arch/m68k/emu/nfcon.c b/arch/m68k/emu/nfcon.c index d41260672e24b..f4657d9f4f153 100644 --- a/arch/m68k/emu/nfcon.c +++ b/arch/m68k/emu/nfcon.c @@ -49,7 +49,7 @@ static void nfcon_write(struct console *con, const char *str, static struct tty_driver *nfcon_device(struct console *con, int *index) { *index = 0; - return console_is_registered(con) ? nfcon_tty_driver : NULL; + return nfcon_tty_driver; } static struct console nf_console = { diff --git a/arch/m68k/kernel/traps.c b/arch/m68k/kernel/traps.c index d2f25e8895e5a..60759ba3cbd45 100644 --- a/arch/m68k/kernel/traps.c +++ b/arch/m68k/kernel/traps.c @@ -936,8 +936,8 @@ void show_stack(struct task_struct *task, unsigned long *stack, int i; if (!stack) { - if (task) - stack = (unsigned long *)task->thread.esp0; + if (task && task != current) + stack = (unsigned long *)task->thread.ksp; else stack = (unsigned long *)&stack; } diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 8555bbf47c633..1f8bdfd7e9a63 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -397,10 +397,7 @@ config ECONET select BOOT_RAW select CPU_BIG_ENDIAN select DEBUG_ZBOOT if DEBUG_KERNEL - select EARLY_PRINTK_8250 select ECONET_EN751221_TIMER - select SERIAL_8250 - select SERIAL_OF_PLATFORM select SYS_SUPPORTS_BIG_ENDIAN select SYS_HAS_CPU_MIPS32_R1 select SYS_HAS_CPU_MIPS32_R2 @@ -662,6 +659,7 @@ config EYEQ select USB_UHCI_BIG_ENDIAN_MMIO if CPU_BIG_ENDIAN select USE_OF select HOTPLUG_PARALLEL if HOTPLUG_CPU + select WEAK_REORDERING_BEYOND_LLSC help Select this to build a kernel supporting EyeQ SoC from Mobileye. diff --git a/arch/mips/cavium-octeon/octeon-platform.c b/arch/mips/cavium-octeon/octeon-platform.c index 47677b5d7ed00..f53c98372e0be 100644 --- a/arch/mips/cavium-octeon/octeon-platform.c +++ b/arch/mips/cavium-octeon/octeon-platform.c @@ -18,7 +18,7 @@ #include #include -#ifdef CONFIG_USB +#if IS_ENABLED(CONFIG_USB) #include #include #include @@ -1080,7 +1080,7 @@ int __init octeon_prune_device_tree(void) ; } -#ifdef CONFIG_USB +#if IS_ENABLED(CONFIG_USB) /* OHCI/UHCI USB */ alias_prop = fdt_getprop(initial_boot_params, aliases, "uctl", NULL); diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c index 3f4c94c881241..87102a03b6eaf 100644 --- a/arch/mips/kernel/ptrace.c +++ b/arch/mips/kernel/ptrace.c @@ -1321,8 +1321,12 @@ long arch_ptrace(struct task_struct *child, long request, */ asmlinkage long syscall_trace_enter(struct pt_regs *regs) { + long syscall; + user_exit(); + syscall = current_thread_info()->syscall; + if (test_thread_flag(TIF_SYSCALL_TRACE)) { if (ptrace_report_syscall_entry(regs)) return -1; @@ -1342,7 +1346,7 @@ asmlinkage long syscall_trace_enter(struct pt_regs *regs) * Negative syscall numbers are mistaken for rejected syscalls, but * won't have had the return value set appropriately, so we do so now. */ - if (current_thread_info()->syscall < 0) + if (syscall < 0) syscall_set_return_value(current, regs, -ENOSYS, 0); return current_thread_info()->syscall; } diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index 40a878b672f5d..15a2a153dc873 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -1111,7 +1111,7 @@ static void emit_jmp_i64(struct jit_context *ctx, emit(ctx, xor, tmp, lo(dst), tmp); } if (imm < 0) { /* Compare sign extension */ - emit(ctx, addu, MIPS_R_T9, hi(dst), 1); + emit(ctx, addiu, MIPS_R_T9, hi(dst), 1); emit(ctx, or, tmp, tmp, MIPS_R_T9); } else { /* Compare zero extension */ emit(ctx, or, tmp, tmp, hi(dst)); diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index fa7e9aa37f498..6681ccac9dd9e 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -305,8 +305,7 @@ static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width) case 16: emit_sext(ctx, dst, dst); emit_bswap_r(ctx, dst, width); - if (cpu_has_mips64r2 || cpu_has_mips64r6) - emit_zext(ctx, dst); + emit_zext(ctx, dst); break; } clobber_reg(ctx, dst); diff --git a/arch/openrisc/kernel/entry.S b/arch/openrisc/kernel/entry.S index c7e90b09645e4..18e68680471eb 100644 --- a/arch/openrisc/kernel/entry.S +++ b/arch/openrisc/kernel/entry.S @@ -1223,15 +1223,50 @@ _no_syscall_trace: * */ +/* Keep this literal; hi()/lo() can't use the UL-suffixed TASK_SIZE. */ +#define OR1K_ATOMIC_ADDR_LIMIT 0x7ffffffc + ENTRY(sys_or1k_atomic) /* FIXME: This ignores r3 and always does an XCHG */ + + /* Check both user pointers before accessing them. */ + l.movhi r13,hi(OR1K_ATOMIC_ADDR_LIMIT) + l.ori r13,r13,lo(OR1K_ATOMIC_ADDR_LIMIT) + l.sfgtu r4,r13 + l.bf 9f + l.nop + l.sfgtu r5,r13 + l.bf 9f + l.nop + DISABLE_INTERRUPTS(r17,r19) - l.lwz r29,0(r4) - l.lwz r27,0(r5) - l.sw 0(r4),r27 - l.sw 0(r5),r29 +10: l.lwz r29,0(r4) +11: l.lwz r27,0(r5) +12: l.sw 0(r4),r27 +13: l.sw 0(r5),r29 ENABLE_INTERRUPTS(r17) l.jr r9 l.or r11,r0,r0 + /* + * Either pointer was outside user space, or turned out to be + * unmapped/inaccessible when we actually touched it. + */ +9: l.jr r9 + l.addi r11,r0,-EFAULT + + .section .fixup, "ax" +14: + ENABLE_INTERRUPTS(r17) + l.j 9b + l.nop + .previous + + .section __ex_table, "a" + .long 10b, 14b + .long 11b, 14b + .long 12b, 14b + .long 13b, 14b + .previous + /* ============================================================[ EOF ]=== */ diff --git a/arch/parisc/include/asm/thread_info.h b/arch/parisc/include/asm/thread_info.h index b283738bb6dab..249370706242c 100644 --- a/arch/parisc/include/asm/thread_info.h +++ b/arch/parisc/include/asm/thread_info.h @@ -24,7 +24,7 @@ struct thread_info { /* thread information allocation */ -#ifdef CONFIG_IRQSTACKS +#if defined(CONFIG_IRQSTACKS) && !defined(CONFIG_64BIT) #define THREAD_SIZE_ORDER 2 /* PA-RISC requires at least 16k stack */ #else #define THREAD_SIZE_ORDER 3 /* PA-RISC requires at least 32k stack */ diff --git a/arch/parisc/kernel/head.S b/arch/parisc/kernel/head.S index 9188c8d874370..e84d7d6e1eb94 100644 --- a/arch/parisc/kernel/head.S +++ b/arch/parisc/kernel/head.S @@ -105,6 +105,7 @@ $iodc_panic: or %r10,%r10,%r10 /* qemu idle sleep */ msg1: .ascii "Can't boot kernel which was built for PA8x00 CPUs on this machine.\r\n" msg1_end: + .align 4 $cpu_ok: #endif diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c index d3e17a7a89016..4d3015a411d6f 100644 --- a/arch/parisc/kernel/setup.c +++ b/arch/parisc/kernel/setup.c @@ -132,6 +132,18 @@ void __init setup_arch(char **cmdline_p) parisc_cache_init(); paging_init(); + /* + * Parse early parameters before mm_core_init_early() runs. + * Several early_param() handlers only record data that is consumed + * from there - for example hugepages=, hugepagesz=, + * default_hugepagesz=, hugetlb_cma= and hugetlb_free_vmemmap= - so + * the generic parse_early_param() call in start_kernel() is too late + * for them. jump_label_init() must come first, since early param + * handlers may enable or disable static keys. + */ + jump_label_init(); + parse_early_param(); + #ifdef CONFIG_PA11 dma_ops_init(); #endif diff --git a/arch/powerpc/configs/85xx-hw.config b/arch/powerpc/configs/85xx-hw.config index 2b19c20a9a2c4..6b84718101432 100644 --- a/arch/powerpc/configs/85xx-hw.config +++ b/arch/powerpc/configs/85xx-hw.config @@ -90,6 +90,7 @@ CONFIG_PPC_EPAPR_HV_BYTECHAN=y CONFIG_QE_GPIO=y CONFIG_QUICC_ENGINE=y CONFIG_RAPIDIO=y +CONFIG_RAS=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_CMOS=y CONFIG_RTC_DRV_DS1307=y diff --git a/arch/powerpc/include/asm/entry-common.h b/arch/powerpc/include/asm/entry-common.h index c5adb50063610..8e91489fdf2bb 100644 --- a/arch/powerpc/include/asm/entry-common.h +++ b/arch/powerpc/include/asm/entry-common.h @@ -222,8 +222,6 @@ static inline void arch_interrupt_enter_prepare(struct pt_regs *regs) if (user_mode(regs)) { kuap_lock(); - account_cpu_user_entry(); - account_stolen_time(); } else { kuap_save_and_lock(regs); /* @@ -270,7 +268,7 @@ static inline void arch_interrupt_exit_prepare(struct pt_regs *regs) } /* irqentry_exit expects to be called with interrupts disabled */ - local_irq_disable(); + hard_irq_disable(); } static inline void arch_interrupt_async_enter_prepare(struct pt_regs *regs) @@ -515,8 +513,14 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, #ifdef CONFIG_PPC_TRANSACTIONAL_MEM local_paca->tm_scratch = regs->msr; #endif - /* Restore user access locks last */ - kuap_user_restore(regs); + /* + * Do not restore KUAP here. Generic entry might treat this as the last + * arch step before userspace but PowerPC still has kernel work after + * irqentry_exit()/syscall_exit_to_user_mode() i.e. in + * interrupt_exit_user_prepare() / syscall_exit_prepare() may enable + * IRQs and retry. Those functions restore KUAP immediately before rfi, + * which is where it should belong. + */ } #define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare diff --git a/arch/powerpc/include/asm/papr-watchdog.h b/arch/powerpc/include/asm/papr-watchdog.h new file mode 100644 index 0000000000000..bf876fc2caae5 --- /dev/null +++ b/arch/powerpc/include/asm/papr-watchdog.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ASM_POWERPC_PAPR_WATCHDOG_H +#define _ASM_POWERPC_PAPR_WATCHDOG_H + +/* + * H_WATCHDOG Input + * + * R4: "flags": + * + * Bits 48-55: "operation" + */ +#define PSERIES_WDTF_OP_START 0x100UL /* start timer */ +#define PSERIES_WDTF_OP_STOP 0x200UL /* stop timer */ +#define PSERIES_WDTF_OP_QUERY 0x300UL /* query timer capabilities */ + +/* + * Bits 56-63: "timeoutAction" (for "Start Watchdog" only) + */ +#define PSERIES_WDTF_ACTION_HARD_POWEROFF 0x1UL /* poweroff */ +#define PSERIES_WDTF_ACTION_HARD_RESTART 0x2UL /* restart */ +#define PSERIES_WDTF_ACTION_DUMP_RESTART 0x3UL /* dump + restart */ + +/* + * R5: "watchdogNumber": + * PAPR says use -1 (all ones) to stop all watchdogs. + */ +#define PSERIES_WDT_NUM_ALL ((unsigned long)-1) + +/* + * H_WATCHDOG Output + * + * R3: Return code + * + * H_SUCCESS The operation completed. + * + * H_BUSY The hypervisor is too busy; retry the operation. + * + * H_PARAMETER The given "flags" are somehow invalid. Either the + * "operation" or "timeoutAction" is invalid, or a + * reserved bit is set. + * + * H_P2 The given "watchdogNumber" is zero or exceeds the + * supported maximum value. + * + * H_P3 The given "timeoutInMs" is below the supported + * minimum value. + * + * H_NOOP The given "watchdogNumber" is already stopped. + * + * H_HARDWARE The operation failed for ineffable reasons. + * + * H_FUNCTION The H_WATCHDOG hypercall is not supported by this + * hypervisor. + * + * R4: + * + * - For the "Query Watchdog Capabilities" operation, a 64-bit + * structure: + */ +#define PSERIES_WDTQ_MIN_TIMEOUT(cap) (((cap) >> 48) & 0xffff) +#define PSERIES_WDTQ_MAX_NUMBER(cap) (((cap) >> 32) & 0xffff) + +#endif /* _ASM_POWERPC_PAPR_WATCHDOG_H */ diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index 3449dd2b577d4..b9ab9df1e2bcc 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -1357,6 +1357,18 @@ #define PVR_ARCH_31 0x0f000006 #define PVR_ARCH_31_P11 0x0f000007 +/* + * Kernel-internal sentinel for invalid processor compatibility modes. + * PAPR specifies that the first byte of a valid logical PVR value is + * 0x0f. So 0xffffffff lies permanently outside the PAPR-defined range + * and is safe to repurpose. KVM stores it in vcpu->arch.arch_compat + * when userspace requests an unsupported compatibility mode (e.g., + * Power11 PVR on a Power11 host booted in Power10 compat). + * kvmppc_sanity_check() detects this and prevents the vCPU from + * running with an unsupported arch_compat. + */ +#define PVR_ARCH_INVALID 0xffffffff + /* Macros for setting and retrieving special purpose registers */ #ifndef __ASSEMBLER__ diff --git a/arch/powerpc/include/asm/syscall.h b/arch/powerpc/include/asm/syscall.h index 834fcc4f7b543..19d1739af0b77 100644 --- a/arch/powerpc/include/asm/syscall.h +++ b/arch/powerpc/include/asm/syscall.h @@ -98,6 +98,12 @@ static inline void syscall_set_return_value(struct task_struct *task, regs->gpr[3] = val; } } + /* + * Mark that a return value has been explicitly set by seccomp or + * ptrace so that system_call_exception() can skip the syscall + * unconditionally, even when the user requested syscall(-1). + */ + set_thread_flag(TIF_SYSCALL_RET); } static inline void syscall_get_arguments(struct task_struct *task, diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index 0487e94d34169..1e069a2e7ce87 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -120,6 +120,7 @@ void arch_setup_new_exec(void); #endif #define TIF_POLLING_NRFLAG 19 /* true if poll_idle() is polling TIF_NEED_RESCHED */ #define TIF_32BIT 20 /* 32 bit binary */ +#define TIF_SYSCALL_RET 21 /* syscall error value set */ /* as above, but as bit values */ #define _TIF_SYSCALL_TRACE (1<rmv_entry, &rmv_data->removed_vf_list); } else { - pci_lock_rescan_remove(); pci_stop_and_remove_bus_device(dev); - pci_unlock_rescan_remove(); } } diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index 5b88bf72786c7..55f9c0c9922ac 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -175,7 +175,7 @@ notrace unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *reg current_thread_info()->exit_flags &= ~_TIF_RESTOREALL; regs->exit_result |= ret; - return ret; + return regs->exit_result; } #endif diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index ee1b5cb557c9a..1ae8384637b5f 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -1076,7 +1076,7 @@ int iommu_tce_check_ioba(unsigned long page_shift, if (ioba < offset) return -EINVAL; - if ((ioba + 1) > (offset + size)) + if ((ioba + npages < ioba) || (ioba - offset + npages > size)) return -EINVAL; return 0; diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index f69de08ad347f..15a3c3fd8e705 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -217,8 +217,12 @@ static __always_inline void call_do_softirq(const void *sp) [sp] "b" (sp), [offset] "i" (THREAD_SIZE - STACK_FRAME_MIN_SIZE), [callee] "i" (__do_softirq) : // Clobbers - "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6", - "cr7", "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", + "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6", "cr7", "r0", + /* r2 may be clobbered by the callee when using PCREL mode in the ELFv2 ABI. */ +#ifdef CONFIG_PPC_KERNEL_PCREL + "r2", +#endif + "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" ); } @@ -275,8 +279,12 @@ static __always_inline void call_do_irq(struct pt_regs *regs, void *sp) [sp] "b" (sp), [offset] "i" (THREAD_SIZE - STACK_FRAME_MIN_SIZE), [callee] "i" (__do_irq) : // Clobbers - "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6", - "cr7", "r0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", + "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6", "cr7", "r0", + /* r2 may be clobbered by the callee when using PCREL mode in the ELFv2 ABI. */ +#ifdef CONFIG_PPC_KERNEL_PCREL + "r2", +#endif + "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" ); } diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index 3c4ca90e2ab77..4fc52c21fe5d4 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -626,19 +626,14 @@ int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, size_t size) return -ENXIO; addr = hose->io_base_virt + port; - /* WARNING: The generic code is idiotic. It gets passed a pointer - * to what can be a 1, 2 or 4 byte quantity and always reads that - * as a u32, which means that we have to correct the location of - * the data read within those 32 bits for size 1 and 2 - */ switch(size) { case 1: - out_8(addr, val >> 24); + out_8(addr, val); return 1; case 2: if (port & 1) return -EINVAL; - out_le16(addr, val >> 16); + out_le16(addr, val); return 2; case 4: if (port & 3) diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 3467f86fd78f2..6a5a5469aaae5 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -289,6 +289,9 @@ void smp_muxed_ipi_set_message(int cpu, int msg) void smp_muxed_ipi_message_pass(int cpu, int msg) { + if (!smp_ops->cause_ipi) + return; + smp_muxed_ipi_set_message(cpu, msg); /* diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c index a9da2af6efa87..9d1b29f44ea0d 100644 --- a/arch/powerpc/kernel/syscall.c +++ b/arch/powerpc/kernel/syscall.c @@ -22,6 +22,9 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0) add_random_kstack_offset(); r0 = syscall_enter_from_user_mode(regs, r0); + if (unlikely(test_and_clear_thread_flag(TIF_SYSCALL_RET))) + return syscall_get_error(current, regs); + if (unlikely(r0 >= NR_syscalls)) { if (unlikely(trap_is_unsupported_scv(regs))) { /* Unsupported scv vector */ diff --git a/arch/powerpc/kexec/crash.c b/arch/powerpc/kexec/crash.c index e6539f213b3d1..a520f851c3a6b 100644 --- a/arch/powerpc/kexec/crash.c +++ b/arch/powerpc/kexec/crash.c @@ -502,7 +502,7 @@ static void update_crash_elfcorehdr(struct kimage *image, struct memory_notify * ret = get_crash_memory_ranges(&cmem); if (ret) { pr_err("Failed to get crash mem range\n"); - return; + goto out; } /* diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index 8c72e12ea44e5..d990880b77dfd 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -57,7 +57,7 @@ int arch_check_excluded_range(struct kimage *image, unsigned long start, emem = image->arch.exclude_ranges; for (i = 0; i < emem->nr_ranges; i++) - if (start < emem->ranges[i].end && end > emem->ranges[i].start) + if (start <= emem->ranges[i].end && end >= emem->ranges[i].start) return 1; return 0; @@ -113,7 +113,7 @@ static int add_usable_mem(struct umem_info *um_info, u64 base, u64 end) loc_end = um_info->ranges[i].end; if (loc_base >= base && loc_end <= end) add = true; - else if (base < loc_end && end > loc_base) { + else if (base <= loc_end && end >= loc_base) { if (loc_base < base) loc_base = base; if (loc_end > end) @@ -664,7 +664,7 @@ unsigned int kexec_extra_fdt_size_ppc64(struct kimage *image, struct crash_mem * extra_size += (cpu_nodes - boot_cpu_node_count) * cpu_node_size(); /* Consider extra space for reserved memory ranges if any */ - if (rmem->nr_ranges > 0) + if (rmem && rmem->nr_ranges > 0) extra_size += sizeof(struct fdt_reserve_entry) * rmem->nr_ranges; return extra_size + kdump_extra_fdt_size_ppc64(image, cpu_nodes); diff --git a/arch/powerpc/kexec/ranges.c b/arch/powerpc/kexec/ranges.c index 867135560e5c8..eb45e89502cae 100644 --- a/arch/powerpc/kexec/ranges.c +++ b/arch/powerpc/kexec/ranges.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -105,19 +106,16 @@ static void __merge_memory_ranges(struct crash_mem *mem_rngs) struct range *ranges; int i, idx; - if (!mem_rngs) + if (!mem_rngs || mem_rngs->nr_ranges <= 1) return; idx = 0; - ranges = &(mem_rngs->ranges[0]); + ranges = mem_rngs->ranges; for (i = 1; i < mem_rngs->nr_ranges; i++) { - if (ranges[i].start <= (ranges[i-1].end + 1)) - ranges[idx].end = ranges[i].end; + if (ranges[i].start <= (ranges[idx].end + 1)) + ranges[idx].end = max(ranges[idx].end, ranges[i].end); else { idx++; - if (i == idx) - continue; - ranges[idx] = ranges[i]; } } diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 61dbeea317f3b..dd128b293992b 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -446,7 +446,19 @@ static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) guest_pcr_bit = PCR_ARCH_300; break; case PVR_ARCH_31: + guest_pcr_bit = PCR_ARCH_31; + break; case PVR_ARCH_31_P11: + /* + * Need to check this for ISA 3.1, as Power10 and + * Power11 share the same PCR. For any subsequent ISA + * versions, this will be taken care of by the guest vs + * host PCR comparison below. + */ + if (!cpu_has_feature(CPU_FTR_P11_PVR)) { + arch_compat = PVR_ARCH_INVALID; + goto out; + } guest_pcr_bit = PCR_ARCH_31; break; default: @@ -469,6 +481,7 @@ static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) return -EINVAL; } +out: spin_lock(&vc->lock); vc->arch_compat = arch_compat; kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_LOGICAL_PVR); @@ -479,7 +492,7 @@ static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) vc->pcr = (host_pcr_bit - guest_pcr_bit) | PCR_MASK; spin_unlock(&vc->lock); - return 0; + return kvmppc_sanity_check(vcpu); } static void kvmppc_dump_regs(struct kvm_vcpu *vcpu) @@ -6111,12 +6124,12 @@ static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons, struct kvm_kernel_irqfd *irqfd = container_of(cons, struct kvm_kernel_irqfd, consumer); - irqfd->producer = prod; - ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi); if (ret) pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n", prod->irq, irqfd->gsi, ret); + else + irqfd->producer = prod; return ret; } diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c index 22e6166622556..a6ff42d7666c1 100644 --- a/arch/powerpc/kvm/book3s_hv_nested.c +++ b/arch/powerpc/kvm/book3s_hv_nested.c @@ -1204,8 +1204,10 @@ static void kvmhv_emulate_tlbie_all_lpid(struct kvm_vcpu *vcpu, int ric) spin_lock(&kvm->mmu_lock); idr_for_each_entry(&kvm->arch.kvm_nested_guest_idr, gp, lpid) { + ++gp->refcnt; spin_unlock(&kvm->mmu_lock); kvmhv_emulate_tlbie_lpid(vcpu, gp, ric); + kvmhv_put_nested(gp); spin_lock(&kvm->mmu_lock); } spin_unlock(&kvm->mmu_lock); diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index 5fbb95d90e996..463aef870c4eb 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -779,8 +779,11 @@ static int kvmppc_svm_page_in(struct vm_area_struct *vma, if (spage) { ret = uv_page_in(kvm->arch.lpid, pfn << page_shift, gpa, 0, page_shift); - if (ret) + if (ret) { + unlock_page(dpage); + put_page(dpage); goto out_finalize; + } } } diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 00302399fc37b..b6b83fe3233f3 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -258,6 +258,12 @@ int kvmppc_sanity_check(struct kvm_vcpu *vcpu) if (!vcpu->arch.pvr) goto out; +#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) + if (vcpu->arch.vcore && + vcpu->arch.vcore->arch_compat == PVR_ARCH_INVALID) + goto out; +#endif + /* PAPR only works with book3s_64 */ if ((vcpu->arch.cpu_type != KVM_CPU_3S_64) && vcpu->arch.papr_enabled) goto out; diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 10aced261cff4..cf692b2b5f7bc 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -1314,7 +1314,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * covering out both edges. */ unsigned long addr; - unsigned long addr_pfn = start_pfn; unsigned long next; pgd_t *pgd; p4d_t *p4d; @@ -1335,7 +1334,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (pmd_leaf(READ_ONCE(*pmd))) { /* existing huge mapping. Skip the range */ - addr_pfn += (PMD_SIZE >> PAGE_SHIFT); next = pmd_addr_end(addr, end); continue; } @@ -1348,11 +1346,11 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * page whose VMEMMAP_RESERVE_NR pages were mapped and * this request fall in those pages. */ - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } else { unsigned long nr_pages = pgmap_vmemmap_nr(pgmap); + unsigned long addr_pfn = page_to_pfn((struct page *)addr); unsigned long pfn_offset = addr_pfn - ALIGN_DOWN(addr_pfn, nr_pages); pte_t *tail_page_pte; @@ -1376,7 +1374,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (!pte) return -ENOMEM; - addr_pfn += 2; next = addr + 2 * PAGE_SIZE; continue; } @@ -1392,7 +1389,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } @@ -1402,7 +1398,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index dab106cae22b5..fc9db691e8203 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -471,8 +471,6 @@ static int bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 f int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func) { unsigned long func_addr = func ? ppc_function_entry((void *)func) : 0; - long __maybe_unused reladdr; - int ret; /* bpf to bpf call, func is not known in the initial pass. Emit 5 nops as a placeholder */ if (!func) { @@ -487,6 +485,8 @@ int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context * } #ifdef CONFIG_PPC_KERNEL_PCREL + long reladdr; + reladdr = func_addr - local_paca->kernelbase; /* @@ -525,7 +525,7 @@ int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context * EMIT(PPC_RAW_BCTRL()); #else if (core_kernel_text(func_addr)) { - ret = bpf_jit_emit_func_call(image, ctx, func_addr, _R12); + int ret = bpf_jit_emit_func_call(image, ctx, func_addr, _R12); if (ret) return ret; } else { diff --git a/arch/powerpc/net/bpf_timed_may_goto.S b/arch/powerpc/net/bpf_timed_may_goto.S index 6fd8b1c9f4ac8..84ecf6fa7f5dc 100644 --- a/arch/powerpc/net/bpf_timed_may_goto.S +++ b/arch/powerpc/net/bpf_timed_may_goto.S @@ -36,7 +36,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) * BPF_REG_FP is r31; BPF_REG_AX is r12 (stack offset in bytes). */ add r3, r31, r12 - bl bpf_check_timed_may_goto + bl CFUNC(bpf_check_timed_may_goto) /* Put return value back into AX */ mr r12, r3 diff --git a/arch/powerpc/platforms/44x/gpio.c b/arch/powerpc/platforms/44x/gpio.c index aea0d913b59d0..4413a94cf7a6a 100644 --- a/arch/powerpc/platforms/44x/gpio.c +++ b/arch/powerpc/platforms/44x/gpio.c @@ -169,6 +169,7 @@ static int ppc4xx_gpio_probe(struct platform_device *ofdev) gc = &chip->gc; + gc->parent = dev; gc->base = -1; gc->ngpio = 32; gc->direction_input = ppc4xx_gpio_dir_in; diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c index da72a30ab8657..973f58771d963 100644 --- a/arch/powerpc/platforms/powermac/low_i2c.c +++ b/arch/powerpc/platforms/powermac/low_i2c.c @@ -1471,7 +1471,7 @@ static int __init pmac_i2c_create_platform_devices(void) if (bus->platform_dev == NULL) return -ENOMEM; bus->platform_dev->dev.platform_data = bus; - bus->platform_dev->dev.of_node = bus->busnode; + bus->platform_dev->dev.of_node = of_node_get(bus->busnode); platform_device_add(bus->platform_dev); } diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 32ecbc46e74b9..728a5610d1675 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -1623,15 +1623,13 @@ int64_t pnv_opal_pci_msi_eoi(struct irq_data *d) return opal_pci_msi_eoi(phb->opal_id, d->parent_data->hwirq); } -static struct irq_chip pnv_pci_msi_irq_chip; - /* * Returns true iff chip is something that we could call * pnv_opal_pci_msi_eoi for. */ bool is_pnv_opal_msi(struct irq_chip *chip) { - return chip == &pnv_pci_msi_irq_chip; + return chip && chip->name && str_has_prefix(chip->name, "PNV-"); } EXPORT_SYMBOL_GPL(is_pnv_opal_msi); @@ -1728,7 +1726,7 @@ static const struct msi_parent_ops pnv_msi_parent_ops = { .chip_flags = MSI_CHIP_FLAG_SET_EOI, .bus_select_token = DOMAIN_BUS_NEXUS, .bus_select_mask = MATCH_PCI_MSI, - .prefix = "PNV-", + .prefix = "PNV-", /* Note: is_pnv_opal_msi() uses this */ .init_dev_msi_info = pnv_init_dev_msi_info, }; diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c index 8f41ef364fc6f..b1201dbafcaf6 100644 --- a/arch/powerpc/platforms/powernv/smp.c +++ b/arch/powerpc/platforms/powernv/smp.c @@ -332,10 +332,12 @@ static void pnv_cause_ipi(int cpu) static void __init pnv_smp_probe(void) { - if (xive_enabled()) - xive_smp_probe(); - else + if (xive_enabled()) { + if (xive_smp_probe() < 0) + return; + } else { xics_smp_probe(); + } if (cpu_has_feature(CPU_FTR_DBELL)) { ic_cause_ipi = smp_ops->cause_ipi; diff --git a/arch/powerpc/platforms/ps3/repository.c b/arch/powerpc/platforms/ps3/repository.c index b8c030eab1384..0cc755ac3e7f5 100644 --- a/arch/powerpc/platforms/ps3/repository.c +++ b/arch/powerpc/platforms/ps3/repository.c @@ -6,6 +6,8 @@ * Copyright 2006 Sony Corp. */ +#include + #include #include "platform.h" @@ -74,8 +76,9 @@ static void _dump_node(unsigned int lpar_id, u64 n1, u64 n2, u64 n3, u64 n4, static u64 make_first_field(const char *text, u64 index) { u64 n = 0; + size_t len = min(strlen(text), sizeof(n)); - memcpy((char *)&n, text, strnlen(text, sizeof(n))); + memcpy(&n, text, len); return PS3_VENDOR_ID_NONE + (n >> 32) + index; } diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index 3e1f915fe4f66..272e9aab66d4f 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -812,18 +812,11 @@ static struct device_node *pci_dma_find(struct device_node *dn, /* parse DMA window property. During normal system boot, only default * DMA window is passed in OF. But, for kdump, a dedicated adapter might - * have both default and DDW in FDT. In this scenario, DDW takes precedence - * over default window. + * have both default and DDW in FDT. In this scenario, default window + * takes precedence over DDW. For a dedicated adapter, default window will + * potentially have more unused TCEs. */ - if (ddw_win) { - struct dynamic_dma_window_prop *p; - - p = (struct dynamic_dma_window_prop *)ddw_prop; - prop->liobn = p->liobn; - prop->dma_base = p->dma_base; - prop->tce_shift = p->tce_shift; - prop->window_shift = p->window_shift; - } else if (default_win) { + if (default_win) { unsigned long offset, size, liobn; of_parse_dma_window(rdn, default_prop, &liobn, &offset, &size); @@ -832,6 +825,14 @@ static struct device_node *pci_dma_find(struct device_node *dn, prop->dma_base = cpu_to_be64(offset); prop->tce_shift = cpu_to_be32(IOMMU_PAGE_SHIFT_4K); prop->window_shift = cpu_to_be32(order_base_2(size)); + } else { + struct dynamic_dma_window_prop *p; + + p = (struct dynamic_dma_window_prop *)ddw_prop; + prop->liobn = p->liobn; + prop->dma_base = p->dma_base; + prop->tce_shift = p->tce_shift; + prop->window_shift = p->window_shift; } return rdn; diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index 1223dc9612429..f29e7547c9956 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -77,6 +77,7 @@ #include #include #include +#include #include "pseries.h" @@ -185,14 +186,37 @@ static void __init fwnmi_init(void) #endif } +static void pseries_crash_stop_watchdogs(void) +{ + long rc; + + rc = plpar_hcall_norets_notrace(H_WATCHDOG, PSERIES_WDTF_OP_STOP, + PSERIES_WDT_NUM_ALL); + if (rc != H_SUCCESS && rc != H_NOOP) + pr_warn("Could not stop watchdogs before kdump rc=%ld\n", rc); +} + /* * Affix a device for the first timer to the platform bus if * we have firmware support for the H_WATCHDOG hypercall. */ static __init int pseries_wdt_init(void) { - if (firmware_has_feature(FW_FEATURE_WATCHDOG)) - platform_device_register_simple("pseries-wdt", 0, NULL, 0); + struct platform_device *pdev; + + if (!firmware_has_feature(FW_FEATURE_WATCHDOG)) + return 0; + + pdev = platform_device_register_simple("pseries-wdt", 0, NULL, 0); + + if (IS_ERR(pdev)) { + pr_err("Failed to register pseries-wdt platform device\n"); + return PTR_ERR(pdev); + } + + if (crash_shutdown_register(pseries_crash_stop_watchdogs)) + pr_warn("Could not register watchdog crash shutdown handler\n"); + return 0; } machine_subsys_initcall(pseries, pseries_wdt_init); diff --git a/arch/powerpc/platforms/pseries/smp.c b/arch/powerpc/platforms/pseries/smp.c index bf3d7ed3be010..9e1bed383e337 100644 --- a/arch/powerpc/platforms/pseries/smp.c +++ b/arch/powerpc/platforms/pseries/smp.c @@ -199,10 +199,12 @@ static int pseries_cause_nmi_ipi(int cpu) static __init void pSeries_smp_probe(void) { - if (xive_enabled()) - xive_smp_probe(); - else + if (xive_enabled()) { + if (xive_smp_probe() < 0) + return; + } else { xics_smp_probe(); + } /* No doorbell facility, must use the interrupt controller for IPIs */ if (!cpu_has_feature(CPU_FTR_DBELL)) diff --git a/arch/powerpc/sysdev/cpm_common.c b/arch/powerpc/sysdev/cpm_common.c index 07ea605ab0e62..b5d200e3ad684 100644 --- a/arch/powerpc/sysdev/cpm_common.c +++ b/arch/powerpc/sysdev/cpm_common.c @@ -181,6 +181,18 @@ static int cpm2_gpio32_dir_in(struct gpio_chip *gc, unsigned int gpio) return 0; } +static int cpm2_gpio32_get_direction(struct gpio_chip *gc, unsigned int gpio) +{ + struct cpm2_gpio32_chip *cpm2_gc = gpiochip_get_data(gc); + struct cpm2_ioports __iomem *iop = cpm2_gc->regs; + u32 pin_mask = 1 << (31 - gpio); + + if (in_be32(&iop->dir) & pin_mask) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; +} + int cpm2_gpiochip_add32(struct device *dev) { struct device_node *np = dev->of_node; @@ -199,6 +211,7 @@ int cpm2_gpiochip_add32(struct device *dev) gc->ngpio = 32; gc->direction_input = cpm2_gpio32_dir_in; gc->direction_output = cpm2_gpio32_dir_out; + gc->get_direction = cpm2_gpio32_get_direction; gc->get = cpm2_gpio32_get; gc->set = cpm2_gpio32_set; gc->parent = dev; diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c index dadd1f46ec939..bbe7c85274ea9 100644 --- a/arch/powerpc/sysdev/xive/common.c +++ b/arch/powerpc/sysdev/xive/common.c @@ -74,6 +74,8 @@ static struct xive_ipi_desc { */ static unsigned int xive_ipi_cpu_to_irq(unsigned int cpu) { + if (!xive_ipis) + return XIVE_BAD_IRQ; return xive_ipis[early_cpu_to_node(cpu)].irq; } #endif @@ -1132,8 +1134,7 @@ static int __init xive_init_ipis(void) if (!ipi_domain) goto out_free_fwnode; - xive_ipis = kzalloc_objs(*xive_ipis, nr_node_ids, - GFP_KERNEL | __GFP_NOFAIL); + xive_ipis = kzalloc_objs(*xive_ipis, nr_node_ids, GFP_KERNEL); if (!xive_ipis) goto out_free_domain; @@ -1158,6 +1159,7 @@ static int __init xive_init_ipis(void) out_free_xive_ipis: kfree(xive_ipis); + xive_ipis = NULL; out_free_domain: irq_domain_remove(ipi_domain); out_free_fwnode: @@ -1190,6 +1192,9 @@ static int xive_setup_cpu_ipi(unsigned int cpu) pr_debug("Setting up IPI for CPU %d\n", cpu); + if (xive_ipi_irq == XIVE_BAD_IRQ) + return -EIO; + xc = per_cpu(xive_cpu, cpu); /* Check if we are already setup */ @@ -1234,6 +1239,9 @@ noinstr static void xive_cleanup_cpu_ipi(unsigned int cpu, struct xive_cpu *xc) /* Disable the IPI and free the IRQ data */ + if (xive_ipi_irq == XIVE_BAD_IRQ) + return; + /* Already cleaned up ? */ if (xc->hw_ipi == XIVE_BAD_IRQ) return; @@ -1257,15 +1265,19 @@ noinstr static void xive_cleanup_cpu_ipi(unsigned int cpu, struct xive_cpu *xc) xive_ops->put_ipi(cpu, xc); } -void __init xive_smp_probe(void) +int __init xive_smp_probe(void) { + int ret; + smp_ops->cause_ipi = xive_cause_ipi; /* Register the IPI */ - xive_init_ipis(); + ret = xive_init_ipis(); + if (ret < 0) + return ret; /* Allocate and setup IPI for the boot CPU */ - xive_setup_cpu_ipi(smp_processor_id()); + return xive_setup_cpu_ipi(smp_processor_id()); } #endif /* CONFIG_SMP */ diff --git a/arch/riscv/boot/dts/sophgo/cv180x.dtsi b/arch/riscv/boot/dts/sophgo/cv180x.dtsi index 06b0ce5a2db7a..ecafdee79ac8e 100644 --- a/arch/riscv/boot/dts/sophgo/cv180x.dtsi +++ b/arch/riscv/boot/dts/sophgo/cv180x.dtsi @@ -417,7 +417,7 @@ }; dmac: dma-controller@4330000 { - compatible = "snps,axi-dma-1.01a"; + compatible = "sophgo,cv1800b-axi-dma", "snps,axi-dma-1.01a"; reg = <0x04330000 0x1000>; interrupts = ; clocks = <&clk CLK_SDMA_AXI>, <&clk CLK_SDMA_AXI>; diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts index 72f77e9edd63d..7f3dd08e74d79 100644 --- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts +++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts @@ -101,7 +101,7 @@ phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -124,7 +124,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; @@ -204,14 +204,14 @@ regulators { buck1 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; buck2 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; diff --git a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts index 2fc8d65337863..c76b91ecb914d 100644 --- a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts +++ b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts @@ -121,7 +121,7 @@ phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -144,7 +144,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; @@ -211,14 +211,14 @@ regulators { buck1 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; buck2 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; diff --git a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts index 0d809e4ad3b1f..246f8f2ab62b9 100644 --- a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts +++ b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts @@ -102,7 +102,7 @@ ð0 { phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; pinctrl-names = "default"; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; @@ -180,14 +180,14 @@ regulators { buck1 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; buck2 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts index b13a8d6a2670e..ae2fd8f7a8572 100644 --- a/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts +++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts @@ -60,7 +60,7 @@ phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -84,7 +84,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; @@ -126,14 +126,14 @@ regulators { buck1 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; buck2 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts index 7c49bce427f30..b907a1396b21f 100644 --- a/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts +++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts @@ -164,14 +164,14 @@ regulators { buck1 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; buck2 { regulator-min-microvolt = <500000>; - regulator-max-microvolt = <3450000>; + regulator-max-microvolt = <1050000>; regulator-ramp-delay = <5000>; regulator-always-on; }; diff --git a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi index 4e9a62d0e85b5..8c57ca05dabdb 100644 --- a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi +++ b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi @@ -27,8 +27,16 @@ , /* gmac0_tx_en */ , /* gmac0_mdc */ , /* gmac0_mdio */ - , /* gmac0_int_n */ - ; /* gmac0_clk_ref */ + ; /* gmac0_int_n */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + gmac0_clk_ref_cfg: gmac0-clk-ref-cfg { + gmac0-clk-ref-pins { + pinmux = ; /* gmac0_clk_ref */ bias-pull-up = <0>; drive-strength = <21>; @@ -51,8 +59,16 @@ , /* gmac1_tx_en */ , /* gmac1_mdc */ , /* gmac1_mdio */ - , /* gmac1_int_n */ - ; /* gmac1_clk_ref */ + ; /* gmac1_int_n */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + gmac1_clk_ref_cfg: gmac1-clk-ref-cfg { + gmac1-clk-ref-pins { + pinmux = ; /* gmac1_clk_ref */ bias-pull-up = <0>; drive-strength = <21>; diff --git a/arch/riscv/boot/dts/spacemit/k3-com260.dtsi b/arch/riscv/boot/dts/spacemit/k3-com260.dtsi index a38d7b738258d..c7a04a338d080 100644 --- a/arch/riscv/boot/dts/spacemit/k3-com260.dtsi +++ b/arch/riscv/boot/dts/spacemit/k3-com260.dtsi @@ -173,12 +173,12 @@ status = "okay"; mdio { - phy1: phy@1 { - compatible = "ethernet-phy-ieee802.3-c22"; + phy1: ethernet-phy@1 { + compatible = "ethernet-phy-id001c.c916"; reg = <1>; reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>; reset-assert-us = <10000>; - reset-deassert-us = <10000>; + reset-deassert-us = <50000>; }; }; }; diff --git a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts index b89c1521e6649..86210cc6f7733 100644 --- a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts +++ b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts @@ -18,7 +18,7 @@ }; chosen { - stdout-path = "serial0"; + stdout-path = "serial0:115200n8"; }; memory@100000000 { @@ -190,12 +190,14 @@ status = "okay"; mdio { - phy0: phy@1 { - compatible = "ethernet-phy-ieee802.3-c22"; + phy0: ethernet-phy@1 { + compatible = "ethernet-phy-id001c.c916"; reg = <1>; reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>; reset-assert-us = <10000>; - reset-deassert-us = <10000>; + reset-deassert-us = <50000>; + realtek,aldps-enable; + realtek,clkout-disable; }; }; }; diff --git a/arch/riscv/boot/dts/thead/th1520.dtsi b/arch/riscv/boot/dts/thead/th1520.dtsi index 94932c51b7e39..db23624696b73 100644 --- a/arch/riscv/boot/dts/thead/th1520.dtsi +++ b/arch/riscv/boot/dts/thead/th1520.dtsi @@ -415,8 +415,7 @@ compatible = "thead,th1520-i2c", "snps,designware-i2c"; reg = <0xff 0xe7f24000 0x0 0x4000>; interrupts = <45 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&clk CLK_I2C1>, <&clk CLK_PERI_APB_PCLK>; - clock-names = "ref", "pclk"; + clocks = <&clk CLK_I2C1>; #address-cells = <1>; #size-cells = <0>; status = "disabled"; diff --git a/arch/riscv/include/asm/acpi.h b/arch/riscv/include/asm/acpi.h index 26ab37c171bcf..f4db04f5bbe75 100644 --- a/arch/riscv/include/asm/acpi.h +++ b/arch/riscv/include/asm/acpi.h @@ -12,6 +12,8 @@ #ifndef _ASM_ACPI_H #define _ASM_ACPI_H +#include + /* Basic configuration for ACPI */ #ifdef CONFIG_ACPI @@ -67,6 +69,23 @@ int acpi_get_riscv_isa(struct acpi_table_header *table, void acpi_get_cbo_block_size(struct acpi_table_header *table, u32 *cbom_size, u32 *cboz_size, u32 *cbop_size); + +/* + * RISC-V Functional Fixed Hardware Specification Version v1.0.1, + * Chapter 3.1.2, Table 4: Arch. Context Lost Flags + */ +#define RISCV_LPI_HART_TIMER_CTXT_LOST BIT(0) + +static inline unsigned int arch_get_idle_state_flags(u32 arch_flags) +{ + if (arch_flags & RISCV_LPI_HART_TIMER_CTXT_LOST) + return CPUIDLE_FLAG_TIMER_STOP; + + return 0; +} + +#define arch_get_idle_state_flags arch_get_idle_state_flags + #else static inline void acpi_init_rintc_map(void) { } static inline struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu) diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 31b8988f4488d..72ff259154e9b 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -183,12 +183,24 @@ #define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT #endif -/* VSIP & HVIP relation */ +/* + * VSIP & HVIP relation + * + * The bit positions are same between VSIP and HVIP for interrupt + * numbers 13-63, where there's a shift for the SSI, STI and SEI. + */ #define VSIP_TO_HVIP_SHIFT (IRQ_VS_SOFT - IRQ_S_SOFT) -#define VSIP_VALID_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ +#define VSIP_BIAS_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ (_AC(1, UL) << IRQ_S_TIMER) | \ - (_AC(1, UL) << IRQ_S_EXT) | \ - (_AC(1, UL) << IRQ_PMU_OVF)) + (_AC(1, UL) << IRQ_S_EXT)) +#define VSIP_NO_BIAS_MASK (_AC(1, UL) << IRQ_PMU_OVF) +#define VSIP_VALID_MASK (VSIP_BIAS_MASK | VSIP_NO_BIAS_MASK) +#define vsip_to_hvip(_vsip) ((((_vsip) & VSIP_BIAS_MASK) << \ + VSIP_TO_HVIP_SHIFT) | \ + ((_vsip) & VSIP_NO_BIAS_MASK)) +#define hvip_to_vsip(_hvip) ((((_hvip) >> VSIP_TO_HVIP_SHIFT) & \ + VSIP_BIAS_MASK) | \ + ((_hvip) & VSIP_NO_BIAS_MASK)) /* AIA CSR bits */ #define TOPI_IID_SHIFT 16 diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index f46aa5602d74d..64c7ec6abbf1f 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -452,7 +452,8 @@ static const unsigned int riscv_c_exts[] = { /* * The canonical order of ISA extension names in the ISA string is defined in - * chapter 27 of the unprivileged specification. + * Chapter 27 of the RISC-V Instruction Set Manual Volume I Unprivileged ISA + * (Document Version 20191213). * * Ordinarily, for in-kernel data structures, this order is unimportant but * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. diff --git a/arch/riscv/kernel/machine_kexec_file.c b/arch/riscv/kernel/machine_kexec_file.c index 59d4bbc848a89..fa2946aa9b8f4 100644 --- a/arch/riscv/kernel/machine_kexec_file.c +++ b/arch/riscv/kernel/machine_kexec_file.c @@ -62,7 +62,7 @@ static int prepare_elf_headers(void **addr, unsigned long *sz) unsigned int nr_ranges; int ret; - nr_ranges = 1; /* For exclusion of crashkernel region */ + nr_ranges = 2; /* For exclusion of crashkernel region */ walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); cmem = kmalloc_flex(*cmem, ranges, nr_ranges); @@ -77,8 +77,16 @@ static int prepare_elf_headers(void **addr, unsigned long *sz) /* Exclude crashkernel region */ ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); - if (!ret) - ret = crash_prepare_elf64_headers(cmem, true, addr, sz); + if (ret) + goto out; + + if (crashk_low_res.end) { + ret = crash_exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end); + if (ret) + goto out; + } + + ret = crash_prepare_elf64_headers(cmem, true, addr, sz); out: kfree(cmem); diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index b2df7f72241a5..7cc5a6a5c0206 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -349,10 +349,8 @@ long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg) if (arg & PR_TAGGED_ADDR_ENABLE && (tagged_addr_disabled || !pmlen)) return -EINVAL; - if (!(arg & PR_TAGGED_ADDR_ENABLE)) { + if (!(arg & PR_TAGGED_ADDR_ENABLE)) pmlen = PMLEN_0; - pmm = ENVCFG_PMM_PMLEN_0; - } if (mmap_write_lock_killable(mm)) return -EINTR; diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index caf6762427c87..d2bb70fed8b9f 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -295,6 +295,8 @@ static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) static void hwprobe_one_pair(struct riscv_hwprobe *pair, const struct cpumask *cpus) { + pair->value = 0; + switch (pair->key) { case RISCV_HWPROBE_KEY_MVENDORID: case RISCV_HWPROBE_KEY_MARCHID: @@ -329,17 +331,14 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, break; case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: - pair->value = 0; if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) pair->value = riscv_cboz_block_size; break; case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE: - pair->value = 0; if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM)) pair->value = riscv_cbom_block_size; break; case RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE: - pair->value = 0; if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOP)) pair->value = riscv_cbop_block_size; break; diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 36201613d2caf..67f1a03716597 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -288,18 +287,9 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus __free_pages(page, MISALIGNED_BUFFER_ORDER); } -/* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) -{ - schedule_on_each_cpu(check_vector_unaligned_access); - riscv_hwprobe_complete_async_probe(); - - return 0; -} #else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ -static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +static void check_vector_unaligned_access(struct work_struct *work __always_unused) { - return 0; } #endif @@ -387,12 +377,7 @@ static int __init check_unaligned_access_all_cpus(void) per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; } else if (!check_vector_unaligned_access_emulated_all_cpus() && IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { - riscv_hwprobe_register_async_probe(); - if (IS_ERR(kthread_run(vec_check_unaligned_access_speed_all_cpus, - NULL, "vec_check_unaligned_access_speed_all_cpus"))) { - pr_warn("Failed to create vec_unalign_check kthread\n"); - riscv_hwprobe_complete_async_probe(); - } + schedule_on_each_cpu(check_vector_unaligned_access); } /* diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c index d38f5de0834c5..74f520e572b1e 100644 --- a/arch/riscv/kvm/aia_imsic.c +++ b/arch/riscv/kvm/aia_imsic.c @@ -969,9 +969,14 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, if (!vcpu) return -ENODEV; + if (mutex_lock_killable(&vcpu->mutex)) + return -EINTR; + imsic = vcpu->arch.aia_context.imsic_state; - if (!imsic) - return -ENODEV; + if (!imsic) { + rc = -ENODEV; + goto out_unlock; + } isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); read_lock_irqsave(&imsic->vsfile_lock, flags); @@ -995,6 +1000,8 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix, isel, write, val); +out_unlock: + mutex_unlock(&vcpu->mutex); return rc; } diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 8a0aa5e0e216e..59e7297dea4a3 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -540,6 +540,7 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, int ret; kvm_pfn_t hfn; bool is_hugetlb; + bool unused = false; bool writable; short vma_pageshift; gfn_t gfn = gpa >> PAGE_SHIFT; @@ -620,6 +621,8 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, vma_pageshift, current); return 0; } + if (is_sigpending_pfn(hfn)) + return -EINTR; if (is_error_noslot_pfn(hfn)) return -EFAULT; @@ -632,8 +635,10 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, write_lock(&kvm->mmu_lock); - if (mmu_invalidate_retry(kvm, mmu_seq)) + if (mmu_invalidate_retry(kvm, mmu_seq)) { + unused = true; goto out_unlock; + } /* * Check if we are backed by a THP and thus use block mapping if @@ -656,7 +661,8 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, kvm_err("Failed to map in G-stage\n"); out_unlock: - kvm_release_faultin_page(kvm, page, ret && ret != -EEXIST, writable); + kvm_release_faultin_page(kvm, page, + unused || (ret && ret != -EEXIST), writable); write_unlock(&kvm->mmu_lock); return ret; } diff --git a/arch/riscv/kvm/nacl.c b/arch/riscv/kvm/nacl.c index 6f9f8963e9ddc..0a2a50c6ce035 100644 --- a/arch/riscv/kvm/nacl.c +++ b/arch/riscv/kvm/nacl.c @@ -42,12 +42,24 @@ void __kvm_riscv_nacl_hfence(void *shmem, } } - entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_CONFIG(i); - *entp = cpu_to_lelong(control); + /* + * Per SBI v3.0 section 15.1.2, the Page_Number and Page_Count + * words must be updated before the Config word with its Pending + * bit set. WRITE_ONCE() stops the compiler from reordering the + * stores and smp_wmb() makes the parameter words globally + * visible to the SBI implementation (or NACL hardware) before + * the Pending bit is set. + */ entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_PNUM(i); - *entp = cpu_to_lelong(page_num); + WRITE_ONCE(*entp, cpu_to_lelong(page_num)); entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_PCOUNT(i); - *entp = cpu_to_lelong(page_count); + WRITE_ONCE(*entp, cpu_to_lelong(page_count)); + + /* Ensure the parameter words are visible before the Pending bit */ + smp_wmb(); + + entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_CONFIG(i); + WRITE_ONCE(*entp, cpu_to_lelong(control)); } int kvm_riscv_nacl_enable(void) diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 977e36ab83d3f..468918309dff3 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -475,8 +475,7 @@ bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) bool ret; raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); - ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) - << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; + ie = vsip_to_hvip(vcpu->arch.guest_csr.vsie) & (unsigned long)mask; ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & (unsigned long)mask; ret = vcpu->arch.irqs_pending[0] & ie; diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 6c8530b9f29ed..8092200c0f478 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -267,7 +267,7 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, } /* Print details in-case of error */ - if (ret < 0) { + if (ret < 0 && ret != -EINTR) { kvm_err("VCPU exit error %d\n", ret); kvm_err("SEPC=0x%lx SSTATUS=0x%lx HSTATUS=0x%lx\n", vcpu->arch.guest_context.sepc, diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index 99b9107b1ac18..9fe829eed1781 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -272,7 +272,7 @@ static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { kvm_riscv_vcpu_flush_interrupts(vcpu); - *out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK; + *out_val = hvip_to_vsip(csr->hvip); *out_val |= csr->hvip & ~IRQ_LOCAL_MASK; } else *out_val = ((unsigned long *)csr)[reg_num]; @@ -293,10 +293,8 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, reg_num = array_index_nospec(reg_num, regs_max); - if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { - reg_val &= VSIP_VALID_MASK; - reg_val <<= VSIP_TO_HVIP_SHIFT; - } + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) + reg_val = vsip_to_hvip(reg_val); ((unsigned long *)csr)[reg_num] = reg_val; diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index 2025b664961c8..73d76fd19896b 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -12,7 +12,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -268,12 +270,13 @@ static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, return -EINVAL; pmc->counter_val = kvpmu->fw_event[fevent_code].value; + *out_val = pmc->counter_val; } else if (pmc->perf_event) { - pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running); + *out_val = pmc->counter_val + + perf_event_read_value(pmc->perf_event, &enabled, &running); } else { return -EINVAL; } - *out_val = pmc->counter_val; return 0; } @@ -479,13 +482,14 @@ int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low unsigned long flags, struct kvm_vcpu_sbi_return *retdata) { struct riscv_pmu_event_info *einfo = NULL; - int shmem_size = num_events * sizeof(*einfo); + size_t shmem_size; gpa_t shmem; u32 eidx, etype; u64 econfig; int ret; - if (flags != 0 || (saddr_low & (SZ_16 - 1) || num_events == 0)) { + if (flags != 0 || (saddr_low & (SZ_16 - 1)) || num_events == 0 || + check_mul_overflow(num_events, sizeof(*einfo), &shmem_size)) { ret = SBI_ERR_INVALID_PARAM; goto out; } @@ -500,7 +504,8 @@ int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low } } - einfo = kzalloc(shmem_size, GFP_KERNEL); + einfo = kvcalloc(num_events, sizeof(*einfo), + GFP_KERNEL_ACCOUNT | __GFP_NOWARN); if (!einfo) { ret = SBI_ERR_FAILURE; goto out; @@ -512,7 +517,7 @@ int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low goto free_mem; } - for (int i = 0; i < num_events; i++) { + for (unsigned long i = 0; i < num_events; i++) { eidx = einfo[i].event_idx; etype = kvm_pmu_get_perf_event_type(eidx); econfig = kvm_pmu_get_perf_event_config(eidx, einfo[i].event_data); @@ -525,7 +530,7 @@ int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low ret = SBI_ERR_INVALID_ADDRESS; free_mem: - kfree(einfo); + kvfree(einfo); out: retdata->err_val = ret; @@ -641,7 +646,6 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, { struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); int i, pmc_index, sbiret = 0; - u64 enabled, running; struct kvm_pmc *pmc; int fevent_code; bool snap_flag_set = flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT; @@ -671,14 +675,19 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, goto out; } - if (!kvpmu->fw_event[fevent_code].started) + if (!kvpmu->fw_event[fevent_code].started) { sbiret = SBI_ERR_ALREADY_STOPPED; - - kvpmu->fw_event[fevent_code].started = false; + } else { + kvpmu->fw_event[fevent_code].started = false; + pmc->counter_val = kvpmu->fw_event[fevent_code].value; + } } else if (pmc->perf_event) { if (pmc->started) { - /* Stop counting the counter */ - perf_event_disable(pmc->perf_event); + /* + * Stop the counter and fold the live count into counter_val. + * Reset the event value to avoid redundant accumulation. + */ + pmc->counter_val += perf_event_pause(pmc->perf_event, true); pmc->started = false; } else { sbiret = SBI_ERR_ALREADY_STOPPED; @@ -692,11 +701,6 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } if (snap_flag_set && !sbiret) { - if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) - pmc->counter_val = kvpmu->fw_event[fevent_code].value; - else if (pmc->perf_event) - pmc->counter_val += perf_event_read_value(pmc->perf_event, - &enabled, &running); /* * The counter and overflow indices in the snapshot region are w.r.to * cbase. Modify the set bit in the counter mask instead of the pmc_index @@ -723,9 +727,10 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } } - if (shmem_needs_update) - kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata, - sizeof(struct riscv_pmu_snapshot_data)); + if (shmem_needs_update && + kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata, + sizeof(struct riscv_pmu_snapshot_data))) + sbiret = SBI_ERR_FAILURE; out: retdata->err_val = sbiret; diff --git a/arch/riscv/kvm/vcpu_sbi_hsm.c b/arch/riscv/kvm/vcpu_sbi_hsm.c index f26207f84bab6..06a15629c26bb 100644 --- a/arch/riscv/kvm/vcpu_sbi_hsm.c +++ b/arch/riscv/kvm/vcpu_sbi_hsm.c @@ -95,9 +95,9 @@ static int kvm_sbi_ext_hsm_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, ret = kvm_sbi_hsm_vcpu_get_status(vcpu); if (ret >= 0) { retdata->out_val = ret; - retdata->err_val = 0; + ret = 0; } - return 0; + break; case SBI_EXT_HSM_HART_SUSPEND: switch (lower_32_bits(cp->a0)) { case SBI_HSM_SUSPEND_RET_DEFAULT: diff --git a/arch/riscv/kvm/vcpu_timer.c b/arch/riscv/kvm/vcpu_timer.c index ae53133c7ab03..a2cd277a40596 100644 --- a/arch/riscv/kvm/vcpu_timer.c +++ b/arch/riscv/kvm/vcpu_timer.c @@ -61,10 +61,13 @@ static enum hrtimer_restart kvm_riscv_vcpu_hrtimer_expired(struct hrtimer *h) static int kvm_riscv_vcpu_timer_cancel(struct kvm_vcpu_timer *t) { - if (!t->init_done || !t->next_set) + if (!t->init_done) return -EINVAL; hrtimer_cancel(&t->hrt); + + if (!t->next_set) + return -EINVAL; t->next_set = false; return 0; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index f9d5347ba966b..a3af5070ac66d 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -18,7 +18,6 @@ #define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) -#define RV_KCFI_NINSNS (IS_ENABLED(CONFIG_CFI) ? 1 : 0) /* imm that allows emit_imm to emit max count insns */ #define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF @@ -272,8 +271,8 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) if (!is_tail_call) emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, - /* kcfi, fentry and TCC init insns will be skipped on tailcall */ - is_tail_call ? (RV_KCFI_NINSNS + RV_FENTRY_NINSNS + 1) * 4 : 0, + /* fentry and TCC init insns will be skipped on tailcall */ + is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, ctx); } @@ -1831,9 +1830,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, for (idx = 0; idx < fm->nr_args; idx++) { u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); + bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG; - if (fm->arg_size[idx] == sizeof(int)) - emit_sextw(reg, reg, ctx); + if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) + return -EINVAL; } } @@ -1986,7 +1986,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, else ret = emit_atomic_rmw(rd, rs, insn, ctx); - ret = ret ?: add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); + /* ret can be 1 (skip-zext); extable entry still needs to be added */ + if (ret >= 0) { + /* + * A load-acquire reads into dst_reg, and a read-modify-write + * carrying BPF_FETCH reads the old value into src_reg, or into + * r0 for a BPF_CMPXCHG. Clear that register on fault, the + * remaining atomics have no destination register. + */ + int load_reg = bpf_atomic_load_reg(insn); + + ret = add_exception_handler(insn, load_reg < 0 ? + REG_DONT_CLEAR_MARKER : regmap[load_reg], + ctx) ?: ret; + } + if (ret) return ret; break; @@ -2033,6 +2047,8 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) /* emit kcfi type preamble immediately before the first insn */ emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); + /* bpf prog starts here as kcfi skipped during prog->bpf_func setup */ + /* nops reserved for auipc+jalr pair */ for (i = 0; i < RV_FENTRY_NINSNS; i++) emit(rv_nop(), ctx); @@ -2114,7 +2130,15 @@ bool bpf_jit_supports_ptr_xchg(void) bool bpf_jit_supports_arena(void) { - return true; + /* + * The arena range tree uses kmalloc_nolock(), which needs + * cmpxchg128, provided by ZACAS on riscv. + */ +#ifdef system_has_cmpxchg128 + return system_has_cmpxchg128(); +#else + return false; +#endif } bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index ce3bd3762e08c..7cce191186190 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -234,6 +234,7 @@ void bpf_jit_free(struct bpf_prog *prog) */ if (jit_data) { bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); + kfree(jit_data->ctx.offset); kfree(jit_data); } hdr = bpf_jit_binary_pack_hdr(prog); diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index 6bc950b92be76..59eabf4a2de05 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -23,6 +23,7 @@ struct parmarea parmarea __section(".parmarea") = { }; char __bootdata(early_command_line)[COMMAND_LINE_SIZE]; +static char command_line_buf[COMMAND_LINE_SIZE]; unsigned int __bootdata_preserved(zlib_dfltcc_support) = ZLIB_DFLTCC_FULL; struct ipl_parameter_block __bootdata_preserved(ipl_block); @@ -135,31 +136,29 @@ static size_t ipl_block_get_ascii_scpdata(char *dest, size_t size, static void append_ipl_block_parm(void) { - char *parm, *delim; - size_t len, rc = 0; + size_t len, extra = 0; + char *delim; len = strlen(early_command_line); - - delim = early_command_line + len; /* '\0' character position */ - parm = early_command_line + len + 1; /* append right after '\0' */ + delim = early_command_line + len; /* '\0' character position */ switch (ipl_block.pb0_hdr.pbt) { case IPL_PBT_CCW: - rc = ipl_block_get_ascii_vmparm( - parm, COMMAND_LINE_SIZE - len - 1, &ipl_block); + extra = ipl_block_get_ascii_vmparm(command_line_buf, sizeof(command_line_buf), &ipl_block); break; case IPL_PBT_FCP: case IPL_PBT_NVME: case IPL_PBT_ECKD: - rc = ipl_block_get_ascii_scpdata( - parm, COMMAND_LINE_SIZE - len - 1, &ipl_block); + extra = ipl_block_get_ascii_scpdata(command_line_buf, sizeof(command_line_buf), &ipl_block); break; } - if (rc) { - if (*parm == '=') - memmove(early_command_line, parm + 1, rc); - else + if (extra) { + if (command_line_buf[0] == '=') { + memmove(early_command_line, command_line_buf + 1, extra); + } else if (len < COMMAND_LINE_SIZE - 2) { *delim = ' '; /* replace '\0' with space */ + sized_strscpy(delim + 1, command_line_buf, COMMAND_LINE_SIZE - len - 1); + } } } @@ -245,7 +244,6 @@ static void modify_fac_list(char *str) check_cleared_facilities(); } -static char command_line_buf[COMMAND_LINE_SIZE]; void parse_boot_command_line(void) { char *param, *val; diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c index 1f2ca5435838e..0ebb2174713f2 100644 --- a/arch/s390/boot/physmem_info.c +++ b/arch/s390/boot/physmem_info.c @@ -141,7 +141,7 @@ static int tprot(unsigned long addr) static unsigned long search_mem_end(void) { - unsigned long range = 1 << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ + unsigned long range = 1UL << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ unsigned long offset = 0; unsigned long pivot; diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 62edc66d54788..0be6fa779d2cc 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -26,14 +26,14 @@ #include #include #include -#include #include +#include #include #include #include static u8 *ctrblk; -static DEFINE_MUTEX(ctrblk_lock); +static DEFINE_SEMAPHORE(ctrblk_sem, 1); static cpacf_mask_t km_functions, kmc_functions, kmctr_functions, kma_functions; @@ -129,7 +129,7 @@ static int ecb_aes_crypt(struct skcipher_request *req, unsigned long modifier) return fallback_skcipher_crypt(sctx, req, modifier); ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(sctx->fc | modifier, sctx->key, @@ -233,7 +233,7 @@ static int cbc_aes_crypt(struct skcipher_request *req, unsigned long modifier) return ret; memcpy(param.iv, walk.iv, AES_BLOCK_SIZE); memcpy(param.key, sctx->key, sctx->key_len); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_kmc(sctx->fc | modifier, ¶m, @@ -359,7 +359,7 @@ static int xts_aes_crypt(struct skcipher_request *req, unsigned long modifier) memcpy(xts_param.key + offset, xts_ctx->key, xts_ctx->key_len); memcpy(xts_param.init, pcc_param.xts, 16); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(xts_ctx->fc | modifier, xts_param.key + offset, @@ -487,7 +487,7 @@ static int fullxts_aes_crypt(struct skcipher_request *req, unsigned long modifi memcpy(fxts_param.tweak, req->iv, AES_BLOCK_SIZE); fxts_param.nap[0] = 0x01; /* initial alpha power (1, little-endian) */ - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(xts_ctx->fc | modifier, fxts_param.key + offset, @@ -562,48 +562,64 @@ static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) return n; } +static int __ctr_aes_crypt(struct s390_aes_ctx *sctx, + struct skcipher_walk *walk, bool locked) +{ + unsigned int n, nbytes; + int ret = 0; + u8 *ctrptr; + + while (!ret && ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE)) { + n = AES_BLOCK_SIZE; + if (nbytes >= 2 * AES_BLOCK_SIZE && locked) + n = __ctrblk_init(ctrblk, walk->iv, nbytes); + ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; + cpacf_kmctr(sctx->fc, sctx->key, walk->dst.virt.addr, + walk->src.virt.addr, n, ctrptr); + if (ctrptr == ctrblk) + memcpy(walk->iv, ctrptr + n - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(walk->iv, AES_BLOCK_SIZE); + ret = skcipher_walk_done(walk, nbytes - n); + } + + return ret; +} + static int ctr_aes_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm); - u8 buf[AES_BLOCK_SIZE], *ctrptr; struct skcipher_walk walk; - unsigned int n, nbytes; - int ret, locked; + u8 buf[AES_BLOCK_SIZE]; + int ret; if (unlikely(!sctx->fc)) return fallback_skcipher_crypt(sctx, req, 0); - locked = mutex_trylock(&ctrblk_lock); - ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { - n = AES_BLOCK_SIZE; + if (ret) + return ret; - if (nbytes >= 2*AES_BLOCK_SIZE && locked) - n = __ctrblk_init(ctrblk, walk.iv, nbytes); - ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv; - cpacf_kmctr(sctx->fc, sctx->key, walk.dst.virt.addr, - walk.src.virt.addr, n, ctrptr); - if (ctrptr == ctrblk) - memcpy(walk.iv, ctrptr + n - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - crypto_inc(walk.iv, AES_BLOCK_SIZE); - ret = skcipher_walk_done(&walk, nbytes - n); + if (down_trylock(&ctrblk_sem) == 0) { + ret = __ctr_aes_crypt(sctx, &walk, true); + up(&ctrblk_sem); + } else { + ret = __ctr_aes_crypt(sctx, &walk, false); } - if (locked) - mutex_unlock(&ctrblk_lock); + /* * final block may be < AES_BLOCK_SIZE, copy only nbytes */ - if (nbytes) { + if (!ret && walk.nbytes > 0) { memset(buf, 0, AES_BLOCK_SIZE); - memcpy(buf, walk.src.virt.addr, nbytes); + memcpy(buf, walk.src.virt.addr, walk.nbytes); cpacf_kmctr(sctx->fc, sctx->key, buf, buf, AES_BLOCK_SIZE, walk.iv); - memcpy(walk.dst.virt.addr, buf, nbytes); + memcpy(walk.dst.virt.addr, buf, walk.nbytes); crypto_inc(walk.iv, AES_BLOCK_SIZE); ret = skcipher_walk_done(&walk, 0); + memzero_explicit(buf, sizeof(buf)); } return ret; @@ -895,10 +911,14 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags) gw_in.ptr, aad_bytes); n = aad_bytes + pc_bytes; - if (gcm_in_walk_done(&gw_in, n) != n) - return -ENOMEM; - if (gcm_out_walk_done(&gw_out, n) != n) - return -ENOMEM; + if (gcm_in_walk_done(&gw_in, n) != n) { + ret = -ENOMEM; + goto out; + } + if (gcm_out_walk_done(&gw_out, n) != n) { + ret = -ENOMEM; + goto out; + } aadlen -= aad_bytes; pclen -= pc_bytes; } while (aadlen + pclen > 0); @@ -910,7 +930,10 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags) } else scatterwalk_map_and_copy(param.t, req->dst, len, taglen, 1); +out: memzero_explicit(¶m, sizeof(param)); + memzero_explicit(gw_in.buf, sizeof(gw_in.buf)); + memzero_explicit(gw_out.buf, sizeof(gw_out.buf)); return ret; } diff --git a/arch/s390/crypto/hmac_s390.c b/arch/s390/crypto/hmac_s390.c index f8cd09f341d4b..445fa7bbd9586 100644 --- a/arch/s390/crypto/hmac_s390.c +++ b/arch/s390/crypto/hmac_s390.c @@ -150,7 +150,10 @@ static int hash_data(const u8 *in, unsigned int inlen, #undef PARAM_INIT - cpacf_klmd(func, ¶m, in, inlen); + if (final) + cpacf_klmd(func, ¶m, in, inlen); + else + cpacf_kimd(func, ¶m, in, inlen); memcpy(digest, ¶m, digestsize); diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 8cfe6166c193c..240c7b87a048b 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -220,6 +220,10 @@ static inline int convert_key(const u8 *key, unsigned int keylen, xflags); } + /* But finally map -EBUSY to -EIO to indicate an IO failure */ + if (rc == -EBUSY) + rc = -EIO; + out: pr_debug("rc=%d\n", rc); return rc; @@ -568,7 +572,7 @@ static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg ecb_paes_alg = { @@ -576,6 +580,7 @@ static struct skcipher_engine_alg ecb_paes_alg = { .base.cra_name = "ecb(paes)", .base.cra_driver_name = "ecb-paes-s390", .base.cra_priority = 401, /* combo: aes + ecb + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -834,7 +839,7 @@ static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg cbc_paes_alg = { @@ -842,6 +847,7 @@ static struct skcipher_engine_alg cbc_paes_alg = { .base.cra_name = "cbc(paes)", .base.cra_driver_name = "cbc-paes-s390", .base.cra_priority = 402, /* cbc-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -1034,6 +1040,7 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, } out: + memzero_explicit(buf, sizeof(buf)); pr_debug("rc=%d\n", rc); return rc; } @@ -1142,7 +1149,7 @@ static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg ctr_paes_alg = { @@ -1150,6 +1157,7 @@ static struct skcipher_engine_alg ctr_paes_alg = { .base.cra_name = "ctr(paes)", .base.cra_driver_name = "ctr-paes-s390", .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -1364,7 +1372,7 @@ static inline int __xts_2keys_prep_param(struct s390_pxts_ctx *ctx, memcpy(param->init, pcc_param.xts, 16); } - memzero_explicit(pcc_param.key, sizeof(pcc_param.key)); + memzero_explicit(&pcc_param, sizeof(pcc_param)); return rc; } @@ -1585,7 +1593,7 @@ static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg xts_paes_alg = { @@ -1593,6 +1601,7 @@ static struct skcipher_engine_alg xts_paes_alg = { .base.cra_name = "xts(paes)", .base.cra_driver_name = "xts-paes-s390", .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_pxts_ctx), .base.cra_module = THIS_MODULE, diff --git a/arch/s390/crypto/phmac_s390.c b/arch/s390/crypto/phmac_s390.c index 03ca33ffe6ccf..44817c3d25cf2 100644 --- a/arch/s390/crypto/phmac_s390.c +++ b/arch/s390/crypto/phmac_s390.c @@ -62,8 +62,10 @@ static inline int hwh_prepare(struct ahash_request *req, */ static inline int hwh_advance(struct hash_walk_helper *hwh, int n) { - if (n < 0) + if (n < 0) { + hwh->walkbytes = n; return crypto_hash_walk_done(&hwh->walk, n); + } hwh->walkbytes -= n; hwh->walkaddr += n; @@ -339,6 +341,10 @@ static inline int convert_key(const u8 *key, unsigned int keylen, xflags); } + /* But finally map -EBUSY to -EIO to indicate an IO failure */ + if (rc == -EBUSY) + rc = -EIO; + out: pr_debug("rc=%d\n", rc); return rc; @@ -606,6 +612,7 @@ static int phmac_update(struct ahash_request *req) struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; + bool cleanup = true; int rc; /* prep the walk in the request context */ @@ -629,12 +636,15 @@ static int phmac_update(struct ahash_request *req) req_ctx->async_op = OP_UPDATE; atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) { - hwh_advance(hwh, rc); + if (cleanup) { + if (hwh->walkbytes > 0) + hwh_advance(hwh, rc); memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); } @@ -649,6 +659,7 @@ static int phmac_final(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; + bool cleanup = true; int rc = 0; /* Try synchronous operation if no active engine usage */ @@ -667,12 +678,14 @@ static int phmac_final(struct ahash_request *req) req_ctx->async_op = OP_FINAL; atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); pr_debug("rc=%d\n", rc); return rc; @@ -685,6 +698,7 @@ static int phmac_finup(struct ahash_request *req) struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; + bool cleanup = true; int rc; /* prep the walk in the request context */ @@ -716,15 +730,17 @@ static int phmac_finup(struct ahash_request *req) /* req->async_op has been set to either OP_FINUP or OP_FINAL */ atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && hwh->walkbytes > 0) hwh_advance(hwh, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); pr_debug("rc=%d\n", rc); return rc; @@ -933,7 +949,7 @@ static int phmac_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&tfm_ctx->via_engine_ctr); crypto_finalize_hash_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } #define S390_ASYNC_PHMAC_ALG(x) \ diff --git a/arch/s390/include/asm/debug.h b/arch/s390/include/asm/debug.h index 39d484c597748..ad438d6352c8e 100644 --- a/arch/s390/include/asm/debug.h +++ b/arch/s390/include/asm/debug.h @@ -460,7 +460,11 @@ static int VNAME(var, active_entries)[EARLY_AREAS] __initdata #define __REGISTER_STATIC_DEBUG_INFO(var, name, pages, areas, view) \ static int __init VNAME(var, reg)(void) \ { \ - debug_register_static(&var, (pages), (areas)); \ + int rc; \ + \ + rc = debug_register_static(&var, (pages), (areas)); \ + if (rc) \ + return rc; \ debug_register_view(&var, (view)); \ return 0; \ } \ @@ -493,7 +497,7 @@ static debug_info_t __refdata var = \ static debug_info_t __used __section(".s390dbf_info") *VNAME(var, info) = &var; \ __REGISTER_STATIC_DEBUG_INFO(var, name, pages, nr_areas, view) -void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas); +int debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas); #endif /* MODULE */ diff --git a/arch/s390/include/asm/irqflags.h b/arch/s390/include/asm/irqflags.h index bcab456dfb803..6eb6de6a99f2d 100644 --- a/arch/s390/include/asm/irqflags.h +++ b/arch/s390/include/asm/irqflags.h @@ -37,18 +37,24 @@ static __always_inline void __arch_local_irq_ssm(unsigned long flags) asm volatile("ssm %0" : : "Q" (flags) : "memory"); } -#ifdef CONFIG_KMSAN -#define arch_local_irq_attributes noinline notrace __no_sanitize_memory __maybe_unused +#if defined(CONFIG_KMSAN) && !defined(__DECOMPRESSOR) +unsigned long arch_local_save_flags(void); +unsigned long arch_local_irq_save(void); +void arch_local_irq_enable_external(void); +void arch_local_irq_enable(void); #else -#define arch_local_irq_attributes __always_inline +#define arch_local_save_flags __arch_local_save_flags +#define arch_local_irq_save __arch_local_irq_save +#define arch_local_irq_enable_external __arch_local_irq_enable_external +#define arch_local_irq_enable __arch_local_irq_enable #endif -static arch_local_irq_attributes unsigned long arch_local_save_flags(void) +static __always_inline unsigned long __arch_local_save_flags(void) { return __arch_local_irq_stnsm(0xff); } -static arch_local_irq_attributes unsigned long arch_local_irq_save(void) +static __always_inline unsigned long __arch_local_irq_save(void) { return __arch_local_irq_stnsm(0xfc); } @@ -58,12 +64,12 @@ static __always_inline void arch_local_irq_disable(void) arch_local_irq_save(); } -static arch_local_irq_attributes void arch_local_irq_enable_external(void) +static __always_inline void __arch_local_irq_enable_external(void) { __arch_local_irq_stosm(0x01); } -static arch_local_irq_attributes void arch_local_irq_enable(void) +static __always_inline void __arch_local_irq_enable(void) { __arch_local_irq_stosm(0x03); } diff --git a/arch/s390/include/asm/nmi.h b/arch/s390/include/asm/nmi.h index 6454c15318544..7919b2b9ac9cc 100644 --- a/arch/s390/include/asm/nmi.h +++ b/arch/s390/include/asm/nmi.h @@ -22,6 +22,7 @@ #define MCCK_CODE_SYSTEM_DAMAGE BIT(63) #define MCCK_CODE_EXT_DAMAGE BIT(63 - 5) #define MCCK_CODE_CP BIT(63 - 9) +#define MCCK_CODE_CK BIT(63 - 11) #define MCCK_CODE_STG_ERROR BIT(63 - 16) #define MCCK_CODE_STG_KEY_ERROR BIT(63 - 18) #define MCCK_CODE_STG_DEGRAD BIT(63 - 19) @@ -33,6 +34,8 @@ #define MCCK_CODE_FC_VALID BIT(63 - 43) #define MCCK_CODE_CPU_TIMER_VALID BIT(63 - 46) +#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE | MCCK_CODE_CK) + #ifndef __ASSEMBLER__ union mci { diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index 534d0320e2aa0..a3456a36aaa72 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -76,7 +76,6 @@ static __always_inline void pai_kernel_exit(struct pt_regs *regs) } #define PAI_SAVE_AREA(x) ((x)->hw.event_base) -#define PAI_CPU_MASK(x) ((x)->hw.addr_filters) #define PAI_PMU_IDX(x) ((x)->hw.last_tag) #define PAI_SWLIST(x) (&(x)->hw.tp_list) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 1d955dd0defa3..feb58acf87d69 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -107,8 +107,8 @@ " .endif\n" \ ".endr\n" \ ".endm\n" \ - ALTERNATIVE("GEN_MVIY " __stringify(disp) " " __stringify(reg) "\n", \ - "GEN_MVIY " __stringify(dispalt) " " __stringify(reg) "\n", \ + ALTERNATIVE("GEN_MVIY " disp ", " reg "\n", \ + "GEN_MVIY " dispalt ", " reg "\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) \ ".purgem GEN_MVIY\n" diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 6c88476d79a31..14ef03cb2f72a 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -72,6 +72,7 @@ obj-$(CONFIG_STACKPROTECTOR) += stackprotector.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o obj-$(CONFIG_CERT_STORE) += cert_store.o +obj-$(CONFIG_KMSAN) += irqflags.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf.o perf_cpum_sf.o diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index dbf430f479bdf..71c966f4c5d03 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -432,7 +432,8 @@ static debug_info_t *debug_info_copy(debug_info_t *in, int mode) debug_info_free(rc); } while (1); - if (mode == NO_AREAS) + /* debug_register_static() failure leaves areas NULL, bounds intact */ + if (mode == NO_AREAS || !in->areas) goto out; for (i = 0; i < in->nr_areas; i++) { @@ -948,8 +949,12 @@ EXPORT_SYMBOL(debug_register); * * Note: This function is called automatically via an initcall generated by * DEFINE_STATIC_DEBUG_INFO. + * + * Return: + * - 0 on success + * - negative error code on failure */ -void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) +int debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) { unsigned long flags; debug_info_t *copy; @@ -957,7 +962,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) if (!initialized) { pr_err("Tried to register debug feature %s too early\n", id->name); - return; + return -EINVAL; } debug_get_param(id->name, &id->level, &pages_per_area); @@ -973,7 +978,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) id->active_entries = NULL; raw_spin_unlock_irqrestore(&id->lock, flags); - return; + return -ENOMEM; } /* Replace static trace area with dynamic copy. */ @@ -991,10 +996,12 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) mutex_lock(&debug_mutex); _debug_register(id); mutex_unlock(&debug_mutex); + + return 0; } -/* Remove debugfs entries and remove from internal list. */ -static void _debug_unregister(debug_info_t *id) +/* Remove debugfs entries. */ +static void _debug_unregister_debugfs(debug_info_t *id) { int i; @@ -1004,6 +1011,11 @@ static void _debug_unregister(debug_info_t *id) debugfs_remove(id->debugfs_entries[i]); } debugfs_remove(id->debugfs_root_entry); +} + +/* Remove from internal list. */ +static void _debug_unregister(debug_info_t *id) +{ if (id == debug_area_first) debug_area_first = id->next; if (id == debug_area_last) @@ -1029,6 +1041,7 @@ void debug_unregister(debug_info_t *id) mutex_lock(&debug_mutex); _debug_unregister(id); mutex_unlock(&debug_mutex); + _debug_unregister_debugfs(id); debug_info_put(id); } @@ -1068,9 +1081,6 @@ static void _debug_set_level(debug_info_t *id, int new_level) { unsigned long flags; - if (!id) - return; - if (new_level == DEBUG_OFF_LEVEL) { pr_info("%s: switched off\n", id->name); } else if ((new_level > DEBUG_MAX_LEVEL) || (new_level < 0)) { @@ -1095,6 +1105,9 @@ static void _debug_set_level(debug_info_t *id, int new_level) */ void debug_set_level(debug_info_t *id, int new_level) { + if (!id) + return; + /* Level specified via kernel parameter takes precedence */ debug_get_param(id->name, &new_level, NULL); diff --git a/arch/s390/kernel/diag/diag324.c b/arch/s390/kernel/diag/diag324.c index fe325c2a2d0dc..3eec0cc8fb9e0 100644 --- a/arch/s390/kernel/diag/diag324.c +++ b/arch/s390/kernel/diag/diag324.c @@ -182,8 +182,7 @@ long diag324_pibbuf(unsigned long arg) goto out; rc = copy_to_user((void __user *)address, data->pib, data->pib->len); rc |= put_user(data->sequence, &udata->sequence); - if (rc) - rc = -EFAULT; + rc = rc ? -EFAULT : data->rc; out: mutex_unlock(&pibmutex); return rc; diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 3c346b02ceb95..68fdd5616dfe1 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -1157,6 +1157,8 @@ static struct attribute_group reipl_nss_attr_group = { void set_os_info_reipl_block(void) { + if (!reipl_block_actual) + return; os_info_entry_add_data(OS_INFO_REIPL_BLOCK, reipl_block_actual, reipl_block_actual->hdr.len); } @@ -1927,7 +1929,8 @@ static struct shutdown_action __refdata dump_action = { static void dump_reipl_run(struct shutdown_trigger *trigger) { struct lowcore *abs_lc; - unsigned int csum; + unsigned long ipib = 0; + unsigned int csum = 0; /* * Set REIPL_CLEAR flag in os_info flags entry indicating @@ -1943,9 +1946,12 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) reipl_type == IPL_TYPE_UNKNOWN) os_info_flags |= OS_INFO_FLAG_REIPL_CLEAR; os_info_entry_add_data(OS_INFO_FLAGS_ENTRY, &os_info_flags, sizeof(os_info_flags)); - csum = (__force unsigned int)cksm(reipl_block_actual, reipl_block_actual->hdr.len, 0); + if (reipl_block_actual) { + ipib = __pa(reipl_block_actual); + csum = (__force unsigned int)cksm(reipl_block_actual, reipl_block_actual->hdr.len, 0); + } abs_lc = get_abs_lowcore(); - abs_lc->ipib = __pa(reipl_block_actual); + abs_lc->ipib = ipib; abs_lc->ipib_checksum = csum; put_abs_lowcore(abs_lc); dump_run(trigger); diff --git a/arch/s390/kernel/irqflags.c b/arch/s390/kernel/irqflags.c new file mode 100644 index 0000000000000..e192f59f8918c --- /dev/null +++ b/arch/s390/kernel/irqflags.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +noinstr unsigned long arch_local_save_flags(void) +{ + return __arch_local_save_flags(); +} +EXPORT_SYMBOL(arch_local_save_flags); + +noinstr unsigned long arch_local_irq_save(void) +{ + return __arch_local_irq_save(); +} +EXPORT_SYMBOL(arch_local_irq_save); + +noinstr void arch_local_irq_enable_external(void) +{ + __arch_local_irq_enable_external(); +} +EXPORT_SYMBOL(arch_local_irq_enable_external); + +noinstr void arch_local_irq_enable(void) +{ + __arch_local_irq_enable(); +} +EXPORT_SYMBOL(arch_local_irq_enable); diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index e17a59d4d5a41..17297a8b63d95 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -344,8 +344,7 @@ static void notrace s390_backup_mcck_info(struct pt_regs *regs) sie_page = container_of(sie_block, struct sie_page, sie_block); mcck_backup = &sie_page->mcck_info; - mcck_backup->mcic = get_lowcore()->mcck_interruption_code & - ~(MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE); + mcck_backup->mcic = get_lowcore()->mcck_interruption_code & ~MCCK_CODE_NO_GUEST; mcck_backup->ext_damage_code = get_lowcore()->external_damage_code; mcck_backup->failing_storage_address = get_lowcore()->failing_storage_address; } @@ -357,8 +356,6 @@ NOKPROBE_SYMBOL(s390_backup_mcck_info); #define ED_STP_ISLAND 6 /* External damage STP island check */ #define ED_STP_SYNC 7 /* External damage STP sync check */ -#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE) - /* * machine check handler. */ diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 2076ac22e2c49..2a8d4cc1d6a2c 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -110,6 +110,7 @@ struct cpu_cf_ptr { static struct cpu_cf_root { /* Anchor to per CPU data */ refcount_t refcnt; /* Overall active events */ + unsigned int tskctx; /* Users tracking all CPUs (cpu == -1) */ struct cpu_cf_ptr __percpu *cfptr; } cpu_cf_root; @@ -118,13 +119,15 @@ static struct cpu_cf_root { /* Anchor to per CPU data */ * user space in task context with perf_event_open() and close() * system calls. * - * This mutex serializes functions cpum_cf_alloc_cpu() called at event - * initialization via cpumf_pmu_event_init() and function cpum_cf_free_cpu() - * called at event removal via call back function hw_perf_event_destroy() - * when the event is deleted. They are serialized to enforce correct - * bookkeeping of pointer and reference counts anchored by - * struct cpu_cf_root and the access to cpu_cf_root::refcnt and the - * per CPU pointers stored in cpu_cf_root::cfptr. + * This mutex serializes the allocation and removal of the per CPU counter + * data via cpum_cf_alloc_cpu() and cpum_cf_free_cpu(). They are called with + * this mutex held at event initialization via cpumf_pmu_event_init(), at + * event removal via call back function hw_perf_event_destroy() when the + * event is deleted, and from the CPU hotplug prepare/dead callbacks. The + * mutex enforces correct bookkeeping of pointer and reference counts + * anchored by struct cpu_cf_root and protects the access to + * cpu_cf_root::refcnt, cpu_cf_root::tskctx and the per CPU pointers + * stored in cpu_cf_root::cfptr. */ static DEFINE_MUTEX(pmc_reserve_mutex); @@ -167,12 +170,14 @@ static void cpum_cf_reset_cpu(void *flags) } /* Free per CPU data when the last event is removed. */ -static void cpum_cf_free_root(void) +static void cpum_cf_free_root(unsigned int num) { - if (!refcount_dec_and_test(&cpu_cf_root.refcnt)) + struct cpu_cf_ptr __percpu *p = cpu_cf_root.cfptr; + + if (!refcount_sub_and_test(num, &cpu_cf_root.refcnt)) return; - free_percpu(cpu_cf_root.cfptr); cpu_cf_root.cfptr = NULL; + free_percpu(p); irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); on_each_cpu(cpum_cf_reset_cpu, NULL, 1); debug_sprintf_event(cf_dbg, 4, "%s root.refcnt %u cfptr %d\n", @@ -186,17 +191,17 @@ static void cpum_cf_free_root(void) * CPUs possible, which might be larger than the number of CPUs currently * online. */ -static int cpum_cf_alloc_root(void) +static int cpum_cf_alloc_root(unsigned int num) { int rc = 0; - if (refcount_inc_not_zero(&cpu_cf_root.refcnt)) + if (refcount_add_not_zero(num, &cpu_cf_root.refcnt)) return rc; /* The memory is already zeroed. */ cpu_cf_root.cfptr = alloc_percpu(struct cpu_cf_ptr); if (cpu_cf_root.cfptr) { - refcount_set(&cpu_cf_root.refcnt, 1); + refcount_set(&cpu_cf_root.refcnt, num); on_each_cpu(cpum_cf_reset_cpu, NULL, 1); irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); } else { @@ -206,20 +211,23 @@ static int cpum_cf_alloc_root(void) return rc; } -/* Free CPU counter data structure for a PMU */ -static void cpum_cf_free_cpu(int cpu) +/* + * Remove num references to the CPU counter data structure of a PMU. + * Called with pmc_reserve_mutex held. + */ +static void cpum_cf_free_cpu(int cpu, unsigned int num) { struct cpu_cf_events *cpuhw; struct cpu_cf_ptr *p; - mutex_lock(&pmc_reserve_mutex); + lockdep_assert_held(&pmc_reserve_mutex); /* * When invoked via CPU hotplug handler, there might be no events * installed or that particular CPU might not have an * event installed. This anchor pointer can be NULL! */ if (!cpu_cf_root.cfptr) - goto out; + return; p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); cpuhw = p->cpucf; /* @@ -227,28 +235,29 @@ static void cpum_cf_free_cpu(int cpu) * installed on that CPU, but on different CPUs. */ if (!cpuhw) - goto out; + return; - if (refcount_dec_and_test(&cpuhw->refcnt)) { - kfree(cpuhw); + if (refcount_sub_and_test(num, &cpuhw->refcnt)) { p->cpucf = NULL; + kfree(cpuhw); } - cpum_cf_free_root(); -out: - mutex_unlock(&pmc_reserve_mutex); + cpum_cf_free_root(num); } -/* Allocate CPU counter data structure for a PMU. Called under mutex lock. */ -static int cpum_cf_alloc_cpu(int cpu) +/* + * Add num references to the CPU counter data structure of a PMU and + * allocate it when necessary. Called with pmc_reserve_mutex held. + */ +static int cpum_cf_alloc_cpu(int cpu, unsigned int num) { struct cpu_cf_events *cpuhw; struct cpu_cf_ptr *p; int rc; - mutex_lock(&pmc_reserve_mutex); - rc = cpum_cf_alloc_root(); + lockdep_assert_held(&pmc_reserve_mutex); + rc = cpum_cf_alloc_root(num); if (rc) - goto unlock; + return rc; p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); cpuhw = p->cpucf; @@ -256,12 +265,12 @@ static int cpum_cf_alloc_cpu(int cpu) cpuhw = kzalloc_obj(*cpuhw); if (cpuhw) { p->cpucf = cpuhw; - refcount_set(&cpuhw->refcnt, 1); + refcount_set(&cpuhw->refcnt, num); } else { rc = -ENOMEM; } } else { - refcount_inc(&cpuhw->refcnt); + refcount_add(num, &cpuhw->refcnt); } if (rc) { /* @@ -269,10 +278,8 @@ static int cpum_cf_alloc_cpu(int cpu) * cpu_cf_event in not created, its destroy() function is not * invoked. Adjust the reference counter for the anchor. */ - cpum_cf_free_root(); + cpum_cf_free_root(num); } -unlock: - mutex_unlock(&pmc_reserve_mutex); return rc; } @@ -284,39 +291,70 @@ static int cpum_cf_alloc_cpu(int cpu) * perf_event_open() with task context and /dev/hwctr interface. * If cpu is non-zero install event on this CPU only. This setup handles * perf_event_open() with CPU context. + * Users with cpu == -1 are counted in cpu_cf_root::tskctx. The CPU hotplug + * prepare and dead callbacks use this count to install and remove the per + * CPU counter data on a new or dying CPU. */ -static int cpum_cf_alloc(int cpu) +static int cpum_cf_alloc_cpuslocked(int cpu) { cpumask_var_t mask; int rc; + lockdep_assert_cpus_held(); if (cpu == -1) { if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) return -ENOMEM; + mutex_lock(&pmc_reserve_mutex); for_each_online_cpu(cpu) { - rc = cpum_cf_alloc_cpu(cpu); + rc = cpum_cf_alloc_cpu(cpu, 1); if (rc) { for_each_cpu(cpu, mask) - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); break; } cpumask_set_cpu(cpu, mask); } + if (!rc) + cpu_cf_root.tskctx++; + mutex_unlock(&pmc_reserve_mutex); free_cpumask_var(mask); } else { - rc = cpum_cf_alloc_cpu(cpu); + mutex_lock(&pmc_reserve_mutex); + rc = cpum_cf_alloc_cpu(cpu, 1); + mutex_unlock(&pmc_reserve_mutex); } return rc; } -static void cpum_cf_free(int cpu) +static int cpum_cf_alloc(int cpu) +{ + int rc; + + cpus_read_lock(); + rc = cpum_cf_alloc_cpuslocked(cpu); + cpus_read_unlock(); + return rc; +} + +static void cpum_cf_free_cpuslocked(int cpu) { + lockdep_assert_cpus_held(); + mutex_lock(&pmc_reserve_mutex); if (cpu == -1) { + cpu_cf_root.tskctx--; for_each_online_cpu(cpu) - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); } else { - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); } + mutex_unlock(&pmc_reserve_mutex); +} + +static void cpum_cf_free(int cpu) +{ + cpus_read_lock(); + cpum_cf_free_cpuslocked(cpu); + cpus_read_unlock(); } #define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ @@ -1090,53 +1128,67 @@ static refcount_t cfset_opencnt = REFCOUNT_INIT(0); /* Access count */ static DEFINE_MUTEX(cfset_ctrset_mutex); /* - * CPU hotplug handles only /dev/hwctr device. - * For perf_event_open() the CPU hotplug handling is done on kernel common - * code: + * CPU hotplug handling: + * + * cpum_cf_prepare_cpu() and cpum_cf_dead_cpu() run while the new or dying + * CPU is offline. They create and remove the per CPU counter data for all + * users tracking every CPU (cpu == -1), that is perf_event_open() events + * with task context and /dev/hwctr device sessions. Each such user holds + * one reference to the per CPU counter data of each CPU. Therefore install + * and remove one reference per user, tracked in cpu_cf_root::tskctx. This + * guarantees the per CPU counter data exists before the new CPU executes + * its first task and is removed only after the dying CPU is gone. + * + * cpum_cf_online_cpu() and cpum_cf_offline_cpu() run while the new or + * dying CPU is online. They handle only the counter set state of open + * /dev/hwctr device sessions on that CPU. For perf_event_open() events + * nothing is done: * - CPU add: Nothing is done since a file descriptor can not be created * and returned to the user. * - CPU delete: Handled by common code via pmu_disable(), pmu_stop() and - * pmu_delete(). The event itself is removed when the file descriptor is - * closed. + * pmu_delete(). During task exit processing of grouped perf events + * triggered by CPU hotplug processing, pmu_disable() is called as part + * of perf context removal process. The event itself is removed when the + * event file descriptor is closed. */ +static int cpum_cf_prepare_cpu(unsigned int cpu) +{ + int rc = 0; + + mutex_lock(&pmc_reserve_mutex); + if (cpu_cf_root.tskctx) + rc = cpum_cf_alloc_cpu(cpu, cpu_cf_root.tskctx); + mutex_unlock(&pmc_reserve_mutex); + return rc; +} + +static int cpum_cf_dead_cpu(unsigned int cpu) +{ + mutex_lock(&pmc_reserve_mutex); + if (cpu_cf_root.tskctx) + cpum_cf_free_cpu(cpu, cpu_cf_root.tskctx); + mutex_unlock(&pmc_reserve_mutex); + return 0; +} + static int cfset_online_cpu(unsigned int cpu); static int cpum_cf_online_cpu(unsigned int cpu) { - int rc = 0; - - /* - * Ignore notification for perf_event_open(). - * Handle only /dev/hwctr device sessions. - */ mutex_lock(&cfset_ctrset_mutex); - if (refcount_read(&cfset_opencnt)) { - rc = cpum_cf_alloc_cpu(cpu); - if (!rc) - cfset_online_cpu(cpu); - } + if (refcount_read(&cfset_opencnt)) + cfset_online_cpu(cpu); mutex_unlock(&cfset_ctrset_mutex); - return rc; + return 0; } static int cfset_offline_cpu(unsigned int cpu); static int cpum_cf_offline_cpu(unsigned int cpu) { - /* - * During task exit processing of grouped perf events triggered by CPU - * hotplug processing, pmu_disable() is called as part of perf context - * removal process. Therefore do not trigger event removal now for - * perf_event_open() created events. Perf common code triggers event - * destruction when the event file descriptor is closed. - * - * Handle only /dev/hwctr device sessions. - */ mutex_lock(&cfset_ctrset_mutex); - if (refcount_read(&cfset_opencnt)) { + if (refcount_read(&cfset_opencnt)) cfset_offline_cpu(cpu); - cpum_cf_free_cpu(cpu); - } mutex_unlock(&cfset_ctrset_mutex); return 0; } @@ -1183,7 +1235,7 @@ static void cpumf_measurement_alert(struct ext_code ext_code, static int cfset_init(void); static int __init cpumf_pmu_init(void) { - int rc; + int state, rc; /* Extract counter measurement facility information */ if (!cpum_cf_avail() || qctri(&cpumf_ctr_info)) @@ -1225,11 +1277,24 @@ static int __init cpumf_pmu_init(void) cfset_init(); } + rc = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, + "perf/s390/cf:prepare", + cpum_cf_prepare_cpu, cpum_cf_dead_cpu); + if (rc < 0) + goto out3; + state = rc; + rc = cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE, "perf/s390/cf:online", cpum_cf_online_cpu, cpum_cf_offline_cpu); - return rc; + if (rc < 0) + goto out4; + return 0; +out4: + cpuhp_remove_state(state); +out3: + perf_pmu_unregister(&cpumf_pmu); out2: debug_unregister_view(cf_dbg, &debug_sprintf_view); debug_unregister(cf_dbg); @@ -1385,6 +1450,7 @@ static void cfset_all_stop(struct cfset_request *req) */ static int cfset_release(struct inode *inode, struct file *file) { + cpus_read_lock(); mutex_lock(&cfset_ctrset_mutex); /* Open followed by close/exit has no private_data */ if (file->private_data) { @@ -1395,9 +1461,10 @@ static int cfset_release(struct inode *inode, struct file *file) } if (refcount_dec_and_test(&cfset_opencnt)) { /* Last close */ on_each_cpu(cfset_release_cpu, NULL, 1); - cpum_cf_free(-1); + cpum_cf_free_cpuslocked(-1); } mutex_unlock(&cfset_ctrset_mutex); + cpus_read_unlock(); return 0; } @@ -1416,15 +1483,17 @@ static int cfset_open(struct inode *inode, struct file *file) return -EPERM; file->private_data = NULL; + cpus_read_lock(); mutex_lock(&cfset_ctrset_mutex); if (!refcount_inc_not_zero(&cfset_opencnt)) { /* First open */ - rc = cpum_cf_alloc(-1); + rc = cpum_cf_alloc_cpuslocked(-1); if (!rc) { cfset_session_init(); refcount_set(&cfset_opencnt, 1); } } mutex_unlock(&cfset_ctrset_mutex); + cpus_read_unlock(); /* nonseekable_open() never fails */ return rc ?: nonseekable_open(inode, file); diff --git a/arch/s390/kernel/perf_pai.c b/arch/s390/kernel/perf_pai.c index cdb8006220ca0..25228b46cdf4e 100644 --- a/arch/s390/kernel/perf_pai.c +++ b/arch/s390/kernel/perf_pai.c @@ -67,6 +67,7 @@ struct pai_mapptr { static struct pai_root { /* Anchor to per CPU data */ refcount_t refcnt; /* Overall active events */ + atomic_t tskctx; /* Overall per-task events */ struct pai_mapptr __percpu *mapptr; } pai_root[PAI_PMU_MAX]; @@ -93,14 +94,15 @@ struct pai_pmu { /* Define PAI PMU characteristics */ static struct pai_pmu pai_pmu[]; /* Forward declaration */ /* Free per CPU data when the last event is removed. */ -static void pai_root_free(int idx) +static void pai_root_free(int idx, int tasks) { - if (refcount_dec_and_test(&pai_root[idx].refcnt)) { + if (refcount_sub_and_test(tasks, &pai_root[idx].refcnt)) { free_percpu(pai_root[idx].mapptr); pai_root[idx].mapptr = NULL; } - debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d\n", __func__, - idx, refcount_read(&pai_root[idx].refcnt)); + debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d tskctx %d\n", + __func__, idx, refcount_read(&pai_root[idx].refcnt), + atomic_read(&pai_root[idx].tskctx)); } /* @@ -137,40 +139,54 @@ static void pai_free(struct pai_mapptr *mp) mp->mapptr = NULL; } -/* Adjust usage counters and remove allocated memory when all users are - * gone. - */ -static void pai_event_destroy_cpu(struct perf_event *event, int cpu) +/* Called under mutex_lock */ +static void pai_event_destroy_cpu(int idx, int cpu, bool hotplug) { - int idx = PAI_PMU_IDX(event); - struct pai_mapptr *mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); - struct pai_map *cpump = mp->mapptr; + struct pai_mapptr *mp; + struct pai_map *cpump; + int tasks = 1; - mutex_lock(&pai_reserve_mutex); - debug_sprintf_event(paidbg, 5, "%s event %#llx idx %d cpu %d users %d " - "refcnt %u\n", __func__, event->attr.config, idx, - event->cpu, cpump->active_events, - refcount_read(&cpump->refcnt)); - if (refcount_dec_and_test(&cpump->refcnt)) + /* Check reference count and return when all gone. + * 1. An event is installed on online CPU X. + * 2. CPU x is offlined and the per-CPU data is removed. + * 3. Event is destroyed via close system call. + */ + if (!refcount_read(&pai_root[idx].refcnt)) + return; /* No events at all */ + mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); + if (!mp || !mp->mapptr) /* No events on that CPU */ + return; + + /* When hotplug is true, invocation is from CPU hotplug callback. + * Delete per-CPU resource and adjust refcnt when per-task events + * are currently active. This can be more than one. + * In this case adjust counters. + */ + if (hotplug) + tasks = atomic_read(&pai_root[idx].tskctx); + + cpump = mp->mapptr; + if (refcount_sub_and_test(tasks, &cpump->refcnt)) pai_free(mp); - pai_root_free(idx); - mutex_unlock(&pai_reserve_mutex); + pai_root_free(idx, tasks); } static void pai_event_destroy(struct perf_event *event) { - int cpu; + int cpu = 0, idx = PAI_PMU_IDX(event); free_page(PAI_SAVE_AREA(event)); + cpus_read_lock(); + mutex_lock(&pai_reserve_mutex); if (event->cpu == -1) { - struct cpumask *mask = PAI_CPU_MASK(event); - - for_each_cpu(cpu, mask) - pai_event_destroy_cpu(event, cpu); - kfree(mask); + atomic_dec(&pai_root[idx].tskctx); + for_each_online_cpu(cpu) + pai_event_destroy_cpu(idx, cpu, false); } else { - pai_event_destroy_cpu(event, event->cpu); + pai_event_destroy_cpu(idx, event->cpu, false); } + mutex_unlock(&pai_reserve_mutex); + cpus_read_unlock(); } static void paicrypt_event_destroy(struct perf_event *event) @@ -234,25 +250,30 @@ static u64 paicrypt_getall(struct perf_event *event) return sum; } -/* Check concurrent access of counting and sampling for crypto events. - * This function is called in process context and it is save to block. - * When the event initialization functions fails, no other call back will - * be invoked. - * - * Allocate the memory for the event. - */ -static int pai_alloc_cpu(struct perf_event *event, int cpu) +/* Called under mutex_lock */ +static int pai_alloc_cpu(int idx, int cpu, bool hotplug) { - int rc, idx = PAI_PMU_IDX(event); struct pai_map *cpump = NULL; bool need_paiext_cb = false; struct pai_mapptr *mp; + int tasks = 1, rc = 0; + + /* When hotplug is true, invocation is from CPU hotplug callback. + * Allocate per-CPU resource when per-task events are currently active. + * This can be more than one. In this case adjust all reference + * counters. Otherwise return, this ensures memory is only allocated + * when needed. + */ + if (hotplug) { + tasks = atomic_read(&pai_root[idx].tskctx); + if (!tasks) + goto out; + } - mutex_lock(&pai_reserve_mutex); /* Allocate root node */ rc = pai_root_alloc(idx); if (rc) - goto unlock; + goto out; /* Allocate node for this event */ mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); @@ -296,28 +317,45 @@ static int pai_alloc_cpu(struct perf_event *event, int cpu) goto undo; } INIT_LIST_HEAD(&cpump->syswide_list); - refcount_set(&cpump->refcnt, 1); + refcount_set(&cpump->refcnt, tasks); rc = 0; } else { - refcount_inc(&cpump->refcnt); + refcount_add(tasks, &cpump->refcnt); } + /* If tasks is greater than 1, we are called from CPU hotplug path + * and need to adjust the pai_root[idx].refcnt by the number of + * per-process events. Function pai_root_alloc(idx) already + * incremented by one. Adjust for the rest. + */ + if (tasks > 1) + refcount_add(tasks - 1, &pai_root[idx].refcnt); undo: if (rc) { /* Error in allocation of event, decrement anchor. Since * the event in not created, its destroy() function is never * invoked. Adjust the reference counter for the anchor. + * The failure happened in the case of variable + * cpump == NULL branch above. The pai_root[XXX].refcnt has + * been incremented by one. Then the per-CPU allocation + * failed, so decrement it by one, regardless of tasks. */ - pai_root_free(idx); + pai_root_free(idx, 1); } -unlock: - mutex_unlock(&pai_reserve_mutex); +out: /* If rc is non-zero, no increment of counter/sampler was done. */ return rc; } +/* Check concurrent access of counting and sampling for PAI events. + * This function is called in process context and it is safe to block. + * When the event initialization functions fails, no other call back will + * be invoked. + * Called under mutex_lock. + */ static int pai_alloc(struct perf_event *event) { + int idx = PAI_PMU_IDX(event); struct cpumask *maskptr; int cpu, rc = -ENOMEM; @@ -326,24 +364,20 @@ static int pai_alloc(struct perf_event *event) goto out; for_each_online_cpu(cpu) { - rc = pai_alloc_cpu(event, cpu); + rc = pai_alloc_cpu(idx, cpu, false); if (rc) { for_each_cpu(cpu, maskptr) - pai_event_destroy_cpu(event, cpu); - kfree(maskptr); - goto out; + pai_event_destroy_cpu(idx, cpu, false); + goto undo; } cpumask_set_cpu(cpu, maskptr); } - /* - * On error all cpumask are freed and all events have been destroyed. - * Save of which CPUs data structures have been allocated for. - * Release them in pai_event_destroy call back function - * for this event. - */ - PAI_CPU_MASK(event) = maskptr; rc = 0; + /* Trace per-task events for CPU hotplug. */ + atomic_inc(&pai_root[idx].tskctx); +undo: + kfree(maskptr); out: return rc; } @@ -391,10 +425,14 @@ static int pai_event_init(struct perf_event *event, int idx) } } + cpus_read_lock(); + mutex_lock(&pai_reserve_mutex); if (event->cpu >= 0) - rc = pai_alloc_cpu(event, event->cpu); + rc = pai_alloc_cpu(idx, event->cpu, false); else rc = pai_alloc(event); + mutex_unlock(&pai_reserve_mutex); + cpus_read_unlock(); if (rc) { free_page(PAI_SAVE_AREA(event)); goto out; @@ -1218,23 +1256,61 @@ static int __init paipmu_setup(void) return install_ok; } +static int pai_online_cpu(unsigned int cpu) +{ + int rc; + + mutex_lock(&pai_reserve_mutex); + rc = pai_alloc_cpu(PAI_PMU_CRYPTO, cpu, true); + if (rc) + goto out; + rc = pai_alloc_cpu(PAI_PMU_EXT, cpu, true); + if (rc) + pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true); +out: + mutex_unlock(&pai_reserve_mutex); + return rc; +} + +static int pai_offline_cpu(unsigned int cpu) +{ + mutex_lock(&pai_reserve_mutex); + pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true); + pai_event_destroy_cpu(PAI_PMU_EXT, cpu, true); + mutex_unlock(&pai_reserve_mutex); + return 0; +} + static int __init pai_init(void) { + int state, rc; + /* Setup s390dbf facility */ - paidbg = debug_register("pai", 32, 256, 128); + paidbg = debug_register("pai", 1, 1, 128); if (!paidbg) { pr_err("Registration of s390dbf pai failed\n"); return -ENOMEM; } debug_register_view(paidbg, &debug_sprintf_view); - if (!paipmu_setup()) { - /* No PMU registration, no need for debug buffer */ - debug_unregister_view(paidbg, &debug_sprintf_view); - debug_unregister(paidbg); - return -ENODEV; - } + /* CPUHP_BP_PREPARE_DYN --> before CPU is brought online */ + state = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "perf/pai:prepare", + pai_online_cpu, pai_offline_cpu); + rc = state < 0 ? state : 0; + if (rc < 0) + goto out_debug; + + rc = -ENODEV; + if (!paipmu_setup()) + goto out_cpuhp; return 0; + +out_cpuhp: + cpuhp_remove_state(state); +out_debug: + debug_unregister_view(paidbg, &debug_sprintf_view); + debug_unregister(paidbg); + return rc; } device_initcall(pai_init); diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index a284f98d97168..e70acad09cd5f 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -760,7 +760,7 @@ static int find_secret_in_page(const u8 secret_id[UV_SECRET_ID_LEN], { u16 i; - for (i = 0; i < list->total_num_secrets; i++) { + for (i = 0; i < list->num_secr_stored; i++) { if (memcmp(secret_id, list->secrets[i].id, UV_SECRET_ID_LEN) == 0) { *secret = list->secrets[i].hdr; return 0; @@ -781,11 +781,14 @@ int uv_find_secret(const u8 secret_id[UV_SECRET_ID_LEN], struct uv_secret_list *list, struct uv_secret_list_item_hdr *secret) { - u16 start_idx = 0; + u16 start_idx; u16 list_rc; int ret; + list->next_secret_idx = 0; + do { + start_idx = list->next_secret_idx; uv_list_secrets(list, start_idx, &list_rc, NULL); if (list_rc != UVC_RC_EXECUTED && list_rc != UVC_RC_MORE_DATA) { if (list_rc == UVC_RC_INV_CMD) @@ -796,7 +799,6 @@ int uv_find_secret(const u8 secret_id[UV_SECRET_ID_LEN], ret = find_secret_in_page(secret_id, list, secret); if (ret == 0) return ret; - start_idx = list->next_secret_idx; } while (list_rc == UVC_RC_MORE_DATA && start_idx < list->next_secret_idx); return -ENOENT; diff --git a/arch/s390/kernel/vdso/Makefile b/arch/s390/kernel/vdso/Makefile index fece5d975eaf8..35c834b895ecb 100644 --- a/arch/s390/kernel/vdso/Makefile +++ b/arch/s390/kernel/vdso/Makefile @@ -30,7 +30,8 @@ KBUILD_CFLAGS_VDSO := $(filter-out -fno-asynchronous-unwind-tables,$(KBUILD_CFLA KBUILD_CFLAGS_VDSO += -fPIC -fno-common -fno-builtin -fasynchronous-unwind-tables KBUILD_CFLAGS_VDSO += -fno-stack-protector $(DISABLE_KSTACK_ERASE) ldflags-y := -shared -soname=linux-vdso.so.1 \ - --hash-style=both --build-id=sha1 -T + --hash-style=both --build-id=sha1 \ + $(call ld-option, --eh-frame-hdr) -T $(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_VDSO) $(targets:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_VDSO) diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index d804e1140c2e9..efcbf406f03e7 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -32,7 +32,7 @@ static atomic64_t virt_timer_elapsed; DEFINE_PER_CPU(u64, mt_cycles[8]); static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 }; static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 }; -static DEFINE_PER_CPU(u64, mt_scaling_jiffies); +static DEFINE_PER_CPU(unsigned long, mt_scaling_jiffies); static inline void set_vtimer(u64 expires) { @@ -81,7 +81,7 @@ static void update_mt_scaling(void) memcpy(cycles_old, cycles_new, sizeof(u64) * (smp_cpu_mtid + 1)); } - __this_cpu_write(mt_scaling_jiffies, jiffies_64); + __this_cpu_write(mt_scaling_jiffies, jiffies); } static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new) @@ -144,7 +144,7 @@ static int do_account_vtime(struct task_struct *tsk) lc->system_timer += timer; /* Update MT utilization calculation */ - if (smp_cpu_mtid && time_after64(jiffies_64, __this_cpu_read(mt_scaling_jiffies))) + if (smp_cpu_mtid && time_after(jiffies, __this_cpu_read(mt_scaling_jiffies))) update_mt_scaling(); /* Calculate cputime delta */ diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 3f2d6e8902d76..a21cb3975e992 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -620,17 +620,20 @@ int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey) union pte *ptep; int rc; +again: skey->skey = 0; rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; if (!ptep) { - union crste crste; + union crste crste = READ_ONCE(*crstep); - crste = READ_ONCE(*crstep); - if (!crste.h.fc || !crste.s.fc1.pr) + if (!crste_leaf(crste) && !crste.h.i) + goto again; + if (!crste.s.fc1.pr) return 0; + skey->skey = page_get_storage_key(large_crste_to_phys(crste, gfn)); return 0; } @@ -661,13 +664,20 @@ int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gf union pte *ptep; int rc; +again: rc = dat_entry_walk(mc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; if (!ptep) { - page_set_storage_key(large_crste_to_phys(*crstep, gfn), skey.skey, !nq); + union crste crste = READ_ONCE(*crstep); + + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste)) + goto again; + + page_set_storage_key(large_crste_to_phys(crste, gfn), skey.skey, !nq); return 0; } @@ -717,14 +727,24 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf union pte *ptep; int rc; +again: rc = dat_entry_walk(mmc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; - if (!ptep) - return page_cond_set_storage_key(large_crste_to_phys(*crstep, gfn), skey, oldkey, + if (!ptep) { + union crste crste = READ_ONCE(*crstep); + + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste)) + goto again; + if (!oldkey) + oldkey = &prev; + + return page_cond_set_storage_key(large_crste_to_phys(crste, gfn), skey, oldkey, nq, mr, mc); + } old = pgste_get_lock(ptep); pgste = old; @@ -763,7 +783,7 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey) int rc; skey->skey = 0; - +again: rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; @@ -771,9 +791,12 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey) if (!ptep) { union crste crste = READ_ONCE(*crstep); - if (!crste.h.fc || !crste.s.fc1.pr) + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste) && !crste.h.i) + goto again; + if (!crste.s.fc1.pr) return 0; - skey->skey = page_reset_referenced(large_crste_to_phys(*crstep, gfn)) << 1; + skey->skey = page_reset_referenced(large_crste_to_phys(crste, gfn)) << 1; return 0; } old = pgste_get_lock(ptep); diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 36102b2727fbf..7c1f614ec3146 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -1593,12 +1593,25 @@ static inline int ___gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg parent = READ_ONCE(sg->parent); if (!parent) return -EAGAIN; +retry: scoped_guard(spinlock, &parent->children_lock) { if (READ_ONCE(sg->parent) != parent) return -EAGAIN; sg->invalidated = false; rc = _gaccess_do_shadow(vcpu->arch.mc, sg, saddr, walk); } + if (rc == -ENOENT) { + struct kvm_memory_slot *slot; + struct guest_fault *entries; + + entries = get_entries(walk); + slot = kvm_vcpu_gfn_to_memslot(vcpu, entries[LEVEL_MEM].gfn); + if (!slot) + return PGM_ADDRESSING; + rc = gmap_link(vcpu->arch.mc, parent, entries + LEVEL_MEM, slot); + if (!rc) + goto retry; + } if (!rc) kvm_s390_release_faultin_array(vcpu->kvm, walk->raw_entries, false); return rc; diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index 8abb4f55b306b..f11d4ecaef7be 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -991,11 +991,13 @@ static long _destroy_pages_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct da static long _destroy_pages_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { phys_addr_t origin, cur, end; + union crste crste; - if (!crstep->h.fc || !crstep->s.fc1.pr) + crste = READ_ONCE(*crstep); + if (!crste.h.fc || !crste.s.fc1.pr) return 0; - origin = crste_origin_large(*crstep); + origin = crste_origin_large(crste); cur = ((max(gfn, walk->start) - gfn) << PAGE_SHIFT) + origin; end = ((min(next, walk->end) - gfn) << PAGE_SHIFT) + origin; for ( ; cur < end; cur += PAGE_SIZE) diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c index 69835e1d4f207..9a6149e310bbf 100644 --- a/arch/s390/kvm/guestdbg.c +++ b/arch/s390/kvm/guestdbg.c @@ -184,7 +184,7 @@ static int __import_wp_info(struct kvm_vcpu *vcpu, if (wp_info->len < 0 || wp_info->len > MAX_WP_SIZE) return -EINVAL; - wp_info->old_data = kmalloc(bp_data->len, GFP_KERNEL_ACCOUNT); + wp_info->old_data = kmalloc(wp_info->len, GFP_KERNEL_ACCOUNT); if (!wp_info->old_data) return -ENOMEM; /* try to backup the original value */ @@ -252,7 +252,7 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, ret = __import_wp_info(vcpu, &bp_data[i], &wp_info[nr_wp]); if (ret) - goto error; + goto error_wp; nr_wp++; break; case KVM_HW_BP: @@ -267,7 +267,12 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, vcpu->arch.guestdbg.hw_bp_info = bp_info; vcpu->arch.guestdbg.nr_hw_wp = nr_wp; vcpu->arch.guestdbg.hw_wp_info = wp_info; + kfree(bp_data); return 0; + +error_wp: + while (nr_wp--) + kfree(wp_info[nr_wp].old_data); error: kfree(bp_data); kfree(wp_info); diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 8f24bcd1a6d36..80f4802934df9 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -1550,23 +1550,21 @@ static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) } #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS) -static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) +static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq, bool *storestatus) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; struct kvm_s390_stop_info *stop = &li->irq.stop; - int rc = 0; vcpu->stat.inject_stop_signal++; trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0); if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS) return -EINVAL; - if (is_vcpu_stopped(vcpu)) { - if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS) - rc = kvm_s390_store_status_unloaded(vcpu, - KVM_S390_STORE_STATUS_NOADDR); - return rc; + if (!(irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)) + return 0; + *storestatus = true; + return -EWOULDBLOCK; } if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) @@ -2102,7 +2100,7 @@ void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu) spin_unlock(&li->lock); } -static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) +static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq, bool *storestatus) { int rc; @@ -2114,7 +2112,7 @@ static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) rc = __inject_set_prefix(vcpu, irq); break; case KVM_S390_SIGP_STOP: - rc = __inject_sigp_stop(vcpu, irq); + rc = __inject_sigp_stop(vcpu, irq, storestatus); break; case KVM_S390_RESTART: rc = __inject_sigp_restart(vcpu); @@ -2150,11 +2148,16 @@ static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; + bool storestatus = false; int rc; spin_lock(&li->lock); - rc = do_inject_vcpu(vcpu, irq); + rc = do_inject_vcpu(vcpu, irq, &storestatus); spin_unlock(&li->lock); + + if (rc == -EWOULDBLOCK && storestatus) + rc = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); + if (!rc) kvm_s390_vcpu_wakeup(vcpu); return rc; @@ -2976,61 +2979,58 @@ static int adapter_indicators_set(struct kvm *kvm, struct s390_io_adapter *adapter, struct kvm_s390_adapter_int *adapter_int) { - unsigned long bit; - int summary_set, idx; struct s390_map_info *ind_info, *summary_info; - void *map; struct page *ind_page, *summary_page; - unsigned long flags; + unsigned long bit; + int summary_set; + void *map; ind_page = NULL; - spin_lock_irqsave(&adapter->maps_lock, flags); - ind_info = get_map_info(adapter, adapter_int->ind_addr); + scoped_guard(spinlock_irqsave, &adapter->maps_lock) { + ind_info = get_map_info(adapter, adapter_int->ind_addr); + if (ind_info) { + map = page_address(ind_info->page); + bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); + set_bit(bit, map); + } + } if (!ind_info) { - spin_unlock_irqrestore(&adapter->maps_lock, flags); ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0); if (!ind_page) return -1; - idx = srcu_read_lock(&kvm->srcu); map = page_address(ind_page); bit = get_ind_bit(adapter_int->ind_addr, adapter_int->ind_offset, adapter->swap); set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT); set_page_dirty_lock(ind_page); - srcu_read_unlock(&kvm->srcu, idx); unpin_user_page(ind_page); - } else { - map = page_address(ind_info->page); - bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); - set_bit(bit, map); - spin_unlock_irqrestore(&adapter->maps_lock, flags); } + scoped_guard(srcu, &kvm->srcu) + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->ind_gaddr)); - spin_lock_irqsave(&adapter->maps_lock, flags); - summary_info = get_map_info(adapter, adapter_int->summary_addr); + scoped_guard(spinlock_irqsave, &adapter->maps_lock) { + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (summary_info) { + map = page_address(summary_info->page); + bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, + adapter->swap); + summary_set = test_and_set_bit(bit, map); + } + } if (!summary_info) { - spin_unlock_irqrestore(&adapter->maps_lock, flags); summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); if (WARN_ON_ONCE(!summary_page)) return -1; - idx = srcu_read_lock(&kvm->srcu); map = page_address(summary_page); bit = get_ind_bit(adapter_int->summary_addr, adapter_int->summary_offset, adapter->swap); summary_set = test_and_set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT); set_page_dirty_lock(summary_page); - srcu_read_unlock(&kvm->srcu, idx); unpin_user_page(summary_page); - } else { - map = page_address(summary_info->page); - bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, - adapter->swap); - summary_set = test_and_set_bit(bit, map); - spin_unlock_irqrestore(&adapter->maps_lock, flags); } + scoped_guard(srcu, &kvm->srcu) + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->summary_gaddr)); return summary_set ? 0 : 1; } @@ -3040,26 +3040,29 @@ static int adapter_indicators_set_fast(struct kvm *kvm, struct kvm_s390_adapter_int *adapter_int, int setbit) { + struct s390_map_info *ind_info, *summary_info; unsigned long bit; int summary_set; - struct s390_map_info *ind_info, *summary_info; void *map; - spin_lock(&adapter->maps_lock); + guard(srcu)(&kvm->srcu); + guard(spinlock)(&adapter->maps_lock); + ind_info = get_map_info(adapter, adapter_int->ind_addr); - if (!ind_info) { - spin_unlock(&adapter->maps_lock); + if (!ind_info) return -EWOULDBLOCK; - } + map = page_address(ind_info->page); bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); - if (setbit) + if (setbit) { set_bit(bit, map); + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->ind_gaddr)); + } + summary_info = get_map_info(adapter, adapter_int->summary_addr); - if (!summary_info) { - spin_unlock(&adapter->maps_lock); + if (!summary_info) return -EWOULDBLOCK; - } + map = page_address(summary_info->page); bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, adapter->swap); @@ -3069,7 +3072,8 @@ static int adapter_indicators_set_fast(struct kvm *kvm, summary_set = test_and_set_bit(bit, map); else summary_set = test_and_clear_bit(bit, map); - spin_unlock(&adapter->maps_lock); + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->summary_gaddr)); + return summary_set ? 0 : 1; } @@ -3108,9 +3112,7 @@ static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, struct mcck_volatile_info *mcck_info) { - struct kvm_s390_interrupt_info inti; - struct kvm_s390_irq irq; - struct kvm_s390_mchk_info *mchk; + struct kvm_s390_irq irq = {}; union mci mci; __u64 cr14 = 0; /* upper bits are not used */ int rc; @@ -3129,20 +3131,14 @@ void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, if (mci.w) cr14 |= CR14_WARNING_SUBMASK; - mchk = mci.ck ? &inti.mchk : &irq.u.mchk; - mchk->cr14 = cr14; - mchk->mcic = mcck_info->mcic; - mchk->ext_damage_code = mcck_info->ext_damage_code; - mchk->failing_storage_address = mcck_info->failing_storage_address; - if (mci.ck) { - /* Inject the floating machine check */ - inti.type = KVM_S390_MCHK; - rc = __inject_vm(vcpu->kvm, &inti); - } else { - /* Inject the machine check to specified vcpu */ - irq.type = KVM_S390_MCHK; - rc = kvm_s390_inject_vcpu(vcpu, &irq); - } + irq.u.mchk.cr14 = cr14; + irq.u.mchk.mcic = mcck_info->mcic; + irq.u.mchk.ext_damage_code = mcck_info->ext_damage_code; + irq.u.mchk.failing_storage_address = mcck_info->failing_storage_address; + + /* Inject the machine check to specified vcpu */ + irq.type = KVM_S390_MCHK; + rc = kvm_s390_inject_vcpu(vcpu, &irq); WARN_ON_ONCE(rc); } @@ -3197,7 +3193,8 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; - struct kvm_s390_irq *buf; + struct kvm_s390_irq *buf __free(kvfree) = NULL; + bool tmp, storestatus = false; int r = 0; int n; @@ -3205,31 +3202,33 @@ int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len if (!buf) return -ENOMEM; - if (copy_from_user((void *) buf, irqstate, len)) { - r = -EFAULT; - goto out_free; - } + if (copy_from_user((void *)buf, irqstate, len)) + return -EFAULT; - /* - * Don't allow setting the interrupt state - * when there are already interrupts pending - */ - spin_lock(&li->lock); - if (li->pending_irqs) { - r = -EBUSY; - goto out_unlock; - } + scoped_guard(spinlock, &li->lock) { + /* + * Don't allow setting the interrupt state + * when there are already interrupts pending + */ + if (li->pending_irqs) + return -EBUSY; - for (n = 0; n < len / sizeof(*buf); n++) { - r = do_inject_vcpu(vcpu, &buf[n]); - if (r) - break; + for (n = 0; n < len / sizeof(*buf); n++) { + tmp = false; + r = do_inject_vcpu(vcpu, &buf[n], &tmp); + if (r == -EWOULDBLOCK && tmp) { + storestatus = true; + r = 0; + } + if (r) + break; + } + } + if (storestatus) { + scoped_guard(srcu, &vcpu->kvm->srcu) + n = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); + return r ? r : n; } - -out_unlock: - spin_unlock(&li->lock); -out_free: - vfree(buf); return r; } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 518a69c55e85c..17aa38710170a 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -562,11 +562,12 @@ static void __kvm_s390_exit(void) static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op, unsigned long addr, union skey skey) { - union asce asce = kvm->arch.gmap->asce; gfn_t gfn = gpa_to_gfn(addr); + union asce asce; int r; guard(read_lock)(&kvm->mmu_lock); + asce = kvm->arch.gmap->asce; switch (op) { case KVM_S390_KEYOP_SSKE: @@ -3410,6 +3411,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); kvm_s390_clear_local_irqs(vcpu); kvm_clear_async_pf_completion_queue(vcpu); + kvm_s390_clear_bp_data(vcpu); if (!kvm_is_ucontrol(vcpu->kvm)) sca_del_vcpu(vcpu); kvm_s390_update_topology_change_report(vcpu->kvm, 1); @@ -4248,8 +4250,10 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, /* enforce guest PER */ kvm_s390_set_cpuflags(vcpu, CPUSTAT_P); - if (dbg->control & KVM_GUESTDBG_USE_HW_BP) - rc = kvm_s390_import_bp_data(vcpu, dbg); + if (dbg->control & KVM_GUESTDBG_USE_HW_BP) { + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_import_bp_data(vcpu, dbg); + } } else { kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P); vcpu->arch.guestdbg.last_bp = 0; @@ -4474,8 +4478,8 @@ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clo static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, unsigned long token) { - struct kvm_s390_interrupt inti; - struct kvm_s390_irq irq; + struct kvm_s390_interrupt inti = {}; + struct kvm_s390_irq irq = {}; struct kvm_s390_interrupt_info *inti_mem = NULL; int ret = 0; @@ -5069,7 +5073,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) pr_err_ratelimited("can't run stopped vcpu %d\n", vcpu->vcpu_id); rc = -EINVAL; - goto out; + goto out_sigset; } kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR); @@ -5099,9 +5103,11 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) store_regs(vcpu); kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR); + vcpu->stat.exit_userspace++; + +out_sigset: kvm_sigset_deactivate(vcpu); - vcpu->stat.exit_userspace++; out: vcpu_put(vcpu); return rc; @@ -5732,7 +5738,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = kvm_s390_handle_pv_vcpu_dump(vcpu, &cmd); /* Always copy over UV rc / rrc data */ - if (copy_to_user((__u8 __user *)argp, &cmd.rc, + if (copy_to_user(argp + offsetof(struct kvm_pv_cmd, rc), &cmd.rc, sizeof(cmd.rc) + sizeof(cmd.rrc))) r = -EFAULT; break; diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index eea24562e7db2..0a7d8dfab6b79 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -173,6 +173,7 @@ static int setup_apcb10(struct kvm_vcpu *vcpu, struct kvm_s390_apcb1 *apcb_s, sizeof(struct kvm_s390_apcb0))) return -EFAULT; + memset(apcb_s, 0, sizeof(*apcb_s)); apcb_s->apm[0] = apcb_h->apm[0] & tmp.apm[0]; apcb_s->aqm[0] = apcb_h->aqm[0] & tmp.aqm[0] & 0xffff000000000000UL; apcb_s->adm[0] = apcb_h->adm[0] & tmp.adm[0] & 0xffff000000000000UL; diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 31749c0362ca2..408ae09be3a59 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -771,6 +771,8 @@ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, struct bpf_insn *insn, struct bpf_jit_probe *probe) { + int load_reg; + if (BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; @@ -780,6 +782,14 @@ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, EMIT4(0xb9080000, REG_W1, insn->dst_reg); probe->arena_reg = REG_W1; probe->prg = jit->prg; + /* + * A read-modify-write carrying BPF_FETCH reads the old value into + * src_reg, or into r0 for a BPF_CMPXCHG. Clear that register on + * fault, the remaining atomics only write memory. + */ + load_reg = bpf_atomic_load_reg(insn); + if (load_reg >= 0) + probe->reg = reg2hex[load_reg]; } static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, @@ -1642,6 +1652,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (load_probe.prg != -1) { probe.prg = jit->prg; probe.arena_reg = load_probe.arena_reg; + probe.reg = load_probe.reg; } loop_start = jit->prg; /* 0: {csy|csg} %w0,%src,off(%arena) */ @@ -1783,8 +1794,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, insn->imm == BPF_FUNC_get_smp_processor_id) { const u32 *cpu_nr = &get_lowcore()->cpu_nr; - /* ly %b0, cpu_nr */ - EMIT6_DISP_LH(0xe3000000, 0x0058, BPF_REG_0, REG_0, REG_0, + /* llgf %b0, cpu_nr */ + EMIT6_DISP_LH(0xe3000000, 0x0016, BPF_REG_0, REG_0, REG_0, (unsigned long)cpu_nr); break; } diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index 839bd91c056e5..c1d6ec9977a5e 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -190,6 +190,7 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) device_lock(&pdev->dev); if (pdev->error_state == pci_channel_io_perm_failure) { ers_res = PCI_ERS_RESULT_DISCONNECT; + status_str = "skipped (permanent failure)"; goto out_unlock; } pdev->error_state = pci_channel_io_frozen; @@ -256,8 +257,8 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev) driver->err_handler->resume(pdev); pci_uevent_ers(pdev, PCI_ERS_RESULT_RECOVERED); out_unlock: + zpci_report_status(zdev, pdev, "recovery", status_str); device_unlock(&pdev->dev); - zpci_report_status(zdev, "recovery", status_str); return ers_res; } @@ -320,8 +321,10 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) pr_err("%s: Event 0x%x reports an error for PCI function 0x%x\n", pdev ? pci_name(pdev) : "n/a", ccdf->pec, ccdf->fid); - if (!pdev) + if (!pdev) { + zpci_report_status(zdev, NULL, "error event", "no pdev bound"); goto no_pdev; + } switch (ccdf->pec) { case 0x002a: /* Error event concerns FMB */ diff --git a/arch/s390/pci/pci_report.c b/arch/s390/pci/pci_report.c index 7030f7052926a..72ecabf7003c4 100644 --- a/arch/s390/pci/pci_report.c +++ b/arch/s390/pci/pci_report.c @@ -87,9 +87,23 @@ static struct debug_view debug_log_view = { NULL }; +static ssize_t zpci_report_pdev(struct pci_dev *pdev, char *buf, size_t size) +{ + struct pci_driver *driver; + const char *start = buf; + char *end = buf + size; + + device_lock_assert(&pdev->dev); + buf += scnprintf(buf, end - buf, "state: %s\n", zpci_state_str(pdev->error_state)); + driver = to_pci_driver(pdev->dev.driver); + buf += scnprintf(buf, end - buf, "driver: %s\n", (driver) ? driver->name : "n/a"); + return buf - start; +} + /** * zpci_report_status - Report the status of operations on a PCI device - * @zdev: The PCI device for which to report status + * @zdev: The zPCI device for which to report status + * @pdev: The PCI device associated with the zdev if any, NULL otherwise * @operation: A string representing the operation reported * @status: A string representing the status of the operation * @@ -103,15 +117,14 @@ static struct debug_view debug_log_view = { * * Return: 0 on success an error code < 0 otherwise. */ -int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char *status) +int zpci_report_status(struct zpci_dev *zdev, struct pci_dev *pdev, + const char *operation, const char *status) { struct zpci_report_error *report; - struct pci_driver *driver = NULL; - struct pci_dev *pdev = NULL; char *buf, *end; int ret; - if (!zdev || !zdev->zbus) + if (!zdev) return -ENODEV; /* Protected virtualization hosts get nothing from us */ @@ -121,18 +134,13 @@ int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char report = (void *)get_zeroed_page(GFP_KERNEL); if (!report) return -ENOMEM; - if (zdev->zbus->bus) - pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); - if (pdev) - driver = to_pci_driver(pdev->dev.driver); buf = report->data.log_data; end = report->data.log_data + ZPCI_REPORT_DATA_SIZE; buf += scnprintf(buf, end - buf, "report: %s\n", operation); buf += scnprintf(buf, end - buf, "status: %s\n", status); - buf += scnprintf(buf, end - buf, "state: %s\n", - (pdev) ? zpci_state_str(pdev->error_state) : "n/a"); - buf += scnprintf(buf, end - buf, "driver: %s\n", (driver) ? driver->name : "n/a"); + if (pdev) + buf += zpci_report_pdev(pdev, buf, end - buf); ret = debug_dump(pci_debug_msg_id, &debug_log_view, buf, end - buf, true); if (ret < 0) pr_err("Reading PCI debug messages failed with code %d\n", ret); diff --git a/arch/s390/pci/pci_report.h b/arch/s390/pci/pci_report.h index e08003d51a972..dd7b0b05001c1 100644 --- a/arch/s390/pci/pci_report.h +++ b/arch/s390/pci/pci_report.h @@ -8,9 +8,11 @@ */ #ifndef __S390_PCI_REPORT_H #define __S390_PCI_REPORT_H +#include struct zpci_dev; -int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char *status); +int zpci_report_status(struct zpci_dev *zdev, struct pci_dev *pdev, + const char *operation, const char *status); #endif /* __S390_PCI_REPORT_H */ diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu index e4654388d794b..6e7a366f07984 100644 --- a/arch/x86/Kconfig.cpu +++ b/arch/x86/Kconfig.cpu @@ -204,10 +204,21 @@ config CC_HAS_MARCH_NATIVE # usage warnings that only appear wth '-march=native'. depends on CC_IS_GCC || CLANG_VERSION >= 190100 +config RUSTC_HAS_APXF + # The kernel isn't ready for in-kernel APX instructions. Without + # explicit frontend gating of APX, the backend may emit those + # instructions in native builds. + # + # Rust 1.88 added the `apxf` feature option, but versions before 1.93 + # emit an `apxf` target attribute that only LLVM 23+ can interpret. + def_bool (RUSTC_VERSION >= 108800 && RUSTC_LLVM_MAJOR_VERSION >= 23) || \ + RUSTC_VERSION >= 109300 + config X86_NATIVE_CPU bool "Build and optimize for local/native CPU" depends on X86_64 depends on CC_HAS_MARCH_NATIVE + depends on !RUST || RUSTC_HAS_APXF help Optimize for the current CPU used to compile the kernel. Use this option if you intend to build the kernel for your diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 598f178102ee4..8af6b80cffdd8 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -161,6 +161,11 @@ else ifdef CONFIG_X86_NATIVE_CPU KBUILD_CFLAGS += -march=native + # Prevent the compiler from generating EGPR use. The kernel is + # not yet prepared for general in-kernel use. + KBUILD_CFLAGS += $(call cc-option,-mno-apx-features=egpr) + + # generate_rust_target.rs handles Rust APX gating. KBUILD_RUSTFLAGS += -Ctarget-cpu=native else KBUILD_CFLAGS += -march=x86-64 -mtune=generic diff --git a/arch/x86/coco/sev/svsm.c b/arch/x86/coco/sev/svsm.c index 916d62cd17dc7..2d493d55ff2e8 100644 --- a/arch/x86/coco/sev/svsm.c +++ b/arch/x86/coco/sev/svsm.c @@ -74,6 +74,14 @@ int svsm_perform_call_protocol(struct svsm_call *call) flags = native_local_irq_save(); + /* + * 'caa' is a per-CPU variable. To avoid using a stale or incorrect + * 'caa' if the task is preempted or migrated to another CPU after it + * is fetched, always fetch 'caa' and then issue the SVSM call with + * interrupts disabled. This ensures the correct 'caa' is used. + */ + call->caa = svsm_get_caa(); + ghcb = __sev_get_ghcb(&state); do { @@ -321,7 +329,6 @@ int snp_svsm_vtpm_send_command(u8 *buffer) { struct svsm_call call = {}; - call.caa = svsm_get_caa(); call.rax = SVSM_VTPM_CALL(SVSM_VTPM_CMD); call.rcx = __pa(buffer); @@ -345,7 +352,6 @@ bool snp_svsm_vtpm_probe(void) if (!snp_vmpl) return false; - call.caa = svsm_get_caa(); call.rax = SVSM_VTPM_CALL(SVSM_VTPM_QUERY); if (svsm_perform_call_protocol(&call)) diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c index 29b6f1ed59ec5..f904a636d449b 100644 --- a/arch/x86/coco/tdx/tdx.c +++ b/arch/x86/coco/tdx/tdx.c @@ -694,8 +694,8 @@ static bool handle_in(struct pt_regs *regs, int size, int port) .r13 = PORT_READ, .r14 = port, }; - u64 mask = GENMASK(BITS_PER_BYTE * size, 0); bool success; + u64 val; /* * Emulate the I/O read via hypercall. More info about ABI can be found @@ -703,18 +703,16 @@ static bool handle_in(struct pt_regs *regs, int size, int port) * "TDG.VP.VMCALL". */ success = !__tdx_hypercall(&args); + val = success ? args.r11 : 0; - /* Update part of the register affected by the emulated instruction */ - regs->ax &= ~mask; - if (success) - regs->ax |= args.r11 & mask; + insn_assign_reg(®s->ax, val, size); return success; } static bool handle_out(struct pt_regs *regs, int size, int port) { - u64 mask = GENMASK(BITS_PER_BYTE * size, 0); + u64 mask = GENMASK(BITS_PER_BYTE * size - 1, 0); /* * Emulate the I/O write via hypercall. More info about ABI can be found diff --git a/arch/x86/crypto/aria-aesni-avx2-asm_64.S b/arch/x86/crypto/aria-aesni-avx2-asm_64.S index ed53d4f46bd7c..fda8cb8a99a8a 100644 --- a/arch/x86/crypto/aria-aesni-avx2-asm_64.S +++ b/arch/x86/crypto/aria-aesni-avx2-asm_64.S @@ -982,6 +982,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_encrypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_encrypt_32way) @@ -1007,6 +1008,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_decrypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_decrypt_32way) @@ -1209,6 +1211,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_ctr_crypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %r10); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_ctr_crypt_32way) @@ -1359,6 +1362,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_encrypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_gfni_encrypt_32way) @@ -1384,6 +1388,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_decrypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_gfni_decrypt_32way) @@ -1428,6 +1433,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_ctr_crypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %r10); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_gfni_ctr_crypt_32way) diff --git a/arch/x86/crypto/aria-gfni-avx512-asm_64.S b/arch/x86/crypto/aria-gfni-avx512-asm_64.S index 860887e5d02ed..ca83eb126e065 100644 --- a/arch/x86/crypto/aria-gfni-avx512-asm_64.S +++ b/arch/x86/crypto/aria-gfni-avx512-asm_64.S @@ -800,6 +800,7 @@ SYM_TYPED_FUNC_START(aria_gfni_avx512_encrypt_64way) %zmm9, %zmm8, %zmm11, %zmm10, %zmm12, %zmm13, %zmm14, %zmm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_gfni_avx512_encrypt_64way) @@ -825,6 +826,7 @@ SYM_TYPED_FUNC_START(aria_gfni_avx512_decrypt_64way) %zmm9, %zmm8, %zmm11, %zmm10, %zmm12, %zmm13, %zmm14, %zmm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_gfni_avx512_decrypt_64way) @@ -966,6 +968,7 @@ SYM_TYPED_FUNC_START(aria_gfni_avx512_ctr_crypt_64way) %zmm9, %zmm8, %zmm11, %zmm10, %zmm12, %zmm13, %zmm14, %zmm15, %r10); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_gfni_avx512_ctr_crypt_64way) diff --git a/arch/x86/entry/entry_64_fred.S b/arch/x86/entry/entry_64_fred.S index 0d2768ab836c4..b98f8945dfff4 100644 --- a/arch/x86/entry/entry_64_fred.S +++ b/arch/x86/entry/entry_64_fred.S @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -19,6 +20,7 @@ UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR PUSH_AND_CLEAR_REGS + ENCODE_FRAME_POINTER movq %rsp, %rdi /* %rdi -> pt_regs */ .endm diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c index fb3594ddf731f..854899bfec5d9 100644 --- a/arch/x86/entry/entry_fred.c +++ b/arch/x86/entry/entry_fred.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -71,7 +72,15 @@ static noinstr void fred_intx(struct pt_regs *regs) #endif default: - return exc_general_protection(regs, 0); + /* + * Reconstruct the #GP fault state that IDT delivery would produce. + * Clear the software event flag so ERETU with TF set does not trap + * before the resumed instruction. See prevent_single_step_upon_eretu(). + */ + regs->ip -= regs->fred_ss.insnlen; + regs->flags |= X86_EFLAGS_RF; + regs->fred_ss.swevent = 0; + return exc_general_protection(regs, (regs->fred_ss.vector << 3) | 2); } } diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index dc564688f3d73..54b13faba116c 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -343,11 +343,10 @@ void amd_brs_drain(void) if (!amd_brs_match_plm(event, from, to)) continue; - perf_clear_branch_entry_bitfields(br+nr); - - br[nr].from = from; - br[nr].to = to; - + br[nr] = (struct perf_branch_entry){ + .from = from, + .to = to, + }; nr++; } empty: diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index 9d9c961989d51..a55646fcb8465 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -184,13 +184,6 @@ void amd_pmu_lbr_read(void) entry.to.split.reserved) continue; - perf_clear_branch_entry_bitfields(br + out); - - br[out].from = sign_ext_branch_ip(entry.from.split.ip); - br[out].to = sign_ext_branch_ip(entry.to.split.ip); - br[out].mispred = entry.from.split.mispredict; - br[out].predicted = !br[out].mispred; - /* * Set branch speculation information using the status of * the valid and spec bits. @@ -208,7 +201,14 @@ void amd_pmu_lbr_read(void) * speculative and took the correct path */ idx = (entry.to.split.valid << 1) | entry.to.split.spec; - br[out].spec = lbr_spec_map[idx]; + + br[out] = (struct perf_branch_entry){ + .from = sign_ext_branch_ip(entry.from.split.ip), + .to = sign_ext_branch_ip(entry.to.split.ip), + .mispred = entry.from.split.mispredict, + .predicted = !entry.from.split.mispredict, + .spec = lbr_spec_map[idx], + }; out++; } diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index dbc00b6dd69ee..222dfab9225fe 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -265,6 +265,29 @@ static void amd_uncore_del(struct perf_event *event, int flags) hwc->idx = -1; } +static bool amd_uncore_group_valid(struct perf_event *event) +{ + struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); + struct perf_event *leader = event->group_leader; + struct perf_event *sibling; + int counters = 0; + + if (leader->pmu == event->pmu) + counters++; + + for_each_sibling_event(sibling, leader) { + if (sibling->pmu == event->pmu && + sibling->state > PERF_EVENT_STATE_OFF) + counters++; + } + + /* + * When pmu->event_init() is called, the event is yet to be linked to + * its leader's sibling list, so it is counted separately + */ + return (counters + 1) <= pmu->num_counters; +} + static int amd_uncore_event_init(struct perf_event *event) { struct amd_uncore_pmu *pmu; @@ -282,6 +305,14 @@ static int amd_uncore_event_init(struct perf_event *event) if (!ctx) return -ENODEV; + /* + * Ensure that all events in a group can be scheduled together so that + * a failure can be reported at perf_event_open() time rather than + * silently at pmu->add() time when no free counter is found + */ + if (event->group_leader != event && !amd_uncore_group_valid(event)) + return -EINVAL; + /* * NB and Last level cache counters (MSRs) are shared across all cores * that share the same NB / Last level cache. On family 16h and below, diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index d1af33d96d0a3..af0b67ffb43d8 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2539,7 +2539,8 @@ static int x86_pmu_event_init(struct perf_event *event) } if (READ_ONCE(x86_pmu.attr_rdpmc) && - !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) + !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS) && + !(event->hw.config & ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE)) event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; return err; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 2b35483e2b707..6437404c5c2cc 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -506,6 +506,8 @@ static struct event_constraint intel_pnc_event_constraints[] = { INTEL_EVENT_CONSTRAINT(0xce, 0x1), INTEL_UEVENT_CONSTRAINT(0x01b1, 0x8), + INTEL_UEVENT_CONSTRAINT(0x01b2, 0xf), + INTEL_UEVENT_CONSTRAINT(0x02b2, 0xf), INTEL_UEVENT_CONSTRAINT(0x0847, 0xf), INTEL_UEVENT_CONSTRAINT(0x0446, 0xf), INTEL_UEVENT_CONSTRAINT(0x0846, 0xf), @@ -3125,6 +3127,27 @@ static void intel_pmu_del_event(struct perf_event *event) this_cpu_ptr(&cpu_hw_events)->n_late_setup--; } +int __intel_pmu_quiesce(void) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int pmu_enabled = cpuc->enabled; + + cpuc->enabled = 0; + if (pmu_enabled) + intel_pmu_disable_all(); + + return pmu_enabled; +} + +void __intel_pmu_resume(int pmu_enabled) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + cpuc->enabled = pmu_enabled; + if (pmu_enabled) + intel_pmu_enable_all(0); +} + static int icl_set_topdown_event_period(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -3316,16 +3339,13 @@ static void intel_pmu_read_event(struct perf_event *event) if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD | PERF_X86_EVENT_TOPDOWN) || is_pebs_counter_event_group(event)) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - bool pmu_enabled = cpuc->enabled; + int pmu_enabled; /* Only need to call update_topdown_event() once for group read. */ if (is_metric_event(event) && (cpuc->txn_flags & PERF_PMU_TXN_READ)) return; - cpuc->enabled = 0; - if (pmu_enabled) - intel_pmu_disable_all(); - + pmu_enabled = __intel_pmu_quiesce(); /* * If the PEBS counters snapshotting is enabled, * the topdown event is available in PEBS records. @@ -3334,10 +3354,7 @@ static void intel_pmu_read_event(struct perf_event *event) static_call(intel_pmu_update_topdown_event)(event, NULL); else intel_pmu_drain_pebs_buffer(); - - cpuc->enabled = pmu_enabled; - if (pmu_enabled) - intel_pmu_enable_all(0); + __intel_pmu_resume(pmu_enabled); return; } @@ -3533,7 +3550,7 @@ static void intel_pmu_update_rdpmc_user_disable(struct perf_event *event) */ if (x86_pmu.attr_rdpmc == X86_USER_RDPMC_ALWAYS_ENABLE || (x86_pmu.attr_rdpmc == X86_USER_RDPMC_CONDITIONAL_ENABLE && - event->ctx->task)) + (event->attach_state & PERF_ATTACH_TASK))) event->hw.config &= ~ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE; else event->hw.config |= ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE; @@ -3547,8 +3564,6 @@ static void intel_pmu_enable_event(struct perf_event *event) struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; - intel_pmu_update_rdpmc_user_disable(event); - if (unlikely(event->attr.precise_ip)) static_call(x86_pmu_pebs_enable)(event); @@ -5147,6 +5162,8 @@ static int intel_pmu_hw_config(struct perf_event *event) leader->hw.flags |= PERF_X86_EVENT_ACR; } + intel_pmu_update_rdpmc_user_disable(event); + if ((event->attr.type == PERF_TYPE_HARDWARE) || (event->attr.type == PERF_TYPE_HW_CACHE)) return 0; @@ -5279,12 +5296,15 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) struct kvm_pmu *kvm_pmu = (struct kvm_pmu *)data; u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); u64 pebs_mask = cpuc->pebs_enabled & x86_pmu.pebs_capable; - int global_ctrl, pebs_enable; + u64 guest_pebs_mask; + int global_ctrl; /* * In addition to obeying exclude_guest/exclude_host, remove bits being * used for PEBS when running a guest, because PEBS writes to virtual - * addresses (not physical addresses). + * addresses (not physical addresses). If the guest wants to utilize + * PEBS, and PEBS can be safely enabled in the guest, bits for the guest's + * PEBS-enabled counters will be OR'd back in as appropriate. */ *nr = 0; global_ctrl = (*nr)++; @@ -5314,41 +5334,68 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) return arr; } - if (!kvm_pmu || !x86_pmu.pebs_ept) + /* + * If the CPU doesn't support PEBS in the guest, then there's nothing + * more to do as disabling PMCs via PERF_GLOBAL_CTRL is sufficient on + * CPUs with guest/host isolation. + */ + if (!x86_pmu.pebs_ept) return arr; + /* + * Restrict guest PEBS events to counters that (a) perf supports, (b) + * the guest wants to use for PEBS, (c) are not excluded from counting + * in the guest, and (d) _are_ excluded from counting in the host. + */ + guest_pebs_mask = pebs_mask & intel_ctrl & kvm_pmu->pebs_enable & + ~cpuc->intel_ctrl_host_mask & + cpuc->intel_ctrl_guest_mask; + + /* + * Disable counters where the guest PMC is different than the host PMC + * being used on behalf of the guest, as the PEBS record includes + * PERF_GLOBAL_STATUS, i.e. the guest will see overflow status for the + * wrong counter(s). + */ + guest_pebs_mask &= ~kvm_pmu->host_cross_mapped_mask; + + /* + * FIXME: Allow guest and host usage of PEBS events to co-exist instead + * of disabling guest PEBS entirely if the host is using PEBS. + * What exactly goes wrong if guest and host are using PEBS is + * unknown. + */ + if (pebs_mask & ~cpuc->intel_ctrl_guest_mask) + guest_pebs_mask = 0; + + /* + * Context switch DS_AREA and PEBS_DATA_CFG if and only if PEBS will be + * active in the guest; if no records will be generated while the guest + * is running, then simply keep the host values resident in hardware. + */ arr[(*nr)++] = (struct perf_guest_switch_msr){ .msr = MSR_IA32_DS_AREA, .host = (unsigned long)cpuc->ds, - .guest = kvm_pmu->ds_area, + .guest = guest_pebs_mask ? kvm_pmu->ds_area : (unsigned long)cpuc->ds, }; if (x86_pmu.intel_cap.pebs_baseline) { arr[(*nr)++] = (struct perf_guest_switch_msr){ .msr = MSR_PEBS_DATA_CFG, .host = cpuc->active_pebs_data_cfg, - .guest = kvm_pmu->pebs_data_cfg, + .guest = guest_pebs_mask ? kvm_pmu->pebs_data_cfg : + cpuc->active_pebs_data_cfg, }; } - pebs_enable = (*nr)++; - arr[pebs_enable] = (struct perf_guest_switch_msr){ - .msr = MSR_IA32_PEBS_ENABLE, - .host = cpuc->pebs_enabled & ~cpuc->intel_ctrl_guest_mask, - .guest = pebs_mask & ~cpuc->intel_ctrl_host_mask & kvm_pmu->pebs_enable, - }; - - if (arr[pebs_enable].host) { - /* Disable guest PEBS if host PEBS is enabled. */ - arr[pebs_enable].guest = 0; - } else { - /* Disable guest PEBS thoroughly for cross-mapped PEBS counters. */ - arr[pebs_enable].guest &= ~kvm_pmu->host_cross_mapped_mask; - arr[global_ctrl].guest &= ~kvm_pmu->host_cross_mapped_mask; - /* Set hw GLOBAL_CTRL bits for PEBS counter when it runs for guest */ - arr[global_ctrl].guest |= arr[pebs_enable].guest; - } - + /* + * Do NOT mess with PEBS_ENABLED. As above, disabling counters via + * PERF_GLOBAL_CTRL is sufficient, and loading a stale PEBS_ENABLED, + * e.g. on VM-Exit, can put the system in a bad state. Simply enable + * counters in PERF_GLOBAL_CTRL, as perf load PEBS_ENABLED with the + * full value, i.e. perf *also* relies on PERF_GLOBAL_CTRL. + */ + arr[global_ctrl].guest |= guest_pebs_mask; return arr; } @@ -7947,12 +7994,6 @@ __init int intel_pmu_init(void) x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */ - if (version >= 5) { - x86_pmu.intel_cap.anythread_deprecated = edx.split.anythread_deprecated; - if (x86_pmu.intel_cap.anythread_deprecated) - pr_cont(" AnyThread deprecated, "); - } - /* The perf side of core PMU is ready to support the mediated vPMU. */ x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_MEDIATED_VPMU; @@ -8746,8 +8787,6 @@ __init int intel_pmu_init(void) /* Initialize Atom core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; intel_pmu_init_arw(&pmu->pmu); - - intel_pmu_pebs_data_source_lnl(); break; default: @@ -8829,8 +8868,10 @@ __init int intel_pmu_init(void) &x86_pmu.intel_ctrl); /* AnyThread may be deprecated on arch perfmon v5 or later */ - if (x86_pmu.intel_cap.anythread_deprecated) + if (version >= 5 && edx.split.anythread_deprecated) { x86_pmu.format_attrs = intel_arch_formats_attr; + pr_cont("AnyThread deprecated, "); + } intel_pmu_check_event_constraints_all(NULL); diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 91a093d8cf2e7..ccd98c4635581 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -277,8 +277,8 @@ static u64 pnc_pebs_l2_hit_data_source[PNC_PEBS_DATA_SOURCE_MAX] = { 0, /* 0x06: Reserved */ OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, HIT), /* 0x07: L2 Hit Snoop HIT */ OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, HITM), /* 0x08: L2 Hit Snoop Hit Modified */ - OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, MISS), /* 0x09: Prefetch Promotion */ - OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, MISS), /* 0x0a: Cross Core Prefetch Promotion */ + OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE), /* 0x09: Prefetch Promotion */ + OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE), /* 0x0a: Cross Core Prefetch Promotion */ 0, /* 0x0b: Reserved */ 0, /* 0x0c: Reserved */ 0, /* 0x0d: Reserved */ @@ -455,6 +455,7 @@ static inline void pebs_set_tlb_lock(u64 *val, bool tlb, bool lock) static u64 __grt_latency_data(struct perf_event *event, u64 status, u8 dse, bool tlb, bool lock, bool blk) { + union perf_mem_data_src src; u64 val; WARN_ON_ONCE(is_hybrid() && @@ -470,7 +471,16 @@ static u64 __grt_latency_data(struct perf_event *event, u64 status, else val |= P(BLK, NA); - return val; + src.val = val; + + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) + src.mem_op = P(OP, STORE); + + return src.val; } u64 grt_latency_data(struct perf_event *event, u64 status) @@ -563,7 +573,11 @@ static u64 lnc_latency_data(struct perf_event *event, u64 status) val |= P(BLK, NA); src.val = val; - if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) src.mem_op = P(OP, STORE); return src.val; @@ -621,7 +635,11 @@ u64 pnc_latency_data(struct perf_event *event, u64 status) val |= P(BLK, NA); src.val = val; - if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) src.mem_op = P(OP, STORE); return src.val; @@ -1242,8 +1260,11 @@ int intel_pmu_drain_bts_buffer(void) void intel_pmu_drain_pebs_buffer(void) { + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_sample_data data; + WARN_ON_ONCE(cpuc->enabled); + static_call(x86_pmu_drain_pebs)(NULL, &data); } @@ -1288,22 +1309,22 @@ struct event_constraint intel_glm_pebs_event_constraints[] = { struct event_constraint intel_grt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3), - INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0x3f), + INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0x3), + INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0x3f), EVENT_CONSTRAINT_END }; struct event_constraint intel_cmt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3), - INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff), + INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0x3), + INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0xff), EVENT_CONSTRAINT_END }; struct event_constraint intel_dkt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0xff), - INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff), + INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0xff), + INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0xff), EVENT_CONSTRAINT_END }; @@ -1503,24 +1524,8 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = { INTEL_FLAGS_UEVENT_CONSTRAINT(0x012a, 0x1), /* OCR.* events */ INTEL_FLAGS_UEVENT_CONSTRAINT(0x012b, 0x1), /* OCR.* events */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x04a4, 0x1), /* TOPDOWN.BAD_SPEC_SLOTS */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x08a4, 0x1), /* TOPDOWN.BR_MISPREDICT_SLOTS */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x10a4, 0x8), /* TOPDOWN.MEMORY_BOUND_SLOTS */ - INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0x3fc), INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3), - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x87d0, 0x3ff), /* MEM_INST_RETIRED.ANY */ - - INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf), - - INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf), /* * Everything else is handled by PMU_FL_PEBS_ALL, because we @@ -1536,18 +1541,6 @@ struct event_constraint intel_pnc_pebs_event_constraints[] = { INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0xfc), INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3), - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */ - - INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf), - - INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf), - INTEL_FLAGS_EVENT_CONSTRAINT(0xd6, 0xf), /* * Everything else is handled by PMU_FL_PEBS_ALL, because we @@ -1877,8 +1870,11 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event) static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc) { if (cpuc->n_pebs == cpuc->n_large_pebs && - cpuc->n_pebs != cpuc->n_pebs_via_pt) + cpuc->n_pebs != cpuc->n_pebs_via_pt) { + int enabled = __intel_pmu_quiesce(); intel_pmu_drain_pebs_buffer(); + __intel_pmu_resume(enabled); + } } static void __intel_pmu_pebs_enable(struct perf_event *event) @@ -2445,7 +2441,7 @@ static inline void __setup_pebs_basic_group(struct perf_event *event, { /* The ip in basic is EventingIP */ set_linear_ip(regs, ip); - regs->flags = PERF_EFLAGS_EXACT; + regs->flags |= PERF_EFLAGS_EXACT; setup_pebs_time(event, data, tsc); if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT) @@ -2457,9 +2453,17 @@ static inline void __setup_pebs_gpr_group(struct perf_event *event, struct pebs_gprs *gprs, u64 sample_type) { + /* + * Update flags with PEBS data. PERF_EFLAGS_EXACT must be set + * in previous basic group handling. + */ + regs->flags = gprs->flags | PERF_EFLAGS_EXACT; + if (event->attr.precise_ip < 2) { set_linear_ip(regs, gprs->ip); regs->flags &= ~PERF_EFLAGS_EXACT; + } else if (regs->flags & X86_VM_MASK) { + regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK); } if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index cae2e02fe6cc4..1ed29d58f119f 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -756,10 +756,10 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); - perf_clear_branch_entry_bitfields(br); - - br->from = msr_lastbranch.from; - br->to = msr_lastbranch.to; + *br = (struct perf_branch_entry){ + .from = msr_lastbranch.from, + .to = msr_lastbranch.to, + }; br++; } cpuc->lbr_stack.nr = i; @@ -847,14 +847,15 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) if (abort && x86_pmu.lbr_double_abort && out > 0) out--; - perf_clear_branch_entry_bitfields(br+out); - br[out].from = from; - br[out].to = to; - br[out].mispred = mis; - br[out].predicted = pred; - br[out].in_tx = in_tx; - br[out].abort = abort; - br[out].cycles = cycles; + br[out] = (struct perf_branch_entry){ + .from = from, + .to = to, + .mispred = mis, + .predicted = pred, + .in_tx = in_tx, + .abort = abort, + .cycles = cycles, + }; out++; } cpuc->lbr_stack.nr = out; @@ -905,6 +906,7 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, struct perf_branch_entry *e; struct lbr_entry *lbr; u64 from, to, info; + bool mispred; int i; for (i = 0; i < x86_pmu.lbr_nr; i++) { @@ -921,24 +923,27 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, to = rdlbr_to(i, lbr); info = rdlbr_info(i, lbr); - perf_clear_branch_entry_bitfields(e); - - e->from = from; - e->to = to; - e->mispred = get_lbr_mispred(info); - e->predicted = !e->mispred; - e->in_tx = !!(info & LBR_INFO_IN_TX); - e->abort = !!(info & LBR_INFO_ABORT); - e->cycles = get_lbr_cycles(info); - e->type = get_lbr_br_type(info); - - /* - * Leverage the reserved field of cpuc->lbr_entries[i] to - * temporarily store the branch counters information. - * The later code will decide what content can be disclosed - * to the perf tool. Pleae see intel_pmu_lbr_counters_reorder(). - */ - e->reserved = (info >> LBR_INFO_BR_CNTR_OFFSET) & LBR_INFO_BR_CNTR_FULL_MASK; + mispred = get_lbr_mispred(info); + + *e = (struct perf_branch_entry){ + .from = from, + .to = to, + .mispred = mispred, + .predicted = !mispred, + .in_tx = !!(info & LBR_INFO_IN_TX), + .abort = !!(info & LBR_INFO_ABORT), + .cycles = get_lbr_cycles(info), + .type = get_lbr_br_type(info), + /* + * Leverage the reserved field of + * cpuc->lbr_entries[i] to temporarily store the + * branch counters information. The later code will + * decide what content can be disclosed to the perf + * tool. Pleae see intel_pmu_lbr_counters_reorder(). + */ + .reserved = (info >> LBR_INFO_BR_CNTR_OFFSET) & + LBR_INFO_BR_CNTR_FULL_MASK, + }; } cpuc->lbr_stack.nr = i; @@ -1213,7 +1218,7 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) { u64 from, to; int br_sel = cpuc->br_sel; - int i, j, type, to_plm; + int i, j, type, from_plm, to_plm; bool compress = false; /* if sampling all branches, then nothing to filter */ @@ -1245,8 +1250,14 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) type |= X86_BR_NO_TX; } - /* if type does not correspond, then discard */ - if (type == X86_BR_NONE || (br_sel & type) != type) { + from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; + /* + * If type does not correspond, then discard. + * Specifically reject entries whose from address is in + * kernel space when only X86_BR_USER is requested. + */ + if (type == X86_BR_NONE || (br_sel & type) != type || + (!(br_sel & X86_BR_KERNEL) && (from_plm & X86_BR_KERNEL))) { cpuc->lbr_entries[i].from = 0; compress = true; } diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index b5726b50e77db..5754cd4055628 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -502,6 +502,29 @@ static u64 pt_config_filters(struct perf_event *event) return rtit_ctl; } +static void pt_config_enable(struct perf_event *event) +{ + struct pt *pt = this_cpu_ptr(&pt_ctx); + + /* + * Allow resume before starting so as not to overwrite a value set by a + * PMI. + */ + barrier(); + WRITE_ONCE(pt->resume_allowed, 1); + /* Configuration is complete, it is now OK to handle an NMI */ + barrier(); + WRITE_ONCE(pt->handle_nmi, 1); + barrier(); + pt_config_start(event); + barrier(); + /* + * Allow pause after starting so its pt_config_stop() doesn't race with + * pt_config_start(). + */ + WRITE_ONCE(pt->pause_allowed, 1); +} + static void pt_config(struct perf_event *event) { struct pt *pt = this_cpu_ptr(&pt_ctx); @@ -541,23 +564,7 @@ static void pt_config(struct perf_event *event) event->hw.aux_config = reg; - /* - * Allow resume before starting so as not to overwrite a value set by a - * PMI. - */ - barrier(); - WRITE_ONCE(pt->resume_allowed, 1); - /* Configuration is complete, it is now OK to handle an NMI */ - barrier(); - WRITE_ONCE(pt->handle_nmi, 1); - barrier(); - pt_config_start(event); - barrier(); - /* - * Allow pause after starting so its pt_config_stop() doesn't race with - * pt_config_start(). - */ - WRITE_ONCE(pt->pause_allowed, 1); + pt_config_enable(event); } static void pt_config_stop(struct perf_event *event) @@ -1533,12 +1540,14 @@ void intel_pt_interrupt(void) perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0)); - if (!event->hw.state) { + event->hw.state |= PERF_HES_UPTODATE; + + if (!(event->hw.state & PERF_HES_STOPPED)) { int ret; buf = perf_aux_output_begin(&pt->handle, event); if (!buf) { - event->hw.state = PERF_HES_STOPPED; + event->hw.state |= PERF_HES_STOPPED; WRITE_ONCE(pt->resume_allowed, 0); return; } @@ -1554,6 +1563,8 @@ void intel_pt_interrupt(void) pt_config_buffer(buf); pt_config_start(event); + + event->hw.state &= ~PERF_HES_UPTODATE; } } @@ -1622,6 +1633,18 @@ static void pt_event_start(struct perf_event *event, int mode) return; } + /* + * Re-start subsequent to a call to pt_event_stop() without the + * PERF_EF_UPDATE flag. Absence of PERF_HES_UPTODATE indicates that + * perf_aux_output_begin() has already been called. This path can + * come about only in snapshot/overwrite mode - see pt_event_stop(). + */ + if (!(hwc->state & PERF_HES_UPTODATE)) { + hwc->state &= ~PERF_HES_STOPPED; + pt_config_enable(event); + return; + } + buf = perf_aux_output_begin(&pt->handle, event); if (!buf) goto fail_stop; @@ -1632,7 +1655,7 @@ static void pt_event_start(struct perf_event *event, int mode) goto fail_end_stop; } - hwc->state = 0; + hwc->state &= ~(PERF_HES_STOPPED | PERF_HES_UPTODATE); pt_config_buffer(buf); pt_config(event); @@ -1642,12 +1665,13 @@ static void pt_event_start(struct perf_event *event, int mode) fail_end_stop: perf_aux_output_end(&pt->handle, 0); fail_stop: - hwc->state = PERF_HES_STOPPED; + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; } static void pt_event_stop(struct perf_event *event, int mode) { struct pt *pt = this_cpu_ptr(&pt_ctx); + struct pt_buffer *buf; if (mode & PERF_EF_PAUSE) { if (READ_ONCE(pt->pause_allowed)) @@ -1673,17 +1697,24 @@ static void pt_event_stop(struct perf_event *event, int mode) pt_config_stop(event); - if (event->hw.state == PERF_HES_STOPPED) - return; - - event->hw.state = PERF_HES_STOPPED; + event->hw.state |= PERF_HES_STOPPED; - if (mode & PERF_EF_UPDATE) { - struct pt_buffer *buf = perf_get_aux(&pt->handle); + if (event->hw.state & PERF_HES_UPTODATE) + return; - if (!buf) - return; + buf = perf_get_aux(&pt->handle); + if (!buf) + return; + /* + * When not in snapshot/overwrite mode, there is a possibility that the + * buffer has run out of space. The accounting for that is handled by + * the update, so always update in that case. Snapshot/overwrite mode is + * treated differently to allow for pt_event_snapshot_aux() which can + * still get called if the AUX-sampling event is not stopped until after + * PT is stopped. + */ + if ((mode & PERF_EF_UPDATE) || !buf->snapshot) { if (WARN_ON_ONCE(pt->handle.event != event)) return; @@ -1698,6 +1729,7 @@ static void pt_event_stop(struct perf_event *event, int mode) local_xchg(&buf->data_size, buf->nr_pages << PAGE_SHIFT); perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0)); + event->hw.state |= PERF_HES_UPTODATE; } } @@ -1768,13 +1800,15 @@ static int pt_event_add(struct perf_event *event, int mode) if (pt->handle.event) goto fail; + event->hw.state |= PERF_HES_UPTODATE; + if (mode & PERF_EF_START) { pt_event_start(event, 0); ret = -EINVAL; - if (hwc->state == PERF_HES_STOPPED) + if (hwc->state & PERF_HES_STOPPED) goto fail; } else { - hwc->state = PERF_HES_STOPPED; + hwc->state |= PERF_HES_STOPPED; } ret = 0; diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index 7857959c6e823..b9ac2f7d31caa 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -757,7 +757,7 @@ static int uncore_pmu_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ @@ -953,16 +953,18 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu) ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); if (!ret) - pmu->registered = true; + uncore_pmu_set_registered(pmu); return ret; } static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu) { - if (!pmu->registered) + if (!uncore_pmu_registered(pmu)) return; perf_pmu_unregister(&pmu->pmu); - pmu->registered = false; + + /* Keep PMU_BROKEN_BIT sticky. */ + uncore_pmu_clear_registered(pmu); } static void uncore_free_boxes(struct intel_uncore_pmu *pmu) @@ -1148,6 +1150,44 @@ uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids) return pmu; } +static int uncore_box_setup(struct intel_uncore_pmu *pmu, + struct intel_uncore_box *box) +{ + int ret; + + if (uncore_pmu_broken(pmu)) + return -ENODEV; + + ret = uncore_box_init(box); + if (ret) + goto err; + + /* First active box registers the pmu. */ + if (atomic_inc_return(&pmu->activeboxes) > 1) + return 0; + + ret = uncore_pmu_register(pmu); + if (ret) { + atomic_dec(&pmu->activeboxes); + goto err; + } + + return 0; +err: + /* + * If any box fails, mark the per-package PMU as broken regardless of + * whether it was registered or not. + * + * Don't decrement refcnt to avoid other in-die CPUs from trying to set + * up the PMU box again. + * + * Don't kfree box; MSR and MMIO boxes are freed at module exit only. + */ + uncore_pmu_set_broken(pmu); + uncore_box_exit(box); + return ret; +} + /* * Register the PMU for a PCI device * @pdev: The PCI device. @@ -1167,26 +1207,22 @@ static int uncore_pci_pmu_register(struct pci_dev *pdev, return -EINVAL; box = uncore_alloc_box(type, NUMA_NO_NODE); - if (!box) + if (!box) { + uncore_pmu_set_broken(pmu); return -ENOMEM; + } atomic_inc(&box->refcnt); box->dieid = die; box->pci_dev = pdev; box->pmu = pmu; - uncore_box_init(box); - pmu->boxes[die] = box; - if (atomic_inc_return(&pmu->activeboxes) > 1) - return 0; - - /* First active box registers the pmu */ - ret = uncore_pmu_register(pmu); - if (ret) { - pmu->boxes[die] = NULL; - uncore_box_exit(box); + ret = uncore_box_setup(pmu, box); + if (!ret) + pmu->boxes[die] = box; + else kfree(box); - } + return ret; } @@ -1248,11 +1284,16 @@ static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die) { struct intel_uncore_box *box = pmu->boxes[die]; + if (!box) + return; + pmu->boxes[die] = NULL; if (atomic_dec_return(&pmu->activeboxes) == 0) uncore_pmu_unregister(pmu); - uncore_box_exit(box); - kfree(box); + if (atomic_dec_return(&box->refcnt) == 0) { + uncore_box_exit(box); + kfree(box); + } } static void uncore_pci_remove(struct pci_dev *pdev) @@ -1272,7 +1313,6 @@ static void uncore_pci_remove(struct pci_dev *pdev) break; } } - WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX); return; } @@ -1486,7 +1526,8 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, if (old_cpu < 0) { WARN_ON_ONCE(box->cpu != -1); - if (uncore_die_has_box(type, die, pmu->pmu_idx)) { + if (uncore_die_has_box(type, die, pmu->pmu_idx) && + !uncore_pmu_broken(pmu)) { box->cpu = new_cpu; cpumask_set_cpu(new_cpu, &pmu->cpu_mask); } @@ -1494,12 +1535,14 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, } WARN_ON_ONCE(box->cpu != -1 && box->cpu != old_cpu); - box->cpu = -1; cpumask_clear_cpu(old_cpu, &pmu->cpu_mask); - if (new_cpu < 0) + if (new_cpu < 0) { + box->cpu = -1; continue; + } - if (!uncore_die_has_box(type, die, pmu->pmu_idx)) + /* An inactive box doesn't need migration. */ + if (box->cpu == -1) continue; uncore_pmu_cancel_hrtimer(box); perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu); @@ -1515,7 +1558,7 @@ static void uncore_change_context(struct intel_uncore_type **uncores, uncore_change_type_ctx(*uncores, old_cpu, new_cpu); } -static void uncore_box_unref(struct intel_uncore_type **types, int id) +static void uncore_box_unref(struct intel_uncore_type **types, int die) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -1526,7 +1569,7 @@ static void uncore_box_unref(struct intel_uncore_type **types, int id) type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; + box = pmu->boxes[die]; if (box && box->cpu >= 0 && atomic_dec_return(&box->refcnt) == 0) uncore_box_exit(box); } @@ -1537,9 +1580,15 @@ static int uncore_event_cpu_offline(unsigned int cpu) { int die, target; + /* Clear the references */ + die = topology_logical_die_id(cpu); + uncore_box_unref(uncore_msr_uncores, die); + uncore_box_unref(uncore_mmio_uncores, die); + /* Check if exiting cpu is used for collecting uncore events */ if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask)) - goto unref; + return 0; + /* Find a new cpu to collect uncore events */ target = cpumask_any_but(topology_die_cpumask(cpu), cpu); @@ -1552,16 +1601,10 @@ static int uncore_event_cpu_offline(unsigned int cpu) uncore_change_context(uncore_msr_uncores, cpu, target); uncore_change_context(uncore_mmio_uncores, cpu, target); uncore_change_context(uncore_pci_uncores, cpu, target); - -unref: - /* Clear the references */ - die = topology_logical_die_id(cpu); - uncore_box_unref(uncore_msr_uncores, die); - uncore_box_unref(uncore_mmio_uncores, die); return 0; } -static int allocate_boxes(struct intel_uncore_type **types, +static void allocate_boxes(struct intel_uncore_type **types, unsigned int die, unsigned int cpu) { struct intel_uncore_box *box, *tmp; @@ -1575,11 +1618,13 @@ static int allocate_boxes(struct intel_uncore_type **types, type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - if (pmu->boxes[die]) + if (pmu->boxes[die] || uncore_pmu_broken(pmu)) continue; box = uncore_alloc_box(type, cpu_to_node(cpu)); - if (!box) + if (!box) { + uncore_pmu_set_broken(pmu); goto cleanup; + } box->pmu = pmu; box->dieid = die; list_add(&box->active_list, &allocated); @@ -1590,33 +1635,28 @@ static int allocate_boxes(struct intel_uncore_type **types, list_del_init(&box->active_list); box->pmu->boxes[die] = box; } - return 0; + return; cleanup: list_for_each_entry_safe(box, tmp, &allocated, active_list) { list_del_init(&box->active_list); kfree(box); } - return -ENOMEM; } static int uncore_box_ref(struct intel_uncore_type **types, - int id, unsigned int cpu) + int die, unsigned int cpu) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; struct intel_uncore_box *box; - int i, ret; - - ret = allocate_boxes(types, id, cpu); - if (ret) - return ret; + int i; for (; *types; types++) { type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; + box = pmu->boxes[die]; if (box && box->cpu >= 0 && atomic_inc_return(&box->refcnt) == 1) uncore_box_init(box); } @@ -1626,29 +1666,26 @@ static int uncore_box_ref(struct intel_uncore_type **types, static int uncore_event_cpu_online(unsigned int cpu) { - int die, target, msr_ret, mmio_ret; + int die, target; die = topology_logical_die_id(cpu); - msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu); - mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu); - if (msr_ret && mmio_ret) - return -ENOMEM; + allocate_boxes(uncore_msr_uncores, die, cpu); + allocate_boxes(uncore_mmio_uncores, die, cpu); /* * Check if there is an online cpu in the package * which collects uncore events already. */ target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu)); - if (target < nr_cpu_ids) - return 0; - - cpumask_set_cpu(cpu, &uncore_cpu_mask); - - if (!msr_ret) + if (target >= nr_cpu_ids) { + cpumask_set_cpu(cpu, &uncore_cpu_mask); uncore_change_context(uncore_msr_uncores, -1, cpu); - if (!mmio_ret) uncore_change_context(uncore_mmio_uncores, -1, cpu); - uncore_change_context(uncore_pci_uncores, -1, cpu); + uncore_change_context(uncore_pci_uncores, -1, cpu); + } + + uncore_box_ref(uncore_msr_uncores, die, cpu); + uncore_box_ref(uncore_mmio_uncores, die, cpu); return 0; } diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index c2e5ccb1d72c4..0adb477d97086 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -129,7 +129,7 @@ struct intel_uncore_type { #define events_group attr_groups[2] struct intel_uncore_ops { - void (*init_box)(struct intel_uncore_box *); + int (*init_box)(struct intel_uncore_box *); void (*exit_box)(struct intel_uncore_box *); void (*disable_box)(struct intel_uncore_box *); void (*enable_box)(struct intel_uncore_box *); @@ -146,13 +146,24 @@ struct intel_uncore_pmu { struct pmu pmu; char name[UNCORE_PMU_NAME_LEN]; int pmu_idx; - bool registered; + unsigned long flags; atomic_t activeboxes; cpumask_t cpu_mask; struct intel_uncore_type *type; struct intel_uncore_box **boxes; }; +#define PMU_REGISTERED_BIT 0 +#define PMU_BROKEN_BIT 1 + +#define uncore_pmu_registered(pmu) test_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_broken(pmu) test_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_available(pmu) (uncore_pmu_registered(pmu) && \ + !uncore_pmu_broken(pmu)) +#define uncore_pmu_set_registered(pmu) set_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_set_broken(pmu) set_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_clear_registered(pmu) clear_bit(PMU_REGISTERED_BIT, &(pmu)->flags) + struct intel_uncore_extra_reg { raw_spinlock_t lock; u64 config, config1, config2; @@ -185,7 +196,7 @@ struct intel_uncore_box { #define CFL_UNC_CBO_7_PERFEVTSEL0 0xf70 #define CFL_UNC_CBO_7_PER_CTR0 0xf76 -#define UNCORE_BOX_FLAG_INITIATED 0 +#define UNCORE_BOX_FLAG_INITIALIZED 0 /* event config registers are 8-byte apart */ #define UNCORE_BOX_FLAG_CTL_OFFS8 1 /* CFL 8th CBOX has different MSR space */ @@ -557,17 +568,23 @@ static inline u64 uncore_read_counter(struct intel_uncore_box *box, return box->pmu->type->ops->read_counter(box, event); } -static inline void uncore_box_init(struct intel_uncore_box *box) +static inline int uncore_box_init(struct intel_uncore_box *box) { - if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { - if (box->pmu->type->ops->init_box) - box->pmu->type->ops->init_box(box); + int ret = 0; + + if (!test_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags) && + box->pmu->type->ops->init_box) { + ret = box->pmu->type->ops->init_box(box); + if (!ret) + __set_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags); } + + return ret; } static inline void uncore_box_exit(struct intel_uncore_box *box) { - if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { + if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)) { if (box->pmu->type->ops->exit_box) box->pmu->type->ops->exit_box(box); } diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index e507762222568..0a22edf4d509a 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -489,14 +489,15 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) return unit->addr; } -void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) { u64 box_ctl = intel_generic_uncore_box_ctl(box); if (!box_ctl) - return; + return -ENODEV; wrmsrq(box_ctl, GENERIC_PMON_BOX_CTL_INT); + return 0; } void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) @@ -578,15 +579,16 @@ static inline int intel_pci_uncore_box_ctl(struct intel_uncore_box *box) return UNCORE_DISCOVERY_PCI_BOX_CTRL(intel_generic_uncore_box_ctl(box)); } -void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) { int box_ctl = intel_pci_uncore_box_ctl(box); if (!box_ctl) - return; + return -ENODEV; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(box->pci_dev, box_ctl, GENERIC_PMON_BOX_CTL_INT); + return pci_write_config_dword(box->pci_dev, box_ctl, + GENERIC_PMON_BOX_CTL_INT); } void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) @@ -648,7 +650,7 @@ static struct intel_uncore_ops generic_uncore_pci_ops = { #define UNCORE_GENERIC_MMIO_SIZE 0x4000 -void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) { static struct intel_uncore_discovery_unit *unit; struct intel_uncore_type *type = box->pmu->type; @@ -658,13 +660,13 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) if (!unit) { pr_warn("Uncore type %d id %d: Cannot find box control address.\n", type->type_id, box->pmu->pmu_idx); - return; + return -ENODEV; } if (!unit->addr) { pr_warn("Uncore type %d box %d: Invalid box control address.\n", type->type_id, unit->id); - return; + return -ENODEV; } addr = unit->addr; @@ -672,10 +674,11 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) if (!box->io_addr) { pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n", type->type_id, unit->id, (unsigned long long)addr); - return; + return -ENOMEM; } writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr); + return 0; } void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box) diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index e1330342b92ee..142e1b56cfc2e 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -148,11 +148,11 @@ void intel_uncore_generic_uncore_cpu_init(void); int intel_uncore_generic_uncore_pci_init(void); void intel_uncore_generic_uncore_mmio_init(void); -void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_msr_init_box(struct intel_uncore_box *box); void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box); -void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, @@ -160,7 +160,7 @@ void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box, struct perf_event *event); -void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_pci_init_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box, diff --git a/arch/x86/events/intel/uncore_nhmex.c b/arch/x86/events/intel/uncore_nhmex.c index 8962e7cb21e3e..7a6855281102f 100644 --- a/arch/x86/events/intel/uncore_nhmex.c +++ b/arch/x86/events/intel/uncore_nhmex.c @@ -199,9 +199,10 @@ DEFINE_UNCORE_FORMAT_ATTR(counter, counter, "config:6-7"); DEFINE_UNCORE_FORMAT_ATTR(match, match, "config1:0-63"); DEFINE_UNCORE_FORMAT_ATTR(mask, mask, "config2:0-63"); -static void nhmex_uncore_msr_init_box(struct intel_uncore_box *box) +static int nhmex_uncore_msr_init_box(struct intel_uncore_box *box) { wrmsrq(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL); + return 0; } static void nhmex_uncore_msr_exit_box(struct intel_uncore_box *box) diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index edddd4f9ab5fc..055131c508ffa 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -295,12 +295,14 @@ static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct pe wrmsrq(event->hw.config_base, 0); } -static void snb_uncore_msr_init_box(struct intel_uncore_box *box) +static int snb_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) { wrmsrq(SNB_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL); } + + return 0; } static void snb_uncore_msr_enable_box(struct intel_uncore_box *box) @@ -394,7 +396,7 @@ void snb_uncore_cpu_init(void) snb_uncore_cbox.num_boxes = topology_num_cores_per_package(); } -static void skl_uncore_msr_init_box(struct intel_uncore_box *box) +static int skl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) { wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, @@ -404,6 +406,8 @@ static void skl_uncore_msr_init_box(struct intel_uncore_box *box) /* The 8th CBOX has different MSR space */ if (box->pmu->pmu_idx == 7) __set_bit(UNCORE_BOX_FLAG_CFL8_CBOX_MSR_OFFS, &box->flags); + + return 0; } static void skl_uncore_msr_enable_box(struct intel_uncore_box *box) @@ -547,10 +551,12 @@ static struct intel_uncore_type *tgl_msr_uncores[] = { NULL, }; -static void rkl_uncore_msr_init_box(struct intel_uncore_box *box) +static int rkl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + + return 0; } void tgl_uncore_cpu_init(void) @@ -707,9 +713,10 @@ static struct intel_uncore_type mtl_uncore_hac_cbox = { .format_group = &adl_uncore_format_group, }; -static void mtl_uncore_msr_init_box(struct intel_uncore_box *box) +static int mtl_uncore_msr_init_box(struct intel_uncore_box *box) { wrmsrq(uncore_msr_box_ctl(box), SNB_UNC_GLOBAL_CTL_EN); + return 0; } static struct intel_uncore_ops mtl_uncore_msr_ops = { @@ -773,10 +780,12 @@ static struct intel_uncore_type *lnl_msr_uncores[] = { #define LNL_UNC_MSR_GLOBAL_CTL 0x240e -static void lnl_uncore_msr_init_box(struct intel_uncore_box *box) +static int lnl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) wrmsrq(LNL_UNC_MSR_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + + return 0; } static struct intel_uncore_ops lnl_uncore_msr_ops = { @@ -874,7 +883,7 @@ static const struct attribute_group snb_uncore_imc_format_group = { .attrs = snb_uncore_imc_formats_attr, }; -static void snb_uncore_imc_init_box(struct intel_uncore_box *box) +static int snb_uncore_imc_init_box(struct intel_uncore_box *box) { struct intel_uncore_type *type = box->pmu->type; struct pci_dev *pdev = box->pci_dev; @@ -893,10 +902,13 @@ static void snb_uncore_imc_init_box(struct intel_uncore_box *box) addr &= ~(PAGE_SIZE - 1); box->io_addr = ioremap(addr, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); + return -ENOMEM; + } box->hrtimer_duration = UNCORE_SNB_IMC_HRTIMER_INTERVAL; + return 0; } static void snb_uncore_imc_enable_box(struct intel_uncore_box *box) @@ -928,7 +940,7 @@ static int snb_uncore_imc_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ @@ -1532,7 +1544,7 @@ static struct pci_dev *tgl_uncore_get_mc_dev(void) #define TGL_UNCORE_MMIO_IMC_MEM_OFFSET 0x10000 #define TGL_UNCORE_PCI_IMC_MAP_SIZE 0xe000 -static void +static int uncore_get_box_mmio_addr(struct intel_uncore_box *box, unsigned int base_offset, int bar_offset, int step) @@ -1541,19 +1553,20 @@ uncore_get_box_mmio_addr(struct intel_uncore_box *box, struct intel_uncore_pmu *pmu = box->pmu; struct intel_uncore_type *type = pmu->type; resource_size_t addr; + int ret = 0; u32 bar; if (!pdev) { pr_warn("perf uncore: Cannot find matched IMC device.\n"); - return; + return -ENODEV; } pci_read_config_dword(pdev, bar_offset, &bar); if (!(bar & BIT(0))) { pr_warn("perf uncore: BAR 0x%x is disabled. Failed to map %s counters.\n", bar_offset, type->name); - pci_dev_put(pdev); - return; + ret = -ENODEV; + goto out; } bar &= ~BIT(0); addr = (resource_size_t)(bar + step * pmu->pmu_idx); @@ -1565,23 +1578,26 @@ uncore_get_box_mmio_addr(struct intel_uncore_box *box, addr += base_offset; box->io_addr = ioremap(addr, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { + ret = -ENOMEM; pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); - + } +out: pci_dev_put(pdev); + return ret; } -static void __uncore_imc_init_box(struct intel_uncore_box *box, +static int __uncore_imc_init_box(struct intel_uncore_box *box, unsigned int base_offset) { - uncore_get_box_mmio_addr(box, base_offset, + return uncore_get_box_mmio_addr(box, base_offset, SNB_UNCORE_PCI_IMC_BAR_OFFSET, TGL_UNCORE_MMIO_IMC_MEM_OFFSET); } -static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, 0); + return __uncore_imc_init_box(box, 0); } static struct intel_uncore_ops tgl_uncore_imc_freerunning_ops = { @@ -1648,13 +1664,15 @@ void tgl_uncore_mmio_init(void) #define ADL_UNCORE_IMC_CTL_INT (ADL_UNCORE_IMC_CTL_RST_CTRL | \ ADL_UNCORE_IMC_CTL_RST_CTRS) -static void adl_uncore_imc_init_box(struct intel_uncore_box *box) +static int adl_uncore_imc_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE); + int ret = __uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE); /* The global control in MC1 can control both MCs. */ - if (box->io_addr && (box->pmu->pmu_idx == 1)) + if (!ret && (box->pmu->pmu_idx == 1)) writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + ADL_UNCORE_IMC_GLOBAL_CTL); + + return ret; } static void adl_uncore_mmio_disable_box(struct intel_uncore_box *box) @@ -1731,9 +1749,9 @@ static struct freerunning_counters adl_uncore_imc_freerunning[] = { [ADL_MMIO_UNCORE_IMC_DATA_WRITE] = { 0xA0, 0x0, 0x0, 1, 64 }, }; -static void adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE); + return __uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE); } static struct intel_uncore_ops adl_uncore_imc_freerunning_ops = { @@ -1803,9 +1821,9 @@ static const struct attribute_group lnl_uncore_format_group = { .attrs = lnl_uncore_formats_attr, }; -static void lnl_uncore_hbo_init_box(struct intel_uncore_box *box) +static int lnl_uncore_hbo_init_box(struct intel_uncore_box *box) { - uncore_get_box_mmio_addr(box, LNL_UNCORE_HBO_BASE, + return uncore_get_box_mmio_addr(box, LNL_UNCORE_HBO_BASE, LNL_UNCORE_PCI_SAFBAR_OFFSET, LNL_UNCORE_HBO_OFFSET); } @@ -1829,14 +1847,16 @@ static struct intel_uncore_type lnl_uncore_hbo = { .format_group = &lnl_uncore_format_group, }; -static void lnl_uncore_sncu_init_box(struct intel_uncore_box *box) +static int lnl_uncore_sncu_init_box(struct intel_uncore_box *box) { - uncore_get_box_mmio_addr(box, LNL_UNCORE_SNCU_BASE, + int ret = uncore_get_box_mmio_addr(box, LNL_UNCORE_SNCU_BASE, LNL_UNCORE_PCI_SAFBAR_OFFSET, 0); - if (box->io_addr) + if (!ret) writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + LNL_UNCORE_GLOBAL_CTL); + + return ret; } static struct intel_uncore_ops lnl_uncore_sncu_ops = { @@ -1887,13 +1907,15 @@ static struct intel_uncore_type ptl_uncore_imc = { .mmio_map_size = 0xf00, }; -static void ptl_uncore_sncu_init_box(struct intel_uncore_box *box) +static int ptl_uncore_sncu_init_box(struct intel_uncore_box *box) { - intel_generic_uncore_mmio_init_box(box); + int ret = intel_generic_uncore_mmio_init_box(box); /* Clear the global freeze bit */ if (box->io_addr) writel(0, box->io_addr + PTL_UNCORE_GLOBAL_CTL_OFFSET); + + return ret; } static struct intel_uncore_ops ptl_uncore_sncu_ops = { diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 334dc384b5b93..a97cd029db366 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -627,12 +627,12 @@ static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box, struct pe return count; } -static void snbep_uncore_pci_init_box(struct intel_uncore_box *box) +static int snbep_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); - pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT); } static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box) @@ -680,12 +680,14 @@ static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box, wrmsrq(hwc->config_base, hwc->config); } -static void snbep_uncore_msr_init_box(struct intel_uncore_box *box) +static int snbep_uncore_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); if (msr) wrmsrq(msr, SNBEP_PMON_BOX_CTL_INT); + + return 0; } static struct attribute *snbep_uncore_formats_attr[] = { @@ -1507,18 +1509,21 @@ int snbep_uncore_pci_init(void) /* end of Sandy Bridge-EP uncore support */ /* IvyTown uncore support */ -static void ivbep_uncore_msr_init_box(struct intel_uncore_box *box) +static int ivbep_uncore_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); if (msr) wrmsrq(msr, IVBEP_PMON_BOX_CTL_INT); + + return 0; } -static void ivbep_uncore_pci_init_box(struct intel_uncore_box *box) +static int ivbep_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; - pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } #define IVBEP_UNCORE_MSR_OPS_COMMON_INIT() \ @@ -2784,7 +2789,7 @@ static struct intel_uncore_type hswep_uncore_cbox = { /* * Write SBOX Initialization register bit by bit to avoid spurious #GPs */ -static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) +static int hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); @@ -2798,6 +2803,8 @@ static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) wrmsrq(msr, flags); } } + + return 0; } static struct intel_uncore_ops hswep_uncore_sbox_msr_ops = { @@ -4162,12 +4169,13 @@ static const struct attribute_group skx_upi_uncore_format_group = { .attrs = skx_upi_uncore_formats_attr, }; -static void skx_upi_uncore_pci_init_box(struct intel_uncore_box *box) +static int skx_upi_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops skx_upi_uncore_pci_ops = { @@ -4323,12 +4331,13 @@ static struct intel_uncore_type skx_uncore_upi = { .cleanup_mapping = skx_upi_cleanup_mapping, }; -static void skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box) +static int skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops skx_m2m_uncore_pci_ops = { @@ -4831,13 +4840,13 @@ void snr_uncore_cpu_init(void) uncore_msr_uncores = snr_msr_uncores; } -static void snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box) +static int snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops snr_m2m_uncore_pci_ops = { @@ -5010,17 +5019,22 @@ static int snr_uncore_mmio_map(struct intel_uncore_box *box, return 0; } -static void __snr_uncore_mmio_init_box(struct intel_uncore_box *box, +static int __snr_uncore_mmio_init_box(struct intel_uncore_box *box, unsigned int box_ctl, int mem_offset, unsigned int device) { - if (!snr_uncore_mmio_map(box, box_ctl, mem_offset, device)) + int ret; + + ret = snr_uncore_mmio_map(box, box_ctl, mem_offset, device); + if (!ret) writel(IVBEP_PMON_BOX_CTL_INT, box->io_addr); + + return ret; } -static void snr_uncore_mmio_init_box(struct intel_uncore_box *box) +static int snr_uncore_mmio_init_box(struct intel_uncore_box *box) { - __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), + return __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), SNR_IMC_MMIO_MEM0_OFFSET, SNR_MC_DEVICE_ID); } @@ -5637,14 +5651,14 @@ int icx_uncore_pci_init(void) return 0; } -static void icx_uncore_imc_init_box(struct intel_uncore_box *box) +static int icx_uncore_imc_init_box(struct intel_uncore_box *box) { unsigned int box_ctl = box->pmu->type->box_ctl + box->pmu->type->mmio_offset * (box->pmu->pmu_idx % ICX_NUMBER_IMC_CHN); int mem_offset = (box->pmu->pmu_idx / ICX_NUMBER_IMC_CHN) * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - __snr_uncore_mmio_init_box(box, box_ctl, mem_offset, + return __snr_uncore_mmio_init_box(box, box_ctl, mem_offset, SNR_MC_DEVICE_ID); } @@ -5701,12 +5715,12 @@ static struct uncore_event_desc icx_uncore_imc_freerunning_events[] = { { /* end: all zeroes */ }, }; -static void icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), + return snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), mem_offset, SNR_MC_DEVICE_ID); } @@ -6003,10 +6017,10 @@ static struct intel_uncore_type spr_uncore_mdf = { .name = "mdf", }; -static void spr_uncore_mmio_offs8_init_box(struct intel_uncore_box *box) +static int spr_uncore_mmio_offs8_init_box(struct intel_uncore_box *box) { __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - intel_generic_uncore_mmio_init_box(box); + return intel_generic_uncore_mmio_init_box(box); } static struct intel_uncore_ops spr_uncore_mmio_offs8_ops = { @@ -6187,12 +6201,11 @@ static struct uncore_event_desc spr_uncore_imc_freerunning_events[] = { #define SPR_MC_DEVICE_ID 0x3251 -static void spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - - snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), - mem_offset, SPR_MC_DEVICE_ID); + return snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), + mem_offset, SPR_MC_DEVICE_ID); } static struct intel_uncore_ops spr_uncore_imc_freerunning_ops = { @@ -6881,20 +6894,24 @@ static unsigned int dmr_iio_freerunning_box_offsets[] = { 0x0, 0x8000, 0x18000, 0x20000 }; -static void dmr_uncore_freerunning_init_box(struct intel_uncore_box *box) +static int dmr_uncore_freerunning_init_box(struct intel_uncore_box *box) { struct intel_uncore_type *type = box->pmu->type; u64 mmio_base; if (box->pmu->pmu_idx >= type->num_boxes) - return; + return -ENODEV; mmio_base = DMR_IMH1_HIOP_MMIO_BASE; mmio_base += dmr_iio_freerunning_box_offsets[box->pmu->pmu_idx]; box->io_addr = ioremap(mmio_base, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); + return -ENOMEM; + } + + return 0; } static struct intel_uncore_ops dmr_uncore_freerunning_ops = { diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index eae24bb35dc14..b577d6048bbbe 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -517,10 +517,6 @@ struct cpu_hw_events { __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST) -#define INTEL_HYBRID_LAT_CONSTRAINT(c, n) \ - __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ - HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID) - #define INTEL_HYBRID_LDLAT_CONSTRAINT(c, n) \ __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_LD_HSW) @@ -668,7 +664,7 @@ union perf_capabilities { u64 perf_metrics:1; u64 pebs_output_pt_available:1; u64 pebs_timing_info:1; - u64 anythread_deprecated:1; + u64 __reserved:1; u64 rdpmc_metrics_clear:1; }; u64 capabilities; @@ -1344,7 +1340,7 @@ static inline u64 x86_pmu_get_event_config(struct perf_event *event) static inline bool x86_pmu_has_rdpmc_user_disable(struct pmu *pmu) { return !!(hybrid(pmu, config_mask) & - ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE); + ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE); } extern struct event_constraint emptyconstraint; @@ -1644,6 +1640,9 @@ static __always_inline void __intel_pmu_lbr_disable(void) wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); } +extern int __intel_pmu_quiesce(void); +extern void __intel_pmu_resume(int pmu_enabled); + int intel_pmu_save_and_restart(struct perf_event *event); struct event_constraint * diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h index db70832232d4a..3e78bb76153ce 100644 --- a/arch/x86/include/asm/barrier.h +++ b/arch/x86/include/asm/barrier.h @@ -17,7 +17,7 @@ #define rmb() asm volatile(ALTERNATIVE("lock addl $0,-4(%%esp)", "lfence", \ X86_FEATURE_XMM2) ::: "memory", "cc") #define wmb() asm volatile(ALTERNATIVE("lock addl $0,-4(%%esp)", "sfence", \ - X86_FEATURE_XMM2) ::: "memory", "cc") + X86_FEATURE_XMM) ::: "memory", "cc") #else #define __mb() asm volatile("mfence":::"memory") #define __rmb() asm volatile("lfence":::"memory") diff --git a/arch/x86/include/asm/div64.h b/arch/x86/include/asm/div64.h index 30fd06ede751c..8a2d343f977ec 100644 --- a/arch/x86/include/asm/div64.h +++ b/arch/x86/include/asm/div64.h @@ -111,7 +111,7 @@ static inline u64 mul_u64_add_u64_div_u64(u64 rax, u64 mul, u64 add, u64 div) if (!statically_true(!add)) asm ("addq %[add], %[lo]; adcq $0, %[hi]" : - [lo] "+r" (rax), [hi] "+r" (rdx) : [add] "irm" (add)); + [lo] "+r" (rax), [hi] "+r" (rdx) : [add] "erm" (add)); asm ("divq %[div]" : "+a" (rax), "+d" (rdx) : [div] "rm" (div)); diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index 4733e9064ee55..ae05647a0afbf 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #define INSN_CODE_SEG_ADDR_SZ(params) ((params >> 4) & 0xf) @@ -46,4 +47,39 @@ enum insn_mmio_type insn_decode_mmio(struct insn *insn, int *bytes); bool insn_is_nop(struct insn *insn); +/* + * Write @val into *@reg following the x86 rules for writes to + * general-purpose registers (Intel SDM Vol. 1, "General-Purpose + * Registers in 64-Bit Mode"): an 8- or 16-bit write leaves the rest of + * the register untouched, a 32-bit write zero-extends the result into + * the upper 32 bits, and a 64-bit write replaces the whole register. + * + * @bytes is the width of the write, not a property of the instruction: + * an instruction that, say, sign-extends a 32-bit immediate into a + * 64-bit register does a 64-bit write here. + * + * @reg need not be 8-byte aligned: KVM's instruction emulator offsets + * the pointer by one byte to address the high-byte registers (AH, CH, + * DH, BH). Use narrow stores for the sub-word cases so the access + * width matches @bytes and the adjacent bytes are left alone. + */ +static inline void insn_assign_reg(unsigned long *reg, u64 val, int bytes) +{ + switch (bytes) { + case 1: + *(u8 *)reg = (u8)val; + break; + case 2: + *(u16 *)reg = (u16)val; + break; + case 4: + /* A 32-bit write zero-extends into the upper 32 bits. */ + *reg = (u32)val; + break; + case 8: + *reg = val; + break; + } +} + #endif /* _ASM_X86_INSN_EVAL_H */ diff --git a/arch/x86/include/asm/kvm-x86-nested-ops.h b/arch/x86/include/asm/kvm-x86-nested-ops.h new file mode 100644 index 0000000000000..4b1be5bcecaa4 --- /dev/null +++ b/arch/x86/include/asm/kvm-x86-nested-ops.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(KVM_X86_NESTED_OP) || \ + !defined(KVM_X86_NESTED_OP_OPTIONAL) || \ + !defined(KVM_X86_NESTED_OP_OPTIONAL_RET0) +#error Missing one or more KVM_X86_NESTED_OP #defines +#else +/* + * KVM_X86_NESTED_OP() and KVM_X86_NESTED_OP_OPTIONAL() are used to help + * generate both DECLARE/DEFINE_STATIC_CALL() invocations and + * "static_call_update()" calls. + * + * KVM_X86_NESTED_OP_OPTIONAL() can be used for those functions that can have + * a NULL definition. KVM_X86_NESTED_OP_OPTIONAL_RET0() can be used likewise + * to make a definition optional, but in this case the default will + * be __static_call_return0. + */ +KVM_X86_NESTED_OP(leave_nested) +KVM_X86_NESTED_OP(is_exception_vmexit) +KVM_X86_NESTED_OP(check_events) +KVM_X86_NESTED_OP_OPTIONAL_RET0(has_events) +KVM_X86_NESTED_OP(triple_fault) +KVM_X86_NESTED_OP(get_state) +KVM_X86_NESTED_OP(set_state) +KVM_X86_NESTED_OP(get_nested_state_pages) +KVM_X86_NESTED_OP_OPTIONAL_RET0(write_log_dirty) +KVM_X86_NESTED_OP(translate_nested_gpa) +#ifdef CONFIG_KVM_HYPERV +KVM_X86_NESTED_OP_OPTIONAL(enable_evmcs) +KVM_X86_NESTED_OP_OPTIONAL_RET0(get_evmcs_version) +KVM_X86_NESTED_OP(hv_inject_synthetic_vmexit_post_tlb_flush) +#endif +#endif + +#undef KVM_X86_NESTED_OP +#undef KVM_X86_NESTED_OP_OPTIONAL +#undef KVM_X86_NESTED_OP_OPTIONAL_RET0 diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 83dc5086138b3..ccf23b3f0e1c2 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -134,6 +134,7 @@ KVM_X86_OP_OPTIONAL(mem_enc_unregister_region) KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from) KVM_X86_OP_OPTIONAL(vm_move_enc_context_from) KVM_X86_OP_OPTIONAL(guest_memory_reclaimed) +KVM_X86_OP_OPTIONAL(reload_vmsa) KVM_X86_OP(get_feature_msr) KVM_X86_OP(check_emulate_instruction) KVM_X86_OP(apic_init_signal_blocked) @@ -148,6 +149,9 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level) KVM_X86_OP_OPTIONAL(gmem_invalidate) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE +KVM_X86_OP_OPTIONAL(gmem_invalidate_range) +#endif #endif #undef KVM_X86_OP diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 6db5b5f79df9b..d02c6c42d2a0a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -128,6 +128,8 @@ KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_HV_TLB_FLUSH \ KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_VMSA_PAGE_RELOAD \ + KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \ KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT) @@ -1985,6 +1987,7 @@ struct kvm_x86_ops { int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd); int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd); void (*guest_memory_reclaimed)(struct kvm *kvm); + void (*reload_vmsa)(struct kvm_vcpu *vcpu); int (*get_feature_msr)(u32 msr, u64 *data); @@ -2009,10 +2012,15 @@ struct kvm_x86_ops { void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); +#endif int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); }; struct kvm_x86_nested_ops { + bool enabled; + void (*leave_nested)(struct kvm_vcpu *vcpu); bool (*is_exception_vmexit)(struct kvm_vcpu *vcpu, u8 vector, u32 error_code); @@ -2069,6 +2077,14 @@ extern struct kvm_x86_ops kvm_x86_ops; #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP #include +#define kvm_nested_call(func) static_call(kvm_x86_nested_##func) + +#define KVM_X86_NESTED_OP(func) \ + DECLARE_STATIC_CALL(kvm_x86_nested_##func, *(((struct kvm_x86_nested_ops *)0)->func)); +#define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP +#define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP +#include + int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops); void kvm_x86_vendor_exit(void); @@ -2427,13 +2443,6 @@ static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); } -#define TSS_IOPB_BASE_OFFSET 0x66 -#define TSS_BASE_SIZE 0x68 -#define TSS_IOPB_SIZE (65536 / 8) -#define TSS_REDIRECTION_SIZE (256 / 8) -#define RMODE_TSS_SIZE \ - (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) - enum { TASK_SWITCH_CALL = 0, TASK_SWITCH_IRET = 1, @@ -2456,12 +2465,7 @@ enum { # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0) #endif -int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v); -int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu); -int kvm_cpu_has_extint(struct kvm_vcpu *v); int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu); -int kvm_cpu_get_extint(struct kvm_vcpu *v); -int kvm_cpu_get_interrupt(struct kvm_vcpu *v); void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index a7294656ad908..c9769a7b6e66c 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -103,7 +103,7 @@ .byte 0xb8 ASM_NL \ .long __kcfi_typeid_##name ASM_NL \ CFI_POST_PADDING \ - SYM_FUNC_END(__cfi_##name) + SYM_END(__cfi_##name, SYM_T_FUNC) /* UML needs to be able to override memcpy() and friends for KASAN. */ #ifdef CONFIG_UML diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index ac295ca6c92f9..b44d24c6eb63f 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -806,7 +806,7 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) pmdval_t val = pmd_val(pmd), oldval = val; pmd_t pmd_result; - val &= (_HPAGE_CHG_MASK & ~_PAGE_DIRTY); + val &= _HPAGE_CHG_MASK; val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h index 06ed2cd2592e8..fcec52132fc7e 100644 --- a/arch/x86/include/asm/pkeys.h +++ b/arch/x86/include/asm/pkeys.h @@ -88,6 +88,9 @@ int mm_pkey_alloc(struct mm_struct *mm) u16 all_pkeys_mask = ((1U << arch_max_pkey()) - 1); int ret; + if (!arch_pkeys_enabled()) + return -1; + /* * Are we out of pkeys? We must handle this specially * because ffz() behavior is undefined if there are no diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index f2d142a0a862e..ea09381070e82 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -164,9 +164,9 @@ unsigned long int3_emulate_pop(struct pt_regs *regs) } static __always_inline -void int3_emulate_call(struct pt_regs *regs, unsigned long func) +void int3_emulate_call(struct pt_regs *regs, unsigned long ip, unsigned long func) { - int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE); + int3_emulate_push(regs, ip); int3_emulate_jmp(regs, func); } diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 62936a3bde19b..f24d7e911d3e5 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -6,6 +6,9 @@ #include #include #include +#include +#include +#include #include #include @@ -1207,6 +1210,41 @@ static bool cfi_debug __ro_after_init; bool cfi_bhi __ro_after_init = false; #endif +#ifdef CONFIG_FINEIBT +/* + * : + * 0: f3 0f 1e fa endbr64 + * 4: 2d 78 56 34 12 sub $0x12345678, %eax + * 9: 2e 0f 85 03 00 00 00 jne,pn 13 + * 10: 0f 1f 40 d6 nopl -0x2a(%rax) + * + * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as + * UDB on x86_64 and raises #UD. + */ +asm( ".pushsection .rodata \n" + "fineibt_preamble_start: \n" + " endbr64 \n" + " subl $0x12345678, %eax \n" + "fineibt_preamble_bhi: \n" + " cs jne.d32 fineibt_preamble_start+0x13 \n" + "#fineibt_func: \n" + " nopl -42(%rax) \n" + "fineibt_preamble_end: \n" + ".popsection\n" +); + +extern u8 fineibt_preamble_start[]; +extern u8 fineibt_preamble_bhi[]; +extern u8 fineibt_preamble_end[]; + +#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) +#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start) +#define fineibt_preamble_ud 0x13 +#define fineibt_preamble_hash 5 + +#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) +#endif /* CONFIG_FINEIBT */ + #ifdef CONFIG_CFI u32 cfi_get_func_hash(void *func) { @@ -1214,9 +1252,11 @@ u32 cfi_get_func_hash(void *func) func -= cfi_get_offset(); switch (cfi_mode) { +#ifdef CONFIG_FINEIBT case CFI_FINEIBT: - func += 7; + func += fineibt_preamble_hash; break; +#endif case CFI_KCFI: func += 1; break; @@ -1358,39 +1398,6 @@ early_param("cfi", cfi_parse_cmdline); * processors. Therefore, it is good to place the most frequent branch first" */ -/* - * : - * 0: f3 0f 1e fa endbr64 - * 4: 2d 78 56 34 12 sub $0x12345678, %eax - * 9: 2e 0f 85 03 00 00 00 jne,pn 13 - * 10: 0f 1f 40 d6 nopl -0x2a(%rax) - * - * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as - * UDB on x86_64 and raises #UD. - */ -asm( ".pushsection .rodata \n" - "fineibt_preamble_start: \n" - " endbr64 \n" - " subl $0x12345678, %eax \n" - "fineibt_preamble_bhi: \n" - " cs jne.d32 fineibt_preamble_start+0x13 \n" - "#fineibt_func: \n" - " nopl -42(%rax) \n" - "fineibt_preamble_end: \n" - ".popsection\n" -); - -extern u8 fineibt_preamble_start[]; -extern u8 fineibt_preamble_bhi[]; -extern u8 fineibt_preamble_end[]; - -#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) -#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start) -#define fineibt_preamble_ud 0x13 -#define fineibt_preamble_hash 5 - -#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) - /* * : * 0: b8 78 56 34 12 mov $0x12345678, %eax @@ -2312,6 +2319,7 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data unsigned long selftest = (unsigned long)&int3_selftest_asm; struct die_args *args = data; struct pt_regs *regs = args->regs; + unsigned long ip; OPTIMIZER_HIDE_VAR(selftest); @@ -2324,7 +2332,8 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data if (regs->ip - INT3_INSN_SIZE != selftest) return NOTIFY_DONE; - int3_emulate_call(regs, (unsigned long)&int3_selftest_callee); + ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE; + int3_emulate_call(regs, ip, (unsigned long)&int3_selftest_callee); return NOTIFY_STOP; } @@ -2543,6 +2552,38 @@ static void text_poke_memset(void *dst, const void *src, size_t len) typedef void text_poke_f(void *dst, const void *src, size_t len); +static void __poke_vmalloc_pages(struct page **pages, void *addr, + bool cross_page_boundary) +{ + pages[0] = vmalloc_to_page(addr); + if (cross_page_boundary) + pages[1] = vmalloc_to_page(addr + PAGE_SIZE); +} + +static void poke_vmalloc_pages(struct page **pages, void *addr, + bool cross_page_boundary) +{ + if (in_dbg_master()) { + /* + * If called from kgdb cannot sleep, but all other CPUs stopped + * anyway so safe to proceed without locks + */ + __poke_vmalloc_pages(pages, addr, cross_page_boundary); + } else { + /* + * execmem ROX ranges are shared between modules and can be + * collapsed to huge PMD entries, and this collapse can happen + * concurrently with a racing set_memory_rox(). + * + * Prevent vmalloc_to_page() from racing by acquiring an + * init_mm read lock which pairs with the init_mm write lock in + * cpa_collapse_large_pages(). + */ + guard(mmap_read_lock)(&init_mm); + __poke_vmalloc_pages(pages, addr, cross_page_boundary); + } +} + static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len) { bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE; @@ -2560,9 +2601,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l BUG_ON(!after_bootmem); if (!core_kernel_text((unsigned long)addr)) { - pages[0] = vmalloc_to_page(addr); - if (cross_page_boundary) - pages[1] = vmalloc_to_page(addr + PAGE_SIZE); + poke_vmalloc_pages(pages, addr, cross_page_boundary); } else { pages[0] = virt_to_page(addr); WARN_ON(!PageReserved(pages[0])); @@ -2892,7 +2931,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) break; case CALL_INSN_OPCODE: - int3_emulate_call(regs, (long)ip + tpl->disp); + int3_emulate_call(regs, (long)ip, (long)ip + tpl->disp); break; case JMP32_INSN_OPCODE: diff --git a/arch/x86/kernel/amd_node.c b/arch/x86/kernel/amd_node.c index 0be01725a2a45..b7926ba3610a3 100644 --- a/arch/x86/kernel/amd_node.c +++ b/arch/x86/kernel/amd_node.c @@ -38,7 +38,6 @@ static struct pci_dev **amd_roots; /* Protect the PCI config register pairs used for SMN. */ static DEFINE_MUTEX(smn_mutex); -static bool smn_exclusive; #define SMN_INDEX_OFFSET 0x60 #define SMN_DATA_OFFSET 0x64 @@ -91,11 +90,16 @@ static int __amd_smn_rw(u8 i_off, u8 d_off, u16 node, u32 address, u32 *value, b if (node >= amd_num_nodes()) return err; - root = amd_roots[node]; - if (!root) + /* + * Uninitialized amd_roots indicates pci_request_config_region_exclusive() + * didn't run or failed and thus the kernel cannot rely on having + * exclusive access to SMN registers so prevent that. + */ + if (!amd_roots) return err; - if (!smn_exclusive) + root = amd_roots[node]; + if (!root) return err; guard(mutex)(&smn_mutex); @@ -247,7 +251,7 @@ __setup("amd_smn_debugfs_enable", amd_smn_enable_dfs); static int __init amd_smn_init(void) { u16 count, num_roots, roots_per_node, node, num_nodes; - struct pci_dev *root; + struct pci_dev *root __free(pci_dev_put) = NULL; if (!cpu_feature_enabled(X86_FEATURE_ZEN)) return 0; @@ -258,7 +262,6 @@ static int __init amd_smn_init(void) return 0; num_roots = 0; - root = NULL; while ((root = get_next_root(root))) { pci_dbg(root, "Reserving PCI config space\n"); @@ -287,17 +290,21 @@ static int __init amd_smn_init(void) return -ENOMEM; roots_per_node = num_roots / num_nodes; + if (!roots_per_node) { + if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) + pr_warn(FW_BUG "Error detecting roots per node.\n"); + roots_per_node = 1; + } count = 0; node = 0; - root = NULL; while (node < num_nodes && (root = get_next_root(root))) { /* Use one root for each node and skip the rest. */ if (count++ % roots_per_node) continue; pci_dbg(root, "is root for AMD node %u\n", node); - amd_roots[node++] = root; + amd_roots[node++] = pci_dev_get(root); } if (enable_dfs) { @@ -308,8 +315,6 @@ static int __init amd_smn_init(void) debugfs_create_file("value", 0600, debugfs_dir, NULL, &smn_value_fops); } - smn_exclusive = true; - return 0; } diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a3caddd411ec6..4c88bbe82a7ef 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1251,9 +1251,6 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { #define VULNBL_INTEL_STEPS(vfm, max_stepping, issues) \ X86_MATCH_VFM_STEPS(vfm, X86_STEP_MIN, max_stepping, issues) -#define VULNBL_INTEL_TYPE(vfm, cpu_type, issues) \ - X86_MATCH_VFM_CPU_TYPE(vfm, INTEL_CPU_TYPE_##cpu_type, issues) - #define VULNBL_AMD(family, blacklist) \ VULNBL(AMD, family, X86_MODEL_ANY, blacklist) @@ -1316,11 +1313,9 @@ static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { VULNBL_INTEL_STEPS(INTEL_TIGERLAKE, X86_STEP_MAX, GDS | ITS | ITS_NATIVE_ONLY), VULNBL_INTEL_STEPS(INTEL_LAKEFIELD, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED), VULNBL_INTEL_STEPS(INTEL_ROCKETLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY), - VULNBL_INTEL_TYPE(INTEL_ALDERLAKE, ATOM, RFDS | VMSCAPE), - VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, VMSCAPE), + VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_ALDERLAKE_L, X86_STEP_MAX, RFDS | VMSCAPE), - VULNBL_INTEL_TYPE(INTEL_RAPTORLAKE, ATOM, RFDS | VMSCAPE), - VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, VMSCAPE), + VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_P, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_S, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_METEORLAKE_L, X86_STEP_MAX, VMSCAPE), @@ -1388,7 +1383,21 @@ static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr) return true; /* Only consult the blacklist when there is no enumeration: */ - return cpu_matches(cpu_vuln_blacklist, RFDS); + if (!cpu_matches(cpu_vuln_blacklist, RFDS)) + return false; + + /* + * ADL and RPL are affected only if they have Atom CPUs. Hybrids have + * both Core and Atom CPUs. Mark unaffected when Atom CPUs are not + * present. + */ + if ((boot_cpu_data.x86_model == 0x97 || + boot_cpu_data.x86_model == 0xB7) && + boot_cpu_data.topo.intel_type != INTEL_CPU_TYPE_ATOM && + !boot_cpu_has(X86_FEATURE_HYBRID_CPU)) + return false; + + return true; } static bool __init vulnerable_to_its(u64 x86_arch_cap_msr) diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 36e0df4d1342d..fd878bc7ff080 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -864,7 +864,7 @@ static void amd_deferred_error_interrupt(void) void mce_amd_handle_storm(unsigned int bank, bool on) { - threshold_restart_bank(bank, on); + threshold_restart_bank(bank, !on); } static void amd_reset_thr_limit(unsigned int bank) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index cfb74be19994d..1aebc8394fca4 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -2108,6 +2108,9 @@ bool filter_mce(struct mce *m) static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) { irqentry_state_t irq_state; + unsigned long dr7; + + dr7 = local_db_save(); WARN_ON_ONCE(user_mode(regs)); @@ -2116,20 +2119,26 @@ static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) * mce_check_crashing_cpu() for details. */ if (mca_cfg.initialized && mce_check_crashing_cpu()) - return; + goto out; irq_state = irqentry_nmi_enter(regs); do_machine_check(regs); irqentry_nmi_exit(regs, irq_state); +out: + local_db_restore(dr7); } static __always_inline void exc_machine_check_user(struct pt_regs *regs) { + unsigned long dr7; + irqentry_enter_from_user_mode(regs); + dr7 = local_db_save(); do_machine_check(regs); + local_db_restore(dr7); irqentry_exit_to_user_mode(regs); } @@ -2138,21 +2147,13 @@ static __always_inline void exc_machine_check_user(struct pt_regs *regs) /* MCE hit kernel mode */ DEFINE_IDTENTRY_MCE(exc_machine_check) { - unsigned long dr7; - - dr7 = local_db_save(); exc_machine_check_kernel(regs); - local_db_restore(dr7); } /* The user mode variant. */ DEFINE_IDTENTRY_MCE_USER(exc_machine_check) { - unsigned long dr7; - - dr7 = local_db_save(); exc_machine_check_user(regs); - local_db_restore(dr7); } #ifdef CONFIG_X86_FRED @@ -2169,28 +2170,20 @@ DEFINE_IDTENTRY_MCE_USER(exc_machine_check) */ DEFINE_FREDENTRY_MCE(exc_machine_check) { - unsigned long dr7; - - dr7 = local_db_save(); if (user_mode(regs)) exc_machine_check_user(regs); else exc_machine_check_kernel(regs); - local_db_restore(dr7); } #endif #else /* 32bit unified entry point */ DEFINE_IDTENTRY_RAW(exc_machine_check) { - unsigned long dr7; - - dr7 = local_db_save(); if (user_mode(regs)) exc_machine_check_user(regs); else exc_machine_check_kernel(regs); - local_db_restore(dr7); } #endif diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index f4a444e6114df..ab87e56788df1 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -345,6 +345,26 @@ static void save_microcode_patch(struct microcode_intel *patch) pr_err("Unable to allocate microcode memory size: %u\n", size); } +static bool revision_is_safe(struct cpu_signature *sig, u32 rev) +{ + u32 vfm = IFM(x86_family(sig->sig), x86_model(sig->sig)); + + /* + * Erratum GNR98 can cause #MCs if "jumping over" revision 0x1000405. + * Avoid the jumps. + */ + if (vfm == INTEL_GRANITERAPIDS_X && + x86_stepping(sig->sig) == 1 && + sig->pf & 0x95 && + sig->rev < 0x1000405 && + rev > 0x1000405) { + pr_err_once("Erratum GNR98: skipping revision 0x%x.\n", rev); + return false; + } + + return true; +} + /* Scan blob for microcode matching the boot CPUs family, model, stepping */ static __init struct microcode_intel *scan_microcode(void *data, size_t size, struct ucode_cpu_info *uci, @@ -366,6 +386,9 @@ static __init struct microcode_intel *scan_microcode(void *data, size_t size, if (!intel_find_matching_signature(data, &uci->cpu_sig)) continue; + if (!revision_is_safe(&uci->cpu_sig, mc_header->rev)) + continue; + /* * For saving the early microcode, find the matching revision which * was loaded on the BSP. @@ -914,6 +937,9 @@ static enum ucode_state parse_microcode_blobs(int cpu, struct iov_iter *iter) if (!intel_find_matching_signature(mc, &uci->cpu_sig)) continue; + if (!revision_is_safe(&uci->cpu_sig, mc_header.rev)) + continue; + is_safe = ucode_validate_minrev(&mc_header); if (force_minrev && !is_safe) continue; diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c index 243a769fdd97b..85ebde361d6ae 100644 --- a/arch/x86/kernel/itmt.c +++ b/arch/x86/kernel/itmt.c @@ -110,18 +110,14 @@ int sched_set_itmt_support(void) arch_debugfs_dir, &sysctl_sched_itmt_enabled, &dfs_sched_itmt_fops); - if (IS_ERR_OR_NULL(dfs_sched_itmt)) { + if (IS_ERR(dfs_sched_itmt)) dfs_sched_itmt = NULL; - return -ENOMEM; - } dfs_sched_core_prio = debugfs_create_file("sched_core_priority", 0644, arch_debugfs_dir, NULL, &sched_core_priority_fops); - if (IS_ERR_OR_NULL(dfs_sched_core_prio)) { + if (IS_ERR(dfs_sched_core_prio)) dfs_sched_core_prio = NULL; - return -ENOMEM; - } sched_itmt_capable = true; diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index c1fac3a9fecc2..03d2710ec3021 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -510,10 +510,9 @@ NOKPROBE_SYMBOL(kprobe_emulate_ret); static void kprobe_emulate_call(struct kprobe *p, struct pt_regs *regs) { - unsigned long func = regs->ip - INT3_INSN_SIZE + p->ainsn.size; + unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; - func += p->ainsn.rel32; - int3_emulate_call(regs, func); + int3_emulate_call(regs, ip, ip + p->ainsn.rel32); } NOKPROBE_SYMBOL(kprobe_emulate_call); diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 3af979fb41d38..65a2de82ecd29 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -276,15 +276,9 @@ static bool is_prefix_bad(struct insn *insn) return false; } -static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64) +static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn) { - enum insn_mode m = x86_64 ? INSN_MODE_64 : INSN_MODE_32; u32 volatile *good_insns; - int ret; - - ret = insn_decode(insn, auprobe->insn, sizeof(auprobe->insn), m); - if (ret < 0) - return -ENOEXEC; if (is_prefix_bad(insn)) return -ENOTSUPP; @@ -293,7 +287,7 @@ static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool if (insn_masking_exception(insn)) return -ENOTSUPP; - if (x86_64) + if (insn->x86_64) good_insns = good_insns_64; else good_insns = good_insns_32; @@ -631,14 +625,29 @@ static struct vm_special_mapping tramp_mapping = { .pages = tramp_mapping_pages, }; -struct uprobe_trampoline { - struct hlist_node node; - unsigned long vaddr; -}; + +#define LEA_INSN_SIZE 5 +#define OPT_INSN_SIZE (LEA_INSN_SIZE + CALL_INSN_SIZE) +#define REDZONE_SIZE 0x80 + +static const u8 lea_rsp[] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; + +static bool is_opt_insns(const uprobe_opcode_t *insn) +{ + return !memcmp(insn, lea_rsp, LEA_INSN_SIZE) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} + +static bool is_swbp_opt_insns(uprobe_opcode_t *insn) +{ + return is_swbp_insn(&insn[0]) && + !memcmp(&insn[1], &lea_rsp[1], LEA_INSN_SIZE - 1) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} static bool is_reachable_by_call(unsigned long vtramp, unsigned long vaddr) { - long delta = (long)(vaddr + 5 - vtramp); + long delta = (long)(vaddr + OPT_INSN_SIZE - vtramp); return delta >= INT_MIN && delta <= INT_MAX; } @@ -651,7 +660,7 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) }; unsigned long low_limit, high_limit; unsigned long low_tramp, high_tramp; - unsigned long call_end = vaddr + 5; + unsigned long call_end = vaddr + OPT_INSN_SIZE; if (check_add_overflow(call_end, INT_MIN, &low_limit)) low_limit = PAGE_SIZE; @@ -682,83 +691,32 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) return high_tramp; } -static struct uprobe_trampoline *create_uprobe_trampoline(unsigned long vaddr) +static struct vm_area_struct *get_uprobe_trampoline(struct mm_struct *mm, unsigned long vaddr, + bool *new_mapping) { - struct pt_regs *regs = task_pt_regs(current); - struct mm_struct *mm = current->mm; - struct uprobe_trampoline *tramp; + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; - if (!user_64bit_mode(regs)) - return NULL; - - vaddr = find_nearest_trampoline(vaddr); - if (IS_ERR_VALUE(vaddr)) - return NULL; - - tramp = kzalloc_obj(*tramp); - if (unlikely(!tramp)) - return NULL; - - tramp->vaddr = vaddr; - vma = _install_special_mapping(mm, tramp->vaddr, PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_DONTCOPY|VM_IO, - &tramp_mapping); - if (IS_ERR(vma)) { - kfree(tramp); - return NULL; - } - return tramp; -} - -static struct uprobe_trampoline *get_uprobe_trampoline(unsigned long vaddr, bool *new) -{ - struct uprobes_state *state = ¤t->mm->uprobes_state; - struct uprobe_trampoline *tramp = NULL; + *new_mapping = false; if (vaddr > TASK_SIZE || vaddr < PAGE_SIZE) - return NULL; + return ERR_PTR(-EINVAL); - hlist_for_each_entry(tramp, &state->head_tramps, node) { - if (is_reachable_by_call(tramp->vaddr, vaddr)) { - *new = false; - return tramp; - } + for_each_vma(vmi, vma) { + if (!vma_is_special_mapping(vma, &tramp_mapping)) + continue; + if (is_reachable_by_call(vma->vm_start, vaddr)) + return vma; } - tramp = create_uprobe_trampoline(vaddr); - if (!tramp) - return NULL; - - *new = true; - hlist_add_head(&tramp->node, &state->head_tramps); - return tramp; -} - -static void destroy_uprobe_trampoline(struct uprobe_trampoline *tramp) -{ - /* - * We do not unmap and release uprobe trampoline page itself, - * because there's no easy way to make sure none of the threads - * is still inside the trampoline. - */ - hlist_del(&tramp->node); - kfree(tramp); -} - -void arch_uprobe_init_state(struct mm_struct *mm) -{ - INIT_HLIST_HEAD(&mm->uprobes_state.head_tramps); -} - -void arch_uprobe_clear_state(struct mm_struct *mm) -{ - struct uprobes_state *state = &mm->uprobes_state; - struct uprobe_trampoline *tramp; - struct hlist_node *n; + vaddr = find_nearest_trampoline(vaddr); + if (IS_ERR_VALUE(vaddr)) + return ERR_PTR(vaddr); - hlist_for_each_entry_safe(tramp, n, &state->head_tramps, node) - destroy_uprobe_trampoline(tramp); + *new_mapping = true; + return _install_special_mapping(mm, vaddr, PAGE_SIZE, + VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_IO, + &tramp_mapping); } static bool __in_uprobe_trampoline(struct mm_struct *mm, unsigned long ip) @@ -810,7 +768,7 @@ SYSCALL_DEFINE0(uprobe) /* Allow execution only from uprobe trampolines. */ if (!in_uprobe_trampoline(regs->ip)) - return -ENXIO; + return -EPROTO; err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args)); if (err) @@ -826,8 +784,8 @@ SYSCALL_DEFINE0(uprobe) regs->ax = args.ax; regs->r11 = args.r11; regs->cx = args.cx; - regs->ip = args.retaddr - 5; - regs->sp += sizeof(args); + regs->ip = args.retaddr - OPT_INSN_SIZE; + regs->sp += sizeof(args) + REDZONE_SIZE; regs->orig_ax = -1; sp = regs->sp; @@ -844,12 +802,12 @@ SYSCALL_DEFINE0(uprobe) */ if (regs->sp != sp) { /* skip the trampoline call */ - if (args.retaddr - 5 == regs->ip) - regs->ip += 5; + if (args.retaddr - OPT_INSN_SIZE == regs->ip) + regs->ip += OPT_INSN_SIZE; return regs->ax; } - regs->sp -= sizeof(args); + regs->sp -= sizeof(args) + REDZONE_SIZE; /* for the case uprobe_consumer has changed ax/r11/cx */ args.ax = regs->ax; @@ -857,7 +815,7 @@ SYSCALL_DEFINE0(uprobe) args.cx = regs->cx; /* keep return address unless we are instructed otherwise */ - if (args.retaddr - 5 != regs->ip) + if (args.retaddr - OPT_INSN_SIZE != regs->ip) args.retaddr = regs->ip; if (shstk_push(args.retaddr) == -EFAULT) @@ -891,7 +849,7 @@ asm ( "pop %rax\n" "pop %r11\n" "pop %rcx\n" - "ret\n" + "ret $" __stringify(REDZONE_SIZE) "\n" "int3\n" ".balign " __stringify(PAGE_SIZE) "\n" ".popsection\n" @@ -909,7 +867,8 @@ late_initcall(arch_uprobes_init); enum { EXPECT_SWBP, - EXPECT_CALL, + EXPECT_OPTIMIZED, + EXPECT_SWBP_OPTIMIZED, }; struct write_opcode_ctx { @@ -917,30 +876,29 @@ struct write_opcode_ctx { int expect; }; -static int is_call_insn(uprobe_opcode_t *insn) -{ - return *insn == CALL_INSN_OPCODE; -} - /* - * Verification callback used by int3_update uprobe_write calls to make sure - * the underlying instruction is as expected - either int3 or call. + * Verification callback used by uprobe_write calls to make sure the underlying + * instruction is in the expected stage of the INT3 update sequence. */ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; - uprobe_opcode_t old_opcode[5]; + uprobe_opcode_t old_opcode[OPT_INSN_SIZE]; - uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5); + uprobe_copy_from_page(page, ctx->base, old_opcode, OPT_INSN_SIZE); switch (ctx->expect) { case EXPECT_SWBP: if (is_swbp_insn(&old_opcode[0])) return 1; break; - case EXPECT_CALL: - if (is_call_insn(&old_opcode[0])) + case EXPECT_OPTIMIZED: + if (is_opt_insns(&old_opcode[0])) + return 1; + break; + case EXPECT_SWBP_OPTIMIZED: + if (is_swbp_opt_insns(&old_opcode[0])) return 1; break; } @@ -949,48 +907,137 @@ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t * } /* - * Modify multi-byte instructions by using INT3 breakpoints on SMP. + * Modify the optimized instruction by using INT3 breakpoints on SMP. * We completely avoid using stop_machine() here, and achieve the * synchronization using INT3 breakpoints and SMP cross-calls. * (borrowed comment from smp_text_poke_batch_finish) * - * The way it is done: - * - Add an INT3 trap to the address that will be patched - * - SMP sync all CPUs - * - Update all but the first byte of the patched range - * - SMP sync all CPUs - * - Replace the first byte (INT3) by the first byte of the replacing opcode - * - SMP sync all CPUs + * For optimization (int3_update_optimize): + * 1) Start with the uprobe INT3 trap already installed + * 2) Update everything but the first byte + * 3) Replace the first INT3 by the first byte of the LEA instruction + * + * For unoptimization (int3_update_unoptimize): + * 1) Start with the optimized uprobe lea/call instructions + * 2) Add an INT3 trap to the address that will be patched + * 3) Restore the NOP bytes before the call opcode + * 4) Replace the first INT3 by the first byte of the NOP instruction + * + * Note that unoptimization deliberately keeps the call opcode and displacement + * in bytes 5..9. Those bytes become operands of the restored 10-byte NOP. + * + * Since there is only a single target uprobe-trampoline for the given nop10 + * instruction address, the CALL instruction will not be changed across + * unoptimization/optimization cycles. + * Therefore, any task that is preempted at the CALL instruction is guaranteed + * to observe that CALL and not anything else. */ -static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, - unsigned long vaddr, char *insn, bool optimize) +static int int3_update_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) { - uprobe_opcode_t int3 = UPROBE_SWBP_INSN; struct write_opcode_ctx ctx = { .base = vaddr, }; int err; /* - * Write int3 trap. + * 1) Initial state after set_swbp() installed the uprobe: + * cc 2e 0f 1f 84 00 00 00 00 00 * - * The swbp_optimize path comes with breakpoint already installed, - * so we can skip this step for optimize == true. + * After a previous unoptimization bytes 5..9 may still contain the + * old call instruction, which remains valid for threads already there. */ - if (!optimize) { - ctx.expect = EXPECT_CALL; - err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, - true /* is_register */, false /* do_update_ref_ctr */, - &ctx); - if (err) - return err; - } + smp_text_poke_sync_each_cpu(); + + /* + * 2) Rewrite the LEA tail and call displacement: + * cc [8d 64 24 80 e8 d0 d1 d2 d3] + */ + ctx.expect = EXPECT_SWBP; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + OPT_INSN_SIZE - 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; smp_text_poke_sync_each_cpu(); - /* Write all but the first byte of the patched range. */ + /* + * 3) Publish the first LEA byte: + * [48] 8d 64 24 80 e8 d0 d1 d2 d3 + * + * From offset 0 this is: + * lea -0x80(%rsp), %rsp + * call + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + goto error; + + smp_text_poke_sync_each_cpu(); + return 0; + +error: + /* + * In all intermediate states byte 0 is INT3, so EXPECT_SWBP covers every + * case. Restore NOP bytes 1..4, but keep the valid CALL at bytes 5..9 + * for a thread that had already executed the LEA before a previous + * unoptimization. + */ ctx.expect = EXPECT_SWBP; - err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, 4, verify_insn, + uprobe_write(auprobe, vma, vaddr + 1, auprobe->insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true, false, &ctx); + smp_text_poke_sync_each_cpu(); + return err; +} + +static int int3_update_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) +{ + uprobe_opcode_t int3 = UPROBE_SWBP_INSN; + struct write_opcode_ctx ctx = { + .base = vaddr, + .expect = EXPECT_OPTIMIZED, + }; + int err; + + /* + * Note the first two uprobe_write calls use is_register=true, because they + * are intermediate patching states while the probe is still active, so + * we force the exclusive anonymous page for the update. + * Also we use do_update_ref_ctr=false because refctr was already updated by + * the initial int3 install. + * + * The last uprobe_write to nop10 instruction is called with is_register=false + * and do_update_ref_ctr=true to trigger the refctr update and to instruct + * uprobe_write to zap the anonymous page if it now matches the file page. + * + * 1) Initial optimized state: + * 48 8d 64 24 80 e8 d0 d1 d2 d3 + * + * 2) Trap new entries before restoring the NOP bytes: + * [cc] 8d 64 24 80 e8 d0 d1 d2 d3 + */ + err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; + + smp_text_poke_sync_each_cpu(); + + /* + * 3) Restore bytes 1..4 of the original NOP while keeping byte 0 trapped + * and byte 5 as CALL: + * cc [2e 0f 1f 84] e8 d0 d1 d2 d3 + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true /* is_register */, false /* do_update_ref_ctr */, &ctx); if (err) @@ -999,13 +1046,16 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, smp_text_poke_sync_each_cpu(); /* - * Write first byte. + * 4) Publish the first byte of the original NOP: + * [66] 2e 0f 1f 84 e8 d0 d1 d2 d3 * - * The swbp_unoptimize needs to finish uprobe removal together - * with ref_ctr update, using uprobe_write with proper flags. + * From offset 0 this is the restored 10-byte NOP; the CALL opcode and + * displacement are now only NOP operands. Offset 5 still decodes as + * CALL for a thread that was already there. */ + ctx.expect = EXPECT_SWBP; err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, - optimize /* is_register */, !optimize /* do_update_ref_ctr */, + false /* is_register */, true /* do_update_ref_ctr */, &ctx); if (err) return err; @@ -1017,17 +1067,25 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int swbp_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr, unsigned long tramp) { - u8 call[5]; + u8 insn[OPT_INSN_SIZE], *call = &insn[LEA_INSN_SIZE]; - __text_gen_insn(call, CALL_INSN_OPCODE, (const void *) vaddr, + /* + * We have nop10 instruction (with first byte overwritten to int3), + * changing it to: + * lea -0x80(%rsp), %rsp + * call tramp + */ + memcpy(insn, lea_rsp, LEA_INSN_SIZE); + __text_gen_insn(call, CALL_INSN_OPCODE, + (const void *) (vaddr + LEA_INSN_SIZE), (const void *) tramp, CALL_INSN_SIZE); - return int3_update(auprobe, vma, vaddr, call, true /* optimize */); + return int3_update_optimize(auprobe, vma, vaddr, insn); } static int swbp_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr) { - return int3_update(auprobe, vma, vaddr, auprobe->insn, false /* optimize */); + return int3_update_unoptimize(auprobe, vma, vaddr, auprobe->insn); } static int copy_from_vaddr(struct mm_struct *mm, unsigned long vaddr, void *dst, int len) @@ -1049,19 +1107,19 @@ static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned struct __packed __arch_relative_insn { u8 op; s32 raddr; - } *call = (struct __arch_relative_insn *) insn; + } *call = (struct __arch_relative_insn *)(insn + LEA_INSN_SIZE); - if (!is_call_insn(insn)) + if (!is_opt_insns(insn)) return false; - return __in_uprobe_trampoline(mm, vaddr + 5 + call->raddr); + return __in_uprobe_trampoline(mm, vaddr + OPT_INSN_SIZE + call->raddr); } static int is_optimized(struct mm_struct *mm, unsigned long vaddr) { - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; int err; - err = copy_from_vaddr(mm, vaddr, &insn, 5); + err = copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE); if (err) return err; return __is_optimized(mm, (uprobe_opcode_t *)&insn, vaddr); @@ -1111,27 +1169,29 @@ int set_orig_insn(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int __arch_uprobe_optimize(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) { - struct uprobe_trampoline *tramp; - struct vm_area_struct *vma; - bool new = false; - int err = 0; + struct pt_regs *regs = task_pt_regs(current); + struct vm_area_struct *vma, *tramp; + bool new_mapping; + int ret; + if (!user_64bit_mode(regs)) + return -EINVAL; vma = find_vma(mm, vaddr); if (!vma) return -EINVAL; - tramp = get_uprobe_trampoline(vaddr, &new); - if (!tramp) - return -EINVAL; - err = swbp_optimize(auprobe, vma, vaddr, tramp->vaddr); - if (WARN_ON_ONCE(err) && new) - destroy_uprobe_trampoline(tramp); - return err; + tramp = get_uprobe_trampoline(mm, vaddr, &new_mapping); + if (IS_ERR(tramp)) + return PTR_ERR(tramp); + ret = swbp_optimize(auprobe, vma, vaddr, tramp->vm_start); + if (WARN_ON_ONCE(ret) && new_mapping) + WARN_ON_ONCE(do_munmap(mm, tramp->vm_start, PAGE_SIZE, NULL)); + return ret; } void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) { struct mm_struct *mm = current->mm; - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; if (!should_optimize(auprobe)) return; @@ -1142,7 +1202,7 @@ void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) * Check if some other thread already optimized the uprobe for us, * if it's the case just go away silently. */ - if (copy_from_vaddr(mm, vaddr, &insn, 5)) + if (copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE)) goto unlock; if (!is_swbp_insn((uprobe_opcode_t*) &insn)) goto unlock; @@ -1158,16 +1218,32 @@ void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) mmap_write_unlock(mm); } +static bool is_optimizable_nop10(struct insn *insn) +{ + static const u8 nop10_prefix[] = { + 0x66, 0x2e, 0x0f, 0x1f, 0x84 + }; + + /* + * Restrict this to the 10-byte NOP form whose last 5 bytes are + * SIB/displacement operands. Unoptimization keeps the call opcode and + * displacement in those bytes, so other NOP encodings are not safe. + */ + return insn->length == OPT_INSN_SIZE && + insn_is_nop(insn) && + !memcmp(insn->kaddr, nop10_prefix, ARRAY_SIZE(nop10_prefix)); +} + static bool can_optimize(struct insn *insn, unsigned long vaddr) { - if (!insn->x86_64 || insn->length != 5) + if (!insn->x86_64) return false; - if (!insn_is_nop(insn)) + if (!is_optimizable_nop10(insn)) return false; /* We can't do cross page atomic writes yet. */ - return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= 5; + return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= OPT_INSN_SIZE; } #else /* 32-bit: */ /* @@ -1549,16 +1625,26 @@ static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn) */ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr) { + enum insn_mode m = is_64bit_mm(mm) ? INSN_MODE_64 : INSN_MODE_32; u8 fix_ip_or_call = UPROBE_FIX_IP; struct insn insn; int ret; - ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm)); - if (ret) - return ret; + ret = insn_decode(&insn, auprobe->insn, sizeof(auprobe->insn), m); + if (ret < 0) + return -ENOEXEC; - if (can_optimize(&insn, addr)) + /* + * No need to check instruction in uprobe_init_insn in case we + * are on top of optimizable nop10. + */ + if (can_optimize(&insn, addr)) { set_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags); + } else { + ret = uprobe_init_insn(auprobe, &insn); + if (ret) + return ret; + } ret = branch_setup_xol_ops(auprobe, &insn); if (ret != -ENOSYS) diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 77337c37324b1..f39c311fd756a 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -5,8 +5,8 @@ ccflags-$(CONFIG_KVM_WERROR) += -Werror include $(srctree)/virt/kvm/Makefile.kvm -kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o pmu.o mtrr.o \ - debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o +kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o pmu.o regs.o \ + mtrr.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o kvm-$(CONFIG_KVM_IOAPIC) += i8259.o i8254.o ioapic.o diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index b566ab5c75156..c6dcb5ac48afe 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -24,6 +24,7 @@ #include "kvm_emulate.h" #include #include +#include #include #include #include @@ -439,25 +440,6 @@ static void assign_masked(ulong *dest, ulong src, ulong mask) *dest = (*dest & ~mask) | (src & mask); } -static void assign_register(unsigned long *reg, u64 val, int bytes) -{ - /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */ - switch (bytes) { - case 1: - *(u8 *)reg = (u8)val; - break; - case 2: - *(u16 *)reg = (u16)val; - break; - case 4: - *reg = (u32)val; - break; /* 64b: zero-extend */ - case 8: - *reg = val; - break; - } -} - static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt) { return (1UL << (ctxt->ad_bytes << 3)) - 1; @@ -505,7 +487,7 @@ register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc) { ulong *preg = reg_rmw(ctxt, reg); - assign_register(preg, *preg + inc, ctxt->ad_bytes); + insn_assign_reg(preg, *preg + inc, ctxt->ad_bytes); } static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc) @@ -1767,7 +1749,7 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt, static void write_register_operand(struct operand *op) { - return assign_register(op->addr.reg, op->val, op->bytes); + return insn_assign_reg(op->addr.reg, op->val, op->bytes); } static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op) @@ -2008,7 +1990,7 @@ static int em_popa(struct x86_emulate_ctxt *ctxt) rc = emulate_pop(ctxt, &val, ctxt->op_bytes); if (rc != X86EMUL_CONTINUE) break; - assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes); + insn_assign_reg(reg_rmw(ctxt, reg), val, ctxt->op_bytes); --reg; } return rc; diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index d8736b5dfcd3a..ddbd35ed255ba 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -626,6 +626,18 @@ static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer) return HRTIMER_NORESTART; } +/* + * Translate a stimer expiry given in 100ns reference ticks into an + * an absolute deadline. Saturates on overflow. + */ +static ktime_t stimer_add_delta(ktime_t now, u64 delta_100ns) +{ + if (delta_100ns >= KTIME_MAX / 100) + return KTIME_MAX; + + return ktime_add_safe(now, 100 * delta_100ns); +} + /* * stimer_start() assumptions: * a) stimer->count is not equal to 0 @@ -635,6 +647,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) { u64 time_now; ktime_t ktime_now; + ktime_t deadline; time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm); ktime_now = ktime_get(); @@ -657,10 +670,8 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) stimer->index, time_now, stimer->exp_time); - hrtimer_start(&stimer->timer, - ktime_add_ns(ktime_now, - 100 * (stimer->exp_time - time_now)), - HRTIMER_MODE_ABS); + deadline = stimer_add_delta(ktime_now, stimer->exp_time - time_now); + hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS); return 0; } stimer->exp_time = stimer->count; @@ -679,9 +690,9 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) stimer->index, time_now, stimer->count); - hrtimer_start(&stimer->timer, - ktime_add_ns(ktime_now, 100 * (stimer->count - time_now)), - HRTIMER_MODE_ABS); + deadline = stimer_add_delta(ktime_now, stimer->count - time_now); + hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS); + return 0; } @@ -2408,7 +2419,7 @@ static int kvm_hv_hypercall_complete(struct kvm_vcpu *vcpu, u64 result) ret = kvm_skip_emulated_instruction(vcpu); if (tlb_lock_count) - kvm_x86_ops.nested_ops->hv_inject_synthetic_vmexit_post_tlb_flush(vcpu); + kvm_nested_call(hv_inject_synthetic_vmexit_post_tlb_flush)(vcpu); return ret; } @@ -2789,8 +2800,8 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, }; int i, nent = ARRAY_SIZE(cpuid_entries); - if (kvm_x86_ops.nested_ops->get_evmcs_version) - evmcs_ver = kvm_x86_ops.nested_ops->get_evmcs_version(vcpu); + if (kvm_x86_ops.nested_ops->enabled) + evmcs_ver = kvm_nested_call(get_evmcs_version)(vcpu); if (cpuid->nent < nent) return -E2BIG; diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h index 34f4a78a7a010..1a84ea31e7fd2 100644 --- a/arch/x86/kvm/irq.h +++ b/arch/x86/kvm/irq.h @@ -112,6 +112,12 @@ static inline int irqchip_in_kernel(struct kvm *kvm) return mode != KVM_IRQCHIP_NONE; } +int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v); +int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu); +int kvm_cpu_has_extint(struct kvm_vcpu *v); +int kvm_cpu_get_extint(struct kvm_vcpu *v); +int kvm_cpu_get_interrupt(struct kvm_vcpu *v); + void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu); void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu); void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index e1bb663ebbd58..bb6e018928527 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -308,9 +308,8 @@ static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu, { if (mmu != &vcpu->arch.nested_mmu) return gpa; - return kvm_x86_ops.nested_ops->translate_nested_gpa(vcpu, gpa, access, - exception, - pte_access); + return kvm_nested_call(translate_nested_gpa)(vcpu, gpa, access, + exception, pte_access); } static inline bool kvm_has_mirrored_tdp(const struct kvm *kvm) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index a61750f8e1e37..49a5fb103194a 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1245,18 +1245,9 @@ struct rmap_iterator { int pos; /* index of the sptep */ }; -/* - * Iteration must be started by this function. This should also be used after - * removing/dropping sptes from the rmap link because in such cases the - * information in the iterator may not be valid. - * - * Returns sptep if found, NULL otherwise. - */ -static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, - struct rmap_iterator *iter) +static u64 *__rmap_get_first(unsigned long rmap_val, + struct rmap_iterator *iter) { - unsigned long rmap_val = kvm_rmap_get(rmap_head); - if (!rmap_val) return NULL; @@ -1270,6 +1261,19 @@ static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, return iter->desc->sptes[iter->pos]; } +/* + * Iteration must be started by this function. This should also be used after + * removing/dropping sptes from the rmap link because in such cases the + * information in the iterator may not be valid. + * + * Returns sptep if found, NULL otherwise. + */ +static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, + struct rmap_iterator *iter) +{ + return __rmap_get_first(kvm_rmap_get(rmap_head), iter); +} + /* * Must be used with a valid iterator: e.g. after rmap_get_first(). * @@ -1304,8 +1308,9 @@ static u64 *rmap_get_next(struct rmap_iterator *iter) __for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \ if (!WARN_ON_ONCE(!is_shadow_present_pte(*(_sptep_)))) \ -#define for_each_rmap_spte_lockless(_rmap_head_, _iter_, _sptep_, _spte_) \ - __for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \ +#define for_each_rmap_spte_lockless(_rmap_val_, _iter_, _sptep_, _spte_) \ + for (_sptep_ = __rmap_get_first(_rmap_val_, _iter_); \ + _sptep_; _sptep_ = rmap_get_next(_iter_)) \ if (is_shadow_present_pte(_spte_ = mmu_spte_get_lockless(sptep))) static void drop_spte(struct kvm *kvm, u64 *sptep) @@ -1743,7 +1748,7 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, rmap_head = gfn_to_rmap(gfn, level, range->slot); rmap_val = kvm_rmap_lock_readonly(rmap_head); - for_each_rmap_spte_lockless(rmap_head, &iter, sptep, spte) { + for_each_rmap_spte_lockless(rmap_val, &iter, sptep, spte) { if (!is_accessed_spte(spte)) continue; @@ -4595,37 +4600,6 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, kvm_vcpu_gfn_to_hva(vcpu, fault->gfn), &arch); } -void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) -{ - int r; - - if (WARN_ON_ONCE(work->arch.error_code & PFERR_PRIVATE_ACCESS)) - return; - - if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) || - work->wakeup_all) - return; - - r = kvm_mmu_reload(vcpu); - if (unlikely(r)) - return; - - if (!vcpu->arch.mmu->root_role.direct && - work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu)) - return; - - r = kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, work->arch.error_code, - true, NULL, NULL); - - /* - * Account fixed page faults, otherwise they'll never be counted, but - * ignore stats for all other return times. Page-ready "faults" aren't - * truly spurious and never trigger emulation - */ - if (r == RET_PF_FIXED) - vcpu->stat.pf_fixed++; -} - static void kvm_mmu_finish_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, int r) { @@ -4983,7 +4957,7 @@ static int kvm_tdp_mmu_page_fault(struct kvm_vcpu *vcpu, } #endif -int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) +static int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { #ifdef CONFIG_X86_64 if (tdp_mmu_enabled) @@ -4993,6 +4967,71 @@ int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) return direct_page_fault(vcpu, fault); } +static int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, + u64 err, bool prefetch, int *emulation_type, + u8 *level) +{ + struct kvm_page_fault fault = { + .addr = cr2_or_gpa, + .error_code = err, + .exec = err & PFERR_FETCH_MASK, + .write = err & PFERR_WRITE_MASK, + .present = err & PFERR_PRESENT_MASK, + .rsvd = err & PFERR_RSVD_MASK, + .user = err & PFERR_USER_MASK, + .prefetch = prefetch, + .is_tdp = likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault), + .nx_huge_page_workaround_enabled = + is_nx_huge_page_enabled(vcpu->kvm), + + .max_level = KVM_MAX_HUGEPAGE_LEVEL, + .req_level = PG_LEVEL_4K, + .goal_level = PG_LEVEL_4K, + .is_private = err & PFERR_PRIVATE_ACCESS, + + .pfn = KVM_PFN_ERR_FAULT, + }; + int r; + + if (vcpu->arch.mmu->root_role.direct) { + /* + * Things like memslots don't understand the concept of a shared + * bit. Strip it so that the GFN can be used like normal, and the + * fault.addr can be used when the shared bit is needed. + */ + fault.gfn = gpa_to_gfn(fault.addr) & ~kvm_gfn_direct_bits(vcpu->kvm); + fault.slot = kvm_vcpu_gfn_to_memslot(vcpu, fault.gfn); + } + + /* + * With retpoline being active an indirect call is rather expensive, + * so do a direct call in the most common case. + */ + if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && fault.is_tdp) + r = kvm_tdp_page_fault(vcpu, &fault); + else + r = vcpu->arch.mmu->page_fault(vcpu, &fault); + + /* + * Not sure what's happening, but punt to userspace and hope that + * they can fix it by changing memory to shared, or they can + * provide a better error. + */ + if (r == RET_PF_EMULATE && fault.is_private) { + pr_warn_ratelimited("kvm: unexpected emulation request on private memory\n"); + kvm_mmu_prepare_memory_fault_exit(vcpu, &fault); + return -EFAULT; + } + + if (fault.write_fault_to_shadow_pgtable && emulation_type) + *emulation_type |= EMULTYPE_WRITE_PF_TO_SP; + if (level) + *level = fault.goal_level; + + return r; +} + + static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code, u8 *level) { @@ -5083,6 +5122,37 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, return min(range->size, end - range->gpa); } +void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) +{ + int r; + + if (WARN_ON_ONCE(work->arch.error_code & PFERR_PRIVATE_ACCESS)) + return; + + if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) || + work->wakeup_all) + return; + + r = kvm_mmu_reload(vcpu); + if (unlikely(r)) + return; + + if (!vcpu->arch.mmu->root_role.direct && + work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu)) + return; + + r = kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, work->arch.error_code, + true, NULL, NULL); + + /* + * Account fixed page faults, otherwise they'll never be counted, but + * ignore stats for all other return times. Page-ready "faults" aren't + * truly spurious and never trigger emulation + */ + if (r == RET_PF_FIXED) + vcpu->stat.pf_fixed++; +} + #ifdef CONFIG_KVM_GUEST_MEMFD static void kvm_assert_gmem_invalidate_lock_held(struct kvm_memory_slot *slot) { @@ -6896,20 +6966,11 @@ static void kvm_zap_obsolete_pages(struct kvm *kvm) kvm_mmu_commit_zap_page(kvm, &invalid_list); } -/* - * Fast invalidate all shadow pages and use lock-break technique - * to zap obsolete pages. - * - * It's required when memslot is being deleted or VM is being - * destroyed, in these cases, we should ensure that KVM MMU does - * not use any resource of the being-deleted slot or all slots - * after calling the function. - */ -static void kvm_mmu_zap_all_fast(struct kvm *kvm) +static void __kvm_mmu_zap_all_fast_front_half(struct kvm *kvm) { lockdep_assert_held(&kvm->slots_lock); + lockdep_assert_held_write(&kvm->mmu_lock); - write_lock(&kvm->mmu_lock); trace_kvm_mmu_zap_all_fast(kvm); /* @@ -6946,8 +7007,12 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS); kvm_zap_obsolete_pages(kvm); +} - write_unlock(&kvm->mmu_lock); +static void __kvm_mmu_zap_all_fast_back_half(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->slots_lock); + lockdep_assert_not_held(&kvm->mmu_lock); /* * Zap the invalidated TDP MMU roots, all SPTEs must be dropped before @@ -6961,6 +7026,24 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) kvm_tdp_mmu_zap_invalidated_roots(kvm, true); } +/* + * Fast invalidate all shadow pages and use lock-break technique + * to zap obsolete pages. + * + * It's required when memslot is being deleted or VM is being + * destroyed, in these cases, we should ensure that KVM MMU does + * not use any resource of the being-deleted slot or all slots + * after calling the function. + */ +static void kvm_mmu_zap_all_fast(struct kvm *kvm) +{ + write_lock(&kvm->mmu_lock); + __kvm_mmu_zap_all_fast_front_half(kvm); + write_unlock(&kvm->mmu_lock); + + __kvm_mmu_zap_all_fast_back_half(kvm); +} + int kvm_mmu_init_vm(struct kvm *kvm) { int r, i; @@ -7535,8 +7618,8 @@ static void kvm_mmu_zap_memslot_pages_and_flush(struct kvm *kvm, kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush); } -static void kvm_mmu_zap_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) { struct kvm_gfn_range range = { .slot = slot, @@ -7545,27 +7628,28 @@ static void kvm_mmu_zap_memslot(struct kvm *kvm, .may_block = true, .attr_filter = KVM_FILTER_PRIVATE | KVM_FILTER_SHARED, }; + bool zap_all = kvm->arch.vm_type == KVM_X86_DEFAULT_VM && + kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); bool flush; write_lock(&kvm->mmu_lock); - flush = kvm_unmap_gfn_range(kvm, &range); - kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush); - write_unlock(&kvm->mmu_lock); -} -static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm) -{ - return kvm->arch.vm_type == KVM_X86_DEFAULT_VM && - kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); -} +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + if (slot->gmem.file) + kvm_arch_gmem_invalidate_range(kvm, &range); +#endif -void kvm_arch_flush_shadow_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) -{ - if (kvm_memslot_flush_zap_all(kvm)) - kvm_mmu_zap_all_fast(kvm); - else - kvm_mmu_zap_memslot(kvm, slot); + if (zap_all) { + __kvm_mmu_zap_all_fast_front_half(kvm); + } else { + flush = kvm_unmap_gfn_range(kvm, &range); + kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush); + } + + write_unlock(&kvm->mmu_lock); + + if (zap_all) + __kvm_mmu_zap_all_fast_back_half(kvm); } void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index 73cdcbccc89e8..c29002c601265 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -290,8 +290,6 @@ struct kvm_page_fault { bool write_fault_to_shadow_pgtable; }; -int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); - /* * Return values of handle_mmio_page_fault(), mmu.page_fault(), fast_page_fault(), * and of course kvm_mmu_do_page_fault(). @@ -337,70 +335,6 @@ static inline void kvm_mmu_prepare_memory_fault_exit(struct kvm_vcpu *vcpu, fault->is_private); } -static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, - u64 err, bool prefetch, - int *emulation_type, u8 *level) -{ - struct kvm_page_fault fault = { - .addr = cr2_or_gpa, - .error_code = err, - .exec = err & PFERR_FETCH_MASK, - .write = err & PFERR_WRITE_MASK, - .present = err & PFERR_PRESENT_MASK, - .rsvd = err & PFERR_RSVD_MASK, - .user = err & PFERR_USER_MASK, - .prefetch = prefetch, - .is_tdp = likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault), - .nx_huge_page_workaround_enabled = - is_nx_huge_page_enabled(vcpu->kvm), - - .max_level = KVM_MAX_HUGEPAGE_LEVEL, - .req_level = PG_LEVEL_4K, - .goal_level = PG_LEVEL_4K, - .is_private = err & PFERR_PRIVATE_ACCESS, - - .pfn = KVM_PFN_ERR_FAULT, - }; - int r; - - if (vcpu->arch.mmu->root_role.direct) { - /* - * Things like memslots don't understand the concept of a shared - * bit. Strip it so that the GFN can be used like normal, and the - * fault.addr can be used when the shared bit is needed. - */ - fault.gfn = gpa_to_gfn(fault.addr) & ~kvm_gfn_direct_bits(vcpu->kvm); - fault.slot = kvm_vcpu_gfn_to_memslot(vcpu, fault.gfn); - } - - /* - * With retpoline being active an indirect call is rather expensive, - * so do a direct call in the most common case. - */ - if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && fault.is_tdp) - r = kvm_tdp_page_fault(vcpu, &fault); - else - r = vcpu->arch.mmu->page_fault(vcpu, &fault); - - /* - * Not sure what's happening, but punt to userspace and hope that - * they can fix it by changing memory to shared, or they can - * provide a better error. - */ - if (r == RET_PF_EMULATE && fault.is_private) { - pr_warn_ratelimited("kvm: unexpected emulation request on private memory\n"); - kvm_mmu_prepare_memory_fault_exit(vcpu, &fault); - return -EFAULT; - } - - if (fault.write_fault_to_shadow_pgtable && emulation_type) - *emulation_type |= EMULTYPE_WRITE_PF_TO_SP; - if (level) - *level = fault.goal_level; - - return r; -} - int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault, const struct kvm_memory_slot *slot, gfn_t gfn); void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 1ba840a73b7ac..8d2c46bc6c0fb 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -233,7 +233,7 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu, !(pte & PT_GUEST_DIRTY_MASK)) { trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte)); #if PTTYPE == PTTYPE_EPT - if (kvm_x86_ops.nested_ops->write_log_dirty(vcpu, addr)) + if (kvm_nested_call(write_log_dirty)(vcpu, addr)) return -EINVAL; #endif pte |= PT_GUEST_DIRTY_MASK; diff --git a/arch/x86/kvm/mmu/tdp_iter.h b/arch/x86/kvm/mmu/tdp_iter.h index 364c5da6c4991..f898d8d0d93c2 100644 --- a/arch/x86/kvm/mmu/tdp_iter.h +++ b/arch/x86/kvm/mmu/tdp_iter.h @@ -19,6 +19,13 @@ static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep) return READ_ONCE(*rcu_dereference(sptep)); } +/* + * WARNING! mmu_lock must be held for write when using the "write atomic" or + * "clear bits atomic" APIs, otherwise KVM could overwrite the "wrong" old SPTE + * value, i.e. clobber an update from a different CPU. The only exception is + * when KVM is freezing a leaf SPTE for removal, in which case KVM doesn't care + * about the exact old SPTE value (KVM will react to the actual old value). + */ static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte) { KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte)); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index c1cbae65d239f..d95af13d922a7 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1334,19 +1334,17 @@ static void kvm_tdp_mmu_age_spte(struct kvm *kvm, struct tdp_iter *iter) if (WARN_ON_ONCE(is_mirror_sptep(iter->sptep))) return; - if (spte_ad_enabled(iter->old_spte)) { - iter->old_spte = tdp_mmu_clear_spte_bits_atomic(iter->sptep, - shadow_accessed_mask); + if (spte_ad_enabled(iter->old_spte)) new_spte = iter->old_spte & ~shadow_accessed_mask; - } else { + else new_spte = mark_spte_for_access_track(iter->old_spte); - /* - * It is safe for the following cmpxchg to fail. Leave the - * Accessed bit set, as the spte is most likely young anyway. - */ - if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte)) - return; - } + + /* + * Don't bother retrying if another CPU modified the SPTE, the SPTE is + * either being zapped or is likely still in-use, i.e. is still young. + */ + if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte)) + return; trace_kvm_tdp_mmu_spte_changed(iter->as_id, iter->gfn, iter->level, iter->old_spte, new_spte); diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index dd1c57593f48b..7fedf25cf46fe 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -812,14 +812,6 @@ void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) { - switch (msr) { - case MSR_CORE_PERF_GLOBAL_STATUS: - case MSR_CORE_PERF_GLOBAL_CTRL: - case MSR_CORE_PERF_GLOBAL_OVF_CTRL: - return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); - default: - break; - } return kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr) || kvm_pmu_call(is_valid_msr)(vcpu, msr); } diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c new file mode 100644 index 0000000000000..94c4e4e41868f --- /dev/null +++ b/arch/x86/kvm/regs.c @@ -0,0 +1,874 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +#include "lapic.h" +#include "mmu.h" +#include "regs.h" +#include "x86.h" + +unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu) +{ + /* Can't read the RIP when guest state is protected, just return 0 */ + if (vcpu->arch.guest_state_protected) + return 0; + + if (is_64_bit_mode(vcpu)) + return kvm_rip_read(vcpu); + return (u32)(kvm_get_segment_base(vcpu, VCPU_SREG_CS) + + kvm_rip_read(vcpu)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_linear_rip); + +bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip) +{ + return kvm_get_linear_rip(vcpu) == linear_rip; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_is_linear_rip); + +unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) +{ + unsigned long rflags; + + rflags = kvm_x86_call(get_rflags)(vcpu); + if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) + rflags &= ~X86_EFLAGS_TF; + return rflags; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_rflags); + +void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) +{ + if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && + kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip)) + rflags |= X86_EFLAGS_TF; + kvm_x86_call(set_rflags)(vcpu, rflags); +} + +void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) +{ + __kvm_set_rflags(vcpu, rflags); + kvm_make_request(KVM_REQ_EVENT, vcpu); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_rflags); + +static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { + /* + * We are here if userspace calls get_regs() in the middle of + * instruction emulation. Registers state needs to be copied + * back from emulation context to vcpu. Userspace shouldn't do + * that usually, but some bad designed PV devices (vmware + * backdoor interface) need this to work + */ + emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); + vcpu->arch.emulate_regs_need_sync_to_vcpu = false; + } + regs->rax = kvm_rax_read_raw(vcpu); + regs->rbx = kvm_rbx_read_raw(vcpu); + regs->rcx = kvm_rcx_read_raw(vcpu); + regs->rdx = kvm_rdx_read_raw(vcpu); + regs->rsi = kvm_rsi_read_raw(vcpu); + regs->rdi = kvm_rdi_read_raw(vcpu); + regs->rsp = kvm_rsp_read(vcpu); + regs->rbp = kvm_rbp_read_raw(vcpu); +#ifdef CONFIG_X86_64 + regs->r8 = kvm_r8_read_raw(vcpu); + regs->r9 = kvm_r9_read_raw(vcpu); + regs->r10 = kvm_r10_read_raw(vcpu); + regs->r11 = kvm_r11_read_raw(vcpu); + regs->r12 = kvm_r12_read_raw(vcpu); + regs->r13 = kvm_r13_read_raw(vcpu); + regs->r14 = kvm_r14_read_raw(vcpu); + regs->r15 = kvm_r15_read_raw(vcpu); +#endif + + regs->rip = kvm_rip_read(vcpu); + regs->rflags = kvm_get_rflags(vcpu); +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + __get_regs(vcpu, regs); + vcpu_put(vcpu); + return 0; +} + +static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + vcpu->arch.emulate_regs_need_sync_from_vcpu = true; + vcpu->arch.emulate_regs_need_sync_to_vcpu = false; + + kvm_rax_write_raw(vcpu, regs->rax); + kvm_rbx_write_raw(vcpu, regs->rbx); + kvm_rcx_write_raw(vcpu, regs->rcx); + kvm_rdx_write_raw(vcpu, regs->rdx); + kvm_rsi_write_raw(vcpu, regs->rsi); + kvm_rdi_write_raw(vcpu, regs->rdi); + kvm_rsp_write(vcpu, regs->rsp); + kvm_rbp_write_raw(vcpu, regs->rbp); +#ifdef CONFIG_X86_64 + kvm_r8_write_raw(vcpu, regs->r8); + kvm_r9_write_raw(vcpu, regs->r9); + kvm_r10_write_raw(vcpu, regs->r10); + kvm_r11_write_raw(vcpu, regs->r11); + kvm_r12_write_raw(vcpu, regs->r12); + kvm_r13_write_raw(vcpu, regs->r13); + kvm_r14_write_raw(vcpu, regs->r14); + kvm_r15_write_raw(vcpu, regs->r15); +#endif + + kvm_rip_write(vcpu, regs->rip); + kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED); + + vcpu->arch.exception.pending = false; + vcpu->arch.exception_vmexit.pending = false; + + kvm_make_request(KVM_REQ_EVENT, vcpu); +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + __set_regs(vcpu, regs); + vcpu_put(vcpu); + return 0; +} + +static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); +} + +/* + * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise. + */ +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; + gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; + gpa_t real_gpa; + int i; + int ret; + u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; + + /* + * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated + * to an L1 GPA. + */ + real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn), + PFERR_USER_MASK | PFERR_WRITE_MASK | + PFERR_GUEST_PAGE_MASK, NULL, 0); + if (real_gpa == INVALID_GPA) + return 0; + + /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */ + ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte, + cr3 & GENMASK(11, 5), sizeof(pdpte)); + if (ret < 0) + return 0; + + for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { + if ((pdpte[i] & PT_PRESENT_MASK) && + (pdpte[i] & pdptr_rsvd_bits(vcpu))) { + return 0; + } + } + + /* + * Marking VCPU_REG_PDPTR dirty doesn't work for !tdp_enabled. + * Shadow page roots need to be reconstructed instead. + */ + if (!tdp_enabled && memcmp(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs))) + kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT); + + memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)); + kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR); + kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); + vcpu->arch.pdptrs_from_userspace = false; + + return 1; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(load_pdptrs); + +static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ +#ifdef CONFIG_X86_64 + if (cr0 & 0xffffffff00000000UL) + return false; +#endif + + if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) + return false; + + if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) + return false; + + return kvm_x86_call(is_valid_cr0)(vcpu, cr0); +} + +void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0) +{ + /* + * CR0.WP is incorporated into the MMU role, but only for non-nested, + * indirect shadow MMUs. If paging is disabled, no updates are needed + * as there are no permission bits to emulate. If TDP is enabled, the + * MMU's metadata needs to be updated, e.g. so that emulating guest + * translations does the right thing, but there's no need to unload the + * root as CR0.WP doesn't affect SPTEs. + */ + if ((cr0 ^ old_cr0) == X86_CR0_WP) { + if (!(cr0 & X86_CR0_PG)) + return; + + if (tdp_enabled) { + kvm_init_mmu(vcpu); + return; + } + } + + if ((cr0 ^ old_cr0) & X86_CR0_PG) { + /* + * Clearing CR0.PG is defined to flush the TLB from the guest's + * perspective. + */ + if (!(cr0 & X86_CR0_PG)) + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + /* + * Check for async #PF completion events when enabling paging, + * as the vCPU may have previously encountered async #PFs (it's + * entirely legal for the guest to toggle paging on/off without + * waiting for the async #PF queue to drain). + */ + else if (kvm_pv_async_pf_enabled(vcpu)) + kvm_make_request(KVM_REQ_APF_READY, vcpu); + } + + if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS) + kvm_mmu_reset_context(vcpu); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr0); + +int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ + unsigned long old_cr0 = kvm_read_cr0(vcpu); + + if (!kvm_is_valid_cr0(vcpu, cr0)) + return 1; + + cr0 |= X86_CR0_ET; + + /* Write to CR0 reserved bits are ignored, even on Intel. */ + cr0 &= ~CR0_RESERVED_BITS; + +#ifdef CONFIG_X86_64 + if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) && + (cr0 & X86_CR0_PG)) { + int cs_db, cs_l; + + if (!is_pae(vcpu)) + return 1; + kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); + if (cs_l) + return 1; + } +#endif + if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) && + is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) && + !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) + return 1; + + if (!(cr0 & X86_CR0_PG) && + (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))) + return 1; + + if (!(cr0 & X86_CR0_WP) && kvm_is_cr4_bit_set(vcpu, X86_CR4_CET)) + return 1; + + kvm_x86_call(set_cr0)(vcpu, cr0); + + kvm_post_set_cr0(vcpu, old_cr0, cr0); + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr0); + +void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) +{ + (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw); + +int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + bool skip_tlb_flush = false; + unsigned long pcid = 0; +#ifdef CONFIG_X86_64 + if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) { + skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH; + cr3 &= ~X86_CR3_PCID_NOFLUSH; + pcid = cr3 & X86_CR3_PCID_MASK; + } +#endif + + /* PDPTRs are always reloaded for PAE paging. */ + if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu)) + goto handle_tlb_flush; + + /* + * Do not condition the GPA check on long mode, this helper is used to + * stuff CR3, e.g. for RSM emulation, and there is no guarantee that + * the current vCPU mode is accurate. + */ + if (!kvm_vcpu_is_legal_cr3(vcpu, cr3)) + return 1; + + if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) + return 1; + + if (cr3 != kvm_read_cr3(vcpu)) + kvm_mmu_new_pgd(vcpu, cr3); + + vcpu->arch.cr3 = cr3; + kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); + /* Do not call post_set_cr3, we do not get here for confidential guests. */ + +handle_tlb_flush: + /* + * A load of CR3 that flushes the TLB flushes only the current PCID, + * even if PCID is disabled, in which case PCID=0 is flushed. It's a + * moot point in the end because _disabling_ PCID will flush all PCIDs, + * and it's impossible to use a non-zero PCID when PCID is disabled, + * i.e. only PCID=0 can be relevant. + */ + if (!skip_tlb_flush) + kvm_invalidate_pcid(vcpu, pcid); + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr3); + +static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + return __kvm_is_valid_cr4(vcpu, cr4) && + kvm_x86_call(is_valid_cr4)(vcpu, cr4); +} + +void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) +{ + if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) + kvm_mmu_reset_context(vcpu); + + /* + * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB + * according to the SDM; however, stale prev_roots could be reused + * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we + * free them all. This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST + * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed, + * so fall through. + */ + if (!tdp_enabled && + (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) + kvm_mmu_unload(vcpu); + + /* + * The TLB has to be flushed for all PCIDs if any of the following + * (architecturally required) changes happen: + * - CR4.PCIDE is changed from 1 to 0 + * - CR4.PGE is toggled + * + * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT. + */ + if (((cr4 ^ old_cr4) & X86_CR4_PGE) || + (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + + /* + * The TLB has to be flushed for the current PCID if any of the + * following (architecturally required) changes happen: + * - CR4.SMEP is changed from 0 to 1 + * - CR4.PAE is toggled + */ + else if (((cr4 ^ old_cr4) & X86_CR4_PAE) || + ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP))) + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); + +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr4); + +int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + unsigned long old_cr4 = kvm_read_cr4(vcpu); + + if (!kvm_is_valid_cr4(vcpu, cr4)) + return 1; + + if (is_long_mode(vcpu)) { + if (!(cr4 & X86_CR4_PAE)) + return 1; + if ((cr4 ^ old_cr4) & X86_CR4_LA57) + return 1; + } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) + && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS) + && !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) + return 1; + + if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) { + /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ + if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) + return 1; + } + + if ((cr4 & X86_CR4_CET) && !kvm_is_cr0_bit_set(vcpu, X86_CR0_WP)) + return 1; + + kvm_x86_call(set_cr4)(vcpu, cr4); + + kvm_post_set_cr4(vcpu, old_cr4, cr4); + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); + +int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) +{ + if (cr8 & CR8_RESERVED_BITS) + return 1; + if (lapic_in_kernel(vcpu)) + kvm_lapic_set_tpr(vcpu, cr8); + else + vcpu->arch.cr8 = cr8; + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr8); + +unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) +{ + if (lapic_in_kernel(vcpu)) + return kvm_lapic_get_cr8(vcpu); + else + return vcpu->arch.cr8; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_cr8); + +static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + struct desc_ptr dt; + + if (vcpu->arch.guest_state_protected) + goto skip_protected_regs; + + kvm_handle_exception_payload_quirk(vcpu); + + kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); + kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + kvm_x86_call(get_idt)(vcpu, &dt); + sregs->idt.limit = dt.size; + sregs->idt.base = dt.address; + kvm_x86_call(get_gdt)(vcpu, &dt); + sregs->gdt.limit = dt.size; + sregs->gdt.base = dt.address; + + sregs->cr2 = vcpu->arch.cr2; + sregs->cr3 = kvm_read_cr3(vcpu); + +skip_protected_regs: + sregs->cr0 = kvm_read_cr0(vcpu); + sregs->cr4 = kvm_read_cr4(vcpu); + sregs->cr8 = kvm_get_cr8(vcpu); + sregs->efer = vcpu->arch.efer; + sregs->apic_base = vcpu->arch.apic_base; +} + +static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + __get_sregs_common(vcpu, sregs); + + if (vcpu->arch.guest_state_protected) + return; + + if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) + set_bit(vcpu->arch.interrupt.nr, + (unsigned long *)sregs->interrupt_bitmap); +} + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + __get_sregs(vcpu, sregs); + vcpu_put(vcpu); + return 0; +} + +void kvm_vcpu_ioctl_x86_get_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2) +{ + int i; + + __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); + + if (vcpu->arch.guest_state_protected) + return; + + if (is_pae_paging(vcpu)) { + kvm_vcpu_srcu_read_lock(vcpu); + for (i = 0 ; i < 4 ; i++) + sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); + sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; + kvm_vcpu_srcu_read_unlock(vcpu); + } +} + +static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) { + /* + * When EFER.LME and CR0.PG are set, the processor is in + * 64-bit mode (though maybe in a 32-bit code segment). + * CR4.PAE and EFER.LMA must be set. + */ + if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA)) + return false; + if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3)) + return false; + } else { + /* + * Not in 64-bit mode: EFER.LMA is clear and the code + * segment cannot be 64-bit. + */ + if (sregs->efer & EFER_LMA || sregs->cs.l) + return false; + } + + return kvm_is_valid_cr4(vcpu, sregs->cr4) && + kvm_is_valid_cr0(vcpu, sregs->cr0) && + kvm_valid_efer(vcpu, sregs->efer); +} + +static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, + int *mmu_reset_needed, bool update_pdptrs) +{ + int idx; + struct desc_ptr dt; + + if (!kvm_is_valid_sregs(vcpu, sregs)) + return -EINVAL; + + if (kvm_apic_set_base(vcpu, sregs->apic_base, true)) + return -EINVAL; + + if (vcpu->arch.guest_state_protected) + return 0; + + dt.size = sregs->idt.limit; + dt.address = sregs->idt.base; + kvm_x86_call(set_idt)(vcpu, &dt); + dt.size = sregs->gdt.limit; + dt.address = sregs->gdt.base; + kvm_x86_call(set_gdt)(vcpu, &dt); + + vcpu->arch.cr2 = sregs->cr2; + *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; + vcpu->arch.cr3 = sregs->cr3; + kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); + kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3); + + *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; + kvm_x86_call(set_efer)(vcpu, sregs->efer); + + *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; + kvm_x86_call(set_cr0)(vcpu, sregs->cr0); + + *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; + kvm_x86_call(set_cr4)(vcpu, sregs->cr4); + + if (update_pdptrs) { + idx = srcu_read_lock(&vcpu->kvm->srcu); + if (is_pae_paging(vcpu)) { + load_pdptrs(vcpu, kvm_read_cr3(vcpu)); + *mmu_reset_needed = 1; + } + srcu_read_unlock(&vcpu->kvm->srcu, idx); + } + + kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); + kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + kvm_set_cr8(vcpu, sregs->cr8); + + /* Older userspace won't unhalt the vcpu on reset. */ + if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && + sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && + !is_protmode(vcpu)) + kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); + + return 0; +} + +static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + int pending_vec, max_bits; + int mmu_reset_needed = 0; + int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); + + if (ret) + return ret; + + if (mmu_reset_needed) { + kvm_mmu_reset_context(vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + } + + max_bits = KVM_NR_INTERRUPTS; + pending_vec = find_first_bit( + (const unsigned long *)sregs->interrupt_bitmap, max_bits); + + if (pending_vec < max_bits) { + kvm_queue_interrupt(vcpu, pending_vec, false); + pr_debug("Set back pending irq %d\n", pending_vec); + kvm_make_request(KVM_REQ_EVENT, vcpu); + } + return 0; +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + int ret; + + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + ret = __set_sregs(vcpu, sregs); + vcpu_put(vcpu); + return ret; +} + +int kvm_vcpu_ioctl_x86_set_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2) +{ + int mmu_reset_needed = 0; + bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID; + bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) && + !(sregs2->efer & EFER_LMA); + int i, ret; + + if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID) + return -EINVAL; + + if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) + return -EINVAL; + + ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, + &mmu_reset_needed, !valid_pdptrs); + if (ret) + return ret; + + if (valid_pdptrs) { + for (i = 0; i < 4 ; i++) + kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); + + kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR); + mmu_reset_needed = 1; + vcpu->arch.pdptrs_from_userspace = true; + } + if (mmu_reset_needed) { + kvm_mmu_reset_context(vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + } + return 0; +} + +void kvm_run_sync_regs_to_user(struct kvm_vcpu *vcpu) +{ + BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES); + + if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS) + __get_regs(vcpu, &vcpu->run->s.regs.regs); + + if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS) + __get_sregs(vcpu, &vcpu->run->s.regs.sregs); +} + +int kvm_run_sync_regs_from_user(struct kvm_vcpu *vcpu) +{ + if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) { + __set_regs(vcpu, &vcpu->run->s.regs.regs); + vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS; + } + + if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) { + struct kvm_sregs sregs = vcpu->run->s.regs.sregs; + + if (__set_sregs(vcpu, &sregs)) + return -EINVAL; + + vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS; + } + + return 0; +} + +void kvm_update_dr0123(struct kvm_vcpu *vcpu) +{ + int i; + + if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { + for (i = 0; i < KVM_NR_DB_REGS; i++) + vcpu->arch.eff_db[i] = vcpu->arch.db[i]; + } +} + +void kvm_update_dr7(struct kvm_vcpu *vcpu) +{ + unsigned long dr7; + + if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) + dr7 = vcpu->arch.guest_debug_dr7; + else + dr7 = vcpu->arch.dr7; + kvm_x86_call(set_dr7)(vcpu, dr7); + vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; + if (dr7 & DR7_BP_EN_MASK) + vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); + +static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) +{ + u64 fixed = DR6_FIXED_1; + + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) + fixed |= DR6_RTM; + + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) + fixed |= DR6_BUS_LOCK; + return fixed; +} + +int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) +{ + size_t size = ARRAY_SIZE(vcpu->arch.db); + + switch (dr) { + case 0 ... 3: + vcpu->arch.db[array_index_nospec(dr, size)] = val; + if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) + vcpu->arch.eff_db[dr] = val; + break; + case 4: + case 6: + if (!kvm_dr6_valid(val)) + return 1; /* #GP */ + vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); + break; + case 5: + default: /* 7 */ + if (!kvm_dr7_valid(val)) + return 1; /* #GP */ + vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; + kvm_update_dr7(vcpu); + break; + } + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_dr); + +unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr) +{ + size_t size = ARRAY_SIZE(vcpu->arch.db); + + switch (dr) { + case 0 ... 3: + return vcpu->arch.db[array_index_nospec(dr, size)]; + case 4: + case 6: + return vcpu->arch.dr6; + case 5: + default: /* 7 */ + return vcpu->arch.dr7; + } +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr); + +int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs) +{ + unsigned int i; + + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + kvm_handle_exception_payload_quirk(vcpu); + + memset(dbgregs, 0, sizeof(*dbgregs)); + + BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db)); + for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) + dbgregs->db[i] = vcpu->arch.db[i]; + + dbgregs->dr6 = vcpu->arch.dr6; + dbgregs->dr7 = vcpu->arch.dr7; + return 0; +} + +int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs) +{ + unsigned int i; + + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + if (dbgregs->flags) + return -EINVAL; + + if (!kvm_dr6_valid(dbgregs->dr6)) + return -EINVAL; + if (!kvm_dr7_valid(dbgregs->dr7)) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) + vcpu->arch.db[i] = dbgregs->db[i]; + + kvm_update_dr0123(vcpu); + vcpu->arch.dr6 = dbgregs->dr6; + vcpu->arch.dr7 = dbgregs->dr7; + kvm_update_dr7(vcpu); + + return 0; +} diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h index 5bda738afb7ca..28f62583545f7 100644 --- a/arch/x86/kvm/regs.h +++ b/arch/x86/kvm/regs.h @@ -397,6 +397,14 @@ static inline bool kvm_dr6_valid(u64 data) return !(data >> 32); } +static inline unsigned long kvm_get_effective_dr7(struct kvm_vcpu *vcpu) +{ + if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) + return vcpu->arch.guest_debug_dr7; + + return vcpu->arch.dr7; +} + static inline void enter_guest_mode(struct kvm_vcpu *vcpu) { vcpu->arch.hflags |= HF_GUEST_MASK; @@ -420,4 +428,26 @@ static inline bool is_guest_mode(struct kvm_vcpu *vcpu) return vcpu->arch.hflags & HF_GUEST_MASK; } +static inline unsigned long kvm_get_segment_base(struct kvm_vcpu *vcpu, int seg) +{ + return kvm_x86_call(get_segment_base)(vcpu, seg); +} + +void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); + +void kvm_vcpu_ioctl_x86_get_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2); +int kvm_vcpu_ioctl_x86_set_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2); + +void kvm_run_sync_regs_to_user(struct kvm_vcpu *vcpu); +int kvm_run_sync_regs_from_user(struct kvm_vcpu *vcpu); + +void kvm_update_dr0123(struct kvm_vcpu *vcpu); +int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs); +int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs); + + #endif diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 3e6c671a8dc26..c1485c3e691c4 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -23,6 +23,7 @@ #include "kvm_emulate.h" #include "trace.h" +#include "irq.h" #include "mmu.h" #include "x86.h" #include "smm.h" diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 0f0ea7896af5b..62561da831075 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -1283,9 +1283,28 @@ static void *sev_dbg_crypt_slow_alloc(struct page *page, unsigned long __va, if (WARN_ON_ONCE((*pa & PAGE_MASK) != ((*pa + *nr_bytes - 1) & PAGE_MASK))) return NULL; + /* + * If SNP is enabled, i.e. the RMP is active, allocate a full page to + * prevent concurrent accesses to the page. As required by firmware, + * the PSP driver updates the RMP to temporarily transfer ownership of + * the page to Firmware while the {DE,EN}CRYPT operation is in-progress, + * and so concurrent software accesses to the page will encounter + * seemingly spurious RMP #PF violations + */ + if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return (void *)__get_free_page(GFP_KERNEL); + return kmalloc(*nr_bytes, GFP_KERNEL); } +static void sev_dbg_crypt_slow_free(void *buf) +{ + if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + free_page((unsigned long)buf); + else + kfree(buf); +} + static int sev_dbg_decrypt_slow(struct kvm *kvm, unsigned long src, struct page *src_p, unsigned long dst, unsigned int len, int *err) @@ -1307,7 +1326,7 @@ static int sev_dbg_decrypt_slow(struct kvm *kvm, unsigned long src, if (copy_to_user((void __user *)dst, buf + (src & 15), len)) r = -EFAULT; out: - kfree(buf); + sev_dbg_crypt_slow_free(buf); return r; } @@ -1340,7 +1359,7 @@ static int sev_dbg_encrypt_slow(struct kvm *kvm, unsigned long src, r = sev_issue_dbg_cmd(kvm, __sme_set(__pa(buf)), dst_pa, nr_bytes, KVM_SEV_DBG_ENCRYPT, err); out: - kfree(buf); + sev_dbg_crypt_slow_free(buf); return r; } @@ -2029,6 +2048,12 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) src->pages_locked = 0; src->es_active = false; + /* + * Do cache maintenance on the source VM as it is no longer an SEV VM, + * i.e. memory reclaim flows won't trigger cache maintenance on the VM. + */ + sev_writeback_caches(src_kvm); + list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); mutex_lock(&sev_mirror_lock); @@ -2168,6 +2193,10 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) * the set of CPUs from the source. If a CPU was used to run a vCPU in * the source VM but is never used for the destination VM, then the CPU * can only have cached memory that was accessible to the source VM. + * Furthermore, KVM *must* perform cache maintenance on the source VM, + * as the source VM may have access to memory that the destination VM + * does not, i.e. KVM could skip flushes if memory is reclaimed from + * the old VM but not the new VM. */ if (!zalloc_cpumask_var(&dst_sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) { ret = -ENOMEM; @@ -2757,8 +2786,12 @@ int sev_mem_enc_register_region(struct kvm *kvm, if (!region) return -ENOMEM; + /* + * Do NOT specify FOLL_WRITE, as KVM isn't using the pinned pages to + * write memory, and FOLL_LONGTERM itself triggers CoW unshare. + */ region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, - FOLL_WRITE | FOLL_LONGTERM); + FOLL_LONGTERM); if (IS_ERR(region->pages)) { ret = PTR_ERR(region->pages); goto e_free; @@ -2958,13 +2991,17 @@ void sev_vm_destroy(struct kvm *kvm) struct list_head *head = &sev->regions_list; struct list_head *pos, *q; + /* + * Free the mask even if the VM is not *currently* an SEV VM, as it may + * have been an SEV VM prior to intra-host migration. + */ + free_cpumask_var(sev->have_run_cpus); + if (!sev_guest(kvm)) return; WARN_ON(!list_empty(&sev->mirror_vms)); - free_cpumask_var(sev->have_run_cpus); - /* * If this is a mirror VM, remove it from the owner's list of a mirrors * and skip ASID cleanup (the ASID is tied to the lifetime of the owner). @@ -3998,30 +4035,25 @@ static int snp_begin_psc(struct vcpu_svm *svm) return snp_do_psc(svm); } -/* - * Invoked as part of svm_vcpu_reset() processing of an init event. - */ -static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) +static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_memory_slot *slot; + struct kvm *kvm = vcpu->kvm; + gfn_t gfn = gpa_to_gfn(gpa); + unsigned long mmu_seq; struct page *page; kvm_pfn_t pfn; - gfn_t gfn; - - guard(mutex)(&svm->sev_es.snp_vmsa_mutex); - if (!svm->sev_es.snp_ap_waiting_for_reset) - return; + lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); - svm->sev_es.snp_ap_waiting_for_reset = false; - - /* Mark the vCPU as offline and not runnable */ - vcpu->arch.pv.pv_unhalted = false; - kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); - - /* Clear use of the VMSA */ + /* + * Clear use of the VMSA. Ensure snp_guest_vmsa_gpa is written exactly + * once, as it is read locklessly when responding to gfn invalidations. + * Pairs with the READ_ONCE() in sev_gmem_invalidate_range(). + */ svm->vmcb->control.vmsa_pa = INVALID_PAGE; + WRITE_ONCE(svm->sev_es.snp_guest_vmsa_gpa, INVALID_PAGE); /* * When replacing the VMSA during SEV-SNP AP creation, @@ -4029,16 +4061,16 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) */ vmcb_mark_all_dirty(svm->vmcb); - if (!VALID_PAGE(svm->sev_es.snp_vmsa_gpa)) + if (!VALID_PAGE(gpa)) return; - gfn = gpa_to_gfn(svm->sev_es.snp_vmsa_gpa); - svm->sev_es.snp_vmsa_gpa = INVALID_PAGE; - slot = gfn_to_memslot(vcpu->kvm, gfn); if (!slot) return; + mmu_seq = kvm->mmu_invalidate_seq; + smp_rmb(); + /* * The new VMSA will be private memory guest memory, so retrieve the * PFN from the gmem backend. @@ -4057,18 +4089,64 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) */ svm->sev_es.snp_has_guest_vmsa = true; - /* Use the new VMSA */ - svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); + read_lock(&kvm->mmu_lock); + /* + * Save the guest-provided GPA. If retry is needed, then KVM will try + * again with the same GPA. If the VMSA is usable, then KVM needs to + * track the GPA so that the VMSA can be reloaded if the backing page + * for the GPA is invalidated. + */ + svm->sev_es.snp_guest_vmsa_gpa = gpa; + if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) + kvm_make_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu); + else + svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); + read_unlock(&kvm->mmu_lock); - /* Mark the vCPU as runnable */ - kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); + kvm_release_page_clean(page); +} + +/* + * Invoked as part of svm_vcpu_reset() processing of an init event. + */ +static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + gpa_t gpa; + + guard(mutex)(&svm->sev_es.snp_vmsa_mutex); + + if (!svm->sev_es.snp_ap_waiting_for_reset) + return; + + svm->sev_es.snp_ap_waiting_for_reset = false; + + /* Mark the vCPU as offline and not runnable */ + vcpu->arch.pv.pv_unhalted = false; + kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); + + gpa = svm->sev_es.snp_pending_vmsa_gpa; + svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; + + __sev_snp_reload_vmsa(vcpu, gpa); /* - * gmem pages aren't currently migratable, but if this ever changes - * then care should be taken to ensure svm->sev_es.vmsa is pinned - * through some other means. + * Mark the vCPU as runnable for CREATE requests, indicated by a valid + * VMSA GPA, even if installing the VMSA failed, so that KVM_RUN will + * fail instead of blocking indefinitely and hanging the vCPU, e.g. if + * the backing guest_memfd page is unavailable. */ - kvm_release_page_clean(page); + if (VALID_PAGE(gpa)) + kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); +} + +void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu) +{ + struct vcpu_sev_es_state *sev_es = &to_svm(vcpu)->sev_es; + + guard(mutex)(&sev_es->snp_vmsa_mutex); + + __sev_snp_reload_vmsa(vcpu, sev_es->snp_guest_vmsa_gpa); } static int sev_snp_ap_creation(struct vcpu_svm *svm) @@ -4124,10 +4202,10 @@ static int sev_snp_ap_creation(struct vcpu_svm *svm) return -EINVAL; } - target_svm->sev_es.snp_vmsa_gpa = svm->vmcb->control.exit_info_2; + target_svm->sev_es.snp_pending_vmsa_gpa = svm->vmcb->control.exit_info_2; break; case SVM_VMGEXIT_AP_DESTROY: - target_svm->sev_es.snp_vmsa_gpa = INVALID_PAGE; + target_svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; break; default: vcpu_unimpl(vcpu, "vmgexit: invalid AP creation request [%#x] from guest\n", @@ -4810,6 +4888,8 @@ int sev_vcpu_create(struct kvm_vcpu *vcpu) return -ENOMEM; svm->sev_es.vmsa = page_address(vmsa_page); + svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; + svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; vcpu->arch.guest_tsc_protected = snp_is_secure_tsc_enabled(vcpu->kvm); @@ -5198,6 +5278,41 @@ void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) } } +void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + struct kvm_vcpu *vcpu; + unsigned long i; + + lockdep_assert_held_write(&kvm->mmu_lock); + + /* + * An unstable result for "is SNP" is a-ok here, thanks to mmu_lock. + * The vCPU's VMSA GPA is invalidated before the vCPU is made visible + * to other tasks, and can only become valid while holding mmu_lock, + * after the VM is fully committed to being an SNP VM. + */ + if (!____sev_snp_guest(kvm)) + return; + + kvm_for_each_vcpu(i, vcpu, kvm) { + /* + * Read snp_guest_vmsa_gpa without taking the vCPU's VMSA mutex + * (or its generic mutex) as mmu_lock is held, i.e. this task + * can't sleep. The VMSA is invalidated outside of mmu_lock, + * but can only become valid inside of mmu_lock, i.e. the below + * can get false positives, but not false negatives. A false + * positive is benign, as a spurious request simply forces the + * vCPU to re-establish its VMSA. + */ + gpa_t gpa = READ_ONCE(to_svm(vcpu)->sev_es.snp_guest_vmsa_gpa); + + if (VALID_PAGE(gpa) && + gpa_to_gfn(gpa) >= range->start && + gpa_to_gfn(gpa) < range->end) + kvm_make_request_and_kick(KVM_REQ_VMSA_PAGE_RELOAD, vcpu); + } +} + int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) { int level, rc; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 60495447f0ca0..d1423bab1a3aa 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5463,9 +5463,15 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .mem_enc_register_region = sev_mem_enc_register_region, .mem_enc_unregister_region = sev_mem_enc_unregister_region, .guest_memory_reclaimed = sev_guest_memory_reclaimed, + .reload_vmsa = sev_snp_reload_vmsa, .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, .vm_move_enc_context_from = sev_vm_move_enc_context_from, + + .gmem_prepare = sev_gmem_prepare, + .gmem_invalidate = sev_gmem_invalidate, + .gmem_invalidate_range = sev_gmem_invalidate_range, + .gmem_max_mapping_level = sev_gmem_max_mapping_level, #endif .check_emulate_instruction = svm_check_emulate_instruction, @@ -5477,10 +5483,6 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, .vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons, .alloc_apic_backing_page = svm_alloc_apic_backing_page, - - .gmem_prepare = sev_gmem_prepare, - .gmem_invalidate = sev_gmem_invalidate, - .gmem_max_mapping_level = sev_gmem_max_mapping_level, }; /* @@ -5656,14 +5658,11 @@ static __init int svm_hardware_setup(void) if (nested) { pr_info("Nested Virtualization enabled\n"); - kvm_enable_efer_bits(EFER_SVME); - if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) - kvm_enable_efer_bits(EFER_LMSLE); - r = nested_svm_init_msrpm_merge_offsets(); if (r) return r; } + svm_nested_ops.enabled = nested; /* * KVM's MMU doesn't support using 2-level paging for itself, and thus diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index d63e5878988aa..cd30740e99849 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -272,7 +272,8 @@ struct vcpu_sev_es_state { u64 ghcb_registered_gpa; struct mutex snp_vmsa_mutex; /* Used to handle concurrent updates of VMSA. */ - gpa_t snp_vmsa_gpa; + gpa_t snp_pending_vmsa_gpa; + gpa_t snp_guest_vmsa_gpa; bool snp_ap_waiting_for_reset; bool snp_has_guest_vmsa; }; @@ -996,6 +997,7 @@ static inline struct page *snp_safe_alloc_page(void) { return snp_safe_alloc_page_node(numa_node_id(), GFP_KERNEL_ACCOUNT); } +void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu); int sev_vcpu_create(struct kvm_vcpu *vcpu); void sev_free_vcpu(struct kvm_vcpu *vcpu); @@ -1010,6 +1012,7 @@ extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu); void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_save_area *vmsa); @@ -1035,16 +1038,6 @@ static inline int sev_cpu_init(struct svm_cpu_data *sd) { return 0; } static inline int sev_dev_get_attr(u32 group, u64 attr, u64 *val) { return -ENXIO; } #define max_sev_asid 0 static inline void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) {} -static inline int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) -{ - return 0; -} -static inline void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) {} -static inline int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) -{ - return 0; -} - static inline struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu) { return NULL; diff --git a/arch/x86/kvm/tss.h b/arch/x86/kvm/tss.h index 3f9150125e701..117bf8bec07dc 100644 --- a/arch/x86/kvm/tss.h +++ b/arch/x86/kvm/tss.h @@ -57,4 +57,11 @@ struct tss_segment_16 { u16 ldt; }; +#define TSS_IOPB_BASE_OFFSET 0x66 +#define TSS_BASE_SIZE 0x68 +#define TSS_IOPB_SIZE (65536 / 8) +#define TSS_REDIRECTION_SIZE (256 / 8) +#define RMODE_TSS_SIZE \ + (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) + #endif diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index ddf6df7bee93b..cec7eb646a1e3 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -11,6 +11,7 @@ #include "x86.h" #include "cpuid.h" #include "hyperv.h" +#include "irq.h" #include "mmu.h" #include "nested.h" #include "pmu.h" @@ -1326,6 +1327,9 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, * is the VPID incorporated into the MMU context. I.e. KVM must assume * that the new vpid12 has never been used and thus represents a new * guest ASID that cannot have entries in the TLB. + * + * Note, last_vpid is initialized as 0, so the first nested VM-Enter + * after VMXON will always flush the TLB to avoid using stale entries. */ if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { vmx->nested.last_vpid = vmcs12->virtual_processor_id; @@ -3760,6 +3764,14 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, vmentry_fail_vmexit_guest_mode: if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) vcpu->arch.tsc_offset -= vmcs12->tsc_offset; + + /* + * Handle any TLB flush requests that were queued for L2 if KVM made it + * far enough along to switch to L2 context. Note, loading host state + * will generate any flushes for L1 required by VM-Exit. + */ + kvm_service_local_tlb_flush_requests(vcpu); + leave_guest_mode(vcpu); vmentry_fail_vmexit: @@ -5073,8 +5085,9 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, /* trying to cancel vmlaunch/vmresume is a bug */ kvm_warn_on_nested_run_pending(vcpu); -#ifdef CONFIG_KVM_HYPERV + /* Note, "checking" the request also clears the request. */ if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { +#ifdef CONFIG_KVM_HYPERV /* * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map * Enlightened VMCS after migration and we still need to @@ -5082,8 +5095,8 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, * the first L2 run. */ (void)nested_get_evmcs_page(vcpu); - } #endif + } /* Service pending TLB flush requests for L2 before switching to L1. */ kvm_service_local_tlb_flush_requests(vcpu); @@ -5432,6 +5445,13 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu) vmx->nested.vpid02 = allocate_vpid(); + /* + * Clear last_vpid to ensure that the VPID is flushed on the first + * nested VM-Enter. Otherwise, stale TLB entries from a previous life of + * the VPID (e.g. different vCPU or even different VM) could be used. + */ + vmx->nested.last_vpid = 0; + vmx->nested.vmcs02_initialized = false; vmx->nested.vmxon = true; @@ -6059,8 +6079,8 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) u64 vpid; u64 gla; } operand; - u16 vpid02; int r, gpr_index; + int cpu; if (!(vmx->nested.msrs.secondary_ctls_high & SECONDARY_EXEC_ENABLE_VPID) || @@ -6094,42 +6114,34 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) return kvm_handle_memory_failure(vcpu, r, &e); if (operand.vpid >> 16) - return nested_vmx_fail(vcpu, - VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + + if (type != VMX_VPID_EXTENT_ALL_CONTEXT && !operand.vpid) + return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + + /* LAM doesn't apply to addresses that are inputs to TLB invalidation. */ + if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR && + is_noncanonical_invlpg_address(operand.gla, vcpu)) + return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); /* * Always flush the effective vpid02, i.e. never flush the current VPID * and never explicitly flush vpid01. INVVPID targets a VPID, not a * VMCS, and so whether or not the current vmcs12 has VPID enabled is * irrelevant (and there may not be a loaded vmcs12). + * + * If vmcs02 was last loaded on a different pCPU, then defer the flush + * by invalidating the nested VPID tracking to ensure that KVM performs + * the invalidation on the correct pCPU. */ - vpid02 = nested_get_vpid02(vcpu); - switch (type) { - case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: - /* - * LAM doesn't apply to addresses that are inputs to TLB - * invalidation. - */ - if (!operand.vpid || - is_noncanonical_invlpg_address(operand.gla, vcpu)) - return nested_vmx_fail(vcpu, - VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - vpid_sync_vcpu_addr(vpid02, operand.gla); - break; - case VMX_VPID_EXTENT_SINGLE_CONTEXT: - case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: - if (!operand.vpid) - return nested_vmx_fail(vcpu, - VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - vpid_sync_context(vpid02); - break; - case VMX_VPID_EXTENT_ALL_CONTEXT: - vpid_sync_context(vpid02); - break; - default: - WARN_ON_ONCE(1); - return kvm_skip_emulated_instruction(vcpu); - } + cpu = get_cpu(); + if (cpu != vmx->nested.vmcs02.cpu) + vmx->nested.last_vpid = 0; + else if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR) + vpid_sync_vcpu_addr(nested_get_vpid02(vcpu), operand.gla); + else + vpid_sync_context(nested_get_vpid02(vcpu)); + put_cpu(); /* * Sync the shadow page tables if EPT is disabled, L1 is invalidating diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 1944939c4139f..e65ba7e856117 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -187,6 +187,9 @@ static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) int ret; switch (msr) { + case MSR_CORE_PERF_GLOBAL_STATUS: + case MSR_CORE_PERF_GLOBAL_CTRL: + case MSR_CORE_PERF_GLOBAL_OVF_CTRL: case MSR_CORE_PERF_FIXED_CTR_CTRL: return kvm_pmu_has_perf_global_ctrl(pmu); case MSR_IA32_PEBS_ENABLE: diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index b8d745f6fd221..61243d16febf0 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -72,6 +72,7 @@ #include "x86.h" #include "x86_ops.h" #include "smm.h" +#include "tss.h" #include "vmx_onhyperv.h" #include "vmenter.h" #include "posted_intr.h" @@ -8770,6 +8771,7 @@ __init int vmx_hardware_setup(void) if (r) return r; } + vmx_nested_ops.enabled = nested; kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 47cb9eba113b1..61db05ff023cd 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -129,13 +129,9 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) static void process_nmi(struct kvm_vcpu *vcpu); -static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); static void store_regs(struct kvm_vcpu *vcpu); static int sync_regs(struct kvm_vcpu *vcpu); -static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); -static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); - static DEFINE_MUTEX(vendor_module_lock); static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); @@ -152,6 +148,13 @@ EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits); EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg); EXPORT_STATIC_CALL_GPL(kvm_x86_get_cpl); +#define KVM_X86_NESTED_OP(func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_nested_##func, \ + *(((struct kvm_x86_nested_ops *)0)->func)); +#define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP +#define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP +#include + static bool __read_mostly ignore_msrs = 0; module_param(ignore_msrs, bool, 0644); @@ -837,7 +840,7 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned int nr, * wants to intercept the exception. */ if (is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) { + kvm_nested_call(is_exception_vmexit)(vcpu, nr, error_code)) { kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code, has_payload, payload); return; @@ -1017,170 +1020,6 @@ bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr); -static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); -} - -/* - * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise. - */ -int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - struct kvm_mmu *mmu = vcpu->arch.walk_mmu; - gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; - gpa_t real_gpa; - int i; - int ret; - u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; - - /* - * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated - * to an L1 GPA. - */ - real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn), - PFERR_USER_MASK | PFERR_WRITE_MASK | - PFERR_GUEST_PAGE_MASK, NULL, 0); - if (real_gpa == INVALID_GPA) - return 0; - - /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */ - ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte, - cr3 & GENMASK(11, 5), sizeof(pdpte)); - if (ret < 0) - return 0; - - for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { - if ((pdpte[i] & PT_PRESENT_MASK) && - (pdpte[i] & pdptr_rsvd_bits(vcpu))) { - return 0; - } - } - - /* - * Marking VCPU_REG_PDPTR dirty doesn't work for !tdp_enabled. - * Shadow page roots need to be reconstructed instead. - */ - if (!tdp_enabled && memcmp(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs))) - kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT); - - memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)); - kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR); - kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); - vcpu->arch.pdptrs_from_userspace = false; - - return 1; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(load_pdptrs); - -static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ -#ifdef CONFIG_X86_64 - if (cr0 & 0xffffffff00000000UL) - return false; -#endif - - if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) - return false; - - if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) - return false; - - return kvm_x86_call(is_valid_cr0)(vcpu, cr0); -} - -void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0) -{ - /* - * CR0.WP is incorporated into the MMU role, but only for non-nested, - * indirect shadow MMUs. If paging is disabled, no updates are needed - * as there are no permission bits to emulate. If TDP is enabled, the - * MMU's metadata needs to be updated, e.g. so that emulating guest - * translations does the right thing, but there's no need to unload the - * root as CR0.WP doesn't affect SPTEs. - */ - if ((cr0 ^ old_cr0) == X86_CR0_WP) { - if (!(cr0 & X86_CR0_PG)) - return; - - if (tdp_enabled) { - kvm_init_mmu(vcpu); - return; - } - } - - if ((cr0 ^ old_cr0) & X86_CR0_PG) { - /* - * Clearing CR0.PG is defined to flush the TLB from the guest's - * perspective. - */ - if (!(cr0 & X86_CR0_PG)) - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - /* - * Check for async #PF completion events when enabling paging, - * as the vCPU may have previously encountered async #PFs (it's - * entirely legal for the guest to toggle paging on/off without - * waiting for the async #PF queue to drain). - */ - else if (kvm_pv_async_pf_enabled(vcpu)) - kvm_make_request(KVM_REQ_APF_READY, vcpu); - } - - if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS) - kvm_mmu_reset_context(vcpu); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr0); - -int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ - unsigned long old_cr0 = kvm_read_cr0(vcpu); - - if (!kvm_is_valid_cr0(vcpu, cr0)) - return 1; - - cr0 |= X86_CR0_ET; - - /* Write to CR0 reserved bits are ignored, even on Intel. */ - cr0 &= ~CR0_RESERVED_BITS; - -#ifdef CONFIG_X86_64 - if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) && - (cr0 & X86_CR0_PG)) { - int cs_db, cs_l; - - if (!is_pae(vcpu)) - return 1; - kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); - if (cs_l) - return 1; - } -#endif - if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) && - is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) && - !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) - return 1; - - if (!(cr0 & X86_CR0_PG) && - (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))) - return 1; - - if (!(cr0 & X86_CR0_WP) && kvm_is_cr4_bit_set(vcpu, X86_CR4_CET)) - return 1; - - kvm_x86_call(set_cr0)(vcpu, cr0); - - kvm_post_set_cr0(vcpu, old_cr0, cr0); - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr0); - -void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) -{ - (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw); - static void kvm_load_xfeatures(struct kvm_vcpu *vcpu, bool load_guest) { if (vcpu->arch.guest_state_protected) @@ -1290,89 +1129,7 @@ int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_xsetbv); -static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - return __kvm_is_valid_cr4(vcpu, cr4) && - kvm_x86_call(is_valid_cr4)(vcpu, cr4); -} - -void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) -{ - if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) - kvm_mmu_reset_context(vcpu); - - /* - * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB - * according to the SDM; however, stale prev_roots could be reused - * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we - * free them all. This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST - * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed, - * so fall through. - */ - if (!tdp_enabled && - (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) - kvm_mmu_unload(vcpu); - - /* - * The TLB has to be flushed for all PCIDs if any of the following - * (architecturally required) changes happen: - * - CR4.PCIDE is changed from 1 to 0 - * - CR4.PGE is toggled - * - * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT. - */ - if (((cr4 ^ old_cr4) & X86_CR4_PGE) || - (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - - /* - * The TLB has to be flushed for the current PCID if any of the - * following (architecturally required) changes happen: - * - CR4.SMEP is changed from 0 to 1 - * - CR4.PAE is toggled - */ - else if (((cr4 ^ old_cr4) & X86_CR4_PAE) || - ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP))) - kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); - -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr4); - -int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - unsigned long old_cr4 = kvm_read_cr4(vcpu); - - if (!kvm_is_valid_cr4(vcpu, cr4)) - return 1; - - if (is_long_mode(vcpu)) { - if (!(cr4 & X86_CR4_PAE)) - return 1; - if ((cr4 ^ old_cr4) & X86_CR4_LA57) - return 1; - } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) - && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS) - && !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) - return 1; - - if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) { - /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ - if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) - return 1; - } - - if ((cr4 & X86_CR4_CET) && !kvm_is_cr0_bit_set(vcpu, X86_CR0_WP)) - return 1; - - kvm_x86_call(set_cr4)(vcpu, cr4); - - kvm_post_set_cr4(vcpu, old_cr4, cr4); - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); - -static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) +void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) { struct kvm_mmu *mmu = vcpu->arch.mmu; unsigned long roots_to_free = 0; @@ -1415,167 +1172,6 @@ static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); } -int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - bool skip_tlb_flush = false; - unsigned long pcid = 0; -#ifdef CONFIG_X86_64 - if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) { - skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH; - cr3 &= ~X86_CR3_PCID_NOFLUSH; - pcid = cr3 & X86_CR3_PCID_MASK; - } -#endif - - /* PDPTRs are always reloaded for PAE paging. */ - if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu)) - goto handle_tlb_flush; - - /* - * Do not condition the GPA check on long mode, this helper is used to - * stuff CR3, e.g. for RSM emulation, and there is no guarantee that - * the current vCPU mode is accurate. - */ - if (!kvm_vcpu_is_legal_cr3(vcpu, cr3)) - return 1; - - if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) - return 1; - - if (cr3 != kvm_read_cr3(vcpu)) - kvm_mmu_new_pgd(vcpu, cr3); - - vcpu->arch.cr3 = cr3; - kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); - /* Do not call post_set_cr3, we do not get here for confidential guests. */ - -handle_tlb_flush: - /* - * A load of CR3 that flushes the TLB flushes only the current PCID, - * even if PCID is disabled, in which case PCID=0 is flushed. It's a - * moot point in the end because _disabling_ PCID will flush all PCIDs, - * and it's impossible to use a non-zero PCID when PCID is disabled, - * i.e. only PCID=0 can be relevant. - */ - if (!skip_tlb_flush) - kvm_invalidate_pcid(vcpu, pcid); - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr3); - -int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) -{ - if (cr8 & CR8_RESERVED_BITS) - return 1; - if (lapic_in_kernel(vcpu)) - kvm_lapic_set_tpr(vcpu, cr8); - else - vcpu->arch.cr8 = cr8; - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr8); - -unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) -{ - if (lapic_in_kernel(vcpu)) - return kvm_lapic_get_cr8(vcpu); - else - return vcpu->arch.cr8; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_cr8); - -static void kvm_update_dr0123(struct kvm_vcpu *vcpu) -{ - int i; - - if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { - for (i = 0; i < KVM_NR_DB_REGS; i++) - vcpu->arch.eff_db[i] = vcpu->arch.db[i]; - } -} - -void kvm_update_dr7(struct kvm_vcpu *vcpu) -{ - unsigned long dr7; - - if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) - dr7 = vcpu->arch.guest_debug_dr7; - else - dr7 = vcpu->arch.dr7; - kvm_x86_call(set_dr7)(vcpu, dr7); - vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; - if (dr7 & DR7_BP_EN_MASK) - vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); - -static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) -{ - u64 fixed = DR6_FIXED_1; - - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) - fixed |= DR6_RTM; - - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) - fixed |= DR6_BUS_LOCK; - return fixed; -} - -int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) -{ - size_t size = ARRAY_SIZE(vcpu->arch.db); - - switch (dr) { - case 0 ... 3: - vcpu->arch.db[array_index_nospec(dr, size)] = val; - if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) - vcpu->arch.eff_db[dr] = val; - break; - case 4: - case 6: - if (!kvm_dr6_valid(val)) - return 1; /* #GP */ - vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); - break; - case 5: - default: /* 7 */ - if (!kvm_dr7_valid(val)) - return 1; /* #GP */ - vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; - kvm_update_dr7(vcpu); - break; - } - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_dr); - -unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr) -{ - size_t size = ARRAY_SIZE(vcpu->arch.db); - - switch (dr) { - case 0 ... 3: - return vcpu->arch.db[array_index_nospec(dr, size)]; - case 4: - case 6: - return vcpu->arch.dr6; - case 5: - default: /* 7 */ - return vcpu->arch.dr7; - } -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr); - -static unsigned long kvm_get_effective_dr7(struct kvm_vcpu *vcpu) -{ - if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) - return vcpu->arch.guest_debug_dr7; - - return vcpu->arch.dr7; -} - int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) { u32 pmc = kvm_ecx_read(vcpu); @@ -4928,15 +4524,16 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST; break; case KVM_CAP_NESTED_STATE: - r = kvm_x86_ops.nested_ops->get_state ? - kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0; + r = kvm_x86_ops.nested_ops->enabled ? + kvm_nested_call(get_state)(NULL, NULL, 0) : 0; break; #ifdef CONFIG_KVM_HYPERV case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: r = kvm_x86_ops.enable_l2_tlb_flush != NULL; break; case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: - r = kvm_x86_ops.nested_ops->enable_evmcs != NULL; + r = kvm_x86_ops.nested_ops->enabled && + kvm_x86_ops.nested_ops->enable_evmcs != NULL; break; #endif case KVM_CAP_SMALLER_MAXPHYADDR: @@ -5532,7 +5129,7 @@ static struct kvm_queued_exception *kvm_get_exception_to_save(struct kvm_vcpu *v return &vcpu->arch.exception; } -static void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu) +void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu) { struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu); @@ -5736,57 +5333,6 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, return 0; } -static int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, - struct kvm_debugregs *dbgregs) -{ - unsigned int i; - - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - kvm_handle_exception_payload_quirk(vcpu); - - memset(dbgregs, 0, sizeof(*dbgregs)); - - BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db)); - for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) - dbgregs->db[i] = vcpu->arch.db[i]; - - dbgregs->dr6 = vcpu->arch.dr6; - dbgregs->dr7 = vcpu->arch.dr7; - return 0; -} - -static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, - struct kvm_debugregs *dbgregs) -{ - unsigned int i; - - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - if (dbgregs->flags) - return -EINVAL; - - if (!kvm_dr6_valid(dbgregs->dr6)) - return -EINVAL; - if (!kvm_dr7_valid(dbgregs->dr7)) - return -EINVAL; - - for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) - vcpu->arch.db[i] = dbgregs->db[i]; - - kvm_update_dr0123(vcpu); - vcpu->arch.dr6 = dbgregs->dr6; - vcpu->arch.dr7 = dbgregs->dr7; - kvm_update_dr7(vcpu); - - return 0; -} - - static int kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu, u8 *state, unsigned int size) { @@ -6029,9 +5575,10 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, uint16_t vmcs_version; void __user *user_ptr; - if (!kvm_x86_ops.nested_ops->enable_evmcs) + if (!kvm_x86_ops.nested_ops->enabled || + !kvm_x86_ops.nested_ops->enable_evmcs) return -ENOTTY; - r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version); + r = kvm_nested_call(enable_evmcs)(vcpu, &vmcs_version); if (!r) { user_ptr = (void __user *)(uintptr_t)cap->args[0]; if (copy_to_user(user_ptr, &vmcs_version, @@ -6529,7 +6076,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, u32 user_data_size; r = -EINVAL; - if (!kvm_x86_ops.nested_ops->get_state) + if (!kvm_x86_ops.nested_ops->enabled) break; BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size)); @@ -6537,8 +6084,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, if (get_user(user_data_size, &user_kvm_nested_state->size)) break; - r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state, - user_data_size); + r = kvm_nested_call(get_state)(vcpu, user_kvm_nested_state, user_data_size); if (r < 0) break; @@ -6559,7 +6105,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, int idx; r = -EINVAL; - if (!kvm_x86_ops.nested_ops->set_state) + if (!kvm_x86_ops.nested_ops->enabled) break; r = -EFAULT; @@ -6582,7 +6128,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, break; idx = srcu_read_lock(&vcpu->kvm->srcu); - r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state); + r = kvm_nested_call(set_state)(vcpu, user_kvm_nested_state, &kvm_state); srcu_read_unlock(&vcpu->kvm->srcu, idx); break; } @@ -6623,7 +6169,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = -ENOMEM; if (!u.sregs2) goto out; - __get_sregs2(vcpu, u.sregs2); + kvm_vcpu_ioctl_x86_get_sregs2(vcpu, u.sregs2); r = -EFAULT; if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2))) goto out; @@ -6642,7 +6188,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, u.sregs2 = NULL; goto out; } - r = __set_sregs2(vcpu, u.sregs2); + r = kvm_vcpu_ioctl_x86_set_sregs2(vcpu, u.sregs2); break; } case KVM_HAS_DEVICE_ATTR: @@ -6736,7 +6282,10 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, break; fallthrough; case KVM_CAP_DISABLE_QUIRKS: - kvm->arch.disabled_quirks |= cap->args[0] & kvm_caps.supported_quirks; + mutex_lock(&kvm->lock); + WRITE_ONCE(kvm->arch.disabled_quirks, + kvm->arch.disabled_quirks | (cap->args[0] & kvm_caps.supported_quirks)); + mutex_unlock(&kvm->lock); r = 0; break; case KVM_CAP_SPLIT_IRQCHIP: { @@ -8492,11 +8041,6 @@ static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count); } -static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) -{ - return kvm_x86_call(get_segment_base)(vcpu, seg); -} - static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address) { kvm_mmu_invlpg(emul_to_vcpu(ctxt), address); @@ -8641,7 +8185,7 @@ static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) static unsigned long emulator_get_cached_segment_base( struct x86_emulate_ctxt *ctxt, int seg) { - return get_segment_base(emul_to_vcpu(ctxt), seg); + return kvm_get_segment_base(emul_to_vcpu(ctxt), seg); } static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector, @@ -10012,10 +9556,18 @@ static void kvm_setup_efer_caps(void) if (kvm_cpu_cap_has(X86_FEATURE_AUTOIBRS)) kvm_enable_efer_bits(EFER_AUTOIBRS); + + if (kvm_cpu_cap_has(X86_FEATURE_SVM)) { + kvm_enable_efer_bits(EFER_SVME); + if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) + kvm_enable_efer_bits(EFER_LMSLE); + } } static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) { + const struct kvm_x86_nested_ops *nested_ops = ops->runtime_ops->nested_ops; + memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); #define __KVM_X86_OP(func) \ @@ -10029,6 +9581,17 @@ static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) #include #undef __KVM_X86_OP +#define __KVM_X86_NESTED_OP(func) \ + static_call_update(kvm_x86_nested_##func, nested_ops->func); +#define KVM_X86_NESTED_OP(func) \ + WARN_ON(!nested_ops->func); __KVM_X86_NESTED_OP(func) +#define KVM_X86_NESTED_OP_OPTIONAL __KVM_X86_NESTED_OP +#define KVM_X86_NESTED_OP_OPTIONAL_RET0(func) \ + static_call_update(kvm_x86_nested_##func, (void *)nested_ops->func ? : \ + (void *)__static_call_return0); +#include +#undef __KVM_X86_NESTED_OP + kvm_pmu_ops_update(ops->pmu_ops); } @@ -10565,11 +10128,11 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu) int kvm_check_nested_events(struct kvm_vcpu *vcpu) { if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { - kvm_x86_ops.nested_ops->triple_fault(vcpu); + kvm_nested_call(triple_fault)(vcpu); return 1; } - return kvm_x86_ops.nested_ops->check_events(vcpu); + return kvm_nested_call(check_events)(vcpu); } static void kvm_inject_exception(struct kvm_vcpu *vcpu) @@ -10807,9 +10370,7 @@ static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu, kvm_x86_call(enable_irq_window)(vcpu); } - if (is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->has_events && - kvm_x86_ops.nested_ops->has_events(vcpu, true)) + if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, true)) *req_immediate_exit = true; /* @@ -11132,7 +10693,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) } if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { - if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) { + if (unlikely(!kvm_nested_call(get_nested_state_pages)(vcpu))) { r = 0; goto out; } @@ -11184,7 +10745,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) } if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { if (is_guest_mode(vcpu)) - kvm_x86_ops.nested_ops->triple_fault(vcpu); + kvm_nested_call(triple_fault)(vcpu); if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; @@ -11278,6 +10839,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) goto out; } } + if (kvm_check_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu)) + kvm_x86_call(reload_vmsa)(vcpu); } if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || @@ -11602,9 +11165,7 @@ bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) if (kvm_hv_has_stimer_pending(vcpu)) return true; - if (is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->has_events && - kvm_x86_ops.nested_ops->has_events(vcpu, false)) + if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, false)) return true; if (kvm_xen_has_pending_events(vcpu)) @@ -12029,8 +11590,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) * a pending VM-Exit if L1 wants to intercept the exception. */ if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector, - ex->error_code)) { + kvm_nested_call(is_exception_vmexit)(vcpu, ex->vector, ex->error_code)) { kvm_queue_exception_vmexit(vcpu, ex->vector, ex->has_error_code, ex->error_code, ex->has_payload, ex->payload); @@ -12073,179 +11633,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) return r; } -static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { - /* - * We are here if userspace calls get_regs() in the middle of - * instruction emulation. Registers state needs to be copied - * back from emulation context to vcpu. Userspace shouldn't do - * that usually, but some bad designed PV devices (vmware - * backdoor interface) need this to work - */ - emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); - vcpu->arch.emulate_regs_need_sync_to_vcpu = false; - } - regs->rax = kvm_rax_read_raw(vcpu); - regs->rbx = kvm_rbx_read_raw(vcpu); - regs->rcx = kvm_rcx_read_raw(vcpu); - regs->rdx = kvm_rdx_read_raw(vcpu); - regs->rsi = kvm_rsi_read_raw(vcpu); - regs->rdi = kvm_rdi_read_raw(vcpu); - regs->rsp = kvm_rsp_read(vcpu); - regs->rbp = kvm_rbp_read_raw(vcpu); -#ifdef CONFIG_X86_64 - regs->r8 = kvm_r8_read_raw(vcpu); - regs->r9 = kvm_r9_read_raw(vcpu); - regs->r10 = kvm_r10_read_raw(vcpu); - regs->r11 = kvm_r11_read_raw(vcpu); - regs->r12 = kvm_r12_read_raw(vcpu); - regs->r13 = kvm_r13_read_raw(vcpu); - regs->r14 = kvm_r14_read_raw(vcpu); - regs->r15 = kvm_r15_read_raw(vcpu); -#endif - - regs->rip = kvm_rip_read(vcpu); - regs->rflags = kvm_get_rflags(vcpu); -} - -int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - __get_regs(vcpu, regs); - vcpu_put(vcpu); - return 0; -} - -static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - vcpu->arch.emulate_regs_need_sync_from_vcpu = true; - vcpu->arch.emulate_regs_need_sync_to_vcpu = false; - - kvm_rax_write_raw(vcpu, regs->rax); - kvm_rbx_write_raw(vcpu, regs->rbx); - kvm_rcx_write_raw(vcpu, regs->rcx); - kvm_rdx_write_raw(vcpu, regs->rdx); - kvm_rsi_write_raw(vcpu, regs->rsi); - kvm_rdi_write_raw(vcpu, regs->rdi); - kvm_rsp_write(vcpu, regs->rsp); - kvm_rbp_write_raw(vcpu, regs->rbp); -#ifdef CONFIG_X86_64 - kvm_r8_write_raw(vcpu, regs->r8); - kvm_r9_write_raw(vcpu, regs->r9); - kvm_r10_write_raw(vcpu, regs->r10); - kvm_r11_write_raw(vcpu, regs->r11); - kvm_r12_write_raw(vcpu, regs->r12); - kvm_r13_write_raw(vcpu, regs->r13); - kvm_r14_write_raw(vcpu, regs->r14); - kvm_r15_write_raw(vcpu, regs->r15); -#endif - - kvm_rip_write(vcpu, regs->rip); - kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED); - - vcpu->arch.exception.pending = false; - vcpu->arch.exception_vmexit.pending = false; - - kvm_make_request(KVM_REQ_EVENT, vcpu); -} - -int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - __set_regs(vcpu, regs); - vcpu_put(vcpu); - return 0; -} - -static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - struct desc_ptr dt; - - if (vcpu->arch.guest_state_protected) - goto skip_protected_regs; - - kvm_handle_exception_payload_quirk(vcpu); - - kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); - kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - kvm_x86_call(get_idt)(vcpu, &dt); - sregs->idt.limit = dt.size; - sregs->idt.base = dt.address; - kvm_x86_call(get_gdt)(vcpu, &dt); - sregs->gdt.limit = dt.size; - sregs->gdt.base = dt.address; - - sregs->cr2 = vcpu->arch.cr2; - sregs->cr3 = kvm_read_cr3(vcpu); - -skip_protected_regs: - sregs->cr0 = kvm_read_cr0(vcpu); - sregs->cr4 = kvm_read_cr4(vcpu); - sregs->cr8 = kvm_get_cr8(vcpu); - sregs->efer = vcpu->arch.efer; - sregs->apic_base = vcpu->arch.apic_base; -} - -static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - __get_sregs_common(vcpu, sregs); - - if (vcpu->arch.guest_state_protected) - return; - - if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) - set_bit(vcpu->arch.interrupt.nr, - (unsigned long *)sregs->interrupt_bitmap); -} - -static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) -{ - int i; - - __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); - - if (vcpu->arch.guest_state_protected) - return; - - if (is_pae_paging(vcpu)) { - kvm_vcpu_srcu_read_lock(vcpu); - for (i = 0 ; i < 4 ; i++) - sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); - sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; - kvm_vcpu_srcu_read_unlock(vcpu); - } -} - -int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - __get_sregs(vcpu, sregs); - vcpu_put(vcpu); - return 0; -} - int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { @@ -12365,173 +11752,6 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_task_switch); -static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) { - /* - * When EFER.LME and CR0.PG are set, the processor is in - * 64-bit mode (though maybe in a 32-bit code segment). - * CR4.PAE and EFER.LMA must be set. - */ - if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA)) - return false; - if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3)) - return false; - } else { - /* - * Not in 64-bit mode: EFER.LMA is clear and the code - * segment cannot be 64-bit. - */ - if (sregs->efer & EFER_LMA || sregs->cs.l) - return false; - } - - return kvm_is_valid_cr4(vcpu, sregs->cr4) && - kvm_is_valid_cr0(vcpu, sregs->cr0); -} - -static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, - int *mmu_reset_needed, bool update_pdptrs) -{ - int idx; - struct desc_ptr dt; - - if (!kvm_is_valid_sregs(vcpu, sregs)) - return -EINVAL; - - if (kvm_apic_set_base(vcpu, sregs->apic_base, true)) - return -EINVAL; - - if (vcpu->arch.guest_state_protected) - return 0; - - dt.size = sregs->idt.limit; - dt.address = sregs->idt.base; - kvm_x86_call(set_idt)(vcpu, &dt); - dt.size = sregs->gdt.limit; - dt.address = sregs->gdt.base; - kvm_x86_call(set_gdt)(vcpu, &dt); - - vcpu->arch.cr2 = sregs->cr2; - *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; - vcpu->arch.cr3 = sregs->cr3; - kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); - kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3); - - *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; - kvm_x86_call(set_efer)(vcpu, sregs->efer); - - *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; - kvm_x86_call(set_cr0)(vcpu, sregs->cr0); - - *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; - kvm_x86_call(set_cr4)(vcpu, sregs->cr4); - - if (update_pdptrs) { - idx = srcu_read_lock(&vcpu->kvm->srcu); - if (is_pae_paging(vcpu)) { - load_pdptrs(vcpu, kvm_read_cr3(vcpu)); - *mmu_reset_needed = 1; - } - srcu_read_unlock(&vcpu->kvm->srcu, idx); - } - - kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); - kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - kvm_set_cr8(vcpu, sregs->cr8); - - /* Older userspace won't unhalt the vcpu on reset. */ - if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && - sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && - !is_protmode(vcpu)) - kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); - - return 0; -} - -static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - int pending_vec, max_bits; - int mmu_reset_needed = 0; - int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); - - if (ret) - return ret; - - if (mmu_reset_needed) { - kvm_mmu_reset_context(vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - } - - max_bits = KVM_NR_INTERRUPTS; - pending_vec = find_first_bit( - (const unsigned long *)sregs->interrupt_bitmap, max_bits); - - if (pending_vec < max_bits) { - kvm_queue_interrupt(vcpu, pending_vec, false); - pr_debug("Set back pending irq %d\n", pending_vec); - kvm_make_request(KVM_REQ_EVENT, vcpu); - } - return 0; -} - -static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) -{ - int mmu_reset_needed = 0; - bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID; - bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) && - !(sregs2->efer & EFER_LMA); - int i, ret; - - if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID) - return -EINVAL; - - if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) - return -EINVAL; - - ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, - &mmu_reset_needed, !valid_pdptrs); - if (ret) - return ret; - - if (valid_pdptrs) { - for (i = 0; i < 4 ; i++) - kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); - - kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR); - mmu_reset_needed = 1; - vcpu->arch.pdptrs_from_userspace = true; - } - if (mmu_reset_needed) { - kvm_mmu_reset_context(vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - } - return 0; -} - -int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - int ret; - - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - ret = __set_sregs(vcpu, sregs); - vcpu_put(vcpu); - return ret; -} - static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm) { bool set = false; @@ -12687,13 +11907,7 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) static void store_regs(struct kvm_vcpu *vcpu) { - BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES); - - if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS) - __get_regs(vcpu, &vcpu->run->s.regs.regs); - - if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS) - __get_sregs(vcpu, &vcpu->run->s.regs.sregs); + kvm_run_sync_regs_to_user(vcpu); if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS) kvm_vcpu_ioctl_x86_get_vcpu_events( @@ -12702,19 +11916,8 @@ static void store_regs(struct kvm_vcpu *vcpu) static int sync_regs(struct kvm_vcpu *vcpu) { - if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) { - __set_regs(vcpu, &vcpu->run->s.regs.regs); - vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS; - } - - if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) { - struct kvm_sregs sregs = vcpu->run->s.regs.sregs; - - if (__set_sregs(vcpu, &sregs)) - return -EINVAL; - - vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS; - } + if (kvm_run_sync_regs_from_user(vcpu)) + return -EINVAL; if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) { struct kvm_vcpu_events events = vcpu->run->s.regs.events; @@ -13822,51 +13025,6 @@ int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu) return kvm_x86_call(interrupt_allowed)(vcpu, false); } -unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu) -{ - /* Can't read the RIP when guest state is protected, just return 0 */ - if (vcpu->arch.guest_state_protected) - return 0; - - if (is_64_bit_mode(vcpu)) - return kvm_rip_read(vcpu); - return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) + - kvm_rip_read(vcpu)); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_linear_rip); - -bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip) -{ - return kvm_get_linear_rip(vcpu) == linear_rip; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_is_linear_rip); - -unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) -{ - unsigned long rflags; - - rflags = kvm_x86_call(get_rflags)(vcpu); - if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) - rflags &= ~X86_EFLAGS_TF; - return rflags; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_rflags); - -static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) -{ - if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && - kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip)) - rflags |= X86_EFLAGS_TF; - kvm_x86_call(set_rflags)(vcpu, rflags); -} - -void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) -{ - __kvm_set_rflags(vcpu, rflags); - kvm_make_request(KVM_REQ_EVENT, vcpu); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_rflags); - static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) { BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU)); @@ -14142,6 +13300,10 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) { kvm_x86_call(gmem_invalidate)(start, end); } +void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + kvm_x86_call(gmem_invalidate_range)(kvm, range); +} #endif #endif diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 9de577ef9c974..9973f905955a7 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -151,7 +151,7 @@ int kvm_check_nested_events(struct kvm_vcpu *vcpu); /* Forcibly leave the nested mode in cases like a vCPU reset */ static inline void kvm_leave_nested(struct kvm_vcpu *vcpu) { - kvm_x86_ops.nested_ops->leave_nested(vcpu); + kvm_nested_call(leave_nested)(vcpu); } /* @@ -367,7 +367,7 @@ static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) { - return !(kvm->arch.disabled_quirks & quirk); + return !(READ_ONCE(kvm->arch.disabled_quirks) & quirk); } static __always_inline void kvm_request_l1tf_flush_l1d(void) @@ -403,6 +403,7 @@ int handle_ud(struct kvm_vcpu *vcpu); void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, struct kvm_queued_exception *ex); +void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu); int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data); int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); @@ -597,6 +598,7 @@ static inline void kvm_machine_check(void) int kvm_spec_ctrl_test_value(u64 value); int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r, struct x86_exception *e); +void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid); int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva); bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type); diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index d023a40a1e034..f8d5cdbb68714 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -49,7 +50,8 @@ struct cpa_data { unsigned int flags; unsigned int force_split : 1, force_static_prot : 1, - force_flush_all : 1; + force_flush_all : 1, + init_mm_read_locked : 1; struct page **pages; }; @@ -62,18 +64,18 @@ enum cpa_warn { static const int cpa_warn_level = CPA_PROTECT; /* - * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings) - * using cpa_lock. So that we don't allow any other cpu, with stale large tlb - * entries change the page attribute in parallel to some other cpu - * splitting a large page entry along with changing the attribute. + * Serialize cpa() using cpa_lock so that we don't allow any other cpu, with + * stale large tlb entries, to change the page attribute in parallel to some + * other cpu splitting a large page entry along with changing the attribute. */ static DEFINE_SPINLOCK(cpa_lock); -#define CPA_FLUSHTLB 1 -#define CPA_ARRAY 2 -#define CPA_PAGES_ARRAY 4 -#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */ -#define CPA_COLLAPSE 16 /* try to collapse large pages */ +#define CPA_FLUSHTLB 0x01 +#define CPA_ARRAY 0x02 +#define CPA_PAGES_ARRAY 0x04 +#define CPA_NO_CHECK_ALIAS 0x08 /* Do not search for aliases */ +#define CPA_COLLAPSE 0x10 /* try to collapse large pages */ +#define CPA_DEBUG_PAGEALLOC 0x20 static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm) { @@ -410,7 +412,7 @@ static void __cpa_flush_tlb(void *data) static int collapse_large_pages(unsigned long addr, struct list_head *pgtables); -static void cpa_collapse_large_pages(struct cpa_data *cpa) +static void __cpa_collapse_large_pages(struct cpa_data *cpa) { unsigned long start, addr, end; struct ptdesc *ptdesc, *tmp; @@ -418,6 +420,8 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) int collapsed = 0; int i; + spin_lock(&cpa_lock); + if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { for (i = 0; i < cpa->numpages; i++) collapsed += collapse_large_pages(__cpa_addr(cpa, i), @@ -431,15 +435,39 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) collapsed += collapse_large_pages(addr, &pgtables); } - if (!collapsed) + if (!collapsed) { + spin_unlock(&cpa_lock); return; + } flush_tlb_all(); list_for_each_entry_safe(ptdesc, tmp, &pgtables, pt_list) { list_del(&ptdesc->pt_list); - pagetable_free(ptdesc); + /* + * Only early alloc'd direct map should not be flagged PG_table + * here and those shouldn't be collapsed. However be abundantly + * cautious and handle the !PG_table case too. + */ + if (PageTable((ptdesc_page(ptdesc)))) + pagetable_dtor_free(ptdesc); + else + pagetable_free(ptdesc); } + + spin_unlock(&cpa_lock); +} + +static void cpa_collapse_large_pages(struct cpa_data *cpa) +{ + /* + * Take the mmap write lock on init_mm to: + * - Avoid a use-after-free if raced by ptdump (which takes its own + * write lock on init_mm). + * - Serialise concurrent CPA walkers. + */ + scoped_guard(mmap_write_lock, &init_mm) + __cpa_collapse_large_pages(cpa); } static void cpa_flush(struct cpa_data *cpa, int cache) @@ -1125,11 +1153,10 @@ static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn, static int __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, - struct ptdesc *ptdesc) + pte_t *pbase) { unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1; - struct page *base = ptdesc_page(ptdesc); - pte_t *pbase = (pte_t *)page_address(base); + struct page *base = virt_to_page(pbase); unsigned int i, level; pgprot_t ref_prot; bool nx, rw; @@ -1233,18 +1260,20 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, static int split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address) { - struct ptdesc *ptdesc; + pte_t *pte; - if (!debug_pagealloc_enabled()) - spin_unlock(&cpa_lock); - ptdesc = pagetable_alloc(GFP_KERNEL, 0); - if (!debug_pagealloc_enabled()) - spin_lock(&cpa_lock); - if (!ptdesc) + spin_unlock(&cpa_lock); + if (cpa->init_mm_read_locked) + mmap_read_unlock(&init_mm); + pte = pte_alloc_one_kernel(&init_mm); + if (cpa->init_mm_read_locked) + mmap_read_lock(&init_mm); + spin_lock(&cpa_lock); + if (!pte) return -ENOMEM; - if (__split_large_page(cpa, kpte, address, ptdesc)) - pagetable_free(ptdesc); + if (__split_large_page(cpa, kpte, address, pte)) + pte_free_kernel(&init_mm, pte); return 0; } @@ -2004,6 +2033,7 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) { unsigned long numpages = cpa->numpages; unsigned long rempages = numpages; + bool lock = true; int ret = 0; /* @@ -2013,6 +2043,29 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) !cpa->force_split) return ret; + /* + * DEBUG_PAGEALLOC is special; it is called from any context the + * page-allocator is, which violates the normal cpa_lock locking + * rules. + * + * However, since it is part of the page-allocator, things are still + * properly serialized by the page-allocator locking and the fact that + * when a page is owned by the page-allocator, it isn't owned by + * anybody else. That is, you *SHOULD NOT* be calling cpa() on memory + * that isn't allocated. + * + * Additionally, DEBUG_PAGEALLOC ensures (per probe_page_size_mask()) + * that the kernel mapping is 4k pages, therefore there are no large + * pages to split/collapse. + * + * Furthermore, the page-allocator strictly manages pages that + * *exist*, avoiding pgd_lock. + * + * Therefore, it is safe to not take cpa_lock. + */ + if (debug_pagealloc_enabled() && (cpa->flags & CPA_DEBUG_PAGEALLOC)) + lock = false; + while (rempages) { /* * Store the remaining nr of pages for the large page @@ -2023,11 +2076,12 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) cpa->numpages = 1; - if (!debug_pagealloc_enabled()) - spin_lock(&cpa_lock); - ret = __change_page_attr(cpa, primary); - if (!debug_pagealloc_enabled()) - spin_unlock(&cpa_lock); + if (lock) { + guard(spinlock)(&cpa_lock); + ret = __change_page_attr(cpa, primary); + } else { + ret = __change_page_attr(cpa, primary); + } if (ret) goto out; @@ -2109,7 +2163,11 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, cpa.curpage = 0; cpa.force_split = force_split; - ret = __change_page_attr_set_clr(&cpa, 1); + /* Avoid race with concurrent CPA collapse. */ + cpa.init_mm_read_locked = true; + scoped_guard(mmap_read_lock, &init_mm) + ret = __change_page_attr_set_clr(&cpa, 1); + cpa.init_mm_read_locked = false; /* * Check whether we really changed something: @@ -2606,7 +2664,7 @@ int set_pages_rw(struct page *page, int numpages) return set_memory_rw(addr, numpages); } -static int __set_pages_p(struct page *page, int numpages) +static int __set_pages_p(struct page *page, int numpages, unsigned int cpa_flags) { unsigned long tempaddr = (unsigned long) page_address(page); struct cpa_data cpa = { .vaddr = &tempaddr, @@ -2614,7 +2672,7 @@ static int __set_pages_p(struct page *page, int numpages) .numpages = numpages, .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW), .mask_clr = __pgprot(0), - .flags = CPA_NO_CHECK_ALIAS }; + .flags = CPA_NO_CHECK_ALIAS | cpa_flags }; /* * No alias checking needed for setting present flag. otherwise, @@ -2625,7 +2683,7 @@ static int __set_pages_p(struct page *page, int numpages) return __change_page_attr_set_clr(&cpa, 1); } -static int __set_pages_np(struct page *page, int numpages) +static int __set_pages_np(struct page *page, int numpages, unsigned int cpa_flags) { unsigned long tempaddr = (unsigned long) page_address(page); struct cpa_data cpa = { .vaddr = &tempaddr, @@ -2633,7 +2691,7 @@ static int __set_pages_np(struct page *page, int numpages) .numpages = numpages, .mask_set = __pgprot(0), .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY), - .flags = CPA_NO_CHECK_ALIAS }; + .flags = CPA_NO_CHECK_ALIAS | cpa_flags }; /* * No alias checking needed for setting not present flag. otherwise, @@ -2646,20 +2704,20 @@ static int __set_pages_np(struct page *page, int numpages) int set_direct_map_invalid_noflush(struct page *page) { - return __set_pages_np(page, 1); + return __set_pages_np(page, 1, 0); } int set_direct_map_default_noflush(struct page *page) { - return __set_pages_p(page, 1); + return __set_pages_p(page, 1, 0); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) { if (valid) - return __set_pages_p(page, nr); + return __set_pages_p(page, nr, 0); - return __set_pages_np(page, nr); + return __set_pages_np(page, nr, 0); } #ifdef CONFIG_DEBUG_PAGEALLOC @@ -2678,15 +2736,23 @@ void __kernel_map_pages(struct page *page, int numpages, int enable) * and hence no memory allocations during large page split. */ if (enable) - __set_pages_p(page, numpages); + __set_pages_p(page, numpages, CPA_DEBUG_PAGEALLOC); else - __set_pages_np(page, numpages); + __set_pages_np(page, numpages, CPA_DEBUG_PAGEALLOC); /* - * We should perform an IPI and flush all tlbs, - * but that can deadlock->flush only current cpu. - * Preemption needs to be disabled around __flush_tlb_all() due to - * CR3 reload in __native_flush_tlb(). + * We should perform an IPI and flush all tlbs, but that can + * deadlock, settle for a local flush. + * + * Not doing a global TLB flush means that remote CPUs will retain + * stale TLB entries. In case of P->NP (on free) this means the remote + * CPUs will not take the faults, making the debug scheme less + * reliable. On the NP->P (on alloc) this means the remote CPUs can + * take a spurious fault. However spurious_kernel_fault() will observe + * *_present() and fix it up. + * + * Preemption needs to be disabled around __flush_tlb_all() due to CR3 + * reload in __native_flush_tlb(). */ preempt_disable(); __flush_tlb_all(); diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index de7515ea1beae..ea74ff1d166cd 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1473,17 +1473,20 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, * * Bit layout of `fixup` (32-bit): * - * +-----------+--------+-----------+---------+----------+ - * | 31 | 30-24 | 23-16 | 15-8 | 7-0 | - * | | | | | | - * | ARENA_ACC | Unused | ARENA_REG | DST_REG | INSN_LEN | - * +-----------+--------+-----------+---------+----------+ + * +-----------+-------------+--------+-----------+---------+----------+ + * | 31 | 30 | 29-24 | 23-16 | 15-8 | 7-0 | + * | | | | | | | + * | ARENA_ACC | ARENA_WRITE | Unused | ARENA_REG | DST_REG | INSN_LEN | + * +-----------+-------------+--------+-----------+---------+----------+ * * - INSN_LEN (8 bits): Length of faulting insn (max x86 insn = 15 bytes (fits in 8 bits)). * - DST_REG (8 bits): Offset of dst_reg from reg2pt_regs[] (max offset = 112 (fits in 8 bits)). - * This is set to DONT_CLEAR if the insn is a store. + * This is set to DONT_CLEAR if the insn does not read into a register. * - ARENA_REG (8 bits): Offset of the register that is used to calculate the * address for load/store when accessing the arena region. + * - ARENA_WRITE (1 bit): This bit is set when the faulting instruction wrote to the arena region. + * It is independent of DST_REG, since a read-modify-write both writes to + * memory and reads the old value into a register. * - ARENA_ACCESS (1 bit): This bit is set when the faulting instruction accessed the arena region. * * Bit layout of `data` (32-bit): @@ -1502,6 +1505,7 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, #define FIXUP_INSN_LEN_MASK GENMASK(7, 0) #define FIXUP_REG_MASK GENMASK(15, 8) #define FIXUP_ARENA_REG_MASK GENMASK(23, 16) +#define FIXUP_ARENA_WRITE BIT(30) #define FIXUP_ARENA_ACCESS BIT(31) #define DATA_ARENA_OFFSET_MASK GENMASK(31, 16) @@ -1510,7 +1514,7 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) u32 reg = FIELD_GET(FIXUP_REG_MASK, x->fixup); u32 insn_len = FIELD_GET(FIXUP_INSN_LEN_MASK, x->fixup); bool is_arena = !!(x->fixup & FIXUP_ARENA_ACCESS); - bool is_write = (reg == DONT_CLEAR); + bool is_write = !!(x->fixup & FIXUP_ARENA_WRITE); unsigned long addr; s16 off; u32 arena_reg; @@ -1887,7 +1891,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_mov(dst_reg, src_reg); #ifdef CONFIG_SMP /* add , gs:[] */ - EMIT2(0x65, add_1mod(0x48, dst_reg)); + EMIT2(0x65, add_2mod(0x48, 0, dst_reg)); EMIT3(0x03, add_2reg(0x04, 0, dst_reg), 0x25); EMIT((u32)(unsigned long)&this_cpu_off, 4); #endif @@ -2304,6 +2308,7 @@ st: insn_off = insn->off; struct exception_table_entry *ex; u8 *_insn = image + proglen + (start_of_ldx - temp); u32 arena_reg, fixup_reg; + bool is_write; s64 delta; if (!bpf_prog->aux->extable) @@ -2331,19 +2336,38 @@ st: insn_off = insn->off; * BPF_PROBE_ATOMIC) before being used for the memory access. Pass * the reg holding the unmodified 32-bit address to * ex_handler_bpf(). + * + * A load-acquire is of BPF_STX class, but reads from src_reg + * into dst_reg like a BPF_LDX does, hence it must not be + * treated as a store here. */ - if (BPF_CLASS(insn->code) == BPF_LDX) { + if (BPF_CLASS(insn->code) == BPF_LDX || + bpf_atomic_is_load_acq(insn)) { arena_reg = reg2pt_regs[src_reg]; fixup_reg = reg2pt_regs[dst_reg]; + is_write = false; } else { + /* + * A store has no destination register to clear, + * except for a read-modify-write with BPF_FETCH, + * which also reads the old value into src_reg, or + * into r0 for a BPF_CMPXCHG. Either way the access + * is still reported as a write. + */ + int load_reg = bpf_atomic_load_reg(insn); + arena_reg = reg2pt_regs[dst_reg]; - fixup_reg = DONT_CLEAR; + fixup_reg = load_reg < 0 ? DONT_CLEAR : + reg2pt_regs[load_reg]; + is_write = true; } ex->fixup = FIELD_PREP(FIXUP_INSN_LEN_MASK, prog - start_of_ldx) | FIELD_PREP(FIXUP_ARENA_REG_MASK, arena_reg) | FIELD_PREP(FIXUP_REG_MASK, fixup_reg); ex->fixup |= FIXUP_ARENA_ACCESS; + if (is_write) + ex->fixup |= FIXUP_ARENA_WRITE; ex->data |= FIELD_PREP(DATA_ARENA_OFFSET_MASK, insn->off); } @@ -3369,11 +3393,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); - /* extra registers for struct arguments */ - for (i = 0; i < m->nr_args; i++) { - if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) - nr_regs += (m->arg_size[i] + 7) / 8 - 1; - } + for (i = 0; i < m->nr_args; i++) + nr_regs += (m->arg_size[i] + 7) / 8 - 1; /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 * are passed through regs, the remains are through stack. diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c index b301c6c8df753..795d81c7a4de4 100644 --- a/arch/x86/pci/fixup.c +++ b/arch/x86/pci/fixup.c @@ -886,6 +886,105 @@ static void quirk_clear_strap_no_soft_reset_dev2_f0(struct pci_dev *dev) } } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x15b8, quirk_clear_strap_no_soft_reset_dev2_f0); + +/* + * Enhanced atomic operations can cause corruption with 64-bit DMA + * on these devices. + */ +#define RX_ENH_ATOMIC_EN BIT(8) + +static const u32 nbio_7_7_pcie_smn_addrs[] = { + 0x111401d0, + 0x111411d0, + 0x111421d0, + 0x111431d0, + 0x111441d0, + 0x112401d0, + 0x112411d0, + 0x112421d0, + 0x112431d0, + 0x112441d0, + 0x112451d0, + 0x113401d0, + 0x114401d0, +}; + +static const u32 nbio_7_11_pcie_smn_addrs[] = { + 0x112401d0, + 0x112411d0, + 0x112421d0, + 0x112431d0, + 0x112441d0, + 0x112451d0, + 0x113401d0, + 0x113411d0, + 0x113421d0, + 0x113431d0, + 0x113441d0, + 0x113451d0, +}; + +static void quirk_amd_nbio_enhanced_atomic(struct pci_dev *host_bridge, + const u32 *smn_addrs, + size_t nr_smn_addrs) +{ + bool changed = false; + size_t i; + u32 data; + int ret; + + for (i = 0; i < nr_smn_addrs; i++) { + ret = amd_smn_read(0, smn_addrs[i], &data); + if (ret) + continue; + if (!(data & RX_ENH_ATOMIC_EN)) + continue; + data = data & ~RX_ENH_ATOMIC_EN; + ret = amd_smn_write(0, smn_addrs[i], data); + if (ret) + continue; + if (changed) + continue; + ret = amd_smn_read(0, smn_addrs[i], &data); + if (ret) + continue; + if (data & RX_ENH_ATOMIC_EN) + continue; + changed = true; + } + + if (changed) + pci_info(host_bridge, "enhanced atomics disabled\n"); +} + +static void quirk_amd_nbio_7_7_disable_enhanced_atomic(struct pci_dev *dev) +{ + quirk_amd_nbio_enhanced_atomic(dev, nbio_7_7_pcie_smn_addrs, + ARRAY_SIZE(nbio_7_7_pcie_smn_addrs)); +} + +static void quirk_amd_nbio_7_11_disable_enhanced_atomic(struct pci_dev *dev) +{ + quirk_amd_nbio_enhanced_atomic(dev, nbio_7_11_pcie_smn_addrs, + ARRAY_SIZE(nbio_7_11_pcie_smn_addrs)); +} + +/* Phoenix, Hawk Point (NBIO 7.7) */ +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x14E8, + quirk_amd_nbio_7_7_disable_enhanced_atomic); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_AMD, 0x14E8, + quirk_amd_nbio_7_7_disable_enhanced_atomic); + +/* Strix, Krackan, Strix Halo (NBIO 7.11) */ +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x1507, + quirk_amd_nbio_7_11_disable_enhanced_atomic); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_AMD, 0x1507, + quirk_amd_nbio_7_11_disable_enhanced_atomic); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x1122, + quirk_amd_nbio_7_11_disable_enhanced_atomic); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_AMD, 0x1122, + quirk_amd_nbio_7_11_disable_enhanced_atomic); + #endif /* diff --git a/block/bio-integrity-auto.c b/block/bio-integrity-auto.c index b1c733ecfd2e8..9456dcffd17a6 100644 --- a/block/bio-integrity-auto.c +++ b/block/bio-integrity-auto.c @@ -44,12 +44,6 @@ static void bio_integrity_verify_fn(struct work_struct *work) bio_endio(bio); } -#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG) -static bool bip_should_check(struct bio_integrity_payload *bip) -{ - return bip->bip_flags & BIP_CHECK_FLAGS; -} - /** * __bio_integrity_endio - Integrity I/O completion function * @bio: Protected bio @@ -66,7 +60,7 @@ bool __bio_integrity_endio(struct bio *bio) container_of(bip, struct bio_integrity_data, bip); if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && - bip_should_check(bip)) { + (bip->bip_flags & BIP_CHECK_FLAGS)) { INIT_WORK(&bid->work, bio_integrity_verify_fn); queue_work(kintegrityd_wq, &bid->work); return false; @@ -99,7 +93,7 @@ void bio_integrity_prep(struct bio *bio, unsigned int action) bio_integrity_setup_default(bio); /* Auto-generate integrity metadata if this is a write */ - if (bio_data_dir(bio) == WRITE && bip_should_check(&bid->bip)) + if (bio_data_dir(bio) == WRITE && (bid->bip.bip_flags & BIP_CHECK_FLAGS)) bio_integrity_generate(bio); else bid->saved_bio_iter = bio->bi_iter; diff --git a/block/bio-integrity-fs.c b/block/bio-integrity-fs.c index 9c5fe5fa8f0da..692403dfa047b 100644 --- a/block/bio-integrity-fs.c +++ b/block/bio-integrity-fs.c @@ -46,7 +46,8 @@ void fs_bio_integrity_free(struct bio *bio) void fs_bio_integrity_generate(struct bio *bio) { - if (fs_bio_integrity_alloc(bio)) + if (fs_bio_integrity_alloc(bio) && + (bio_integrity(bio)->bip_flags & BIP_CHECK_FLAGS)) bio_integrity_generate(bio); } EXPORT_SYMBOL_GPL(fs_bio_integrity_generate); @@ -60,6 +61,9 @@ int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size) .bi_size = size, }; + if (!bip || !(bip->bip_flags & BIP_CHECK_FLAGS)) + return 0; + /* * Reinitialize bip->bip_iter. * diff --git a/block/bio.c b/block/bio.c index 6a2f6fc3413e8..bdcb60cb8d88f 100644 --- a/block/bio.c +++ b/block/bio.c @@ -859,6 +859,7 @@ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp) bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; bio->bi_write_stream = bio_src->bi_write_stream; + bio->bi_bvec_gap_bit = bio_src->bi_bvec_gap_bit; bio->bi_iter = bio_src->bi_iter; if (bio->bi_bdev) { @@ -1221,10 +1222,45 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_DEBUG_KERNEL +static inline bool bio_iov_bvec_aligned(const struct bio *bio, + unsigned mem_align_mask) +{ + struct bvec_iter iter; + struct bio_vec bv; + + /* + * Correct callers never break the alignment requirements, so this + * exhaustive check is only paid for in debug builds. + */ + for_each_mp_bvec(bv, bio->bi_io_vec, iter, bio->bi_iter) + if ((bv.bv_offset | bv.bv_len) & mem_align_mask) + return false; + return true; +} +#else +static inline bool bio_iov_bvec_aligned(const struct bio *bio, + unsigned mem_align_mask) +{ + /* + * We forward the bio_vec as-is, so ITER_BVEC callers must provide + * segments already aligned to the device's DMA alignment. The only + * unchecked user-controllable offset that reaches here is an io_uring + * registered buffer where just the first segment can be unaligned + * (the rest is virtually contiguous), so checking only that one is + * sufficient to know if the entire vector is valid. + */ + return !(mp_bvec_iter_offset(bio->bi_io_vec, bio->bi_iter) & + mem_align_mask); +} +#endif + /** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to * @iter: iov iterator describing the region to be added + * @mem_align_mask: the mask the source address and length must be aligned to, + * 0 for no requirement * @len_align_mask: the mask to align the total size to, 0 for any length * * This takes either an iterator pointing to user memory, or one pointing to @@ -1243,7 +1279,7 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, * is returned only if 0 pages could be pinned. */ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, - unsigned len_align_mask) + unsigned mem_align_mask, unsigned len_align_mask) { iov_iter_extraction_t flags = 0; @@ -1252,6 +1288,10 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, if (iov_iter_is_bvec(iter)) { bio_iov_bvec_set(bio, iter); + + if (!bio_iov_bvec_aligned(bio, mem_align_mask)) + return -EINVAL; + iov_iter_advance(iter, bio->bi_iter.bi_size); return 0; } @@ -1266,8 +1306,19 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec, BIO_MAX_SIZE - bio->bi_iter.bi_size, - &bio->bi_vcnt, bio->bi_max_vecs, flags); + &bio->bi_vcnt, bio->bi_max_vecs, + mem_align_mask, flags); if (ret <= 0) { + /* + * A misaligned vector fails the whole I/O. Release any + * pages pinned by earlier iterations before returning + * since this bio won't be submitted to release them. + */ + if (ret == -EINVAL) { + bio_release_pages(bio, false); + bio_clear_flag(bio, BIO_PAGE_PINNED); + bio->bi_vcnt = 0; + } if (!bio->bi_vcnt) return ret; break; @@ -1380,7 +1431,7 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter, do { ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec + 1, len, - &bio->bi_vcnt, bio->bi_max_vecs - 1, 0); + &bio->bi_vcnt, bio->bi_max_vecs - 1, 0, 0); if (ret <= 0) { if (!bio->bi_vcnt) goto out_folio_put; @@ -1863,6 +1914,14 @@ struct bio *bio_split(struct bio *bio, int sectors, bio_advance(bio, split->bi_iter.bi_size); + /* + * The gap bit is set when splitting to limits and only applies to the + * front bio that was split off. The remaining bio will calcualte its + * gap value when it is subsequently split to limits, so it is safe to + * re-initialize the value back to 0. + */ + bio->bi_bvec_gap_bit = 0; + if (bio_flagged(bio, BIO_TRACE_COMPLETION)) bio_set_flag(split, BIO_TRACE_COMPLETION); diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index d9676126c5b5d..52080c7c1a14b 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1563,6 +1563,8 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (queue_is_mq(q)) memflags = blk_mq_freeze_queue(q); + + mutex_lock(&q->blkcg_mutex); retry: spin_lock_irq(&q->queue_lock); @@ -1572,6 +1574,8 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (blkg->pd[pol->plid]) continue; + if (hlist_unhashed(&blkg->blkcg_node)) + continue; /* If prealloc matches, use it; otherwise try GFP_NOWAIT */ if (blkg == pinned_blkg) { @@ -1625,6 +1629,7 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) spin_unlock_irq(&q->queue_lock); out: + mutex_unlock(&q->blkcg_mutex); if (queue_is_mq(q)) blk_mq_unfreeze_queue(q, memflags); if (pinned_blkg) diff --git a/block/blk-iocost.c b/block/blk-iocost.c index 8b2aeba2e1e39..64b92aa3e5d48 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -1592,7 +1592,7 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p u64 rq_wait_ns = 0; int cpu, rw; - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct ioc_pcpu_stat *stat = per_cpu_ptr(ioc->pcpu_stat, cpu); u64 this_rq_wait_ns; @@ -3063,6 +3063,7 @@ static void iocg_release(struct rcu_head *rcu) static void ioc_pd_free(struct blkg_policy_data *pd) { struct ioc_gq *iocg = pd_to_iocg(pd); + struct blkcg_gq *blkg = pd_to_blkg(pd); struct ioc *ioc = iocg->ioc; unsigned long flags; @@ -3085,6 +3086,12 @@ static void ioc_pd_free(struct blkg_policy_data *pd) hrtimer_cancel(&iocg->waitq_timer); } + /* off ->active_iocgs and timer gone, so nothing can re-arm the delay */ + iocg->delay = 0; + iocg->indelay_since = 0; + if (blkg) + blkcg_clear_delay(blkg); + call_rcu(&pd->rcu_head, iocg_release); } diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index cef02b6c5fa91..2caa79a008ad1 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -523,7 +523,7 @@ static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now) latency_stat_init(iolat, &stat); preempt_disable(); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct latency_stat *s; s = per_cpu_ptr(iolat->stats, cpu); latency_stat_sum(iolat, &stat, s); @@ -925,7 +925,7 @@ static void iolatency_ssd_stat(struct iolatency_grp *iolat, struct seq_file *s) latency_stat_init(iolat, &stat); preempt_disable(); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct latency_stat *s; s = per_cpu_ptr(iolat->stats, cpu); latency_stat_sum(iolat, &stat, s); @@ -1043,6 +1043,15 @@ static void iolat_release(struct rcu_head *rcu) static void iolatency_pd_free(struct blkg_policy_data *pd) { + struct blkcg_gq *blkg = pd_to_blkg(pd); + + /* + * Groups throttled as collateral have min_lat_nsec == 0, so + * iolatency_pd_offline() leaves their delay set. Drop it here, where + * no in-flight bio can re-arm it via check_scale_change(). + */ + if (blkg) + blkcg_clear_delay(blkg); call_rcu(&pd->rcu_head, iolat_release); } diff --git a/block/blk-map.c b/block/blk-map.c index d1d6bbe0ecf1f..615d29bb840eb 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -274,7 +274,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, * No alignment requirements on our part to support arbitrary * passthrough commands. */ - ret = bio_iov_iter_get_pages(bio, iter, 0); + ret = bio_iov_iter_get_pages(bio, iter, 0, 0); if (ret) goto out_put; ret = blk_rq_append_bio(rq, bio); diff --git a/block/blk-stat.c b/block/blk-stat.c index de126e1ea5ac4..d57c2fc6bf064 100644 --- a/block/blk-stat.c +++ b/block/blk-stat.c @@ -83,7 +83,7 @@ static void blk_stat_timer_fn(struct timer_list *t) for (bucket = 0; bucket < cb->buckets; bucket++) blk_rq_stat_init(&cb->stat[bucket]); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct blk_rq_stat *cpu_stat; cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); diff --git a/block/blk-zoned.c b/block/blk-zoned.c index ca30caec838e7..dd6bc679210c8 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -2048,12 +2048,17 @@ static int disk_revalidate_zone_resources(struct gendisk *disk, struct blk_revalidate_zone_args *args) { struct queue_limits *lim = &disk->queue->limits; + unsigned long long nr_zones; unsigned int pool_size; int ret = 0; args->disk = disk; - args->nr_zones = - DIV_ROUND_UP_ULL(get_capacity(disk), lim->chunk_sectors); + nr_zones = DIV_ROUND_UP_ULL(get_capacity(disk), lim->chunk_sectors); + if (nr_zones > UINT_MAX) { + pr_warn("%s: Too many zones (%llu)\n", disk->disk_name, nr_zones); + return -EINVAL; + } + args->nr_zones = nr_zones; /* Cached zone conditions: 1 byte per zone */ args->zones_cond = kzalloc(args->nr_zones, GFP_NOIO); @@ -2161,6 +2166,12 @@ static int blk_revalidate_zone_cond(struct blk_zone *zone, unsigned int idx, { enum blk_zone_cond cond = zone->cond; + if (idx >= args->nr_zones) { + pr_warn("%s: Zone report index %u exceeds zone count %u\n", + args->disk->disk_name, idx, args->nr_zones); + return -EINVAL; + } + /* Check that the zone condition is consistent with the zone type. */ switch (cond) { case BLK_ZONE_COND_NOT_WP: diff --git a/block/fops.c b/block/fops.c index 15783a6180dec..cb919dea00411 100644 --- a/block/fops.c +++ b/block/fops.c @@ -46,7 +46,7 @@ static bool blkdev_dio_invalid(struct block_device *bdev, struct kiocb *iocb, static inline int blkdev_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, struct block_device *bdev) { - return bio_iov_iter_get_pages(bio, iter, + return bio_iov_iter_get_pages(bio, iter, bdev_dma_alignment(bdev), bdev_logical_block_size(bdev) - 1); } @@ -238,8 +238,10 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, } if (iocb->ki_flags & IOCB_HAS_METADATA) { ret = bio_integrity_map_iter(bio, iocb->private); - if (unlikely(ret)) - goto fail; + if (unlikely(ret)) { + bio_endio_status(bio, errno_to_blk_status(ret)); + break; + } } if (is_read) { diff --git a/block/genhd.c b/block/genhd.c index e8ce0cabf392c..10ca8b4d6eea7 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -447,6 +447,13 @@ static int __add_disk(struct device *parent, struct gendisk *disk, bdev_set_flag(disk->part0, BD_HAS_SUBMIT_BIO); } + /* + * We do not support partitions with zoned block devices, so do not try + * to scan the partitions table. + */ + if (blk_queue_is_zoned(disk->queue)) + disk->flags |= GENHD_FL_NO_PART; + /* * If the driver provides an explicit major number it also must provide * the number of minors numbers supported, and those will be used to @@ -681,6 +688,7 @@ static bool __blk_mark_disk_dead(struct gendisk *disk) */ void blk_mark_disk_dead(struct gendisk *disk) { + blk_queue_flag_set(QUEUE_FLAG_DYING, disk->queue); __blk_mark_disk_dead(disk); blk_report_disk_dead(disk, true); } diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index 971818bcdc9db..2ee552ab81351 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -275,7 +275,7 @@ static void kyber_timer_fn(struct timer_list *t) bool bad = false; /* Sum all of the per-cpu latency histograms. */ - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct kyber_cpu_latency *cpu_latency; cpu_latency = per_cpu_ptr(kqd->cpu_latency, cpu); diff --git a/crypto/acompress.c b/crypto/acompress.c index 032de704eb2ca..4de1a2ad577f2 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -559,12 +559,22 @@ EXPORT_SYMBOL_GPL(acomp_walk_virt); struct acomp_req *acomp_request_clone(struct acomp_req *req, size_t total, gfp_t gfp) { + struct crypto_tfm *tfm = req->base.tfm; struct acomp_req *nreq; + size_t len; - nreq = container_of(crypto_request_clone(&req->base, total, gfp), - struct acomp_req, base); - if (nreq == req) + len = sizeof(*req) + + crypto_acomp_reqsize(crypto_acomp_reqtfm(req)); + len = ALIGN(len, CRYPTO_MINALIGN); + + nreq = kzalloc(len, gfp); + if (!nreq) { + req->base.tfm = tfm->fb; return req; + } + + memcpy(nreq, req, sizeof(*req)); + nreq->base.flags &= ~CRYPTO_TFM_REQ_ON_STACK; if (req->src == &req->chain.ssg) nreq->src = &nreq->chain.ssg; diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index 03395b89cc61e..b377f4f7558da 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -263,10 +263,10 @@ struct crypto_aead *crypto_krb5_prepare_encryption(const struct krb5_enctype *kr goto err; } - kfree(keys.data); + kfree_sensitive(keys.data); return ci; err: - kfree(keys.data); + kfree_sensitive(keys.data); return ERR_PTR(ret); } EXPORT_SYMBOL(crypto_krb5_prepare_encryption); @@ -333,10 +333,10 @@ struct crypto_shash *crypto_krb5_prepare_checksum(const struct krb5_enctype *krb goto err; } - kfree(keys.data); + kfree_sensitive(keys.data); return ci; err: - kfree(keys.data); + kfree_sensitive(keys.data); return ERR_PTR(ret); } EXPORT_SYMBOL(crypto_krb5_prepare_checksum); diff --git a/crypto/lskcipher.c b/crypto/lskcipher.c index e4328df6e26c0..a79c9a246e5d7 100644 --- a/crypto/lskcipher.c +++ b/crypto/lskcipher.c @@ -95,7 +95,6 @@ static int crypto_lskcipher_crypt_unaligned( while (len >= bs) { unsigned chunk = min((unsigned)PAGE_SIZE, len); - int err; if (chunk > cs) chunk &= ~(cs - 1); diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c index dfe0fbdf066d2..b4c49259a1a23 100644 --- a/drivers/accel/amdxdna/aie2_message.c +++ b/drivers/accel/amdxdna/aie2_message.c @@ -994,7 +994,7 @@ int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, } ccnt = payload->command_count; - if (payload_len < struct_size(payload, data, ccnt)) { + if (!ccnt || payload_len < struct_size(payload, data, ccnt)) { XDNA_DBG(xdna, "Invalid command count %d", ccnt); return -EINVAL; } diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c index 31a414c3f0d96..888e857ec5582 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.c +++ b/drivers/accel/amdxdna/amdxdna_ctx.c @@ -183,8 +183,10 @@ int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo, if (!abo) return -EINVAL; cmd = amdxdna_gem_vmap(abo); - if (!cmd) + if (!cmd) { + amdxdna_gem_put_obj(abo); return -ENOMEM; + } } memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd)); diff --git a/drivers/accel/amdxdna/amdxdna_ctx.h b/drivers/accel/amdxdna/amdxdna_ctx.h index b6bef3af7dab4..6e78bab8a02c0 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.h +++ b/drivers/accel/amdxdna/amdxdna_ctx.h @@ -55,7 +55,7 @@ struct amdxdna_cmd_chain { u32 submit_index; u32 error_index; u32 reserved[3]; - u64 data[] __counted_by(command_count); + u64 data[]; }; /* diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c index 1c63eff0a4a85..4b0d58d0329b7 100644 --- a/drivers/accel/amdxdna/amdxdna_gem.c +++ b/drivers/accel/amdxdna/amdxdna_gem.c @@ -758,9 +758,15 @@ static int amdxdna_gem_dev_obj_vmap(struct drm_gem_object *obj, struct iosys_map return 0; } +static struct dma_buf *amdxdna_gem_dev_obj_export(struct drm_gem_object *gobj, int flags) +{ + return ERR_PTR(-EOPNOTSUPP); +} + static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = { .free = amdxdna_gem_dev_obj_free, .vmap = amdxdna_gem_dev_obj_vmap, + .export = amdxdna_gem_dev_obj_export, }; static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = { @@ -1240,6 +1246,9 @@ static int amdxdna_flush_bo(struct amdxdna_gem_obj *abo, u64 offset, u64 size) { u64 end; + if (is_import_bo(abo)) + return -EOPNOTSUPP; + if (offset >= abo->mem.size) return -EINVAL; @@ -1247,9 +1256,10 @@ static int amdxdna_flush_bo(struct amdxdna_gem_obj *abo, u64 offset, u64 size) return -EINVAL; size = min(abo->mem.size, end) - offset; - if (is_import_bo(abo)) - drm_clflush_sg(abo->base.sgt); - else if (amdxdna_gem_vmap(abo)) + if (!size) + return 0; + + if (amdxdna_gem_vmap(abo)) drm_clflush_virt_range(amdxdna_gem_vmap(abo) + offset, size); else if (abo->base.pages) drm_clflush_pages(abo->base.pages, abo->mem.size >> PAGE_SHIFT); diff --git a/drivers/accel/ethosu/ethosu_device.h b/drivers/accel/ethosu/ethosu_device.h index b189fa783d6a3..cf53964a4f04b 100644 --- a/drivers/accel/ethosu/ethosu_device.h +++ b/drivers/accel/ethosu/ethosu_device.h @@ -68,6 +68,7 @@ struct gen_pool; #define PROT_ACTIVE_CSL BIT(1) enum ethosu_cmds { + NPU_OP_STOP = 0x0, NPU_OP_CONV = 0x2, NPU_OP_DEPTHWISE = 0x3, NPU_OP_POOL = 0x5, diff --git a/drivers/accel/ethosu/ethosu_drv.c b/drivers/accel/ethosu/ethosu_drv.c index ed9c748a54ad5..ff11d7b279530 100644 --- a/drivers/accel/ethosu/ethosu_drv.c +++ b/drivers/accel/ethosu/ethosu_drv.c @@ -274,8 +274,6 @@ static int ethosu_device_suspend(struct device *dev) static int ethosu_sram_init(struct ethosu_device *ethosudev) { - ethosudev->npu_info.sram_size = 0; - ethosudev->srampool = of_gen_pool_get(ethosudev->base.dev->of_node, "sram", 0); if (!ethosudev->srampool) return 0; @@ -286,6 +284,7 @@ static int ethosu_sram_init(struct ethosu_device *ethosudev) ethosudev->npu_info.sram_size, ðosudev->sramphys); if (!ethosudev->sram) { + ethosudev->npu_info.sram_size = 0; dev_err(ethosudev->base.dev, "failed to allocate from SRAM pool\n"); return -ENOMEM; } @@ -342,6 +341,8 @@ static int ethosu_probe(struct platform_device *pdev) dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); ethosudev->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ethosudev->regs)) + return PTR_ERR(ethosudev->regs); ethosudev->num_clks = devm_clk_bulk_get_all(&pdev->dev, ðosudev->clks); if (ethosudev->num_clks < 0) diff --git a/drivers/accel/ethosu/ethosu_gem.c b/drivers/accel/ethosu/ethosu_gem.c index d50fed64d4d93..9afe2549ec84d 100644 --- a/drivers/accel/ethosu/ethosu_gem.c +++ b/drivers/accel/ethosu/ethosu_gem.c @@ -204,7 +204,7 @@ static u64 feat_matrix_length(struct ethosu_device *edev, struct feat_matrix *fm, u32 x, u32 y, u32 c, bool ofm) { - u32 element_size, storage = fm->precision >> 14; + u32 element_size, storage = ethosu_is_u65(edev) ? 0 : fm->precision >> 14; int tile = 0; u64 addr; @@ -390,6 +390,7 @@ static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev, struct ethosu_validated_cmdstream_info __free(kfree) *info = kzalloc_obj(*info); struct ethosu_device *edev = to_ethosu_device(ddev); u32 *bocmds = bo->base.vaddr; + bool ends_with_stop = false; struct cmd_state st; int i, ret; @@ -426,6 +427,11 @@ static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev, } switch (cmd) { + case NPU_OP_STOP: + if (i != size / 4 - 1) + return -EINVAL; + ends_with_stop = true; + break; case NPU_OP_DMA_START: srclen = dma_length(info, &st.dma, &st.dma.src); dstlen = dma_length(info, &st.dma, &st.dma.dst); @@ -688,6 +694,9 @@ static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev, } } + if (!ends_with_stop) + return -EINVAL; + for (i = 0; i < NPU_BASEP_REGION_MAX; i++) { if (!info->region_size[i]) continue; diff --git a/drivers/accel/ethosu/ethosu_job.c b/drivers/accel/ethosu/ethosu_job.c index 1e2465279aae6..3365d2e6d51b2 100644 --- a/drivers/accel/ethosu/ethosu_job.c +++ b/drivers/accel/ethosu/ethosu_job.c @@ -152,6 +152,13 @@ static void ethosu_job_err_cleanup(struct ethosu_job *job) drm_gem_object_put(job->cmd_bo); + if (job->done_fence) { + if (dma_fence_was_initialized(job->done_fence)) + dma_fence_put(job->done_fence); + else + dma_fence_free(job->done_fence); + } + kfree(job); } @@ -162,7 +169,6 @@ static void ethosu_job_cleanup(struct kref *ref) pm_runtime_put_autosuspend(job->dev->base.dev); - dma_fence_put(job->done_fence); dma_fence_put(job->inference_done_fence); ethosu_job_err_cleanup(job); @@ -319,7 +325,7 @@ int ethosu_job_init(struct ethosu_device *edev) ret = devm_request_threaded_irq(dev, edev->irq, ethosu_job_irq_handler, ethosu_job_irq_handler_thread, - IRQF_SHARED, KBUILD_MODNAME, + 0, KBUILD_MODNAME, edev); if (ret) { dev_err(dev, "failed to request irq\n"); @@ -350,12 +356,10 @@ int ethosu_job_open(struct ethosu_file_priv *ethosu_priv) { struct ethosu_device *dev = ethosu_priv->edev; struct drm_gpu_scheduler *sched = &dev->sched; - int ret; - ret = drm_sched_entity_init(ðosu_priv->sched_entity, - DRM_SCHED_PRIORITY_NORMAL, - &sched, 1, NULL); - return WARN_ON(ret); + return drm_sched_entity_init(ðosu_priv->sched_entity, + DRM_SCHED_PRIORITY_NORMAL, + &sched, 1, NULL); } void ethosu_job_close(struct ethosu_file_priv *ethosu_priv) @@ -393,7 +397,7 @@ static int ethosu_ioctl_submit_job(struct drm_device *dev, struct drm_file *file ejob->done_fence = kzalloc_obj(*ejob->done_fence); if (!ejob->done_fence) { ret = -ENOMEM; - goto out_cleanup_job; + goto out_put_job; } ret = drm_sched_job_init(&ejob->base, @@ -421,13 +425,13 @@ static int ethosu_ioctl_submit_job(struct drm_device *dev, struct drm_file *file if (!cmd_info->region_size[i]) continue; if (i == ETHOSU_SRAM_REGION) { - if (cmd_info->region_size[i] <= edev->npu_info.sram_size) + if (cmd_info->region_size[i] <= ejob->sram_size) continue; dev_err(dev->dev, - "cmd stream region %d size greater than SRAM size (%llu > %u)\n", + "cmd stream region %d size greater than job SRAM size (%llu > %u)\n", i, cmd_info->region_size[i], - edev->npu_info.sram_size); + ejob->sram_size); ret = -EINVAL; goto out_cleanup_job; } diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c index 35e506074d5f1..8c1c87e69f91f 100644 --- a/drivers/accel/ivpu/ivpu_drv.c +++ b/drivers/accel/ivpu/ivpu_drv.c @@ -307,6 +307,11 @@ static int ivpu_open(struct drm_device *dev, struct drm_file *file) return -ENODEV; limits = ivpu_user_limits_get(vdev); + if (IS_ERR(limits) && PTR_ERR(limits) == -EMFILE) { + /* Context limit may be held by jobs pending deferred cleanup */ + flush_work(&vdev->job_destroy_work); + limits = ivpu_user_limits_get(vdev); + } if (IS_ERR(limits)) { ret = PTR_ERR(limits); goto err_dev_exit; @@ -510,9 +515,10 @@ void ivpu_prepare_for_reset(struct ivpu_device *vdev) { ivpu_hw_irq_disable(vdev); disable_irq(vdev->irq); - flush_work(&vdev->irq_ipc_work); + atomic_set(&vdev->job_timeout_detected, 0); flush_work(&vdev->irq_dct_work); flush_work(&vdev->context_abort_work); + flush_work(&vdev->job_destroy_work); ivpu_ipc_disable(vdev); ivpu_mmu_disable(vdev); } @@ -584,6 +590,11 @@ static const struct drm_driver driver = { .major = 1, }; +static void ivpu_destroy_workqueue(void *wq) +{ + destroy_workqueue(wq); +} + static int ivpu_irq_init(struct ivpu_device *vdev) { struct pci_dev *pdev = to_pci_dev(vdev->drm.dev); @@ -595,16 +606,26 @@ static int ivpu_irq_init(struct ivpu_device *vdev) return ret; } - INIT_WORK(&vdev->irq_ipc_work, ivpu_ipc_irq_work_fn); INIT_WORK(&vdev->irq_dct_work, ivpu_pm_irq_dct_work_fn); INIT_WORK(&vdev->context_abort_work, ivpu_context_abort_work_fn); + init_llist_head(&vdev->job_destroy_list); + INIT_WORK(&vdev->job_destroy_work, ivpu_job_destroy_work_fn); + + vdev->job_destroy_wq = alloc_workqueue("ivpu_job_destroy", WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!vdev->job_destroy_wq) + return -ENOMEM; + + ret = devm_add_action_or_reset(vdev->drm.dev, ivpu_destroy_workqueue, vdev->job_destroy_wq); + if (ret) + return ret; ivpu_irq_handlers_init(vdev); vdev->irq = pci_irq_vector(pdev, 0); - ret = devm_request_irq(vdev->drm.dev, vdev->irq, ivpu_hw_irq_handler, - IRQF_NO_AUTOEN, DRIVER_NAME, vdev); + ret = devm_request_threaded_irq(vdev->drm.dev, vdev->irq, ivpu_hw_irq_handler, + ivpu_ipc_irq_thread_handler, IRQF_NO_AUTOEN, + DRIVER_NAME, vdev); if (ret) ivpu_err(vdev, "Failed to request an IRQ %d\n", ret); @@ -690,7 +711,7 @@ static int ivpu_dev_init(struct ivpu_device *vdev) vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID; atomic64_set(&vdev->unique_id_counter, 0); atomic_set(&vdev->job_timeout_counter, 0); - atomic_set(&vdev->faults_detected, 0); + atomic_set(&vdev->job_timeout_detected, 0); xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ); xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1); xa_init_flags(&vdev->db_xa, XA_FLAGS_ALLOC1); diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h index 9eefbbb7ba118..6f40129264789 100644 --- a/drivers/accel/ivpu/ivpu_drv.h +++ b/drivers/accel/ivpu/ivpu_drv.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -157,9 +158,11 @@ struct ivpu_device { struct xa_limit db_limit; u32 db_next; - struct work_struct irq_ipc_work; struct work_struct irq_dct_work; struct work_struct context_abort_work; + struct llist_head job_destroy_list; + struct work_struct job_destroy_work; + struct workqueue_struct *job_destroy_wq; struct mutex bo_list_lock; /* Protects bo_list */ struct list_head bo_list; @@ -168,7 +171,7 @@ struct ivpu_device { struct xarray submitted_jobs_xa; struct ivpu_ipc_consumer job_done_consumer; atomic_t job_timeout_counter; - atomic_t faults_detected; + atomic_t job_timeout_detected; atomic64_t unique_id_counter; diff --git a/drivers/accel/ivpu/ivpu_fw_log.c b/drivers/accel/ivpu/ivpu_fw_log.c index 716467aa31567..9eafc42120b6a 100644 --- a/drivers/accel/ivpu/ivpu_fw_log.c +++ b/drivers/accel/ivpu/ivpu_fw_log.c @@ -26,10 +26,17 @@ MODULE_PARM_DESC(fw_log_level, " error=" __stringify(IVPU_FW_LOG_ERROR) " fatal=" __stringify(IVPU_FW_LOG_FATAL)); +struct ivpu_fw_log_desc { + struct vpu_tracing_buffer_header *log; + u32 header_size; + u32 size; +}; + static int fw_log_from_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, u32 *offset, - struct vpu_tracing_buffer_header **out_log) + struct ivpu_fw_log_desc *desc) { struct vpu_tracing_buffer_header *log; + u32 header_size, size; if ((*offset + sizeof(*log)) > ivpu_bo_size(bo)) return -EINVAL; @@ -39,26 +46,32 @@ static int fw_log_from_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, u32 *off if (log->vpu_canary_start != VPU_TRACING_BUFFER_CANARY) return -EINVAL; - if (log->header_size < sizeof(*log) || log->header_size > 1024) { - ivpu_dbg(vdev, FW_BOOT, "Invalid header size 0x%x\n", log->header_size); + header_size = READ_ONCE(log->header_size); + size = READ_ONCE(log->size); + + if (header_size < sizeof(*log) || header_size > 1024) { + ivpu_dbg(vdev, FW_BOOT, "Invalid header size 0x%x\n", header_size); return -EINVAL; } - if (log->size < log->header_size) { - ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size); + if ((char *)log + size > (char *)ivpu_bo_vaddr(bo) + ivpu_bo_size(bo)) { + ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", size); return -EINVAL; } - if ((char *)log + log->size > (char *)ivpu_bo_vaddr(bo) + ivpu_bo_size(bo)) { - ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size); + if (size < header_size) { + ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x < header size 0x%x\n", + size, header_size); return -EINVAL; } - *out_log = log; - *offset += log->size; + desc->log = log; + desc->header_size = header_size; + desc->size = size; + *offset += size; ivpu_dbg(vdev, FW_BOOT, - "FW log name \"%s\", write offset 0x%x size 0x%x, wrap count %d, hdr version %d size %d format %d, alignment %d", - log->name, log->write_index, log->size, log->wrap_count, log->header_version, - log->header_size, log->format, log->alignment); + "FW log name \"%.*s\", write offset 0x%x size 0x%x, wrap count %d, hdr version %d size %d format %d, alignment %d", + (int)ARRAY_SIZE(log->name), log->name, log->write_index, size, log->wrap_count, + log->header_version, header_size, log->format, log->alignment); return 0; } @@ -94,11 +107,12 @@ static void fw_log_print_lines(char *buffer, u32 size, struct drm_printer *p) drm_printf(p, "%s", line); } -static void fw_log_print_buffer(struct vpu_tracing_buffer_header *log, const char *prefix, +static void fw_log_print_buffer(struct ivpu_fw_log_desc *desc, const char *prefix, bool only_new_msgs, struct drm_printer *p) { - char *log_data = (void *)log + log->header_size; - u32 data_size = log->size - log->header_size; + struct vpu_tracing_buffer_header *log = desc->log; + char *log_data = (void *)log + desc->header_size; + u32 data_size = desc->size - desc->header_size; u32 log_start = only_new_msgs ? READ_ONCE(log->read_index) : 0; u32 log_end = READ_ONCE(log->write_index); @@ -109,7 +123,8 @@ static void fw_log_print_buffer(struct vpu_tracing_buffer_header *log, const cha if (log->wrap_count == log->read_wrap_count) { if (log_end <= log_start) { - drm_printf(p, "==== %s \"%s\" log empty ====\n", prefix, log->name); + drm_printf(p, "==== %s \"%.*s\" log empty ====\n", prefix, + (int)ARRAY_SIZE(log->name), log->name); return; } } else if (log->wrap_count == log->read_wrap_count + 1) { @@ -119,7 +134,8 @@ static void fw_log_print_buffer(struct vpu_tracing_buffer_header *log, const cha log_start = log_end; } - drm_printf(p, "==== %s \"%s\" log start ====\n", prefix, log->name); + drm_printf(p, "==== %s \"%.*s\" log start ====\n", prefix, (int)ARRAY_SIZE(log->name), + log->name); if (log_end > log_start) { fw_log_print_lines(log_data + log_start, log_end - log_start, p); } else { @@ -127,18 +143,19 @@ static void fw_log_print_buffer(struct vpu_tracing_buffer_header *log, const cha fw_log_print_lines(log_data, log_end, p); } drm_printf(p, "\n\x1b[0m"); /* add new line and clear formatting */ - drm_printf(p, "==== %s \"%s\" log end ====\n", prefix, log->name); + drm_printf(p, "==== %s \"%.*s\" log end ====\n", prefix, (int)ARRAY_SIZE(log->name), + log->name); } static void fw_log_print_all_in_bo(struct ivpu_device *vdev, const char *name, struct ivpu_bo *bo, bool only_new_msgs, struct drm_printer *p) { - struct vpu_tracing_buffer_header *log; + struct ivpu_fw_log_desc desc; u32 next = 0; - while (fw_log_from_bo(vdev, bo, &next, &log) == 0) - fw_log_print_buffer(log, name, only_new_msgs, p); + while (fw_log_from_bo(vdev, bo, &next, &desc) == 0) + fw_log_print_buffer(&desc, name, only_new_msgs, p); } void ivpu_fw_log_print(struct ivpu_device *vdev, bool only_new_msgs, struct drm_printer *p) @@ -149,36 +166,36 @@ void ivpu_fw_log_print(struct ivpu_device *vdev, bool only_new_msgs, struct drm_ void ivpu_fw_log_mark_read(struct ivpu_device *vdev) { - struct vpu_tracing_buffer_header *log; + struct ivpu_fw_log_desc desc; u32 next; next = 0; - while (fw_log_from_bo(vdev, vdev->fw->mem_log_crit, &next, &log) == 0) { - log->read_index = READ_ONCE(log->write_index); - log->read_wrap_count = READ_ONCE(log->wrap_count); + while (fw_log_from_bo(vdev, vdev->fw->mem_log_crit, &next, &desc) == 0) { + desc.log->read_index = READ_ONCE(desc.log->write_index); + desc.log->read_wrap_count = READ_ONCE(desc.log->wrap_count); } next = 0; - while (fw_log_from_bo(vdev, vdev->fw->mem_log_verb, &next, &log) == 0) { - log->read_index = READ_ONCE(log->write_index); - log->read_wrap_count = READ_ONCE(log->wrap_count); + while (fw_log_from_bo(vdev, vdev->fw->mem_log_verb, &next, &desc) == 0) { + desc.log->read_index = READ_ONCE(desc.log->write_index); + desc.log->read_wrap_count = READ_ONCE(desc.log->wrap_count); } } void ivpu_fw_log_reset(struct ivpu_device *vdev) { - struct vpu_tracing_buffer_header *log; + struct ivpu_fw_log_desc desc; u32 next; next = 0; - while (fw_log_from_bo(vdev, vdev->fw->mem_log_crit, &next, &log) == 0) { - log->read_index = 0; - log->read_wrap_count = 0; + while (fw_log_from_bo(vdev, vdev->fw->mem_log_crit, &next, &desc) == 0) { + desc.log->read_index = 0; + desc.log->read_wrap_count = 0; } next = 0; - while (fw_log_from_bo(vdev, vdev->fw->mem_log_verb, &next, &log) == 0) { - log->read_index = 0; - log->read_wrap_count = 0; + while (fw_log_from_bo(vdev, vdev->fw->mem_log_verb, &next, &desc) == 0) { + desc.log->read_index = 0; + desc.log->read_wrap_count = 0; } } diff --git a/drivers/accel/ivpu/ivpu_gem.h b/drivers/accel/ivpu/ivpu_gem.h index 0c3350f22b558..b1ae020a4fc21 100644 --- a/drivers/accel/ivpu/ivpu_gem.h +++ b/drivers/accel/ivpu/ivpu_gem.h @@ -87,15 +87,23 @@ static inline bool ivpu_bo_is_resident(struct ivpu_bo *bo) return !!bo->base.pages; } -static inline void *ivpu_to_cpu_addr(struct ivpu_bo *bo, u32 vpu_addr) +static inline void *ivpu_to_cpu_addr(struct ivpu_bo *bo, u64 vpu_addr, u64 size) { + u64 bo_size = ivpu_bo_size(bo); + u64 offset; + if (vpu_addr < bo->vpu_addr) return NULL; - if (vpu_addr >= (bo->vpu_addr + ivpu_bo_size(bo))) + if (size > bo_size) + return NULL; + + offset = vpu_addr - bo->vpu_addr; + + if (offset > bo_size - size) return NULL; - return ivpu_bo_vaddr(bo) + (vpu_addr - bo->vpu_addr); + return ivpu_bo_vaddr(bo) + offset; } static inline u32 cpu_to_vpu_addr(struct ivpu_bo *bo, void *cpu_addr) diff --git a/drivers/accel/ivpu/ivpu_hw.c b/drivers/accel/ivpu/ivpu_hw.c index d4a9bcda4100f..647dc045c231a 100644 --- a/drivers/accel/ivpu/ivpu_hw.c +++ b/drivers/accel/ivpu/ivpu_hw.c @@ -399,6 +399,10 @@ irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr) return IRQ_NONE; pm_runtime_mark_last_busy(vdev->drm.dev); + + if (ip_handled) + return IRQ_WAKE_THREAD; + return IRQ_HANDLED; } diff --git a/drivers/accel/ivpu/ivpu_ipc.c b/drivers/accel/ivpu/ivpu_ipc.c index 978bc3d8704fb..459e740873abb 100644 --- a/drivers/accel/ivpu/ivpu_ipc.c +++ b/drivers/accel/ivpu/ivpu_ipc.c @@ -79,7 +79,7 @@ ivpu_ipc_tx_prepare(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, return -ENOMEM; } - tx_buf = ivpu_to_cpu_addr(ipc->mem_tx, tx_buf_vpu_addr); + tx_buf = ivpu_to_cpu_addr(ipc->mem_tx, tx_buf_vpu_addr, sizeof(*tx_buf)); if (drm_WARN_ON(&vdev->drm, !tx_buf)) { gen_pool_free(ipc->mm_tx, tx_buf_vpu_addr, sizeof(*tx_buf)); return -EIO; @@ -420,7 +420,7 @@ void ivpu_ipc_irq_handler(struct ivpu_device *vdev) return; } - ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr); + ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr, sizeof(*ipc_hdr)); if (!ipc_hdr) { ivpu_warn_ratelimited(vdev, "IPC msg 0x%x out of range\n", vpu_addr); continue; @@ -429,7 +429,8 @@ void ivpu_ipc_irq_handler(struct ivpu_device *vdev) jsm_msg = NULL; if (ipc_hdr->channel != IVPU_IPC_CHAN_BOOT_MSG) { - jsm_msg = ivpu_to_cpu_addr(ipc->mem_rx, ipc_hdr->data_addr); + jsm_msg = ivpu_to_cpu_addr(ipc->mem_rx, ipc_hdr->data_addr, + sizeof(*jsm_msg)); if (!jsm_msg) { ivpu_warn_ratelimited(vdev, "JSM msg 0x%x out of range\n", ipc_hdr->data_addr); @@ -462,13 +463,11 @@ void ivpu_ipc_irq_handler(struct ivpu_device *vdev) ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg); } } - - queue_work(system_percpu_wq, &vdev->irq_ipc_work); } -void ivpu_ipc_irq_work_fn(struct work_struct *work) +irqreturn_t ivpu_ipc_irq_thread_handler(int irq, void *ptr) { - struct ivpu_device *vdev = container_of(work, struct ivpu_device, irq_ipc_work); + struct ivpu_device *vdev = ptr; struct ivpu_ipc_info *ipc = vdev->ipc; struct ivpu_ipc_rx_msg *rx_msg, *r; struct list_head cb_msg_list; @@ -483,6 +482,8 @@ void ivpu_ipc_irq_work_fn(struct work_struct *work) rx_msg->callback(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg); ivpu_ipc_rx_msg_del(vdev, rx_msg); } + + return IRQ_HANDLED; } int ivpu_ipc_init(struct ivpu_device *vdev) diff --git a/drivers/accel/ivpu/ivpu_ipc.h b/drivers/accel/ivpu/ivpu_ipc.h index b524a1985b9de..1c9cf67f1e5bd 100644 --- a/drivers/accel/ivpu/ivpu_ipc.h +++ b/drivers/accel/ivpu/ivpu_ipc.h @@ -90,7 +90,7 @@ void ivpu_ipc_disable(struct ivpu_device *vdev); void ivpu_ipc_reset(struct ivpu_device *vdev); void ivpu_ipc_irq_handler(struct ivpu_device *vdev); -void ivpu_ipc_irq_work_fn(struct work_struct *work); +irqreturn_t ivpu_ipc_irq_thread_handler(int irq, void *ptr); void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, u32 channel, ivpu_ipc_rx_callback_t callback); diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c index b24f31a8b5672..4689b8ab519d7 100644 --- a/drivers/accel/ivpu/ivpu_job.c +++ b/drivers/accel/ivpu/ivpu_job.c @@ -535,6 +535,20 @@ static void ivpu_job_destroy(struct ivpu_job *job) kfree(job); } +void ivpu_job_destroy_work_fn(struct work_struct *work) +{ + struct ivpu_device *vdev = container_of(work, struct ivpu_device, job_destroy_work); + struct ivpu_job *job, *tmp; + struct llist_node *list; + + list = llist_del_all(&vdev->job_destroy_list); + + llist_for_each_entry_safe(job, tmp, list, destroy_node) { + ivpu_job_destroy(job); + ivpu_rpm_put(vdev); + } +} + static struct ivpu_job * ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count) { @@ -607,7 +621,6 @@ bool ivpu_job_handle_engine_error(struct ivpu_device *vdev, u32 job_id, u32 job_ * status and ensure both are handled in the same way */ job->file_priv->has_mmu_faults = true; - atomic_set(&vdev->faults_detected, 1); queue_work(system_percpu_wq, &vdev->context_abort_work); return true; } @@ -619,7 +632,7 @@ bool ivpu_job_handle_engine_error(struct ivpu_device *vdev, u32 job_id, u32 job_ return false; } -static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status) +static struct ivpu_job *ivpu_job_signal(struct ivpu_device *vdev, u32 job_id, u32 job_status) { struct ivpu_job *job; @@ -627,7 +640,7 @@ static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job = xa_load(&vdev->submitted_jobs_xa, job_id); if (!job) - return -ENOENT; + return NULL; ivpu_job_remove_from_submitted_jobs(vdev, job_id); @@ -646,14 +659,37 @@ static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job->job_id, job->file_priv->ctx.id, job->cmdq_id, job->engine_idx, job->job_status); - ivpu_job_destroy(job); ivpu_stop_job_timeout_detection(vdev); - ivpu_rpm_put(vdev); - if (!xa_empty(&vdev->submitted_jobs_xa)) ivpu_start_job_timeout_detection(vdev); + return job; +} + +static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status) +{ + struct ivpu_job *job = ivpu_job_signal(vdev, job_id, job_status); + + if (!job) + return -ENOENT; + + ivpu_job_destroy(job); + ivpu_rpm_put(vdev); + + return 0; +} + +static int ivpu_job_signal_and_defer_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status) +{ + struct ivpu_job *job = ivpu_job_signal(vdev, job_id, job_status); + + if (!job) + return -ENOENT; + + llist_add(&job->destroy_node, &vdev->job_destroy_list); + queue_work(vdev->job_destroy_wq, &vdev->job_destroy_work); + return 0; } @@ -689,6 +725,7 @@ static int ivpu_job_submit(struct ivpu_job *job, u8 priority, u32 cmdq_id) struct ivpu_file_priv *file_priv = job->file_priv; struct ivpu_device *vdev = job->vdev; struct ivpu_cmdq *cmdq; + bool flushed = false; bool is_first_job; int ret; @@ -696,6 +733,7 @@ static int ivpu_job_submit(struct ivpu_job *job, u8 priority, u32 cmdq_id) if (ret < 0) return ret; +retry: mutex_lock(&vdev->submitted_jobs_lock); mutex_lock(&file_priv->lock); @@ -709,6 +747,14 @@ static int ivpu_job_submit(struct ivpu_job *job, u8 priority, u32 cmdq_id) } ret = ivpu_cmdq_register(file_priv, cmdq); + if (ret == -EBUSY && !flushed) { + /* Doorbell may be held by jobs pending deferred cleanup */ + mutex_unlock(&file_priv->lock); + mutex_unlock(&vdev->submitted_jobs_lock); + flush_work(&vdev->job_destroy_work); + flushed = true; + goto retry; + } if (ret) { ivpu_err(vdev, "Failed to register command queue: %d\n", ret); goto err_unlock; @@ -1101,7 +1147,7 @@ ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr, mutex_lock(&vdev->submitted_jobs_lock); if (!ivpu_job_handle_engine_error(vdev, payload->job_id, payload->job_status)) /* No engine error, complete the job normally */ - ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status); + ivpu_job_signal_and_defer_destroy(vdev, payload->job_id, payload->job_status); mutex_unlock(&vdev->submitted_jobs_lock); } @@ -1128,10 +1174,10 @@ static int reset_engine_and_mark_faulty_contexts(struct ivpu_device *vdev) return ret; /* - * If faults are detected, ignore guilty contexts from engine reset as NPU may not be stuck - * and could return currently running good context and faulty contexts are already marked + * If job timeout is detected, read guilty context from engine reset, for other reasons + * faulty context is already known */ - if (atomic_cmpxchg(&vdev->faults_detected, 1, 0) == 1) + if (atomic_cmpxchg(&vdev->job_timeout_detected, 1, 0) == 0) return 0; num_impacted_contexts = resp.payload.engine_reset_done.num_impacted_contexts; diff --git a/drivers/accel/ivpu/ivpu_job.h b/drivers/accel/ivpu/ivpu_job.h index 3ab61e6a5616c..d8dbce82447a1 100644 --- a/drivers/accel/ivpu/ivpu_job.h +++ b/drivers/accel/ivpu/ivpu_job.h @@ -6,8 +6,10 @@ #ifndef __IVPU_JOB_H__ #define __IVPU_JOB_H__ -#include #include +#include +#include +#include #include "ivpu_gem.h" @@ -47,6 +49,7 @@ struct ivpu_cmdq { * @vdev: Pointer to the VPU device * @file_priv: The client context that submitted this job * @done_fence: Fence signaled when job completes + * @destroy_node: List node for deferred resource cleanup after job completion * @cmd_buf_vpu_addr: VPU address of the command buffer for this job * @cmdq_id: Command queue ID used for submission * @job_id: Unique job ID for tracking and status reporting @@ -61,6 +64,7 @@ struct ivpu_job { struct ivpu_device *vdev; struct ivpu_file_priv *file_priv; struct dma_fence *done_fence; + struct llist_node destroy_node; u64 cmd_buf_vpu_addr; u32 cmdq_id; u32 job_id; @@ -87,6 +91,7 @@ void ivpu_job_done_consumer_init(struct ivpu_device *vdev); void ivpu_job_done_consumer_fini(struct ivpu_device *vdev); bool ivpu_job_handle_engine_error(struct ivpu_device *vdev, u32 job_id, u32 job_status); void ivpu_context_abort_work_fn(struct work_struct *work); +void ivpu_job_destroy_work_fn(struct work_struct *work); void ivpu_jobs_abort_all(struct ivpu_device *vdev); diff --git a/drivers/accel/ivpu/ivpu_mmu.c b/drivers/accel/ivpu/ivpu_mmu.c index 41efd8985fa67..b2025274f91de 100644 --- a/drivers/accel/ivpu/ivpu_mmu.c +++ b/drivers/accel/ivpu/ivpu_mmu.c @@ -964,7 +964,6 @@ void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev) file_priv = xa_load(&vdev->context_xa, ssid); if (file_priv) { if (!READ_ONCE(file_priv->has_mmu_faults)) { - atomic_set(&vdev->faults_detected, 1); ivpu_mmu_dump_event(vdev, event); WRITE_ONCE(file_priv->has_mmu_faults, true); } diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c index c1ce8329790e0..de0becbfdffb2 100644 --- a/drivers/accel/ivpu/ivpu_pm.c +++ b/drivers/accel/ivpu/ivpu_pm.c @@ -229,6 +229,7 @@ static void ivpu_job_timeout_work(struct work_struct *work) ivpu_jsm_state_dump(vdev); ivpu_dev_coredump(vdev); + atomic_set(&vdev->job_timeout_detected, 1); queue_work(system_percpu_wq, &vdev->context_abort_work); } diff --git a/drivers/accel/qaic/qaic_control.c b/drivers/accel/qaic/qaic_control.c index 50bf3340e49ce..2ccc55486aac0 100644 --- a/drivers/accel/qaic/qaic_control.c +++ b/drivers/accel/qaic/qaic_control.c @@ -963,11 +963,13 @@ static int decode_status(struct qaic_device *qdev, void *trans, struct manage_ms static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg, struct wire_msg *msg, struct ioctl_resources *resources, - struct qaic_user *usr) + struct qaic_user *usr, bool orphaned_deactivate) { + u32 msg_hdr_count = le32_to_cpu(msg->hdr.count); u32 msg_hdr_len = le32_to_cpu(msg->hdr.len); struct wire_trans_hdr *trans_hdr; u32 msg_len = 0; + int trans_type; int ret; int i; @@ -975,10 +977,12 @@ static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg, msg_hdr_len > QAIC_MANAGE_MAX_MSG_LENGTH) return -EINVAL; - user_msg->len = 0; - user_msg->count = le32_to_cpu(msg->hdr.count); + if (user_msg) { + user_msg->len = 0; + user_msg->count = msg_hdr_count; + } - for (i = 0; i < user_msg->count; ++i) { + for (i = 0; i < msg_hdr_count; ++i) { u32 hdr_len; if (msg_len > msg_hdr_len - sizeof(*trans_hdr)) @@ -990,7 +994,20 @@ static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg, size_add(msg_len, hdr_len) > msg_hdr_len) return -EINVAL; - switch (le32_to_cpu(trans_hdr->type)) { + trans_type = le32_to_cpu(trans_hdr->type); + /* + * orphaned_deactivate is the case where a deactivate response + * is received from the device after the user owning the DBC, + * and the message requesting deactivation, has gone away. + * In this case, only process QAIC_TRANS_DEACTIVATE_FROM_DEV + * transaction and skip the others. + */ + if (orphaned_deactivate && trans_type != QAIC_TRANS_DEACTIVATE_FROM_DEV) { + msg_len += hdr_len; + continue; + } + + switch (trans_type) { case QAIC_TRANS_PASSTHROUGH_FROM_DEV: ret = decode_passthrough(qdev, trans_hdr, user_msg, &msg_len); break; @@ -1281,7 +1298,7 @@ static int qaic_manage(struct qaic_device *qdev, struct qaic_user *usr, struct m goto dma_cont_failed; } - ret = decode_message(qdev, user_msg, rsp, &resources, usr); + ret = decode_message(qdev, user_msg, rsp, &resources, usr, false); dma_cont_failed: free_dbc_buf(qdev, &resources); @@ -1446,22 +1463,7 @@ static void resp_worker(struct work_struct *work) * response to the QAIC_TRANS_TERMINATE_TO_DEV transaction, * otherwise, the user can issue an soc_reset to the device. */ - u32 msg_count = le32_to_cpu(msg->hdr.count); - u32 msg_len = le32_to_cpu(msg->hdr.len); - u32 len = 0; - int j; - - for (j = 0; j < msg_count && len < msg_len; ++j) { - struct wire_trans_hdr *trans_hdr; - - trans_hdr = (struct wire_trans_hdr *)(msg->data + len); - if (le32_to_cpu(trans_hdr->type) == QAIC_TRANS_DEACTIVATE_FROM_DEV) { - if (decode_deactivate(qdev, trans_hdr, &len, NULL)) - len += le32_to_cpu(trans_hdr->len); - } else { - len += le32_to_cpu(trans_hdr->len); - } - } + decode_message(qdev, NULL, msg, NULL, NULL, true); /* request must have timed out, drop packet */ kfree(msg); } diff --git a/drivers/accel/rocket/rocket_job.c b/drivers/accel/rocket/rocket_job.c index ac51bff39833f..3141f210fcd1b 100644 --- a/drivers/accel/rocket/rocket_job.c +++ b/drivers/accel/rocket/rocket_job.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -102,6 +103,7 @@ rocket_copy_tasks(struct drm_device *dev, fail: kvfree(rjob->tasks); + rjob->tasks = NULL; return ret; } @@ -188,14 +190,19 @@ static int rocket_job_push(struct rocket_job *job) struct rocket_device *rdev = job->rdev; struct drm_gem_object **bos; struct ww_acquire_ctx acquire_ctx; + u32 bo_count; int ret = 0; - bos = kvmalloc_array(job->in_bo_count + job->out_bo_count, sizeof(void *), - GFP_KERNEL); + if (check_add_overflow(job->in_bo_count, job->out_bo_count, &bo_count)) + return -EINVAL; + + bos = kvmalloc_array(bo_count, sizeof(*bos), GFP_KERNEL); + if (!bos) + return -ENOMEM; memcpy(bos, job->in_bos, job->in_bo_count * sizeof(void *)); memcpy(&bos[job->in_bo_count], job->out_bos, job->out_bo_count * sizeof(void *)); - ret = drm_gem_lock_reservations(bos, job->in_bo_count + job->out_bo_count, &acquire_ctx); + ret = drm_gem_lock_reservations(bos, bo_count, &acquire_ctx); if (ret) goto err; @@ -220,7 +227,7 @@ static int rocket_job_push(struct rocket_job *job) rocket_attach_object_fences(job->out_bos, job->out_bo_count, job->inference_done_fence); err_unlock: - drm_gem_unlock_reservations(bos, job->in_bo_count + job->out_bo_count, &acquire_ctx); + drm_gem_unlock_reservations(bos, bo_count, &acquire_ctx); err: kvfree(bos); @@ -310,13 +317,13 @@ static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job) dma_fence_put(job->done_fence); job->done_fence = dma_fence_get(fence); - ret = pm_runtime_get_sync(core->dev); + ret = pm_runtime_resume_and_get(core->dev); if (ret < 0) - return fence; + goto err_put_fences; ret = iommu_attach_group(job->domain->domain, core->iommu_group); if (ret < 0) - return fence; + goto err_put_pm; scoped_guard(mutex, &core->job_lock) { core->in_flight_job = job; @@ -324,6 +331,14 @@ static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job) } return fence; + +err_put_pm: + pm_runtime_put(core->dev); +err_put_fences: + dma_fence_put(job->done_fence); + job->done_fence = NULL; + dma_fence_put(fence); + return ERR_PTR(ret); } static void rocket_job_handle_irq(struct rocket_core *core) @@ -549,6 +564,7 @@ static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file kref_init(&rjob->refcount); rjob->rdev = rdev; + rjob->domain = rocket_iommu_domain_get(file_priv); ret = drm_sched_job_init(&rjob->base, &file_priv->sched_entity, @@ -574,8 +590,6 @@ static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file rjob->out_bo_count = job->out_bo_handle_count; - rjob->domain = rocket_iommu_domain_get(file_priv); - ret = rocket_job_push(rjob); if (ret) goto out_cleanup_job; diff --git a/drivers/accessibility/speakup/keyhelp.c b/drivers/accessibility/speakup/keyhelp.c index 822ceac830683..e632c53d6246e 100644 --- a/drivers/accessibility/speakup/keyhelp.c +++ b/drivers/accessibility/speakup/keyhelp.c @@ -8,6 +8,7 @@ */ #include +#include #include "spk_priv.h" #include "speakup.h" @@ -111,7 +112,7 @@ static void say_key(int key) spk_msg_get(MSG_KEYNAMES_START + (key - 1))); } -static int help_init(void) +static void help_init(void) { char start = SPACE; int i; @@ -120,13 +121,19 @@ static int help_init(void) state_tbl = spk_our_keys[0] + SHIFT_TBL_SIZE + 2; for (i = 0; i < num_funcs; i++) { char *cur_funcname = spk_msg_get(MSG_FUNCNAMES_START + i); + char first_letter; - if (start == *cur_funcname) + first_letter = tolower(*cur_funcname); + + /* Accept only 'a'..'z' to index letter_offsets[] safely */ + if (first_letter < 'a' || first_letter > 'z') + continue; + + if (start == first_letter) continue; - start = *cur_funcname; + start = first_letter; letter_offsets[(start & 31) - 1] = i; } - return 0; } int spk_handle_help(struct vc_data *vc, u_char type, u_char ch, u_short key) @@ -144,7 +151,7 @@ int spk_handle_help(struct vc_data *vc, u_char type, u_char ch, u_short key) synth_printf("%s\n", spk_msg_get(MSG_LEAVING_HELP)); return 1; } - ch |= 32; /* lower case */ + ch = tolower(ch); if (ch < 'a' || ch > 'z') return -1; if (letter_offsets[ch - 'a'] == -1) { diff --git a/drivers/accessibility/speakup/kobjects.c b/drivers/accessibility/speakup/kobjects.c index 0dfdb6608e022..943ef71b1329b 100644 --- a/drivers/accessibility/speakup/kobjects.c +++ b/drivers/accessibility/speakup/kobjects.c @@ -92,9 +92,9 @@ static void report_char_chartab_status(int reset, int received, int used, if (reset) { pr_info("%s reset to defaults\n", object_type[do_characters]); } else if (received) { - len = snprintf(buf, sizeof(buf), - " updated %d of %d %s\n", - used, received, object_type[do_characters]); + len = scnprintf(buf, sizeof(buf), + " updated %d of %d %s\n", + used, received, object_type[do_characters]); if (rejected) snprintf(buf + (len - 1), sizeof(buf) - (len - 1), " with %d reject%s\n", diff --git a/drivers/accessibility/speakup/main.c b/drivers/accessibility/speakup/main.c index 78a77dd789a2e..a632a65e90371 100644 --- a/drivers/accessibility/speakup/main.c +++ b/drivers/accessibility/speakup/main.c @@ -2444,6 +2444,7 @@ static int __init speakup_init(void) mutex_lock(&spk_mutex); synth_release(); mutex_unlock(&spk_mutex); + spk_ttyio_unregister_ldisc(); speakup_kobj_exit(); error_kobjects: diff --git a/drivers/acpi/acpi_platform.c b/drivers/acpi/acpi_platform.c index a09636a4168ec..80ea7121744d0 100644 --- a/drivers/acpi/acpi_platform.c +++ b/drivers/acpi/acpi_platform.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -71,18 +72,51 @@ static struct notifier_block acpi_platform_notifier = { .notifier_call = acpi_platform_device_remove_notify, }; -static void acpi_platform_fill_resource(struct acpi_device *adev, - const struct resource *src, struct resource *dest) +static unsigned int acpi_platform_adjust_resources(struct acpi_device *adev, + struct resource *new_res, + struct resource *resources, + unsigned int count) { - struct device *parent; + unsigned int i; + + if (!(new_res->flags & (IORESOURCE_IO | IORESOURCE_MEM))) + return count; + + for (i = 0; i < count; ) { + struct resource *res = &resources[i]; + + /* + * Look for overlaps of resources of the same type that would + * cause resource insertion to fail down the road. + */ + if (__resource_contains_unbound(res, new_res) || + __resource_contains_unbound(new_res, res) || + resource_type(new_res) != resource_type(res) || + !resource_union(new_res, res, new_res)) { + i++; + continue; + } - *dest = *src; + dev_info(&adev->dev, "%pR expanded due to overlap\n", new_res); + /* + * Eliminate the previously processed resource that overlapped + * with the new one because it is not necessary any more. + */ + memmove(res, res + 1, (--count - i) * sizeof(*res)); + } + + return count; +} +static void acpi_platform_fill_resource(struct device *parent, + const struct resource *src, + struct resource *dest) +{ + *dest = *src; /* * If the device has parent we need to take its resources into * account as well because this device might consume part of those. */ - parent = acpi_get_first_physical_node(acpi_dev_parent(adev)); if (parent && dev_is_pci(parent)) dest->parent = pci_find_resource(to_pci_dev(parent), dest); } @@ -110,7 +144,8 @@ static unsigned int acpi_platform_resource_count(struct acpi_resource *ares, voi struct platform_device *acpi_create_platform_device(struct acpi_device *adev, const struct property_entry *properties) { - struct acpi_device *parent = acpi_dev_parent(adev); + struct acpi_device *p = acpi_dev_parent(adev); + struct device *parent __free(put_device) = acpi_bus_get_primary_device(p); struct platform_device *pdev = NULL; struct platform_device_info pdevinfo; const struct acpi_device_id *match; @@ -151,10 +186,14 @@ struct platform_device *acpi_create_platform_device(struct acpi_device *adev, return ERR_PTR(-ENOMEM); } count = 0; - list_for_each_entry(rentry, &resource_list, node) - acpi_platform_fill_resource(adev, rentry->res, + list_for_each_entry(rentry, &resource_list, node) { + count = acpi_platform_adjust_resources(adev, + rentry->res, + resources, + count); + acpi_platform_fill_resource(parent, rentry->res, &resources[count++]); - + } acpi_dev_free_resource_list(&resource_list); } } @@ -165,7 +204,7 @@ struct platform_device *acpi_create_platform_device(struct acpi_device *adev, * attached to it, that physical device should be the parent of the * platform device we are about to create. */ - pdevinfo.parent = parent ? acpi_get_first_physical_node(parent) : NULL; + pdevinfo.parent = parent; pdevinfo.name = dev_name(&adev->dev); pdevinfo.id = PLATFORM_DEVID_NONE; pdevinfo.res = resources; diff --git a/drivers/acpi/acpi_tad.c b/drivers/acpi/acpi_tad.c index fc43df083738c..45459a4cafae3 100644 --- a/drivers/acpi/acpi_tad.c +++ b/drivers/acpi/acpi_tad.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -86,6 +87,8 @@ static bool acpi_tad_rt_is_invalid(struct acpi_tad_rt *rt) rt->daylight > 3; } +static DEFINE_MUTEX(acpi_tad_aml_lock); + static int acpi_tad_set_real_time(struct device *dev, struct acpi_tad_rt *rt) { acpi_handle handle = ACPI_HANDLE(dev); @@ -113,6 +116,8 @@ static int acpi_tad_set_real_time(struct device *dev, struct acpi_tad_rt *rt) if (PM_RUNTIME_ACQUIRE_ERR(&pm)) return -ENXIO; + guard(mutex)(&acpi_tad_aml_lock); + status = acpi_evaluate_integer(handle, "_SRT", &arg_list, &retval); if (ACPI_FAILURE(status) || retval) return -EIO; @@ -124,30 +129,27 @@ static int acpi_tad_evaluate_grt(struct device *dev, struct acpi_tad_rt *rt) { acpi_handle handle = ACPI_HANDLE(dev); struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER }; - union acpi_object *out_obj; - struct acpi_tad_rt *data; acpi_status status; int ret = -EIO; - status = acpi_evaluate_object(handle, "_GRT", NULL, &output); - if (ACPI_FAILURE(status)) - goto out_free; - - out_obj = output.pointer; - if (out_obj->type != ACPI_TYPE_BUFFER) - goto out_free; - - if (out_obj->buffer.length != sizeof(*rt)) - goto out_free; - - data = (struct acpi_tad_rt *)(out_obj->buffer.pointer); - if (!data->valid) - goto out_free; - - memcpy(rt, data, sizeof(*rt)); - ret = 0; + guard(mutex)(&acpi_tad_aml_lock); -out_free: + status = acpi_evaluate_object(handle, "_GRT", NULL, &output); + if (ACPI_SUCCESS(status)) { + union acpi_object *out_obj; + + out_obj = output.pointer; + if (out_obj->type == ACPI_TYPE_BUFFER && + out_obj->buffer.length == sizeof(*rt)) { + struct acpi_tad_rt *data; + + data = (struct acpi_tad_rt *)(out_obj->buffer.pointer); + if (data->valid) { + memcpy(rt, data, sizeof(*rt)); + ret = 0; + } + } + } ACPI_FREE(output.pointer); return ret; } @@ -193,6 +195,8 @@ static int __acpi_tad_wake_set(struct device *dev, char *method, u32 timer_id, args[0].integer.value = timer_id; args[1].integer.value = value; + guard(mutex)(&acpi_tad_aml_lock); + status = acpi_evaluate_integer(handle, method, &arg_list, &retval); if (ACPI_FAILURE(status) || retval) return -EIO; @@ -215,6 +219,8 @@ static int __acpi_tad_wake_read(struct device *dev, char *method, u32 timer_id, args[0].integer.value = timer_id; + guard(mutex)(&acpi_tad_aml_lock); + status = acpi_evaluate_integer(handle, method, &arg_list, retval); if (ACPI_FAILURE(status)) return -EIO; @@ -416,6 +422,8 @@ static int acpi_tad_clear_status(struct device *dev, u32 timer_id) if (PM_RUNTIME_ACQUIRE_ERR(&pm)) return -ENXIO; + guard(mutex)(&acpi_tad_aml_lock); + status = acpi_evaluate_integer(handle, "_CWS", &arg_list, &retval); if (ACPI_FAILURE(status) || retval) return -EIO; @@ -456,6 +464,8 @@ static ssize_t acpi_tad_status_read(struct device *dev, char *buf, u32 timer_id) if (PM_RUNTIME_ACQUIRE_ERR(&pm)) return -ENXIO; + guard(mutex)(&acpi_tad_aml_lock); + status = acpi_evaluate_integer(handle, "_GWS", &arg_list, &retval); if (ACPI_FAILURE(status)) return -EIO; diff --git a/drivers/acpi/apei/erst.c b/drivers/acpi/apei/erst.c index bf65e34615314..15ab797641cfc 100644 --- a/drivers/acpi/apei/erst.c +++ b/drivers/acpi/apei/erst.c @@ -108,7 +108,7 @@ static inline u64 erst_get_timeout(void) if (erst_erange.attr & ERST_RANGE_SLOW) { timeout = ((erst_erange.timings & ERST_EXEC_TIMING_MAX_MASK) >> - ERST_EXEC_TIMING_MAX_SHIFT) * NSEC_PER_MSEC; + ERST_EXEC_TIMING_MAX_SHIFT) * NSEC_PER_USEC; if (timeout < FIRMWARE_TIMEOUT) timeout = FIRMWARE_TIMEOUT; } diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 3236a3ce79d6b..c40df9a3b37cd 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -576,7 +576,7 @@ static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata, return false; p = (char *)(err + 1); - length -= sizeof(err); + length -= sizeof(*err); for (i = 0; i < err->err_info_num; i++) { struct cper_arm_err_info *err_info; @@ -749,7 +749,7 @@ static DEFINE_KFIFO(cxl_cper_prot_err_fifo, struct cxl_cper_prot_err_work_data, CXL_CPER_PROT_ERR_FIFO_DEPTH); /* Synchronize schedule_work() with cxl_cper_prot_err_work changes */ -static DEFINE_SPINLOCK(cxl_cper_prot_err_work_lock); +static DEFINE_RAW_SPINLOCK(cxl_cper_prot_err_work_lock); struct work_struct *cxl_cper_prot_err_work; static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, @@ -761,7 +761,7 @@ static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, if (cxl_cper_sec_prot_err_valid(prot_err)) return; - guard(spinlock_irqsave)(&cxl_cper_prot_err_work_lock); + guard(raw_spinlock_irqsave)(&cxl_cper_prot_err_work_lock); if (!cxl_cper_prot_err_work) return; @@ -780,10 +780,11 @@ static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, int cxl_cper_register_prot_err_work(struct work_struct *work) { - if (cxl_cper_prot_err_work) - return -EINVAL; + guard(raw_spinlock_irqsave)(&cxl_cper_prot_err_work_lock); - guard(spinlock)(&cxl_cper_prot_err_work_lock); + if (WARN_ONCE(cxl_cper_prot_err_work, + "CPER-CXL kfifo consumer already registered\n")) + return -EINVAL; cxl_cper_prot_err_work = work; return 0; } @@ -791,11 +792,18 @@ EXPORT_SYMBOL_NS_GPL(cxl_cper_register_prot_err_work, "CXL"); int cxl_cper_unregister_prot_err_work(struct work_struct *work) { - if (cxl_cper_prot_err_work != work) - return -EINVAL; + scoped_guard(raw_spinlock_irqsave, &cxl_cper_prot_err_work_lock) { + if (WARN_ONCE(cxl_cper_prot_err_work != work, + "CPER-CXL kfifo consumer mismatch on unregister\n")) + return -EINVAL; + cxl_cper_prot_err_work = NULL; + } + + cancel_work_sync(work); + + /* Discard stale entries so they are not replayed on next module load */ + kfifo_reset(&cxl_cper_prot_err_fifo); - guard(spinlock)(&cxl_cper_prot_err_work_lock); - cxl_cper_prot_err_work = NULL; return 0; } EXPORT_SYMBOL_NS_GPL(cxl_cper_unregister_prot_err_work, "CXL"); @@ -811,7 +819,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_cper_prot_err_kfifo_get, "CXL"); DEFINE_KFIFO(cxl_cper_fifo, struct cxl_cper_work_data, CXL_CPER_FIFO_DEPTH); /* Synchronize schedule_work() with cxl_cper_work changes */ -static DEFINE_SPINLOCK(cxl_cper_work_lock); +static DEFINE_RAW_SPINLOCK(cxl_cper_work_lock); struct work_struct *cxl_cper_work; static void cxl_cper_post_event(enum cxl_event_type event_type, @@ -831,7 +839,7 @@ static void cxl_cper_post_event(enum cxl_event_type event_type, return; } - guard(spinlock_irqsave)(&cxl_cper_work_lock); + guard(raw_spinlock_irqsave)(&cxl_cper_work_lock); if (!cxl_cper_work) return; @@ -849,10 +857,11 @@ static void cxl_cper_post_event(enum cxl_event_type event_type, int cxl_cper_register_work(struct work_struct *work) { - if (cxl_cper_work) + guard(raw_spinlock_irqsave)(&cxl_cper_work_lock); + if (WARN_ONCE(cxl_cper_work, + "CXL CPER kfifo consumer already registered\n")) return -EINVAL; - guard(spinlock)(&cxl_cper_work_lock); cxl_cper_work = work; return 0; } @@ -860,11 +869,18 @@ EXPORT_SYMBOL_NS_GPL(cxl_cper_register_work, "CXL"); int cxl_cper_unregister_work(struct work_struct *work) { - if (cxl_cper_work != work) - return -EINVAL; + scoped_guard(raw_spinlock_irqsave, &cxl_cper_work_lock) { + if (WARN_ONCE(cxl_cper_work != work, + "CXL CPER kfifo consumer mismatch on unregister\n")) + return -EINVAL; + cxl_cper_work = NULL; + } + + cancel_work_sync(work); + + /* Discard stale entries so they are not replayed on next module load */ + kfifo_reset(&cxl_cper_fifo); - guard(spinlock)(&cxl_cper_work_lock); - cxl_cper_work = NULL; return 0; } EXPORT_SYMBOL_NS_GPL(cxl_cper_unregister_work, "CXL"); diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index f5e0eb299610b..365626827d61c 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -97,6 +98,9 @@ struct acpi_battery { struct device *phys_dev; struct notifier_block pm_nb; struct list_head list; + unsigned long flags; + + struct mutex property_lock; /* Protects properties below. */ unsigned long update_time; int revision; int rate_now; @@ -123,7 +127,6 @@ struct acpi_battery { char oem_info[MAX_STRING_LENGTH]; int state; int power_unit; - unsigned long flags; }; #define to_acpi_battery(x) power_supply_get_drvdata(x) @@ -150,27 +153,28 @@ static int acpi_battery_technology(struct acpi_battery *battery) static int acpi_battery_get_state(struct acpi_battery *battery); -static int acpi_battery_is_charged(struct acpi_battery *battery) +static bool acpi_battery_is_full(struct acpi_battery *battery) { - /* charging, discharging, critical low or charge limited */ - if (battery->state != 0) - return 0; - /* battery not reporting charge */ if (battery->capacity_now == ACPI_BATTERY_VALUE_UNKNOWN || battery->capacity_now == 0) - return 0; + return false; /* good batteries update full_charge as the batteries degrade */ if (battery->full_charge_capacity == battery->capacity_now) - return 1; + return true; /* fallback to using design values for broken batteries */ - if (battery->design_capacity <= battery->capacity_now) - return 1; + return battery->design_capacity <= battery->capacity_now; +} - /* we don't do any sort of metric based on percentages */ - return 0; +static int acpi_battery_is_charged(struct acpi_battery *battery) +{ + /* charging, discharging, critical low or charge limited */ + if (battery->state != 0) + return 0; + + return acpi_battery_is_full(battery); } static bool acpi_battery_is_degraded(struct acpi_battery *battery) @@ -180,20 +184,6 @@ static bool acpi_battery_is_degraded(struct acpi_battery *battery) battery->full_charge_capacity < battery->design_capacity; } -static int acpi_battery_handle_discharging(struct acpi_battery *battery) -{ - /* - * Some devices wrongly report discharging if the battery's charge level - * was above the device's start charging threshold atm the AC adapter - * was plugged in and the device thus did not start a new charge cycle. - */ - if ((battery_ac_is_broken || power_supply_is_system_supplied()) && - battery->rate_now == 0) - return POWER_SUPPLY_STATUS_NOT_CHARGING; - - return POWER_SUPPLY_STATUS_DISCHARGING; -} - static int acpi_battery_get_property(struct power_supply *psy, enum power_supply_property psp, union power_supply_propval *val) @@ -201,23 +191,50 @@ static int acpi_battery_get_property(struct power_supply *psy, int full_capacity = ACPI_BATTERY_VALUE_UNKNOWN, ret = 0; struct acpi_battery *battery = to_acpi_battery(psy); - if (acpi_battery_present(battery)) { - /* run battery update only if it is present */ - acpi_battery_get_state(battery); - } else if (psp != POWER_SUPPLY_PROP_PRESENT) - return -ENODEV; + /* run battery update only if it is present */ + if (!acpi_battery_present(battery)) { + switch (psp) { + case POWER_SUPPLY_PROP_PRESENT: + val->intval = 0; + return 0; + default: + return -ENODEV; + } + } + + mutex_lock(&battery->property_lock); + + acpi_battery_get_state(battery); + switch (psp) { case POWER_SUPPLY_PROP_STATUS: + /* + * Some devices wrongly report discharging if the battery's charge level + * was above the device's start charging threshold atm the AC adapter + * was plugged in and the device thus did not start a new charge cycle. + */ if (battery->state & ACPI_BATTERY_STATE_DISCHARGING) - val->intval = acpi_battery_handle_discharging(battery); - else if (battery->state & ACPI_BATTERY_STATE_CHARGING) - /* Validate the status by checking the current. */ - if (battery->rate_now != ACPI_BATTERY_VALUE_UNKNOWN && - battery->rate_now == 0) { - /* On charge but no current (0W/0mA). */ + if (battery->rate_now != 0) { + val->intval = POWER_SUPPLY_STATUS_DISCHARGING; + } else if (battery_ac_is_broken) { val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; } else { + mutex_unlock(&battery->property_lock); + + val->intval = power_supply_is_system_supplied() + ? POWER_SUPPLY_STATUS_NOT_CHARGING + : POWER_SUPPLY_STATUS_DISCHARGING; + return 0; + } + else if (battery->state & ACPI_BATTERY_STATE_CHARGING) + /* Check the rate and capacity to validate the status. */ + if (!acpi_battery_is_full(battery) || + (battery->rate_now != ACPI_BATTERY_VALUE_UNKNOWN && + battery->rate_now > 0)) { val->intval = POWER_SUPPLY_STATUS_CHARGING; + } else { + /* Full and zero rate. */ + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; } else if (battery->state & ACPI_BATTERY_STATE_CHARGE_LIMITING) val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; @@ -311,6 +328,8 @@ static int acpi_battery_get_property(struct power_supply *psy, default: ret = -EINVAL; } + + mutex_unlock(&battery->property_lock); return ret; } @@ -533,6 +552,8 @@ static int acpi_battery_get_info(struct acpi_battery *battery) int use_bix; int result = -ENODEV; + lockdep_assert_held(&battery->property_lock); + if (!acpi_battery_present(battery)) return 0; @@ -572,6 +593,8 @@ static int acpi_battery_get_state(struct acpi_battery *battery) acpi_status status = 0; struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + lockdep_assert_held(&battery->property_lock); + if (!acpi_battery_present(battery)) return 0; @@ -625,6 +648,8 @@ static int acpi_battery_set_alarm(struct acpi_battery *battery) { acpi_status status = 0; + lockdep_assert_held(&battery->property_lock); + if (!acpi_battery_present(battery) || !test_bit(ACPI_BATTERY_ALARM_PRESENT, &battery->flags)) return -ENODEV; @@ -642,6 +667,8 @@ static int acpi_battery_set_alarm(struct acpi_battery *battery) static int acpi_battery_init_alarm(struct acpi_battery *battery) { + lockdep_assert_held(&battery->property_lock); + /* See if alarms are supported, and if so, set default */ if (!acpi_has_method(battery->device->handle, "_BTP")) { clear_bit(ACPI_BATTERY_ALARM_PRESENT, &battery->flags); @@ -659,6 +686,8 @@ static ssize_t acpi_battery_alarm_show(struct device *dev, { struct acpi_battery *battery = to_acpi_battery(dev_get_drvdata(dev)); + guard(mutex)(&battery->property_lock); + return sysfs_emit(buf, "%d\n", battery->alarm * 1000); } @@ -668,9 +697,15 @@ static ssize_t acpi_battery_alarm_store(struct device *dev, { unsigned long x; struct acpi_battery *battery = to_acpi_battery(dev_get_drvdata(dev)); + int err; + + err = kstrtoul(buf, 10, &x); + if (err) + return err; - if (sscanf(buf, "%lu\n", &x) == 1) - battery->alarm = x/1000; + guard(mutex)(&battery->property_lock); + + battery->alarm = x / 1000; if (acpi_battery_present(battery)) acpi_battery_set_alarm(battery); return count; @@ -854,12 +889,17 @@ static int sysfs_add_battery(struct acpi_battery *battery) .no_wakeup_source = true, }; bool full_cap_broken = false; + int power_unit; - if (!ACPI_BATTERY_CAPACITY_VALID(battery->full_charge_capacity) && - !ACPI_BATTERY_CAPACITY_VALID(battery->design_capacity)) - full_cap_broken = true; + scoped_guard(mutex, &battery->property_lock) { + power_unit = battery->power_unit; - if (battery->power_unit == ACPI_BATTERY_POWER_UNIT_MA) { + if (!ACPI_BATTERY_CAPACITY_VALID(battery->full_charge_capacity) && + !ACPI_BATTERY_CAPACITY_VALID(battery->design_capacity)) + full_cap_broken = true; + } + + if (power_unit == ACPI_BATTERY_POWER_UNIT_MA) { if (full_cap_broken) { battery->bat_desc.properties = charge_battery_full_cap_broken_props; @@ -913,6 +953,9 @@ static void sysfs_remove_battery(struct acpi_battery *battery) static void find_battery(const struct dmi_header *dm, void *private) { struct acpi_battery *battery = (struct acpi_battery *)private; + + lockdep_assert_held(&battery->property_lock); + /* Note: the hardcoded offsets below have been extracted from * the source code of dmidecode. */ @@ -944,6 +987,8 @@ static void find_battery(const struct dmi_header *dm, void *private) */ static void acpi_battery_quirks(struct acpi_battery *battery) { + lockdep_assert_held(&battery->property_lock); + if (test_bit(ACPI_BATTERY_QUIRK_PERCENTAGE_CAPACITY, &battery->flags)) return; @@ -996,30 +1041,38 @@ static void acpi_battery_quirks(struct acpi_battery *battery) static int acpi_battery_update(struct acpi_battery *battery, bool resume) { int result = acpi_battery_get_status(battery); + bool wakeup; if (result) return result; if (!acpi_battery_present(battery)) { sysfs_remove_battery(battery); - battery->update_time = 0; + scoped_guard(mutex, &battery->property_lock) + battery->update_time = 0; return 0; } if (resume) return 0; - if (!battery->update_time) { - result = acpi_battery_get_info(battery); + scoped_guard(mutex, &battery->property_lock) { + if (!battery->update_time) { + result = acpi_battery_get_info(battery); + if (result) + return result; + acpi_battery_init_alarm(battery); + } + + result = acpi_battery_get_state(battery); if (result) return result; - acpi_battery_init_alarm(battery); - } + acpi_battery_quirks(battery); - result = acpi_battery_get_state(battery); - if (result) - return result; - acpi_battery_quirks(battery); + wakeup = ((battery->state & ACPI_BATTERY_STATE_CRITICAL) || + (test_bit(ACPI_BATTERY_ALARM_PRESENT, &battery->flags) && + (battery->capacity_now <= battery->alarm))); + } if (!battery->bat) { result = sysfs_add_battery(battery); @@ -1031,9 +1084,7 @@ static int acpi_battery_update(struct acpi_battery *battery, bool resume) * Wakeup the system if battery is critical low * or lower than the alarm level */ - if ((battery->state & ACPI_BATTERY_STATE_CRITICAL) || - (test_bit(ACPI_BATTERY_ALARM_PRESENT, &battery->flags) && - (battery->capacity_now <= battery->alarm))) + if (wakeup) acpi_pm_wakeup_event(battery->phys_dev); return result; @@ -1046,12 +1097,14 @@ static void acpi_battery_refresh(struct acpi_battery *battery) if (!battery->bat) return; - power_unit = battery->power_unit; + scoped_guard(mutex, &battery->property_lock) { + power_unit = battery->power_unit; - acpi_battery_get_info(battery); + acpi_battery_get_info(battery); - if (power_unit == battery->power_unit) - return; + if (power_unit == battery->power_unit) + return; + } /* The battery has changed its reporting units. */ sysfs_remove_battery(battery); @@ -1106,17 +1159,21 @@ static int battery_notify(struct notifier_block *nb, } else { int result; - result = acpi_battery_get_info(battery); - if (result) - return result; + scoped_guard(mutex, &battery->property_lock) { + result = acpi_battery_get_info(battery); + if (result) + return result; + } result = sysfs_add_battery(battery); if (result) return result; } - acpi_battery_init_alarm(battery); - acpi_battery_get_state(battery); + scoped_guard(mutex, &battery->property_lock) { + acpi_battery_init_alarm(battery); + acpi_battery_get_state(battery); + } } return 0; @@ -1258,6 +1315,10 @@ static int acpi_battery_probe(struct platform_device *pdev) if (result) return result; + result = devm_mutex_init(&pdev->dev, &battery->property_lock); + if (result) + return result; + if (acpi_has_method(battery->device->handle, "_BIX")) set_bit(ACPI_BATTERY_XINFO_PRESENT, &battery->flags); diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index a30a904f6535f..7a2458b00a661 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -335,7 +335,7 @@ static int acpi_osc_handshake(acpi_handle handle, const char *uuid_str, .length = bufsize * sizeof(u32), }; struct acpi_buffer output; - u32 *retbuf, test; + u32 *retbuf, test, errors; guid_t guid; int ret, i; @@ -395,10 +395,18 @@ static int acpi_osc_handshake(acpi_handle handle, const char *uuid_str, * Clear the feature bits in capbuf[] that have not been acknowledged. * After that, capbuf[] contains the resultant feature mask. */ - for (i = OSC_QUERY_DWORD + 1; i < bufsize; i++) + for (i = OSC_QUERY_DWORD + 1, test = 0; i < bufsize; i++) { + test |= capbuf[i] & ~retbuf[i]; capbuf[i] &= retbuf[i]; + } - if (retbuf[OSC_QUERY_DWORD] & OSC_ERROR_MASK) { + errors = retbuf[OSC_QUERY_DWORD] & OSC_ERROR_MASK; + /* + * Some platforms set OSC_CAPABILITIES_MASK_ERROR even though they + * acknowledge all of the requested features, so avoid complaining in + * those cases unless any other error bits are also set. + */ + if (errors && (test || errors != OSC_CAPABILITIES_MASK_ERROR)) { /* * Complain about the unexpected errors and print diagnostic * information related to them. @@ -801,30 +809,52 @@ static int __init acpi_setup_sb_notify_handler(void) Device Matching -------------------------------------------------------------------------- */ + +static struct device *primary_physical_device(struct acpi_device *adev) +{ + struct acpi_device_physical_node *pn; + + pn = list_first_entry_or_null(&adev->physical_node_list, + struct acpi_device_physical_node, node); + if (pn) + return pn->dev; + + return NULL; +} + +/** + * acpi_bus_get_primary_device - Get first physical device for a given ACPI one + * @adev: ACPI device to get the first physical device for. + * + * Find the first physical device for which @adev is the ACPI companion and + * reference count it if present. + * + * Return: Pointer to the first physical counterpart of @adev or NULL if there + * are none. Callers are responsible for invoking put_device() on the returned + * device. + */ +struct device *acpi_bus_get_primary_device(struct acpi_device *adev) +{ + if (!adev) + return NULL; + + guard(mutex)(&adev->physical_node_lock); + + return get_device(primary_physical_device(adev)); +} +EXPORT_SYMBOL_GPL(acpi_bus_get_primary_device); + /** - * acpi_get_first_physical_node - Get first physical node of an ACPI device + * acpi_get_first_physical_node - Find first physical node of an ACPI device * @adev: ACPI device in question * * Return: First physical node of ACPI device @adev */ struct device *acpi_get_first_physical_node(struct acpi_device *adev) { - struct mutex *physical_node_lock = &adev->physical_node_lock; - struct device *phys_dev; - - mutex_lock(physical_node_lock); - if (list_empty(&adev->physical_node_list)) { - phys_dev = NULL; - } else { - const struct acpi_device_physical_node *node; - - node = list_first_entry(&adev->physical_node_list, - struct acpi_device_physical_node, node); + guard(mutex)(&adev->physical_node_lock); - phys_dev = node->dev; - } - mutex_unlock(physical_node_lock); - return phys_dev; + return primary_physical_device(adev); } EXPORT_SYMBOL_GPL(acpi_get_first_physical_node); diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 53d09ca98f066..becb7e442b30e 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1316,15 +1316,30 @@ static int cppc_set_reg_val(int cpu, enum cppc_regs reg_idx, u64 val) return cpc_write(cpu, reg, val); } +static bool cppc_desired_perf_readable(const struct cpc_desc *cpc_desc) +{ + return cpc_desc->version < CPPC_V4_REV; +} + /** * cppc_get_desired_perf - Get the desired performance register value. * @cpunum: CPU from which to get desired performance. * @desired_perf: Return address. * - * Return: 0 for success, -EIO otherwise. + * Return: 0 for success, -EOPNOTSUPP for _CPC revision 4 or later, and a + * negative errno otherwise. */ int cppc_get_desired_perf(int cpunum, u64 *desired_perf) { + struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); + + if (!cpc_desc) + return -ENODEV; + + /* _CPC revision 4 no longer specifies Desired Performance as readable. */ + if (!cppc_desired_perf_readable(cpc_desc)) + return -EOPNOTSUPP; + return cppc_get_reg_val(cpunum, DESIRED_PERF, desired_perf); } EXPORT_SYMBOL_GPL(cppc_get_desired_perf); @@ -1830,12 +1845,14 @@ int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) u64 desired_perf = 0, min = 0, max = 0, energy_perf = 0, auto_sel = 0; int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); struct cppc_pcc_data *pcc_ss_data = NULL; + bool read_desired_perf; int ret = 0, regs_in_pcc = 0; if (!cpc_desc) { pr_debug("No CPC descriptor for CPU:%d\n", cpu); return -ENODEV; } + read_desired_perf = cppc_desired_perf_readable(cpc_desc); if (!perf_ctrls) { pr_debug("Invalid perf_ctrls pointer\n"); @@ -1849,7 +1866,8 @@ int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; /* Are any of the regs PCC ?*/ - if (CPC_IN_PCC(desired_perf_reg) || CPC_IN_PCC(min_perf_reg) || + if ((read_desired_perf && CPC_IN_PCC(desired_perf_reg)) || + CPC_IN_PCC(min_perf_reg) || CPC_IN_PCC(max_perf_reg) || CPC_IN_PCC(energy_perf_reg) || CPC_IN_PCC(auto_sel_reg)) { if (pcc_ss_id < 0) { @@ -1881,7 +1899,7 @@ int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) } perf_ctrls->min_perf = min; - if (CPC_SUPPORTED(desired_perf_reg)) { + if (read_desired_perf && CPC_SUPPORTED(desired_perf_reg)) { ret = cpc_read(cpu, desired_perf_reg, &desired_perf); if (ret) goto out_err; diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 64ad4cfa6208b..a89f10256dbb8 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -1510,6 +1510,24 @@ static bool install_gpio_irq_event_handler(struct acpi_ec *ec) IRQF_SHARED | IRQF_ONESHOT, "ACPI EC", ec) >= 0; } +static int ec_prepare_gpio_irq(struct acpi_ec *ec, struct acpi_device *device) +{ + int irq; + + if (!device || ec->gpe >= 0 || ec->irq >= 0) + return 0; + + /* ACPI reduced hardware platforms use a GpioInt from _CRS. */ + irq = acpi_dev_gpio_irq_get(device, 0); + if (irq == -EPROBE_DEFER) + return irq; + + if (irq >= 0) + ec->irq = irq; + + return 0; +} + /** * ec_install_handlers - Install service callbacks and register query methods. * @ec: Target EC. @@ -1524,7 +1542,6 @@ static bool install_gpio_irq_event_handler(struct acpi_ec *ec) * Return: * -ENODEV if the address space handler cannot be installed, which means * "unable to handle transactions", - * -EPROBE_DEFER if GPIO IRQ acquisition needs to be deferred, * or 0 (success) otherwise. */ static int ec_install_handlers(struct acpi_ec *ec, struct acpi_device *device, @@ -1557,19 +1574,6 @@ static int ec_install_handlers(struct acpi_ec *ec, struct acpi_device *device, if (!device) return 0; - if (ec->gpe < 0) { - /* ACPI reduced hardware platforms use a GpioInt from _CRS. */ - int irq = acpi_dev_gpio_irq_get(device, 0); - /* - * Bail out right away for deferred probing or complete the - * initialization regardless of any other errors. - */ - if (irq == -EPROBE_DEFER) - return -EPROBE_DEFER; - else if (irq >= 0) - ec->irq = irq; - } - if (!test_bit(EC_FLAGS_QUERY_METHODS_INSTALLED, &ec->flags)) { /* Find and register all query methods */ acpi_walk_namespace(ACPI_TYPE_METHOD, ec->handle, 1, @@ -1647,6 +1651,14 @@ static int acpi_ec_setup(struct acpi_ec *ec, struct acpi_device *device, bool ca { int ret; + /* + * GPIO IRQ lookup can defer. Do it before publishing the EC + * OpRegion to AML to avoid a spurious _REG(disconnect). + */ + ret = ec_prepare_gpio_irq(ec, device); + if (ret) + return ret; + /* First EC capable of handling transactions */ if (!first_ec) first_ec = ec; diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index 4c06c3ffd0cbb..d836f46319f26 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -574,6 +574,13 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm) return; } + if (!is_pcie(root) && !is_cxl(root) && !acpi_has_method(handle, "_OSC")) { + dev_dbg(&device->dev, "Non-PCIe host bridge without _OSC, skipping\n"); + + *no_aspm = 1; + return; + } + support = calculate_support(); decode_osc_support(root, "OS supports", support); @@ -615,10 +622,6 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm) */ *no_aspm = 1; - /* _OSC is optional for PCI host bridges */ - if (status == AE_NOT_FOUND && !is_pcie(root)) - return; - if (control) { decode_osc_control(root, "OS requested", requested); decode_osc_control(root, "platform willing to grant", control); @@ -725,7 +728,6 @@ static int acpi_pci_root_add(struct acpi_device *device, dev_err(&device->dev, "Bus %04x:%02x not present in PCI namespace\n", root->segment, (unsigned int)root->secondary.start); - device->driver_data = NULL; result = -ENODEV; goto remove_dmar; } @@ -765,6 +767,7 @@ static int acpi_pci_root_add(struct acpi_device *device, if (hotadd) dmar_device_remove(handle); end: + device->driver_data = NULL; kfree(root); return result; } @@ -788,6 +791,7 @@ static void acpi_pci_root_remove(struct acpi_device *device) pci_unlock_rescan_remove(); + device->driver_data = NULL; kfree(root); } diff --git a/drivers/acpi/pfr_update.c b/drivers/acpi/pfr_update.c index 6283105bb0e8b..9afd2c52fdbd8 100644 --- a/drivers/acpi/pfr_update.c +++ b/drivers/acpi/pfr_update.c @@ -120,7 +120,7 @@ static int query_capability(struct pfru_update_cap_info *cap_hdr, struct pfru_device *pfru_dev) { acpi_handle handle = ACPI_HANDLE(pfru_dev->parent_dev); - union acpi_object *out_obj; + union acpi_object *out_obj, *elem; int ret = -EINVAL; out_obj = acpi_evaluate_dsm_typed(handle, &pfru_guid, @@ -150,7 +150,9 @@ static int query_capability(struct pfru_update_cap_info *cap_hdr, goto free_acpi_buffer; } - cap_hdr->status = out_obj->package.elements[CAP_STATUS_IDX].integer.value; + elem = out_obj->package.elements; + + cap_hdr->status = elem[CAP_STATUS_IDX].integer.value; if (cap_hdr->status != DSM_SUCCEED) { ret = -EBUSY; dev_dbg(pfru_dev->parent_dev, "Query cap Error Status:%d\n", @@ -158,29 +160,30 @@ static int query_capability(struct pfru_update_cap_info *cap_hdr, goto free_acpi_buffer; } - cap_hdr->update_cap = out_obj->package.elements[CAP_UPDATE_IDX].integer.value; + if (elem[CAP_CODE_TYPE_IDX].buffer.length > sizeof(cap_hdr->code_type) || + elem[CAP_DRV_TYPE_IDX].buffer.length > sizeof(cap_hdr->drv_type) || + elem[CAP_PLAT_ID_IDX].buffer.length > sizeof(cap_hdr->platform_id) || + elem[CAP_OEM_ID_IDX].buffer.length > sizeof(cap_hdr->oem_id)) + goto free_acpi_buffer; + + cap_hdr->update_cap = elem[CAP_UPDATE_IDX].integer.value; memcpy(&cap_hdr->code_type, - out_obj->package.elements[CAP_CODE_TYPE_IDX].buffer.pointer, - out_obj->package.elements[CAP_CODE_TYPE_IDX].buffer.length); - cap_hdr->fw_version = - out_obj->package.elements[CAP_FW_VER_IDX].integer.value; - cap_hdr->code_rt_version = - out_obj->package.elements[CAP_CODE_RT_VER_IDX].integer.value; + elem[CAP_CODE_TYPE_IDX].buffer.pointer, + elem[CAP_CODE_TYPE_IDX].buffer.length); + cap_hdr->fw_version = elem[CAP_FW_VER_IDX].integer.value; + cap_hdr->code_rt_version = elem[CAP_CODE_RT_VER_IDX].integer.value; memcpy(&cap_hdr->drv_type, - out_obj->package.elements[CAP_DRV_TYPE_IDX].buffer.pointer, - out_obj->package.elements[CAP_DRV_TYPE_IDX].buffer.length); - cap_hdr->drv_rt_version = - out_obj->package.elements[CAP_DRV_RT_VER_IDX].integer.value; - cap_hdr->drv_svn = - out_obj->package.elements[CAP_DRV_SVN_IDX].integer.value; + elem[CAP_DRV_TYPE_IDX].buffer.pointer, + elem[CAP_DRV_TYPE_IDX].buffer.length); + cap_hdr->drv_rt_version = elem[CAP_DRV_RT_VER_IDX].integer.value; + cap_hdr->drv_svn = elem[CAP_DRV_SVN_IDX].integer.value; memcpy(&cap_hdr->platform_id, - out_obj->package.elements[CAP_PLAT_ID_IDX].buffer.pointer, - out_obj->package.elements[CAP_PLAT_ID_IDX].buffer.length); + elem[CAP_PLAT_ID_IDX].buffer.pointer, + elem[CAP_PLAT_ID_IDX].buffer.length); memcpy(&cap_hdr->oem_id, - out_obj->package.elements[CAP_OEM_ID_IDX].buffer.pointer, - out_obj->package.elements[CAP_OEM_ID_IDX].buffer.length); - cap_hdr->oem_info_len = - out_obj->package.elements[CAP_OEM_INFO_IDX].buffer.length; + elem[CAP_OEM_ID_IDX].buffer.pointer, + elem[CAP_OEM_ID_IDX].buffer.length); + cap_hdr->oem_info_len = elem[CAP_OEM_INFO_IDX].buffer.length; ret = 0; diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c index a4498357bd165..3bf076c150fa1 100644 --- a/drivers/acpi/processor_core.c +++ b/drivers/acpi/processor_core.c @@ -336,11 +336,26 @@ int acpi_get_cpuid(acpi_handle handle, int type, u32 acpi_id) EXPORT_SYMBOL_GPL(acpi_get_cpuid); #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC -static int get_ioapic_id(struct acpi_subtable_header *entry, u32 gsi_base, +static bool madt_entry_is_valid(struct acpi_subtable_header *entry, + unsigned long end) +{ + unsigned long start = (unsigned long)entry; + + if (start >= end || end - start < sizeof(*entry)) + return false; + + return entry->length >= sizeof(*entry) && entry->length <= end - start; +} + +static int get_ioapic_id(struct acpi_subtable_header *entry, + const unsigned long end, u32 gsi_base, u64 *phys_addr, int *ioapic_id) { struct acpi_madt_io_apic *ioapic = (struct acpi_madt_io_apic *)entry; + if (!madt_entry_is_valid(entry, end) || BAD_MADT_ENTRY(ioapic, end)) + return 0; + if (ioapic->global_irq_base != gsi_base) return 0; @@ -361,17 +376,19 @@ static int parse_madt_ioapic_entry(u32 gsi_base, u64 *phys_addr) return apic_id; entry = (unsigned long)madt; + if (madt->header.length < sizeof(*madt)) + return apic_id; madt_end = entry + madt->header.length; /* Parse all entries looking for a match. */ entry += sizeof(struct acpi_table_madt); - while (entry + sizeof(struct acpi_subtable_header) < madt_end) { + while (madt_entry_is_valid((struct acpi_subtable_header *)entry, + madt_end)) { hdr = (struct acpi_subtable_header *)entry; if (hdr->type == ACPI_MADT_TYPE_IO_APIC && - get_ioapic_id(hdr, gsi_base, phys_addr, &apic_id)) + get_ioapic_id(hdr, madt_end, gsi_base, phys_addr, &apic_id)) break; - else - entry += hdr->length; + entry += hdr->length; } return apic_id; @@ -398,7 +415,9 @@ static int parse_mat_ioapic_entry(acpi_handle handle, u32 gsi_base, header = (struct acpi_subtable_header *)obj->buffer.pointer; if (header->type == ACPI_MADT_TYPE_IO_APIC) - get_ioapic_id(header, gsi_base, phys_addr, &apic_id); + get_ioapic_id(header, + (unsigned long)header + obj->buffer.length, + gsi_base, phys_addr, &apic_id); exit: kfree(buffer.pointer); diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index cda8fd7200004..cdc2ae1632b21 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -285,6 +285,12 @@ static int __init acpi_processor_driver_init(void) unregister_idle_drv: acpi_processor_unregister_idle_driver(); + if (acpi_processor_cpufreq_init) { + cpufreq_unregister_notifier(&acpi_processor_notifier_block, + CPUFREQ_POLICY_NOTIFIER); + acpi_processor_cpufreq_init = false; + } + return result; } diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 4482cf28f56ae..d573f201295ae 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -927,6 +927,13 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle, if (obj->type == ACPI_TYPE_BUFFER) { struct acpi_power_register *reg; + if (obj->buffer.length < sizeof(*reg)) { + acpi_handle_debug(handle, + "Invalid register data for _LPI state %d\n", + state_idx); + continue; + } + reg = (struct acpi_power_register *)obj->buffer.pointer; if (reg->space_id != ACPI_ADR_SPACE_SYSTEM_IO && reg->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) @@ -945,13 +952,6 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle, continue; } - /* elements[7,8] skipped for now i.e. Residency/Usage counter*/ - - obj = pkg_elem + 9; - if (obj->type == ACPI_TYPE_STRING) - strscpy(lpi_state->desc, obj->string.pointer, - ACPI_CX_DESC_LEN); - lpi_state->index = state_idx; if (obj_get_integer(pkg_elem + 0, &lpi_state->min_residency)) { pr_debug("No min. residency found, assuming 10 us\n"); @@ -974,6 +974,20 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle, if (obj_get_integer(pkg_elem + 5, &lpi_state->enable_parent_state)) lpi_state->enable_parent_state = 0; + + /* Skip elements [7-8] i.e. Residency/Usage counters. */ + + /* + * Avoid out-of-bounds access if the size of the package is less + * than expected. + */ + if (element->package.count < 10) + continue; + + obj = pkg_elem + 9; + if (obj->type == ACPI_TYPE_STRING) + strscpy(lpi_state->desc, obj->string.pointer, + ACPI_CX_DESC_LEN); } acpi_handle_debug(handle, "Found %d power states\n", state_idx); diff --git a/drivers/acpi/riscv/irq.c b/drivers/acpi/riscv/irq.c index 9b88d0993e880..0cdec5dd575eb 100644 --- a/drivers/acpi/riscv/irq.c +++ b/drivers/acpi/riscv/irq.c @@ -299,6 +299,7 @@ static acpi_status riscv_acpi_irq_get_parent(struct acpi_resource *ares, void *c return AE_OK; ctx->handle = riscv_acpi_get_gsi_handle(eirq->interrupts[ctx->index]); + ctx->rc = 0; return AE_CTRL_TERMINATE; } @@ -314,10 +315,22 @@ static int riscv_acpi_irq_get_dep(acpi_handle handle, unsigned int index, acpi_h acpi_walk_resources(handle, METHOD_NAME__CRS, riscv_acpi_irq_get_parent, &ctx); *gsi_handle = ctx.handle; - if (*gsi_handle) - return 1; - return 0; + return ctx.rc; +} + +static bool acpi_prt_entry_valid(void *prt_entry) +{ + struct acpi_pci_routing_table *entry = prt_entry; + + return entry && entry->length > 0; +} + +static void *acpi_prt_next_entry(void *prt_entry) +{ + struct acpi_pci_routing_table *entry = prt_entry; + + return prt_entry + entry->length; } static u32 riscv_acpi_add_prt_dep(acpi_handle handle) @@ -338,9 +351,11 @@ static u32 riscv_acpi_add_prt_dep(acpi_handle handle) } entry = buffer.pointer; - while (entry && (entry->length > 0)) { + for (; acpi_prt_entry_valid(entry); entry = acpi_prt_next_entry(entry)) { if (entry->source[0]) { - acpi_get_handle(handle, entry->source, &link_handle); + status = acpi_get_handle(handle, entry->source, &link_handle); + if (ACPI_FAILURE(status)) + continue; dep_devices.count = 1; dep_devices.handles = kzalloc_objs(*dep_devices.handles, 1); @@ -364,9 +379,6 @@ static u32 riscv_acpi_add_prt_dep(acpi_handle handle) dep_devices.handles[0] = gsi_handle; count += acpi_scan_add_dep(handle, &dep_devices); } - - entry = (struct acpi_pci_routing_table *) - ((unsigned long)entry + entry->length); } kfree(buffer.pointer); @@ -381,8 +393,11 @@ static u32 riscv_acpi_add_irq_dep(acpi_handle handle) int i; for (i = 0; - riscv_acpi_irq_get_dep(handle, i, &gsi_handle); + !riscv_acpi_irq_get_dep(handle, i, &gsi_handle); i++) { + if (!gsi_handle) + continue; + dep_devices.count = 1; dep_devices.handles = kzalloc_objs(*dep_devices.handles, 1); if (!dep_devices.handles) { diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 9a7ac2eb9ce0b..99af95f338652 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -524,12 +524,10 @@ static void acpi_device_release(struct device *dev) kfree(acpi_dev); } -static void acpi_device_del(struct acpi_device *device) +static void acpi_device_cleanup(struct acpi_device *device) { struct acpi_device_bus_id *acpi_device_bus_id; - mutex_lock(&acpi_device_lock); - list_for_each_entry(acpi_device_bus_id, &acpi_bus_id_list, node) if (!strcmp(acpi_device_bus_id->bus_id, acpi_device_hid(device))) { @@ -544,6 +542,13 @@ static void acpi_device_del(struct acpi_device *device) } list_del(&device->wakeup_list); +} + +static void acpi_device_del(struct acpi_device *device) +{ + mutex_lock(&acpi_device_lock); + + acpi_device_cleanup(device); mutex_unlock(&acpi_device_lock); @@ -803,7 +808,7 @@ int acpi_device_add(struct acpi_device *device) err: mutex_lock(&acpi_device_lock); - list_del(&device->wakeup_list); + acpi_device_cleanup(device); err_unlock: mutex_unlock(&acpi_device_lock); @@ -2208,29 +2213,27 @@ static void acpi_create_video_bus_device(struct acpi_device *adev, struct auxiliary_device *aux_dev; static unsigned int aux_dev_id; + struct device *phys_parent __free(put_device) = acpi_bus_get_primary_device(parent); + if (!phys_parent) + return; + aux_dev = kzalloc_obj(*aux_dev); if (!aux_dev) return; aux_dev->id = aux_dev_id++; aux_dev->name = "video_bus"; - aux_dev->dev.parent = acpi_get_first_physical_node(parent); - if (!aux_dev->dev.parent) - goto err; - + aux_dev->dev.parent = phys_parent; aux_dev->dev.release = acpi_video_bus_device_release; - if (auxiliary_device_init(aux_dev)) - goto err; + if (auxiliary_device_init(aux_dev)) { + kfree(aux_dev); + return; + } ACPI_COMPANION_SET(&aux_dev->dev, adev); if (__auxiliary_device_add(aux_dev, "acpi")) auxiliary_device_uninit(aux_dev); - - return; - -err: - kfree(aux_dev); } struct acpi_scan_system_dev { diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 458efa4fe9d4b..4c1c3b0b5499d 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -137,8 +137,10 @@ static int video_detect_portege_r100(const struct dmi_system_id *d) struct pci_dev *dev; /* Search for Trident CyberBlade XP4m32 to confirm Portégé R100 */ dev = pci_get_device(PCI_VENDOR_ID_TRIDENT, 0x2100, NULL); - if (dev) + if (dev) { acpi_backlight_dmi = acpi_backlight_vendor; + pci_dev_put(dev); + } return 0; } diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 58f512f8952a9..9b8c0935001cb 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -2614,6 +2614,51 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance) } #endif +/* + * The Marvell 88SE6111/6121/6145 ("Thor") family stops reporting interrupts + * for a port when HOST_IRQ_STAT is cleared while PxIS still holds bits: PxIS + * keeps its content, HOST_IRQ_STAT reads back as 0, the port is never looked + * at again and the command in flight only ends in a timeout. On a 88SE6121 + * this makes every SATA-2 or SATA-3 disk fail to IDENTIFY, while SATA-1 disks + * happen to win the race often enough to work. + * + * Clearing the host status before servicing the ports avoids it. Marvell's + * own driver for these chips does the same and says so ("clear global before + * channel"), and ahci_xgene handles its broken edge latch the same way. The + * price is at most one spurious interrupt per valid one, which is why this is + * not the generic behaviour - see AHCI 1.1 section 10.6.2. + * + * Link: https://bugzilla.kernel.org/show_bug.cgi?id=216094 + */ +static irqreturn_t ahci_mv_irq_handler(int irq, void *dev_instance) +{ + struct ata_host *host = dev_instance; + struct ahci_host_priv *hpriv = host->private_data; + void __iomem *mmio = hpriv->mmio; + unsigned int rc; + u32 irq_stat, irq_masked; + + irq_stat = readl(mmio + HOST_IRQ_STAT); + if (!irq_stat) + return IRQ_NONE; + + irq_masked = irq_stat & hpriv->port_map; + + spin_lock(&host->lock); + + /* + * Use the unmasked value to clear the interrupt, as a spurious pending + * event on a dummy port might cause a screaming IRQ. + */ + writel(irq_stat, mmio + HOST_IRQ_STAT); + + rc = ahci_handle_port_intr(host, irq_masked); + + spin_unlock(&host->lock); + + return IRQ_RETVAL(rc); +} + static void ahci_remap_check(struct pci_dev *pdev, int bar, struct ahci_host_priv *hpriv) { @@ -2917,6 +2962,10 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; hpriv->flags |= (unsigned long)pi.private_data; + /* the Marvell "Thor" family needs HOST_IRQ_STAT cleared first */ + if (board_id == board_ahci_mv) + hpriv->irq_handler = ahci_mv_irq_handler; + /* MCP65 revision A1 and A2 can't do MSI */ if (board_id == board_ahci_mcp65 && (pdev->revision == 0xa1 || pdev->revision == 0xa2)) diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index 6d72eb017b498..08ab56e955662 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -744,15 +744,28 @@ void ahci_start_fis_rx(struct ata_port *ap) struct ahci_port_priv *pp = ap->private_data; u32 tmp; - /* set FIS registers */ + /* + * On HBAs that only support 32-bit addressing PxCLBU is read only '0'. + * When applying the AHCI_HFLAG_32BIT_ONLY quirk, PxCLBU is RW, and the + * reset value is Implementation Specific, so we need to clear it to 0. + */ if (hpriv->cap & HOST_CAP_64) writel((pp->cmd_slot_dma >> 16) >> 16, port_mmio + PORT_LST_ADDR_HI); + else if (hpriv->flags & AHCI_HFLAG_32BIT_ONLY) + writel(0, port_mmio + PORT_LST_ADDR_HI); writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR); + /* + * On HBAs that only support 32-bit addressing PxFBU is read only '0'. + * When applying the AHCI_HFLAG_32BIT_ONLY quirk, PxFBU is RW, and the + * reset value is Implementation Specific, so we need to clear it to 0. + */ if (hpriv->cap & HOST_CAP_64) writel((pp->rx_fis_dma >> 16) >> 16, port_mmio + PORT_FIS_ADDR_HI); + else if (hpriv->flags & AHCI_HFLAG_32BIT_ONLY) + writel(0, port_mmio + PORT_FIS_ADDR_HI); writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR); /* enable FIS reception */ diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c index 6e072d681341a..f6524135ea117 100644 --- a/drivers/ata/libahci_platform.c +++ b/drivers/ata/libahci_platform.c @@ -620,10 +620,10 @@ struct ahci_host_priv *ahci_platform_get_resources(struct platform_device *pdev, of_platform_device_create(child, NULL, NULL); port_dev = of_find_device_by_node(child); - if (port_dev) { rc = ahci_platform_get_regulator(hpriv, port, &port_dev->dev); + put_device(&port_dev->dev); if (rc == -EPROBE_DEFER) goto err_out; } diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index d2160ee7ca7d4..3559485b50349 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -261,12 +261,18 @@ static void ata_scsi_set_passthru_sense_fields(struct ata_queued_cmd *qc) /* descriptor format */ len = sb[7]; - desc = (char *)scsi_sense_desc_find(sb, len + 8, 9); + desc = (char *)scsi_sense_desc_find(sb, SCSI_SENSE_BUFFERSIZE, 9); if (!desc) { - if (SCSI_SENSE_BUFFERSIZE < len + 14) + /* + * The descriptor is written at sb[8 + len] and is 14 + * bytes long, so it needs len + 22 bytes of buffer. + */ + if (len + 22 > SCSI_SENSE_BUFFERSIZE) return; sb[7] = len + 14; desc = sb + 8 + len; + } else if (desc - sb > SCSI_SENSE_BUFFERSIZE - 14) { + return; } desc[0] = 9; desc[1] = 12; @@ -3545,17 +3551,13 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc) static size_t ata_format_dsm_trim_descr(struct scsi_cmnd *cmd, u32 trmax, u64 sector, u32 count) { - struct scsi_device *sdp = cmd->device; - size_t len = sdp->sector_size; + size_t len = ATA_SECT_SIZE; size_t r; __le64 *buf; u32 i = 0; unsigned long flags; - WARN_ON(len > ATA_SCSI_RBUF_SIZE); - - if (len > ATA_SCSI_RBUF_SIZE) - len = ATA_SCSI_RBUF_SIZE; + BUILD_BUG_ON(ATA_SECT_SIZE > ATA_SCSI_RBUF_SIZE); spin_lock_irqsave(&ata_scsi_rbuf_lock, flags); buf = ((void *)ata_scsi_rbuf); @@ -3590,13 +3592,11 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc) { struct ata_taskfile *tf = &qc->tf; struct scsi_cmnd *scmd = qc->scsicmd; - struct scsi_device *sdp = scmd->device; - size_t len = sdp->sector_size; struct ata_device *dev = qc->dev; const u8 *cdb = scmd->cmnd; u64 block; u32 n_block; - const u32 trmax = len >> 3; + const u32 trmax = ATA_MAX_TRIM_RNUM; u32 size; u16 fp; u8 bp = 0xff; @@ -3640,13 +3640,13 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc) goto invalid_param_len; /* - * size must match sector size in bytes - * For DATA SET MANAGEMENT TRIM in ACS-2 nsect (aka count) - * is defined as number of 512 byte blocks to be transferred. + * The TRIM descriptor is a single 512-byte page, which is the maximum + * WRITE SAME length advertised in the Block Limits VPD page. For DATA + * SET MANAGEMENT TRIM the COUNT field (aka nsect) is the number of + * 512-byte blocks to be transferred. */ - size = ata_format_dsm_trim_descr(scmd, trmax, block, n_block); - if (size != len) + if (size != ATA_SECT_SIZE) goto invalid_param_len; if (ata_ncq_enabled(dev) && ata_fpdma_dsm_supported(dev)) { @@ -3672,6 +3672,12 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc) ATA_TFLAG_WRITE; ata_qc_set_pc_nbytes(qc); + /* + * The DSM TRIM payload is a single 512-byte page, which may be smaller + * than the WRITE SAME data-out buffer (one logical block); only + * transfer that page so the length matches the COUNT field. + */ + qc->nbytes = size; return 0; diff --git a/drivers/auxdisplay/charlcd.c b/drivers/auxdisplay/charlcd.c index 09020bb8ad15f..ae2692ec39639 100644 --- a/drivers/auxdisplay/charlcd.c +++ b/drivers/auxdisplay/charlcd.c @@ -595,6 +595,16 @@ static int charlcd_init(struct charlcd *lcd) return 0; } +static void charlcd_deinit(struct charlcd *lcd) +{ + struct charlcd_priv *priv = charlcd_to_priv(lcd); + + if (lcd->ops->backlight) { + cancel_delayed_work_sync(&priv->bl_work); + lcd->ops->backlight(lcd, CHARLCD_OFF); + } +} + struct charlcd *charlcd_alloc(unsigned int drvdata_size) { struct charlcd_priv *priv; @@ -654,8 +664,10 @@ int charlcd_register(struct charlcd *lcd) return ret; ret = misc_register(&charlcd_dev); - if (ret) + if (ret) { + charlcd_deinit(lcd); return ret; + } the_charlcd = lcd; register_reboot_notifier(&panel_notifier); @@ -665,16 +677,11 @@ EXPORT_SYMBOL_GPL(charlcd_register); int charlcd_unregister(struct charlcd *lcd) { - struct charlcd_priv *priv = charlcd_to_priv(lcd); - unregister_reboot_notifier(&panel_notifier); charlcd_puts(lcd, "\x0cLCD driver unloaded.\x1b[Lc\x1b[Lb\x1b[L-"); misc_deregister(&charlcd_dev); the_charlcd = NULL; - if (lcd->ops->backlight) { - cancel_delayed_work_sync(&priv->bl_work); - priv->lcd.ops->backlight(&priv->lcd, CHARLCD_OFF); - } + charlcd_deinit(lcd); return 0; } diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c index c99f2ab105e5b..a36c939b8e905 100644 --- a/drivers/base/arch_numa.c +++ b/drivers/base/arch_numa.c @@ -105,6 +105,18 @@ static void __init setup_node_to_cpumask_map(void) if (nr_node_ids == MAX_NUMNODES) setup_nr_node_ids(); + /* + * This check should never be true but it makes it clear to compilers + * that node_to_cpumask_map is bound by nr_node_ids, avoiding false + * positive fortify warnings when accessing node_to_cpumask_map in the + * for loop below. + */ + if (unlikely(nr_node_ids > MAX_NUMNODES)) { + pr_err("nr_node_ids (%u) is larger than MAX_NUMNODES (%u)\n", + nr_node_ids, MAX_NUMNODES); + return; + } + /* allocate and clear the mapping */ for (node = 0; node < nr_node_ids; node++) { alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index 70701d3bc81c6..7a47a392568a0 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -401,9 +401,14 @@ static int cache_setup_properties(unsigned int cpu) else if (!acpi_disabled) ret = cache_setup_acpi(cpu); - // Assume there is no cache information available in DT/ACPI from now. - if (ret && use_arch_cache_info()) + /* + * No DT/ACPI cache nodes; fall back to arch-derived topology (e.g. + * arm64 CLIDR_EL1) and clear the error to avoid a spurious warning. + */ + if (ret && use_arch_cache_info()) { use_arch_info = true; + ret = 0; + } return ret; } @@ -1035,9 +1040,10 @@ static int cacheinfo_cpu_online(unsigned int cpu) rc = cache_add_dev(cpu); if (rc) goto err; - if (cpu_map_shared_cache(true, cpu, &cpu_map)) + if (cpu_map_shared_cache(true, cpu, &cpu_map)) { update_per_cpu_data_slice_size(true, cpu, cpu_map); - sched_update_llc_bytes(cpu); + sched_update_llc_bytes(cpu_map); + } return 0; err: free_cache_attributes(cpu); @@ -1054,10 +1060,10 @@ static int cacheinfo_cpu_pre_down(unsigned int cpu) cpu_cache_sysfs_exit(cpu); free_cache_attributes(cpu); - if (nr_shared > 1) + if (nr_shared > 1) { update_per_cpu_data_slice_size(false, cpu, cpu_map); - - sched_update_llc_bytes(cpu); + sched_update_llc_bytes(cpu_map); + } return 0; } diff --git a/drivers/base/firmware_loader/fallback.c b/drivers/base/firmware_loader/fallback.c index 3ef0b312ae719..00d6b6e594616 100644 --- a/drivers/base/firmware_loader/fallback.c +++ b/drivers/base/firmware_loader/fallback.c @@ -95,6 +95,16 @@ static int fw_load_sysfs_fallback(struct fw_sysfs *fw_sysfs, long timeout) retval = -EINTR; goto out; } + + /* + * device_add() exposes the loading interface before pending_list is + * linked into pending_fw_head, so fw_state_done() may run first. + */ + if (fw_state_is_done(fw_priv)) { + mutex_unlock(&fw_lock); + goto out; + } + list_add(&fw_priv->pending_list, &pending_fw_head); mutex_unlock(&fw_lock); diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index f71467f6ada4e..4ed2e9b7e99b7 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -2234,6 +2234,7 @@ static int device_prepare(struct device *dev, pm_message_t state) if (ret < 0) { suspend_report_result(dev, callback, ret); + pm_runtime_unblock(dev); pm_runtime_put(dev); return ret; } diff --git a/drivers/base/property.c b/drivers/base/property.c index e08eadd66f4f9..e0f570f48bc32 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -807,18 +807,31 @@ struct fwnode_handle * fwnode_get_next_child_node(const struct fwnode_handle *fwnode, struct fwnode_handle *child) { + const struct fwnode_handle *parent; + struct fwnode_handle *child_parent __free(fwnode_handle) = NULL; struct fwnode_handle *next; - if (IS_ERR_OR_NULL(fwnode)) + /* + * If this function is in a loop and the previous iteration returned + * an child from fwnode->secondary, then we need to use the secondary + * as parent rather than @fwnode. + */ + if (child) { + child_parent = fwnode_get_parent(child); + parent = child_parent; + } else { + parent = fwnode; + } + if (IS_ERR_OR_NULL(parent)) return NULL; /* Try to find a child in primary fwnode */ - next = fwnode_call_ptr_op(fwnode, get_next_child_node, child); + next = fwnode_call_ptr_op(parent, get_next_child_node, child); if (next) return next; /* When no more children in primary, continue with secondary */ - return fwnode_call_ptr_op(fwnode->secondary, get_next_child_node, child); + return fwnode_get_next_child_node(parent->secondary, NULL); } EXPORT_SYMBOL_GPL(fwnode_get_next_child_node); diff --git a/drivers/base/soc.c b/drivers/base/soc.c index 65ce72d492303..af7d71393774b 100644 --- a/drivers/base/soc.c +++ b/drivers/base/soc.c @@ -191,6 +191,7 @@ EXPORT_SYMBOL_GPL(soc_device_unregister); static int __init soc_bus_register(void) { + struct soc_device *soc_dev; int ret; ret = bus_register(&soc_bus_type); @@ -198,10 +199,20 @@ static int __init soc_bus_register(void) return ret; soc_bus_registered = true; - if (early_soc_dev_attr) - return PTR_ERR(soc_device_register(early_soc_dev_attr)); + if (early_soc_dev_attr) { + soc_dev = soc_device_register(early_soc_dev_attr); + if (IS_ERR(soc_dev)) { + ret = PTR_ERR(soc_dev); + goto err_unregister_bus; + } + } return 0; + +err_unregister_bus: + soc_bus_registered = false; + bus_unregister(&soc_bus_type); + return ret; } core_initcall(soc_bus_register); diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c index 869228a65cb36..2bc76f01eb77d 100644 --- a/drivers/base/swnode.c +++ b/drivers/base/swnode.c @@ -537,7 +537,7 @@ software_node_get_reference_args(const struct fwnode_handle *fwnode, if (prop->is_inline) return -EINVAL; - if ((index + 1) * sizeof(*ref) > prop->length) + if (index >= prop->length / sizeof(*ref)) return -ENOENT; ref_array = prop->pointer; diff --git a/drivers/base/transport_class.c b/drivers/base/transport_class.c index 416e9f819df51..351c3d3ce6a08 100644 --- a/drivers/base/transport_class.c +++ b/drivers/base/transport_class.c @@ -168,11 +168,14 @@ static int transport_add_class_device(struct attribute_container *cont, if (tcont->encryption) { error = sysfs_create_group(&classdev->kobj, tcont->encryption); if (error) - goto err_del; + goto err_del_statistics; } return 0; +err_del_statistics: + if (tcont->statistics) + sysfs_remove_group(&classdev->kobj, tcont->statistics); err_del: attribute_container_class_device_del(classdev); err_remove: diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index f214a616386c8..113bdb868c469 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3048,6 +3048,8 @@ static int mtip_block_ioctl(struct block_device *dev, if (!dd) return -ENOTTY; + guard(mutex)(&dd->ioctl_mutex); + if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) return -ENOTTY; @@ -3086,6 +3088,8 @@ static int mtip_block_compat_ioctl(struct block_device *dev, if (!dd) return -ENOTTY; + guard(mutex)(&dd->ioctl_mutex); + if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) return -ENOTTY; @@ -3721,6 +3725,7 @@ static int mtip_pci_probe(struct pci_dev *pdev, dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); if (!dd) return -ENOMEM; + mutex_init(&dd->ioctl_mutex); /* Attach the private data to this PCI device. */ pci_set_drvdata(pdev, dd); @@ -3887,6 +3892,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) } set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); + mutex_lock(&dd->ioctl_mutex); if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) del_gendisk(dd->disk); @@ -3915,6 +3921,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) /* De-initialize the protocol layer. */ mtip_hw_exit(dd); + mutex_unlock(&dd->ioctl_mutex); if (dd->isr_workq) { destroy_workqueue(dd->isr_workq); diff --git a/drivers/block/mtip32xx/mtip32xx.h b/drivers/block/mtip32xx/mtip32xx.h index f7328f19ac5c2..0963c07b58457 100644 --- a/drivers/block/mtip32xx/mtip32xx.h +++ b/drivers/block/mtip32xx/mtip32xx.h @@ -12,6 +12,7 @@ #define __MTIP32XX_H__ #include +#include #include #include #include @@ -432,6 +433,7 @@ struct driver_data { struct request_queue *queue; /* Our request queue. */ struct blk_mq_tag_set tags; /* blk_mq tags */ + struct mutex ioctl_mutex; struct mtip_port *port; /* Pointer to the port data structure. */ diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index f8c0fd57e041a..4a9beec211b92 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -66,7 +66,7 @@ struct nullb_page { #define NULLB_PAGE_FREE (MAP_SZ - 2) static LIST_HEAD(nullb_list); -static struct mutex lock; +static DEFINE_MUTEX(lock); static int null_major; static DEFINE_IDA(nullb_indexes); static struct blk_mq_tag_set tag_set; @@ -340,7 +340,15 @@ static ssize_t nullb_device_bool_attr_store(bool *val, const char *page, return count; } -/* The following macro should only be used with TYPE = {uint, ulong, bool}. */ +/* + * The following macro should only be used with TYPE = {uint, ulong, bool}. + * + * The device configuration is modified under the global lock to serialize + * attribute changes against null_add_dev() and null_del_dev(): without this, + * an attribute could be changed while null_add_dev() is running, that is, + * before NULLB_DEV_FL_CONFIGURED is set, which would let null_add_dev() + * observe inconsistent values for the device configuration. + */ #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \ static ssize_t \ nullb_device_##NAME##_show(struct config_item *item, char *page) \ @@ -360,6 +368,7 @@ nullb_device_##NAME##_store(struct config_item *item, const char *page, \ ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\ if (ret < 0) \ return ret; \ + guard(mutex)(&lock); \ if (apply_fn) \ ret = apply_fn(dev, new_value); \ else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \ @@ -379,9 +388,20 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, struct blk_mq_tag_set *set; int ret, nr_hw_queues; + lockdep_assert_held(&lock); + if (!dev->nullb) return 0; + /* + * A shared tag_set is mapped via the module-wide queue counts, so a + * per-device resize is meaningless. On shrink it would also leave + * mq_map[] pointing at NULLed hctx slots, causing a NULL deref in + * blk_mq_map_swqueue(). Reject it. + */ + if (dev->shared_tags) + return -EINVAL; + /* * Make sure at least one submit queue exists. */ @@ -421,25 +441,13 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, static int nullb_apply_submit_queues(struct nullb_device *dev, unsigned int submit_queues) { - int ret; - - mutex_lock(&lock); - ret = nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); - mutex_unlock(&lock); - - return ret; + return nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); } static int nullb_apply_poll_queues(struct nullb_device *dev, unsigned int poll_queues) { - int ret; - - mutex_lock(&lock); - ret = nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); - mutex_unlock(&lock); - - return ret; + return nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); } NULLB_DEVICE_ATTR(size, ulong, NULL); @@ -579,8 +587,13 @@ static ssize_t nullb_device_zone_readonly_store(struct config_item *item, const char *page, size_t count) { struct nullb_device *dev = to_nullb_device(item); + ssize_t ret; - return zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); + mutex_lock(&lock); + ret = zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); + mutex_unlock(&lock); + + return ret; } CONFIGFS_ATTR_WO(nullb_device_, zone_readonly); @@ -588,8 +601,13 @@ static ssize_t nullb_device_zone_offline_store(struct config_item *item, const char *page, size_t count) { struct nullb_device *dev = to_nullb_device(item); + ssize_t ret; + + mutex_lock(&lock); + ret = zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); + mutex_unlock(&lock); - return zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); + return ret; } CONFIGFS_ATTR_WO(nullb_device_, zone_offline); @@ -836,7 +854,6 @@ static void null_free_dev(struct nullb_device *dev) if (!dev) return; - null_free_zoned_dev(dev); badblocks_exit(&dev->badblocks); kfree(dev); } @@ -1777,6 +1794,7 @@ static void null_del_dev(struct nullb *nullb) } put_disk(nullb->disk); + null_free_zoned_dev(dev); if (nullb->tag_set == &nullb->__tag_set) blk_mq_free_tag_set(nullb->tag_set); kfree(nullb->queues); @@ -2162,17 +2180,9 @@ static int __init null_init(void) config_group_init(&nullb_subsys.su_group); mutex_init(&nullb_subsys.su_mutex); - ret = configfs_register_subsystem(&nullb_subsys); - if (ret) - return ret; - - mutex_init(&lock); - null_major = register_blkdev(0, "nullb"); - if (null_major < 0) { - ret = null_major; - goto err_conf; - } + if (null_major < 0) + return null_major; for (i = 0; i < nr_devices; i++) { ret = null_create_dev(); @@ -2180,6 +2190,10 @@ static int __init null_init(void) goto err_dev; } + ret = configfs_register_subsystem(&nullb_subsys); + if (ret) + goto err_dev; + pr_info("module loaded\n"); return 0; @@ -2189,8 +2203,8 @@ static int __init null_init(void) null_destroy_dev(nullb); } unregister_blkdev(null_major, "nullb"); -err_conf: - configfs_unregister_subsystem(&nullb_subsys); + if (tag_set.ops) + blk_mq_free_tag_set(&tag_set); return ret; } @@ -2211,8 +2225,6 @@ static void __exit null_exit(void) if (tag_set.ops) blk_mq_free_tag_set(&tag_set); - - mutex_destroy(&lock); } module_init(null_init); diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index 384bdce6a9b7f..7e9b5ffb9e841 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -18,6 +18,8 @@ static inline sector_t mb_to_sects(unsigned long mb) static inline unsigned int null_zone_no(struct nullb_device *dev, sector_t sect) { + if (WARN_ON_ONCE(!dev->zone_size_sects)) + return 0; return sect >> ilog2(dev->zone_size_sects); } @@ -56,8 +58,8 @@ int null_init_zoned_dev(struct nullb_device *dev, sector_t sector = 0; unsigned int i; - if (!is_power_of_2(dev->zone_size)) { - pr_err("zone_size must be power-of-two\n"); + if (!dev->zone_size || !is_power_of_2(dev->zone_size)) { + pr_err("zone_size must be non-zero power-of-two\n"); return -EINVAL; } if (dev->zone_size > dev->size) { @@ -88,6 +90,10 @@ int null_init_zoned_dev(struct nullb_device *dev, zone_capacity_sects = mb_to_sects(dev->zone_capacity); dev_capacity_sects = mb_to_sects(dev->size); dev->zone_size_sects = mb_to_sects(dev->zone_size); + if (!dev->zone_size_sects) { + pr_err("zone_size too large or too small, leads to zero sectors\n"); + return -EINVAL; + } dev->nr_zones = round_up(dev_capacity_sects, dev->zone_size_sects) >> ilog2(dev->zone_size_sects); diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c index 020bd9f1a7b6a..2be8231dcd5b4 100644 --- a/drivers/block/sunvdc.c +++ b/drivers/block/sunvdc.c @@ -525,6 +525,23 @@ static int __send_request(struct request *req) err = __vdc_tx_trigger(port); if (err < 0) { printk(KERN_ERR PFX "vdc_tx_trigger() failure, err=%d\n", err); + /* + * If the port was reset (-ENOTCONN), the dring and the + * LDC channel including all of its mappings are already + * torn down and reallocated - there is nothing to undo + * and @desc must not be touched. + * + * For any other failure the descriptor was never handed + * to the peer: unmap the cookies and free the descriptor + * again, so that a later retry of the request does not + * leak LDC map table entries. + */ + if (err != -ENOTCONN) { + ldc_unmap(port->vio.lp, desc->cookies, + desc->ncookies); + desc->hdr.state = VIO_DESC_FREE; + rqe->req = NULL; + } } else { port->req_id++; dr->prod = vio_dring_next(dr, dr->prod); @@ -539,6 +556,7 @@ static blk_status_t vdc_queue_rq(struct blk_mq_hw_ctx *hctx, struct vdc_port *port = hctx->queue->queuedata; struct vio_dring_state *dr; unsigned long flags; + int ret; dr = &port->vio.drings[VIO_DRIVER_TX_RING]; @@ -560,7 +578,13 @@ static blk_status_t vdc_queue_rq(struct blk_mq_hw_ctx *hctx, return BLK_STS_DEV_RESOURCE; } - if (__send_request(bd->rq) < 0) { + ret = __send_request(bd->rq); + if (ret == -EAGAIN) { + spin_unlock_irqrestore(&port->vio.lock, flags); + /* already spun for 10msec, defer 10msec and retry */ + blk_mq_delay_kick_requeue_list(hctx->queue, 10); + return BLK_STS_DEV_RESOURCE; + } else if (ret < 0) { spin_unlock_irqrestore(&port->vio.lock, flags); return BLK_STS_IOERR; } diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 2a22f9dc1f2f1..d879e8492d59c 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -970,7 +970,7 @@ static int ublk_validate_params(const struct ublk_device *ub) if (p->max_sectors < PAGE_SECTORS) return -EINVAL; - if (ublk_dev_is_zoned(ub) && !p->chunk_sectors) + if (ublk_dev_is_zoned(ub) && !is_power_of_2(p->chunk_sectors)) return -EINVAL; } else return -EINVAL; @@ -1475,7 +1475,10 @@ static unsigned int ublk_map_io(const struct ublk_queue *ubq, struct iov_iter iter; const int dir = ITER_DEST; - import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, &iter); + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, + &iter) < 0) + return 0; + return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; @@ -1496,7 +1499,10 @@ static unsigned int ublk_unmap_io(bool need_map, WARN_ON_ONCE(io->res > rq_bytes); - import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter); + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, + &iter) < 0) + return 0; + return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; @@ -1584,8 +1590,14 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io, * * Re-read simply for this unlikely case. */ - if (unlikely(unmapped_bytes < io->res)) + if (unlikely(unmapped_bytes < io->res)) { + if (unlikely(!unmapped_bytes)) { + res = BLK_STS_IOERR; + goto exit; + } + io->res = unmapped_bytes; + } /* * Run bio->bi_end_io() with softirqs disabled. If the final fput @@ -2651,6 +2663,12 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) if (vma->vm_flags & VM_WRITE) return -EPERM; + /* + * The per-queue command buffer is kernel-written ABI; prevent + * the daemon from upgrading to writable via mprotect(). + */ + vm_flags_clear(vma, VM_MAYWRITE); + end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz; if (phys_off < UBLKSRV_CMD_BUF_OFFSET || phys_off >= end) return -EINVAL; @@ -3022,6 +3040,7 @@ static void ublk_queue_reset_io_flags(struct ublk_queue *ubq) ubq->canceling = false; spin_unlock(&ubq->cancel_lock); ubq->fail_io = false; + ubq->force_abort = false; } /* device can only be started after all IOs are ready */ @@ -3075,18 +3094,19 @@ static inline int ublk_check_cmd_op(u32 cmd_op) return 0; } -static inline int ublk_set_auto_buf_reg(struct ublk_io *io, struct io_uring_cmd *cmd) +/* Must run before ublk_fill_io_cmd() / __ublk_fetch(). */ +static inline int ublk_validate_io_buf(const struct ublk_device *ub, + struct io_uring_cmd *cmd, + struct ublk_auto_buf_reg *buf) { - struct ublk_auto_buf_reg buf; - - buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); + if (!ublk_dev_support_auto_buf_reg(ub)) + return 0; - if (buf.reserved0 || buf.reserved1) + *buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); + if (buf->reserved0 || buf->reserved1) return -EINVAL; - - if (buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK) + if (buf->flags & ~UBLK_AUTO_BUF_REG_F_MASK) return -EINVAL; - io->buf.auto_reg = buf; return 0; } @@ -3107,17 +3127,25 @@ static void ublk_clear_auto_buf_reg(struct ublk_io *io, * responsibility for unregistering the buffer, otherwise * this ublk request gets stuck. */ - if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) + if (buf_idx && + io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) *buf_idx = io->buf.auto_reg.index; } } -static int ublk_handle_auto_buf_reg(struct ublk_io *io, - struct io_uring_cmd *cmd, - u16 *buf_idx) +static inline void ublk_apply_io_buf(const struct ublk_device *ub, + struct ublk_io *io, + struct io_uring_cmd *cmd, + unsigned long buf_addr, + const struct ublk_auto_buf_reg *auto_buf, + u16 *buf_idx) { - ublk_clear_auto_buf_reg(io, cmd, buf_idx); - return ublk_set_auto_buf_reg(io, cmd); + if (ublk_dev_support_auto_buf_reg(ub)) { + ublk_clear_auto_buf_reg(io, cmd, buf_idx); + io->buf.auto_reg = *auto_buf; + } else { + io->buf.addr = buf_addr; + } } /* Once we return, `io->req` can't be used any more */ @@ -3134,18 +3162,6 @@ ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd) return req; } -static inline int -ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io, - struct io_uring_cmd *cmd, unsigned long buf_addr, - u16 *buf_idx) -{ - if (ublk_dev_support_auto_buf_reg(ub)) - return ublk_handle_auto_buf_reg(io, cmd, buf_idx); - - io->buf.addr = buf_addr; - return 0; -} - static inline void ublk_prep_cancel(struct io_uring_cmd *cmd, unsigned int issue_flags, struct ublk_queue *ubq, unsigned int tag) @@ -3286,6 +3302,7 @@ static int __ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, struct ublk_io *io, __u64 buf_addr, u16 q_id) { + struct ublk_auto_buf_reg auto_buf; int ret; /* @@ -3294,11 +3311,13 @@ static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, * FETCH, so it is fine even for IO_URING_F_NONBLOCK. */ mutex_lock(&ub->mutex); - ret = __ublk_fetch(cmd, ub, io, q_id); - if (!ret) - ret = ublk_config_io_buf(ub, io, cmd, buf_addr, NULL); + ret = ublk_validate_io_buf(ub, cmd, &auto_buf); if (!ret) + ret = __ublk_fetch(cmd, ub, io, q_id); + if (!ret) { + ublk_apply_io_buf(ub, io, cmd, buf_addr, &auto_buf, NULL); ublk_mark_io_ready(ub, q_id, io); + } mutex_unlock(&ub->mutex); return ret; } @@ -3441,13 +3460,18 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, case UBLK_IO_REGISTER_IO_BUF: return ublk_daemon_register_io_buf(cmd, ub, q_id, tag, io, addr, issue_flags); - case UBLK_IO_COMMIT_AND_FETCH_REQ: + case UBLK_IO_COMMIT_AND_FETCH_REQ: { + struct ublk_auto_buf_reg auto_buf; + ret = ublk_check_commit_and_fetch(ub, io, addr); + if (ret) + goto out; + ret = ublk_validate_io_buf(ub, cmd, &auto_buf); if (ret) goto out; io->res = result; req = ublk_fill_io_cmd(io, cmd); - ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx); + ublk_apply_io_buf(ub, io, cmd, addr, &auto_buf, &buf_idx); if (buf_idx != UBLK_INVALID_BUF_IDX) io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); compl = ublk_need_complete_req(ub, io); @@ -3456,10 +3480,8 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, req->__sector = addr; if (compl) __ublk_complete_rq(req, io, ublk_dev_need_map_io(ub), NULL); - - if (ret) - goto out; break; + } case UBLK_IO_NEED_GET_DATA: /* * ublk_get_data() may fail and fallback to requeue, so keep @@ -3467,8 +3489,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, * request */ req = ublk_fill_io_cmd(io, cmd); - ret = ublk_config_io_buf(ub, io, cmd, addr, NULL); - WARN_ON_ONCE(ret); + io->buf.addr = addr; if (likely(ublk_get_data(ubq, io, req))) { __ublk_prep_compl_io_cmd(io, req); return UBLK_IO_RES_OK; @@ -3780,11 +3801,11 @@ static int ublk_batch_commit_io(struct ublk_queue *ubq, ret = ublk_batch_commit_io_check(ubq, io, &buf); if (!ret) { io->res = elem->result; - io->buf = buf; req = ublk_fill_io_cmd(io, data->cmd); if (auto_reg) ublk_clear_auto_buf_reg(io, data->cmd, &buf_idx); + io->buf = buf; compl = ublk_need_complete_req(data->ub, io); } ublk_io_unlock(io); @@ -5502,39 +5523,36 @@ static void ublk_unpin_range_pages(unsigned long base_pfn, /* * Inner loop: erase up to UBLK_REMOVE_BATCH matching ranges under - * mas_lock, collecting them into an xarray. Then drop the lock and - * unpin pages + free ranges outside spinlock context. + * mas_lock, collecting the page ranges in a fixed-size array. Then + * drop the lock and unpin pages + free ranges outside spinlock context. * * Returns true if the tree walk completed, false if more ranges remain. - * Xarray key is the base PFN, value encodes nr_pages via xa_mk_value(). */ #define UBLK_REMOVE_BATCH 64 +struct ublk_unpin_range { + unsigned long base_pfn; + unsigned long nr_pages; +}; + static bool __ublk_shmem_remove_ranges(struct ublk_device *ub, int buf_index, int *ret) { MA_STATE(mas, &ub->buf_tree, 0, ULONG_MAX); struct ublk_buf_range *range; - struct xarray to_unpin; - unsigned long idx; + struct ublk_unpin_range to_unpin[UBLK_REMOVE_BATCH]; unsigned int count = 0; + unsigned int i; bool done = false; - void *entry; - - xa_init(&to_unpin); mas_lock(&mas); mas_for_each(&mas, range, ULONG_MAX) { - unsigned long nr; - if (buf_index >= 0 && range->buf_index != buf_index) continue; *ret = 0; - nr = mas.last - mas.index + 1; - if (xa_err(xa_store(&to_unpin, mas.index, - xa_mk_value(nr), GFP_ATOMIC))) - goto unlock; + to_unpin[count].base_pfn = mas.index; + to_unpin[count].nr_pages = mas.last - mas.index + 1; mas_erase(&mas); kfree(range); if (++count >= UBLK_REMOVE_BATCH) @@ -5544,9 +5562,9 @@ static bool __ublk_shmem_remove_ranges(struct ublk_device *ub, unlock: mas_unlock(&mas); - xa_for_each(&to_unpin, idx, entry) - ublk_unpin_range_pages(idx, xa_to_value(entry)); - xa_destroy(&to_unpin); + for (i = 0; i < count; i++) + ublk_unpin_range_pages(to_unpin[i].base_pfn, + to_unpin[i].nr_pages); return done; } diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index 55eeb6aac0ea3..1333fae05ac5d 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -478,7 +478,8 @@ static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no) zone->cond == BLK_ZONE_COND_FULL) goto unlock; - if (vfs_truncate(&zone->file->f_path, zlo->zone_size << SECTOR_SHIFT)) { + if (vfs_truncate(&zone->file->f_path, + zlo->zone_capacity << SECTOR_SHIFT)) { set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); ret = -EIO; goto unlock; diff --git a/drivers/block/zram/backend_deflate.c b/drivers/block/zram/backend_deflate.c index f92a52a720d1e..b3f7d08b49d93 100644 --- a/drivers/block/zram/backend_deflate.c +++ b/drivers/block/zram/backend_deflate.c @@ -24,8 +24,16 @@ static int deflate_setup_params(struct zcomp_params *params) { if (params->level == ZCOMP_PARAM_NOT_SET) params->level = Z_DEFAULT_COMPRESSION; - if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) + if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) { params->deflate.winbits = DEFLATE_DEF_WINBITS; + } else { + s32 wb = params->deflate.winbits; + + if ((wb < -15 || wb > -9) && (wb < 9 || wb > 15)) { + pr_err("invalid deflate winbits: %d\n", wb); + return -EINVAL; + } + } return 0; } diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index ace65c586072b..6cb44e229f1ec 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -57,14 +57,12 @@ static size_t huge_class_size; static const struct block_device_operations zram_devops; static void slot_free(struct zram *zram, u32 index); -#define slot_dep_map(zram, index) (&(zram)->table[(index)].dep_map) -static void slot_lock_init(struct zram *zram, u32 index) +static void slot_lock_init(struct zram *zram) { static struct lock_class_key __key; - lockdep_init_map(slot_dep_map(zram, index), "zram->table[index].lock", - &__key, 0); + lockdep_init_map(&zram->table_lock_map, "zram->table[index].lock", &__key, 0); } /* @@ -83,9 +81,9 @@ static __must_check bool slot_trylock(struct zram *zram, u32 index) { unsigned long *lock = &zram->table[index].__lock; - if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK, lock)) { - mutex_acquire(slot_dep_map(zram, index), 0, 1, _RET_IP_); - lock_acquired(slot_dep_map(zram, index), _RET_IP_); + if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK_BIT, lock)) { + mutex_acquire(&zram->table_lock_map, 0, 1, _RET_IP_); + lock_acquired(&zram->table_lock_map, _RET_IP_); return true; } @@ -96,17 +94,17 @@ static void slot_lock(struct zram *zram, u32 index) { unsigned long *lock = &zram->table[index].__lock; - mutex_acquire(slot_dep_map(zram, index), 0, 0, _RET_IP_); - wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK, TASK_UNINTERRUPTIBLE); - lock_acquired(slot_dep_map(zram, index), _RET_IP_); + mutex_acquire(&zram->table_lock_map, 0, 0, _RET_IP_); + wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK_BIT, TASK_UNINTERRUPTIBLE); + lock_acquired(&zram->table_lock_map, _RET_IP_); } static void slot_unlock(struct zram *zram, u32 index) { unsigned long *lock = &zram->table[index].__lock; - mutex_release(slot_dep_map(zram, index), _RET_IP_); - clear_and_wake_up_bit(ZRAM_ENTRY_LOCK, lock); + mutex_release(&zram->table_lock_map, _RET_IP_); + clear_and_wake_up_bit(ZRAM_ENTRY_LOCK_BIT, lock); } static inline bool init_done(struct zram *zram) @@ -1244,8 +1242,8 @@ static ssize_t writeback_store(struct device *dev, const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); - u64 nr_pages = zram->disksize >> PAGE_SHIFT; - unsigned long lo = 0, hi = nr_pages; + u64 nr_pages; + unsigned long lo = 0, hi; struct zram_pp_ctl *pp_ctl = NULL; struct zram_wb_ctl *wb_ctl = NULL; char *args, *param, *val; @@ -1259,6 +1257,9 @@ static ssize_t writeback_store(struct device *dev, if (!zram->backing_dev) return -ENODEV; + nr_pages = zram->disksize >> PAGE_SHIFT; + hi = nr_pages; + pp_ctl = init_pp_ctl(); if (!pp_ctl) return -ENOMEM; @@ -1549,7 +1550,7 @@ static ssize_t read_block_state(struct file *file, char __user *buf, char *kbuf; ssize_t index, written = 0; struct zram *zram = file->private_data; - unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; + unsigned long nr_pages; kbuf = kvmalloc(count, GFP_KERNEL); if (!kbuf) @@ -1561,6 +1562,8 @@ static ssize_t read_block_state(struct file *file, char __user *buf, return -EINVAL; } + nr_pages = zram->disksize >> PAGE_SHIFT; + for (index = *ppos; index < nr_pages; index++) { int copied; @@ -1984,7 +1987,7 @@ static void zram_meta_free(struct zram *zram, u64 disksize) static bool zram_meta_alloc(struct zram *zram, u64 disksize) { - size_t num_pages, index; + size_t num_pages; num_pages = disksize >> PAGE_SHIFT; zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table))); @@ -2001,8 +2004,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize) if (!huge_class_size) huge_class_size = zs_huge_class_size(zram->mem_pool); - for (index = 0; index < num_pages; index++) - slot_lock_init(zram, index); + slot_lock_init(zram); return true; } @@ -2836,6 +2838,7 @@ static void zram_destroy_comps(struct zram *zram) zram->comp_algs[prio] = NULL; zram_comp_params_reset(zram); + comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); } static void zram_reset_device(struct zram *zram) @@ -2853,8 +2856,6 @@ static void zram_reset_device(struct zram *zram) zram_destroy_comps(zram); memset(&zram->stats, 0, sizeof(zram->stats)); reset_bdev(zram); - - comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); } static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 08d1774c15db6..996bb99ea8bbf 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -15,6 +15,7 @@ #ifndef _ZRAM_DRV_H_ #define _ZRAM_DRV_H_ +#include #include #include @@ -57,6 +58,19 @@ enum zram_pageflags { __NR_ZRAM_PAGEFLAGS, }; +/* + * The slot lock is a bit-wait lock on the whole __lock word, while + * flags and ac_time alias that word as two u32s. The lock bit must + * land in the slot that ZRAM_ENTRY_LOCK reserves in attr.flags; on + * 64-bit big-endian the flags word maps to the upper half of __lock, + * so the bit position has to be shifted up. + */ +#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN) +#define ZRAM_ENTRY_LOCK_BIT (ZRAM_ENTRY_LOCK + 32) +#else +#define ZRAM_ENTRY_LOCK_BIT ZRAM_ENTRY_LOCK +#endif + /* * Allocated for each disk page. We use bit-lock (ZRAM_ENTRY_LOCK bit * of flags) to save memory. There can be plenty of entries and standard @@ -74,7 +88,6 @@ struct zram_table_entry { #endif } attr; }; - struct lockdep_map dep_map; }; struct zram_stats { @@ -107,6 +120,7 @@ struct zram_stats { struct zram { struct zram_table_entry *table; + struct lockdep_map table_lock_map; struct zs_pool *mem_pool; struct zcomp *comps[ZRAM_MAX_COMPS]; struct zcomp_params params[ZRAM_MAX_COMPS]; diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index bf567b7c5f00b..96a55d5badda1 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -570,12 +570,44 @@ int btintel_version_info_tlv(struct hci_dev *hdev, } EXPORT_SYMBOL_GPL(btintel_version_info_tlv); +static u8 btintel_version_tlv_min_len(u8 type) +{ + switch (type) { + case INTEL_TLV_CNVI_TOP: + case INTEL_TLV_CNVR_TOP: + case INTEL_TLV_CNVI_BT: + case INTEL_TLV_CNVR_BT: + case INTEL_TLV_BUILD_NUM: + case INTEL_TLV_GIT_SHA1: + return sizeof(u32); + case INTEL_TLV_DEV_REV_ID: + case INTEL_TLV_TIME_STAMP: + return sizeof(u16); + case INTEL_TLV_IMAGE_TYPE: + case INTEL_TLV_BUILD_TYPE: + case INTEL_TLV_SECURE_BOOT: + case INTEL_TLV_OTP_LOCK: + case INTEL_TLV_API_LOCK: + case INTEL_TLV_DEBUG_LOCK: + case INTEL_TLV_LIMITED_CCE: + case INTEL_TLV_SBE_TYPE: + return sizeof(u8); + case INTEL_TLV_MIN_FW: + return 3; + case INTEL_TLV_OTP_BDADDR: + return sizeof(bdaddr_t); + default: + return 0; + } +} + int btintel_parse_version_tlv(struct hci_dev *hdev, struct intel_version_tlv *version, struct sk_buff *skb) { /* Consume Command Complete Status field */ - skb_pull(skb, 1); + if (!skb_pull(skb, 1)) + return -EINVAL; /* Event parameters contain multiple TLVs. Read each of them * and only keep the required data. Also, it use existing legacy @@ -595,6 +627,9 @@ int btintel_parse_version_tlv(struct hci_dev *hdev, if (skb->len < tlv->len + sizeof(*tlv)) return -EINVAL; + if (tlv->len < btintel_version_tlv_min_len(tlv->type)) + return -EINVAL; + switch (tlv->type) { case INTEL_TLV_CNVI_TOP: version->cnvi_top = get_unaligned_le32(tlv->val); @@ -666,7 +701,7 @@ int btintel_parse_version_tlv(struct hci_dev *hdev, break; case INTEL_TLV_FW_ID: snprintf(version->fw_id, sizeof(version->fw_id), - "%s", tlv->val); + "%.*s", tlv->len, tlv->val); break; default: /* Ignore rest of information */ @@ -3834,7 +3869,7 @@ int btintel_recv_event(struct hci_dev *hdev, struct sk_buff *skb) /* Handle all diagnostics events separately. May still call * hci_recv_frame. */ - if (len >= sizeof(diagnostics_hdr) && + if (len + 1 >= sizeof(diagnostics_hdr) && memcmp(&skb->data[2], diagnostics_hdr, sizeof(diagnostics_hdr)) == 0) { return btintel_diagnostics(hdev, skb); diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c index 2b7231be5973d..9381895dc3cf0 100644 --- a/drivers/bluetooth/btintel_pcie.c +++ b/drivers/bluetooth/btintel_pcie.c @@ -404,6 +404,12 @@ static int btintel_pcie_send_sync(struct btintel_pcie_data *data, if (tfd_index > txq->count) return -ERANGE; + if (skb->len > BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN) { + bt_dev_err(hdev, "TX skb too large (%u > %u)", skb->len, + BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN); + return -EMSGSIZE; + } + /* Firmware raises alive interrupt on HCI_OP_RESET or * BTINTEL_HCI_OP_RESET */ @@ -502,7 +508,7 @@ static int btintel_pcie_submit_rx(struct btintel_pcie_data *data) frbd_index = data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM]; - if (frbd_index > rxq->count) + if (frbd_index >= rxq->count) return -ERANGE; /* Prepare for RX submit. It updates the FRBD with the address of DMA @@ -1093,13 +1099,18 @@ static void btintel_pcie_msix_tx_handle(struct btintel_pcie_data *data) txq = &data->txq; + if (cr_hia >= txq->count) { + bt_dev_err(data->hdev, "TXQ: invalid cr_hia %u", cr_hia); + return; + } + while (cr_tia != cr_hia) { data->tx_wait_done = true; wake_up(&data->tx_wait_q); urbd0 = &txq->urbd0s[cr_tia]; - if (urbd0->tfd_index > txq->count) + if (urbd0->tfd_index >= txq->count) return; cr_tia = (cr_tia + 1) % txq->count; @@ -1537,7 +1548,9 @@ static int btintel_pcie_submit_rx_work(struct btintel_pcie_data *data, u8 status rfh_hdr = buf; len = rfh_hdr->packet_len; - if (len <= 0) { + if (len == 0 || len > BTINTEL_PCIE_BUFFER_SIZE - sizeof(*rfh_hdr)) { + bt_dev_err(data->hdev, "Invalid packet_len %d (max %zu)", len, + BTINTEL_PCIE_BUFFER_SIZE - sizeof(*rfh_hdr)); ret = -EINVAL; goto resubmit; } @@ -1580,6 +1593,11 @@ static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) rxq = &data->rxq; + if (cr_hia >= rxq->count) { + bt_dev_err(hdev, "RXQ: invalid cr_hia %u", cr_hia); + return; + } + /* The firmware sends multiple CD in a single MSI-X and it needs to * process all received CDs in this interrupt. */ @@ -1587,6 +1605,12 @@ static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) urbd1 = &rxq->urbd1s[cr_tia]; ipc_print_urbd1(data->hdev, urbd1, cr_tia); + if (urbd1->frbd_tag >= rxq->count) { + bt_dev_err(hdev, "RXQ: invalid frbd_tag %u", + urbd1->frbd_tag); + return; + } + buf = &rxq->bufs[urbd1->frbd_tag]; if (!buf) { bt_dev_err(hdev, "RXQ: failed to get the DMA buffer for %d", @@ -1634,6 +1658,9 @@ static irqreturn_t btintel_pcie_irq_msix_handler(int irq, void *dev_id) if (unlikely(!(intr_fh | intr_hw))) { /* Ignore interrupt, inta == 0 */ + bt_warn_ratelimited("Bluetooth: btintel_pcie: Received spurious interrupt\n"); + btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_AUTOMASK_ST, + BIT(entry->entry)); return IRQ_NONE; } diff --git a/drivers/bluetooth/btmtk.c b/drivers/bluetooth/btmtk.c index 66b3467610436..7ea8bcd8a7eca 100644 --- a/drivers/bluetooth/btmtk.c +++ b/drivers/bluetooth/btmtk.c @@ -721,7 +721,12 @@ static int btmtk_usb_hci_wmt_sync(struct hci_dev *hdev, case BTMTK_WMT_FUNC_CTRL: if (!skb_pull_data(data->evt_skb, sizeof(wmt_evt_funcc->status))) { - status = BTMTK_WMT_ON_UNDONE; + /* A plain enable/disable request is acked with just + * the WMT header and no trailing status word; the + * result is carried in the header's own flag byte. + */ + status = wmt_evt->whdr.flag ? BTMTK_WMT_ON_UNDONE : + BTMTK_WMT_ON_DONE; break; } @@ -860,6 +865,7 @@ static u32 btmtk_usb_reset_done(struct hci_dev *hdev) int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) { + int reset_err = 0; u32 val; int err; @@ -958,8 +964,10 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) err = readx_poll_timeout(btmtk_usb_reset_done, hdev, val, val & MTK_BT_RST_DONE, 20000, 1000000); - if (err < 0) + if (err < 0) { bt_dev_err(hdev, "Reset timeout"); + reset_err = err; + } if (dev_id == 0x7922) { err = btmtk_usb_uhw_reg_write(hdev, MTK_UDMA_INT_STA_BT, 0x000000FF); @@ -968,10 +976,12 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) } err = btmtk_usb_id_get(hdev, 0x70010200, &val); - if (err || (!val && dev_id != 0x6639)) + if (err || (!val && dev_id != 0x6639)) { bt_dev_err(hdev, "Can't get device id, subsys reset fail."); + return err ? err : -ENODEV; + } - return err; + return reset_err; } EXPORT_SYMBOL_GPL(btmtk_usb_subsys_reset); @@ -1369,16 +1379,6 @@ int btmtk_usb_setup(struct hci_dev *hdev) break; case 0x7922: case 0x7925: - /* - * A remote wakeup could cause the device completely unresponsive, and - * recovering from such a state needs a power cycle. - * - * Since the remote wakeup capability is super broken, just disable it - * to get rid of the troubles. The device can still be autosuspended - * when the bluetooth interface is closed. - */ - device_set_wakeup_capable(&btmtk_data->udev->dev, false); - fallthrough; case 0x7961: case 0x7902: case 0x6639: @@ -1582,5 +1582,6 @@ MODULE_FIRMWARE(FIRMWARE_MT7663); MODULE_FIRMWARE(FIRMWARE_MT7668); MODULE_FIRMWARE(FIRMWARE_MT7922); MODULE_FIRMWARE(FIRMWARE_MT7961); +MODULE_FIRMWARE(FIRMWARE_MT7920); MODULE_FIRMWARE(FIRMWARE_MT7925); MODULE_FIRMWARE(FIRMWARE_MT7927); diff --git a/drivers/bluetooth/btmtk.h b/drivers/bluetooth/btmtk.h index c83c24897c954..bc26148ec5442 100644 --- a/drivers/bluetooth/btmtk.h +++ b/drivers/bluetooth/btmtk.h @@ -7,6 +7,7 @@ #define FIRMWARE_MT7922 "mediatek/BT_RAM_CODE_MT7922_1_1_hdr.bin" #define FIRMWARE_MT7902 "mediatek/BT_RAM_CODE_MT7902_1_1_hdr.bin" #define FIRMWARE_MT7961 "mediatek/BT_RAM_CODE_MT7961_1_2_hdr.bin" +#define FIRMWARE_MT7920 "mediatek/BT_RAM_CODE_MT7961_1a_2_hdr.bin" #define FIRMWARE_MT7925 "mediatek/mt7925/BT_RAM_CODE_MT7925_1_1_hdr.bin" #define FIRMWARE_MT7927 "mediatek/mt7927/BT_RAM_CODE_MT6639_2_1_hdr.bin" diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c index c6f80c419e901..a15ae6598c665 100644 --- a/drivers/bluetooth/btmtksdio.c +++ b/drivers/bluetooth/btmtksdio.c @@ -217,7 +217,14 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev, } /* Parse and handle the return WMT event */ - wmt_evt = (struct btmtk_hci_wmt_evt *)bdev->evt_skb->data; + wmt_evt = skb_pull_data(bdev->evt_skb, sizeof(*wmt_evt)); + if (!wmt_evt) { + bt_dev_err(hdev, "WMT event too short (%u bytes)", + bdev->evt_skb->len); + err = -EINVAL; + goto err_free_skb; + } + if (wmt_evt->whdr.op != hdr->op) { bt_dev_err(hdev, "Wrong op received %d expected %d", wmt_evt->whdr.op, hdr->op); @@ -233,6 +240,17 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev, status = BTMTK_WMT_PATCH_DONE; break; case BTMTK_WMT_FUNC_CTRL: + if (!skb_pull_data(bdev->evt_skb, + sizeof(wmt_evt_funcc->status))) { + /* A plain enable/disable request is acked with just + * the WMT header and no trailing status word; the + * result is carried in the header's own flag byte. + */ + status = wmt_evt->whdr.flag ? BTMTK_WMT_ON_UNDONE : + BTMTK_WMT_ON_DONE; + break; + } + wmt_evt_funcc = (struct btmtk_hci_wmt_evt_funcc *)wmt_evt; if (be16_to_cpu(wmt_evt_funcc->status) == 0x404) status = BTMTK_WMT_ON_DONE; @@ -272,12 +290,24 @@ static int btmtksdio_tx_packet(struct btmtksdio_dev *bdev, struct sk_buff *skb) { struct mtkbtsdio_hdr *sdio_hdr; + unsigned int len, pad_len; int err; - /* Make sure that there are enough rooms for SDIO header */ - if (unlikely(skb_headroom(skb) < sizeof(*sdio_hdr))) { - err = pskb_expand_head(skb, sizeof(*sdio_hdr), 0, - GFP_ATOMIC); + /* Make sure that the data buffer is not shared with anyone else and + * that there is enough room for the SDIO header + */ + err = skb_cow_head(skb, sizeof(*sdio_hdr)); + if (err < 0) + return err; + + /* The transfer is rounded up to the SDIO block size, so the buffer + * has to provide tailroom for the padding as well + */ + len = skb->len + sizeof(*sdio_hdr); + pad_len = round_up(len, MTK_SDIO_BLOCK_SIZE) - len; + + if (unlikely(skb_tailroom(skb) < pad_len)) { + err = pskb_expand_head(skb, 0, pad_len, GFP_ATOMIC); if (err < 0) return err; } @@ -290,19 +320,22 @@ static int btmtksdio_tx_packet(struct btmtksdio_dev *bdev, sdio_hdr->reserved = cpu_to_le16(0); sdio_hdr->bt_type = hci_skb_pkt_type(skb); + /* Zero the padding so that no uninitialised memory is sent out */ + skb_put_zero(skb, pad_len); + clear_bit(BTMTKSDIO_HW_TX_READY, &bdev->tx_state); - err = sdio_writesb(bdev->func, MTK_REG_CTDR, skb->data, - round_up(skb->len, MTK_SDIO_BLOCK_SIZE)); + err = sdio_writesb(bdev->func, MTK_REG_CTDR, skb->data, skb->len); if (err < 0) - goto err_skb_pull; + goto err_skb_restore; - bdev->hdev->stat.byte_tx += skb->len; + bdev->hdev->stat.byte_tx += len; kfree_skb(skb); return 0; -err_skb_pull: +err_skb_restore: + skb_trim(skb, len); skb_pull(skb, sizeof(*sdio_hdr)); return err; @@ -1229,10 +1262,8 @@ static int btmtksdio_shutdown(struct hci_dev *hdev) wmt_params.status = NULL; err = mtk_hci_wmt_sync(hdev, &wmt_params); - if (err < 0) { + if (err < 0) bt_dev_err(hdev, "Failed to send wmt func ctrl (%d)", err); - return err; - } ignore_wmt_cmd: pm_runtime_put_noidle(bdev->dev); @@ -1480,6 +1511,9 @@ static void btmtksdio_remove(struct sdio_func *func) if (test_bit(BTMTKSDIO_FUNC_ENABLED, &bdev->tx_state)) btmtksdio_close(hdev); + if (bdev->data->pm_runtime_supported) + pm_runtime_dont_use_autosuspend(bdev->dev); + /* Be consistent the state in btmtksdio_probe */ pm_runtime_get_noresume(bdev->dev); diff --git a/drivers/bluetooth/btmtkuart.c b/drivers/bluetooth/btmtkuart.c index 27aa48ff3ac2c..4af6fbbbd302a 100644 --- a/drivers/bluetooth/btmtkuart.c +++ b/drivers/bluetooth/btmtkuart.c @@ -151,7 +151,14 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev, } /* Parse and handle the return WMT event */ - wmt_evt = (struct btmtk_hci_wmt_evt *)bdev->evt_skb->data; + wmt_evt = skb_pull_data(bdev->evt_skb, sizeof(*wmt_evt)); + if (!wmt_evt) { + bt_dev_err(hdev, "WMT event too short (%u bytes)", + bdev->evt_skb->len); + err = -EINVAL; + goto err_free_wc; + } + if (wmt_evt->whdr.op != hdr->op) { bt_dev_err(hdev, "Wrong op received %d expected %d", wmt_evt->whdr.op, hdr->op); @@ -167,6 +174,17 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev, status = BTMTK_WMT_PATCH_DONE; break; case BTMTK_WMT_FUNC_CTRL: + if (!skb_pull_data(bdev->evt_skb, + sizeof(wmt_evt_funcc->status))) { + /* A plain enable/disable request is acked with just + * the WMT header and no trailing status word; the + * result is carried in the header's own flag byte. + */ + status = wmt_evt->whdr.flag ? BTMTK_WMT_ON_UNDONE : + BTMTK_WMT_ON_DONE; + break; + } + wmt_evt_funcc = (struct btmtk_hci_wmt_evt_funcc *)wmt_evt; if (be16_to_cpu(wmt_evt_funcc->status) == 0x404) status = BTMTK_WMT_ON_DONE; diff --git a/drivers/bluetooth/btnxpuart.c b/drivers/bluetooth/btnxpuart.c index 6a1cffe08d5f7..73b319c7e1765 100644 --- a/drivers/bluetooth/btnxpuart.c +++ b/drivers/bluetooth/btnxpuart.c @@ -1368,12 +1368,21 @@ static int nxp_process_fw_dump(struct hci_dev *hdev, struct sk_buff *skb) { struct hci_acl_hdr *acl_hdr = (struct hci_acl_hdr *)skb_pull_data(skb, sizeof(*acl_hdr)); - struct nxp_fw_dump_hdr *fw_dump_hdr = (struct nxp_fw_dump_hdr *)skb->data; + struct nxp_fw_dump_hdr *fw_dump_hdr; struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev); - __u16 seq_num = __le16_to_cpu(fw_dump_hdr->seq_num); - __u16 buf_len = __le16_to_cpu(fw_dump_hdr->buf_len); + __u16 seq_num; + __u16 buf_len; int err; + fw_dump_hdr = skb_pull_data(skb, sizeof(*fw_dump_hdr)); + if (!fw_dump_hdr) { + bt_dev_warn(hdev, "FW dump: invalid or corrupt fw dump chunk"); + goto free_skb; + } + + seq_num = __le16_to_cpu(fw_dump_hdr->seq_num); + buf_len = __le16_to_cpu(fw_dump_hdr->buf_len); + if (seq_num == 0x0001) { if (test_and_set_bit(BTNXPUART_FW_DUMP_IN_PROGRESS, &nxpdev->tx_state)) { bt_dev_err(hdev, "FW dump already in progress"); @@ -1388,9 +1397,11 @@ static int nxp_process_fw_dump(struct hci_dev *hdev, struct sk_buff *skb) msecs_to_jiffies(20000)); } - err = hci_devcd_append(hdev, skb_clone(skb, GFP_ATOMIC)); - if (err < 0) - goto free_skb; + if (IS_ENABLED(CONFIG_DEV_COREDUMP)) { + err = hci_devcd_append(hdev, skb_clone(skb, GFP_ATOMIC)); + if (err < 0) + goto free_skb; + } if (buf_len == 0) { bt_dev_warn(hdev, "==== FW dump complete ==="); diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c index 10c496eaea2c1..4b0d83858229b 100644 --- a/drivers/bluetooth/btqca.c +++ b/drivers/bluetooth/btqca.c @@ -1029,8 +1029,7 @@ int qca_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr) baswap(&bdaddr_swapped, bdaddr); skb = __hci_cmd_sync_ev(hdev, EDL_WRITE_BD_ADDR_OPCODE, 6, - &bdaddr_swapped, HCI_EV_VENDOR, - HCI_INIT_TIMEOUT); + &bdaddr_swapped, 0, HCI_INIT_TIMEOUT); if (IS_ERR(skb)) { err = PTR_ERR(skb); bt_dev_err(hdev, "QCA Change address cmd failed (%d)", err); diff --git a/drivers/bluetooth/btqcomsmd.c b/drivers/bluetooth/btqcomsmd.c index d2e13fcb6babf..d669ea4eb3eb0 100644 --- a/drivers/bluetooth/btqcomsmd.c +++ b/drivers/bluetooth/btqcomsmd.c @@ -188,7 +188,10 @@ static int btqcomsmd_probe(struct platform_device *pdev) return 0; hci_free_dev: + rpmsg_destroy_ept(btq->cmd_channel); + rpmsg_destroy_ept(btq->acl_channel); hci_free_dev(hdev); + return ret; destroy_cmd_channel: rpmsg_destroy_ept(btq->cmd_channel); destroy_acl_channel: @@ -202,10 +205,11 @@ static void btqcomsmd_remove(struct platform_device *pdev) struct btqcomsmd *btq = platform_get_drvdata(pdev); hci_unregister_dev(btq->hdev); - hci_free_dev(btq->hdev); rpmsg_destroy_ept(btq->cmd_channel); rpmsg_destroy_ept(btq->acl_channel); + + hci_free_dev(btq->hdev); } static const struct of_device_id btqcomsmd_of_match[] = { diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c index 7f54d2d2d13a0..d29813331603b 100644 --- a/drivers/bluetooth/btrtl.c +++ b/drivers/bluetooth/btrtl.c @@ -591,7 +591,7 @@ static int rtlbt_parse_firmware_v2(struct hci_dev *hdev, * headers. */ if (!key_id) - break; + continue; rc = btrtl_parse_section(hdev, btrtl_dev, opcode, ptr, section_len); break; @@ -600,8 +600,7 @@ static int rtlbt_parse_firmware_v2(struct hci_dev *hdev, ptr, section_len); break; default: - rc = 0; - break; + continue; } if (rc < 0) { rtl_dev_err(hdev, "RTL: Parse section (%u) err %d", @@ -1343,19 +1342,6 @@ void btrtl_set_quirks(struct hci_dev *hdev, struct btrtl_device_info *btrtl_dev) if (!btrtl_dev->ic_info) return; - switch (btrtl_dev->project_id) { - case CHIP_ID_8761B: - /* RTL8761B/BU reports HCI version 5.1 but does not support - * the LE Extended Scan commands (Opcode 0x2042), causing - * repeated -EBUSY failures when BlueZ attempts extended - * scanning while a connection is active. - */ - hci_set_quirk(hdev, HCI_QUIRK_BROKEN_EXT_SCAN); - break; - default: - break; - } - switch (btrtl_dev->ic_info->lmp_subver) { case RTL_ROM_LMP_8703B: /* 8723CS reports two pages for local ext features, diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index 184e95c1625e5..87d50efcd2715 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -6,6 +6,7 @@ * Copyright (C) 2005-2008 Marcel Holtmann */ +#include #include #include #include @@ -67,8 +68,18 @@ static struct usb_driver btusb_driver; #define BTUSB_INTEL_NO_WBS_SUPPORT BIT(26) #define BTUSB_ACTIONS_SEMI BIT(27) #define BTUSB_BARROT BIT(28) +#define BTUSB_BROKEN_EXT_SCAN BIT(29) static const struct usb_device_id btusb_table[] = { + /* + * NXP IW610 (0471:0215): the composite device reports Bluetooth + * class at the whole-device level, so the generic entry below + * would also match this WiFi vendor interface. Ignore it here + * first so mwifiex-nxp can bind it instead. + */ + { USB_DEVICE_AND_INTERFACE_INFO(0x0471, 0x0215, 0xff, 0xff, 0xff), + .driver_info = BTUSB_IGNORE }, + /* Generic Bluetooth USB device */ { USB_DEVICE_INFO(0xe0, 0x01, 0x01) }, @@ -473,6 +484,14 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x1286, 0x2046), .driver_info = BTUSB_MARVELL }, { USB_DEVICE(0x1286, 0x204e), .driver_info = BTUSB_MARVELL }, + /* + * NXP IW610 BT interfaces (Marvell-lineage silicon, same quirk as + * the 0x1286 entries above). Scoped to the BT interface class, + * not just VID/PID -- see the btusb_table entry above. + */ + { USB_DEVICE_AND_INTERFACE_INFO(0x0471, 0x0215, 0xe0, 0x01, 0x01), + .driver_info = BTUSB_MARVELL }, + /* Intel Bluetooth devices */ { USB_DEVICE(0x8087, 0x0025), .driver_info = BTUSB_INTEL_COMBINED }, { USB_DEVICE(0x8087, 0x0026), .driver_info = BTUSB_INTEL_COMBINED }, @@ -615,6 +634,10 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x0489, 0xe130), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + /* Realtek 8761BU Bluetooth devices */ + { USB_DEVICE(0x0bda, 0xa728), .driver_info = BTUSB_REALTEK | + BTUSB_BROKEN_EXT_SCAN }, + /* Realtek Bluetooth devices */ { USB_VENDOR_AND_INTERFACE_INFO(0x0bda, 0xe0, 0x01, 0x01), .driver_info = BTUSB_REALTEK }, @@ -850,6 +873,12 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x37ad, 0x0600), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + /* Additional Realtek 8761CU Bluetooth devices */ + { USB_DEVICE(0x0b05, 0x1bef), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0b05, 0x1d70), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + /* Additional Realtek 8821AE Bluetooth devices */ { USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK }, { USB_DEVICE(0x13d3, 0x3414), .driver_info = BTUSB_REALTEK }, @@ -957,6 +986,8 @@ struct qca_dump_info { #define BTUSB_USE_ALT3_FOR_WBS 15 #define BTUSB_ALT6_CONTINUOUS_TX 16 #define BTUSB_HW_SSR_ACTIVE 17 +#define BTUSB_WAKEUP_BROKEN 18 +#define BTUSB_RESET 19 struct btusb_data { struct hci_dev *hdev; @@ -1010,6 +1041,7 @@ struct btusb_data { bool usb_alt6_packet_flow; int isoc_altsetting; int suspend_count; + const struct usb_device_id *match_id; int (*recv_event)(struct hci_dev *hdev, struct sk_buff *skb); int (*recv_acl)(struct hci_dev *hdev, struct sk_buff *skb); @@ -1032,13 +1064,15 @@ static void btusb_reset(struct hci_dev *hdev) int err; data = hci_get_drvdata(hdev); - /* This is not an unbalanced PM reference since the device will reset */ err = usb_autopm_get_interface(data->intf); if (err) { bt_dev_err(hdev, "Failed usb_autopm_get_interface: %d", err); return; } + if (test_and_set_bit(BTUSB_RESET, &data->flags)) + usb_autopm_put_interface_no_suspend(data->intf); + bt_dev_err(hdev, "Resetting usb device."); usb_queue_reset_device(data->intf); } @@ -2060,18 +2094,24 @@ static int btusb_close(struct hci_dev *hdev) BT_DBG("%s", hdev->name); - cancel_delayed_work(&data->rx_work); cancel_work_sync(&data->work); cancel_work_sync(&data->waker); - skb_queue_purge(&data->acl_q); - clear_bit(BTUSB_ISOC_RUNNING, &data->flags); clear_bit(BTUSB_BULK_RUNNING, &data->flags); clear_bit(BTUSB_INTR_RUNNING, &data->flags); clear_bit(BTUSB_DIAG_RUNNING, &data->flags); btusb_stop_traffic(data); + + /* rx_work must only be canceled once the URBs that can rearm it are + * gone, and it must be canceled synchronously since btusb_disconnect() + * frees the btusb_data it dereferences right after hci_unregister_dev(). + */ + cancel_delayed_work_sync(&data->rx_work); + + skb_queue_purge(&data->acl_q); + btusb_free_frags(data); err = usb_autopm_get_interface(data->intf); @@ -2097,7 +2137,7 @@ static int btusb_flush(struct hci_dev *hdev) BT_DBG("%s", hdev->name); - cancel_delayed_work(&data->rx_work); + cancel_delayed_work_sync(&data->rx_work); skb_queue_purge(&data->acl_q); @@ -2891,8 +2931,11 @@ static int btusb_mtk_reset(struct hci_dev *hdev, void *rst_data) } err = usb_autopm_get_interface(data->intf); - if (err < 0) + if (err < 0) { + bt_dev_err(hdev, "Failed usb_autopm_get_interface: %d", err); + clear_bit(BTMTK_HW_RESET_ACTIVE, &btmtk_data->flags); return err; + } /* Release MediaTek ISO data interface */ btusb_mtk_release_iso_intf(hdev); @@ -2915,6 +2958,11 @@ static int btusb_mtk_reset(struct hci_dev *hdev, void *rst_data) err = btmtk_usb_subsys_reset(hdev, btmtk_data->dev_id); + if (test_and_set_bit(BTUSB_RESET, &data->flags)) { + bt_dev_err(hdev, "last usb reset failed? Resetting again"); + usb_autopm_put_interface_no_suspend(data->intf); + } + usb_queue_reset_device(data->intf); clear_bit(BTMTK_HW_RESET_ACTIVE, &btmtk_data->flags); @@ -2938,10 +2986,25 @@ static int btusb_send_frame_mtk(struct hci_dev *hdev, struct sk_buff *skb) } } +static inline bool platform_is_ryzen(void) +{ +#ifdef CONFIG_X86 + return boot_cpu_has(X86_FEATURE_ZEN); +#else + return false; +#endif +} + +static inline bool is_direct_child_of_root_hub(struct usb_device *udev) +{ + return udev->parent == udev->bus->root_hub; +} + static int btusb_mtk_setup(struct hci_dev *hdev) { struct btusb_data *data = hci_get_drvdata(hdev); struct btmtk_data *btmtk_data = hci_get_priv(hdev); + int err; /* MediaTek WMT vendor cmd requiring below USB resources to * complete the handshake. @@ -2958,7 +3021,40 @@ static int btusb_mtk_setup(struct hci_dev *hdev) btusb_mtk_claim_iso_intf(data); } - return btmtk_usb_setup(hdev); + err = btmtk_usb_setup(hdev); + if (err) + return err; + + switch (btmtk_data->dev_id) { + case 0x7922: + case 0x7925: + /* + * All reports seen to be relevant to Ryzen-based laptops. These + * NICs are usually used as OEM components thanks to some sort + * of reference designs. + * + * Their popularity on other platforms is unclear. While there + * is still a chance that the quirk may exist on other + * platforms, be cautious and only apply the quirk to direct + * children of Ryzen platforms's root hubs for the time being. + * + * In most cases the root hub is on the SoC or PCH, which needs + * the quirk. Unfortunately, this can't distinguish root hubs on + * PCIe add-in cards. Such roughness should be acceptable, as + * PCIe USB controller add-in cards are less commonly used + * nowadays. On the other hand, applying the quirk doesn't hurt + * any functionalities either, as the device can still be used + * as a wakeup source if desired. + * + * Theoretically, we could retrieve the root hub's PCI vendor ID + * with some hierarchy magic, but that's too intrusive... + */ + if (platform_is_ryzen() && is_direct_child_of_root_hub(data->udev)) + set_bit(BTUSB_WAKEUP_BROKEN, &data->flags); + break; + } + + return 0; } static int btusb_mtk_shutdown(struct hci_dev *hdev) @@ -3072,14 +3168,15 @@ static int btusb_set_bdaddr_ath3012(struct hci_dev *hdev, static int btusb_set_bdaddr_wcn6855(struct hci_dev *hdev, const bdaddr_t *bdaddr) { + bdaddr_t bdaddr_swapped; struct sk_buff *skb; - u8 buf[6]; long ret; - memcpy(buf, bdaddr, sizeof(bdaddr_t)); + baswap(&bdaddr_swapped, bdaddr); - skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(buf), buf, - HCI_EV_CMD_COMPLETE, HCI_INIT_TIMEOUT); + skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(bdaddr_swapped), + &bdaddr_swapped, HCI_EV_CMD_COMPLETE, + HCI_INIT_TIMEOUT); if (IS_ERR(skb)) { ret = PTR_ERR(skb); bt_dev_err(hdev, "Change address command failed (%ld)", ret); @@ -3695,8 +3792,10 @@ static int btusb_setup_qca(struct hci_dev *hdev) if (err) return err; - btdata->qca_dump.fw_version = le32_to_cpu(ver.patch_version); - btdata->qca_dump.controller_id = le32_to_cpu(ver.rom_version); + if (btdata->match_id->driver_info & BTUSB_QCA_WCN6855) { + btdata->qca_dump.fw_version = le32_to_cpu(ver.patch_version); + btdata->qca_dump.controller_id = le32_to_cpu(ver.rom_version); + } if (!(status & QCA_SYSCFG_UPDATED)) { err = btusb_setup_qca_load_nvm(hdev, &ver, info); @@ -4107,6 +4206,7 @@ static int btusb_probe(struct usb_interface *intf, if (!data) return -ENOMEM; + data->match_id = id; err = usb_find_common_endpoints(intf->cur_altsetting, &data->bulk_rx_ep, &data->bulk_tx_ep, &data->intr_ep, NULL); if (err) @@ -4366,6 +4466,9 @@ static int btusb_probe(struct usb_interface *intf, if (id->driver_info & BTUSB_INVALID_LE_STATES) hci_set_quirk(hdev, HCI_QUIRK_BROKEN_LE_STATES); + if (id->driver_info & BTUSB_BROKEN_EXT_SCAN) + hci_set_quirk(hdev, HCI_QUIRK_BROKEN_EXT_SCAN); + if (id->driver_info & BTUSB_DIGIANSWER) { data->cmdreq_type = USB_TYPE_VENDOR; hci_set_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE); @@ -4497,6 +4600,9 @@ static void btusb_disconnect(struct usb_interface *intf) if (data->reset_gpio) gpiod_put(data->reset_gpio); + if (test_and_clear_bit(BTUSB_RESET, &data->flags)) + usb_autopm_put_interface_no_suspend(data->intf); + if (intf == data->intf) { if (data->isoc) usb_driver_release_interface(&btusb_driver, data->isoc); @@ -4522,11 +4628,26 @@ static int btusb_suspend(struct usb_interface *intf, pm_message_t message) BT_DBG("intf %p", intf); - /* Don't auto-suspend if there are connections or discovery in - * progress; external suspend calls shall never fail. + /* + * It is reported that remote wakeup events could sometimes cause some + * adapters completely unresponsive. Resetting the xHCI root hub doesn't + * help at all, and recovering from such a state needs a power cycle. + * Since disabling remote wakeup simply causes the USB core to gate + * runtime autosuspend as well due to needs_remote_wakeup == 1, let's do + * this ourselves to make our life easier. The interface can be safely + * autosuspended as long as remote wakeup is disabled, i.e., after + * closing the HCI device. + * + * Don't auto-suspend if there are connections or discovery in progress. + * + * External suspend calls shall never fail. Specifically, a device with + * broken remote wakeup may still take the advantage of remote wakeup in + * order to wake up the system from sleep if userspace has enabled it as + * a wakeup source. */ if (PMSG_IS_AUTO(message) && - (hci_conn_count(data->hdev) || hci_discovery_active(data->hdev))) + ((test_bit(BTUSB_WAKEUP_BROKEN, &data->flags) && data->intf->needs_remote_wakeup) || + hci_conn_count(data->hdev) || hci_discovery_active(data->hdev))) return -EBUSY; if (data->suspend_count++) diff --git a/drivers/bluetooth/hci_aml.c b/drivers/bluetooth/hci_aml.c index 959d9e67b669c..067fbf278b44c 100644 --- a/drivers/bluetooth/hci_aml.c +++ b/drivers/bluetooth/hci_aml.c @@ -247,7 +247,7 @@ static int aml_download_firmware(struct hci_dev *hdev, const char *fw_name) struct hci_uart *hu = hci_get_drvdata(hdev); struct aml_serdev *amldev = serdev_device_get_drvdata(hu->serdev); const struct firmware *firmware = NULL; - struct aml_fw_len *fw_len = NULL; + const struct aml_fw_len *fw_len = NULL; u8 *iccm_start = NULL, *dccm_start = NULL; u32 iccm_len, dccm_len; u32 value = 0; @@ -281,7 +281,21 @@ static int aml_download_firmware(struct hci_dev *hdev, const char *fw_name) goto exit; } - fw_len = (struct aml_fw_len *)firmware->data; + if (firmware->size < sizeof(*fw_len)) { + bt_dev_err(hdev, "Firmware is too small for its header"); + ret = -EINVAL; + goto exit; + } + + fw_len = (const struct aml_fw_len *)firmware->data; + if (fw_len->iccm_len < amldev->aml_dev_data->iccm_offset || + fw_len->iccm_len > firmware->size - sizeof(*fw_len) || + fw_len->dccm_len > firmware->size - sizeof(*fw_len) - + fw_len->iccm_len) { + bt_dev_err(hdev, "Invalid firmware segment lengths"); + ret = -EINVAL; + goto exit; + } /* Download ICCM */ iccm_start = (u8 *)(firmware->data) + sizeof(struct aml_fw_len) diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c index 1a4fc3882fd2a..cc419e1960de5 100644 --- a/drivers/bluetooth/hci_bcm.c +++ b/drivers/bluetooth/hci_bcm.c @@ -547,6 +547,7 @@ static int bcm_close(struct hci_uart *hu) if (IS_ENABLED(CONFIG_PM) && bdev->irq_acquired) { devm_free_irq(bdev->dev, bdev->irq, bdev); device_init_wakeup(bdev->dev, false); + pm_runtime_dont_use_autosuspend(bdev->dev); pm_runtime_disable(bdev->dev); } diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 925d0a6359453..66d49b4715440 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -2490,6 +2490,7 @@ static const struct bcm4377_hw bcm4377_hw_variants[] = { .has_bar0_core2_window2 = true, .broken_mws_transport_config = true, .broken_le_coded = true, + .broken_le_ext_adv_report_phy = true, .send_calibration = bcm4378_send_calibration, .send_ptb = bcm4378_send_ptb, }, diff --git a/drivers/bluetooth/hci_h5.c b/drivers/bluetooth/hci_h5.c index 93cdde9818401..a5e7e9b0ea746 100644 --- a/drivers/bluetooth/hci_h5.c +++ b/drivers/bluetooth/hci_h5.c @@ -1023,8 +1023,10 @@ static void h5_btrtl_open(struct h5 *h5) static void h5_btrtl_close(struct h5 *h5) { - if (!test_bit(H5_WAKEUP_DISABLE, &h5->flags)) + if (!test_bit(H5_WAKEUP_DISABLE, &h5->flags)) { + pm_runtime_dont_use_autosuspend(&h5->hu->serdev->dev); pm_runtime_disable(&h5->hu->serdev->dev); + } gpiod_set_value_cansleep(h5->device_wake_gpio, 0); gpiod_set_value_cansleep(h5->enable_gpio, 0); diff --git a/drivers/bluetooth/hci_intel.c b/drivers/bluetooth/hci_intel.c index c31105b91e476..1919547ff2a1c 100644 --- a/drivers/bluetooth/hci_intel.c +++ b/drivers/bluetooth/hci_intel.c @@ -345,6 +345,7 @@ static int intel_set_power(struct hci_uart *hu, bool powered) devm_free_irq(&idev->pdev->dev, idev->irq, idev); device_wakeup_disable(&idev->pdev->dev); + pm_runtime_dont_use_autosuspend(&idev->pdev->dev); pm_runtime_disable(&idev->pdev->dev); } } diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index 2ad42c3bbaac7..27644d2e53f8c 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -451,7 +451,7 @@ static int hci_uart_setup(struct hci_dev *hdev) if (IS_ERR(skb)) { BT_ERR("%s: Reading local version information failed (%ld)", hdev->name, PTR_ERR(skb)); - return 0; + return PTR_ERR(skb); } if (skb->len != sizeof(*ver)) { diff --git a/drivers/bluetooth/hci_mrvl.c b/drivers/bluetooth/hci_mrvl.c index 516b8f74c4340..5798a8db016ee 100644 --- a/drivers/bluetooth/hci_mrvl.c +++ b/drivers/bluetooth/hci_mrvl.c @@ -307,9 +307,8 @@ static int mrvl_load_firmware(struct hci_dev *hdev, const char *name) err = wait_on_bit_timeout(&mrvl->flags, STATE_FW_REQ_PENDING, TASK_INTERRUPTIBLE, msecs_to_jiffies(2000)); - if (err == 1) { + if (err == -EINTR) { bt_dev_err(hdev, "Firmware load interrupted"); - err = -EINTR; break; } else if (err) { bt_dev_err(hdev, "Firmware request timeout"); diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index 1222f97800f4a..875093e61c681 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -2228,8 +2228,8 @@ static void qca_power_off(struct hci_uart *hu) bool sw_ctrl_state; struct qca_power *power; - /* From this point we go into power off state. But serial port is - * still open, stop queueing the IBS data and flush all the buffered + /* From this point we go into power off state. But serial port may + * still be open, stop queueing the IBS data and flush all the buffered * data in skb's. */ spin_lock_irqsave(&qca->hci_ibs_lock, flags); @@ -2251,8 +2251,14 @@ static void qca_power_off(struct hci_uart *hu) case QCA_WCN3990: case QCA_WCN3991: case QCA_WCN3998: - host_set_baudrate(hu, 2400); - qca_send_power_pulse(hu, false); + /* Both of these write to the serial port which may have + * already been closed by hci_uart_close(), which closes + * the port if HCI_QUIRK_NON_PERSISTENT_SETUP is set. + */ + if (test_bit(HCI_UART_PROTO_READY, &hu->flags)) { + host_set_baudrate(hu, 2400); + qca_send_power_pulse(hu, false); + } break; default: break; diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c index 593d9cefbbf92..d2eaf2f12aa27 100644 --- a/drivers/bluetooth/hci_serdev.c +++ b/drivers/bluetooth/hci_serdev.c @@ -221,7 +221,7 @@ static int hci_uart_setup(struct hci_dev *hdev) if (IS_ERR(skb)) { bt_dev_err(hdev, "Reading local version info failed (%ld)", PTR_ERR(skb)); - return 0; + return PTR_ERR(skb); } if (skb->len != sizeof(*ver)) diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c index 140ab55c9fc5a..c20d54088c8c4 100644 --- a/drivers/bluetooth/virtio_bt.c +++ b/drivers/bluetooth/virtio_bt.c @@ -120,9 +120,13 @@ static int virtbt_setup_zephyr(struct hci_dev *hdev) if (IS_ERR(skb)) return PTR_ERR(skb); - bt_dev_info(hdev, "%s", (char *)(skb->data + 1)); + /* Bounded print: the backend controls skb->len. */ + if (skb->len > 1) { + int len = skb->len - 1; - hci_set_fw_info(hdev, "%s", skb->data + 1); + bt_dev_info(hdev, "%.*s", len, (char *)(skb->data + 1)); + hci_set_fw_info(hdev, "%.*s", len, skb->data + 1); + } kfree_skb(skb); return 0; diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c index b1213786f72c6..21bc2c50170ff 100644 --- a/drivers/bus/mhi/ep/main.c +++ b/drivers/bus/mhi/ep/main.c @@ -1340,14 +1340,19 @@ static int mhi_ep_create_device(struct mhi_ep_cntrl *mhi_cntrl, u32 ch_id) ret = dev_set_name(&mhi_dev->dev, "%s_%s", dev_name(&mhi_cntrl->mhi_dev->dev), mhi_dev->name); - if (ret) { - put_device(&mhi_dev->dev); - return ret; - } + if (ret) + goto err_put_channels; ret = device_add(&mhi_dev->dev); if (ret) - put_device(&mhi_dev->dev); + goto err_put_channels; + + return 0; + +err_put_channels: + put_device(&mhi_dev->dev); /* DL channel reference */ + put_device(&mhi_dev->dev); /* UL channel reference */ + put_device(&mhi_dev->dev); /* device_initialize() reference */ return ret; } diff --git a/drivers/bus/mhi/host/init.c b/drivers/bus/mhi/host/init.c index 12dcb1a2753c7..fd3050889412d 100644 --- a/drivers/bus/mhi/host/init.c +++ b/drivers/bus/mhi/host/init.c @@ -1029,7 +1029,7 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, if (mhi_cntrl->edl_trigger) { ret = sysfs_create_file(&mhi_dev->dev.kobj, &dev_attr_trigger_edl.attr); if (ret) - goto err_release_dev; + goto err_del_dev; } mhi_cntrl->mhi_dev = mhi_dev; @@ -1038,6 +1038,8 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, return 0; +err_del_dev: + device_del(&mhi_dev->dev); err_release_dev: put_device(&mhi_dev->dev); error_setup_irq: diff --git a/drivers/bus/mhi/host/main.c b/drivers/bus/mhi/host/main.c index 53c0ffe300702..4d458396233ab 100644 --- a/drivers/bus/mhi/host/main.c +++ b/drivers/bus/mhi/host/main.c @@ -170,6 +170,9 @@ EXPORT_SYMBOL_GPL(mhi_get_mhi_state); void mhi_soc_reset(struct mhi_controller *mhi_cntrl) { + int __maybe_unused ret; + u32 tmp; + if (mhi_cntrl->reset) { mhi_cntrl->reset(mhi_cntrl); return; @@ -178,6 +181,9 @@ void mhi_soc_reset(struct mhi_controller *mhi_cntrl) /* Generic MHI SoC reset */ mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, MHI_SOC_RESET_REQ); + /* Flush the posted write to the device (ignore return value) */ + ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, + &tmp); } EXPORT_SYMBOL_GPL(mhi_soc_reset); diff --git a/drivers/bus/qcom-ebi2.c b/drivers/bus/qcom-ebi2.c index be8166565e7cc..8d2eb955dc921 100644 --- a/drivers/bus/qcom-ebi2.c +++ b/drivers/bus/qcom-ebi2.c @@ -302,41 +302,23 @@ static int qcom_ebi2_probe(struct platform_device *pdev) u32 val; int ret; - ebi2xclk = devm_clk_get(dev, "ebi2x"); + ebi2xclk = devm_clk_get_enabled(dev, "ebi2x"); if (IS_ERR(ebi2xclk)) return PTR_ERR(ebi2xclk); - ret = clk_prepare_enable(ebi2xclk); - if (ret) { - dev_err(dev, "could not enable EBI2X clk (%d)\n", ret); - return ret; - } - - ebi2clk = devm_clk_get(dev, "ebi2"); - if (IS_ERR(ebi2clk)) { - ret = PTR_ERR(ebi2clk); - goto err_disable_2x_clk; - } - - ret = clk_prepare_enable(ebi2clk); - if (ret) { - dev_err(dev, "could not enable EBI2 clk\n"); - goto err_disable_2x_clk; - } + ebi2clk = devm_clk_get_enabled(dev, "ebi2"); + if (IS_ERR(ebi2clk)) + return PTR_ERR(ebi2clk); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ebi2_base = devm_ioremap_resource(dev, res); - if (IS_ERR(ebi2_base)) { - ret = PTR_ERR(ebi2_base); - goto err_disable_clk; - } + if (IS_ERR(ebi2_base)) + return PTR_ERR(ebi2_base); res = platform_get_resource(pdev, IORESOURCE_MEM, 1); ebi2_xmem = devm_ioremap_resource(dev, res); - if (IS_ERR(ebi2_xmem)) { - ret = PTR_ERR(ebi2_xmem); - goto err_disable_clk; - } + if (IS_ERR(ebi2_xmem)) + return PTR_ERR(ebi2_xmem); /* Allegedly this turns the power save mode off */ writel(0UL, ebi2_xmem + EBI2_XMEM_CFG); @@ -373,15 +355,9 @@ static int qcom_ebi2_probe(struct platform_device *pdev) } if (have_children) - return of_platform_default_populate(np, NULL, dev); - return 0; + return devm_of_platform_populate(dev); -err_disable_clk: - clk_disable_unprepare(ebi2clk); -err_disable_2x_clk: - clk_disable_unprepare(ebi2xclk); - - return ret; + return 0; } static const struct of_device_id qcom_ebi2_of_match[] = { diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c index a5b9507de37c3..e118b900c9ac8 100644 --- a/drivers/bus/ti-sysc.c +++ b/drivers/bus/ti-sysc.c @@ -682,6 +682,7 @@ static struct device_node *stdout_path; static void sysc_init_stdout_path(struct sysc *ddata) { + struct device_node *chosen; struct device_node *np = NULL; const char *uart; @@ -691,15 +692,18 @@ static void sysc_init_stdout_path(struct sysc *ddata) if (stdout_path) return; - np = of_find_node_by_path("/chosen"); - if (!np) + chosen = of_find_node_by_path("/chosen"); + if (!chosen) goto err; - uart = of_get_property(np, "stdout-path", NULL); - if (!uart) + uart = of_get_property(chosen, "stdout-path", NULL); + if (!uart) { + of_node_put(chosen); goto err; + } np = of_find_node_by_path(uart); + of_node_put(chosen); if (!np) goto err; diff --git a/drivers/cdx/cdx.c b/drivers/cdx/cdx.c index d3d2302472621..a118a89a7779d 100644 --- a/drivers/cdx/cdx.c +++ b/drivers/cdx/cdx.c @@ -738,7 +738,6 @@ static int cdx_create_res_attr(struct cdx_device *cdx_dev, int num) sysfs_bin_attr_init(res_attr); - cdx_dev->res_attr[num] = res_attr; sprintf(res_attr_name, "resource%d", num); res_attr->mmap = cdx_mmap_resource; @@ -747,8 +746,12 @@ static int cdx_create_res_attr(struct cdx_device *cdx_dev, int num) res_attr->size = cdx_resource_len(cdx_dev, num); res_attr->private = (void *)(unsigned long)num; ret = sysfs_create_bin_file(&cdx_dev->dev.kobj, res_attr); - if (ret) + if (ret) { kfree(res_attr); + return ret; + } + + cdx_dev->res_attr[num] = res_attr; return ret; } diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 6931657ad2caa..e77af6578ab50 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -596,11 +596,13 @@ int hwrng_register(struct hwrng *rng) */ err = set_current_rng(rng); if (err) - goto out_unlock; + goto out_list_del; } } mutex_unlock(&rng_mutex); return 0; +out_list_del: + list_del_init(&rng->list); out_unlock: mutex_unlock(&rng_mutex); out: diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index 28c56c2d1bf6d..a2db26d7f790e 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -313,8 +313,10 @@ static int __init imx_rngc_probe(struct platform_device *pdev) devm_pm_runtime_enable(&pdev->dev); ret = devm_hwrng_register(&pdev->dev, &rngc->rng); - if (ret) + if (ret) { + clk_disable_unprepare(rngc->clk); return dev_err_probe(&pdev->dev, ret, "hwrng registration failed\n"); + } dev_info(&pdev->dev, "Freescale RNG%c registered (HW revision %d.%02d)\n", diff --git a/drivers/char/hw_random/ks-sa-rng.c b/drivers/char/hw_random/ks-sa-rng.c index 9e408144a10c1..4494f1e4ab4db 100644 --- a/drivers/char/hw_random/ks-sa-rng.c +++ b/drivers/char/hw_random/ks-sa-rng.c @@ -242,7 +242,14 @@ static int ks_sa_rng_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "Failed to enable SA power-domain\n"); } - return devm_hwrng_register(&pdev->dev, &ks_sa_rng->rng); + ret = devm_hwrng_register(dev, &ks_sa_rng->rng); + if (ret) { + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + return ret; + } + + return 0; } static void ks_sa_rng_remove(struct platform_device *pdev) diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index 5e8b50f15db75..a8c0b3dfb133c 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -455,32 +455,40 @@ static int omap_rng_probe(struct platform_device *pdev) ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { dev_err(&pdev->dev, "Failed to runtime_get device: %d\n", ret); - goto err_ioremap; + goto err_pm_disable; } priv->clk = devm_clk_get(&pdev->dev, NULL); - if (PTR_ERR(priv->clk) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (PTR_ERR(priv->clk) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + goto err_pm_put; + } if (!IS_ERR(priv->clk)) { ret = clk_prepare_enable(priv->clk); if (ret) { dev_err(&pdev->dev, "Unable to enable the clk: %d\n", ret); - goto err_register; + goto err_pm_put; } + } else { + priv->clk = NULL; } priv->clk_reg = devm_clk_get(&pdev->dev, "reg"); - if (PTR_ERR(priv->clk_reg) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (PTR_ERR(priv->clk_reg) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + goto err_clk; + } if (!IS_ERR(priv->clk_reg)) { ret = clk_prepare_enable(priv->clk_reg); if (ret) { dev_err(&pdev->dev, "Unable to enable the register clk: %d\n", ret); - goto err_register; + goto err_clk; } + } else { + priv->clk_reg = NULL; } ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) : @@ -498,12 +506,14 @@ static int omap_rng_probe(struct platform_device *pdev) return 0; err_register: + clk_disable_unprepare(priv->clk_reg); +err_clk: + clk_disable_unprepare(priv->clk); +err_pm_put: priv->base = NULL; pm_runtime_put_sync(&pdev->dev); +err_pm_disable: pm_runtime_disable(&pdev->dev); - - clk_disable_unprepare(priv->clk_reg); - clk_disable_unprepare(priv->clk); err_ioremap: dev_err(dev, "initialization failed.\n"); return ret; diff --git a/drivers/char/hw_random/stm32-rng.c b/drivers/char/hw_random/stm32-rng.c index 9a8c00586ab09..f5bfe54c01dc7 100644 --- a/drivers/char/hw_random/stm32-rng.c +++ b/drivers/char/hw_random/stm32-rng.c @@ -368,11 +368,6 @@ static int stm32_rng_init(struct hwrng *rng) return 0; } -static void stm32_rng_remove(struct platform_device *ofdev) -{ - pm_runtime_disable(&ofdev->dev); -} - static int __maybe_unused stm32_rng_runtime_suspend(struct device *dev) { struct stm32_rng_private *priv = dev_get_drvdata(dev); @@ -590,7 +585,9 @@ static int stm32_rng_probe(struct platform_device *ofdev) pm_runtime_set_autosuspend_delay(dev, 100); pm_runtime_use_autosuspend(dev); - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; return devm_hwrng_register(dev, &priv->rng); } @@ -602,7 +599,6 @@ static struct platform_driver stm32_rng_driver = { .of_match_table = stm32_rng_match, }, .probe = stm32_rng_probe, - .remove = stm32_rng_remove, }; module_platform_driver(stm32_rng_driver); diff --git a/drivers/char/hw_random/xilinx-trng.c b/drivers/char/hw_random/xilinx-trng.c index 0fbc22c38fbc7..290bf5bc82db8 100644 --- a/drivers/char/hw_random/xilinx-trng.c +++ b/drivers/char/hw_random/xilinx-trng.c @@ -86,8 +86,8 @@ static void xtrng_softreset(struct xilinx_rng *rng) xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSRST_MASK, 0); } -/* Return no. of bytes read */ -static size_t xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks32, bool wait) +/* Return no. of bytes read or a negative error before any data is read. */ +static int xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks32, bool wait) { int read = 0, ret; int timeout = 1; @@ -102,8 +102,11 @@ static size_t xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks3 ret = readl_poll_timeout(rng_base + TRNG_STATUS_OFFSET, val, (val & TRNG_STATUS_QCNT_MASK) == TRNG_STATUS_QCNT_16_BYTES, !!wait, timeout); - if (ret) + if (ret) { + if (!read) + return ret; break; + } for (idx = 0; idx < TRNG_READ_4_WORD; idx++) { *(buf + read) = cpu_to_be32(ioread32(rng_base + TRNG_CORE_OUTPUT_OFFSET)); @@ -118,27 +121,40 @@ static int xtrng_collect_random_data(struct xilinx_rng *rng, u8 *rand_gen_buf, { u8 randbuf[TRNG_SEC_STRENGTH_BYTES]; int byteleft, blocks, count = 0; + int full_blocks_bytes; int ret; byteleft = no_of_random_bytes & (TRNG_SEC_STRENGTH_BYTES - 1); blocks = no_of_random_bytes >> TRNG_SEC_STRENGTH_SHIFT; + full_blocks_bytes = blocks * TRNG_SEC_STRENGTH_BYTES; xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSTART_MASK, TRNG_CTRL_PRNGSTART_MASK); if (blocks) { ret = xtrng_readblock32(rng->rng_base, (__be32 *)rand_gen_buf, blocks, wait); - if (!ret) - return 0; + if (ret <= 0) { + count = ret; + goto out_stop; + } count += ret; + if (ret < full_blocks_bytes) + goto out_stop; } if (byteleft) { ret = xtrng_readblock32(rng->rng_base, (__be32 *)randbuf, 1, wait); + if (ret < 0) { + if (!count) + count = ret; + goto out_stop; + } if (!ret) - return count; - memcpy(rand_gen_buf + (blocks * TRNG_SEC_STRENGTH_BYTES), randbuf, byteleft); - count += byteleft; + goto out_stop; + ret = min(ret, no_of_random_bytes - count); + memcpy(rand_gen_buf + count, randbuf, ret); + count += ret; } +out_stop: xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGMODE_MASK | TRNG_CTRL_PRNGSTART_MASK, 0U); diff --git a/drivers/char/ipmi/ipmb_dev_int.c b/drivers/char/ipmi/ipmb_dev_int.c index 680ff15c30ab9..e4c50d9ae3e13 100644 --- a/drivers/char/ipmi/ipmb_dev_int.c +++ b/drivers/char/ipmi/ipmb_dev_int.c @@ -141,13 +141,14 @@ static ssize_t ipmb_write(struct file *file, const char __user *buf, u8 msg[MAX_MSG_LEN]; ssize_t ret; - if (count > sizeof(msg)) + if (!count || count > sizeof(msg)) return -EINVAL; if (copy_from_user(&msg, buf, count)) return -EFAULT; - if (count < msg[0]) + if (msg[IPMB_MSG_LEN_IDX] < IPMB_REQUEST_LEN_MIN || + count < (size_t)msg[IPMB_MSG_LEN_IDX] + 1) return -EINVAL; rq_sa = GET_7BIT_ADDR(msg[RQ_SA_8BIT_IDX]); diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index ab4c85f3d6fe8..0330d8412a0e7 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -1391,6 +1391,7 @@ static void _ipmi_destroy_user(struct ipmi_user *user) } } mutex_unlock(&intf->cmd_rcvrs_mutex); + synchronize_rcu(); while (rcvrs) { rcvr = rcvrs; rcvrs = rcvr->next; @@ -3740,6 +3741,7 @@ int ipmi_add_smi(struct module *owner, sysfs_attr_init(&intf->maintenance_mode_devattr.attr); rv = device_create_file(intf->si_dev, &intf->maintenance_mode_devattr); if (rv) { + device_remove_file(intf->si_dev, &intf->nr_msgs_devattr); device_remove_file(intf->si_dev, &intf->nr_users_devattr); goto out_err_bmc_reg; } @@ -3757,12 +3759,14 @@ int ipmi_add_smi(struct module *owner, out_err_bmc_reg: ipmi_bmc_unregister(intf); out_err_started: + intf->in_shutdown = true; if (intf->handlers->shutdown) intf->handlers->shutdown(intf->send_info); out_err: list_del(&intf->link); mutex_unlock(&ipmi_interfaces_mutex); mutex_unlock(&smi_watchers_mutex); + cancel_work_sync(&intf->smi_work); kref_put(&intf->refcount, intf_free); return rv; diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c index 6da817b9849f3..8803268b4cdc8 100644 --- a/drivers/char/ppdev.c +++ b/drivers/char/ppdev.c @@ -340,15 +340,17 @@ static enum ieee1284_phase init_phase(int mode) return IEEE1284_PH_FWD_IDLE; } -static int pp_set_timeout(struct pardevice *pdev, long tv_sec, int tv_usec) +static int pp_set_timeout(struct pardevice *pdev, s64 tv_sec, s64 tv_usec) { + struct timespec64 ts; long to_jiffies; - if ((tv_sec < 0) || (tv_usec < 0)) + if (tv_sec < 0 || tv_usec < 0 || tv_usec >= USEC_PER_SEC) return -EINVAL; - to_jiffies = usecs_to_jiffies(tv_usec); - to_jiffies += tv_sec * HZ; + ts.tv_sec = tv_sec; + ts.tv_nsec = tv_usec * NSEC_PER_USEC; + to_jiffies = timespec64_to_jiffies(&ts); if (to_jiffies <= 0) return -EINVAL; diff --git a/drivers/char/tlclk.c b/drivers/char/tlclk.c index dd45fe5eb6f27..255f69123af5f 100644 --- a/drivers/char/tlclk.c +++ b/drivers/char/tlclk.c @@ -328,7 +328,7 @@ static DEVICE_ATTR(alarms, S_IRUGO, show_alarms, NULL); static ssize_t store_received_ref_clk3a(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -350,7 +350,7 @@ static DEVICE_ATTR(received_ref_clk3a, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_received_ref_clk3b(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -372,7 +372,7 @@ static DEVICE_ATTR(received_ref_clk3b, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_enable_clk3b_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -394,7 +394,7 @@ static ssize_t store_enable_clk3a_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -415,7 +415,7 @@ static ssize_t store_enable_clkb1_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -437,7 +437,7 @@ static ssize_t store_enable_clka1_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -458,7 +458,7 @@ static ssize_t store_enable_clkb0_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -479,7 +479,7 @@ static ssize_t store_enable_clka0_output(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -500,7 +500,7 @@ static ssize_t store_select_amcb2_transmit_clock(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { unsigned long flags; - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; sscanf(buf, "%lX", &tmp); @@ -541,7 +541,7 @@ static DEVICE_ATTR(select_amcb2_transmit_clock, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_select_amcb1_transmit_clock(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -583,7 +583,7 @@ static DEVICE_ATTR(select_amcb1_transmit_clock, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_select_redundant_clock(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -604,7 +604,7 @@ static DEVICE_ATTR(select_redundant_clock, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_select_ref_frequency(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -625,7 +625,7 @@ static DEVICE_ATTR(select_ref_frequency, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_filter_select(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -645,7 +645,7 @@ static DEVICE_ATTR(filter_select, (S_IWUSR|S_IWGRP), NULL, store_filter_select); static ssize_t store_hardware_switching_mode(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -666,7 +666,7 @@ static DEVICE_ATTR(hardware_switching_mode, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_hardware_switching(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -687,7 +687,7 @@ static DEVICE_ATTR(hardware_switching, (S_IWUSR|S_IWGRP), NULL, static ssize_t store_refalign (struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned long flags; sscanf(buf, "%lX", &tmp); @@ -706,7 +706,7 @@ static DEVICE_ATTR(refalign, (S_IWUSR|S_IWGRP), NULL, store_refalign); static ssize_t store_mode_select (struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; @@ -726,7 +726,7 @@ static DEVICE_ATTR(mode_select, (S_IWUSR|S_IWGRP), NULL, store_mode_select); static ssize_t store_reset (struct device *d, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long tmp; + unsigned long tmp = 0; unsigned char val; unsigned long flags; diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c index e2b7451ea7ccd..0e2deff94c367 100644 --- a/drivers/char/tpm/st33zp24/st33zp24.c +++ b/drivers/char/tpm/st33zp24/st33zp24.c @@ -93,7 +93,9 @@ static u8 st33zp24_status(struct tpm_chip *chip) struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u8 data; - tpm_dev->ops->recv(tpm_dev->phy_id, TPM_STS, &data, 1); + if (tpm_dev->ops->recv(tpm_dev->phy_id, TPM_STS, &data, 1) != 1) + return 0; + return data; } @@ -104,10 +106,10 @@ static bool check_locality(struct tpm_chip *chip) { struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u8 data; - u8 status; + int status; status = tpm_dev->ops->recv(tpm_dev->phy_id, TPM_ACCESS, &data, 1); - if (status && (data & + if (status == 1 && (data & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) == (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) return true; diff --git a/drivers/char/tpm/tpm_i2c_nuvoton.c b/drivers/char/tpm/tpm_i2c_nuvoton.c index d44903b299294..aa3673002f3aa 100644 --- a/drivers/char/tpm/tpm_i2c_nuvoton.c +++ b/drivers/char/tpm/tpm_i2c_nuvoton.c @@ -182,8 +182,10 @@ static int i2c_nuvoton_wait_for_stat(struct tpm_chip *chip, u8 mask, u8 value, timeout); if (rc > 0) return 0; - /* At this point we know that the SINT pin is asserted, so we - * do not need to do i2c_nuvoton_check_status */ + + disable_irq(priv->irq); + if (rc < 0) + return rc; } else { unsigned long ten_msec, stop; bool status_valid; diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index 198b973141680..1ec5b25d1a9ae 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -1891,13 +1891,28 @@ static const struct file_operations portdev_fops = { static void remove_vqs(struct ports_device *portdev) { struct virtqueue *vq; + bool multiport = use_multiport(portdev); virtio_device_for_each_vq(portdev->vdev, vq) { struct port_buffer *buf; + unsigned int len; - flush_bufs(vq, true); - while ((buf = virtqueue_detach_unused_buf(vq))) - free_buf(buf, true); + /* + * c_ovq cookies are &portdev->cpkt, not port_buffer. + * Detach them but do not free_buf(). + */ + if (multiport && vq == portdev->c_ovq) { + spin_lock(&portdev->c_ovq_lock); + while (virtqueue_get_buf(vq, &len)) + ; + while (virtqueue_detach_unused_buf(vq)) + ; + spin_unlock(&portdev->c_ovq_lock); + } else { + flush_bufs(vq, true); + while ((buf = virtqueue_detach_unused_buf(vq))) + free_buf(buf, true); + } cond_resched(); } portdev->vdev->config->del_vqs(portdev->vdev); diff --git a/drivers/char/xilinx_hwicap/xilinx_hwicap.c b/drivers/char/xilinx_hwicap/xilinx_hwicap.c index 34a345dc5e724..9bb5fa642fd88 100644 --- a/drivers/char/xilinx_hwicap/xilinx_hwicap.c +++ b/drivers/char/xilinx_hwicap/xilinx_hwicap.c @@ -760,7 +760,7 @@ static int __init hwicap_module_init(void) HWICAP_DEVICES, DRIVER_NAME); if (retval < 0) - return retval; + goto failed_class; retval = platform_driver_register(&hwicap_platform_driver); if (retval) @@ -771,6 +771,9 @@ static int __init hwicap_module_init(void) failed: unregister_chrdev_region(devt, HWICAP_DEVICES); + failed_class: + class_unregister(&icap_class); + return retval; } diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c index 994d5bc5168be..697d102449c10 100644 --- a/drivers/clk/clk-devres.c +++ b/drivers/clk/clk-devres.c @@ -99,6 +99,11 @@ struct clk *devm_clk_get_optional_enabled(struct device *dev, const char *id) } EXPORT_SYMBOL_GPL(devm_clk_get_optional_enabled); +static void devm_clk_disable_unprepare(void *data) +{ + clk_disable_unprepare(data); +} + struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev, const char *id, unsigned long rate) @@ -106,8 +111,7 @@ struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev, struct clk *clk; int ret; - clk = __devm_clk_get(dev, id, clk_get_optional, NULL, - clk_disable_unprepare); + clk = devm_clk_get_optional(dev, id); if (IS_ERR(clk)) return ERR_CAST(clk); @@ -119,6 +123,10 @@ struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev, if (ret) goto out_put_clk; + ret = devm_add_action_or_reset(dev, devm_clk_disable_unprepare, clk); + if (ret) + goto out_put_clk; + return clk; out_put_clk: diff --git a/drivers/clk/clk-k230.c b/drivers/clk/clk-k230.c index cfc437038e4ed..f34a3e6d3bca0 100644 --- a/drivers/clk/clk-k230.c +++ b/drivers/clk/clk-k230.c @@ -2227,6 +2227,7 @@ static int k230_clk_set_rate_mul(struct clk_hw *hw, unsigned long rate, guard(spinlock)(rate_self->lock); mul_reg = readl(rate_self->reg + clk->mul_reg_off); + mul_reg &= ~(rate_self->mul_mask << rate_self->mul_shift); mul_reg |= ((mul - 1) & rate_self->mul_mask) << (rate_self->mul_shift); mul_reg |= BIT(rate_self->write_enable_bit); writel(mul_reg, rate_self->reg + clk->mul_reg_off); @@ -2257,6 +2258,7 @@ static int k230_clk_set_rate_div(struct clk_hw *hw, unsigned long rate, guard(spinlock)(rate_self->lock); div_reg = readl(rate_self->reg + clk->div_reg_off); + div_reg &= ~(rate_self->div_mask << rate_self->div_shift); div_reg |= ((div - 1) & rate_self->div_mask) << (rate_self->div_shift); div_reg |= BIT(rate_self->write_enable_bit); writel(div_reg, rate_self->reg + clk->div_reg_off); @@ -2287,11 +2289,13 @@ static int k230_clk_set_rate_mul_div(struct clk_hw *hw, unsigned long rate, guard(spinlock)(rate_self->lock); div_reg = readl(rate_self->reg + clk->div_reg_off); + div_reg &= ~(rate_self->div_mask << rate_self->div_shift); div_reg |= ((div - 1) & rate_self->div_mask) << (rate_self->div_shift); div_reg |= BIT(rate_self->write_enable_bit); writel(div_reg, rate_self->reg + clk->div_reg_off); mul_reg = readl(rate_self->reg + clk->mul_reg_off); + mul_reg &= ~(rate_self->mul_mask << rate_self->mul_shift); mul_reg |= ((mul - 1) & rate_self->mul_mask) << (rate_self->mul_shift); mul_reg |= BIT(rate_self->write_enable_bit); writel(mul_reg, rate_self->reg + clk->mul_reg_off); diff --git a/drivers/clk/clk-moxart.c b/drivers/clk/clk-moxart.c index 3786a0153ad17..40663ef3ef0ae 100644 --- a/drivers/clk/clk-moxart.c +++ b/drivers/clk/clk-moxart.c @@ -17,7 +17,6 @@ static void __init moxart_of_pll_clk_init(struct device_node *node) { void __iomem *base; struct clk_hw *hw; - struct clk *ref_clk; unsigned int mul; const char *name = node->name; const char *parent_name; @@ -34,12 +33,6 @@ static void __init moxart_of_pll_clk_init(struct device_node *node) mul = readl(base + 0x30) >> 3 & 0x3f; iounmap(base); - ref_clk = of_clk_get(node, 0); - if (IS_ERR(ref_clk)) { - pr_err("%pOF: of_clk_get failed\n", node); - return; - } - hw = clk_hw_register_fixed_factor(NULL, name, parent_name, 0, mul, 1); if (IS_ERR(hw)) { pr_err("%pOF: failed to register clock\n", node); @@ -56,7 +49,6 @@ static void __init moxart_of_apb_clk_init(struct device_node *node) { void __iomem *base; struct clk_hw *hw; - struct clk *pll_clk; unsigned int div, val; unsigned int div_idx[] = { 2, 3, 4, 6, 8}; const char *name = node->name; @@ -78,12 +70,6 @@ static void __init moxart_of_apb_clk_init(struct device_node *node) val = 0; div = div_idx[val] * 2; - pll_clk = of_clk_get(node, 0); - if (IS_ERR(pll_clk)) { - pr_err("%pOF: of_clk_get failed\n", node); - return; - } - hw = clk_hw_register_fixed_factor(NULL, name, parent_name, 0, 1, div); if (IS_ERR(hw)) { pr_err("%pOF: failed to register clock\n", node); diff --git a/drivers/clk/clk-palmas.c b/drivers/clk/clk-palmas.c index 39049f62dbbb3..86a51edac8272 100644 --- a/drivers/clk/clk-palmas.c +++ b/drivers/clk/clk-palmas.c @@ -194,6 +194,13 @@ static void palmas_clks_get_clk_data(struct platform_device *pdev, cinfo->ext_control_pin = prop; } +static void palmas_clks_unprepare_ext_control(void *data) +{ + struct palmas_clock_info *cinfo = data; + + clk_unprepare(cinfo->hw.clk); +} + static int palmas_clks_init_configure(struct palmas_clock_info *cinfo) { int ret; @@ -214,13 +221,18 @@ static int palmas_clks_init_configure(struct palmas_clock_info *cinfo) return ret; } + ret = devm_add_action_or_reset(cinfo->dev, + palmas_clks_unprepare_ext_control, + cinfo); + if (ret) + return ret; + ret = palmas_ext_control_req_config(cinfo->palmas, cinfo->clk_desc->sleep_reqstr_id, cinfo->ext_control_pin, true); if (ret < 0) { dev_err(cinfo->dev, "Ext config for %s failed, %d\n", cinfo->clk_desc->clk_name, ret); - clk_unprepare(cinfo->hw.clk); return ret; } } diff --git a/drivers/clk/clk-scpi.c b/drivers/clk/clk-scpi.c index 24cee7c9fda6c..e1891351c221a 100644 --- a/drivers/clk/clk-scpi.c +++ b/drivers/clk/clk-scpi.c @@ -85,7 +85,7 @@ static unsigned long scpi_dvfs_recalc_rate(struct clk_hw *hw, int idx = clk->scpi_ops->dvfs_get_idx(clk->id); const struct scpi_opp *opp; - if (idx < 0) + if (idx < 0 || idx >= clk->info->count) return 0; opp = clk->info->opps + idx; @@ -284,10 +284,14 @@ static int scpi_clocks_probe(struct platform_device *pdev) if (match->data != &scpi_dvfs_ops) continue; /* Add the virtual cpufreq device if it's DVFS clock provider */ + if (cpufreq_dev) + continue; cpufreq_dev = platform_device_register_simple("scpi-cpufreq", -1, NULL, 0); - if (IS_ERR(cpufreq_dev)) + if (IS_ERR(cpufreq_dev)) { pr_warn("unable to register cpufreq device"); + cpufreq_dev = NULL; + } } return 0; } diff --git a/drivers/clk/clk-versaclock7.c b/drivers/clk/clk-versaclock7.c index adcc603e32593..e3a36dcd98b80 100644 --- a/drivers/clk/clk-versaclock7.c +++ b/drivers/clk/clk-versaclock7.c @@ -1197,7 +1197,7 @@ static int vc7_probe(struct i2c_client *client) if (ret) { dev_err_probe(&client->dev, ret, "unable to register output %d\n", i); - return ret; + goto err_clk; } switch (bank_src_map.type) { diff --git a/drivers/clk/eswin/clk-eic7700.c b/drivers/clk/eswin/clk-eic7700.c index be81d74192dae..43a47fe16ab14 100644 --- a/drivers/clk/eswin/clk-eic7700.c +++ b/drivers/clk/eswin/clk-eic7700.c @@ -791,7 +791,8 @@ static struct eswin_clk_info eic7700_clks[] = { EIC7700_CLK_MUX_CPU_ROOT_3MUX1_GFREE, CLK_SET_RATE_PARENT, EIC7700_REG_OFFSET_U84, 27, 0), ESWIN_GATE_TYPE(EIC7700_CLK_GATE_NOC_NSP_CLK, "gate_noc_nsp_clk", - EIC7700_CLK_DIV_NOC_NSP_DYNM, CLK_SET_RATE_PARENT, + EIC7700_CLK_DIV_NOC_NSP_DYNM, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, EIC7700_REG_OFFSET_NOC, 31, 0), ESWIN_GATE_TYPE(EIC7700_CLK_GATE_BOOTSPI, "gate_clk_bootspi", EIC7700_CLK_MUX_BOOTSPI_CLK_2MUX1_GFREE, diff --git a/drivers/clk/eswin/clk.c b/drivers/clk/eswin/clk.c index e09a52cc35872..79d1e4c5e637a 100644 --- a/drivers/clk/eswin/clk.c +++ b/drivers/clk/eswin/clk.c @@ -204,7 +204,7 @@ int eswin_clk_register_pll(struct device *dev, struct eswin_pll_clock *clks, int nums, struct eswin_clock_data *data) { struct eswin_clk_pll *p_clk = NULL; - struct clk_init_data init; + struct clk_init_data init = {}; struct clk_hw *clk_hw; int i, ret; @@ -419,7 +419,7 @@ struct clk_hw *eswin_register_clkdiv(struct device *dev, unsigned int id, unsigned long priv_flag, spinlock_t *lock) { struct eswin_divider_clock *dclk; - struct clk_init_data init; + struct clk_init_data init = {}; struct clk_hw *clk_hw; int ret; diff --git a/drivers/clk/hisilicon/reset.c b/drivers/clk/hisilicon/reset.c index 93cee17db8b16..c3b7daac93132 100644 --- a/drivers/clk/hisilicon/reset.c +++ b/drivers/clk/hisilicon/reset.c @@ -91,7 +91,7 @@ struct hisi_reset_controller *hisi_reset_init(struct platform_device *pdev) { struct hisi_reset_controller *rstc; - rstc = devm_kmalloc(&pdev->dev, sizeof(*rstc), GFP_KERNEL); + rstc = devm_kzalloc(&pdev->dev, sizeof(*rstc), GFP_KERNEL); if (!rstc) return NULL; diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c index 9b33df9967ece..658b6d94de9b4 100644 --- a/drivers/clk/imx/clk-scu.c +++ b/drivers/clk/imx/clk-scu.c @@ -475,7 +475,6 @@ struct clk_hw *__imx_clk_scu(struct device *dev, const char *name, clk->clk_type = clk_type; init.name = name; - init.ops = &clk_scu_ops; if (rsrc_id == IMX_SC_R_A35 || rsrc_id == IMX_SC_R_A53 || rsrc_id == IMX_SC_R_A72) init.ops = &clk_scu_cpu_ops; else if (rsrc_id == IMX_SC_R_PI_0_PLL) diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index 2c09fd729bab0..fd8440122ec26 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -1006,6 +1006,7 @@ config COMMON_CLK_MT8196 tristate "Clock driver for MediaTek MT8196" depends on ARM64 || COMPILE_TEST select COMMON_CLK_MEDIATEK + select REGMAP_MMIO default ARCH_MEDIATEK help This driver supports MediaTek MT8196 basic clocks. diff --git a/drivers/clk/mediatek/clk-mt6735-apmixedsys.c b/drivers/clk/mediatek/clk-mt6735-apmixedsys.c index 9e30c089a2092..b6eb6a581c31e 100644 --- a/drivers/clk/mediatek/clk-mt6735-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt6735-apmixedsys.c @@ -102,9 +102,12 @@ static int clk_mt6735_apmixed_probe(struct platform_device *pdev) ret = devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_onecell_get, clk_data); - if (ret) + if (ret) { dev_err(&pdev->dev, "Failed to register clock provider: %d\n", ret); + mtk_clk_unregister_plls(apmixedsys_plls, ARRAY_SIZE(apmixedsys_plls), + clk_data); + } return ret; } diff --git a/drivers/clk/mediatek/clk-mt8135.c b/drivers/clk/mediatek/clk-mt8135.c index 084e48a554c26..1d20e15608f77 100644 --- a/drivers/clk/mediatek/clk-mt8135.c +++ b/drivers/clk/mediatek/clk-mt8135.c @@ -409,6 +409,9 @@ static const struct mtk_gate_regs infra_cg_regs = { GATE_MTK_FLAGS(_id, _name, _parent, &infra_cg_regs, _shift, \ &mtk_clk_gate_ops_setclr, CLK_IS_CRITICAL) +#define GATE_ICG_INV(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &infra_cg_regs, _shift, &mtk_clk_gate_ops_setclr_inv) + static const struct mtk_gate infra_clks[] = { GATE_DUMMY(CLK_DUMMY, "infra_dummy"), GATE_ICG(CLK_INFRA_PMIC_WRAP, "pmic_wrap_ck", "axi_sel", 23), @@ -419,7 +422,7 @@ static const struct mtk_gate infra_clks[] = { GATE_ICG(CLK_INFRA_CPUM, "cpum_ck", "cpum_tck_in", 15), GATE_ICG_AO(CLK_INFRA_M4U, "m4u_ck", "mem_sel", 8), GATE_ICG(CLK_INFRA_MFGAXI, "mfgaxi_ck", "axi_sel", 7), - GATE_ICG(CLK_INFRA_DEVAPC, "devapc_ck", "axi_sel", 6), + GATE_ICG_INV(CLK_INFRA_DEVAPC, "devapc_ck", "axi_sel", 6), GATE_ICG(CLK_INFRA_AUDIO, "audio_ck", "aud_intbus_sel", 5), GATE_ICG(CLK_INFRA_MFG_BUS, "mfg_bus_ck", "axi_sel", 2), GATE_ICG(CLK_INFRA_SMI, "smi_ck", "smi_sel", 1), diff --git a/drivers/clk/mediatek/clk-pllfh.c b/drivers/clk/mediatek/clk-pllfh.c index aa95cd9197b3c..6249fb87b1f53 100644 --- a/drivers/clk/mediatek/clk-pllfh.c +++ b/drivers/clk/mediatek/clk-pllfh.c @@ -197,12 +197,56 @@ static void mtk_clk_unregister_pllfh(struct clk_hw *hw) kfree(fh); } +static void mtk_clk_cleanup_pllfhs(void __iomem *iomem_base, + const struct mtk_pll_data *plls, int num_plls, + void __iomem *iomem_fhctl_base, + struct mtk_pllfh_data *pllfhs, int num_fhs, + struct clk_hw_onecell_data *clk_data) +{ + void __iomem *base = iomem_base; + void __iomem *fhctl_base = iomem_fhctl_base; + int i; + + for (i = num_plls - 1; i >= 0; i--) { + const struct mtk_pll_data *pll = &plls[i]; + struct mtk_pllfh_data *pllfh; + bool use_fhctl; + + if (IS_ERR_OR_NULL(clk_data->hws[pll->id])) + continue; + + pllfh = get_pllfh_by_id(pllfhs, num_fhs, pll->id); + use_fhctl = fhctl_is_supported_and_enabled(pllfh); + + if (!base) + base = mtk_clk_pll_get_base(clk_data->hws[pll->id], + pll); + + if (use_fhctl) { + if (!fhctl_base) + fhctl_base = pllfh->state.base; + mtk_clk_unregister_pllfh(clk_data->hws[pll->id]); + } else { + mtk_clk_unregister_pll(clk_data->hws[pll->id]); + } + + clk_data->hws[pll->id] = ERR_PTR(-ENOENT); + } + + if (fhctl_base) + iounmap(fhctl_base); + + if (base) + iounmap(base); +} + + int mtk_clk_register_pllfhs(struct device *dev, const struct mtk_pll_data *plls, int num_plls, struct mtk_pllfh_data *pllfhs, int num_fhs, struct clk_hw_onecell_data *clk_data) { - void __iomem *base; + void __iomem *base, *fhctl_base = NULL; int i; struct clk_hw *hw; @@ -238,24 +282,8 @@ int mtk_clk_register_pllfhs(struct device *dev, return 0; err: - while (--i >= 0) { - const struct mtk_pll_data *pll = &plls[i]; - struct mtk_pllfh_data *pllfh; - bool use_fhctl; - - pllfh = get_pllfh_by_id(pllfhs, num_fhs, pll->id); - use_fhctl = fhctl_is_supported_and_enabled(pllfh); - - if (use_fhctl) - mtk_clk_unregister_pllfh(clk_data->hws[pll->id]); - else - mtk_clk_unregister_pll(clk_data->hws[pll->id]); - - clk_data->hws[pll->id] = ERR_PTR(-ENOENT); - } - - iounmap(base); - + mtk_clk_cleanup_pllfhs(base, plls, i, fhctl_base, pllfhs, num_fhs, + clk_data); return PTR_ERR(hw); } EXPORT_SYMBOL_GPL(mtk_clk_register_pllfhs); @@ -264,38 +292,10 @@ void mtk_clk_unregister_pllfhs(const struct mtk_pll_data *plls, int num_plls, struct mtk_pllfh_data *pllfhs, int num_fhs, struct clk_hw_onecell_data *clk_data) { - void __iomem *base = NULL, *fhctl_base = NULL; - int i; - if (!clk_data) return; - for (i = num_plls; i > 0; i--) { - const struct mtk_pll_data *pll = &plls[i - 1]; - struct mtk_pllfh_data *pllfh; - bool use_fhctl; - - if (IS_ERR_OR_NULL(clk_data->hws[pll->id])) - continue; - - pllfh = get_pllfh_by_id(pllfhs, num_fhs, pll->id); - use_fhctl = fhctl_is_supported_and_enabled(pllfh); - - if (use_fhctl) { - fhctl_base = pllfh->state.base; - mtk_clk_unregister_pllfh(clk_data->hws[pll->id]); - } else { - base = mtk_clk_pll_get_base(clk_data->hws[pll->id], - pll); - mtk_clk_unregister_pll(clk_data->hws[pll->id]); - } - - clk_data->hws[pll->id] = ERR_PTR(-ENOENT); - } - - if (fhctl_base) - iounmap(fhctl_base); - - iounmap(base); + mtk_clk_cleanup_pllfhs(NULL, plls, num_plls, NULL, pllfhs, + num_fhs, clk_data); } EXPORT_SYMBOL_GPL(mtk_clk_unregister_pllfhs); diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index 39af7b1868e32..7a7b104182ff8 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -1393,7 +1393,7 @@ static struct clk_regmap gxbb_32k_clk_sel = { .name = "32k_clk_sel", .ops = &clk_regmap_mux_ops, .parent_data = gxbb_32k_clk_parents, - .num_parents = 4, + .num_parents = ARRAY_SIZE(gxbb_32k_clk_parents), .flags = CLK_SET_RATE_PARENT, }, }; diff --git a/drivers/clk/microchip/clk-mpfs.c b/drivers/clk/microchip/clk-mpfs.c index ee58304913ef1..7f044775aea81 100644 --- a/drivers/clk/microchip/clk-mpfs.c +++ b/drivers/clk/microchip/clk-mpfs.c @@ -285,7 +285,7 @@ static int mpfs_cfg_clk_set_rate(struct clk_hw *hw, unsigned long rate, unsigned mask = clk_div_mask(cfg->width) << cfg->shift; val = divider_setting << cfg->shift; - regmap_update_bits(cfg->map, cfg->map_offset, val, mask); + regmap_update_bits(cfg->map, cfg->map_offset, mask, val); return 0; } diff --git a/drivers/clk/nuvoton/clk-ma35d1-pll.c b/drivers/clk/nuvoton/clk-ma35d1-pll.c index 4620acfe47e85..c7c0dc91a012c 100644 --- a/drivers/clk/nuvoton/clk-ma35d1-pll.c +++ b/drivers/clk/nuvoton/clk-ma35d1-pll.c @@ -48,7 +48,7 @@ #define PLL_CTL1_PD BIT(0) #define PLL_CTL1_BP BIT(1) #define PLL_CTL1_OUTDIV GENMASK(6, 4) -#define PLL_CTL1_FRAC GENMASK(31, 24) +#define PLL_CTL1_FRAC GENMASK(31, 8) #define PLL_CTL2_SLOPE GENMASK(23, 0) #define INDIV_MIN 1 @@ -92,7 +92,7 @@ static unsigned long ma35d1_calc_smic_pll_freq(u32 pll0_ctl0, p = FIELD_GET(SPLL0_CTL0_OUTDIV, pll0_ctl0); outdiv = 1 << p; pll_freq = (u64)parent_rate * n; - div_u64(pll_freq, m * outdiv); + pll_freq = div_u64(pll_freq, m * outdiv); return pll_freq; } @@ -110,12 +110,12 @@ static unsigned long ma35d1_calc_pll_freq(u8 mode, u32 *reg_ctl, unsigned long p if (mode == PLL_MODE_INT) { pll_freq = (u64)parent_rate * n; - div_u64(pll_freq, m * p); + pll_freq = div_u64(pll_freq, m * p); } else { x = FIELD_GET(PLL_CTL1_FRAC, reg_ctl[1]); - /* 2 decimal places floating to integer (ex. 1.23 to 123) */ - n = n * 100 + ((x * 100) / FIELD_MAX(PLL_CTL1_FRAC)); - pll_freq = div_u64(parent_rate * n, 100 * m * p); + /* convert 24-bit fraction to 3 decimal digits, rounding to closest */ + n = n * 1000 + DIV_ROUND_CLOSEST_ULL((u64)x * 1000, 1ULL << 24); + pll_freq = div_u64((u64)parent_rate * n, 1000 * m * p); } return pll_freq; } @@ -255,32 +255,32 @@ static int ma35d1_clk_pll_determine_rate(struct clk_hw *hw, if (req->best_parent_rate < PLL_FREF_MIN_FREQ || req->best_parent_rate > PLL_FREF_MAX_FREQ) return -EINVAL; - ret = ma35d1_pll_find_closest(pll, req->rate, req->best_parent_rate, - reg_ctl, &pll_freq); - if (ret < 0) - return ret; - switch (pll->id) { case CAPLL: + case DDRPLL: + /* Read-only PLLs: return current rate */ reg_ctl[0] = readl_relaxed(pll->ctl0_base); - pll_freq = ma35d1_calc_smic_pll_freq(reg_ctl[0], req->best_parent_rate); + if (pll->id == CAPLL) { + pll_freq = ma35d1_calc_smic_pll_freq(reg_ctl[0], req->best_parent_rate); + } else { + reg_ctl[1] = readl_relaxed(pll->ctl1_base); + pll_freq = ma35d1_calc_pll_freq(pll->mode, reg_ctl, req->best_parent_rate); + } req->rate = pll_freq; - return 0; - case DDRPLL: case APLL: case EPLL: case VPLL: - reg_ctl[0] = readl_relaxed(pll->ctl0_base); - reg_ctl[1] = readl_relaxed(pll->ctl1_base); - pll_freq = ma35d1_calc_pll_freq(pll->mode, reg_ctl, req->best_parent_rate); + /* Configurable PLLs: find closest achievable rate */ + ret = ma35d1_pll_find_closest(pll, req->rate, req->best_parent_rate, + reg_ctl, &pll_freq); + if (ret < 0) + return ret; req->rate = pll_freq; - return 0; } req->rate = 0; - return 0; } diff --git a/drivers/clk/qcom/camcc-sc8280xp.c b/drivers/clk/qcom/camcc-sc8280xp.c index e97b8d4f3c844..660d8655d3919 100644 --- a/drivers/clk/qcom/camcc-sc8280xp.c +++ b/drivers/clk/qcom/camcc-sc8280xp.c @@ -1753,24 +1753,6 @@ static struct clk_branch camcc_csiphy3_clk = { }, }; -static struct clk_branch camcc_gdsc_clk = { - .halt_reg = 0xc1e4, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0xc1e4, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "camcc_gdsc_clk", - .parent_hws = (const struct clk_hw*[]){ - &camcc_xo_clk_src.clkr.hw, - }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch camcc_icp_ahb_clk = { .halt_reg = 0xc0d8, .halt_check = BRANCH_HALT, @@ -2839,7 +2821,6 @@ static struct clk_regmap *camcc_sc8280xp_clocks[] = { [CAMCC_CSIPHY2_CLK] = &camcc_csiphy2_clk.clkr, [CAMCC_CSIPHY3_CLK] = &camcc_csiphy3_clk.clkr, [CAMCC_FAST_AHB_CLK_SRC] = &camcc_fast_ahb_clk_src.clkr, - [CAMCC_GDSC_CLK] = &camcc_gdsc_clk.clkr, [CAMCC_ICP_AHB_CLK] = &camcc_icp_ahb_clk.clkr, [CAMCC_ICP_CLK] = &camcc_icp_clk.clkr, [CAMCC_ICP_CLK_SRC] = &camcc_icp_clk_src.clkr, diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index eec369d2173b5..0e8f380873af0 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -169,7 +169,7 @@ static int _qcom_cc_register_board_clk(struct device *dev, const char *path, if (!node) { fixed = devm_kzalloc(dev, sizeof(*fixed), GFP_KERNEL); if (!fixed) - return -EINVAL; + return -ENOMEM; fixed->fixed_rate = rate; fixed->hw.init = &init_data; @@ -186,7 +186,7 @@ static int _qcom_cc_register_board_clk(struct device *dev, const char *path, if (add_factor) { factor = devm_kzalloc(dev, sizeof(*factor), GFP_KERNEL); if (!factor) - return -EINVAL; + return -ENOMEM; factor->mult = factor->div = 1; factor->hw.init = &init_data; diff --git a/drivers/clk/qcom/dispcc-qcm2290.c b/drivers/clk/qcom/dispcc-qcm2290.c index 4d6aad280ae17..d32fd3f8d7acd 100644 --- a/drivers/clk/qcom/dispcc-qcm2290.c +++ b/drivers/clk/qcom/dispcc-qcm2290.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2020, The Linux Foundation. All rights reserved. * Copyright (c) 2021, Linaro Ltd. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -48,6 +49,7 @@ static const struct alpha_pll_config disp_cc_pll0_config = { static struct clk_alpha_pll disp_cc_pll0 = { .offset = 0x0, + .config = &disp_cc_pll0_config, .vco_table = spark_vco, .num_vco = ARRAY_SIZE(spark_vco), .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], @@ -449,11 +451,14 @@ static const struct qcom_reset_map disp_cc_qcm2290_resets[] = { static struct gdsc mdss_gdsc = { .gdscr = 0x3000, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0xf, .pd = { .name = "mdss_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = HW_CTRL, + .flags = HW_CTRL | POLL_CFG_GDSCR | RETAIN_FF_ENABLE, }; static struct gdsc *disp_cc_qcm2290_gdscs[] = { @@ -482,6 +487,14 @@ static struct clk_regmap *disp_cc_qcm2290_clocks[] = { [DISP_CC_SLEEP_CLK_SRC] = &disp_cc_sleep_clk_src.clkr, }; +static struct clk_alpha_pll *disp_cc_qcm2290_plls[] = { + &disp_cc_pll0, +}; + +static const u32 disp_cc_qcm2290_critical_cbcrs[] = { + 0x604c, /* DISP_CC_XO_CLK */ +}; + static const struct regmap_config disp_cc_qcm2290_regmap_config = { .reg_bits = 32, .reg_stride = 4, @@ -490,6 +503,13 @@ static const struct regmap_config disp_cc_qcm2290_regmap_config = { .fast_io = true, }; +static const struct qcom_cc_driver_data disp_cc_qcm2290_driver_data = { + .alpha_plls = disp_cc_qcm2290_plls, + .num_alpha_plls = ARRAY_SIZE(disp_cc_qcm2290_plls), + .clk_cbcrs = disp_cc_qcm2290_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(disp_cc_qcm2290_critical_cbcrs), +}; + static const struct qcom_cc_desc disp_cc_qcm2290_desc = { .config = &disp_cc_qcm2290_regmap_config, .clks = disp_cc_qcm2290_clocks, @@ -498,6 +518,8 @@ static const struct qcom_cc_desc disp_cc_qcm2290_desc = { .num_gdscs = ARRAY_SIZE(disp_cc_qcm2290_gdscs), .resets = disp_cc_qcm2290_resets, .num_resets = ARRAY_SIZE(disp_cc_qcm2290_resets), + .use_rpm = true, + .driver_data = &disp_cc_qcm2290_driver_data, }; static const struct of_device_id disp_cc_qcm2290_match_table[] = { @@ -508,25 +530,7 @@ MODULE_DEVICE_TABLE(of, disp_cc_qcm2290_match_table); static int disp_cc_qcm2290_probe(struct platform_device *pdev) { - struct regmap *regmap; - int ret; - - regmap = qcom_cc_map(pdev, &disp_cc_qcm2290_desc); - if (IS_ERR(regmap)) - return PTR_ERR(regmap); - - clk_alpha_pll_configure(&disp_cc_pll0, regmap, &disp_cc_pll0_config); - - /* Keep some clocks always-on */ - qcom_branch_set_clk_en(regmap, 0x604c); /* DISP_CC_XO_CLK */ - - ret = qcom_cc_really_probe(&pdev->dev, &disp_cc_qcm2290_desc, regmap); - if (ret) { - dev_err(&pdev->dev, "Failed to register DISP CC clocks\n"); - return ret; - } - - return ret; + return qcom_cc_probe(pdev, &disp_cc_qcm2290_desc); } static struct platform_driver disp_cc_qcm2290_driver = { diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c index f4ede4a3a1c07..2ee4820b6fdfb 100644 --- a/drivers/clk/qcom/gcc-glymur.c +++ b/drivers/clk/qcom/gcc-glymur.c @@ -3668,21 +3668,6 @@ static struct clk_branch gcc_disp_hf_axi_clk = { }, }; -static struct clk_branch gcc_eva_ahb_clk = { - .halt_reg = 0x9b004, - .halt_check = BRANCH_HALT_VOTED, - .hwcg_reg = 0x9b004, - .hwcg_bit = 1, - .clkr = { - .enable_reg = 0x9b004, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "gcc_eva_ahb_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_eva_axi0_clk = { .halt_reg = 0x9b008, .halt_check = BRANCH_HALT_SKIP, @@ -3713,19 +3698,6 @@ static struct clk_branch gcc_eva_axi0c_clk = { }, }; -static struct clk_branch gcc_eva_xo_clk = { - .halt_reg = 0x9b024, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0x9b024, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "gcc_eva_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_gp1_clk = { .halt_reg = 0x64000, .halt_check = BRANCH_HALT, @@ -7992,10 +7964,8 @@ static struct clk_regmap *gcc_glymur_clocks[] = { [GCC_CFG_NOC_USB_ANOC_AHB_CLK] = &gcc_cfg_noc_usb_anoc_ahb_clk.clkr, [GCC_CFG_NOC_USB_ANOC_SOUTH_AHB_CLK] = &gcc_cfg_noc_usb_anoc_south_ahb_clk.clkr, [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, - [GCC_EVA_AHB_CLK] = &gcc_eva_ahb_clk.clkr, [GCC_EVA_AXI0_CLK] = &gcc_eva_axi0_clk.clkr, [GCC_EVA_AXI0C_CLK] = &gcc_eva_axi0c_clk.clkr, - [GCC_EVA_XO_CLK] = &gcc_eva_xo_clk.clkr, [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, @@ -8544,6 +8514,8 @@ static const u32 gcc_glymur_critical_cbcrs[] = { 0x71004, /* GCC_GPU_CFG_AHB_CLK */ 0x32004, /* GCC_VIDEO_AHB_CLK */ 0x32058, /* GCC_VIDEO_XO_CLK */ + 0x9b004, /* GCC_EVA_AHB_CLK */ + 0x9b024, /* GCC_EVA_XO_CLK */ }; static const struct regmap_config gcc_glymur_regmap_config = { @@ -8576,6 +8548,7 @@ static const struct qcom_cc_desc gcc_glymur_desc = { .num_resets = ARRAY_SIZE(gcc_glymur_resets), .gdscs = gcc_glymur_gdscs, .num_gdscs = ARRAY_SIZE(gcc_glymur_gdscs), + .use_rpm = true, .driver_data = &gcc_glymur_driver_data, }; diff --git a/drivers/clk/qcom/gcc-mdm9607.c b/drivers/clk/qcom/gcc-mdm9607.c index 07f1b78d737a1..4b749c99586b7 100644 --- a/drivers/clk/qcom/gcc-mdm9607.c +++ b/drivers/clk/qcom/gcc-mdm9607.c @@ -26,7 +26,6 @@ enum { P_XO, - P_BIMC, P_GPLL0, P_GPLL1, P_GPLL2, @@ -121,7 +120,7 @@ static struct clk_alpha_pll gpll2_early = { .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], .clkr = { .enable_reg = 0x45000, - .enable_mask = BIT(3), /* Yeah, apparently it's not 2 */ + .enable_mask = BIT(3), /* BIT(2) is used for BIMC PLL */ .hw.init = &(struct clk_init_data) { .name = "gpll2_early", @@ -158,20 +157,6 @@ static const struct clk_parent_data gcc_xo_gpll0_gpll2[] = { { .hw = &gpll2.clkr.hw }, }; -static const struct parent_map gcc_xo_gpll0_gpll1_gpll2_map[] = { - { P_XO, 0 }, - { P_GPLL0, 1 }, - { P_GPLL1, 2 }, - { P_GPLL2, 3 }, -}; - -static const struct clk_parent_data gcc_xo_gpll0_gpll1_gpll2[] = { - { .fw_name = "xo" }, - { .hw = &gpll0.clkr.hw }, - { .hw = &gpll1_vote.hw }, - { .hw = &gpll2.clkr.hw }, -}; - static const struct freq_tbl ftbl_apss_ahb_clk[] = { F(19200000, P_XO, 1, 0, 0), F(50000000, P_GPLL0, 16, 0, 0), @@ -192,80 +177,6 @@ static struct clk_rcg2 apss_ahb_clk_src = { }, }; -static struct clk_pll bimc_pll = { - .l_reg = 0x23004, - .m_reg = 0x23008, - .n_reg = 0x2300c, - .config_reg = 0x23010, - .mode_reg = 0x23000, - .status_reg = 0x2301c, - .status_bit = 17, - .clkr.hw.init = &(struct clk_init_data){ - .name = "bimc_pll", - .parent_data = &(const struct clk_parent_data){ - .fw_name = "xo", - }, - .num_parents = 1, - .ops = &clk_pll_ops, - }, -}; - -static struct clk_regmap bimc_pll_vote = { - .enable_reg = 0x45000, - .enable_mask = BIT(3), - .hw.init = &(struct clk_init_data){ - .name = "bimc_pll_vote", - .parent_hws = (const struct clk_hw *[]){ &bimc_pll.clkr.hw }, - .num_parents = 1, - .ops = &clk_pll_vote_ops, - }, -}; - -static const struct parent_map gcc_xo_gpll0_bimc_map[] = { - { P_XO, 0 }, - { P_GPLL0, 1 }, - { P_BIMC, 2 }, -}; - -static const struct clk_parent_data gcc_xo_gpll0_bimc[] = { - { .fw_name = "xo" }, - { .hw = &gpll0.clkr.hw }, - { .hw = &bimc_pll_vote.hw }, -}; - -static const struct freq_tbl ftbl_pcnoc_bfdcd_clk_src[] = { - F(19200000, P_XO, 1, 0, 0), - F(50000000, P_GPLL0, 16, 0, 0), - F(100000000, P_GPLL0, 8, 0, 0), - { } -}; - -static struct clk_rcg2 pcnoc_bfdcd_clk_src = { - .cmd_rcgr = 0x27000, - .freq_tbl = ftbl_pcnoc_bfdcd_clk_src, - .hid_width = 5, - .parent_map = gcc_xo_gpll0_bimc_map, - .clkr.hw.init = &(struct clk_init_data){ - .name = "pcnoc_bfdcd_clk_src", - .parent_data = gcc_xo_gpll0_bimc, - .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc), - .ops = &clk_rcg2_ops, - .flags = CLK_IS_CRITICAL, - }, -}; - -static struct clk_rcg2 system_noc_bfdcd_clk_src = { - .cmd_rcgr = 0x26004, - .hid_width = 5, - .parent_map = gcc_xo_gpll0_bimc_map, - .clkr.hw.init = &(struct clk_init_data){ - .name = "system_noc_bfdcd_clk_src", - .parent_data = gcc_xo_gpll0_bimc, - .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc), - .ops = &clk_rcg2_ops, - }, -}; - static const struct freq_tbl ftbl_gcc_blsp1_qup1_6_i2c_apps_clk[] = { F(19200000, P_XO, 1, 0, 0), F(50000000, P_GPLL0, 16, 0, 0), @@ -674,26 +585,6 @@ static struct clk_rcg2 sdcc2_apps_clk_src = { }, }; -static const struct freq_tbl ftbl_gcc_apss_tcu_clk[] = { - F(155000000, P_GPLL2, 6, 0, 0), - F(310000000, P_GPLL2, 3, 0, 0), - F(400000000, P_GPLL0, 2, 0, 0), - { } -}; - -static struct clk_rcg2 apss_tcu_clk_src = { - .cmd_rcgr = 0x1207c, - .hid_width = 5, - .parent_map = gcc_xo_gpll0_gpll1_gpll2_map, - .freq_tbl = ftbl_gcc_apss_tcu_clk, - .clkr.hw.init = &(struct clk_init_data){ - .name = "apss_tcu_clk_src", - .parent_data = gcc_xo_gpll0_gpll1_gpll2, - .num_parents = 4, - .ops = &clk_rcg2_ops, - }, -}; - static const struct freq_tbl ftbl_gcc_usb_hs_system_clk[] = { F(19200000, P_XO, 1, 0, 0), F(57140000, P_GPLL0, 14, 0, 0), @@ -781,8 +672,6 @@ static struct clk_branch gcc_blsp1_ahb_clk = { .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, .ops = &clk_branch2_ops, }, }, @@ -790,9 +679,10 @@ static struct clk_branch gcc_blsp1_ahb_clk = { static struct clk_branch gcc_blsp1_sleep_clk = { .halt_reg = 0x1004, + .halt_check = BRANCH_HALT_VOTED, .clkr = { - .enable_reg = 0x1004, - .enable_mask = BIT(0), + .enable_reg = 0x45004, + .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_sleep_clk", .parent_data = &(const struct clk_parent_data){ @@ -1083,8 +973,6 @@ static struct clk_branch gcc_boot_rom_ahb_clk = { .enable_mask = BIT(7), .hw.init = &(struct clk_init_data){ .name = "gcc_boot_rom_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, .ops = &clk_branch2_ops, }, }, @@ -1098,9 +986,6 @@ static struct clk_branch gcc_crypto_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_crypto_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1114,9 +999,6 @@ static struct clk_branch gcc_crypto_axi_clk = { .enable_mask = BIT(1), .hw.init = &(struct clk_init_data){ .name = "gcc_crypto_axi_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1190,9 +1072,6 @@ static struct clk_branch gcc_mss_cfg_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mss_cfg_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1220,9 +1099,6 @@ static struct clk_branch gcc_pdm_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_pdm_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1236,9 +1112,6 @@ static struct clk_branch gcc_prng_ahb_clk = { .enable_mask = BIT(8), .hw.init = &(struct clk_init_data){ .name = "gcc_prng_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1251,9 +1124,6 @@ static struct clk_branch gcc_sdcc1_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc1_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1281,9 +1151,6 @@ static struct clk_branch gcc_sdcc2_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc2_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1304,19 +1171,6 @@ static struct clk_branch gcc_sdcc2_apps_clk = { }, }; -static struct clk_rcg2 bimc_ddr_clk_src = { - .cmd_rcgr = 0x32004, - .hid_width = 5, - .parent_map = gcc_xo_gpll0_bimc_map, - .clkr.hw.init = &(struct clk_init_data){ - .name = "bimc_ddr_clk_src", - .parent_data = gcc_xo_gpll0_bimc, - .num_parents = 3, - .ops = &clk_rcg2_ops, - .flags = CLK_GET_RATE_NOCACHE, - }, -}; - static struct clk_branch gcc_mss_q6_bimc_axi_clk = { .halt_reg = 0x49004, .clkr = { @@ -1324,9 +1178,6 @@ static struct clk_branch gcc_mss_q6_bimc_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mss_q6_bimc_axi_clk", - .parent_hws = (const struct clk_hw *[]){ &bimc_ddr_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1340,8 +1191,6 @@ static struct clk_branch gcc_apss_tcu_clk = { .enable_mask = BIT(1), .hw.init = &(struct clk_init_data){ .name = "gcc_apss_tcu_clk", - .parent_hws = (const struct clk_hw *[]){ &bimc_ddr_clk_src.clkr.hw }, - .num_parents = 1, .ops = &clk_branch2_ops, }, }, @@ -1355,9 +1204,6 @@ static struct clk_branch gcc_smmu_cfg_clk = { .enable_mask = BIT(12), .hw.init = &(struct clk_init_data){ .name = "gcc_smmu_cfg_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1405,9 +1251,6 @@ static struct clk_branch gcc_usb_hs_phy_cfg_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_hs_phy_cfg_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1420,9 +1263,6 @@ static struct clk_branch gcc_usb_hs_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_hs_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1451,23 +1291,19 @@ static struct clk_branch gcc_apss_ahb_clk = { .enable_mask = BIT(14), .hw.init = &(struct clk_init_data){ .name = "gcc_apss_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, .ops = &clk_branch2_ops, }, }, }; static struct clk_branch gcc_apss_axi_clk = { - .halt_reg = 0x4601c, + .halt_reg = 0x46020, .halt_check = BRANCH_HALT_VOTED, .clkr = { .enable_reg = 0x45004, .enable_mask = BIT(13), .hw.init = &(struct clk_init_data){ .name = "gcc_apss_axi_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, .ops = &clk_branch2_ops, }, }, @@ -1480,11 +1316,6 @@ static struct clk_regmap *gcc_mdm9607_clocks[] = { [GPLL1_VOTE] = &gpll1_vote, [GPLL2] = &gpll2.clkr, [GPLL2_EARLY] = &gpll2_early.clkr, - [BIMC_PLL] = &bimc_pll.clkr, - [BIMC_PLL_VOTE] = &bimc_pll_vote, - [BIMC_DDR_CLK_SRC] = &bimc_ddr_clk_src.clkr, - [PCNOC_BFDCD_CLK_SRC] = &pcnoc_bfdcd_clk_src.clkr, - [SYSTEM_NOC_BFDCD_CLK_SRC] = &system_noc_bfdcd_clk_src.clkr, [APSS_AHB_CLK_SRC] = &apss_ahb_clk_src.clkr, [BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr, [BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr, @@ -1511,7 +1342,6 @@ static struct clk_regmap *gcc_mdm9607_clocks[] = { [PDM2_CLK_SRC] = &pdm2_clk_src.clkr, [SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr, [SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr, - [APSS_TCU_CLK_SRC] = &apss_tcu_clk_src.clkr, [USB_HS_SYSTEM_CLK_SRC] = &usb_hs_system_clk_src.clkr, [GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr, [GCC_BLSP1_SLEEP_CLK] = &gcc_blsp1_sleep_clk.clkr, @@ -1567,7 +1397,7 @@ static const struct qcom_reset_map gcc_mdm9607_resets[] = { [USB_HS_HSIC_BCR] = { 0x3d05c }, [GCC_MSS_RESTART] = { 0x3e000 }, [USB_HS_BCR] = { 0x41000 }, - [USB2_HS_PHY_ONLY_BCR] = { 0x41034 }, + [USB2_HS_PHY_ONLY_BCR] = { .reg = 0x41034, .udelay = 15 }, [QUSB2_PHY_BCR] = { 0x4103c }, }; diff --git a/drivers/clk/qcom/gcc-msm8916.c b/drivers/clk/qcom/gcc-msm8916.c index 9c7c6b23ce327..e91406e6f5273 100644 --- a/drivers/clk/qcom/gcc-msm8916.c +++ b/drivers/clk/qcom/gcc-msm8916.c @@ -1589,9 +1589,10 @@ static struct clk_branch gcc_blsp1_ahb_clk = { static struct clk_branch gcc_blsp1_sleep_clk = { .halt_reg = 0x01004, + .halt_check = BRANCH_HALT_VOTED, .clkr = { - .enable_reg = 0x01004, - .enable_mask = BIT(0), + .enable_reg = 0x45004, + .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_sleep_clk", .parent_data = &(const struct clk_parent_data){ diff --git a/drivers/clk/qcom/gcc-msm8939.c b/drivers/clk/qcom/gcc-msm8939.c index 45193b3d714ba..ffd7f14fcbaf8 100644 --- a/drivers/clk/qcom/gcc-msm8939.c +++ b/drivers/clk/qcom/gcc-msm8939.c @@ -1929,9 +1929,10 @@ static struct clk_branch gcc_blsp1_ahb_clk = { static struct clk_branch gcc_blsp1_sleep_clk = { .halt_reg = 0x01004, + .halt_check = BRANCH_HALT_VOTED, .clkr = { - .enable_reg = 0x01004, - .enable_mask = BIT(0), + .enable_reg = 0x45004, + .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_sleep_clk", .ops = &clk_branch2_ops, diff --git a/drivers/clk/qcom/gcc-nord.c b/drivers/clk/qcom/gcc-nord.c index 7c7c2171ac965..5c9d25f53a6d2 100644 --- a/drivers/clk/qcom/gcc-nord.c +++ b/drivers/clk/qcom/gcc-nord.c @@ -701,7 +701,7 @@ static struct clk_branch gcc_pcie_a_phy_rchng_clk = { static struct clk_branch gcc_pcie_a_pipe_clk = { .halt_reg = 0x49068, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0x49068, .hwcg_bit = 1, .clkr = { @@ -850,7 +850,7 @@ static struct clk_branch gcc_pcie_b_phy_rchng_clk = { static struct clk_branch gcc_pcie_b_pipe_clk = { .halt_reg = 0x4a068, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x9d008, .enable_mask = BIT(24), @@ -995,7 +995,7 @@ static struct clk_branch gcc_pcie_c_phy_rchng_clk = { static struct clk_branch gcc_pcie_c_pipe_clk = { .halt_reg = 0x4b068, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x9d010, .enable_mask = BIT(1), @@ -1140,7 +1140,7 @@ static struct clk_branch gcc_pcie_d_phy_rchng_clk = { static struct clk_branch gcc_pcie_d_pipe_clk = { .halt_reg = 0x4c068, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x9d010, .enable_mask = BIT(10), @@ -1184,34 +1184,6 @@ static struct clk_branch gcc_pcie_d_slv_q2a_axi_clk = { }, }; -static struct clk_branch gcc_pcie_link_ahb_clk = { - .halt_reg = 0x52464, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0x52464, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "gcc_pcie_link_ahb_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - -static struct clk_branch gcc_pcie_link_xo_clk = { - .halt_reg = 0x52468, - .halt_check = BRANCH_HALT_VOTED, - .hwcg_reg = 0x52468, - .hwcg_bit = 1, - .clkr = { - .enable_reg = 0x52468, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "gcc_pcie_link_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_pcie_noc_async_bridge_clk = { .halt_reg = 0x52048, .halt_check = BRANCH_HALT_SKIP, @@ -1757,8 +1729,6 @@ static struct clk_regmap *gcc_nord_clocks[] = { [GCC_PCIE_D_PIPE_CLK_SRC] = &gcc_pcie_d_pipe_clk_src.clkr, [GCC_PCIE_D_SLV_AXI_CLK] = &gcc_pcie_d_slv_axi_clk.clkr, [GCC_PCIE_D_SLV_Q2A_AXI_CLK] = &gcc_pcie_d_slv_q2a_axi_clk.clkr, - [GCC_PCIE_LINK_AHB_CLK] = &gcc_pcie_link_ahb_clk.clkr, - [GCC_PCIE_LINK_XO_CLK] = &gcc_pcie_link_xo_clk.clkr, [GCC_PCIE_NOC_ASYNC_BRIDGE_CLK] = &gcc_pcie_noc_async_bridge_clk.clkr, [GCC_PCIE_NOC_CNOC_SF_QX_CLK] = &gcc_pcie_noc_cnoc_sf_qx_clk.clkr, [GCC_PCIE_NOC_M_CFG_CLK] = &gcc_pcie_noc_m_cfg_clk.clkr, @@ -1849,9 +1819,16 @@ static const struct regmap_config gcc_nord_regmap_config = { .fast_io = true, }; +static const u32 gcc_nord_critical_cbcrs[] = { + 0x52464, /* GCC_PCIE_LINK_AHB_CLK */ + 0x52468, /* GCC_PCIE_LINK_XO_CLK */ +}; + static const struct qcom_cc_driver_data gcc_nord_driver_data = { .dfs_rcgs = gcc_nord_dfs_clocks, .num_dfs_rcgs = ARRAY_SIZE(gcc_nord_dfs_clocks), + .clk_cbcrs = gcc_nord_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(gcc_nord_critical_cbcrs), }; static const struct qcom_cc_desc gcc_nord_desc = { diff --git a/drivers/clk/qcom/gcc-qcm2290.c b/drivers/clk/qcom/gcc-qcm2290.c index 6684cab63ae11..77cff0e9af93b 100644 --- a/drivers/clk/qcom/gcc-qcm2290.c +++ b/drivers/clk/qcom/gcc-qcm2290.c @@ -116,7 +116,7 @@ static const struct alpha_pll_config gpll10_config = { .vco_mask = GENMASK(21, 20), .main_output_mask = BIT(0), .config_ctl_val = 0x4001055B, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, }; static struct clk_alpha_pll gpll10 = { @@ -148,7 +148,7 @@ static const struct alpha_pll_config gpll11_config = { .vco_mask = GENMASK(21, 20), .main_output_mask = BIT(0), .config_ctl_val = 0x4001055B, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, }; static struct clk_alpha_pll gpll11 = { @@ -309,7 +309,7 @@ static const struct alpha_pll_config gpll8_config = { .post_div_val = 0x1 << 8, .post_div_mask = GENMASK(11, 8), .config_ctl_val = 0x4001055B, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, }; static struct clk_alpha_pll gpll8 = { @@ -1082,7 +1082,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = { .name = "gcc_qupv3_wrap0_s0_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { @@ -1098,7 +1098,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = { .name = "gcc_qupv3_wrap0_s1_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { @@ -1114,7 +1114,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = { .name = "gcc_qupv3_wrap0_s2_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { @@ -1130,7 +1130,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = { .name = "gcc_qupv3_wrap0_s3_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { @@ -1146,7 +1146,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = { .name = "gcc_qupv3_wrap0_s4_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { @@ -1162,7 +1162,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = { .name = "gcc_qupv3_wrap0_s5_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { diff --git a/drivers/clk/qcom/gcc-qcs8300.c b/drivers/clk/qcom/gcc-qcs8300.c index 07218d9c96a7d..31fd870b10f7a 100644 --- a/drivers/clk/qcom/gcc-qcs8300.c +++ b/drivers/clk/qcom/gcc-qcs8300.c @@ -3267,7 +3267,7 @@ static struct gdsc gcc_pcie_0_gdsc = { .pd = { .name = "gcc_pcie_0_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = VOTABLE | RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -3281,7 +3281,7 @@ static struct gdsc gcc_pcie_1_gdsc = { .pd = { .name = "gcc_pcie_1_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = VOTABLE | RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -3305,7 +3305,7 @@ static struct gdsc gcc_usb20_prim_gdsc = { .pd = { .name = "gcc_usb20_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -3317,7 +3317,7 @@ static struct gdsc gcc_usb30_prim_gdsc = { .pd = { .name = "gcc_usb30_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; diff --git a/drivers/clk/qcom/gcc-sm6115.c b/drivers/clk/qcom/gcc-sm6115.c index 4c3804701e243..4bd2ebc158946 100644 --- a/drivers/clk/qcom/gcc-sm6115.c +++ b/drivers/clk/qcom/gcc-sm6115.c @@ -120,7 +120,7 @@ static const struct alpha_pll_config gpll10_config = { .vco_mask = GENMASK(21, 20), .main_output_mask = BIT(0), .config_ctl_val = 0x4001055b, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, .test_ctl_hi_mask = 0x1, }; @@ -173,7 +173,7 @@ static const struct alpha_pll_config gpll11_config = { .vco_val = 0x2 << 20, .vco_mask = GENMASK(21, 20), .config_ctl_val = 0x4001055b, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, .test_ctl_hi_mask = 0x1, }; @@ -367,7 +367,7 @@ static const struct alpha_pll_config gpll8_config = { .post_div_val = 0x1 << 8, .post_div_mask = GENMASK(11, 8), .config_ctl_val = 0x4001055b, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, .test_ctl_hi_mask = 0x1, }; diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c index ee5f86ca50cb7..b99d70149af34 100644 --- a/drivers/clk/qcom/gdsc.c +++ b/drivers/clk/qcom/gdsc.c @@ -104,14 +104,21 @@ static int gdsc_hwctrl(struct gdsc *sc, bool en) static int gdsc_poll_status(struct gdsc *sc, enum gdsc_status status) { ktime_t start; + int ret; start = ktime_get(); do { - if (gdsc_check_status(sc, status)) + ret = gdsc_check_status(sc, status); + if (ret < 0) + return ret; + if (ret) return 0; } while (ktime_us_delta(ktime_get(), start) < STATUS_POLL_TIMEOUT_US); - if (gdsc_check_status(sc, status)) + ret = gdsc_check_status(sc, status); + if (ret < 0) + return ret; + if (ret) return 0; return -ETIMEDOUT; @@ -493,7 +500,9 @@ static int gdsc_init(struct gdsc *sc) } else if (sc->flags & ALWAYS_ON) { /* If ALWAYS_ON GDSCs are not ON, turn them ON */ - gdsc_enable(&sc->pd); + ret = gdsc_enable(&sc->pd); + if (ret) + return ret; on = true; } @@ -669,10 +678,18 @@ int gdsc_register(struct gdsc_desc *desc, void gdsc_unregister(struct gdsc_desc *desc) { struct device *dev = desc->dev; + struct gdsc **scs = desc->scs; size_t num = desc->num; + int i; - gdsc_pm_subdomain_remove(desc, num); of_genpd_del_provider(dev->of_node); + gdsc_pm_subdomain_remove(desc, num); + + for (i = 0; i < num; i++) { + if (!scs[i]) + continue; + pm_genpd_remove(&scs[i]->pd); + } } /* @@ -708,3 +725,25 @@ int gdsc_gx_do_nothing_enable(struct generic_pm_domain *domain) return ret; } EXPORT_SYMBOL_GPL(gdsc_gx_do_nothing_enable); + +/* + * GX GDSC is a special power domain. Normally, its disable sequence + * is managed by the GMU firmware, and high level OS must not attempt + * to disable it. The only exception is during GMU recovery, where the + * GMU driver can set GenPD’s synced_poweroff flag to allow explicitly + * disable GX GDSC in hardware. + */ +int gdsc_gx_disable(struct generic_pm_domain *domain) +{ + struct gdsc *sc = domain_to_gdsc(domain); + + if (domain->synced_poweroff) + return gdsc_disable(domain); + + /* Remove parent-supply placed in enable */ + if (sc->rsupply) + return regulator_disable(sc->rsupply); + + return 0; +} +EXPORT_SYMBOL_GPL(gdsc_gx_disable); diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h index 92ff6bcce7b1c..2f9665b664e60 100644 --- a/drivers/clk/qcom/gdsc.h +++ b/drivers/clk/qcom/gdsc.h @@ -93,6 +93,7 @@ int gdsc_register(struct gdsc_desc *desc, struct reset_controller_dev *, struct regmap *); void gdsc_unregister(struct gdsc_desc *desc); int gdsc_gx_do_nothing_enable(struct generic_pm_domain *domain); +int gdsc_gx_disable(struct generic_pm_domain *domain); #else static inline int gdsc_register(struct gdsc_desc *desc, struct reset_controller_dev *rcdev, diff --git a/drivers/clk/qcom/gpucc-qcm2290.c b/drivers/clk/qcom/gpucc-qcm2290.c index 66dea9d2a0e51..fc33d82bcfb68 100644 --- a/drivers/clk/qcom/gpucc-qcm2290.c +++ b/drivers/clk/qcom/gpucc-qcm2290.c @@ -2,13 +2,12 @@ /* * Copyright (c) 2020, The Linux Foundation. All rights reserved. * Copyright (c) 2024, Linaro Limited + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include #include #include -#include -#include #include #include @@ -20,6 +19,7 @@ #include "clk-regmap-divider.h" #include "clk-regmap-mux.h" #include "clk-regmap-phy-mux.h" +#include "common.h" #include "gdsc.h" #include "reset.h" @@ -56,6 +56,7 @@ static const struct alpha_pll_config gpu_cc_pll0_config = { static struct clk_alpha_pll gpu_cc_pll0 = { .offset = 0x0, + .config = &gpu_cc_pll0_config, .vco_table = huayra_vco, .num_vco = ARRAY_SIZE(huayra_vco), .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_HUAYRA_2290], @@ -143,21 +144,7 @@ static struct clk_rcg2 gpu_cc_gx_gfx3d_clk_src = { .parent_data = gpu_cc_parent_data_1, .num_parents = ARRAY_SIZE(gpu_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, - }, -}; - -static struct clk_branch gpu_cc_ahb_clk = { - .halt_reg = 0x1078, - .halt_check = BRANCH_HALT_DELAY, - .clkr = { - .enable_reg = 0x1078, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gpu_cc_ahb_clk", - .flags = CLK_IS_CRITICAL, - .ops = &clk_branch2_ops, - }, + .ops = &clk_rcg2_shared_ops, }, }; @@ -296,11 +283,14 @@ static struct clk_branch gpu_cc_hlos1_vote_gpu_smmu_clk = { static struct gdsc gpu_cx_gdsc = { .gdscr = 0x106c, .gds_hw_ctrl = 0x1540, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x2, .pd = { .name = "gpu_cx_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = VOTABLE, + .flags = RETAIN_FF_ENABLE | VOTABLE, }; static struct gdsc gpu_gx_gdsc = { @@ -308,16 +298,18 @@ static struct gdsc gpu_gx_gdsc = { .clamp_io_ctrl = 0x1508, .resets = (unsigned int []){ GPU_GX_BCR }, .reset_count = 1, + .en_rest_wait_val = 0x2, + .en_few_wait_val = 0x2, + .clk_dis_wait_val = 0x2, .pd = { .name = "gpu_gx_gdsc", }, .parent = &gpu_cx_gdsc.pd, .pwrsts = PWRSTS_OFF_ON, - .flags = CLAMP_IO | AON_RESET | SW_RESET, + .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR | CLAMP_IO | AON_RESET | SW_RESET, }; static struct clk_regmap *gpu_cc_qcm2290_clocks[] = { - [GPU_CC_AHB_CLK] = &gpu_cc_ahb_clk.clkr, [GPU_CC_CRC_AHB_CLK] = &gpu_cc_crc_ahb_clk.clkr, [GPU_CC_CX_GFX3D_CLK] = &gpu_cc_cx_gfx3d_clk.clkr, [GPU_CC_CX_GMU_CLK] = &gpu_cc_cx_gmu_clk.clkr, @@ -341,6 +333,15 @@ static struct gdsc *gpu_cc_qcm2290_gdscs[] = { [GPU_GX_GDSC] = &gpu_gx_gdsc, }; +static struct clk_alpha_pll *gpu_cc_qcm2290_plls[] = { + &gpu_cc_pll0, +}; + +static const u32 gpu_cc_qcm2290_critical_cbcrs[] = { + 0x1078, /* GPU_CC_AHB_CLK */ + 0x1060, /* GPU_CC_GX_CXO_CLK */ +}; + static const struct regmap_config gpu_cc_qcm2290_regmap_config = { .reg_bits = 32, .reg_stride = 4, @@ -349,6 +350,12 @@ static const struct regmap_config gpu_cc_qcm2290_regmap_config = { .fast_io = true, }; +static const struct qcom_cc_driver_data gpu_cc_qcm2290_driver_data = { + .alpha_plls = gpu_cc_qcm2290_plls, + .num_alpha_plls = ARRAY_SIZE(gpu_cc_qcm2290_plls), + .clk_cbcrs = gpu_cc_qcm2290_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(gpu_cc_qcm2290_critical_cbcrs), +}; static const struct qcom_cc_desc gpu_cc_qcm2290_desc = { .config = &gpu_cc_qcm2290_regmap_config, @@ -358,6 +365,8 @@ static const struct qcom_cc_desc gpu_cc_qcm2290_desc = { .num_resets = ARRAY_SIZE(gpu_cc_qcm2290_resets), .gdscs = gpu_cc_qcm2290_gdscs, .num_gdscs = ARRAY_SIZE(gpu_cc_qcm2290_gdscs), + .use_rpm = true, + .driver_data = &gpu_cc_qcm2290_driver_data, }; static const struct of_device_id gpu_cc_qcm2290_match_table[] = { @@ -368,45 +377,7 @@ MODULE_DEVICE_TABLE(of, gpu_cc_qcm2290_match_table); static int gpu_cc_qcm2290_probe(struct platform_device *pdev) { - struct regmap *regmap; - int ret; - - regmap = qcom_cc_map(pdev, &gpu_cc_qcm2290_desc); - if (IS_ERR(regmap)) - return PTR_ERR(regmap); - - ret = devm_pm_runtime_enable(&pdev->dev); - if (ret) - return ret; - - ret = devm_pm_clk_create(&pdev->dev); - if (ret) - return ret; - - ret = pm_clk_add(&pdev->dev, NULL); - if (ret < 0) { - dev_err(&pdev->dev, "failed to acquire ahb clock\n"); - return ret; - } - - ret = pm_runtime_resume_and_get(&pdev->dev); - if (ret) - return ret; - - clk_huayra_2290_pll_configure(&gpu_cc_pll0, regmap, &gpu_cc_pll0_config); - - regmap_update_bits(regmap, 0x1060, BIT(0), BIT(0)); /* GPU_CC_GX_CXO_CLK */ - - ret = qcom_cc_really_probe(&pdev->dev, &gpu_cc_qcm2290_desc, regmap); - if (ret) { - dev_err(&pdev->dev, "Failed to register display clock controller\n"); - goto out_pm_runtime_put; - } - -out_pm_runtime_put: - pm_runtime_put_sync(&pdev->dev); - - return 0; + return qcom_cc_probe(pdev, &gpu_cc_qcm2290_desc); } static struct platform_driver gpu_cc_qcm2290_driver = { diff --git a/drivers/clk/qcom/gxclkctl-kaanapali.c b/drivers/clk/qcom/gxclkctl-kaanapali.c index 10c1a8976c56c..a03da61489b4d 100644 --- a/drivers/clk/qcom/gxclkctl-kaanapali.c +++ b/drivers/clk/qcom/gxclkctl-kaanapali.c @@ -25,6 +25,7 @@ static struct gdsc gx_clkctl_gx_gdsc = { .pd = { .name = "gx_clkctl_gx_gdsc", .power_on = gdsc_gx_do_nothing_enable, + .power_off = gdsc_gx_disable, }, .pwrsts = PWRSTS_OFF_ON, .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, diff --git a/drivers/clk/qcom/negcc-nord.c b/drivers/clk/qcom/negcc-nord.c index 355850a875acb..0db284edc4e41 100644 --- a/drivers/clk/qcom/negcc-nord.c +++ b/drivers/clk/qcom/negcc-nord.c @@ -16,7 +16,6 @@ #include "clk-rcg.h" #include "clk-regmap.h" #include "clk-regmap-divider.h" -#include "clk-regmap-mux.h" #include "clk-regmap-phy-mux.h" #include "common.h" #include "gdsc.h" @@ -41,8 +40,6 @@ enum { P_UFS_PHY_RX_SYMBOL_0_CLK, P_UFS_PHY_RX_SYMBOL_1_CLK, P_UFS_PHY_TX_SYMBOL_0_CLK, - P_USB3_PHY_SEC_WRAPPER_NE_GCC_USB31_PIPE_CLK, - P_USB3_PHY_WRAPPER_NE_GCC_USB31_PIPE_CLK, }; static struct clk_alpha_pll ne_gcc_gpll0 = { @@ -165,26 +162,6 @@ static const struct clk_parent_data ne_gcc_parent_data_5[] = { { .index = DT_BI_TCXO }, }; -static const struct parent_map ne_gcc_parent_map_6[] = { - { P_USB3_PHY_WRAPPER_NE_GCC_USB31_PIPE_CLK, 0 }, - { P_BI_TCXO, 2 }, -}; - -static const struct clk_parent_data ne_gcc_parent_data_6[] = { - { .index = DT_USB3_PHY_WRAPPER_NE_GCC_USB31_PIPE_CLK }, - { .index = DT_BI_TCXO }, -}; - -static const struct parent_map ne_gcc_parent_map_7[] = { - { P_USB3_PHY_SEC_WRAPPER_NE_GCC_USB31_PIPE_CLK, 0 }, - { P_BI_TCXO, 2 }, -}; - -static const struct clk_parent_data ne_gcc_parent_data_7[] = { - { .index = DT_USB3_PHY_SEC_WRAPPER_NE_GCC_USB31_PIPE_CLK }, - { .index = DT_BI_TCXO }, -}; - static struct clk_regmap_phy_mux ne_gcc_ufs_phy_rx_symbol_0_clk_src = { .reg = 0x33068, .clkr = { @@ -227,32 +204,30 @@ static struct clk_regmap_phy_mux ne_gcc_ufs_phy_tx_symbol_0_clk_src = { }, }; -static struct clk_regmap_mux ne_gcc_usb3_prim_phy_pipe_clk_src = { +static struct clk_regmap_phy_mux ne_gcc_usb3_prim_phy_pipe_clk_src = { .reg = 0x2a078, - .shift = 0, - .width = 2, - .parent_map = ne_gcc_parent_map_6, .clkr = { .hw.init = &(const struct clk_init_data) { .name = "ne_gcc_usb3_prim_phy_pipe_clk_src", - .parent_data = ne_gcc_parent_data_6, - .num_parents = ARRAY_SIZE(ne_gcc_parent_data_6), - .ops = &clk_regmap_mux_closest_ops, + .parent_data = &(const struct clk_parent_data){ + .index = DT_USB3_PHY_WRAPPER_NE_GCC_USB31_PIPE_CLK, + }, + .num_parents = 1, + .ops = &clk_regmap_phy_mux_ops, }, }, }; -static struct clk_regmap_mux ne_gcc_usb3_sec_phy_pipe_clk_src = { +static struct clk_regmap_phy_mux ne_gcc_usb3_sec_phy_pipe_clk_src = { .reg = 0x2c078, - .shift = 0, - .width = 2, - .parent_map = ne_gcc_parent_map_7, .clkr = { .hw.init = &(const struct clk_init_data) { .name = "ne_gcc_usb3_sec_phy_pipe_clk_src", - .parent_data = ne_gcc_parent_data_7, - .num_parents = ARRAY_SIZE(ne_gcc_parent_data_7), - .ops = &clk_regmap_mux_closest_ops, + .parent_data = &(const struct clk_parent_data){ + .index = DT_USB3_PHY_SEC_WRAPPER_NE_GCC_USB31_PIPE_CLK, + }, + .num_parents = 1, + .ops = &clk_regmap_phy_mux_ops, }, }, }; @@ -951,21 +926,6 @@ static struct clk_branch ne_gcc_gp2_clk = { }, }; -static struct clk_branch ne_gcc_gpu_2_cfg_clk = { - .halt_reg = 0x34004, - .halt_check = BRANCH_HALT_VOTED, - .hwcg_reg = 0x34004, - .hwcg_bit = 1, - .clkr = { - .enable_reg = 0x34004, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "ne_gcc_gpu_2_cfg_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch ne_gcc_gpu_2_gpll0_clk_src = { .halt_check = BRANCH_HALT_DELAY, .clkr = { @@ -1641,7 +1601,7 @@ static struct clk_branch ne_gcc_usb3_prim_phy_com_aux_clk = { static struct clk_branch ne_gcc_usb3_prim_phy_pipe_clk = { .halt_reg = 0x2a074, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0x2a074, .hwcg_bit = 1, .clkr = { @@ -1697,7 +1657,7 @@ static struct clk_branch ne_gcc_usb3_sec_phy_com_aux_clk = { static struct clk_branch ne_gcc_usb3_sec_phy_pipe_clk = { .halt_reg = 0x2c074, - .halt_check = BRANCH_HALT_VOTED, + .halt_check = BRANCH_HALT_SKIP, .hwcg_reg = 0x2c074, .hwcg_bit = 1, .clkr = { @@ -1816,7 +1776,6 @@ static struct clk_regmap *ne_gcc_nord_clocks[] = { [NE_GCC_GPLL0] = &ne_gcc_gpll0.clkr, [NE_GCC_GPLL0_OUT_EVEN] = &ne_gcc_gpll0_out_even.clkr, [NE_GCC_GPLL2] = &ne_gcc_gpll2.clkr, - [NE_GCC_GPU_2_CFG_CLK] = &ne_gcc_gpu_2_cfg_clk.clkr, [NE_GCC_GPU_2_GPLL0_CLK_SRC] = &ne_gcc_gpu_2_gpll0_clk_src.clkr, [NE_GCC_GPU_2_GPLL0_DIV_CLK_SRC] = &ne_gcc_gpu_2_gpll0_div_clk_src.clkr, [NE_GCC_GPU_2_HSCNOC_GFX_CLK] = &ne_gcc_gpu_2_hscnoc_gfx_clk.clkr, @@ -1945,10 +1904,16 @@ static void clk_nord_regs_configure(struct device *dev, struct regmap *regmap) qcom_branch_set_force_mem_core(regmap, ne_gcc_ufs_phy_axi_clk, true); } +static const u32 ne_gcc_nord_critical_cbcrs[] = { + 0x34004, /* NE_GCC_GPU_2_CFG_CLK */ +}; + static const struct qcom_cc_driver_data ne_gcc_nord_driver_data = { .dfs_rcgs = ne_gcc_nord_dfs_clocks, .num_dfs_rcgs = ARRAY_SIZE(ne_gcc_nord_dfs_clocks), .clk_regs_configure = clk_nord_regs_configure, + .clk_cbcrs = ne_gcc_nord_critical_cbcrs, + .num_clk_cbcrs = ARRAY_SIZE(ne_gcc_nord_critical_cbcrs), }; static const struct qcom_cc_desc ne_gcc_nord_desc = { diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c index 6b853800cb6bc..706ca4b344d39 100644 --- a/drivers/clk/rockchip/clk-pll.c +++ b/drivers/clk/rockchip/clk-pll.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include "clk.h" @@ -900,6 +901,14 @@ static void rockchip_rk3588_pll_get_params(struct rockchip_clk_pll *pll, rate->k = ((pllcon >> RK3588_PLLCON2_K_SHIFT) & RK3588_PLLCON2_K_MASK); } +/* + * 2250 MHz <= Fvco <= 4500 MHz + * For Fvco > 3 GHz: period jitter +-1% frac PLL, +-0.75% int PLL + * For Fvco < 3 GHz: period jitter +-2% frac PLL, +-1.50% int PLL + * Fvco = ((m + k / 65536) * Fin) / p + * Fout = ((m + k / 65536) * Fin) / (p * 2^s) + * -32768 <= k <= 32767 (only available in frac PLLs, not int PLLs) + */ static unsigned long rockchip_rk3588_pll_recalc_rate(struct clk_hw *hw, unsigned long prate) { struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); @@ -913,11 +922,10 @@ static unsigned long rockchip_rk3588_pll_recalc_rate(struct clk_hw *hw, unsigned if (cur.k) { /* fractional mode */ - u64 frac_rate64 = prate * cur.k; + s64 frac_rate64 = (s64)prate * cur.k; - postdiv = cur.p * 65535; - do_div(frac_rate64, postdiv); - rate64 += frac_rate64; + postdiv = cur.p * 65536; + rate64 += div_s64(frac_rate64, postdiv); } rate64 = rate64 >> cur.s; diff --git a/drivers/clk/rockchip/clk-rk3576.c b/drivers/clk/rockchip/clk-rk3576.c index 2557358e0b9d8..2cdd667f40041 100644 --- a/drivers/clk/rockchip/clk-rk3576.c +++ b/drivers/clk/rockchip/clk-rk3576.c @@ -79,13 +79,13 @@ static struct rockchip_pll_rate_table rk3576_pll_rates[] = { RK3588_PLL_RATE(1008000000, 2, 336, 2, 0), RK3588_PLL_RATE(1000000000, 3, 500, 2, 0), RK3588_PLL_RATE(983040000, 4, 655, 2, 23592), - RK3588_PLL_RATE(955520000, 3, 477, 2, 49806), + RK3588_PLL_RATE(955520000, 3, 478, 2, -15730), RK3588_PLL_RATE(903168000, 6, 903, 2, 11009), RK3588_PLL_RATE(900000000, 2, 300, 2, 0), RK3588_PLL_RATE(816000000, 2, 272, 2, 0), RK3588_PLL_RATE(786432000, 2, 262, 2, 9437), RK3588_PLL_RATE(786000000, 1, 131, 2, 0), - RK3588_PLL_RATE(785560000, 3, 392, 2, 51117), + RK3588_PLL_RATE(785560000, 3, 393, 2, -14419), RK3588_PLL_RATE(722534400, 8, 963, 2, 24850), RK3588_PLL_RATE(600000000, 2, 200, 2, 0), RK3588_PLL_RATE(594000000, 2, 198, 2, 0), @@ -315,6 +315,7 @@ PNAME(mux_100m_24m_lclk0_p) = { "clk_cpll_div10", "xin24m", "lclk_asrc_src_0" } PNAME(mux_100m_24m_lclk1_p) = { "clk_cpll_div10", "xin24m", "lclk_asrc_src_1" }; PNAME(mux_150m_100m_50m_24m_p) = { "clk_gpll_div8", "clk_cpll_div10", "clk_cpll_div20", "xin24m" }; PNAME(mux_200m_100m_50m_24m_p) = { "clk_gpll_div6", "clk_cpll_div10", "clk_cpll_div20", "xin24m" }; +PNAME(mux_200m_150m_100m_24m_p) = { "clk_gpll_div6", "clk_gpll_div8", "clk_cpll_div10", "xin24m" }; PNAME(mux_400m_200m_100m_24m_p) = { "clk_gpll_div3", "clk_gpll_div6", "clk_cpll_div10", "xin24m" }; PNAME(mux_500m_250m_100m_24m_p) = { "clk_cpll_div2", "clk_cpll_div4", "clk_cpll_div10", "xin24m" }; PNAME(mux_600m_400m_300m_24m_p) = { "clk_gpll_div2", "clk_gpll_div3", "clk_gpll_div4", "xin24m" }; @@ -706,19 +707,19 @@ static struct rockchip_clk_branch rk3576_clk_branches[] __initdata = { RK3576_CLKGATE_CON(16), 0, GFLAGS), GATE(PCLK_SPI4, "pclk_spi4", "pclk_bus_root", 0, RK3576_CLKGATE_CON(16), 1, GFLAGS), - COMPOSITE_NODIV(CLK_SPI0, "clk_spi0", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI0, "clk_spi0", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(70), 13, 2, MFLAGS, RK3576_CLKGATE_CON(16), 2, GFLAGS), - COMPOSITE_NODIV(CLK_SPI1, "clk_spi1", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI1, "clk_spi1", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 0, 2, MFLAGS, RK3576_CLKGATE_CON(16), 3, GFLAGS), - COMPOSITE_NODIV(CLK_SPI2, "clk_spi2", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI2, "clk_spi2", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 2, 2, MFLAGS, RK3576_CLKGATE_CON(16), 4, GFLAGS), - COMPOSITE_NODIV(CLK_SPI3, "clk_spi3", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI3, "clk_spi3", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 4, 2, MFLAGS, RK3576_CLKGATE_CON(16), 5, GFLAGS), - COMPOSITE_NODIV(CLK_SPI4, "clk_spi4", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI4, "clk_spi4", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 6, 2, MFLAGS, RK3576_CLKGATE_CON(16), 6, GFLAGS), GATE(PCLK_WDT0, "pclk_wdt0", "pclk_bus_root", 0, diff --git a/drivers/clk/rockchip/clk-rk3588.c b/drivers/clk/rockchip/clk-rk3588.c index 86953f9ffee38..517e30e249d7b 100644 --- a/drivers/clk/rockchip/clk-rk3588.c +++ b/drivers/clk/rockchip/clk-rk3588.c @@ -79,14 +79,14 @@ static struct rockchip_pll_rate_table rk3588_pll_rates[] = { RK3588_PLL_RATE(1008000000, 2, 336, 2, 0), RK3588_PLL_RATE(1000000000, 3, 500, 2, 0), RK3588_PLL_RATE(983040000, 4, 655, 2, 23592), - RK3588_PLL_RATE(955520000, 3, 477, 2, 49806), + RK3588_PLL_RATE(955520000, 3, 478, 2, -15730), RK3588_PLL_RATE(903168000, 6, 903, 2, 11009), RK3588_PLL_RATE(900000000, 2, 300, 2, 0), RK3588_PLL_RATE(850000000, 3, 425, 2, 0), RK3588_PLL_RATE(816000000, 2, 272, 2, 0), RK3588_PLL_RATE(786432000, 2, 262, 2, 9437), RK3588_PLL_RATE(786000000, 1, 131, 2, 0), - RK3588_PLL_RATE(785560000, 3, 392, 2, 51117), + RK3588_PLL_RATE(785560000, 3, 393, 2, -14419), RK3588_PLL_RATE(722534400, 8, 963, 2, 24850), RK3588_PLL_RATE(600000000, 2, 200, 2, 0), RK3588_PLL_RATE(594000000, 2, 198, 2, 0), @@ -2102,7 +2102,7 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = { COMPOSITE(DCLK_VOP1_SRC, "dclk_vop1_src", gpll_cpll_v0pll_aupll_p, 0, RK3588_CLKSEL_CON(111), 14, 2, MFLAGS, 9, 5, DFLAGS, RK3588_CLKGATE_CON(52), 11, GFLAGS), - COMPOSITE(DCLK_VOP2_SRC, "dclk_vop2_src", gpll_cpll_v0pll_aupll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + COMPOSITE(DCLK_VOP2_SRC, "dclk_vop2_src", gpll_cpll_v0pll_aupll_p, CLK_SET_RATE_NO_REPARENT, RK3588_CLKSEL_CON(112), 5, 2, MFLAGS, 0, 5, DFLAGS, RK3588_CLKGATE_CON(52), 12, GFLAGS), COMPOSITE_NODIV(DCLK_VOP0, "dclk_vop0", dclk_vop0_p, diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h index 9e3503e2ffc23..72b36bba31523 100644 --- a/drivers/clk/rockchip/clk.h +++ b/drivers/clk/rockchip/clk.h @@ -635,10 +635,10 @@ struct rockchip_pll_rate_table { }; struct { /* for RK3588 */ - unsigned int m; - unsigned int p; - unsigned int s; - unsigned int k; + unsigned int m; /* main divider, 10 bit unsigned */ + unsigned int p; /* pre-divider, 6 bit unsigned */ + unsigned int s; /* scaler, 3 bit unsigned */ + s16 k; /* fractional part, 16 bit two's complement */ }; }; }; diff --git a/drivers/clk/spacemit/ccu-k3.c b/drivers/clk/spacemit/ccu-k3.c index 764426359540c..e94a5d2687d57 100644 --- a/drivers/clk/spacemit/ccu-k3.c +++ b/drivers/clk/spacemit/ccu-k3.c @@ -236,11 +236,11 @@ CCU_MUX_DEFINE(i2s3_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 12, 2, CCU_MUX_DEFINE(i2s4_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 16, 2, 0); CCU_MUX_DEFINE(i2s5_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 20, 2, 0); -CCU_DDN_DEFINE(i2s0_sysclk_div, i2s0_sysclk_sel, MPMU_I2S0_SYSCLK, 0, 16, 16, 16, 1, 0); -CCU_DDN_DEFINE(i2s2_sysclk_div, i2s2_sysclk_sel, MPMU_I2S2_SYSCLK, 0, 16, 16, 16, 1, 0); -CCU_DDN_DEFINE(i2s3_sysclk_div, i2s3_sysclk_sel, MPMU_I2S3_SYSCLK, 0, 16, 16, 16, 1, 0); -CCU_DDN_DEFINE(i2s4_sysclk_div, i2s4_sysclk_sel, MPMU_I2S4_SYSCLK, 0, 16, 16, 16, 1, 0); -CCU_DDN_DEFINE(i2s5_sysclk_div, i2s5_sysclk_sel, MPMU_I2S5_SYSCLK, 0, 16, 16, 16, 1, 0); +CCU_DDN_DEFINE(i2s0_sysclk_div, i2s0_sysclk_sel, MPMU_I2S0_SYSCLK, 0, 16, 16, 16, 2, 0); +CCU_DDN_DEFINE(i2s2_sysclk_div, i2s2_sysclk_sel, MPMU_I2S2_SYSCLK, 0, 16, 16, 16, 2, 0); +CCU_DDN_DEFINE(i2s3_sysclk_div, i2s3_sysclk_sel, MPMU_I2S3_SYSCLK, 0, 16, 16, 16, 2, 0); +CCU_DDN_DEFINE(i2s4_sysclk_div, i2s4_sysclk_sel, MPMU_I2S4_SYSCLK, 0, 16, 16, 16, 2, 0); +CCU_DDN_DEFINE(i2s5_sysclk_div, i2s5_sysclk_sel, MPMU_I2S5_SYSCLK, 0, 16, 16, 16, 2, 0); static const struct clk_parent_data i2s2_sysclk_parents[] = { CCU_PARENT_HW(i2s1_sysclk), @@ -926,11 +926,10 @@ CCU_MUX_DIV_GATE_FC_DEFINE(dpu_aclk, dpu_aclk_parents, APMU_LCD_CLK_RES_CTRL5, 1 20, 3, BIT(16), 0); static const struct clk_parent_data ufs_aclk_parents[] = { - CCU_PARENT_HW(pll1_d6_409p6), CCU_PARENT_HW(pll1_d5_491p52), - CCU_PARENT_HW(pll1_d4_614p4), - CCU_PARENT_HW(pll1_d8_307p2), - CCU_PARENT_HW(pll2_d4), + CCU_PARENT_HW(pll1_d6_409p6), + CCU_PARENT_HW(pll2_d6), + CCU_PARENT_HW(pll2_d5), }; CCU_MUX_DIV_GATE_FC_DEFINE(ufs_aclk, ufs_aclk_parents, APMU_UFS_CLK_RES_CTRL, 5, 3, BIT(8), 2, 3, BIT(1), 0); diff --git a/drivers/clk/stm32/clk-stm32mp21.c b/drivers/clk/stm32/clk-stm32mp21.c index c8a37b716bd55..bdb17419908c8 100644 --- a/drivers/clk/stm32/clk-stm32mp21.c +++ b/drivers/clk/stm32/clk-stm32mp21.c @@ -4,6 +4,7 @@ * Author: Gabriel Fernandez for STMicroelectronics. */ +#include #include #include #include diff --git a/drivers/clk/stm32/clk-stm32mp25.c b/drivers/clk/stm32/clk-stm32mp25.c index 52f0e8a129262..eb0bc918ecee0 100644 --- a/drivers/clk/stm32/clk-stm32mp25.c +++ b/drivers/clk/stm32/clk-stm32mp25.c @@ -4,6 +4,7 @@ * Author: Gabriel Fernandez for STMicroelectronics. */ +#include #include #include #include diff --git a/drivers/clk/sunxi-ng/ccu_mux.c b/drivers/clk/sunxi-ng/ccu_mux.c index 766f27cff748e..e56a3005548e3 100644 --- a/drivers/clk/sunxi-ng/ccu_mux.c +++ b/drivers/clk/sunxi-ng/ccu_mux.c @@ -93,66 +93,65 @@ int ccu_mux_helper_determine_rate(struct ccu_common *common, struct clk_rate_request adj_req = *req; best_parent = clk_hw_get_parent(hw); - best_parent_rate = clk_hw_get_rate(best_parent); - + adj_req.best_parent_rate = clk_hw_get_rate(best_parent); adj_req.best_parent_hw = best_parent; - adj_req.best_parent_rate = ccu_mux_helper_apply_prediv(common, cm, -1, - best_parent_rate); + + /* + * This effectively treats the predivider as a postdivider. + * It stays mathematically correct and ensures whatever + * round() will do stays correct while walking the tree. + * It may query the parent rate too while handling rate + * propagation. + */ + adj_req.rate = ccu_mux_helper_unapply_prediv(common, cm, -1, + req->rate); ret = round(cm, &adj_req, data); if (ret) return ret; - best_rate = adj_req.rate; - /* - * best_parent_rate might have been modified by our clock. - * Unapply the pre-divider if there's one, and give - * the actual frequency the parent needs to run at. + * parent_rate might have been modified by our clock as part + * of the rate propagation mechanism. Same goes below. */ - best_parent_rate = ccu_mux_helper_unapply_prediv(common, cm, -1, - adj_req.best_parent_rate); + best_parent_rate = adj_req.best_parent_rate; + best_rate = ccu_mux_helper_apply_prediv(common, cm, -1, + adj_req.rate); goto out; } for (i = 0; i < clk_hw_get_num_parents(hw); i++) { struct clk_rate_request tmp_req = *req; - unsigned long parent_rate; + unsigned long rate; struct clk_hw *parent; parent = clk_hw_get_parent_by_index(hw, i); if (!parent) continue; - parent_rate = ccu_mux_helper_apply_prediv(common, cm, i, - clk_hw_get_rate(parent)); - tmp_req.best_parent_hw = parent; - tmp_req.best_parent_rate = parent_rate; + tmp_req.best_parent_rate = clk_hw_get_rate(parent); + tmp_req.rate = ccu_mux_helper_unapply_prediv(common, cm, i, + req->rate); ret = round(cm, &tmp_req, data); if (ret) continue; - /* - * parent_rate might have been modified by our clock. - * Unapply the pre-divider if there's one, and give - * the actual frequency the parent needs to run at. - */ - parent_rate = ccu_mux_helper_unapply_prediv(common, cm, i, - tmp_req.best_parent_rate); + rate = ccu_mux_helper_apply_prediv(common, cm, i, + tmp_req.rate); - if (tmp_req.rate == req->rate) { + if (rate == req->rate) { best_parent = parent; - best_parent_rate = parent_rate; - best_rate = tmp_req.rate; + best_parent_rate = tmp_req.best_parent_rate; + best_rate = rate; goto out; } - if (ccu_is_better_rate(common, req->rate, tmp_req.rate, best_rate)) { - best_rate = tmp_req.rate; - best_parent_rate = parent_rate; + if (ccu_is_better_rate(common, req->rate, rate, best_rate)) { + best_rate = rate; + best_parent_rate = tmp_req.best_parent_rate; best_parent = parent; } } diff --git a/drivers/clk/tegra/clk-tegra124-emc.c b/drivers/clk/tegra/clk-tegra124-emc.c index f3b2c96fdcfc2..94ac24ea1e6ad 100644 --- a/drivers/clk/tegra/clk-tegra124-emc.c +++ b/drivers/clk/tegra/clk-tegra124-emc.c @@ -537,6 +537,7 @@ struct clk *tegra124_clk_register_emc(void __iomem *base, struct device_node *np clk = clk_register(NULL, &tegra->hw); if (IS_ERR(clk)) { + of_node_put(tegra->emc_node); kfree(tegra); return clk; } diff --git a/drivers/clk/ti/adpll.c b/drivers/clk/ti/adpll.c index e305fcbac6475..8885d28face50 100644 --- a/drivers/clk/ti/adpll.c +++ b/drivers/clk/ti/adpll.c @@ -483,7 +483,7 @@ static const struct clk_ops ti_adpll_ops = { static int ti_adpll_init_dco(struct ti_adpll_data *d) { - struct clk_init_data init; + struct clk_init_data init = {}; struct clk *clock; const char *postfix; int width, err; @@ -576,7 +576,7 @@ static int ti_adpll_init_clkout(struct ti_adpll_data *d, struct clk *clk1) { struct ti_adpll_clkout_data *co; - struct clk_init_data init; + struct clk_init_data init = {}; struct clk_ops *ops; const char *parent_names[2]; const char *child_name; diff --git a/drivers/clk/ti/divider.c b/drivers/clk/ti/divider.c index af923b8cb0ed8..3b438c2d68aac 100644 --- a/drivers/clk/ti/divider.c +++ b/drivers/clk/ti/divider.c @@ -311,7 +311,7 @@ static struct clk *_register_divider(struct device_node *node, u32 flags, struct clk_omap_divider *div) { - struct clk_init_data init; + struct clk_init_data init = {}; const char *parent_name; const char *name; diff --git a/drivers/clk/ti/mux.c b/drivers/clk/ti/mux.c index d6a0ccfd81dbe..0fef60e82107a 100644 --- a/drivers/clk/ti/mux.c +++ b/drivers/clk/ti/mux.c @@ -119,14 +119,14 @@ const struct clk_ops ti_clk_mux_ops = { }; static struct clk *_register_mux(struct device_node *node, const char *name, - const char * const *parent_names, + const struct clk_parent_data *parent_data, u8 num_parents, unsigned long flags, struct clk_omap_reg *reg, u8 shift, u32 mask, s8 latch, u8 clk_mux_flags, u32 *table) { + struct clk_init_data init = {}; struct clk_omap_mux *mux; struct clk *clk; - struct clk_init_data init; /* allocate the mux */ mux = kzalloc_obj(*mux); @@ -136,7 +136,7 @@ static struct clk *_register_mux(struct device_node *node, const char *name, init.name = name; init.ops = &ti_clk_mux_ops; init.flags = flags; - init.parent_names = parent_names; + init.parent_data = parent_data; init.num_parents = num_parents; /* struct clk_mux assignments */ @@ -167,24 +167,26 @@ static void of_mux_clk_setup(struct device_node *node) struct clk *clk; struct clk_omap_reg reg; unsigned int num_parents; - const char **parent_names; + struct clk_parent_data *parent_data; const char *name; u8 clk_mux_flags = 0; u32 mask = 0; u32 shift = 0; s32 latch = -EINVAL; u32 flags = CLK_SET_RATE_NO_REPARENT; + int i; num_parents = of_clk_get_parent_count(node); if (num_parents < 2) { pr_err("mux-clock %pOFn must have parents\n", node); return; } - parent_names = kcalloc(num_parents, sizeof(char *), GFP_KERNEL); - if (!parent_names) - goto cleanup; + parent_data = kcalloc(num_parents, sizeof(*parent_data), GFP_KERNEL); + if (!parent_data) + return; - of_clk_parent_fill(node, parent_names, num_parents); + for (i = 0; i < num_parents; i++) + parent_data[i].index = i; if (ti_clk_get_reg_addr(node, 0, ®)) goto cleanup; @@ -207,7 +209,7 @@ static void of_mux_clk_setup(struct device_node *node) mask = (1 << fls(mask)) - 1; name = ti_dt_clk_name(node); - clk = _register_mux(node, name, parent_names, num_parents, + clk = _register_mux(node, name, parent_data, num_parents, flags, ®, shift, mask, latch, clk_mux_flags, NULL); @@ -215,7 +217,7 @@ static void of_mux_clk_setup(struct device_node *node) of_clk_add_provider(node, of_clk_src_simple_get, clk); cleanup: - kfree(parent_names); + kfree(parent_data); } CLK_OF_DECLARE(mux_clk, "ti,mux-clock", of_mux_clk_setup); diff --git a/drivers/clk/ux500/clk-prcmu.c b/drivers/clk/ux500/clk-prcmu.c index ddc86551bf574..ac96c46bd1bbf 100644 --- a/drivers/clk/ux500/clk-prcmu.c +++ b/drivers/clk/ux500/clk-prcmu.c @@ -7,7 +7,7 @@ */ #include -#include +#include #include #include #include @@ -35,13 +35,13 @@ static int clk_prcmu_prepare(struct clk_hw *hw) { struct clk_prcmu *clk = to_clk_prcmu(hw); - return prcmu_request_clock(clk->cg_sel, true); + return db8500_prcmu_request_clock(clk->cg_sel, true); } static void clk_prcmu_unprepare(struct clk_hw *hw) { struct clk_prcmu *clk = to_clk_prcmu(hw); - if (prcmu_request_clock(clk->cg_sel, false)) + if (db8500_prcmu_request_clock(clk->cg_sel, false)) pr_err("clk_prcmu: %s failed to disable %s.\n", __func__, clk_hw_get_name(hw)); } @@ -86,7 +86,7 @@ static int clk_prcmu_opp_prepare(struct clk_hw *hw) clk->opp_requested = 1; } - err = prcmu_request_clock(clk->cg_sel, true); + err = db8500_prcmu_request_clock(clk->cg_sel, true); if (err) { prcmu_qos_remove_requirement(PRCMU_QOS_APE_OPP, (char *)clk_hw_get_name(hw)); @@ -101,7 +101,7 @@ static void clk_prcmu_opp_unprepare(struct clk_hw *hw) { struct clk_prcmu *clk = to_clk_prcmu(hw); - if (prcmu_request_clock(clk->cg_sel, false)) { + if (db8500_prcmu_request_clock(clk->cg_sel, false)) { pr_err("clk_prcmu: %s failed to disable %s.\n", __func__, clk_hw_get_name(hw)); return; @@ -120,7 +120,7 @@ static int clk_prcmu_opp_volt_prepare(struct clk_hw *hw) struct clk_prcmu *clk = to_clk_prcmu(hw); if (!clk->opp_requested) { - err = prcmu_request_ape_opp_100_voltage(true); + err = db8500_prcmu_request_ape_opp_100_voltage(true); if (err) { pr_err("clk_prcmu: %s fail req APE OPP VOLT for %s.\n", __func__, clk_hw_get_name(hw)); @@ -129,9 +129,9 @@ static int clk_prcmu_opp_volt_prepare(struct clk_hw *hw) clk->opp_requested = 1; } - err = prcmu_request_clock(clk->cg_sel, true); + err = db8500_prcmu_request_clock(clk->cg_sel, true); if (err) { - prcmu_request_ape_opp_100_voltage(false); + db8500_prcmu_request_ape_opp_100_voltage(false); clk->opp_requested = 0; return err; } @@ -143,14 +143,14 @@ static void clk_prcmu_opp_volt_unprepare(struct clk_hw *hw) { struct clk_prcmu *clk = to_clk_prcmu(hw); - if (prcmu_request_clock(clk->cg_sel, false)) { + if (db8500_prcmu_request_clock(clk->cg_sel, false)) { pr_err("clk_prcmu: %s failed to disable %s.\n", __func__, clk_hw_get_name(hw)); return; } if (clk->opp_requested) { - prcmu_request_ape_opp_100_voltage(false); + db8500_prcmu_request_ape_opp_100_voltage(false); clk->opp_requested = 0; } } diff --git a/drivers/clk/ux500/u8500_of_clk.c b/drivers/clk/ux500/u8500_of_clk.c index 6f78808387b10..d2499815226f0 100644 --- a/drivers/clk/ux500/u8500_of_clk.c +++ b/drivers/clk/ux500/u8500_of_clk.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include "clk.h" #include "prcc.h" diff --git a/drivers/clk/visconti/clkc.c b/drivers/clk/visconti/clkc.c index d0b193b5d0b35..4018d1298880d 100644 --- a/drivers/clk/visconti/clkc.c +++ b/drivers/clk/visconti/clkc.c @@ -81,9 +81,9 @@ static struct clk_hw *visconti_clk_register_gate(struct device *dev, u8 rs_idx, spinlock_t *lock) { + struct clk_init_data init = {}; struct visconti_clk_gate *gate; struct clk_parent_data *pdata; - struct clk_init_data init; struct clk_hw *hw; int ret; diff --git a/drivers/clk/x86/clk-pmc-atom.c b/drivers/clk/x86/clk-pmc-atom.c index 99291ba65da73..08c83e0abc41d 100644 --- a/drivers/clk/x86/clk-pmc-atom.c +++ b/drivers/clk/x86/clk-pmc-atom.c @@ -160,6 +160,9 @@ static struct clk_plt *plt_clk_register(struct platform_device *pdev, int id, return ERR_PTR(-ENOMEM); init.name = kasprintf(GFP_KERNEL, "%s_%d", PLT_CLK_NAME_BASE, id); + if (!init.name) + return ERR_PTR(-ENOMEM); + init.ops = &plt_clk_ops; init.flags = 0; init.parent_names = parent_names; diff --git a/drivers/clk/xilinx/clk-xlnx-clock-wizard.c b/drivers/clk/xilinx/clk-xlnx-clock-wizard.c index 4a0136349f71a..dbef983eb4252 100644 --- a/drivers/clk/xilinx/clk-xlnx-clock-wizard.c +++ b/drivers/clk/xilinx/clk-xlnx-clock-wizard.c @@ -663,8 +663,8 @@ static int clk_wzrd_determine_rate_all(struct clk_hw *hw, d = divider->d; o = divider->o; - req->rate = div_u64(req->best_parent_rate * (m * 1000 + divider->m_frac), - d * (o * 1000 + divider->o_frac)); + req->rate = mult_frac(req->best_parent_rate, m * 1000 + divider->m_frac, + d * (o * 1000 + divider->o_frac)); return 0; } diff --git a/drivers/clocksource/clps711x-timer.c b/drivers/clocksource/clps711x-timer.c index bb0a44adaf28a..63ae3a691b148 100644 --- a/drivers/clocksource/clps711x-timer.c +++ b/drivers/clocksource/clps711x-timer.c @@ -94,7 +94,7 @@ static int __init clps711x_timer_init(struct device_node *np) switch (of_alias_get_id(np, "timer")) { case CLPS711X_CLKSRC_CLOCKSOURCE: clps711x_clksrc_init(clock, base); - break; + return 0; case CLPS711X_CLKSRC_CLOCKEVENT: ret = _clps711x_clkevt_init(clock, base, irq); break; diff --git a/drivers/clocksource/samsung_pwm_timer.c b/drivers/clocksource/samsung_pwm_timer.c index b9561e3f196c4..0544124cf5cea 100644 --- a/drivers/clocksource/samsung_pwm_timer.c +++ b/drivers/clocksource/samsung_pwm_timer.c @@ -56,7 +56,7 @@ #define TCON_AUTORELOAD(chan) \ ((chan < 5) ? _TCON_AUTORELOAD(chan) : _TCON_AUTORELOAD4(chan)) -DEFINE_SPINLOCK(samsung_pwm_lock); +DEFINE_RAW_SPINLOCK(samsung_pwm_lock); EXPORT_SYMBOL(samsung_pwm_lock); struct samsung_pwm_clocksource { @@ -87,14 +87,14 @@ static void samsung_timer_set_prescale(unsigned int channel, u16 prescale) if (channel >= 2) shift = TCFG0_PRESCALER1_SHIFT; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); reg = readl(pwm.base + REG_TCFG0); reg &= ~(TCFG0_PRESCALER_MASK << shift); reg |= (prescale - 1) << shift; writel(reg, pwm.base + REG_TCFG0); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static void samsung_timer_set_divisor(unsigned int channel, u8 divisor) @@ -106,14 +106,14 @@ static void samsung_timer_set_divisor(unsigned int channel, u8 divisor) bits = (fls(divisor) - 1) - pwm.variant.div_base; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); reg = readl(pwm.base + REG_TCFG1); reg &= ~(TCFG1_MUX_MASK << shift); reg |= bits << shift; writel(reg, pwm.base + REG_TCFG1); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static void samsung_time_stop(unsigned int channel) @@ -124,13 +124,13 @@ static void samsung_time_stop(unsigned int channel) if (channel > 0) ++channel; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl_relaxed(pwm.base + REG_TCON); tcon &= ~TCON_START(channel); writel_relaxed(tcon, pwm.base + REG_TCON); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static void samsung_time_setup(unsigned int channel, unsigned long tcnt) @@ -142,7 +142,7 @@ static void samsung_time_setup(unsigned int channel, unsigned long tcnt) if (tcon_chan > 0) ++tcon_chan; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl_relaxed(pwm.base + REG_TCON); @@ -153,7 +153,7 @@ static void samsung_time_setup(unsigned int channel, unsigned long tcnt) writel_relaxed(tcnt, pwm.base + REG_TCMPB(channel)); writel_relaxed(tcon, pwm.base + REG_TCON); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static void samsung_time_start(unsigned int channel, bool periodic) @@ -164,7 +164,7 @@ static void samsung_time_start(unsigned int channel, bool periodic) if (channel > 0) ++channel; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl_relaxed(pwm.base + REG_TCON); @@ -178,7 +178,7 @@ static void samsung_time_start(unsigned int channel, bool periodic) writel_relaxed(tcon, pwm.base + REG_TCON); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static int samsung_set_next_event(unsigned long cycles, diff --git a/drivers/clocksource/timer-armada-370-xp.c b/drivers/clocksource/timer-armada-370-xp.c index a405a084cf72d..b5a984aa1cbbb 100644 --- a/drivers/clocksource/timer-armada-370-xp.c +++ b/drivers/clocksource/timer-armada-370-xp.c @@ -349,7 +349,11 @@ static int __init armada_xp_timer_init(struct device_node *np) timer_clk = clk_get_rate(clk); - return armada_370_xp_timer_common_init(np); + ret = armada_370_xp_timer_common_init(np); + if (ret) + clk_disable_unprepare(clk); + + return ret; } TIMER_OF_DECLARE(armada_xp, "marvell,armada-xp-timer", armada_xp_timer_init); @@ -387,7 +391,11 @@ static int __init armada_375_timer_init(struct device_node *np) timer25Mhz = false; } - return armada_370_xp_timer_common_init(np); + ret = armada_370_xp_timer_common_init(np); + if (ret) + clk_disable_unprepare(clk); + + return ret; } TIMER_OF_DECLARE(armada_375, "marvell,armada-375-timer", armada_375_timer_init); @@ -410,7 +418,11 @@ static int __init armada_370_timer_init(struct device_node *np) timer_clk = clk_get_rate(clk) / TIMER_DIVIDER; timer25Mhz = false; - return armada_370_xp_timer_common_init(np); + ret = armada_370_xp_timer_common_init(np); + if (ret) + clk_disable_unprepare(clk); + + return ret; } TIMER_OF_DECLARE(armada_370, "marvell,armada-370-timer", armada_370_timer_init); diff --git a/drivers/clocksource/timer-nxp-pit.c b/drivers/clocksource/timer-nxp-pit.c index bc5157e2ba57c..2f70d1d5e21b2 100644 --- a/drivers/clocksource/timer-nxp-pit.c +++ b/drivers/clocksource/timer-nxp-pit.c @@ -328,8 +328,10 @@ static int pit_timer_init(struct device_node *np) if (pit_instances == max_pit_instances) { ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "PIT timer:starting", pit_clockevent_starting_cpu, NULL); - if (ret < 0) + if (ret < 0) { + pit_clockevent_per_cpu_exit(pit, pit_instances); goto out_pit_clocksource_unregister; + } } return 0; diff --git a/drivers/clocksource/timer-sun4i.c b/drivers/clocksource/timer-sun4i.c index 7bdcc60ad43cb..c2d04ab7cf2d3 100644 --- a/drivers/clocksource/timer-sun4i.c +++ b/drivers/clocksource/timer-sun4i.c @@ -208,7 +208,7 @@ static int __init sun4i_timer_init(struct device_node *node) sun4i_timer_clear_interrupt(timer_of_base(&to)); clockevents_config_and_register(&to.clkevt, timer_of_rate(&to), - TIMER_SYNC_TICKS, 0xffffffff); + TIMER_SYNC_TICKS + 1, 0xffffffff); /* Enable timer0 interrupt */ val = readl(timer_of_base(&to) + TIMER_IRQ_EN_REG); diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 735b29f76438a..2142838ad6cc4 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -560,6 +560,11 @@ static int amd_pstate_ut_check_freq_attrs(u32 index) static int __init amd_pstate_ut_init(void) { u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases); + enum amd_pstate_mode mode = amd_pstate_get_status(); + + /* don't test if no running amd-pstate driver */ + if (mode == AMD_PSTATE_UNDEFINED || mode == AMD_PSTATE_DISABLE) + return -EOPNOTSUPP; for (i = 0; i < arr_size; i++) { int ret; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 3a6b4b224a66d..80d99ba1902c9 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -199,7 +199,7 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) static DEFINE_MUTEX(amd_pstate_driver_lock); -static u8 msr_get_epp(struct amd_cpudata *cpudata) +static int msr_get_epp(struct amd_cpudata *cpudata) { u64 value; int ret; @@ -215,12 +215,12 @@ static u8 msr_get_epp(struct amd_cpudata *cpudata) DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); -static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) +static inline int amd_pstate_get_epp(struct amd_cpudata *cpudata) { return static_call(amd_pstate_get_epp)(cpudata); } -static u8 shmem_get_epp(struct amd_cpudata *cpudata) +static int shmem_get_epp(struct amd_cpudata *cpudata) { u64 epp; int ret; @@ -526,9 +526,6 @@ static int shmem_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->perf, perf); WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); - if (cppc_state == AMD_PSTATE_ACTIVE) - return 0; - ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel); if (ret) { pr_warn("failed to get auto_sel, ret: %d\n", ret); @@ -702,9 +699,12 @@ static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) WRITE_ONCE(cpudata->max_limit_freq, policy->max); if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { + u8 min_limit_perf = perf.bios_min_perf ?: perf.nominal_perf; + u32 min_limit_freq; + /* - * For performance policy, set MinPerf to nominal_perf rather than - * highest_perf or lowest_nonlinear_perf. + * For performance policy, set MinPerf to nominal_perf / bios_min_perf + * rather than highest_perf or lowest_nonlinear_perf. * * Per commit 0c411b39e4f4c, using highest_perf was observed * to cause frequency throttling on power-limited platforms, leading to @@ -712,11 +712,18 @@ static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) * performance too much for HPC workloads requiring high frequency * operation and minimal wakeup latency from idle states. * - * nominal_perf therefore provides a balance by avoiding throttling - * while still maintaining enough performance for HPC workloads. + * nominal_perf therefore provides a balanced default by avoiding + * throttling while still maintaining enough performance for HPC + * workloads when bios_min_perf is not available. + * + * When bios_min_perf is available, users have profiled their workloads + * to understand the best idling frequency. Use that instead. */ - perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); - WRITE_ONCE(cpudata->min_limit_freq, min(cpudata->nominal_freq, cpudata->max_limit_freq)); + min_limit_perf = min(min_limit_perf, perf.max_limit_perf); + min_limit_freq = perf_to_freq(perf, cpudata->nominal_freq, min_limit_perf); + perf.min_limit_perf = min_limit_perf; + + WRITE_ONCE(cpudata->min_limit_freq, min(min_limit_freq, cpudata->max_limit_freq)); } else { perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); WRITE_ONCE(cpudata->min_limit_freq, policy->min); @@ -1173,6 +1180,9 @@ static int amd_pstate_power_supply_notifier(struct notifier_block *nb, if (cpudata->current_profile != PLATFORM_PROFILE_BALANCED) return 0; + if (!policy) + return NOTIFY_OK; + epp = amd_pstate_get_balanced_epp(policy); ret = amd_pstate_set_epp(policy, epp); @@ -1208,6 +1218,9 @@ static int amd_pstate_profile_set(struct device *dev, struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); int ret; + if (!policy) + return -ENODEV; + switch (profile) { case PLATFORM_PROFILE_LOW_POWER: ret = amd_pstate_set_epp(policy, AMD_CPPC_EPP_POWERSAVE); @@ -1876,6 +1889,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) struct amd_cpudata *cpudata; union perf_cached perf; struct device *dev; + int default_epp; int ret; /* @@ -1924,6 +1938,13 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->boost_supported = READ_ONCE(cpudata->boost_supported); + /* Fetch the firmware programmed default EPP value */ + default_epp = amd_pstate_get_epp(cpudata); + if (default_epp < 0) { + ret = default_epp; + goto free_cpudata1; + } + /* * Set the policy to provide a valid fallback value in case * the default cpufreq governor is neither powersave nor performance. @@ -1931,7 +1952,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (amd_pstate_acpi_pm_profile_server() || amd_pstate_acpi_pm_profile_undefined()) { policy->policy = CPUFREQ_POLICY_PERFORMANCE; - cpudata->epp_default_ac = cpudata->epp_default_dc = amd_pstate_get_epp(cpudata); + cpudata->epp_default_ac = cpudata->epp_default_dc = default_epp; cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE; } else { policy->policy = CPUFREQ_POLICY_POWERSAVE; diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 638e5bf72185a..3f64f266e695b 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -249,21 +249,19 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) return -ENODEV; } - ret = dev_pm_opp_of_add_table(cpu_dev); - if (ret < 0) { - dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); - return ret; - } + priv = kzalloc_obj(*priv); + if (!priv) + return -ENOMEM; ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); if (ret) { dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); - return ret; + goto out_free_priv; } - ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); - if (ret) { - dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret); + ret = dev_pm_opp_of_cpumask_add_table(policy->cpus); + if (ret < 0) { + dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); goto out_iounmap; } @@ -271,19 +269,13 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) if (ret <= 0) { dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); ret = -EPROBE_DEFER; - goto out_free_opp; - } - - priv = kzalloc_obj(*priv); - if (!priv) { - ret = -ENOMEM; - goto out_free_opp; + goto out_free_table; } ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); if (ret) { dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); - goto out_free_priv; + goto out_free_table; } /* Get OPP levels (p-state indexes) and stash them in driver_data */ @@ -318,12 +310,12 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) out_free_cpufreq_table: dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); -out_free_priv: - kfree(priv); -out_free_opp: - dev_pm_opp_remove_all_dynamic(cpu_dev); +out_free_table: + dev_pm_opp_of_cpumask_remove_table(policy->cpus); out_iounmap: iounmap(reg_base); +out_free_priv: + kfree(priv); return ret; } @@ -332,7 +324,7 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) struct apple_cpu_priv *priv = policy->driver_data; dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); - dev_pm_opp_remove_all_dynamic(priv->cpu_dev); + dev_pm_opp_of_cpumask_remove_table(policy->cpus); iounmap(priv->reg_base); kfree(priv); } diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 6fe0e972952af..80893844353ca 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -836,7 +836,7 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) * value first as some platforms may update the actual delivered perf * there; if failed, resort to the cached desired perf. */ - if (cppc_get_desired_perf(cpu, &delivered_perf)) + if (cppc_get_desired_perf(cpu, &delivered_perf) || !delivered_perf) delivered_perf = cpu_data->perf_ctrls.desired_perf; return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index b898b6544069f..196c50f26fe0d 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1249,7 +1249,7 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) if (!policy) return NULL; - if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) + if (!zalloc_cpumask_var(&policy->cpus, GFP_KERNEL)) goto err_free_policy; if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) @@ -1258,6 +1258,8 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL)) goto err_free_rcpumask; + init_rwsem(&policy->rwsem); + init_completion(&policy->kobj_unregister); ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, cpufreq_global_kobject, "policy%u", cpu); @@ -1272,8 +1274,6 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) goto err_free_real_cpus; } - init_rwsem(&policy->rwsem); - freq_constraints_init(&policy->constraints); policy->nb_min.notifier_call = cpufreq_notifier_min; diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index e93697d3edfd9..731f5721ff1ed 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -55,7 +55,6 @@ static unsigned int max_freq; static unsigned int transition_latency; static u32 *imx6_soc_volt; -static u32 soc_opp_count; static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index) { @@ -330,6 +329,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) const struct property *prop; const __be32 *val; u32 nr, i, j; + u32 soc_opp_count = 0; cpu_dev = get_cpu_device(0); if (!cpu_dev) { @@ -400,7 +400,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) } /* Make imx6_soc_volt array's size same as arm opp number */ - imx6_soc_volt = devm_kcalloc(cpu_dev, num, sizeof(*imx6_soc_volt), + imx6_soc_volt = devm_kcalloc(&pdev->dev, num, sizeof(*imx6_soc_volt), GFP_KERNEL); if (imx6_soc_volt == NULL) { ret = -ENOMEM; @@ -485,6 +485,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) return 0; free_freq_table: + imx6_soc_volt = NULL; dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); out_free_opp: dev_pm_opp_of_remove_table(cpu_dev); @@ -506,6 +507,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) static void imx6q_cpufreq_remove(struct platform_device *pdev) { cpufreq_unregister_driver(&imx6q_cpufreq_driver); + imx6_soc_volt = NULL; dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); dev_pm_opp_of_remove_table(cpu_dev); regulator_put(arm_reg); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 6e984c114d96f..311087197d7fa 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2356,8 +2356,6 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) if (pstate_funcs.get_vid) pstate_funcs.get_vid(cpu); - - intel_pstate_set_min_pstate(cpu); } /* @@ -3063,6 +3061,7 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) static int intel_pstate_cpu_init(struct cpufreq_policy *policy) { int ret = __intel_pstate_cpu_init(policy); + struct cpudata *cpu; if (ret) return ret; @@ -3073,11 +3072,11 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy) */ policy->policy = CPUFREQ_POLICY_POWERSAVE; - if (hwp_active) { - struct cpudata *cpu = all_cpu_data[policy->cpu]; - + cpu = all_cpu_data[policy->cpu]; + if (hwp_active) cpu->epp_cached = intel_pstate_get_epp(cpu, 0); - } + else + intel_pstate_set_min_pstate(cpu); return 0; } @@ -3301,8 +3300,6 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) return ret; policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; - /* This reflects the intel_pstate_get_cpu_pstates() setting. */ - policy->cur = policy->cpuinfo.min_freq; req = kzalloc_objs(*req, 2); if (!req) { @@ -3323,9 +3320,15 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) WRITE_ONCE(cpu->hwp_req_cached, value); cpu->epp_cached = intel_pstate_get_epp(cpu, value); + + intel_cpufreq_hwp_update(cpu, cpu->pstate.min_pstate, + cpu->pstate.max_pstate, + cpu->pstate.min_pstate, false); } else { policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY; + intel_pstate_set_min_pstate(cpu); } + policy->cur = policy->cpuinfo.min_freq; freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.min_perf_pct, 100); diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c index 81a0780b2ebf0..ffe5eda82f0b1 100644 --- a/drivers/cpufreq/spear-cpufreq.c +++ b/drivers/cpufreq/spear-cpufreq.c @@ -79,9 +79,9 @@ static int spear1340_set_cpu_rate(struct clk *sys_pclk, unsigned long newfreq) int ret = 0; sys_clk = clk_get_parent(spear_cpufreq.clk); - if (IS_ERR(sys_clk)) { + if (!sys_clk) { pr_err("failed to get cpu's parent (sys) clock\n"); - return PTR_ERR(sys_clk); + return -EINVAL; } /* Set the rate of the source clock before changing the parent */ diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c index dcf20ea5ef5e3..b250d0dde7607 100644 --- a/drivers/cpuidle/cpuidle-psci.c +++ b/drivers/cpuidle/cpuidle-psci.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include @@ -428,14 +428,14 @@ static int psci_idle_init_cpu(struct device *dev, int cpu) * to register cpuidle driver then rollback to cancel all CPUs * registration. */ -static int psci_cpuidle_probe(struct faux_device *fdev) +static int psci_cpuidle_probe(struct platform_device *pdev) { int cpu, ret; struct cpuidle_driver *drv; struct cpuidle_device *dev; for_each_present_cpu(cpu) { - ret = psci_idle_init_cpu(&fdev->dev, cpu); + ret = psci_idle_init_cpu(&pdev->dev, cpu); if (ret) goto out_fail; } @@ -455,36 +455,26 @@ static int psci_cpuidle_probe(struct faux_device *fdev) return ret; } -static struct faux_device_ops psci_cpuidle_ops = { +static struct platform_driver psci_cpuidle_driver = { .probe = psci_cpuidle_probe, + .driver = { + .name = "psci-cpuidle", + }, }; -static bool __init dt_idle_state_present(void) -{ - struct device_node *cpu_node __free(device_node) = - of_cpu_device_node_get(cpumask_first(cpu_possible_mask)); - if (!cpu_node) - return false; - - struct device_node *state_node __free(device_node) = - of_get_cpu_state_node(cpu_node, 0); - if (!state_node) - return false; - - return !!of_match_node(psci_idle_state_match, state_node); -} - static int __init psci_idle_init(void) { - struct faux_device *fdev; + struct platform_device *pdev; + int ret; - if (!dt_idle_state_present()) - return 0; + ret = platform_driver_register(&psci_cpuidle_driver); + if (ret) + return ret; - fdev = faux_device_create("psci-cpuidle", NULL, &psci_cpuidle_ops); - if (!fdev) { - pr_err("Failed to create psci-cpuidle device\n"); - return -ENODEV; + pdev = platform_device_register_simple("psci-cpuidle", -1, NULL, 0); + if (IS_ERR(pdev)) { + platform_driver_unregister(&psci_cpuidle_driver); + return PTR_ERR(pdev); } return 0; diff --git a/drivers/cpuidle/cpuidle-ux500.c b/drivers/cpuidle/cpuidle-ux500.c index f7d778580e9be..6d6c52c0bcc2d 100644 --- a/drivers/cpuidle/cpuidle-ux500.c +++ b/drivers/cpuidle/cpuidle-ux500.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include @@ -66,7 +66,7 @@ static inline int ux500_enter_idle(struct cpuidle_device *dev, /* Go to the retention state, the prcmu will wait for the * cpu to go WFI and this is what happens after exiting this * 'master' critical section */ - if (prcmu_set_power_state(PRCMU_AP_IDLE, true, true)) + if (db8500_prcmu_set_power_state(PRCMU_AP_IDLE, true, true)) goto out; /* When we switch to retention, the prcmu is in charge @@ -109,7 +109,7 @@ static struct cpuidle_driver ux500_idle_driver = { static int dbx500_cpuidle_probe(struct platform_device *pdev) { /* Configure wake up reasons */ - prcmu_enable_wakeups(PRCMU_WAKEUP(ARM) | PRCMU_WAKEUP(RTC) | + db8500_prcmu_enable_wakeups(PRCMU_WAKEUP(ARM) | PRCMU_WAKEUP(RTC) | PRCMU_WAKEUP(ABB)); return cpuidle_register(&ux500_idle_driver, NULL); diff --git a/drivers/cpuidle/dt_idle_genpd.c b/drivers/cpuidle/dt_idle_genpd.c index d292975cc4681..ed41a90eeeb71 100644 --- a/drivers/cpuidle/dt_idle_genpd.c +++ b/drivers/cpuidle/dt_idle_genpd.c @@ -99,7 +99,7 @@ struct generic_pm_domain *dt_idle_pd_alloc(struct device_node *np, if (!pd) goto out; - pd->name = kasprintf(GFP_KERNEL, "%pOF", np); + pd->name = kstrdup(kbasename(of_node_full_name(np)), GFP_KERNEL); if (!pd->name) goto free_pd; @@ -112,7 +112,6 @@ struct generic_pm_domain *dt_idle_pd_alloc(struct device_node *np, goto free_name; pd->free_states = pd_free_states; - pd->name = kbasename(pd->name); pd->states = states; pd->state_count = state_count; diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 216a00bad5d70..eb834d15d6149 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -641,7 +641,6 @@ config CRYPTO_DEV_QCOM_RNG tristate "Qualcomm Random Number Generator Driver" depends on ARCH_QCOM || COMPILE_TEST depends on HW_RANDOM - select CRYPTO_RNG help This driver provides support for the Random Number Generator hardware found on Qualcomm SoCs. diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 06ea0e9fe6f22..d86ae005fbe2e 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -70,14 +70,6 @@ config CRYPTO_DEV_SUN8I_CE_HASH help Say y to enable support for hash algorithms. -config CRYPTO_DEV_SUN8I_CE_PRNG - bool "Support for Allwinner Crypto Engine PRNG" - depends on CRYPTO_DEV_SUN8I_CE - select CRYPTO_RNG - help - Select this option if you want to provide kernel-side support for - the Pseudo-Random Number Generator found in the Crypto Engine. - config CRYPTO_DEV_SUN8I_CE_TRNG bool "Support for Allwinner Crypto Engine TRNG" depends on CRYPTO_DEV_SUN8I_CE @@ -113,14 +105,6 @@ config CRYPTO_DEV_SUN8I_SS_DEBUG This will create /sys/kernel/debug/sun8i-ss/stats for displaying the number of requests per flow and per algorithm. -config CRYPTO_DEV_SUN8I_SS_PRNG - bool "Support for Allwinner Security System PRNG" - depends on CRYPTO_DEV_SUN8I_SS - select CRYPTO_RNG - help - Select this option if you want to provide kernel-side support for - the Pseudo-Random Number Generator found in the Security System. - config CRYPTO_DEV_SUN8I_SS_HASH bool "Enable support for hash on sun8i-ss" depends on CRYPTO_DEV_SUN8I_SS diff --git a/drivers/crypto/allwinner/sun8i-ce/Makefile b/drivers/crypto/allwinner/sun8i-ce/Makefile index 0842eb2d9408d..ea708b427e2e4 100644 --- a/drivers/crypto/allwinner/sun8i-ce/Makefile +++ b/drivers/crypto/allwinner/sun8i-ce/Makefile @@ -1,5 +1,4 @@ obj-$(CONFIG_CRYPTO_DEV_SUN8I_CE) += sun8i-ce.o sun8i-ce-y += sun8i-ce-core.o sun8i-ce-cipher.o sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_HASH) += sun8i-ce-hash.o -sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG) += sun8i-ce-prng.o sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) += sun8i-ce-trng.o diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index f3b58ed6aed0d..c6402e87f8a07 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include @@ -49,7 +48,6 @@ static const struct ce_variant ce_h3_variant = { { "mod", 50000000, 0 }, }, .esr = ESR_H3, - .prng = CE_ALG_PRNG, .trng = CE_ID_NOTSUPP, }; @@ -66,7 +64,6 @@ static const struct ce_variant ce_h5_variant = { { "mod", 300000000, 0 }, }, .esr = ESR_H5, - .prng = CE_ALG_PRNG, .trng = CE_ID_NOTSUPP, }; @@ -80,7 +77,6 @@ static const struct ce_variant ce_h6_variant = { }, .cipher_t_dlen_in_bytes = true, .hash_t_dlen_in_bits = true, - .prng_t_dlen_in_bytes = true, .trng_t_dlen_in_bytes = true, .ce_clks = { { "bus", 0, 200000000 }, @@ -88,7 +84,6 @@ static const struct ce_variant ce_h6_variant = { { "ram", 0, 400000000 }, }, .esr = ESR_H6, - .prng = CE_ALG_PRNG_V2, .trng = CE_ALG_TRNG_V2, }; @@ -102,7 +97,6 @@ static const struct ce_variant ce_h616_variant = { }, .cipher_t_dlen_in_bytes = true, .hash_t_dlen_in_bits = true, - .prng_t_dlen_in_bytes = true, .trng_t_dlen_in_bytes = true, .needs_word_addresses = true, .ce_clks = { @@ -112,7 +106,6 @@ static const struct ce_variant ce_h616_variant = { { "trng", 0, 0 }, }, .esr = ESR_H6, - .prng = CE_ALG_PRNG_V2, .trng = CE_ALG_TRNG_V2, }; @@ -129,7 +122,6 @@ static const struct ce_variant ce_a64_variant = { { "mod", 300000000, 0 }, }, .esr = ESR_A64, - .prng = CE_ALG_PRNG, .trng = CE_ID_NOTSUPP, }; @@ -148,7 +140,6 @@ static const struct ce_variant ce_d1_variant = { { "trng", 0, 0 }, }, .esr = ESR_D1, - .prng = CE_ALG_PRNG, .trng = CE_ALG_TRNG, }; @@ -165,7 +156,6 @@ static const struct ce_variant ce_r40_variant = { { "mod", 300000000, 0 }, }, .esr = ESR_R40, - .prng = CE_ALG_PRNG, .trng = CE_ID_NOTSUPP, }; @@ -614,25 +604,6 @@ static struct sun8i_ce_alg_template ce_algs[] = { }, }, #endif -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG -{ - .type = CRYPTO_ALG_TYPE_RNG, - .alg.rng = { - .base = { - .cra_name = "stdrng", - .cra_driver_name = "sun8i-ce-prng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct sun8i_ce_rng_tfm_ctx), - .cra_module = THIS_MODULE, - .cra_init = sun8i_ce_prng_init, - .cra_exit = sun8i_ce_prng_exit, - }, - .generate = sun8i_ce_prng_generate, - .seed = sun8i_ce_prng_seed, - .seedsize = PRNG_SEED_SIZE, - } -}, -#endif }; static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) @@ -693,14 +664,6 @@ static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) seq_printf(seq, "\tFallback due to SG numbers: %lu\n", ce_algs[i].stat_fb_maxsg); break; -#endif -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG - case CRYPTO_ALG_TYPE_RNG: - seq_printf(seq, "%s %s reqs=%lu bytes=%lu\n", - ce_algs[i].alg.rng.base.cra_driver_name, - ce_algs[i].alg.rng.base.cra_name, - ce_algs[i].stat_req, ce_algs[i].stat_bytes); - break; #endif } } @@ -930,25 +893,6 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) return err; } break; -#endif -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG - case CRYPTO_ALG_TYPE_RNG: - if (ce->variant->prng == CE_ID_NOTSUPP) { - dev_info(ce->dev, - "DEBUG: Algo of %s not supported\n", - ce_algs[i].alg.rng.base.cra_name); - ce_algs[i].ce = NULL; - break; - } - dev_info(ce->dev, "Register %s\n", - ce_algs[i].alg.rng.base.cra_name); - err = crypto_register_rng(&ce_algs[i].alg.rng); - if (err) { - dev_err(ce->dev, "Fail to register %s\n", - ce_algs[i].alg.rng.base.cra_name); - ce_algs[i].ce = NULL; - } - break; #endif default: ce_algs[i].ce = NULL; @@ -977,13 +921,6 @@ static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce) ce_algs[i].alg.hash.base.halg.base.cra_name); crypto_engine_unregister_ahash(&ce_algs[i].alg.hash); break; -#endif -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG - case CRYPTO_ALG_TYPE_RNG: - dev_info(ce->dev, "Unregister %d %s\n", i, - ce_algs[i].alg.rng.base.cra_name); - crypto_unregister_rng(&ce_algs[i].alg.rng); - break; #endif } } diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c deleted file mode 100644 index d0a1ac66738bf..0000000000000 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c +++ /dev/null @@ -1,159 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * sun8i-ce-prng.c - hardware cryptographic offloader for - * Allwinner H3/A64/H5/H2+/H6/R40 SoC - * - * Copyright (C) 2015-2020 Corentin Labbe - * - * This file handle the PRNG - * - * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst - */ -#include "sun8i-ce.h" -#include -#include -#include - -int sun8i_ce_prng_init(struct crypto_tfm *tfm) -{ - struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); - - memset(ctx, 0, sizeof(struct sun8i_ce_rng_tfm_ctx)); - return 0; -} - -void sun8i_ce_prng_exit(struct crypto_tfm *tfm) -{ - struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); - - kfree_sensitive(ctx->seed); - ctx->seed = NULL; - ctx->slen = 0; -} - -int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); - - if (ctx->seed && ctx->slen != slen) { - kfree_sensitive(ctx->seed); - ctx->slen = 0; - ctx->seed = NULL; - } - if (!ctx->seed) - ctx->seed = kmalloc(slen, GFP_KERNEL | GFP_DMA); - if (!ctx->seed) - return -ENOMEM; - - memcpy(ctx->seed, seed, slen); - ctx->slen = slen; - - return 0; -} - -int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen) -{ - struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); - struct rng_alg *alg = crypto_rng_alg(tfm); - struct sun8i_ce_alg_template *algt; - struct sun8i_ce_dev *ce; - dma_addr_t dma_iv, dma_dst; - int err = 0; - int flow = 3; - unsigned int todo; - struct sun8i_ce_flow *chan; - struct ce_task *cet; - u32 common, sym; - void *d; - - algt = container_of(alg, struct sun8i_ce_alg_template, alg.rng); - ce = algt->ce; - - if (ctx->slen == 0) { - dev_err(ce->dev, "not seeded\n"); - return -EINVAL; - } - - /* we want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE */ - todo = dlen + ctx->slen + PRNG_DATA_SIZE * 2; - todo -= todo % PRNG_DATA_SIZE; - - d = kzalloc(todo, GFP_KERNEL | GFP_DMA); - if (!d) { - err = -ENOMEM; - goto err_mem; - } - - dev_dbg(ce->dev, "%s PRNG slen=%u dlen=%u todo=%u multi=%u\n", __func__, - slen, dlen, todo, todo / PRNG_DATA_SIZE); - -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_req++; - algt->stat_bytes += todo; -#endif - - dma_iv = dma_map_single(ce->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE); - if (dma_mapping_error(ce->dev, dma_iv)) { - dev_err(ce->dev, "Cannot DMA MAP IV\n"); - err = -EFAULT; - goto err_iv; - } - - dma_dst = dma_map_single(ce->dev, d, todo, DMA_FROM_DEVICE); - if (dma_mapping_error(ce->dev, dma_dst)) { - dev_err(ce->dev, "Cannot DMA MAP DST\n"); - err = -EFAULT; - goto err_dst; - } - - err = pm_runtime_resume_and_get(ce->dev); - if (err < 0) - goto err_pm; - - mutex_lock(&ce->rnglock); - chan = &ce->chanlist[flow]; - - cet = &chan->tl[0]; - memset(cet, 0, sizeof(struct ce_task)); - - cet->t_id = cpu_to_le32(flow); - common = ce->variant->prng | CE_COMM_INT; - cet->t_common_ctl = cpu_to_le32(common); - - /* recent CE (H6) need length in bytes, in word otherwise */ - if (ce->variant->prng_t_dlen_in_bytes) - cet->t_dlen = cpu_to_le32(todo); - else - cet->t_dlen = cpu_to_le32(todo / 4); - - sym = PRNG_LD; - cet->t_sym_ctl = cpu_to_le32(sym); - cet->t_asym_ctl = 0; - - cet->t_key = desc_addr_val_le32(ce, dma_iv); - cet->t_iv = desc_addr_val_le32(ce, dma_iv); - - cet->t_dst[0].addr = desc_addr_val_le32(ce, dma_dst); - cet->t_dst[0].len = cpu_to_le32(todo / 4); - - err = sun8i_ce_run_task(ce, 3, "PRNG"); - mutex_unlock(&ce->rnglock); - - pm_runtime_put(ce->dev); - -err_pm: - dma_unmap_single(ce->dev, dma_dst, todo, DMA_FROM_DEVICE); -err_dst: - dma_unmap_single(ce->dev, dma_iv, ctx->slen, DMA_TO_DEVICE); - - if (!err) { - memcpy(dst, d, dlen); - memcpy(ctx->seed, d + dlen, ctx->slen); - } -err_iv: - kfree_sensitive(d); -err_mem: - return err; -} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 71f5a0cd3d458..468d99bf5bf66 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -58,9 +57,7 @@ #define CE_ALG_SHA384 20 #define CE_ALG_SHA512 21 #define CE_ALG_TRNG 48 -#define CE_ALG_PRNG 49 #define CE_ALG_TRNG_V2 0x1c -#define CE_ALG_PRNG_V2 0x1d /* Used in ce_variant */ #define CE_ID_NOTSUPP 0xFF @@ -96,10 +93,6 @@ #define ESR_H6 4 #define ESR_D1 5 -#define PRNG_DATA_SIZE (160 / 8) -#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8) -#define PRNG_LD BIT(17) - #define CE_DIE_ID_SHIFT 16 #define CE_DIE_ID_MASK 0x07 @@ -136,13 +129,10 @@ struct ce_clock { * bytes or words * @hash_t_dlen_in_bytes: Does the request size for hash is in * bits or words - * @prng_t_dlen_in_bytes: Does the request size for PRNG is in - * bytes or words * @trng_t_dlen_in_bytes: Does the request size for TRNG is in * bytes or words * @ce_clks: list of clocks needed by this variant * @esr: The type of error register - * @prng: The CE_ALG_XXX value for the PRNG * @trng: The CE_ALG_XXX value for the TRNG */ struct ce_variant { @@ -151,12 +141,10 @@ struct ce_variant { u32 op_mode[CE_ID_OP_MAX]; bool cipher_t_dlen_in_bytes; bool hash_t_dlen_in_bits; - bool prng_t_dlen_in_bytes; bool trng_t_dlen_in_bytes; bool needs_word_addresses; struct ce_clock ce_clks[CE_MAX_CLOCKS]; int esr; - unsigned char prng; unsigned char trng; }; @@ -327,16 +315,6 @@ struct sun8i_ce_hash_reqctx { struct ahash_request fallback_req; // keep at the end }; -/* - * struct sun8i_ce_prng_ctx - context for PRNG TFM - * @seed: The seed to use - * @slen: The size of the seed - */ -struct sun8i_ce_rng_tfm_ctx { - void *seed; - unsigned int slen; -}; - /* * struct sun8i_ce_alg_template - crypto_alg template * @type: the CRYPTO_ALG_TYPE for this template @@ -357,7 +335,6 @@ struct sun8i_ce_alg_template { union { struct skcipher_engine_alg skcipher; struct ahash_engine_alg hash; - struct rng_alg rng; } alg; unsigned long stat_req; unsigned long stat_fb; @@ -398,11 +375,5 @@ int sun8i_ce_hash_finup(struct ahash_request *areq); int sun8i_ce_hash_digest(struct ahash_request *areq); int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq); -int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen); -int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); -void sun8i_ce_prng_exit(struct crypto_tfm *tfm); -int sun8i_ce_prng_init(struct crypto_tfm *tfm); - int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce); void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce); diff --git a/drivers/crypto/allwinner/sun8i-ss/Makefile b/drivers/crypto/allwinner/sun8i-ss/Makefile index aabfd893c8174..2d6458a42e583 100644 --- a/drivers/crypto/allwinner/sun8i-ss/Makefile +++ b/drivers/crypto/allwinner/sun8i-ss/Makefile @@ -1,4 +1,3 @@ obj-$(CONFIG_CRYPTO_DEV_SUN8I_SS) += sun8i-ss.o sun8i-ss-y += sun8i-ss-core.o sun8i-ss-cipher.o -sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG) += sun8i-ss-prng.o sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_HASH) += sun8i-ss-hash.o diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c index 59c9bc45ec0f2..0b22fcddb882c 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c @@ -11,7 +11,6 @@ */ #include -#include #include #include #include @@ -283,25 +282,6 @@ static struct sun8i_ss_alg_template ss_algs[] = { .do_one_request = sun8i_ss_handle_cipher_request, }, }, -#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG -{ - .type = CRYPTO_ALG_TYPE_RNG, - .alg.rng = { - .base = { - .cra_name = "stdrng", - .cra_driver_name = "sun8i-ss-prng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct sun8i_ss_rng_tfm_ctx), - .cra_module = THIS_MODULE, - .cra_init = sun8i_ss_prng_init, - .cra_exit = sun8i_ss_prng_exit, - }, - .generate = sun8i_ss_prng_generate, - .seed = sun8i_ss_prng_seed, - .seedsize = PRNG_SEED_SIZE, - } -}, -#endif #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH { .type = CRYPTO_ALG_TYPE_AHASH, .ss_algo_id = SS_ID_HASH_MD5, @@ -501,14 +481,6 @@ static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v) seq_printf(seq, "\tFallback due to SG numbers: %lu\n", ss_algs[i].stat_fb_sgnum); break; -#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG - case CRYPTO_ALG_TYPE_RNG: - seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n", - ss_algs[i].alg.rng.base.cra_driver_name, - ss_algs[i].alg.rng.base.cra_name, - ss_algs[i].stat_req, ss_algs[i].stat_bytes); - break; -#endif #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH case CRYPTO_ALG_TYPE_AHASH: seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", @@ -711,16 +683,6 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) return err; } break; -#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG - case CRYPTO_ALG_TYPE_RNG: - err = crypto_register_rng(&ss_algs[i].alg.rng); - if (err) { - dev_err(ss->dev, "Fail to register %s\n", - ss_algs[i].alg.rng.base.cra_name); - ss_algs[i].ss = NULL; - } - break; -#endif #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH case CRYPTO_ALG_TYPE_AHASH: id = ss_algs[i].ss_algo_id; @@ -764,13 +726,6 @@ static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss) ss_algs[i].alg.skcipher.base.base.cra_name); crypto_engine_unregister_skcipher(&ss_algs[i].alg.skcipher); break; -#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG - case CRYPTO_ALG_TYPE_RNG: - dev_info(ss->dev, "Unregister %d %s\n", i, - ss_algs[i].alg.rng.base.cra_name); - crypto_unregister_rng(&ss_algs[i].alg.rng); - break; -#endif #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH case CRYPTO_ALG_TYPE_AHASH: dev_info(ss->dev, "Unregister %d %s\n", i, diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c deleted file mode 100644 index a923cfc6553f4..0000000000000 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * sun8i-ss-prng.c - hardware cryptographic offloader for - * Allwinner A80/A83T SoC - * - * Copyright (C) 2015-2020 Corentin Labbe - * - * This file handle the PRNG found in the SS - * - * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst - */ -#include "sun8i-ss.h" -#include -#include -#include -#include -#include - -int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); - - if (ctx->seed && ctx->slen != slen) { - kfree_sensitive(ctx->seed); - ctx->slen = 0; - ctx->seed = NULL; - } - if (!ctx->seed) - ctx->seed = kmalloc(slen, GFP_KERNEL); - if (!ctx->seed) - return -ENOMEM; - - memcpy(ctx->seed, seed, slen); - ctx->slen = slen; - - return 0; -} - -int sun8i_ss_prng_init(struct crypto_tfm *tfm) -{ - struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); - - memset(ctx, 0, sizeof(struct sun8i_ss_rng_tfm_ctx)); - return 0; -} - -void sun8i_ss_prng_exit(struct crypto_tfm *tfm) -{ - struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); - - kfree_sensitive(ctx->seed); - ctx->seed = NULL; - ctx->slen = 0; -} - -int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen) -{ - struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); - struct rng_alg *alg = crypto_rng_alg(tfm); - struct sun8i_ss_alg_template *algt; - unsigned int todo_with_padding; - struct sun8i_ss_dev *ss; - dma_addr_t dma_iv, dma_dst; - unsigned int todo; - int err = 0; - int flow; - void *d; - u32 v; - - algt = container_of(alg, struct sun8i_ss_alg_template, alg.rng); - ss = algt->ss; - - if (ctx->slen == 0) { - dev_err(ss->dev, "The PRNG is not seeded\n"); - return -EINVAL; - } - - /* The SS does not give an updated seed, so we need to get a new one. - * So we will ask for an extra PRNG_SEED_SIZE data. - * We want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE - */ - todo = dlen + PRNG_SEED_SIZE + PRNG_DATA_SIZE; - todo -= todo % PRNG_DATA_SIZE; - - todo_with_padding = ALIGN(todo, dma_get_cache_alignment()); - if (todo_with_padding < todo || todo < dlen) - return -EOVERFLOW; - - d = kzalloc(todo_with_padding, GFP_KERNEL); - if (!d) - return -ENOMEM; - - flow = sun8i_ss_get_engine_number(ss); - -#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG - algt->stat_req++; - algt->stat_bytes += todo; -#endif - - v = SS_ALG_PRNG | SS_PRNG_CONTINUE | SS_START; - if (flow) - v |= SS_FLOW1; - else - v |= SS_FLOW0; - - dma_iv = dma_map_single(ss->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE); - if (dma_mapping_error(ss->dev, dma_iv)) { - dev_err(ss->dev, "Cannot DMA MAP IV\n"); - err = -EFAULT; - goto err_free; - } - - dma_dst = dma_map_single(ss->dev, d, todo, DMA_FROM_DEVICE); - if (dma_mapping_error(ss->dev, dma_dst)) { - dev_err(ss->dev, "Cannot DMA MAP DST\n"); - err = -EFAULT; - goto err_iv; - } - - err = pm_runtime_resume_and_get(ss->dev); - if (err < 0) - goto err_pm; - err = 0; - - mutex_lock(&ss->mlock); - writel(dma_iv, ss->base + SS_IV_ADR_REG); - /* the PRNG act badly (failing rngtest) without SS_KEY_ADR_REG set */ - writel(dma_iv, ss->base + SS_KEY_ADR_REG); - writel(dma_dst, ss->base + SS_DST_ADR_REG); - writel(todo / 4, ss->base + SS_LEN_ADR_REG); - - reinit_completion(&ss->flows[flow].complete); - ss->flows[flow].status = 0; - /* Be sure all data is written before enabling the task */ - wmb(); - - writel(v, ss->base + SS_CTL_REG); - - wait_for_completion_interruptible_timeout(&ss->flows[flow].complete, - msecs_to_jiffies(todo)); - if (ss->flows[flow].status == 0) { - dev_err(ss->dev, "DMA timeout for PRNG (size=%u)\n", todo); - err = -EFAULT; - } - /* Since cipher and hash use the linux/cryptoengine and that we have - * a cryptoengine per flow, we are sure that they will issue only one - * request per flow. - * Since the cryptoengine wait for completion before submitting a new - * one, the mlock could be left just after the final writel. - * But cryptoengine cannot handle crypto_rng, so we need to be sure - * nothing will use our flow. - * The easiest way is to grab mlock until the hardware end our requests. - * We could have used a per flow lock, but this would increase - * complexity. - * The drawback is that no request could be handled for the other flow. - */ - mutex_unlock(&ss->mlock); - - pm_runtime_put(ss->dev); - -err_pm: - dma_unmap_single(ss->dev, dma_dst, todo, DMA_FROM_DEVICE); -err_iv: - dma_unmap_single(ss->dev, dma_iv, ctx->slen, DMA_TO_DEVICE); - - if (!err) { - memcpy(dst, d, dlen); - /* Update seed */ - memcpy(ctx->seed, d + dlen, ctx->slen); - } -err_free: - kfree_sensitive(d); - - return err; -} diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h index 3fc86225edaf3..289fb22abfa27 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -27,7 +26,6 @@ #define SS_ALG_DES (1 << 2) #define SS_ALG_3DES (2 << 2) #define SS_ALG_MD5 (3 << 2) -#define SS_ALG_PRNG (4 << 2) #define SS_ALG_SHA1 (6 << 2) #define SS_ALG_SHA224 (7 << 2) #define SS_ALG_SHA256 (8 << 2) @@ -68,8 +66,6 @@ #define SS_FLOW0 BIT(30) #define SS_FLOW1 BIT(31) -#define SS_PRNG_CONTINUE BIT(18) - #define MAX_SG 8 #define MAXFLOW 2 @@ -79,9 +75,6 @@ #define SS_DIE_ID_SHIFT 20 #define SS_DIE_ID_MASK 0x07 -#define PRNG_DATA_SIZE (160 / 8) -#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8) - #define MAX_PAD_SIZE 4096 /* @@ -213,16 +206,6 @@ struct sun8i_cipher_tfm_ctx { struct crypto_skcipher *fallback_tfm; }; -/* - * struct sun8i_ss_prng_ctx - context for PRNG TFM - * @seed: The seed to use - * @slen: The size of the seed - */ -struct sun8i_ss_rng_tfm_ctx { - void *seed; - unsigned int slen; -}; - /* * struct sun8i_ss_hash_tfm_ctx - context for an ahash TFM * @fallback_tfm: pointer to the fallback TFM @@ -274,7 +257,6 @@ struct sun8i_ss_alg_template { struct sun8i_ss_dev *ss; union { struct skcipher_engine_alg skcipher; - struct rng_alg rng; struct ahash_engine_alg hash; } alg; unsigned long stat_req; @@ -300,11 +282,6 @@ int sun8i_ss_skencrypt(struct skcipher_request *areq); int sun8i_ss_get_engine_number(struct sun8i_ss_dev *ss); int sun8i_ss_run_task(struct sun8i_ss_dev *ss, struct sun8i_cipher_req_ctx *rctx, const char *name); -int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen); -int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); -int sun8i_ss_prng_init(struct crypto_tfm *tfm); -void sun8i_ss_prng_exit(struct crypto_tfm *tfm); int sun8i_ss_hash_init_tfm(struct crypto_ahash *tfm); void sun8i_ss_hash_exit_tfm(struct crypto_ahash *tfm); diff --git a/drivers/crypto/aspeed/aspeed-acry.c b/drivers/crypto/aspeed/aspeed-acry.c index 5993bcba97163..301612556a769 100644 --- a/drivers/crypto/aspeed/aspeed-acry.c +++ b/drivers/crypto/aspeed/aspeed-acry.c @@ -728,7 +728,7 @@ static int aspeed_acry_probe(struct platform_device *pdev) /* Get irq number and register it */ acry_dev->irq = platform_get_irq(pdev, 0); if (acry_dev->irq < 0) - return -ENXIO; + return acry_dev->irq; rc = devm_request_irq(dev, acry_dev->irq, aspeed_acry_irq, 0, dev_name(dev), acry_dev); diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c index 3fe644bfe0373..1f9afa002ae8c 100644 --- a/drivers/crypto/aspeed/aspeed-hace.c +++ b/drivers/crypto/aspeed/aspeed-hace.c @@ -127,7 +127,7 @@ static int aspeed_hace_probe(struct platform_device *pdev) /* Get irq number and register it */ hace_dev->irq = platform_get_irq(pdev, 0); if (hace_dev->irq < 0) - return -ENXIO; + return hace_dev->irq; rc = devm_request_irq(&pdev->dev, hace_dev->irq, aspeed_hace_irq, 0, dev_name(&pdev->dev), hace_dev); diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index 9da9dd6585dfc..bd1664ad5c0ea 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -27,15 +27,14 @@ static struct atmel_ecc_driver_data driver_data; /** * struct atmel_ecdh_ctx - transformation context - * @client : pointer to i2c client device - * @fallback : used for unsupported curves or when user wants to use its own - * private key. - * @public_key : generated when calling set_secret(). It's the responsibility - * of the user to not call set_secret() while - * generate_public_key() or compute_shared_secret() are in flight. - * @curve_id : elliptic curve id - * @do_fallback: true when the device doesn't support the curve or when the user - * wants to use its own private key. + * @client: I2C client device + * @fallback: ECDH fallback used for caller-provided private keys + * @public_key: cached public key for the device-generated private key + * @curve_id: elliptic curve id + * @do_fallback: true when ECDH operations should use @fallback + * + * The caller must not invoke set_secret() while generate_public_key() + * or compute_shared_secret() are in flight. */ struct atmel_ecdh_ctx { struct i2c_client *client; @@ -55,7 +54,7 @@ static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq, if (status) goto free_work_data; - /* might want less than we've got */ + /* copy only as much as requested, capped at 32 bytes */ n_sz = min(ATMEL_ECC_NIST_P256_N_SIZE, req->dst_len); /* copy the shared secret */ @@ -64,15 +63,15 @@ static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq, if (copied != n_sz) status = -EINVAL; - /* fall through */ free_work_data: kfree_sensitive(work_data); kpp_request_complete(req, status); } /* - * A random private key is generated and stored in the device. The device - * returns the pair public key. + * If no private key is provided, generate one in the device and cache + * the corresponding public key. The generated private key never leaves + * the device. */ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, unsigned int len) @@ -83,10 +82,9 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, struct ecdh params; int ret = -ENOMEM; - /* free the old public key, if any */ kfree(ctx->public_key); - /* make sure you don't free the old public key twice */ ctx->public_key = NULL; + ctx->do_fallback = false; if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { dev_err(&ctx->client->dev, "crypto_ecdh_decode_key failed\n"); @@ -94,33 +92,25 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, } if (params.key_size) { - /* fallback to ecdh software implementation */ - ctx->do_fallback = true; - return crypto_kpp_set_secret(ctx->fallback, buf, len); + ret = crypto_kpp_set_secret(ctx->fallback, buf, len); + ctx->do_fallback = !ret; + return ret; } cmd = kmalloc_obj(*cmd); if (!cmd) return -ENOMEM; - /* - * The device only supports NIST P256 ECC keys. The public key size will - * always be the same. Use a macro for the key size to avoid unnecessary - * computations. - */ public_key = kmalloc(ATMEL_ECC_PUBKEY_SIZE, GFP_KERNEL); if (!public_key) goto free_cmd; - ctx->do_fallback = false; - atmel_i2c_init_genkey_cmd(cmd, DATA_SLOT_2); ret = atmel_i2c_send_receive(ctx->client, cmd); if (ret) goto free_public_key; - /* save the public key */ memcpy(public_key, &cmd->data[RSP_DATA_IDX], ATMEL_ECC_PUBKEY_SIZE); ctx->public_key = public_key; @@ -149,7 +139,7 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req) if (!ctx->public_key) return -EINVAL; - /* might want less than we've got */ + /* copy only as much as requested, capped at 64 bytes */ nbytes = min(ATMEL_ECC_PUBKEY_SIZE, req->dst_len); /* public key was saved at private key generation */ @@ -175,7 +165,10 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req) return crypto_kpp_compute_shared_secret(req); } - /* must have exactly two points to be on the curve */ + if (!ctx->public_key) + return -EINVAL; + + /* A P-256 public key must contain two 32-byte coordinates */ if (req->src_len != ATMEL_ECC_PUBKEY_SIZE) return -EINVAL; diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index 4c9af737b33aa..5eb76245347d8 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -31,10 +31,14 @@ static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data, struct atmel_i2c_client_priv *i2c_priv = work_data->ctx; struct hwrng *rng = areq; - if (status) + if (status) { dev_warn_ratelimited(&i2c_priv->client->dev, "i2c transaction failed (%d)\n", status); + kfree(work_data); + atomic_dec(&i2c_priv->tfm_count); + return; + } rng->priv = (unsigned long)work_data; atomic_dec(&i2c_priv->tfm_count); diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 3c0eacacfc87b..48cd537f001e8 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -462,14 +462,13 @@ static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd) IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size); fast = in && out; - if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg)) + if (dd->in_sg->length != dd->out_sg->length) fast = 0; } if (fast) { - count = min_t(size_t, dd->total, sg_dma_len(dd->in_sg)); - count = min_t(size_t, count, sg_dma_len(dd->out_sg)); + count = min_t(size_t, dd->total, dd->in_sg->length); err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); if (!err) { diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index ca473ca198b81..e7db638c25276 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1545,7 +1545,7 @@ static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev) if (sev_init_ex_buffer) return 0; - page = alloc_pages(GFP_KERNEL, get_order(NV_LENGTH)); + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(NV_LENGTH)); if (!page) { dev_err(sev->dev, "SEV: INIT_EX NV memory allocation failed\n"); return -ENOMEM; @@ -1555,20 +1555,31 @@ static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev) rc = sev_read_init_ex_file(); if (rc) - return rc; + goto err_free; /* If SEV-SNP is initialized, transition to firmware page. */ if (sev->snp_initialized) { unsigned long npages; npages = 1UL << get_order(NV_LENGTH); - if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, false)) { + if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, true)) { dev_err(sev->dev, "SEV: INIT_EX NV memory page state change failed.\n"); - return -ENOMEM; + rc = -ENOMEM; + /* + * Pages can be in an inconsistent state, don't release them back to the + * system. + */ + goto err_reset; } } return 0; + +err_free: + __free_pages(page, get_order(NV_LENGTH)); +err_reset: + sev_init_ex_buffer = NULL; + return rc; } static int __sev_platform_init_locked(int *error) diff --git a/drivers/crypto/gemini/sl3516-ce-cipher.c b/drivers/crypto/gemini/sl3516-ce-cipher.c index 583010b2d0071..02ec4282333b6 100644 --- a/drivers/crypto/gemini/sl3516-ce-cipher.c +++ b/drivers/crypto/gemini/sl3516-ce-cipher.c @@ -56,10 +56,6 @@ static bool sl3516_ce_need_fallback(struct skcipher_request *areq) ce->fallback_mod16++; return true; } - if ((sg_dma_len(sg) % 16) != 0) { - ce->fallback_mod16++; - return true; - } if (!IS_ALIGNED(sg->offset, 16)) { ce->fallback_align16++; return true; @@ -72,10 +68,6 @@ static bool sl3516_ce_need_fallback(struct skcipher_request *areq) ce->fallback_mod16++; return true; } - if ((sg_dma_len(sg) % 16) != 0) { - ce->fallback_mod16++; - return true; - } if (!IS_ALIGNED(sg->offset, 16)) { ce->fallback_align16++; return true; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 77e0e03cbcab9..a7354fd10dc0f 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -96,7 +96,6 @@ #define IV_FLAGS_OFFSET 0x6 #define IV_CM_OFFSET 0x3 #define IV_LAST_BYTE1 1 -#define IV_LAST_BYTE2 2 #define IV_LAST_BYTE_MASK 0xFF #define IV_CTR_INIT 0x1 #define IV_BYTE_OFFSET 0x8 @@ -1696,7 +1695,7 @@ static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) struct sec_cipher_req *c_req = &req->c_req; u32 data_size = aead_req->cryptlen; u8 flage = 0; - u8 cm, cl; + u8 cm, cl, i; /* the specification has been checked in aead_iv_demension_check() */ cl = c_req->c_ivin[0] + 1; @@ -1720,15 +1719,16 @@ static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) * the last 32bit is counter's initial number, * but the nonce uses the first 16bit * the tail 16bit fill with the cipher length + * When CL is 3, the tail 24bit fill with the cipher length. */ if (!c_req->encrypt) data_size = aead_req->cryptlen - authsize; - a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = - data_size & IV_LAST_BYTE_MASK; - data_size >>= IV_BYTE_OFFSET; - a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] = + for (i = 1; i <= cl; i++) { + a_req->a_ivin[ctx->c_ctx.ivsize - i] = data_size & IV_LAST_BYTE_MASK; + data_size >>= IV_BYTE_OFFSET; + } } static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index f62b994e18e58..cdfe00464fa36 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -2,6 +2,7 @@ /* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ #include +#include #include #include #include @@ -996,6 +997,19 @@ static int deflate_generic_decompress(struct acomp_req *req) return ret; } +static int deflate_generic_compress(struct acomp_req *req) +{ + ACOMP_FBREQ_ON_STACK(fbreq, req); + int ret; + + ret = crypto_acomp_compress(fbreq); + req->dlen = fbreq->dlen; + + update_total_sw_comp_calls(); + + return ret; +} + static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, struct acomp_req *req, dma_addr_t *src_addr, dma_addr_t *dst_addr); @@ -1044,13 +1058,17 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, pr_warn("%s: falling back to deflate-generic decompress, " "analytics error code %x\n", __func__, idxd_desc->iax_completion->error_code); + dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), + DMA_FROM_DEVICE); + dma_unmap_sg(dev, ctx->req->src, 1, DMA_TO_DEVICE); + ret = deflate_generic_decompress(ctx->req); if (ret) { dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", __func__, ret); err = -EIO; - goto err; } + goto out; } else { err = -EIO; goto err; @@ -1433,19 +1451,9 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, ret = check_completion(dev, idxd_desc->iax_completion, false, false); if (ret) { dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); - if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) { - pr_warn("%s: falling back to deflate-generic decompress, " - "analytics error code %x\n", __func__, - idxd_desc->iax_completion->error_code); - ret = deflate_generic_decompress(req); - if (ret) { - dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", - __func__, ret); - goto err; - } - } else { - goto err; - } + if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) + ret = -EAGAIN; + goto err; } else { req->dlen = idxd_desc->iax_completion->output_size; } @@ -1472,7 +1480,7 @@ static int iaa_comp_acompress(struct acomp_req *req) struct iaa_compression_ctx *compression_ctx; struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; - int nr_sgs, cpu, ret = 0; + int cpu, ret = 0; struct iaa_wq *iaa_wq; struct idxd_wq *wq; struct device *dev; @@ -1484,11 +1492,15 @@ static int iaa_comp_acompress(struct acomp_req *req) return -ENODEV; } - if (!req->src || !req->slen) { - pr_debug("invalid src, not compressing\n"); + if (!req->src || !req->slen || !req->dst) { + pr_debug("invalid req, not compressing\n"); return -EINVAL; } + /* Fall back to software if src or dst has multiple sg entries */ + if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + return deflate_generic_compress(req); + cpu = get_cpu(); wq = wq_table_next_wq(cpu); put_cpu(); @@ -1507,30 +1519,25 @@ static int iaa_comp_acompress(struct acomp_req *req) dev = &wq->idxd->pdev->dev; - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_compress(req); } src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, req->src, req->slen, sg_dma_len(req->src)); - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; + if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); + iaa_wq_put(wq); + return deflate_generic_compress(req); } dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + dev_dbg(dev, "map dst %llx req->dst %p dlen %d sg_len %d\n", dst_addr, req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, @@ -1550,8 +1557,8 @@ static int iaa_comp_acompress(struct acomp_req *req) if (ret) dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->dst, 1, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_FROM_DEVICE); goto out; } @@ -1559,9 +1566,8 @@ static int iaa_comp_acompress(struct acomp_req *req) if (ret) dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); out: iaa_wq_put(wq); @@ -1572,7 +1578,7 @@ static int iaa_comp_adecompress(struct acomp_req *req) { struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; - int nr_sgs, cpu, ret = 0; + int cpu, ret = 0; struct iaa_wq *iaa_wq; struct device *dev; struct idxd_wq *wq; @@ -1582,11 +1588,15 @@ static int iaa_comp_adecompress(struct acomp_req *req) return -ENODEV; } - if (!req->src || !req->slen) { - pr_debug("invalid src, not decompressing\n"); + if (!req->src || !req->slen || !req->dst) { + pr_debug("invalid req, not decompressing\n"); return -EINVAL; } + /* Fall back to software if src or dst has multiple sg entries */ + if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + return deflate_generic_decompress(req); + cpu = get_cpu(); wq = wq_table_next_wq(cpu); put_cpu(); @@ -1605,30 +1615,25 @@ static int iaa_comp_adecompress(struct acomp_req *req) dev = &wq->idxd->pdev->dev; - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_decompress(req); } src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, req->src, req->slen, sg_dma_len(req->src)); - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; + if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); + iaa_wq_put(wq); + return deflate_generic_decompress(req); } dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + dev_dbg(dev, "map dst %llx req->dst %p dlen %d sg_len %d\n", dst_addr, req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, @@ -1636,15 +1641,16 @@ static int iaa_comp_adecompress(struct acomp_req *req) if (ret == -EINPROGRESS) return ret; - if (ret != 0) + if (ret != 0 && ret != -EAGAIN) dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); -out: + dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); iaa_wq_put(wq); + if (ret == -EAGAIN) + ret = deflate_generic_decompress(req); + return ret; } diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.c b/drivers/crypto/intel/iaa/iaa_crypto_stats.c index f5cc3d29ca19e..2f2ed88c88120 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_stats.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.c @@ -19,6 +19,7 @@ static atomic64_t total_comp_calls; static atomic64_t total_decomp_calls; +static atomic64_t total_sw_comp_calls; static atomic64_t total_sw_decomp_calls; static atomic64_t total_comp_bytes_out; static atomic64_t total_decomp_bytes_in; @@ -43,6 +44,11 @@ void update_total_decomp_calls(void) atomic64_inc(&total_decomp_calls); } +void update_total_sw_comp_calls(void) +{ + atomic64_inc(&total_sw_comp_calls); +} + void update_total_sw_decomp_calls(void) { atomic64_inc(&total_sw_decomp_calls); @@ -104,6 +110,7 @@ static void reset_iaa_crypto_stats(void) { atomic64_set(&total_comp_calls, 0); atomic64_set(&total_decomp_calls, 0); + atomic64_set(&total_sw_comp_calls, 0); atomic64_set(&total_sw_decomp_calls, 0); atomic64_set(&total_comp_bytes_out, 0); atomic64_set(&total_decomp_bytes_in, 0); @@ -174,6 +181,8 @@ static int global_stats_show(struct seq_file *m, void *v) atomic64_read(&total_comp_calls)); seq_printf(m, " total_decomp_calls: %llu\n", atomic64_read(&total_decomp_calls)); + seq_printf(m, " total_sw_comp_calls: %llu\n", + atomic64_read(&total_sw_comp_calls)); seq_printf(m, " total_sw_decomp_calls: %llu\n", atomic64_read(&total_sw_decomp_calls)); seq_printf(m, " total_comp_bytes_out: %llu\n", diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.h b/drivers/crypto/intel/iaa/iaa_crypto_stats.h index 3787a5f507eb2..6e0c6f9939bfa 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_stats.h +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.h @@ -11,6 +11,7 @@ void iaa_crypto_debugfs_cleanup(void); void update_total_comp_calls(void); void update_total_comp_bytes_out(int n); void update_total_decomp_calls(void); +void update_total_sw_comp_calls(void); void update_total_sw_decomp_calls(void); void update_total_decomp_bytes_in(int n); void update_completion_einval_errs(void); @@ -29,6 +30,7 @@ static inline void iaa_crypto_debugfs_cleanup(void) {} static inline void update_total_comp_calls(void) {} static inline void update_total_comp_bytes_out(int n) {} static inline void update_total_decomp_calls(void) {} +static inline void update_total_sw_comp_calls(void) {} static inline void update_total_sw_decomp_calls(void) {} static inline void update_total_decomp_bytes_in(int n) {} static inline void update_completion_einval_errs(void) {} diff --git a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c index 8a8f6c81e010c..419f88af1031b 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c @@ -1541,7 +1541,7 @@ static int register_aes_algs(struct ocs_aes_dev *aes_dev) /* * If any algorithm fails to register, all preceding algorithms that - * were successfully registered will be automatically unregistered. + * were registered in the same call are automatically unregistered. */ ret = crypto_engine_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); if (ret) @@ -1549,7 +1549,7 @@ static int register_aes_algs(struct ocs_aes_dev *aes_dev) ret = crypto_engine_register_skciphers(algs, ARRAY_SIZE(algs)); if (ret) - crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); + crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); return ret; } @@ -1561,6 +1561,7 @@ static const struct of_device_id kmb_ocs_aes_of_match[] = { }, {} }; +MODULE_DEVICE_TABLE(of, kmb_ocs_aes_of_match); static void kmb_ocs_aes_remove(struct platform_device *pdev) { @@ -1602,6 +1603,8 @@ static int kmb_ocs_aes_probe(struct platform_device *pdev) if (IS_ERR(aes_dev->base_reg)) return PTR_ERR(aes_dev->base_reg); + init_completion(&aes_dev->irq_completion); + /* Get and request IRQ */ aes_dev->irq = platform_get_irq(pdev, 0); if (aes_dev->irq < 0) @@ -1619,8 +1622,6 @@ static int kmb_ocs_aes_probe(struct platform_device *pdev) list_add_tail(&aes_dev->list, &ocs_aes.dev_list); spin_unlock(&ocs_aes.lock); - init_completion(&aes_dev->irq_completion); - /* Initialize crypto engine */ aes_dev->engine = crypto_engine_alloc_init(dev, true); if (!aes_dev->engine) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index d58cd7fbf7077..afded3030e9a3 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -189,6 +189,8 @@ static void adf_device_reset_worker(struct work_struct *work) queue_work(device_sriov_wq, &sriov_data.sriov_work); if (wait_for_completion_timeout(&sriov_data.compl, wait_jiffies)) adf_pf2vf_notify_restarted(accel_dev); + else + cancel_work_sync(&sriov_data.sriov_work); adf_dev_restarted_notify(accel_dev); clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.c b/drivers/crypto/intel/qat/qat_common/adf_cfg.c index ea5d72d5090c0..b88febf53a19d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.c @@ -2,6 +2,7 @@ /* Copyright(c) 2014 - 2020 Intel Corporation */ #include #include +#include #include #include #include "adf_accel_devices.h" @@ -284,16 +285,13 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, return -ENOMEM; INIT_LIST_HEAD(&key_val->list); - strscpy(key_val->key, key, sizeof(key_val->key)); + strscpy(key_val->key, key); if (type == ADF_DEC) { snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, "%ld", (*((long *)val))); } else if (type == ADF_STR) { - strscpy(key_val->val, (char *)val, sizeof(key_val->val)); - } else if (type == ADF_HEX) { - snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, - "0x%lx", (unsigned long)val); + strscpy(key_val->val, (char *)val); } else { dev_err(&GET_DEV(accel_dev), "Unknown type given.\n"); kfree(key_val); @@ -350,7 +348,7 @@ int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name) if (!sec) return -ENOMEM; - strscpy(sec->name, name, sizeof(sec->name)); + strscpy(sec->name, name); INIT_LIST_HEAD(&sec->param_head); down_write(&cfg->lock); list_add_tail(&sec->list, &cfg->sec_list); diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h index d63f4dcccbb5d..5922d018f5b98 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h @@ -28,7 +28,6 @@ enum adf_cfg_service_type { enum adf_cfg_val_type { ADF_DEC, - ADF_HEX, ADF_STR }; diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c index 1af6da8b263fb..0cb6cb63e9950 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c @@ -60,7 +60,7 @@ static int adf_service_string_to_mask(struct adf_accel_dev *accel_dev, const cha if (len > ADF_CFG_MAX_VAL_LEN_IN_BYTES - 1) return -EINVAL; - strscpy(services, buf, ADF_CFG_MAX_VAL_LEN_IN_BYTES); + strscpy(services, buf); substr = services; while ((token = strsep(&substr, ADF_SERVICES_DELIMITER))) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c index f9017e03ec0f2..32aeb795cc036 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c @@ -2,6 +2,7 @@ /* Copyright(c) 2024 Intel Corporation */ #include +#include #include #include "adf_mstate_mgr.h" @@ -158,7 +159,7 @@ static struct adf_mstate_sect_h *adf_mstate_sect_add_header(struct adf_mstate_mg return NULL; } - strscpy(sect->id, id, sizeof(sect->id)); + strscpy(sect->id, id); sect->size = 0; sect->sub_sects = 0; mgr->state += sizeof(*sect); diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c index a8f853516a3f9..fc5d88a2bb172 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c @@ -2,6 +2,7 @@ /* Copyright(c) 2014 - 2020 Intel Corporation */ #include #include +#include #include #include "adf_accel_devices.h" #include "adf_transport_internal.h" @@ -103,7 +104,7 @@ int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name) if (!ring_debug) return -ENOMEM; - strscpy(ring_debug->ring_name, name, sizeof(ring_debug->ring_name)); + strscpy(ring_debug->ring_name, name); snprintf(entry_name, sizeof(entry_name), "ring_%02d", ring->ring_number); diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs.c b/drivers/crypto/intel/qat/qat_common/qat_algs.c index 7f638a62e3ade..91663805d9e60 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_algs.c @@ -405,6 +405,7 @@ static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen, memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE, AES_BLOCK_SIZE); } + memzero_explicit(&aes_expanded, sizeof(aes_expanded)); } static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c index 1424d7a9bcd36..8129ad0c32d84 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.c +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c @@ -2,6 +2,7 @@ /* Copyright(c) 2022 Intel Corporation */ #include #include +#include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_transport.h" @@ -144,7 +145,7 @@ static int qat_compression_create_instances(struct adf_accel_dev *accel_dev) int i; INIT_LIST_HEAD(&accel_dev->compression_list); - strscpy(key, ADF_NUM_DC, sizeof(key)); + strscpy(key, ADF_NUM_DC); ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); if (ret) return ret; diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index 133ebc9982362..595b2fd84667e 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -353,7 +353,7 @@ static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq) for_each_sg(req->src, src, sg_nents(req->src), i) { src_buf = sg_virt(src); - len = sg_dma_len(src); + len = src->length; tlen += len; limit_hit = tlen > req->cryptlen; diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c index 9cb11fada2c4d..a9642c67f4761 100644 --- a/drivers/crypto/qce/aead.c +++ b/drivers/crypto/qce/aead.c @@ -9,8 +9,6 @@ #include #include #include -#include -#include #include #include #include "aead.h" @@ -198,7 +196,7 @@ qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req) /* Get the msg */ msg_sg = scatterwalk_ffwd(__sg, req->src, req->assoclen); - rctx->adata = kzalloc((ALIGN(assoclen, 16) + MAX_CCM_ADATA_HEADER_LEN) * + rctx->adata = kzalloc(ALIGN(assoclen + MAX_CCM_ADATA_HEADER_LEN, 16) * sizeof(unsigned char), GFP_ATOMIC); if (!rctx->adata) return -ENOMEM; @@ -592,7 +590,6 @@ static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_authenc_keys authenc_keys; unsigned long flags = to_aead_tmpl(tfm)->alg_flags; - u32 _key[6]; int err; err = crypto_authenc_extractkeys(&authenc_keys, key, keylen); @@ -603,26 +600,7 @@ static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int authenc_keys.authkeylen > QCE_MAX_KEY_SIZE) return -EINVAL; - if (IS_DES(flags)) { - err = verify_aead_des_key(tfm, authenc_keys.enckey, authenc_keys.enckeylen); - if (err) - return err; - } else if (IS_3DES(flags)) { - err = verify_aead_des3_key(tfm, authenc_keys.enckey, authenc_keys.enckeylen); - if (err) - return err; - /* - * The crypto engine does not support any two keys - * being the same for triple des algorithms. The - * verify_skcipher_des3_key does not check for all the - * below conditions. Schedule fallback in this case. - */ - memcpy(_key, authenc_keys.enckey, DES3_EDE_KEY_SIZE); - if (!((_key[0] ^ _key[2]) | (_key[1] ^ _key[3])) || - !((_key[2] ^ _key[4]) | (_key[3] ^ _key[5])) || - !((_key[0] ^ _key[4]) | (_key[1] ^ _key[5]))) - ctx->need_fallback = true; - } else if (IS_AES(flags)) { + if (IS_AES(flags)) { /* No random key sizes */ if (authenc_keys.enckeylen != AES_KEYSIZE_128 && authenc_keys.enckeylen != AES_KEYSIZE_192 && @@ -693,38 +671,6 @@ struct qce_aead_def { }; static const struct qce_aead_def aead_def[] = { - { - .flags = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC, - .name = "authenc(hmac(sha1),cbc(des))", - .drv_name = "authenc-hmac-sha1-cbc-des-qce", - .blocksize = DES_BLOCK_SIZE, - .ivsize = DES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - }, - { - .flags = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC, - .name = "authenc(hmac(sha1),cbc(des3_ede))", - .drv_name = "authenc-hmac-sha1-cbc-3des-qce", - .blocksize = DES3_EDE_BLOCK_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - }, - { - .flags = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, - .name = "authenc(hmac(sha256),cbc(des))", - .drv_name = "authenc-hmac-sha256-cbc-des-qce", - .blocksize = DES_BLOCK_SIZE, - .ivsize = DES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - }, - { - .flags = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, - .name = "authenc(hmac(sha256),cbc(des3_ede))", - .drv_name = "authenc-hmac-sha256-cbc-3des-qce", - .blocksize = DES3_EDE_BLOCK_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - }, { .flags = QCE_ALG_AES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, .name = "authenc(hmac(sha256),cbc(aes))", diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c index 54a78a57f6302..3081d765a0f2c 100644 --- a/drivers/crypto/qce/common.c +++ b/drivers/crypto/qce/common.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include "cipher.h" @@ -115,18 +114,16 @@ static u32 qce_auth_cfg(unsigned long flags, u32 key_size, u32 auth_size) cfg |= AUTH_KEY_SZ_AES256 << AUTH_KEY_SIZE_SHIFT; } - if (IS_SHA1(flags) || IS_SHA1_HMAC(flags)) - cfg |= AUTH_SIZE_SHA1 << AUTH_SIZE_SHIFT; - else if (IS_SHA256(flags) || IS_SHA256_HMAC(flags)) + if (IS_SHA256(flags) || IS_SHA256_HMAC(flags)) cfg |= AUTH_SIZE_SHA256 << AUTH_SIZE_SHIFT; else if (IS_CMAC(flags)) cfg |= AUTH_SIZE_ENUM_16_BYTES << AUTH_SIZE_SHIFT; else if (IS_CCM(flags)) cfg |= (auth_size - 1) << AUTH_SIZE_SHIFT; - if (IS_SHA1(flags) || IS_SHA256(flags)) + if (IS_SHA256(flags)) cfg |= AUTH_MODE_HASH << AUTH_MODE_SHIFT; - else if (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags)) + else if (IS_SHA256_HMAC(flags)) cfg |= AUTH_MODE_HMAC << AUTH_MODE_SHIFT; else if (IS_CCM(flags)) cfg |= AUTH_MODE_CCM << AUTH_MODE_SHIFT; @@ -191,7 +188,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req) else qce_cpu_to_be32p_array(auth, rctx->digest, digestsize); - iv_words = (IS_SHA1(rctx->flags) || IS_SHA1_HMAC(rctx->flags)) ? 5 : 8; + iv_words = 8; qce_write_array(qce, REG_AUTH_IV0, (u32 *)auth, iv_words); if (rctx->first_blk) @@ -243,19 +240,8 @@ static u32 qce_encr_cfg(unsigned long flags, u32 aes_key_size) if (IS_AES(flags)) cfg |= ENCR_ALG_AES << ENCR_ALG_SHIFT; - else if (IS_DES(flags) || IS_3DES(flags)) - cfg |= ENCR_ALG_DES << ENCR_ALG_SHIFT; - - if (IS_DES(flags)) - cfg |= ENCR_KEY_SZ_DES << ENCR_KEY_SZ_SHIFT; - - if (IS_3DES(flags)) - cfg |= ENCR_KEY_SZ_3DES << ENCR_KEY_SZ_SHIFT; switch (flags & QCE_MODE_MASK) { - case QCE_MODE_ECB: - cfg |= ENCR_MODE_ECB << ENCR_MODE_SHIFT; - break; case QCE_MODE_CBC: cfg |= ENCR_MODE_CBC << ENCR_MODE_SHIFT; break; @@ -340,13 +326,7 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req) encr_cfg = qce_encr_cfg(flags, keylen); - if (IS_DES(flags)) { - enciv_words = 2; - enckey_words = 2; - } else if (IS_3DES(flags)) { - enciv_words = 2; - enckey_words = 6; - } else if (IS_AES(flags)) { + if (IS_AES(flags)) { if (IS_XTS(flags)) qce_xtskey(qce, ctx->enc_key, ctx->enc_keylen, rctx->cryptlen); @@ -357,14 +337,12 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req) qce_write_array(qce, REG_ENCR_KEY0, (u32 *)enckey, enckey_words); - if (!IS_ECB(flags)) { - if (IS_XTS(flags)) - qce_xts_swapiv(enciv, rctx->iv, ivsize); - else - qce_cpu_to_be32p_array(enciv, rctx->iv, ivsize); + if (IS_XTS(flags)) + qce_xts_swapiv(enciv, rctx->iv, ivsize); + else + qce_cpu_to_be32p_array(enciv, rctx->iv, ivsize); - qce_write_array(qce, REG_CNTR0_IV0, (u32 *)enciv, enciv_words); - } + qce_write_array(qce, REG_CNTR0_IV0, (u32 *)enciv, enciv_words); if (IS_ENCRYPT(flags)) encr_cfg |= BIT(ENCODE_SHIFT); @@ -393,10 +371,6 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req) #endif #ifdef CONFIG_CRYPTO_DEV_QCE_AEAD -static const u32 std_iv_sha1[SHA256_DIGEST_SIZE / sizeof(u32)] = { - SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 0, 0, 0 -}; - static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 @@ -473,13 +447,8 @@ static int qce_setup_regs_aead(struct crypto_async_request *async_req) /* Write initial authentication IV only for HMAC algorithms */ if (IS_SHA_HMAC(rctx->flags)) { /* Write default authentication iv */ - if (IS_SHA1_HMAC(rctx->flags)) { - auth_ivsize = SHA1_DIGEST_SIZE; - memcpy(authiv, std_iv_sha1, auth_ivsize); - } else if (IS_SHA256_HMAC(rctx->flags)) { - auth_ivsize = SHA256_DIGEST_SIZE; - memcpy(authiv, std_iv_sha256, auth_ivsize); - } + auth_ivsize = SHA256_DIGEST_SIZE; + memcpy(authiv, std_iv_sha256, auth_ivsize); authiv_words = auth_ivsize / sizeof(u32); qce_write_array(qce, REG_AUTH_IV0, (u32 *)authiv, authiv_words); } else if (IS_CCM(rctx->flags)) { diff --git a/drivers/crypto/qce/common.h b/drivers/crypto/qce/common.h index 02e63ad9f2455..9cd2e6ed8bbb0 100644 --- a/drivers/crypto/qce/common.h +++ b/drivers/crypto/qce/common.h @@ -22,7 +22,7 @@ /* IV length in bytes */ #define QCE_AES_IV_LENGTH AES_BLOCK_SIZE -/* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ +/* max of AES_BLOCK_SIZE */ #define QCE_MAX_IV_SIZE AES_BLOCK_SIZE /* maximum nonce bytes */ @@ -33,14 +33,10 @@ #define QCE_MAX_ALIGN_SIZE 64 /* cipher algorithms */ -#define QCE_ALG_DES BIT(0) -#define QCE_ALG_3DES BIT(1) #define QCE_ALG_AES BIT(2) /* hash and hmac algorithms */ -#define QCE_HASH_SHA1 BIT(3) #define QCE_HASH_SHA256 BIT(4) -#define QCE_HASH_SHA1_HMAC BIT(5) #define QCE_HASH_SHA256_HMAC BIT(6) #define QCE_HASH_AES_CMAC BIT(7) @@ -58,21 +54,15 @@ #define QCE_ENCRYPT BIT(30) #define QCE_DECRYPT BIT(31) -#define IS_DES(flags) (flags & QCE_ALG_DES) -#define IS_3DES(flags) (flags & QCE_ALG_3DES) #define IS_AES(flags) (flags & QCE_ALG_AES) -#define IS_SHA1(flags) (flags & QCE_HASH_SHA1) #define IS_SHA256(flags) (flags & QCE_HASH_SHA256) -#define IS_SHA1_HMAC(flags) (flags & QCE_HASH_SHA1_HMAC) #define IS_SHA256_HMAC(flags) (flags & QCE_HASH_SHA256_HMAC) #define IS_CMAC(flags) (flags & QCE_HASH_AES_CMAC) -#define IS_SHA(flags) (IS_SHA1(flags) || IS_SHA256(flags)) -#define IS_SHA_HMAC(flags) \ - (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags)) +#define IS_SHA(flags) IS_SHA256(flags) +#define IS_SHA_HMAC(flags) IS_SHA256_HMAC(flags) #define IS_CBC(mode) (mode & QCE_MODE_CBC) -#define IS_ECB(mode) (mode & QCE_MODE_ECB) #define IS_CTR(mode) (mode & QCE_MODE_CTR) #define IS_XTS(mode) (mode & QCE_MODE_XTS) #define IS_CCM(mode) (mode & QCE_MODE_CCM) diff --git a/drivers/crypto/qce/regs-v5.h b/drivers/crypto/qce/regs-v5.h index d59ed27989062..431a7db1a4e72 100644 --- a/drivers/crypto/qce/regs-v5.h +++ b/drivers/crypto/qce/regs-v5.h @@ -203,7 +203,6 @@ #define AUTH_SIZE_SHIFT 9 #define AUTH_SIZE_MASK GENMASK(13, 9) -#define AUTH_SIZE_SHA1 0 #define AUTH_SIZE_SHA256 1 #define AUTH_SIZE_ENUM_1_BYTES 0 #define AUTH_SIZE_ENUM_2_BYTES 1 @@ -284,15 +283,12 @@ #define ENCR_KEY_SZ_SHIFT 3 #define ENCR_KEY_SZ_MASK GENMASK(5, 3) -#define ENCR_KEY_SZ_DES 0 -#define ENCR_KEY_SZ_3DES 1 #define ENCR_KEY_SZ_AES128 0 #define ENCR_KEY_SZ_AES256 2 #define ENCR_ALG_SHIFT 0 #define ENCR_ALG_MASK GENMASK(2, 0) #define ENCR_ALG_NONE 0 -#define ENCR_ALG_DES 1 #define ENCR_ALG_AES 2 #define ENCR_ALG_KASUMI 4 #define ENCR_ALG_SNOW_3G 5 diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 1b37121cbcdcb..cd9391219b1f3 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -25,10 +25,6 @@ struct qce_sha_saved_state { static LIST_HEAD(ahash_algs); -static const u32 std_iv_sha1[SHA256_DIGEST_SIZE / sizeof(u32)] = { - SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 0, 0, 0 -}; - static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 @@ -349,9 +345,7 @@ static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, return 0; } - if (digestsize == SHA1_DIGEST_SIZE) - alg_name = "sha1-qce"; - else if (digestsize == SHA256_DIGEST_SIZE) + if (digestsize == SHA256_DIGEST_SIZE) alg_name = "sha256-qce"; else return -EINVAL; @@ -412,15 +406,6 @@ struct qce_ahash_def { }; static const struct qce_ahash_def ahash_def[] = { - { - .flags = QCE_HASH_SHA1, - .name = "sha1", - .drv_name = "sha1-qce", - .digestsize = SHA1_DIGEST_SIZE, - .blocksize = SHA1_BLOCK_SIZE, - .statesize = sizeof(struct qce_sha_saved_state), - .std_iv = std_iv_sha1, - }, { .flags = QCE_HASH_SHA256, .name = "sha256", @@ -430,15 +415,6 @@ static const struct qce_ahash_def ahash_def[] = { .statesize = sizeof(struct qce_sha_saved_state), .std_iv = std_iv_sha256, }, - { - .flags = QCE_HASH_SHA1_HMAC, - .name = "hmac(sha1)", - .drv_name = "hmac-sha1-qce", - .digestsize = SHA1_DIGEST_SIZE, - .blocksize = SHA1_BLOCK_SIZE, - .statesize = sizeof(struct qce_sha_saved_state), - .std_iv = std_iv_sha1, - }, { .flags = QCE_HASH_SHA256_HMAC, .name = "hmac(sha256)", @@ -476,9 +452,7 @@ static int qce_ahash_register_one(const struct qce_ahash_def *def, alg->halg.digestsize = def->digestsize; alg->halg.statesize = def->statesize; - if (IS_SHA1(def->flags)) - tmpl->hash_zero = sha1_zero_message_hash; - else if (IS_SHA256(def->flags)) + if (IS_SHA256(def->flags)) tmpl->hash_zero = sha256_zero_message_hash; base = &alg->halg.base; diff --git a/drivers/crypto/qce/sha.h b/drivers/crypto/qce/sha.h index a22695361f165..cb822fc334dc1 100644 --- a/drivers/crypto/qce/sha.h +++ b/drivers/crypto/qce/sha.h @@ -7,7 +7,6 @@ #define _SHA_H_ #include -#include #include #include "common.h" diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c index db0b648a56eb1..ff4ee9541f7cb 100644 --- a/drivers/crypto/qce/skcipher.c +++ b/drivers/crypto/qce/skcipher.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include "cipher.h" @@ -209,51 +208,6 @@ static int qce_skcipher_setkey(struct crypto_skcipher *ablk, const u8 *key, return ret; } -static int qce_des_setkey(struct crypto_skcipher *ablk, const u8 *key, - unsigned int keylen) -{ - struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(ablk); - int err; - - err = verify_skcipher_des_key(ablk, key); - if (err) - return err; - - ctx->enc_keylen = keylen; - memcpy(ctx->enc_key, key, keylen); - return 0; -} - -static int qce_des3_setkey(struct crypto_skcipher *ablk, const u8 *key, - unsigned int keylen) -{ - struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(ablk); - u32 _key[6]; - int err; - - err = verify_skcipher_des3_key(ablk, key); - if (err) - return err; - - /* - * The crypto engine does not support any two keys - * being the same for triple des algorithms. The - * verify_skcipher_des3_key does not check for all the - * below conditions. Return -ENOKEY in case any two keys - * are the same. Revisit to see if a fallback cipher - * is needed to handle this condition. - */ - memcpy(_key, key, DES3_EDE_KEY_SIZE); - if (!((_key[0] ^ _key[2]) | (_key[1] ^ _key[3])) || - !((_key[2] ^ _key[4]) | (_key[3] ^ _key[5])) || - !((_key[0] ^ _key[4]) | (_key[1] ^ _key[5]))) - return -ENOKEY; - - ctx->enc_keylen = keylen; - memcpy(ctx->enc_key, key, keylen); - return 0; -} - static int qce_skcipher_crypt(struct skcipher_request *req, int encrypt) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -276,7 +230,7 @@ static int qce_skcipher_crypt(struct skcipher_request *req, int encrypt) * ECB and CBC algorithms require message lengths to be * multiples of block size. */ - if (IS_ECB(rctx->flags) || IS_CBC(rctx->flags)) + if (IS_CBC(rctx->flags)) if (!IS_ALIGNED(req->cryptlen, blocksize)) return -EINVAL; @@ -359,15 +313,6 @@ struct qce_skcipher_def { }; static const struct qce_skcipher_def skcipher_def[] = { - { - .flags = QCE_ALG_AES | QCE_MODE_ECB, - .name = "ecb(aes)", - .drv_name = "ecb-aes-qce", - .blocksize = AES_BLOCK_SIZE, - .ivsize = 0, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - }, { .flags = QCE_ALG_AES | QCE_MODE_CBC, .name = "cbc(aes)", @@ -396,42 +341,6 @@ static const struct qce_skcipher_def skcipher_def[] = { .min_keysize = AES_MIN_KEY_SIZE * 2, .max_keysize = AES_MAX_KEY_SIZE * 2, }, - { - .flags = QCE_ALG_DES | QCE_MODE_ECB, - .name = "ecb(des)", - .drv_name = "ecb-des-qce", - .blocksize = DES_BLOCK_SIZE, - .ivsize = 0, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - }, - { - .flags = QCE_ALG_DES | QCE_MODE_CBC, - .name = "cbc(des)", - .drv_name = "cbc-des-qce", - .blocksize = DES_BLOCK_SIZE, - .ivsize = DES_BLOCK_SIZE, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - }, - { - .flags = QCE_ALG_3DES | QCE_MODE_ECB, - .name = "ecb(des3_ede)", - .drv_name = "ecb-3des-qce", - .blocksize = DES3_EDE_BLOCK_SIZE, - .ivsize = 0, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - }, - { - .flags = QCE_ALG_3DES | QCE_MODE_CBC, - .name = "cbc(des3_ede)", - .drv_name = "cbc-3des-qce", - .blocksize = DES3_EDE_BLOCK_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - }, }; static int qce_skcipher_register_one(const struct qce_skcipher_def *def, @@ -455,9 +364,7 @@ static int qce_skcipher_register_one(const struct qce_skcipher_def *def, alg->ivsize = def->ivsize; alg->min_keysize = def->min_keysize; alg->max_keysize = def->max_keysize; - alg->setkey = IS_3DES(def->flags) ? qce_des3_setkey : - IS_DES(def->flags) ? qce_des_setkey : - qce_skcipher_setkey; + alg->setkey = qce_skcipher_setkey; alg->encrypt = qce_skcipher_encrypt; alg->decrypt = qce_skcipher_decrypt; diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c index 150e5802e3515..4d046caafe5bb 100644 --- a/drivers/crypto/qcom-rng.c +++ b/drivers/crypto/qcom-rng.c @@ -3,10 +3,8 @@ // // Based on msm-rng.c and downstream driver -#include #include #include -#include #include #include #include @@ -32,24 +30,15 @@ #define QCOM_TRNG_QUALITY 1024 struct qcom_rng { - struct mutex lock; void __iomem *base; struct clk *clk; struct hwrng hwrng; - struct qcom_rng_match_data *match_data; -}; - -struct qcom_rng_ctx { - struct qcom_rng *rng; }; struct qcom_rng_match_data { - bool skip_init; bool hwrng_support; }; -static struct qcom_rng *qcom_rng_dev; - static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max) { unsigned int currsize = 0; @@ -65,8 +54,6 @@ static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max) return ret; val = readl_relaxed(rng->base + PRNG_DATA_OUT); - if (!val) - return -EINVAL; if ((max - currsize) >= WORD_SZ) { memcpy(data, &val, WORD_SZ); @@ -82,35 +69,11 @@ static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max) return currsize; } -static int qcom_rng_generate(struct crypto_rng *tfm, - const u8 *src, unsigned int slen, - u8 *dstn, unsigned int dlen) +static int qcom_hwrng_init(struct hwrng *hwrng) { - struct qcom_rng_ctx *ctx = crypto_rng_ctx(tfm); - struct qcom_rng *rng = ctx->rng; - int ret; - - ret = clk_prepare_enable(rng->clk); - if (ret) - return ret; - - mutex_lock(&rng->lock); - - ret = qcom_rng_read(rng, dstn, dlen); - - mutex_unlock(&rng->lock); - clk_disable_unprepare(rng->clk); - - if (ret >= 0) - ret = 0; - - return ret; -} + struct qcom_rng *qrng = container_of(hwrng, struct qcom_rng, hwrng); -static int qcom_rng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - return 0; + return clk_prepare_enable(qrng->clk); } static int qcom_hwrng_read(struct hwrng *hwrng, void *data, size_t max, bool wait) @@ -120,74 +83,33 @@ static int qcom_hwrng_read(struct hwrng *hwrng, void *data, size_t max, bool wai return qcom_rng_read(qrng, data, max); } -static int qcom_rng_enable(struct qcom_rng *rng) +static void qcom_hwrng_cleanup(struct hwrng *hwrng) { - u32 val; - int ret; - - ret = clk_prepare_enable(rng->clk); - if (ret) - return ret; - - /* Enable PRNG only if it is not already enabled */ - val = readl_relaxed(rng->base + PRNG_CONFIG); - if (val & PRNG_CONFIG_HW_ENABLE) - goto already_enabled; - - val = readl_relaxed(rng->base + PRNG_LFSR_CFG); - val &= ~PRNG_LFSR_CFG_MASK; - val |= PRNG_LFSR_CFG_CLOCKS; - writel(val, rng->base + PRNG_LFSR_CFG); - - val = readl_relaxed(rng->base + PRNG_CONFIG); - val |= PRNG_CONFIG_HW_ENABLE; - writel(val, rng->base + PRNG_CONFIG); - -already_enabled: - clk_disable_unprepare(rng->clk); - - return 0; -} - -static int qcom_rng_init(struct crypto_tfm *tfm) -{ - struct qcom_rng_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->rng = qcom_rng_dev; - - if (!ctx->rng->match_data->skip_init) - return qcom_rng_enable(ctx->rng); + struct qcom_rng *qrng = container_of(hwrng, struct qcom_rng, hwrng); - return 0; + clk_disable_unprepare(qrng->clk); } -static struct rng_alg qcom_rng_alg = { - .generate = qcom_rng_generate, - .seed = qcom_rng_seed, - .seedsize = 0, - .base = { - .cra_name = "stdrng", - .cra_driver_name = "qcom-rng", - .cra_flags = CRYPTO_ALG_TYPE_RNG, - .cra_priority = 300, - .cra_ctxsize = sizeof(struct qcom_rng_ctx), - .cra_module = THIS_MODULE, - .cra_init = qcom_rng_init, - } -}; - static int qcom_rng_probe(struct platform_device *pdev) { + const struct qcom_rng_match_data *match_data; struct qcom_rng *rng; int ret; + match_data = device_get_match_data(&pdev->dev); + if (match_data == NULL || !match_data->hwrng_support) { + dev_info(&pdev->dev, "TRNG support not detected\n"); + /* + * In this case the driver does nothing except the dev_info(), + * but bind the device anyway to avoid effects on GCC state. + */ + return 0; + } + rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); if (!rng) return -ENOMEM; - platform_set_drvdata(pdev, rng); - mutex_init(&rng->lock); - rng->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(rng->base)) return PTR_ERR(rng->base); @@ -196,53 +118,26 @@ static int qcom_rng_probe(struct platform_device *pdev) if (IS_ERR(rng->clk)) return PTR_ERR(rng->clk); - rng->match_data = (struct qcom_rng_match_data *)device_get_match_data(&pdev->dev); - - qcom_rng_dev = rng; - ret = crypto_register_rng(&qcom_rng_alg); - if (ret) { - dev_err(&pdev->dev, "Register crypto rng failed: %d\n", ret); - qcom_rng_dev = NULL; - return ret; - } - - if (rng->match_data->hwrng_support) { - rng->hwrng.name = "qcom_hwrng"; - rng->hwrng.read = qcom_hwrng_read; - rng->hwrng.quality = QCOM_TRNG_QUALITY; - ret = devm_hwrng_register(&pdev->dev, &rng->hwrng); - if (ret) { - dev_err(&pdev->dev, "Register hwrng failed: %d\n", ret); - qcom_rng_dev = NULL; - goto fail; - } - } - - return ret; -fail: - crypto_unregister_rng(&qcom_rng_alg); + rng->hwrng.name = "qcom_hwrng"; + rng->hwrng.init = qcom_hwrng_init; + rng->hwrng.read = qcom_hwrng_read; + rng->hwrng.cleanup = qcom_hwrng_cleanup; + rng->hwrng.quality = QCOM_TRNG_QUALITY; + ret = devm_hwrng_register(&pdev->dev, &rng->hwrng); + if (ret) + dev_err(&pdev->dev, "Register hwrng failed: %d\n", ret); return ret; } -static void qcom_rng_remove(struct platform_device *pdev) -{ - crypto_unregister_rng(&qcom_rng_alg); - - qcom_rng_dev = NULL; -} - static struct qcom_rng_match_data qcom_prng_match_data = { - .skip_init = false, .hwrng_support = false, }; static struct qcom_rng_match_data qcom_prng_ee_match_data = { - .skip_init = true, .hwrng_support = false, }; static struct qcom_rng_match_data qcom_trng_match_data = { - .skip_init = true, .hwrng_support = true, }; @@ -262,7 +157,6 @@ MODULE_DEVICE_TABLE(of, qcom_rng_of_match); static struct platform_driver qcom_rng_driver = { .probe = qcom_rng_probe, - .remove = qcom_rng_remove, .driver = { .name = KBUILD_MODNAME, .of_match_table = qcom_rng_of_match, diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index b9f5a8b42e661..d3482619aa2f1 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -324,7 +324,12 @@ static int rk_hash_run(struct crypto_engine *engine, void *breq) * efficiency, and make it response quickly when dma * complete. */ - readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v, v == 0, 10, 1000); + err = readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v, + v == 0, 10, 1000); + if (err) { + dev_err(rkc->dev, "HASH idle timeout\n"); + goto theend; + } for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) { v = readl(rkc->reg + RK_CRYPTO_HASH_DOUT_0 + i * 4); diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 965a03d5b27ae..d865fd4a098cb 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -2395,7 +2395,10 @@ static int sa_ul_probe(struct platform_device *pdev) return ret; } - sa_init_mem(dev_data); + ret = sa_init_mem(dev_data); + if (ret) + goto disable_pm; + ret = sa_dma_init(dev_data); if (ret) goto destroy_dma_pool; @@ -2430,6 +2433,7 @@ static int sa_ul_probe(struct platform_device *pdev) destroy_dma_pool: dma_pool_destroy(dev_data->sc_pool); +disable_pm: pm_runtime_put_sync(dev); pm_runtime_disable(dev); diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index d8d452cac3910..64ea141f018cb 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -88,7 +88,8 @@ static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request * } /* actual length may be less than dst buffer */ - akcipher_req->dst_len = len - sizeof(vc_req->status); + akcipher_req->dst_len = min_t(unsigned int, len - sizeof(vc_req->status), + akcipher_req->dst_len); sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst), vc_akcipher_req->dst_buf, akcipher_req->dst_len); virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error); diff --git a/drivers/cxl/core/features.c b/drivers/cxl/core/features.c index 85185af46b72d..ba6d2a5acb74a 100644 --- a/drivers/cxl/core/features.c +++ b/drivers/cxl/core/features.c @@ -225,7 +225,7 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, void *feat_out, size_t feat_out_size, u16 offset, u16 *return_code) { - size_t data_to_rd_size, size_out; + size_t data_to_rd_size; struct cxl_mbox_get_feat_in pi; struct cxl_mbox_cmd mbox_cmd; size_t data_rcvd_size = 0; @@ -237,9 +237,10 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, if (!feat_out || !feat_out_size) return 0; - size_out = min(feat_out_size, cxl_mbox->payload_size); uuid_copy(&pi.uuid, feat_uuid); pi.selection = selection; + + guard(mutex)(&cxl_mbox->feat_mutex); do { data_to_rd_size = min(feat_out_size - data_rcvd_size, cxl_mbox->payload_size); @@ -250,7 +251,7 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, .opcode = CXL_MBOX_OP_GET_FEATURE, .size_in = sizeof(pi), .payload_in = &pi, - .size_out = size_out, + .size_out = data_to_rd_size, .payload_out = feat_out + data_rcvd_size, .min_out = data_to_rd_size, }; @@ -314,6 +315,7 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox, data_in_size = cxl_mbox->payload_size - hdr_size; } + guard(mutex)(&cxl_mbox->feat_mutex); do { int rc; @@ -471,6 +473,10 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs, if (!count) return ERR_PTR(-EINVAL); + if (out_size < offsetof(struct fwctl_rpc_cxl_out, payload) || + count > out_size - offsetof(struct fwctl_rpc_cxl_out, payload)) + return ERR_PTR(-EINVAL); + struct fwctl_rpc_cxl_out *rpc_out __free(kvfree) = kvzalloc(out_size, GFP_KERNEL); if (!rpc_out) @@ -516,6 +522,9 @@ static void *cxlctl_set_feature(struct cxl_features_state *cxlfs, flags = le32_to_cpu(feat_in->flags); out_size = *out_len; + if (out_size < offsetof(struct fwctl_rpc_cxl_out, payload)) + return ERR_PTR(-EINVAL); + struct fwctl_rpc_cxl_out *rpc_out __free(kvfree) = kvzalloc(out_size, GFP_KERNEL); if (!rpc_out) @@ -649,7 +658,13 @@ static void *cxlctl_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope, struct cxl_memdev *cxlmd = fwctl_to_memdev(fwctl_dev); struct cxl_features_state *cxlfs = to_cxlfs(cxlmd->cxlds); const struct fwctl_rpc_cxl *rpc_in = in; - u16 opcode = rpc_in->opcode; + u16 opcode; + + if (in_len < sizeof(rpc_in->hdr) || + rpc_in->op_size > in_len - sizeof(rpc_in->hdr)) + return ERR_PTR(-EINVAL); + + opcode = rpc_in->opcode; if (!cxlctl_validate_hw_command(cxlfs, rpc_in, scope, opcode)) return ERR_PTR(-EINVAL); diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 7c6c5b7450a54..cc479f4322e73 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -11,7 +11,6 @@ #include "core.h" #include "trace.h" -#include "mce.h" static bool cxl_raw_allow_all; @@ -380,11 +379,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd, } } - /* Prepare to handle a full payload for variable sized output */ - if (out_size == CXL_VARIABLE_PAYLOAD) - mbox_cmd->size_out = cxl_mbox->payload_size; - else - mbox_cmd->size_out = out_size; + mbox_cmd->size_out = min_t(size_t, out_size, cxl_mbox->payload_size); if (mbox_cmd->size_out) { mbox_cmd->payload_out = kvzalloc(mbox_cmd->size_out, GFP_KERNEL); @@ -1455,6 +1450,11 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, if (rc) break; + if (!le16_to_cpu(po->count)) { + dev_dbg(&cxlmd->dev, "Poison empty payload!\n"); + break; + } + for (int i = 0; i < le16_to_cpu(po->count); i++) trace_cxl_poison(cxlmd, cxlr, &po->record[i], po->flags, po->overflow_ts, @@ -1516,6 +1516,7 @@ int cxl_mailbox_init(struct cxl_mailbox *cxl_mbox, struct device *host) cxl_mbox->host = host; mutex_init(&cxl_mbox->mbox_mutex); + mutex_init(&cxl_mbox->feat_mutex); rcuwait_init(&cxl_mbox->mbox_wait); return 0; @@ -1526,7 +1527,6 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial, u16 dvsec) { struct cxl_memdev_state *mds; - int rc; mds = devm_cxl_dev_state_create(dev, CXL_DEVTYPE_CLASSMEM, serial, dvsec, struct cxl_memdev_state, cxlds, @@ -1538,12 +1538,6 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial, mutex_init(&mds->event.log_lock); - rc = devm_cxl_register_mce_notifier(dev, &mds->mce_notifier); - if (rc == -EOPNOTSUPP) - dev_warn(dev, "CXL MCE unsupported\n"); - else if (rc) - return ERR_PTR(rc); - return mds; } EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, "CXL"); diff --git a/drivers/cxl/core/mce.c b/drivers/cxl/core/mce.c index ff8d078c6ca1e..65fed913b2217 100644 --- a/drivers/cxl/core/mce.c +++ b/drivers/cxl/core/mce.c @@ -4,16 +4,16 @@ #include #include #include -#include +#include +#include "core.h" #include "mce.h" static int cxl_handle_mce(struct notifier_block *nb, unsigned long val, void *data) { - struct cxl_memdev_state *mds = container_of(nb, struct cxl_memdev_state, - mce_notifier); - struct cxl_memdev *cxlmd = mds->cxlds.cxlmd; - struct cxl_port *endpoint = cxlmd->endpoint; + struct cxl_region *cxlr = container_of(nb, struct cxl_region, + mce_notifier); + struct cxl_region_params *p = &cxlr->params; struct mce *mce = data; u64 spa, spa_alias; unsigned long pfn; @@ -21,26 +21,25 @@ static int cxl_handle_mce(struct notifier_block *nb, unsigned long val, if (!mce || !mce_usable_address(mce)) return NOTIFY_DONE; - if (!endpoint) - return NOTIFY_DONE; - spa = mce->addr & MCI_ADDR_PHYSADDR; - pfn = spa >> PAGE_SHIFT; - if (!pfn_valid(pfn)) + if (!cxl_resource_contains_addr(p->res, spa)) return NOTIFY_DONE; - spa_alias = cxl_port_get_spa_cache_alias(endpoint, spa); - if (spa_alias == ~0ULL) - return NOTIFY_DONE; + if (spa >= p->res->start + p->cache_size) + spa_alias = spa - p->cache_size; + else + spa_alias = spa + p->cache_size; pfn = spa_alias >> PAGE_SHIFT; + if (!pfn_valid(pfn)) + return NOTIFY_DONE; /* * Take down the aliased memory page. The original memory page flagged * by the MCE will be taken cared of by the standard MCE handler. */ - dev_emerg(mds->cxlds.dev, "Offlining aliased SPA address0: %#llx\n", + dev_emerg(&cxlr->dev, "Offlining aliased SPA address0: %#llx\n", spa_alias); if (!memory_failure(pfn, 0)) set_mce_nospec(pfn); diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index 33a3d2e7b13af..45e7d2be17e09 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -921,7 +921,7 @@ static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, if (!size) return FW_UPLOAD_ERR_INVALID_SIZE; - mds->fw.oneshot = struct_size(transfer, data, size) < + mds->fw.oneshot = struct_size(transfer, data, size) <= cxl_mbox->payload_size; if (cxl_mem_get_fw_info(mds)) diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c index 68462e38a977b..5a3bb7e8a1f14 100644 --- a/drivers/cxl/core/pmem.c +++ b/drivers/cxl/core/pmem.c @@ -219,12 +219,14 @@ static struct cxl_nvdimm *cxl_nvdimm_alloc(struct cxl_nvdimm_bridge *cxl_nvb, dev->bus = &cxl_bus_type; dev->type = &cxl_nvdimm_type; /* - * A "%llx" string is 17-bytes vs dimm_id that is max - * NVDIMM_KEY_DESC_LEN + * dev_id is the nvdimm dimm_id used for security key lookup. + * It must match id_show(), which emits the CXL serial as an + * unsigned decimal. A u64 decimal string is at most 20 digits + * plus NUL. */ - BUILD_BUG_ON(sizeof(cxl_nvd->dev_id) < 17 || + BUILD_BUG_ON(sizeof(cxl_nvd->dev_id) < 21 || sizeof(cxl_nvd->dev_id) > NVDIMM_KEY_DESC_LEN); - sprintf(cxl_nvd->dev_id, "%llx", cxlmd->cxlds->serial); + sprintf(cxl_nvd->dev_id, "%llu", cxlmd->cxlds->serial); return cxl_nvd; } diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 1215ee4f40351..65f2d2f1eb003 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -1749,8 +1749,8 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, parent_dport, uport_dev, dport_dev); if (IS_ERR(dport)) { - /* Port already exists, restart iteration */ - if (PTR_ERR(dport) == -EAGAIN) + /* Port or dport already exists, restart iteration */ + if (PTR_ERR(dport) == -EAGAIN || PTR_ERR(dport) == -EBUSY) return 0; return PTR_ERR(dport); } diff --git a/drivers/cxl/core/ras.c b/drivers/cxl/core/ras.c index 99fb00949c2fa..bc74d4848132e 100644 --- a/drivers/cxl/core/ras.c +++ b/drivers/cxl/core/ras.c @@ -137,7 +137,6 @@ int cxl_ras_init(void) void cxl_ras_exit(void) { cxl_cper_unregister_prot_err_work(&cxl_cper_prot_err_work); - cancel_work_sync(&cxl_cper_prot_err_work); } static void cxl_dport_map_ras(struct cxl_dport *dport) diff --git a/drivers/cxl/core/ras_rch.c b/drivers/cxl/core/ras_rch.c index 0a8b3b9b63884..e0e01aa5eba6c 100644 --- a/drivers/cxl/core/ras_rch.c +++ b/drivers/cxl/core/ras_rch.c @@ -58,13 +58,28 @@ void cxl_disable_rch_root_ints(struct cxl_dport *dport) static bool cxl_rch_get_aer_info(void __iomem *aer_base, struct aer_capability_regs *aer_regs) { - int read_cnt = sizeof(struct aer_capability_regs) / sizeof(u32); + /* + * Bound the copy to the physically-defined AER registers (header + * through the 16-byte Header Log). struct aer_capability_regs is a + * software layout whose embedded struct pcie_tlp_log is larger than + * the on-wire AER capability; copying sizeof(*aer_regs) would + * over-read the RCRB-mapped MMIO block. + */ + int read_cnt = (PCI_ERR_HEADER_LOG + 16) / sizeof(u32); u32 *aer_regs_buf = (u32 *)aer_regs; int n; if (!aer_base) return false; + /* + * Zero the destination so the software-only tail fields + * (e.g. header_log.header_len) are deterministic rather than + * left as uninitialized stack, which could drive a bogus loop + * length in pcie_print_tlp_log(). + */ + memset(aer_regs, 0, sizeof(*aer_regs)); + /* Use readl() to guarantee 32-bit accesses */ for (n = 0; n < read_cnt; n++) aer_regs_buf[n] = readl(aer_base + n * sizeof(u32)); @@ -79,11 +94,11 @@ static bool cxl_rch_get_aer_info(void __iomem *aer_base, static bool cxl_rch_get_aer_severity(struct aer_capability_regs *aer_regs, int *severity) { - if (aer_regs->uncor_status & ~aer_regs->uncor_mask) { - if (aer_regs->uncor_status & PCI_ERR_ROOT_FATAL_RCV) - *severity = AER_FATAL; - else - *severity = AER_NONFATAL; + u32 uncor_status = aer_regs->uncor_status & ~aer_regs->uncor_mask; + + if (uncor_status) { + *severity = (uncor_status & aer_regs->uncor_severity) ? + AER_FATAL : AER_NONFATAL; return true; } diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 1e211542b6b64..8b0005a57d03c 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -15,6 +15,7 @@ #include #include #include "core.h" +#include "mce.h" /** * DOC: cxl core region @@ -1938,14 +1939,13 @@ static int find_pos_and_ways(struct cxl_port *port, struct range *range, break; } } - put_device(dev); - if (rc) dev_err(port->uport_dev, "failed to find %s:%s in target list of %s\n", dev_name(&port->dev), - dev_name(port->parent_dport->dport_dev), - dev_name(&cxlsd->cxld.dev)); + dev_name(port->parent_dport->dport_dev), dev_name(dev)); + + put_device(dev); return rc; } @@ -3859,34 +3859,6 @@ int cxl_add_to_region(struct cxl_endpoint_decoder *cxled) } EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL"); -u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa) -{ - struct cxl_region_ref *iter; - unsigned long index; - - if (!endpoint) - return ~0ULL; - - guard(rwsem_write)(&cxl_rwsem.region); - - xa_for_each(&endpoint->regions, index, iter) { - struct cxl_region_params *p = &iter->region->params; - - if (cxl_resource_contains_addr(p->res, spa)) { - if (!p->cache_size) - return ~0ULL; - - if (spa >= p->res->start + p->cache_size) - return spa - p->cache_size; - - return spa + p->cache_size; - } - } - - return ~0ULL; -} -EXPORT_SYMBOL_NS_GPL(cxl_port_get_spa_cache_alias, "CXL"); - static int is_system_ram(struct resource *res, void *arg) { struct cxl_region *cxlr = arg; @@ -4217,6 +4189,20 @@ static int cxl_region_probe(struct device *dev) if (rc) return rc; + /* + * Regions fronted by an extended linear cache need the MCE notifier to + * offline the aliased page on a memory error. + */ + if (p->cache_size) { + rc = devm_cxl_register_mce_notifier(&cxlr->dev, + &cxlr->mce_notifier); + if (rc == -EOPNOTSUPP) + dev_warn(&cxlr->dev, + "CONFIG_CXL_MCE disabled, MCE notifier not registered\n"); + else if (rc) + return rc; + } + rc = cxl_region_setup_poison(cxlr); if (rc) return rc; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index c0e5308e4d1b8..cab8ce39f4653 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -478,6 +478,7 @@ struct cxl_region_params { * @coord: QoS access coordinates for the region * @node_notifier: notifier for setting the access coordinates to node * @adist_notifier: notifier for calculating the abstract distance of node + * @mce_notifier: notifier for MCE */ struct cxl_region { struct device dev; @@ -493,6 +494,7 @@ struct cxl_region { struct access_coordinate coord[ACCESS_COORDINATE_MAX]; struct notifier_block node_notifier; struct notifier_block adist_notifier; + struct notifier_block mce_notifier; }; struct cxl_nvdimm_bridge { @@ -503,7 +505,8 @@ struct cxl_nvdimm_bridge { struct nvdimm_bus_descriptor nd_desc; }; -#define CXL_DEV_ID_LEN 19 +/* Holds a u64 serial as a decimal string: up to 20 digits + NUL */ +#define CXL_DEV_ID_LEN 21 enum { CXL_NVD_F_INVALIDATED = 0, @@ -870,7 +873,6 @@ bool is_cxl_pmem_region(struct device *dev); struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev); int cxl_add_to_region(struct cxl_endpoint_decoder *cxled); struct cxl_dax_region *to_cxl_dax_region(struct device *dev); -u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa); bool cxl_region_contains_resource(const struct resource *res); #else static inline bool is_cxl_pmem_region(struct device *dev) @@ -889,11 +891,6 @@ static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev) { return NULL; } -static inline u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, - u64 spa) -{ - return 0; -} static inline bool cxl_region_contains_resource(const struct resource *res) { return false; diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index ed419d0c59f2f..c288e624c1f68 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -431,7 +431,6 @@ static inline struct cxl_dev_state *mbox_to_cxlds(struct cxl_mailbox *cxl_mbox) * @poison: poison driver state info * @security: security driver state info * @fw: firmware upload / activation state - * @mce_notifier: MCE notifier * * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for * details on capacity parameters. @@ -451,7 +450,6 @@ struct cxl_memdev_state { struct cxl_poison_state poison; struct cxl_security_state security; struct cxl_fw_state fw; - struct notifier_block mce_notifier; }; static inline struct cxl_memdev_state * diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 267c679b0b3c2..d2f761f15a115 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -691,12 +691,6 @@ static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds) { int rc; - /* - * Fail the init if there's no mailbox. For a type3 this is out of spec. - */ - if (!cxlds->reg_map.device_map.mbox.valid) - return -ENODEV; - rc = cxl_mailbox_init(&cxlds->cxl_mbox, cxlds->dev); if (rc) return rc; @@ -829,10 +823,13 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) */ rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &cxlds->reg_map); - if (rc) + if (rc) { + if (rc == -EPROBE_DEFER) + return rc; dev_warn(&pdev->dev, "No component registers (%d)\n", rc); - else if (!cxlds->reg_map.component_map.ras.valid) + } else if (!cxlds->reg_map.component_map.ras.valid) { dev_dbg(&pdev->dev, "RAS registers not found\n"); + } rc = cxl_pci_type3_init_mailbox(cxlds); if (rc) @@ -1083,7 +1080,6 @@ static int __init cxl_pci_driver_init(void) static void __exit cxl_pci_driver_exit(void) { cxl_cper_unregister_work(&cxl_cper_work); - cancel_work_sync(&cxl_cper_work); pci_unregister_driver(&cxl_pci_driver); } diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index 261dff7ced9f1..a9f50281875d8 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -52,7 +52,7 @@ static ssize_t id_show(struct device *dev, struct device_attribute *attr, char * struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds; - return sysfs_emit(buf, "%lld\n", cxlds->serial); + return sysfs_emit(buf, "%llu\n", cxlds->serial); } static DEVICE_ATTR_RO(id); diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h index 81e4af49e39c1..607a53a91f58b 100644 --- a/drivers/dax/dax-private.h +++ b/drivers/dax/dax-private.h @@ -69,7 +69,6 @@ struct dev_dax_range { * data while the device is activated in the driver. * @region: parent region * @dax_dev: core dax functionality - * @virt_addr: kva from memremap; used by fsdev_dax * @cached_size: size of daxdev cached by fsdev_dax * @align: alignment of this instance * @target_node: effective numa node if dev_dax memory range is onlined @@ -85,7 +84,6 @@ struct dev_dax_range { struct dev_dax { struct dax_region *region; struct dax_device *dax_dev; - void *virt_addr; u64 cached_size; unsigned int align; int target_node; diff --git a/drivers/dax/fsdev.c b/drivers/dax/fsdev.c index 188b2526bee45..d50891d6dc135 100644 --- a/drivers/dax/fsdev.c +++ b/drivers/dax/fsdev.c @@ -51,9 +51,7 @@ static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, struct dev_dax *dev_dax = dax_get_private(dax_dev); size_t size = nr_pages << PAGE_SHIFT; size_t offset = pgoff << PAGE_SHIFT; - void *virt_addr = dev_dax->virt_addr + offset; phys_addr_t phys; - unsigned long local_pfn; phys = dax_pgoff_to_phys(dev_dax, pgoff, size); if (phys == -1) { @@ -63,11 +61,10 @@ static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, } if (kaddr) - *kaddr = virt_addr; + *kaddr = __va(phys); - local_pfn = PHYS_PFN(phys); if (pfn) - *pfn = local_pfn; + *pfn = PHYS_PFN(phys); /* * Use cached_size which was computed at probe time. The size cannot @@ -127,6 +124,23 @@ static void fsdev_clear_ops(void *data) dax_set_ops(dev_dax->dax_dev, NULL); } +static void fsdev_clear_pgmap_ops(void *data) +{ + struct dev_pagemap *pgmap = data; + + /* + * fsdev installs pgmap->ops and ->owner at probe. For a static device + * the pgmap is shared and long-lived (owned by the dax bus), so + * leaving fsdev's ops behind on unbind would let a later + * memory_failure -- after rebind to another driver, or after this + * module is unloaded -- dispatch through a stale or freed + * ->memory_failure handler. Clear them so the pgmap carries no fsdev + * state once we are unbound. + */ + pgmap->ops = NULL; + pgmap->owner = NULL; +} + /* * Page map operations for FS-DAX mode * Similar to fsdax_pagemap_ops in drivers/nvdimm/pmem.c @@ -135,11 +149,26 @@ static void fsdev_clear_ops(void *data) * The core mm code in free_zone_device_folio() handles the wake_up_var() * directly for this memory type. */ +static u64 fsdev_pfn_to_offset(struct dev_dax *dev_dax, unsigned long pfn) +{ + phys_addr_t phys = PFN_PHYS(pfn); + u64 offset = 0; + + for (int i = 0; i < dev_dax->nr_range; i++) { + struct range *range = &dev_dax->ranges[i].range; + + if (phys >= range->start && phys <= range->end) + return offset + (phys - range->start); + offset += range_len(range); + } + return -1ULL; +} + static int fsdev_pagemap_memory_failure(struct dev_pagemap *pgmap, unsigned long pfn, unsigned long nr_pages, int mf_flags) { struct dev_dax *dev_dax = pgmap->owner; - u64 offset = PFN_PHYS(pfn) - dev_dax->ranges[0].range.start; + u64 offset = fsdev_pfn_to_offset(dev_dax, pfn); u64 len = nr_pages << PAGE_SHIFT; return dax_holder_notify_failure(dev_dax->dax_dev, offset, @@ -204,46 +233,62 @@ static const struct file_operations fsdev_fops = { .release = fsdev_release, }; -static int fsdev_dax_probe(struct dev_dax *dev_dax) +/* + * Acquire the dev_pagemap for probe: the static (pre-populated) one if + * present, or a devm-allocated one for the dynamic case. Note that + * dev_dax->pgmap is not set here; fsdev_dax_probe() sets it only once + * probe succeeds, so a failed probe never leaves a dangling pointer + * to a devres-freed pgmap. + */ +static struct dev_pagemap *fsdev_acquire_pgmap(struct dev_dax *dev_dax) { - struct dax_device *dax_dev = dev_dax->dax_dev; struct device *dev = &dev_dax->dev; struct dev_pagemap *pgmap; - struct inode *inode; - u64 data_offset = 0; - struct cdev *cdev; - void *addr; - int rc, i; + size_t pgmap_size; if (static_dev_dax(dev_dax)) { if (dev_dax->nr_range > 1) { - dev_warn(dev, "static pgmap / multi-range device conflict\n"); - return -EINVAL; + dev_warn(dev, + "static pgmap / multi-range device conflict\n"); + return ERR_PTR(-EINVAL); } pgmap = dev_dax->pgmap; - } else { - size_t pgmap_size; + pgmap->vmemmap_shift = 0; + return pgmap; + } - if (dev_dax->pgmap) { - dev_warn(dev, "dynamic-dax with pre-populated page map\n"); - return -EINVAL; - } + if (dev_dax->pgmap) { + dev_warn(dev, "dynamic-dax with pre-populated page map\n"); + return ERR_PTR(-EINVAL); + } - pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1); - pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL); - if (!pgmap) - return -ENOMEM; + pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1); + pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL); + if (!pgmap) + return ERR_PTR(-ENOMEM); - pgmap->nr_range = dev_dax->nr_range; - dev_dax->pgmap = pgmap; + pgmap->nr_range = dev_dax->nr_range; + for (int i = 0; i < dev_dax->nr_range; i++) + pgmap->ranges[i] = dev_dax->ranges[i].range; - for (i = 0; i < dev_dax->nr_range; i++) { - struct range *range = &dev_dax->ranges[i].range; + return pgmap; +} - pgmap->ranges[i] = *range; - } - } +static int fsdev_dax_probe(struct dev_dax *dev_dax) +{ + struct dax_device *dax_dev = dev_dax->dax_dev; + struct device *dev = &dev_dax->dev; + struct dev_pagemap *pgmap; + struct inode *inode; + u64 data_offset = 0; + struct cdev *cdev; + void *addr; + int rc, i; + + pgmap = fsdev_acquire_pgmap(dev_dax); + if (IS_ERR(pgmap)) + return PTR_ERR(pgmap); for (i = 0; i < dev_dax->nr_range; i++) { struct range *range = &dev_dax->ranges[i].range; @@ -275,6 +320,11 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax) if (IS_ERR(addr)) return PTR_ERR(addr); + /* Drop fsdev's pgmap->ops/owner on unbind so no stale ops survive. */ + rc = devm_add_action_or_reset(dev, fsdev_clear_pgmap_ops, pgmap); + if (rc) + return rc; + /* * Clear any stale compound folio state left over from a previous * driver (e.g., device_dax with vmemmap_shift). Also register this @@ -290,15 +340,18 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax) /* Detect whether the data is at a non-zero offset into the memory */ if (pgmap->range.start != dev_dax->ranges[0].range.start) { u64 phys = dev_dax->ranges[0].range.start; - u64 pgmap_phys = dev_dax->pgmap[0].range.start; + u64 pgmap_phys = pgmap[0].range.start; - if (!WARN_ON(pgmap_phys > phys)) - data_offset = phys - pgmap_phys; + if (pgmap_phys > phys) { + dev_err(dev, "pgmap start %#llx exceeds data start %#llx\n", + pgmap_phys, phys); + return -EINVAL; + } + data_offset = phys - pgmap_phys; pr_debug("%s: offset detected phys=%llx pgmap_phys=%llx offset=%llx\n", __func__, phys, pgmap_phys, data_offset); } - dev_dax->virt_addr = addr + data_offset; inode = dax_inode(dax_dev); cdev = inode->i_cdev; @@ -323,7 +376,13 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax) return rc; run_dax(dax_dev); - return devm_add_action_or_reset(dev, fsdev_kill, dev_dax); + rc = devm_add_action_or_reset(dev, fsdev_kill, dev_dax); + if (rc) + return rc; + + /* Probe can no longer fail; expose the pgmap via dev_dax */ + dev_dax->pgmap = pgmap; + return 0; } static struct dax_device_driver fsdev_dax_driver = { diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 25cf99dd9360b..45f84b0eb909a 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -116,11 +116,47 @@ EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev); #if IS_ENABLED(CONFIG_FS_DAX) +/** + * fs_put_dax() - release holder ownership of a dax_device + * @dax_dev: dax device to release (may be NULL) + * @holder: the holder pointer previously passed to fs_dax_get() or + * fs_dax_get_by_bdev(); must match exactly, as it is used + * in a cmpxchg to atomically release ownership + * + * Must only be called by the current holder. Clears holder_ops before + * holder_data to avoid a race where a concurrent fs_dax_get() could have + * its newly installed holder_ops overwritten. + */ void fs_put_dax(struct dax_device *dax_dev, void *holder) { - if (dax_dev && holder && - cmpxchg(&dax_dev->holder_data, holder, NULL) == holder) - dax_dev->holder_ops = NULL; + if (dax_dev && holder) { + void *prev; + + /* + * Clear holder_ops before releasing holder_data. A concurrent + * dax_holder_notify_failure() that sees NULL ops returns + * -EOPNOTSUPP cleanly. A concurrent fs_dax_get() that acquires + * holder_data after the cmpxchg below is guaranteed to observe + * holder_ops=NULL first (cmpxchg provides release ordering), so + * its subsequent store of new ops will not be overwritten. + */ + WRITE_ONCE(dax_dev->holder_ops, NULL); + prev = cmpxchg(&dax_dev->holder_data, holder, NULL); + + /* + * prev == holder: normal release. + * prev == NULL: already released by kill_dax() when the + * device was removed under a live holder; + * not a bug. + * prev != holder (non-NULL): fs_put_dax() called by something + * that is not the current holder; an API + * contract violation. A lock would be needed + * to guard against this, but we WARN_ON() + * instead since violating the contract is + * a bug. + */ + WARN_ON(prev && prev != holder); + } put_dax(dax_dev); } EXPORT_SYMBOL_GPL(fs_put_dax); @@ -303,6 +339,7 @@ EXPORT_SYMBOL_GPL(dax_recovery_write); int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len, int mf_flags) { + const struct dax_holder_operations *ops; int rc, id; id = dax_read_lock(); @@ -311,12 +348,19 @@ int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, goto out; } - if (!dax_dev->holder_ops) { + /* + * Read holder_ops once: a concurrent fs_put_dax() can clear it without + * synchronizing against readers. Without the single fetch the compiler + * could reload between the NULL check and the call and dereference a + * NULL ops. + */ + ops = READ_ONCE(dax_dev->holder_ops); + if (!ops) { rc = -EOPNOTSUPP; goto out; } - rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags); + rc = ops->notify_failure(dax_dev, off, len, mf_flags); out: dax_read_unlock(id); return rc; diff --git a/drivers/dibs/dibs_main.c b/drivers/dibs/dibs_main.c index 2b53a9d277dca..20c50997a7cf2 100644 --- a/drivers/dibs/dibs_main.c +++ b/drivers/dibs/dibs_main.c @@ -251,13 +251,19 @@ static int __init dibs_init(void) rc = class_register(&dibs_class); if (rc) - return rc; + goto err; rc = dibs_loopback_init(); if (rc) - pr_err("%s fails with %d\n", __func__, rc); + goto err_unregister; return rc; + +err_unregister: + class_unregister(&dibs_class); +err: + pr_err("%s fails with %d\n", __func__, rc); + return rc; } static void __exit dibs_exit(void) diff --git a/drivers/dma-buf/dma-buf-mapping.c b/drivers/dma-buf/dma-buf-mapping.c index 794acff2546a3..833be519e1e6a 100644 --- a/drivers/dma-buf/dma-buf-mapping.c +++ b/drivers/dma-buf/dma-buf-mapping.c @@ -5,16 +5,18 @@ */ #include #include +#include +#include + +#define MAX_SG_ENT_SZ ALIGN_DOWN(UINT_MAX, PAGE_SIZE) static struct scatterlist *fill_sg_entry(struct scatterlist *sgl, size_t length, dma_addr_t addr) { - unsigned int len, nents; - int i; + size_t len; - nents = DIV_ROUND_UP(length, UINT_MAX); - for (i = 0; i < nents; i++) { - len = min_t(size_t, length, UINT_MAX); + while (length) { + len = min(length, MAX_SG_ENT_SZ); length -= len; /* * DMABUF abuses scatterlist to create a scatterlist @@ -24,8 +26,10 @@ static struct scatterlist *fill_sg_entry(struct scatterlist *sgl, size_t length, * does not require the CPU list for mapping or unmapping. */ sg_set_page(sgl, NULL, 0, 0); - sg_dma_address(sgl) = addr + (dma_addr_t)i * UINT_MAX; + sg_dma_address(sgl) = addr; sg_dma_len(sgl) = len; + addr += len; + /* Unconditionally advance. On last segment, this becomes NULL */ sgl = sg_next(sgl); } @@ -40,15 +44,19 @@ static unsigned int calc_sg_nents(struct dma_iova_state *state, size_t i; if (!state || !dma_use_iova(state)) { - for (i = 0; i < nr_ranges; i++) - nents += DIV_ROUND_UP(phys_vec[i].len, UINT_MAX); + for (i = 0; i < nr_ranges; i++) { + unsigned int added = DIV_ROUND_UP(phys_vec[i].len, MAX_SG_ENT_SZ); + + if (check_add_overflow(nents, added, &nents)) + return 0; + } } else { /* * In IOVA case, there is only one SG entry which spans * for whole IOVA address space, but we need to make sure * that it fits sg->length, maybe we need more. */ - nents = DIV_ROUND_UP(size, UINT_MAX); + nents = DIV_ROUND_UP(size, MAX_SG_ENT_SZ); } return nents; @@ -95,9 +103,10 @@ struct sg_table *dma_buf_phys_vec_to_sgt(struct dma_buf_attachment *attach, size_t nr_ranges, size_t size, enum dma_data_direction dir) { - unsigned int nents, mapped_len = 0; struct dma_buf_dma *dma; struct scatterlist *sgl; + size_t mapped_len = 0; + unsigned int nents; dma_addr_t addr; size_t i; int ret; @@ -133,6 +142,8 @@ struct sg_table *dma_buf_phys_vec_to_sgt(struct dma_buf_attachment *attach, } nents = calc_sg_nents(dma->state, phys_vec, nr_ranges, size); + + /* sg_alloc_table will cleanly fail and return -EINVAL if nents == 0 */ ret = sg_alloc_table(&dma->sgt, nents, GFP_KERNEL | __GFP_ZERO); if (ret) goto err_free_state; diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index d504c636dc294..4c9add51f9ef5 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -803,6 +803,26 @@ int dma_buf_fd(struct dma_buf *dmabuf, int flags) } EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF"); +/** + * dma_buf_fd_install - install a reserved fd for a dma-buf + * @dmabuf: [in] pointer to dma_buf + * @fd: [in] fd reserved with get_unused_fd_flags() + * + * Publishes a previously reserved fd into the caller's fd table. + * Must only be called after all fallible work (e.g. copy_to_user) + * has succeeded, as it cannot be undone safely once called. + * + * The caller is responsible for having emitted the trace event + * (via dma_buf_fd() or get_unused_fd_flags() + this function) + * before calling this. + */ +void dma_buf_fd_install(struct dma_buf *dmabuf, int fd) +{ + DMA_BUF_TRACE(trace_dma_buf_fd, dmabuf, fd); + fd_install(fd, dmabuf->file); +} +EXPORT_SYMBOL_NS_GPL(dma_buf_fd_install, "DMA_BUF"); + /** * dma_buf_get - returns the struct dma_buf related to an fd * @fd: [in] fd associated with the struct dma_buf to be returned diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 87797bea91cbd..1037a815aa0e5 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -1168,7 +1168,12 @@ const char __rcu *dma_fence_driver_name(struct dma_fence *fence) /* RCU protection is required for safe access to returned string */ ops = rcu_dereference(fence->ops); - if (ops) + + /* + * Make load ordering irrelevant by checking both signaled state and ops + * pointer and ops pointer is only set to NULL on newer implementations. + */ + if (!dma_fence_test_signaled_flag(fence) && ops) return (const char __rcu *)ops->get_driver_name(fence); else return (const char __rcu *)"detached-driver"; @@ -1201,7 +1206,12 @@ const char __rcu *dma_fence_timeline_name(struct dma_fence *fence) /* RCU protection is required for safe access to returned string */ ops = rcu_dereference(fence->ops); - if (ops) + + /* + * Make load ordering irrelevant by checking both signaled state and ops + * pointer and ops pointer is only set to NULL on newer implementations. + */ + if (!dma_fence_test_signaled_flag(fence) && ops) return (const char __rcu *)ops->get_timeline_name(fence); else return (const char __rcu *)"signaled-timeline"; diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c index a76bf3f8b071a..43c32fb283130 100644 --- a/drivers/dma-buf/dma-heap.c +++ b/drivers/dma-buf/dma-heap.c @@ -55,33 +55,6 @@ MODULE_PARM_DESC(mem_accounting, "Enable cgroup-based memory accounting for dma-buf heap allocations (default=false)."); EXPORT_SYMBOL_NS_GPL(mem_accounting, "DMA_BUF_HEAP"); -static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, - u32 fd_flags, - u64 heap_flags) -{ - struct dma_buf *dmabuf; - int fd; - - /* - * Allocations from all heaps have to begin - * and end on page boundaries. - */ - len = PAGE_ALIGN(len); - if (!len) - return -EINVAL; - - dmabuf = heap->ops->allocate(heap, len, fd_flags, heap_flags); - if (IS_ERR(dmabuf)) - return PTR_ERR(dmabuf); - - fd = dma_buf_fd(dmabuf, fd_flags); - if (fd < 0) { - dma_buf_put(dmabuf); - /* just return, as put will call release and that will free */ - } - return fd; -} - static int dma_heap_open(struct inode *inode, struct file *file) { struct dma_heap *heap; @@ -99,30 +72,42 @@ static int dma_heap_open(struct inode *inode, struct file *file) return 0; } -static long dma_heap_ioctl_allocate(struct file *file, void *data) +static struct dma_buf *dma_heap_ioctl_allocate(struct file *file, void *data) { struct dma_heap_allocation_data *heap_allocation = data; struct dma_heap *heap = file->private_data; + struct dma_buf *dmabuf; int fd; + size_t len; if (heap_allocation->fd) - return -EINVAL; + return ERR_PTR(-EINVAL); if (heap_allocation->fd_flags & ~DMA_HEAP_VALID_FD_FLAGS) - return -EINVAL; + return ERR_PTR(-EINVAL); if (heap_allocation->heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS) - return -EINVAL; + return ERR_PTR(-EINVAL); + + len = PAGE_ALIGN(heap_allocation->len); + if (!len) + return ERR_PTR(-EINVAL); + + dmabuf = heap->ops->allocate(heap, len, heap_allocation->fd_flags, + heap_allocation->heap_flags); - fd = dma_heap_buffer_alloc(heap, heap_allocation->len, - heap_allocation->fd_flags, - heap_allocation->heap_flags); - if (fd < 0) - return fd; + if (IS_ERR(dmabuf)) + return dmabuf; + + fd = get_unused_fd_flags(heap_allocation->fd_flags); + if (fd < 0) { + dma_buf_put(dmabuf); + return ERR_PTR(fd); + } heap_allocation->fd = fd; - return 0; + return dmabuf; } static unsigned int dma_heap_ioctl_cmds[] = { @@ -138,6 +123,8 @@ static long dma_heap_ioctl(struct file *file, unsigned int ucmd, unsigned int in_size, out_size, drv_size, ksize; int nr = _IOC_NR(ucmd); int ret = 0; + int fd; + struct dma_buf *dmabuf; if (nr >= ARRAY_SIZE(dma_heap_ioctl_cmds)) return -EINVAL; @@ -174,15 +161,28 @@ static long dma_heap_ioctl(struct file *file, unsigned int ucmd, switch (kcmd) { case DMA_HEAP_IOCTL_ALLOC: - ret = dma_heap_ioctl_allocate(file, kdata); + dmabuf = dma_heap_ioctl_allocate(file, kdata); + + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); + break; + } + + fd = ((struct dma_heap_allocation_data *)kdata)->fd; + if (copy_to_user((void __user *)arg, kdata, out_size) != 0) { + put_unused_fd(fd); + dma_buf_put(dmabuf); + ret = -EFAULT; + } else { + dma_buf_fd_install(dmabuf, fd); + } + break; default: ret = -ENOTTY; goto err; } - if (copy_to_user((void __user *)arg, kdata, out_size) != 0) - ret = -EFAULT; err: if (kdata != stack_kdata) kfree(kdata); diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 9049171df8578..5b5673fc7344f 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -428,6 +428,7 @@ static void dma_device_release(struct kref *ref) list_del_rcu(&device->global_node); dma_channel_rebalance(); + synchronize_rcu(); if (device->device_release) device->device_release(device); @@ -495,10 +496,13 @@ static int dma_chan_get(struct dma_chan *chan) */ static void dma_chan_put(struct dma_chan *chan) { + struct module *owner; + /* This channel is not in use, bail out */ if (!chan->client_count) return; + owner = dma_chan_to_owner(chan); chan->client_count--; /* This channel is not in use anymore, free it */ @@ -515,8 +519,10 @@ static void dma_chan_put(struct dma_chan *chan) chan->route_data = NULL; } - dma_device_put(chan->device); - module_put(dma_chan_to_owner(chan)); + /* This channel is not in use anymore, drop the device ref */ + if (!chan->client_count) + dma_device_put(chan->device); + module_put(owner); } enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 89a4c498a17b8..5fc025ab4a04a 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -201,6 +202,35 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan) return 1; } +static void dw_edma_terminate_vdesc(struct virt_dma_desc *vd) +{ + list_del(&vd->node); + dma_cookie_complete(&vd->tx); + vchan_terminate_vdesc(vd); +} + +static void dw_edma_terminate_vdesc_list(struct list_head *head) +{ + struct virt_dma_desc *vd, *_vd; + + list_for_each_entry_safe(vd, _vd, head, node) + dw_edma_terminate_vdesc(vd); +} + +/* Must be called with vc.lock held. */ +static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan) +{ + /* + * This order must not be reversed. Cookies are assigned when + * descriptors are submitted, so desc_issued contains older cookies + * than desc_submitted. Completing desc_submitted first could move + * chan->vc.chan.completed_cookie backwards when desc_issued is + * terminated afterwards. + */ + dw_edma_terminate_vdesc_list(&chan->vc.desc_issued); + dw_edma_terminate_vdesc_list(&chan->vc.desc_submitted); +} + static void dw_edma_device_caps(struct dma_chan *dchan, struct dma_slave_caps *caps) { @@ -272,6 +302,8 @@ static int dw_edma_device_pause(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) err = -EPERM; else if (chan->status != EDMA_ST_BUSY) @@ -289,6 +321,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) { err = -EPERM; } else if (chan->status != EDMA_ST_PAUSE) { @@ -297,7 +331,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan) err = -EPERM; } else { chan->status = EDMA_ST_BUSY; - dw_edma_start_transfer(chan); + if (!dw_edma_start_transfer(chan)) + chan->status = EDMA_ST_IDLE; } return err; @@ -308,25 +343,29 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) { - /* Do nothing */ + dw_edma_terminate_all_descs(chan); } else if (chan->status == EDMA_ST_PAUSE) { + dw_edma_terminate_all_descs(chan); chan->status = EDMA_ST_IDLE; - chan->configured = false; } else if (chan->status == EDMA_ST_IDLE) { - chan->configured = false; + dw_edma_terminate_all_descs(chan); } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { /* * The channel is in a false BUSY state, probably didn't * receive or lost an interrupt */ + dw_edma_terminate_all_descs(chan); chan->status = EDMA_ST_IDLE; - chan->configured = false; } else if (chan->request > EDMA_REQ_PAUSE) { err = -EPERM; } else { chan->request = EDMA_REQ_STOP; } + if (chan->status == EDMA_ST_IDLE) + chan->request = EDMA_REQ_NONE; return err; } @@ -336,11 +375,9 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long flags; - if (!chan->configured) - return; - spin_lock_irqsave(&chan->vc.lock, flags); - if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && + if (chan->configured && vchan_issue_pending(&chan->vc) && + chan->request == EDMA_REQ_NONE && chan->status == EDMA_ST_IDLE) { chan->status = EDMA_ST_BUSY; dw_edma_start_transfer(chan); @@ -655,10 +692,16 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) unsigned long flags; spin_lock_irqsave(&chan->vc.lock, flags); + if (chan->status == EDMA_ST_PAUSE) { + spin_unlock_irqrestore(&chan->vc.lock, flags); + return; + } + vd = vchan_next_desc(&chan->vc); if (vd) { switch (chan->request) { case EDMA_REQ_NONE: + case EDMA_REQ_PAUSE: desc = vd2dw_edma_desc(vd); if (!desc->chunks_alloc) { dw_hdma_set_callback_result(vd, @@ -667,23 +710,23 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) vchan_cookie_complete(vd); } + if (chan->request == EDMA_REQ_PAUSE) { + chan->request = EDMA_REQ_NONE; + chan->status = EDMA_ST_PAUSE; + break; + } + /* Continue transferring if there are remaining chunks or issued requests. */ chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; break; case EDMA_REQ_STOP: - list_del(&vd->node); - vchan_cookie_complete(vd); + dw_edma_terminate_all_descs(chan); chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; break; - case EDMA_REQ_PAUSE: - chan->request = EDMA_REQ_NONE; - chan->status = EDMA_ST_PAUSE; - break; - default: break; } @@ -698,14 +741,16 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) spin_lock_irqsave(&chan->vc.lock, flags); vd = vchan_next_desc(&chan->vc); - if (vd) { + if (vd && chan->request == EDMA_REQ_STOP) { + dw_edma_terminate_all_descs(chan); + } else if (vd) { dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED); list_del(&vd->node); vchan_cookie_complete(vd); } - spin_unlock_irqrestore(&chan->vc.lock, flags); chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; + spin_unlock_irqrestore(&chan->vc.lock, flags); } static void dw_edma_emul_irq_ack(struct irq_data *d) @@ -755,6 +800,7 @@ static int dw_edma_emul_irq_alloc(struct dw_edma *dw) return virq; irq_set_chip_and_handler(virq, &dw_edma_emul_irqchip, handle_level_irq); + irq_set_status_flags(virq, IRQ_LEVEL); irq_set_chip_data(virq, dw); irq_set_noprobe(virq); @@ -856,21 +902,49 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) return 0; } -static void dw_edma_free_chan_resources(struct dma_chan *dchan) +static void dw_edma_wait_termination(struct dma_chan *dchan) { + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long timeout = jiffies + msecs_to_jiffies(5000); - int ret; + bool stopping; + /* + * A STOP may be deferred to a later interrupt while the channel is still + * running. Wait until that handler completes the termination. + */ while (time_before(jiffies, timeout)) { - ret = dw_edma_device_terminate_all(dchan); - if (!ret) - break; + scoped_guard(spinlock_irqsave, &chan->vc.lock) + stopping = chan->request == EDMA_REQ_STOP; - if (time_after_eq(jiffies, timeout)) + if (!stopping) return; - cpu_relax(); + fsleep(1000); } + + dev_warn(chan->dw->chip->dev, + "timeout waiting for channel termination\n"); +} + +static void dw_edma_device_synchronize(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + dw_edma_wait_termination(dchan); + vchan_synchronize(&chan->vc); +} + +static void dw_edma_free_chan_resources(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + dw_edma_device_terminate_all(dchan); + dw_edma_device_synchronize(dchan); + + scoped_guard(spinlock_irqsave, &chan->vc.lock) + chan->configured = false; + + vchan_free_chan_resources(&chan->vc); } static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) @@ -929,7 +1003,6 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) else irq->rd_mask |= BIT(chan->id); - irq->dw = dw; memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", @@ -968,6 +1041,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) dma->device_pause = dw_edma_device_pause; dma->device_resume = dw_edma_device_resume; dma->device_terminate_all = dw_edma_device_terminate_all; + dma->device_synchronize = dw_edma_device_synchronize; dma->device_issue_pending = dw_edma_device_issue_pending; dma->device_tx_status = dw_edma_device_tx_status; dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; @@ -1010,6 +1084,7 @@ static int dw_edma_irq_request(struct dw_edma *dw, if (chip->nr_irqs == 1) { /* Common IRQ shared among all channels */ irq = chip->ops->irq_vector(dev, 0); + dw->irq[0].dw = dw; err = request_irq(irq, dw_edma_interrupt_common, IRQF_SHARED, dw->name, &dw->irq[0]); if (err) { @@ -1032,6 +1107,7 @@ static int dw_edma_irq_request(struct dw_edma *dw, for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { irq = chip->ops->irq_vector(dev, i); + dw->irq[i].dw = dw; err = request_irq(irq, i < *wr_alloc ? dw_edma_interrupt_write : diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 632abb8b481cf..2beec876b1842 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -79,7 +79,7 @@ static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan) u32 tmp; tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK, - GET_CH_32(dw, chan->id, chan->dir, ch_stat)); + GET_CH_32(dw, chan->dir, chan->id, ch_stat)); if (tmp == 1) return DMA_IN_PROGRESS; diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index 36155ab1602a9..d9fb717b5b53c 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -414,6 +414,8 @@ static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engi errirq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s-err", dev_name(&pdev->dev)); + if (!errirq_name) + return -ENOMEM; ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, fsl_edma3_err_handler_shared, 0, errirq_name, fsl_edma); diff --git a/drivers/dma/fsl-edma-trace.h b/drivers/dma/fsl-edma-trace.h index d3541301a2470..45d964a3726d6 100644 --- a/drivers/dma/fsl-edma-trace.h +++ b/drivers/dma/fsl-edma-trace.h @@ -19,14 +19,16 @@ DECLARE_EVENT_CLASS(edma_log_io, __field(struct fsl_edma_engine *, edma) __field(void __iomem *, addr) __field(u32, value) + __field(void __iomem *, membase) ), TP_fast_assign( __entry->edma = edma; __entry->addr = addr; __entry->value = value; + __entry->membase = edma->membase; ), TP_printk("offset %08x: value %08x", - (u32)(__entry->addr - __entry->edma->membase), __entry->value) + (u32)(__entry->addr - __entry->membase), __entry->value) ); DEFINE_EVENT(edma_log_io, edma_readl, diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c index 28bf818f9aa63..c751a2e49e6dc 100644 --- a/drivers/dma/hisi_dma.c +++ b/drivers/dma/hisi_dma.c @@ -983,7 +983,7 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num), GFP_KERNEL); if (!hdma_dev) - return -EINVAL; + return -ENOMEM; hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2]; hdma_dev->pdev = pdev; diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index c269d84d7bd2b..f74e9a3285887 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -531,7 +531,7 @@ static int mtk_uart_apdma_probe(struct platform_device *pdev) for (i = 0; i < mtkd->dma_requests; i++) { c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL); if (!c) { - rc = -ENODEV; + rc = -ENOMEM; goto err_no_dma; } diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index 386e85cd4882a..ed520737882bc 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -52,7 +52,6 @@ #define DCSR_EORINTR BIT(9) /* The end of Receive */ #define DRCMR_BASE 0x0100 -#define DRCMR_EXT_BASE_K3 0x1000 #define DRCMR_EXT_BASE_DEFAULT 0x1100 #define DRCMR_REQ_LIMIT 64 #define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */ @@ -713,7 +712,7 @@ mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, for_each_sg(sgl, sg, sg_len, i) { addr = sg_dma_address(sg); - avail = sg_dma_len(sgl); + avail = sg_dma_len(sg); do { len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES); @@ -1219,7 +1218,7 @@ static const struct mmp_pdma_ops spacemit_k3_pdma_ops = { .get_desc_dst_addr = get_desc_dst_addr_64, .run_bits = (DCSR_RUN | DCSR_LPAEEN | DCSR_EORIRQEN | DCSR_EORSTOPEN), .dma_width = 64, - .drcmr_ext_base = DRCMR_EXT_BASE_K3, + .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT, }; static const struct of_device_id mmp_pdma_dt_ids[] = { diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index fa2ee0b3e09f8..fc43124fefa89 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -744,6 +744,7 @@ pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc) sw_desc = kzalloc_flex(*sw_desc, hw_desc, nb_hw_desc, GFP_NOWAIT); if (!sw_desc) return NULL; + sw_desc->nb_desc = nb_hw_desc; sw_desc->desc_pool = chan->desc_pool; for (i = 0; i < nb_hw_desc; i++) { @@ -752,10 +753,10 @@ pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc) dev_err(&chan->vc.chan.dev->device, "%s(): Couldn't allocate the %dth hw_desc from dma_pool %p\n", __func__, i, sw_desc->desc_pool); + sw_desc->nb_desc = i; goto err; } - sw_desc->nb_desc++; sw_desc->hw_desc[i] = desc; if (i == 0) diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 1bb26af0405f3..05a3b1f9e0c23 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -1401,6 +1401,7 @@ static void bam_dma_remove(struct platform_device *pdev) struct bam_device *bdev = platform_get_drvdata(pdev); u32 i; + pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_force_suspend(&pdev->dev); of_dma_controller_free(pdev->dev.of_node); diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c index 087fea3af2e41..19b32a23c882d 100644 --- a/drivers/dma/sprd-dma.c +++ b/drivers/dma/sprd-dma.c @@ -1212,7 +1212,7 @@ static int sprd_dma_probe(struct platform_device *pdev) ret = pm_runtime_get_sync(&pdev->dev); if (ret < 0) - goto err_rpm; + goto err_register; ret = dma_async_device_register(&sdev->dma_dev); if (ret < 0) { @@ -1234,7 +1234,6 @@ static int sprd_dma_probe(struct platform_device *pdev) err_register: pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); -err_rpm: sprd_dma_disable(sdev); return ret; } diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index f47a326dd7ffa..7704b016aed8c 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -354,8 +354,10 @@ static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan) size_t bytes; dma_addr_t pos; - pos = readl(pchan->base + DMA_CHAN_LLI_ADDR); - bytes = readl(pchan->base + DMA_CHAN_CUR_CNT); + do { + pos = readl(pchan->base + DMA_CHAN_LLI_ADDR); + bytes = readl(pchan->base + DMA_CHAN_CUR_CNT); + } while (pos != readl(pchan->base + DMA_CHAN_LLI_ADDR)); if (pos == LLI_LAST_ITEM) return bytes; @@ -979,7 +981,6 @@ static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan, struct sun6i_pchan *pchan = vchan->phy; struct sun6i_dma_lli *lli; struct virt_dma_desc *vd; - struct sun6i_desc *txd; enum dma_status ret; unsigned long flags; size_t bytes = 0; @@ -991,9 +992,9 @@ static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan, spin_lock_irqsave(&vchan->vc.lock, flags); vd = vchan_find_desc(&vchan->vc, cookie); - txd = to_sun6i_desc(&vd->tx); if (vd) { + struct sun6i_desc *txd = to_sun6i_desc(&vd->tx); for (lli = txd->v_lli; lli != NULL; lli = lli->v_lli_next) bytes += lli->len; } else if (!pchan || !pchan->desc) { diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index 686dc140293eb..70eaf7ee57e68 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -1243,8 +1243,9 @@ void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) rx_chn->psil_paired = false; } - for (i = 0; i < rx_chn->flow_num; i++) - k3_udma_glue_release_rx_flow(rx_chn, i); + if (rx_chn->flows) + for (i = 0; i < rx_chn->flow_num; i++) + k3_udma_glue_release_rx_flow(rx_chn, i); if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) xudma_free_gp_rflow_range(rx_chn->common.udmax, diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 404235c173538..12cf1ca3a98a1 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -167,6 +167,8 @@ /* Delay loop counter to prevent hardware failure */ #define XILINX_DMA_LOOP_COUNT 1000000 +/* Delay between polls (avoid a delay of 0 to prevent CPU stalls) */ +#define XILINX_DMA_POLL_DELAY_US 10 /* AXI DMA Specific Registers/Offsets */ #define XILINX_DMA_REG_SRCDSTADDR 0x18 @@ -754,15 +756,25 @@ xilinx_aximcdma_alloc_tx_segment(struct xilinx_dma_chan *chan) return segment; } -static void xilinx_dma_clean_hw_desc(struct xilinx_axidma_desc_hw *hw) +static void xilinx_dma_clean_hw_desc(struct xilinx_dma_chan *chan, + struct xilinx_axidma_tx_segment *segment) { - u32 next_desc = hw->next_desc; - u32 next_desc_msb = hw->next_desc_msb; + dma_addr_t next; + u32 i; - memset(hw, 0, sizeof(struct xilinx_axidma_desc_hw)); + /* + * Restore the buffer descriptor's next descriptor pointer to the value + * set up in xilinx_dma_alloc_chan_resources(). Otherwise using the DMA + * in cyclic mode leaves the next descriptor pointer altered and + * prevents subsequent non-cyclic transfers. + */ + i = segment - chan->seg_v; + next = chan->seg_p + + sizeof(*chan->seg_v) * ((i + 1) % XILINX_DMA_NUM_DESCS); - hw->next_desc = next_desc; - hw->next_desc_msb = next_desc_msb; + memset(&segment->hw, 0, sizeof(segment->hw)); + segment->hw.next_desc = lower_32_bits(next); + segment->hw.next_desc_msb = upper_32_bits(next); } static void xilinx_mcdma_clean_hw_desc(struct xilinx_aximcdma_desc_hw *hw) @@ -784,7 +796,7 @@ static void xilinx_mcdma_clean_hw_desc(struct xilinx_aximcdma_desc_hw *hw) static void xilinx_dma_free_tx_segment(struct xilinx_dma_chan *chan, struct xilinx_axidma_tx_segment *segment) { - xilinx_dma_clean_hw_desc(&segment->hw); + xilinx_dma_clean_hw_desc(chan, segment); list_add_tail(&segment->node, &chan->free_seg_list); } @@ -918,9 +930,9 @@ static void xilinx_dma_free_descriptors(struct xilinx_dma_chan *chan) spin_lock_irqsave(&chan->lock, flags); - xilinx_dma_free_desc_list(chan, &chan->pending_list); xilinx_dma_free_desc_list(chan, &chan->done_list); xilinx_dma_free_desc_list(chan, &chan->active_list); + xilinx_dma_free_desc_list(chan, &chan->pending_list); spin_unlock_irqrestore(&chan->lock, flags); } @@ -1324,7 +1336,8 @@ static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan) /* Wait for the hardware to halt */ return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - val & XILINX_DMA_DMASR_HALTED, 0, + val & XILINX_DMA_DMASR_HALTED, + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); } @@ -1339,7 +1352,8 @@ static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan) u32 val; return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - val & XILINX_DMA_DMASR_IDLE, 0, + val & XILINX_DMA_DMASR_IDLE, + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); } @@ -1356,7 +1370,8 @@ static void xilinx_dma_start(struct xilinx_dma_chan *chan) /* Wait for the hardware to start */ err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - !(val & XILINX_DMA_DMASR_HALTED), 0, + !(val & XILINX_DMA_DMASR_HALTED), + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); if (err) { @@ -1794,7 +1809,8 @@ static int xilinx_dma_reset(struct xilinx_dma_chan *chan) /* Wait for the hardware to finish reset */ err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp, - !(tmp & XILINX_DMA_DMACR_RESET), 0, + !(tmp & XILINX_DMA_DMACR_RESET), + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); if (err) { @@ -1893,8 +1909,10 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) if (status & XILINX_MCDMA_IRQ_IOC_MASK) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); - chan->idle = true; - chan->start_transfer(chan); + if (list_empty(&chan->active_list)) { + chan->idle = true; + chan->start_transfer(chan); + } spin_unlock(&chan->lock); } @@ -1950,8 +1968,10 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) XILINX_DMA_DMASR_DLY_CNT_IRQ)) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); - chan->idle = true; - chan->start_transfer(chan); + if (list_empty(&chan->active_list)) { + chan->idle = true; + chan->start_transfer(chan); + } spin_unlock(&chan->lock); } diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c index f6a812e49ddca..ca2dee0012c87 100644 --- a/drivers/dma/xilinx/zynqmp_dma.c +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -1170,9 +1170,9 @@ static void zynqmp_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&zdev->common); zynqmp_dma_chan_remove(zdev->chan); - if (pm_runtime_active(zdev->dev)) - zynqmp_dma_runtime_suspend(zdev->dev); pm_runtime_disable(zdev->dev); + if (!pm_runtime_status_suspended(zdev->dev)) + zynqmp_dma_runtime_suspend(zdev->dev); } static const struct of_device_id zynqmp_dma_of_match[] = { diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c index 43d51d942eadd..a320eeb829ad2 100644 --- a/drivers/dpll/dpll_core.c +++ b/drivers/dpll/dpll_core.c @@ -876,19 +876,25 @@ int dpll_pin_register(struct dpll_device *dpll, struct dpll_pin *pin, const struct dpll_pin_ops *ops, void *priv) { + const struct dpll_device_ops *dev_ops; int ret; if (WARN_ON(!ops) || WARN_ON(!ops->state_on_dpll_get) || WARN_ON(!ops->direction_get) || - WARN_ON(ops->measured_freq_get && - (!dpll_device_ops(dpll)->freq_monitor_get || - !dpll_device_ops(dpll)->freq_monitor_set)) || WARN_ON(ops->supported_ffo && !ops->ffo_get)) return -EINVAL; mutex_lock(&dpll_lock); + dev_ops = dpll_device_ops(dpll); + if (WARN_ON(ops->measured_freq_get && + (!dev_ops || !dev_ops->freq_monitor_get || + !dev_ops->freq_monitor_set))) { + ret = -EINVAL; + goto out_unlock; + } + /* * For pins identified via firmware (pin->fwnode), allow registration * even if the pin's (module, clock_id) differs from the target DPLL. @@ -1081,12 +1087,8 @@ EXPORT_SYMBOL_GPL(dpll_pin_ref_sync_pair_add); static struct dpll_device_registration * dpll_device_registration_first(struct dpll_device *dpll) { - struct dpll_device_registration *reg; - - reg = list_first_entry_or_null((struct list_head *)&dpll->registration_list, - struct dpll_device_registration, list); - WARN_ON(!reg); - return reg; + return list_first_entry_or_null((struct list_head *)&dpll->registration_list, + struct dpll_device_registration, list); } void *dpll_priv(struct dpll_device *dpll) @@ -1094,6 +1096,8 @@ void *dpll_priv(struct dpll_device *dpll) struct dpll_device_registration *reg; reg = dpll_device_registration_first(dpll); + if (!reg) + return NULL; return reg->priv; } @@ -1102,6 +1106,8 @@ const struct dpll_device_ops *dpll_device_ops(struct dpll_device *dpll) struct dpll_device_registration *reg; reg = dpll_device_registration_first(dpll); + if (!reg) + return NULL; return reg->ops; } diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index afb31c0040382..9f274c6253c66 100644 --- a/drivers/dpll/dpll_netlink.c +++ b/drivers/dpll/dpll_netlink.c @@ -66,6 +66,22 @@ static bool dpll_pin_available(struct dpll_pin *pin) return false; } +static bool dpll_device_registered(struct dpll_device *dpll) +{ + return dpll_device_ops(dpll); +} + +static struct dpll_pin_ref *dpll_pin_first_registered_ref(struct dpll_pin *pin) +{ + struct dpll_pin_ref *ref; + unsigned long i; + + xa_for_each(&pin->dpll_refs, i, ref) + if (dpll_device_registered(ref->dpll)) + return ref; + return NULL; +} + /** * dpll_msg_add_pin_handle - attach pin handle attribute to a given message * @msg: pointer to sk_buff message to attach a pin handle @@ -656,6 +672,8 @@ dpll_msg_add_pin_dplls(struct sk_buff *msg, struct dpll_pin *pin, int ret; xa_for_each(&pin->dpll_refs, index, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; attr = nla_nest_start(msg, DPLL_A_PIN_PARENT_DEVICE); if (!attr) return -EMSGSIZE; @@ -700,9 +718,10 @@ dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin, int ret; ref = dpll_pin_own_dpll_ref_first(pin); + if (!ref || !dpll_device_registered(ref->dpll)) + ref = dpll_pin_first_registered_ref(pin); if (!ref) - ref = dpll_xa_ref_dpll_first(&pin->dpll_refs); - ASSERT_NOT_NULL(ref); + return -ENODEV; ret = dpll_msg_add_pin_handle(msg, pin); if (ret) @@ -1091,6 +1110,8 @@ dpll_pin_freq_set(struct dpll_pin *pin, struct nlattr *a, } xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if ((!ops->frequency_set || !ops->frequency_get) && ref->dpll->module == pin->module && @@ -1101,7 +1122,7 @@ dpll_pin_freq_set(struct dpll_pin *pin, struct nlattr *a, } } ref = dpll_pin_own_dpll_ref_first(pin); - if (!ref) { + if (!ref || !dpll_device_registered(ref->dpll)) { NL_SET_ERR_MSG(extack, "pin owner dpll not found"); return -ENODEV; } @@ -1117,6 +1138,8 @@ dpll_pin_freq_set(struct dpll_pin *pin, struct nlattr *a, return 0; xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->frequency_set) continue; @@ -1138,6 +1161,8 @@ dpll_pin_freq_set(struct dpll_pin *pin, struct nlattr *a, xa_for_each(&pin->dpll_refs, i, ref) { if (ref == failed) break; + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->frequency_set) continue; @@ -1163,6 +1188,8 @@ dpll_pin_esync_set(struct dpll_pin *pin, struct nlattr *a, int ret; xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if ((!ops->esync_set || !ops->esync_get) && ref->dpll->module == pin->module && @@ -1173,7 +1200,7 @@ dpll_pin_esync_set(struct dpll_pin *pin, struct nlattr *a, } } ref = dpll_pin_own_dpll_ref_first(pin); - if (!ref) { + if (!ref || !dpll_device_registered(ref->dpll)) { NL_SET_ERR_MSG(extack, "pin owner dpll not found"); return -ENODEV; } @@ -1199,6 +1226,8 @@ dpll_pin_esync_set(struct dpll_pin *pin, struct nlattr *a, xa_for_each(&pin->dpll_refs, i, ref) { void *pin_dpll_priv; + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->esync_set) continue; @@ -1224,6 +1253,8 @@ dpll_pin_esync_set(struct dpll_pin *pin, struct nlattr *a, if (ref == failed) break; + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->esync_set) continue; @@ -1251,8 +1282,7 @@ dpll_pin_ref_sync_state_set(struct dpll_pin *pin, unsigned long i; int ret; - ref_sync_pin = xa_find(&pin->ref_sync_pins, &ref_sync_pin_idx, - ULONG_MAX, XA_PRESENT); + ref_sync_pin = xa_load(&pin->ref_sync_pins, ref_sync_pin_idx); if (!ref_sync_pin) { NL_SET_ERR_MSG(extack, "reference sync pin not found"); return -EINVAL; @@ -1262,7 +1292,7 @@ dpll_pin_ref_sync_state_set(struct dpll_pin *pin, return -EINVAL; } ref = dpll_pin_own_dpll_ref_first(pin); - if (!ref) { + if (!ref || !dpll_device_registered(ref->dpll)) { NL_SET_ERR_MSG(extack, "pin owner dpll not found"); return -ENODEV; } @@ -1283,6 +1313,8 @@ dpll_pin_ref_sync_state_set(struct dpll_pin *pin, if (state == old_state) return 0; xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->ref_sync_set) continue; @@ -1307,6 +1339,8 @@ dpll_pin_ref_sync_state_set(struct dpll_pin *pin, xa_for_each(&pin->dpll_refs, i, ref) { if (ref == failed) break; + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->ref_sync_set) continue; @@ -1500,6 +1534,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr, } xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if ((!ops->phase_adjust_set || !ops->phase_adjust_get) && ref->dpll->module == pin->module && @@ -1509,7 +1545,7 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr, } } ref = dpll_pin_own_dpll_ref_first(pin); - if (!ref) { + if (!ref || !dpll_device_registered(ref->dpll)) { NL_SET_ERR_MSG(extack, "pin owner dpll not found"); return -ENODEV; } @@ -1526,6 +1562,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr, return 0; xa_for_each(&pin->dpll_refs, i, ref) { + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->phase_adjust_set) continue; @@ -1550,6 +1588,8 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr, xa_for_each(&pin->dpll_refs, i, ref) { if (ref == failed) break; + if (!dpll_device_registered(ref->dpll)) + continue; ops = dpll_pin_ops(ref); if (!ops->phase_adjust_set) continue; @@ -1581,7 +1621,7 @@ dpll_pin_parent_device_set(struct dpll_pin *pin, struct nlattr *parent_nest, return -EINVAL; } pdpll_idx = nla_get_u32(tb[DPLL_A_PIN_PARENT_ID]); - dpll = xa_load(&dpll_device_xa, pdpll_idx); + dpll = dpll_device_get_by_id(pdpll_idx); if (!dpll) { NL_SET_ERR_MSG(extack, "parent device not found"); return -EINVAL; @@ -1873,6 +1913,10 @@ int dpll_nl_pin_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) ret = dpll_cmd_pin_get_one(skb, pin, cb->extack); if (ret) { genlmsg_cancel(skb, hdr); + if (ret == -ENODEV) { + ret = 0; + continue; + } break; } genlmsg_end(skb, hdr); diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 4edd2088c2db6..5914b2fd94d9c 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -1533,7 +1533,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) altdev = dci->pvt_info; *altdev = *device; - if (!devres_open_group(&altdev->ddev, altr_portb_setup, GFP_KERNEL)) + if (!devres_open_group(device->edac->dev, altr_portb_setup, GFP_KERNEL)) return -ENOMEM; /* Update PortB specific values */ @@ -1562,7 +1562,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) rc = -ENODEV; goto err_release_group_1; } - rc = devm_request_irq(&altdev->ddev, altdev->sb_irq, + rc = devm_request_irq(device->edac->dev, altdev->sb_irq, prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { @@ -1585,7 +1585,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) rc = -ENODEV; goto err_release_group_1; } - rc = devm_request_irq(&altdev->ddev, altdev->db_irq, + rc = devm_request_irq(device->edac->dev, altdev->db_irq, prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { @@ -1605,13 +1605,13 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) list_add(&altdev->next, &altdev->edac->a10_ecc_devices); - devres_remove_group(&altdev->ddev, altr_portb_setup); + devres_remove_group(device->edac->dev, altr_portb_setup); return 0; err_release_group_1: edac_device_free_ctl_info(dci); - devres_release_group(&altdev->ddev, altr_portb_setup); + devres_release_group(device->edac->dev, altr_portb_setup); edac_printk(KERN_ERR, EDAC_DEVICE, "%s:Error setting up EDAC device: %d\n", ecc_name, rc); return rc; diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index c6aa69dbd9fb1..475235c402e88 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -4173,6 +4173,8 @@ static int __init amd64_edac_init(void) goto err_pci; } + request_module_nowait("amd_atl"); + /* register stuff with EDAC MCE */ if (boot_cpu_data.x86 >= 0x17) { amd_register_ecc_decoder(decode_umc_error); diff --git a/drivers/edac/edac_device_sysfs.c b/drivers/edac/edac_device_sysfs.c index b1c2717cd0237..6995ce039db97 100644 --- a/drivers/edac/edac_device_sysfs.c +++ b/drivers/edac/edac_device_sysfs.c @@ -88,14 +88,21 @@ static ssize_t edac_device_ctl_poll_msec_store(struct edac_device_ctl_info *ctl_info, const char *data, size_t count) { - unsigned long value; + unsigned int value; + int ret; /* get the value and enforce that it is non-zero, must be at least * one millisecond for the delay period, between scans * Then cancel last outstanding delay for the work request * and set a new one. */ - value = simple_strtoul(data, NULL, 0); + ret = kstrtouint(data, 0, &value); + if (ret < 0) + return ret; + + if (value < 1) + return -EINVAL; + edac_device_reset_delay_period(ctl_info, value); return count; diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c index f1fc20d4ebf61..12d718a50e1c9 100644 --- a/drivers/edac/igen6_edac.c +++ b/drivers/edac/igen6_edac.c @@ -175,8 +175,6 @@ static struct res_config { /* Set imc->dimm_{l_size,s_size,l_map}[chan]. */ void (*set_dimm_params)(struct igen6_imc *imc, int chan); bool (*ibecc_available)(struct pci_dev *pdev); - /* Extract error address logged in IBECC */ - u64 (*err_addr)(u64 ecclog); /* Convert error address logged in IBECC to system physical address */ u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc); /* Convert error address logged in IBECC to integrated memory controller address */ @@ -522,11 +520,6 @@ static u64 adl_err_addr_to_imc_addr(u64 eaddr, int mc) return imc_addr; } -static u64 rpl_p_err_addr(u64 ecclog) -{ - return field_get(res_cfg->reg_eccerrlog_addr_mask, ecclog); -} - static enum mem_type ptl_h_get_mem_type(struct igen6_imc *imc) { u32 mtype, val; @@ -716,22 +709,6 @@ static struct res_config adl_n_cfg = { .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, }; -static struct res_config rpl_p_cfg = { - .machine_check = true, - .num_imc = 2, - .reg_mchbar_mask = GENMASK_ULL(41, 17), - .reg_tom_mask = GENMASK_ULL(41, 20), - .reg_touud_mask = GENMASK_ULL(41, 20), - .reg_eccerrlog_addr_mask = GENMASK_ULL(45, 5), - .imc_base = 0xd800, - .ibecc_base = 0xd400, - .ibecc_error_log_offset = 0x68, - .ibecc_available = tgl_ibecc_available, - .err_addr = rpl_p_err_addr, - .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, - .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, -}; - static struct res_config mtl_ps_cfg = { .machine_check = true, .num_imc = 2, @@ -877,11 +854,11 @@ static struct pci_device_id igen6_pci_tbl[] = { { PCI_VDEVICE(INTEL, DID_ASL_SKU1), .driver_data = (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_ASL_SKU2), .driver_data = (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_ASL_SKU3), .driver_data = (kernel_ulong_t)&adl_n_cfg }, - { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_RPL_P_SKU4), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, - { PCI_VDEVICE(INTEL, DID_RPL_P_SKU5), .driver_data = (kernel_ulong_t)&rpl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg }, + { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg }, + { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg }, + { PCI_VDEVICE(INTEL, DID_RPL_P_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg }, + { PCI_VDEVICE(INTEL, DID_RPL_P_SKU5), .driver_data = (kernel_ulong_t)&adl_cfg }, { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU1), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU2), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU3), .driver_data = (kernel_ulong_t)&mtl_ps_cfg }, @@ -1009,14 +986,22 @@ static void set_dimm_params(struct igen6_imc *imc, int chan) static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit) { - u64 hash_addr = addr & mask, hash = 0; - u64 intlv = (addr >> intlv_bit) & 1; + u64 hash_addr, hash = 0; int i; + /* + * In hash mode, the @intlv_bit is the lowest selected bit of @addr + * to be XORed. While @mask may or may not include this @intlv_bit, + * we enforce that @mask includes @intlv_bit to ensure @intlv_bit is + * XORed exactly once. + */ + mask |= 1 << intlv_bit; + hash_addr = addr & mask; + for (i = 6; i < 20; i++) hash ^= (hash_addr >> i) & 1; - return (int)hash ^ intlv; + return (int)hash; } static u64 decode_channel_addr(u64 addr, int intlv_bit) @@ -1035,19 +1020,18 @@ static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map, { int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6; - if (addr > 2 * s_size) { + if (addr >= 2 * s_size) { *sub_addr = addr - s_size; *idx = l_map; return; } - if (CHANNEL_HASH_MODE(hash)) { - *sub_addr = decode_channel_addr(addr, intlv_bit); + *sub_addr = decode_channel_addr(addr, intlv_bit); + + if (CHANNEL_HASH_MODE(hash)) *idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit); - } else { - *sub_addr = decode_channel_addr(addr, 6); - *idx = GET_BITFIELD(addr, 6, 6); - } + else + *idx = GET_BITFIELD(addr, intlv_bit, intlv_bit); } static int igen6_decode(struct decoded_addr *res) @@ -1230,11 +1214,7 @@ static void ecclog_work_cb(struct work_struct *work) llist_for_each_entry_safe(node, tmp, head, llnode) { memset(&res, 0, sizeof(res)); - if (res_cfg->err_addr) - eaddr = res_cfg->err_addr(node->ecclog); - else - eaddr = node->ecclog & res_cfg->reg_eccerrlog_addr_mask; - + eaddr = node->ecclog & res_cfg->reg_eccerrlog_addr_mask; res.mc = node->mc; res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr, res.mc); res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr, res.mc); diff --git a/drivers/firewire/.kunitconfig b/drivers/firewire/.kunitconfig index 21b7e9eef63d1..7406acb004782 100644 --- a/drivers/firewire/.kunitconfig +++ b/drivers/firewire/.kunitconfig @@ -6,3 +6,4 @@ CONFIG_FIREWIRE_KUNIT_DEVICE_ATTRIBUTE_TEST=y CONFIG_FIREWIRE_KUNIT_PACKET_SERDES_TEST=y CONFIG_FIREWIRE_KUNIT_SELF_ID_SEQUENCE_HELPER_TEST=y CONFIG_FIREWIRE_KUNIT_OHCI_SERDES_TEST=y +CONFIG_FIREWIRE_KUNIT_NODE_TREE_TEST=y diff --git a/drivers/firewire/Kconfig b/drivers/firewire/Kconfig index a5f5e250223a1..b5abe00accc9d 100644 --- a/drivers/firewire/Kconfig +++ b/drivers/firewire/Kconfig @@ -81,6 +81,21 @@ config FIREWIRE_KUNIT_SELF_ID_SEQUENCE_HELPER_TEST For more information on KUnit and unit tests in general, refer to the KUnit documentation in Documentation/dev-tools/kunit/. +config FIREWIRE_KUNIT_NODE_TREE_TEST + tristate "KUnit tests for node tree" if !KUNIT_ALL_TESTS + depends on FIREWIRE && KUNIT + default KUNIT_ALL_TESTS + help + This builds the KUnit tests for node tree. + + KUnit tests run during boot and output the results to the debug + log in TAP format (https://testanything.org/). Only useful for + kernel devs running KUnit test harness and are not for inclusion + into a production build. + + For more information on KUnit and unit tests in general, refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + config FIREWIRE_OHCI tristate "OHCI-1394 controllers" depends on PCI && FIREWIRE diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c index e49d8a58be09e..664952a67a11a 100644 --- a/drivers/firewire/core-cdev.c +++ b/drivers/firewire/core-cdev.c @@ -1397,8 +1397,10 @@ static void iso_resource_auto_work(struct work_struct *work) } else { // Transit from allocation to reallocation, except if the client requested // deallocation in the meantime. - scoped_guard(spinlock_irq, &client->lock) - r->todo = ISO_RES_AUTO_REALLOC; + scoped_guard(spinlock_irq, &client->lock) { + if (r->todo == ISO_RES_AUTO_ALLOC) + r->todo = ISO_RES_AUTO_REALLOC; + } if (channel >= 0) r->params.channels_mask = BIT_ULL(channel); diff --git a/drivers/firewire/core-topology.c b/drivers/firewire/core-topology.c index df2ac0dab1060..ee6b54f89859e 100644 --- a/drivers/firewire/core-topology.c +++ b/drivers/firewire/core-topology.c @@ -88,6 +88,17 @@ static inline struct fw_node *fw_node(struct list_head *l) return list_entry(l, struct fw_node, link); } +typedef void (*fw_node_callback_t)(struct fw_card *card, struct fw_node *node, + struct fw_node *parent); + +static void for_each_fw_node(struct fw_card *card, struct fw_node *root, + fw_node_callback_t callback); + +static void free_fw_node(struct fw_card *card, struct fw_node *node, struct fw_node *parent) +{ + kfree(node); +} + /* * This function builds the tree representation of the topology given * by the self IDs from the latest bus reset. During the construction @@ -119,8 +130,8 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self while (enumerator.quadlet_count > 0) { unsigned int child_port_count = 0; + unsigned int parent_port_count = 0; unsigned int total_port_count = 0; - unsigned int parent_count = 0; unsigned int quadlet_count; const u32 *self_id_sequence; unsigned int port_capacity; @@ -134,7 +145,7 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self if (PTR_ERR(self_id_sequence) != -ENODATA) { fw_err(card, "inconsistent extended self IDs: %ld\n", PTR_ERR(self_id_sequence)); - return NULL; + goto error; } break; } @@ -148,26 +159,38 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self switch (port_status) { case PHY_PACKET_SELF_ID_PORT_STATUS_CHILD: ++child_port_count; - fallthrough; + break; case PHY_PACKET_SELF_ID_PORT_STATUS_PARENT: + ++parent_port_count; + break; case PHY_PACKET_SELF_ID_PORT_STATUS_NCONN: ++total_port_count; - fallthrough; + break; case PHY_PACKET_SELF_ID_PORT_STATUS_NONE: default: break; } } + total_port_count += child_port_count + parent_port_count; + + // Check that the node reports exactly one parent port, except for the root, which + // of course should have no parents. + if ((enumerator.quadlet_count == 0 && parent_port_count != 0) || + (enumerator.quadlet_count > 0 && parent_port_count != 1)) { + fw_err(card, "parent port inconsistency for node %d: parent_count=%d\n", + phy_id, parent_port_count); + goto error; + } if (phy_id != phy_packet_self_id_get_phy_id(self_id_sequence[0])) { fw_err(card, "PHY ID mismatch in self ID: %d != %d\n", phy_id, phy_packet_self_id_get_phy_id(self_id_sequence[0])); - return NULL; + goto error; } if (child_port_count > stack_depth) { fw_err(card, "topology stack underflow\n"); - return NULL; + goto error; } /* @@ -185,7 +208,7 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self node = fw_node_create(self_id_sequence[0], total_port_count, card->color); if (node == NULL) { fw_err(card, "out of memory while building topology\n"); - return NULL; + goto error; } if (phy_id == (card->node_id & 0x3f)) @@ -203,7 +226,6 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self // we temporarily abuse node->color for remembering the entry in // the node->ports array where the parent node should be. Later, // when we handle the parent node, we fix up the reference. - ++parent_count; node->color = port_index; break; @@ -221,21 +243,12 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self } } - // Check that the node reports exactly one parent port, except for the root, which - // of course should have no parents. - if ((enumerator.quadlet_count == 0 && parent_count != 0) || - (enumerator.quadlet_count > 0 && parent_count != 1)) { - fw_err(card, "parent port inconsistency for node %d: " - "parent_count=%d\n", phy_id, parent_count); - return NULL; - } - /* Pop the child nodes off the stack and push the new node. */ __list_del(h->prev, &stack); list_add_tail(&node->link, &stack); stack_depth += 1 - child_port_count; - if (node->phy_speed == SCODE_BETA && parent_count + child_port_count > 1) + if (node->phy_speed == SCODE_BETA && parent_port_count + child_port_count > 1) beta_repeaters_present = true; // If PHYs report different gap counts, set an invalid count which will force a gap @@ -254,12 +267,13 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self card->beta_repeaters_present = beta_repeaters_present; return local_node; +error: + ++card->color; + list_for_each_entry_safe(node, child, &stack, link) + for_each_fw_node(card, node, free_fw_node); + return NULL; } -typedef void (*fw_node_callback_t)(struct fw_card * card, - struct fw_node * node, - struct fw_node * parent); - static void for_each_fw_node(struct fw_card *card, struct fw_node *root, fw_node_callback_t callback) { @@ -507,3 +521,7 @@ void fw_core_handle_bus_reset(struct fw_card *card, int node_id, int generation, } } EXPORT_SYMBOL(fw_core_handle_bus_reset); + +#ifdef CONFIG_FIREWIRE_KUNIT_NODE_TREE_TEST +#include "node-tree-test.c" +#endif diff --git a/drivers/firewire/node-tree-test.c b/drivers/firewire/node-tree-test.c new file mode 100644 index 0000000000000..5e6bb58537e94 --- /dev/null +++ b/drivers/firewire/node-tree-test.c @@ -0,0 +1,607 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// node-tree-test.c - An application of Kunit to test node tree. +// +// Copyright (c) 2026 Takashi Sakamoto +// +// This file can not be built independently since it is intentionally included in core-topology.c. + +#include +#include +#include + +struct private_data { + struct fw_card *card; + unsigned int release_count; +}; + +static int node_tree_test_init(struct kunit *test) +{ + struct private_data *data; + + data = kunit_kzalloc(test, sizeof(*data), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + + data->card = kunit_kzalloc(test, sizeof(struct fw_card), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data->card); + + data->card->device = kunit_device_register(test, "dummy-device"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, data->card->device); + + test->priv = data; + + return 0; +} + +static void node_tree_test_exit(struct kunit *test) +{ + struct private_data *data = test->priv; + + kunit_device_unregister(test, data->card->device); + kunit_kfree(test, data->card); + kunit_kfree(test, data); +} + +static void release_fw_node(struct fw_card *card, struct fw_node *node, struct fw_node *parent) +{ + struct private_data *data = kunit_get_current_test()->priv; + + fw_node_put(node); + ++data->release_count; +} + +static void node_tree_test_two_nodes(struct kunit *test) +{ + // root + // ++============++ + // || phy 1 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | + // +-----+ + // | + // ++===|==x==x==++ + // || P0 P1 P2 || + // || phy 0 || + // ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000080, + 0x8100005e, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x01; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 2); +} + +static void node_tree_test_two_nodes_1394a(struct kunit *test) +{ + // root + // ++===============++ + // || phy 0 || + // || P0 P1 P2 P3 || + // ++===|==|==|==|==++ + // | + // +--+ + // | + // ++===|==|==|==|==|==++ + // || P0 P1 P2 P3 P4 || + // || phy 1 || + // ++==================++ + // + // NOTE: Just for Self-ID Packets Zero and One. + static const u32 self_id_sequence[] = { + 0x80000065, 0x80814000, + 0x8100005d, 0x81810000, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x01; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 4); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + KUNIT_EXPECT_NULL(test, node->ports[3]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 5); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + KUNIT_EXPECT_NULL(test, node->ports[2]); + KUNIT_EXPECT_NULL(test, node->ports[3]); + KUNIT_EXPECT_NULL(test, node->ports[4]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 2); +} + +static void node_tree_test_three_nodes_case0(struct kunit *test) +{ + // root + // ++============++ + // || phy 2 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | | + // +--+ +-----------------+ + // | | + // ++===|==|==x==++ ++===|==|==|==++ + // || P0 P1 P2 || || P0 P1 P2 || + // || phy 0 || || phy 1 || + // ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000060, + 0x81000058, + 0x820000dc, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x02; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[2], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 3); +} + +static void node_tree_test_three_nodes_case1(struct kunit *test) +{ + // root + // ++============++ + // || phy 2 || + // || P0 P1 P2 || + // ++===|==|==x==++ + // | + // | +-----------+ + // | | | + // ++===|==|==|==++ ++===|==x==x==++ + // || P0 P1 P2 || || P0 P1 P2 || + // || phy 1 || || phy 0 || + // ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000080, + 0x8100006c, + 0x82000070, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x02; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + + struct fw_node *parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 3); +} + +static void node_tree_test_four_nodes_case0(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | + // | +-----------+ +--------------+ + // | | | | | + // ++===|==|==|==++ ++===|==|==x==++ ++===|==x==x==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 2 || || phy 1 || || phy 0 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000080, + 0x810000b0, + 0x8200006c, + 0x83000074, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + + parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_four_nodes_case1(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==x==++ + // | + // | +--------------------------------+ + // | | +-----------+ | + // ++===|==|==|==++ ++===|==x==x==++ ++===|==|==|==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 2 || || phy 1 || || phy 0 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000d0, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + + struct fw_node *parent = node; + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NULL(test, node->ports[2]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_four_nodes_case2(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | | + // | +-----------------------------+ + // | +--------------+ | + // ++===|==|==x==++ ++===|==|==|==++ ++===|==x==x==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 1 || || phy 0 || || phy 2 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000094, + 0x810000b0, + 0x82000080, + 0x830000dc, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + + parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NULL(test, node->ports[2]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_four_nodes_case3(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | | +--------------------------------+ + // | +--------------------+ | + // | | | + // ++===|==|==x==++ ++===|==|==|==++ ++===|==|==x==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 0 || || phy 1 || || phy 2 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000090, + 0x81000058, + 0x82000060, + 0x830000fc, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[2], parent); + + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NULL(test, node->ports[1]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_invalid_extended_self_id_sequence(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // content of self ID packet for the phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000d1, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static void node_tree_test_invalid_phy_id(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // phy ID for phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x8f0000d0, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static void node_tree_test_invalid_child_port_count(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // count of child ports for phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000fc, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static void node_tree_test_invalid_parent_port_count(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // count of parent ports for phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000e8, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static struct kunit_case node_tree_test_cases[] = { + KUNIT_CASE(node_tree_test_two_nodes), + KUNIT_CASE(node_tree_test_two_nodes_1394a), + KUNIT_CASE(node_tree_test_three_nodes_case0), + KUNIT_CASE(node_tree_test_three_nodes_case1), + KUNIT_CASE(node_tree_test_four_nodes_case0), + KUNIT_CASE(node_tree_test_four_nodes_case1), + KUNIT_CASE(node_tree_test_four_nodes_case2), + KUNIT_CASE(node_tree_test_four_nodes_case3), + KUNIT_CASE(node_tree_test_invalid_extended_self_id_sequence), + KUNIT_CASE(node_tree_test_invalid_phy_id), + KUNIT_CASE(node_tree_test_invalid_child_port_count), + KUNIT_CASE(node_tree_test_invalid_parent_port_count), + {} +}; + +static struct kunit_suite node_tree_test_suite = { + .name = "firewire-node-tree", + .init = node_tree_test_init, + .exit = node_tree_test_exit, + .test_cases = node_tree_test_cases, +}; +kunit_test_suite(node_tree_test_suite); diff --git a/drivers/firmware/arm_scmi/bus.c b/drivers/firmware/arm_scmi/bus.c index 793be9eabaedd..e060edbe7e832 100644 --- a/drivers/firmware/arm_scmi/bus.c +++ b/drivers/firmware/arm_scmi/bus.c @@ -7,7 +7,6 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include #include #include #include @@ -33,8 +32,8 @@ struct scmi_requested_dev { struct list_head node; }; -/* Track globally the creation of SCMI SystemPower related devices */ -static atomic_t scmi_syspower_registered = ATOMIC_INIT(0); +/* Track globally the SCMI SystemPower protocol device. */ +static struct scmi_device *scmi_syspower_registered; /** * scmi_protocol_device_request - Helper to request a device @@ -136,17 +135,6 @@ static int scmi_protocol_device_request(const struct scmi_device_id *id_table) return ret; } -static int scmi_protocol_table_register(const struct scmi_device_id *id_table) -{ - int ret = 0; - const struct scmi_device_id *entry; - - for (entry = id_table; entry->name && ret == 0; entry++) - ret = scmi_protocol_device_request(entry); - - return ret; -} - /** * scmi_protocol_device_unrequest - Helper to unrequest a device * @@ -159,6 +147,7 @@ static int scmi_protocol_table_register(const struct scmi_device_id *id_table) */ static void scmi_protocol_device_unrequest(const struct scmi_device_id *id_table) { + struct scmi_requested_dev *rdev, *victim = NULL; struct list_head *phead; pr_debug("Unrequesting SCMI device (%s) for protocol %x\n", @@ -167,29 +156,48 @@ static void scmi_protocol_device_unrequest(const struct scmi_device_id *id_table mutex_lock(&scmi_requested_devices_mtx); phead = idr_find(&scmi_requested_devices, id_table->protocol_id); if (phead) { - struct scmi_requested_dev *victim, *tmp; - - list_for_each_entry_safe(victim, tmp, phead, node) { - if (!strcmp(victim->id_table->name, id_table->name)) { - list_del(&victim->node); - - mutex_unlock(&scmi_requested_devices_mtx); - blocking_notifier_call_chain(&scmi_requested_devices_nh, - SCMI_BUS_NOTIFY_DEVICE_UNREQUEST, - (void *)victim->id_table); - kfree(victim); - mutex_lock(&scmi_requested_devices_mtx); + list_for_each_entry(rdev, phead, node) { + if (!strcmp(rdev->id_table->name, id_table->name)) { + victim = rdev; + list_del(&rdev->node); break; } } - if (list_empty(phead)) { + if (victim && list_empty(phead)) { idr_remove(&scmi_requested_devices, id_table->protocol_id); kfree(phead); } } mutex_unlock(&scmi_requested_devices_mtx); + + if (victim) { + blocking_notifier_call_chain(&scmi_requested_devices_nh, + SCMI_BUS_NOTIFY_DEVICE_UNREQUEST, + (void *)victim->id_table); + kfree(victim); + } +} + +static int scmi_protocol_table_register(const struct scmi_device_id *id_table) +{ + const struct scmi_device_id *entry; + int ret; + + for (entry = id_table; entry->name; entry++) { + ret = scmi_protocol_device_request(entry); + if (ret) + goto err_unrequest; + } + + return 0; + +err_unrequest: + while (entry != id_table) + scmi_protocol_device_unrequest(--entry); + + return ret; } static void @@ -201,21 +209,33 @@ scmi_protocol_table_unregister(const struct scmi_device_id *id_table) scmi_protocol_device_unrequest(entry); } -static int scmi_dev_match_by_id_table(struct scmi_device *scmi_dev, - const struct scmi_device_id *id_table) +static bool scmi_device_is_transport(const struct scmi_device *scmi_dev) +{ + return !strncmp(scmi_dev->name, SCMI_TRANSPORT_DEVNAME_PREFIX, + strlen(SCMI_TRANSPORT_DEVNAME_PREFIX)); +} + +static int __scmi_dev_match_by_id_table(struct scmi_device *scmi_dev, + const struct scmi_device_id *id_table, + bool skip_transport) { if (!id_table || !id_table->name) return 0; - /* Always skip transport devices from matching */ for (; id_table->protocol_id && id_table->name; id_table++) if (id_table->protocol_id == scmi_dev->protocol_id && - strncmp(scmi_dev->name, "__scmi_transport_device", 23) && + !(skip_transport && scmi_device_is_transport(scmi_dev)) && !strcmp(id_table->name, scmi_dev->name)) return 1; return 0; } +static int scmi_dev_match_by_id_table(struct scmi_device *scmi_dev, + const struct scmi_device_id *id_table) +{ + return __scmi_dev_match_by_id_table(scmi_dev, id_table, true); +} + static int scmi_dev_match_id(struct scmi_device *scmi_dev, const struct scmi_driver *scmi_drv) { @@ -235,11 +255,12 @@ static int scmi_match_by_id_table(struct device *dev, const void *data) struct scmi_device *scmi_dev = to_scmi_dev(dev); const struct scmi_device_id *id_table = data; - return scmi_dev_match_by_id_table(scmi_dev, id_table); + return __scmi_dev_match_by_id_table(scmi_dev, id_table, false); } -static struct scmi_device *scmi_child_dev_find(struct device *parent, - int prot_id, const char *name) +/* Returns a device_find_child() reference which must be dropped by caller. */ +static struct scmi_device * +scmi_child_dev_find_get(struct device *parent, int prot_id, const char *name) { struct scmi_device_id id_table[2] = { 0 }; struct device *dev; @@ -251,9 +272,6 @@ static struct scmi_device *scmi_child_dev_find(struct device *parent, if (!dev) return NULL; - /* Drop the refcnt bumped implicitly by device_find_child */ - put_device(dev); - return to_scmi_dev(dev); } @@ -377,10 +395,14 @@ int scmi_driver_register(struct scmi_driver *driver, struct module *owner, driver->driver.mod_name = mod_name; retval = driver_register(&driver->driver); - if (!retval) - pr_debug("Registered new scmi driver %s\n", driver->name); + if (retval) { + scmi_protocol_table_unregister(driver->id_table); + return retval; + } - return retval; + pr_debug("Registered new scmi driver %s\n", driver->name); + + return 0; } EXPORT_SYMBOL_GPL(scmi_driver_register); @@ -391,10 +413,23 @@ void scmi_driver_unregister(struct scmi_driver *driver) } EXPORT_SYMBOL_GPL(scmi_driver_unregister); +static void scmi_device_release_resources(struct scmi_device *scmi_dev) +{ + if (scmi_dev->protocol_id == SCMI_PROTOCOL_SYSTEM) + cmpxchg(&scmi_syspower_registered, scmi_dev, NULL); + + if (scmi_dev->id) { + ida_free(&scmi_bus_id, scmi_dev->id); + scmi_dev->id = 0; + } +} + static void scmi_device_release(struct device *dev) { struct scmi_device *scmi_dev = to_scmi_dev(dev); + scmi_device_release_resources(scmi_dev); + of_node_put(dev->of_node); kfree_const(scmi_dev->name); kfree(scmi_dev); } @@ -406,11 +441,9 @@ static void __scmi_device_destroy(struct scmi_device *scmi_dev) dev_name(&scmi_dev->dev), scmi_dev->protocol_id, scmi_dev->name); - if (scmi_dev->protocol_id == SCMI_PROTOCOL_SYSTEM) - atomic_set(&scmi_syspower_registered, 0); - - ida_free(&scmi_bus_id, scmi_dev->id); - device_unregister(&scmi_dev->dev); + device_del(&scmi_dev->dev); + scmi_device_release_resources(scmi_dev); + put_device(&scmi_dev->dev); } static struct scmi_device * @@ -419,6 +452,7 @@ __scmi_device_create(struct device_node *np, struct device *parent, { int id, retval; struct scmi_device *scmi_dev; + bool syspower = (protocol == SCMI_PROTOCOL_SYSTEM); /* * If the same protocol/name device already exist under the same parent @@ -427,45 +461,41 @@ __scmi_device_create(struct device_node *np, struct device *parent, * each DT defined protocol at probe time, and the concurrent * registration of SCMI drivers. */ - scmi_dev = scmi_child_dev_find(parent, protocol, name); - if (scmi_dev) + scmi_dev = scmi_child_dev_find_get(parent, protocol, name); + if (scmi_dev) { + put_device(&scmi_dev->dev); return scmi_dev; - - /* - * Ignore any possible subsequent failures while creating the device - * since we are doomed anyway at that point; not using a mutex which - * spans across this whole function to keep things simple and to avoid - * to serialize all the __scmi_device_create calls across possibly - * different SCMI server instances (parent) - */ - if (protocol == SCMI_PROTOCOL_SYSTEM && - atomic_cmpxchg(&scmi_syspower_registered, 0, 1)) { - dev_warn(parent, - "SCMI SystemPower protocol device must be unique !\n"); - return NULL; } scmi_dev = kzalloc_obj(*scmi_dev); if (!scmi_dev) return NULL; - scmi_dev->name = kstrdup_const(name ?: "unknown", GFP_KERNEL); - if (!scmi_dev->name) { + scmi_dev->protocol_id = protocol; + + /* + * Reserve the singleton SystemPower protocol device using the device + * pointer itself, so delayed release of an older device cannot clear + * a reservation owned by a newer device. + */ + if (syspower && cmpxchg(&scmi_syspower_registered, NULL, scmi_dev)) { + dev_warn(parent, + "SCMI SystemPower protocol device must be unique !\n"); kfree(scmi_dev); return NULL; } + scmi_dev->name = kstrdup_const(name ?: "unknown", GFP_KERNEL); + if (!scmi_dev->name) + goto free_dev; + id = ida_alloc_min(&scmi_bus_id, 1, GFP_KERNEL); - if (id < 0) { - kfree_const(scmi_dev->name); - kfree(scmi_dev); - return NULL; - } + if (id < 0) + goto free_name; scmi_dev->id = id; - scmi_dev->protocol_id = protocol; scmi_dev->dev.parent = parent; - device_set_node(&scmi_dev->dev, of_fwnode_handle(np)); + device_set_node(&scmi_dev->dev, of_fwnode_handle(of_node_get(np))); scmi_dev->dev.bus = &scmi_bus_type; scmi_dev->dev.release = scmi_device_release; dev_set_name(&scmi_dev->dev, "scmi_dev.%d", id); @@ -479,8 +509,14 @@ __scmi_device_create(struct device_node *np, struct device *parent, return scmi_dev; put_dev: + scmi_device_release_resources(scmi_dev); put_device(&scmi_dev->dev); - ida_free(&scmi_bus_id, id); + return NULL; +free_name: + kfree_const(scmi_dev->name); +free_dev: + scmi_device_release_resources(scmi_dev); + kfree(scmi_dev); return NULL; } @@ -561,9 +597,11 @@ void scmi_device_destroy(struct device *parent, int protocol, const char *name) { struct scmi_device *scmi_dev; - scmi_dev = scmi_child_dev_find(parent, protocol, name); - if (scmi_dev) + scmi_dev = scmi_child_dev_find_get(parent, protocol, name); + if (scmi_dev) { __scmi_device_destroy(scmi_dev); + put_device(&scmi_dev->dev); + } } EXPORT_SYMBOL_GPL(scmi_device_destroy); diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h index b9723c105fc1b..fe8c22cfb9f7d 100644 --- a/drivers/firmware/arm_scmi/common.h +++ b/drivers/firmware/arm_scmi/common.h @@ -34,6 +34,8 @@ #define SCMI_SHMEM_MAX_PAYLOAD_SIZE 104 +#define SCMI_TRANSPORT_DEVNAME_PREFIX "__scmi_transport_device" + enum scmi_error_codes { SCMI_SUCCESS = 0, /* Success */ SCMI_ERR_SUPPORT = -1, /* Not supported */ diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 3e0d975ec94c4..ef29fd223287d 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -2628,21 +2629,31 @@ static int scmi_handle_put(const struct scmi_handle *handle) return 0; } -static void scmi_device_link_add(struct device *consumer, +static bool scmi_device_link_add(struct device *consumer, struct device *supplier) { struct device_link *link; link = device_link_add(consumer, supplier, DL_FLAG_AUTOREMOVE_CONSUMER); - WARN_ON(!link); + return !WARN_ON(!link); +} + +static void scmi_clear_handle(struct scmi_device *scmi_dev) +{ + if (!scmi_dev->handle) + return; + + scmi_handle_put(scmi_dev->handle); + scmi_dev->handle = NULL; } static void scmi_set_handle(struct scmi_device *scmi_dev) { scmi_dev->handle = scmi_handle_get(&scmi_dev->dev); - if (scmi_dev->handle) - scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev); + if (scmi_dev->handle && + !scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev)) + scmi_clear_handle(scmi_dev); } static int __scmi_xfer_info_init(struct scmi_info *sinfo, @@ -2751,6 +2762,9 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, idx = tx ? 0 : 1; idr = tx ? &info->tx_idr : &info->rx_idr; + if (idr_find(idr, prot_id)) + return -EEXIST; + if (!info->desc->ops->chan_available(of_node, idx)) { cinfo = idr_find(idr, SCMI_PROTOCOL_BASE); if (unlikely(!cinfo)) /* Possible only if platform has no Rx */ @@ -2768,7 +2782,7 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, cinfo->no_completion_irq = info->desc->no_completion_irq; /* Create a unique name for this transport device */ - snprintf(name, 32, "__scmi_transport_device_%s_%02X", + snprintf(name, sizeof(name), SCMI_TRANSPORT_DEVNAME_PREFIX "_%s_%02X", idx ? "rx" : "tx", prot_id); /* Create a uniquely named, dedicated transport device for this chan */ tdev = scmi_device_create(of_node, info->dev, prot_id, name); @@ -2778,13 +2792,12 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, devm_kfree(info->dev, cinfo); return -EINVAL; } - of_node_get(of_node); cinfo->id = prot_id; cinfo->dev = &tdev->dev; + cinfo->handle = &info->handle; ret = info->desc->ops->chan_setup(cinfo, info->dev, tx); if (ret) { - of_node_put(of_node); scmi_device_destroy(info->dev, prot_id, name); devm_kfree(info->dev, cinfo); return ret; @@ -2807,14 +2820,13 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, "unable to allocate SCMI idr slot err %d\n", ret); /* Destroy channel and device only if created by this call. */ if (tdev) { - of_node_put(of_node); + info->desc->ops->chan_free(prot_id, cinfo, idr); scmi_device_destroy(info->dev, prot_id, name); devm_kfree(info->dev, cinfo); } return ret; } - cinfo->handle = &info->handle; return 0; } @@ -2872,9 +2884,11 @@ static int scmi_channels_setup(struct scmi_info *info) if (of_property_read_u32(child, "reg", &prot_id)) continue; - if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) + if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) { dev_err(info->dev, "Out of range protocol %d\n", prot_id); + continue; + } ret = scmi_txrx_setup(info, child, prot_id); if (ret) @@ -2884,7 +2898,7 @@ static int scmi_channels_setup(struct scmi_info *info) return 0; } -static int scmi_chan_destroy(int id, void *p, void *idr) +static int scmi_chan_destroy(int id, void *p, void *data) { struct scmi_chan_info *cinfo = p; @@ -2892,13 +2906,10 @@ static int scmi_chan_destroy(int id, void *p, void *idr) struct scmi_info *info = handle_to_scmi_info(cinfo->handle); struct scmi_device *sdev = to_scmi_dev(cinfo->dev); - of_node_put(cinfo->dev->of_node); - scmi_device_destroy(info->dev, id, sdev->name); + scmi_device_destroy(info->dev, cinfo->id, sdev->name); cinfo->dev = NULL; } - idr_remove(idr, id); - return 0; } @@ -2925,6 +2936,7 @@ static int scmi_bus_notifier(struct notifier_block *nb, { struct scmi_info *info = bus_nb_to_scmi_info(nb); struct scmi_device *sdev = to_scmi_dev(data); + const char *status; /* Skip devices of different SCMI instances */ if (sdev->dev.parent != info->dev) @@ -2934,18 +2946,22 @@ static int scmi_bus_notifier(struct notifier_block *nb, case BUS_NOTIFY_BIND_DRIVER: /* setup handle now as the transport is ready */ scmi_set_handle(sdev); + status = "about to be BOUND."; + break; + case BUS_NOTIFY_DRIVER_NOT_BOUND: + scmi_clear_handle(sdev); + status = "NOT BOUND."; break; case BUS_NOTIFY_UNBOUND_DRIVER: - scmi_handle_put(sdev->handle); - sdev->handle = NULL; + scmi_clear_handle(sdev); + status = "UNBOUND."; break; default: return NOTIFY_DONE; } dev_dbg(info->dev, "Device %s (%s) is now %s\n", dev_name(&sdev->dev), - sdev->name, action == BUS_NOTIFY_BIND_DRIVER ? - "about to be BOUND." : "UNBOUND."); + sdev->name, status); return NOTIFY_OK; } @@ -2957,7 +2973,9 @@ static int scmi_device_request_notifier(struct notifier_block *nb, struct scmi_device_id *id_table = data; struct scmi_info *info = req_nb_to_scmi_info(nb); + rcu_read_lock(); np = idr_find(&info->active_protocols, id_table->protocol_id); + rcu_read_unlock(); if (!np) return NOTIFY_DONE; @@ -3263,7 +3281,7 @@ static int scmi_probe(struct platform_device *pdev) ret = scmi_channels_setup(info); if (ret) { err_str = "failed to setup channels\n"; - goto clear_ida; + goto clear_txrx_setup; } ret = bus_register_notifier(&scmi_bus_type, &info->bus_nb); @@ -3325,7 +3343,7 @@ static int scmi_probe(struct platform_device *pdev) dev_err(dev, "%s", err_str); return 0; } - goto notification_exit; + goto raw_mode_cleanup; } mutex_lock(&scmi_list_mutex); @@ -3340,8 +3358,10 @@ static int scmi_probe(struct platform_device *pdev) if (of_property_read_u32(child, "reg", &prot_id)) continue; - if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) + if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) { dev_err(dev, "Out of range protocol %d\n", prot_id); + continue; + } if (!scmi_is_protocol_implemented(handle, prot_id)) { dev_err(dev, "SCMI protocol %d not implemented\n", @@ -3367,18 +3387,18 @@ static int scmi_probe(struct platform_device *pdev) return 0; -notification_exit: +raw_mode_cleanup: if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) scmi_raw_mode_cleanup(info->raw); - scmi_notification_exit(&info->handle); clear_dev_req_notifier: blocking_notifier_chain_unregister(&scmi_requested_devices_nh, &info->dev_req_nb); clear_bus_notifier: bus_unregister_notifier(&scmi_bus_type, &info->bus_nb); clear_txrx_setup: + scmi_notification_quiesce(&info->handle); scmi_cleanup_txrx_channels(info); -clear_ida: + scmi_notification_exit(&info->handle); ida_free(&scmi_id, info->id); out_err: @@ -3401,6 +3421,12 @@ static void scmi_remove(struct platform_device *pdev) list_del(&info->node); mutex_unlock(&scmi_list_mutex); + blocking_notifier_chain_unregister(&scmi_requested_devices_nh, + &info->dev_req_nb); + + /* Stop transport callbacks before tearing down notifications. */ + scmi_notification_quiesce(&info->handle); + scmi_cleanup_txrx_channels(info); scmi_notification_exit(&info->handle); mutex_lock(&info->protocols_mtx); @@ -3411,13 +3437,8 @@ static void scmi_remove(struct platform_device *pdev) of_node_put(child); idr_destroy(&info->active_protocols); - blocking_notifier_chain_unregister(&scmi_requested_devices_nh, - &info->dev_req_nb); bus_unregister_notifier(&scmi_bus_type, &info->bus_nb); - /* Safe to free channels since no more users */ - scmi_cleanup_txrx_channels(info); - ida_free(&scmi_id, info->id); } diff --git a/drivers/firmware/arm_scmi/notify.c b/drivers/firmware/arm_scmi/notify.c index 0a192cf2deab6..dfe2aa89c5004 100644 --- a/drivers/firmware/arm_scmi/notify.c +++ b/drivers/firmware/arm_scmi/notify.c @@ -1706,6 +1706,25 @@ int scmi_notification_init(struct scmi_handle *handle) return -ENOMEM; } +/** + * scmi_notification_quiesce() - Stop notification late initialization + * @handle: The handle identifying the platform instance to quiesce + * + * Prevent new late-init work from being queued and wait for any already queued + * or running late-init work to complete before transport channels are torn + * down. + */ +void scmi_notification_quiesce(struct scmi_handle *handle) +{ + struct scmi_notify_instance *ni; + + ni = scmi_notification_instance_data_get(handle); + if (!ni) + return; + + disable_work_sync(&ni->init_work); +} + /** * scmi_notification_exit() - Shutdown and clean Notification core * @handle: The handle identifying the platform instance to shutdown @@ -1717,6 +1736,8 @@ void scmi_notification_exit(struct scmi_handle *handle) ni = scmi_notification_instance_data_get(handle); if (!ni) return; + + scmi_notification_quiesce(handle); scmi_notification_instance_data_set(handle, NULL); /* Destroy while letting pending work complete */ diff --git a/drivers/firmware/arm_scmi/notify.h b/drivers/firmware/arm_scmi/notify.h index 76758a736cf47..f18f98c5ab3ba 100644 --- a/drivers/firmware/arm_scmi/notify.h +++ b/drivers/firmware/arm_scmi/notify.h @@ -82,6 +82,7 @@ struct scmi_protocol_events { }; int scmi_notification_init(struct scmi_handle *handle); +void scmi_notification_quiesce(struct scmi_handle *handle); void scmi_notification_exit(struct scmi_handle *handle); int scmi_register_protocol_events(const struct scmi_handle *handle, u8 proto_id, const struct scmi_protocol_handle *ph, diff --git a/drivers/firmware/arm_scmi/transports/mailbox.c b/drivers/firmware/arm_scmi/transports/mailbox.c index ae0f67e6cc45f..308736c3ead9c 100644 --- a/drivers/firmware/arm_scmi/transports/mailbox.c +++ b/drivers/firmware/arm_scmi/transports/mailbox.c @@ -211,13 +211,18 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, cl->tx_block = false; cl->knows_txdone = tx; + cinfo->transport_info = smbox; + smbox->cinfo = cinfo; + mutex_init(&smbox->chan_lock); + smbox->chan = mbox_request_channel(cl, tx ? 0 : p2a_chan); if (IS_ERR(smbox->chan)) { ret = PTR_ERR(smbox->chan); + smbox->chan = NULL; if (ret != -EPROBE_DEFER) dev_err(cdev, "failed to request SCMI %s mailbox\n", desc); - return ret; + goto err_clear_cinfo; } /* Additional unidirectional channel for TX if needed */ @@ -225,9 +230,10 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, smbox->chan_receiver = mbox_request_channel(cl, a2p_rx_chan); if (IS_ERR(smbox->chan_receiver)) { ret = PTR_ERR(smbox->chan_receiver); + smbox->chan_receiver = NULL; if (ret != -EPROBE_DEFER) dev_err(cdev, "failed to request SCMI Tx Receiver mailbox\n"); - return ret; + goto err_free_chan; } } @@ -235,17 +241,23 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, smbox->chan_platform_receiver = mbox_request_channel(cl, p2a_rx_chan); if (IS_ERR(smbox->chan_platform_receiver)) { ret = PTR_ERR(smbox->chan_platform_receiver); + smbox->chan_platform_receiver = NULL; if (ret != -EPROBE_DEFER) dev_err(cdev, "failed to request SCMI P2A Receiver mailbox\n"); - return ret; + goto err_free_chan; } } - cinfo->transport_info = smbox; - smbox->cinfo = cinfo; - mutex_init(&smbox->chan_lock); - return 0; + +err_free_chan: + mbox_free_channel(smbox->chan); +err_clear_cinfo: + cinfo->transport_info = NULL; + smbox->cinfo = NULL; + devm_iounmap(dev, smbox->shmem); + devm_kfree(dev, smbox); + return ret; } static int mailbox_chan_free(int id, void *p, void *data) diff --git a/drivers/firmware/arm_scmi/transports/smc.c b/drivers/firmware/arm_scmi/transports/smc.c index 21abb571e4f2f..1fce3ccdeb7fc 100644 --- a/drivers/firmware/arm_scmi/transports/smc.c +++ b/drivers/firmware/arm_scmi/transports/smc.c @@ -172,6 +172,13 @@ static int smc_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, scmi_info->param_page = SHMEM_PAGE(res.start); scmi_info->param_offset = SHMEM_OFFSET(res.start); } + + scmi_info->func_id = func_id; + scmi_info->cap_id = cap_id; + scmi_info->cinfo = cinfo; + smc_channel_lock_init(scmi_info); + cinfo->transport_info = scmi_info; + /* * If there is an interrupt named "a2p", then the service and * completion of a message is signaled by an interrupt rather than by @@ -183,18 +190,14 @@ static int smc_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, IRQF_NO_SUSPEND, dev_name(dev), scmi_info); if (ret) { dev_err(dev, "failed to setup SCMI smc irq\n"); + cinfo->transport_info = NULL; + scmi_info->cinfo = NULL; return ret; } } else { cinfo->no_completion_irq = true; } - scmi_info->func_id = func_id; - scmi_info->cap_id = cap_id; - scmi_info->cinfo = cinfo; - smc_channel_lock_init(scmi_info); - cinfo->transport_info = scmi_info; - return 0; } diff --git a/drivers/firmware/arm_scpi.c b/drivers/firmware/arm_scpi.c index 2acad5fa5a286..6eeb0ad9b0085 100644 --- a/drivers/firmware/arm_scpi.c +++ b/drivers/firmware/arm_scpi.c @@ -631,8 +631,8 @@ static struct scpi_dvfs_info *scpi_dvfs_get_info(u8 domain) if (ret) return ERR_PTR(ret); - if (!buf.opp_count) - return ERR_PTR(-ENOENT); + if (!buf.opp_count || buf.opp_count > MAX_DVFS_OPPS) + return ERR_PTR(-EINVAL); info = kmalloc_obj(*info); if (!info) diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index e63933ff67470..96e68ae3f6acc 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -170,6 +170,7 @@ static int coreboot_table_populate(struct device *dev, void *ptr, resource_size_ static int coreboot_table_probe(struct platform_device *pdev) { resource_size_t len; + resource_size_t table_span; struct coreboot_table_header *header; struct resource *res; struct device *dev = &pdev->dev; @@ -181,7 +182,7 @@ static int coreboot_table_probe(struct platform_device *pdev) return -EINVAL; len = resource_size(res); - if (!res->start || !len) + if (!res->start || len < sizeof(*header)) return -EINVAL; /* Check just the header first to make sure things are sane */ @@ -189,19 +190,27 @@ static int coreboot_table_probe(struct platform_device *pdev) if (!header) return -ENOMEM; - len = header->header_bytes + header->table_bytes; ret = strncmp(header->signature, "LBIO", sizeof(header->signature)); + + if (!ret && + (header->header_bytes < sizeof(*header) || + check_add_overflow((resource_size_t)header->header_bytes, + (resource_size_t)header->table_bytes, + &table_span) || + table_span > len)) + ret = -EINVAL; + memunmap(header); if (ret) { dev_warn(dev, "coreboot table missing or corrupt!\n"); return -ENODEV; } - ptr = memremap(res->start, len, MEMREMAP_WB); + ptr = memremap(res->start, table_span, MEMREMAP_WB); if (!ptr) return -ENOMEM; - ret = coreboot_table_populate(dev, ptr, len); + ret = coreboot_table_populate(dev, ptr, table_span); memunmap(ptr); diff --git a/drivers/firmware/imx/sm-misc.c b/drivers/firmware/imx/sm-misc.c index ac9af824c2d40..fb8d7bdb5b084 100644 --- a/drivers/firmware/imx/sm-misc.c +++ b/drivers/firmware/imx/sm-misc.c @@ -79,6 +79,9 @@ static int syslog_show(struct seq_file *file, void *priv) u16 size = SZ_4K / 4; int ret; + if (!syslog) + return -ENOMEM; + if (!ph) return -ENODEV; diff --git a/drivers/firmware/qcom/Makefile b/drivers/firmware/qcom/Makefile index 0be40a1abc13c..b679d3fc2c267 100644 --- a/drivers/firmware/qcom/Makefile +++ b/drivers/firmware/qcom/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_QCOM_SCM) += qcom-scm.o qcom-scm-objs += qcom_scm.o qcom_scm-smc.o qcom_scm-legacy.o +CFLAGS_qcom_scm-smc.o := -I$(src) obj-$(CONFIG_QCOM_TZMEM) += qcom_tzmem.o obj-$(CONFIG_QCOM_QSEECOM) += qcom_qseecom.o obj-$(CONFIG_QCOM_QSEECOM_UEFISECAPP) += qcom_qseecom_uefisecapp.o diff --git a/drivers/firmware/qcom/qcom_scm-smc.c b/drivers/firmware/qcom/qcom_scm-smc.c index 574930729ddd7..127365ab11fc2 100644 --- a/drivers/firmware/qcom/qcom_scm-smc.c +++ b/drivers/firmware/qcom/qcom_scm-smc.c @@ -24,6 +24,9 @@ struct arm_smccc_args { unsigned long args[8]; }; +#define CREATE_TRACE_POINTS +#include "qcom_scm_trace.h" + static DEFINE_MUTEX(qcom_scm_lock); #define QCOM_SCM_EBUSY_WAIT_MS 30 @@ -44,6 +47,7 @@ static void __scm_smc_do_quirk(const struct arm_smccc_args *smc, quirk.state.a6 = 0; do { + trace_scm_smc_request(a0, smc); arm_smccc_smc_quirk(a0, smc->args[1], smc->args[2], smc->args[3], smc->args[4], smc->args[5], quirk.state.a6, smc->args[7], res, &quirk); @@ -83,6 +87,7 @@ int scm_get_wq_ctx(u32 *wq_ctx, u32 *flags, u32 *more_pending) if (ret) return ret; + trace_scm_waitq_get_wq_ctx(get_wq_res.a1, get_wq_res.a2, get_wq_res.a3); *wq_ctx = get_wq_res.a1; *flags = get_wq_res.a2; *more_pending = get_wq_res.a3; @@ -105,10 +110,12 @@ static int __scm_smc_do_quirk_handle_waitq(struct device *dev, struct arm_smccc_ wq_ctx = res->a1; smc_call_ctx = res->a2; - ret = qcom_scm_wait_for_wq_completion(wq_ctx); + trace_scm_waitq_sleep(wq_ctx, smc_call_ctx); + ret = qcom_scm_wait_for_wq_completion(dev, wq_ctx); if (ret) return ret; + trace_scm_waitq_resume(smc_call_ctx); fill_wq_resume_args(&resume, smc_call_ctx); smc = &resume; } @@ -201,6 +208,9 @@ int __scm_smc_call(struct device *dev, const struct qcom_scm_desc *desc, } ret = __scm_smc_do(dev, &smc, &smc_res, atomic); + + trace_scm_smc_done(ret, smc.args[0], &smc_res); + if (ret) return ret; diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c index 6b601a4b89dbf..26bf87247afa1 100644 --- a/drivers/firmware/qcom/qcom_scm.c +++ b/drivers/firmware/qcom/qcom_scm.c @@ -2630,23 +2630,20 @@ static int qcom_scm_get_waitq_irq(struct qcom_scm *scm) return irq_create_fwspec_mapping(&fwspec); } -static struct completion *qcom_scm_get_completion(u32 wq_ctx) +static struct completion *qcom_scm_get_completion(struct qcom_scm *scm, u32 wq_ctx) { - struct completion *wq; - - if (WARN_ON_ONCE(wq_ctx >= __scm->wq_cnt)) + if (WARN_ON_ONCE(wq_ctx >= scm->wq_cnt)) return ERR_PTR(-EINVAL); - wq = &__scm->waitq_comps[wq_ctx]; - - return wq; + return &scm->waitq_comps[wq_ctx]; } -int qcom_scm_wait_for_wq_completion(u32 wq_ctx) +int qcom_scm_wait_for_wq_completion(struct device *dev, u32 wq_ctx) { + struct qcom_scm *scm = dev_get_drvdata(dev); struct completion *wq; - wq = qcom_scm_get_completion(wq_ctx); + wq = qcom_scm_get_completion(scm, wq_ctx); if (IS_ERR(wq)) return PTR_ERR(wq); @@ -2655,11 +2652,11 @@ int qcom_scm_wait_for_wq_completion(u32 wq_ctx) return 0; } -static int qcom_scm_waitq_wakeup(unsigned int wq_ctx) +static int qcom_scm_waitq_wakeup(struct qcom_scm *scm, unsigned int wq_ctx) { struct completion *wq; - wq = qcom_scm_get_completion(wq_ctx); + wq = qcom_scm_get_completion(scm, wq_ctx); if (IS_ERR(wq)) return PTR_ERR(wq); @@ -2686,7 +2683,7 @@ static irqreturn_t qcom_scm_irq_handler(int irq, void *data) goto out; } - ret = qcom_scm_waitq_wakeup(wq_ctx); + ret = qcom_scm_waitq_wakeup(scm, wq_ctx); if (ret) goto out; } while (more_pending); @@ -2746,6 +2743,7 @@ static int qcom_scm_probe(struct platform_device *pdev) return -ENOMEM; scm->dev = &pdev->dev; + platform_set_drvdata(pdev, scm); ret = qcom_scm_find_dload_address(&pdev->dev, &scm->dload_mode_addr); if (ret < 0) return ret; @@ -2787,9 +2785,11 @@ static int qcom_scm_probe(struct platform_device *pdev) "Failed to setup the reserved memory region for TZ mem\n"); ret = qcom_tzmem_enable(scm->dev); - if (ret) - return dev_err_probe(scm->dev, ret, - "Failed to enable the TrustZone memory allocator\n"); + if (ret) { + ret = dev_err_probe(scm->dev, ret, + "Failed to enable the TrustZone memory allocator\n"); + goto err_rmem; + } memset(&pool_config, 0, sizeof(pool_config)); pool_config.initial_size = 0; @@ -2797,9 +2797,11 @@ static int qcom_scm_probe(struct platform_device *pdev) pool_config.max_size = SZ_256K; scm->mempool = devm_qcom_tzmem_pool_new(scm->dev, &pool_config); - if (IS_ERR(scm->mempool)) - return dev_err_probe(scm->dev, PTR_ERR(scm->mempool), - "Failed to create the SCM memory pool\n"); + if (IS_ERR(scm->mempool)) { + ret = dev_err_probe(scm->dev, PTR_ERR(scm->mempool), + "Failed to create the SCM memory pool\n"); + goto err_rmem; + } ret = qcom_scm_query_waitq_count(scm); scm->wq_cnt = ret < 0 ? QCOM_SCM_DEFAULT_WAITQ_COUNT : ret; @@ -2870,6 +2872,10 @@ static int qcom_scm_probe(struct platform_device *pdev) qcom_scm_gunyah_wdt_init(scm); return 0; + +err_rmem: + of_reserved_mem_device_release(scm->dev); + return ret; } static void qcom_scm_shutdown(struct platform_device *pdev) diff --git a/drivers/firmware/qcom/qcom_scm.h b/drivers/firmware/qcom/qcom_scm.h index caab80a73e17f..cf90a565fdfbd 100644 --- a/drivers/firmware/qcom/qcom_scm.h +++ b/drivers/firmware/qcom/qcom_scm.h @@ -66,7 +66,7 @@ struct qcom_scm_res { u64 result[MAX_QCOM_SCM_RETS]; }; -int qcom_scm_wait_for_wq_completion(u32 wq_ctx); +int qcom_scm_wait_for_wq_completion(struct device *dev, u32 wq_ctx); int scm_get_wq_ctx(u32 *wq_ctx, u32 *flags, u32 *more_pending); #define SCM_SMC_FNID(s, c) ((((s) & 0xFF) << 8) | ((c) & 0xFF)) diff --git a/drivers/firmware/qcom/qcom_scm_trace.h b/drivers/firmware/qcom/qcom_scm_trace.h new file mode 100644 index 0000000000000..6c911124fc56b --- /dev/null +++ b/drivers/firmware/qcom/qcom_scm_trace.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM qcom_scm + +#if !defined(_TRACE_SCM_SMC_INTERFACE_H) || defined(TRACE_HEADER_MULTI_READ) + +#define _TRACE_SCM_SMC_INTERFACE_H + +#include + +TRACE_EVENT(scm_smc_request, + + TP_PROTO(unsigned long a0, const struct arm_smccc_args *smc), + + TP_ARGS(a0, smc), + + TP_STRUCT__entry( + __field(u64, smc_id) + __field(u8, svc_id) + __field(u8, cmd_id) + __field(u8, args_cnt) + __dynamic_array(unsigned long, args, + min_t(u8, (smc->args[1] & 0xF), (u8)6)) + ), + + TP_fast_assign( + __entry->smc_id = a0; + __entry->svc_id = (smc->args[0] >> 8) & 0xFF; + __entry->cmd_id = smc->args[0] & 0xFF; + u8 n = min_t(u8, (smc->args[1] & 0xF), (u8)6); + + __entry->args_cnt = n; + + unsigned long *dst = __get_dynamic_array(args); + + for (int i = 0; i < n; i++) + dst[i] = smc->args[2 + i]; + ), + + TP_printk("smc_id:0x%08llx svc_id:0x%02x cmd_id:0x%02x args_cnt:%u args:%s", + __entry->smc_id, __entry->svc_id, __entry->cmd_id, __entry->args_cnt, + __print_dynamic_array(args, sizeof(unsigned long))) +); + +TRACE_EVENT(scm_waitq_sleep, + + TP_PROTO(u32 wq_ctx, u32 smc_ctx), + + TP_ARGS(wq_ctx, smc_ctx), + + TP_STRUCT__entry( + __field(u32, wq_ctx) + __field(u32, smc_call_ctx) + ), + + TP_fast_assign( + __entry->wq_ctx = wq_ctx; + __entry->smc_call_ctx = smc_ctx; + ), + + TP_printk("wq_ctx:%u, smc_call_ctx:%u", __entry->wq_ctx, __entry->smc_call_ctx) +); + +TRACE_EVENT(scm_waitq_resume, + + TP_PROTO(u32 smc_ctx), + + TP_ARGS(smc_ctx), + + TP_STRUCT__entry( + __field(u32, smc_call_ctx) + ), + + TP_fast_assign( + __entry->smc_call_ctx = smc_ctx; + ), + + TP_printk("smc_call_ctx:%u", __entry->smc_call_ctx) +); + +TRACE_EVENT(scm_waitq_get_wq_ctx, + + TP_PROTO(u32 wq_ctx, u32 flags, u32 pending), + + TP_ARGS(wq_ctx, flags, pending), + + TP_STRUCT__entry( + __field(u32, wq_ctx) + __field(u32, flags) + __field(u32, more_pending) + ), + + TP_fast_assign( + __entry->wq_ctx = wq_ctx; + __entry->flags = flags; + __entry->more_pending = pending; + ), + + TP_printk("wq_ctx:%u, flags:%u, more_pending:%u", + __entry->wq_ctx, __entry->flags, __entry->more_pending) +); + +TRACE_EVENT(scm_smc_done, + + TP_PROTO(int ret, u64 smc_id, struct arm_smccc_res *smc_res), + + TP_ARGS(ret, smc_id, smc_res), + + TP_STRUCT__entry( + __field(int, ret) + __field(u64, smc_id) + __field(unsigned long, res) + __field(unsigned long, res0) + __field(unsigned long, res1) + __field(unsigned long, res2) + ), + + TP_fast_assign( + __entry->ret = ret; + __entry->smc_id = smc_id; + __entry->res = smc_res->a0; + __entry->res0 = smc_res->a1; + __entry->res1 = smc_res->a2; + __entry->res2 = smc_res->a3; + ), + + TP_printk("smc_id:0x%08llx, ret:%d res_to_callee:0x%lx res0:0x%lx res1:0x%lx res2:0x%lx", + __entry->smc_id, __entry->ret, __entry->res, + __entry->res0, __entry->res1, __entry->res2) +); + +#endif /* _TRACE_SCM_SMC_INTERFACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE qcom_scm_trace + +#include + diff --git a/drivers/firmware/qcom/qcom_tzmem.c b/drivers/firmware/qcom/qcom_tzmem.c index 0635cbeacfc8e..0fd9581275f17 100644 --- a/drivers/firmware/qcom/qcom_tzmem.c +++ b/drivers/firmware/qcom/qcom_tzmem.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -507,14 +508,18 @@ phys_addr_t qcom_tzmem_to_phys(void *vaddr) } EXPORT_SYMBOL_GPL(qcom_tzmem_to_phys); +static void qcom_tzmem_do_init(int *result) +{ + *result = qcom_tzmem_init(); +} + int qcom_tzmem_enable(struct device *dev) { - if (qcom_tzmem_dev) - return -EBUSY; + static int result; qcom_tzmem_dev = dev; - - return qcom_tzmem_init(); + DO_ONCE(qcom_tzmem_do_init, &result); + return result; } EXPORT_SYMBOL_GPL(qcom_tzmem_enable); diff --git a/drivers/fpga/altera-cvp.c b/drivers/fpga/altera-cvp.c index 44badfd11e1b0..070c025ee2154 100644 --- a/drivers/fpga/altera-cvp.c +++ b/drivers/fpga/altera-cvp.c @@ -16,6 +16,7 @@ #include #include #include +#include #define CVP_BAR 0 /* BAR used for data transfer in memory mode */ #define CVP_DUMMY_WR 244 /* dummy writes to clear CvP state machine */ @@ -261,7 +262,7 @@ static int altera_cvp_v2_wait_for_credit(struct fpga_manager *mgr, static int altera_cvp_send_block(struct altera_cvp_conf *conf, const u32 *data, size_t len) { - u32 mask, words = len / sizeof(u32); + u32 words = len / sizeof(u32); int i, remainder; for (i = 0; i < words; i++) @@ -270,9 +271,10 @@ static int altera_cvp_send_block(struct altera_cvp_conf *conf, /* write up to 3 trailing bytes, if any */ remainder = len % sizeof(u32); if (remainder) { - mask = BIT(remainder * 8) - 1; - if (mask) - conf->write_data(conf, *data & mask); + u32 word = 0; + + memcpy(&word, data, remainder); + conf->write_data(conf, word); } return 0; diff --git a/drivers/fpga/dfl-fme-perf.c b/drivers/fpga/dfl-fme-perf.c index 7422d2bc6f374..db56d52411efc 100644 --- a/drivers/fpga/dfl-fme-perf.c +++ b/drivers/fpga/dfl-fme-perf.c @@ -925,6 +925,8 @@ static int fme_perf_pmu_register(struct platform_device *pdev, PERF_PMU_CAP_NO_EXCLUDE; name = devm_kasprintf(priv->dev, GFP_KERNEL, "dfl_fme%d", pdev->id); + if (!name) + return -ENOMEM; ret = perf_pmu_register(pmu, name, -1); if (ret) diff --git a/drivers/fpga/stratix10-soc.c b/drivers/fpga/stratix10-soc.c index 0a295ccf1644a..b8ec2e6f615fa 100644 --- a/drivers/fpga/stratix10-soc.c +++ b/drivers/fpga/stratix10-soc.c @@ -195,20 +195,18 @@ static int s10_ops_write_init(struct fpga_manager *mgr, ret = s10_svc_send_msg(priv, COMMAND_RECONFIG, &ctype, sizeof(ctype)); if (ret < 0) - goto init_done; + goto init_error; - ret = wait_for_completion_timeout( - &priv->status_return_completion, S10_RECONFIG_TIMEOUT); - if (!ret) { + if (!wait_for_completion_timeout(&priv->status_return_completion, + S10_RECONFIG_TIMEOUT)) { dev_err(dev, "timeout waiting for RECONFIG_REQUEST\n"); ret = -ETIMEDOUT; - goto init_done; + goto init_error; } - ret = 0; if (!test_and_clear_bit(SVC_STATUS_OK, &priv->status)) { ret = -ETIMEDOUT; - goto init_done; + goto init_error; } /* Allocate buffers from the service layer's pool. */ @@ -217,14 +215,16 @@ static int s10_ops_write_init(struct fpga_manager *mgr, if (IS_ERR(kbuf)) { s10_free_buffers(mgr); ret = PTR_ERR(kbuf); - goto init_done; + goto init_error; } priv->svc_bufs[i].buf = kbuf; priv->svc_bufs[i].lock = 0; } -init_done: + return 0; + +init_error: stratix10_svc_done(priv->chan); return ret; } @@ -342,6 +342,9 @@ static int s10_ops_write(struct fpga_manager *mgr, const char *buf, if (!s10_free_buffers(mgr)) dev_err(dev, "%s not all buffers were freed\n", __func__); + if (ret < 0) + stratix10_svc_done(priv->chan); + return ret; } diff --git a/drivers/gpib/common/gpib_os.c b/drivers/gpib/common/gpib_os.c index 69f6aa73ab9a3..384800e6bf975 100644 --- a/drivers/gpib/common/gpib_os.c +++ b/drivers/gpib/common/gpib_os.c @@ -289,18 +289,19 @@ int autopoll_all_devices(struct gpib_board *board) dev_dbg(board->gpib_dev, "autopoll has board lock\n"); retval = serial_poll_all(board, serial_timeout); - if (retval < 0) { - mutex_unlock(&board->big_gpib_mutex); - mutex_unlock(&board->user_mutex); - return retval; + if (retval >= 0) { + dev_dbg(board->gpib_dev, "complete\n"); + /* + * need to wake wait queue in case someone is + * waiting on RQS + */ + wake_up_interruptible(&board->wait); } - dev_dbg(board->gpib_dev, "complete\n"); - /* - * need to wake wait queue in case someone is - * waiting on RQS - */ - wake_up_interruptible(&board->wait); + if (retval <= 0) { + atomic_set(&board->stuck_srq, 1); + set_bit(SRQI_NUM, &board->status); + } mutex_unlock(&board->big_gpib_mutex); mutex_unlock(&board->user_mutex); diff --git a/drivers/gpib/common/iblib.c b/drivers/gpib/common/iblib.c index b672dd6aad25f..511e1d61c1fb2 100644 --- a/drivers/gpib/common/iblib.c +++ b/drivers/gpib/common/iblib.c @@ -193,9 +193,6 @@ static int autospoll_thread(void *board_void) } if (retval <= 0) { dev_err(board->gpib_dev, "stuck SRQ\n"); - - atomic_set(&board->stuck_srq, 1); // XXX could be better - set_bit(SRQI_NUM, &board->status); } } return retval; diff --git a/drivers/gpib/include/amccs5933.h b/drivers/gpib/include/amccs5933.h index d7f63c7950963..f8a80bdc70dcc 100644 --- a/drivers/gpib/include/amccs5933.h +++ b/drivers/gpib/include/amccs5933.h @@ -13,7 +13,7 @@ enum { }; // incoming mailbox 0-3 register offsets -extern inline int INCOMING_MAILBOX_REG(unsigned int mailbox) +static inline int INCOMING_MAILBOX_REG(unsigned int mailbox) { return (0x10 + 4 * mailbox); }; @@ -29,25 +29,25 @@ enum { }; // select byte 0 to 3 of incoming mailbox -extern inline int INBOX_BYTE_BITS(unsigned int byte) +static inline int INBOX_BYTE_BITS(unsigned int byte) { return (byte & 0x3) << 8; }; // select incoming mailbox 0 to 3 -extern inline int INBOX_SELECT_BITS(unsigned int mailbox) +static inline int INBOX_SELECT_BITS(unsigned int mailbox) { return (mailbox & 0x3) << 10; }; // select byte 0 to 3 of outgoing mailbox -extern inline int OUTBOX_BYTE_BITS(unsigned int byte) +static inline int OUTBOX_BYTE_BITS(unsigned int byte) { return (byte & 0x3); }; // select outgoing mailbox 0 to 3 -extern inline int OUTBOX_SELECT_BITS(unsigned int mailbox) +static inline int OUTBOX_SELECT_BITS(unsigned int mailbox) { return (mailbox & 0x3) << 2; }; diff --git a/drivers/gpib/tnt4882/mite.h b/drivers/gpib/tnt4882/mite.h index a1fdba9672a03..dd251afa90e34 100644 --- a/drivers/gpib/tnt4882/mite.h +++ b/drivers/gpib/tnt4882/mite.h @@ -45,12 +45,12 @@ struct mite_struct { extern struct mite_struct *mite_devices; -extern inline unsigned int mite_irq(struct mite_struct *mite) +static inline unsigned int mite_irq(struct mite_struct *mite) { return mite->pcidev->irq; }; -extern inline unsigned int mite_device_id(struct mite_struct *mite) +static inline unsigned int mite_device_id(struct mite_struct *mite) { return mite->pcidev->device; }; diff --git a/drivers/gpio/gpio-arizona.c b/drivers/gpio/gpio-arizona.c index a7e98d395d8e5..88cf013f0d90d 100644 --- a/drivers/gpio/gpio-arizona.c +++ b/drivers/gpio/gpio-arizona.c @@ -115,8 +115,12 @@ static int arizona_gpio_direction_out(struct gpio_chip *chip, if (value) value = ARIZONA_GPN_LVL; - return regmap_update_bits(arizona->regmap, ARIZONA_GPIO1_CTRL + offset, - ARIZONA_GPN_DIR | ARIZONA_GPN_LVL, value); + ret = regmap_update_bits(arizona->regmap, ARIZONA_GPIO1_CTRL + offset, + ARIZONA_GPN_DIR | ARIZONA_GPN_LVL, value); + if (ret < 0 && (val & ARIZONA_GPN_DIR) && persistent) + pm_runtime_put_autosuspend(chip->parent); + + return ret; } static int arizona_gpio_set(struct gpio_chip *chip, unsigned int offset, diff --git a/drivers/gpio/gpio-tps65219.c b/drivers/gpio/gpio-tps65219.c index 457fd8a589e89..6958466455d04 100644 --- a/drivers/gpio/gpio-tps65219.c +++ b/drivers/gpio/gpio-tps65219.c @@ -79,7 +79,7 @@ static int tps65219_gpio_get(struct gpio_chip *gc, unsigned int offset) if (ret) return ret; - ret = !!(val & BIT(TPS65219_MFP_GPIO_STATUS_MASK)); + ret = !!(val & TPS65219_MFP_GPIO_STATUS_MASK); dev_warn(dev, "GPIO%d = %d, MULTI_DEVICE_ENABLE, not a standard GPIO\n", offset, ret); /* @@ -87,7 +87,7 @@ static int tps65219_gpio_get(struct gpio_chip *gc, unsigned int offset) * status bit. */ - if (tps65219_gpio_get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT) + if (gc->get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT) return -ENOTSUPP; return ret; @@ -158,8 +158,10 @@ static int tps65214_gpio_change_direction(struct gpio_chip *gc, unsigned int off if (ret) dev_err(dev, "GPIO%d configured as VSEL, not GPIO\n", offset); + val = direction == GPIO_LINE_DIRECTION_OUT ? + TPS65214_GPIO0_DIR_MASK : 0; ret = regmap_update_bits(gpio->tps->regmap, TPS65219_REG_GENERAL_CONFIG, - TPS65214_GPIO0_DIR_MASK, direction); + TPS65214_GPIO0_DIR_MASK, val); if (ret) dev_err(dev, "Fail to change direction to %u for GPIO%d.\n", direction, offset); @@ -176,7 +178,7 @@ static int tps65219_gpio_direction_input(struct gpio_chip *gc, unsigned int offs return -ENOTSUPP; } - if (tps65219_gpio_get_direction(gc, offset) == GPIO_LINE_DIRECTION_IN) + if (gc->get_direction(gc, offset) == GPIO_LINE_DIRECTION_IN) return 0; return gpio->change_dir(gc, offset, GPIO_LINE_DIRECTION_IN); @@ -190,7 +192,7 @@ static int tps65219_gpio_direction_output(struct gpio_chip *gc, unsigned int off if (offset != TPS6521X_GPIO0_IDX) return 0; - if (tps65219_gpio_get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT) + if (gc->get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT) return 0; return gpio->change_dir(gc, offset, GPIO_LINE_DIRECTION_OUT); diff --git a/drivers/gpio/gpio-virtuser.c b/drivers/gpio/gpio-virtuser.c index 7d0d366be37a4..d5876c19cfefa 100644 --- a/drivers/gpio/gpio-virtuser.c +++ b/drivers/gpio/gpio-virtuser.c @@ -692,7 +692,8 @@ static int gpio_virtuser_interrupts_set(void *data, u64 val) atomic_set(&ld->irq, irq); } else { irq = atomic_xchg(&ld->irq, 0); - free_irq(irq, ld); + if (irq) + free_irq(irq, ld); } return 0; diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c index 15a79d9a2e9eb..13e4c5f0e297c 100644 --- a/drivers/gpio/gpio-zynq.c +++ b/drivers/gpio/gpio-zynq.c @@ -798,15 +798,7 @@ static int zynq_gpio_runtime_resume(struct device *dev) static int zynq_gpio_request(struct gpio_chip *chip, unsigned int offset) { - int ret; - - ret = pm_runtime_get_sync(chip->parent); - - /* - * If the device is already active pm_runtime_get() will return 1 on - * success, but gpio_request still needs to return 0. - */ - return ret < 0 ? ret : 0; + return pm_runtime_resume_and_get(chip->parent); } static void zynq_gpio_free(struct gpio_chip *chip, unsigned int offset) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 82f27db0b2304..4d0a94e19ebab 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -2172,18 +2172,19 @@ static void gpio_v2_line_info_changed_to_v1( #endif /* CONFIG_GPIO_CDEV_V1 */ -static void gpio_desc_to_lineinfo(struct gpio_desc *desc, - struct gpio_v2_line_info *info, bool atomic) +static int gpio_desc_to_lineinfo(struct gpio_desc *desc, + struct gpio_v2_line_info *info, bool atomic) { u32 debounce_period_us; unsigned long dflags; const char *label; + memset(info, 0, sizeof(*info)); + CLASS(gpio_chip_guard, guard)(desc); if (!guard.gc) - return; + return -ENODEV; - memset(info, 0, sizeof(*info)); info->offset = gpiod_hwgpio(desc); if (desc->name) @@ -2258,6 +2259,8 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, debounce_period_us; info->num_attrs++; } + + return 0; } struct gpio_chardev_data { @@ -2309,6 +2312,7 @@ static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip, struct gpio_desc *desc; struct gpioline_info lineinfo; struct gpio_v2_line_info lineinfo_v2; + int ret; if (copy_from_user(&lineinfo, ip, sizeof(lineinfo))) return -EFAULT; @@ -2326,7 +2330,10 @@ static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip, return -EBUSY; } - gpio_desc_to_lineinfo(desc, &lineinfo_v2, false); + ret = gpio_desc_to_lineinfo(desc, &lineinfo_v2, false); + if (ret) + return ret; + gpio_v2_line_info_to_v1(&lineinfo_v2, &lineinfo); if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) { @@ -2344,6 +2351,7 @@ static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip, { struct gpio_desc *desc; struct gpio_v2_line_info lineinfo; + int ret; if (copy_from_user(&lineinfo, ip, sizeof(lineinfo))) return -EFAULT; @@ -2363,7 +2371,10 @@ static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip, if (test_and_set_bit(lineinfo.offset, cdev->watched_lines)) return -EBUSY; } - gpio_desc_to_lineinfo(desc, &lineinfo, false); + + ret = gpio_desc_to_lineinfo(desc, &lineinfo, false); + if (ret) + return ret; if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) { if (watch) @@ -2489,6 +2500,7 @@ static int lineinfo_changed_notify(struct notifier_block *nb, struct lineinfo_changed_ctx *ctx; struct gpio_desc *desc = data; struct file *fp; + int ret; if (!test_bit(gpiod_hwgpio(desc), cdev->watched_lines)) return NOTIFY_DONE; @@ -2519,7 +2531,13 @@ static int lineinfo_changed_notify(struct notifier_block *nb, ctx->chg.event_type = action; ctx->chg.timestamp_ns = ktime_get_ns(); - gpio_desc_to_lineinfo(desc, &ctx->chg.info, true); + + ret = gpio_desc_to_lineinfo(desc, &ctx->chg.info, true); + if (ret) { + fput(fp); + return NOTIFY_DONE; + } + /* Keep the GPIO device alive until we emit the event. */ ctx->gdev = gpio_device_get(desc->gdev); ctx->cdev = cdev; diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 940b566946ce3..f36e4b171fa7d 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -788,13 +788,13 @@ static int of_gpio_notify(struct notifier_block *nb, unsigned long action, if (!of_property_read_bool(rd->dn, "gpio-hog")) return NOTIFY_DONE; /* not for us */ - if (of_node_test_and_set_flag(rd->dn, OF_POPULATED)) - return NOTIFY_DONE; - gdev = of_find_gpio_device_by_node(rd->dn->parent); if (!gdev) return NOTIFY_DONE; /* not for us */ + if (of_node_test_and_set_flag(rd->dn, OF_POPULATED)) + return NOTIFY_DONE; + ret = gpiochip_add_hog(gpio_device_get_chip(gdev), of_fwnode_handle(rd->dn)); if (ret < 0) { pr_err("%s: failed to add hogs for %pOF\n", __func__, diff --git a/drivers/gpio/gpiolib-shared.c b/drivers/gpio/gpiolib-shared.c index 495bd3d0ddf02..5f9623e40b0fb 100644 --- a/drivers/gpio/gpiolib-shared.c +++ b/drivers/gpio/gpiolib-shared.c @@ -261,10 +261,13 @@ static int gpio_shared_of_traverse(struct device_node *curr) con_id[con_id_len - suffix_len] = '\0'; } - ref = gpio_shared_make_ref(fwnode_handle_get(of_fwnode_handle(curr)), - con_id, args.args[1]); - if (!ref) + struct fwnode_handle *curr_fwnode = + fwnode_handle_get(of_fwnode_handle(curr)); + ref = gpio_shared_make_ref(curr_fwnode, con_id, args.args[1]); + if (!ref) { + fwnode_handle_put(curr_fwnode); return -ENOMEM; + } if (!list_empty(&entry->refs)) pr_debug("GPIO %u at %s is shared by multiple firmware nodes\n", diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index ef8ccaf17c9ce..28f7265c5d569 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -1029,6 +1029,13 @@ int gpiochip_add_hog(struct gpio_chip *gc, struct fwnode_handle *fwnode) ret = of_gpiochip_get_lflags(gc, &gpiospec, &lflags); if (ret) return ret; + + /* + * If no line-name property is present, fall back to the OF + * node name as in the previous implementation. + */ + if (!name) + name = to_of_node(fwnode)->name; } else { /* * GPIO_ACTIVE_LOW is currently the only lookup flag diff --git a/drivers/gpu/drm/adp/Kconfig b/drivers/gpu/drm/adp/Kconfig index 9fcc27eb200db..acfa21ee06d22 100644 --- a/drivers/gpu/drm/adp/Kconfig +++ b/drivers/gpu/drm/adp/Kconfig @@ -6,7 +6,6 @@ config DRM_ADP select DRM_KMS_HELPER select DRM_BRIDGE_CONNECTOR select DRM_DISPLAY_HELPER - select DRM_KMS_DMA_HELPER select DRM_GEM_DMA_HELPER select DRM_PANEL_BRIDGE select VIDEOMODE_HELPERS diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 7b09410d6d8f0..6b0c1ad5a0534 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -822,6 +822,9 @@ enum amdgpu_enforce_isolation_mode { struct amdgpu_device { struct device *dev; struct pci_dev *pdev; + /* The two ends of the physical PCIe link outside the device. */ + struct pci_dev *link_dev; + struct pci_dev *link_partner; struct drm_device ddev; #ifdef CONFIG_DRM_AMD_ACP diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c index 516ab9cf88fc0..8d3f1f5a141e1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c @@ -1167,8 +1167,10 @@ int amdgpu_acpi_enumerate_xcc(void) } xcc_info = kzalloc_obj(struct amdgpu_acpi_xcc_info); - if (!xcc_info) + if (!xcc_info) { + acpi_dev_put(acpi_dev); return -ENOMEM; + } INIT_LIST_HEAD(&xcc_info->list); xcc_info->handle = acpi_device_handle(acpi_dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index da325863ad764..4b29e7eee6105 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -329,7 +329,7 @@ void amdgpu_amdkfd_clear_kfd_mapping(struct amdgpu_device *adev) struct kfd_dev *kfd = adev->kfd.dev; unsigned int i; - if (!kfd) + if (!kfd || !kfd->init_complete) return; for (i = 0; i < kfd->num_nodes; i++) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index f0f516a79424f..194e53880eddf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1795,6 +1795,12 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( size >>= 1; aligned_size = PAGE_ALIGN(size); + /* reject AQL queue with size < 2 */ + if (!aligned_size) { + ret = -EINVAL; + goto err_alignment_size; + } + (*mem)->alloc_flags = flags; amdgpu_sync_create(&(*mem)->sync); @@ -1886,6 +1892,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id); err_reserve_limit: amdgpu_sync_free(&(*mem)->sync); +err_alignment_size: mutex_destroy(&(*mem)->lock); if (gobj) drm_gem_object_put(gobj); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index df25886428174..4536d46b7f21b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -1773,8 +1773,10 @@ static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused) /* Avoid accidently unparking the sched thread during GPU reset */ r = down_write_killable(&adev->reset_domain->sem); - if (r) + if (r) { + pm_runtime_put_autosuspend(dev->dev); return r; + } /* hold on the scheduler */ for (i = 0; i < AMDGPU_MAX_RINGS; i++) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c index 45dfe1c8a11b5..a4d79484f6d91 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c @@ -298,10 +298,10 @@ amdgpu_devcoredump_print_ibs(struct drm_printer *p, amdgpu_res_first(abo->tbo.resource, offset, coredump->ibs[i].ib_size_dw * 4, &cursor); while (cursor.remaining) { - amdgpu_device_mm_access(adev, cursor.start / 4, - &ib_content[off], cursor.size / 4, + amdgpu_device_mm_access(adev, cursor.start, + &ib_content[off], cursor.size, false); - off += cursor.size; + off += cursor.size / 4; amdgpu_res_next(&cursor, cursor.size); } emit_content = true; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index e84b75c03c6f9..ea9a6ed4bcc9c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -607,6 +607,13 @@ void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev) int bamaco_support; adev->pm.rpm_mode = AMDGPU_RUNPM_NONE; + if (pci_is_thunderbolt_attached(adev->pdev) || + dev_is_removable(&adev->pdev->dev)) { + dev_info(adev->dev, + "Runtime PM disabled for externally attached device\n"); + return; + } + bamaco_support = amdgpu_device_supports_baco(adev); switch (amdgpu_runtime_pm) { @@ -743,6 +750,67 @@ void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos, drm_dev_exit(idx); } +#ifdef CONFIG_64BIT +/* + * During early SR-IOV VF init, host-provided init data can live in FB before + * the normal VRAM aperture mapping is ready. Use a temporary BAR0 mapping for + * reads only, and verify it matches the VRAM aperture when aperture information + * is already available. + */ +static int amdgpu_device_read_fb_via_bar0(struct amdgpu_device *adev, + u64 offset, void *buf, size_t size) +{ + resource_size_t aper_base, aper_size, bar_start, bar_size, map_base; + void __iomem *vram; + size_t map_offset, map_size; + unsigned long flags; + u64 end; + + if (!buf || !size) + return -EINVAL; + + if (!amdgpu_sriov_vf(adev)) + return -EINVAL; + + flags = pci_resource_flags(adev->pdev, 0); + if ((flags & IORESOURCE_UNSET) || !(flags & IORESOURCE_MEM)) + return -EINVAL; + + bar_start = pci_resource_start(adev->pdev, 0); + bar_size = pci_resource_len(adev->pdev, 0); + if (!bar_size) + return -ENODEV; + + aper_base = adev->gmc.aper_base; + aper_size = adev->gmc.visible_vram_size ? adev->gmc.visible_vram_size : + adev->gmc.aper_size; + + if (aper_base || aper_size) { + if (aper_base != bar_start || aper_size > bar_size) + return -EINVAL; + } else { + aper_base = bar_start; + aper_size = bar_size; + } + + if (check_add_overflow(offset, size, &end) || end > aper_size) + return -EINVAL; + + map_offset = offset_in_page(offset); + map_base = aper_base + (offset & PAGE_MASK); + map_size = PAGE_ALIGN(map_offset + size); + + vram = ioremap_wc(map_base, map_size); + if (!vram) + return -ENOMEM; + + memcpy_fromio(buf, (u8 __iomem *)vram + map_offset, size); + iounmap(vram); + + return 0; +} +#endif + /** * amdgpu_device_aper_access - access vram by vram aperture * @@ -762,8 +830,12 @@ size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos, size_t count = 0; uint64_t last; - if (!adev->mman.aper_base_kaddr) + if (!adev->mman.aper_base_kaddr) { + /* Writes still require the regular aperture/MM path. */ + if (!write && !amdgpu_device_read_fb_via_bar0(adev, pos, buf, size)) + return size; return 0; + } last = min(pos + size, adev->gmc.visible_vram_size); if (last > pos) { @@ -1976,18 +2048,17 @@ static void amdgpu_uid_fini(struct amdgpu_device *adev) adev->uid_info = NULL; } -static struct pci_dev *amdgpu_device_find_parent(struct amdgpu_device *adev) +static void amdgpu_device_init_pcie_links(struct amdgpu_device *adev) { - struct pci_dev *parent = adev->pdev; + adev->link_dev = adev->pdev; + adev->link_partner = pci_upstream_bridge(adev->link_dev); - /* skip upstream/downstream switches internal to dGPU */ - while ((parent = pci_upstream_bridge(parent))) { - if (parent->vendor == PCI_VENDOR_ID_ATI) - continue; - break; + /* Skip upstream/downstream switches internal to the dGPU. */ + while (adev->link_partner && + adev->link_partner->vendor == PCI_VENDOR_ID_ATI) { + adev->link_dev = adev->link_partner; + adev->link_partner = pci_upstream_bridge(adev->link_dev); } - - return parent; } /** @@ -2003,17 +2074,25 @@ static struct pci_dev *amdgpu_device_find_parent(struct amdgpu_device *adev) static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) { struct amdgpu_ip_block *ip_block; - struct pci_dev *parent; - bool total, skip_bios; + bool total, skip_bios, early_full_gpu_access = false; uint32_t bios_flags; int i, r; amdgpu_device_enable_virtual_display(adev); if (amdgpu_sriov_vf(adev)) { - r = amdgpu_virt_request_full_gpu(adev, true); - if (r) - return r; + /* + * Legacy hosts do not provide init data before early init, so + * keep the original early full GPU access request for them. Newer + * hosts publish the init data through VF FB, which lets us defer + * full GPU access until after non-GPU early init work is done. + */ + early_full_gpu_access = (adev->virt.req_init_data_ver == 0); + if (early_full_gpu_access) { + r = amdgpu_virt_request_full_gpu(adev, true); + if (r) + return r; + } r = amdgpu_virt_init_critical_region(adev); if (r) @@ -2090,10 +2169,9 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) !dev_is_removable(&adev->pdev->dev)) adev->flags |= AMD_IS_PX; - if (!(adev->flags & AMD_IS_APU)) { - parent = amdgpu_device_find_parent(adev); - adev->has_pr3 = parent ? pci_pr3_present(parent) : false; - } + if (!(adev->flags & AMD_IS_APU)) + adev->has_pr3 = adev->link_partner && + pci_pr3_present(adev->link_partner); adev->pm.pp_feature = amdgpu_pp_feature_mask; if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS) @@ -2176,6 +2254,13 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) if (!total) return -ENODEV; + /* Request full GPU access only for the remaining SR-IOV init work. */ + if (amdgpu_sriov_vf(adev) && !early_full_gpu_access) { + r = amdgpu_virt_request_full_gpu(adev, true); + if (r) + return r; + } + if (adev->gmc.xgmi.supported) amdgpu_xgmi_early_init(adev); @@ -2537,7 +2622,11 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) if (r) goto init_failed; - amdgpu_ttm_enable_buffer_funcs(adev); + /* If SDMA is not brought up during hwini, the ttm buffer funcs enablement + * is delayed after reset-on-init completes. + */ + if (amdgpu_ip_member_of_hwini(adev, AMD_IP_BLOCK_TYPE_SDMA)) + amdgpu_ttm_enable_buffer_funcs(adev); /* Don't init kfd if whole hive need to be reset during init */ if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { @@ -3745,6 +3834,14 @@ static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev) amdgpu_ptl_sysfs_fini(adev); } +static bool +amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px) +{ + return !pci_is_thunderbolt_attached(adev->pdev) && + (px || (!dev_is_removable(&adev->pdev->dev) && + apple_gmux_detect(NULL, NULL))); +} + /** * amdgpu_device_init - initialize the driver * @@ -3766,6 +3863,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, adev->shutdown = false; adev->flags = flags; + amdgpu_device_init_pcie_links(adev); if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST) adev->asic_type = amdgpu_force_asic_type; @@ -4194,8 +4292,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, px = amdgpu_device_supports_px(adev); - if (px || (!dev_is_removable(&adev->pdev->dev) && - apple_gmux_detect(NULL, NULL))) + if (amdgpu_device_should_register_switcheroo(adev, px)) vga_switcheroo_register_client(adev->pdev, &amdgpu_switcheroo_ops, px); @@ -4326,7 +4423,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) void amdgpu_device_fini_sw(struct amdgpu_device *adev) { - int i, idx; + int i; bool px; amdgpu_device_ip_fini(adev); @@ -4359,8 +4456,7 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) px = amdgpu_device_supports_px(adev); - if (px || (!dev_is_removable(&adev->pdev->dev) && - apple_gmux_detect(NULL, NULL))) + if (amdgpu_device_should_register_switcheroo(adev, px)) vga_switcheroo_unregister_client(adev->pdev); if (px) @@ -4369,11 +4465,9 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) vga_client_unregister(adev->pdev); - if (drm_dev_enter(adev_to_drm(adev), &idx)) { - + if (adev->rmmio) { iounmap(adev->rmmio); adev->rmmio = NULL; - drm_dev_exit(idx); } if (IS_ENABLED(CONFIG_PERF_EVENTS)) @@ -5202,6 +5296,31 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, amdgpu_fence_driver_force_completion(ring, fence); } + /* + * KIQ rings are polling-fence/no_scheduler like MES, so realign their + * fence too (one ring per XCC), otherwise the first post-reset KIQ + * submission polls forever on a stale seq. + */ + for (i = 0; i < AMDGPU_MAX_GC_INSTANCES; i++) { + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[i].ring; + + if (kiq_ring->fence_drv.initialized && kiq_ring->sched.ready) + amdgpu_fence_driver_force_completion(kiq_ring, fence); + } + + /* + * MES scheduler rings have no drm scheduler, so they are missed by the + * loop above. Realign their polling fence too (one per XCC), otherwise the + * first post-reset submission polls forever on a stale seq. sched.ready is + * only set while the driver owns the ring. + */ + for (i = 0; i < AMDGPU_MAX_MES_INST_PIPES; i++) { + struct amdgpu_ring *mes_ring = &adev->mes.ring[i]; + + if (mes_ring->fence_drv.initialized && mes_ring->sched.ready) + amdgpu_fence_driver_force_completion(mes_ring, fence); + } + amdgpu_fence_driver_isr_toggle(adev, false); r = amdgpu_reset_prepare_hwcontext(adev, reset_context); @@ -5315,8 +5434,6 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context) if (r) goto out; - amdgpu_ttm_enable_buffer_funcs(tmp_adev); - r = amdgpu_device_ip_resume_phase3(tmp_adev); if (r) goto out; @@ -6008,11 +6125,9 @@ static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev, *width = PCIE_LNK_WIDTH_UNKNOWN; if (amdgpu_device_pcie_dynamic_switching_supported(adev)) { - struct pci_dev *parent = amdgpu_device_find_parent(adev); - - if (parent) { - *speed = pcie_get_speed_cap(parent); - *width = pcie_get_width_cap(parent); + if (adev->link_partner) { + *speed = pcie_get_speed_cap(adev->link_partner); + *width = pcie_get_width_cap(adev->link_partner); } } else { /* use the current speeds rather than max if switching is not supported */ @@ -6034,21 +6149,11 @@ static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev, enum pci_bus_speed *speed, enum pcie_link_width *width) { - struct pci_dev *parent = adev->pdev; - if (!speed || !width) return; - /* use the device itself */ - *speed = pcie_get_speed_cap(adev->pdev); - *width = pcie_get_width_cap(adev->pdev); - - /* use the link outside the device */ - parent = amdgpu_device_find_parent(adev); - if (parent) { - *speed = pcie_get_speed_cap(parent); - *width = pcie_get_width_cap(parent); - } + *speed = pcie_get_speed_cap(adev->link_dev); + *width = pcie_get_width_cap(adev->link_dev); } /** @@ -6826,8 +6931,8 @@ struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev, struct amdgpu_ring *ring, struct amdgpu_job *job) { - struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id]; struct drm_sched_fence *f = job->base.s_fence; + struct amdgpu_isolation *isolation; struct dma_fence *dep; void *owner; int r; @@ -6840,6 +6945,9 @@ struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev, ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE) return NULL; + isolation = &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ? + 0 : ring->xcp_id]; + /* * All submissions where enforce isolation is false are handled as if * they come from a single client. Use ~0l as the owner to distinct it diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index b33c300e26e2e..9adf3eed8822c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -43,6 +43,7 @@ #include #include #include +#include static const struct dma_buf_attach_ops amdgpu_dma_buf_attach_ops; @@ -100,15 +101,54 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf, pci_p2pdma_distance(adev->pdev, attach->dev, false) < 0) attach->peer2peer = false; + /* + * Only allow P2P while the exporter is active, and keep it active + * until detach. With runtime PM disabled take a plain reference so + * the put in detach stays balanced. + */ + if (attach->peer2peer) { + struct device *dev = adev_to_drm(adev)->dev; + int ret = pm_runtime_get_if_active(dev); + + if (!ret) + attach->peer2peer = false; + else if (ret < 0) + pm_runtime_get_noresume(dev); + } + r = dma_resv_lock(bo->tbo.base.resv, NULL); if (r) - return r; + goto err_pm_put; amdgpu_vm_bo_update_shared(bo); dma_resv_unlock(bo->tbo.base.resv); return 0; + +err_pm_put: + if (attach->peer2peer) + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); + return r; +} + +/** + * amdgpu_dma_buf_detach - &dma_buf_ops.detach implementation + * + * @dmabuf: DMA-buf where we remove the attachment from + * @attach: the attachment to remove + * + * Drop the runtime PM reference taken in amdgpu_dma_buf_attach(). + */ +static void amdgpu_dma_buf_detach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = dmabuf->priv; + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + + if (attach->peer2peer) + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); } /** @@ -350,6 +390,7 @@ static void amdgpu_dma_buf_vunmap(struct dma_buf *dma_buf, struct iosys_map *map const struct dma_buf_ops amdgpu_dmabuf_ops = { .attach = amdgpu_dma_buf_attach, + .detach = amdgpu_dma_buf_detach, .pin = amdgpu_dma_buf_pin, .unpin = amdgpu_dma_buf_unpin, .map_dma_buf = amdgpu_dma_buf_map, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 1aed121f4ddb6..e814701bc8fd4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2548,6 +2548,7 @@ amdgpu_pci_remove(struct pci_dev *pdev) if (adev->pm.rpm_mode != AMDGPU_RUNPM_NONE) { pm_runtime_get_sync(dev->dev); pm_runtime_forbid(dev->dev); + pm_runtime_dont_use_autosuspend(dev->dev); } amdgpu_driver_unload_kms(dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index f6b7522c3c82f..f8652fd0525df 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -68,6 +68,9 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work) mutex_lock(&uq_mgr->userq_mutex); + /* Fence waits are not allowed in a fence signalling critical section. */ + amdgpu_userq_wait_for_signal(uq_mgr); + /* * This is intentionally after taking the userq_mutex since we do * allocate memory while holding this lock, but only after ensuring that diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c index e3972673fd641..c47e5ffd3d0c7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c @@ -237,7 +237,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev) } if (adev->mes.hung_queue_db_array_size) { - for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) { + for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { r = amdgpu_bo_create_kernel(adev, adev->mes.hung_queue_db_array_size * sizeof(u32), PAGE_SIZE, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 7139cb3b2db5b..5c5c0fc76aeb8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -396,6 +396,12 @@ static bool psp_get_runtime_db_entry(struct amdgpu_device *adev, bool ret = false; int i; + /* + * Runtime DB is for dGPUs only. + */ + if (adev->flags & AMD_IS_APU) + return false; + if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14) || diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index b97fa35bac232..4d5d95e74bb5d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -254,7 +254,6 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, ring->adev = adev; ring->num_hw_submission = sched_hw_submission; ring->sched_score = sched_score; - ring->vmid_wait = dma_fence_get_stub(); ring->idx = adev->num_rings++; adev->rings[ring->idx] = ring; @@ -374,6 +373,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, ring->max_dw = max_dw; ring->hw_prio = hw_prio; + ring->vmid_wait = dma_fence_get_stub(); if (!ring->no_scheduler && ring->funcs->type < AMDGPU_HW_IP_NUM) { hw_ip = ring->funcs->type; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index eaa86e32912e5..ea416749138d8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -190,6 +190,8 @@ amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entit * @tmz: if we should setup a TMZ enabled mapping * @size: in number of bytes to map, out number of bytes mapped * @addr: resulting address inside the MC address space + * @vm_needs_flush: out, set true if a GART window was programmed (VMID 0 flush + * needed) or false for a direct address * * Setup one of the GART windows to access a specific piece of memory or return * the physical address for local memory. @@ -199,7 +201,8 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity, struct ttm_resource *mem, struct amdgpu_res_cursor *mm_cur, unsigned int window, - bool tmz, uint64_t *size, uint64_t *addr) + bool tmz, uint64_t *size, uint64_t *addr, + bool *vm_needs_flush) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); unsigned int offset, num_pages, num_dw, num_bytes; @@ -220,9 +223,12 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity, if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) { *addr = amdgpu_ttm_domain_start(adev, mem->mem_type) + mm_cur->start; + *vm_needs_flush = false; return 0; } + /* A GART window is programmed below, so its VMID 0 TLB needs a flush */ + *vm_needs_flush = true; /* * If start begins at an offset inside the page, then adjust the size @@ -322,6 +328,7 @@ static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, while (src_mm.remaining) { uint64_t from, to, cur_size, tiling_flags; uint32_t num_type, data_format, max_com, write_compress_disable; + bool src_vm_flush, dst_vm_flush; struct dma_fence *next; /* Never copy more than 256MiB at once to avoid a timeout */ @@ -329,12 +336,12 @@ static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, /* Map src to window 0 and dst to window 1. */ r = amdgpu_ttm_map_buffer(entity, src->bo, src->mem, &src_mm, - 0, tmz, &cur_size, &from); + 0, tmz, &cur_size, &from, &src_vm_flush); if (r) goto error; r = amdgpu_ttm_map_buffer(entity, dst->bo, dst->mem, &dst_mm, - 1, tmz, &cur_size, &to); + 1, tmz, &cur_size, &to, &dst_vm_flush); if (r) goto error; @@ -362,7 +369,7 @@ static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, } r = amdgpu_copy_buffer(adev, entity, from, to, cur_size, resv, - &next, true, copy_flags); + &next, src_vm_flush || dst_vm_flush, copy_flags); if (r) goto error; @@ -2580,6 +2587,7 @@ int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity, struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); struct dma_fence *fence = NULL; struct amdgpu_res_cursor dst; + bool vm_needs_flush = false; int r; if (!entity) @@ -2601,13 +2609,13 @@ int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity, cur_size = min(dst.size, 256ULL << 20); r = amdgpu_ttm_map_buffer(entity, &bo->tbo, bo->tbo.resource, &dst, - 0, false, &cur_size, &to); + 0, false, &cur_size, &to, &vm_needs_flush); if (r) goto error; r = amdgpu_ttm_fill_mem(adev, entity, 0, to, cur_size, resv, - &next, true, k_job_id); + &next, vm_needs_flush, k_job_id); if (r) goto error; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index b18d787206562..655429c403518 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -167,27 +167,27 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work) void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue) { struct amdgpu_device *adev; - unsigned long timeout_ms; + unsigned long timeout_jiffies; adev = queue->userq_mgr->adev; /* Determine timeout based on queue type */ switch (queue->queue_type) { case AMDGPU_RING_TYPE_GFX: - timeout_ms = adev->gfx_timeout; + timeout_jiffies = adev->gfx_timeout; break; case AMDGPU_RING_TYPE_COMPUTE: - timeout_ms = adev->compute_timeout; + timeout_jiffies = adev->compute_timeout; break; case AMDGPU_RING_TYPE_SDMA: - timeout_ms = adev->sdma_timeout; + timeout_jiffies = adev->sdma_timeout; break; default: - timeout_ms = adev->gfx_timeout; + timeout_jiffies = adev->gfx_timeout; break; } queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work, - msecs_to_jiffies(timeout_ms)); + timeout_jiffies); } void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell) @@ -1168,7 +1168,7 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr) return ret; } -static void +void amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) { struct amdgpu_usermode_queue *queue; @@ -1187,8 +1187,6 @@ amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr) { - /* Wait for any pending userqueue fence work to finish */ - amdgpu_userq_wait_for_signal(uq_mgr); amdgpu_userq_evict_all(uq_mgr); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h index d1751febaefe4..ff9f203a320b2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h @@ -156,6 +156,7 @@ void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev); void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr); void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr); +void amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr); void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr); void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index de3dbc95e376c..5ca302e62af7b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -517,7 +517,8 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev) } memset_io(ptr, 0, size); /* to restore uvd fence seq */ - amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring, NULL); + if (adev->uvd.inst[i].ring.fence_drv.initialized) + amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring, NULL); } } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 1baad7624f1f9..5edf00a8273b2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -773,7 +773,9 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync) { struct amdgpu_device *adev = ring->adev; - struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id]; + struct amdgpu_isolation *isolation = + &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ? + 0 : ring->xcp_id]; unsigned vmhub = ring->vm_hub; struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; @@ -2062,7 +2064,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, after->start = eaddr + 1; after->last = tmp->last; after->offset = tmp->offset; - after->offset += (after->start - tmp->start) << PAGE_SHIFT; + after->offset += (after->start - tmp->start) << AMDGPU_GPU_PAGE_SHIFT; after->flags = tmp->flags; after->bo_va = tmp->bo_va; list_add(&after->list, &tmp->bo_va->invalids); @@ -2663,6 +2665,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, amdgpu_bo_unref(&root_bo); error_free_delayed: + dma_fence_put(vm->last_update); dma_fence_put(vm->last_tlb_flush); dma_fence_put(vm->last_unlocked); ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move); @@ -3109,7 +3112,7 @@ static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m, id = 0; seq_puts(m, "\tIdle BOs:\n"); - list_for_each_entry(base, &lists->needs_update, vm_status) { + list_for_each_entry(base, &lists->idle, vm_status) { if (!base->bo) continue; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index 2a241a5b12c42..ac3f71d77140a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -918,9 +918,6 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev) struct ttm_resource_manager *man = &mgr->manager; int err; - man->cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram", adev->gmc.real_vram_size); - if (IS_ERR(man->cg)) - return PTR_ERR(man->cg); ttm_resource_manager_init(man, &adev->mman.bdev, adev->gmc.real_vram_size); @@ -935,6 +932,10 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev) if (err) return err; + man->cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram", adev->gmc.real_vram_size); + if (IS_ERR(man->cg)) + return PTR_ERR(man->cg); + ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); ttm_resource_manager_set_used(man, true); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c index e63d05c477a08..adbfdee01e804 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c @@ -1659,6 +1659,9 @@ static void amdgpu_xgmi_reset_on_init_work(struct work_struct *work) amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { + /* Enable ttm buffers funcs after the reset lock has been dropped. */ + amdgpu_ttm_enable_buffer_funcs(tmp_adev); + r = amdgpu_ras_init_badpage_info(tmp_adev); if (r && r != -EHWPOISON) dev_err(tmp_adev->dev, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 3b12eb27a2533..047f5f891b8a4 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -1646,10 +1646,10 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(11, 0, 2): case IP_VERSION(11, 0, 3): if (!adev->gfx.disable_uq && - adev->gfx.me_fw_version >= 2420 && - adev->gfx.pfp_fw_version >= 2580 && - adev->gfx.mec_fw_version >= 2650 && - adev->mes.fw_version[0] >= 120) { + adev->gfx.me_fw_version >= 3090 && + adev->gfx.pfp_fw_version >= 3190 && + adev->gfx.mec_fw_version >= 3450 && + adev->mes.fw_version[0] >= 147) { adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index 9d7a75670beee..e55c9002113c7 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -1434,10 +1434,10 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(12, 0, 0): case IP_VERSION(12, 0, 1): if (!adev->gfx.disable_uq && - adev->gfx.me_fw_version >= 2780 && - adev->gfx.pfp_fw_version >= 2840 && - adev->gfx.mec_fw_version >= 3050 && - adev->mes.fw_version[0] >= 123) { + adev->gfx.me_fw_version >= 3090 && + adev->gfx.pfp_fw_version >= 3190 && + adev->gfx.mec_fw_version >= 3450 && + adev->mes.fw_version[0] >= 147) { adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; } @@ -1823,6 +1823,21 @@ static void gfx_v12_0_constants_init(struct amdgpu_device *adev) gfx_v12_0_get_tcc_info(adev); adev->gfx.config.pa_sc_tile_steering_override = 0; + /* Set whether texture coordinate truncation is conformant. */ + tmp = RREG32_SOC15(GC, 0, regTA_CNTL2); + adev->gfx.config.ta_cntl2_truncate_coord_mode = + REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE); + + /* Program DB_RING_CONTROL for multiple GFX pipes + * Default power up value is 1. + * Possible values: + * 0 - split occlusion counters between gfx pipes + * 1 - all occlusion counters to pipe 0 + * 2 - all occlusion counters to pipe 1 + */ + WREG32_FIELD15_PREREG(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL, + (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1); + /* Set whether texture coordinate truncation is conformant. */ tmp = RREG32_SOC15(GC, 0, regTA_CNTL2); adev->gfx.config.ta_cntl2_truncate_coord_mode = diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c index ac90d8e9d86a8..8f3f90d5ce375 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c @@ -1881,11 +1881,13 @@ static int gfx_v6_0_ring_test_ring(struct amdgpu_ring *ring) return r; } -static void gfx_v6_0_ring_emit_vgt_flush(struct amdgpu_ring *ring) +static void gfx_v6_0_ring_emit_event_write(struct amdgpu_ring *ring, + uint32_t event_type, + uint32_t event_index) { amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); - amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) | - EVENT_INDEX(0)); + amdgpu_ring_write(ring, EVENT_TYPE(event_type) | + EVENT_INDEX(event_index)); } static void gfx_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, @@ -1924,12 +1926,6 @@ static void gfx_v6_0_ring_emit_ib(struct amdgpu_ring *ring, unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 header, control = 0; - /* insert SWITCH_BUFFER packet before first IB in the ring frame */ - if (flags & AMDGPU_HAVE_CTX_SWITCH) { - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - } - if (ib->flags & AMDGPU_IB_FLAG_CE) header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); else @@ -2351,7 +2347,7 @@ static int gfx_v6_0_cp_resume(struct amdgpu_device *adev) static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) { - int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); + int usepfp = 1; uint32_t seq = ring->fence_drv.sync_seq; uint64_t addr = ring->fence_drv.gpu_addr; @@ -2364,20 +2360,12 @@ static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) amdgpu_ring_write(ring, seq); amdgpu_ring_write(ring, 0xffffffff); amdgpu_ring_write(ring, 4); /* poll interval */ - - if (usepfp) { - /* synce CE with ME to prevent CE fetch CEIB before context switch done */ - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - } } static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring, unsigned vmid, uint64_t pd_addr) { - int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); + int usepfp = 1; amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); @@ -2395,19 +2383,13 @@ static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring, /* sync PFP to ME, otherwise we might get invalid PFP reads */ amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); amdgpu_ring_write(ring, 0x0); - - /* synce CE with ME to prevent CE fetch CEIB before context switch done */ - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); } } static void gfx_v6_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val) { - int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); + int usepfp = 1; amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | @@ -2996,12 +2978,30 @@ static uint64_t gfx_v6_0_get_gpu_clock_counter(struct amdgpu_device *adev) return clock; } +static void gfx_v6_0_ring_emit_sb(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); + amdgpu_ring_write(ring, 0); +} + static void gfx_v6_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) { - if (flags & AMDGPU_HAVE_CTX_SWITCH) - gfx_v6_0_ring_emit_vgt_flush(ring); + u32 dw2 = 0x80000000; /* set load_enable otherwise this package is just NOPs */ + + if (flags & AMDGPU_HAVE_CTX_SWITCH) { + gfx_v6_0_ring_emit_event_write(ring, VS_PARTIAL_FLUSH, 4); + gfx_v6_0_ring_emit_event_write(ring, VGT_FLUSH, 0); + + /* set load_global_config (load_global_uconfig doesn't exist on GFX6) */ + dw2 |= 0x1; + /* set load_cs_sh_regs */ + dw2 |= 0x01000000; + /* set load_per_context_state & load_gfx_sh_regs */ + dw2 |= 0x10002; + } + amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); - amdgpu_ring_write(ring, 0x80000000); + amdgpu_ring_write(ring, dw2); amdgpu_ring_write(ring, 0); } @@ -3527,11 +3527,12 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = { .emit_frame_size = 5 + 5 + /* hdp flush / invalidate */ 14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */ - 7 + 4 + /* gfx_v6_0_ring_emit_pipeline_sync */ - SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 6 + /* gfx_v6_0_ring_emit_vm_flush */ - 3 + 2 + /* gfx_v6_ring_emit_cntxcntl including vgt flush */ + 7 + /* gfx_v6_0_ring_emit_pipeline_sync */ + SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 2 + /* gfx_v6_0_ring_emit_vm_flush */ + 3 * 2 + /* gfx_v6_0_ring_emit_sb x3 (from amdgpu_vm_flush, amdgpu_ib_schedule) */ + 3 + 2 + 2 + /* gfx_v6_ring_emit_cntxcntl including VGT flush */ 5, /* SURFACE_SYNC */ - .emit_ib_size = 6, /* gfx_v6_0_ring_emit_ib */ + .emit_ib_size = 4, /* gfx_v6_0_ring_emit_ib */ .emit_ib = gfx_v6_0_ring_emit_ib, .emit_fence = gfx_v6_0_ring_emit_fence, .emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync, @@ -3539,6 +3540,7 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = { .test_ring = gfx_v6_0_ring_test_ring, .test_ib = gfx_v6_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, + .emit_switch_buffer = gfx_v6_0_ring_emit_sb, .emit_cntxcntl = gfx_v6_ring_emit_cntxcntl, .emit_wreg = gfx_v6_0_ring_emit_wreg, .emit_mem_sync = gfx_v6_0_emit_mem_sync, @@ -3554,10 +3556,11 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_compute = { .emit_frame_size = 5 + 5 + /* hdp flush / invalidate */ 7 + /* gfx_v6_0_ring_emit_pipeline_sync */ - SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v6_0_ring_emit_vm_flush */ + SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 2 + /* gfx_v6_0_ring_emit_vm_flush */ 14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */ + 3 * 2 + /* gfx_v6_0_ring_emit_sb x3 (from amdgpu_vm_flush, amdgpu_ib_schedule) */ 5, /* SURFACE_SYNC */ - .emit_ib_size = 6, /* gfx_v6_0_ring_emit_ib */ + .emit_ib_size = 4, /* gfx_v6_0_ring_emit_ib */ .emit_ib = gfx_v6_0_ring_emit_ib, .emit_fence = gfx_v6_0_ring_emit_fence, .emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync, @@ -3565,6 +3568,7 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_compute = { .test_ring = gfx_v6_0_ring_test_ring, .test_ib = gfx_v6_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, + .emit_switch_buffer = gfx_v6_0_ring_emit_sb, .emit_wreg = gfx_v6_0_ring_emit_wreg, .emit_mem_sync = gfx_v6_0_emit_mem_sync, }; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 70ba81e6b4d43..180ed00bb9590 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -4533,9 +4533,11 @@ static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring) /* set static priority for a queue/ring */ gfx_v8_0_mqd_set_priority(ring, mqd); tmp = RREG32(mmCP_HQD_QUANTUM); - tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1); - tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); - tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10); + if (ring != &adev->gfx.kiq[0].ring) { + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10); + } mqd->cp_hqd_quantum = tmp; /* map_queues packet doesn't need activate the queue, diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c index 9a40107a0869d..fec08f235b920 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c @@ -185,8 +185,11 @@ static int xgpu_ai_send_access_requests(struct amdgpu_device *adev, } else if (req == IDH_REQ_GPU_INIT_DATA){ /* Dummy REQ_GPU_INIT_DATA handling */ r = xgpu_ai_poll_msg(adev, IDH_REQ_GPU_INIT_DATA_READY); - /* version set to 0 since dummy */ - adev->virt.req_init_data_ver = 0; + /* + * AI uses the GPU_CRIT_REGION_V1 layout in practice, so fix the + * dummy version value to match the actual init-data format. + */ + adev->virt.req_init_data_ver = GPU_CRIT_REGION_V1; } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c index bdfd2917e3cab..def02993b7cfd 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c @@ -535,7 +535,7 @@ static void nbio_v7_9_handle_ras_controller_intr_no_bifring(struct amdgpu_device RAS_CNTLR_INTERRUPT_CLEAR, 1); WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl); - if (!ras->disable_ras_err_cnt_harvest) { + if (ras && !ras->disable_ras_err_cnt_harvest && obj) { /* * clear error status after ras_controller_intr * according to hw team and count ue number diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c index 81bba3ec2a937..00d8f35846f22 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c @@ -1964,9 +1964,13 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, len_dw = msg[1] / 4; num_buffers = msg[2]; - /* Verify that all indices fit within the claimed length. Each index is 4 DWORDs */ - if (num_buffers > len_dw || 6 + num_buffers * 4 > len_dw) { - DRM_ERROR("VCN message has too many buffers!\n"); + /* Verify that all indices fit within the claimed length. + * There are 6 dwords in the header before the first buffer. + * Each buffer has 4 dwords. Any trailing dwords after the + * last buffer are ignored. + */ + if (len_dw < 6 || num_buffers > (len_dw - 6) / 4) { + DRM_ERROR("Invalid VCN message!\n"); r = -EINVAL; goto out; } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c index 894780669f9cf..a12652dc3a380 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c @@ -1880,9 +1880,13 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, len_dw = msg[1] / 4; num_buffers = msg[2]; - /* Verify that all indices fit within the claimed length. Each index is 4 DWORDs */ - if (num_buffers > len_dw || 6 + num_buffers * 4 > len_dw) { - DRM_ERROR("VCN message has too many buffers!\n"); + /* Verify that all indices fit within the claimed length. + * There are 6 dwords in the header before the first buffer. + * Each buffer has 4 dwords. Any trailing dwords after the + * last buffer are ignored. + */ + if (len_dw < 6 || num_buffers > (len_dw - 6) / 4) { + DRM_ERROR("Invalid VCN message!\n"); r = -EINVAL; goto out; } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c index 7f001c32e9113..55acb544b1b6d 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c @@ -1672,7 +1672,8 @@ static int vcn_v4_0_3_reset_jpeg_pre_helper(struct amdgpu_device *adev, int inst /* if Jobs are still pending after timeout, * We'll handle them in the bottom helper */ - amdgpu_fence_wait_polling(ring, wait_seq, adev->video_timeout); + amdgpu_fence_wait_polling(ring, wait_seq, + jiffies_to_usecs(adev->video_timeout)); } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c index d3db0494341e3..d25a775b3efd1 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c @@ -1318,7 +1318,8 @@ static int vcn_v5_0_1_reset_jpeg_pre_helper(struct amdgpu_device *adev, int inst /* if Jobs are still pending after timeout, * We'll handle them in the bottom helper */ - amdgpu_fence_wait_polling(ring, wait_seq, adev->video_timeout); + amdgpu_fence_wait_polling(ring, wait_seq, + jiffies_to_usecs(adev->video_timeout)); } return 0; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index e1689b3d2add9..2e85a1de51c01 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -70,18 +71,54 @@ static const struct class kfd_class = { }; /* - * Cache the address space of the chardev on first open so that the reset - * path can drop all userspace mappings of doorbell and MMIO ranges via - * unmap_mapping_range(). + * Private pseudo-filesystem for KFD, Provides a stable, module-owned + * inode whose address_space is the unmap target for all /dev/kfd + * openers during GPU reset. */ -static struct address_space *kfd_dev_mapping; +static struct vfsmount *kfd_fs_mnt; +static int kfd_fs_cnt; -void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen) +static int kfd_fs_init_fs_context(struct fs_context *fc) +{ + return init_pseudo(fc, 0x4b464400 /* "KFD" */) ? 0 : -ENOMEM; +} + +static struct file_system_type kfd_fs_type = { + .name = "kfd", + .init_fs_context = kfd_fs_init_fs_context, + .kill_sb = kill_anon_super, +}; + +static struct inode *kfd_fs_inode_new(void) { - struct address_space *mapping = READ_ONCE(kfd_dev_mapping); + struct inode *inode; + int r; + + r = simple_pin_fs(&kfd_fs_type, &kfd_fs_mnt, &kfd_fs_cnt); + if (r < 0) + return ERR_PTR(r); + + inode = alloc_anon_inode(kfd_fs_mnt->mnt_sb); + if (IS_ERR(inode)) + simple_release_fs(&kfd_fs_mnt, &kfd_fs_cnt); - if (mapping) - unmap_mapping_range(mapping, holebegin, holelen, 1); + return inode; +} + +static void kfd_fs_inode_free(struct inode *inode) +{ + if (inode) { + iput(inode); + simple_release_fs(&kfd_fs_mnt, &kfd_fs_cnt); + } +} + +static struct inode *kfd_anon_inode; + +void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen) +{ + if (kfd_anon_inode) + unmap_mapping_range(kfd_anon_inode->i_mapping, holebegin, holelen, 1); } static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id) @@ -107,6 +144,13 @@ int kfd_chardev_init(void) { int err = 0; + kfd_anon_inode = kfd_fs_inode_new(); + if (IS_ERR(kfd_anon_inode)) { + err = PTR_ERR(kfd_anon_inode); + kfd_anon_inode = NULL; + return err; + } + kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); err = kfd_char_dev_major; if (err < 0) @@ -130,6 +174,8 @@ int kfd_chardev_init(void) err_class_create: unregister_chrdev(kfd_char_dev_major, kfd_dev_name); err_register_chrdev: + kfd_fs_inode_free(kfd_anon_inode); + kfd_anon_inode = NULL; return err; } @@ -138,6 +184,8 @@ void kfd_chardev_exit(void) device_destroy(&kfd_class, MKDEV(kfd_char_dev_major, 0)); class_unregister(&kfd_class); unregister_chrdev(kfd_char_dev_major, kfd_dev_name); + kfd_fs_inode_free(kfd_anon_inode); + kfd_anon_inode = NULL; kfd_device = NULL; } @@ -150,12 +198,7 @@ static int kfd_open(struct inode *inode, struct file *filep) if (iminor(inode) != 0) return -ENODEV; - /* - * /dev/kfd is a single chardev so all opens share one inode. Cache - * its address_space on the first open for use by the reset path. - */ - if (!READ_ONCE(kfd_dev_mapping)) - cmpxchg(&kfd_dev_mapping, NULL, inode->i_mapping); + filep->f_mapping = kfd_anon_inode->i_mapping; is_32bit_user_mode = in_compat_syscall(); @@ -1200,7 +1243,8 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, if (flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM) size >>= 1; - atomic64_add(PAGE_ALIGN(size), &pdd->vram_usage); + size = PAGE_ALIGN(size); + atomic64_add(size, &pdd->vram_usage); } mutex_unlock(&p->mutex); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 67137679a901b..487cf746c4504 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -796,6 +796,13 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm, mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( q->properties.type)]; + if (qd && !mqd_mgr->restore_mqd) { + pr_debug("restore_mqd not implemented for queue type %d\n", + q->properties.type); + retval = -EOPNOTSUPP; + goto deallocate_vmid; + } + if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { retval = allocate_hqd(dqm, q); if (retval) @@ -1375,6 +1382,14 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm, } } + /* + * Heavy-weight TLB flush after MES removes queues to ensure + * in-flight memory accesses complete before memory is freed/migrated. + * HWS does this automatically, MES does not. + */ + if (dqm->dev->kfd->shared_resources.enable_mes) + kfd_flush_tlb(pdd); + if (!dqm->dev->kfd->shared_resources.enable_mes) { pdd->last_evict_timestamp = get_jiffies_64(); retval = execute_queues_cpsch(dqm, @@ -2154,6 +2169,12 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( q->properties.type)]; + if (qd && !mqd_mgr->restore_mqd) { + pr_debug("restore_mqd not implemented for queue type %d\n", + q->properties.type); + retval = -EOPNOTSUPP; + goto out_deallocate_doorbell; + } if (q->properties.type == KFD_QUEUE_TYPE_SDMA || q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) @@ -3651,8 +3672,11 @@ int suspend_queues(struct kfd_process *p, if (!per_device_suspended) { dqm_unlock(dqm); mutex_unlock(&p->event_mutex); - if (total_suspended) + if (total_suspended) { amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev); + /* Heavy-weight TLB flush after MES suspends queues */ + kfd_flush_tlb(pdd); + } continue; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 7cd236c1ff75f..f0dc0c00de822 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -134,7 +134,7 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys, u64 gart_s, gart_d; struct dma_fence *next; u64 size; - int r; + int r = 0; ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); entity = &adev->mman.move_entities[0]; @@ -258,15 +258,6 @@ svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr) return page; } -static void svm_migrate_put_sys_page(unsigned long addr) -{ - struct page *page; - - page = pfn_to_page(addr >> PAGE_SHIFT); - unlock_page(page); - put_page(page); -} - static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate) { unsigned long mpages = 0; @@ -293,7 +284,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, dma_addr_t *src; u64 *dst; u64 i, j; - int r; + int r = 0; pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start, prange->last, ttm_res_offset); @@ -319,6 +310,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, DMA_BIDIRECTIONAL); r = dma_mapping_error(dev, src[i]); if (r) { + src[i] = 0; dev_err(dev, "%s: fail %d dma_map_page\n", __func__, r); goto out_free_vram_pages; @@ -343,7 +335,8 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n", src[i] >> PAGE_SHIFT, page_to_pfn(spage)); - if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) { + /* accumulated j + 1 pages reach end of current drm_buddy_block */ + if (j + 1 >= (cursor.size >> PAGE_SHIFT)) { r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j + 1, FROM_RAM_TO_VRAM, @@ -357,7 +350,8 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, } } - r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j, + if (j > 0) + r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j, FROM_RAM_TO_VRAM, mfence); out_free_vram_pages: @@ -591,9 +585,10 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, dma_addr_t *scratch, u64 npages) { struct device *dev = adev->dev; - u64 *src; + struct page *dpage = NULL; dma_addr_t *dst; - struct page *dpage; + u64 *src; + u64 i = 0, j; u64 addr; int r = 0; @@ -647,6 +642,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, r = dma_mapping_error(dev, dst[i]); if (r) { dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r); + dst[i] = 0; goto out_oom; } @@ -654,17 +650,39 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, dst[i] >> PAGE_SHIFT, page_to_pfn(dpage)); migrate->dst[i] = migrate_pfn(page_to_pfn(dpage)); + + dpage = NULL; j++; } - r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j, - FROM_VRAM_TO_RAM, mfence); - + if (j > 0) + r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j, + FROM_VRAM_TO_RAM, mfence); out_oom: if (r) { pr_debug("failed %d copy to ram\n", r); + + /* first release current dpage when dma_map_page fail */ + if (dpage) { + unlock_page(dpage); + put_page(dpage); + } + + /* release previous allocated sys pages and unmap dma address */ while (i--) { - svm_migrate_put_sys_page(dst[i]); + + if (dst[i]) { + dma_unmap_page(dev, dst[i], PAGE_SIZE, + DMA_BIDIRECTIONAL); + dst[i] = 0; + } + + dpage = migrate_pfn_to_page(migrate->dst[i]); + if (!dpage) + continue; + + unlock_page(dpage); + put_page(dpage); migrate->dst[i] = 0; } } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c index e034da638c07a..4f8a8a1a61860 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c @@ -204,7 +204,7 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * is safe, giving a maximum field value of 0xA. */ m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; + ffs(q->eop_ring_buffer_size / sizeof(unsigned int) / 4)) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c index 350fcbbba4b29..bf015dc5b8684 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c @@ -242,7 +242,7 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * is safe, giving a maximum field value of 0xA. */ m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; + ffs(q->eop_ring_buffer_size / sizeof(unsigned int) / 4)) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c index 7c387fa900766..6ea09b031caf8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c @@ -217,7 +217,7 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * is safe, giving a maximum field value of 0xA. */ m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; + ffs(q->eop_ring_buffer_size / sizeof(unsigned int) / 4)) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = @@ -380,6 +380,63 @@ static int debugfs_show_mqd_sdma(struct seq_file *m, void *data) #endif +static void restore_mqd(struct mqd_manager *mm, void **mqd, + struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, + struct queue_properties *qp, const void *mqd_src, + const void *ctl_stack_src, const u32 ctl_stack_size) +{ + u64 addr; + struct v12_compute_mqd *m; + + m = (struct v12_compute_mqd *)mqd_mem_obj->cpu_ptr; + addr = mqd_mem_obj->gpu_addr; + + memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size)); + memcpy(m, mqd_src, sizeof(*m)); + + /* Update MQD base address to the newly allocated location */ + m->cp_mqd_base_addr_lo = lower_32_bits(addr); + m->cp_mqd_base_addr_hi = upper_32_bits(addr); + + m->cp_hqd_pq_doorbell_control &= + ~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK; + m->cp_hqd_pq_doorbell_control |= + qp->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; + pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell_control); + + *mqd = m; + if (gart_addr) + *gart_addr = addr; + + qp->is_active = 0; +} + +static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd, + struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, + struct queue_properties *qp, + const void *mqd_src, + const void *ctl_stack_src, + const u32 ctl_stack_size) +{ + u64 addr; + struct v12_sdma_mqd *m; + + m = (struct v12_sdma_mqd *)mqd_mem_obj->cpu_ptr; + addr = mqd_mem_obj->gpu_addr; + + memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size)); + memcpy(m, mqd_src, sizeof(*m)); + + m->sdmax_rlcx_doorbell_offset = + qp->doorbell_off << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT; + + *mqd = m; + if (gart_addr) + *gart_addr = addr; + + qp->is_active = 0; +} + struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type, struct kfd_node *dev) { @@ -407,6 +464,7 @@ struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type, mqd->mqd_size = sizeof(struct v12_compute_mqd); mqd->get_wave_state = get_wave_state; mqd->mqd_stride = kfd_mqd_stride; + mqd->restore_mqd = restore_mqd; #if defined(CONFIG_DEBUG_FS) mqd->debugfs_show_mqd = debugfs_show_mqd; #endif @@ -453,6 +511,7 @@ struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type, mqd->is_occupied = kfd_is_occupied_sdma; mqd->mqd_size = sizeof(struct v12_sdma_mqd); mqd->mqd_stride = kfd_mqd_stride; + mqd->restore_mqd = restore_mqd_sdma; #if defined(CONFIG_DEBUG_FS) mqd->debugfs_show_mqd = debugfs_show_mqd_sdma; #endif diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c index 431a940f91f3b..c709db0210ced 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c @@ -295,7 +295,7 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * is safe, giving a maximum field value of 0xA. */ m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, - ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1) : 0; + ffs(q->eop_ring_buffer_size / sizeof(unsigned int) / 4)) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = @@ -641,6 +641,72 @@ static int debugfs_show_mqd_sdma(struct seq_file *m, void *data) #endif +static void restore_mqd_v12_1(struct mqd_manager *mm, void **mqd, + struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, + struct queue_properties *qp, const void *mqd_src, + const void *ctl_stack_src, const u32 ctl_stack_size) +{ + u64 addr; + struct v12_1_compute_mqd *m; + + /* + * GFX12.1 is multi-XCC capable but this restore handles XCC0 only. + * Multi-XCC CRIU restore is currently unreachable because + * kfd_criu_restore_queue() validates against unscaled mqd_size. + */ + if (NUM_XCC(mm->dev->xcc_mask) > 1) + pr_warn_once("GFX12.1 multi-XCC CRIU restore not fully supported\n"); + + m = (struct v12_1_compute_mqd *)mqd_mem_obj->cpu_ptr; + addr = mqd_mem_obj->gpu_addr; + + memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size) * + NUM_XCC(mm->dev->xcc_mask)); + memcpy(m, mqd_src, sizeof(*m)); + + /* Update MQD base address to the newly allocated location */ + m->cp_mqd_base_addr_lo = lower_32_bits(addr); + m->cp_mqd_base_addr_hi = upper_32_bits(addr); + + m->cp_hqd_pq_doorbell_control &= + ~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK; + m->cp_hqd_pq_doorbell_control |= + qp->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; + pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell_control); + + *mqd = m; + if (gart_addr) + *gart_addr = addr; + + qp->is_active = 0; +} + +static void restore_mqd_sdma_v12_1(struct mqd_manager *mm, void **mqd, + struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, + struct queue_properties *qp, + const void *mqd_src, + const void *ctl_stack_src, + const u32 ctl_stack_size) +{ + u64 addr; + struct v12_sdma_mqd *m; + + m = (struct v12_sdma_mqd *)mqd_mem_obj->cpu_ptr; + addr = mqd_mem_obj->gpu_addr; + + memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size)); + memcpy(m, mqd_src, sizeof(*m)); + + m->sdmax_rlcx_doorbell_offset = + qp->doorbell_off << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT; + + *mqd = m; + if (gart_addr) + *gart_addr = addr; + + qp->is_active = 0; +} + struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type, struct kfd_node *dev) { @@ -668,6 +734,7 @@ struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type, mqd->mqd_size = sizeof(struct v12_1_compute_mqd); mqd->get_wave_state = get_wave_state_v12_1; mqd->mqd_stride = kfd_mqd_stride; + mqd->restore_mqd = restore_mqd_v12_1; #if defined(CONFIG_DEBUG_FS) mqd->debugfs_show_mqd = debugfs_show_mqd; #endif @@ -714,6 +781,7 @@ struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type, mqd->is_occupied = kfd_is_occupied_sdma; mqd->mqd_size = sizeof(struct v12_sdma_mqd); mqd->mqd_stride = kfd_mqd_stride; + mqd->restore_mqd = restore_mqd_sdma_v12_1; #if defined(CONFIG_DEBUG_FS) mqd->debugfs_show_mqd = debugfs_show_mqd_sdma; #endif diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c index be99f0d53b188..1f7224a9c5cbd 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c @@ -299,6 +299,10 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, 1 << CP_HQD_IB_CONTROL__IB_EXE_DISABLE__SHIFT; /* + * The lowest 6 bits of eop_control store the EOP ring size. If + * their value is X, the ring size is 2^(X + 1) dwords, or + * 2^(X + 3) bytes. + * * HW does not clamp this field correctly. Maximum EOP queue size * is constrained by per-SE EOP done signal count, which is 8-bit. * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit @@ -310,7 +314,7 @@ static void update_mqd(struct mqd_manager *mm, void *mqd, * */ m->cp_hqd_eop_control = q->eop_ring_buffer_size ? - min(0xA, order_base_2(q->eop_ring_buffer_size / 4) - 1) : 0; + min(0xA, order_base_2(q->eop_ring_buffer_size / 8)) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c index 60b87a5006983..029572548c149 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c @@ -208,6 +208,9 @@ static void __update_mqd(struct mqd_manager *mm, void *mqd, mtype << CP_HQD_IB_CONTROL__MTYPE__SHIFT; /* + * The lowest 6 bits of eop_control store the EOP ring size. If + * their value is X, the ring size is 2^(X + 1) dwords, or + * 2^(X + 3) bytes. * HW does not clamp this field correctly. Maximum EOP queue size * is constrained by per-SE EOP done signal count, which is 8-bit. * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit @@ -215,7 +218,7 @@ static void __update_mqd(struct mqd_manager *mm, void *mqd, * is safe, giving a maximum field value of 0xA. */ m->cp_hqd_eop_control |= q->eop_ring_buffer_size ? min(0xA, - order_base_2(q->eop_ring_buffer_size / 4) - 1) : 0; + order_base_2(q->eop_ring_buffer_size / 8)) : 0; m->cp_hqd_eop_base_addr_lo = lower_32_bits(q->eop_ring_buffer_address >> 8); m->cp_hqd_eop_base_addr_hi = diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 503176f002040..236089aef75a6 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -1148,13 +1148,13 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data) mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type]; size = mqd_mgr->mqd_stride(mqd_mgr, &q->properties); - } - for (xcc = 0; xcc < num_xccs; xcc++) { - mqd = q->mqd + size * xcc; - r = mqd_mgr->debugfs_show_mqd(m, mqd); - if (r != 0) - break; + for (xcc = 0; xcc < num_xccs; xcc++) { + mqd = q->mqd + size * xcc; + r = mqd_mgr->debugfs_show_mqd(m, mqd); + if (r != 0) + break; + } } } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 941c1a3128249..e5945485ec59a 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -3284,17 +3284,26 @@ static int dm_hw_init(struct amdgpu_ip_block *ip_block) struct amdgpu_device *adev = ip_block->adev; int r; + adev->dm.i2c_devres_group = devres_open_group(adev->dev, NULL, GFP_KERNEL); + if (!adev->dm.i2c_devres_group) + return -ENOMEM; + /* Create DAL display manager */ r = amdgpu_dm_init(adev); if (r) - return r; + goto err_release_i2c; amdgpu_dm_hpd_init(adev); r = dm_oem_i2c_hw_init(adev); if (r) drm_info(adev_to_drm(adev), "Failed to add OEM i2c bus\n"); + devres_close_group(adev->dev, adev->dm.i2c_devres_group); return 0; + +err_release_i2c: + devres_release_group(adev->dev, adev->dm.i2c_devres_group); + return r; } /** @@ -3309,6 +3318,9 @@ static int dm_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; + if (adev->dm.i2c_devres_group) + devres_release_group(adev->dev, adev->dm.i2c_devres_group); + amdgpu_dm_hpd_fini(adev); amdgpu_dm_irq_fini(adev); @@ -3332,7 +3344,7 @@ static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev, if (acrtc && state->stream_status[i].plane_count != 0 && amdgpu_ip_version(adev, DCE_HWIP, 0) == 0) { irq_source = IRQ_TYPE_PFLIP + acrtc->otg_inst; - rc = dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY; + rc = amdgpu_dm_irq_set(adev, irq_source, enable) ? 0 : -EBUSY; if (rc) drm_warn(adev_to_drm(adev), "Failed to %s pflip interrupts\n", enable ? "enable" : "disable"); @@ -3356,7 +3368,7 @@ static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev, /* During gpu-reset we disable and then enable vblank irq, so * don't use amdgpu_irq_get/put() to avoid refcount change. */ - if (!dc_interrupt_set(adev->dm.dc, irq_source, enable)) + if (!amdgpu_dm_irq_set(adev, irq_source, enable)) drm_warn(adev_to_drm(adev), "Failed to %sable vblank interrupt\n", enable ? "en" : "dis"); } else if (acrtc && state->stream_status[i].plane_count != 0) { @@ -3511,6 +3523,9 @@ static int dm_suspend(struct amdgpu_ip_block *ip_block) res = amdgpu_dm_commit_zero_streams(dm->dc); if (res != DC_OK) { drm_err(adev_to_drm(adev), "Failed to commit zero streams: %d\n", res); + dc_state_release(dm->cached_dc_state); + dm->cached_dc_state = NULL; + mutex_unlock(&dm->dc_lock); return -EINVAL; } @@ -3824,6 +3839,9 @@ static int dm_resume(struct amdgpu_ip_block *ip_block) r = dm_dmub_hw_init(adev); if (r) { drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r); + dc_state_release(dm->cached_dc_state); + dm->cached_dc_state = NULL; + mutex_unlock(&dm->dc_lock); return r; } @@ -3907,6 +3925,10 @@ static int dm_resume(struct amdgpu_ip_block *ip_block) /* On resume we need to rewrite the MSTM control bits to enable MST*/ s3_handle_mst(ddev, false); + /* Exit IPS before the detection loop's first AUX/DDC access. */ + scoped_guard(mutex, &dm->dc_lock) + dc_exit_ips_for_hw_access(dm->dc); + /* Do detection*/ drm_connector_list_iter_begin(ddev, &iter); drm_for_each_connector_iter(connector, &iter) { @@ -5265,10 +5287,10 @@ static int get_brightness_range(const struct amdgpu_dm_backlight_caps *caps, return 1; } -/* Rescale from [min..max] to [0..AMDGPU_MAX_BL_LEVEL] */ -static inline u32 scale_input_to_fw(int min, int max, u64 input) +/* Rescale userspace [0..max] to the firmware curve's [0..255]. */ +static inline u32 scale_input_to_fw(int max, u64 input) { - return DIV_ROUND_CLOSEST_ULL(input * AMDGPU_MAX_BL_LEVEL, max - min); + return DIV_ROUND_CLOSEST_ULL(input * AMDGPU_MAX_BL_LEVEL, max); } /* Rescale from [0..AMDGPU_MAX_BL_LEVEL] to [min..max] */ @@ -5281,7 +5303,7 @@ static void convert_custom_brightness(const struct amdgpu_dm_backlight_caps *cap unsigned int min, unsigned int max, uint32_t *user_brightness) { - u32 brightness = scale_input_to_fw(min, max, *user_brightness); + u32 brightness = scale_input_to_fw(max, *user_brightness); u8 lower_signal, upper_signal, upper_lum, lower_lum, lum; int left, right; @@ -6873,10 +6895,14 @@ get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing, break; case DRM_MODE_COLORIMETRY_BT2020_RGB: case DRM_MODE_COLORIMETRY_BT2020_YCC: - if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) - color_space = COLOR_SPACE_2020_RGB_FULLRANGE; - else + if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) { + if (connector_state->hdmi.broadcast_rgb == DRM_HDMI_BROADCAST_RGB_LIMITED) + color_space = COLOR_SPACE_2020_RGB_LIMITEDRANGE; + else + color_space = COLOR_SPACE_2020_RGB_FULLRANGE; + } else { color_space = COLOR_SPACE_2020_YCBCR_LIMITED; + } break; case DRM_MODE_COLORIMETRY_DEFAULT: // ITU601 default: @@ -7846,6 +7872,39 @@ amdgpu_dm_connector_poll(struct amdgpu_dm_connector *aconnector, bool force) return status; } +/* + * Apple Studio Display exposes two SST DP links for a 2x1 tiled panel. + * The primary tile advertises the full 5120x2880 mode (with DSC on the + * bandwidth-sufficient link) while the secondary carries a per-tile + * 2560x2880 timing on a insufficient bandwidth link. Hide the secondary + * connector from userspace so compositors configure a single 5K stream + * on the primary link only. + */ +static bool amdgpu_dm_hide_secondary_tile_from_userspace(struct drm_connector *connector) +{ + struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); + + if (!aconnector->dc_sink) + return false; + + if (!aconnector->dc_sink->edid_caps.panel_patch.disable_second_tile) + return false; + + drm_edid_connector_update(connector, aconnector->drm_edid); + + if (!connector->has_tile) + return false; + + if (!connector->tile_h_loc && !connector->tile_v_loc) + return false; + + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] hiding secondary Apple Studio Display tile from userspace\n", + connector->base.id, connector->name); + + return true; +} + /** * amdgpu_dm_connector_detect() - Detect whether a DRM connector is connected to a display * @@ -7889,6 +7948,9 @@ amdgpu_dm_connector_detect(struct drm_connector *connector, bool force) (!aconnector->dc_sink || aconnector->dc_sink->edid_caps.analog)) return amdgpu_dm_connector_poll(aconnector, force); + if (amdgpu_dm_hide_secondary_tile_from_userspace(connector)) + return connector_status_disconnected; + return (aconnector->dc_sink ? connector_status_connected : connector_status_disconnected); } @@ -9080,6 +9142,47 @@ static void amdgpu_dm_connector_add_common_modes(struct drm_encoder *encoder, } } +/* + * The Apple Studio Display primary tile advertises both the full 5120x2880 + * mode and the per-tile 2560x2880 timing. As the secondary tile is hidden from + * userspace (see amdgpu_dm_hide_secondary_tile_from_userspace()), drop the + * per-tile timing from the primary connector so compositors only pick the full + * 5K mode. + */ +static void amdgpu_dm_prune_primary_tile_modes(struct drm_connector *connector) +{ + struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); + struct drm_display_mode *mode, *t; + + if (!aconnector->dc_sink) + return; + + if (!aconnector->dc_sink->edid_caps.panel_patch.disable_second_tile) + return; + + if (!connector->has_tile) + return; + + /* Only prune the per-tile timing from the primary tile. */ + if (connector->tile_h_loc || connector->tile_v_loc) + return; + + list_for_each_entry_safe(mode, t, &connector->probed_modes, head) { + if (mode->hdisplay != connector->tile_h_size || + mode->vdisplay != connector->tile_v_size) + continue; + + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] pruning per-tile %dx%d timing from primary Apple Studio Display tile\n", + connector->base.id, connector->name, + mode->hdisplay, mode->vdisplay); + + list_del(&mode->head); + drm_mode_destroy(connector->dev, mode); + aconnector->num_modes--; + } +} + static void amdgpu_set_panel_orientation(struct drm_connector *connector) { struct drm_encoder *encoder; @@ -9122,6 +9225,8 @@ static void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *connector, amdgpu_dm_connector->num_modes = drm_edid_connector_add_modes(connector); + amdgpu_dm_prune_primary_tile_modes(connector); + /* sorting the probed modes before calling function * amdgpu_dm_get_native_mode() since EDID can have * more than one preferred mode. The modes that are @@ -12243,8 +12348,10 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm, skip_modeset: /* Release extra reference */ - if (new_stream) + if (new_stream) { dc_stream_release(new_stream); + new_stream = NULL; + } new_stream = NULL; /* diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index dd199e0b79226..d2602c0310b20 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -550,6 +550,18 @@ struct amdgpu_display_manager { struct common_irq_params vupdate_params[DC_IRQ_SOURCE_VUPDATE6 - DC_IRQ_SOURCE_VUPDATE1 + 1]; + /** + * @irq_reg_lock: + * + * Serializes the read-modify-writes of the HW interrupt control + * registers. Several interrupt sources share one register - e.g. the + * enable and clear bits of both VSTARTUP (vblank) and VUPDATE_NO_LOCK + * live in OTG_GLOBAL_SYNC_STATUS. Therefore, enabling one source must + * not race with acking another. Held only across amdgpu_dm_irq_set() + * and amdgpu_dm_irq_ack(). + */ + spinlock_t irq_reg_lock; + /** * @dmub_trace_params: * @@ -688,6 +700,13 @@ struct amdgpu_display_manager { */ void *bb_from_dmub; + /** + * @i2c_devres_group: + * + * Devres group for DM i2c adapter lifetime management. + */ + void *i2c_devres_group; + /** * @oem_i2c: * diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c index 86086d10c5435..46a74f8954303 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c @@ -470,6 +470,12 @@ bool __is_lut_linear(const struct drm_color_lut *lut, uint32_t size) uint32_t expected; int delta; + /* A LUT with fewer than two entries can't be interpolated and would + * divide by zero below (size - 1); it can't be treated as linear. + */ + if (size < 2) + return false; + for (i = 0; i < size; i++) { /* All color values should equal */ if ((lut[i].red != lut[i].green) || (lut[i].green != lut[i].blue)) @@ -1467,6 +1473,13 @@ __set_dm_plane_degamma(struct drm_plane_state *plane_state, degamma_lut = __extract_blob_lut(dm_plane_state->degamma_lut, °amma_size); + if (degamma_lut && degamma_size != MAX_COLOR_LUT_ENTRIES) { + drm_dbg(plane_state->state->dev, + "Invalid Plane Degamma LUT size. Should be %u but got %u.\n", + MAX_COLOR_LUT_ENTRIES, degamma_size); + return -EINVAL; + } + has_degamma_lut = degamma_lut && !__is_lut_linear(degamma_lut, degamma_size); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 56bf907f1f6cc..7206439ea5d52 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -31,6 +31,7 @@ #include "amdgpu_dm_psr.h" #include "amdgpu_dm_replay.h" #include "amdgpu_dm_crtc.h" +#include "amdgpu_dm_irq.h" #include "amdgpu_dm_plane.h" #include "amdgpu_dm_trace.h" #include "amdgpu_dm_debugfs.h" @@ -87,7 +88,7 @@ int amdgpu_dm_crtc_set_vupdate_irq(struct drm_crtc *crtc, bool enable) irq_source = IRQ_TYPE_VUPDATE + acrtc->otg_inst; - rc = dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY; + rc = amdgpu_dm_irq_set(adev, irq_source, enable) ? 0 : -EBUSY; DRM_DEBUG_VBL("crtc %d - vupdate irq %sabling: r=%d\n", acrtc->crtc_id, enable ? "en" : "dis", rc); @@ -464,13 +465,13 @@ static void amdgpu_dm_crtc_reset_state(struct drm_crtc *crtc) { struct dm_crtc_state *state; - if (crtc->state) - amdgpu_dm_crtc_destroy_state(crtc, crtc->state); - state = kzalloc_obj(*state); - if (WARN_ON(!state)) + if (!state) return; + if (crtc->state) + amdgpu_dm_crtc_destroy_state(crtc, crtc->state); + __drm_atomic_helper_crtc_reset(crtc, &state->base); } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c index 7db38ad3f8487..8b0fe5c3a3e8c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c @@ -196,7 +196,6 @@ static ssize_t dp_link_settings_read(struct file *f, char __user *buf, char *rd_buf_ptr = NULL; const uint32_t rd_buf_size = 100; uint32_t result = 0; - uint8_t str_len = 0; int r; if (*pos & 3 || size & 3) @@ -208,29 +207,26 @@ static ssize_t dp_link_settings_read(struct file *f, char __user *buf, rd_buf_ptr = rd_buf; - str_len = strlen("Current: %d 0x%x %d "); - snprintf(rd_buf_ptr, str_len, "Current: %d 0x%x %d ", + rd_buf_ptr += scnprintf(rd_buf_ptr, rd_buf_size - (rd_buf_ptr - rd_buf), + "Current: %d 0x%x %d ", link->cur_link_settings.lane_count, link->cur_link_settings.link_rate, link->cur_link_settings.link_spread); - rd_buf_ptr += str_len; - str_len = strlen("Verified: %d 0x%x %d "); - snprintf(rd_buf_ptr, str_len, "Verified: %d 0x%x %d ", + rd_buf_ptr += scnprintf(rd_buf_ptr, rd_buf_size - (rd_buf_ptr - rd_buf), + "Verified: %d 0x%x %d ", link->verified_link_cap.lane_count, link->verified_link_cap.link_rate, link->verified_link_cap.link_spread); - rd_buf_ptr += str_len; - str_len = strlen("Reported: %d 0x%x %d "); - snprintf(rd_buf_ptr, str_len, "Reported: %d 0x%x %d ", + rd_buf_ptr += scnprintf(rd_buf_ptr, rd_buf_size - (rd_buf_ptr - rd_buf), + "Reported: %d 0x%x %d ", link->reported_link_cap.lane_count, link->reported_link_cap.link_rate, link->reported_link_cap.link_spread); - rd_buf_ptr += str_len; - str_len = strlen("Preferred: %d 0x%x %d "); - snprintf(rd_buf_ptr, str_len, "Preferred: %d 0x%x %d\n", + rd_buf_ptr += scnprintf(rd_buf_ptr, rd_buf_size - (rd_buf_ptr - rd_buf), + "Preferred: %d 0x%x %d\n", link->preferred_link_setting.lane_count, link->preferred_link_setting.link_rate, link->preferred_link_setting.link_spread); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index 4c164ae4a4f9c..acb38051c96e4 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -294,7 +294,7 @@ void hdcp_reset_display(struct hdcp_workqueue *hdcp_work, unsigned int link_inde cancel_delayed_work(&hdcp_w->property_validate_dwork); - for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_INDEX; conn_index++) { + for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_COUNT; conn_index++) { hdcp_w->encryption_status[conn_index] = MOD_HDCP_ENCRYPTION_STATUS_HDCP_OFF; if (hdcp_w->aconnector[conn_index]) { @@ -341,7 +341,7 @@ static void event_property_update(struct work_struct *work) struct drm_connector *connector; struct drm_connector_state *conn_state; - for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_INDEX; conn_index++) { + for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_COUNT; conn_index++) { aconnector = hdcp_work->aconnector[conn_index]; if (!aconnector) @@ -410,7 +410,7 @@ static void event_property_validate(struct work_struct *work) guard(mutex)(&hdcp_work->mutex); - for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_INDEX; + for (conn_index = 0; conn_index < AMDGPU_DM_MAX_DISPLAY_COUNT; conn_index++) { aconnector = hdcp_work->aconnector[conn_index]; @@ -535,15 +535,27 @@ static bool enable_assr(void *handle, struct dc_link *link) static void update_config(void *handle, struct cp_psp_stream_config *config) { struct hdcp_workqueue *hdcp_work = handle; - struct amdgpu_dm_connector *aconnector = config->dm_stream_ctx; - int link_index = aconnector->dc_link->link_index; - unsigned int conn_index = aconnector->base.index; - struct mod_hdcp_display *display = &hdcp_work[link_index].display; - struct mod_hdcp_link *link = &hdcp_work[link_index].link; - struct hdcp_workqueue *hdcp_w = &hdcp_work[link_index]; + struct amdgpu_dm_connector *aconnector; + const struct dc *dc; + int link_index; + unsigned int conn_index; + struct mod_hdcp_display *display; + struct mod_hdcp_link *link; + struct hdcp_workqueue *hdcp_w; struct dc_sink *sink = NULL; bool link_is_hdcp14 = false; - const struct dc *dc = aconnector->dc_link->dc; + + aconnector = config->dm_stream_ctx; + if (!aconnector || !aconnector->dc_link) + return; + + link_index = aconnector->dc_link->link_index; + display = &hdcp_work[link_index].display; + link = &hdcp_work[link_index].link; + hdcp_w = &hdcp_work[link_index]; + + conn_index = aconnector->base.index; + dc = aconnector->dc_link->dc; if (config->dpms_off) { hdcp_remove_display(hdcp_work, link_index, aconnector); @@ -581,7 +593,7 @@ static void update_config(void *handle, struct cp_psp_stream_config *config) link->dp.mst_enabled = config->mst_enabled; link->dp.dp2_enabled = config->dp2_enabled; link->dp.usb4_enabled = config->usb4_enabled; - if (aconnector->dc_sink->sink_signal == SIGNAL_TYPE_HDMI_FRL) + if (sink && sink->sink_signal == SIGNAL_TYPE_HDMI_FRL) link->hdmi.frl_enabled = config->frl_enabled; display->adjust.disable = MOD_HDCP_DISPLAY_DISABLE_AUTHENTICATION; link->adjust.auth_delay = 2; @@ -805,10 +817,10 @@ struct hdcp_workqueue *hdcp_create_workqueue(struct amdgpu_device *adev, memset(hdcp_work[i].aconnector, 0, sizeof(struct amdgpu_dm_connector *) * - AMDGPU_DM_MAX_DISPLAY_INDEX); + AMDGPU_DM_MAX_DISPLAY_COUNT); memset(hdcp_work[i].encryption_status, 0, sizeof(enum mod_hdcp_encryption_status) * - AMDGPU_DM_MAX_DISPLAY_INDEX); + AMDGPU_DM_MAX_DISPLAY_COUNT); } cp_psp->funcs.update_stream_config = update_config; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h index 90b18c450ca61..1c6436eebb063 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h @@ -36,7 +36,7 @@ * Minimal declarations needed by this header. * Full amdgpu/DM definitions come from amdgpu_dm.h included by each .c file. */ -#define AMDGPU_DM_MAX_DISPLAY_INDEX 31 +#define AMDGPU_DM_MAX_DISPLAY_COUNT 32 struct amdgpu_dm_connector; struct mod_hdcp; @@ -51,7 +51,7 @@ struct hdcp_workqueue { struct delayed_work callback_dwork; struct delayed_work watchdog_timer_dwork; struct delayed_work property_validate_dwork; - struct amdgpu_dm_connector *aconnector[AMDGPU_DM_MAX_DISPLAY_INDEX]; + struct amdgpu_dm_connector *aconnector[AMDGPU_DM_MAX_DISPLAY_COUNT]; struct mutex mutex; struct mod_hdcp hdcp; @@ -59,7 +59,7 @@ struct hdcp_workqueue { struct mod_hdcp_display display; struct mod_hdcp_link link; - enum mod_hdcp_encryption_status encryption_status[AMDGPU_DM_MAX_DISPLAY_INDEX]; + enum mod_hdcp_encryption_status encryption_status[AMDGPU_DM_MAX_DISPLAY_COUNT]; /* when display is unplugged from mst hub, connctor will be * destroyed within dm_dp_mst_connector_destroy. connector * hdcp perperties, like type, undesired, desired, enabled, @@ -69,9 +69,9 @@ struct hdcp_workqueue { * will be retrieved from hdcp_work within dm_dp_mst_get_modes */ /* un-desired, desired, enabled */ - unsigned int content_protection[AMDGPU_DM_MAX_DISPLAY_INDEX]; + unsigned int content_protection[AMDGPU_DM_MAX_DISPLAY_COUNT]; /* hdcp1.x, hdcp2.x */ - unsigned int hdcp_content_type[AMDGPU_DM_MAX_DISPLAY_INDEX]; + unsigned int hdcp_content_type[AMDGPU_DM_MAX_DISPLAY_COUNT]; uint8_t max_link; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index 6be7f6edd0b29..ee7140d8dda8b 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -134,6 +134,17 @@ static void apply_edid_quirks(struct dc_link *link, struct edid *edid, drm_dbg_driver(dev, "Skip PHY SSC reduction on panel id %X\n", panel_id); link->wa_flags.skip_phy_ssc_reduction = true; break; + /* + * Workaround for Apple Studio Display which exposes a 2x1 tiled panel + * over two SST DP links. Hide the secondary tile from userspace so + * compositors drive a single 5K stream on the primary link only. + */ + case drm_edid_encode_panel_id('A', 'P', 'P', 0xAE3A): + case drm_edid_encode_panel_id('A', 'P', 'P', 0xAE42): + case drm_edid_encode_panel_id('A', 'P', 'P', 0xAE46): + drm_dbg_driver(dev, "Hiding secondary tile on panel id %X\n", panel_id); + edid_caps->panel_patch.disable_second_tile = true; + break; default: return; } @@ -186,6 +197,7 @@ enum dc_edid_status dm_helpers_parse_edid_caps( edid_caps->edid_hdmi = connector->display_info.is_hdmi; if (edid_caps->edid_hdmi) { + edid_caps->qs_bit = connector->display_info.rgb_quant_range_selectable; populate_hdmi_info_from_connector(link->dc->config.enable_frl, &connector->display_info.hdmi, edid_caps); drm_dbg_driver(connector->dev, "%s: HDMI_FRL [%s] max_frl_rate %d\n", __func__, connector->name, edid_caps->max_frl_rate); if (edid_caps->frl_dsc_support) @@ -1366,12 +1378,13 @@ void dm_helpers_free_gpu_mem( bool dm_helpers_dmub_outbox_interrupt_control(struct dc_context *ctx, bool enable) { + struct amdgpu_device *adev = ctx->driver_context; enum dc_irq_source irq_source; bool ret; irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX; - ret = dc_interrupt_set(ctx->dc, irq_source, enable); + ret = amdgpu_dm_irq_set(adev, irq_source, enable); DRM_DEBUG_DRIVER("Dmub trace irq %sabling: r=%d\n", enable ? "en" : "dis", ret); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c index e49803a90edad..d59434e01dfdb 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c @@ -424,6 +424,7 @@ int amdgpu_dm_irq_init(struct amdgpu_device *adev) DRM_DEBUG_KMS("DM_IRQ\n"); spin_lock_init(&adev->dm.irq_handler_list_table_lock); + spin_lock_init(&adev->dm.irq_reg_lock); for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) { /* low context handler list init */ @@ -496,7 +497,7 @@ void amdgpu_dm_irq_suspend(struct amdgpu_device *adev) hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; hnd_list_h = &adev->dm.irq_handler_list_high_tab[src]; if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h)) - dc_interrupt_set(adev->dm.dc, src, false); + amdgpu_dm_irq_set(adev, src, false); DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); @@ -533,7 +534,7 @@ void amdgpu_dm_irq_resume_early(struct amdgpu_device *adev) hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; hnd_list_h = &adev->dm.irq_handler_list_high_tab[src]; if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h)) - dc_interrupt_set(adev->dm.dc, src, true); + amdgpu_dm_irq_set(adev, src, true); } DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); @@ -558,7 +559,7 @@ void amdgpu_dm_irq_resume_late(struct amdgpu_device *adev) hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; hnd_list_h = &adev->dm.irq_handler_list_high_tab[src]; if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h)) - dc_interrupt_set(adev->dm.dc, src, true); + amdgpu_dm_irq_set(adev, src, true); } DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); @@ -645,6 +646,21 @@ static void amdgpu_dm_irq_immediate_work(struct amdgpu_device *adev, DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); } +bool amdgpu_dm_irq_set(struct amdgpu_device *adev, enum dc_irq_source src, + bool enable) +{ + guard(spinlock_irqsave)(&adev->dm.irq_reg_lock); + + return dc_interrupt_set(adev->dm.dc, src, enable); +} + +void amdgpu_dm_irq_ack(struct amdgpu_device *adev, enum dc_irq_source src) +{ + guard(spinlock_irqsave)(&adev->dm.irq_reg_lock); + + dc_interrupt_ack(adev->dm.dc, src); +} + /** * amdgpu_dm_irq_handler - Generic DM IRQ handler * @adev: amdgpu base driver device containing the DM device @@ -665,7 +681,7 @@ static int amdgpu_dm_irq_handler(struct amdgpu_device *adev, entry->src_id, entry->src_data[0]); - dc_interrupt_ack(adev->dm.dc, src); + amdgpu_dm_irq_ack(adev, src); /* Call high irq work immediately */ amdgpu_dm_irq_immediate_work(adev, src); @@ -703,7 +719,7 @@ static int amdgpu_dm_set_hpd_irq_state(struct amdgpu_device *adev, enum dc_irq_source src = amdgpu_dm_hpd_to_dal_irq_source(type); bool st = (state == AMDGPU_IRQ_STATE_ENABLE); - dc_interrupt_set(adev->dm.dc, src, st); + amdgpu_dm_irq_set(adev, src, st); return 0; } @@ -737,7 +753,7 @@ static inline int dm_irq_state(struct amdgpu_device *adev, if (dc && dc->caps.ips_support && dc->idle_optimizations_allowed) dc_allow_idle_optimizations(dc, false); - dc_interrupt_set(adev->dm.dc, irq_source, st); + amdgpu_dm_irq_set(adev, irq_source, st); return 0; } @@ -791,7 +807,7 @@ static int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev, enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX; bool st = (state == AMDGPU_IRQ_STATE_ENABLE); - dc_interrupt_set(adev->dm.dc, irq_source, st); + amdgpu_dm_irq_set(adev, irq_source, st); return 0; } @@ -817,7 +833,7 @@ static int amdgpu_dm_set_dmub_trace_irq_state(struct amdgpu_device *adev, enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX0; bool st = (state == AMDGPU_IRQ_STATE_ENABLE); - dc_interrupt_set(adev->dm.dc, irq_source, st); + amdgpu_dm_irq_set(adev, irq_source, st); return 0; } @@ -881,9 +897,7 @@ void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev) } void amdgpu_dm_outbox_init(struct amdgpu_device *adev) { - dc_interrupt_set(adev->dm.dc, - DC_IRQ_SOURCE_DMCUB_OUTBOX, - true); + amdgpu_dm_irq_set(adev, DC_IRQ_SOURCE_DMCUB_OUTBOX, true); } /** @@ -905,7 +919,7 @@ void amdgpu_dm_hpd_init(struct amdgpu_device *adev) /* First, clear all hpd and hpdrx interrupts */ for (i = DC_IRQ_SOURCE_HPD1; i <= DC_IRQ_SOURCE_HPD6RX; i++) { - if (!dc_interrupt_set(adev->dm.dc, i, false)) + if (!amdgpu_dm_irq_set(adev, i, false)) drm_err(dev, "Failed to clear hpd(rx) source=%d on init\n", i); } @@ -934,7 +948,7 @@ void amdgpu_dm_hpd_init(struct amdgpu_device *adev) * of dm. Note that only hpd interrupt types are registered with * base driver; hpd_rx types aren't. IOW, amdgpu_irq_get/put on * hpd_rx isn't available. DM currently controls hpd_rx - * explicitly with dc_interrupt_set() + * explicitly with amdgpu_dm_irq_set() */ if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) { irq_type = dc_link->irq_source_hpd - DC_IRQ_SOURCE_HPD1; @@ -943,23 +957,21 @@ void amdgpu_dm_hpd_init(struct amdgpu_device *adev) * and what bios reports as the # of connectors with hpd * sources. Since the # of hpd source types registered * with base driver == mode_info.num_hpd, we have to - * fallback to dc_interrupt_set for the remaining types. + * fallback to amdgpu_dm_irq_set for the remaining types. */ if (irq_type < adev->mode_info.num_hpd) { if (amdgpu_irq_get(adev, &adev->hpd_irq, irq_type)) drm_err(dev, "DM_IRQ: Failed get HPD for source=%d)!\n", dc_link->irq_source_hpd); } else { - dc_interrupt_set(adev->dm.dc, - dc_link->irq_source_hpd, - true); + amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd, + true); } } if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) { - dc_interrupt_set(adev->dm.dc, - dc_link->irq_source_hpd_rx, - true); + amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd_rx, + true); } } drm_connector_list_iter_end(&iter); @@ -1003,16 +1015,14 @@ void amdgpu_dm_hpd_fini(struct amdgpu_device *adev) drm_err(dev, "DM_IRQ: Failed put HPD for source=%d!\n", dc_link->irq_source_hpd); } else { - dc_interrupt_set(adev->dm.dc, - dc_link->irq_source_hpd, - false); + amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd, + false); } } if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) { - dc_interrupt_set(adev->dm.dc, - dc_link->irq_source_hpd_rx, - false); + amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd_rx, + false); } } drm_connector_list_iter_end(&iter); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h index 4f6b58f4f90d7..f0a4577848f35 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h @@ -81,6 +81,34 @@ void amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device *adev, enum dc_irq_source irq_source, void *ih_index); +/** + * amdgpu_dm_irq_set - enable or disable a DC interrupt source. + * + * @adev: AMD DRM device + * @src: DC interrupt source to toggle + * @enable: true to enable the source, false to disable it + * + * DM-wide replacement for dc_interrupt_set(). As locking is DM's + * responsibility, this is a thin wrapper serializes the underlying + * read-modify-write against the other interrupt sources sharing HW control + * registers with @src, so DM must never call dc_interrupt_set() directly. + * + * Returns: true if the source was toggled. + */ +bool amdgpu_dm_irq_set(struct amdgpu_device *adev, enum dc_irq_source src, + bool enable); + +/** + * amdgpu_dm_irq_ack - acknowledge a DC interrupt source. + * + * @adev: AMD DRM device + * @src: DC interrupt source to acknowledge + * + * DM-wide replacement for dc_interrupt_ack(), serialized the same way as + * amdgpu_dm_irq_set(). + */ +void amdgpu_dm_irq_ack(struct amdgpu_device *adev, enum dc_irq_source src); + void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev); void amdgpu_dm_outbox_init(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index c7f8e08feaf4b..cfd76c54f652b 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -1488,17 +1488,15 @@ static const struct drm_plane_helper_funcs dm_primary_plane_helper_funcs = { static void amdgpu_dm_plane_drm_plane_reset(struct drm_plane *plane) { - struct dm_plane_state *amdgpu_state = NULL; - - if (plane->state) - plane->funcs->atomic_destroy_state(plane, plane->state); + struct dm_plane_state *amdgpu_state; amdgpu_state = kzalloc_obj(*amdgpu_state); - WARN_ON(amdgpu_state == NULL); - if (!amdgpu_state) return; + if (plane->state) + plane->funcs->atomic_destroy_state(plane, plane->state); + __drm_atomic_helper_plane_reset(plane, &amdgpu_state->base); amdgpu_state->degamma_tf = AMDGPU_TRANSFER_FUNCTION_DEFAULT; amdgpu_state->hdr_mult = AMDGPU_HDR_MULT_DEFAULT; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index e25b94b65daca..ebfed18e780c5 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -3122,6 +3122,7 @@ static struct surface_update_descriptor check_update_surfaces_for_stream( } if ((stream_update->hdr_static_metadata && !stream_update->stream->use_dynamic_meta) || + stream_update->output_color_space || stream_update->vrr_infopacket || stream_update->vsc_infopacket || stream_update->vsp_infopacket || @@ -4090,6 +4091,7 @@ static void commit_planes_do_stream_update_sequence(struct dc *dc, hwss_add_setup_periodic_interrupt(&seq_state, dc, pipe_ctx); if ((stream_update->hdr_static_metadata && !stream->use_dynamic_meta) || + stream_update->output_color_space || stream_update->vrr_infopacket || stream_update->vsc_infopacket || stream_update->vsp_infopacket || @@ -4272,6 +4274,7 @@ static void commit_planes_do_stream_update(struct dc *dc, dc->hwss.setup_periodic_interrupt(dc, pipe_ctx); if ((stream_update->hdr_static_metadata && !stream->use_dynamic_meta) || + stream_update->output_color_space || stream_update->vrr_infopacket || stream_update->vsc_infopacket || stream_update->vsp_infopacket || diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h index 4ed1efa172709..6482e275e63bf 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_types.h @@ -187,6 +187,7 @@ struct dc_panel_patch { unsigned int skip_audio_sab_check; unsigned int mst_start_top_delay; unsigned int remove_sink_ext_caps; + bool disable_second_tile; unsigned int disable_colorimetry; uint8_t blankstream_before_otg_off; bool oled_optimize_display_on; diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c index 483cd9ab7eb76..42066b8a03623 100644 --- a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c +++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c @@ -572,7 +572,7 @@ static void dccg35_set_hdmistreamclk_src_new( case 0: REG_UPDATE_2(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_EN, (src == HDMI_STREAM_REFCLK) ? 0 : 1, - DPSTREAMCLK0_SRC_SEL, + HDMISTREAMCLK0_SRC_SEL, (src == HDMI_STREAM_REFCLK) ? 0 : src); break; default: diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile index 10d4ace04d4f1..60bca8d14771a 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile @@ -29,14 +29,18 @@ dml_ccflags := $(CC_FLAGS_FPU) dml_rcflags := $(CC_FLAGS_NO_FPU) ifneq ($(CONFIG_FRAME_WARN),0) - ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)),y) + ifneq ($(filter y,$(CONFIG_KASAN) $(CONFIG_KCSAN) $(CONFIG_UBSAN)),) ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_COMPILE_TEST),yy) frame_warn_limit := 4096 else frame_warn_limit := 3072 endif else - frame_warn_limit := 2048 + ifeq ($(CONFIG_CC_IS_CLANG),y) + frame_warn_limit := 3072 + else + frame_warn_limit := 2048 + endif endif ifeq ($(call test-lt, $(CONFIG_FRAME_WARN), $(frame_warn_limit)),y) diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile b/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile index 44e00c2b7ac79..5f6bde11c3ca2 100644 --- a/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile @@ -28,7 +28,7 @@ dml2_ccflags := $(CC_FLAGS_FPU) dml2_rcflags := $(CC_FLAGS_NO_FPU) ifneq ($(CONFIG_FRAME_WARN),0) - ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)),y) + ifneq ($(filter y,$(CONFIG_KASAN) $(CONFIG_KCSAN) $(CONFIG_UBSAN)),) ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_COMPILE_TEST),yy) frame_warn_limit := 4096 else diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h index 11e295253f72e..51725ca16a28f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h +++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h @@ -1748,8 +1748,8 @@ struct dml2_core_calcs_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport_param bool UnboundedRequestEnabled; unsigned int CompressedBufferSizeInkByte; bool max_outstanding_when_urgent_expected; - const unsigned int max_outstanding_requests; - const unsigned int max_request_size_bytes; + unsigned int max_outstanding_requests; + unsigned int max_request_size_bytes; const unsigned int *meta_row_height_l; const unsigned int *meta_row_height_c; const enum dml2_pstate_method *uclk_pstate_switch_modes; diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c index 166f10b8862f1..c82886323a518 100644 --- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c +++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c @@ -301,6 +301,7 @@ void dml2_init_socbb_params(struct dml2_context *dml2, const struct dc *in_dc, s out->smn_latency_us = 2; out->dispclk_dppclk_vco_speed_mhz = 3600; out->pct_ideal_dram_bw_after_urgent_pixel_only = 65.0; + out->gpuvm_min_page_size_kbytes = 4; break; diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c index 7c97a774141ff..d8eb5996b5774 100644 --- a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c @@ -772,8 +772,7 @@ bool hubp1_is_flip_pending(struct hubp *hubp) if (flip_pending) return true; - if (hubp && - earliest_inuse_address.grph.addr.quad_part != hubp->request_address.grph.addr.quad_part) + if (earliest_inuse_address.grph.addr.quad_part != hubp->request_address.grph.addr.quad_part) return true; return false; diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c index 2d3587a31bc8b..2ecd3a4068c03 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c @@ -1058,10 +1058,12 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable) */ unsigned int denom = refresh_hz * 6528; unsigned int stutter_period = dc->current_state->perf_params.stutter_period_us; + uint64_t num = (1000000LL + 2 * stutter_period * refresh_hz) * + (100LL + dc->debug.mall_additional_timer_percent); + uint64_t tmr_ticks; - tmr_delay = (uint32_t)(div_u64(((1000000LL + 2 * stutter_period * refresh_hz) * - (100LL + dc->debug.mall_additional_timer_percent) + denom - 1), - denom) - 64LL); + tmr_ticks = div_u64(num + denom - 1, denom); + tmr_delay = tmr_ticks > 64 ? (uint32_t)(tmr_ticks - 64) : 0; /* In some cases the stutter period is really big (tiny modes) in these * cases MALL cant be enabled, So skip these cases to avoid a ASSERT() @@ -1083,9 +1085,8 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable) } denom *= 2; - tmr_delay = (uint32_t)(div_u64(((1000000LL + 2 * stutter_period * refresh_hz) * - (100LL + dc->debug.mall_additional_timer_percent) + denom - 1), - denom) - 64LL); + tmr_ticks = div_u64(num + denom - 1, denom); + tmr_delay = tmr_ticks > 64 ? (uint32_t)(tmr_ticks - 64) : 0; } /* Copy HW cursor */ diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c index ca59977487994..f49dba6b4e426 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c @@ -2296,6 +2296,8 @@ static enum dc_status enable_link( static bool allocate_usb4_bandwidth_for_stream(struct dc_stream_state *stream, int stream_bw) { + ASSERT(stream->sink); + struct dc_link *link = stream->sink->link; int req_bw = stream_bw; @@ -2342,6 +2344,8 @@ static bool allocate_usb4_bandwidth_for_stream(struct dc_stream_state *stream, i static bool allocate_usb4_bandwidth(struct dc_stream_state *stream) { + ASSERT(stream->sink); + bool ret; int bw = dc_bandwidth_in_kbps_from_timing(&stream->timing, @@ -2361,38 +2365,43 @@ static bool deallocate_usb4_bandwidth(struct dc_stream_state *stream) return ret; } +static struct vpg *get_vpg(struct pipe_ctx *pipe_ctx) +{ + if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal)) + return pipe_ctx->stream_res.hpo_frl_stream_enc->vpg; + else if (dp_is_128b_132b_signal(pipe_ctx)) + return pipe_ctx->stream_res.hpo_dp_stream_enc->vpg; + else + return pipe_ctx->stream_res.stream_enc->vpg; +} + void link_set_dpms_off(struct pipe_ctx *pipe_ctx) { - struct dc *dc = pipe_ctx->stream->ctx->dc; + DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger); struct dc_stream_state *stream = pipe_ctx->stream; - struct dc_link *link = stream->sink->link; - struct vpg *vpg = pipe_ctx->stream_res.stream_enc->vpg; + struct dc *dc = stream->ctx->dc; + struct dc_link *link = stream->link; + struct vpg *vpg = get_vpg(pipe_ctx); enum dp_panel_mode panel_mode_dp = dp_get_panel_mode(link); - DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger); - ASSERT(is_master_pipe_for_link(link, pipe_ctx)); - if (dp_is_128b_132b_signal(pipe_ctx)) - vpg = pipe_ctx->stream_res.hpo_dp_stream_enc->vpg; - if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal)) - vpg = pipe_ctx->stream_res.hpo_frl_stream_enc->vpg; - if (dc_is_virtual_signal(pipe_ctx->stream->signal)) + if (dc_is_virtual_signal(stream->signal)) return; - if (pipe_ctx->stream->sink) { - if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL && - pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) { + if (stream->sink) { + if (stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL && + stream->sink->sink_signal != SIGNAL_TYPE_NONE) { DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__, - pipe_ctx->stream->sink->edid_caps.display_name, - pipe_ctx->stream->signal, link->link_index, link->sink_count); + stream->sink->edid_caps.display_name, + stream->signal, link->link_index, link->sink_count); } } link_wait_for_unlocked(link); - if (!pipe_ctx->stream->sink->edid_caps.panel_patch.skip_avmute) { - if (dc_is_hdmi_signal(pipe_ctx->stream->signal)) + if (stream->sink && !stream->sink->edid_caps.panel_patch.skip_avmute) { + if (dc_is_hdmi_signal(stream->signal)) set_avmute(pipe_ctx, true); } @@ -2402,15 +2411,15 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx) dc->hwss.blank_stream(pipe_ctx); if (pipe_ctx->link_config.dp_tunnel_settings.should_use_dp_bw_allocation) - deallocate_usb4_bandwidth(pipe_ctx->stream); + deallocate_usb4_bandwidth(stream); - if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) + if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) deallocate_mst_payload(pipe_ctx); - else if (dc_is_dp_sst_signal(pipe_ctx->stream->signal) && + else if (dc_is_dp_sst_signal(stream->signal) && dp_is_128b_132b_signal(pipe_ctx)) update_sst_payload(pipe_ctx, false); - if (dc_is_hdmi_signal(pipe_ctx->stream->signal)) { + if (dc_is_hdmi_signal(stream->signal)) { struct ext_hdmi_settings settings = {0}; enum engine_id eng_id = pipe_ctx->stream_res.stream_enc->id; @@ -2433,7 +2442,7 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx) } } - if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT && + if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT && !dp_is_128b_132b_signal(pipe_ctx)) { /* In DP1.x SST mode, our encoder will go to TPS1 @@ -2443,18 +2452,18 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx) * state machine. * In DP2 or MST mode, our encoder will stay video active */ - disable_link(pipe_ctx->stream->link, &pipe_ctx->link_res, pipe_ctx->stream->signal); + disable_link(link, &pipe_ctx->link_res, stream->signal); dc->hwss.disable_stream(pipe_ctx); } else { dc->hwss.disable_stream(pipe_ctx); - disable_link(pipe_ctx->stream->link, &pipe_ctx->link_res, pipe_ctx->stream->signal); + disable_link(link, &pipe_ctx->link_res, stream->signal); } edp_set_panel_assr(link, pipe_ctx, &panel_mode_dp, false); - if (pipe_ctx->stream->timing.flags.DSC) { - if (dc_is_dp_signal(pipe_ctx->stream->signal)) + if (stream->timing.flags.DSC) { + if (dc_is_dp_signal(stream->signal)) link_set_dsc_enable(pipe_ctx, false); - else if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal)) + else if (dc_is_hdmi_frl_signal(stream->signal)) link_set_dsc_on_stream(pipe_ctx, false); } if (dp_is_128b_132b_signal(pipe_ctx)) { @@ -2468,7 +2477,7 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx) /* for psp not exist case */ if (link->connector_signal == SIGNAL_TYPE_EDP && dc->debug.psp_disabled_wa) { /* reset internal save state to default since eDP is off */ - enum dp_panel_mode panel_mode = dp_get_panel_mode(pipe_ctx->stream->link); + enum dp_panel_mode panel_mode = dp_get_panel_mode(link); /* since current psp not loaded, we need to reset it to default */ link->panel_mode = panel_mode; } @@ -2478,62 +2487,57 @@ void link_set_dpms_on( struct dc_state *state, struct pipe_ctx *pipe_ctx) { - struct dc *dc = pipe_ctx->stream->ctx->dc; + DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger); struct dc_stream_state *stream = pipe_ctx->stream; - struct dc_link *link = stream->sink->link; + struct dc *dc = stream->ctx->dc; + struct dc_link *link = stream->link; enum dc_status status; struct link_encoder *link_enc = pipe_ctx->link_res.dio_link_enc; enum otg_out_mux_dest otg_out_dest = OUT_MUX_DIO; - struct vpg *vpg = pipe_ctx->stream_res.stream_enc->vpg; + struct vpg *vpg = get_vpg(pipe_ctx); const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res); bool apply_edp_fast_boot_optimization = - pipe_ctx->stream->apply_edp_fast_boot_optimization; - - DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger); + stream->apply_edp_fast_boot_optimization; ASSERT(is_master_pipe_for_link(link, pipe_ctx)); - if (dp_is_128b_132b_signal(pipe_ctx)) - vpg = pipe_ctx->stream_res.hpo_dp_stream_enc->vpg; - if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal)) - vpg = pipe_ctx->stream_res.hpo_frl_stream_enc->vpg; - if (dc_is_virtual_signal(pipe_ctx->stream->signal)) + if (dc_is_virtual_signal(stream->signal)) return; - if (pipe_ctx->stream->sink) { - if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL && - pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) { + if (stream->sink) { + if (stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL && + stream->sink->sink_signal != SIGNAL_TYPE_NONE) { DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__, - pipe_ctx->stream->sink->edid_caps.display_name, - pipe_ctx->stream->signal, + stream->sink->edid_caps.display_name, + stream->signal, link->link_index, link->sink_count); } } - link_wait_for_unlocked(stream->link); + link_wait_for_unlocked(link); if (!dc->config.unify_link_enc_assignment) link_enc = link_enc_cfg_get_link_enc(link); ASSERT(link_enc); - if (!dc_is_virtual_signal(pipe_ctx->stream->signal) - && !dc_is_hdmi_frl_signal(pipe_ctx->stream->signal) + if (!dc_is_virtual_signal(stream->signal) + && !dc_is_hdmi_frl_signal(stream->signal) && !dp_is_128b_132b_signal(pipe_ctx)) { if (link_enc) link_enc->funcs->setup( link_enc, - pipe_ctx->stream->signal); + stream->signal); } - pipe_ctx->stream->link->link_state_valid = true; + link->link_state_valid = true; - if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal)) - hdmi_frl_decide_link_settings(stream, &stream->link->frl_link_settings, &pipe_ctx->dsc_padding_params); + if (dc_is_hdmi_frl_signal(stream->signal)) + hdmi_frl_decide_link_settings(stream, &link->frl_link_settings, &pipe_ctx->dsc_padding_params); if (pipe_ctx->stream_res.tg->funcs->set_out_mux) { if (dp_is_128b_132b_signal(pipe_ctx)) otg_out_dest = OUT_MUX_HPO_DP; - else if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal)) + else if (dc_is_hdmi_frl_signal(stream->signal)) otg_out_dest = OUT_MUX_HPO_FRL; else otg_out_dest = OUT_MUX_DIO; @@ -2542,7 +2546,7 @@ void link_set_dpms_on( link_hwss->setup_stream_attribute(pipe_ctx); - pipe_ctx->stream->apply_edp_fast_boot_optimization = false; + stream->apply_edp_fast_boot_optimization = false; // Enable VPG before building infoframe if (vpg && vpg->funcs->vpg_poweron) @@ -2551,15 +2555,15 @@ void link_set_dpms_on( resource_build_info_frame(pipe_ctx); dc->hwss.update_info_frame(pipe_ctx); - if (dc_is_dp_signal(pipe_ctx->stream->signal)) + if (dc_is_dp_signal(stream->signal)) dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME); /* Do not touch link on seamless boot optimization. */ - if (pipe_ctx->stream->apply_seamless_boot_optimization) { - pipe_ctx->stream->dpms_off = false; + if (stream->apply_seamless_boot_optimization) { + stream->dpms_off = false; /* Still enable stream features & audio on seamless boot for DP external displays */ - if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT) { + if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT) { enable_stream_features(pipe_ctx); dc->hwss.enable_audio_stream(pipe_ctx); } @@ -2569,11 +2573,11 @@ void link_set_dpms_on( } /* eDP lit up by bios already, no need to enable again. */ - if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP && + if (stream->signal == SIGNAL_TYPE_EDP && apply_edp_fast_boot_optimization && - !pipe_ctx->stream->timing.flags.DSC && + !stream->timing.flags.DSC && !pipe_ctx->next_odm_pipe) { - pipe_ctx->stream->dpms_off = false; + stream->dpms_off = false; update_psp_stream_config(pipe_ctx, false); if (link->is_dds) { @@ -2586,7 +2590,7 @@ void link_set_dpms_on( return; } - if (pipe_ctx->stream->dpms_off) + if (stream->dpms_off) return; /* For Dp tunneling link, a pending HPD means that we have a race condition between processing @@ -2604,9 +2608,9 @@ void link_set_dpms_on( * will be automatically set at a later time when the video is enabled * (DP_VID_STREAM_EN = 1). */ - if (pipe_ctx->stream->timing.flags.DSC) { - if (dc_is_dp_signal(pipe_ctx->stream->signal) || - dc_is_virtual_signal(pipe_ctx->stream->signal)) + if (stream->timing.flags.DSC) { + if (dc_is_dp_signal(stream->signal) || + dc_is_virtual_signal(stream->signal)) link_set_dsc_enable(pipe_ctx, true); } @@ -2617,7 +2621,7 @@ void link_set_dpms_on( if (status != DC_OK) { DC_LOG_WARNING("enabling link %u failed: %d\n", - pipe_ctx->stream->link->link_index, + link->link_index, status); /* Abort stream enable *unless* the failure was due to @@ -2627,19 +2631,18 @@ void link_set_dpms_on( */ if ((status != DC_FAIL_DP_LINK_TRAINING && status != DC_FAIL_HDMI_FRL_LINK_TRAINING) || - pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { - if (false == stream->link->link_status.link_active) - disable_link(stream->link, &pipe_ctx->link_res, - pipe_ctx->stream->signal); + stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { + if (false == link->link_status.link_active) + disable_link(link, &pipe_ctx->link_res, + stream->signal); BREAK_TO_DEBUGGER(); return; } } - if (pipe_ctx->stream->timing.flags.DSC && - dc_is_hdmi_frl_signal(pipe_ctx->stream->signal)) - //TODO: bring HDMI FRL in line with DP - link_set_dsc_on_stream(pipe_ctx, true); + if (stream->timing.flags.DSC && dc_is_hdmi_frl_signal(stream->signal)) + //TODO: bring HDMI FRL in line with DP + link_set_dsc_on_stream(pipe_ctx, true); /* turn off otg test pattern if enable */ if (pipe_ctx->stream_res.tg->funcs->set_test_pattern) @@ -2651,37 +2654,37 @@ void link_set_dpms_on( * as a workaround for the incorrect value being applied * from transmitter control. */ - if (!(dc_is_virtual_signal(pipe_ctx->stream->signal) || - dc_is_hdmi_frl_signal(pipe_ctx->stream->signal) || + if (!(dc_is_virtual_signal(stream->signal) || + dc_is_hdmi_frl_signal(stream->signal) || dp_is_128b_132b_signal(pipe_ctx))) { if (link_enc) link_enc->funcs->setup( link_enc, - pipe_ctx->stream->signal); + stream->signal); } dc->hwss.enable_stream(pipe_ctx); /* Set DPS PPS SDP (AKA "info frames") */ - if (pipe_ctx->stream->timing.flags.DSC) { - if (dc_is_dp_signal(pipe_ctx->stream->signal) || - dc_is_virtual_signal(pipe_ctx->stream->signal)) { + if (stream->timing.flags.DSC) { + if (dc_is_dp_signal(stream->signal) || + dc_is_virtual_signal(stream->signal)) { dp_set_dsc_on_rx(pipe_ctx, true); link_set_dsc_pps_packet(pipe_ctx, true, true); } } - if (dc_is_dp_signal(pipe_ctx->stream->signal)) + if (dc_is_dp_signal(stream->signal)) dp_set_hblank_reduction_on_rx(pipe_ctx); if (pipe_ctx->link_config.dp_tunnel_settings.should_use_dp_bw_allocation) - allocate_usb4_bandwidth(pipe_ctx->stream); + allocate_usb4_bandwidth(stream); - if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) + if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) allocate_mst_payload(pipe_ctx); - else if (dc_is_dp_sst_signal(pipe_ctx->stream->signal) && + else if (dc_is_dp_sst_signal(stream->signal) && dp_is_128b_132b_signal(pipe_ctx)) update_sst_payload(pipe_ctx, true); @@ -2690,23 +2693,22 @@ void link_set_dpms_on( * training and stream unblank resolves the corruption issue. * This is workaround. */ - if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP && + if (stream->signal == SIGNAL_TYPE_EDP && link->is_display_mux_present) msleep(20); dc->hwss.unblank_stream(pipe_ctx, - &pipe_ctx->stream->link->cur_link_settings); + &link->cur_link_settings); if (stream->sink_patches.delay_ignore_msa > 0) msleep(stream->sink_patches.delay_ignore_msa); - if (dc_is_dp_signal(pipe_ctx->stream->signal)) + if (dc_is_dp_signal(stream->signal)) enable_stream_features(pipe_ctx); update_psp_stream_config(pipe_ctx, false); dc->hwss.enable_audio_stream(pipe_ctx); - if (dc_is_hdmi_signal(pipe_ctx->stream->signal)) { + if (dc_is_hdmi_signal(stream->signal)) set_avmute(pipe_ctx, false); - } } diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c index b6262e43ca02a..4c27052b62f73 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c @@ -632,7 +632,7 @@ static bool construct_phy(struct dc_link *link, DC_LOG_DC("BIOS object table - hpd_gpio id: %d", enc_init_data.hpd_gpio->id); DC_LOG_DC("BIOS object table - hpd_gpio en: %d", enc_init_data.hpd_gpio->en); - } else { + } else if (link->ctx->dce_version > DCN_VERSION_4_01) { struct graphics_object_hpd_info hpd_info; if (link->ctx->dc_bios->funcs->get_hpd_info(link->ctx->dc_bios, link->link_id, &hpd_info) == BP_RESULT_OK) { diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c index dd854d9926925..f43fc4b78a8d3 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c @@ -103,6 +103,11 @@ static int get_estimated_bw(struct dc_link *link) { uint8_t bw_estimated_bw = 0; + if (link->dpia_bw_alloc_config.bw_granularity == 0) { + DC_LOG_ERROR("%s: BW granularity is zero!\n", __func__); + return 0; + } + core_link_read_dpcd( link, ESTIMATED_BW, diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c index 54ce768ae6ad9..da679fb7d89c7 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c @@ -39,7 +39,6 @@ #include "link/link_dpms.h" #include "dm_helpers.h" #include "link_dp_dpia_bw.h" -#include "link_dp_panel_replay.h" #define DC_LOGGER \ link->ctx->logger diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c index e5e89294958a8..68a1e78cd4b82 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c @@ -348,7 +348,8 @@ int amdgpu_dpm_switch_power_profile(struct amdgpu_device *adev, const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; int ret = 0; - if (amdgpu_sriov_vf(adev)) + if (amdgpu_sriov_vf(adev) || + amdgpu_in_reset(adev)) return 0; if (pp_funcs && pp_funcs->switch_power_profile) { @@ -367,7 +368,8 @@ int amdgpu_dpm_pause_power_profile(struct amdgpu_device *adev, const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; int ret = 0; - if (amdgpu_sriov_vf(adev)) + if (amdgpu_sriov_vf(adev) || + amdgpu_in_reset(adev)) return 0; if (pp_funcs && pp_funcs->pause_power_profile) { diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c index ce166a7f8e420..1fff7567bca27 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c @@ -268,15 +268,21 @@ static const ATOM_VOLTAGE_OBJECT_V3 *atomctrl_lookup_voltage_type_v3( unsigned int offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO_V3_1, asVoltageObj[0]); uint8_t *start = (uint8_t *)voltage_object_info_table; - while (offset < size) { + while (offset + sizeof(ATOM_VOLTAGE_OBJECT_HEADER_V3) <= size) { const ATOM_VOLTAGE_OBJECT_V3 *voltage_object = (const ATOM_VOLTAGE_OBJECT_V3 *)(start + offset); + u16 obj_size; + + obj_size = le16_to_cpu(voltage_object->asGpioVoltageObj.sHeader.usSize); + if (obj_size < sizeof(voltage_object->asGpioVoltageObj.sHeader) || + offset + obj_size > size) + break; if (voltage_type == voltage_object->asGpioVoltageObj.sHeader.ucVoltageType && voltage_mode == voltage_object->asGpioVoltageObj.sHeader.ucVoltageMode) return voltage_object; - offset += le16_to_cpu(voltage_object->asGpioVoltageObj.sHeader.usSize); + offset += obj_size; } return NULL; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c index 6120f14caab08..69aee8661d1e5 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c @@ -36,16 +36,21 @@ static const union atom_voltage_object_v4 *pp_atomfwctrl_lookup_voltage_type_v4( offsetof(struct atom_voltage_objects_info_v4_1, voltage_object[0]); unsigned long start = (unsigned long)voltage_object_info_table; - while (offset < size) { + while (offset + sizeof(struct atom_voltage_object_header_v4) <= size) { const union atom_voltage_object_v4 *voltage_object = (const union atom_voltage_object_v4 *)(start + offset); + u16 obj_size; + + obj_size = le16_to_cpu(voltage_object->gpio_voltage_obj.header.object_size); + if (obj_size < sizeof(voltage_object->gpio_voltage_obj.header) || + offset + obj_size > size) + break; if (voltage_type == voltage_object->gpio_voltage_obj.header.voltage_type && voltage_mode == voltage_object->gpio_voltage_obj.header.voltage_mode) return voltage_object; - offset += le16_to_cpu(voltage_object->gpio_voltage_obj.header.object_size); - + offset += obj_size; } return NULL; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index f8a5648095d17..1e9fa0a250750 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -2216,12 +2216,24 @@ static int smu7_patch_voltage_dependency_tables_with_lookup_table( if (data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) { for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) { voltage_id = sclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddgfx_lookup_table->count) { + pr_err("amdgpu: sclk[%u] vddgfx index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddgfx_lookup_table->count); + return -EINVAL; + } sclk_table->entries[entry_id].vddgfx = table_info->vddgfx_lookup_table->entries[voltage_id].us_vdd; } } else { for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) { voltage_id = sclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: sclk[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } sclk_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } @@ -2229,12 +2241,24 @@ static int smu7_patch_voltage_dependency_tables_with_lookup_table( for (entry_id = 0; entry_id < mclk_table->count; ++entry_id) { voltage_id = mclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } for (entry_id = 0; entry_id < mm_table->count; ++entry_id) { voltage_id = mm_table->entries[entry_id].vddcInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mm[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mm_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c index c283309efe87f..ae8e44b796a89 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c @@ -685,10 +685,18 @@ static int vega10_patch_voltage_dependency_tables_with_lookup_table( case 3: vdt = table_info->vdd_dep_on_pixclk; break; case 4: vdt = table_info->vdd_dep_on_dispclk; break; case 5: vdt = table_info->vdd_dep_on_phyclk; break; + default: + continue; } for (entry_id = 0; entry_id < vdt->count; entry_id++) { voltage_id = vdt->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: clk_dep[%u][%u] vddc index %u out of bounds (%u)\n", + i, entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } vdt->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } @@ -696,23 +704,48 @@ static int vega10_patch_voltage_dependency_tables_with_lookup_table( for (entry_id = 0; entry_id < mm_table->count; ++entry_id) { voltage_id = mm_table->entries[entry_id].vddcInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mm[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mm_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } for (entry_id = 0; entry_id < mclk_table->count; ++entry_id) { voltage_id = mclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; + voltage_id = mclk_table->entries[entry_id].vddciInd; + if (voltage_id >= table_info->vddci_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddci index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddci_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].vddci = table_info->vddci_lookup_table->entries[voltage_id].us_vdd; + voltage_id = mclk_table->entries[entry_id].mvddInd; + if (voltage_id >= table_info->vddmem_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddmem index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddmem_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].mvdd = table_info->vddmem_lookup_table->entries[voltage_id].us_vdd; } - return 0; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index 0be34fbdac266..4177211f32dca 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -2094,8 +2094,6 @@ static ssize_t smu_v13_0_0_get_gpu_metrics(struct smu_context *smu, if ((mp1_ver == IP_VERSION(13, 0, 0) && smu->smc_fw_version <= 0x004e1e00) || (mp1_ver == IP_VERSION(13, 0, 10) && smu->smc_fw_version <= 0x00500800)) gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; - else - gpu_metrics->energy_accumulator = UINT_MAX; if (metrics->AverageGfxActivity <= SMU_13_0_0_BUSY_THRESHOLD) gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c index 5d11ed8be4c32..845bd00c9df85 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c @@ -2096,8 +2096,8 @@ static ssize_t smu_v13_0_7_get_gpu_metrics(struct smu_context *smu, metrics->Vcn1ActivityPercentage); gpu_metrics->average_socket_power = metrics->AverageSocketPower; - gpu_metrics->energy_accumulator = smu->smc_fw_version <= 0x00521400 ? - metrics->EnergyAccumulator : UINT_MAX; + if (smu->smc_fw_version <= 0x00521400) + gpu_metrics->energy_accumulator = metrics->EnergyAccumulator; if (metrics->AverageGfxActivity <= SMU_13_0_7_BUSY_THRESHOLD) gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c index 118eda5429ff7..68bd673c0573d 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c @@ -1915,7 +1915,7 @@ static int smu_v15_0_8_set_performance_level(struct smu_context *smu, struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table; struct smu_dpm_table *uclk_table = &dpm_context->dpm_tables.uclk_table; struct smu_umd_pstate_table *pstate_table = &smu->pstate_table; - int ret; + int ret = 0; switch (level) { case AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM: @@ -1956,9 +1956,6 @@ static int smu_v15_0_8_set_performance_level(struct smu_context *smu, SMU_DPM_TABLE_MAX(uclk_table); } - if (ret) - goto out; - smu_cmn_reset_custom_level(smu); break; diff --git a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c index cf90d4468b5c1..344c3f4660185 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c @@ -1015,7 +1015,7 @@ cdns_dsi_bridge_atomic_reset(struct drm_bridge *bridge) dsi_state = kzalloc_obj(*dsi_state); if (!dsi_state) - return NULL; + return ERR_PTR(-ENOMEM); memset(dsi_state, 0, sizeof(*dsi_state)); dsi_state->base.bridge = bridge; diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c index 36c07b71fe04b..46779b49545bd 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c @@ -1927,7 +1927,7 @@ cdns_mhdp_bridge_atomic_reset(struct drm_bridge *bridge) cdns_mhdp_state = kzalloc_obj(*cdns_mhdp_state); if (!cdns_mhdp_state) - return NULL; + return ERR_PTR(-ENOMEM); __drm_atomic_helper_bridge_reset(bridge, &cdns_mhdp_state->base); diff --git a/drivers/gpu/drm/bridge/display-connector.c b/drivers/gpu/drm/bridge/display-connector.c index 6b128fabe3a97..441355ace2bb8 100644 --- a/drivers/gpu/drm/bridge/display-connector.c +++ b/drivers/gpu/drm/bridge/display-connector.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ struct display_connector { struct regulator *supply; struct gpio_desc *ddc_en; + + struct work_struct hpd_work; }; static inline struct display_connector * @@ -94,6 +97,34 @@ display_connector_bridge_detect(struct drm_bridge *bridge, struct drm_connector return display_connector_detect(bridge); } +static void display_connector_hpd_enable(struct drm_bridge *bridge) +{ + struct display_connector *conn = to_display_connector(bridge); + + enable_irq(conn->hpd_irq); + + if (conn->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) + schedule_work(&conn->hpd_work); +} + +static void display_connector_hpd_disable(struct drm_bridge *bridge) +{ + struct display_connector *conn = to_display_connector(bridge); + + if (conn->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) + cancel_work_sync(&conn->hpd_work); + + disable_irq(conn->hpd_irq); +} + +static void display_connector_hpd_work(struct work_struct *work) +{ + struct display_connector *conn = container_of(work, struct display_connector, hpd_work); + struct drm_bridge *bridge = &conn->bridge; + + drm_bridge_hpd_notify(bridge, display_connector_detect(bridge)); +} + static const struct drm_edid *display_connector_edid_read(struct drm_bridge *bridge, struct drm_connector *connector) { @@ -186,6 +217,8 @@ static const struct drm_bridge_funcs display_connector_bridge_funcs = { .attach = display_connector_attach, .destroy = display_connector_destroy, .detect = display_connector_bridge_detect, + .hpd_enable = display_connector_hpd_enable, + .hpd_disable = display_connector_hpd_disable, .edid_read = display_connector_edid_read, .atomic_get_output_bus_fmts = display_connector_get_output_bus_fmts, .atomic_get_input_bus_fmts = display_connector_get_input_bus_fmts, @@ -315,6 +348,7 @@ static int display_connector_probe(struct platform_device *pdev) NULL, display_connector_hpd_irq, IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | + IRQF_NO_AUTOEN | IRQF_ONESHOT, "HPD", conn); if (ret) { @@ -386,6 +420,8 @@ static int display_connector_probe(struct platform_device *pdev) conn->bridge.ops |= DRM_BRIDGE_OP_DETECT; if (conn->hpd_irq >= 0) conn->bridge.ops |= DRM_BRIDGE_OP_HPD; + if (conn->hpd_irq >= 0 && type == DRM_MODE_CONNECTOR_DisplayPort) + INIT_WORK(&conn->hpd_work, display_connector_hpd_work); dev_dbg(&pdev->dev, "Found %s display connector '%s' %s DDC bus and %s HPD GPIO (ops 0x%x)\n", diff --git a/drivers/gpu/drm/bridge/of-display-mode-bridge.c b/drivers/gpu/drm/bridge/of-display-mode-bridge.c index cb15713f3a796..e66dae168fd0d 100644 --- a/drivers/gpu/drm/bridge/of-display-mode-bridge.c +++ b/drivers/gpu/drm/bridge/of-display-mode-bridge.c @@ -51,7 +51,7 @@ static int of_display_mode_bridge_get_modes(struct drm_bridge *bridge, return 0; } -struct drm_bridge_funcs of_display_mode_bridge_funcs = { +static const struct drm_bridge_funcs of_display_mode_bridge_funcs = { .attach = of_display_mode_bridge_attach, .get_modes = of_display_mode_bridge_get_modes, }; diff --git a/drivers/gpu/drm/bridge/samsung-dsim.c b/drivers/gpu/drm/bridge/samsung-dsim.c index 9ee0515074c78..7e320eac2e4ad 100644 --- a/drivers/gpu/drm/bridge/samsung-dsim.c +++ b/drivers/gpu/drm/bridge/samsung-dsim.c @@ -1862,7 +1862,7 @@ static int samsung_dsim_register_te_irq(struct samsung_dsim *dsi, struct device int te_gpio_irq; int ret; - dsi->te_gpio = devm_gpiod_get_optional(dev, "te", GPIOD_IN); + dsi->te_gpio = gpiod_get_optional(dev, "te", GPIOD_IN); if (!dsi->te_gpio) return 0; else if (IS_ERR(dsi->te_gpio)) diff --git a/drivers/gpu/drm/bridge/synopsys/dw-dp.c b/drivers/gpu/drm/bridge/synopsys/dw-dp.c index 21541be094c47..36ee6e027af52 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-dp.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-dp.c @@ -2093,6 +2093,12 @@ struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder, } EXPORT_SYMBOL_GPL(dw_dp_bind); +void dw_dp_unbind(struct dw_dp *dp) +{ + drm_dp_aux_unregister(&dp->aux); +} +EXPORT_SYMBOL_GPL(dw_dp_unbind); + MODULE_AUTHOR("Andy Yan "); MODULE_DESCRIPTION("DW DP Core Library"); MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c index 17d5caedb32e0..9eb6cf8a321f2 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c @@ -3389,7 +3389,8 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, break; default: dev_err(dev, "reg-io-width must be 1 or 4\n"); - return ERR_PTR(-EINVAL); + ret = -EINVAL; + goto err_res; } iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c index 7188935fdb826..948bb7b2867a0 100644 --- a/drivers/gpu/drm/bridge/tc358767.c +++ b/drivers/gpu/drm/bridge/tc358767.c @@ -527,7 +527,7 @@ static ssize_t tc_aux_transfer(struct drm_dp_aux *aux, * address-only transfer */ if (size) - size = FIELD_GET(AUX_BYTES, auxstatus); + size = min_t(size_t, size, FIELD_GET(AUX_BYTES, auxstatus)); msg->reply = FIELD_GET(AUX_STATUS, auxstatus); switch (request) { diff --git a/drivers/gpu/drm/bridge/tc358768.c b/drivers/gpu/drm/bridge/tc358768.c index 0f2820b373029..d18136a4b5c6d 100644 --- a/drivers/gpu/drm/bridge/tc358768.c +++ b/drivers/gpu/drm/bridge/tc358768.c @@ -1262,10 +1262,13 @@ tc358768_atomic_get_input_bus_fmts(struct drm_bridge *bridge, return input_fmts; } -static bool tc358768_mode_fixup(struct drm_bridge *bridge, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) +static int tc358768_bridge_atomic_check(struct drm_bridge *bridge, + struct drm_bridge_state *bridge_state, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state) { + struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode; + /* Default to positive sync */ if (!(adjusted_mode->flags & @@ -1276,13 +1279,15 @@ static bool tc358768_mode_fixup(struct drm_bridge *bridge, (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC))) adjusted_mode->flags |= DRM_MODE_FLAG_PVSYNC; - return true; + bridge_state->input_bus_cfg.flags = bridge->timings->input_bus_flags; + + return 0; } static const struct drm_bridge_funcs tc358768_bridge_funcs = { .attach = tc358768_bridge_attach, .mode_valid = tc358768_bridge_mode_valid, - .mode_fixup = tc358768_mode_fixup, + .atomic_check = tc358768_bridge_atomic_check, .atomic_pre_enable = tc358768_bridge_atomic_pre_enable, .atomic_enable = tc358768_bridge_atomic_enable, .atomic_disable = tc358768_bridge_atomic_disable, diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c index 42b451432bbb6..9f530cd05d2f3 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c @@ -403,7 +403,7 @@ static int sn65dsi83_reset_pipe(struct sn65dsi83 *sn65dsi83) drm_modeset_drop_locks(&ctx); drm_modeset_acquire_fini(&ctx); - return 0; + return err; } static void sn65dsi83_reset_work(struct work_struct *ws) @@ -419,11 +419,13 @@ static void sn65dsi83_reset_work(struct work_struct *ws) ret = sn65dsi83_reset_pipe(ctx); if (ret) { dev_err(ctx->dev, "reset pipe failed %pe\n", ERR_PTR(ret)); - return; + goto bridge_exit; } + if (ctx->irq) enable_irq(ctx->irq); +bridge_exit: drm_bridge_exit(idx); } diff --git a/drivers/gpu/drm/clients/drm_fbdev_client.c b/drivers/gpu/drm/clients/drm_fbdev_client.c index 91d196a397cf2..1c16bc1084c40 100644 --- a/drivers/gpu/drm/clients/drm_fbdev_client.c +++ b/drivers/gpu/drm/clients/drm_fbdev_client.c @@ -42,6 +42,14 @@ static int drm_fbdev_client_restore(struct drm_client_dev *client, bool force) { struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client); + /* + * The client is registered before the initial fbdev probe. + * If probing failed, the client remains registered but there + * is no valid fbdev framebuffer to restore. + */ + if (!fb_helper->info || !fb_helper->fb) + return 0; + drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper, force); return 0; diff --git a/drivers/gpu/drm/display/drm_hdmi_state_helper.c b/drivers/gpu/drm/display/drm_hdmi_state_helper.c index cae0d85fb4407..a3aee41385548 100644 --- a/drivers/gpu/drm/display/drm_hdmi_state_helper.c +++ b/drivers/gpu/drm/display/drm_hdmi_state_helper.c @@ -910,8 +910,21 @@ drm_hdmi_connector_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode) { unsigned long long clock; + enum drm_output_color_format fmt; + + if (drm_mode_is_420_only(&connector->display_info, mode)) { + if (connector->ycbcr_420_allowed) + fmt = DRM_OUTPUT_COLOR_FORMAT_YCBCR420; + else + return MODE_NO_420; + } else if (drm_mode_is_420_also(&connector->display_info, mode) && + connector->ycbcr_420_allowed) { + fmt = DRM_OUTPUT_COLOR_FORMAT_YCBCR420; + } else { + fmt = DRM_OUTPUT_COLOR_FORMAT_RGB444; + } - clock = drm_hdmi_compute_mode_clock(mode, 8, DRM_OUTPUT_COLOR_FORMAT_RGB444); + clock = drm_hdmi_compute_mode_clock(mode, 8, fmt); if (!clock) return MODE_ERROR; diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c index cc70508d4fdba..a82568d87e4f7 100644 --- a/drivers/gpu/drm/drm_atomic_state_helper.c +++ b/drivers/gpu/drm/drm_atomic_state_helper.c @@ -731,8 +731,6 @@ void __drm_atomic_helper_private_obj_create_state(struct drm_private_obj *obj, { if (state) state->obj = obj; - - obj->state = state; } EXPORT_SYMBOL(__drm_atomic_helper_private_obj_create_state); diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index e997917819e8e..66b591b39a513 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -1461,10 +1461,12 @@ static int prepare_signaling(struct drm_device *dev, struct dma_fence *fence; struct drm_out_fence_state *f; + ret = -ENOMEM; + f = krealloc(*fence_state, sizeof(**fence_state) * (*num_fences + 1), GFP_KERNEL); if (!f) - return -ENOMEM; + goto err_free_event; memset(&f[*num_fences], 0, sizeof(*f)); @@ -1473,12 +1475,12 @@ static int prepare_signaling(struct drm_device *dev, fence = drm_crtc_create_fence(crtc); if (!fence) - return -ENOMEM; + goto err_free_event; ret = setup_out_fence(&f[(*num_fences)++], fence); if (ret) { dma_fence_put(fence); - return ret; + goto err_free_event; } crtc_state->event->base.fence = fence; @@ -1534,6 +1536,11 @@ static int prepare_signaling(struct drm_device *dev, } return 0; + +err_free_event: + drm_event_cancel_free(dev, &crtc_state->event->base); + crtc_state->event = NULL; + return ret; } static void complete_signaling(struct drm_device *dev, @@ -1563,6 +1570,8 @@ static void complete_signaling(struct drm_device *dev, * to prevent a double free in drm_atomic_commit_clear. */ if (event && (event->base.fence || event->base.file_priv)) { + if (crtc_state->commit && crtc_state->commit->abort_completion) + drm_crtc_commit_put(crtc_state->commit); drm_event_cancel_free(dev, &event->base); crtc_state->event = NULL; } diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c index 675675480da49..e890052061f30 100644 --- a/drivers/gpu/drm/drm_drv.c +++ b/drivers/gpu/drm/drm_drv.c @@ -473,6 +473,22 @@ void drm_dev_exit(int idx) } EXPORT_SYMBOL(drm_dev_exit); +/* + * Mark the device as unplugged and wait for any in-flight drm_dev_enter() + * critical sections to complete. + */ +static void drm_dev_synchronize_unplug(struct drm_device *dev) +{ + /* + * After synchronizing any critical read section is guaranteed to see + * the new value of ->unplugged, and any critical section which might + * still have seen the old value of ->unplugged is guaranteed to have + * finished. + */ + dev->unplugged = true; + synchronize_srcu(&drm_unplug_srcu); +} + /** * drm_dev_unplug - unplug a DRM device * @dev: DRM device @@ -485,15 +501,7 @@ EXPORT_SYMBOL(drm_dev_exit); */ void drm_dev_unplug(struct drm_device *dev) { - /* - * After synchronizing any critical read section is guaranteed to see - * the new value of ->unplugged, and any critical section which might - * still have seen the old value of ->unplugged is guaranteed to have - * finished. - */ - dev->unplugged = true; - synchronize_srcu(&drm_unplug_srcu); - + drm_dev_synchronize_unplug(dev); drm_dev_unregister(dev); /* Clear all CPU mappings pointing to this device */ @@ -1091,6 +1099,7 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags) goto err_minors; dev->registered = true; + dev->unplugged = false; if (driver->load) { ret = driver->load(dev, flags); @@ -1118,6 +1127,13 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags) if (dev->driver->unload) dev->driver->unload(dev); err_minors: + /* + * If a minor was registered before the failure, userspace could have + * opened it and entered a drm_dev_enter() critical section. Ensure all + * such sections complete before we clean up. + */ + drm_dev_synchronize_unplug(dev); + remove_compat_control_link(dev); drm_minor_unregister(dev, DRM_MINOR_ACCEL); drm_minor_unregister(dev, DRM_MINOR_PRIMARY); diff --git a/drivers/gpu/drm/drm_dumb_buffers.c b/drivers/gpu/drm/drm_dumb_buffers.c index 2156dbe601c9c..8e9ff17538e73 100644 --- a/drivers/gpu/drm/drm_dumb_buffers.c +++ b/drivers/gpu/drm/drm_dumb_buffers.c @@ -32,7 +32,6 @@ #include #include "drm_crtc_internal.h" -#include "drm_internal.h" /** * DOC: overview diff --git a/drivers/gpu/drm/drm_exec.c b/drivers/gpu/drm/drm_exec.c index 7988f5e7d56a3..2abc5c623fffd 100644 --- a/drivers/gpu/drm/drm_exec.c +++ b/drivers/gpu/drm/drm_exec.c @@ -322,6 +322,19 @@ int drm_exec_prepare_array(struct drm_exec *exec, { int ret; + /* + * Make sure to lock a contended object even when no objects are + * given, otherwise drm_exec_retry_on_contention() would loop + * forever on patterns like: + * + * ret = drm_exec_prepare_array(exec, objs, num_objects, ...); + * drm_exec_retry_on_contention(exec); + * + * with num_objects == 0. + */ + if (!num_objects) + return drm_exec_lock_contended(exec); + for (unsigned int i = 0; i < num_objects; ++i) { ret = drm_exec_prepare_obj(exec, objects[i], num_fences); if (unlikely(ret)) diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index 892b325fa99b8..a0546955d0b9f 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -383,6 +383,58 @@ drm_pagemap_migrate_map_system_pages(struct device *dev, return 0; } +/** + * drm_pagemap_migrate_populate_src_pages() - Populate the source page array + * @pages: Array of source pages to populate + * @src_mpfn: Source array of migrate PFNs + * @dst_mpfn: Destination array of migrate PFNs + * @npages: Number of pages in the arrays + * + * Populate @pages with the device pages the copy callback is to read from. + * + * Entries are normally only populated at the head of each source folio, with + * the copy callback deriving the rest of the folio from the order recorded in + * the corresponding drm_pagemap_addr. That does not work where + * drm_pagemap_migrate_populate_ram_pfn() had to demote a higher-order source + * folio to order-0 destination folios: the drm_pagemap_addr entries are then + * per-page, and the copy callback needs a source page for each of them. + * Populate every entry for those ranges. + * + * Note that the source folio itself is only split later, by + * migrate_vma_pages() / migrate_device_pages(), so its order cannot be used to + * detect the demotion - the destination has to be inspected instead. + */ +static void drm_pagemap_migrate_populate_src_pages(struct page **pages, + unsigned long *src_mpfn, + unsigned long *dst_mpfn, + unsigned long npages) +{ + unsigned long i; + + for (i = 0; i < npages;) { + struct page *page = migrate_pfn_to_page(src_mpfn[i]); + unsigned int order = 0; + unsigned long j, nr; + + if (!page) { + i++; + continue; + } + + order = folio_order(page_folio(page)); + nr = NR_PAGES(order); + + if (order && !(dst_mpfn[i] & MIGRATE_PFN_COMPOUND)) { + for (j = 0; j < nr && i + j < npages; j++) + pages[i + j] = folio_page(page_folio(page), j); + } else { + pages[i] = page; + } + + i += nr; + } +} + /** * drm_pagemap_migrate_unmap_pages() - Unmap pages previously mapped for GPU SVM migration * @dev: The device for which the pages were mapped @@ -875,6 +927,7 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas, struct page *page = NULL, *src_page; struct folio *folio; unsigned int order = 0; + gfp_t gfp = GFP_HIGHUSER; if (!(src_mpfn[i] & MIGRATE_PFN_MIGRATE)) goto next; @@ -891,11 +944,51 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas, order = folio_order(page_folio(src_page)); - /* TODO: Support fallback to single pages if THP allocation fails */ + /* + * A large source folio is always collected whole, at its head + * page, PMD aligned and flagged MIGRATE_PFN_COMPOUND: anything + * else is split before it reaches us, either by + * migrate_vma_collect_pmd() or, for the eviction path, by + * migrate_device_pfns(). Both the order-0 fallback below and + * drm_pagemap_migrate_populate_src_pages() rely on that, as + * they index the folio from @i. + */ + WARN_ON_ONCE(order && + (src_page != folio_page(page_folio(src_page), 0) || + !(src_mpfn[i] & MIGRATE_PFN_COMPOUND))); + + if (order) + gfp |= __GFP_NOWARN; + if (vas) - folio = vma_alloc_folio(GFP_HIGHUSER, order, vas, addr); + folio = vma_alloc_folio(gfp, order, vas, addr); else - folio = folio_alloc(GFP_HIGHUSER, order); + folio = folio_alloc(gfp, order); + + if (!folio && order) { + /* + * Higher-order allocation failed, fall back to + * order-0 allocations for the entire range covered + * by the original higher-order allocation, without + * setting MIGRATE_PFN_COMPOUND, until we move past + * that range. + */ + unsigned long nr = NR_PAGES(order); + unsigned long j; + + gfp &= ~__GFP_NOWARN; + for (j = 0; j < nr && i < npages; j++, i++, addr += PAGE_SIZE) { + folio = vas ? + vma_alloc_folio(gfp, 0, vas, addr) : + folio_alloc(gfp, 0); + if (!folio) + goto free_pages; + + page = folio_page(folio, 0); + mpfn[i] = migrate_pfn(page_to_pfn(page)); + } + continue; + } if (!folio) goto free_pages; @@ -940,11 +1033,11 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas, if (!page) goto next_put; + order = folio_order(page_folio(page)); + put_page(page); mpfn[i] = 0; - order = folio_order(page_folio(page)); - next_put: i += NR_PAGES(order); } @@ -1102,12 +1195,117 @@ void drm_pagemap_put(struct drm_pagemap *dpagemap) } EXPORT_SYMBOL(drm_pagemap_put); +/** + * drm_pagemap_page_get_flags() - Read flags from a device-private folio + * @page: Pointer to a page of the device-private folio + * + * Return: The DRM_PAGEMAP_ZDD_FLAG_* bits encoded in zone_device_data. + */ +static unsigned long drm_pagemap_page_get_flags(struct page *page) +{ + struct folio *folio = page_folio(page); + + return (unsigned long)folio_zone_device_data(folio) & + DRM_PAGEMAP_ZDD_FLAG_MASK; +} + +/** + * drm_pagemap_page_set_flags() - Set flags on a device-private folio + * @page: Pointer to a page of the device-private folio + * @flags: DRM_PAGEMAP_ZDD_FLAG_* bits to set + * + * Preserve any flags already encoded alongside the ZDD pointer. + */ +static void drm_pagemap_page_set_flags(struct page *page, + unsigned long flags) +{ + struct folio *folio = page_folio(page); + unsigned long old; + + if (WARN_ON_ONCE(flags & ~DRM_PAGEMAP_ZDD_FLAG_MASK)) + return; + + old = (unsigned long)folio_zone_device_data(folio); + folio_set_zone_device_data(folio, (void *)(old | flags)); +} + +/** + * drm_pagemap_retire_migrated_pages() - Record migrated device-private folios + * @src_pfns: source array after migrate_vma_pages() or migrate_device_pages() + * @npages: number of entries in @src_pfns + * + * Flag device-private folios successfully migrated to RAM before finalize + * unlocks the sources. The migrated state is stored in the physical folio, so + * it survives later folio splits and subsequent migrations can skip it. + */ +static void drm_pagemap_retire_migrated_pages(unsigned long *src_pfns, + unsigned long npages) +{ + unsigned long i = 0; + + while (i < npages) { + struct page *page = migrate_pfn_to_page(src_pfns[i]); + unsigned long nr = 1; + + if (!page) { + i++; + continue; + } + + if (src_pfns[i] & MIGRATE_PFN_COMPOUND) + nr = folio_nr_pages(page_folio(page)); + + if ((src_pfns[i] & MIGRATE_PFN_MIGRATE) && + is_device_private_page(page)) + drm_pagemap_page_set_flags(page, + DRM_PAGEMAP_ZDD_FLAG_MIGRATED); + + i += nr; + } +} + +/** + * drm_pagemap_skip_retired_pages() - Skip retired device-private folios + * @src_pfns: MIGRATE_PFN-encoded source array + * @npages: number of entries in @src_pfns + * + * Skip source folios already migrated to RAM, identified by the migrated flag + * stored in the physical folio's zone_device_data. + */ +static void drm_pagemap_skip_retired_pages(unsigned long *src_pfns, + unsigned long npages) +{ + unsigned long i = 0; + + while (i < npages) { + struct page *page = migrate_pfn_to_page(src_pfns[i]); + unsigned long nr = 1; + + if (!page) { + i++; + continue; + } + + if (src_pfns[i] & MIGRATE_PFN_COMPOUND) + nr = folio_nr_pages(page_folio(page)); + + if ((src_pfns[i] & MIGRATE_PFN_MIGRATE) && + is_device_private_page(page) && + (drm_pagemap_page_get_flags(page) & + DRM_PAGEMAP_ZDD_FLAG_MIGRATED)) + src_pfns[i] &= ~MIGRATE_PFN_MIGRATE; + + i += nr; + } +} + /** * drm_pagemap_evict_to_ram() - Evict GPU SVM range to RAM * @devmem_allocation: Pointer to the device memory allocation * - * Similar to __drm_pagemap_migrate_to_ram but does not require mmap lock and - * migration done via migrate_device_* functions. + * Similar to __drm_pagemap_migrate_to_ram(), but uses the + * migrate_device_* helpers and does not require the mmap lock. + * Device-private PFNs already migrated to RAM by either path are skipped. * * Return: 0 on success, negative error code on failure. */ @@ -1115,17 +1313,18 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) { const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops; struct drm_pagemap_iova_state state = {}; - unsigned long npages, mpages = 0; + unsigned long npages, mpages; struct page **pages; unsigned long *src, *dst; struct drm_pagemap_addr *pagemap_addr; void *buf; - int i, err = 0; + int err = 0; unsigned int retry_count = 2; npages = devmem_allocation->size >> PAGE_SHIFT; retry: + mpages = 0; if (!mmget_not_zero(devmem_allocation->mm)) return -EFAULT; @@ -1148,6 +1347,8 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) if (err) goto err_free; + drm_pagemap_skip_retired_pages(src, npages); + err = drm_pagemap_migrate_populate_ram_pfn(NULL, NULL, npages, &mpages, src, dst, 0); if (err || !mpages) @@ -1160,27 +1361,20 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) if (err) goto err_finalize; - for (i = 0; i < npages;) { - unsigned int order = 0; - - pages[i] = migrate_pfn_to_page(src[i]); - if (pages[i]) - order = folio_order(page_folio(pages[i])); - - i += NR_PAGES(order); - } + drm_pagemap_migrate_populate_src_pages(pages, src, dst, npages); err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL); if (err) goto err_finalize; err_finalize: + drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, dst, npages, + DMA_FROM_DEVICE, &state); if (err) drm_pagemap_migration_unlock_put_pages(npages, dst); migrate_device_pages(src, dst, npages); + drm_pagemap_retire_migrated_pages(src, npages); migrate_device_finalize(src, dst, npages); - drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, dst, npages, - DMA_FROM_DEVICE, &state); err_free: kvfree(buf); @@ -1235,7 +1429,7 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, struct drm_pagemap_addr *pagemap_addr; unsigned long start, end; void *buf; - int i, err = 0; + int err = 0; zdd = drm_pagemap_page_zone_device_data(page); if (time_before64(get_jiffies_64(), zdd->devmem_allocation->timeslice_expiration)) @@ -1275,13 +1469,15 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, if (!migrate.cpages) goto err_free; + drm_pagemap_skip_retired_pages(migrate.src, npages); + ops = zdd->devmem_allocation->ops; dev = zdd->devmem_allocation->dev; err = drm_pagemap_migrate_populate_ram_pfn(vas, page, npages, &mpages, migrate.src, migrate.dst, start); - if (err) + if (err || !mpages) goto err_finalize; err = drm_pagemap_migrate_map_system_pages(dev, pagemap_addr, @@ -1290,29 +1486,23 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, if (err) goto err_finalize; - for (i = 0; i < npages;) { - unsigned int order = 0; - - pages[i] = migrate_pfn_to_page(migrate.src[i]); - if (pages[i]) - order = folio_order(page_folio(pages[i])); - - i += NR_PAGES(order); - } + drm_pagemap_migrate_populate_src_pages(pages, migrate.src, migrate.dst, + npages); err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL); if (err) goto err_finalize; err_finalize: - if (err) - drm_pagemap_migration_unlock_put_pages(npages, migrate.dst); - migrate_vma_pages(&migrate); - migrate_vma_finalize(&migrate); if (dev) drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, migrate.dst, npages, DMA_FROM_DEVICE, &state); + if (err) + drm_pagemap_migration_unlock_put_pages(npages, migrate.dst); + migrate_vma_pages(&migrate); + drm_pagemap_retire_migrated_pages(migrate.src, npages); + migrate_vma_finalize(&migrate); err_free: kvfree(buf); err_out: @@ -1360,13 +1550,19 @@ static vm_fault_t drm_pagemap_migrate_to_ram(struct vm_fault *vmf) static void drm_pagemap_folio_split(struct folio *orig_folio, struct folio *new_folio) { struct drm_pagemap_zdd *zdd; + unsigned long orig_data, new_data; if (!new_folio) return; new_folio->pgmap = orig_folio->pgmap; - zdd = folio_zone_device_data(orig_folio); - folio_set_zone_device_data(new_folio, drm_pagemap_zdd_get(zdd)); + + orig_data = (unsigned long)folio_zone_device_data(orig_folio); + zdd = (struct drm_pagemap_zdd *)(orig_data & ~DRM_PAGEMAP_ZDD_FLAG_MASK); + + new_data = (unsigned long)drm_pagemap_zdd_get(zdd); + new_data |= orig_data & DRM_PAGEMAP_ZDD_FLAG_MASK; + folio_set_zone_device_data(new_folio, (void *)new_data); } static const struct dev_pagemap_ops drm_pagemap_pagemap_ops = { diff --git a/drivers/gpu/drm/drm_panic_qr.rs b/drivers/gpu/drm/drm_panic_qr.rs index ac27e86c601c8..4d7eb75a3afc1 100644 --- a/drivers/gpu/drm/drm_panic_qr.rs +++ b/drivers/gpu/drm/drm_panic_qr.rs @@ -407,8 +407,8 @@ impl DecFifo { for i in (0..self.len).rev() { self.decimals[i + len] = self.decimals[i]; } - for i in 0..len { - self.decimals[i] = (chunk % 10) as u8; + for decimal in &mut self.decimals[..len] { + *decimal = (chunk % 10) as u8; chunk = div10(chunk); } self.len += len; diff --git a/drivers/gpu/drm/gud/gud_connector.c b/drivers/gpu/drm/gud/gud_connector.c index ea0cca58b7c89..8141c3a1e30ad 100644 --- a/drivers/gpu/drm/gud/gud_connector.c +++ b/drivers/gpu/drm/gud/gud_connector.c @@ -396,8 +396,16 @@ static int gud_connector_add_tv_mode(struct gud_device *gdrm, struct drm_connect } num_modes = ret / GUD_CONNECTOR_TV_MODE_NAME_LEN; - for (i = 0; i < num_modes; i++) - modes[i] = &buf[i * GUD_CONNECTOR_TV_MODE_NAME_LEN]; + for (i = 0; i < num_modes; i++) { + char *mode = &buf[i * GUD_CONNECTOR_TV_MODE_NAME_LEN]; + + if (!memchr(mode, '\0', GUD_CONNECTOR_TV_MODE_NAME_LEN)) { + ret = -EIO; + goto free; + } + + modes[i] = mode; + } ret = drm_mode_create_tv_properties_legacy(connector->dev, num_modes, modes); free: diff --git a/drivers/gpu/drm/gud/gud_drv.c b/drivers/gpu/drm/gud/gud_drv.c index 89bd6ca36003f..3a1b9e2a2eaa1 100644 --- a/drivers/gpu/drm/gud/gud_drv.c +++ b/drivers/gpu/drm/gud/gud_drv.c @@ -289,6 +289,8 @@ static int gud_plane_add_properties(struct gud_device *gdrm) * but mask out any additions on future devices. */ val &= GUD_ROTATION_MASK; + if (!(val & GUD_ROTATION_0)) + continue; ret = drm_plane_create_rotation_property(&gdrm->plane, DRM_MODE_ROTATE_0, val); break; diff --git a/drivers/gpu/drm/gud/gud_pipe.c b/drivers/gpu/drm/gud/gud_pipe.c index 5ef887d8485a3..aa7792966287b 100644 --- a/drivers/gpu/drm/gud/gud_pipe.c +++ b/drivers/gpu/drm/gud/gud_pipe.c @@ -482,6 +482,9 @@ int gud_plane_atomic_check(struct drm_plane *plane, if (!new_plane_state->visible) return 0; + if (gdrm->flags & GUD_DISPLAY_FLAG_FULL_UPDATE) + new_plane_state->ignore_damage_clips = true; + if (old_plane_state->rotation != new_plane_state->rotation) crtc_state->mode_changed = true; @@ -562,8 +565,8 @@ int gud_plane_atomic_check(struct drm_plane *plane, goto out; } - req->properties[num_properties + i].prop = cpu_to_le16(prop); - req->properties[num_properties + i].val = cpu_to_le64(val); + req->properties[num_properties].prop = cpu_to_le16(prop); + req->properties[num_properties].val = cpu_to_le64(val); num_properties++; } diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c index 3066dc9ebc647..2e6e189bec1a0 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c @@ -72,46 +72,28 @@ static int hibmc_get_best_clock_idx(const struct drm_display_mode *mode) static int hibmc_plane_atomic_check(struct drm_plane *plane, struct drm_atomic_commit *state) { - struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, - plane); - struct drm_framebuffer *fb = new_plane_state->fb; - struct drm_crtc *crtc = new_plane_state->crtc; - struct drm_crtc_state *crtc_state; - u32 src_w = new_plane_state->src_w >> 16; - u32 src_h = new_plane_state->src_h >> 16; - - if (!crtc || !fb) - return 0; + struct drm_plane_state *new_plane_state = + drm_atomic_get_new_plane_state(state, plane); + struct drm_crtc_state *new_crtc_state = NULL; + int ret; - crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); + if (new_plane_state->crtc) + new_crtc_state = drm_atomic_get_new_crtc_state(state, new_plane_state->crtc); - if (src_w != new_plane_state->crtc_w || src_h != new_plane_state->crtc_h) { - drm_dbg_atomic(plane->dev, "scale not support\n"); - return -EINVAL; - } - - if (new_plane_state->crtc_x < 0 || new_plane_state->crtc_y < 0) { - drm_dbg_atomic(plane->dev, "crtc_x/y of drm_plane state is invalid\n"); - return -EINVAL; - } - - if (!crtc_state->enable) + ret = drm_atomic_helper_check_plane_state(new_plane_state, new_crtc_state, + DRM_PLANE_NO_SCALING, + DRM_PLANE_NO_SCALING, + false, true); + if (ret) + return ret; + else if (!new_plane_state->visible) return 0; - if (new_plane_state->crtc_x + new_plane_state->crtc_w > - crtc_state->adjusted_mode.hdisplay || - new_plane_state->crtc_y + new_plane_state->crtc_h > - crtc_state->adjusted_mode.vdisplay) { - drm_dbg_atomic(plane->dev, "visible portion of plane is invalid\n"); - return -EINVAL; - } - if (new_plane_state->fb->pitches[0] % 128 != 0) { drm_dbg_atomic(plane->dev, "wrong stride with 128-byte aligned\n"); return -EINVAL; } + return 0; } @@ -153,10 +135,8 @@ static void hibmc_plane_atomic_update(struct drm_plane *plane, } static const u32 channel_formats1[] = { - DRM_FORMAT_RGB565, DRM_FORMAT_BGR565, DRM_FORMAT_RGB888, - DRM_FORMAT_BGR888, DRM_FORMAT_XRGB8888, DRM_FORMAT_XBGR8888, - DRM_FORMAT_RGBA8888, DRM_FORMAT_BGRA8888, DRM_FORMAT_ARGB8888, - DRM_FORMAT_ABGR8888 + DRM_FORMAT_XRGB8888, + DRM_FORMAT_RGB565, }; static const struct drm_plane_funcs hibmc_plane_funcs = { diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index 7bc9b956554ba..40ecad20d32be 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -2366,8 +2366,10 @@ static void bxt_sanitize_cdclk(struct intel_display *display) * dividers both syncing to an active pipe, or asynchronously * (PIPE_NONE). */ - cdctl &= ~bxt_cdclk_cd2x_pipe(display, INVALID_PIPE); - cdctl |= bxt_cdclk_cd2x_pipe(display, INVALID_PIPE); + if (DISPLAY_VER(display) < 30) { + cdctl &= ~bxt_cdclk_cd2x_pipe(display, INVALID_PIPE); + cdctl |= bxt_cdclk_cd2x_pipe(display, INVALID_PIPE); + } if (cdctl != expected) { if (DISPLAY_VER(display) < 20) { @@ -2697,8 +2699,8 @@ static void intel_set_cdclk(struct intel_display *display, } } -static bool dg2_power_well_count(struct intel_display *display, - const struct intel_cdclk_state *cdclk_state) +static int dg2_power_well_count(struct intel_display *display, + const struct intel_cdclk_state *cdclk_state) { return display->platform.dg2 ? hweight8(cdclk_state->active_pipes) : 0; } diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c index 52347668f27d6..dad0e0eb0e0c0 100644 --- a/drivers/gpu/drm/i915/display/intel_cursor.c +++ b/drivers/gpu/drm/i915/display/intel_cursor.c @@ -536,7 +536,8 @@ static void i9xx_cursor_disable_sel_fetch_arm(struct intel_dsb *dsb, struct intel_display *display = to_intel_display(plane); enum pipe pipe = plane->pipe; - if (!crtc_state->enable_psr2_sel_fetch) + if (!crtc_state->enable_psr2_sel_fetch && + !crtc_state->clear_psr2_sel_fetch) return; intel_de_write_dsb(display, dsb, SEL_FETCH_CUR_CTL(pipe), 0); @@ -569,8 +570,10 @@ static void i9xx_cursor_update_sel_fetch_arm(struct intel_dsb *dsb, struct intel_display *display = to_intel_display(plane); enum pipe pipe = plane->pipe; - if (!crtc_state->enable_psr2_sel_fetch) + if (!crtc_state->enable_psr2_sel_fetch) { + i9xx_cursor_disable_sel_fetch_arm(dsb, plane, crtc_state); return; + } if (drm_rect_height(&plane_state->psr2_sel_fetch_area) > 0) { if (crtc_state->enable_psr2_su_region_et) { diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index c21e0c0ef0b12..18e6b01efd380 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -1177,6 +1177,8 @@ struct intel_crtc_state { bool has_sel_update; bool enable_psr2_sel_fetch; bool enable_psr2_su_region_et; + /* Drop the stale selective fetch enable bits as selective fetch is turned off */ + bool clear_psr2_sel_fetch; bool req_psr2_sdp_prior_scanline; bool has_panel_replay; bool link_off_after_as_sdp_when_pr_active; @@ -1947,6 +1949,8 @@ struct intel_dp { bool colorimetry_support; + bool sst_split_sdp_support; + struct { enum transcoder transcoder; struct mutex lock; diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 5733d2e7ac7f1..ba0b19d1163cf 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -3518,12 +3518,22 @@ intel_dp_audio_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + pipe_config->has_audio = intel_dp_has_audio(encoder, conn_state) && intel_audio_compute_config(encoder, pipe_config, conn_state); pipe_config->sdp_split_enable = pipe_config->has_audio && intel_dp_is_uhbr(pipe_config); + + /* + * SDP splitting for UHBR audio requires explicit sink capability in + * SST mode, whereas in MST mode it is inherently supported. + */ + if (pipe_config->sdp_split_enable && + !intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DP_MST)) + pipe_config->sdp_split_enable = intel_dp->sst_split_sdp_support; } void @@ -4566,16 +4576,25 @@ void intel_dp_configure_protocol_converter(struct intel_dp *intel_dp, str_enable_disable(tmp)); } -static bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp) +static u8 intel_dp_read_dprx_feature_enum(struct intel_dp *intel_dp) { u8 dprx = 0; - if (drm_dp_dpcd_readb(&intel_dp->aux, DP_DPRX_FEATURE_ENUMERATION_LIST, - &dprx) != 1) - return false; + drm_dp_dpcd_read_data(&intel_dp->aux, DP_DPRX_FEATURE_ENUMERATION_LIST, + &dprx, sizeof(dprx)); + return dprx; +} + +static bool intel_dp_get_colorimetry_status(u8 dprx) +{ return dprx & DP_VSC_SDP_EXT_FOR_COLORIMETRY_SUPPORTED; } +static bool intel_dp_get_sst_split_sdp_status(u8 dprx) +{ + return dprx & DP_SST_SPLIT_SDP_CAP; +} + static int intel_dp_read_dsc_dpcd(struct drm_dp_aux *aux, u8 dsc_dpcd[DP_DSC_RECEIVER_CAP_SIZE]) { @@ -4875,6 +4894,7 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector { struct intel_display *display = to_intel_display(intel_dp); int ret; + u8 dprx; /* this function is meant to be called only once */ drm_WARN_ON(display->drm, intel_dp->dpcd[DP_DPCD_REV] != 0); @@ -4886,8 +4906,13 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector drm_dp_is_branch(intel_dp->dpcd)); intel_init_dpcd_quirks(intel_dp, &intel_dp->desc.ident); + dprx = intel_dp_read_dprx_feature_enum(intel_dp); + intel_dp->colorimetry_support = - intel_dp_get_colorimetry_status(intel_dp); + intel_dp_get_colorimetry_status(dprx); + + intel_dp->sst_split_sdp_support = + intel_dp_get_sst_split_sdp_status(dprx); /* * Read the eDP display control registers. @@ -4978,13 +5003,20 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) * the OUI/ID since we know it won't change. */ if (!intel_dp_is_edp(intel_dp)) { + u8 dprx; + drm_dp_read_desc(&intel_dp->aux, &intel_dp->desc, drm_dp_is_branch(intel_dp->dpcd)); intel_init_dpcd_quirks(intel_dp, &intel_dp->desc.ident); + dprx = intel_dp_read_dprx_feature_enum(intel_dp); + intel_dp->colorimetry_support = - intel_dp_get_colorimetry_status(intel_dp); + intel_dp_get_colorimetry_status(dprx); + + intel_dp->sst_split_sdp_support = + intel_dp_get_sst_split_sdp_status(dprx); intel_dp_update_sink_caps(intel_dp); } diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index 0aa3e6b4c781d..a1922f95e5d55 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -722,10 +722,6 @@ static int mst_stream_compute_config(struct intel_encoder *encoder, pipe_config->sink_format = INTEL_OUTPUT_FORMAT_RGB; pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; - ret = intel_pfit_compute_config(pipe_config, conn_state); - if (ret) - return ret; - ret = intel_pfit_compute_config(pipe_config, conn_state); if (ret) return ret; @@ -817,7 +813,8 @@ static u8 get_pipes_downstream_of_mst_port(struct intel_atomic_state *state, if (&connector->mst.dp->mst.mgr != mst_mgr) continue; - if (connector->mst.port != parent_port && + if (parent_port && + connector->mst.port != parent_port && !drm_dp_mst_port_downstream_of_parent(mst_mgr, connector->mst.port, parent_port)) @@ -2125,6 +2122,27 @@ bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state, return false; } +bool intel_dp_mst_stream_disconnected(struct intel_atomic_state *state, + const struct intel_crtc *crtc) +{ + struct intel_connector *connector; + + connector = get_connector_in_state_for_crtc(state, crtc); + if (!connector) + return false; + + if (!connector->mst.dp) + return false; + + if (!connector->mst.dp->mst.mgr.mst_state) + return true; + + if (drm_connector_is_unregistered(&connector->base)) + return true; + + return false; +} + /** * intel_dp_mst_prepare_probe - Prepare an MST link for topology probing * @intel_dp: DP port object diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.h b/drivers/gpu/drm/i915/display/intel_dp_mst.h index ab09b487c6bb5..8ce89242c05c9 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.h +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.h @@ -28,6 +28,8 @@ int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state, struct intel_link_bw_limits *limits); bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state, struct intel_crtc *crtc); +bool intel_dp_mst_stream_disconnected(struct intel_atomic_state *state, + const struct intel_crtc *crtc); void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp); bool intel_dp_mst_verify_dpcd_state(struct intel_dp *intel_dp); diff --git a/drivers/gpu/drm/i915/display/intel_link_bw.c b/drivers/gpu/drm/i915/display/intel_link_bw.c index b47474a3e9fec..e71e76d6fd3e0 100644 --- a/drivers/gpu/drm/i915/display/intel_link_bw.c +++ b/drivers/gpu/drm/i915/display/intel_link_bw.c @@ -64,7 +64,8 @@ void intel_link_bw_init_limits(struct intel_atomic_state *state, intel_atomic_get_new_crtc_state(state, crtc); int forced_bpp_x16 = get_forced_link_bpp_x16(state, crtc); - if (state->base.duplicated && crtc_state) { + if ((state->base.duplicated && crtc_state) || + intel_dp_mst_stream_disconnected(state, crtc)) { limits->max_bpp_x16[pipe] = crtc_state->max_link_bpp_x16; if (intel_dsc_enabled_on_link(crtc_state)) limits->link_dsc_pipes |= BIT(pipe); diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index beaa1d62613db..87128104476a3 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -2933,6 +2933,8 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct intel_display *display = to_intel_display(state); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); struct intel_plane_state *new_plane_state, *old_plane_state; struct intel_plane *plane; @@ -2945,6 +2947,19 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, bool full_update = false, su_area_changed; int i, ret; + /* + * Selective fetch is not always usable, for instance it is dropped + * while pipe CRC is active. The planes keep their selective fetch + * enable bit set in hardware over that, and a plane disabled while + * selective fetch is off never gets the bit cleared. Once selective + * fetch comes back the hardware would resume fetching for a plane that + * is no longer enabled and keep its DDB range reserved, so have the + * plane update drop the bit for every plane of the pipe as selective + * fetch is turned off. + */ + crtc_state->clear_psr2_sel_fetch = old_crtc_state->enable_psr2_sel_fetch && + !crtc_state->enable_psr2_sel_fetch; + if (!crtc_state->enable_psr2_sel_fetch) return 0; diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c index 33245f44c0d50..7d7db774d8c7b 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.c +++ b/drivers/gpu/drm/i915/display/intel_quirks.c @@ -257,6 +257,9 @@ static struct intel_quirk intel_quirks[] = { /* Dell XPS 13 7390 2-in-1 */ { 0x8a52, 0x1028, 0x08b0, quirk_edp_limit_rate_hbr2 }, + /* HP Pavilion Plus Laptop 14-ew1xxx */ + { 0x7d55, 0x103c, 0x8c31, quirk_edp_limit_rate_hbr2 }, + /* Xiaomi Book Pro 14 2026 */ { 0xb081, 0x1d72, 0x2424, quirk_disable_psr2 }, }; diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index 164b7d61c9a31..3dad2da4c3aaf 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -885,7 +885,8 @@ static void icl_plane_disable_sel_fetch_arm(struct intel_dsb *dsb, struct intel_display *display = to_intel_display(plane); enum pipe pipe = plane->pipe; - if (!crtc_state->enable_psr2_sel_fetch) + if (!crtc_state->enable_psr2_sel_fetch && + !crtc_state->clear_psr2_sel_fetch) return; intel_de_write_dsb(display, dsb, SEL_FETCH_PLANE_CTL(pipe, plane->id), 0); @@ -1634,10 +1635,8 @@ static void icl_plane_update_sel_fetch_arm(struct intel_dsb *dsb, struct intel_display *display = to_intel_display(plane); enum pipe pipe = plane->pipe; - if (!crtc_state->enable_psr2_sel_fetch) - return; - - if (drm_rect_height(&plane_state->psr2_sel_fetch_area) > 0) + if (crtc_state->enable_psr2_sel_fetch && + drm_rect_height(&plane_state->psr2_sel_fetch_area) > 0) intel_de_write_dsb(display, dsb, SEL_FETCH_PLANE_CTL(pipe, plane->id), SEL_FETCH_PLANE_CTL_ENABLE); else diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index 5172d39826548..9e01f8b2079ab 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -89,6 +89,7 @@ struct drm_i915_gem_object *i915_gem_object_alloc(void) void i915_gem_object_free(struct drm_i915_gem_object *obj) { + dma_resv_fini(&obj->base._resv); return kmem_cache_free(slab_objects, obj); } @@ -144,7 +145,6 @@ void __i915_gem_object_fini(struct drm_i915_gem_object *obj) { mutex_destroy(&obj->mm.get_page.lock); mutex_destroy(&obj->mm.get_dma_page.lock); - dma_resv_fini(&obj->base._resv); } /** diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 82415af47d540..2f03f95945f1b 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -958,6 +958,9 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) (struct intel_device_info *) ent->driver_data; int err; + if (!intel_info) + return -ENODEV; + if (intel_info->require_force_probe && !id_forced(pdev->device)) { dev_info(&pdev->dev, "Your graphics device %04x is not properly supported by i915 in this\n" diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index 0c55fb6e9727d..11157fb14db31 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -403,8 +403,10 @@ static int query_perf_config_list(struct drm_i915_private *i915, ids = krealloc(oa_config_ids, n_configs * sizeof(*oa_config_ids), GFP_KERNEL); - if (!ids) + if (!ids) { + kfree(oa_config_ids); return -ENOMEM; + } alloc = fetch_and_zero(&n_configs); diff --git a/drivers/gpu/drm/imagination/pvr_free_list.c b/drivers/gpu/drm/imagination/pvr_free_list.c index e85cac83834c6..faf5e586d8dc1 100644 --- a/drivers/gpu/drm/imagination/pvr_free_list.c +++ b/drivers/gpu/drm/imagination/pvr_free_list.c @@ -8,6 +8,7 @@ #include "pvr_vm.h" #include +#include #include #include #include @@ -612,13 +613,21 @@ pvr_free_list_process_reconstruct_req(struct pvr_device *pvr_dev, }; struct rogue_fwif_freelists_reconstruction_data *resp = &resp_cmd.cmd_data.free_lists_reconstruction_data; + u32 count = min_t(u32, req->freelist_count, + ARRAY_SIZE(req->freelist_ids)); - for (u32 i = 0; i < req->freelist_count; i++) + if (count != req->freelist_count) { + drm_warn_once(from_pvr_device(pvr_dev), + "Requested reconstruction of %u freelists, limiting to %u\n", + req->freelist_count, count); + } + + for (u32 i = 0; i < count; i++) pvr_free_list_reconstruct(pvr_dev, req->freelist_ids[i]); - resp->freelist_count = req->freelist_count; + resp->freelist_count = count; memcpy(resp->freelist_ids, req->freelist_ids, - req->freelist_count * sizeof(resp->freelist_ids[0])); + count * sizeof(resp->freelist_ids[0])); WARN_ON(pvr_kccb_send_cmd(pvr_dev, &resp_cmd, NULL)); } diff --git a/drivers/gpu/drm/imagination/pvr_mmu.c b/drivers/gpu/drm/imagination/pvr_mmu.c index 3cac482e10347..62eae7fcd5a26 100644 --- a/drivers/gpu/drm/imagination/pvr_mmu.c +++ b/drivers/gpu/drm/imagination/pvr_mmu.c @@ -12,6 +12,7 @@ #include "pvr_rogue_mmu_defs.h" #include +#include #include #include #include @@ -2335,6 +2336,7 @@ void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx) * pvr_mmu_op_context_create() - Create an MMU op context. * @ctx: MMU context associated with owning VM context. * @sgt: Scatter gather table containing pages pinned for use by this context. + * @device_addr: Virtual device address at the start of the requested mapping. * @sgt_offset: Start offset of the requested device-virtual memory mapping. * @size: Size in bytes of the requested device-virtual memory mapping. For an * unmapping, this should be zero so that no page tables are allocated. @@ -2346,8 +2348,9 @@ void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx) */ struct pvr_mmu_op_context * pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, struct sg_table *sgt, - u64 sgt_offset, u64 size) + u64 device_addr, u64 sgt_offset, u64 size) { + u64 start_addr = device_addr + sgt_offset; int err; struct pvr_mmu_op_context *op_ctx = kzalloc_obj(*op_ctx); @@ -2363,16 +2366,16 @@ pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, struct sg_table *sgt, if (size) { /* * The number of page table objects we need to prealloc is - * indicated by the mapping size, start offset and the sizes + * indicated by the mapping size, start address and the sizes * of the areas mapped per PT or PD. The range calculation is * identical to that for the index into a table for a device * address, so we reuse those functions here. */ - const u32 l1_start_idx = pvr_page_table_l2_idx(sgt_offset); - const u32 l1_end_idx = pvr_page_table_l2_idx(sgt_offset + size); + const u32 l1_start_idx = pvr_page_table_l2_idx(start_addr); + const u32 l1_end_idx = pvr_page_table_l2_idx(start_addr + size); const u32 l1_count = l1_end_idx - l1_start_idx + 1; - const u32 l0_start_idx = pvr_page_table_l1_idx(sgt_offset); - const u32 l0_end_idx = pvr_page_table_l1_idx(sgt_offset + size); + const u32 l0_start_idx = pvr_page_table_l1_idx(start_addr); + const u32 l0_end_idx = pvr_page_table_l1_idx(start_addr + size); const u32 l0_count = l0_end_idx - l0_start_idx + 1; /* @@ -2553,7 +2556,9 @@ pvr_mmu_map_sgl(struct pvr_mmu_op_context *op_ctx, struct scatterlist *sgl, err_destroy_pages: memcpy(&op_ctx->curr_page, &ptr_copy, sizeof(op_ctx->curr_page)); - err = pvr_mmu_op_context_unmap_curr_page(op_ctx, page); + if (pvr_mmu_op_context_unmap_curr_page(op_ctx, page)) + drm_err(from_pvr_device(op_ctx->mmu_ctx->pvr_dev), + "%s : Failure in unmapping pages\n", __func__); return err; } diff --git a/drivers/gpu/drm/imagination/pvr_mmu.h b/drivers/gpu/drm/imagination/pvr_mmu.h index a8ecd460168dc..2c02d61ba0a29 100644 --- a/drivers/gpu/drm/imagination/pvr_mmu.h +++ b/drivers/gpu/drm/imagination/pvr_mmu.h @@ -99,7 +99,7 @@ dma_addr_t pvr_mmu_get_root_table_dma_addr(struct pvr_mmu_context *ctx); void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx); struct pvr_mmu_op_context * pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, - struct sg_table *sgt, u64 sgt_offset, u64 size); + struct sg_table *sgt, u64 device_addr, u64 sgt_offset, u64 size); int pvr_mmu_map(struct pvr_mmu_op_context *op_ctx, u64 size, u64 flags, u64 device_addr); diff --git a/drivers/gpu/drm/imagination/pvr_vm.c b/drivers/gpu/drm/imagination/pvr_vm.c index ceb78694cd987..55cc999f3708b 100644 --- a/drivers/gpu/drm/imagination/pvr_vm.c +++ b/drivers/gpu/drm/imagination/pvr_vm.c @@ -276,7 +276,7 @@ pvr_vm_bind_op_map_init(struct pvr_vm_bind_op *bind_op, goto err_bind_op_fini; bind_op->mmu_op_ctx = - pvr_mmu_op_context_create(vm_ctx->mmu_ctx, sgt, offset, size); + pvr_mmu_op_context_create(vm_ctx->mmu_ctx, sgt, device_addr, offset, size); err = PTR_ERR_OR_ZERO(bind_op->mmu_op_ctx); if (err) { bind_op->mmu_op_ctx = NULL; @@ -318,7 +318,7 @@ pvr_vm_bind_op_unmap_init(struct pvr_vm_bind_op *bind_op, } bind_op->mmu_op_ctx = - pvr_mmu_op_context_create(vm_ctx->mmu_ctx, NULL, 0, 0); + pvr_mmu_op_context_create(vm_ctx->mmu_ctx, NULL, device_addr, 0, 0); err = PTR_ERR_OR_ZERO(bind_op->mmu_op_ctx); if (err) { bind_op->mmu_op_ctx = NULL; diff --git a/drivers/gpu/drm/lima/lima_device.c b/drivers/gpu/drm/lima/lima_device.c index 0bf7105c8748b..7c873e62c16da 100644 --- a/drivers/gpu/drm/lima/lima_device.c +++ b/drivers/gpu/drm/lima/lima_device.c @@ -368,12 +368,6 @@ int lima_device_init(struct lima_device *ldev) if (err) goto err_out0; - ldev->empty_vm = lima_vm_create(ldev); - if (!ldev->empty_vm) { - err = -ENOMEM; - goto err_out1; - } - ldev->va_start = 0; if (ldev->id == lima_gpu_mali450) { ldev->va_end = LIMA_VA_RESERVE_START; @@ -387,6 +381,12 @@ int lima_device_init(struct lima_device *ldev) } else ldev->va_end = LIMA_VA_RESERVE_END; + ldev->empty_vm = lima_vm_create(ldev); + if (!ldev->empty_vm) { + err = -ENOMEM; + goto err_out1; + } + ldev->iomem = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ldev->iomem)) { dev_err(ldev->dev, "fail to ioremap iomem\n"); diff --git a/drivers/gpu/drm/logicvc/Kconfig b/drivers/gpu/drm/logicvc/Kconfig index 579a358ed5cf3..11aae1626199b 100644 --- a/drivers/gpu/drm/logicvc/Kconfig +++ b/drivers/gpu/drm/logicvc/Kconfig @@ -4,7 +4,6 @@ config DRM_LOGICVC depends on OF || COMPILE_TEST select DRM_CLIENT_SELECTION select DRM_KMS_HELPER - select DRM_KMS_DMA_HELPER select DRM_GEM_DMA_HELPER select REGMAP select REGMAP_MMIO diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c index 2c0bbac43c52b..f1df2514c6132 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c @@ -1275,6 +1275,11 @@ static irqreturn_t a5xx_irq(struct msm_gpu *gpu) status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR); if (priv->disable_err_irq) { + /* Turn off interrupts to avoid interrupt storm */ + gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, + A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | + A5XX_RBBM_INT_0_MASK_CP_SW); + status &= A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | A5XX_RBBM_INT_0_MASK_CP_SW; } diff --git a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c index 3e6f409d13a2a..a98d550b72d0e 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c @@ -1199,7 +1199,7 @@ static const struct adreno_reglist a730_hwcg[] = { { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 }, { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 }, { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 }, - { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 }, + { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22220222 }, { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 }, { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 }, { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 }, diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 2e5d7b53a0c38..a2f6918c4f7f2 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -642,7 +642,7 @@ static void a6xx_rpmh_stop(struct a6xx_gmu *gmu) int ret; u32 val; - if (test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status)) + if (!test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status)) return; if (adreno_is_a840(adreno_gpu)) @@ -1465,6 +1465,9 @@ static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu) /* Stop the interrupts and mask the hardware */ a6xx_gmu_irq_disable(gmu); + /* Halt the gmu cm3 core */ + gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); + /* Tell RPMh to power off the GPU */ a6xx_rpmh_stop(gmu); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index 8b3bb2fd433ba..e293b4ca808a4 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -1911,8 +1911,11 @@ static irqreturn_t a6xx_irq(struct msm_gpu *gpu) gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status); - if (priv->disable_err_irq) + if (priv->disable_err_irq) { + /* Turn off interrupts to avoid interrupt storm */ + gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS); status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS; + } if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT) a6xx_fault_detect_irq(gpu); @@ -2770,8 +2773,9 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) adreno_gpu->ubwc_config = qcom_ubwc_config_get_data(); if (IS_ERR(adreno_gpu->ubwc_config)) { + ret = PTR_ERR(adreno_gpu->ubwc_config); a6xx_destroy(&(a6xx_gpu->base.base)); - return ERR_CAST(adreno_gpu->ubwc_config); + return ERR_PTR(ret); } /* Set up the preemption specific bits and pieces for each ringbuffer */ diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c index 166365359fa6d..3ea8ff8c74044 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c @@ -1244,9 +1244,12 @@ static void a6xx_get_gmu_registers(struct msm_gpu *gpu, _a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gmu_reglist[1], &a6xx_state->gmu_registers[1], true); - if (adreno_is_a621(adreno_gpu) || adreno_is_a623(adreno_gpu)) + if (adreno_is_a621(adreno_gpu)) _a6xx_get_gmu_registers(gpu, a6xx_state, &a621_gpucc_reg, &a6xx_state->gmu_registers[2], false); + else if (adreno_is_a623(adreno_gpu) || adreno_is_a663(adreno_gpu)) + _a6xx_get_gmu_registers(gpu, a6xx_state, &a623_gpucc_reg, + &a6xx_state->gmu_registers[2], false); else _a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gpucc_reg, &a6xx_state->gmu_registers[2], false); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h index b49d8427b59e6..0a13a65f89ac8 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h @@ -377,6 +377,17 @@ static const u32 a6xx_gmu_gpucc_registers[] = { }; static const u32 a621_gmu_gpucc_registers[] = { + /* GPU CC */ + 0x24000, 0x2400e, 0x24400, 0x2440e, 0x24800, 0x24805, 0x24c00, 0x24cff, + 0x25800, 0x25804, 0x25c00, 0x25c04, 0x26000, 0x26004, 0x26400, 0x26405, + 0x26414, 0x2641d, 0x2642a, 0x26430, 0x26432, 0x26432, 0x26441, 0x26455, + 0x26466, 0x26468, 0x26478, 0x2647a, 0x26489, 0x2648a, 0x2649c, 0x2649e, + 0x264a0, 0x264a3, 0x264b3, 0x264b5, 0x264c5, 0x264c7, 0x264d6, 0x264d8, + 0x264e8, 0x264e9, 0x264f9, 0x264fc, 0x2650b, 0x2650c, 0x2651c, 0x2651e, + 0x26540, 0x26570, 0x26600, 0x26616, 0x26620, 0x2662d, +}; + +static const u32 a623_gmu_gpucc_registers[] = { /* GPU CC */ 0x24000, 0x2400e, 0x24400, 0x2440e, 0x25800, 0x25804, 0x25c00, 0x25c04, 0x26000, 0x26004, 0x26400, 0x26405, 0x26414, 0x2641d, 0x2642a, 0x26430, @@ -402,6 +413,7 @@ static const struct a6xx_registers a6xx_gmu_reglist[] = { static const struct a6xx_registers a6xx_gpucc_reg = REGS(a6xx_gmu_gpucc_registers, 0, 0); static const struct a6xx_registers a621_gpucc_reg = REGS(a621_gmu_gpucc_registers, 0, 0); +static const struct a6xx_registers a623_gpucc_reg = REGS(a623_gmu_gpucc_registers, 0, 0); static u32 a6xx_get_cp_roq_size(struct msm_gpu *gpu); static u32 a7xx_get_cp_roq_size(struct msm_gpu *gpu); diff --git a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c index 9e44fd1ae6346..0f6fd35bd5878 100644 --- a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c @@ -1211,8 +1211,11 @@ irqreturn_t a8xx_irq(struct msm_gpu *gpu) gpu_write(gpu, REG_A8XX_RBBM_INT_CLEAR_CMD, status); - if (priv->disable_err_irq) + if (priv->disable_err_irq) { + /* Turn off interrupts to avoid interrupt storm */ + gpu_write(gpu, REG_A8XX_RBBM_INT_0_MASK, A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS); status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS; + } if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT) a8xx_fault_detect_irq(gpu); diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c index 7f20320ef66af..05c77fe27e620 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_device.c +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c @@ -25,7 +25,7 @@ MODULE_PARM_DESC(disable_acd, "Forcefully disable GPU ACD"); module_param_unsafe(disable_acd, bool, 0400); static bool skip_gpu; -MODULE_PARM_DESC(no_gpu, "Disable GPU driver register (0=enable GPU driver register (default), 1=skip GPU driver register"); +MODULE_PARM_DESC(skip_gpu, "Disable GPU driver register (0=enable GPU driver register (default), 1=skip GPU driver register"); module_param(skip_gpu, bool, 0400); extern const struct adreno_gpulist a2xx_gpulist; diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index c62c45bb0ddbb..29c59899fdefb 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -1252,6 +1252,8 @@ void adreno_gpu_cleanup(struct adreno_gpu *adreno_gpu) for (i = 0; i < ARRAY_SIZE(adreno_gpu->info->fw); i++) release_firmware(adreno_gpu->fw[i]); + pm_runtime_dont_use_autosuspend(&gpu->pdev->dev); + if (priv && pm_runtime_enabled(&priv->gpu_pdev->dev)) pm_runtime_disable(&priv->gpu_pdev->dev); diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h index 06da1583fb1eb..85f455a9f29c8 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h @@ -86,42 +86,42 @@ static const struct dpu_sspp_cfg kaanapali_sspp[] = { .name = "sspp_8", .id = SSPP_DMA0, .base = 0x97000, .len = 0x84, .features = DMA_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 1, .type = SSPP_TYPE_DMA, }, { .name = "sspp_9", .id = SSPP_DMA1, .base = 0xa0000, .len = 0x84, .features = DMA_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 5, .type = SSPP_TYPE_DMA, }, { .name = "sspp_10", .id = SSPP_DMA2, .base = 0xa9000, .len = 0x84, .features = DMA_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 9, .type = SSPP_TYPE_DMA, }, { .name = "sspp_11", .id = SSPP_DMA3, .base = 0xb2000, .len = 0x84, .features = DMA_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 13, .type = SSPP_TYPE_DMA, }, { .name = "sspp_12", .id = SSPP_DMA4, .base = 0xbb000, .len = 0x84, .features = DMA_CURSOR_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 14, .type = SSPP_TYPE_DMA, }, { .name = "sspp_13", .id = SSPP_DMA5, .base = 0xc4000, .len = 0x84, .features = DMA_CURSOR_SDM845_MASK_SDMA, - .sblk = &dpu_dma_sblk, + .sblk = &dpu_dma_sblk_v13, .xin_id = 15, .type = SSPP_TYPE_DMA, }, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c index 13cc658065c56..6524531bd8bdc 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c @@ -394,6 +394,10 @@ int dpu_core_perf_crtc_update(struct drm_crtc *crtc, trace_dpu_core_perf_update_clk(kms->dev, !crtc->enabled, clk_rate); + /* If we're going offline, PM callbacks will disable the clocks instead */ + if (!clk_rate) + return 0; + clk_rate = min(clk_rate, kms->perf.max_core_clk_rate); ret = dev_pm_opp_set_rate(&kms->pdev->dev, clk_rate); if (ret) { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index 778e231d49677..1f20695f81e35 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -710,8 +710,7 @@ void dpu_encoder_update_topology(struct drm_encoder *drm_enc, if (fb && MSM_FORMAT_IS_YUV(msm_framebuffer_format(fb))) topology->num_cdm++; } else if (disp_info->intf_type == INTF_DP) { - if (msm_dp_is_yuv_420_enabled(priv->kms->dp[disp_info->h_tile_instance[0]], - adj_mode)) + if (drm_mode_is_420_only(&connector->display_info, adj_mode)) topology->num_cdm++; } } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c index 2e10add84fd71..9a993cdfab856 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c @@ -303,6 +303,16 @@ static const u32 wb2_formats_rgb_yuv[] = { .num_formats = ARRAY_SIZE(plane_formats), \ } +#define _DMA_SBLK_V13() \ + { \ + .sspp_rec0_blk = {.name = "sspp_rec0", \ + .base = 0x1000, .len = 0x180,}, \ + .sspp_rec1_blk = {.name = "sspp_rec1", \ + .base = 0x3000, .len = 0x180,}, \ + .format_list = plane_formats, \ + .num_formats = ARRAY_SIZE(plane_formats), \ + } + static const struct dpu_rotation_cfg dpu_rot_sc7280_cfg_v2 = { .rot_maxheight = 1088, .rot_num_formats = ARRAY_SIZE(rotation_v2_formats), @@ -353,6 +363,8 @@ static const struct dpu_sspp_sub_blks dpu_rgb_sblk = _RGB_SBLK(); static const struct dpu_sspp_sub_blks dpu_dma_sblk = _DMA_SBLK(); +static const struct dpu_sspp_sub_blks dpu_dma_sblk_v13 = _DMA_SBLK_V13(); + /************************************************************* * MIXER sub blocks config *************************************************************/ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c index 36a497f1d6c12..e8729fe1cfefa 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c @@ -118,6 +118,7 @@ static inline void dpu_hw_ctl_clear_pending_flush(struct dpu_hw_ctl *ctx) ctx->pending_intf_flush_mask = 0; ctx->pending_wb_flush_mask = 0; ctx->pending_cwb_flush_mask = 0; + ctx->pending_periph_flush_mask = 0; ctx->pending_merge_3d_flush_mask = 0; ctx->pending_dsc_flush_mask = 0; ctx->pending_cdm_flush_mask = 0; diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c index 7726edb0d4ede..6ae49f94fea7b 100644 --- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c @@ -398,7 +398,7 @@ static int mdp4_kms_init(struct drm_device *dev) ret = mdp_kms_init(&mdp4_kms->base, &kms_funcs); if (ret) { DRM_DEV_ERROR(dev->dev, "failed to init kms\n"); - goto fail; + return ret; } kms = priv->kms; @@ -409,7 +409,7 @@ static int mdp4_kms_init(struct drm_device *dev) ret = regulator_enable(mdp4_kms->vdd); if (ret) { DRM_DEV_ERROR(dev->dev, "failed to enable regulator vdd: %d\n", ret); - goto fail; + return ret; } } @@ -421,7 +421,7 @@ static int mdp4_kms_init(struct drm_device *dev) DRM_DEV_ERROR(dev->dev, "unexpected MDP version: v%d.%d\n", major, minor); ret = -ENXIO; - goto fail; + return ret; } mdp4_kms->rev = minor; @@ -430,7 +430,7 @@ static int mdp4_kms_init(struct drm_device *dev) if (!mdp4_kms->lut_clk) { DRM_DEV_ERROR(dev->dev, "failed to get lut_clk\n"); ret = -ENODEV; - goto fail; + return ret; } clk_set_rate(mdp4_kms->lut_clk, max_clk); } @@ -452,7 +452,7 @@ static int mdp4_kms_init(struct drm_device *dev) vm = msm_kms_init_vm(mdp4_kms->dev, NULL); if (IS_ERR(vm)) { ret = PTR_ERR(vm); - goto fail; + return ret; } kms->vm = vm; @@ -460,7 +460,7 @@ static int mdp4_kms_init(struct drm_device *dev) ret = modeset_init(mdp4_kms); if (ret) { DRM_DEV_ERROR(dev->dev, "modeset_init failed: %d\n", ret); - goto fail; + return ret; } mdp4_kms->blank_cursor_bo = msm_gem_new(dev, SZ_16K, MSM_BO_WC | MSM_BO_SCANOUT); @@ -468,14 +468,14 @@ static int mdp4_kms_init(struct drm_device *dev) ret = PTR_ERR(mdp4_kms->blank_cursor_bo); DRM_DEV_ERROR(dev->dev, "could not allocate blank-cursor bo: %d\n", ret); mdp4_kms->blank_cursor_bo = NULL; - goto fail; + return ret; } ret = msm_gem_get_and_pin_iova(mdp4_kms->blank_cursor_bo, kms->vm, &mdp4_kms->blank_cursor_iova); if (ret) { DRM_DEV_ERROR(dev->dev, "could not pin blank-cursor bo: %d\n", ret); - goto fail; + return ret; } dev->mode_config.min_width = 0; @@ -484,12 +484,6 @@ static int mdp4_kms_init(struct drm_device *dev) dev->mode_config.max_height = 2048; return 0; - -fail: - if (kms) - mdp4_destroy(kms); - - return ret; } static const struct dev_pm_ops mdp4_pm_ops = { diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c index 0a004ab9fc856..3934cd060b276 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c @@ -517,7 +517,7 @@ static int mdp5_kms_init(struct drm_device *dev) ret = mdp_kms_init(&mdp5_kms->base, &kms_funcs); if (ret) { DRM_DEV_ERROR(&pdev->dev, "failed to init kms\n"); - goto fail; + return ret; } config = mdp5_cfg_get_config(mdp5_kms->cfg); @@ -540,7 +540,7 @@ static int mdp5_kms_init(struct drm_device *dev) vm = msm_kms_init_vm(mdp5_kms->dev, pdev->dev.parent); if (IS_ERR(vm)) { ret = PTR_ERR(vm); - goto fail; + return ret; } kms->vm = vm; @@ -550,7 +550,7 @@ static int mdp5_kms_init(struct drm_device *dev) ret = modeset_init(mdp5_kms); if (ret) { DRM_DEV_ERROR(&pdev->dev, "modeset_init failed: %d\n", ret); - goto fail; + return ret; } dev->mode_config.min_width = 0; @@ -562,11 +562,6 @@ static int mdp5_kms_init(struct drm_device *dev) dev->vblank_disable_immediate = true; return 0; -fail: - if (kms) - mdp5_kms_destroy(kms); - - return ret; } static void mdp5_destroy(struct mdp5_kms *mdp5_kms) diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index 86ef8c89ad443..9024abdc2c569 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -1950,13 +1950,12 @@ static int msm_dp_ctrl_reinitialize_mainlink(struct msm_dp_ctrl_private *ctrl) msm_dp_ctrl_mainlink_disable(ctrl); ctrl->phy_opts.dp.lanes = ctrl->link->link_params.num_lanes; phy_configure(phy, &ctrl->phy_opts); + /* * Disable and re-enable the mainlink clock since the * link clock might have been adjusted as part of the * link maintenance. */ - dev_pm_opp_set_rate(ctrl->dev, 0); - msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); phy_power_off(phy); @@ -1982,7 +1981,6 @@ static int msm_dp_ctrl_deinitialize_mainlink(struct msm_dp_ctrl_private *ctrl) msm_dp_ctrl_reset(&ctrl->msm_dp_ctrl); - dev_pm_opp_set_rate(ctrl->dev, 0); msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); phy_power_off(phy); @@ -2602,7 +2600,6 @@ void msm_dp_ctrl_off(struct msm_dp_ctrl *msm_dp_ctrl) ctrl->stream_clks_on = false; } - dev_pm_opp_set_rate(ctrl->dev, 0); msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); phy_power_off(phy); diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index dc6f33809ca5f..22eade11c6150 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -698,6 +698,8 @@ enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *bridge, u32 mode_rate_khz = 0, supported_rate_khz = 0, mode_bpp = 0; struct msm_dp *dp; int mode_pclk_khz = mode->clock; + int link_pclk_khz; + bool is_yuv_420; dp = to_dp_bridge(bridge)->msm_dp_display; @@ -709,9 +711,18 @@ enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *bridge, msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display); link_info = &msm_dp_display->panel->link_info; - if ((drm_mode_is_420_only(&dp->connector->display_info, mode) && - msm_dp_display->panel->vsc_sdp_supported) || - msm_dp_wide_bus_available(dp)) + is_yuv_420 = drm_mode_is_420_only(&dp->connector->display_info, mode); + + /* + * YUV 420 is carried over DP by signalling the colorimetry through a + * VSC SDP, so a 420-only mode cannot be driven without VSC SDP support. + */ + if (is_yuv_420 && !msm_dp_display->panel->vsc_sdp_supported) + return MODE_NO_420; + + link_pclk_khz = is_yuv_420 ? mode_pclk_khz / 2 : mode_pclk_khz; + + if (is_yuv_420 || msm_dp_display->wide_bus_supported) mode_pclk_khz /= 2; if (mode_pclk_khz > DP_MAX_PIXEL_CLK_KHZ) @@ -722,9 +733,9 @@ enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *bridge, mode_bpp = default_bpp; mode_bpp = msm_dp_panel_get_mode_bpp(msm_dp_display->panel, - mode_bpp, mode_pclk_khz); + mode_bpp, link_pclk_khz); - mode_rate_khz = mode_pclk_khz * mode_bpp; + mode_rate_khz = link_pclk_khz * mode_bpp; supported_rate_khz = link_info->num_lanes * link_info->rate * 8; if (mode_rate_khz > supported_rate_khz) @@ -1277,22 +1288,10 @@ void __exit msm_dp_unregister(void) platform_driver_unregister(&msm_dp_display_driver); } -bool msm_dp_is_yuv_420_enabled(const struct msm_dp *msm_dp_display, - const struct drm_display_mode *mode) -{ - struct msm_dp_display_private *dp; - const struct drm_display_info *info; - - dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - info = &msm_dp_display->connector->display_info; - - return dp->panel->vsc_sdp_supported && drm_mode_is_420_only(info, mode); -} - bool msm_dp_needs_periph_flush(const struct msm_dp *msm_dp_display, const struct drm_display_mode *mode) { - return msm_dp_is_yuv_420_enabled(msm_dp_display, mode); + return drm_mode_is_420_only(&msm_dp_display->connector->display_info, mode); } bool msm_dp_wide_bus_available(const struct msm_dp *msm_dp_display) @@ -1416,6 +1415,20 @@ void msm_dp_bridge_atomic_disable(struct drm_bridge *drm_bridge, msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display); + /* + * If .atomic_enable() bailed out - link training failure is the common + * case - the mainlink was never brought up and ->power_on stayed false. + * Driving the PUSH_IDLE pattern into a controller that was never + * enabled times out, and .atomic_post_disable() then drops the + * controller's runtime-PM reference without tearing the PHY back down, + * because msm_dp_display_disable() returns early on !power_on. On + * glymur (Snapdragon X2 Elite) that combination is answered by a + * TrustZone-level SOCCP/ADSP force-stop and a silent SoC reset. + * There is nothing to push idle, so leave it alone. + */ + if (!dp->power_on) + return; + msm_dp_ctrl_push_idle(msm_dp_display->ctrl); } diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c index 6bb021820d7c5..5b4954e7cb748 100644 --- a/drivers/gpu/drm/msm/dp/dp_panel.c +++ b/drivers/gpu/drm/msm/dp/dp_panel.c @@ -332,8 +332,10 @@ int msm_dp_panel_get_modes(struct msm_dp_panel *msm_dp_panel, return -EINVAL; } - if (msm_dp_panel->drm_edid) + if (msm_dp_panel->drm_edid) { + drm_edid_connector_update(connector, msm_dp_panel->drm_edid); return drm_edid_connector_add_modes(connector); + } return 0; } diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c index 982abaaac00d5..89face2670187 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_host.c +++ b/drivers/gpu/drm/msm/dsi/dsi_host.c @@ -129,7 +129,7 @@ struct msm_dsi_host { struct clk *dsi_pll_pixel_clk; unsigned long byte_clk_rate; - unsigned long byte_intf_clk_rate; + bool byte_intf_clk_div_2; unsigned long pixel_clk_rate; unsigned long esc_clk_rate; @@ -381,8 +381,20 @@ int msm_dsi_runtime_resume(struct device *dev) int dsi_link_clk_set_rate_6g(struct msm_dsi_host *msm_host) { + unsigned long byte_intf_clk_rate; + long rounded_byte_clk_rate; int ret; + rounded_byte_clk_rate = clk_round_rate(msm_host->byte_clk, + msm_host->byte_clk_rate); + if (rounded_byte_clk_rate < 0) { + pr_err("%s: failed to round byte clock rate, %ld\n", + __func__, rounded_byte_clk_rate); + return rounded_byte_clk_rate; + } + + msm_host->byte_clk_rate = rounded_byte_clk_rate; + DBG("Set clk rates: pclk=%lu, byteclk=%lu", msm_host->pixel_clk_rate, msm_host->byte_clk_rate); @@ -400,7 +412,11 @@ int dsi_link_clk_set_rate_6g(struct msm_dsi_host *msm_host) } if (msm_host->byte_intf_clk) { - ret = clk_set_rate(msm_host->byte_intf_clk, msm_host->byte_intf_clk_rate); + byte_intf_clk_rate = msm_host->byte_clk_rate; + if (msm_host->byte_intf_clk_div_2) + byte_intf_clk_rate /= 2; + + ret = clk_set_rate(msm_host->byte_intf_clk, byte_intf_clk_rate); if (ret) { pr_err("%s: Failed to set rate byte intf clk, %d\n", __func__, ret); @@ -549,8 +565,6 @@ int dsi_link_clk_enable_v2(struct msm_dsi_host *msm_host) void dsi_link_clk_disable_6g(struct msm_dsi_host *msm_host) { - /* Drop the performance state vote */ - dev_pm_opp_set_rate(&msm_host->pdev->dev, 0); clk_disable_unprepare(msm_host->esc_clk); clk_disable_unprepare(msm_host->pixel_clk); clk_disable_unprepare(msm_host->byte_intf_clk); @@ -2490,9 +2504,7 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host, goto unlock_ret; } - msm_host->byte_intf_clk_rate = msm_host->byte_clk_rate; - if (phy_shared_timings->byte_intf_clk_div_2) - msm_host->byte_intf_clk_rate /= 2; + msm_host->byte_intf_clk_div_2 = phy_shared_timings->byte_intf_clk_div_2; msm_dsi_sfpb_config(msm_host, true); diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h index 21a59d66e8dc5..f5d3e806f8fd5 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h @@ -111,6 +111,7 @@ struct msm_dsi_phy { struct msm_dsi_dphy_timing timing; const struct msm_dsi_phy_cfg *cfg; void *tuning_cfg; + void *pll_data; enum msm_dsi_phy_usecase usecase; bool regulator_ldo_mode; diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c index 984a66085dfbf..5d805a797abdb 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c @@ -426,8 +426,11 @@ static void dsi_pll_enable_pll_bias(struct dsi_pll_7nm *pll) u32 data; spin_lock_irqsave(&pll->pll_enable_lock, flags); - pll->pll_enable_cnt++; - WARN_ON(pll->pll_enable_cnt == INT_MAX); + if (pll->pll_enable_cnt++) { + spin_unlock_irqrestore(&pll->pll_enable_lock, flags); + WARN_ON(pll->pll_enable_cnt == INT_MAX); + return; + } data = readl(pll->phy->base + REG_DSI_7nm_PHY_CMN_CTRL_0); data |= DSI_7nm_PHY_CMN_CTRL_0_PLL_SHUTDOWNB; @@ -873,6 +876,7 @@ static int dsi_pll_7nm_init(struct msm_dsi_phy *phy) spin_lock_init(&pll_7nm->pll_enable_lock); pll_7nm->phy = phy; + phy->pll_data = pll_7nm; ret = pll_7nm_register(pll_7nm, phy->provided_clocks->hws); if (ret) { @@ -961,8 +965,10 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, u32 const delay_us = 5; u32 const timeout_us = 1000; struct msm_dsi_dphy_timing *timing = &phy->timing; + struct dsi_pll_7nm *pll = phy->pll_data; void __iomem *base = phy->base; bool less_than_1500_mhz; + unsigned long flags; u32 vreg_ctrl_0, vreg_ctrl_1, lane_ctrl0; u32 glbl_pemph_ctrl_0; u32 glbl_str_swi_cal_sel_ctrl, glbl_hstx_str_ctrl_0; @@ -1084,10 +1090,13 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, glbl_rescode_bot_ctrl = 0x3c; } + spin_lock_irqsave(&pll->pll_enable_lock, flags); + pll->pll_enable_cnt = 1; /* de-assert digital and pll power down */ data = DSI_7nm_PHY_CMN_CTRL_0_DIGTOP_PWRDN_B | DSI_7nm_PHY_CMN_CTRL_0_PLL_SHUTDOWNB; writel(data, base + REG_DSI_7nm_PHY_CMN_CTRL_0); + spin_unlock_irqrestore(&pll->pll_enable_lock, flags); /* Assert PLL core reset */ writel(0x00, base + REG_DSI_7nm_PHY_CMN_PLL_CNTRL); @@ -1200,7 +1209,9 @@ static bool dsi_7nm_set_continuous_clock(struct msm_dsi_phy *phy, bool enable) static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy) { + struct dsi_pll_7nm *pll = phy->pll_data; void __iomem *base = phy->base; + unsigned long flags; u32 data; DBG(""); @@ -1227,8 +1238,11 @@ static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy) writel(data, base + REG_DSI_7nm_PHY_CMN_CTRL_0); writel(0, base + REG_DSI_7nm_PHY_CMN_LANE_CTRL0); + spin_lock_irqsave(&pll->pll_enable_lock, flags); + pll->pll_enable_cnt = 0; /* Turn off all PHY blocks */ writel(0x00, base + REG_DSI_7nm_PHY_CMN_CTRL_0); + spin_unlock_irqrestore(&pll->pll_enable_lock, flags); /* make sure phy is turned off */ wmb(); diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_phy.c b/drivers/gpu/drm/msm/hdmi/hdmi_phy.c index eb1088755cb3a..77dce35cd45e0 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi_phy.c +++ b/drivers/gpu/drm/msm/hdmi/hdmi_phy.c @@ -168,13 +168,13 @@ static int msm_hdmi_phy_probe(struct platform_device *pdev) ret = msm_hdmi_phy_resource_enable(phy); if (ret) - return ret; + goto err_pm_disable; ret = msm_hdmi_phy_pll_init(pdev, phy->cfg->type); if (ret) { DRM_DEV_ERROR(dev, "couldn't init PLL\n"); msm_hdmi_phy_resource_disable(phy); - return ret; + goto err_pm_disable; } msm_hdmi_phy_resource_disable(phy); @@ -182,6 +182,10 @@ static int msm_hdmi_phy_probe(struct platform_device *pdev) platform_set_drvdata(pdev, phy); return 0; + +err_pm_disable: + pm_runtime_disable(dev); + return ret; } static void msm_hdmi_phy_remove(struct platform_device *pdev) diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 32d5ebea2596f..fa46ab3063e6f 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -55,7 +55,7 @@ MODULE_PARM_DESC(modeset, "Use kernel modesetting [KMS] (1=on (default), 0=disab module_param(modeset, bool, 0600); static bool separate_gpu_kms; -MODULE_PARM_DESC(separate_gpu_drm, "Use separate DRM device for the GPU (0=single DRM device for both GPU and display (default), 1=two DRM devices)"); +MODULE_PARM_DESC(separate_gpu_kms, "Use separate DRM device for the GPU (0=single DRM device for both GPU and display (default), 1=two DRM devices)"); module_param(separate_gpu_kms, bool, 0400); DECLARE_FAULT_ATTR(fail_gem_alloc); diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 3787db8770ada..3d5679be488e7 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -356,8 +356,6 @@ void __exit msm_dp_unregister(void); int msm_dp_modeset_init(struct msm_dp *dp_display, struct drm_device *dev, struct drm_encoder *encoder, bool yuv_supported); void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm_dp *dp_display); -bool msm_dp_is_yuv_420_enabled(const struct msm_dp *dp_display, - const struct drm_display_mode *mode); bool msm_dp_needs_periph_flush(const struct msm_dp *dp_display, const struct drm_display_mode *mode); bool msm_dp_wide_bus_available(const struct msm_dp *dp_display); @@ -382,12 +380,6 @@ static inline void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm { } -static inline bool msm_dp_is_yuv_420_enabled(const struct msm_dp *dp_display, - const struct drm_display_mode *mode) -{ - return false; -} - static inline bool msm_dp_needs_periph_flush(const struct msm_dp *dp_display, const struct drm_display_mode *mode) { diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h index 762e546d25ef3..43a938e8f2880 100644 --- a/drivers/gpu/drm/msm/msm_gem.h +++ b/drivers/gpu/drm/msm/msm_gem.h @@ -68,6 +68,9 @@ struct msm_gem_vm { /** @base: Inherit from drm_gpuvm. */ struct drm_gpuvm base; + /** @rcu: RCU-delayed free so an exported sched fence->sched stays valid. */ + struct rcu_head rcu; + /** * @sched: Scheduler used for asynchronous VM_BIND request. * diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c index c4cfe036066b7..2e16f3ebcf3fd 100644 --- a/drivers/gpu/drm/msm/msm_gem_vma.c +++ b/drivers/gpu/drm/msm/msm_gem_vma.c @@ -166,7 +166,7 @@ msm_gem_vm_free(struct drm_gpuvm *gpuvm) dma_fence_put(vm->last_fence); put_pid(vm->pid); kfree(vm->log); - kfree(vm); + kfree_rcu(vm, rcu); } /** @@ -458,6 +458,8 @@ msm_gem_vm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec) return ret; } + drm_gpuvm_bo_evict(vm_bo, false); + return 0; } diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index 18ed00e5f143b..03c0578560658 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -505,6 +505,8 @@ static void recover_worker(struct kthread_work *work) */ if (!vm->managed) msm_gem_vm_unusable(submit->vm); + + put_task_struct(task); } noreclaim_flag = memalloc_noreclaim_save(); @@ -552,11 +554,13 @@ static void recover_worker(struct kthread_work *work) msm_update_fence(ring->fctx, fence); } - /* retire completed submits, plus the one that hung: */ - retire_submits(gpu); + priv->disable_err_irq = false; gpu->funcs->recover(gpu); + /* retire completed submits, plus the one that hung: */ + retire_submits(gpu); + /* * Replay all remaining submits starting with highest priority * ring diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c index 2d6b930b766ec..38e1e6866301e 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.c +++ b/drivers/gpu/drm/msm/msm_ringbuffer.c @@ -109,9 +109,9 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id, ring->memptrs_iova = memptrs_iova; ret = drm_sched_init(&ring->sched, &args); - if (ret) { + if (ret) goto fail; - } + ring->sched_initialized = true; INIT_LIST_HEAD(&ring->submits); spin_lock_init(&ring->submit_lock); @@ -133,11 +133,12 @@ void msm_ringbuffer_destroy(struct msm_ringbuffer *ring) if (IS_ERR_OR_NULL(ring)) return; - drm_sched_fini(&ring->sched); + if (ring->sched_initialized) + drm_sched_fini(&ring->sched); msm_fence_context_free(ring->fctx); msm_gem_kernel_put(ring->bo, ring->gpu->vm); - kfree(ring); + kfree_rcu(ring, rcu); } diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.h b/drivers/gpu/drm/msm/msm_ringbuffer.h index 28ca8c9f7463d..0bfb6b0f42faa 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.h +++ b/drivers/gpu/drm/msm/msm_ringbuffer.h @@ -55,7 +55,9 @@ struct msm_ringbuffer { /* * The job scheduler for this ring. */ + struct rcu_head rcu; struct drm_gpu_scheduler sched; + bool sched_initialized; /* * List of in-flight submits on this ring. Protected by submit_lock. diff --git a/drivers/gpu/drm/mxsfb/lcdif_kms.c b/drivers/gpu/drm/mxsfb/lcdif_kms.c index ade76c3f4e4e9..68b1eb7a8ff13 100644 --- a/drivers/gpu/drm/mxsfb/lcdif_kms.c +++ b/drivers/gpu/drm/mxsfb/lcdif_kms.c @@ -374,14 +374,23 @@ static void lcdif_disable_controller(struct lcdif_drm_private *lcdif) int ret; reg = readl(lcdif->base + LCDC_V8_CTRLDESCL0_5); + /* Disable the layer for DMA. */ reg &= ~CTRLDESCL0_5_EN; + /* + * It is necessary to wait for the full frame to finish streaming + * through the DMA engine before we can safely disable it by removing + * the DISP_PARA_DISP_ON bit. Disabling it in-flight can leave the + * hardware confused and unable to resume streaming for the next frame. + */ + reg |= CTRLDESCL0_5_SHADOW_LOAD_EN; writel(reg, lcdif->base + LCDC_V8_CTRLDESCL0_5); + /* Wait for the frame to finish or timeout after 50 ms. */ ret = readl_poll_timeout(lcdif->base + LCDC_V8_CTRLDESCL0_5, - reg, !(reg & CTRLDESCL0_5_EN), - 0, 36000); /* Wait ~2 frame times max */ + reg, !(reg & CTRLDESCL0_5_SHADOW_LOAD_EN), + 200, 50000); if (ret) - drm_err(lcdif->drm, "Failed to disable controller!\n"); + drm_err(lcdif->drm, "Timed out waiting for final vblank!\n"); reg = readl(lcdif->base + LCDC_V8_DISP_PARA); reg &= ~DISP_PARA_DISP_ON; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h index 7903d7470d194..01145db32c534 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h @@ -87,4 +87,5 @@ int gp102_disp_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct int gv100_disp_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_disp **); int tu102_disp_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_disp **); int ga102_disp_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_disp **); +int gb202_disp_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_disp **); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 0e8de6d4b36f7..6dcb92575eb46 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -578,8 +578,9 @@ int nouveau_bo_pin_locked(struct nouveau_bo *nvbo, uint32_t domain, bool contig) "0x%08x vs 0x%08x\n", bo, bo->resource->mem_type, domain); ret = -EBUSY; + } else { + ttm_bo_pin(&nvbo->bo); } - ttm_bo_pin(&nvbo->bo); goto out; } diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index 598513f604493..f142f63105966 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -90,6 +90,14 @@ nouveau_channel_del(struct nouveau_channel **pchan) { struct nouveau_channel *chan = *pchan; if (chan) { + /* + * Drop the kill-event subscription first. Its handler + * dereferences chan->fence, which the fence context teardown + * below frees, so leaving it armed across the teardown leaves + * a window for a use-after-free. + */ + nvif_event_dtor(&chan->kill); + if (chan->fence) nouveau_fence(chan->cli->drm)->context_del(chan); @@ -100,7 +108,6 @@ nouveau_channel_del(struct nouveau_channel **pchan) nvif_object_dtor(&chan->nvsw); nvif_object_dtor(&chan->gart); nvif_object_dtor(&chan->vram); - nvif_event_dtor(&chan->kill); nvif_object_dtor(&chan->user); nvif_mem_dtor(&chan->mem_userd); nouveau_vma_del(&chan->sema.vma); diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c index b0b0ad9a0c248..4cfc9c7c2ae0c 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.c +++ b/drivers/gpu/drm/nouveau/nouveau_connector.c @@ -600,8 +600,11 @@ nouveau_connector_detect(struct drm_connector *connector, bool force) new_edid = drm_get_edid(connector, nv_encoder->i2c); } else { ret = nvif_outp_edid_get(&nv_encoder->outp, (u8 **)&new_edid); - if (ret < 0) + if (ret < 0) { + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); return connector_status_disconnected; + } } nouveau_connector_set_edid(nv_connector, new_edid); diff --git a/drivers/gpu/drm/nouveau/nouveau_dmem.c b/drivers/gpu/drm/nouveau/nouveau_dmem.c index 9442ec6e1f6c1..e74d7bb975a86 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dmem.c +++ b/drivers/gpu/drm/nouveau/nouveau_dmem.c @@ -267,7 +267,7 @@ static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) nouveau_fence_new(&fence, dmem->migrate.chan); migrate_vma_pages(&args); nouveau_dmem_fence_done(&fence); - dma_unmap_page(drm->dev->dev, dma_info.dma_addr, PAGE_SIZE, + dma_unmap_page(drm->dev->dev, dma_info.dma_addr, dma_info.size, DMA_BIDIRECTIONAL); done: migrate_vma_finalize(&args); @@ -279,11 +279,25 @@ static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) static void nouveau_dmem_folio_split(struct folio *head, struct folio *tail) { + struct nouveau_dmem_chunk *chunk; + struct nouveau_dmem *dmem; + if (tail == NULL) return; tail->pgmap = head->pgmap; tail->mapping = head->mapping; folio_set_zone_device_data(tail, folio_zone_device_data(head)); + + /* + * The split hands out a new independently-freeable folio that will + * later be released via nouveau_dmem_folio_free(); account for it so + * chunk->callocated stays balanced. + */ + chunk = nouveau_page_to_chunk(&head->page); + dmem = chunk->drm->dmem; + spin_lock(&dmem->lock); + chunk->callocated++; + spin_unlock(&dmem->lock); } static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = { @@ -325,8 +339,8 @@ nouveau_dmem_chunk_alloc(struct nouveau_drm *drm, struct page **ppage, chunk->pagemap.ops = &nouveau_dmem_pagemap_ops; chunk->pagemap.owner = drm->dev; - ret = nouveau_bo_new_pin(&drm->client, NOUVEAU_GEM_DOMAIN_VRAM, DMEM_CHUNK_SIZE, - &chunk->bo); + ret = nouveau_bo_new_pin(&drm->client, NOUVEAU_GEM_DOMAIN_VRAM, + DMEM_CHUNK_SIZE * NR_CHUNKS, &chunk->bo); if (ret) goto out_release; @@ -772,7 +786,7 @@ static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm, return mpfn; out_dma_unmap: - dma_unmap_page(dev, dma_info->dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + dma_unmap_page(dev, dma_info->dma_addr, dma_info->size, DMA_BIDIRECTIONAL); out_free_page: nouveau_dmem_page_free_locked(drm, dpage); out: diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index e16f59b00f6fa..c03638ebd352b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -585,6 +585,7 @@ nouveau_drm_device_fini(struct nouveau_drm *drm) if (nouveau_pmops_runtime()) { pm_runtime_get_sync(dev->dev); pm_runtime_forbid(dev->dev); + pm_runtime_dont_use_autosuspend(dev->dev); } nouveau_led_fini(dev); @@ -1251,10 +1252,8 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv) mutex_unlock(&drm->clients_lock); done: - if (ret && cli) { - nouveau_cli_fini(cli); + if (ret && cli) kfree(cli); - } pm_runtime_mark_last_busy(dev->dev); pm_runtime_put_autosuspend(dev->dev); diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 20dba02d6175b..1958bd52dbbc4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -313,11 +313,20 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, rep->offset = nvbo->offset; if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50 && !nouveau_cli_uvmm(cli)) { + int ret; + + ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); + if (ret) + return ret; + vma = nouveau_vma_find(nvbo, vmm); - if (!vma) + if (!vma) { + ttm_bo_unreserve(&nvbo->bo); return -EINVAL; + } rep->offset = vma->addr; + ttm_bo_unreserve(&nvbo->bo); } else rep->offset = 0; @@ -513,6 +522,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, if (unlikely(ret)) { if (ret != -ERESTARTSYS) NV_PRINTK(err, cli, "fail reserve\n"); + drm_gem_object_put(gem); break; } } @@ -522,6 +532,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); if (!vma) { NV_PRINTK(err, cli, "vma not found!\n"); + drm_gem_object_put(gem); ret = -EINVAL; break; } diff --git a/drivers/gpu/drm/nouveau/nouveau_sched.c b/drivers/gpu/drm/nouveau/nouveau_sched.c index 8b9f935afe094..b3f02c490ecb6 100644 --- a/drivers/gpu/drm/nouveau/nouveau_sched.c +++ b/drivers/gpu/drm/nouveau/nouveau_sched.c @@ -517,7 +517,7 @@ nouveau_sched_destroy(struct nouveau_sched **psched) struct nouveau_sched *sched = *psched; nouveau_sched_fini(sched); - kfree(sched); + kfree_rcu(sched, rcu); *psched = NULL; } diff --git a/drivers/gpu/drm/nouveau/nouveau_sched.h b/drivers/gpu/drm/nouveau/nouveau_sched.h index 20cd1da8db73c..51ce8dcf62858 100644 --- a/drivers/gpu/drm/nouveau/nouveau_sched.h +++ b/drivers/gpu/drm/nouveau/nouveau_sched.h @@ -98,6 +98,7 @@ void nouveau_job_free(struct nouveau_job *job); struct nouveau_sched { struct drm_gpu_scheduler base; + struct rcu_head rcu; struct drm_sched_entity entity; struct workqueue_struct *wq; struct mutex mutex; diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c index fa3b4ebf38a83..2bd0376193aee 100644 --- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c +++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c @@ -72,9 +72,7 @@ nouveau_sgdma_create_ttm(struct ttm_buffer_object *bo, uint32_t page_flags) struct nouveau_sgdma_be *nvbe; enum ttm_caching caching; - if (nvbo->force_coherent) - caching = ttm_uncached; - else if (drm->agp.bridge) + if (nvbo->force_coherent || drm->agp.bridge) caching = ttm_write_combined; else caching = ttm_cached; diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.c b/drivers/gpu/drm/nouveau/nouveau_uvmm.c index f5e4756b4de4a..2026fe6b48c69 100644 --- a/drivers/gpu/drm/nouveau/nouveau_uvmm.c +++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.c @@ -846,6 +846,9 @@ op_map(struct nouveau_uvma *uvma) { struct nouveau_bo *nvbo = nouveau_gem_object(uvma->va.gem.obj); + if (drm_gpuva_invalidated(&uvma->va)) + return; + nouveau_uvma_map(uvma, nouveau_mem(nvbo->bo.resource)); } @@ -1232,6 +1235,7 @@ bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec, drm_gpuva_for_each_op(va_op, op->ops) { struct drm_gem_object *obj = op_gem_obj(va_op); + struct nouveau_bo *nvbo; if (unlikely(!obj)) continue; @@ -1246,8 +1250,13 @@ bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec, if (va_op->op == DRM_GPUVA_OP_UNMAP) continue; - ret = nouveau_bo_validate(nouveau_gem_object(obj), - true, false); + nvbo = nouveau_gem_object(obj); + if (!(nvbo->valid_domains & + (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) + return -EINVAL; + + nouveau_bo_placement_set(nvbo, nvbo->valid_domains, 0); + ret = nouveau_bo_validate(nvbo, true, false); if (ret) return ret; } @@ -1319,6 +1328,7 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job, op->va.range); if (!op->reg || op->reg->dirty) { ret = -ENOENT; + op->reg = NULL; goto unwind_continue; } @@ -1327,6 +1337,7 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job, op->va.range); if (IS_ERR(op->ops)) { ret = PTR_ERR(op->ops); + op->reg = NULL; goto unwind_continue; } @@ -1473,6 +1484,7 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job, op->va.range); break; case OP_UNMAP_SPARSE: + op->reg->dirty = false; __nouveau_uvma_region_insert(uvmm, op->reg); nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new, op->ops); @@ -1489,7 +1501,8 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job, break; } - drm_gpuva_ops_free(&uvmm->base, op->ops); + if (!IS_ERR_OR_NULL(op->ops)) + drm_gpuva_ops_free(&uvmm->base, op->ops); op->ops = NULL; op->reg = NULL; } diff --git a/drivers/gpu/drm/nouveau/nvif/vmm.c b/drivers/gpu/drm/nouveau/nvif/vmm.c index 65c3e883b1193..579af70766f2d 100644 --- a/drivers/gpu/drm/nouveau/nvif/vmm.c +++ b/drivers/gpu/drm/nouveau/nvif/vmm.c @@ -192,6 +192,7 @@ void nvif_vmm_dtor(struct nvif_vmm *vmm) { kfree(vmm->page); + vmm->page = NULL; nvif_object_dtor(&vmm->object); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index ea62dc97f1189..96c8a5b299999 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2846,7 +2846,7 @@ nv1b2_chipset = { .pci = { 0x00000001, gh100_pci_new }, .timer = { 0x00000001, gk20a_timer_new }, .vfn = { 0x00000001, ga100_vfn_new }, - .disp = { 0x00000001, ga102_disp_new }, + .disp = { 0x00000001, gb202_disp_new }, .fifo = { 0x00000001, ga102_fifo_new }, }; @@ -2862,7 +2862,7 @@ nv1b3_chipset = { .pci = { 0x00000001, gh100_pci_new }, .timer = { 0x00000001, gk20a_timer_new }, .vfn = { 0x00000001, ga100_vfn_new }, - .disp = { 0x00000001, ga102_disp_new }, + .disp = { 0x00000001, gb202_disp_new }, .fifo = { 0x00000001, ga102_fifo_new }, }; @@ -2878,7 +2878,7 @@ nv1b5_chipset = { .pci = { 0x00000001, gh100_pci_new }, .timer = { 0x00000001, gk20a_timer_new }, .vfn = { 0x00000001, ga100_vfn_new }, - .disp = { 0x00000001, ga102_disp_new }, + .disp = { 0x00000001, gb202_disp_new }, .fifo = { 0x00000001, ga102_fifo_new }, }; @@ -2894,7 +2894,7 @@ nv1b6_chipset = { .pci = { 0x00000001, gh100_pci_new }, .timer = { 0x00000001, gk20a_timer_new }, .vfn = { 0x00000001, ga100_vfn_new }, - .disp = { 0x00000001, ga102_disp_new }, + .disp = { 0x00000001, gb202_disp_new }, .fifo = { 0x00000001, ga102_fifo_new }, }; @@ -2910,7 +2910,7 @@ nv1b7_chipset = { .pci = { 0x00000001, gh100_pci_new }, .timer = { 0x00000001, gk20a_timer_new }, .vfn = { 0x00000001, ga100_vfn_new }, - .disp = { 0x00000001, ga102_disp_new }, + .disp = { 0x00000001, gb202_disp_new }, .fifo = { 0x00000001, ga102_fifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild index e1aecd3fe96c1..98d6ca5ac311e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild @@ -27,6 +27,7 @@ nvkm-y += nvkm/engine/disp/gp102.o nvkm-y += nvkm/engine/disp/gv100.o nvkm-y += nvkm/engine/disp/tu102.o nvkm-y += nvkm/engine/disp/ga102.o +nvkm-y += nvkm/engine/disp/gb202.o nvkm-y += nvkm/engine/disp/udisp.o nvkm-y += nvkm/engine/disp/uconn.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c index ab0a85c924304..820834b5ee9b2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c @@ -144,12 +144,23 @@ ga102_disp = { }, }; +static const struct nvkm_disp_func +ga102_gsp_disp = { + .uevent = &gv100_disp_chan_uevent, + .ramht_size = 0x2000, + .gsp.intr = tu102_disp_intr, + .gsp.head = &tu102_gsp_head, + .gsp.hdmi_gcp = tu102_sor_hdmi_gcp, + .gsp.hdmi_infoframe_avi = gv100_sor_hdmi_infoframe_avi, + .gsp.hdmi_infoframe_vsi = gv100_sor_hdmi_infoframe_vsi, +}; + int ga102_disp_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_disp **pdisp) { if (nvkm_gsp_rm(device->gsp)) - return r535_disp_new(&ga102_disp, device, type, inst, pdisp); + return r535_disp_new(&ga102_gsp_disp, device, type, inst, pdisp); return nvkm_disp_new_(&ga102_disp, device, type, inst, pdisp); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gb202.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gb202.c new file mode 100644 index 0000000000000..d0360610f9fad --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gb202.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright 2026 Valve Corp. + */ +#include "priv.h" +#include "head.h" +#include "ior.h" + +#include + +/* GB20x (NVD5.0) reorganised the SF HDMI packet units. The AVI unit is + * unchanged from GV100, but the legacy VSI unit is gone. Vendor infoframes + * are sent through the shared generic infoframe units instead. Register + * layout per NVIDIA's clc971.h/clca71.h, programming sequence per + * nvhdmipkt_C971.c:programAdvancedInfoframeC971(). + */ +static void +gb202_sor_hdmi_infoframe_vsi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + const u32 hoff = head * 0x400; + /* Generic infoframe unit 1, the slot NVIDIA's driver uses for the VSI. */ + const u32 ctrl = 0x6f0138 + hoff; + u8 buf[36] = {}; + int i; + + /* Disable the unit and wait for it to go idle. */ + nvkm_mask(device, ctrl, 0x00000001, 0x00000000); + if (nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, ctrl) & 0x00400000)) + break; + ) < 0) + return; + + if (!size) + return; + + /* Clear SENT status, and point the data port at unit 1's slot. */ + nvkm_mask(device, ctrl, 0x00800000, 0x00800000); + nvkm_wr32(device, 0x6f03f0 + hoff, 0x00000001); + + /* The data port takes the raw packet, except that a zero is inserted + * in HB3 after the three header bytes. A slot is 9 dwords (HB0-3 plus + * up to 32 payload bytes). An HDMI infoframe carries at most PB0-27, + * so the tail stays zero, and we always write the whole slot. + */ + size = min_t(u32, size, 31); + memcpy(buf, data, min_t(u32, size, 3)); + if (size > 3) + memcpy(&buf[4], (u8 *)data + 3, size - 3); + + for (i = 0; i < 36; i += 4) { + nvkm_wr32(device, 0x6f03f4 + hoff, buf[i + 0] | buf[i + 1] << 8 | + buf[i + 2] << 16 | + (u32)buf[i + 3] << 24); + } + + /* No flip ID or scanline matching. */ + nvkm_wr32(device, 0x6f013c + hoff, 0x00000000); + + /* ENABLE | RUN_MODE=ALWAYS | LOC=VBLANK | OFFSET=1 | SIZE=0. */ + nvkm_wr32(device, ctrl, 0x00000041); + + /* Audio priority low (the init value). */ + nvkm_wr32(device, 0x6f03f8 + hoff, 0x00000002); +} + +/* General Control Packet AVMute bracket. The GCP unit moved to slot 1 on + * NVD5.0. Only SB0 (the AVMute bit) is ours to write so we must not do a + * full write here: SB1 carries the deep-color CD/PP fields, and SB1_CTRL + * (bit 24, new with clc871.h) controls where their generation happens (HW + * or driver) on these chips, with the default being HW. + */ +static void +gb202_sor_hdmi_gcp(struct nvkm_ior *sor, int head, bool enable) +{ + struct nvkm_device *device = sor->disp->engine.subdev.device; + const u32 hdmi = head * 0x400; + + nvkm_mask(device, 0x6f0040 + hdmi, 0x00000001, 0x00000000); + nvkm_mask(device, 0x6f004c + hdmi, 0x000000ff, !enable ? 0x00000001 : + 0x00000010); + nvkm_mask(device, 0x6f0040 + hdmi, 0x00000001, 0x00000001); +} + +/* Same core-channel state mirror as gv100_head_state() (assembly at 0x680000, + * armed at +0x8000, per-head method offsets unchanged), but NVD5.0 spaces + * heads 0x800 apart (see NVCA7D_HEAD_SET_*(a) in clca7d.h). + */ +static void +gb202_head_state(struct nvkm_head *head, struct nvkm_head_state *state) +{ + struct nvkm_device *device = head->disp->engine.subdev.device; + const u32 hoff = (state == &head->arm) * 0x8000 + head->id * 0x800; + u32 data; + + data = nvkm_rd32(device, 0x682064 + hoff); + state->vtotal = (data & 0xffff0000) >> 16; + state->htotal = (data & 0x0000ffff); + data = nvkm_rd32(device, 0x682068 + hoff); + state->vsynce = (data & 0xffff0000) >> 16; + state->hsynce = (data & 0x0000ffff); + data = nvkm_rd32(device, 0x68206c + hoff); + state->vblanke = (data & 0xffff0000) >> 16; + state->hblanke = (data & 0x0000ffff); + data = nvkm_rd32(device, 0x682070 + hoff); + state->vblanks = (data & 0xffff0000) >> 16; + state->hblanks = (data & 0x0000ffff); + /* Bit 31 is ADJ1000DIV1001, not a HERTZ bit. We don't have enough bits + * to add the full clock in hz on Blackwell (35 bits), but state->hz + * is unused and obsolete under GSP so this is fine. + */ + state->hz = nvkm_rd32(device, 0x68200c + hoff) & 0x7fffffff; + + data = nvkm_rd32(device, 0x682004 + hoff); + switch ((data & 0x000000f0) >> 4) { + case 5: state->or.depth = 30; break; + case 4: state->or.depth = 24; break; + case 1: state->or.depth = 18; break; + default: + state->or.depth = 18; + WARN_ON(1); + break; + } +} + +/* NVD5.0 (GB20x and later) moved the RM head-timing interrupt enable to + * the low-latency vector's EN1 block. The event latch is unchanged. + */ +static void +gb202_head_vblank_put(struct nvkm_head *head) +{ + struct nvkm_device *device = head->disp->engine.subdev.device; + + nvkm_mask(device, 0x611ef0 + (head->id * 4), 0x00000002, 0x00000000); +} + +static void +gb202_head_vblank_get(struct nvkm_head *head) +{ + struct nvkm_device *device = head->disp->engine.subdev.device; + + nvkm_wr32(device, 0x611800 + (head->id * 4), 0x00000002); + nvkm_mask(device, 0x611ef0 + (head->id * 4), 0x00000002, 0x00000002); +} + +static irqreturn_t +gb202_disp_intr(struct nvkm_inth *inth) +{ + struct nvkm_disp *disp = container_of(inth, typeof(*disp), engine.subdev.inth); + irqreturn_t ret = tu102_disp_intr(inth); + + /* The FE interrupt vectors are message-based on NVD5.0. Re-arm the + * low-latency vector so it fires again for any event that latched + * while we were servicing. + */ + nvkm_wr32(disp->engine.subdev.device, 0x611f34, 0x00000001); + return ret; +} + +static const struct nvkm_head_func +gb202_gsp_head = { + .state = gb202_head_state, + .rgpos = gv100_head_rgpos, + .vblank_get = gb202_head_vblank_get, + .vblank_put = gb202_head_vblank_put, +}; + +/* GB20x is GSP-only. This table supplies the register programming the + * GSP-RM display path needs from the chip. + */ +static const struct nvkm_disp_func +gb202_gsp_disp = { + .uevent = &gv100_disp_chan_uevent, + .ramht_size = 0x2000, + /* Head timing arrives on the dedicated low-latency vector. */ + .gsp.intr = gb202_disp_intr, + .gsp.intr_low_latency = true, + .gsp.head = &gb202_gsp_head, + .gsp.hdmi_gcp = gb202_sor_hdmi_gcp, + /* The legacy AVI unit is unchanged on GB20x. */ + .gsp.hdmi_infoframe_avi = gv100_sor_hdmi_infoframe_avi, + .gsp.hdmi_infoframe_vsi = gb202_sor_hdmi_infoframe_vsi, +}; + +int +gb202_disp_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_disp **pdisp) +{ + return r535_disp_new(&gb202_gsp_disp, device, type, inst, pdisp); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c index dbd984da75014..0608266188d3e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c @@ -253,7 +253,7 @@ gv100_head_vblank_get(struct nvkm_head *head) nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000004, 0x00000004); } -static void +void gv100_head_rgpos(struct nvkm_head *head, u16 *hline, u16 *vline) { struct nvkm_device *device = head->disp->engine.subdev.device; @@ -263,7 +263,7 @@ gv100_head_rgpos(struct nvkm_head *head, u16 *hline, u16 *vline) *hline = nvkm_rd32(device, 0x616334 + hoff) & 0x0000ffff; } -static void +void gv100_head_state(struct nvkm_head *head, struct nvkm_head_state *state) { struct nvkm_device *device = head->disp->engine.subdev.device; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h index 856252bf559a4..5976498da9094 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h @@ -53,6 +53,10 @@ void gf119_head_rgclk(struct nvkm_head *, int); int gv100_head_cnt(struct nvkm_disp *, unsigned long *); int gv100_head_new(struct nvkm_disp *, int id); +void gv100_head_state(struct nvkm_head *head, struct nvkm_head_state *state); +void gv100_head_rgpos(struct nvkm_head *head, u16 *hline, u16 *vline); + +extern const struct nvkm_head_func tu102_gsp_head; #define HEAD_MSG(h,l,f,a...) do { \ struct nvkm_head *_h = (h); \ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h index 3ba04bead2f9c..5d682a774f2dd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h @@ -194,6 +194,7 @@ void gv100_sor_dp_audio_sym(struct nvkm_ior *, int, u16, u32); void gv100_sor_dp_watermark(struct nvkm_ior *, int, u8); extern const struct nvkm_ior_func_hda gv100_sor_hda; +void tu102_sor_hdmi_gcp(struct nvkm_ior *, int, bool); void tu102_sor_dp_vcpi(struct nvkm_ior *, int, u8, u8, u16, u16); int nv50_pior_cnt(struct nvkm_disp *, unsigned long *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h index a3fd7cb7c4883..fde321dbd7c8f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h @@ -5,6 +5,8 @@ #include #include struct nvkm_head; +struct nvkm_head_func; +struct nvkm_ior; struct nvkm_outp; struct dcb_output; @@ -34,6 +36,20 @@ struct nvkm_disp_func { int (*new)(struct nvkm_disp *, int id); } wndw, head, dac, sor, pior; + /* Register programming that the GSP-RM display path (rm/r535) needs from + * the chip, everything else on that path goes through RM. The hooks are + * called unconditionally and the head table is handed to nvkm_head_new_(). + */ + struct { + irqreturn_t (*intr)(struct nvkm_inth *); + /* Head-timing interrupts arrive on a second DISP vector. */ + bool intr_low_latency; + const struct nvkm_head_func *head; + void (*hdmi_gcp)(struct nvkm_ior *, int head, bool enable); + void (*hdmi_infoframe_avi)(struct nvkm_ior *, int head, void *data, u32 size); + void (*hdmi_infoframe_vsi)(struct nvkm_ior *, int head, void *data, u32 size); + } gsp; + u16 ramht_size; struct nvkm_sclass root; @@ -72,6 +88,7 @@ int gv100_disp_wndw_cnt(struct nvkm_disp *, unsigned long *); int gv100_disp_caps_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); int tu102_disp_init(struct nvkm_disp *); +irqreturn_t tu102_disp_intr(struct nvkm_inth *); void nv50_disp_dptmds_war_2(struct nvkm_disp *, struct dcb_output *); void nv50_disp_dptmds_war_3(struct nvkm_disp *, struct dcb_output *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c index dcb9f8ba374ca..f6c163072ff66 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c @@ -30,6 +30,21 @@ #include +/* General Control Packet: bracket an audio enable/disable with AVMute + * through the legacy GCP SF unit. Used by the GSP-RM path, which sends the + * equivalent packet via RM as well but keeps the direct write in sync. + */ +void +tu102_sor_hdmi_gcp(struct nvkm_ior *sor, int head, bool enable) +{ + struct nvkm_device *device = sor->disp->engine.subdev.device; + const u32 hdmi = head * 0x400; + + nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000000); + nvkm_wr32(device, 0x6f00cc + hdmi, !enable ? 0x00000001 : 0x00000010); + nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000001); +} + void tu102_sor_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned) { @@ -104,6 +119,64 @@ tu102_sor_new(struct nvkm_disp *disp, int id) return nvkm_ior_new_(&tu102_sor, disp, SOR, id, hda & BIT(id)); } +/* The GSP-RM display path leaves head-timing (vblank) interrupts and their + * enables to us. These program the RM head-timing line (bit 1 of the + * per-head enable, not the bit nvkm's own gv100 path uses). + */ +static void +tu102_head_vblank_put(struct nvkm_head *head) +{ + struct nvkm_device *device = head->disp->engine.subdev.device; + + nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000002, 0x00000000); +} + +static void +tu102_head_vblank_get(struct nvkm_head *head) +{ + struct nvkm_device *device = head->disp->engine.subdev.device; + + nvkm_wr32(device, 0x611800 + (head->id * 4), 0x00000002); + nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000002, 0x00000002); +} + +const struct nvkm_head_func +tu102_gsp_head = { + .state = gv100_head_state, + .rgpos = gv100_head_rgpos, + .vblank_get = tu102_head_vblank_get, + .vblank_put = tu102_head_vblank_put, +}; + +static void +tu102_disp_intr_head_timing(struct nvkm_disp *disp, int head) +{ + struct nvkm_subdev *subdev = &disp->engine.subdev; + struct nvkm_device *device = subdev->device; + u32 stat = nvkm_rd32(device, 0x611c00 + (head * 0x04)); + + if (stat & 0x00000002) { + nvkm_disp_vblank(disp, head); + + nvkm_wr32(device, 0x611800 + (head * 0x04), 0x00000002); + } +} + +irqreturn_t +tu102_disp_intr(struct nvkm_inth *inth) +{ + struct nvkm_disp *disp = container_of(inth, typeof(*disp), engine.subdev.inth); + struct nvkm_subdev *subdev = &disp->engine.subdev; + struct nvkm_device *device = subdev->device; + unsigned long mask = nvkm_rd32(device, 0x611ec0) & 0x000000ff; + int head; + + for_each_set_bit(head, &mask, 8) + tu102_disp_intr_head_timing(disp, head); + + return IRQ_HANDLED; +} + int tu102_disp_init(struct nvkm_disp *disp) { @@ -230,12 +303,23 @@ tu102_disp = { }, }; +static const struct nvkm_disp_func +tu102_gsp_disp = { + .uevent = &gv100_disp_chan_uevent, + .ramht_size = 0x2000, + .gsp.intr = tu102_disp_intr, + .gsp.head = &tu102_gsp_head, + .gsp.hdmi_gcp = tu102_sor_hdmi_gcp, + .gsp.hdmi_infoframe_avi = gv100_sor_hdmi_infoframe_avi, + .gsp.hdmi_infoframe_vsi = gv100_sor_hdmi_infoframe_vsi, +}; + int tu102_disp_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_disp **pdisp) { if (nvkm_gsp_rm(device->gsp)) - return r535_disp_new(&tu102_disp, device, type, inst, pdisp); + return r535_disp_new(&tu102_gsp_disp, device, type, inst, pdisp); return nvkm_disp_new_(&tu102_disp, device, type, inst, pdisp); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c index 377d0e0cef848..9887b3898505b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c @@ -253,8 +253,7 @@ nvkm_uoutp_mthd_hdmi(struct nvkm_outp *outp, void *argv, u32 argc) if (!ior->func->hdmi || args->v0.max_ac_packet > 0x1f || - args->v0.rekey > 0x7f || - (args->v0.scdc && !ior->func->hdmi->scdc)) + args->v0.rekey > 0x7f) return -EINVAL; if (!args->v0.enable) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c index 572e638463159..fd2a9d64caba9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c @@ -199,16 +199,18 @@ nvkm_cstate_prog(struct nvkm_clk *clk, struct nvkm_pstate *pstate, int cstatei) } if (volt) { - ret = nvkm_volt_set_id(volt, cstate->voltage, - pstate->base.voltage, clk->temp, -1); - if (ret && ret != -ENODEV) - nvkm_error(subdev, "failed to lower voltage: %d\n", ret); + int err = nvkm_volt_set_id(volt, cstate->voltage, + pstate->base.voltage, clk->temp, -1); + + if (err && err != -ENODEV) + nvkm_error(subdev, "failed to lower voltage: %d\n", err); } if (therm) { - ret = nvkm_therm_cstate(therm, pstate->fanspeed, -1); - if (ret && ret != -ENODEV) - nvkm_error(subdev, "failed to lower fan speed: %d\n", ret); + int err = nvkm_therm_cstate(therm, pstate->fanspeed, -1); + + if (err && err != -ENODEV) + nvkm_error(subdev, "failed to lower fan speed: %d\n", err); } return ret; @@ -473,6 +475,7 @@ static int nvkm_clk_ustate_update(struct nvkm_clk *clk, int req) { struct nvkm_pstate *pstate; + bool found = false; int i = 0; if (!clk->allow_reclock) @@ -480,12 +483,14 @@ nvkm_clk_ustate_update(struct nvkm_clk *clk, int req) if (req != -1 && req != -2) { list_for_each_entry(pstate, &clk->states, head) { - if (pstate->pstate == req) + if (pstate->pstate == req) { + found = true; break; + } i++; } - if (pstate->pstate != req) + if (!found) return -EINVAL; req = i; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c index 18241c6ba5fa3..4d52a158f3205 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c @@ -51,6 +51,8 @@ nv1a_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) mib = ((mem >> 4) & 127) + 1; } + pci_dev_put(bridge); + return nvkm_ram_new_(&nv04_ram_func, fb, NVKM_RAM_TYPE_STOLEN, mib * 1024 * 1024, pram); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c index 1155f079b0c3d..f5f22173fc2c1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c @@ -400,16 +400,16 @@ r535_sor_dp_audio(struct nvkm_ior *sor, int head, bool enable) r535_sor_dp_audio_mute(sor, false); } -static void -r535_sor_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned_pbn) +static int +r535_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned_pbn) { struct nvkm_disp *disp = sor->disp; struct NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS *ctrl; ctrl = nvkm_gsp_rm_ctrl_get(&disp->rm.objcom, NV0073_CTRL_CMD_DP_CONFIG_STREAM, sizeof(*ctrl)); - if (WARN_ON(IS_ERR(ctrl))) - return; + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); ctrl->subDeviceInstance = 0; ctrl->head = head; @@ -429,12 +429,20 @@ r535_sor_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u ctrl->MST.sendACT = 0; ctrl->MST.singleHeadMSTPipeline = 0; ctrl->MST.bEnableAudioOverRightPanel = 0; - WARN_ON(nvkm_gsp_rm_ctrl_wr(&disp->rm.objcom, ctrl)); + return nvkm_gsp_rm_ctrl_wr(&disp->rm.objcom, ctrl); +} + +static void +r535_sor_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned_pbn) +{ + const struct nvkm_rm_api *rmapi = sor->disp->engine.subdev.device->gsp->rm->api; + + WARN_ON(rmapi->disp->dp.vcpi(sor, head, slot, slot_nr, pbn, aligned_pbn)); } static int -r535_sor_dp_sst(struct nvkm_ior *sor, int head, bool ef, - u32 watermark, u32 hblanksym, u32 vblanksym) +r535_dp_sst(struct nvkm_ior *sor, int head, bool ef, + u32 watermark, u32 hblanksym, u32 vblanksym) { struct nvkm_disp *disp = sor->disp; struct NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS *ctrl; @@ -461,6 +469,15 @@ r535_sor_dp_sst(struct nvkm_ior *sor, int head, bool ef, return nvkm_gsp_rm_ctrl_wr(&disp->rm.objcom, ctrl); } +static int +r535_sor_dp_sst(struct nvkm_ior *sor, int head, bool ef, + u32 watermark, u32 hblanksym, u32 vblanksym) +{ + const struct nvkm_rm_api *rmapi = sor->disp->engine.subdev.device->gsp->rm->api; + + return rmapi->disp->dp.sst(sor, head, ef, watermark, hblanksym, vblanksym); +} + static const struct nvkm_ior_func_dp r535_sor_dp = { .sst = r535_sor_dp_sst, @@ -545,16 +562,21 @@ r535_sor_hdmi_ctrl_audio(struct nvkm_outp *outp, bool enable) static void r535_sor_hdmi_audio(struct nvkm_ior *sor, int head, bool enable) { - struct nvkm_device *device = sor->disp->engine.subdev.device; - const u32 hdmi = head * 0x400; - r535_sor_hdmi_ctrl_audio(sor->asy.outp, enable); r535_sor_hdmi_ctrl_audio_mute(sor->asy.outp, !enable); + sor->disp->func->gsp.hdmi_gcp(sor, head, enable); +} + +static void +r535_sor_hdmi_infoframe_avi(struct nvkm_ior *sor, int head, void *data, u32 size) +{ + sor->disp->func->gsp.hdmi_infoframe_avi(sor, head, data, size); +} - /* General Control (GCP). */ - nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000000); - nvkm_wr32(device, 0x6f00cc + hdmi, !enable ? 0x00000001 : 0x00000010); - nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000001); +static void +r535_sor_hdmi_infoframe_vsi(struct nvkm_ior *sor, int head, void *data, u32 size) +{ + sor->disp->func->gsp.hdmi_infoframe_vsi(sor, head, data, size); } static void @@ -582,8 +604,8 @@ r535_sor_hdmi = { .ctrl = r535_sor_hdmi_ctrl, .scdc = r535_sor_hdmi_scdc, /*TODO: SF_USER -> KMS. */ - .infoframe_avi = gv100_sor_hdmi_infoframe_avi, - .infoframe_vsi = gv100_sor_hdmi_infoframe_vsi, + .infoframe_avi = r535_sor_hdmi_infoframe_avi, + .infoframe_vsi = r535_sor_hdmi_infoframe_vsi, .audio = r535_sor_hdmi_audio, }; @@ -608,35 +630,6 @@ r535_sor_cnt(struct nvkm_disp *disp, unsigned long *pmask) return 4; } -static void -r535_head_vblank_put(struct nvkm_head *head) -{ - struct nvkm_device *device = head->disp->engine.subdev.device; - - nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000002, 0x00000000); -} - -static void -r535_head_vblank_get(struct nvkm_head *head) -{ - struct nvkm_device *device = head->disp->engine.subdev.device; - - nvkm_wr32(device, 0x611800 + (head->id * 4), 0x00000002); - nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000002, 0x00000002); -} - -static void -r535_head_state(struct nvkm_head *head, struct nvkm_head_state *state) -{ -} - -static const struct nvkm_head_func -r535_head = { - .state = r535_head_state, - .vblank_get = r535_head_vblank_get, - .vblank_put = r535_head_vblank_put, -}; - static struct nvkm_conn * r535_conn_new(struct nvkm_disp *disp, u32 id) { @@ -1408,35 +1401,6 @@ static const struct nvkm_event_func r535_disp_event = { }; -static void -r535_disp_intr_head_timing(struct nvkm_disp *disp, int head) -{ - struct nvkm_subdev *subdev = &disp->engine.subdev; - struct nvkm_device *device = subdev->device; - u32 stat = nvkm_rd32(device, 0x611c00 + (head * 0x04)); - - if (stat & 0x00000002) { - nvkm_disp_vblank(disp, head); - - nvkm_wr32(device, 0x611800 + (head * 0x04), 0x00000002); - } -} - -static irqreturn_t -r535_disp_intr(struct nvkm_inth *inth) -{ - struct nvkm_disp *disp = container_of(inth, typeof(*disp), engine.subdev.inth); - struct nvkm_subdev *subdev = &disp->engine.subdev; - struct nvkm_device *device = subdev->device; - unsigned long mask = nvkm_rd32(device, 0x611ec0) & 0x000000ff; - int head; - - for_each_set_bit(head, &mask, 8) - r535_disp_intr_head_timing(disp, head); - - return IRQ_HANDLED; -} - static void r535_disp_fini(struct nvkm_disp *disp, bool suspend) { @@ -1663,7 +1627,7 @@ r535_disp_oneinit(struct nvkm_disp *disp) nvkm_gsp_rm_ctrl_done(&disp->rm.objcom, ctrl); for_each_set_bit(i, &disp->head.mask, disp->head.nr) { - ret = nvkm_head_new_(&r535_head, disp, i); + ret = nvkm_head_new_(disp->func->gsp.head, disp, i); if (ret) return ret; } @@ -1707,12 +1671,20 @@ r535_disp_oneinit(struct nvkm_disp *disp) if (ret) return ret; - ret = nvkm_gsp_intr_stall(gsp, disp->engine.subdev.type, disp->engine.subdev.inst); + /* Chips that raise head-timing interrupts on a separate low-latency + * vector report it as a second DISP interrupt table entry, exposed + * as instance 1 by the RM engine-index translation (see + * r570_gsp_xlat_mc_engine_idx()). Their high-latency vector + * (instance 0) is left unhandled as no event nouveau enables is + * routed to it, and without a handler it stays masked. + */ + ret = nvkm_gsp_intr_stall(gsp, disp->engine.subdev.type, + disp->func->gsp.intr_low_latency ? 1 : disp->engine.subdev.inst); if (ret < 0) return ret; ret = nvkm_inth_add(&device->vfn->intr, ret, NVKM_INTR_PRIO_NORMAL, &disp->engine.subdev, - r535_disp_intr, &disp->engine.subdev.inth); + disp->func->gsp.intr, &disp->engine.subdev.inth); if (ret) return ret; @@ -1745,6 +1717,7 @@ r535_disp_new(const struct nvkm_disp_func *hw, struct nvkm_device *device, rm->uevent = hw->uevent; rm->sor.cnt = r535_sor_cnt; rm->sor.new = r535_sor_new; + rm->gsp = hw->gsp; rm->ramht_size = hw->ramht_size; rm->root.oclass = gpu->disp.class.root; @@ -1786,6 +1759,8 @@ r535_disp = { .dp = { .get_caps = r535_dp_get_caps, .set_indexed_link_rates = r535_dp_set_indexed_link_rates, + .sst = r535_dp_sst, + .vcpi = r535_dp_vcpi, }, .chan = { .set_pushbuf = r535_disp_chan_set_pushbuf, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c index 700cea5def357..035b575722db7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c @@ -208,7 +208,7 @@ r535_fbsr_resume(struct nvkm_gsp *gsp) } static int -r535_fbsr_suspend(struct nvkm_gsp *gsp, bool runtime) +r535_fbsr_suspend(struct nvkm_gsp *gsp) { struct nvkm_subdev *subdev = &gsp->subdev; struct nvkm_device *device = subdev->device; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c index f544afa12b6bb..94925f1590ea4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c @@ -1749,7 +1749,7 @@ r535_gsp_fini(struct nvkm_gsp *gsp, enum nvkm_suspend_state suspend) sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.lvl0.addr; sr->sizeOfSuspendResumeData = len; - ret = rm->api->fbsr->suspend(gsp, suspend == NVKM_RUNTIME_SUSPEND); + ret = rm->api->fbsr->suspend(gsp); if (ret) { nvkm_gsp_mem_dtor(&gsp->sr.meta); nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3); @@ -1761,8 +1761,12 @@ r535_gsp_fini(struct nvkm_gsp *gsp, enum nvkm_suspend_state suspend) * TODO: Debug the GSP firmware / RPC handling to find out why * without this Turing (but none of the other architectures) * ends up resetting all channels after resume. + * Additionally, runtime suspend on other architectures quickly + * becomes unreliable without this sleep. If you're experiencing + * issues with runtime suspend, try bumping this delay up and + * sending a patch if it fixes your GPU. */ - msleep(50); + msleep(200); } ret = r535_gsp_rpc_unloading_guest_driver(gsp, suspend); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/disp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/disp.c index a96e31c2d80b2..8a23837f356e9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/disp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/disp.c @@ -5,6 +5,7 @@ #include #include +#include #include #include "nvhw/drf.h" @@ -74,6 +75,67 @@ r570_disp_chan_set_pushbuf(struct nvkm_disp *disp, s32 oclass, int inst, struct return nvkm_gsp_rm_ctrl_wr(&gsp->internal.device.subdevice, ctrl); } +static int +r570_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned_pbn) +{ + struct nvkm_disp *disp = sor->disp; + NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS *ctrl; + + ctrl = nvkm_gsp_rm_ctrl_get(&disp->rm.objcom, + NV0073_CTRL_CMD_DP_CONFIG_STREAM, sizeof(*ctrl)); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); + + ctrl->subDeviceInstance = 0; + ctrl->head = head; + ctrl->sorIndex = sor->id; + ctrl->dpLink = sor->asy.link == 2; + ctrl->bEnableOverride = 1; + ctrl->bMST = 1; + ctrl->hBlankSym = 0; + ctrl->vBlankSym = 0; + ctrl->colorFormat = 0; + ctrl->bEnableTwoHeadOneOr = 0; + ctrl->singleHeadMultistreamMode = 0; + ctrl->MST.slotStart = slot; + ctrl->MST.slotEnd = slot + slot_nr - 1; + ctrl->MST.PBN = pbn; + ctrl->MST.Timeslice = aligned_pbn; + ctrl->MST.sendACT = 0; + ctrl->MST.singleHeadMSTPipeline = 0; + ctrl->MST.bEnableAudioOverRightPanel = 0; + return nvkm_gsp_rm_ctrl_wr(&disp->rm.objcom, ctrl); +} + +static int +r570_dp_sst(struct nvkm_ior *sor, int head, bool ef, + u32 watermark, u32 hblanksym, u32 vblanksym) +{ + struct nvkm_disp *disp = sor->disp; + NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS *ctrl; + + ctrl = nvkm_gsp_rm_ctrl_get(&disp->rm.objcom, + NV0073_CTRL_CMD_DP_CONFIG_STREAM, sizeof(*ctrl)); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); + + ctrl->subDeviceInstance = 0; + ctrl->head = head; + ctrl->sorIndex = sor->id; + ctrl->dpLink = sor->asy.link == 2; + ctrl->bEnableOverride = 1; + ctrl->bMST = 0; + ctrl->hBlankSym = hblanksym; + ctrl->vBlankSym = vblanksym; + ctrl->colorFormat = 0; + ctrl->bEnableTwoHeadOneOr = 0; + ctrl->SST.bEnhancedFraming = ef; + ctrl->SST.tuSize = 64; + ctrl->SST.waterMark = watermark; + ctrl->SST.bEnableAudioOverRightPanel = 0; + return nvkm_gsp_rm_ctrl_wr(&disp->rm.objcom, ctrl); +} + static int r570_dp_set_indexed_link_rates(struct nvkm_outp *outp) { @@ -255,6 +317,8 @@ r570_disp = { .dp = { .get_caps = r570_dp_get_caps, .set_indexed_link_rates = r570_dp_set_indexed_link_rates, + .sst = r570_dp_sst, + .vcpi = r570_dp_vcpi, }, .chan = { .set_pushbuf = r570_disp_chan_set_pushbuf, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c index 8ef8b4f655883..af5aa5065c3dd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c @@ -62,7 +62,7 @@ r570_fbsr_resume(struct nvkm_gsp *gsp) } static int -r570_fbsr_init(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size, bool runtime) +r570_fbsr_init(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size) { NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS *ctrl; struct nvkm_gsp_object memlist; @@ -81,7 +81,7 @@ r570_fbsr_init(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size, bool runtim ctrl->hClient = gsp->internal.client.object.handle; ctrl->hSysMem = memlist.handle; ctrl->sysmemAddrOfSuspendResumeData = gsp->sr.meta.addr; - ctrl->bEnteringGcoffState = runtime ? 1 : 0; + ctrl->bEnteringGcoffState = 0; ret = nvkm_gsp_rm_ctrl_wr(&gsp->internal.device.subdevice, ctrl); if (ret) @@ -92,7 +92,7 @@ r570_fbsr_init(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size, bool runtim } static int -r570_fbsr_suspend(struct nvkm_gsp *gsp, bool runtime) +r570_fbsr_suspend(struct nvkm_gsp *gsp) { struct nvkm_subdev *subdev = &gsp->subdev; struct nvkm_device *device = subdev->device; @@ -133,7 +133,7 @@ r570_fbsr_suspend(struct nvkm_gsp *gsp, bool runtime) return ret; /* Initialise FBSR on RM. */ - ret = r570_fbsr_init(gsp, &gsp->sr.fbsr, size, runtime); + ret = r570_fbsr_init(gsp, &gsp->sr.fbsr, size); if (ret) { nvkm_gsp_sg_free(device, &gsp->sr.fbsr); return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c index 996941c668ba9..b45781cd0dfdc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c @@ -44,6 +44,15 @@ r570_gsp_xlat_mc_engine_idx(u32 mc_engine_idx, enum nvkm_subdev_type *ptype, int *ptype = NVKM_ENGINE_DISP; *pinst = 0; return true; + case MC_ENGINE_IDX_DISP_LOW: + /* GB20x+ report a separate low-latency display vector, used + * for head-timing interrupts. Expose it as a second DISP + * interrupt instance. r535_disp_oneinit() attaches the + * handler to it when the chip's gsp.intr_low_latency is set. + */ + *ptype = NVKM_ENGINE_DISP; + *pinst = 1; + return true; case MC_ENGINE_IDX_CE0 ... MC_ENGINE_IDX_CE19: *ptype = NVKM_ENGINE_CE; *pinst = mc_engine_idx - MC_ENGINE_IDX_CE0; @@ -198,7 +207,8 @@ r570_gsp_set_rmargs(struct nvkm_gsp *gsp, bool resume) args->srInitArguments.bInPMTransition = 0; } else { args->srInitArguments.oldLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3; - args->srInitArguments.flags = 0; + args->srInitArguments.flags = + GPU_STATE_FLAGS_PRESERVING | GPU_STATE_FLAGS_PM_TRANSITION; args->srInitArguments.bInPMTransition = 1; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/disp.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/disp.h index 06e972835d778..742b25a2a12df 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/disp.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/disp.h @@ -256,6 +256,8 @@ typedef struct NV0073_CTRL_DP_CTRL_PARAMS { NvU32 eightLaneDpcdBaseAddr; } NV0073_CTRL_DP_CTRL_PARAMS; +#define NV0073_CTRL_CMD_DP_CONFIG_STREAM (0x731362U) /* finn: Evaluated from "(FINN_NV04_DISPLAY_COMMON_DP_INTERFACE_ID << 8) | NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS_MESSAGE_ID" */ + typedef struct NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS { NvU32 subDeviceInstance; NvU32 head; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h index b6075021e74f5..c458569af9d72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h @@ -523,6 +523,14 @@ typedef struct #define NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3 (0x00000003U) +#define GPU_STATE_FLAGS_PRESERVING BIT(0) // GPU state is preserved +#define GPU_STATE_FLAGS_VGA_TRANSITION BIT(1) // To be used with GPU_STATE_FLAGS_PRESERVING. +#define GPU_STATE_FLAGS_PM_TRANSITION BIT(2) // To be used with GPU_STATE_FLAGS_PRESERVING. +#define GPU_STATE_FLAGS_PM_SUSPEND BIT(3) +#define GPU_STATE_FLAGS_PM_HIBERNATE BIT(4) +#define GPU_STATE_FLAGS_GC6_TRANSITION BIT(5) // To be used with GPU_STATE_FLAGS_PRESERVING. +#define GPU_STATE_FLAGS_FAST_UNLOAD BIT(6) // Used during windows restart, skips stateDestroy steps + typedef struct { // Magic for verification by secure ucode diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h index a9af94adf9efc..e9ac47d86b69a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h @@ -6,6 +6,7 @@ #ifndef __NVKM_RM_H__ #define __NVKM_RM_H__ #include "handles.h" +struct nvkm_ior; struct nvkm_outp; struct r535_gr; @@ -78,7 +79,7 @@ struct nvkm_rm_api { } *device; const struct nvkm_rm_api_fbsr { - int (*suspend)(struct nvkm_gsp *, bool runtime); + int (*suspend)(struct nvkm_gsp *); void (*resume)(struct nvkm_gsp *); } *fbsr; @@ -93,6 +94,10 @@ struct nvkm_rm_api { struct { int (*get_caps)(struct nvkm_disp *, int *link_bw, bool *mst, bool *wm); int (*set_indexed_link_rates)(struct nvkm_outp *); + int (*sst)(struct nvkm_ior *, int head, bool ef, + u32 watermark, u32 hblanksym, u32 vblanksym); + int (*vcpi)(struct nvkm_ior *, int head, + u8 slot, u8 slot_nr, u16 pbn, u16 aligned_pbn); } dp; struct { diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c index 27fe7bca9e2cf..70cfb779d6ae5 100644 --- a/drivers/gpu/drm/omapdrm/dss/dsi.c +++ b/drivers/gpu/drm/omapdrm/dss/dsi.c @@ -455,15 +455,10 @@ static irqreturn_t omap_dsi_irq_handler(int irq, void *arg) timer_delete(&dsi->te_timer); #endif - /* make a copy and unlock, so that isrs can unregister - * themselves */ - memcpy(&dsi->isr_tables_copy, &dsi->isr_tables, - sizeof(dsi->isr_tables)); + dsi_handle_isrs(&dsi->isr_tables, irqstatus, vcstatus, ciostatus); spin_unlock(&dsi->irq_lock); - dsi_handle_isrs(&dsi->isr_tables_copy, irqstatus, vcstatus, ciostatus); - dsi_handle_irq_errors(dsi, irqstatus, vcstatus, ciostatus); dsi_collect_irq_stats(dsi, irqstatus, vcstatus, ciostatus); diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.h b/drivers/gpu/drm/omapdrm/dss/dsi.h index 601707c0ecc4e..2b25247ea8935 100644 --- a/drivers/gpu/drm/omapdrm/dss/dsi.h +++ b/drivers/gpu/drm/omapdrm/dss/dsi.h @@ -379,8 +379,6 @@ struct dsi_data { spinlock_t irq_lock; struct dsi_isr_tables isr_tables; - /* space for a copy used by the interrupt handler */ - struct dsi_isr_tables isr_tables_copy; int update_vc; #ifdef DSI_PERF_MEASURE diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c index aa27d6cd932e8..ac815729aa831 100644 --- a/drivers/gpu/drm/panel/panel-edp.c +++ b/drivers/gpu/drm/panel/panel-edp.c @@ -831,6 +831,13 @@ static int generic_edp_panel_probe(struct device *dev, struct panel_edp *panel) return 0; } +static void panel_edp_put_adapter(void *_adap) +{ + struct i2c_adapter *adap = _adap; + + put_device(&adap->dev); +} + static int panel_edp_probe(struct device *dev, const struct panel_desc *desc, struct drm_dp_aux *aux) { @@ -878,6 +885,11 @@ static int panel_edp_probe(struct device *dev, const struct panel_desc *desc, if (!panel->ddc) return -EPROBE_DEFER; + + err = devm_add_action_or_reset(dev, panel_edp_put_adapter, + panel->ddc); + if (err) + return err; } else if (aux) { panel->ddc = &aux->ddc; } @@ -889,7 +901,7 @@ static int panel_edp_probe(struct device *dev, const struct panel_desc *desc, err = drm_panel_of_backlight(&panel->base); if (err) - goto err_finished_ddc_init; + return err; /* * We use runtime PM for prepare / unprepare since those power the panel @@ -936,9 +948,6 @@ static int panel_edp_probe(struct device *dev, const struct panel_desc *desc, err_finished_pm_runtime: pm_runtime_dont_use_autosuspend(dev); pm_runtime_disable(dev); -err_finished_ddc_init: - if (panel->ddc && (!panel->aux || panel->ddc != &panel->aux->ddc)) - put_device(&panel->ddc->dev); return err; } @@ -982,8 +991,6 @@ static void panel_edp_remove(struct device *dev) pm_runtime_dont_use_autosuspend(dev); pm_runtime_disable(dev); - if (panel->ddc && (!panel->aux || panel->ddc != &panel->aux->ddc)) - put_device(&panel->ddc->dev); drm_edid_free(panel->drm_edid); panel->drm_edid = NULL; diff --git a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c index 1b14aa4efe359..54a65abf7e890 100644 --- a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c +++ b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c @@ -88,16 +88,22 @@ static int s6d16d0_prepare(struct drm_panel *panel) MIPI_DSI_DCS_TEAR_MODE_VBLANK); if (ret) { dev_err(s6->dev, "failed to enable vblank TE (%d)\n", ret); - return ret; + goto err_power_off; } /* Exit sleep mode and power on */ ret = mipi_dsi_dcs_exit_sleep_mode(dsi); if (ret) { dev_err(s6->dev, "failed to exit sleep mode (%d)\n", ret); - return ret; + goto err_power_off; } return 0; + +err_power_off: + gpiod_set_value_cansleep(s6->reset_gpio, 1); + regulator_disable(s6->supply); + + return ret; } static int s6d16d0_enable(struct drm_panel *panel) diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c index e8dc4096c1d25..74ec417115a3d 100644 --- a/drivers/gpu/drm/panthor/panthor_drv.c +++ b/drivers/gpu/drm/panthor/panthor_drv.c @@ -1814,7 +1814,7 @@ static int panthor_probe(struct platform_device *pdev) ptdev = devm_drm_dev_alloc(&pdev->dev, &panthor_drm_driver, struct panthor_device, base); if (IS_ERR(ptdev)) - return -ENOMEM; + return PTR_ERR(ptdev); platform_set_drvdata(pdev, ptdev); diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c index fa88b2f9da76d..10b7e12c1f1ed 100644 --- a/drivers/gpu/drm/panthor/panthor_fw.c +++ b/drivers/gpu/drm/panthor/panthor_fw.c @@ -707,7 +707,8 @@ static int panthor_fw_read_build_info(struct panthor_device *ptdev, return ret; if (hdr.meta_start > fw->size || - hdr.meta_start + hdr.meta_size > fw->size) { + hdr.meta_size > fw->size - hdr.meta_start || + hdr.meta_size <= header_len) { drm_err(&ptdev->base, "Firmware build info corrupt\n"); /* We don't need the build info, so continue */ return 0; @@ -894,14 +895,15 @@ static int panthor_init_cs_iface(struct panthor_device *ptdev, struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, csg_idx); struct panthor_fw_cs_iface *cs_iface = &ptdev->fw->iface.streams[csg_idx][cs_idx]; u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); - u32 iface_offset = CSF_GROUP_CONTROL_OFFSET + - (csg_idx * glb_iface->control->group_stride) + + u64 iface_offset = CSF_GROUP_CONTROL_OFFSET + + ((u64)csg_idx * glb_iface->control->group_stride) + CSF_STREAM_CONTROL_OFFSET + - (cs_idx * csg_iface->control->stream_stride); + ((u64)cs_idx * csg_iface->control->stream_stride); struct panthor_fw_cs_iface *first_cs_iface = panthor_fw_get_cs_iface(ptdev, 0, 0); - if (iface_offset + sizeof(*cs_iface) >= shared_section_sz) + if (iface_offset > shared_section_sz || + sizeof(*cs_iface->control) > shared_section_sz - iface_offset) return -EINVAL; spin_lock_init(&cs_iface->lock); @@ -951,10 +953,12 @@ static int panthor_init_csg_iface(struct panthor_device *ptdev, struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); struct panthor_fw_csg_iface *csg_iface = &ptdev->fw->iface.groups[csg_idx]; u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); - u32 iface_offset = CSF_GROUP_CONTROL_OFFSET + (csg_idx * glb_iface->control->group_stride); + u64 iface_offset = CSF_GROUP_CONTROL_OFFSET + + ((u64)csg_idx * glb_iface->control->group_stride); unsigned int i; - if (iface_offset + sizeof(*csg_iface) >= shared_section_sz) + if (iface_offset > shared_section_sz || + sizeof(*csg_iface->control) > shared_section_sz - iface_offset) return -EINVAL; spin_lock_init(&csg_iface->lock); @@ -1006,11 +1010,15 @@ static u32 panthor_get_instr_features(struct panthor_device *ptdev) static int panthor_fw_init_ifaces(struct panthor_device *ptdev) { struct panthor_fw_global_iface *glb_iface = &ptdev->fw->iface.global; + u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); unsigned int i; if (!ptdev->fw->shared_section->mem->kmap) return -EINVAL; + if (sizeof(*glb_iface->control) > shared_section_sz) + return -EINVAL; + spin_lock_init(&glb_iface->lock); glb_iface->control = ptdev->fw->shared_section->mem->kmap; diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c index 54535bae2b0cf..772121ece3180 100644 --- a/drivers/gpu/drm/panthor/panthor_gem.c +++ b/drivers/gpu/drm/panthor/panthor_gem.c @@ -1351,6 +1351,24 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, return ERR_PTR(ret); } +/** + * panthor_dummy_bo_create() - Create a Panthor BO meant to back sparse bindings. + * @ptdev: Device. + * + * Return: A valid pointer in case of success, an ERR_PTR() otherwise. + */ +struct panthor_gem_object * +panthor_dummy_bo_create(struct panthor_device *ptdev) +{ + /* Since even when the DRM device's mount point has enabled THP we have no guarantee + * that drm_gem_get_pages() will return a single 2MiB PMD, and also we cannot be sure + * that the 2MiB won't be reclaimed and re-allocated later on as 4KiB chunks, it doesn't + * make sense to pre-populate this object's page array, nor to fall back on a BO size + * of 4KiB. Sticking to a dummy object size of 2MiB lets us keep things simple for now. + */ + return panthor_gem_create(&ptdev->base, SZ_2M, DRM_PANTHOR_BO_NO_MMAP, NULL, 0); +} + static bool can_swap(void) { return get_nr_swap_pages() > 0; diff --git a/drivers/gpu/drm/panthor/panthor_gem.h b/drivers/gpu/drm/panthor/panthor_gem.h index 56d63137b4ebb..5ae37d0d3646f 100644 --- a/drivers/gpu/drm/panthor/panthor_gem.h +++ b/drivers/gpu/drm/panthor/panthor_gem.h @@ -325,6 +325,8 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo); +struct panthor_gem_object *panthor_dummy_bo_create(struct panthor_device *ptdev); + #ifdef CONFIG_DEBUG_FS void panthor_gem_debugfs_init(struct drm_minor *minor); #endif diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c index e10dbd18d8cf9..fba2fff94ce32 100644 --- a/drivers/gpu/drm/panthor/panthor_mmu.c +++ b/drivers/gpu/drm/panthor/panthor_mmu.c @@ -116,6 +116,17 @@ struct panthor_mmu { struct panthor_vm_pool { /** @xa: Array used for VM handle tracking. */ struct xarray xa; + + /** + * @dummy: Dummy object used for sparse mappings + * + * Sparse bindings map virtual address ranges onto a dummy + * BO in a modulo fashion. Even though sparse writes are meant + * to be discarded and reads undefined, writes are still reflected + * in the dummy buffer. That means we must keep a dummy object per + * file context, to avoid data leaks between them. + */ + struct panthor_gem_object *dummy; }; /** @@ -307,6 +318,9 @@ struct panthor_vm { u64 end; } kernel_auto_va; + /** @user_va_range: Upper boundary of VAs VM users can map objects against. */ + u64 user_va_range; + /** @as: Address space related fields. */ struct { /** @@ -403,6 +417,15 @@ struct panthor_vm { */ struct list_head lru_node; } reclaim; + + /** + * @dummy: Dummy object used for sparse mappings. + * + * VM's must keep a reference to the file context-wide dummy BO because + * they can outlive the file context, which includes the VM pool holding + * the original dummy BO reference. + */ + struct panthor_gem_object *dummy; }; /** @@ -1035,6 +1058,30 @@ panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot, return 0; } +static int +panthor_vm_map_sparse(struct panthor_vm *vm, u64 iova, int prot, + struct sg_table *sgt, u64 size) +{ + u64 mapped = 0; + int ret; + + while (mapped < size) { + u64 addr = iova + mapped; + u32 chunk_size = min(size - mapped, SZ_2M - (addr & (SZ_2M - 1))); + + ret = panthor_vm_map_pages(vm, addr, prot, sgt, + addr % SZ_2M, chunk_size); + if (ret) { + panthor_vm_unmap_pages(vm, iova, mapped); + return ret; + } + + mapped += chunk_size; + } + + return 0; +} + static int flags_to_prot(u32 flags) { int prot = 0; @@ -1277,15 +1324,15 @@ static int panthor_vm_op_ctx_prealloc_pts(struct panthor_vm_op_ctx *op_ctx) (DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \ + DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE | \ DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, struct panthor_vm *vm, struct panthor_gem_object *bo, - u64 offset, - u64 size, u64 va, - u32 flags) + const struct drm_panthor_vm_bind_op *op) { + bool is_sparse = op->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE; struct drm_gpuvm_bo *preallocated_vm_bo; struct sg_table *sgt = NULL; int ret; @@ -1293,12 +1340,25 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, if (!bo) return -EINVAL; - if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) || - (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP) + if ((op->flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) || + (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP) + return -EINVAL; + + /* uAPI mandates sparsely bound regions must not be executable. */ + if (is_sparse && !(op->flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC)) return -EINVAL; - /* Make sure the VA and size are in-bounds. */ - if (size > bo->base.size || offset > bo->base.size - size) + /* For non-sparse, make sure the VA and size are in-bounds. + * For sparse, this is not applicable, because the dummy BO is + * repeatedly mapped over a potentially wider VA range. + */ + if (!is_sparse && (op->size > bo->base.size || op->bo_offset > bo->base.size - op->size)) + return -EINVAL; + + /* For sparse, we don't expect any user BO, the BO we get passed + * is the dummy BO attached to the VM pool. + */ + if (is_sparse && (op->bo_handle || op->bo_offset)) return -EINVAL; /* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */ @@ -1306,7 +1366,7 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, bo->exclusive_vm_root_gem != panthor_vm_root_gem(vm)) return -EINVAL; - panthor_vm_init_op_ctx(op_ctx, size, va, flags); + panthor_vm_init_op_ctx(op_ctx, op->size, op->va, op->flags); ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx); if (ret) @@ -1335,7 +1395,7 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, } op_ctx->map.vm_bo = drm_gpuvm_bo_obtain_prealloc(preallocated_vm_bo); - op_ctx->map.bo_offset = offset; + op_ctx->map.bo_offset = op->bo_offset; ret = panthor_vm_op_ctx_prealloc_pts(op_ctx); if (ret) @@ -1446,7 +1506,9 @@ panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset) if (vma && vma->base.gem.obj) { drm_gem_object_get(vma->base.gem.obj); bo = to_panthor_bo(vma->base.gem.obj); - *bo_offset = vma->base.gem.offset + (va - vma->base.va.addr); + *bo_offset = !(vma->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE) ? + vma->base.gem.offset + (va - vma->base.va.addr) : + va & (SZ_2M - 1); } mutex_unlock(&vm->op_lock); @@ -1551,6 +1613,9 @@ int panthor_vm_pool_create_vm(struct panthor_device *ptdev, if (IS_ERR(vm)) return PTR_ERR(vm); + drm_gem_object_get(&pool->dummy->base); + vm->dummy = pool->dummy; + ret = xa_alloc(&pool->xa, &id, vm, XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL); @@ -1650,6 +1715,8 @@ void panthor_vm_pool_destroy(struct panthor_file *pfile) xa_for_each(&pfile->vms->xa, i, vm) panthor_vm_destroy(vm); + if (pfile->vms->dummy) + drm_gem_object_put(&pfile->vms->dummy->base); xa_destroy(&pfile->vms->xa); kfree(pfile->vms); } @@ -1662,12 +1729,28 @@ void panthor_vm_pool_destroy(struct panthor_file *pfile) */ int panthor_vm_pool_create(struct panthor_file *pfile) { + struct panthor_gem_object *dummy; + int ret; + pfile->vms = kzalloc_obj(*pfile->vms); if (!pfile->vms) return -ENOMEM; xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1); + + dummy = panthor_dummy_bo_create(pfile->ptdev); + if (IS_ERR(dummy)) { + ret = PTR_ERR(dummy); + goto err_destroy_vm_pool; + } + + pfile->vms->dummy = dummy; + return 0; + +err_destroy_vm_pool: + panthor_vm_pool_destroy(pfile); + return ret; } /* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */ @@ -2004,6 +2087,9 @@ static void panthor_vm_free(struct drm_gpuvm *gpuvm) free_io_pgtable_ops(vm->pgtbl_ops); + if (vm->dummy) + drm_gem_object_put(&vm->dummy->base); + drm_mm_takedown(&vm->mm); kfree(vm); } @@ -2163,7 +2249,30 @@ static void panthor_vma_init(struct panthor_vma *vma, u32 flags) #define PANTHOR_VM_MAP_FLAGS \ (DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \ - DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED) + DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \ + DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE) + +static void +panthor_fix_sparse_map_offset(struct drm_gpuva_op_map *op, u32 flags) +{ + if (op && (flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE)) + op->gem.offset = op->va.addr & (SZ_2M - 1); +} + +static int +panthor_vm_exec_map_op(struct panthor_vm *vm, u32 flags, + const struct drm_gpuva_op_map *op) +{ + struct panthor_gem_object *bo = to_panthor_bo(op->gem.obj); + int prot = flags_to_prot(flags); + + if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE) + return panthor_vm_map_sparse(vm, op->va.addr, prot, + bo->dmap.sgt, op->va.range); + + return panthor_vm_map_pages(vm, op->va.addr, prot, bo->dmap.sgt, + op->gem.offset, op->va.range); +} static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv) { @@ -2176,10 +2285,9 @@ static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv) return -EINVAL; panthor_vma_init(vma, op_ctx->flags & PANTHOR_VM_MAP_FLAGS); + panthor_fix_sparse_map_offset(&op->map, vma->flags); - ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags), - op_ctx->map.bo->dmap.sgt, op->map.gem.offset, - op->map.va.range); + ret = panthor_vm_exec_map_op(vm, vma->flags, &op->map); if (ret) { panthor_vm_op_ctx_return_vma(op_ctx, vma); return ret; @@ -2211,6 +2319,8 @@ static void unmap_hugepage_align(const struct drm_gpuva_op_remap *op, u64 *unmap_start, u64 *unmap_range) { + struct panthor_vma *unmap_vma = container_of(op->unmap->va, struct panthor_vma, base); + bool is_sparse = unmap_vma->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE; u64 aligned_unmap_start, aligned_unmap_end, unmap_end; unmap_end = *unmap_start + *unmap_range; @@ -2218,11 +2328,15 @@ unmap_hugepage_align(const struct drm_gpuva_op_remap *op, aligned_unmap_end = ALIGN(unmap_end, SZ_2M); /* If we're dealing with a huge page, make sure the unmap region is - * aligned on the start of the page. + * aligned on the start of the page. If the unmapped VMA stands for + * a sparse mapping, always assume the backing storage is a THP, since + * the overhead of unmapping 2MiB worth of 4KiB pages and remapping + * some of them is offset by the logic of working out whether it's + * the opposite case right below. This also holds true for op->next. */ if (op->prev && aligned_unmap_start < *unmap_start && op->prev->va.addr <= aligned_unmap_start && - iova_mapped_as_huge_page(op->prev, *unmap_start)) { + (is_sparse || iova_mapped_as_huge_page(op->prev, *unmap_start))) { *unmap_range += *unmap_start - aligned_unmap_start; *unmap_start = aligned_unmap_start; } @@ -2232,7 +2346,7 @@ unmap_hugepage_align(const struct drm_gpuva_op_remap *op, */ if (op->next && aligned_unmap_end > unmap_end && op->next->va.addr + op->next->va.range >= aligned_unmap_end && - iova_mapped_as_huge_page(op->next, unmap_end - 1)) { + (is_sparse || iova_mapped_as_huge_page(op->next, unmap_end - 1))) { *unmap_range += aligned_unmap_end - unmap_end; } } @@ -2249,33 +2363,43 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op, drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range); - /* - * ARM IOMMU page table management code disallows partial unmaps of huge pages, - * so when a partial unmap is requested, we must first unmap the entire huge - * page and then remap the difference between the huge page minus the requested - * unmap region. Calculating the right start address and range for the expanded - * unmap operation is the responsibility of the following function. + /* op->remap.prev's BO offset is always the same as the unmap va's, but + * that of op->remap.next must be adjusted so as to remain < SZ_2M */ - unmap_hugepage_align(&op->remap, &unmap_start, &unmap_range); + panthor_fix_sparse_map_offset(op->remap.next, unmap_vma->flags); - /* If the range changed, we might have to lock a wider region to guarantee - * atomicity. panthor_vm_lock_region() bails out early if the new region - * is already part of the locked region, so no need to do this check here. - */ if (!unmap_vma->evicted) { + /* + * ARM IOMMU page table management code disallows partial unmaps of huge pages, + * so when a partial unmap is requested, we must first unmap the entire huge + * page and then remap the difference between the huge page minus the requested + * unmap region. Calculating the right start address and range for the expanded + * unmap operation is the responsibility of the following function. + */ + unmap_hugepage_align(&op->remap, &unmap_start, &unmap_range); + + /* If the range changed, we might have to lock a wider region to guarantee + * atomicity. panthor_vm_lock_region() bails out early if the new region + * is already part of the locked region, so no need to do this check here. + */ panthor_vm_lock_region(vm, unmap_start, unmap_range); panthor_vm_unmap_pages(vm, unmap_start, unmap_range); } if (op->remap.prev) { - struct panthor_gem_object *bo = to_panthor_bo(op->remap.prev->gem.obj); u64 offset = op->remap.prev->gem.offset + unmap_start - op->remap.prev->va.addr; u64 size = op->remap.prev->va.addr + op->remap.prev->va.range - unmap_start; - if (!unmap_vma->evicted) { - ret = panthor_vm_map_pages(vm, unmap_start, - flags_to_prot(unmap_vma->flags), - bo->dmap.sgt, offset, size); + if (!unmap_vma->evicted && size > 0) { + struct drm_gpuva_op_map map_op = { + .va.addr = unmap_start, + .va.range = size, + .gem.obj = op->remap.prev->gem.obj, + .gem.offset = offset, + }; + panthor_fix_sparse_map_offset(&map_op, unmap_vma->flags); + + ret = panthor_vm_exec_map_op(vm, unmap_vma->flags, &map_op); if (ret) return ret; } @@ -2286,14 +2410,19 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op, } if (op->remap.next) { - struct panthor_gem_object *bo = to_panthor_bo(op->remap.next->gem.obj); u64 addr = op->remap.next->va.addr; u64 size = unmap_start + unmap_range - op->remap.next->va.addr; - if (!unmap_vma->evicted) { - ret = panthor_vm_map_pages(vm, addr, flags_to_prot(unmap_vma->flags), - bo->dmap.sgt, op->remap.next->gem.offset, - size); + if (!unmap_vma->evicted && size > 0) { + struct drm_gpuva_op_map map_op = { + .va.addr = addr, + .va.range = size, + .gem.obj = op->remap.next->gem.obj, + .gem.offset = op->remap.next->gem.offset, + }; + panthor_fix_sparse_map_offset(&map_op, unmap_vma->flags); + + ret = panthor_vm_exec_map_op(vm, unmap_vma->flags, &map_op); if (ret) return ret; } @@ -2490,11 +2619,17 @@ static int remap_evicted_vma(struct drm_gpuvm_bo *vm_bo, ret = panthor_vm_lock_region(vm, evicted_vma->base.va.addr, evicted_vma->base.va.range); if (!ret) { - ret = panthor_vm_map_pages(vm, evicted_vma->base.va.addr, - flags_to_prot(evicted_vma->flags), - bo->dmap.sgt, - evicted_vma->base.gem.offset, - evicted_vma->base.va.range); + struct drm_gpuva_op_map map_op = { + .va.addr = evicted_vma->base.va.addr, + .va.range = evicted_vma->base.va.range, + .gem.obj = &bo->base, + .gem.offset = evicted_vma->base.gem.offset, + }; + if (evicted_vma->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE) + drm_WARN_ON_ONCE(&vm->ptdev->base, map_op.gem.offset != + (map_op.va.addr & (SZ_2M - 1))); + + ret = panthor_vm_exec_map_op(vm, evicted_vma->flags, &map_op); if (!ret) evicted_vma->evicted = false; @@ -2769,6 +2904,8 @@ panthor_vm_create(struct panthor_device *ptdev, bool for_mcu, va_range = full_va_range; } + vm->user_va_range = kernel_va_start; + mutex_init(&vm->mm_lock); drm_mm_init(&vm->mm, kernel_va_start, kernel_va_size); vm->kernel_auto_va.start = auto_kernel_va_start; @@ -2857,15 +2994,22 @@ panthor_vm_bind_prepare_op_ctx(struct drm_file *file, if (!IS_ALIGNED(op->va | op->size | op->bo_offset, vm_pgsz)) return -EINVAL; + /* We don't allow mappings that overlap with kbo's reserved range */ + if (range_overflows(op->va, op->size, vm->user_va_range)) + return -EINVAL; + switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) { case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP: - gem = drm_gem_object_lookup(file, op->bo_handle); + if (!(op->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE)) { + gem = drm_gem_object_lookup(file, op->bo_handle); + } else { + gem = &vm->dummy->base; + drm_gem_object_get(&vm->dummy->base); + } + ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm, gem ? to_panthor_bo(gem) : NULL, - op->bo_offset, - op->size, - op->va, - op->flags); + op); drm_gem_object_put(gem); return ret; @@ -3061,10 +3205,19 @@ int panthor_vm_bind_exec_sync_op(struct drm_file *file, int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo, u64 offset, u64 size, u64 va, u32 flags) { + struct drm_panthor_vm_bind_op op = { + .bo_offset = offset, + .size = size, + .va = va, + .flags = flags, + }; struct panthor_vm_op_ctx op_ctx; int ret; - ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags); + if (drm_WARN_ON(&vm->ptdev->base, flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE)) + return -EINVAL; + + ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, &op); if (ret) return ret; diff --git a/drivers/gpu/drm/rockchip/Kconfig b/drivers/gpu/drm/rockchip/Kconfig index e7f49fe845eae..4e58685f58ff9 100644 --- a/drivers/gpu/drm/rockchip/Kconfig +++ b/drivers/gpu/drm/rockchip/Kconfig @@ -68,6 +68,7 @@ config ROCKCHIP_CDN_DP config ROCKCHIP_DW_DP bool "Rockchip specific extensions for Synopsys DW DP" + select DRM_BRIDGE_CONNECTOR help This selects support for Rockchip SoC specific extensions to enable Synopsys DesignWare Cores based DisplayPort transmit @@ -145,6 +146,8 @@ config ROCKCHIP_RGB config ROCKCHIP_RK3066_HDMI bool "Rockchip specific extensions for RK3066 HDMI" depends on DRM_ROCKCHIP + select DRM_DISPLAY_HELPER + select DRM_BRIDGE_CONNECTOR help This selects support for Rockchip SoC specific extensions for the RK3066 HDMI driver. If you want to enable diff --git a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c index 06072efd7fca3..d638785ee5abc 100644 --- a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c +++ b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c @@ -8,6 +8,7 @@ * Jeff Chen */ +#include #include #include #include @@ -206,7 +207,6 @@ static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder, struct drm_crtc *crtc; struct drm_crtc_state *old_crtc_state; struct of_endpoint endpoint; - struct device_node *remote_port, *remote_port_parent; char name[32]; u32 port_id; int ret; @@ -230,18 +230,23 @@ static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder, if (ret < 0) return; - remote_port_parent = of_graph_get_remote_port_parent(endpoint.local_node); + struct device_node *remote_port_parent __free(device_node) = + of_graph_get_remote_port_parent(endpoint.local_node); if (remote_port_parent) { - if (of_get_child_by_name(remote_port_parent, "ports")) { - remote_port = of_graph_get_remote_port(endpoint.local_node); + struct device_node *ports __free(device_node) = + of_get_child_by_name(remote_port_parent, "ports"); + + if (ports) { + struct device_node *remote_port __free(device_node) = + of_graph_get_remote_port(endpoint.local_node); + of_property_read_u32(remote_port, "reg", &port_id); - of_node_put(remote_port); - sprintf(name, "%s vp%d", remote_port_parent->full_name, port_id); + snprintf(name, sizeof(name), "%s vp%d", + remote_port_parent->full_name, port_id); } else { - sprintf(name, "%s %s", - remote_port_parent->full_name, endpoint.id ? "vopl" : "vopb"); + snprintf(name, sizeof(name), "%s %s", + remote_port_parent->full_name, endpoint.id ? "vopl" : "vopb"); } - of_node_put(remote_port_parent); DRM_DEV_DEBUG(dp->dev, "vop %s output to dp\n", (ret) ? "LIT" : "BIG"); } @@ -311,6 +316,7 @@ static int rockchip_dp_of_probe(struct rockchip_dp_device *dp) { struct device *dev = dp->dev; struct device_node *np = dev->of_node; + struct clk *clk; dp->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf"); if (IS_ERR(dp->grf)) @@ -327,6 +333,11 @@ static int rockchip_dp_of_probe(struct rockchip_dp_device *dp) return dev_err_probe(dev, PTR_ERR(dp->pclk), "failed to get pclk property\n"); + clk = devm_clk_get_optional_enabled(dev, "hclk"); + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), + "failed to get hclk property\n"); + dp->rst = devm_reset_control_get(dev, "dp"); if (IS_ERR(dp->rst)) return dev_err_probe(dev, PTR_ERR(dp->rst), diff --git a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c index 32bc73a1d5e45..0de822360c8db 100644 --- a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c @@ -107,15 +107,26 @@ static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void * return PTR_ERR(dp->base); connector = drm_bridge_connector_init(drm_dev, encoder); - if (IS_ERR(connector)) + if (IS_ERR(connector)) { + dw_dp_unbind(dp->base); return dev_err_probe(dev, PTR_ERR(connector), - "Failed to init bridge connector"); + "Failed to init bridge connector\n"); + } return 0; } +static void dw_dp_rockchip_unbind(struct device *dev, struct device *master, + void *data) +{ + struct rockchip_dw_dp *dp = dev_get_drvdata(dev); + + dw_dp_unbind(dp->base); +} + static const struct component_ops dw_dp_rockchip_component_ops = { .bind = dw_dp_rockchip_bind, + .unbind = dw_dp_rockchip_unbind, }; static int dw_dp_probe(struct platform_device *pdev) diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index a160077a507f2..a268bd2fbaaab 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -337,7 +337,8 @@ static bool vop2_output_uv_swap(u32 bus_format, u32 output_mode) static bool vop2_output_rg_swap(struct vop2 *vop2, u32 bus_format) { - if (vop2->version == VOP_VERSION_RK3588) { + if (vop2->version == VOP_VERSION_RK3588 || + vop2->version == VOP_VERSION_RK3576) { if (bus_format == MEDIA_BUS_FMT_YUV8_1X24 || bus_format == MEDIA_BUS_FMT_YUV10_1X30) return true; @@ -351,6 +352,8 @@ static bool is_yuv_output(u32 bus_format) switch (bus_format) { case MEDIA_BUS_FMT_YUV8_1X24: case MEDIA_BUS_FMT_YUV10_1X30: + case MEDIA_BUS_FMT_YUYV10_1X20: + case MEDIA_BUS_FMT_UYVY10_1X20: case MEDIA_BUS_FMT_UYYVYY8_0_5X24: case MEDIA_BUS_FMT_UYYVYY10_0_5X30: case MEDIA_BUS_FMT_YUYV8_2X8: diff --git a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c index 17eda592b1833..322a303d3f1a5 100644 --- a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c +++ b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c @@ -2288,15 +2288,6 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp) * lead to the configuration of the previous VP being take effect along with the VSYNC * of the new VP. */ - if (layer_sel != old_layer_sel || port_sel != old_port_sel) - ovl_ctrl |= FIELD_PREP(RK3568_OVL_CTRL__LAYERSEL_REGDONE_SEL, vp->id); - vop2_writel(vop2, RK3568_OVL_CTRL, ovl_ctrl); - - if (port_sel != old_port_sel) { - vop2_writel(vop2, RK3568_OVL_PORT_SEL, port_sel); - vop2_cfg_done(vp); - rk3568_vop2_wait_for_port_mux_done(vop2); - } if (layer_sel != old_layer_sel && atv_layer_sel != old_layer_sel) { cfg_done = vop2_readl(vop2, RK3568_REG_CFG_DONE); @@ -2306,7 +2297,17 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp) * Changes of other VPs' overlays have not taken effect */ if (cfg_done) - rk3568_vop2_wait_for_layer_cfg_done(vop2, vop2->old_layer_sel); + rk3568_vop2_wait_for_layer_cfg_done(vop2, old_layer_sel); + } + + if (layer_sel != old_layer_sel || port_sel != old_port_sel) + ovl_ctrl |= FIELD_PREP(RK3568_OVL_CTRL__LAYERSEL_REGDONE_SEL, vp->id); + vop2_writel(vop2, RK3568_OVL_CTRL, ovl_ctrl); + + if (port_sel != old_port_sel) { + vop2_writel(vop2, RK3568_OVL_PORT_SEL, port_sel); + vop2_cfg_done(vp); + rk3568_vop2_wait_for_port_mux_done(vop2); } vop2_writel(vop2, RK3568_OVL_LAYER_SEL, layer_sel); diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index 672b5c57ed8e9..fd1824d797066 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -566,9 +566,10 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity) */ smp_wmb(); + spin_lock(&entity->lock); spsc_queue_pop(&entity->job_queue); - drm_sched_rq_pop_entity(entity); + spin_unlock(&entity->lock); /* Jobs and entities might have different lifecycles. Since we're * removing the job from the entities queue, set the jobs entity pointer @@ -654,6 +655,9 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) * Make sure to set the submit_ts first, to avoid a race. */ sched_job->submit_ts = submit_ts = ktime_get(); + + spin_lock(&entity->lock); + first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node); /* first job wakes up scheduler */ @@ -664,5 +668,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) if (sched) drm_sched_wakeup(sched); } + + spin_unlock(&entity->lock); } EXPORT_SYMBOL(drm_sched_entity_push_job); diff --git a/drivers/gpu/drm/scheduler/sched_rq.c b/drivers/gpu/drm/scheduler/sched_rq.c index 0464d324d98d5..23f46ec610e78 100644 --- a/drivers/gpu/drm/scheduler/sched_rq.c +++ b/drivers/gpu/drm/scheduler/sched_rq.c @@ -257,19 +257,17 @@ static ktime_t drm_sched_entity_get_job_ts(struct drm_sched_entity *entity) struct drm_gpu_scheduler * drm_sched_rq_add_entity(struct drm_sched_entity *entity, ktime_t ts) { + struct drm_sched_rq *rq = entity->rq; struct drm_gpu_scheduler *sched; - struct drm_sched_rq *rq; /* Add the entity to the run queue */ - spin_lock(&entity->lock); - if (entity->stopped) { - spin_unlock(&entity->lock); + lockdep_assert_held(&entity->lock); + if (entity->stopped) { DRM_ERROR("Trying to push to a killed entity\n"); return NULL; } - rq = entity->rq; spin_lock(&rq->lock); sched = rq->sched; @@ -289,7 +287,6 @@ drm_sched_rq_add_entity(struct drm_sched_entity *entity, ktime_t ts) drm_sched_rq_update_fifo_locked(entity, rq, ts); spin_unlock(&rq->lock); - spin_unlock(&entity->lock); return sched; } @@ -343,16 +340,17 @@ drm_sched_rq_next_rr_ts(struct drm_sched_rq *rq, */ void drm_sched_rq_pop_entity(struct drm_sched_entity *entity) { + struct drm_sched_rq *rq = entity->rq; struct drm_sched_job *next_job; - struct drm_sched_rq *rq; + + lockdep_assert_held(&entity->lock); + + spin_lock(&rq->lock); /* * Update the entity's location in the min heap according to * the timestamp of the next job, if any. */ - spin_lock(&entity->lock); - rq = entity->rq; - spin_lock(&rq->lock); next_job = drm_sched_entity_queue_peek(entity); if (next_job) { ktime_t ts; @@ -375,8 +373,8 @@ void drm_sched_rq_pop_entity(struct drm_sched_entity *entity) drm_sched_entity_save_vruntime(entity, min_vruntime); } } + spin_unlock(&rq->lock); - spin_unlock(&entity->lock); } /** diff --git a/drivers/gpu/drm/scheduler/tests/mock_scheduler.c b/drivers/gpu/drm/scheduler/tests/mock_scheduler.c index 8e9ae7d980eb2..2dfa3efef2100 100644 --- a/drivers/gpu/drm/scheduler/tests/mock_scheduler.c +++ b/drivers/gpu/drm/scheduler/tests/mock_scheduler.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2025 Valve Corporation */ +#include + #include "sched_tests.h" /* @@ -288,6 +290,7 @@ static const struct drm_sched_backend_ops drm_mock_scheduler_ops = { */ struct drm_mock_scheduler *drm_mock_sched_new(struct kunit *test, long timeout) { + static unsigned int instance; struct drm_sched_init_args args = { .ops = &drm_mock_scheduler_ops, .num_rqs = DRM_SCHED_PRIORITY_COUNT, @@ -297,11 +300,19 @@ struct drm_mock_scheduler *drm_mock_sched_new(struct kunit *test, long timeout) .name = "drm-mock-scheduler", }; struct drm_mock_scheduler *sched; + struct device *dev; + char name[64]; int ret; sched = kunit_kzalloc(test, sizeof(*sched), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, sched); + snprintf(name, sizeof(name), "%s-%u", args.name, instance++); + dev = kunit_device_register(test, name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + args.dev = dev; + ret = drm_sched_init(&sched->base, &args); KUNIT_ASSERT_EQ(test, ret, 0); diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c index cae92a3ae8a41..df67a1759bb3c 100644 --- a/drivers/gpu/drm/solomon/ssd130x.c +++ b/drivers/gpu/drm/solomon/ssd130x.c @@ -864,12 +864,13 @@ static int ssd132x_update_rect(struct ssd130x_device *ssd130x, */ /* Set column start and end */ - ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_COL_RANGE, x / segment_width, columns - 1); + ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_COL_RANGE, x / segment_width, + x / segment_width + columns - 1); if (ret < 0) return ret; /* Set row start and end */ - ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_ROW_RANGE, y, rows - 1); + ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_ROW_RANGE, y, y + rows - 1); if (ret < 0) return ret; @@ -915,12 +916,12 @@ static int ssd133x_update_rect(struct ssd130x_device *ssd130x, */ /* Set column start and end */ - ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_COL_RANGE, x, columns - 1); + ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_COL_RANGE, x, x + columns - 1); if (ret < 0) return ret; /* Set row start and end */ - ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_ROW_RANGE, y, rows - 1); + ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_ROW_RANGE, y, y + rows - 1); if (ret < 0) return ret; @@ -1007,7 +1008,6 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb, struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev); struct iosys_map dst; unsigned int dst_pitch; - int ret = 0; /* Align y to display page boundaries */ rect->y1 = round_down(rect->y1, SSD130X_PAGE_HEIGHT); @@ -1020,7 +1020,7 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb, ssd130x_update_rect(ssd130x, rect, buf, data_array); - return ret; + return 0; } static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb, @@ -1032,7 +1032,6 @@ static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb, struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev); unsigned int dst_pitch; struct iosys_map dst; - int ret = 0; /* Align x to display segment boundaries */ rect->x1 = round_down(rect->x1, SSD132X_SEGMENT_WIDTH); @@ -1046,7 +1045,7 @@ static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb, ssd132x_update_rect(ssd130x, rect, buf, data_array); - return ret; + return 0; } static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb, @@ -1058,7 +1057,6 @@ static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb, const struct drm_format_info *fi = drm_format_info(DRM_FORMAT_RGB332); unsigned int dst_pitch; struct iosys_map dst; - int ret = 0; if (!fi) return -EINVAL; @@ -1070,7 +1068,7 @@ static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb, ssd133x_update_rect(ssd130x, rect, data_array, dst_pitch); - return ret; + return 0; } static int ssd130x_primary_plane_atomic_check(struct drm_plane *plane, diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c index 03e46a2af6463..adf5b6e09ca64 100644 --- a/drivers/gpu/drm/sun4i/sun4i_backend.c +++ b/drivers/gpu/drm/sun4i/sun4i_backend.c @@ -686,7 +686,7 @@ static int sun4i_backend_init_sat(struct device *dev) { ret = clk_prepare_enable(backend->sat_clk); if (ret) { dev_err(dev, "Couldn't enable the SAT clock\n"); - return ret; + goto err_assert_reset; } return 0; diff --git a/drivers/gpu/drm/sun4i/sun4i_crtc.c b/drivers/gpu/drm/sun4i/sun4i_crtc.c index c2f7f69baf75a..3fa1f4fda0f0d 100644 --- a/drivers/gpu/drm/sun4i/sun4i_crtc.c +++ b/drivers/gpu/drm/sun4i/sun4i_crtc.c @@ -208,7 +208,7 @@ struct sun4i_crtc *sun4i_crtc_init(struct drm_device *drm, planes = sunxi_engine_layers_init(drm, engine); if (IS_ERR(planes)) { dev_err(drm->dev, "Couldn't create the planes\n"); - return NULL; + return ERR_CAST(planes); } /* find primary and cursor planes for drm_crtc_init_with_planes */ diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c index 8a409eee1dca5..e9baca360160b 100644 --- a/drivers/gpu/drm/sun4i/sun4i_drv.c +++ b/drivers/gpu/drm/sun4i/sun4i_drv.c @@ -289,6 +289,8 @@ static void sun4i_drv_traverse_endpoints(struct endpoint_list *list, kfifo_put(&list->fifo, remote); } + + of_node_put(port); } static int sun4i_drv_add_endpoints(struct device *dev, @@ -394,6 +396,7 @@ static int sun4i_drv_probe(struct platform_device *pdev) /* process this endpoint */ ret = sun4i_drv_add_endpoints(&pdev->dev, &list, &match, endpoint); + of_node_put(endpoint); /* sun4i_drv_add_endpoints can fail to allocate memory */ if (ret < 0) diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c index 74c7c3720ba82..a27d394d05708 100644 --- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c +++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c @@ -161,7 +161,7 @@ static void sun4i_hdmi_enable(struct drm_encoder *encoder, drm_atomic_helper_connector_hdmi_update_infoframes(connector, state); - val |= SUN4I_HDMI_PKT_CTRL_TYPE(0, SUN4I_HDMI_PKT_AVI); + val = SUN4I_HDMI_PKT_CTRL_TYPE(0, SUN4I_HDMI_PKT_AVI); val |= SUN4I_HDMI_PKT_CTRL_TYPE(1, SUN4I_HDMI_PKT_END); writel(val, hdmi->base + SUN4I_HDMI_PKT_CTRL_REG(0)); diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c index 960e83c8291da..67047e3b657ff 100644 --- a/drivers/gpu/drm/sun4i/sun4i_tcon.c +++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c @@ -717,9 +717,11 @@ void sun4i_tcon_mode_set(struct sun4i_tcon *tcon, case DRM_MODE_ENCODER_DSI: /* DSI is tied to special case of CPU interface */ sun4i_tcon0_mode_set_cpu(tcon, encoder, mode); + sun4i_tcon_set_mux(tcon, 0, encoder); break; case DRM_MODE_ENCODER_LVDS: sun4i_tcon0_mode_set_lvds(tcon, encoder, mode); + sun4i_tcon_set_mux(tcon, 0, encoder); break; case DRM_MODE_ENCODER_NONE: sun4i_tcon0_mode_set_rgb(tcon, encoder, mode); @@ -970,6 +972,7 @@ static int sun4i_tcon_of_get_id_from_port(struct device_node *port) continue; ret = of_property_read_u32(remote, "reg", ®); + of_node_put(remote); if (ret) continue; @@ -1407,7 +1410,7 @@ static int sun8i_r40_tcon_tv_set_mux(struct sun4i_tcon *tcon, { struct device_node *port, *remote; struct platform_device *pdev; - int id, ret; + int id, ret = 0; /* find TCON TOP platform device and TCON id */ @@ -1430,21 +1433,20 @@ static int sun8i_r40_tcon_tv_set_mux(struct sun4i_tcon *tcon, if (IS_ENABLED(CONFIG_DRM_SUN8I_TCON_TOP) && encoder->encoder_type == DRM_MODE_ENCODER_TMDS) { ret = sun8i_tcon_top_set_hdmi_src(&pdev->dev, id); - if (ret) { - put_device(&pdev->dev); - return ret; - } + if (ret) + goto out_put_device; } if (IS_ENABLED(CONFIG_DRM_SUN8I_TCON_TOP)) { ret = sun8i_tcon_top_de_config(&pdev->dev, tcon->id, id); - if (ret) { - put_device(&pdev->dev); - return ret; - } + if (ret) + goto out_put_device; } - return 0; +out_put_device: + put_device(&pdev->dev); + + return ret; } static const struct sun4i_tcon_quirks sun4i_a10_quirks = { diff --git a/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c b/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c index 96532709c2a7e..51cdfa1ae3bf4 100644 --- a/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c +++ b/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c @@ -83,6 +83,8 @@ static u32 sun8i_dw_hdmi_find_possible_crtcs(struct drm_device *drm, of_node_put(remote_port); } } + + of_node_put(port); } else { crtcs = drm_of_find_possible_crtcs(drm, node); } diff --git a/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c b/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c index 4fa69c463dc46..489ea94693ffa 100644 --- a/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c +++ b/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c @@ -91,7 +91,7 @@ static const struct dw_hdmi_mpll_config sun50i_h6_mpll_cfg[] = { }, }, { 594000000, { - { 0x1a40, 0x0003 }, + { 0x1a7c, 0x0003 }, { 0x3b4c, 0x0003 }, { 0x5a64, 0x0003 }, }, diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.c b/drivers/gpu/drm/sun4i/sun8i_mixer.c index 4e42262950104..b6cd7352e76c5 100644 --- a/drivers/gpu/drm/sun4i/sun8i_mixer.c +++ b/drivers/gpu/drm/sun4i/sun8i_mixer.c @@ -785,7 +785,7 @@ static const struct sun8i_mixer_cfg sun8i_v3s_mixer_cfg = { .de_type = SUN8I_MIXER_DE2, .vi_scaler_num = 2, .scaler_mask = 0x3, - .scanline_yuv = 2048, + .scanline_yuv = 1024, }, .de_type = SUN8I_MIXER_DE2, .mod_rate = 150000000, diff --git a/drivers/gpu/drm/sun4i/sun8i_tcon_top.c b/drivers/gpu/drm/sun4i/sun8i_tcon_top.c index 8adda578c51ba..9cbd655518b20 100644 --- a/drivers/gpu/drm/sun4i/sun8i_tcon_top.c +++ b/drivers/gpu/drm/sun4i/sun8i_tcon_top.c @@ -25,6 +25,49 @@ static bool sun8i_tcon_top_node_is_tcon_top(struct device_node *node) return !!of_match_node(sun8i_tcon_top_of_table, node); } +static unsigned int sun8i_tcon_top_get_tcon_map(struct device_node *node) +{ + static const u32 out_ports[] = { + TCON_TOP_MIXER0_OUT_PORT, + TCON_TOP_MIXER1_OUT_PORT, + }; + unsigned int i, map = 0; + + for (i = 0; i < ARRAY_SIZE(out_ports); i++) { + struct device_node *port; + + port = of_graph_get_port_by_id(node, out_ports[i]); + if (!port) + continue; + + for_each_of_graph_port_endpoint(port, ep) { + struct of_endpoint endpoint; + + if (of_graph_parse_endpoint(ep, &endpoint)) + continue; + + if (endpoint.id < TCON_TOP_PORT_TCON_NUM) + map |= BIT(endpoint.id); + } + + of_node_put(port); + } + + return map; +} + +static unsigned int sun8i_tcon_top_park_index(struct sun8i_tcon_top *tcon_top, + int tcon) +{ + unsigned int candidates; + + candidates = tcon_top->tcon_map & ~BIT(tcon); + if (!candidates) + candidates = GENMASK(TCON_TOP_PORT_TCON_NUM - 1, 0) & ~BIT(tcon); + + return ffs(candidates) - 1; +} + int sun8i_tcon_top_set_hdmi_src(struct device *dev, int tcon) { struct sun8i_tcon_top *tcon_top = dev_get_drvdata(dev); @@ -57,6 +100,7 @@ EXPORT_SYMBOL(sun8i_tcon_top_set_hdmi_src); int sun8i_tcon_top_de_config(struct device *dev, int mixer, int tcon) { struct sun8i_tcon_top *tcon_top = dev_get_drvdata(dev); + u32 mixer_msk, other_msk; unsigned long flags; u32 reg; @@ -70,21 +114,27 @@ int sun8i_tcon_top_de_config(struct device *dev, int mixer, int tcon) return -EINVAL; } - if (tcon > 3) { - dev_err(dev, "TCON index is too high!\n"); + if (tcon < 0 || tcon >= TCON_TOP_PORT_TCON_NUM) { + dev_err(dev, "TCON index is invalid!\n"); return -EINVAL; } + mixer_msk = mixer ? TCON_TOP_PORT_DE1_MSK : TCON_TOP_PORT_DE0_MSK; + other_msk = mixer ? TCON_TOP_PORT_DE0_MSK : TCON_TOP_PORT_DE1_MSK; + spin_lock_irqsave(&tcon_top->reg_lock, flags); reg = readl(tcon_top->regs + TCON_TOP_PORT_SEL_REG); - if (mixer == 0) { - reg &= ~TCON_TOP_PORT_DE0_MSK; - reg |= FIELD_PREP(TCON_TOP_PORT_DE0_MSK, tcon); - } else { - reg &= ~TCON_TOP_PORT_DE1_MSK; - reg |= FIELD_PREP(TCON_TOP_PORT_DE1_MSK, tcon); + + reg &= ~mixer_msk; + reg |= field_prep(mixer_msk, tcon); + + if (field_get(other_msk, reg) == tcon) { + reg &= ~other_msk; + reg |= field_prep(other_msk, + sun8i_tcon_top_park_index(tcon_top, tcon)); } + writel(reg, tcon_top->regs + TCON_TOP_PORT_SEL_REG); spin_unlock_irqrestore(&tcon_top->reg_lock, flags); @@ -143,6 +193,7 @@ static int sun8i_tcon_top_bind(struct device *dev, struct device *master, return -ENOMEM; clk_data->num = CLK_NUM; tcon_top->clk_data = clk_data; + tcon_top->tcon_map = sun8i_tcon_top_get_tcon_map(dev->of_node); spin_lock_init(&tcon_top->reg_lock); @@ -175,10 +226,6 @@ static int sun8i_tcon_top_bind(struct device *dev, struct device *master, goto err_assert_reset; } - /* - * At least on H6, some registers have some bits set by default - * which may cause issues. Clear them here. - */ writel(0, regs + TCON_TOP_PORT_SEL_REG); writel(0, regs + TCON_TOP_GATE_SRC_REG); diff --git a/drivers/gpu/drm/sun4i/sun8i_tcon_top.h b/drivers/gpu/drm/sun4i/sun8i_tcon_top.h index 0390584a330e5..2b887470a49f8 100644 --- a/drivers/gpu/drm/sun4i/sun8i_tcon_top.h +++ b/drivers/gpu/drm/sun4i/sun8i_tcon_top.h @@ -14,6 +14,10 @@ #define TCON_TOP_PORT_SEL_REG 0x1C #define TCON_TOP_PORT_DE0_MSK GENMASK(1, 0) #define TCON_TOP_PORT_DE1_MSK GENMASK(5, 4) +#define TCON_TOP_PORT_TCON_NUM 4 + +#define TCON_TOP_MIXER0_OUT_PORT 1 +#define TCON_TOP_MIXER1_OUT_PORT 3 #define TCON_TOP_GATE_SRC_REG 0x20 #define TCON_TOP_HDMI_SRC_MSK GENMASK(29, 28) @@ -29,6 +33,8 @@ struct sun8i_tcon_top { void __iomem *regs; struct reset_control *rst; + unsigned int tcon_map; + /* * spinlock is used to synchronize access to same * register where multiple clock gates can be set. diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c index 3dec4eeb1ba21..a5cc99981006a 100644 --- a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c +++ b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c @@ -894,20 +894,26 @@ static void sun8i_vi_scaler_set_coeff(struct regmap *map, u32 base, lan3coefftab32_left[offset + i]); regmap_write(map, SUN8I_SCALER_VSU_YHCOEFF1(base, i), lan3coefftab32_right[offset + i]); + } + offset = sun8i_vi_scaler_coef_index(vstep) * + SUN8I_VI_SCALER_COEFF_COUNT; + for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) + regmap_write(map, SUN8I_SCALER_VSU_YVCOEFF(base, i), + lan2coefftab32[offset + i]); + + offset = sun8i_vi_scaler_coef_index(hstep / format->hsub) * + SUN8I_VI_SCALER_COEFF_COUNT; + for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) { regmap_write(map, SUN8I_SCALER_VSU_CHCOEFF0(base, i), ch_left[offset + i]); regmap_write(map, SUN8I_SCALER_VSU_CHCOEFF1(base, i), ch_right[offset + i]); } - - offset = sun8i_vi_scaler_coef_index(hstep) * + offset = sun8i_vi_scaler_coef_index(vstep / format->vsub) * SUN8I_VI_SCALER_COEFF_COUNT; - for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) { - regmap_write(map, SUN8I_SCALER_VSU_YVCOEFF(base, i), - lan2coefftab32[offset + i]); + for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) regmap_write(map, SUN8I_SCALER_VSU_CVCOEFF(base, i), cy[offset + i]); - } } void sun8i_vi_scaler_enable(struct sun8i_layer *layer, bool enable) @@ -969,6 +975,8 @@ void sun8i_vi_scaler_setup(struct sun8i_layer *layer, regmap_write(layer->regs, SUN50I_SCALER_VSU_SCALE_MODE(base), val); + regmap_write(layer->regs, + SUN50I_SCALER_VSU_GLB_ALPHA(base), 0xff); } regmap_write(layer->regs, diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h index 245fe2f431c3c..d89c0cbe188dd 100644 --- a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h +++ b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h @@ -36,6 +36,7 @@ #define SUN50I_SCALER_VSU_EDSCL_CTRL(base) ((base) + 0x28) #define SUN50I_SCALER_VSU_ANGLE_THR(base) ((base) + 0x2c) #define SUN8I_SCALER_VSU_OUTSIZE(base) ((base) + 0x40) +#define SUN50I_SCALER_VSU_GLB_ALPHA(base) ((base) + 0x44) #define SUN8I_SCALER_VSU_YINSIZE(base) ((base) + 0x80) #define SUN8I_SCALER_VSU_YHSTEP(base) ((base) + 0x88) #define SUN8I_SCALER_VSU_YVSTEP(base) ((base) + 0x8c) diff --git a/drivers/gpu/drm/sysfb/ofdrm.c b/drivers/gpu/drm/sysfb/ofdrm.c index 819aed466727e..9d60db45139c3 100644 --- a/drivers/gpu/drm/sysfb/ofdrm.c +++ b/drivers/gpu/drm/sysfb/ofdrm.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -238,7 +239,7 @@ static bool is_avivo(u32 vendor, u32 device) /* This will match most R5xx */ return (vendor == PCI_VENDOR_ID_ATI) && ((device >= PCI_VENDOR_ID_ATI_R520 && device < 0x7800) || - (PCI_VENDOR_ID_ATI_R600 >= 0x9400)); + (device >= PCI_VENDOR_ID_ATI_R600)); } static enum ofdrm_model display_get_model_of(struct drm_device *dev, struct device_node *of_node) @@ -913,7 +914,10 @@ static struct ofdrm_device *ofdrm_device_create(struct drm_driver *drv, return ERR_PTR(-EINVAL); } - fb_size = linebytes * height; + if (check_mul_overflow(linebytes, height, &fb_size)) { + drm_err(dev, "framebuffer size exceeds maximum\n"); + return ERR_PTR(-EINVAL); + } /* * Try to figure out the address of the framebuffer. Unfortunately, Open diff --git a/drivers/gpu/drm/sysfb/simpledrm.c b/drivers/gpu/drm/sysfb/simpledrm.c index fc168920f2c64..ca8a8d86094e4 100644 --- a/drivers/gpu/drm/sysfb/simpledrm.c +++ b/drivers/gpu/drm/sysfb/simpledrm.c @@ -48,13 +48,6 @@ simplefb_get_validated_int(struct drm_device *dev, const char *name, return drm_sysfb_get_validated_int(dev, name, value, INT_MAX); } -static int -simplefb_get_validated_int0(struct drm_device *dev, const char *name, - uint32_t value) -{ - return drm_sysfb_get_validated_int0(dev, name, value, INT_MAX); -} - static const struct drm_format_info * simplefb_get_validated_format(struct drm_device *dev, const char *format_name) { @@ -88,14 +81,14 @@ static int simplefb_get_width_pd(struct drm_device *dev, const struct simplefb_platform_data *pd) { - return simplefb_get_validated_int0(dev, "width", pd->width); + return drm_sysfb_get_validated_int0(dev, "width", pd->width, U16_MAX); } static int simplefb_get_height_pd(struct drm_device *dev, const struct simplefb_platform_data *pd) { - return simplefb_get_validated_int0(dev, "height", pd->height); + return drm_sysfb_get_validated_int0(dev, "height", pd->height, U16_MAX); } static int @@ -144,7 +137,7 @@ simplefb_get_width_of(struct drm_device *dev, struct device_node *of_node) if (ret) return ret; - return simplefb_get_validated_int0(dev, "width", width); + return drm_sysfb_get_validated_int0(dev, "width", width, U16_MAX); } static int @@ -155,7 +148,7 @@ simplefb_get_height_of(struct drm_device *dev, struct device_node *of_node) if (ret) return ret; - return simplefb_get_validated_int0(dev, "height", height); + return drm_sysfb_get_validated_int0(dev, "height", height, U16_MAX); } static int @@ -200,6 +193,39 @@ simplefb_get_memory_of(struct drm_device *dev, struct device_node *of_node) return res; } +static int __simplefb_get_panel_size_mm_of(struct drm_device *dev, + struct device_node *of_panel_node, + const char *name) +{ + int ret; + u32 value; + + ret = of_property_read_u32(of_panel_node, name, &value); + if (ret) { + drm_dbg(dev, "simplefb: cannot parse panel %s: error %d\n", + name, ret); + return ret; + } else if (value > U16_MAX) { + drm_dbg(dev, "simplefb: panel %s of %u exceeds maximum value\n", + name, value); + return -EINVAL; + } + + return value; +} + +static int simplefb_get_panel_width_mm_of(struct drm_device *dev, + struct device_node *of_panel_node) +{ + return __simplefb_get_panel_size_mm_of(dev, of_panel_node, "width-mm"); +} + +static int simplefb_get_panel_height_mm_of(struct drm_device *dev, + struct device_node *of_panel_node) +{ + return __simplefb_get_panel_size_mm_of(dev, of_panel_node, "height-mm"); +} + /* * Simple Framebuffer device */ @@ -601,7 +627,7 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv, struct drm_sysfb_device *sysfb; struct drm_device *dev; int width, height, stride; - int width_mm = 0, height_mm = 0; + u16 width_mm = 0, height_mm = 0; struct device_node *panel_node; const struct drm_format_info *format; struct resource *res, *mem = NULL; @@ -665,8 +691,18 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv, return ERR_CAST(mem); panel_node = of_parse_phandle(of_node, "panel", 0); if (panel_node) { - simplefb_read_u32_of(dev, panel_node, "width-mm", &width_mm); - simplefb_read_u32_of(dev, panel_node, "height-mm", &height_mm); + /* + * Ignore errors from parsing the physical panel + * size. Using the pre-initialized sizes of 0 will + * make drm_sysfb_mode() calculate a default physical + * size based on a resolution of 96 dpi. + */ + ret = simplefb_get_panel_width_mm_of(dev, panel_node); + if (ret > 0) + width_mm = ret; + ret = simplefb_get_panel_height_mm_of(dev, panel_node); + if (ret > 0) + height_mm = ret; of_node_put(panel_node); } } else { @@ -674,9 +710,15 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv, return ERR_PTR(-ENODEV); } if (!stride) { - stride = drm_format_info_min_pitch(format, 0, width); - if (drm_WARN_ON(dev, !stride)) + u64 pitch = drm_format_info_min_pitch(format, 0, width); + + if (drm_WARN_ON(dev, !pitch)) { + return ERR_PTR(-EINVAL); /* driver bug */ + } else if (pitch > INT_MAX) { + drm_warn(dev, "stride of %llu exceeds maximum\n", pitch); return ERR_PTR(-EINVAL); + } + stride = pitch; } sysfb->fb_mode = drm_sysfb_mode(width, height, width_mm, height_mm); diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c index 7f25c50621c94..690e4488de3f6 100644 --- a/drivers/gpu/drm/tegra/dsi.c +++ b/drivers/gpu/drm/tegra/dsi.c @@ -922,6 +922,15 @@ static void tegra_dsi_encoder_enable(struct drm_encoder *encoder) return; } + /* If the bootloader enabled DSI it needs to be disabled + * in order for the panel initialization commands to be + * properly sent. + */ + value = tegra_dsi_readl(dsi, DSI_POWER_CONTROL); + + if (value & DSI_POWER_CONTROL_ENABLE) + tegra_dsi_disable(dsi); + state = tegra_dsi_get_state(dsi); tegra_dsi_set_timeout(dsi, state->bclk, state->vrefresh); diff --git a/drivers/gpu/drm/tiny/cirrus-qemu.c b/drivers/gpu/drm/tiny/cirrus-qemu.c index 075221b431d37..3bf23fcf65749 100644 --- a/drivers/gpu/drm/tiny/cirrus-qemu.c +++ b/drivers/gpu/drm/tiny/cirrus-qemu.c @@ -582,6 +582,9 @@ static int cirrus_pci_probe(struct pci_dev *pdev, struct cirrus_device *cirrus; int ret; + if (pci_resource_len(pdev, 0) < CIRRUS_VRAM_SIZE) + return -ENODEV; + ret = aperture_remove_conflicting_pci_devices(pdev, cirrus_driver.name); if (ret) return ret; diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index bcd76f6bb7f02..ca6fc5cddacbf 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1178,7 +1178,7 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) if (ttm_tt_is_populated(tt)) { ret = ttm_tt_swapout(bdev, tt, swapout_walk->gfp_flags); - if (!ret) { + if (ret > 0) { spin_lock(&bdev->lru_lock); ttm_resource_del_bulk_move_unevictable(bo->resource, bo); ttm_resource_move_to_lru_tail(bo->resource); diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index e7648ad3e6159..6142f90d43e2e 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -917,6 +917,7 @@ int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt, { struct ttm_pool_tt_restore *restore = tt->restore; struct ttm_pool_alloc_state alloc; + int ret; if (WARN_ON(!ttm_tt_is_backed_up(tt))) return -EINVAL; @@ -940,14 +941,22 @@ int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt, } else { alloc = restore->snapshot_alloc; if (ttm_pool_restore_valid(restore)) { - int ret = ttm_pool_restore_commit(restore, tt->backup, - ctx, &alloc); + ret = ttm_pool_restore_commit(restore, tt->backup, + ctx, &alloc); if (ret) return ret; } - if (!alloc.remaining_pages) + if (!alloc.remaining_pages) { + ret = ttm_pool_apply_caching(&alloc); + if (ret) + return ret; + + kfree(tt->restore); + tt->restore = NULL; + return 0; + } } return __ttm_pool_alloc(pool, tt, ctx, &alloc, restore); diff --git a/drivers/gpu/drm/tve200/tve200_drv.c b/drivers/gpu/drm/tve200/tve200_drv.c index 562f3f11812a3..f5ef468538f9b 100644 --- a/drivers/gpu/drm/tve200/tve200_drv.c +++ b/drivers/gpu/drm/tve200/tve200_drv.c @@ -263,6 +263,7 @@ static const struct of_device_id tve200_of_match[] = { }, {}, }; +MODULE_DEVICE_TABLE(of, tve200_of_match); static struct platform_driver tve200_driver = { .driver = { diff --git a/drivers/gpu/drm/v3d/Kconfig b/drivers/gpu/drm/v3d/Kconfig index ce62c5908e1db..6a33e0ab30de6 100644 --- a/drivers/gpu/drm/v3d/Kconfig +++ b/drivers/gpu/drm/v3d/Kconfig @@ -5,6 +5,7 @@ config DRM_V3D depends on DRM depends on COMMON_CLK depends on MMU + select DRM_EXEC select DRM_SCHED select DRM_GEM_SHMEM_HELPER help diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index 8779a42c65081..c64ba176b81cf 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -7,7 +7,8 @@ #include #include -#include +#include +#include #include #include #include @@ -294,6 +295,27 @@ to_v3d_fence(struct dma_fence *fence) #define V3D_CORE_READ(core, offset) readl(v3d->core_regs[core] + offset) #define V3D_CORE_WRITE(core, offset, val) writel(val, v3d->core_regs[core] + offset) +#define V3D_MAX_JOBS_PER_SUBMISSION 3 + +/* Per-ioctl submission context */ +struct v3d_submit { + struct v3d_dev *v3d; + + struct drm_file *file_priv; + + /* DRM exec context for this submission. */ + struct drm_exec exec; + + /* Ordered array of jobs forming the submission chain. Jobs are + * appended via v3d_submit_add_job(), then chained and pushed to + * the scheduler by v3d_submit_jobs(). + */ + struct v3d_job *jobs[V3D_MAX_JOBS_PER_SUBMISSION]; + + /* Number of jobs currently in @jobs. */ + u32 job_count; +}; + struct v3d_job { struct drm_sched_job base; @@ -301,6 +323,9 @@ struct v3d_job { struct v3d_dev *v3d; + /* The queue that the job was submitted on. */ + enum v3d_queue queue; + /* This is the array of BOs that were looked up at the start * of submission. */ @@ -333,6 +358,11 @@ struct v3d_job { void (*free)(struct kref *ref); bool has_pm_ref; + + /* Whether the job needs implicit dependencies, i.e. must wait for + * other contexts still writing its BOs. + */ + bool has_implicit_dep; }; struct v3d_bin_job { @@ -410,6 +440,11 @@ struct v3d_indirect_csd_info { /* Clean cache job associated to the Indirect CSD job */ struct v3d_job *clean_job; + /* Indirect CSD args, stashed by the extension parser and later used + * to create the CSD job from them. + */ + struct drm_v3d_submit_csd args; + /* Offset within the BO where the workgroup counts are stored */ u32 offset; @@ -425,7 +460,7 @@ struct v3d_indirect_csd_info { struct drm_gem_object *indirect; /* Context of the Indirect CSD job */ - struct ww_acquire_ctx acquire_ctx; + struct drm_exec exec; }; struct v3d_timestamp_query_info { diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index bea46298b69e8..663561ee8d818 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -189,15 +189,11 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) struct v3d_dev *v3d = job->base.v3d; struct v3d_queue_state *queue = &v3d->queue[V3D_BIN]; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; unsigned long irqflags; - if (unlikely(job->base.base.s_fence->finished.error)) { - spin_lock_irqsave(&queue->queue_lock, irqflags); - queue->active_job = NULL; - spin_unlock_irqrestore(&queue->queue_lock, irqflags); - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; /* Lock required around bin_job update vs * v3d_overflow_mem_work(). @@ -214,7 +210,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_BIN); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -242,6 +238,12 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end); return fence; + +out_clean_job: + spin_lock_irqsave(&queue->queue_lock, irqflags); + queue->active_job = NULL; + spin_unlock_irqrestore(&queue->queue_lock, irqflags); + return fence; } static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) @@ -249,12 +251,10 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) struct v3d_render_job *job = to_render_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_RENDER].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; v3d->queue[V3D_RENDER].active_job = &job->base; @@ -268,7 +268,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_RENDER); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -289,6 +289,10 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end); return fence; + +out_clean_job: + v3d->queue[V3D_RENDER].active_job = NULL; + return fence; } static struct dma_fence * @@ -297,18 +301,16 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job) struct v3d_tfu_job *job = to_tfu_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_TFU].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; v3d->queue[V3D_TFU].active_job = &job->base; fence = v3d_fence_create(v3d, V3D_TFU); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -336,6 +338,10 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job) V3D_WRITE(V3D_TFU_ICFG(v3d->ver), job->args.icfg | V3D_TFU_ICFG_IOC); return fence; + +out_clean_job: + v3d->queue[V3D_TFU].active_job = NULL; + return fence; } static struct dma_fence * @@ -344,13 +350,11 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) struct v3d_csd_job *job = to_csd_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; int i, csd_cfg0_reg; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_CSD].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; /* The HW interprets a workgroup size of 0 as 65536; however, the * user-space driver exposes a maximum of 65535. Therefore, a 0 in @@ -368,7 +372,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_CSD); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -395,6 +399,10 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, csd_cfg0_reg, job->args.cfg[0]); return fence; + +out_clean_job: + v3d->queue[V3D_CSD].active_job = NULL; + return fence; } static void diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index 7682b24f13ec5..fdf27d7205877 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -20,37 +20,57 @@ * to v3d, so we don't attach dma-buf fences to them. */ static int -v3d_lock_bo_reservations(struct v3d_job *job, - struct ww_acquire_ctx *acquire_ctx) +v3d_submit_lock_reservations(struct v3d_submit *submit) { - int i, ret; - - ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx); - if (ret) - return ret; - - for (i = 0; i < job->bo_count; i++) { - ret = dma_resv_reserve_fences(job->bo[i]->resv, 1); + int i, j, ret; + + drm_exec_init(&submit->exec, + DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 0); + drm_exec_until_all_locked(&submit->exec) { + for (i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + ret = drm_exec_prepare_array(&submit->exec, job->bo, + job->bo_count, 1); + if (ret) + break; + } + drm_exec_retry_on_contention(&submit->exec); if (ret) goto fail; + } - ret = drm_sched_job_add_implicit_dependencies(&job->base, - job->bo[i], true); - if (ret) - goto fail; + for (i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + if (!job->has_implicit_dep) + continue; + + for (j = 0; j < job->bo_count; j++) { + ret = drm_sched_job_add_implicit_dependencies(&job->base, + job->bo[j], + true); + if (ret) + goto fail; + } } return 0; fail: - drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); + drm_exec_fini(&submit->exec); return ret; } +static void +v3d_submit_unlock_reservations(struct v3d_submit *submit) +{ + drm_exec_fini(&submit->exec); +} + /** * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects * referenced by the job. - * @dev: DRM device * @file_priv: DRM file for this fd * @job: V3D job being set up * @bo_handles: GEM handles @@ -64,27 +84,46 @@ v3d_lock_bo_reservations(struct v3d_job *job, * failure, because that will happen at `v3d_job_free()`. */ static int -v3d_lookup_bos(struct drm_device *dev, - struct drm_file *file_priv, - struct v3d_job *job, - u64 bo_handles, - u32 bo_count) +v3d_lookup_bos(struct drm_file *file_priv, struct v3d_job *job, + u64 bo_handles, u32 bo_count) { - job->bo_count = bo_count; - - if (!job->bo_count) { - /* See comment on bo_index for why we have to check - * this. - */ - drm_warn(dev, "Rendering requires BOs\n"); + if (!bo_count) { + drm_warn(&job->v3d->drm, "Rendering requires BOs\n"); return -EINVAL; } + job->bo_count = bo_count; + return drm_gem_objects_lookup(file_priv, (void __user *)(uintptr_t)bo_handles, job->bo_count, &job->bo); } +/** + * v3d_job_reference_bos() - Share another job's BOs with @dst + * @dst: job that acquires references to the BOs + * @src: job whose already-resolved BO list is shared + * + * For submissions with multiple jobs that use the same BOs, a trailing job + * shouldn't look the handles up again, as it could cause inconsistencies. + * Instead, it should reference the previous job's BOs. + */ +static int +v3d_job_reference_bos(struct v3d_job *dst, struct v3d_job *src) +{ + dst->bo = kvmalloc_objs(*dst->bo, src->bo_count); + if (!dst->bo) + return -ENOMEM; + + dst->bo_count = src->bo_count; + for (int i = 0; i < dst->bo_count; i++) { + dst->bo[i] = src->bo[i]; + drm_gem_object_get(dst->bo[i]); + } + + return 0; +} + static void v3d_job_free(struct kref *ref) { @@ -162,74 +201,88 @@ void v3d_job_put(struct v3d_job *job) } static int -v3d_job_allocate(struct v3d_dev *v3d, void **container, size_t size) +v3d_job_add_syncobjs(struct v3d_job *job, struct drm_file *file_priv, + u32 in_sync, struct v3d_submit_ext *se) { - *container = kcalloc(1, size, GFP_KERNEL); - if (!*container) { - drm_err(&v3d->drm, "Cannot allocate memory for V3D job.\n"); - return -ENOMEM; + bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); + struct v3d_dev *v3d = job->v3d; + int ret = 0; + + if (!has_multisync) { + ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, + in_sync, 0); + // TODO: Investigate why this was filtered out for the IOCTL. + if (ret && ret != -ENOENT) + return ret; + return 0; + } + + if (se->in_sync_count && se->wait_stage == job->queue) { + struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); + + for (int i = 0; i < se->in_sync_count; i++) { + struct drm_v3d_sem in; + + if (copy_from_user(&in, handle++, sizeof(in))) { + drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); + return -EFAULT; + } + + ret = drm_sched_job_add_syncobj_dependency(&job->base, + file_priv, in.handle, 0); + // TODO: Investigate why this was filtered out for the IOCTL. + if (ret && ret != -ENOENT) + return ret; + } } return 0; } -static void -v3d_job_deallocate(void **container) -{ - kfree(*container); - *container = NULL; -} +static const struct { + size_t size; + void (*free)(struct kref *ref); + bool has_implicit_dep; +} v3d_job_types[] = { + [V3D_BIN] = { sizeof(struct v3d_bin_job), v3d_job_free, false }, + [V3D_RENDER] = { sizeof(struct v3d_render_job), v3d_render_job_free, true }, + [V3D_TFU] = { sizeof(struct v3d_tfu_job), v3d_job_free, true }, + [V3D_CSD] = { sizeof(struct v3d_csd_job), v3d_job_free, true }, + [V3D_CACHE_CLEAN] = { sizeof(struct v3d_job), v3d_job_free, false }, + [V3D_CPU] = { sizeof(struct v3d_cpu_job), v3d_cpu_job_free, true }, +}; -static int -v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, - struct v3d_job *job, void (*free)(struct kref *ref), - u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue) +static struct v3d_job * +v3d_submit_add_job(struct v3d_submit *submit, enum v3d_queue queue) { - struct v3d_file_priv *v3d_priv = file_priv->driver_priv; - bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); - int ret, i; + struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv; + struct v3d_dev *v3d = submit->v3d; + struct v3d_job *job; + int ret; + + if (queue >= V3D_MAX_QUEUES) + return ERR_PTR(-EINVAL); + + job = kzalloc(v3d_job_types[queue].size, GFP_KERNEL); + if (!job) + return ERR_PTR(-ENOMEM); job->v3d = v3d; - job->free = free; + job->queue = queue; job->file_priv = v3d_priv; + job->free = v3d_job_types[queue].free; + job->has_implicit_dep = v3d_job_types[queue].has_implicit_dep; ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], - 1, v3d_priv, file_priv->client_id); + 1, v3d_priv, submit->file_priv->client_id); if (ret) - return ret; - - if (has_multisync) { - if (se->in_sync_count && se->wait_stage == queue) { - struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); - - for (i = 0; i < se->in_sync_count; i++) { - struct drm_v3d_sem in; - - if (copy_from_user(&in, handle++, sizeof(in))) { - ret = -EFAULT; - drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n"); - goto fail_job_init; - } - ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0); - - // TODO: Investigate why this was filtered out for the IOCTL. - if (ret && ret != -ENOENT) - goto fail_job_init; - } - } - } else { - ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0); - - // TODO: Investigate why this was filtered out for the IOCTL. - if (ret && ret != -ENOENT) - goto fail_job_init; - } + goto fail_free; /* CPU jobs don't require hardware resources */ if (queue != V3D_CPU) { ret = v3d_pm_runtime_get(v3d); if (ret) - goto fail_job_init; + goto fail_sched_job; job->has_pm_ref = true; } @@ -238,52 +291,86 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, job->client_stats = v3d_stats_get(v3d_priv->stats[queue]); job->global_stats = v3d_stats_get(v3d->queue[queue].stats); - return 0; + submit->jobs[submit->job_count++] = job; -fail_job_init: + return job; + +fail_sched_job: drm_sched_job_cleanup(&job->base); - return ret; +fail_free: + kfree(job); + return ERR_PTR(ret); } static void -v3d_push_job(struct v3d_job *job) +v3d_submit_put_jobs(struct v3d_submit *submit) { - drm_sched_job_arm(&job->base); + for (int i = 0; i < submit->job_count; i++) + v3d_job_put(submit->jobs[i]); +} - job->done_fence = dma_fence_get(&job->base.s_fence->finished); +static void +v3d_submit_cleanup_jobs(struct v3d_submit *submit) +{ + for (int i = 0; i < submit->job_count; i++) + v3d_job_cleanup(submit->jobs[i]); +} - /* put by scheduler job completion */ - kref_get(&job->refcount); +static int +v3d_attach_perfmon_to_jobs(struct v3d_submit *submit, u32 perfmon_id) +{ + struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv; + struct v3d_dev *v3d = submit->v3d; + struct v3d_perfmon *perfmon; - drm_sched_entity_push_job(&job->base); + if (!perfmon_id) + return 0; + + if (v3d->global_perfmon) + return -EAGAIN; + + perfmon = v3d_perfmon_find(v3d_priv, perfmon_id); + if (!perfmon) + return -ENOENT; + + for (int i = 0; i < submit->job_count; i++) { + submit->jobs[i]->perfmon = perfmon; + if (i != 0) + v3d_perfmon_get(perfmon); + } + + return 0; } static void -v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, - struct v3d_job *job, - struct ww_acquire_ctx *acquire_ctx, - u32 out_sync, - struct v3d_submit_ext *se, - struct dma_fence *done_fence) +v3d_attach_fences_and_unlock_reservation(struct v3d_submit *submit, + u32 out_sync, struct v3d_submit_ext *se) { - struct drm_syncobj *sync_out; bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); - int i; + struct v3d_job *last_job = submit->jobs[submit->job_count - 1]; + struct drm_syncobj *sync_out; - for (i = 0; i < job->bo_count; i++) { - /* XXX: Use shared fences for read-only objects. */ - dma_resv_add_fence(job->bo[i]->resv, job->done_fence, - DMA_RESV_USAGE_WRITE); + /* The submission's last fence covers the entire submission. Attach it + * to every BO touched by any job in the submission. + */ + for (int i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + for (int j = 0; j < job->bo_count; j++) { + /* XXX: Use shared fences for read-only objects. */ + dma_resv_add_fence(job->bo[j]->resv, last_job->done_fence, + DMA_RESV_USAGE_WRITE); + } } - drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); + v3d_submit_unlock_reservations(submit); /* Update the return sync object for the job */ /* If it only supports a single signal semaphore*/ if (!has_multisync) { - sync_out = drm_syncobj_find(file_priv, out_sync); + sync_out = drm_syncobj_find(submit->file_priv, out_sync); if (sync_out) { - drm_syncobj_replace_fence(sync_out, done_fence); + drm_syncobj_replace_fence(sync_out, last_job->done_fence); drm_syncobj_put(sync_out); } return; @@ -291,56 +378,91 @@ v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, /* If multiple semaphores extension is supported */ if (se->out_sync_count) { - for (i = 0; i < se->out_sync_count; i++) { + for (int i = 0; i < se->out_sync_count; i++) { drm_syncobj_replace_fence(se->out_syncs[i].syncobj, - done_fence); + last_job->done_fence); drm_syncobj_put(se->out_syncs[i].syncobj); } kvfree(se->out_syncs); } } +static void +v3d_push_job(struct v3d_job *job) +{ + drm_sched_job_arm(&job->base); + + job->done_fence = dma_fence_get(&job->base.s_fence->finished); + + /* put by scheduler job completion */ + kref_get(&job->refcount); + + drm_sched_entity_push_job(&job->base); +} + static int -v3d_setup_csd_jobs_and_bos(struct drm_file *file_priv, - struct v3d_dev *v3d, - struct drm_v3d_submit_csd *args, - struct v3d_csd_job **job, - struct v3d_job **clean_job, - struct v3d_submit_ext *se, - struct ww_acquire_ctx *acquire_ctx) +v3d_submit_jobs(struct v3d_submit *submit, u32 out_sync, + struct v3d_submit_ext *se) { - int ret; + struct v3d_dev *v3d = submit->v3d; + int ret = 0; - ret = v3d_job_allocate(v3d, (void *)job, sizeof(**job)); - if (ret) - return ret; + mutex_lock(&v3d->sched_lock); - ret = v3d_job_init(v3d, file_priv, &(*job)->base, - v3d_job_free, args->in_sync, se, V3D_CSD); - if (ret) { - v3d_job_deallocate((void *)job); - return ret; + for (int i = 0; i < submit->job_count; i++) { + struct v3d_job *job = submit->jobs[i]; + + v3d_push_job(job); + + if (i + 1 < submit->job_count) { + ret = drm_sched_job_add_dependency(&submit->jobs[i + 1]->base, + dma_fence_get(job->done_fence)); + if (ret) + goto err; + } } - ret = v3d_job_allocate(v3d, (void *)clean_job, sizeof(**clean_job)); + mutex_unlock(&v3d->sched_lock); + + v3d_attach_fences_and_unlock_reservation(submit, out_sync, se); + v3d_submit_put_jobs(submit); + + return 0; + +err: + mutex_unlock(&v3d->sched_lock); + return ret; +} + +static int +v3d_setup_csd_jobs_and_bos(struct v3d_submit *submit, + struct drm_v3d_submit_csd *args, + struct v3d_submit_ext *se) +{ + struct v3d_csd_job *job; + struct v3d_job *clean_job; + int ret; + + job = (struct v3d_csd_job *)v3d_submit_add_job(submit, V3D_CSD); + if (IS_ERR(job)) + return PTR_ERR(job); + + ret = v3d_job_add_syncobjs(&job->base, submit->file_priv, args->in_sync, se); if (ret) return ret; - ret = v3d_job_init(v3d, file_priv, *clean_job, - v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); - if (ret) { - v3d_job_deallocate((void *)clean_job); + ret = v3d_lookup_bos(submit->file_priv, &job->base, args->bo_handles, + args->bo_handle_count); + if (ret) return ret; - } - (*job)->args = *args; + job->args = *args; - ret = v3d_lookup_bos(&v3d->drm, file_priv, *clean_job, - args->bo_handles, args->bo_handle_count); - if (ret) - return ret; + clean_job = v3d_submit_add_job(submit, V3D_CACHE_CLEAN); + if (IS_ERR(clean_job)) + return PTR_ERR(clean_job); - return v3d_lock_bo_reservations(*clean_job, acquire_ctx); + return v3d_job_reference_bos(clean_job, &job->base); } static void @@ -489,6 +611,7 @@ v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, } job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD; + info->args = indirect_csd.submit; info->offset = indirect_csd.offset; info->wg_size = indirect_csd.wg_size; memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets, @@ -498,9 +621,7 @@ v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, if (!info->indirect) return -ENOENT; - return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit, - &info->job, &info->clean_job, - NULL, &info->acquire_ctx); + return 0; } /* Get data for the query timestamp job submission. */ @@ -907,18 +1028,15 @@ int v3d_submit_cl_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct v3d_dev *v3d = to_v3d_dev(dev); - struct v3d_file_priv *v3d_priv = file_priv->driver_priv; + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_cl *args = data; struct v3d_submit_ext se = {0}; struct v3d_bin_job *bin = NULL; - struct v3d_render_job *render = NULL; - struct v3d_job *clean_job = NULL; - struct v3d_job *last_job; - struct ww_acquire_ctx acquire_ctx; - int ret = 0; + struct v3d_render_job *render; + struct v3d_job *clean_job; + int ret; - trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end); + trace_v3d_submit_cl_ioctl(dev, args->rcl_start, args->rcl_end); if (args->pad) return -EINVAL; @@ -938,30 +1056,10 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data, } } - ret = v3d_job_allocate(v3d, (void *)&render, sizeof(*render)); - if (ret) - return ret; - - ret = v3d_job_init(v3d, file_priv, &render->base, - v3d_render_job_free, args->in_sync_rcl, &se, V3D_RENDER); - if (ret) { - v3d_job_deallocate((void *)&render); - goto fail; - } - - render->start = args->rcl_start; - render->end = args->rcl_end; - INIT_LIST_HEAD(&render->unref_list); - if (args->bcl_start != args->bcl_end) { - ret = v3d_job_allocate(v3d, (void *)&bin, sizeof(*bin)); - if (ret) - goto fail; - - ret = v3d_job_init(v3d, file_priv, &bin->base, - v3d_job_free, args->in_sync_bcl, &se, V3D_BIN); - if (ret) { - v3d_job_deallocate((void *)&bin); + bin = (struct v3d_bin_job *)v3d_submit_add_job(&submit, V3D_BIN); + if (IS_ERR(bin)) { + ret = PTR_ERR(bin); goto fail; } @@ -970,100 +1068,71 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data, bin->qma = args->qma; bin->qms = args->qms; bin->qts = args->qts; - bin->render = render; - } - if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { - ret = v3d_job_allocate(v3d, (void *)&clean_job, sizeof(*clean_job)); + ret = v3d_job_add_syncobjs(&bin->base, file_priv, args->in_sync_bcl, + &se); if (ret) goto fail; + } - ret = v3d_job_init(v3d, file_priv, clean_job, - v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); - if (ret) { - v3d_job_deallocate((void *)&clean_job); - goto fail; - } - - last_job = clean_job; - } else { - last_job = &render->base; + render = (struct v3d_render_job *)v3d_submit_add_job(&submit, V3D_RENDER); + if (IS_ERR(render)) { + ret = PTR_ERR(render); + goto fail; } - ret = v3d_lookup_bos(dev, file_priv, last_job, - args->bo_handles, args->bo_handle_count); + INIT_LIST_HEAD(&render->unref_list); + render->start = args->rcl_start; + render->end = args->rcl_end; + + if (bin) + bin->render = render; + + ret = v3d_job_add_syncobjs(&render->base, file_priv, args->in_sync_rcl, &se); if (ret) goto fail; - ret = v3d_lock_bo_reservations(last_job, &acquire_ctx); + /* + * We don't associate the BOs with the BIN job. Fences are only + * attached to the last job in the submission chain, and BIN jobs + * don't need implicit dependencies since depending on results from + * another context is not a realistic scenario for binning. + */ + ret = v3d_lookup_bos(submit.file_priv, &render->base, + args->bo_handles, args->bo_handle_count); if (ret) goto fail; - if (args->perfmon_id) { - if (v3d->global_perfmon) { - ret = -EAGAIN; - goto fail_perfmon; - } - - render->base.perfmon = v3d_perfmon_find(v3d_priv, - args->perfmon_id); - - if (!render->base.perfmon) { - ret = -ENOENT; - goto fail_perfmon; + if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { + clean_job = v3d_submit_add_job(&submit, V3D_CACHE_CLEAN); + if (IS_ERR(clean_job)) { + ret = PTR_ERR(clean_job); + goto fail; } - } - mutex_lock(&v3d->sched_lock); - if (bin) { - bin->base.perfmon = render->base.perfmon; - v3d_perfmon_get(bin->base.perfmon); - v3d_push_job(&bin->base); - - ret = drm_sched_job_add_dependency(&render->base.base, - dma_fence_get(bin->base.done_fence)); + ret = v3d_job_reference_bos(clean_job, &render->base); if (ret) - goto fail_unreserve; - } - - v3d_push_job(&render->base); - - if (clean_job) { - struct dma_fence *render_fence = - dma_fence_get(render->base.done_fence); - ret = drm_sched_job_add_dependency(&clean_job->base, - render_fence); - if (ret) - goto fail_unreserve; - clean_job->perfmon = render->base.perfmon; - v3d_perfmon_get(clean_job->perfmon); - v3d_push_job(clean_job); + goto fail; } - mutex_unlock(&v3d->sched_lock); + ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id); + if (ret) + goto fail; - v3d_attach_fences_and_unlock_reservation(file_priv, - last_job, - &acquire_ctx, - args->out_sync, - &se, - last_job->done_fence); + ret = v3d_submit_lock_reservations(&submit); + if (ret) + goto fail; - v3d_job_put(&bin->base); - v3d_job_put(&render->base); - v3d_job_put(clean_job); + ret = v3d_submit_jobs(&submit, args->out_sync, &se); + if (ret) + goto fail_unreserve; return 0; fail_unreserve: - mutex_unlock(&v3d->sched_lock); -fail_perfmon: - drm_gem_unlock_reservations(last_job->bo, - last_job->bo_count, &acquire_ctx); + v3d_submit_unlock_reservations(&submit); fail: - v3d_job_cleanup((void *)bin); - v3d_job_cleanup((void *)render); - v3d_job_cleanup(clean_job); + v3d_submit_cleanup_jobs(&submit); v3d_put_multisync_post_deps(&se); return ret; @@ -1082,14 +1151,13 @@ int v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct v3d_dev *v3d = to_v3d_dev(dev); + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_tfu *args = data; struct v3d_submit_ext se = {0}; - struct v3d_tfu_job *job = NULL; - struct ww_acquire_ctx acquire_ctx; + struct v3d_tfu_job *job; int ret = 0; - trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia); + trace_v3d_submit_tfu_ioctl(dev, args->iia); if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { drm_dbg(dev, "invalid flags: %d\n", args->flags); @@ -1104,17 +1172,16 @@ v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, } } - ret = v3d_job_allocate(v3d, (void *)&job, sizeof(*job)); - if (ret) - return ret; - - ret = v3d_job_init(v3d, file_priv, &job->base, - v3d_job_free, args->in_sync, &se, V3D_TFU); - if (ret) { - v3d_job_deallocate((void *)&job); + job = (struct v3d_tfu_job *)v3d_submit_add_job(&submit, V3D_TFU); + if (IS_ERR(job)) { + ret = PTR_ERR(job); goto fail; } + ret = v3d_job_add_syncobjs(&job->base, file_priv, args->in_sync, &se); + if (ret) + goto fail; + job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles)); if (!job->base.bo) { ret = -ENOMEM; @@ -1142,26 +1209,20 @@ v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, job->base.bo[job->base.bo_count] = bo; } - ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx); + ret = v3d_submit_lock_reservations(&submit); if (ret) goto fail; - mutex_lock(&v3d->sched_lock); - v3d_push_job(&job->base); - mutex_unlock(&v3d->sched_lock); - - v3d_attach_fences_and_unlock_reservation(file_priv, - &job->base, &acquire_ctx, - args->out_sync, - &se, - job->base.done_fence); - - v3d_job_put(&job->base); + ret = v3d_submit_jobs(&submit, args->out_sync, &se); + if (ret) + goto fail_unreserve; return 0; +fail_unreserve: + v3d_submit_unlock_reservations(&submit); fail: - v3d_job_cleanup((void *)job); + v3d_submit_cleanup_jobs(&submit); v3d_put_multisync_post_deps(&se); return ret; @@ -1180,21 +1241,17 @@ int v3d_submit_csd_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct v3d_dev *v3d = to_v3d_dev(dev); - struct v3d_file_priv *v3d_priv = file_priv->driver_priv; + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_csd *args = data; struct v3d_submit_ext se = {0}; - struct v3d_csd_job *job = NULL; - struct v3d_job *clean_job = NULL; - struct ww_acquire_ctx acquire_ctx; int ret; - trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]); + trace_v3d_submit_csd_ioctl(dev, args->cfg[5], args->cfg[6]); if (args->pad) return -EINVAL; - if (!v3d_has_csd(v3d)) { + if (!v3d_has_csd(submit.v3d)) { drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n"); return -EINVAL; } @@ -1212,57 +1269,28 @@ v3d_submit_csd_ioctl(struct drm_device *dev, void *data, } } - ret = v3d_setup_csd_jobs_and_bos(file_priv, v3d, args, - &job, &clean_job, &se, - &acquire_ctx); + ret = v3d_setup_csd_jobs_and_bos(&submit, args, &se); if (ret) goto fail; - if (args->perfmon_id) { - if (v3d->global_perfmon) { - ret = -EAGAIN; - goto fail_perfmon; - } - - job->base.perfmon = v3d_perfmon_find(v3d_priv, - args->perfmon_id); - if (!job->base.perfmon) { - ret = -ENOENT; - goto fail_perfmon; - } - } + ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id); + if (ret) + goto fail; - mutex_lock(&v3d->sched_lock); - v3d_push_job(&job->base); + ret = v3d_submit_lock_reservations(&submit); + if (ret) + goto fail; - ret = drm_sched_job_add_dependency(&clean_job->base, - dma_fence_get(job->base.done_fence)); + ret = v3d_submit_jobs(&submit, args->out_sync, &se); if (ret) goto fail_unreserve; - v3d_push_job(clean_job); - mutex_unlock(&v3d->sched_lock); - - v3d_attach_fences_and_unlock_reservation(file_priv, - clean_job, - &acquire_ctx, - args->out_sync, - &se, - clean_job->done_fence); - - v3d_job_put(&job->base); - v3d_job_put(clean_job); - return 0; fail_unreserve: - mutex_unlock(&v3d->sched_lock); -fail_perfmon: - drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, - &acquire_ctx); + v3d_submit_unlock_reservations(&submit); fail: - v3d_job_cleanup((void *)job); - v3d_job_cleanup(clean_job); + v3d_submit_cleanup_jobs(&submit); v3d_put_multisync_post_deps(&se); return ret; @@ -1291,13 +1319,14 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct v3d_dev *v3d = to_v3d_dev(dev); + struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; + struct v3d_submit indirect_submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv }; struct drm_v3d_submit_cpu *args = data; struct v3d_submit_ext se = {0}; struct v3d_submit_ext *out_se = NULL; struct v3d_cpu_job *cpu_job = NULL; struct v3d_csd_job *csd_job = NULL; struct v3d_job *clean_job = NULL; - struct ww_acquire_ctx acquire_ctx; int ret; if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { @@ -1305,9 +1334,9 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, return -EINVAL; } - ret = v3d_job_allocate(v3d, (void *)&cpu_job, sizeof(*cpu_job)); - if (ret) - return ret; + cpu_job = (struct v3d_cpu_job *)v3d_submit_add_job(&submit, V3D_CPU); + if (IS_ERR(cpu_job)) + return PTR_ERR(cpu_job); if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job); @@ -1332,23 +1361,36 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type); - ret = v3d_job_init(v3d, file_priv, &cpu_job->base, - v3d_cpu_job_free, 0, &se, V3D_CPU); - if (ret) { - v3d_job_deallocate((void *)&cpu_job); + ret = v3d_job_add_syncobjs(&cpu_job->base, file_priv, 0, &se); + if (ret) goto fail; - } - clean_job = cpu_job->indirect_csd.clean_job; - csd_job = cpu_job->indirect_csd.job; + if (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) { + ret = v3d_setup_csd_jobs_and_bos(&indirect_submit, + &cpu_job->indirect_csd.args, + NULL); + if (ret) + goto fail; + + ret = v3d_submit_lock_reservations(&indirect_submit); + if (ret) + goto fail; + + cpu_job->indirect_csd.job = container_of(indirect_submit.jobs[0], + struct v3d_csd_job, base); + cpu_job->indirect_csd.clean_job = indirect_submit.jobs[1]; + + clean_job = cpu_job->indirect_csd.clean_job; + csd_job = cpu_job->indirect_csd.job; + } if (args->bo_handle_count) { - ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base, + ret = v3d_lookup_bos(submit.file_priv, &cpu_job->base, args->bo_handles, args->bo_handle_count); if (ret) goto fail; - ret = v3d_lock_bo_reservations(&cpu_job->base, &acquire_ctx); + ret = v3d_submit_lock_reservations(&submit); if (ret) goto fail; } @@ -1380,41 +1422,28 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, out_se = (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) ? NULL : &se; - v3d_attach_fences_and_unlock_reservation(file_priv, - &cpu_job->base, - &acquire_ctx, 0, - out_se, cpu_job->base.done_fence); + v3d_attach_fences_and_unlock_reservation(&submit, 0, out_se); switch (cpu_job->job_type) { case V3D_CPU_JOB_TYPE_INDIRECT_CSD: - v3d_attach_fences_and_unlock_reservation(file_priv, - clean_job, - &cpu_job->indirect_csd.acquire_ctx, - 0, &se, clean_job->done_fence); + v3d_attach_fences_and_unlock_reservation(&indirect_submit, 0, &se); break; default: break; } - v3d_job_put(&cpu_job->base); - v3d_job_put(&csd_job->base); - v3d_job_put(clean_job); + v3d_submit_put_jobs(&submit); + v3d_submit_put_jobs(&indirect_submit); return 0; fail_unreserve: mutex_unlock(&v3d->sched_lock); - - drm_gem_unlock_reservations(cpu_job->base.bo, cpu_job->base.bo_count, - &acquire_ctx); - - drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, - &cpu_job->indirect_csd.acquire_ctx); - + v3d_submit_unlock_reservations(&submit); + v3d_submit_unlock_reservations(&indirect_submit); fail: - v3d_job_cleanup((void *)cpu_job); - v3d_job_cleanup((void *)csd_job); - v3d_job_cleanup(clean_job); + v3d_submit_cleanup_jobs(&submit); + v3d_submit_cleanup_jobs(&indirect_submit); v3d_put_multisync_post_deps(&se); return ret; diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c index b17e73bce3841..82a7f2ac4c7e2 100644 --- a/drivers/gpu/drm/vc4/vc4_kms.c +++ b/drivers/gpu/drm/vc4/vc4_kms.c @@ -1188,7 +1188,7 @@ int vc4_kms_load(struct drm_device *dev) drm_mode_config_reset(dev); - drm_kms_helper_poll_init(dev); + drmm_kms_helper_poll_init(dev); return 0; } diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c index 44ffffec550fd..85ea252c658e3 100644 --- a/drivers/gpu/drm/virtio/virtgpu_display.c +++ b/drivers/gpu/drm/virtio/virtgpu_display.c @@ -344,7 +344,7 @@ virtio_gpu_user_framebuffer_create(struct drm_device *dev, if (ret) { kfree(virtio_gpu_fb); drm_gem_object_put(obj); - return NULL; + return ERR_PTR(ret); } return &virtio_gpu_fb->base; diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h index 7449907754a43..5610add1da470 100644 --- a/drivers/gpu/drm/virtio/virtgpu_drv.h +++ b/drivers/gpu/drm/virtio/virtgpu_drv.h @@ -43,6 +43,8 @@ #include #include +#include + #define DRIVER_NAME "virtio_gpu" #define DRIVER_DESC "virtio GPU" @@ -60,6 +62,24 @@ /* See virtio_gpu_ctx_create. One additional character for NULL terminator. */ #define DEBUG_NAME_MAX_LEN 65 +/* + * Whether the host must be told about resource backing pages by DMA address + * rather than guest-physical address. + * + * This mirrors vring_use_map_api() in drivers/virtio/virtio_ring.c, including + * its xen_domain() case. + */ +static inline bool virtio_gpu_use_dma_api(const struct virtio_device *vdev) +{ + if (!virtio_has_dma_quirk(vdev)) + return true; + + if (xen_domain()) + return true; + + return false; +} + struct virtio_gpu_object_params { unsigned long size; bool dumb; @@ -94,6 +114,8 @@ struct virtio_gpu_object { bool dumb; bool created; bool attached; + /* a guest-bound transfer is queued and its mapping not yet synced */ + bool from_host_pending; bool host3d_blob, guest_blob; uint32_t blob_mem, blob_flags; @@ -172,6 +194,9 @@ struct virtio_gpu_vbuffer { struct list_head list; uint32_t seqno; + + /* guest-bound transfer whose shmem backing needs a CPU sync */ + bool sync_for_cpu; }; struct virtio_gpu_output { diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c index 66c3f6f74e9c6..d2f0b8a3f172b 100644 --- a/drivers/gpu/drm/virtio/virtgpu_gem.c +++ b/drivers/gpu/drm/virtio/virtgpu_gem.c @@ -45,7 +45,7 @@ static int virtio_gpu_gem_create(struct drm_file *file, ret = drm_gem_handle_create(file, &obj->base.base, &handle); if (ret) { - drm_gem_object_release(&obj->base.base); + drm_gem_object_put(&obj->base.base); return ret; } diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c index 3d8e4ccdb7c1f..81e70a12b3569 100644 --- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c +++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c @@ -185,7 +185,7 @@ static int virtio_gpu_resource_create_ioctl(struct drm_device *dev, void *data, ret = drm_gem_handle_create(file, obj, &handle); if (ret) { - drm_gem_object_release(obj); + drm_gem_object_put(obj); return ret; } @@ -261,6 +261,27 @@ static int virtio_gpu_transfer_from_host_ioctl(struct drm_device *dev, if (ret != 0) goto err_put_free; + if (virtio_gpu_is_shmem(bo) && virtio_gpu_use_dma_api(vgdev->vdev)) { + /* + * The sync on completion restores the whole mapping, so an + * earlier transfer has to be done before this one snapshots it. + * Otherwise the snapshot predates anything the CPU wrote once + * that transfer's fence signalled, and the later sync would + * discard it. Nothing can add a fence behind our back here, + * since doing so takes the reservation we already hold. + * This writes the pages, so it waits as a writer does. READ + * usage covers existing readers. + */ + long wait = dma_resv_wait_timeout(objs->objs[0]->resv, + DMA_RESV_USAGE_READ, true, + MAX_SCHEDULE_TIMEOUT); + + if (wait < 0) { + ret = wait; + goto err_unlock; + } + } + fence = virtio_gpu_fence_alloc(vgdev, vgdev->fence_drv.context, 0); if (!fence) { ret = -ENOMEM; @@ -320,6 +341,28 @@ static int virtio_gpu_transfer_to_host_ioctl(struct drm_device *dev, void *data, if (ret != 0) goto err_put_free; + /* + * A transfer the other way may have queued without yet syncing + * its mapping. Pushing the guest pages into it now would + * discard what the device wrote there, so wait for that sync: + * it runs before the fence it belongs to is signalled. The + * flag is only set under this reservation, so it cannot appear + * behind our back, and the acquire pairs with the release in + * that sync, so finding it clear means the pages it wrote are + * visible here too. + */ + if (smp_load_acquire(&bo->from_host_pending)) { + long wait = dma_resv_wait_timeout(objs->objs[0]->resv, + DMA_RESV_USAGE_WRITE, + true, + MAX_SCHEDULE_TIMEOUT); + + if (wait < 0) { + ret = wait; + goto err_unlock; + } + } + ret = -ENOMEM; fence = virtio_gpu_fence_alloc(vgdev, vgdev->fence_drv.context, 0); @@ -557,14 +600,14 @@ static int virtio_gpu_resource_create_blob_ioctl(struct drm_device *dev, if (params.blob_flags & VIRTGPU_BLOB_FLAG_USE_CROSS_DEVICE) { ret = virtio_gpu_resource_assign_uuid(vgdev, bo); if (ret) { - drm_gem_object_release(obj); + drm_gem_object_put(obj); return ret; } } ret = drm_gem_handle_create(file, obj, &handle); if (ret) { - drm_gem_object_release(obj); + drm_gem_object_put(obj); return ret; } diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c b/drivers/gpu/drm/virtio/virtgpu_object.c index ec9efacc69195..1527c62be88ba 100644 --- a/drivers/gpu/drm/virtio/virtgpu_object.c +++ b/drivers/gpu/drm/virtio/virtgpu_object.c @@ -163,7 +163,7 @@ static int virtio_gpu_object_shmem_init(struct virtio_gpu_device *vgdev, struct virtio_gpu_mem_entry **ents, unsigned int *nents) { - bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev); + bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev); struct scatterlist *sg; struct sg_table *pages; int si; diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c index 70b3b836e1c99..c2748378a8ad6 100644 --- a/drivers/gpu/drm/virtio/virtgpu_prime.c +++ b/drivers/gpu/drm/virtio/virtgpu_prime.c @@ -310,7 +310,7 @@ struct drm_gem_object *virtgpu_gem_prime_import(struct drm_device *dev, } } - if (!vgdev->has_resource_blob) + if (!vgdev->has_resource_blob || vgdev->has_virgl_3d) return drm_gem_prime_import(dev, buf); bo = kzalloc_obj(*bo); diff --git a/drivers/gpu/drm/virtio/virtgpu_submit.c b/drivers/gpu/drm/virtio/virtgpu_submit.c index 32cb1e4aa4258..3d35326dd9040 100644 --- a/drivers/gpu/drm/virtio/virtgpu_submit.c +++ b/drivers/gpu/drm/virtio/virtgpu_submit.c @@ -389,10 +389,13 @@ static int virtio_gpu_init_submit(struct virtio_gpu_submit *submit, if ((exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) || exbuf->num_out_syncobjs || exbuf->num_bo_handles || - drm_fence_event) + drm_fence_event) { out_fence = virtio_gpu_fence_alloc(vgdev, fence_ctx, ring_idx); - else + if (!out_fence) + return -ENOMEM; + } else { out_fence = NULL; + } if (drm_fence_event) { err = virtio_gpu_fence_event_create(dev, file, out_fence, ring_idx); @@ -538,6 +541,10 @@ int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data, virtio_gpu_process_post_deps(&submit); virtio_gpu_complete_submit(&submit); cleanup: + if (ret && submit.out_fence && submit.out_fence->e) { + drm_event_cancel_free(dev, &submit.out_fence->e->base); + submit.out_fence->e = NULL; + } virtio_gpu_cleanup_submit(&submit); return ret; diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c index e5e1af8b8e8a0..28682a6d5937d 100644 --- a/drivers/gpu/drm/virtio/virtgpu_vq.c +++ b/drivers/gpu/drm/virtio/virtgpu_vq.c @@ -241,6 +241,33 @@ void virtio_gpu_dequeue_ctrl_func(struct work_struct *work) } while (!virtqueue_enable_cb(vgdev->ctrlq.vq)); spin_unlock(&vgdev->ctrlq.qlock); + /* + * Sync guest-bound transfers before signalling anything, so that a + * waiter cannot read the backing pages while what the device wrote is + * still in a bounce buffer. This cannot be folded into the loop below: + * virtio_gpu_fence_event_process() also signals every earlier fence in + * the same context, so any entry there may signal this entry's fence. + */ + list_for_each_entry(entry, &reclaim_list, list) { + if (entry->sync_for_cpu) { + struct virtio_gpu_object *bo = + gem_to_virtio_gpu_obj(entry->objs->objs[0]); + + dma_sync_sgtable_for_cpu(vgdev->vdev->dev.parent, + bo->base.sgt, DMA_FROM_DEVICE); + /* + * Release, so a transfer the other way that skips its + * wait on the strength of this cannot go on to read + * the backing pages before the sync above is visible. + * Nothing orders the two otherwise: where the mapping + * bounces on a coherent device the sync is a plain + * copy, and dma_direct_sync_sg_for_cpu() emits its + * barrier only for the non-coherent case. + */ + smp_store_release(&bo->from_host_pending, false); + } + } + list_for_each_entry(entry, &reclaim_list, list) { resp = (struct virtio_gpu_ctrl_hdr *)entry->resp_buf; @@ -724,7 +751,7 @@ int virtio_gpu_panic_cmd_transfer_to_host_2d(struct virtio_gpu_device *vgdev, struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(objs->objs[0]); struct virtio_gpu_transfer_to_host_2d *cmd_p; struct virtio_gpu_vbuffer *vbuf; - bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev); + bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev); if (virtio_gpu_is_shmem(bo) && use_dma_api) dma_sync_sgtable_for_device(vgdev->vdev->dev.parent, @@ -755,7 +782,7 @@ void virtio_gpu_cmd_transfer_to_host_2d(struct virtio_gpu_device *vgdev, struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(objs->objs[0]); struct virtio_gpu_transfer_to_host_2d *cmd_p; struct virtio_gpu_vbuffer *vbuf; - bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev); + bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev); if (virtio_gpu_is_shmem(bo) && use_dma_api) dma_sync_sgtable_for_device(vgdev->vdev->dev.parent, @@ -1188,7 +1215,7 @@ void virtio_gpu_cmd_transfer_to_host_3d(struct virtio_gpu_device *vgdev, struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(objs->objs[0]); struct virtio_gpu_transfer_host_3d *cmd_p; struct virtio_gpu_vbuffer *vbuf; - bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev); + bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev); if (virtio_gpu_is_shmem(bo) && use_dma_api) dma_sync_sgtable_for_device(vgdev->vdev->dev.parent, @@ -1223,12 +1250,31 @@ void virtio_gpu_cmd_transfer_from_host_3d(struct virtio_gpu_device *vgdev, struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(objs->objs[0]); struct virtio_gpu_transfer_host_3d *cmd_p; struct virtio_gpu_vbuffer *vbuf; + bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev); cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p)); memset(cmd_p, 0, sizeof(*cmd_p)); vbuf->objs = objs; + if (virtio_gpu_is_shmem(bo) && use_dma_api) { + /* + * The device writes only the requested box, so prime the + * mapping with the current contents: otherwise the sync on + * completion would hand back whatever a bounce buffer held for + * the regions the device does not touch. + */ + dma_sync_sgtable_for_device(vgdev->vdev->dev.parent, + bo->base.sgt, DMA_TO_DEVICE); + vbuf->sync_for_cpu = true; + /* + * Set under the reservation the caller holds, so a transfer + * the other way cannot miss it and push the guest pages into + * the mapping while the device still owns it. + */ + WRITE_ONCE(bo->from_host_pending, true); + } + cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D); cmd_p->hdr.ctx_id = cpu_to_le32(ctx_id); cmd_p->resource_id = cpu_to_le32(bo->hw_res_handle); diff --git a/drivers/gpu/drm/virtio/virtgpu_vram.c b/drivers/gpu/drm/virtio/virtgpu_vram.c index 4ae3cbc35dd36..683ea17454c2e 100644 --- a/drivers/gpu/drm/virtio/virtgpu_vram.c +++ b/drivers/gpu/drm/virtio/virtgpu_vram.c @@ -212,16 +212,12 @@ int virtio_gpu_vram_create(struct virtio_gpu_device *vgdev, /* Create fake offset */ ret = drm_gem_create_mmap_offset(obj); - if (ret) { - kfree(vram); - return ret; - } + if (ret) + goto err_release_obj; ret = virtio_gpu_resource_id_get(vgdev, &vram->base.hw_res_handle); - if (ret) { - kfree(vram); - return ret; - } + if (ret) + goto err_release_obj; virtio_gpu_cmd_resource_create_blob(vgdev, &vram->base, params, NULL, 0); @@ -237,6 +233,11 @@ int virtio_gpu_vram_create(struct virtio_gpu_device *vgdev, *bo_ptr = &vram->base; return 0; + +err_release_obj: + drm_gem_object_release(obj); + kfree(vram); + return ret; } void virtio_gpu_vram_map_deferred(struct virtio_gpu_object_vram *vram) diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 8e7b146880f46..a11c619080659 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -152,7 +152,9 @@ xe-y += xe_bb.o \ xe_wait_user_fence.o \ xe_wopcm.o -xe-$(CONFIG_I2C) += xe_i2c.o +xe-$(CONFIG_I2C) += xe_i2c.o \ + xe_amc.o + xe-$(CONFIG_DRM_XE_GPUSVM) += xe_svm.o xe-$(CONFIG_DRM_GPUSVM) += xe_userptr.o diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h index 18d0fde8c98f9..faf8d7e2c5c11 100644 --- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h @@ -46,6 +46,7 @@ #define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2)) #define PIPE_CONTROL0_QUEUE_DRAIN_MODE BIT(12) +#define PIPE_CONTROL0_UNTYPED_DATAPORT_CACHE_FLUSH BIT(11) /* gen12 */ #define PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE BIT(10) /* gen12 */ #define PIPE_CONTROL0_HDC_PIPELINE_FLUSH BIT(9) /* gen12 */ diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index 08251c7a1a4b6..247a736a54aaa 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -651,6 +651,7 @@ #define MEM_THERMAL_MASK REG_BIT(2) #define VR_THERMAL_MASK REG_BIT(3) #define ICCMAX_MASK REG_BIT(4) +#define PWRBRK_MASK REG_BIT(5) #define SOC_AVG_THERMAL_MASK REG_BIT(6) #define FASTVMODE_MASK REG_BIT(7) #define PSYS_PL1_MASK REG_BIT(12) diff --git a/drivers/gpu/drm/xe/regs/xe_i2c_regs.h b/drivers/gpu/drm/xe/regs/xe_i2c_regs.h index f2e455e2bfe45..37550e4a20f80 100644 --- a/drivers/gpu/drm/xe/regs/xe_i2c_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_i2c_regs.h @@ -20,4 +20,6 @@ #define I2C_CONFIG_CMD XE_REG(I2C_CONFIG_SPACE_OFFSET + PCI_COMMAND) #define I2C_CONFIG_PMCSR XE_REG(I2C_CONFIG_SPACE_OFFSET + 0x84) +#define I2C_REG(reg) XE_REG((reg) + I2C_MEM_SPACE_OFFSET) + #endif /* _XE_I2C_REGS_H_ */ diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c index 3c1be809be82d..f10d9513747b4 100644 --- a/drivers/gpu/drm/xe/tests/xe_migrate.c +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -198,8 +198,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test, err = xe_bo_vmap(bo); if (err) { - KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n", - PTR_ERR(bo)); + KUNIT_FAIL(test, "Failed to vmap our pagetables: %d\n", err); return; } diff --git a/drivers/gpu/drm/xe/xe_amc.c b/drivers/gpu/drm/xe/xe_amc.c new file mode 100644 index 0000000000000..8ecadee6eea39 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_amc.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 Intel Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "regs/xe_i2c_regs.h" + +#include "xe_amc.h" +#include "xe_device.h" +#include "xe_i2c.h" +#include "xe_mmio.h" + +/** + * DOC: Add-In Management Controller (AMC) + * + * Handler for the SMBus Alerts from the AMC. All the alerts from AMC will cause + * the device to be declared wedged. + */ + +#define AMC_COMMAND 0x0f +#define AMC_GPU_I2C_ADDR 0x8f +#define AMC_VERSION_V1 0x01 +#define AMC_DESTINATION_ID 12 +#define AMC_SOURCE_ID 8 +#define AMC_FLAGS 0xc8 + +#define AMC_MSG_TYPE 0x7e +#define AMC_GET_ALERT_REASON 0x01 + +enum xe_amc_alert { + AMC_ALERT_UNKNOWN, + AMC_ALERT_FW_DOWNLOAD, + AMC_ALERT_THERMAL_TRIP, + AMC_ALERT_OOB_REQUEST, + AMC_ALERT_OOB_RESET, + AMC_ALERT_CATERR, +}; + +static const char * const amc_alert[] = { + [AMC_ALERT_FW_DOWNLOAD] = "Firmware Download", + [AMC_ALERT_THERMAL_TRIP] = "Thermal Trip", + [AMC_ALERT_OOB_REQUEST] = "OOB Request", + [AMC_ALERT_OOB_RESET] = "OOB Reset", + [AMC_ALERT_CATERR] = "Catastrophic", +}; + +struct xe_amc { + struct xe_i2c *i2c; + struct work_struct work; +}; + +struct amc_header { + u8 command; + u8 len; + u8 address; + u8 version; + u8 destination; + u8 source; + u8 flags; +} __packed; + +struct amc_message { + u8 type; + u16 vendor; + u8 command; +} __packed; + +struct amc_request { + struct amc_header header; + struct amc_message message; + u32 reserved; +} __packed; + +struct amc_response { + struct amc_header header; + struct amc_message message; + u8 error; + u8 value; +} __packed; + +static const struct amc_request amc_get_alert_reason = { + .header = { + .command = AMC_COMMAND, + .len = sizeof(struct amc_request) - 2, + .address = AMC_GPU_I2C_ADDR, + .version = AMC_VERSION_V1, + .destination = AMC_DESTINATION_ID, + .source = AMC_SOURCE_ID, + .flags = AMC_FLAGS, + }, + .message = { + .type = AMC_MSG_TYPE, + .vendor = htons(PCI_VENDOR_ID_INTEL), + .command = AMC_GET_ALERT_REASON, + }, +}; + +static void xe_amc_work(struct work_struct *work) +{ + const struct amc_request *request = &amc_get_alert_reason; + struct xe_amc *amc = from_work(amc, work, work); + u8 alert_reason = AMC_ALERT_UNKNOWN; + struct amc_response response; + struct i2c_client *client; + int ret; + + client = amc->i2c->client[XE_I2C_CLIENT_AMC]; + if (IS_ERR_OR_NULL(client)) + goto out_reassert_interrupt; + + ret = i2c_master_send(client, (u8 *)request, sizeof(*request)); + if (ret < 0) { + dev_err(&client->dev, "failed to send request (%d)\n", ret); + goto out_reassert_interrupt; + } + + /* AMC needs 20ms to generate the response. */ + fsleep(20 * USEC_PER_MSEC); + + ret = i2c_master_recv(client, (u8 *)&response, sizeof(response)); + if (ret < 0) { + dev_err(&client->dev, "failed to read response (%d)\n", ret); + goto out_reassert_interrupt; + } + + if (!response.header.len) { + dev_err(&client->dev, "empty response from AMC\n"); + goto out_reassert_interrupt; + } + + if (memcmp(&response.message, &request->message, sizeof(struct amc_message))) { + dev_err(&client->dev, "response does not match the request\n"); + goto out_reassert_interrupt; + } + + if (response.error) { + dev_err(&client->dev, "AMC error 0x%02x\n", response.error); + goto out_reassert_interrupt; + } + + alert_reason = response.value; + dev_dbg(&client->dev, "Alert reason: %d\n", alert_reason); + +out_reassert_interrupt: + xe_mmio_rmw32(amc->i2c->mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0); + + switch (alert_reason) { + case AMC_ALERT_FW_DOWNLOAD: + case AMC_ALERT_THERMAL_TRIP: + case AMC_ALERT_OOB_REQUEST: + case AMC_ALERT_OOB_RESET: + case AMC_ALERT_CATERR: + dev_warn(amc->i2c->drm_dev, "AMC Alert: %s\n", amc_alert[alert_reason]); + xe_device_declare_wedged(i2c_client_to_xe_device(client)); + break; + default: + dev_warn(amc->i2c->drm_dev, "unknown AMC alert: %d\n", alert_reason); + break; + } +} + +void xe_amc_handle_alert(struct xe_i2c *i2c) +{ + queue_work(system_long_wq, &i2c->amc->work); +} + +int xe_amc_init(struct xe_i2c *i2c) +{ + struct xe_amc *amc; + + amc = kzalloc(sizeof(*amc), GFP_KERNEL); + if (!amc) + return -ENOMEM; + + INIT_WORK(&amc->work, xe_amc_work); + i2c->amc = amc; + amc->i2c = i2c; + + return 0; +} + +void xe_amc_exit(struct xe_i2c *i2c) +{ + if (i2c->amc) { + cancel_work_sync(&i2c->amc->work); + kfree(i2c->amc); + } +} diff --git a/drivers/gpu/drm/xe/xe_amc.h b/drivers/gpu/drm/xe/xe_amc.h new file mode 100644 index 0000000000000..b1d5311fee536 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_amc.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _XE_AMC_H_ +#define _XE_AMC_H_ + +#include + +#include "xe_device.h" + +struct xe_i2c; + +static inline struct xe_device *i2c_adapter_to_xe_device(struct i2c_adapter *adapter) +{ + return kdev_to_xe_device(adapter->dev.parent->parent); +} + +static inline struct xe_device *i2c_client_to_xe_device(struct i2c_client *client) +{ + return i2c_adapter_to_xe_device(client->adapter); +} + +int xe_amc_init(struct xe_i2c *i2c); +void xe_amc_exit(struct xe_i2c *i2c); +void xe_amc_handle_alert(struct xe_i2c *i2c); + +#endif /* _XE_AMC_H_ */ diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 7ed76349075fd..2c60e99249f93 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1025,6 +1025,13 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, } else { drm_dbg(&xe->drm, "Evict system allocator BO failed=%pe\n", ERR_PTR(ret)); + /* + * The semantic we want upon SVM eviction failure + * because of racing access is keep walking for + * eviction, which is -ENOSPC. + */ + if (ret == -EBUSY) + ret = -ENOSPC; } goto out; diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 57039cf42ea7f..a9e5e1bb8e4f7 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -9,6 +9,8 @@ #include #include +#include + #include "xe_bo_types.h" #include "xe_ggtt.h" #include "xe_macros.h" @@ -574,6 +576,23 @@ static inline unsigned int xe_sg_segment_size(struct device *dev) struct scatterlist __maybe_unused sg; size_t max = BIT_ULL(sizeof(sg.length) * 8) - 1; + /* + * For Xen PV guests pages aren't contiguous in DMA (machine) address + * space. The DMA API takes care of that both in dma_alloc_* (by + * calling into the hypervisor to make the pages contiguous) and in + * dma_map_* (by bounce buffering). But xe (like i915, see commit + * 78a07fe777c4) ignores the coherency aspects of the DMA API and thus + * can't cope with bounce buffering actually happening, so add a hack + * here to force small allocations and mappings when running in PV + * mode on Xen. + * + * Note this will still break if bounce buffering is required for other + * reasons, like confidential computing hypervisors or PCIe root ports + * with addressing limitations. + */ + if (xen_pv_domain()) + return PAGE_SIZE; + max = min_t(size_t, max, dma_max_mapping_size(dev)); /* diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c index 04b24e1c8b78e..7dc9873aef545 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.c +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -248,7 +248,8 @@ int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p) pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS); } - if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) { + if (gt->info.engine_mask & + (XE_HW_ENGINE_RCS_MASK | XE_HW_ENGINE_CCS_MASK)) { drm_printf(p, "Render Power Gating Enabled: %s\n", str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE)); diff --git a/drivers/gpu/drm/xe/xe_gt_throttle.c b/drivers/gpu/drm/xe/xe_gt_throttle.c index 1e7e3a31aa698..c0af5484611d1 100644 --- a/drivers/gpu/drm/xe/xe_gt_throttle.c +++ b/drivers/gpu/drm/xe/xe_gt_throttle.c @@ -39,7 +39,7 @@ * - ``reason_mem_thermal``: Memory thermal * - ``reason_vr_thermal``: VR thermal * - ``reason_iccmax``: ICCMAX - * - ``reason_ratl``: RATL thermal algorithm + * - ``reason_pwrbrk``: Power brake * - ``reason_soc_avg_thermal``: SoC average temp * - ``reason_fastvmode``: VR is hitting FastVMode * - ``reason_psys_pl1``: PSYS PL1 @@ -200,6 +200,7 @@ static THROTTLE_ATTR_RO(reason_psys_pl1, PSYS_PL1_MASK); static THROTTLE_ATTR_RO(reason_psys_pl2, PSYS_PL2_MASK); static THROTTLE_ATTR_RO(reason_p0_freq, P0_FREQ_MASK); static THROTTLE_ATTR_RO(reason_psys_crit, PSYS_CRIT_MASK); +static THROTTLE_ATTR_RO(reason_pwrbrk, PWRBRK_MASK); static struct attribute *cri_throttle_attrs[] = { /* Common */ @@ -209,12 +210,12 @@ static struct attribute *cri_throttle_attrs[] = { &attr_reason_pl2.attr.attr, &attr_reason_pl4.attr.attr, &attr_reason_prochot.attr.attr, - &attr_reason_ratl.attr.attr, /* CRI */ &attr_reason_vr_thermal.attr.attr, &attr_reason_soc_thermal.attr.attr, &attr_reason_mem_thermal.attr.attr, &attr_reason_iccmax.attr.attr, + &attr_reason_pwrbrk.attr.attr, &attr_reason_soc_avg_thermal.attr.attr, &attr_reason_fastvmode.attr.attr, &attr_reason_psys_pl1.attr.attr, diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index 886bafd31f45a..2b42a1bb83b22 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -791,7 +791,7 @@ static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads, } } - if (XE_GT_WA(hwe->gt, 16023105232)) + if (XE_GT_WA(hwe->gt, 16023105232) || XE_GT_WA(hwe->gt, 14025941587)) guc_mmio_regset_write_one(ads, regset_map, RING_IDLEDLY(hwe->mmio_base), count++); diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 1c92b96e4c324..eb426e8585887 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -1537,8 +1537,14 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) if (!skip_timeout_check && !check_timeout(q, job)) goto rearm; + /* + * Killed queues must not newly wedge the device, but preserve an + * already-wedged state to avoid warning on teardown timeouts. + */ if (!exec_queue_killed(q)) wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); + else + wedged = xe_device_wedged(xe); set_exec_queue_banned(q); diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index 0b193c451a11d..ecf691923c7ce 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -577,28 +577,102 @@ static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe, xe_reg_whitelist_process_engine(hwe); } +static u32 idledly_floor_ticks(u32 idledly_ns, u32 idledly_units_ps) +{ + return DIV_ROUND_DOWN_ULL((u64)idledly_ns * 1000, idledly_units_ps); +} + static void adjust_idledly(struct xe_hw_engine *hwe) { struct xe_gt *gt = hwe->gt; - u32 idledly, maxcnt; + u32 idledly, idledly_hw, idledly_reg_val, maxcnt; u32 idledly_units_ps = 8 * gt->info.timestamp_base; u32 maxcnt_units_ns = 640; - bool inhibit_switch = 0; + bool inhibit_switch = false; + bool wa_applied = false; + bool clamped_below_maxcnt = false; + + if ((!IS_SRIOV_VF(gt_to_xe(gt)) && XE_GT_WA(gt, 16023105232)) || + XE_GT_WA(gt, 14025941587)) { + u32 mincnt_idledly_ns = 5000; + + /* xe_gt_clock_init() warns and zeroes timestamp_base on unknown crystal clock. */ + if (!idledly_units_ps) + return; - if (!IS_SRIOV_VF(gt_to_xe(hwe->gt)) && XE_GT_WA(gt, 16023105232)) { - idledly = xe_mmio_read32(>->mmio, RING_IDLEDLY(hwe->mmio_base)); + idledly_reg_val = xe_mmio_read32(>->mmio, RING_IDLEDLY(hwe->mmio_base)); maxcnt = xe_mmio_read32(>->mmio, RING_PWRCTX_MAXCNT(hwe->mmio_base)); - inhibit_switch = idledly & INHIBIT_SWITCH_UNTIL_PREEMPTED; - idledly = REG_FIELD_GET(IDLE_DELAY, idledly); - idledly = DIV_ROUND_CLOSEST(idledly * idledly_units_ps, 1000); + inhibit_switch = idledly_reg_val & INHIBIT_SWITCH_UNTIL_PREEMPTED; + idledly = REG_FIELD_GET(IDLE_DELAY, idledly_reg_val); + idledly = DIV_ROUND_CLOSEST_ULL((u64)idledly * idledly_units_ps, 1000); + idledly_hw = idledly; maxcnt = REG_FIELD_GET(IDLE_WAIT_TIME, maxcnt); maxcnt *= maxcnt_units_ns; - if (xe_gt_WARN_ON(gt, idledly >= maxcnt || inhibit_switch)) { - idledly = DIV_ROUND_CLOSEST(((maxcnt - 1) * 1000), - idledly_units_ps); - xe_mmio_write32(>->mmio, RING_IDLEDLY(hwe->mmio_base), idledly); + /* + * Wa_14025941587 is applied before Wa_16023105232, which takes + * priority if the two ever conflict (not expected in practice). + */ + if (XE_GT_WA(gt, 14025941587) && + idledly < mincnt_idledly_ns) { + idledly = mincnt_idledly_ns; + wa_applied = true; + } + + if (XE_GT_WA(gt, 16023105232)) { + /* Clear the inhibit switch without disturbing a valid delay. */ + if (inhibit_switch) { + idledly_reg_val &= ~INHIBIT_SWITCH_UNTIL_PREEMPTED; + wa_applied = true; + } + + /* Warn only on the value read from hardware. */ + xe_gt_WARN_ON(gt, idledly_hw >= maxcnt); + + if (idledly >= maxcnt) { + /* maxcnt may be 0 if IDLE_WAIT_TIME is unprogrammed. */ + idledly = maxcnt ? maxcnt - 1 : 0; + clamped_below_maxcnt = true; + wa_applied = true; + } + } + + if (wa_applied) { + u32 idledly_ticks; + + /* + * Wa_16023105232 requires idledly < maxcnt, so floor + * that clamp; otherwise round up to guarantee the + * Wa_14025941587 minimum survives tick quantization. + */ + if (clamped_below_maxcnt) + idledly_ticks = idledly_floor_ticks(idledly, idledly_units_ps); + else + idledly_ticks = DIV_ROUND_UP_ULL((u64)idledly * 1000, + idledly_units_ps); + + /* + * Tick quantization can still push the rounded-up value + * to/above maxcnt; re-floor here so Wa_16023105232 keeps + * priority even in that case. + */ + if (!clamped_below_maxcnt && XE_GT_WA(gt, 16023105232) && + (u64)idledly_ticks * idledly_units_ps >= (u64)maxcnt * 1000) { + xe_gt_dbg(gt, "idledly %s: %u ticks would exceed maxcnt=%u, so flooring\n", + hwe->name, idledly_ticks, maxcnt); + idledly = maxcnt ? maxcnt - 1 : 0; + idledly_ticks = idledly_floor_ticks(idledly, idledly_units_ps); + } + + idledly_reg_val &= ~IDLE_DELAY; + idledly_reg_val |= REG_FIELD_PREP(IDLE_DELAY, idledly_ticks); + xe_gt_dbg(gt, "idledly %s: set %u max=%u inh=%u ts=%u\n", + hwe->name, idledly, maxcnt, + !!inhibit_switch, gt->info.timestamp_base); + xe_mmio_write32(>->mmio, + RING_IDLEDLY(hwe->mmio_base), + idledly_reg_val); } } } diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c index f05f23221c1b7..6fce3de272965 100644 --- a/drivers/gpu/drm/xe/xe_i2c.c +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -12,8 +12,6 @@ #include #include #include -#include -#include #include #include #include @@ -24,9 +22,12 @@ #include #include +#include + #include "regs/xe_i2c_regs.h" #include "regs/xe_irq_regs.h" +#include "xe_amc.h" #include "xe_device.h" #include "xe_i2c.h" #include "xe_mmio.h" @@ -61,16 +62,32 @@ static inline void xe_i2c_read_endpoint(struct xe_mmio *mmio, void *ep) val[1] = xe_mmio_read32(mmio, REG_SG_REMAP_ADDR_POSTFIX); } +static void xe_i2c_handle_smbus_alert(struct xe_i2c *i2c) +{ + u32 stat; + + stat = xe_mmio_read32(i2c->mmio, I2C_REG(DW_IC_SMBUS_INTR_STAT)); + if (!stat) + return; + + xe_mmio_write32(i2c->mmio, I2C_REG(DW_IC_CLR_SMBUS_INTR), stat); + + if (stat & DW_IC_SMBUS_INTR_ALERT && i2c->amc) + xe_amc_handle_alert(i2c); + else + xe_mmio_rmw32(i2c->mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0); +} + static void xe_i2c_client_work(struct work_struct *work) { struct xe_i2c *i2c = container_of(work, struct xe_i2c, work); struct i2c_board_info info = { .type = "amc", .flags = I2C_CLIENT_HOST_NOTIFY, - .addr = i2c->ep.addr[1], + .addr = i2c->ep.addr[XE_I2C_CLIENT_AMC], }; - i2c->client[0] = i2c_new_client_device(i2c->adapter, &info); + i2c->client[XE_I2C_CLIENT_AMC] = i2c_new_client_device(i2c->adapter, &info); } static int xe_i2c_notifier(struct notifier_block *nb, unsigned long action, void *data) @@ -115,16 +132,6 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) goto err_fwnode_remove; } - if (i2c->adapter_irq) { - struct resource res; - - res = DEFINE_RES_IRQ_NAMED(i2c->adapter_irq, "xe_i2c"); - - ret = platform_device_add_resources(pdev, &res, 1); - if (ret) - goto err_pdev_put; - } - pdev->dev.parent = i2c->drm_dev; pdev->dev.fwnode = fwnode; i2c->adapter_node = fwnode; @@ -166,7 +173,8 @@ bool xe_i2c_present(struct xe_device *xe) static bool xe_i2c_irq_present(struct xe_device *xe) { - return xe->i2c && xe->i2c->adapter_irq; + return xe->i2c && xe->i2c->ep.capabilities & XE_I2C_EP_CAP_IRQ && + !xe_survivability_mode_is_boot_enabled(xe); } /** @@ -179,18 +187,10 @@ static bool xe_i2c_irq_present(struct xe_device *xe) */ void xe_i2c_irq_handler(struct xe_device *xe, u32 master_ctl) { - struct xe_mmio *mmio = xe_root_tile_mmio(xe); - if (!(master_ctl & I2C_IRQ) || !xe_i2c_irq_present(xe)) return; - /* Forward interrupt to I2C adapter */ - generic_handle_irq_safe(xe->i2c->adapter_irq); - - /* Deassert after I2C adapter clears the interrupt */ - xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, 0, PCI_COMMAND_INTX_DISABLE); - /* Reassert to allow subsequent interrupt generation */ - xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0); + xe_i2c_handle_smbus_alert(xe->i2c); } void xe_i2c_irq_reset(struct xe_device *xe) @@ -215,45 +215,6 @@ void xe_i2c_irq_postinstall(struct xe_device *xe) xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0); } -static int xe_i2c_irq_map(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hw_irq_num) -{ - irq_set_chip_and_handler(virq, &dummy_irq_chip, handle_simple_irq); - return 0; -} - -static const struct irq_domain_ops xe_i2c_irq_ops = { - .map = xe_i2c_irq_map, -}; - -static int xe_i2c_create_irq(struct xe_device *xe) -{ - struct xe_i2c *i2c = xe->i2c; - struct irq_domain *domain; - - if (!(i2c->ep.capabilities & XE_I2C_EP_CAP_IRQ) || - xe_survivability_mode_is_boot_enabled(xe)) - return 0; - - domain = irq_domain_create_linear(dev_fwnode(i2c->drm_dev), 1, &xe_i2c_irq_ops, NULL); - if (!domain) - return -ENOMEM; - - i2c->adapter_irq = irq_create_mapping(domain, 0); - i2c->irqdomain = domain; - - return 0; -} - -static void xe_i2c_remove_irq(struct xe_i2c *i2c) -{ - if (!i2c->irqdomain) - return; - - irq_dispose_mapping(i2c->adapter_irq); - irq_domain_remove(i2c->irqdomain); -} - static int xe_i2c_read(void *context, unsigned int reg, unsigned int *val) { struct xe_i2c *i2c = context; @@ -267,8 +228,16 @@ static int xe_i2c_write(void *context, unsigned int reg, unsigned int val) { struct xe_i2c *i2c = context; - xe_mmio_write32(i2c->mmio, XE_REG(reg + I2C_MEM_SPACE_OFFSET), val); + switch (reg) { + case DW_IC_SMBUS_INTR_MASK: + /* Make sure the Alert is never masked. */ + val |= DW_IC_SMBUS_INTR_ALERT; + break; + default: + break; + } + xe_mmio_write32(i2c->mmio, I2C_REG(reg), val); return 0; } @@ -308,14 +277,20 @@ void xe_i2c_pm_resume(struct xe_device *xe, bool d3cold) static void xe_i2c_remove(void *data) { struct xe_i2c *i2c = data; + struct xe_device *xe = tile_to_xe(i2c->mmio->tile); unsigned int i; - for (i = 0; i < XE_I2C_MAX_CLIENTS; i++) + xe_i2c_irq_reset(xe); + xe_amc_exit(i2c); + + for (i = 0; i < XE_I2C_MAX_CLIENTS; i++) { i2c_unregister_device(i2c->client[i]); + i2c->client[i] = NULL; + } bus_unregister_notifier(&i2c_bus_type, &i2c->bus_notifier); xe_i2c_unregister_adapter(i2c); - xe_i2c_remove_irq(i2c); + xe->i2c = NULL; } /** @@ -366,22 +341,18 @@ int xe_i2c_probe(struct xe_device *xe) if (ret) return ret; - ret = xe_i2c_create_irq(xe); - if (ret) - goto err_unregister_notifier; - ret = xe_i2c_register_adapter(i2c); - if (ret) - goto err_remove_irq; + if (ret) { + bus_unregister_notifier(&i2c_bus_type, &i2c->bus_notifier); + return ret; + } + + ret = xe_amc_init(i2c); + if (ret) { + xe_i2c_remove(i2c); + return ret; + } xe_i2c_irq_postinstall(xe); return devm_add_action_or_reset(drm_dev, xe_i2c_remove, i2c); - -err_remove_irq: - xe_i2c_remove_irq(i2c); - -err_unregister_notifier: - bus_unregister_notifier(&i2c_bus_type, &i2c->bus_notifier); - - return ret; } diff --git a/drivers/gpu/drm/xe/xe_i2c.h b/drivers/gpu/drm/xe/xe_i2c.h index 425d8160835f4..b200966b00484 100644 --- a/drivers/gpu/drm/xe/xe_i2c.h +++ b/drivers/gpu/drm/xe/xe_i2c.h @@ -11,18 +11,21 @@ struct device; struct fwnode_handle; struct i2c_adapter; struct i2c_client; -struct irq_domain; struct platform_device; +struct xe_amc; struct xe_device; struct xe_mmio; -#define XE_I2C_MAX_CLIENTS 3 - #define XE_I2C_EP_COOKIE_DEVICE 0xde /* Endpoint Capabilities */ #define XE_I2C_EP_CAP_IRQ BIT(0) +enum XE_I2C_CLIENT { + XE_I2C_CLIENT_AMC = 1, + XE_I2C_MAX_CLIENTS = 3, +}; + struct xe_i2c_endpoint { u8 cookie; u8 capabilities; @@ -38,13 +41,11 @@ struct xe_i2c { struct notifier_block bus_notifier; struct work_struct work; - struct irq_domain *irqdomain; - int adapter_irq; - struct xe_i2c_endpoint ep; struct device *drm_dev; struct xe_mmio *mmio; + struct xe_amc *amc; }; #if IS_ENABLED(CONFIG_I2C) diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c index 8c803ef233cc4..ad3c9fa05223d 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.c +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -39,10 +39,20 @@ struct xe_mmio_gem { phys_addr_t phys_addr; }; +static int xe_mmio_gem_vm_may_split(struct vm_area_struct *area, unsigned long addr) +{ + /* + * Forbid splitting. Together with VM_DONTEXPAND, this keeps the VMA + * matching the GEM object exactly. + */ + return -EINVAL; +} + static const struct vm_operations_struct vm_ops = { .open = drm_gem_vm_open, .close = drm_gem_vm_close, .fault = xe_mmio_gem_vm_fault, + .may_split = xe_mmio_gem_vm_may_split, }; static const struct drm_gem_object_funcs xe_mmio_gem_funcs = { @@ -128,14 +138,16 @@ static void xe_mmio_gem_free(struct drm_gem_object *base) /** * xe_mmio_gem_destroy - Destroy the GEM object that exposes an MMIO region * @gem: the GEM object to destroy + * @file: DRM file descriptor previously passed to xe_mmio_gem_create() * * This function releases resources associated with the GEM object created by * xe_mmio_gem_create(). * * See: "Exposing MMIO regions to userspace" */ -void xe_mmio_gem_destroy(struct xe_mmio_gem *gem) +void xe_mmio_gem_destroy(struct xe_mmio_gem *gem, struct drm_file *file) { + drm_vma_node_revoke(&gem->base.vma_node, file); xe_mmio_gem_free(&gem->base); } @@ -162,14 +174,13 @@ static void xe_mmio_gem_release_dummy_page(struct drm_device *dev, void *res) __free_page((struct page *)res); } -static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_area_struct *vma) +static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_fault *vmf) { + struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *base = vma->vm_private_data; struct drm_device *dev = base->dev; - vm_fault_t ret = VM_FAULT_NOPAGE; struct page *page; unsigned long pfn; - unsigned long i; page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) @@ -180,16 +191,8 @@ static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_area_struct *vma) pfn = page_to_pfn(page); - /* Map the entire VMA to the same dummy page */ - for (i = 0; i < base->size; i += PAGE_SIZE) { - unsigned long addr = vma->vm_start + i; - - ret = vmf_insert_pfn(vma, addr, pfn); - if (ret & VM_FAULT_ERROR) - break; - } - - return ret; + return vmf_insert_pfn_prot(vma, vmf->address, pfn, + vm_get_page_prot(vma->vm_flags)); } static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) @@ -209,7 +212,7 @@ static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) * It is assumed the userspace will receive the notification via some * other channel (e.g. drm uevent). */ - return xe_mmio_gem_vm_fault_dummy_page(vma); + return xe_mmio_gem_vm_fault_dummy_page(vmf); } for (i = 0; i < base->size; i += PAGE_SIZE) { diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.h b/drivers/gpu/drm/xe/xe_mmio_gem.h index 4b76d5586ebb8..80d7795f07c8e 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.h +++ b/drivers/gpu/drm/xe/xe_mmio_gem.h @@ -15,6 +15,6 @@ struct xe_mmio_gem; struct xe_mmio_gem *xe_mmio_gem_create(struct xe_device *xe, struct drm_file *file, phys_addr_t phys_addr, size_t size); u64 xe_mmio_gem_mmap_offset(struct xe_mmio_gem *gem); -void xe_mmio_gem_destroy(struct xe_mmio_gem *gem); +void xe_mmio_gem_destroy(struct xe_mmio_gem *gem, struct drm_file *file); #endif /* _XE_MMIO_GEM_H_ */ diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 18d990c5d4ec4..e5b0f3e2f3894 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -2431,9 +2431,9 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi oa_config->id = idr_alloc(&oa->metrics_idr, oa_config, 1, 0, GFP_KERNEL); if (oa_config->id < 0) { - drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n"); + drm_dbg(&oa->xe->drm, "Failed to allocate id for OA config\n"); err = oa_config->id; - goto sysfs_err; + goto id_alloc_err; } id = oa_config->id; @@ -2444,6 +2444,8 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi return id; +id_alloc_err: + sysfs_remove_group(oa->metrics_kobj, &oa_config->sysfs_metric); sysfs_err: mutex_unlock(&oa->metrics_lock); reg_err: diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index dd3c068e1a399..a4986df8328d6 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -16,6 +16,7 @@ #include "xe_hw_engine.h" #include "xe_pagefault.h" #include "xe_pagefault_types.h" +#include "xe_pm.h" #include "xe_svm.h" #include "xe_trace_bo.h" #include "xe_vm.h" @@ -158,7 +159,7 @@ static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid) down_read(&xe->usm.lock); vm = xa_load(&xe->usm.asid_to_vm, asid); - if (vm && (xe_vm_in_fault_mode(vm) || xe_vm_has_scratch(vm))) + if (vm && xe_vm_in_fault_mode(vm)) xe_vm_get(vm); else vm = ERR_PTR(-EINVAL); @@ -292,9 +293,17 @@ static void xe_pagefault_queue_work(struct work_struct *w) { struct xe_pagefault_queue *pf_queue = container_of(w, typeof(*pf_queue), worker); + struct xe_device *xe = pf_queue->xe; struct xe_pagefault pf; unsigned long threshold; + /* + * A live VM holds a PM reference, but a torn-down VM does not. + * Guard the entire worker loop to safely drain stale faults and + * prevent autosuspends from desyncing batched CT flushes. + */ + guard(xe_pm_runtime)(xe); + #define USM_QUEUE_MAX_RUNTIME_MS 20 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS); @@ -365,6 +374,7 @@ static int xe_pagefault_queue_init(struct xe_device *xe, drm_dbg(&xe->drm, "xe_pagefault_entry_size=%d, total_num_eus=%d, pf_queue->size=%u", xe_pagefault_entry_size(), total_num_eus, pf_queue->size); + pf_queue->xe = xe; spin_lock_init(&pf_queue->lock); INIT_WORK(&pf_queue->worker, xe_pagefault_queue_work); diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h index c4ee625b93ddd..f63a12aa0d4f7 100644 --- a/drivers/gpu/drm/xe/xe_pagefault_types.h +++ b/drivers/gpu/drm/xe/xe_pagefault_types.h @@ -8,6 +8,7 @@ #include +struct xe_device; struct xe_gt; struct xe_pagefault; @@ -118,6 +119,8 @@ struct xe_pagefault { * queue to absorb the device’s worst-case number of outstanding faults. */ struct xe_pagefault_queue { + /** @xe: Back-pointer to the Xe device */ + struct xe_device *xe; /** * @data: Data in queue containing struct xe_pagefault, protected by * @lock diff --git a/drivers/gpu/drm/xe/xe_ras.c b/drivers/gpu/drm/xe/xe_ras.c index 4cb16b419b0c6..96702234d7ece 100644 --- a/drivers/gpu/drm/xe/xe_ras.c +++ b/drivers/gpu/drm/xe/xe_ras.c @@ -4,11 +4,14 @@ */ #include "xe_device.h" +#include "xe_pm.h" #include "xe_printk.h" #include "xe_ras.h" #include "xe_ras_types.h" #include "xe_sysctrl.h" #include "xe_sysctrl_event_types.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_mailbox_types.h" /* Severity of detected errors */ enum xe_ras_severity { @@ -50,6 +53,36 @@ static const char *const xe_ras_components[] = { }; static_assert(ARRAY_SIZE(xe_ras_components) == XE_RAS_COMP_MAX); +static u8 drm_to_xe_ras_severity(u8 severity) +{ + switch (severity) { + case DRM_XE_RAS_ERR_SEV_CORRECTABLE: + return XE_RAS_SEV_CORRECTABLE; + case DRM_XE_RAS_ERR_SEV_UNCORRECTABLE: + return XE_RAS_SEV_UNCORRECTABLE; + default: + return XE_RAS_SEV_NOT_SUPPORTED; + } +} + +static u8 drm_to_xe_ras_component(u8 component) +{ + switch (component) { + case DRM_XE_RAS_ERR_COMP_CORE_COMPUTE: + return XE_RAS_COMP_CORE_COMPUTE; + case DRM_XE_RAS_ERR_COMP_SOC_INTERNAL: + return XE_RAS_COMP_SOC_INTERNAL; + case DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY: + return XE_RAS_COMP_DEVICE_MEMORY; + case DRM_XE_RAS_ERR_COMP_PCIE: + return XE_RAS_COMP_PCIE; + case DRM_XE_RAS_ERR_COMP_FABRIC: + return XE_RAS_COMP_FABRIC; + default: + return XE_RAS_COMP_NOT_SUPPORTED; + } +} + static inline const char *sev_to_str(u8 severity) { if (severity >= XE_RAS_SEV_MAX) @@ -91,3 +124,61 @@ void xe_ras_counter_threshold_crossed(struct xe_device *xe, comp_to_str(component), sev_to_str(severity)); } } + +static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, u32 *value) +{ + struct xe_ras_get_counter_response response = {0}; + struct xe_ras_get_counter_request request = {0}; + struct xe_sysctrl_mailbox_command command = {0}; + struct xe_ras_error_common *common; + size_t rlen; + int ret; + + request.counter = *counter; + + xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_COUNTER, + &request, sizeof(request), &response, sizeof(response)); + + ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen); + if (ret) { + xe_err(xe, "sysctrl: failed to get counter %d\n", ret); + return ret; + } + + if (rlen != sizeof(response)) { + xe_err(xe, "sysctrl: unexpected get counter response length %zu (expected %zu)\n", + rlen, sizeof(response)); + return -EIO; + } + + common = &response.counter.common; + *value = response.value; + + xe_dbg(xe, "[RAS]: get counter %u for %s %s\n", *value, comp_to_str(common->component), + sev_to_str(common->severity)); + + return 0; +} + +/** + * xe_ras_get_counter() - Get error counter value + * @xe: Xe device instance + * @severity: Error severity to be queried (&enum drm_xe_ras_error_severity) + * @component: Error component to be queried (&enum drm_xe_ras_error_component) + * @value: Counter value + * + * This function retrieves the value of a specific error counter based on + * the error severity and component. + * + * Return: 0 on success, negative error code on failure. + */ +int xe_ras_get_counter(struct xe_device *xe, u8 severity, u8 component, u32 *value) +{ + struct xe_ras_error_class counter = {0}; + + counter.common.severity = drm_to_xe_ras_severity(severity); + counter.common.component = drm_to_xe_ras_component(component); + + guard(xe_pm_runtime)(xe); + return get_counter(xe, &counter, value); +} diff --git a/drivers/gpu/drm/xe/xe_ras.h b/drivers/gpu/drm/xe/xe_ras.h index ea90593b62dc1..e148debd5d418 100644 --- a/drivers/gpu/drm/xe/xe_ras.h +++ b/drivers/gpu/drm/xe/xe_ras.h @@ -6,10 +6,13 @@ #ifndef _XE_RAS_H_ #define _XE_RAS_H_ +#include + struct xe_device; struct xe_sysctrl_event_response; void xe_ras_counter_threshold_crossed(struct xe_device *xe, struct xe_sysctrl_event_response *response); +int xe_ras_get_counter(struct xe_device *xe, u8 severity, u8 component, u32 *value); #endif diff --git a/drivers/gpu/drm/xe/xe_ras_types.h b/drivers/gpu/drm/xe/xe_ras_types.h index 4e63c67f806a2..fdfebaeb5ed26 100644 --- a/drivers/gpu/drm/xe/xe_ras_types.h +++ b/drivers/gpu/drm/xe/xe_ras_types.h @@ -70,4 +70,30 @@ struct xe_ras_threshold_crossed { struct xe_ras_error_class counters[XE_RAS_NUM_COUNTERS]; } __packed; +/** + * struct xe_ras_get_counter_request - Request structure for get counter + */ +struct xe_ras_get_counter_request { + /** @counter: Error counter to be queried */ + struct xe_ras_error_class counter; + /** @reserved: Reserved for future use */ + u32 reserved; +} __packed; + +/** + * struct xe_ras_get_counter_response - Response structure for get counter + */ +struct xe_ras_get_counter_response { + /** @counter: Error counter that was queried */ + struct xe_ras_error_class counter; + /** @value: Current counter value */ + u32 value; + /** @timestamp: Timestamp when counter was last updated */ + u64 timestamp; + /** @threshold: Threshold value for the counter */ + u32 threshold; + /** @reserved: Reserved */ + u32 reserved[57]; +} __packed; + #endif diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index 39a670e91ba72..08b4a4283e96a 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -212,6 +212,7 @@ static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) { struct xe_exec_queue *q = job->q; struct xe_gt *gt = q->gt; + struct xe_device *xe = gt_to_xe(gt); bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); u32 flags0, flags1; @@ -220,6 +221,16 @@ static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH; + /* + * Prior to MTL, HDC Pipeline Flush reliably also flushes the LSC + * untyped L1 dataport cache, provided HDC_CHICKEN0 is programmed + * correctly. Starting with MTL that coupling no longer holds + * regardless of how HDC_CHICKEN0 is programmed, but explicitly + * requesting the flush via PIPE_CONTROL is itself only reliable + * from Xe2 onward, so only gate it in on Xe2+. + */ + if (GRAPHICS_VERx100(xe) >= 2000) + flags0 |= PIPE_CONTROL0_UNTYPED_DATAPORT_CACHE_FLUSH; flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH | PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | PIPE_CONTROL_DEPTH_CACHE_FLUSH | diff --git a/drivers/gpu/drm/xe/xe_shrinker.c b/drivers/gpu/drm/xe/xe_shrinker.c index 83374cd576608..deb4378c1ec15 100644 --- a/drivers/gpu/drm/xe/xe_shrinker.c +++ b/drivers/gpu/drm/xe/xe_shrinker.c @@ -54,13 +54,40 @@ xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgea write_unlock(&shrinker->lock); } -static s64 __xe_shrinker_walk(struct xe_device *xe, +static bool __xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker) +{ + struct xe_device *xe = shrinker->xe; + + if (xe_pm_runtime_get_if_active(xe)) + return true; + + if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) { + xe_pm_runtime_get(xe); + return true; + } + + queue_work(xe->unordered_wq, &shrinker->pm_worker); + + return false; +} + +static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm) +{ + if (runtime_pm) + xe_pm_runtime_put(shrinker->xe); +} + +static int __xe_shrinker_walk(struct xe_shrinker *shrinker, struct ttm_operation_ctx *ctx, const struct xe_bo_shrink_flags flags, - unsigned long to_scan, unsigned long *scanned) + unsigned long to_scan, unsigned long *scanned, + unsigned long *freed) { + struct xe_device *xe = shrinker->xe; unsigned int mem_type; - s64 freed = 0, lret; + bool rpm = false; + int ret = 0; + s64 lret; for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) { struct ttm_resource_manager *man = ttm_manager_type(&xe->ttm, mem_type); @@ -74,23 +101,35 @@ static s64 __xe_shrinker_walk(struct xe_device *xe, if (!man || !man->use_tt) continue; + if (mem_type != XE_PL_SYSTEM && !rpm && + xe_device_is_l2_flush_optimized(xe)) { + if (!__xe_shrinker_runtime_pm_get(shrinker)) + break; + rpm = true; + } + ttm_bo_lru_for_each_reserved_guarded(&curs, man, &arg, ttm_bo) { if (!ttm_bo_shrink_suitable(ttm_bo, ctx)) continue; lret = xe_bo_shrink(ctx, ttm_bo, flags, scanned); - if (lret < 0) - return lret; + if (lret < 0) { + ret = lret; + goto out; + } - freed += lret; + *freed += lret; if (*scanned >= to_scan) - break; + goto out; } /* Trylocks should never error, just fail. */ xe_assert(xe, !IS_ERR(ttm_bo)); } - return freed; +out: + xe_shrinker_runtime_pm_put(shrinker, rpm); + + return ret; } /* @@ -99,40 +138,36 @@ static s64 __xe_shrinker_walk(struct xe_device *xe, * add writeback. This avoids stalls and explicit writebacks with light or * moderate memory pressure. */ -static s64 xe_shrinker_walk(struct xe_device *xe, +static int xe_shrinker_walk(struct xe_shrinker *shrinker, struct ttm_operation_ctx *ctx, const struct xe_bo_shrink_flags flags, - unsigned long to_scan, unsigned long *scanned) + unsigned long to_scan, unsigned long *scanned, + unsigned long *freed) { bool no_wait_gpu = true; struct xe_bo_shrink_flags save_flags = flags; - s64 lret, freed; + int ret; swap(no_wait_gpu, ctx->no_wait_gpu); save_flags.writeback = false; - lret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned); + ret = __xe_shrinker_walk(shrinker, ctx, save_flags, to_scan, scanned, + freed); swap(no_wait_gpu, ctx->no_wait_gpu); - if (lret < 0 || *scanned >= to_scan) - return lret; + if (ret || *scanned >= to_scan) + return ret; - freed = lret; if (!ctx->no_wait_gpu) { - lret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned); - if (lret < 0) - return lret; - freed += lret; - if (*scanned >= to_scan) - return freed; + ret = __xe_shrinker_walk(shrinker, ctx, save_flags, to_scan, scanned, + freed); + if (ret || *scanned >= to_scan) + return ret; } - if (flags.writeback) { - lret = __xe_shrinker_walk(xe, ctx, flags, to_scan, scanned); - if (lret < 0) - return lret; - freed += lret; - } + if (flags.writeback) + ret = __xe_shrinker_walk(shrinker, ctx, flags, to_scan, scanned, + freed); - return freed; + return ret; } static unsigned long @@ -180,22 +215,7 @@ static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force, return false; } - if (!xe_pm_runtime_get_if_active(xe)) { - if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) { - xe_pm_runtime_get(xe); - return true; - } - queue_work(xe->unordered_wq, &shrinker->pm_worker); - return false; - } - - return true; -} - -static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm) -{ - if (runtime_pm) - xe_pm_runtime_put(shrinker->xe); + return __xe_shrinker_runtime_pm_get(shrinker); } static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) @@ -214,7 +234,6 @@ static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_con bool runtime_pm; bool purgeable; bool can_backup = !!(sc->gfp_mask & __GFP_FS); - s64 lret; nr_to_scan = sc->nr_to_scan; @@ -225,12 +244,9 @@ static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_con /* Might need runtime PM. Try to wake early if it looks like it. */ runtime_pm = xe_shrinker_runtime_pm_get(shrinker, false, nr_to_scan, can_backup); - if (purgeable && nr_scanned < nr_to_scan) { - lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, - nr_to_scan, &nr_scanned); - if (lret >= 0) - freed += lret; - } + if (purgeable && nr_scanned < nr_to_scan) + xe_shrinker_walk(shrinker, &ctx, shrink_flags, + nr_to_scan, &nr_scanned, &freed); sc->nr_scanned = nr_scanned; if (nr_scanned >= nr_to_scan || !can_backup) @@ -242,10 +258,8 @@ static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_con shrink_flags.purge = false; - lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, - nr_to_scan, &nr_scanned); - if (lret >= 0) - freed += lret; + xe_shrinker_walk(shrinker, &ctx, shrink_flags, + nr_to_scan, &nr_scanned, &freed); sc->nr_scanned = nr_scanned; out: diff --git a/drivers/gpu/drm/xe/xe_sysctrl.c b/drivers/gpu/drm/xe/xe_sysctrl.c index 1db20be8158b5..62ccc9be71b45 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl.c +++ b/drivers/gpu/drm/xe/xe_sysctrl.c @@ -85,7 +85,6 @@ int xe_sysctrl_init(struct xe_device *xe) return ret; xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); - xe_sysctrl_mailbox_init(sc); INIT_WORK(&sc->work, xe_sysctrl_work); return devm_add_action_or_reset(xe->drm.dev, sysctrl_fini, xe); @@ -114,12 +113,10 @@ void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl) * @xe: xe device instance * * Invoked during system resume (S3/S4 to S0) and runtime resume from D3cold. - * Restores SoC remapper configuration and reinitializes mailbox interface. + * Restores SoC remapper configuration. */ void xe_sysctrl_pm_resume(struct xe_device *xe) { - struct xe_sysctrl *sc = &xe->sc; - if (!xe->info.has_soc_remapper_sysctrl) return; @@ -127,6 +124,4 @@ void xe_sysctrl_pm_resume(struct xe_device *xe) return; xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); - - xe_sysctrl_mailbox_init(sc); } diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c index 3caa9f15875f5..72baf1aa4b3ad 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c @@ -145,6 +145,7 @@ static int sysctrl_send_frames(struct xe_sysctrl *sc, struct xe_device *xe = sc_to_xe(sc); u32 ctrl_reg, total_frames, frame; size_t bytes_sent, frame_size; + bool phase; total_frames = DIV_ROUND_UP(cmd_size, XE_SYSCTRL_MB_FRAME_SIZE); @@ -153,7 +154,8 @@ static int sysctrl_send_frames(struct xe_sysctrl *sc, return -EBUSY; } - sc->phase_bit ^= 1; + ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL); + phase = !(ctrl_reg & SYSCTRL_FRAME_PHASE); bytes_sent = 0; for (frame = 0; frame < total_frames; frame++) { @@ -161,7 +163,6 @@ static int sysctrl_send_frames(struct xe_sysctrl *sc, if (sysctrl_write_frame(sc, mbox_cmd + bytes_sent, frame_size)) { xe_err(xe, "sysctrl: Failed to write frame %u\n", frame); - sc->phase_bit = 0; return -EIO; } @@ -169,13 +170,12 @@ static int sysctrl_send_frames(struct xe_sysctrl *sc, REG_FIELD_PREP(SYSCTRL_FRAME_CURRENT_MASK, frame) | REG_FIELD_PREP(SYSCTRL_FRAME_TOTAL_MASK, total_frames - 1) | SYSCTRL_MB_CTRL_CMD | - (sc->phase_bit ? SYSCTRL_FRAME_PHASE : 0); + (phase ? SYSCTRL_FRAME_PHASE : 0); xe_mmio_write32(sc->mmio, SYSCTRL_MB_CTRL, ctrl_reg); if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) { xe_err(xe, "sysctrl: Frame %u acknowledgment timeout\n", frame); - sc->phase_bit = 0; return -ETIMEDOUT; } @@ -294,17 +294,31 @@ static int sysctrl_send_command(struct xe_sysctrl *sc, } /** - * xe_sysctrl_mailbox_init - Initialize System Controller mailbox interface - * @sc: System controller structure + * xe_sysctrl_create_command() - Create system controller command + * @command: Sysctrl command structure + * @group_id: Command group ID + * @cmd_id: Command ID + * @request: Pointer to request buffer (can be NULL) + * @request_len: Size of request buffer + * @response: Pointer to response buffer + * @response_len: Size of response buffer * - * Initialize system controller mailbox interface for communication. + * Helper function to create sysctrl command to be sent via %xe_sysctrl_send_command() */ -void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc) +void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 group_id, u8 cmd_id, + void *request, size_t request_len, void *response, + size_t response_len) { - u32 ctrl_reg; + struct xe_sysctrl_app_msg_hdr header = {0}; - ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL); - sc->phase_bit = (ctrl_reg & SYSCTRL_FRAME_PHASE) ? 1 : 0; + header.data = FIELD_PREP(APP_HDR_GROUP_ID_MASK, group_id) | + FIELD_PREP(APP_HDR_COMMAND_MASK, cmd_id); + + command->header = header; + command->data_in = request; + command->data_in_len = request_len; + command->data_out = response; + command->data_out_len = response_len; } /** diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h index f67e9234de482..f0d5e3d7f5e31 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h @@ -23,7 +23,9 @@ struct xe_sysctrl_mailbox_command; #define XE_SYSCTRL_APP_HDR_VERSION(hdr) \ FIELD_GET(APP_HDR_VERSION_MASK, (hdr)->data) -void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc); +void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 group_id, u8 cmd_id, + void *request, size_t request_len, void *response, + size_t response_len); int xe_sysctrl_send_command(struct xe_sysctrl *sc, struct xe_sysctrl_mailbox_command *cmd, size_t *rdata_len); diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h index 84d7c647e7432..b315847cbf64f 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h @@ -22,9 +22,11 @@ enum xe_sysctrl_group { /** * enum xe_sysctrl_gfsp_cmd - Commands supported by GFSP group * + * @XE_SYSCTRL_CMD_GET_COUNTER: Get error counter value * @XE_SYSCTRL_CMD_GET_PENDING_EVENT: Retrieve pending event */ enum xe_sysctrl_gfsp_cmd { + XE_SYSCTRL_CMD_GET_COUNTER = 0x03, XE_SYSCTRL_CMD_GET_PENDING_EVENT = 0x07, }; diff --git a/drivers/gpu/drm/xe/xe_sysctrl_types.h b/drivers/gpu/drm/xe/xe_sysctrl_types.h index 66ba24f43017d..98c2f473f7c6a 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_types.h +++ b/drivers/gpu/drm/xe/xe_sysctrl_types.h @@ -26,9 +26,6 @@ struct xe_sysctrl { /** @cmd_lock: Mutex protecting mailbox command operations */ struct mutex cmd_lock; - /** @phase_bit: Message boundary phase toggle bit (0 or 1) */ - bool phase_bit; - /** @work: Pending events worker */ struct work_struct work; diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 67819deb45e3c..84ced696f46b3 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1933,21 +1933,13 @@ void xe_vm_close_and_put(struct xe_vm *vm) vma->gpuva.flags |= XE_VMA_DESTROYED; } - /* - * All vm operations will add shared fences to resv. - * The only exception is eviction for a shared object, - * but even so, the unbind when evicted would still - * install a fence to resv. Hence it's safe to - * destroy the pagetables immediately. - */ - xe_vm_free_scratch(vm); - xe_vm_pt_destroy(vm); xe_vm_unlock(vm); /* - * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL - * Since we hold a refcount to the bo, we can remove and free - * the members safely without locking. + * Unlink and destroy all contested external-BO VMAs before destroying + * the page tables. Otherwise, concurrent eviction holding only bo->resv + * can walk the BO's VMAs and attempt to invalidate/zap page tables that + * have already been freed. */ list_for_each_entry_safe(vma, next_vma, &contested, combined_links.destroy) { @@ -1955,6 +1947,11 @@ void xe_vm_close_and_put(struct xe_vm *vm) xe_vma_destroy_unlocked(vma); } + xe_vm_lock(vm, false); + xe_vm_free_scratch(vm); + xe_vm_pt_destroy(vm); + xe_vm_unlock(vm); + xe_svm_fini(vm); up_write(&vm->lock); diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index 23eb7edbdd572..56cff1e44530b 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -89,12 +89,28 @@ static int get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size, u64 *poffset) offset = offset_hi << 32; /* HW view bits 39:32 */ offset |= offset_lo << 6; /* HW view bits 31:6 */ offset *= num_enabled; /* convert to SW view */ - offset = round_up(offset, SZ_128K); /* SW must round up to nearest 128K */ - /* We don't expect any holes */ - xe_assert_msg(xe, offset == (xe_mmio_read64_2x32(>_to_tile(gt)->mmio, GSMBASE) - - ccs_size), - "Hole between CCS and GSM.\n"); + drm_info(&xe->drm, "FLAT_CCS base:%llx, aligned:%s\n", offset, + str_yes_no(IS_ALIGNED(offset, SZ_128K))); + + /* + * Everything below this offset is handed to the VRAM + * allocator, so it has to be the *first* address the + * compression hardware owns, rounded down. Rounding it up + * publishes CCS storage as free memory. + */ + offset = round_down(offset, SZ_4K); + + /* + * CCS storage must not run into GSM. The old check compared + * the offset against GSMBASE - ccs_size for equality, which + * could not fail: that value is 128K aligned, so it agreed + * with the rounded-up offset even when the base was not 128K + * aligned - exactly the case this fixes. + */ + xe_assert_msg(xe, offset + ccs_size <= + xe_mmio_read64_2x32(>_to_tile(gt)->mmio, GSMBASE), + "CCS overlaps GSM.\n"); } else { reg = xe_gt_mcr_unicast_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR); offset = (u64)REG_FIELD_GET(XEHP_FLAT_CCS_PTR, reg) * SZ_64K; diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index f8a185103b805..00c58b3ca9c2b 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -63,5 +63,7 @@ 16026007364 MEDIA_VERSION(3000) 14020316580 MEDIA_VERSION(1301) -14025883347 MEDIA_VERSION_RANGE(1301, 3503) +14025883347 MEDIA_VERSION_RANGE(1301, 3500) GRAPHICS_VERSION_RANGE(2004, 3005) +14025941587 GRAPHICS_VERSION_RANGE(2001, 3511), FUNC(xe_rtp_match_not_sriov_vf) + MEDIA_VERSION_RANGE(1301, 3503), FUNC(xe_rtp_match_not_sriov_vf) diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c index ea3b584990c9a..733f53e69eacc 100644 --- a/drivers/gpu/host1x/bus.c +++ b/drivers/gpu/host1x/bus.c @@ -508,7 +508,7 @@ static int host1x_device_add(struct host1x *host1x, * Add device even if there are no subdevs to ensure syncpoint functionality * is available regardless of whether any engine subdevices are present */ - if (list_empty(&device->subdevs)) { + if (list_empty(&device->subdevs) && !device->registered) { err = device_add(&device->dev); if (err < 0) dev_err(&device->dev, "failed to add device: %d\n", err); diff --git a/drivers/gpu/host1x/debug.c b/drivers/gpu/host1x/debug.c index 6433c00d5d7e0..b828f773fc065 100644 --- a/drivers/gpu/host1x/debug.c +++ b/drivers/gpu/host1x/debug.c @@ -31,7 +31,7 @@ void host1x_debug_output(struct output *o, const char *fmt, ...) int len; va_start(args, fmt); - len = vsnprintf(o->buf, sizeof(o->buf), fmt, args); + len = vscnprintf(o->buf, sizeof(o->buf), fmt, args); va_end(args); o->fn(o->ctx, o->buf, len, false); @@ -43,7 +43,7 @@ void host1x_debug_cont(struct output *o, const char *fmt, ...) int len; va_start(args, fmt); - len = vsnprintf(o->buf, sizeof(o->buf), fmt, args); + len = vscnprintf(o->buf, sizeof(o->buf), fmt, args); va_end(args); o->fn(o->ctx, o->buf, len, true); diff --git a/drivers/gpu/host1x/hw/channel_hw.c b/drivers/gpu/host1x/hw/channel_hw.c index 2df6a16d484e0..9dda73199889c 100644 --- a/drivers/gpu/host1x/hw/channel_hw.c +++ b/drivers/gpu/host1x/hw/channel_hw.c @@ -36,10 +36,9 @@ static void trace_write_gather(struct host1x_cdma *cdma, struct host1x_bo *bo, for (i = 0; i < words; i += TRACE_MAX_LENGTH) { u32 num_words = min(words - i, TRACE_MAX_LENGTH); - offset += i * sizeof(u32); - trace_host1x_cdma_push_gather(dev_name(dev), bo, - num_words, offset, + num_words, + offset + i * sizeof(u32), mem); } diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index 6e0eba101ca15..51bd0c0bede69 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -78,6 +78,7 @@ fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded) { ); } +// This PMU reservation size is r570-specific. pub(super) const fn pmu_reserved_size_gb100() -> u32 { usize_into_u32::<{ const_align_up(SZ_8M + SZ_16M + SZ_4K, Alignment::new::()).unwrap() }>( ) diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs index 038d1278c6341..9978a5d2dec1c 100644 --- a/drivers/gpu/nova-core/fb/hal/gb202.rs +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -83,6 +83,7 @@ impl FbHal for Gb202 { fn non_wpr_heap_size(&self) -> u32 { // Non-WPR heap for GB20x (see Open RM: kgspGetNonWprHeapSize, GB202+). + // This size is r570-specific. u32::SZ_2M + u32::SZ_128K } diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 8fc243c66e356..04af98d2c1f81 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -57,12 +57,35 @@ struct NvdmPayloadCommandResponse { error_code: u32, } -/// Complete FSP response structure with MCTP and NVDM headers. +/// Common MCTP and NVDM headers shared by all FSP messages. #[repr(C, packed)] #[derive(Clone, Copy)] -struct FspResponse { +struct FspMessageHeader { mctp_header: MctpHeader, nvdm_header: NvdmHeader, +} + +// SAFETY: FspMessageHeader is a packed C struct with only integral fields. +unsafe impl AsBytes for FspMessageHeader {} + +// SAFETY: FspMessageHeader is a packed C struct with only integral fields. +unsafe impl FromBytes for FspMessageHeader {} + +impl FspMessageHeader { + /// Construct a standard FSP message header for the given NVDM type. + fn new(nvdm_type: NvdmType) -> Self { + Self { + mctp_header: MctpHeader::single_packet(), + nvdm_header: NvdmHeader::new(nvdm_type), + } + } +} + +/// Complete FSP response structure with MCTP and NVDM headers. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspResponse { + header: FspMessageHeader, response: NvdmPayloadCommandResponse, } @@ -94,29 +117,41 @@ struct NvdmPayloadCot { gsp_boot_args_sysmem_offset: u64, } -/// Complete FSP message structure with MCTP and NVDM headers. +/// Complete FSP COT (Chain of Trust) message structure. #[repr(C)] #[derive(Clone, Copy)] -struct FspMessage { - mctp_header: MctpHeader, - nvdm_header: NvdmHeader, +struct FspCotMessage { + header: FspMessageHeader, cot: NvdmPayloadCot, } -impl FspMessage { - /// Returns an in-place initializer for [`FspMessage`]. +impl FspCotMessage { + /// Computes the FRTS vidmem offset for the Chain-of-Trust message. It is measured backwards + /// from the end of the framebuffer. + fn frts_vidmem_offset(hal: &dyn hal::FspHal, fb_layout: &FbLayout) -> Result { + let mut offset = hal.fb_end_reserved_size(); + + // As per OpenRM's `kfspPrepareBootCommands_GH100`. + if fb_layout.pmu_reserved_size != 0 { + offset = (offset + u64::from(fb_layout.pmu_reserved_size)) + // The 2 MiB alignment is r570-specific. + .align_up(Alignment::new::()) + .ok_or(EINVAL)?; + } + + Ok(offset) + } + + /// Returns an in-place initializer for [`FspCotMessage`]. fn new<'a>( fb_layout: &FbLayout, fsp_fw: &'a FspFirmware, args: &'a FmcBootArgs, ) -> Result + 'a> { - // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset - let frts_vidmem_offset = if !args.resume { - let frts_reserved_size = fb_layout.heap.len() + u64::from(fb_layout.pmu_reserved_size); + let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?; - frts_reserved_size - .align_up(Alignment::new::()) - .ok_or(EINVAL)? + let frts_vidmem_offset = if !args.resume { + Self::frts_vidmem_offset(hal, fb_layout)? } else { 0 }; @@ -127,12 +162,11 @@ impl FspMessage { 0 }; - let version = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?.cot_version(); + let version = hal.cot_version(); let size = num::usize_into_u16::<{ core::mem::size_of::() }>(); Ok(init!(Self { - mctp_header: MctpHeader::single_packet(), - nvdm_header: NvdmHeader::new(NvdmType::Cot), + header: FspMessageHeader::new(NvdmType::Cot), // The payload is packed, so we cannot use `init!`. Initialize it member-by-member using // `chain`. cot <- pin_init::init_zeroed(), @@ -143,8 +177,8 @@ impl FspMessage { msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_handle(); msg.cot.frts_vidmem_offset = frts_vidmem_offset; msg.cot.frts_vidmem_size = frts_size; - // frts_sysmem_* intentionally left at zero for now, but will be needed for e.g. - // systems without VRAM. + // frts_sysmem_* are left at zero because this path places FRTS in vidmem. The sysmem + // fields point to an FRTS buffer in sysmem instead, for systems without VRAM. msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_handle(); msg.cot.sigs = *fsp_fw.fmc_sigs; @@ -153,11 +187,11 @@ impl FspMessage { } } -// SAFETY: `FspMessage` is `#[repr(C)]` with no padding, so all of its +// SAFETY: `FspCotMessage` is `#[repr(C)]` with no padding, so all of its // bytes are initialized. -unsafe impl AsBytes for FspMessage {} +unsafe impl AsBytes for FspCotMessage {} -impl MessageToFsp for FspMessage { +impl MessageToFsp for FspCotMessage { const NVDM_TYPE: NvdmType = NvdmType::Cot; } @@ -251,8 +285,8 @@ impl Fsp { EIO })?; - let mctp_header = response.mctp_header; - let nvdm_header = response.nvdm_header; + let mctp_header = response.header.mctp_header; + let nvdm_header = response.header.nvdm_header; let command_nvdm_type = response.response.command_nvdm_type; let error_code = response.response.error_code; @@ -310,7 +344,10 @@ impl Fsp { ) -> Result { dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); - let msg = KBox::init(FspMessage::new(fb_layout, &self.fsp_fw, args)?, GFP_KERNEL)?; + let msg = KBox::init( + FspCotMessage::new(fb_layout, &self.fsp_fw, args)?, + GFP_KERNEL, + )?; self.send_sync_fsp(dev, bar, &*msg)?; diff --git a/drivers/gpu/nova-core/fsp/hal.rs b/drivers/gpu/nova-core/fsp/hal.rs index b6f2624bb13d4..eaf5837ac5a88 100644 --- a/drivers/gpu/nova-core/fsp/hal.rs +++ b/drivers/gpu/nova-core/fsp/hal.rs @@ -19,6 +19,10 @@ pub(super) trait FspHal { /// Returns the FSP Chain of Trust protocol version this chipset advertises. fn cot_version(&self) -> u16; + + // TODO: consider moving this into the TLV firmware metadata when ready + /// Returns the size reserved at the end of the framebuffer, in bytes. + fn fb_end_reserved_size(&self) -> u64; } /// Returns the FSP HAL, or `None` if the architecture doesn't support FSP. diff --git a/drivers/gpu/nova-core/fsp/hal/gb100.rs b/drivers/gpu/nova-core/fsp/hal/gb100.rs index 42f5ecfc64004..7cf53aa3d1ff0 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb100.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +use kernel::sizes::SizeConstants; + use crate::{ driver::Bar0, fsp::hal::FspHal, // @@ -17,6 +19,10 @@ impl FspHal for Gb100 { fn cot_version(&self) -> u16 { 2 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + u64::SZ_128K + } } const GB100: Gb100 = Gb100; diff --git a/drivers/gpu/nova-core/fsp/hal/gb202.rs b/drivers/gpu/nova-core/fsp/hal/gb202.rs index 1091b169a645a..e380bbc5d58d7 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb202.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb202.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -use kernel::io::Io; +use kernel::{ + io::Io, + sizes::SizeConstants, // +}; use crate::{ driver::Bar0, @@ -21,6 +24,10 @@ impl FspHal for Gb202 { fn cot_version(&self) -> u16 { 2 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + u64::SZ_128K + } } const GB202: Gb202 = Gb202; diff --git a/drivers/gpu/nova-core/fsp/hal/gh100.rs b/drivers/gpu/nova-core/fsp/hal/gh100.rs index 291acaf2845a7..9a8563799da85 100644 --- a/drivers/gpu/nova-core/fsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gh100.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -use kernel::io::Io; +use kernel::{ + io::Io, + sizes::SizeConstants, // +}; use crate::{ driver::Bar0, @@ -26,6 +29,10 @@ impl FspHal for Gh100 { fn cot_version(&self) -> u16 { 1 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + } } const GH100: Gh100 = Gh100; diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index b3c91731db45d..b603b0bd2692b 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -285,10 +285,10 @@ pub(crate) struct Gpu<'gpu> { } impl<'gpu> Gpu<'gpu> { - pub(crate) fn new( - pdev: &'gpu pci::Device>, + pub(crate) fn new<'a>( + pdev: &'gpu pci::Device>, bar: Bar0<'gpu>, - ) -> impl PinInit + 'gpu { + ) -> impl PinInit + use<'gpu, 'a> { try_pin_init!(Self { device: pdev.as_ref(), spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 8afb62d689cb4..fbf81eb6c34dc 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -146,7 +146,7 @@ impl super::Gsp { self.cmdq .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new())?; + .send_command_no_wait(bar, commands::SetRegistry::new()?)?; hal.post_boot(&self, dev, bar, &gsp_fw, gsp_falcon, sec2_falcon)?; diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index f84de9f4f0450..0c2919b0980d8 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -66,37 +66,44 @@ struct RegistryEntry { /// The `SetRegistry` command. pub(crate) struct SetRegistry { - entries: [RegistryEntry; Self::NUM_ENTRIES], + entries: KVec, } impl SetRegistry { - // For now we hard-code the registry entries. Future work will allow others to - // be added as module parameters. - const NUM_ENTRIES: usize = 3; - /// Creates a new `SetRegistry` command, using a set of hardcoded entries. - pub(crate) fn new() -> Self { - Self { - entries: [ - // RMSecBusResetEnable - enables PCI secondary bus reset - RegistryEntry { - key: "RMSecBusResetEnable", - value: 1, - }, - // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on - // any PCI reset. - RegistryEntry { - key: "RMForcePcieConfigSave", - value: 1, - }, - // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found - // in the internal product name database. - RegistryEntry { - key: "RMDevidCheckIgnore", - value: 1, - }, - ], - } + pub(crate) fn new() -> Result { + let mut entries = KVec::new(); + + // RMSecBusResetEnable - enables PCI secondary bus reset + entries.push( + RegistryEntry { + key: "RMSecBusResetEnable", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on + // any PCI reset. + entries.push( + RegistryEntry { + key: "RMForcePcieConfigSave", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found + // in the internal product name database. + entries.push( + RegistryEntry { + key: "RMDevidCheckIgnore", + value: 1, + }, + GFP_KERNEL, + )?; + + Ok(Self { entries }) } } @@ -107,15 +114,15 @@ impl CommandToGsp for SetRegistry { type InitError = Infallible; fn init(&self) -> impl Init { - Self::Command::init(Self::NUM_ENTRIES as u32, self.variable_payload_len() as u32) + Self::Command::init(self.entries.len() as u32, self.size() as u32) } fn variable_payload_len(&self) -> usize { let mut key_size = 0; - for i in 0..Self::NUM_ENTRIES { - key_size += self.entries[i].key.len() + 1; // +1 for NULL terminator + for entry in self.entries.iter() { + key_size += entry.key.len() + 1; // +1 for NULL terminator } - Self::NUM_ENTRIES * size_of::() + key_size + self.entries.len() * size_of::() + key_size } fn init_variable_payload( @@ -123,12 +130,12 @@ impl CommandToGsp for SetRegistry { dst: &mut SBufferIter>, ) -> Result { let string_data_start_offset = size_of::() - + Self::NUM_ENTRIES * size_of::(); + + self.entries.len() * size_of::(); // Array for string data. let mut string_data = KVec::new(); - for entry in self.entries.iter().take(Self::NUM_ENTRIES) { + for entry in self.entries.iter() { dst.write_all( fw::commands::PackedRegistryEntry::new( (string_data_start_offset + string_data.len()) as u32, diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 2f6301af71131..eb7166148cc9a 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -134,11 +134,19 @@ impl UnloadBundle for Sec2UnloadBundle { sec2_falcon: &Falcon, ) -> Result { // Run FWSEC-SB to reset the GSP falcon to its pre-libos state. - self.fwsec_sb.run(dev, bar, gsp_falcon)?; + // Log errors but keep going if it fails. + let fwsec_sb_res = self + .fwsec_sb + .run(dev, bar, gsp_falcon) + .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e)); // Remove WPR2 region if set. let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); - if wpr2_hi.is_wpr2_set() { + let booter_unloader_res = (|| { + if !wpr2_hi.is_wpr2_set() { + return Ok(()); + } + sec2_falcon.reset(bar)?; sec2_falcon.load(dev, bar, &self.booter_unloader)?; @@ -160,9 +168,12 @@ impl UnloadBundle for Sec2UnloadBundle { ); return Err(EBUSY); } - } - Ok(()) + Ok(()) + })() + .inspect_err(|e| dev_err!(dev, "Booter Unloader failed to run: {:?}\n", e)); + + fwsec_sb_res.and(booter_unloader_res) } } diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 0f49c1ab83ad4..a1af4a8bd2365 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -570,7 +570,7 @@ register! { /// GA102 and later. pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ PFalcon2Base + 0x00000388 { 7:7 active_stat => bool; - 0:0 halted => bool; + 4:4 halted => bool; } /// GA102 and later. diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index f9bcaeb66385b..2f4a43d949aea 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -205,6 +205,23 @@ config HID_AUREAL help Support for Aureal Cy se W-01RN Remote Controller and other Aureal derived remotes. +config HID_AYANEO + tristate "AYANEO 3 detachable controller support" + depends on USB_HID + depends on DMI + depends on LEDS_CLASS_MULTICOLOR + help + Provides support for the detachable controller ("Magic Modules") + of the AYANEO 3 handheld: module identification, software eject + and RGB control of the joystick rings. Complements the ayaneo-ec + platform driver, which handles module attach state and controller + power. + + Say Y or M here if you have an AYANEO 3. + + To compile this driver as a module, choose M here: the + module will be called hid-ayaneo. + config HID_BELKIN tristate "Belkin Flip KVM and Wireless keyboard" help @@ -1048,6 +1065,7 @@ config HID_PXRC config HID_RAPOO tristate "Rapoo non-fully HID-compliant devices" + depends on USB_HID help Support for Rapoo devices that are not fully compliant with the HID standard. diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 23e6e3dd0c56a..6dced58a2b6fc 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -35,6 +35,7 @@ obj-$(CONFIG_HID_APPLETB_KBD) += hid-appletb-kbd.o obj-$(CONFIG_HID_CREATIVE_SB0540) += hid-creative-sb0540.o obj-$(CONFIG_HID_ASUS) += hid-asus.o obj-$(CONFIG_HID_AUREAL) += hid-aureal.o +obj-$(CONFIG_HID_AYANEO) += hid-ayaneo.o obj-$(CONFIG_HID_BELKIN) += hid-belkin.o obj-$(CONFIG_HID_BETOP_FF) += hid-betopff.o obj-$(CONFIG_HID_BIGBEN_FF) += hid-bigbenff.o diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_common.h b/drivers/hid/amd-sfh-hid/amd_sfh_common.h index 78f830c133e5c..bd8dc16feb614 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_common.h +++ b/drivers/hid/amd-sfh-hid/amd_sfh_common.h @@ -12,11 +12,15 @@ #include #include +#include #include "amd_sfh_hid.h" #define PCI_DEVICE_ID_AMD_MP2 0x15E4 #define PCI_DEVICE_ID_AMD_MP2_1_1 0x164A +/* The BAR 2 size must cover the highest register offset (0x10958) */ +#define AMD_SFH_MIN_BAR_SIZE SZ_128K + #define AMD_C2P_MSG(regno) (0x10500 + ((regno) * 4)) #define AMD_P2C_MSG(regno) (0x10680 + ((regno) * 4)) diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c index 4b81cebdc3359..039b6ac327d3d 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c +++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c @@ -451,6 +451,16 @@ static int amd_mp2_pci_probe(struct pci_dev *pdev, const struct pci_device_id *i if (rc) return rc; + if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { + dev_err(&pdev->dev, "BAR 2 is not IORESOURCE_MEM\n"); + return -ENODEV; + } + + if (pci_resource_len(pdev, 2) < AMD_SFH_MIN_BAR_SIZE) { + dev_err(&pdev->dev, "BAR 2 is too small\n"); + return -EINVAL; + } + rc = pcim_iomap_regions(pdev, BIT(2), DRIVER_NAME); if (rc) return rc; diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c index 536f6d01fd14c..b1de1dd0f21d0 100644 --- a/drivers/hid/bpf/hid_bpf_dispatch.c +++ b/drivers/hid/bpf/hid_bpf_dispatch.c @@ -359,7 +359,7 @@ hid_bpf_release_context(struct hid_bpf_ctx *ctx) static int __hid_bpf_hw_check_params(struct hid_bpf_ctx *ctx, __u8 *buf, size_t *buf__sz, - enum hid_report_type rtype) + enum hid_report_type rtype, bool hw_request) { struct hid_report_enum *report_enum; struct hid_report *report; @@ -388,6 +388,10 @@ __hid_bpf_hw_check_params(struct hid_bpf_ctx *ctx, __u8 *buf, size_t *buf__sz, report_len = hid_report_len(report); + /* unnumbered reports need to have a report ID reserved in the first byte */ + if (hw_request && report_enum->numbered == 0) + report_len += 1; + if (*buf__sz > report_len) *buf__sz = report_len; @@ -420,7 +424,7 @@ hid_bpf_hw_request(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz, return -EDEADLOCK; /* check arguments */ - ret = __hid_bpf_hw_check_params(ctx, buf, &size, rtype); + ret = __hid_bpf_hw_check_params(ctx, buf, &size, rtype, true); if (ret) return ret; @@ -480,7 +484,7 @@ hid_bpf_hw_output_report(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz) return -EDEADLOCK; /* check arguments */ - ret = __hid_bpf_hw_check_params(ctx, buf, &size, HID_OUTPUT_REPORT); + ret = __hid_bpf_hw_check_params(ctx, buf, &size, HID_OUTPUT_REPORT, true); if (ret) return ret; @@ -506,7 +510,7 @@ __hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum hid_report_type type, u8 *b return -EDEADLOCK; /* check arguments */ - ret = __hid_bpf_hw_check_params(ctx, buf, &size, type); + ret = __hid_bpf_hw_check_params(ctx, buf, &size, type, false); if (ret) return ret; diff --git a/drivers/hid/bpf/hid_bpf_struct_ops.c b/drivers/hid/bpf/hid_bpf_struct_ops.c index 702c22fae136a..049df06477610 100644 --- a/drivers/hid/bpf/hid_bpf_struct_ops.c +++ b/drivers/hid/bpf/hid_bpf_struct_ops.c @@ -250,6 +250,11 @@ static void hid_bpf_unreg(void *kdata, struct bpf_link *link) mutex_lock(&hdev->bpf.prog_list_lock); + if (!ops->hdev) { + mutex_unlock(&hdev->bpf.prog_list_lock); + return; + } + list_del_rcu(&ops->list); synchronize_srcu(&hdev->bpf.srcu); ops->hdev = NULL; @@ -310,13 +315,17 @@ static struct bpf_struct_ops bpf_hid_bpf_ops = { void __hid_bpf_ops_destroy_device(struct hid_device *hdev) { struct hid_bpf_ops *e; + int count = 0; - rcu_read_lock(); - list_for_each_entry_rcu(e, &hdev->bpf.prog_list, list) { - hid_put_device(hdev); + mutex_lock(&hdev->bpf.prog_list_lock); + list_for_each_entry(e, &hdev->bpf.prog_list, list) { e->hdev = NULL; + count++; } - rcu_read_unlock(); + mutex_unlock(&hdev->bpf.prog_list_lock); + + while (count--) + hid_put_device(hdev); } static int __init hid_bpf_struct_ops_init(void) diff --git a/drivers/hid/hid-alps.c b/drivers/hid/hid-alps.c index 67179e3fe39b7..0556cb5645ebb 100644 --- a/drivers/hid/hid-alps.c +++ b/drivers/hid/hid-alps.c @@ -407,6 +407,8 @@ static int u1_raw_event(struct alps_dev *hdata, u8 *data, int size) return 1; case U1_SP_ABSOLUTE_REPORT_ID: + if (!hdata->input2) + return 0; sp_x = get_unaligned_le16(data+2); sp_y = get_unaligned_le16(data+4); @@ -738,7 +740,6 @@ static int alps_input_configured(struct hid_device *hdev, struct hid_input *hi) goto exit; } - data->input2 = input2; input2->phys = input->phys; input2->name = "DualPoint Stick"; input2->id.bustype = BUS_I2C; @@ -762,11 +763,12 @@ static int alps_input_configured(struct hid_device *hdev, struct hid_input *hi) __set_bit(INPUT_PROP_POINTER, input2->propbit); __set_bit(INPUT_PROP_POINTING_STICK, input2->propbit); - if (input_register_device(data->input2)) { + if (input_register_device(input2)) { input_free_device(input2); ret = -ENOENT; goto exit; } + data->input2 = input2; } exit: @@ -823,6 +825,24 @@ static int alps_probe(struct hid_device *hdev, const struct hid_device_id *id) return 0; } +static void alps_remove(struct hid_device *hdev) +{ + struct alps_dev *data = hid_get_drvdata(hdev); + + /* + * input2 ("DualPoint Stick") is allocated separately and is not + * tracked in hdev->inputs, so the default remove path + * (hid_hw_stop -> hidinput_disconnect) does not unregister it. + * + * Stop the device first so that no URB callback can touch input2 + * while it is being unregistered, then drop it explicitly. + */ + hid_hw_stop(hdev); + + if (data->input2) + input_unregister_device(data->input2); +} + static const struct hid_device_id alps_id[] = { { HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY, USB_VENDOR_ID_ALPS_JP, HID_DEVICE_ID_ALPS_U1_DUAL) }, @@ -845,6 +865,7 @@ static struct hid_driver alps_driver = { .input_configured = alps_input_configured, .resume = pm_ptr(alps_post_resume), .reset_resume = pm_ptr(alps_post_reset), + .remove = alps_remove, }; module_hid_driver(alps_driver); diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index bf7dd0fbf2496..a5f232cc2b66e 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -91,6 +91,9 @@ struct apple_sc_backlight { struct hid_device *hdev; }; +/* T2 VHCI re-enumerates the internal keyboard across system resume. */ +static int apple_backlight_resume_brightness = -1; + struct apple_backlight_config_report { u8 report_id; u8 version; @@ -825,6 +828,7 @@ static int apple_backlight_led_set(struct led_classdev *led_cdev, static int apple_backlight_init(struct hid_device *hdev) { int ret; + int brightness; struct apple_sc *asc = hid_get_drvdata(hdev); struct apple_backlight_config_report *rep; @@ -860,13 +864,20 @@ static int apple_backlight_init(struct hid_device *hdev) asc->backlight->cdev.name = "apple::kbd_backlight"; asc->backlight->cdev.max_brightness = rep->backlight_on_max; asc->backlight->cdev.brightness_set_blocking = apple_backlight_led_set; - asc->backlight->cdev.flags = LED_CORE_SUSPENDRESUME; + /* VHCI re-enumeration restores the cached brightness in the next probe. */ + + brightness = READ_ONCE(apple_backlight_resume_brightness); + if (brightness < 0) + brightness = LED_OFF; + else + brightness = min_t(int, brightness, rep->backlight_on_max); - ret = apple_backlight_set(hdev, 0, 0); + ret = apple_backlight_set(hdev, brightness, 0); if (ret < 0) { hid_err(hdev, "backlight set request failed: %d\n", ret); goto cleanup_and_exit; } + asc->backlight->cdev.brightness = brightness; ret = devm_led_classdev_register(&hdev->dev, &asc->backlight->cdev); @@ -999,6 +1010,9 @@ static void apple_remove(struct hid_device *hdev) if (asc->quirks & APPLE_RDESC_BATTERY) timer_delete_sync(&asc->battery_timer); + if (asc->backlight) + WRITE_ONCE(apple_backlight_resume_brightness, + asc->backlight->cdev.brightness); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index 3f5e96900b67a..c22c1021ecc38 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -109,11 +109,36 @@ MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad"); #define TRKID_SGN ((TRKID_MAX + 1) >> 1) -struct asus_kbd_leds { - struct asus_hid_listener listener; +enum asus_work_action_type { + FN_LOCK_SYNC, + BRIGHTNESS_SET, + WMI_FAN, +}; + +struct hid_raw_event_data { + u8 report_data[FEATURE_KBD_REPORT_SIZE]; + size_t report_size; +}; + +struct asus_work_action { + struct list_head node; + enum asus_work_action_type type; + union { + /* Data for BRIGHTNESS_SET */ + unsigned int brightness; + + /* Data for FN_LOCK_SYNC */ + bool fn_lock; + + /* Data for WMI_FAN */ + struct hid_raw_event_data fan_hid_data; + } data; +}; + +struct asus_worker { struct hid_device *hdev; struct work_struct work; - unsigned int brightness; + struct list_head actions; spinlock_t lock; bool removed; }; @@ -133,7 +158,8 @@ struct asus_drvdata { struct hid_device *hdev; struct input_dev *input; struct input_dev *tp_kbd_input; - struct asus_kbd_leds *kbd_backlight; + struct asus_worker *worker; + unsigned int kbd_backlight_brightness; const struct asus_touchpad_info *tp; struct power_supply *battery; struct power_supply_desc battery_desc; @@ -141,7 +167,7 @@ struct asus_drvdata { int battery_stat; bool battery_in_query; unsigned long battery_next_query; - struct work_struct fn_lock_sync_work; + struct asus_hid_listener listener; bool fn_lock; }; @@ -211,6 +237,29 @@ static const u8 asus_report_id_init[] = { FEATURE_KBD_LED_REPORT_ID2 }; +/* + * Send events to asus-wmi driver for handling special keys + */ +static int asus_wmi_send_event(struct asus_drvdata *drvdata, u8 code) +{ + int err; + u32 retval; + + err = asus_wmi_evaluate_method(ASUS_WMI_METHODID_DEVS, + ASUS_WMI_METHODID_NOTIF, code, &retval); + if (err) { + pr_warn("Failed to notify asus-wmi: %d\n", err); + return err; + } + + if (retval != 0) { + pr_warn("Failed to notify asus-wmi (retval): 0x%x\n", retval); + return -EIO; + } + + return 0; +} + static void asus_report_contact_down(struct asus_drvdata *drvdat, int toolType, u8 *data) { @@ -331,25 +380,71 @@ static int asus_e1239t_event(struct asus_drvdata *drvdat, u8 *data, int size) } /* - * Send events to asus-wmi driver for handling special keys + * Used in atomic contexts to schedule work involving sleeps operations or + * asus-wmi interactions. + * + * Caller is responsible to store relevant data in the structure to carry out + * the required action. + * + * This function must be called while the spin lock protecting the workqueue + * is already being held. */ -static int asus_wmi_send_event(struct asus_drvdata *drvdata, u8 code) +static void asus_worker_schedule(struct asus_worker *worker, struct asus_work_action *action) { - int err; - u32 retval; - - err = asus_wmi_evaluate_method(ASUS_WMI_METHODID_DEVS, - ASUS_WMI_METHODID_NOTIF, code, &retval); - if (err) { - pr_warn("Failed to notify asus-wmi: %d\n", err); - return err; + if (worker->removed) { + kfree(action); + return; } - if (retval != 0) { - pr_warn("Failed to notify asus-wmi (retval): 0x%x\n", retval); - return -EIO; + list_add_tail(&action->node, &worker->actions); + schedule_work(&worker->work); +} + +static int asus_kbd_fn_lock_set(struct asus_drvdata *drvdata, bool enabled) +{ + struct asus_work_action *action; + unsigned long flags; + + action = kzalloc(sizeof(struct asus_work_action), GFP_ATOMIC); + if (!action) + return -ENOMEM; + + drvdata->fn_lock = enabled; + action->type = FN_LOCK_SYNC; + action->data.fn_lock = drvdata->fn_lock; + INIT_LIST_HEAD(&action->node); + + spin_lock_irqsave(&drvdata->worker->lock, flags); + asus_worker_schedule(drvdata->worker, action); + spin_unlock_irqrestore(&drvdata->worker->lock, flags); + + return 0; +} + +static int asus_kbd_wmi_fan_send(struct asus_drvdata *drvdata, u8 *report_data, + size_t report_size) +{ + struct asus_work_action *action; + unsigned long flags; + + if (report_size > FEATURE_KBD_REPORT_SIZE) { + hid_err(drvdata->hdev, "Invalid report size for fan event: %zu\n", report_size); + return -EINVAL; } + action = kzalloc(sizeof(struct asus_work_action), GFP_NOWAIT); + if (!action) + return -ENOMEM; + + action->type = WMI_FAN; + action->data.fan_hid_data.report_size = report_size; + memcpy(action->data.fan_hid_data.report_data, report_data, report_size); + INIT_LIST_HEAD(&action->node); + + spin_lock_irqsave(&drvdata->worker->lock, flags); + asus_worker_schedule(drvdata->worker, action); + spin_unlock_irqrestore(&drvdata->worker->lock, flags); + return 0; } @@ -357,6 +452,7 @@ static int asus_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); + int ret; if ((usage->hid & HID_USAGE_PAGE) == HID_UP_ASUSVENDOR && (usage->hid & HID_USAGE) != 0x00 && @@ -375,8 +471,11 @@ static int asus_event(struct hid_device *hdev, struct hid_field *field, return !asus_hid_event(ASUS_EV_BRTTOGGLE); case KEY_FN_ESC: if (drvdata->quirks & QUIRK_HID_FN_LOCK) { - drvdata->fn_lock = !drvdata->fn_lock; - schedule_work(&drvdata->fn_lock_sync_work); + ret = asus_kbd_fn_lock_set(drvdata, !drvdata->fn_lock); + if (ret) { + hid_err(hdev, "Error while toggling FN lock: %d\n", ret); + return ret; + } } break; } @@ -389,6 +488,12 @@ static int asus_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); + int ret; + + if (size < 2) { + hid_dbg(hdev, "Unexpected keyboard report size %d\n", size); + return 0; + } if (drvdata->battery && data[0] == BATTERY_REPORT_ID) return asus_report_battery(drvdata, data, size); @@ -414,19 +519,13 @@ static int asus_raw_event(struct hid_device *hdev, * pass to userspace so it can implement its own fan control. */ if (data[1] == ASUS_FAN_CTRL_KEY_CODE) { - int ret = asus_wmi_send_event(drvdata, ASUS_FAN_CTRL_KEY_CODE); + ret = asus_kbd_wmi_fan_send(drvdata, data, size); - if (ret == 0) { - /* Successfully handled by asus-wmi, block event */ + /* if execution deferred successfully block event */ + if (ret == 0) return -1; - } - /* - * Warn if asus-wmi failed (but not if it's unavailable). - * Let the event reach userspace in all failure cases. - */ - if (ret != -ENODEV) - hid_warn(hdev, "Failed to notify asus-wmi: %d\n", ret); + return ret; } /* @@ -569,59 +668,157 @@ static int asus_kbd_disable_oobe(struct hid_device *hdev) return 0; } -static int asus_kbd_set_fn_lock(struct hid_device *hdev, bool enabled) +static void asus_kbd_set_fn_lock(struct hid_device *hdev, bool enabled) { - u8 buf[] = { FEATURE_KBD_REPORT_ID, 0xd0, 0x4e, !!enabled }; + const u8 buf[FEATURE_KBD_REPORT_SIZE] = { FEATURE_KBD_REPORT_ID, 0xd0, 0x4e, !!enabled }; + int ret; - return asus_kbd_set_report(hdev, buf, sizeof(buf)); + ret = asus_kbd_set_report(hdev, buf, sizeof(buf)); + if (ret < 0) + hid_err(hdev, "Asus failed to set fn lock: %d\n", ret); } -static void asus_sync_fn_lock(struct work_struct *work) +static void asus_kbd_set_brightness(struct hid_device *hdev, u8 brightness) { - struct asus_drvdata *drvdata = - container_of(work, struct asus_drvdata, fn_lock_sync_work); + const u8 buf[FEATURE_KBD_REPORT_SIZE] = { + FEATURE_KBD_REPORT_ID, 0xba, 0xc5, 0xc4, brightness + }; + int ret; - asus_kbd_set_fn_lock(drvdata->hdev, drvdata->fn_lock); + ret = asus_kbd_set_report(hdev, buf, sizeof(buf)); + if (ret < 0) + hid_err(hdev, "Asus failed to set keyboard backlight: %d\n", ret); } -static void asus_schedule_work(struct asus_kbd_leds *led) +static void asus_kbd_wmi_fan(struct hid_device *hdev, struct hid_raw_event_data *data) { + struct asus_drvdata *drvdata = hid_get_drvdata(hdev); + int ret; + + ret = asus_wmi_send_event(drvdata, ASUS_FAN_CTRL_KEY_CODE); + + /* + * Warn if asus-wmi failed (but not if it's unavailable). + * Let the event reach userspace in all failure cases. + */ + switch (ret) { + case -ENODEV: + break; + case 0: + return; + default: + hid_warn(hdev, "Failed to notify asus-wmi: %d\n", ret); + break; + } + + /* + * Fallback: pass the raw event to the HID core; to avoid + * racing against the hid_report_raw_event() that generated + * this event use the same locking mechanism and wait for + * that function to terminate and signal the deferred execution + * before raising the stored event. + */ + down(&hdev->driver_input_lock); + hid_report_raw_event(hdev, HID_INPUT_REPORT, + data->report_data, data->report_size, + data->report_size, 1); + up(&hdev->driver_input_lock); +} + +static void asus_kbd_backlight_set(struct asus_hid_listener *listener, int brightness) +{ + struct asus_drvdata *drvdata = container_of(listener, struct asus_drvdata, listener); + struct asus_worker *worker = drvdata->worker; + struct asus_work_action *action; unsigned long flags; - spin_lock_irqsave(&led->lock, flags); - if (!led->removed) - schedule_work(&led->work); - spin_unlock_irqrestore(&led->lock, flags); + drvdata->kbd_backlight_brightness = brightness; + + action = kzalloc(sizeof(struct asus_work_action), GFP_NOWAIT); + if (!action) + return; + + action->type = BRIGHTNESS_SET; + action->data.brightness = brightness; + INIT_LIST_HEAD(&action->node); + + spin_lock_irqsave(&worker->lock, flags); + asus_worker_schedule(worker, action); + spin_unlock_irqrestore(&worker->lock, flags); } -static void asus_kbd_backlight_set(struct asus_hid_listener *listener, - int brightness) +static void asus_work(struct work_struct *work) { - struct asus_kbd_leds *led = container_of(listener, struct asus_kbd_leds, - listener); + struct asus_worker *worker = container_of(work, struct asus_worker, work); + struct asus_work_action *action = NULL; unsigned long flags; - spin_lock_irqsave(&led->lock, flags); - led->brightness = brightness; - spin_unlock_irqrestore(&led->lock, flags); + /* Save the action to be performed and clear the flag */ + spin_lock_irqsave(&worker->lock, flags); + if (!list_empty(&worker->actions)) { + action = list_first_entry(&worker->actions, + struct asus_work_action, node); + list_del(&action->node); + } + spin_unlock_irqrestore(&worker->lock, flags); + + if (!action) + return; + + switch (action->type) { + case BRIGHTNESS_SET: + asus_kbd_set_brightness(worker->hdev, action->data.brightness); + break; + case FN_LOCK_SYNC: + asus_kbd_set_fn_lock(worker->hdev, action->data.fn_lock); + break; + case WMI_FAN: + asus_kbd_wmi_fan(worker->hdev, &action->data.fan_hid_data); + break; + default: + hid_err(worker->hdev, "Invalid action type: %d\n", action->type); + break; + } + + kfree(action); - asus_schedule_work(led); + /* Re-schedule if there are more pending actions */ + spin_lock_irqsave(&worker->lock, flags); + if (!list_empty(&worker->actions)) + schedule_work(&worker->work); + spin_unlock_irqrestore(&worker->lock, flags); } -static void asus_kbd_backlight_work(struct work_struct *work) +static int asus_worker_create(struct hid_device *hdev, struct asus_drvdata *drvdata) { - struct asus_kbd_leds *led = container_of(work, struct asus_kbd_leds, work); - u8 buf[] = { FEATURE_KBD_REPORT_ID, 0xba, 0xc5, 0xc4, 0x00 }; - int ret; + drvdata->worker = devm_kzalloc(&hdev->dev, sizeof(struct asus_worker), GFP_KERNEL); + if (!drvdata->worker) + return -ENOMEM; + + drvdata->worker->removed = false; + drvdata->worker->hdev = hdev; + INIT_LIST_HEAD(&drvdata->worker->actions); + + INIT_WORK(&drvdata->worker->work, asus_work); + spin_lock_init(&drvdata->worker->lock); + + return 0; +} + +static void asus_worker_stop(struct asus_worker *worker) +{ + struct asus_work_action *action, *tmp; unsigned long flags; - spin_lock_irqsave(&led->lock, flags); - buf[4] = led->brightness; - spin_unlock_irqrestore(&led->lock, flags); + spin_lock_irqsave(&worker->lock, flags); + worker->removed = true; + list_for_each_entry_safe(action, tmp, &worker->actions, node) { + list_del(&action->node); + kfree(action); + } + spin_unlock_irqrestore(&worker->lock, flags); - ret = asus_kbd_set_report(led->hdev, buf, sizeof(buf)); - if (ret < 0) - hid_err(led->hdev, "Asus failed to set keyboard backlight: %d\n", ret); + cancel_work_sync(&worker->work); } /* @@ -644,7 +841,7 @@ static int mcu_parse_version_string(const u8 *response, size_t response_size) dots++; } - if (dots != 2 || p >= end || (p + 3) >= end) + if (dots != 2 || end - p < 3) return -EINVAL; memcpy(buf, p, 3); @@ -753,30 +950,18 @@ static int asus_kbd_register_leds(struct hid_device *hdev) return ret; } - if (drvdata->quirks & QUIRK_ROG_ALLY_XPAD) { + if ((drvdata->quirks & QUIRK_ROG_ALLY_XPAD) && hid_is_usb(hdev)) { intf = to_usb_interface(hdev->dev.parent); udev = interface_to_usbdev(intf); validate_mcu_fw_version(hdev, le16_to_cpu(udev->descriptor.idProduct)); } - drvdata->kbd_backlight = devm_kzalloc(&hdev->dev, - sizeof(struct asus_kbd_leds), - GFP_KERNEL); - if (!drvdata->kbd_backlight) - return -ENOMEM; - - drvdata->kbd_backlight->removed = false; - drvdata->kbd_backlight->brightness = 0; - drvdata->kbd_backlight->hdev = hdev; - drvdata->kbd_backlight->listener.brightness_set = asus_kbd_backlight_set; - INIT_WORK(&drvdata->kbd_backlight->work, asus_kbd_backlight_work); - spin_lock_init(&drvdata->kbd_backlight->lock); - - ret = asus_hid_register_listener(&drvdata->kbd_backlight->listener); + drvdata->listener.brightness_set = asus_kbd_backlight_set; + ret = asus_hid_register_listener(&drvdata->listener); if (ret < 0) { - /* No need to have this still around */ - devm_kfree(&hdev->dev, drvdata->kbd_backlight); + hid_err(hdev, "Unable to register kbd brightness listener: %d\n", ret); + drvdata->listener.brightness_set = NULL; } return ret; @@ -998,11 +1183,9 @@ static int asus_input_configured(struct hid_device *hdev, struct hid_input *hi) drvdata->input = input; - if (drvdata->quirks & QUIRK_HID_FN_LOCK) { - drvdata->fn_lock = true; - INIT_WORK(&drvdata->fn_lock_sync_work, asus_sync_fn_lock); - asus_kbd_set_fn_lock(hdev, true); - } + if ((drvdata->quirks & QUIRK_HID_FN_LOCK) && + (asus_kbd_fn_lock_set(drvdata, true))) + hid_warn(hdev, "Error while setting FN lock to ON\n"); return 0; } @@ -1165,20 +1348,16 @@ static int asus_start_multitouch(struct hid_device *hdev) static int __maybe_unused asus_resume(struct hid_device *hdev) { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); - int ret = 0; - if (drvdata->kbd_backlight) { - const u8 buf[] = { FEATURE_KBD_REPORT_ID, 0xba, 0xc5, 0xc4, - drvdata->kbd_backlight->brightness }; - ret = asus_kbd_set_report(hdev, buf, sizeof(buf)); - if (ret < 0) { - hid_err(hdev, "Asus failed to set keyboard backlight: %d\n", ret); - goto asus_resume_err; - } - } + /* + * If we have a backlight listener registered, restore the previous state, + * in case of error do not fail: most models restore the backlight + * automatically, and the error is non-fatal. + */ + if (drvdata->listener.brightness_set) + asus_kbd_backlight_set(&drvdata->listener, drvdata->kbd_backlight_brightness); -asus_resume_err: - return ret; + return 0; } static int __maybe_unused asus_reset_resume(struct hid_device *hdev) @@ -1288,8 +1467,15 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id) is_vendor = true; } + ret = asus_worker_create(hdev, drvdata); + if (ret) { + hid_warn(hdev, "Failed to initialize worker: %d\n", ret); + return ret; + } + ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT); if (ret) { + asus_worker_stop(drvdata->worker); hid_err(hdev, "Asus hw start failed: %d\n", ret); return ret; } @@ -1337,6 +1523,10 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id) return 0; err_stop_hw: + if (drvdata->listener.brightness_set) + asus_hid_unregister_listener(&drvdata->listener); + + asus_worker_stop(drvdata->worker); hid_hw_stop(hdev); return ret; } @@ -1344,21 +1534,11 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id) static void asus_remove(struct hid_device *hdev) { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); - unsigned long flags; - - if (drvdata->kbd_backlight) { - asus_hid_unregister_listener(&drvdata->kbd_backlight->listener); - - spin_lock_irqsave(&drvdata->kbd_backlight->lock, flags); - drvdata->kbd_backlight->removed = true; - spin_unlock_irqrestore(&drvdata->kbd_backlight->lock, flags); - - cancel_work_sync(&drvdata->kbd_backlight->work); - } - if (drvdata->quirks & QUIRK_HID_FN_LOCK) - cancel_work_sync(&drvdata->fn_lock_sync_work); + if (drvdata->listener.brightness_set) + asus_hid_unregister_listener(&drvdata->listener); + asus_worker_stop(drvdata->worker); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-ayaneo.c b/drivers/hid/hid-ayaneo.c new file mode 100644 index 0000000000000..bfd7b8dae362d --- /dev/null +++ b/drivers/hid/hid-ayaneo.c @@ -0,0 +1,885 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * HID driver for the AYANEO 3 detachable controller ("Magic Modules"). + * + * The AYANEO 3 controller exposes three USB HID interfaces behind + * VID 0x1c4f PID 0x0002 (a generic SigmaMicro ID): a gamepad, a + * keyboard for the extra buttons, and a vendor interface (application + * usage 0xff000001) accepting 65-byte commands. + * + * The driver claims every interface of the shared ID because + * hid-generic yields any device another driver's id table matches. + * Interfaces other than the AYANEO 3 vendor interface are started as + * transparent generic devices. On the vendor interface it provides: + * - module identification (which module type is inserted on each side) + * - software eject of the left/right modules + * - RGB control of the joystick rings as a multicolor LED class device + * - rumble intensity selection + * - configuration reset + * + * It complements the ayaneo-ec platform driver, which exposes module + * attach state and controller power. A full eject is: write to this + * driver's "eject" attribute, then power the controller off through + * ayaneo-ec's controller_power once the eject completes. + * + * The protocol was reverse engineered in the Handheld Daemon project by + * Antheas Kapenekakis. + * + * Command format (65 bytes, unnumbered report): + * [0] report id (0) + * [1:3] little-endian sum of bytes 7..64 + * [3] command + * [4] subcommand + * [5:] payload + * The device replies with a 64-byte report echoing the subcommand at + * byte 3. + * + * Copyright (C) 2026 Matías Martínez + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hid-ids.h" + +#define AYA3_REPORT_SIZE 65 +#define AYA3_RESP_SIZE 64 + +/* + * Empirical timings, inherited from the Handheld Daemon + * implementation of this protocol and validated on hardware: the + * device answers well within 300ms or not at all, needs about half + * a second to settle after a reset before it accepts a new + * configuration, and completes an eject handshake within a few + * seconds (polled below at a rate that keeps the sysfs write + * responsive). The firmware also wants a minimum settle time between + * an eject command and the subsequent power cut (value inherited from + * Handheld Daemon's AYA_MIN_EJECT). + */ +#define AYA3_CMD_TIMEOUT_MS 300 +#define AYA3_CMD_ATTEMPTS 3 +#define AYA3_RESET_SETTLE_MS 500 +#define AYA3_EJECT_POLL_MS 400 +#define AYA3_EJECT_POLLS 20 +#define AYA3_EJECT_MIN_MS 3000 + +/* Subcommands (byte 4). Byte 3 is 0x00 except for the config command. */ +#define AYA3_SUBCMD_CHECK 0x08 +#define AYA3_CMD_CONFIG 0x21 +#define AYA3_SUBCMD_CONFIG 0x09 + +/* Bits that stay set in the eject status byte after an eject completes */ +#define AYA3_EJECT_DONE_MASK 0x11 + +/* Application usage of the vendor interface's first collection */ +#define AYA3_APP_USAGE (HID_UP_MSVENDOR | 0x0001) + +/* Config command eject/reset field */ +#define AYA3_EJECT_LEFT 0x07 +#define AYA3_EJECT_RIGHT 0x70 +#define AYA3_RESET 0x88 + +/* Config command RGB modes */ +#define AYA3_RGB_SOLID 0x01 +#define AYA3_RGB_PULSE 0x02 +#define AYA3_RGB_RAINBOW 0x03 +#define AYA3_RGB_OFF 0xff + +/* Config command vibration levels, stored in the high nibble */ +enum aya3_vibration { + AYA3_VIBRATION_LOW = 0x1, + AYA3_VIBRATION_MEDIUM = 0x2, + AYA3_VIBRATION_HIGH = 0x3, + AYA3_VIBRATION_OFF = 0x4, +}; + +/* Firmware LED effects selectable through the LED "effect" attribute */ +enum aya3_effect { + AYA3_EFFECT_MONOCOLOR, + AYA3_EFFECT_BREATHE, + AYA3_EFFECT_RAINBOW, +}; + +static const char * const ayaneo_effect_names[] = { + [AYA3_EFFECT_MONOCOLOR] = "monocolor", + [AYA3_EFFECT_BREATHE] = "breathe", + [AYA3_EFFECT_RAINBOW] = "rainbow", +}; + +struct aya3_rgb { + u8 mode; + u8 r; + u8 g; + u8 b; +} __packed; + +/* + * The 65-byte config command. The checksum is the little-endian sum of + * bytes 7..64. The unk* fields are sent as zero. + */ +struct aya3_config { + u8 report_id; + __le16 csum; + u8 cmd; + u8 subcmd; + u8 unk5[3]; + struct aya3_rgb right; + struct aya3_rgb left; + u8 unk16[4]; + u8 eject; + u8 unk21; + u8 sensitivity[2]; + u8 vibration; + u8 unk25[7]; + u8 magic; + u8 unk33[32]; +} __packed; +static_assert(sizeof(struct aya3_config) == AYA3_REPORT_SIZE); + +/* Replies echo the subcommand they answer at byte 3 */ +struct aya3_resp { + u8 unk0[3]; + u8 subcmd; + u8 unk4[15]; + u8 eject_status; + u8 unk20[12]; + u8 module_left; + u8 module_right; + u8 unk34[30]; +} __packed; +static_assert(sizeof(struct aya3_resp) == AYA3_RESP_SIZE); + +struct ayaneo { + struct hid_device *hdev; + /* DMA-safe command buffer; guarded by lock */ + u8 *xfer; + /* Serializes commands and cached-config access */ + struct mutex lock; + /* Protects the reply-matching state shared with raw_event */ + spinlock_t resp_lock; + struct completion resp_done; + struct aya3_resp resp; + u8 resp_expect; + bool resp_pending; + + /* + * True once a configuration was successfully applied. The driver + * sends no traffic unless userspace asked, and that policy + * extends across resets. + */ + bool configured; + + u8 rgb[3]; + u8 brightness; + u8 effect; + u8 vibration; + + struct led_classdev_mc mcled; + struct mc_subled subleds[3]; +}; + +static int ayaneo_send(struct ayaneo *aya) +{ + int ret; + + ret = hid_hw_output_report(aya->hdev, aya->xfer, AYA3_REPORT_SIZE); + if (ret == -ENOSYS) + ret = hid_hw_raw_request(aya->hdev, aya->xfer[0], aya->xfer, + AYA3_REPORT_SIZE, HID_OUTPUT_REPORT, + HID_REQ_SET_REPORT); + if (ret < 0) + return ret; + return 0; +} + +/** + * ayaneo_cmd() - send the command in aya->xfer and wait for the reply + * @aya: driver data with the fully built 65-byte command in @aya->xfer + * @resp: destination for the reply, or NULL to discard it + * @attempts: how many times to send before giving up + * + * The device echoes the subcommand byte of the command it is answering, + * which ayaneo_raw_event() uses to match replies. Unanswered commands are + * retried up to @attempts times. + * + * Context: process context. The caller must hold @aya->lock, which + * protects @aya->xfer and the reply state. + * Return: 0 on success, -ETIMEDOUT if every attempt went unanswered, or + * a negative errno if sending failed. + */ +static int ayaneo_cmd(struct ayaneo *aya, struct aya3_resp *resp, int attempts) +{ + int attempt, ret; + + lockdep_assert_held(&aya->lock); + + for (attempt = 0; attempt < attempts; attempt++) { + scoped_guard(spinlock_irqsave, &aya->resp_lock) { + reinit_completion(&aya->resp_done); + aya->resp_expect = aya->xfer[4]; + aya->resp_pending = true; + } + + ret = ayaneo_send(aya); + if (ret) { + scoped_guard(spinlock_irqsave, &aya->resp_lock) + aya->resp_pending = false; + return ret; + } + + if (wait_for_completion_timeout(&aya->resp_done, + msecs_to_jiffies(AYA3_CMD_TIMEOUT_MS))) { + /* + * raw_event cleared resp_pending before completing, + * so nothing else writes aya->resp: the copy is + * safe unlocked. + */ + if (resp) + memcpy(resp, &aya->resp, sizeof(*resp)); + return 0; + } + } + scoped_guard(spinlock_irqsave, &aya->resp_lock) + aya->resp_pending = false; + return -ETIMEDOUT; +} + +static void ayaneo_checksum(u8 *buf) +{ + u16 sum = 0; + int i; + + for (i = 7; i < AYA3_REPORT_SIZE; i++) + sum += buf[i]; + put_unaligned_le16(sum, buf + 1); +} + +static int ayaneo_check(struct ayaneo *aya, struct aya3_resp *resp, + int attempts) +{ + memset(aya->xfer, 0, AYA3_REPORT_SIZE); + aya->xfer[4] = AYA3_SUBCMD_CHECK; + return ayaneo_cmd(aya, resp, attempts); +} + +/* + * The config command sets everything at once: RGB for both rings, + * vibration strength and the eject/reset field. The command can also + * carry joystick sensitivity. Those bytes are left zero so the + * firmware setting is not clobbered on every RGB update. + */ +static int ayaneo_send_config(struct ayaneo *aya, u8 eject) +{ + static const struct aya3_config template = { + .cmd = AYA3_CMD_CONFIG, + .subcmd = AYA3_SUBCMD_CONFIG, + .magic = 0x01, + }; + struct aya3_config *cfg = (struct aya3_config *)aya->xfer; + u8 mode = AYA3_RGB_OFF; + int ret; + + if (aya->effect == AYA3_EFFECT_RAINBOW) { + if (aya->brightness) + mode = AYA3_RGB_RAINBOW; + } else if (aya->rgb[0] || aya->rgb[1] || aya->rgb[2]) { + mode = aya->effect == AYA3_EFFECT_BREATHE ? + AYA3_RGB_PULSE : AYA3_RGB_SOLID; + } + + *cfg = template; + cfg->right.mode = mode; + if (mode == AYA3_RGB_RAINBOW) { + /* + * The firmware generates hue and tempo itself. The RGB + * payload only scales the amplitude. Handheld Daemon + * derives it as HSV(275, 100, brightness), which is + * (7/12 * v, 0, v) in RGB. + */ + cfg->right.r = aya->brightness * 7 / 12; + cfg->right.g = 0; + cfg->right.b = aya->brightness; + } else { + cfg->right.r = aya->rgb[0]; + cfg->right.g = aya->rgb[1]; + cfg->right.b = aya->rgb[2]; + } + cfg->left = cfg->right; + cfg->eject = eject; + cfg->vibration = aya->vibration << 4; + ayaneo_checksum(aya->xfer); + + ret = ayaneo_cmd(aya, NULL, AYA3_CMD_ATTEMPTS); + if (!ret) + aya->configured = true; + return ret; +} + +static int ayaneo_raw_event(struct hid_device *hdev, struct hid_report *report, + u8 *data, int size) +{ + struct ayaneo *aya = hid_get_drvdata(hdev); + const struct aya3_resp *resp = (const struct aya3_resp *)data; + + /* Transparent interfaces have no driver data */ + if (!aya) + return 0; + + guard(spinlock_irqsave)(&aya->resp_lock); + + if (!aya->resp_pending || size < AYA3_RESP_SIZE) + return 0; + /* + * Replies carry no sequence number, only the subcommand echo. A + * late reply to a timed-out command can thus complete a newer + * command with the same subcommand. Such replies are snapshots + * of the same query milliseconds apart, so this is harmless. + * Replies to a different subcommand are dropped here. + */ + if (resp->subcmd != aya->resp_expect) + return 0; + + memcpy(&aya->resp, data, sizeof(aya->resp)); + aya->resp_pending = false; + complete(&aya->resp_done); + return 0; +} + +static ssize_t ayaneo_module_show(struct device *dev, char *buf, bool right) +{ + struct ayaneo *aya = dev_get_drvdata(dev); + struct aya3_resp resp; + int ret = 0; + + scoped_cond_guard(mutex_intr, return -EINTR, &aya->lock) + ret = ayaneo_check(aya, &resp, AYA3_CMD_ATTEMPTS); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%02x\n", + right ? resp.module_right : resp.module_left); +} + +static ssize_t module_left_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return ayaneo_module_show(dev, buf, false); +} +static DEVICE_ATTR_RO(module_left); + +static ssize_t module_right_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return ayaneo_module_show(dev, buf, true); +} +static DEVICE_ATTR_RO(module_right); + +static ssize_t eject_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ayaneo *aya = dev_get_drvdata(dev); + struct aya3_resp resp; + u8 eject; + int ret = 0, err, i; + + if (sysfs_streq(buf, "left")) + eject = AYA3_EJECT_LEFT; + else if (sysfs_streq(buf, "right")) + eject = AYA3_EJECT_RIGHT; + else if (sysfs_streq(buf, "both")) + eject = AYA3_EJECT_LEFT | AYA3_EJECT_RIGHT; + else + return -EINVAL; + + scoped_cond_guard(mutex_intr, return -EINTR, &aya->lock) { + unsigned long deadline = jiffies + + msecs_to_jiffies(AYA3_EJECT_MIN_MS); + + ret = ayaneo_send_config(aya, eject); + if (ret) + break; + + /* + * Wait for the firmware to report the eject as done: no + * bits outside AYA3_EJECT_DONE_MASK set, matching + * Handheld Daemon's verification. The firmware reports + * the in-progress state from the first poll, and the + * minimum-time floor below guards the power-cut step + * regardless. Userspace must then cut power through + * ayaneo-ec's controller_power for the module to be + * physically released. + */ + ret = -ETIMEDOUT; + for (i = 0; i < AYA3_EJECT_POLLS; i++) { + if (msleep_interruptible(AYA3_EJECT_POLL_MS)) { + ret = -EINTR; + break; + } + err = ayaneo_check(aya, &resp, 1); + if (err == -ETIMEDOUT) + continue; /* busy mid-eject, keep polling */ + if (err) { + ret = err; + break; + } + if (!(resp.eject_status & ~AYA3_EJECT_DONE_MASK)) { + ret = 0; + break; + } + } + + /* + * The firmware wants a minimum time between the eject + * command and the power cut that follows this write. + */ + if (!ret && time_before(jiffies, deadline)) { + if (msleep_interruptible(jiffies_to_msecs(deadline - + jiffies))) + ret = -EINTR; + } + } + return ret ? ret : count; +} +static DEVICE_ATTR_WO(eject); + +static ssize_t reset_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ayaneo *aya = dev_get_drvdata(dev); + bool value; + int ret; + + ret = kstrtobool(buf, &value); + if (ret) + return ret; + if (!value) + return count; + + scoped_cond_guard(mutex_intr, return -EINTR, &aya->lock) { + ret = ayaneo_send_config(aya, AYA3_RESET); + if (!ret) { + msleep(AYA3_RESET_SETTLE_MS); + ret = ayaneo_send_config(aya, 0); + } + } + return ret ? ret : count; +} +static DEVICE_ATTR_WO(reset); + +/* + * Rumble intensity names, index-aligned with the wire values through + * ayaneo_rumble_levels below. + */ +static const char * const ayaneo_rumble_names[] = { + "off", "low", "medium", "high", +}; + +static const u8 ayaneo_rumble_levels[] = { + AYA3_VIBRATION_OFF, + AYA3_VIBRATION_LOW, + AYA3_VIBRATION_MEDIUM, + AYA3_VIBRATION_HIGH, +}; + +static ssize_t rumble_intensity_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ayaneo *aya = dev_get_drvdata(dev); + u8 level; + int i; + + /* + * Single-byte snapshot; writers serialize on aya->lock, and a + * racing update is harmless, while taking the mutex here would + * block trivial reads behind a multi-second eject. + */ + level = READ_ONCE(aya->vibration); + + for (i = 0; i < ARRAY_SIZE(ayaneo_rumble_levels); i++) { + if (ayaneo_rumble_levels[i] == level) + return sysfs_emit(buf, "%s\n", ayaneo_rumble_names[i]); + } + return -EINVAL; +} + +static ssize_t rumble_intensity_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ayaneo *aya = dev_get_drvdata(dev); + u8 previous; + int ret; + + ret = sysfs_match_string(ayaneo_rumble_names, buf); + if (ret < 0) + return ret; + + scoped_cond_guard(mutex_intr, return -EINTR, &aya->lock) { + previous = aya->vibration; + aya->vibration = ayaneo_rumble_levels[ret]; + ret = ayaneo_send_config(aya, 0); + if (ret) { + aya->vibration = previous; + return ret; + } + } + return count; +} +static DEVICE_ATTR_RW(rumble_intensity); + +static ssize_t rumble_intensity_index_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + ssize_t count = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(ayaneo_rumble_names); i++) + count += sysfs_emit_at(buf, count, "%s ", + ayaneo_rumble_names[i]); + + if (count) + buf[count - 1] = '\n'; + + return count; +} +static DEVICE_ATTR_RO(rumble_intensity_index); + +static struct attribute *ayaneo_attrs[] = { + &dev_attr_module_left.attr, + &dev_attr_module_right.attr, + &dev_attr_eject.attr, + &dev_attr_reset.attr, + &dev_attr_rumble_intensity.attr, + &dev_attr_rumble_intensity_index.attr, + NULL +}; + +static const struct attribute_group ayaneo_group = { + .attrs = ayaneo_attrs, +}; + +static int ayaneo_led_set(struct led_classdev *cdev, enum led_brightness value) +{ + struct led_classdev_mc *mc = lcdev_to_mccdev(cdev); + struct ayaneo *aya = container_of(mc, struct ayaneo, mcled); + int ret = 0, i; + + scoped_cond_guard(mutex_intr, return -EINTR, &aya->lock) { + led_mc_calc_color_components(mc, value); + aya->brightness = min_t(unsigned int, value, 255); + for (i = 0; i < 3; i++) + aya->rgb[i] = min_t(unsigned int, + aya->subleds[i].brightness, 255); + + ret = ayaneo_send_config(aya, 0); + if (ret) + hid_err(aya->hdev, + "failed to update RGB config: %d\n", ret); + } + return ret; +} + +static ssize_t effect_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct led_classdev *cdev = dev_get_drvdata(dev); + struct ayaneo *aya = container_of(lcdev_to_mccdev(cdev), + struct ayaneo, mcled); + u8 effect; + + /* + * Single-byte snapshot; writers serialize on aya->lock, and a + * racing update is harmless, while taking the mutex here would + * block trivial reads behind a multi-second eject. + */ + effect = READ_ONCE(aya->effect); + + return sysfs_emit(buf, "%s\n", ayaneo_effect_names[effect]); +} + +static ssize_t effect_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct led_classdev *cdev = dev_get_drvdata(dev); + struct ayaneo *aya = container_of(lcdev_to_mccdev(cdev), + struct ayaneo, mcled); + u8 previous; + int ret; + + ret = sysfs_match_string(ayaneo_effect_names, buf); + if (ret < 0) + return ret; + + scoped_cond_guard(mutex_intr, return -EINTR, &aya->lock) { + previous = aya->effect; + if (previous == ret) + return count; + aya->effect = ret; + ret = ayaneo_send_config(aya, 0); + if (ret) { + aya->effect = previous; + return ret; + } + } + return count; +} +static DEVICE_ATTR_RW(effect); + +static ssize_t effect_index_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t count = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(ayaneo_effect_names); i++) + count += sysfs_emit_at(buf, count, "%s ", + ayaneo_effect_names[i]); + + if (count) + buf[count - 1] = '\n'; + + return count; +} +static DEVICE_ATTR_RO(effect_index); + +static struct attribute *ayaneo_led_attrs[] = { + &dev_attr_effect.attr, + &dev_attr_effect_index.attr, + NULL +}; + +static const struct attribute_group ayaneo_led_group = { + .attrs = ayaneo_led_attrs, +}; + +/* + * The final LED write issued by the unregister needs its reply. Callers + * guarantee IO is working: probe leaves IO started after hid_hw_open, + * and remove re-enables it explicitly. + */ +static void ayaneo_unregister_led(struct ayaneo *aya) +{ + led_classdev_multicolor_unregister(&aya->mcled); + flush_work(&aya->mcled.led_cdev.set_brightness_work); +} + +static int ayaneo_register_led(struct ayaneo *aya) +{ + struct led_classdev *cdev = &aya->mcled.led_cdev; + int ret, i; + + aya->subleds[0].color_index = LED_COLOR_ID_RED; + aya->subleds[1].color_index = LED_COLOR_ID_GREEN; + aya->subleds[2].color_index = LED_COLOR_ID_BLUE; + /* + * Full intensity by default so brightness writes produce light + * before userspace sets multi_intensity (querying the firmware + * for its value would break the no-probe-traffic policy). + */ + for (i = 0; i < 3; i++) + aya->subleds[i].intensity = 255; + aya->mcled.subled_info = aya->subleds; + aya->mcled.num_colors = 3; + + /* OGC-only Steam compatibility name; the generic ABI remains unchanged. */ + cdev->name = "go:rgb:joystick_rings"; + cdev->color = LED_COLOR_ID_RGB; + cdev->brightness = 0; + cdev->max_brightness = 255; + cdev->brightness_set_blocking = ayaneo_led_set; + + /* + * Not devm: the LED must be unregistered before hid_hw_stop() in + * remove, or a concurrent brightness write could reach a torn + * down transport. + */ + ret = led_classdev_multicolor_register(&aya->hdev->dev, &aya->mcled); + if (ret) + return ret; + + /* + * The multicolor class installs its own groups during + * registration, so the effect attributes are added afterwards. + * They are tied to the LED device and vanish with it. + */ + ret = devm_device_add_group(cdev->dev, &ayaneo_led_group); + if (ret) + ayaneo_unregister_led(aya); + return ret; +} + +static const struct dmi_system_id ayaneo_dmi_table[] = { + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "AYANEO"), + DMI_MATCH(DMI_BOARD_NAME, "AYANEO 3"), + }, + }, + {} +}; + +static int ayaneo_probe(struct hid_device *hdev, const struct hid_device_id *id) +{ + struct ayaneo *aya; + int ret; + + ret = hid_parse(hdev); + if (ret) + return ret; + + /* + * The VID/PID is a generic SigmaMicro ID shared by the gamepad, + * keyboard and vendor interfaces, and by unrelated devices on + * other machines. Refusing those probes would strand them, as + * hid-generic yields any device that another driver's id table + * matches. Everything that is not the AYANEO 3 vendor interface + * is therefore started as a transparent generic device, exactly + * as hid-generic would have started it. + */ + if (!dmi_check_system(ayaneo_dmi_table) || !hid_is_usb(hdev) || + !hdev->maxcollection || + hdev->collection->type != HID_COLLECTION_APPLICATION || + hdev->collection->usage != AYA3_APP_USAGE) { + hdev->quirks |= HID_QUIRK_INPUT_PER_APP; + return hid_hw_start(hdev, HID_CONNECT_DEFAULT); + } + + aya = devm_kzalloc(&hdev->dev, sizeof(*aya), GFP_KERNEL); + if (!aya) + return -ENOMEM; + + aya->xfer = devm_kzalloc(&hdev->dev, AYA3_REPORT_SIZE, GFP_KERNEL); + if (!aya->xfer) + return -ENOMEM; + + aya->hdev = hdev; + aya->vibration = AYA3_VIBRATION_MEDIUM; + init_completion(&aya->resp_done); + spin_lock_init(&aya->resp_lock); + ret = devm_mutex_init(&hdev->dev, &aya->lock); + if (ret) + return ret; + hid_set_drvdata(hdev, aya); + + ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW); + if (ret) + return ret; + + ret = hid_hw_open(hdev); + if (ret) + goto err_stop; + + /* + * Report delivery is off during probe by default, but the LED + * and attributes go live below and their first commands need + * replies. Leave IO started (the HID core supports probe + * returning with IO started). + */ + hid_device_io_start(hdev); + + ret = ayaneo_register_led(aya); + if (ret) + goto err_close; + + ret = sysfs_create_group(&hdev->dev.kobj, &ayaneo_group); + if (ret) + goto err_led; + + return 0; + +err_led: + ayaneo_unregister_led(aya); +err_close: + hid_hw_close(hdev); +err_stop: + hid_hw_stop(hdev); + return ret; +} + +static void ayaneo_remove(struct hid_device *hdev) +{ + struct ayaneo *aya = hid_get_drvdata(hdev); + + /* Transparent interfaces carry no driver state */ + if (!aya) { + hid_hw_stop(hdev); + return; + } + + /* + * The HID core stops report delivery before calling remove, but + * the final LED write issued by the unregister below needs its + * reply, and the group removal waits for in-flight attribute + * callers, which need working replies to finish promptly. + * Re-enable IO for the teardown writes. + */ + hid_device_io_start(hdev); + sysfs_remove_group(&hdev->dev.kobj, &ayaneo_group); + led_classdev_multicolor_unregister(&aya->mcled); + /* + * A brightness store racing with the unregister can requeue + * set_brightness_work after the flush inside + * led_classdev_unregister() runs but before the sysfs node is + * removed. Flush again now that nothing can requeue it, while + * the transport is still up. This double flush papers over a + * led-class ordering gap (the flush runs before + * device_unregister) that affects any brightness_set_blocking + * driver and deserves a core fix separately. + */ + flush_work(&aya->mcled.led_cdev.set_brightness_work); + hid_device_io_stop(hdev); + hid_hw_close(hdev); + hid_hw_stop(hdev); +} + +static int ayaneo_reset_resume(struct hid_device *hdev) +{ + struct ayaneo *aya = hid_get_drvdata(hdev); + int ret = 0; + + /* Transparent interfaces resume exactly as hid-generic would */ + if (!aya) { + if (hdev->claimed & HID_CLAIMED_INPUT) + hidinput_reset_resume(hdev); + return 0; + } + + scoped_cond_guard(mutex_intr, return -EINTR, &aya->lock) { + if (aya->configured) + ret = ayaneo_send_config(aya, 0); + } + return ret; +} + +static const struct hid_device_id ayaneo_devices[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_SIGMA_MICRO, + USB_DEVICE_ID_SIGMA_MICRO_KEYBOARD) }, + {} +}; +MODULE_DEVICE_TABLE(hid, ayaneo_devices); + +static struct hid_driver ayaneo_driver = { + .name = "hid-ayaneo", + .id_table = ayaneo_devices, + .probe = ayaneo_probe, + .remove = ayaneo_remove, + .raw_event = ayaneo_raw_event, + .reset_resume = ayaneo_reset_resume, +}; +module_hid_driver(ayaneo_driver); + +MODULE_AUTHOR("Matías Martínez "); +MODULE_DESCRIPTION("AYANEO 3 detachable controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index cf123347a2af7..14a06ab435a23 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -379,6 +379,9 @@ static int hid_add_field(struct hid_parser *parser, unsigned report_type, unsign static u32 item_udata(struct hid_item *item) { + if (item->format != HID_ITEM_FORMAT_SHORT) + return 0; + switch (item->size) { case 1: return item->data.u8; case 2: return item->data.u16; @@ -389,6 +392,9 @@ static u32 item_udata(struct hid_item *item) static s32 item_sdata(struct hid_item *item) { + if (item->format != HID_ITEM_FORMAT_SHORT) + return 0; + switch (item->size) { case 1: return item->data.s8; case 2: return item->data.s16; @@ -1933,13 +1939,14 @@ int hid_set_field(struct hid_field *field, unsigned offset, __s32 value) size = field->report_size; - hid_dump_input(field->report->device, field->usage + offset, value); - if (offset >= field->report_count) { hid_err(field->report->device, "offset (%d) exceeds report_count (%d)\n", offset, field->report_count); return -1; } + + hid_dump_input(field->report->device, field->usage + offset, value); + if (field->logical_minimum < 0) { if (value != snto32(s32ton(value, size), size)) { hid_err(field->report->device, "value %d is out of range\n", value); @@ -2447,9 +2454,16 @@ EXPORT_SYMBOL_GPL(hid_hw_start); * * This is usually called from remove function or from probe when something * failed and hid_hw_start was called already. + * + * If the caller enabled HID input via hid_device_io_start() and is unwinding + * without an explicit hid_device_io_stop(), quiesce input first so that + * in-flight reports cannot reach handlers (e.g. hidraw_report_event) whose + * backing objects hid_disconnect() is about to free. */ void hid_hw_stop(struct hid_device *hdev) { + if (hdev->io_started) + hid_device_io_stop(hdev); hid_disconnect(hdev); hdev->ll_driver->stop(hdev); } @@ -2842,6 +2856,8 @@ static int __hid_device_probe(struct hid_device *hdev, struct hid_driver *hdrv) */ if (ret) { + if (hdev->io_started) + hid_device_io_stop(hdev); devres_release_group(&hdev->dev, hdev->devres_group_id); hid_close_report(hdev); hdev->driver = NULL; diff --git a/drivers/hid/hid-corsair-void.c b/drivers/hid/hid-corsair-void.c index 5e9a5b8f7f167..071a663a6c268 100644 --- a/drivers/hid/hid-corsair-void.c +++ b/drivers/hid/hid-corsair-void.c @@ -92,6 +92,9 @@ #define CORSAIR_VOID_STATUS_REPORT_ID 0x64 #define CORSAIR_VOID_FIRMWARE_REPORT_ID 0x66 +#define CORSAIR_VOID_STATUS_REPORT_SIZE 5 +#define CORSAIR_VOID_FIRMWARE_REPORT_SIZE 5 + #define CORSAIR_VOID_USB_SIDETONE_REQUEST 0x1 #define CORSAIR_VOID_USB_SIDETONE_REQUEST_TYPE 0x21 #define CORSAIR_VOID_USB_SIDETONE_VALUE 0x200 @@ -742,6 +745,13 @@ static int corsair_void_raw_event(struct hid_device *hid_dev, /* Description of packets are documented at the top of this file */ if (hid_report->id == CORSAIR_VOID_STATUS_REPORT_ID) { + if (size < CORSAIR_VOID_STATUS_REPORT_SIZE) { + hid_warn_ratelimited(hid_dev, + "unexpected status report of size %d", + size); + return 1; + } + drvdata->mic_up = FIELD_GET(CORSAIR_VOID_MIC_MASK, data[2]); drvdata->connected = (data[3] == CORSAIR_VOID_WIRELESS_CONNECTED) || drvdata->is_wired; @@ -750,6 +760,13 @@ static int corsair_void_raw_event(struct hid_device *hid_dev, FIELD_GET(CORSAIR_VOID_CAPACITY_MASK, data[2]), data[3], data[4]); } else if (hid_report->id == CORSAIR_VOID_FIRMWARE_REPORT_ID) { + if (size < CORSAIR_VOID_FIRMWARE_REPORT_SIZE) { + hid_warn_ratelimited(hid_dev, + "unexpected firmware report of size %d", + size); + return 1; + } + drvdata->fw_receiver_major = data[1]; drvdata->fw_receiver_minor = data[2]; drvdata->fw_headset_major = data[3]; diff --git a/drivers/hid/hid-ft260.c b/drivers/hid/hid-ft260.c index 70e2eedb465af..f479459544aee 100644 --- a/drivers/hid/hid-ft260.c +++ b/drivers/hid/hid-ft260.c @@ -240,6 +240,8 @@ struct ft260_device { struct mutex lock; u8 write_buf[FT260_REPORT_MAX_LENGTH]; unsigned long need_wakeup_at; + /* Protects read_buf, read_idx and read_len against ft260_raw_event() */ + spinlock_t read_lock; u8 *read_buf; u16 read_idx; u16 read_len; @@ -501,6 +503,7 @@ static int ft260_i2c_read(struct ft260_device *dev, u8 addr, u8 *data, int timeout, ret = 0; struct ft260_i2c_read_request_report rep; struct hid_device *hdev = dev->hdev; + unsigned long irqflags; u8 bus_busy = 0; if ((flag & FT260_FLAG_START_REPEATED) == FT260_FLAG_START_REPEATED) @@ -526,9 +529,11 @@ static int ft260_i2c_read(struct ft260_device *dev, u8 addr, u8 *data, reinit_completion(&dev->wait); + spin_lock_irqsave(&dev->read_lock, irqflags); dev->read_idx = 0; dev->read_buf = data; dev->read_len = rd_len; + spin_unlock_irqrestore(&dev->read_lock, irqflags); ret = ft260_hid_output_report(hdev, (u8 *)&rep, sizeof(rep)); if (ret < 0) { @@ -543,7 +548,9 @@ static int ft260_i2c_read(struct ft260_device *dev, u8 addr, u8 *data, goto ft260_i2c_read_exit; } + spin_lock_irqsave(&dev->read_lock, irqflags); dev->read_buf = NULL; + spin_unlock_irqrestore(&dev->read_lock, irqflags); if (flag & FT260_FLAG_STOP) bus_busy = FT260_I2C_STATUS_BUS_BUSY; @@ -562,7 +569,9 @@ static int ft260_i2c_read(struct ft260_device *dev, u8 addr, u8 *data, } while (len > 0); ft260_i2c_read_exit: + spin_lock_irqsave(&dev->read_lock, irqflags); dev->read_buf = NULL; + spin_unlock_irqrestore(&dev->read_lock, irqflags); return ret; } @@ -1018,6 +1027,7 @@ static int ft260_probe(struct hid_device *hdev, const struct hid_device_id *id) "FT260 usb-i2c bridge"); mutex_init(&dev->lock); + spin_lock_init(&dev->read_lock); init_completion(&dev->wait); ret = ft260_xfer_status(dev, FT260_I2C_STATUS_BUS_BUSY); @@ -1067,6 +1077,7 @@ static int ft260_raw_event(struct hid_device *hdev, struct hid_report *report, { struct ft260_device *dev = hid_get_drvdata(hdev); struct ft260_i2c_input_report *xfer = (void *)data; + unsigned long irqflags; if (size < offsetof(struct ft260_i2c_input_report, data)) { hid_err(hdev, "short report %d\n", size); @@ -1075,6 +1086,8 @@ static int ft260_raw_event(struct hid_device *hdev, struct hid_report *report, if (xfer->report >= FT260_I2C_REPORT_MIN && xfer->report <= FT260_I2C_REPORT_MAX) { + bool complete_read; + ft260_dbg("i2c resp: rep %#02x len %d size %d\n", xfer->report, xfer->length, size); @@ -1085,8 +1098,15 @@ static int ft260_raw_event(struct hid_device *hdev, struct hid_report *report, return -1; } + /* + * Hold read_lock so a timed-out ft260_i2c_read() cannot + * clear read_buf between the NULL check and the memcpy. + */ + spin_lock_irqsave(&dev->read_lock, irqflags); + if ((dev->read_buf == NULL) || (xfer->length > dev->read_len - dev->read_idx)) { + spin_unlock_irqrestore(&dev->read_lock, irqflags); hid_err(hdev, "unexpected report %#02x, length %d\n", xfer->report, xfer->length); return -1; @@ -1095,8 +1115,11 @@ static int ft260_raw_event(struct hid_device *hdev, struct hid_report *report, memcpy(&dev->read_buf[dev->read_idx], &xfer->data, xfer->length); dev->read_idx += xfer->length; + complete_read = dev->read_idx == dev->read_len; + + spin_unlock_irqrestore(&dev->read_lock, irqflags); - if (dev->read_idx == dev->read_len) + if (complete_read) complete(&dev->wait); } else { diff --git a/drivers/hid/hid-haptic.c b/drivers/hid/hid-haptic.c index deadab28cdbef..66d90f0f78a3b 100644 --- a/drivers/hid/hid-haptic.c +++ b/drivers/hid/hid-haptic.c @@ -187,7 +187,7 @@ static void fill_effect_buf(struct hid_haptic_device *haptic, value = waveform_ordinal; break; default: - break; + continue; } field->value[j] = value; diff --git a/drivers/hid/hid-huawei.c b/drivers/hid/hid-huawei.c index 6a616bf21b38f..ee3fc6f68475a 100644 --- a/drivers/hid/hid-huawei.c +++ b/drivers/hid/hid-huawei.c @@ -44,11 +44,12 @@ static const __u8 huawei_cd30_kbd_rdesc_fixed[] = { static const __u8 *huawei_report_fixup(struct hid_device *hdev, __u8 *rdesc, unsigned int *rsize) { - struct usb_interface *intf = to_usb_interface(hdev->dev.parent); + struct usb_interface *intf = hid_is_usb(hdev) ? + to_usb_interface(hdev->dev.parent) : NULL; switch (hdev->product) { case USB_DEVICE_ID_HUAWEI_CD30KBD: - if (intf->cur_altsetting->desc.bInterfaceNumber == 1) { + if (!intf || intf->cur_altsetting->desc.bInterfaceNumber == 1) { if (*rsize != sizeof(huawei_cd30_kbd_rdesc_fixed) || memcmp(huawei_cd30_kbd_rdesc_fixed, rdesc, sizeof(huawei_cd30_kbd_rdesc_fixed)) != 0) { diff --git a/drivers/hid/hid-hyperv.c b/drivers/hid/hid-hyperv.c index 7d2b0063df151..fd90196430e29 100644 --- a/drivers/hid/hid-hyperv.c +++ b/drivers/hid/hid-hyperv.c @@ -171,18 +171,32 @@ static void mousevsc_free_device(struct mousevsc_dev *device) } static void mousevsc_on_receive_device_info(struct mousevsc_dev *input_device, - struct synthhid_device_info *device_info) + struct synthhid_device_info *device_info, + u32 device_info_size) { int ret = 0; struct hid_descriptor *desc; struct mousevsc_prt_msg ack; + size_t desc_offset; + size_t desc_size; input_device->dev_info_status = -ENOMEM; + if (device_info_size < sizeof(*device_info)) { + input_device->dev_info_status = -EINVAL; + goto cleanup; + } + input_device->hid_dev_info = device_info->hid_dev_info; desc = &device_info->hid_descriptor; + desc_offset = offsetof(struct synthhid_device_info, hid_descriptor); + desc_size = device_info_size - desc_offset; if (desc->bLength == 0) goto cleanup; + if (desc->bLength < sizeof(*desc) || desc->bLength > desc_size) { + input_device->dev_info_status = -EINVAL; + goto cleanup; + } /* The pointer is not NULL when we resume from hibernation */ kfree(input_device->hid_desc); @@ -197,6 +211,10 @@ static void mousevsc_on_receive_device_info(struct mousevsc_dev *input_device, input_device->dev_info_status = -EINVAL; goto cleanup; } + if (input_device->report_desc_size > desc_size - desc->bLength) { + input_device->dev_info_status = -EINVAL; + goto cleanup; + } /* The pointer is not NULL when we resume from hibernation */ kfree(input_device->report_desc); @@ -273,14 +291,17 @@ static void mousevsc_on_receive(struct hv_device *device, break; case SYNTH_HID_INITIAL_DEVICE_INFO: - WARN_ON(pipe_msg->size < sizeof(struct hv_input_dev_info)); + if (WARN_ON_ONCE(pipe_msg->size < + sizeof(struct synthhid_device_info))) + break; /* * Parse out the device info into device attr, * hid desc and report desc */ mousevsc_on_receive_device_info(input_dev, - (struct synthhid_device_info *)pipe_msg->data); + (struct synthhid_device_info *)pipe_msg->data, + pipe_msg->size); break; case SYNTH_HID_INPUT_REPORT: input_report = diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 1059922baaacd..dd31757601b61 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -1283,6 +1283,9 @@ #define USB_DEVICE_ID_SAMSUNG_WIRELESS_UNIVERSAL_KBD 0xa006 #define USB_DEVICE_ID_SAMSUNG_WIRELESS_MULTI_HOGP_KBD 0xa064 +#define USB_VENDOR_ID_SDINNOVATION 0x36ae +#define USB_DEVICE_ID_SDINNOVATION_GAMING_KBD 0xfeab + #define USB_VENDOR_ID_SEMICO 0x1a2c #define USB_DEVICE_ID_SEMICO_USB_KEYKOARD 0x0023 #define USB_DEVICE_ID_SEMICO_USB_KEYKOARD2 0x0027 diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c index 3487600cadb42..d05cf790bbf65 100644 --- a/drivers/hid/hid-input.c +++ b/drivers/hid/hid-input.c @@ -375,6 +375,9 @@ static const struct hid_device_id hid_battery_quirks[] = { { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD), HID_BATTERY_QUIRK_IGNORE }, + { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), + HID_BATTERY_QUIRK_AVOID_QUERY }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084), HID_BATTERY_QUIRK_IGNORE }, @@ -432,17 +435,25 @@ static int hidinput_scale_battery_capacity(struct hid_battery *bat, static int hidinput_query_battery_capacity(struct hid_battery *bat) { int ret; + /* + * The capacity field may not be the first field in the report: some + * devices (e.g. the Apple Magic Trackpad 2 over Bluetooth) precede it + * with status flags. Read it from its actual byte offset in the report + * (report_offset is in bits; the leading byte is the report id). + */ + int offset = 1 + bat->report_offset / 8; + int len = offset + 1; - u8 *buf __free(kfree) = kmalloc(4, GFP_KERNEL); + u8 *buf __free(kfree) = kmalloc(max(len, 4), GFP_KERNEL); if (!buf) return -ENOMEM; - ret = hid_hw_raw_request(bat->dev, bat->report_id, buf, 4, + ret = hid_hw_raw_request(bat->dev, bat->report_id, buf, max(len, 4), bat->report_type, HID_REQ_GET_REPORT); - if (ret < 2) + if (ret < len) return -ENODATA; - return hidinput_scale_battery_capacity(bat, buf[1]); + return hidinput_scale_battery_capacity(bat, buf[offset]); } static int hidinput_get_battery_property(struct power_supply *psy, @@ -593,6 +604,7 @@ static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type, bat->max = max; bat->report_type = report_type; bat->report_id = field->report->id; + bat->report_offset = field->report_offset; bat->charge_status = POWER_SUPPLY_STATUS_DISCHARGING; bat->status = HID_BATTERY_UNKNOWN; diff --git a/drivers/hid/hid-lg4ff.c b/drivers/hid/hid-lg4ff.c index e901fdb7d033c..62f99a93210c9 100644 --- a/drivers/hid/hid-lg4ff.c +++ b/drivers/hid/hid-lg4ff.c @@ -336,6 +336,8 @@ int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report, if (entry->wdata.combine) { switch (entry->wdata.product_id) { case USB_DEVICE_ID_LOGITECH_WHEEL: + if (size < 7) + return 0; rd[5] = rd[3]; rd[6] = 0x7F; return 1; @@ -343,10 +345,14 @@ int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report, case USB_DEVICE_ID_LOGITECH_WINGMAN_FFG: case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL: case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL2: + if (size < 6) + return 0; rd[4] = rd[3]; rd[5] = 0x7F; return 1; case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: + if (size < 7) + return 0; rd[5] = rd[4]; rd[6] = 0x7F; return 1; @@ -366,6 +372,8 @@ int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report, } /* Compute a combined axis when wheel does not supply it */ + if (size <= offset + 1) + return 0; rd[offset] = (0xFF + rd[offset] - rd[offset+1]) >> 1; rd[offset+1] = 0x7F; return 1; diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index 90b0184df777f..fb2062233df22 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -2861,18 +2861,19 @@ static int hidpp_ff_init(struct hidpp_device *hidpp, * ownership to FF core */ data = kmemdup(data, sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; + if (!data) { + error = -ENOMEM; + goto err_destroy_ff; + } data->effect_ids = kzalloc_objs(int, num_slots); if (!data->effect_ids) { - kfree(data); - return -ENOMEM; + error = -ENOMEM; + goto err_free_data; } data->wq = create_singlethread_workqueue("hidpp-ff-sendqueue"); if (!data->wq) { - kfree(data->effect_ids); - kfree(data); - return -ENOMEM; + error = -ENOMEM; + goto err_free_effect_ids; } data->hidpp = hidpp; @@ -2902,6 +2903,14 @@ static int hidpp_ff_init(struct hidpp_device *hidpp, version); return 0; + +err_free_effect_ids: + kfree(data->effect_ids); +err_free_data: + kfree(data); +err_destroy_ff: + input_ff_destroy(dev); + return error; } /* ************************************************************************** */ diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 802a3479e24b9..d637c04773799 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -383,8 +383,8 @@ static void magicmouse_emit_touch(struct magicmouse_sc *msc, int raw_id, u8 *tda } } -static int magicmouse_raw_event(struct hid_device *hdev, - struct hid_report *report, u8 *data, int size) +static int __magicmouse_raw_event(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size, bool nested) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); struct input_dev *input = msc->input; @@ -495,6 +495,15 @@ static int magicmouse_raw_event(struct hid_device *hdev, * packet. */ + /* + * A double report only ever wraps two normal reports, so it is + * never nested. Refuse to recurse a second time; otherwise a + * malicious device could chain DOUBLE_REPORT_ID packets to drive + * unbounded recursion and overflow the kernel stack. + */ + if (nested) + return 0; + /* Ensure that we have at least 2 elements (report type and size) */ if (size < 2) return 0; @@ -506,9 +515,9 @@ static int magicmouse_raw_event(struct hid_device *hdev, return 0; } - magicmouse_raw_event(hdev, report, data + 2, data[1]); - magicmouse_raw_event(hdev, report, data + 2 + data[1], - size - 2 - data[1]); + __magicmouse_raw_event(hdev, report, data + 2, data[1], true); + __magicmouse_raw_event(hdev, report, data + 2 + data[1], + size - 2 - data[1], true); return 0; default: return 0; @@ -534,6 +543,12 @@ static int magicmouse_raw_event(struct hid_device *hdev, return 1; } +static int magicmouse_raw_event(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + return __magicmouse_raw_event(hdev, report, data, size, false); +} + static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { @@ -828,6 +843,12 @@ static bool is_usb_magictrackpad2(__u32 vendor, __u32 product) product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC; } +static bool is_bt_magictrackpad2(__u32 vendor, __u32 product) +{ + return vendor == BT_VENDOR_ID_APPLE && + product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC; +} + static int magicmouse_fetch_battery(struct hid_device *hdev) { #ifdef CONFIG_HID_BATTERY_STRENGTH @@ -838,7 +859,8 @@ static int magicmouse_fetch_battery(struct hid_device *hdev) bat = hid_get_battery(hdev); if (!bat || (!is_usb_magicmouse2(hdev->vendor, hdev->product) && - !is_usb_magictrackpad2(hdev->vendor, hdev->product))) + !is_usb_magictrackpad2(hdev->vendor, hdev->product) && + !is_bt_magictrackpad2(hdev->vendor, hdev->product))) return -1; report_enum = &hdev->report_enum[bat->report_type]; @@ -900,6 +922,16 @@ static int magicmouse_probe(struct hid_device *hdev, return ret; } + /* + * When hidinput_connect() fails it frees every input device it + * created, but that does not fail hid_hw_start(): the core simply + * does not claim an input. msc->input, cached in ->input_mapping + * while the report descriptor was parsed, would then be a dangling + * pointer that passes every NULL check. Trust the core's claim. + */ + if (!(hdev->claimed & HID_CLAIMED_INPUT)) + msc->input = NULL; + if (is_usb_magicmouse2(id->vendor, id->product) || is_usb_magictrackpad2(id->vendor, id->product)) { timer_setup(&msc->battery_timer, magicmouse_battery_timer_tick, 0); @@ -971,6 +1003,16 @@ static int magicmouse_probe(struct hid_device *hdev, schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); } + /* + * Query the Bluetooth Magic Trackpad USB-C battery as done for USB. + * Start io first: probe holds driver_input_lock and the synchronous + * GET_REPORT reply would otherwise be dropped. + */ + if (is_bt_magictrackpad2(id->vendor, id->product)) { + hid_device_io_start(hdev); + magicmouse_fetch_battery(hdev); + } + return 0; err_stop_hw: if (is_usb_magicmouse2(id->vendor, id->product) || @@ -995,6 +1037,22 @@ static void magicmouse_remove(struct hid_device *hdev) hid_hw_stop(hdev); } +#ifdef CONFIG_PM +static int magicmouse_reset_resume(struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + /* The device drops out of multitouch mode on resume; re-send the + * enable report. Only the HID_TYPE_USBMOUSE interface accepts it, and + * it must be deferred. Sending it inline here is too early. + */ + if (msc && hdev->type == HID_TYPE_USBMOUSE) + schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + + return 0; +} +#endif + static const __u8 *magicmouse_report_fixup(struct hid_device *hdev, __u8 *rdesc, unsigned int *rsize) { @@ -1058,6 +1116,9 @@ static struct hid_driver magicmouse_driver = { .event = magicmouse_event, .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, +#ifdef CONFIG_PM + .reset_resume = magicmouse_reset_resume, +#endif }; module_hid_driver(magicmouse_driver); diff --git a/drivers/hid/hid-mcp2221.c b/drivers/hid/hid-mcp2221.c index e4ddd8e9293b6..d52ce3531ab70 100644 --- a/drivers/hid/hid-mcp2221.c +++ b/drivers/hid/hid-mcp2221.c @@ -343,7 +343,7 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp, ret = mcp_send_data_req_status(mcp, mcp->txbuf, 4); if (ret) - return ret; + goto out; mcp->rxbuf_idx = 0; @@ -365,7 +365,7 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp, } else { usleep_range(980, 1000); mcp_cancel_last_cmd(mcp); - return ret; + goto out; } } else { retries = 0; @@ -375,6 +375,10 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp, usleep_range(980, 1000); ret = mcp_chk_last_cmd_status_free_bus(mcp); +out: + mcp->rxbuf = NULL; + mcp->rxbuf_size = 0; + return ret; } @@ -861,6 +865,9 @@ static int mcp2221_raw_event(struct hid_device *hdev, u8 *buf; struct mcp2221 *mcp = hid_get_drvdata(hdev); + if (size < 4) + return 0; + switch (data[0]) { case MCP2221_I2C_WR_DATA: @@ -926,6 +933,10 @@ static int mcp2221_raw_event(struct hid_device *hdev, mcp->status = -EINVAL; break; } + if (4 + data[3] > size) { + mcp->status = -EINVAL; + break; + } buf = mcp->rxbuf; memcpy(&buf[mcp->rxbuf_idx], &data[4], data[3]); mcp->rxbuf_idx = mcp->rxbuf_idx + data[3]; @@ -1049,6 +1060,8 @@ static void mcp2221_hid_unregister(void *ptr) { struct hid_device *hdev = ptr; + if (hdev->io_started) + hid_device_io_stop(hdev); hid_hw_close(hdev); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index edb37b4c867e4..e9da59e4bf795 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -79,6 +79,7 @@ MODULE_LICENSE("GPL"); #define MT_QUIRK_APPLE_TOUCHBAR BIT(23) #define MT_QUIRK_YOGABOOK9I BIT(24) #define MT_QUIRK_KEEP_LATENCY_ON_CLOSE BIT(25) +#define MT_QUIRK_IGNORE_FEATURE_ID_MISMATCH BIT(26) #define MT_INPUTMODE_TOUCHSCREEN 0x02 #define MT_INPUTMODE_TOUCHPAD 0x03 @@ -235,6 +236,7 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); #define MT_CLS_APPLE_TOUCHBAR 0x0114 #define MT_CLS_YOGABOOK9I 0x0115 #define MT_CLS_EGALAX_P80H84 0x0116 +#define MT_CLS_ASUS_ROG_Z13_FOLIO 0x0117 #define MT_CLS_SIS 0x0457 #define MT_DEFAULT_MAXCONTACT 10 @@ -405,6 +407,16 @@ static const struct mt_class mt_classes[] = { .quirks = MT_QUIRK_ALWAYS_VALID | MT_QUIRK_CONTACT_CNT_ACCURATE | MT_QUIRK_ASUS_CUSTOM_UP }, + { .name = MT_CLS_ASUS_ROG_Z13_FOLIO, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_IGNORE_DUPLICATES | + MT_QUIRK_HOVERING | + MT_QUIRK_CONTACT_CNT_ACCURATE | + MT_QUIRK_STICKY_FINGERS | + MT_QUIRK_WIN8_PTP_BUTTONS | + MT_QUIRK_CONFIDENCE | + MT_QUIRK_IGNORE_FEATURE_ID_MISMATCH, + .export_all_inputs = true }, { .name = MT_CLS_VTL, .quirks = MT_QUIRK_ALWAYS_VALID | MT_QUIRK_CONTACT_CNT_ACCURATE | @@ -507,6 +519,7 @@ static const struct attribute_group mt_attribute_group = { static void mt_get_feature(struct hid_device *hdev, struct hid_report *report) { + struct mt_device *td = hid_get_drvdata(hdev); int ret; u32 size = hid_report_len(report); u8 *buf; @@ -528,8 +541,14 @@ static void mt_get_feature(struct hid_device *hdev, struct hid_report *report) dev_warn(&hdev->dev, "failed to fetch feature %d\n", report->id); } else { - /* The report ID in the request and the response should match */ - if (report->id != buf[0]) { + /* + * The report ID in the request and the response should match. + * Some firmware (e.g. the ASUS ROG Z13 Folio + * touchpad) returns a mismatched ID on this specific fetch; + * tolerate it only for devices explicitly flagged as such. + */ + if (report->id != buf[0] && + !(td->mtclass.quirks & MT_QUIRK_IGNORE_FEATURE_ID_MISMATCH)) { hid_err(hdev, "Returned feature report did not match the request\n"); goto free; } @@ -2722,10 +2741,16 @@ static const struct hid_device_id mt_devices[] = { HID_ANY_ID) }, /* Hantick */ - { .driver_data = MT_CLS_NSMU, + { .driver_data = MT_CLS_WIN_8_FORCE_MULTI_INPUT_NSMU, HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8, I2C_VENDOR_ID_HANTICK, I2C_PRODUCT_ID_HANTICK_5288) }, + /* Asus ROG Flow Z13 (2025) GZ302EA keyboard-cover touchpad */ + { .driver_data = MT_CLS_ASUS_ROG_Z13_FOLIO, + HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_ASUSTEK, + USB_DEVICE_ID_ASUSTEK_ROG_Z13_FOLIO) }, + /* Generic MT device */ { HID_DEVICE(HID_BUS_ANY, HID_GROUP_MULTITOUCH, HID_ANY_ID, HID_ANY_ID) }, diff --git a/drivers/hid/hid-nintendo.c b/drivers/hid/hid-nintendo.c index e7302ec01ff1f..4ee402e2f9cab 100644 --- a/drivers/hid/hid-nintendo.c +++ b/drivers/hid/hid-nintendo.c @@ -1474,7 +1474,6 @@ static void joycon_parse_imu_report(struct joycon_ctlr *ctlr, dropped_threshold = ctlr->imu_avg_delta_ms * 3 / 2; dropped_pkts = (delta - min(delta, dropped_threshold)) / ctlr->imu_avg_delta_ms; - ctlr->imu_timestamp_us += 1000 * ctlr->imu_avg_delta_ms; if (dropped_pkts > JC_IMU_DROPPED_PKT_WARNING) { hid_warn_ratelimited(ctlr->hdev, "compensating for %u dropped IMU reports\n", @@ -2162,10 +2161,6 @@ static int joycon_input_create(struct joycon_ctlr *ctlr) ctlr->input->phys = hdev->phys; input_set_drvdata(ctlr->input, ctlr); - ret = input_register_device(ctlr->input); - if (ret) - return ret; - if (joycon_type_is_right_joycon(ctlr)) { joycon_config_right_stick(ctlr->input); joycon_config_buttons(ctlr->input, right_joycon_button_mappings); @@ -2208,6 +2203,10 @@ static int joycon_input_create(struct joycon_ctlr *ctlr) if (joycon_has_rumble(ctlr)) joycon_config_rumble(ctlr); + ret = input_register_device(ctlr->input); + if (ret) + return ret; + return 0; } @@ -2607,7 +2606,12 @@ static int joycon_ctlr_read_handler(struct joycon_ctlr *ctlr, u8 *data, { if (data[0] == JC_INPUT_SUBCMD_REPLY || data[0] == JC_INPUT_IMU_DATA || data[0] == JC_INPUT_MCU_DATA) { - if (size >= 12) /* make sure it contains the input report */ + /* + * The whole struct is cast and parsed below, including the + * IMU/subcmd union, not just the 12-byte partial header this + * used to check for. + */ + if (size >= sizeof(struct joycon_input_report)) joycon_parse_report(ctlr, (struct joycon_input_report *)data); } @@ -2736,14 +2740,14 @@ static int nintendo_hid_probe(struct hid_device *hdev, ret = joycon_init(hdev); if (ret) { hid_err(hdev, "Failed to initialize controller; ret=%d\n", ret); - goto err_close; + goto err_io_stop; } /* Initialize the leds */ ret = joycon_leds_create(ctlr); if (ret) { hid_err(hdev, "Failed to create leds; ret=%d\n", ret); - goto err_close; + goto err_io_stop; } /* Initialize the battery power supply */ @@ -2766,7 +2770,8 @@ static int nintendo_hid_probe(struct hid_device *hdev, err_ida: ida_free(&nintendo_player_id_allocator, ctlr->player_id); -err_close: +err_io_stop: + hid_device_io_stop(hdev); hid_hw_close(hdev); err_stop: hid_hw_stop(hdev); diff --git a/drivers/hid/hid-oxp.c b/drivers/hid/hid-oxp.c index 20a54f337220d..d8fb6a69d40d4 100644 --- a/drivers/hid/hid-oxp.c +++ b/drivers/hid/hid-oxp.c @@ -1552,9 +1552,9 @@ static int oxp_hid_probe(struct hid_device *hdev, static void oxp_hid_remove(struct hid_device *hdev) { - cancel_delayed_work(&drvdata.oxp_rgb_queue); - cancel_delayed_work(&drvdata.oxp_btn_queue); - cancel_delayed_work(&drvdata.oxp_mcu_init); + cancel_delayed_work_sync(&drvdata.oxp_rgb_queue); + cancel_delayed_work_sync(&drvdata.oxp_btn_queue); + cancel_delayed_work_sync(&drvdata.oxp_mcu_init); hid_hw_close(hdev); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-picolcd_debugfs.c b/drivers/hid/hid-picolcd_debugfs.c index 085847a92e07d..1f7dfe60e9ba2 100644 --- a/drivers/hid/hid-picolcd_debugfs.c +++ b/drivers/hid/hid-picolcd_debugfs.c @@ -99,6 +99,15 @@ static ssize_t picolcd_debug_eeprom_read(struct file *f, char __user *u, ret = resp->raw_data[2]; if (ret > s) ret = s; + /* + * raw_data[2] is a device-supplied length; also clamp it to + * what picolcd_raw_event() actually stored (raw_size), or a + * hostile device overruns the raw_data[] buffer. + */ + if (ret > resp->raw_size - 3) + ret = resp->raw_size - 3; + if (ret < 0) + ret = 0; if (copy_to_user(u, resp->raw_data+3, ret)) ret = -EFAULT; else diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index 57d8efdd9b890..dcfa8278447ff 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -183,6 +183,7 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_X52_2), HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE }, { HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_X52_PRO), HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE }, { HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_X65), HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE }, + { HID_USB_DEVICE(USB_VENDOR_ID_SDINNOVATION, USB_DEVICE_ID_SDINNOVATION_GAMING_KBD), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD2), HID_QUIRK_NO_INIT_REPORTS }, { HID_USB_DEVICE(USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD), HID_QUIRK_NO_INIT_REPORTS }, { HID_USB_DEVICE(USB_VENDOR_ID_SENNHEISER, USB_DEVICE_ID_SENNHEISER_BTD500USB), HID_QUIRK_NOGET }, @@ -422,7 +423,7 @@ static const struct hid_device_id hid_have_special_driver[] = { #endif #if IS_ENABLED(CONFIG_HID_ELECOM) { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) }, - { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) }, diff --git a/drivers/hid/hid-rapoo.c b/drivers/hid/hid-rapoo.c index 4c81f3086de43..5c9c396fabf77 100644 --- a/drivers/hid/hid-rapoo.c +++ b/drivers/hid/hid-rapoo.c @@ -36,7 +36,7 @@ static int rapoo_probe(struct hid_device *hdev, const struct hid_device_id *id) return ret; } - if (hdev->bus == BUS_USB) { + if (hid_is_usb(hdev)) { struct usb_interface *intf = to_usb_interface(hdev->dev.parent); if (intf->cur_altsetting->desc.bInterfaceNumber != 1) diff --git a/drivers/hid/hid-rmi.c b/drivers/hid/hid-rmi.c index d4af17fdba467..d55a0388895fb 100644 --- a/drivers/hid/hid-rmi.c +++ b/drivers/hid/hid-rmi.c @@ -235,7 +235,23 @@ static int rmi_hid_read_block(struct rmi_transport_dev *xport, u16 addr, break; } - read_input_count = data->readReport[1]; + read_input_count = min_t(int, data->readReport[1], + data->input_report_size - 2); + if (!read_input_count) { + /* + * A zero length reply advances neither + * bytes_read nor bytes_needed, and because a + * reply did arrive the wait above does not + * time out either, so a device answering 0 + * forever would spin here indefinitely with + * page_mutex held. + */ + hid_warn(hdev, "%s: zero-length read reply\n", + __func__); + clear_bit(RMI_READ_DATA_PENDING, &data->flags); + ret = -EIO; + break; + } memcpy(buf + bytes_read, &data->readReport[2], min(read_input_count, bytes_needed)); @@ -271,6 +287,11 @@ static int rmi_hid_write_block(struct rmi_transport_dev *xport, u16 addr, goto exit; } + if (len + 4 > data->output_report_size) { + ret = -EINVAL; + goto exit; + } + data->writeReport[0] = RMI_WRITE_REPORT_ID; data->writeReport[1] = len; data->writeReport[2] = addr & 0xFF; @@ -666,8 +687,16 @@ static int rmi_probe(struct hid_device *hdev, const struct hid_device_id *id) return ret; } - if (id->driver_data) - data->device_flags = id->driver_data; + /* + * RMI_DEVICE can only mean "this probe validated the RMI reports and + * allocated writeReport": every bail-out to start below skips that + * allocation, and device_flags left carrying RMI_DEVICE from + * driver_data would send rmi_input_configured() into rmi_set_page() + * with writeReport still NULL. A bind through the new_id sysfs + * attribute can supply driver_data with the bit set, so do not let + * driver_data grant it. + */ + data->device_flags = id->driver_data & ~RMI_DEVICE; /* * Check for the RMI specific report ids. If they are misisng @@ -696,6 +725,17 @@ static int rmi_probe(struct hid_device *hdev, const struct hid_device_id *id) data->output_report_size = hid_report_len(output_report); + /* + * The write reports built by this driver occupy 6 bytes and the read + * handshake looks at the first 3 bytes of an input report, so refuse + * to drive a device whose reports cannot hold them. + */ + if (data->output_report_size < 6 || data->input_report_size < 3) { + hid_err(hdev, "rmi reports too small (out=%u in=%u)\n", + data->output_report_size, data->input_report_size); + goto start; + } + data->device_flags |= RMI_DEVICE; alloc_size = data->output_report_size + data->input_report_size; diff --git a/drivers/hid/hid-roccat-kone.c b/drivers/hid/hid-roccat-kone.c index 58654cf78f0df..17495fcc8b7da 100644 --- a/drivers/hid/hid-roccat-kone.c +++ b/drivers/hid/hid-roccat-kone.c @@ -36,6 +36,8 @@ static uint profile_numbers[5] = {0, 1, 2, 3, 4}; static void kone_profile_activated(struct kone_device *kone, uint new_profile) { + if (new_profile < 1 || new_profile > ARRAY_SIZE(kone->profiles)) + new_profile = 1; kone->actual_profile = new_profile; kone->actual_dpi = kone->profiles[new_profile - 1].startup_dpi; } @@ -793,8 +795,10 @@ static void kone_keep_values_up_to_date(struct kone_device *kone, { switch (event->event) { case kone_mouse_event_switch_profile: - kone->actual_dpi = kone->profiles[event->value - 1]. - startup_dpi; + if (event->value >= 1 && + event->value <= ARRAY_SIZE(kone->profiles)) + kone->actual_dpi = + kone->profiles[event->value - 1].startup_dpi; fallthrough; case kone_mouse_event_osd_profile: kone->actual_profile = event->value; diff --git a/drivers/hid/hid-roccat.c b/drivers/hid/hid-roccat.c index d6fff53d4ee7a..4f15eb9510396 100644 --- a/drivers/hid/hid-roccat.c +++ b/drivers/hid/hid-roccat.c @@ -70,6 +70,15 @@ static struct roccat_device *devices[ROCCAT_MAX_DEVICES]; /* protects modifications of devices array */ static DEFINE_MUTEX(devices_lock); +static void roccat_free_device(struct roccat_device *device) +{ + int i; + + for (i = 0; i < ROCCAT_CBUF_SIZE; i++) + kfree(device->cbuf[i].value); + kfree(device); +} + static ssize_t roccat_read(struct file *file, char __user *buffer, size_t count, loff_t *ppos) { @@ -226,7 +235,7 @@ static int roccat_release(struct inode *inode, struct file *file) hid_hw_power(device->hid, PM_HINT_NORMAL); hid_hw_close(device->hid); } else { - kfree(device); + roccat_free_device(device); } } @@ -374,7 +383,7 @@ void roccat_disconnect(int minor) hid_hw_close(device->hid); wake_up_interruptible(&device->wait); } else { - kfree(device); + roccat_free_device(device); } } EXPORT_SYMBOL_GPL(roccat_disconnect); diff --git a/drivers/hid/hid-sensor-custom.c b/drivers/hid/hid-sensor-custom.c index afffea8940214..c2b425afd951b 100644 --- a/drivers/hid/hid-sensor-custom.c +++ b/drivers/hid/hid-sensor-custom.c @@ -609,7 +609,7 @@ static int hid_sensor_custom_add_attributes(struct hid_sensor_custom &sensor_inst->fields[i]. hid_custom_attribute_group); if (ret) - break; + goto err_remove_groups; /* For power or report field store indexes */ if (sensor_inst->fields[i].attribute.attrib_id == @@ -621,6 +621,13 @@ static int hid_sensor_custom_add_attributes(struct hid_sensor_custom } return ret; + +err_remove_groups: + while (--i >= 0) + sysfs_remove_group(&sensor_inst->pdev->dev.kobj, + &sensor_inst->fields[i].hid_custom_attribute_group); + kfree(sensor_inst->fields); + return ret; } static void hid_sensor_custom_remove_attributes(struct hid_sensor_custom * @@ -1005,26 +1012,26 @@ static int hid_sensor_custom_probe(struct platform_device *pdev) return ret; } - ret = sysfs_create_group(&sensor_inst->pdev->dev.kobj, - &enable_sensor_attr_group); + ret = hid_sensor_custom_add_attributes(sensor_inst); if (ret) goto err_remove_callback; - ret = hid_sensor_custom_add_attributes(sensor_inst); + ret = sysfs_create_group(&sensor_inst->pdev->dev.kobj, + &enable_sensor_attr_group); if (ret) - goto err_remove_group; + goto err_remove_attributes; ret = hid_sensor_custom_dev_if_add(sensor_inst); if (ret) - goto err_remove_attributes; + goto err_remove_group; return 0; -err_remove_attributes: - hid_sensor_custom_remove_attributes(sensor_inst); err_remove_group: sysfs_remove_group(&sensor_inst->pdev->dev.kobj, &enable_sensor_attr_group); +err_remove_attributes: + hid_sensor_custom_remove_attributes(sensor_inst); err_remove_callback: sensor_hub_remove_callback(hsdev, hsdev->usage); @@ -1042,9 +1049,10 @@ static void hid_sensor_custom_remove(struct platform_device *pdev) } hid_sensor_custom_dev_if_remove(sensor_inst); - hid_sensor_custom_remove_attributes(sensor_inst); + /* Remove enable_sensor first as it uses fields via power_state/report_state. */ sysfs_remove_group(&sensor_inst->pdev->dev.kobj, &enable_sensor_attr_group); + hid_sensor_custom_remove_attributes(sensor_inst); sensor_hub_remove_callback(hsdev, hsdev->usage); } diff --git a/drivers/hid/hid-sensor-hub.c b/drivers/hid/hid-sensor-hub.c index 34f710c465b80..6470a290ebfc5 100644 --- a/drivers/hid/hid-sensor-hub.c +++ b/drivers/hid/hid-sensor-hub.c @@ -239,12 +239,17 @@ int sensor_hub_get_feature(struct hid_sensor_hub_device *hsdev, u32 report_id, u32 field_index, int buffer_size, void *buffer) { struct hid_report *report; + struct hid_field *field; struct sensor_hub_data *data = hid_get_drvdata(hsdev->hdev); - int report_size; + size_t field_size; + size_t report_size; + size_t copied = 0; + size_t to_copy; int ret = 0; - u8 *val_ptr; - int buffer_index = 0; - int i; + unsigned int i; + + if (!buffer || buffer_size <= 0) + return -EINVAL; memset(buffer, 0, buffer_size); @@ -258,26 +263,29 @@ int sensor_hub_get_feature(struct hid_sensor_hub_device *hsdev, u32 report_id, hid_hw_request(hsdev->hdev, report, HID_REQ_GET_REPORT); hid_hw_wait(hsdev->hdev); + field = report->field[field_index]; + /* calculate number of bytes required to read this field */ - report_size = DIV_ROUND_UP(report->field[field_index]->report_size, - 8) * - report->field[field_index]->report_count; - if (!report_size) { + field_size = DIV_ROUND_UP(field->report_size, 8); + /* HID core stores each parsed report value in a __s32 slot. */ + if (!field_size || field_size > sizeof(field->value[0])) { + ret = -EINVAL; + goto done_proc; + } + if (field->report_count > SIZE_MAX / field_size) { ret = -EINVAL; goto done_proc; } - ret = min(report_size, buffer_size); - val_ptr = (u8 *)report->field[field_index]->value; - for (i = 0; i < report->field[field_index]->report_count; ++i) { - if (buffer_index >= ret) - break; + report_size = field_size * field->report_count; + report_size = min_t(size_t, report_size, buffer_size); - memcpy(&((u8 *)buffer)[buffer_index], val_ptr, - report->field[field_index]->report_size / 8); - val_ptr += sizeof(__s32); - buffer_index += (report->field[field_index]->report_size / 8); + for (i = 0; i < field->report_count && copied < report_size; ++i) { + to_copy = min(field_size, report_size - copied); + memcpy(&((u8 *)buffer)[copied], &field->value[i], to_copy); + copied += to_copy; } + ret = copied; done_proc: mutex_unlock(&data->mutex); diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c index e75246d29e16d..a8b3a1039f0ca 100644 --- a/drivers/hid/hid-sony.c +++ b/drivers/hid/hid-sony.c @@ -29,6 +29,7 @@ * There will be no PIN request from the device. */ +#include #include #include #include @@ -503,7 +504,7 @@ struct motion_output_report_02 { u8 r, g, b; u8 zero2; u8 rumble; -}; +} __packed; static_assert(sizeof(struct motion_output_report_02) == 7); #define SIXAXIS_REPORT_0xF2_SIZE 17 @@ -569,14 +570,12 @@ static void sony_set_leds(struct sony_sc *sc); static inline void sony_schedule_work(struct sony_sc *sc, enum sony_worker which) { - unsigned long flags; - switch (which) { case SONY_WORKER_STATE: - spin_lock_irqsave(&sc->lock, flags); - if (!sc->defer_initialization && sc->state_worker_initialized) - schedule_work(&sc->state_worker); - spin_unlock_irqrestore(&sc->lock, flags); + scoped_guard(spinlock_irqsave, &sc->lock) { + if (!sc->defer_initialization && sc->state_worker_initialized) + schedule_work(&sc->state_worker); + } break; } } @@ -949,7 +948,6 @@ static const u8 *sony_report_fixup(struct hid_device *hdev, u8 *rdesc, static void sixaxis_parse_report(struct sony_sc *sc, u8 *rd, int size) { static const u8 sixaxis_battery_capacity[] = { 0, 1, 25, 50, 75, 100 }; - unsigned long flags; int offset; u8 index; u8 battery_capacity; @@ -972,10 +970,10 @@ static void sixaxis_parse_report(struct sony_sc *sc, u8 *rd, int size) battery_status = POWER_SUPPLY_STATUS_DISCHARGING; } - spin_lock_irqsave(&sc->lock, flags); - sc->battery_capacity = battery_capacity; - sc->battery_status = battery_status; - spin_unlock_irqrestore(&sc->lock, flags); + scoped_guard(spinlock_irqsave, &sc->lock) { + sc->battery_capacity = battery_capacity; + sc->battery_status = battery_status; + } if (sc->quirks & SIXAXIS_CONTROLLER) { int val; @@ -1092,7 +1090,6 @@ static void rb4_ps5_guitar_parse_report(struct sony_sc *sc, u8 *rd, int size) u8 battery_data; u8 battery_capacity; u8 battery_status; - unsigned long flags; /* * Rock Band 4 PS5 guitars have whammy and @@ -1132,10 +1129,10 @@ static void rb4_ps5_guitar_parse_report(struct sony_sc *sc, u8 *rd, int size) break; } - spin_lock_irqsave(&sc->lock, flags); - sc->battery_capacity = battery_capacity; - sc->battery_status = battery_status; - spin_unlock_irqrestore(&sc->lock, flags); + scoped_guard(spinlock_irqsave, &sc->lock) { + sc->battery_capacity = battery_capacity; + sc->battery_status = battery_status; + } input_sync(sc->input_dev); } @@ -1869,15 +1866,14 @@ static int sony_battery_get_property(struct power_supply *psy, union power_supply_propval *val) { struct sony_sc *sc = power_supply_get_drvdata(psy); - unsigned long flags; int ret = 0; u8 battery_capacity; int battery_status; - spin_lock_irqsave(&sc->lock, flags); - battery_capacity = sc->battery_capacity; - battery_status = sc->battery_status; - spin_unlock_irqrestore(&sc->lock, flags); + scoped_guard(spinlock_irqsave, &sc->lock) { + battery_capacity = sc->battery_capacity; + battery_status = sc->battery_status; + } switch (psp) { case POWER_SUPPLY_PROP_PRESENT: @@ -1959,10 +1955,9 @@ static inline int sony_compare_connection_type(struct sony_sc *sc0, static int sony_check_add_dev_list(struct sony_sc *sc) { struct sony_sc *entry; - unsigned long flags; int ret; - spin_lock_irqsave(&sony_dev_list_lock, flags); + guard(spinlock_irqsave)(&sony_dev_list_lock); list_for_each_entry(entry, &sony_device_list, list_node) { ret = memcmp(sc->mac_address, entry->mac_address, @@ -1976,27 +1971,23 @@ static int sony_check_add_dev_list(struct sony_sc *sc) "controller with MAC address %pMR already connected\n", sc->mac_address); } - goto unlock; + goto out; } } ret = 0; list_add(&(sc->list_node), &sony_device_list); -unlock: - spin_unlock_irqrestore(&sony_dev_list_lock, flags); +out: return ret; } static void sony_remove_dev_list(struct sony_sc *sc) { - unsigned long flags; + guard(spinlock_irqsave)(&sony_dev_list_lock); - if (sc->list_node.next) { - spin_lock_irqsave(&sony_dev_list_lock, flags); - list_del(&(sc->list_node)); - spin_unlock_irqrestore(&sony_dev_list_lock, flags); - } + if (!list_empty(&sc->list_node)) + list_del_init(&sc->list_node); } static int sony_get_bt_devaddr(struct sony_sc *sc) @@ -2123,16 +2114,21 @@ static inline void sony_init_output_report(struct sony_sc *sc, static inline void sony_cancel_work_sync(struct sony_sc *sc) { - unsigned long flags; - if (sc->state_worker_initialized) { - spin_lock_irqsave(&sc->lock, flags); - sc->state_worker_initialized = 0; - spin_unlock_irqrestore(&sc->lock, flags); + scoped_guard(spinlock_irqsave, &sc->lock) { + sc->state_worker_initialized = 0; + } cancel_work_sync(&sc->state_worker); } } +static void sony_cleanup(struct sony_sc *sc) +{ + sony_cancel_work_sync(sc); + sony_remove_dev_list(sc); + sony_release_device_id(sc); +} + static int sony_input_configured(struct hid_device *hdev, struct hid_input *hidinput) { @@ -2306,9 +2302,7 @@ static int sony_input_configured(struct hid_device *hdev, err_close: hid_hw_close(hdev); err_stop: - sony_cancel_work_sync(sc); - sony_remove_dev_list(sc); - sony_release_device_id(sc); + sony_cleanup(sc); return ret; } @@ -2332,6 +2326,8 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id) return -ENOMEM; spin_lock_init(&sc->lock); + INIT_LIST_HEAD(&sc->list_node); + sc->device_id = -1; sc->quirks = quirks; hid_set_drvdata(hdev, sc); @@ -2360,6 +2356,7 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id) ret = hid_hw_start(hdev, connect_mask); if (ret) { hid_err(hdev, "hw start failed\n"); + sony_cleanup(sc); return ret; } @@ -2414,7 +2411,7 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id) err: usb_free_urb(sc->ghl_urb); - + sony_cleanup(sc); hid_hw_stop(hdev); return ret; } @@ -2424,18 +2421,14 @@ static void sony_remove(struct hid_device *hdev) struct sony_sc *sc = hid_get_drvdata(hdev); if (sc->quirks & (GHL_GUITAR_PS3WIIU | GHL_GUITAR_PS4)) { - timer_delete_sync(&sc->ghl_poke_timer); + /* poison, not kill: a pending timer must not re-submit during teardown */ + usb_poison_urb(sc->ghl_urb); + timer_shutdown_sync(&sc->ghl_poke_timer); usb_free_urb(sc->ghl_urb); } hid_hw_close(hdev); - - sony_cancel_work_sync(sc); - - sony_remove_dev_list(sc); - - sony_release_device_id(sc); - + sony_cleanup(sc); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-steam.c b/drivers/hid/hid-steam.c index 197126d6e0810..80d2eabc930ec 100644 --- a/drivers/hid/hid-steam.c +++ b/drivers/hid/hid-steam.c @@ -43,6 +43,7 @@ #include #include #include +#include #include "hid-ids.h" MODULE_DESCRIPTION("HID driver for Valve Steam Controller"); @@ -68,13 +69,14 @@ static LIST_HEAD(steam_devices); /* Joystick runs are about 5 mm and 32768 units */ #define STEAM_DECK_JOYSTICK_RESOLUTION 6553 /* Accelerometer has 16 bit resolution and a range of +/- 2g */ -#define STEAM_DECK_ACCEL_RES_PER_G 16384 -#define STEAM_DECK_ACCEL_RANGE 32768 +#define STEAM_ACCEL_RES_PER_G 16384 +#define STEAM_ACCEL_RANGE 32768 +#define STEAM_ACCEL_FUZZ 128 #define STEAM_DECK_ACCEL_FUZZ 32 /* Gyroscope has 16 bit resolution and a range of +/- 2000 dps */ -#define STEAM_DECK_GYRO_RES_PER_DPS 16 -#define STEAM_DECK_GYRO_RANGE 32768 -#define STEAM_DECK_GYRO_FUZZ 1 +#define STEAM_GYRO_RES_PER_DPS 16 +#define STEAM_GYRO_RANGE 32768 +#define STEAM_GYRO_FUZZ 0 #define STEAM_PAD_FUZZ 256 @@ -149,7 +151,7 @@ enum { SETTING_USB_DEBUG_MODE, SETTING_LEFT_TRACKPAD_MODE, SETTING_RIGHT_TRACKPAD_MODE, - SETTING_MOUSE_POINTER_ENABLED, + SETTING_LIZARD_MODE, /* 10 */ SETTING_DPAD_DEADZONE, @@ -243,14 +245,39 @@ enum { /* Input report identifiers */ enum { - ID_CONTROLLER_STATE = 1, - ID_CONTROLLER_DEBUG = 2, - ID_CONTROLLER_WIRELESS = 3, - ID_CONTROLLER_STATUS = 4, - ID_CONTROLLER_DEBUG2 = 5, - ID_CONTROLLER_SECONDARY_STATE = 6, - ID_CONTROLLER_BLE_STATE = 7, - ID_CONTROLLER_DECK_STATE = 9 + ID_CONTROLLER_STATE = 1, + ID_CONTROLLER_DEBUG = 2, + ID_CONTROLLER_WIRELESS = 3, + ID_CONTROLLER_STATUS = 4, + ID_CONTROLLER_DEBUG2 = 5, + ID_CONTROLLER_SECONDARY_STATE = 6, + ID_CONTROLLER_BLE_STATE = 7, + ID_CONTROLLER_DECK_STATE = 9, +}; + +/* Read-only attributes */ +enum { + ATTRIB_UNIQUE_ID, // deprecated + ATTRIB_PRODUCT_ID, + ATTRIB_PRODUCT_REVISON, // deprecated + ATTRIB_CAPABILITIES = ATTRIB_PRODUCT_REVISON, // intentional aliasing + ATTRIB_FIRMWARE_VERSION, // deprecated + ATTRIB_FIRMWARE_BUILD_TIME, + ATTRIB_RADIO_FIRMWARE_BUILD_TIME, + ATTRIB_RADIO_DEVICE_ID0, + ATTRIB_RADIO_DEVICE_ID1, + ATTRIB_DONGLE_FIRMWARE_BUILD_TIME, + ATTRIB_HW_ID, // AKA BOARD_REVISION, + ATTRIB_BOOTLOADER_BUILD_TIME, + ATTRIB_CONNECTION_INTERVAL_IN_US, + ATTRIB_SECONDARY_FIRMWARE_BUILD_TIME, + ATTRIB_SECONDARY_BOOTLOADER_BUILD_TIME, + ATTRIB_SECONDARY_HW_ID, // AKA BOARD_REVISION, + ATTRIB_STREAMING, + ATTRIB_TRACKPAD_ID, + ATTRIB_SECONDARY_TRACKPAD_ID, + + ATTRIB_COUNT }; /* String attribute identifiers */ @@ -259,14 +286,14 @@ enum { ATTRIB_STR_UNIT_SERIAL, }; -/* Values for GYRO_MODE (bitmask) */ +/* Values for IMU_MODE (bitmask) */ enum { - SETTING_GYRO_MODE_OFF = 0, - SETTING_GYRO_MODE_STEERING = BIT(0), - SETTING_GYRO_MODE_TILT = BIT(1), - SETTING_GYRO_MODE_SEND_ORIENTATION = BIT(2), - SETTING_GYRO_MODE_SEND_RAW_ACCEL = BIT(3), - SETTING_GYRO_MODE_SEND_RAW_GYRO = BIT(4), + SETTING_IMU_MODE_OFF = 0, + SETTING_IMU_MODE_STEERING = BIT(0), + SETTING_IMU_MODE_TILT = BIT(1), + SETTING_IMU_MODE_SEND_ORIENTATION = BIT(2), + SETTING_IMU_MODE_SEND_RAW_ACCEL = BIT(3), + SETTING_IMU_MODE_SEND_RAW_GYRO = BIT(4), }; /* Trackpad modes */ @@ -282,6 +309,11 @@ enum { TRACKPAD_GESTURE_KEYBOARD, }; +struct steam_controller_attribute { + unsigned char tag; + __le32 value; +} __packed; + /* Pad identifiers for the deck */ #define STEAM_PAD_LEFT 0 #define STEAM_PAD_RIGHT 1 @@ -313,6 +345,7 @@ struct steam_device { u16 rumble_left; u16 rumble_right; unsigned int sensor_timestamp_us; + unsigned int sensor_update_rate_us; struct work_struct unregister_work; }; @@ -323,6 +356,13 @@ static int steam_recv_report(struct steam_device *steam, u8 *buf; int ret; + /* + * All reports start with a two byte header. + * We must read at least two bytes to get a sensible output. + */ + if (size < 2) + return -EINVAL; + r = steam->hdev->report_enum[HID_FEATURE_REPORT].report_id_hash[0]; if (!r) { hid_err(steam->hdev, "No HID_FEATURE_REPORT submitted - nothing to read\n"); @@ -345,10 +385,31 @@ static int steam_recv_report(struct steam_device *steam, ret = hid_hw_raw_request(steam->hdev, 0x00, buf, hid_report_len(r) + 1, HID_FEATURE_REPORT, HID_REQ_GET_REPORT); - if (ret > 0) - memcpy(data, buf + 1, min(size, ret - 1)); + if (ret > 0) { + /* Remove the report ID from the return buffer */ + ret--; + size = min(size, ret); + memcpy(data, buf + 1, size); + } kfree(buf); - return ret; + + if (ret < 0) + hid_err(steam->hdev, "%s: error %d\n", __func__, ret); + else + hid_dbg(steam->hdev, "Received report %*ph\n", size, data); + if (ret < 0) + return ret; + + if (ret < 2) { + hid_err(steam->hdev, "%s: reply too short\n", __func__); + return -EPROTO; + } + if (ret < data[1] + 2) { + hid_err(steam->hdev, "%s: expected %u bytes, read %i\n", + __func__, data[1] + 2, ret); + return -EPROTO; + } + return size; } static int steam_send_report(struct steam_device *steam, @@ -375,6 +436,8 @@ static int steam_send_report(struct steam_device *steam, /* The report ID is always 0 */ memcpy(buf + 1, cmd, size); + hid_dbg(steam->hdev, "Sending report %*ph\n", size, cmd); + /* * Sometimes the wireless controller fails with EPIPE * when sending a feature report. @@ -447,25 +510,59 @@ static int steam_get_serial(struct steam_device *steam) u8 cmd[] = {ID_GET_STRING_ATTRIBUTE, sizeof(steam->serial_no), ATTRIB_STR_UNIT_SERIAL}; u8 reply[3 + STEAM_SERIAL_LEN + 1]; - mutex_lock(&steam->report_mutex); + guard(mutex)(&steam->report_mutex); ret = steam_send_report(steam, cmd, sizeof(cmd)); if (ret < 0) - goto out; + return ret; ret = steam_recv_report(steam, reply, sizeof(reply)); if (ret < 0) - goto out; + return ret; if (reply[0] != ID_GET_STRING_ATTRIBUTE || reply[1] < 1 || reply[1] > sizeof(steam->serial_no) || reply[2] != ATTRIB_STR_UNIT_SERIAL) { - ret = -EIO; - goto out; + hid_err(steam->hdev, "%s: invalid reply (%*ph)\n", __func__, + (int)sizeof(reply), reply); + return -EIO; } reply[3 + STEAM_SERIAL_LEN] = 0; strscpy(steam->serial_no, reply + 3, reply[1]); -out: - mutex_unlock(&steam->report_mutex); return ret; } +static int steam_get_attributes(struct steam_device *steam) +{ + int ret = 0; + u8 cmd[] = {ID_GET_ATTRIBUTES_VALUES, 0}; + u8 reply[64] = {}; + u8 size; + int i; + struct steam_controller_attribute *attr; + + guard(mutex)(&steam->report_mutex); + ret = steam_send_report(steam, cmd, sizeof(cmd)); + if (ret < 0) + return ret; + ret = steam_recv_report(steam, reply, sizeof(reply)); + if (ret < 0) + return ret; + if (reply[0] != ID_GET_ATTRIBUTES_VALUES || reply[1] < 2) { + hid_err(steam->hdev, "%s: invalid reply (%*ph)\n", __func__, + (int)sizeof(reply), reply); + return -EIO; + } + + size = min(reply[1], sizeof(reply) - 2); + for (i = 0; i + sizeof(*attr) <= size; i += sizeof(*attr)) { + attr = (struct steam_controller_attribute *)&reply[i + 2]; + if (attr->tag == ATTRIB_CONNECTION_INTERVAL_IN_US) { + steam->sensor_update_rate_us = get_unaligned_le32(&attr->value); + hid_dbg(steam->hdev, "Sensor update rate: %uus\n", + steam->sensor_update_rate_us); + } + } + + return 0; +} + /* * This command requests the wireless adaptor to post an event * with the connection status. Useful if this driver is loaded when @@ -473,11 +570,8 @@ static int steam_get_serial(struct steam_device *steam) */ static inline int steam_request_conn_status(struct steam_device *steam) { - int ret; - mutex_lock(&steam->report_mutex); - ret = steam_send_report_byte(steam, ID_DONGLE_GET_WIRELESS_STATE); - mutex_unlock(&steam->report_mutex); - return ret; + guard(mutex)(&steam->report_mutex); + return steam_send_report_byte(steam, ID_DONGLE_GET_WIRELESS_STATE); } /* @@ -624,6 +718,42 @@ static void steam_input_close(struct input_dev *dev) } } +static int steam_sensor_open(struct input_dev *dev) +{ + struct steam_device *steam = input_get_drvdata(dev); + unsigned long flags; + bool client_opened; + + spin_lock_irqsave(&steam->lock, flags); + client_opened = steam->client_opened; + spin_unlock_irqrestore(&steam->lock, flags); + if (client_opened) + return 0; + + guard(mutex)(&steam->report_mutex); + steam_write_settings(steam, SETTING_IMU_MODE, + SETTING_IMU_MODE_SEND_RAW_ACCEL | SETTING_IMU_MODE_SEND_RAW_GYRO, + 0); + + return 0; +} + +static void steam_sensor_close(struct input_dev *dev) +{ + struct steam_device *steam = input_get_drvdata(dev); + unsigned long flags; + bool client_opened; + + spin_lock_irqsave(&steam->lock, flags); + client_opened = steam->client_opened; + spin_unlock_irqrestore(&steam->lock, flags); + if (client_opened) + return; + + guard(mutex)(&steam->report_mutex); + steam_write_settings(steam, SETTING_IMU_MODE, 0, 0); +} + static enum power_supply_property steam_battery_props[] = { POWER_SUPPLY_PROP_PRESENT, POWER_SUPPLY_PROP_SCOPE, @@ -837,9 +967,6 @@ static int steam_sensors_register(struct steam_device *steam) struct input_dev *sensors; int ret; - if (!(steam->quirks & STEAM_QUIRK_DECK)) - return 0; - rcu_read_lock(); sensors = rcu_dereference(steam->sensors); rcu_read_unlock(); @@ -854,8 +981,14 @@ static int steam_sensors_register(struct steam_device *steam) input_set_drvdata(sensors, steam); sensors->dev.parent = &hdev->dev; + if (!(steam->quirks & STEAM_QUIRK_DECK)) { + sensors->open = steam_sensor_open; + sensors->close = steam_sensor_close; + } - sensors->name = "Steam Deck Motion Sensors"; + sensors->name = steam->quirks & STEAM_QUIRK_DECK ? + "Steam Deck Motion Sensors" : + "Steam Controller Motion Sensors"; sensors->phys = hdev->phys; sensors->uniq = steam->serial_no; sensors->id.bustype = hdev->bus; @@ -867,25 +1000,34 @@ static int steam_sensors_register(struct steam_device *steam) __set_bit(EV_MSC, sensors->evbit); __set_bit(MSC_TIMESTAMP, sensors->mscbit); - input_set_abs_params(sensors, ABS_X, -STEAM_DECK_ACCEL_RANGE, - STEAM_DECK_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); - input_set_abs_params(sensors, ABS_Y, -STEAM_DECK_ACCEL_RANGE, - STEAM_DECK_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); - input_set_abs_params(sensors, ABS_Z, -STEAM_DECK_ACCEL_RANGE, - STEAM_DECK_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); - input_abs_set_res(sensors, ABS_X, STEAM_DECK_ACCEL_RES_PER_G); - input_abs_set_res(sensors, ABS_Y, STEAM_DECK_ACCEL_RES_PER_G); - input_abs_set_res(sensors, ABS_Z, STEAM_DECK_ACCEL_RES_PER_G); - - input_set_abs_params(sensors, ABS_RX, -STEAM_DECK_GYRO_RANGE, - STEAM_DECK_GYRO_RANGE, STEAM_DECK_GYRO_FUZZ, 0); - input_set_abs_params(sensors, ABS_RY, -STEAM_DECK_GYRO_RANGE, - STEAM_DECK_GYRO_RANGE, STEAM_DECK_GYRO_FUZZ, 0); - input_set_abs_params(sensors, ABS_RZ, -STEAM_DECK_GYRO_RANGE, - STEAM_DECK_GYRO_RANGE, STEAM_DECK_GYRO_FUZZ, 0); - input_abs_set_res(sensors, ABS_RX, STEAM_DECK_GYRO_RES_PER_DPS); - input_abs_set_res(sensors, ABS_RY, STEAM_DECK_GYRO_RES_PER_DPS); - input_abs_set_res(sensors, ABS_RZ, STEAM_DECK_GYRO_RES_PER_DPS); + if (steam->quirks & STEAM_QUIRK_DECK) { + input_set_abs_params(sensors, ABS_X, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Y, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Z, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); + } else { + input_set_abs_params(sensors, ABS_X, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Y, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Z, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_ACCEL_FUZZ, 0); + } + input_abs_set_res(sensors, ABS_X, STEAM_ACCEL_RES_PER_G); + input_abs_set_res(sensors, ABS_Y, STEAM_ACCEL_RES_PER_G); + input_abs_set_res(sensors, ABS_Z, STEAM_ACCEL_RES_PER_G); + + input_set_abs_params(sensors, ABS_RX, -STEAM_GYRO_RANGE, + STEAM_GYRO_RANGE, STEAM_GYRO_FUZZ, 0); + input_set_abs_params(sensors, ABS_RY, -STEAM_GYRO_RANGE, + STEAM_GYRO_RANGE, STEAM_GYRO_FUZZ, 0); + input_set_abs_params(sensors, ABS_RZ, -STEAM_GYRO_RANGE, + STEAM_GYRO_RANGE, STEAM_GYRO_FUZZ, 0); + input_abs_set_res(sensors, ABS_RX, STEAM_GYRO_RES_PER_DPS); + input_abs_set_res(sensors, ABS_RY, STEAM_GYRO_RES_PER_DPS); + input_abs_set_res(sensors, ABS_RZ, STEAM_GYRO_RES_PER_DPS); ret = input_register_device(sensors); if (ret) @@ -916,9 +1058,6 @@ static void steam_sensors_unregister(struct steam_device *steam) { struct input_dev *sensors; - if (!(steam->quirks & STEAM_QUIRK_DECK)) - return; - rcu_read_lock(); sensors = rcu_dereference(steam->sensors); rcu_read_unlock(); @@ -966,6 +1105,12 @@ static int steam_register(struct steam_device *steam) strscpy(steam->serial_no, "XXXXXXXXXX", sizeof(steam->serial_no)); + ret = steam_get_attributes(steam); + if (ret < 0) + hid_err(steam->hdev, + "%s:steam_get_attributes failed with error %d\n", + __func__, ret); + hid_info(steam->hdev, "Steam Controller '%s' connected", steam->serial_no); @@ -1049,6 +1194,7 @@ static void steam_mode_switch_cb(struct work_struct *work) return; steam->gamepad_mode = !steam->gamepad_mode; + hid_dbg(steam->hdev, "%s: switching gamepad mode to %i\n", __func__, steam->gamepad_mode); if (steam->gamepad_mode) steam_set_lizard_mode(steam, false); else { @@ -1244,6 +1390,10 @@ static int steam_probe(struct hid_device *hdev, INIT_LIST_HEAD(&steam->list); INIT_WORK(&steam->rumble_work, steam_haptic_rumble_cb); steam->sensor_timestamp_us = 0; + if (steam->quirks & STEAM_QUIRK_DECK) + steam->sensor_update_rate_us = 4000; + else + steam->sensor_update_rate_us = 9000; INIT_WORK(&steam->unregister_work, steam_work_unregister_cb); /* @@ -1354,13 +1504,45 @@ static void steam_do_connect_event(struct steam_device *steam, bool connected) * Clamp the values to 32767..-32767 so that the range is * symmetrical and can be negated safely. */ -static inline s16 steam_le16(u8 *data) +static inline s16 steam_le16(const u8 *data) { - s16 x = (s16) le16_to_cpup((__le16 *)data); + s16 x = (s16) get_unaligned_le16((const __le16 *)data); return x == -32768 ? -32767 : x; } +struct steam_button_mapping { + int code; + u8 byte; + u8 bit; +}; + +struct steam_axis_mapping { + int code; + s8 sign; + u8 byte; +}; + +static void steam_map_buttons(struct input_dev *input, + const struct steam_button_mapping *mappings, const u8 *data) +{ + const struct steam_button_mapping *mapping; + + for (mapping = mappings; mapping->code; mapping++) + input_report_key(input, mapping->code, + data[mapping->byte] & BIT(mapping->bit)); +} + +static void steam_map_axes(struct input_dev *input, + const struct steam_axis_mapping *mappings, const u8 *data) +{ + const struct steam_axis_mapping *mapping; + + for (mapping = mappings; mapping->sign; mapping++) + input_report_abs(input, mapping->code, + mapping->sign * steam_le16(&data[mapping->byte])); +} + /* * The size for this message payload is 60. * The known values are: @@ -1427,18 +1609,52 @@ static inline s16 steam_le16(u8 *data) * 10.7 | -- | lpad_and_joy */ +static const struct steam_button_mapping steam_controller_button_mappings[] = { + { BTN_TR2, 8, 0 }, + { BTN_TL2, 8, 1 }, + { BTN_TR, 8, 2 }, + { BTN_TL, 8, 3 }, + { BTN_Y, 8, 4 }, + { BTN_B, 8, 5 }, + { BTN_X, 8, 6 }, + { BTN_A, 8, 7 }, + { BTN_SELECT, 9, 4 }, + { BTN_MODE, 9, 5 }, + { BTN_START, 9, 6 }, + { BTN_GRIPL, 9, 7 }, + { BTN_GRIPR, 10, 0 }, + { BTN_THUMBR, 10, 2 }, + { BTN_THUMBL, 10, 6 }, + { BTN_THUMB2, 10, 4 }, + { BTN_DPAD_UP, 9, 0 }, + { BTN_DPAD_RIGHT, 9, 1 }, + { BTN_DPAD_LEFT, 9, 2 }, + { BTN_DPAD_DOWN, 9, 3 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_controller_axis_mappings[] = { + { ABS_RX, 1, 20 }, + { ABS_RY, -1, 22 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_controller_imu_mappings[] = { + { ABS_X, 1, 28 }, + { ABS_Z, -1, 30 }, + { ABS_Y, 1, 32 }, + { ABS_RX, 1, 34 }, + { ABS_RZ, 1, 36 }, + { ABS_RY, 1, 38 }, + { /* sentinel */ }, +}; + static void steam_do_input_event(struct steam_device *steam, struct input_dev *input, u8 *data) { - /* 24 bits of buttons */ - u8 b8, b9, b10; s16 x, y; bool lpad_touched, lpad_and_joy; - b8 = data[8]; - b9 = data[9]; - b10 = data[10]; - input_report_abs(input, ABS_HAT2Y, data[11]); input_report_abs(input, ABS_HAT2X, data[12]); @@ -1450,8 +1666,8 @@ static void steam_do_input_event(struct steam_device *steam, * joystick values. * (lpad_touched || lpad_and_joy) tells if the lpad is really touched. */ - lpad_touched = b10 & BIT(3); - lpad_and_joy = b10 & BIT(7); + lpad_touched = data[10] & BIT(3); + lpad_and_joy = data[10] & BIT(7); x = steam_le16(data + 16); y = -steam_le16(data + 18); @@ -1467,35 +1683,25 @@ static void steam_do_input_event(struct steam_device *steam, input_report_abs(input, ABS_HAT0X, 0); input_report_abs(input, ABS_HAT0Y, 0); } + input_report_key(input, BTN_THUMB, lpad_touched || lpad_and_joy); - input_report_abs(input, ABS_RX, steam_le16(data + 20)); - input_report_abs(input, ABS_RY, -steam_le16(data + 22)); - - input_event(input, EV_KEY, BTN_TR2, !!(b8 & BIT(0))); - input_event(input, EV_KEY, BTN_TL2, !!(b8 & BIT(1))); - input_event(input, EV_KEY, BTN_TR, !!(b8 & BIT(2))); - input_event(input, EV_KEY, BTN_TL, !!(b8 & BIT(3))); - input_event(input, EV_KEY, BTN_Y, !!(b8 & BIT(4))); - input_event(input, EV_KEY, BTN_B, !!(b8 & BIT(5))); - input_event(input, EV_KEY, BTN_X, !!(b8 & BIT(6))); - input_event(input, EV_KEY, BTN_A, !!(b8 & BIT(7))); - input_event(input, EV_KEY, BTN_SELECT, !!(b9 & BIT(4))); - input_event(input, EV_KEY, BTN_MODE, !!(b9 & BIT(5))); - input_event(input, EV_KEY, BTN_START, !!(b9 & BIT(6))); - input_event(input, EV_KEY, BTN_GRIPL, !!(b9 & BIT(7))); - input_event(input, EV_KEY, BTN_GRIPR, !!(b10 & BIT(0))); - input_event(input, EV_KEY, BTN_THUMBR, !!(b10 & BIT(2))); - input_event(input, EV_KEY, BTN_THUMBL, !!(b10 & BIT(6))); - input_event(input, EV_KEY, BTN_THUMB, lpad_touched || lpad_and_joy); - input_event(input, EV_KEY, BTN_THUMB2, !!(b10 & BIT(4))); - input_event(input, EV_KEY, BTN_DPAD_UP, !!(b9 & BIT(0))); - input_event(input, EV_KEY, BTN_DPAD_RIGHT, !!(b9 & BIT(1))); - input_event(input, EV_KEY, BTN_DPAD_LEFT, !!(b9 & BIT(2))); - input_event(input, EV_KEY, BTN_DPAD_DOWN, !!(b9 & BIT(3))); + steam_map_buttons(input, steam_controller_button_mappings, data); + steam_map_axes(input, steam_controller_axis_mappings, data); input_sync(input); } +static void steam_do_sensors_event(struct steam_device *steam, + struct input_dev *sensors, u8 *data) +{ + steam->sensor_timestamp_us += steam->sensor_update_rate_us; + + input_event(sensors, EV_MSC, MSC_TIMESTAMP, steam->sensor_timestamp_us); + steam_map_axes(sensors, steam_controller_imu_mappings, data); + + input_sync(sensors); +} + /* * The size for this message payload is 56. * The known values are: @@ -1594,23 +1800,68 @@ static void steam_do_input_event(struct steam_device *steam, * 15.6 | -- | unknown * 15.7 | -- | unknown */ + +static const struct steam_button_mapping steam_deck_button_mappings[] = { + { BTN_TR2, 8, 0 }, + { BTN_TL2, 8, 1 }, + { BTN_TR, 8, 2 }, + { BTN_TL, 8, 3 }, + { BTN_Y, 8, 4 }, + { BTN_B, 8, 5 }, + { BTN_X, 8, 6 }, + { BTN_A, 8, 7 }, + { BTN_SELECT, 9, 4 }, + { BTN_MODE, 9, 5 }, + { BTN_START, 9, 6 }, + { BTN_GRIPL2, 9, 7 }, + { BTN_GRIPR2, 10, 0 }, + { BTN_THUMBL, 10, 6 }, + { BTN_THUMBR, 11, 2 }, + { BTN_DPAD_UP, 9, 0 }, + { BTN_DPAD_RIGHT, 9, 1 }, + { BTN_DPAD_LEFT, 9, 2 }, + { BTN_DPAD_DOWN, 9, 3 }, + { BTN_THUMB, 10, 1 }, + { BTN_THUMB2, 10, 2 }, + { BTN_GRIPL, 13, 1 }, + { BTN_GRIPR, 13, 2 }, + { BTN_BASE, 14, 2 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_deck_axis_mappings[] = { + { ABS_X, 1, 48 }, + { ABS_Y, -1, 50 }, + { ABS_RX, 1, 52 }, + { ABS_RY, -1, 54 }, + { ABS_HAT2Y, 1, 44 }, + { ABS_HAT2X, 1, 46 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_deck_imu_mappings[] = { + { ABS_X, 1, 24 }, + { ABS_Z, -1, 26 }, + { ABS_Y, 1, 28 }, + { ABS_RX, 1, 30 }, + { ABS_RZ, -1, 32 }, + { ABS_RY, 1, 34 }, + { /* sentinel */ }, +}; + static void steam_do_deck_input_event(struct steam_device *steam, struct input_dev *input, u8 *data) { - u8 b8, b9, b10, b11, b13, b14; + bool start_pressed; bool lpad_touched, rpad_touched; - b8 = data[8]; - b9 = data[9]; - b10 = data[10]; - b11 = data[11]; - b13 = data[13]; - b14 = data[14]; + start_pressed = data[9] & BIT(6); - if (!(b9 & BIT(6)) && steam->did_mode_switch) { + if (!start_pressed && steam->did_mode_switch) { steam->did_mode_switch = false; cancel_delayed_work(&steam->mode_switch); - } else if (!steam->client_opened && (b9 & BIT(6)) && !steam->did_mode_switch) { + } else if (!steam->client_opened && start_pressed && !steam->did_mode_switch) { + hid_dbg(steam->hdev, "%s: doing mode switch\n", __func__); steam->did_mode_switch = true; schedule_delayed_work(&steam->mode_switch, 45 * HZ / 100); } @@ -1618,8 +1869,8 @@ static void steam_do_deck_input_event(struct steam_device *steam, if (!steam->gamepad_mode && lizard_mode) return; - lpad_touched = b10 & BIT(3); - rpad_touched = b10 & BIT(4); + lpad_touched = data[10] & BIT(3); + rpad_touched = data[10] & BIT(4); if (lpad_touched) { input_report_abs(input, ABS_HAT0X, steam_le16(data + 16)); @@ -1637,38 +1888,8 @@ static void steam_do_deck_input_event(struct steam_device *steam, input_report_abs(input, ABS_HAT1Y, 0); } - input_report_abs(input, ABS_X, steam_le16(data + 48)); - input_report_abs(input, ABS_Y, -steam_le16(data + 50)); - input_report_abs(input, ABS_RX, steam_le16(data + 52)); - input_report_abs(input, ABS_RY, -steam_le16(data + 54)); - - input_report_abs(input, ABS_HAT2Y, steam_le16(data + 44)); - input_report_abs(input, ABS_HAT2X, steam_le16(data + 46)); - - input_event(input, EV_KEY, BTN_TR2, !!(b8 & BIT(0))); - input_event(input, EV_KEY, BTN_TL2, !!(b8 & BIT(1))); - input_event(input, EV_KEY, BTN_TR, !!(b8 & BIT(2))); - input_event(input, EV_KEY, BTN_TL, !!(b8 & BIT(3))); - input_event(input, EV_KEY, BTN_Y, !!(b8 & BIT(4))); - input_event(input, EV_KEY, BTN_B, !!(b8 & BIT(5))); - input_event(input, EV_KEY, BTN_X, !!(b8 & BIT(6))); - input_event(input, EV_KEY, BTN_A, !!(b8 & BIT(7))); - input_event(input, EV_KEY, BTN_SELECT, !!(b9 & BIT(4))); - input_event(input, EV_KEY, BTN_MODE, !!(b9 & BIT(5))); - input_event(input, EV_KEY, BTN_START, !!(b9 & BIT(6))); - input_event(input, EV_KEY, BTN_GRIPL2, !!(b9 & BIT(7))); - input_event(input, EV_KEY, BTN_GRIPR2, !!(b10 & BIT(0))); - input_event(input, EV_KEY, BTN_THUMBL, !!(b10 & BIT(6))); - input_event(input, EV_KEY, BTN_THUMBR, !!(b11 & BIT(2))); - input_event(input, EV_KEY, BTN_DPAD_UP, !!(b9 & BIT(0))); - input_event(input, EV_KEY, BTN_DPAD_RIGHT, !!(b9 & BIT(1))); - input_event(input, EV_KEY, BTN_DPAD_LEFT, !!(b9 & BIT(2))); - input_event(input, EV_KEY, BTN_DPAD_DOWN, !!(b9 & BIT(3))); - input_event(input, EV_KEY, BTN_THUMB, !!(b10 & BIT(1))); - input_event(input, EV_KEY, BTN_THUMB2, !!(b10 & BIT(2))); - input_event(input, EV_KEY, BTN_GRIPL, !!(b13 & BIT(1))); - input_event(input, EV_KEY, BTN_GRIPR, !!(b13 & BIT(2))); - input_event(input, EV_KEY, BTN_BASE, !!(b14 & BIT(2))); + steam_map_buttons(input, steam_deck_button_mappings, data); + steam_map_axes(input, steam_deck_axis_mappings, data); input_sync(input); } @@ -1676,25 +1897,13 @@ static void steam_do_deck_input_event(struct steam_device *steam, static void steam_do_deck_sensors_event(struct steam_device *steam, struct input_dev *sensors, u8 *data) { - /* - * The deck input report is received every 4 ms on average, - * with a jitter of +/- 4 ms even though the USB descriptor claims - * that it uses 1 kHz. - * Since the HID report does not include a sensor timestamp, - * use a fixed increment here. - */ - steam->sensor_timestamp_us += 4000; + steam->sensor_timestamp_us += steam->sensor_update_rate_us; if (!steam->gamepad_mode && lizard_mode) return; input_event(sensors, EV_MSC, MSC_TIMESTAMP, steam->sensor_timestamp_us); - input_report_abs(sensors, ABS_X, steam_le16(data + 24)); - input_report_abs(sensors, ABS_Z, -steam_le16(data + 26)); - input_report_abs(sensors, ABS_Y, steam_le16(data + 28)); - input_report_abs(sensors, ABS_RX, steam_le16(data + 30)); - input_report_abs(sensors, ABS_RZ, -steam_le16(data + 32)); - input_report_abs(sensors, ABS_RY, steam_le16(data + 34)); + steam_map_axes(sensors, steam_deck_imu_mappings, data); input_sync(sensors); } @@ -1773,6 +1982,9 @@ static int steam_raw_event(struct hid_device *hdev, input = rcu_dereference(steam->input); if (likely(input)) steam_do_input_event(steam, input, data); + sensors = rcu_dereference(steam->sensors); + if (likely(sensors)) + steam_do_sensors_event(steam, sensors, data); rcu_read_unlock(); break; case ID_CONTROLLER_DECK_STATE: diff --git a/drivers/hid/hid-tmff.c b/drivers/hid/hid-tmff.c index 423f395d01ac5..319e7b6703846 100644 --- a/drivers/hid/hid-tmff.c +++ b/drivers/hid/hid-tmff.c @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -47,9 +48,9 @@ struct tmff_device { /* Changes values from 0 to 0xffff into values from minimum to maximum */ static inline int tmff_scale_u16(unsigned int in, int minimum, int maximum) { - int ret; + s64 ret; - ret = (in * (maximum - minimum) / 0xffff) + minimum; + ret = div_s64((s64)in * ((s64)maximum - minimum), 0xffff) + minimum; if (ret < minimum) return minimum; if (ret > maximum) @@ -60,9 +61,9 @@ static inline int tmff_scale_u16(unsigned int in, int minimum, int maximum) /* Changes values from -0x80 to 0x7f into values from minimum to maximum */ static inline int tmff_scale_s8(int in, int minimum, int maximum) { - int ret; + s64 ret; - ret = (((in + 0x80) * (maximum - minimum)) / 0xff) + minimum; + ret = div_s64((s64)(in + 0x80) * ((s64)maximum - minimum), 0xff) + minimum; if (ret < minimum) return minimum; if (ret > maximum) diff --git a/drivers/hid/hid-uclogic-core.c b/drivers/hid/hid-uclogic-core.c index b73f09d26688a..0b8a83fa6c5be 100644 --- a/drivers/hid/hid-uclogic-core.c +++ b/drivers/hid/hid-uclogic-core.c @@ -548,7 +548,17 @@ static void uclogic_remove(struct hid_device *hdev) { struct uclogic_drvdata *drvdata = hid_get_drvdata(hdev); - timer_delete_sync(&drvdata->inrange_timer); + /* + * Shut the in-range timer down before stopping the device. + * uclogic_raw_event_pen() re-arms inrange_timer on every pen report + * and keeps running until hid_hw_stop() stops the transport, so a + * plain timer_delete_sync() here can be undone by a report landing in + * the window before hid_hw_stop(). timer_shutdown_sync() cancels the + * timer and makes any later re-arm a no-op, so it is provably dead + * before hid_hw_stop() frees the input device drvdata->pen_input + * points at. + */ + timer_shutdown_sync(&drvdata->inrange_timer); hid_hw_stop(hdev); kfree(drvdata->desc_ptr); uclogic_params_cleanup(&drvdata->params); diff --git a/drivers/hid/hid-universal-pidff.c b/drivers/hid/hid-universal-pidff.c index 549dac555d408..60180467a0bfd 100644 --- a/drivers/hid/hid-universal-pidff.c +++ b/drivers/hid/hid-universal-pidff.c @@ -104,12 +104,14 @@ static int universal_pidff_probe(struct hid_device *hdev, error = init_function(hdev, id->driver_data); if (error) { hid_warn(hdev, "Error initialising force feedback\n"); - goto err; + goto err_stop; } hid_info(hdev, "Universal pidff driver loaded successfully!"); return 0; +err_stop: + hid_hw_stop(hdev); err: return error; } diff --git a/drivers/hid/hid-winwing.c b/drivers/hid/hid-winwing.c index 9cd25a77999e6..19b92c2c6579a 100644 --- a/drivers/hid/hid-winwing.c +++ b/drivers/hid/hid-winwing.c @@ -315,7 +315,8 @@ static void winwing_haptic_rumble_cb(struct work_struct *work) static int winwing_play_effect(struct input_dev *dev, void *context, struct ff_effect *effect) { - struct winwing_drv_data *data = (struct winwing_drv_data *) context; + struct hid_device *hdev = input_get_drvdata(dev); + struct winwing_drv_data *data = hid_get_drvdata(hdev); if (effect->type != FF_RUMBLE) return 0; @@ -342,7 +343,12 @@ static int winwing_init_ff(struct hid_device *hdev, struct hid_input *hidinput) input_set_capability(hidinput->input, EV_FF, FF_RUMBLE); - return input_ff_create_memless(hidinput->input, data, + /* + * input_ff_create_memless() takes ownership of the context pointer + * and frees it on teardown; do not hand it the devm-managed drvdata. + * winwing_play_effect() fetches it from the input device instead. + */ + return input_ff_create_memless(hidinput->input, NULL, winwing_play_effect); } diff --git a/drivers/hid/i2c-hid/Makefile b/drivers/hid/i2c-hid/Makefile index 55bd5e0f35af3..38d5d827f3ce4 100644 --- a/drivers/hid/i2c-hid/Makefile +++ b/drivers/hid/i2c-hid/Makefile @@ -8,7 +8,7 @@ obj-$(CONFIG_I2C_HID_CORE) += i2c-hid.o i2c-hid-objs = i2c-hid-core.o i2c-hid-$(CONFIG_DMI) += i2c-hid-dmi-quirks.o -obj-$(CONFIG_I2C_HID_ACPI) += i2c-hid-acpi.o +obj-$(CONFIG_I2C_HID_ACPI) += i2c-hid-acpi.o i2c-hid-acpi-prp0001.o obj-$(CONFIG_I2C_HID_OF) += i2c-hid-of.o obj-$(CONFIG_I2C_HID_OF_ELAN) += i2c-hid-of-elan.o obj-$(CONFIG_I2C_HID_OF_GOODIX) += i2c-hid-of-goodix.o diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi-prp0001.c b/drivers/hid/i2c-hid/i2c-hid-acpi-prp0001.c new file mode 100644 index 0000000000000..d2cf4714ae7f1 --- /dev/null +++ b/drivers/hid/i2c-hid/i2c-hid-acpi-prp0001.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * HID over I2C driver for PRP0001 devices missing hid-descr-addr + * + * Some devices, for example the Lenovo KaiTian N60d and Inspur CP300L3, use + * _HID "PRP0001" with _DSD compatible "hid-over-i2c" but lack "hid-descr-addr" + * from the _DSD. The HID descriptor address is provided only through an ACPI + * _DSM. The TPD0 node in the DSDT shows _DSM Function 1 returning 0x20. + * + * Copyright (C) 2026 谢致邦 (XIE Zhibang) + */ + +#include +#include +#include +#include +#include + +#include "i2c-hid.h" +#include "i2c-hid-acpi.h" + +static int i2c_hid_acpi_prp0001_power_up(struct i2chid_ops *ops) +{ + /* give the device time to power up */ + msleep(750); + return 0; +} + +static struct i2chid_ops i2c_hid_acpi_prp0001_ops = { + .power_up = i2c_hid_acpi_prp0001_power_up, + /* + * No .restore_sequence needed: the _DSM on these devices returns a + * constant (0x20) with no side effects, unlike some PNP0C50 _DSM + * implementations that switch the hardware between PS/2 and I2C modes. + */ +}; + +static int i2c_hid_acpi_prp0001_probe(struct i2c_client *client) +{ + struct device *dev = &client->dev; + struct acpi_device *adev; + u16 hid_descriptor_address; + int ret; + + /* If hid-descr-addr is present, let i2c-hid-of handle it */ + if (device_property_present(dev, "hid-descr-addr")) + return -ENODEV; + + adev = ACPI_COMPANION(dev); + if (!adev) + return -ENODEV; + + ret = i2c_hid_acpi_get_descriptor(adev); + if (ret < 0) + return ret; + dev_warn(dev, + "hid-descr-addr device property NOT found, using ACPI _DSM fallback. Contact vendor for firmware update!\n"); + hid_descriptor_address = ret; + + /* + * No acpi_device_fix_up_power() needed: TPD0 has no _PS0, _PS3, _PSC + * or _PRx methods and follows I2C bus power. + */ + return i2c_hid_core_probe(client, &i2c_hid_acpi_prp0001_ops, + hid_descriptor_address, 0); +} + +static const struct of_device_id i2c_hid_acpi_prp0001_of_match[] = { + { .compatible = "hid-over-i2c" }, + {}, +}; +MODULE_DEVICE_TABLE(of, i2c_hid_acpi_prp0001_of_match); + +static const struct i2c_device_id i2c_hid_acpi_prp0001_id[] = { + { .name = "hid-over-i2c" }, + { } +}; +MODULE_DEVICE_TABLE(i2c, i2c_hid_acpi_prp0001_id); + +static struct i2c_driver i2c_hid_acpi_prp0001_driver = { + .driver = { + .name = "i2c_hid_acpi_prp0001", + .pm = &i2c_hid_core_pm, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + /* + * of_match_ptr() makes this NULL when CONFIG_OF=n, but that's + * fine: the I2C id_table with "hid-over-i2c" handles matching + * via client->name (set by acpi_set_modalias() from the _DSD + * compatible property). + */ + .of_match_table = of_match_ptr(i2c_hid_acpi_prp0001_of_match), + }, + + .probe = i2c_hid_acpi_prp0001_probe, + .remove = i2c_hid_core_remove, + .shutdown = i2c_hid_core_shutdown, + .id_table = i2c_hid_acpi_prp0001_id, +}; + +module_i2c_driver(i2c_hid_acpi_prp0001_driver); + +MODULE_DESCRIPTION("HID over I2C driver for PRP0001 devices missing hid-descr-addr"); +MODULE_AUTHOR("谢致邦 (XIE Zhibang) "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi.c b/drivers/hid/i2c-hid/i2c-hid-acpi.c index abd700a101f46..13f977d6aab61 100644 --- a/drivers/hid/i2c-hid/i2c-hid-acpi.c +++ b/drivers/hid/i2c-hid/i2c-hid-acpi.c @@ -25,9 +25,9 @@ #include #include #include -#include #include "i2c-hid.h" +#include "i2c-hid-acpi.h" struct i2c_hid_acpi { struct i2chid_ops ops; @@ -48,39 +48,11 @@ static const struct acpi_device_id i2c_hid_acpi_blacklist[] = { { } }; -/* HID I²C Device: 3cdff6f7-4267-4555-ad05-b30a3d8938de */ -static guid_t i2c_hid_guid = - GUID_INIT(0x3CDFF6F7, 0x4267, 0x4555, - 0xAD, 0x05, 0xB3, 0x0A, 0x3D, 0x89, 0x38, 0xDE); - -static int i2c_hid_acpi_get_descriptor(struct i2c_hid_acpi *ihid_acpi) -{ - struct acpi_device *adev = ihid_acpi->adev; - acpi_handle handle = acpi_device_handle(adev); - union acpi_object *obj; - u16 hid_descriptor_address; - - if (acpi_match_device_ids(adev, i2c_hid_acpi_blacklist) == 0) - return -ENODEV; - - obj = acpi_evaluate_dsm_typed(handle, &i2c_hid_guid, 1, 1, NULL, - ACPI_TYPE_INTEGER); - if (!obj) { - acpi_handle_err(handle, "Error _DSM call to get HID descriptor address failed\n"); - return -ENODEV; - } - - hid_descriptor_address = obj->integer.value; - ACPI_FREE(obj); - - return hid_descriptor_address; -} - static void i2c_hid_acpi_restore_sequence(struct i2chid_ops *ops) { struct i2c_hid_acpi *ihid_acpi = container_of(ops, struct i2c_hid_acpi, ops); - i2c_hid_acpi_get_descriptor(ihid_acpi); + i2c_hid_acpi_get_descriptor(ihid_acpi->adev); } static void i2c_hid_acpi_shutdown_tail(struct i2chid_ops *ops) @@ -93,24 +65,28 @@ static void i2c_hid_acpi_shutdown_tail(struct i2chid_ops *ops) static int i2c_hid_acpi_probe(struct i2c_client *client) { struct device *dev = &client->dev; + struct acpi_device *adev = ACPI_COMPANION(dev); struct i2c_hid_acpi *ihid_acpi; u16 hid_descriptor_address; int ret; - ihid_acpi = devm_kzalloc(&client->dev, sizeof(*ihid_acpi), GFP_KERNEL); + if (acpi_match_device_ids(adev, i2c_hid_acpi_blacklist) == 0) + return -ENODEV; + + ret = i2c_hid_acpi_get_descriptor(adev); + if (ret < 0) + return ret; + hid_descriptor_address = ret; + + ihid_acpi = devm_kzalloc(dev, sizeof(*ihid_acpi), GFP_KERNEL); if (!ihid_acpi) return -ENOMEM; - ihid_acpi->adev = ACPI_COMPANION(dev); + ihid_acpi->adev = adev; ihid_acpi->ops.shutdown_tail = i2c_hid_acpi_shutdown_tail; ihid_acpi->ops.restore_sequence = i2c_hid_acpi_restore_sequence; - ret = i2c_hid_acpi_get_descriptor(ihid_acpi); - if (ret < 0) - return ret; - hid_descriptor_address = ret; - - acpi_device_fix_up_power(ihid_acpi->adev); + acpi_device_fix_up_power(adev); return i2c_hid_core_probe(client, &ihid_acpi->ops, hid_descriptor_address, 0); diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi.h b/drivers/hid/i2c-hid/i2c-hid-acpi.h new file mode 100644 index 0000000000000..0bbed1853313d --- /dev/null +++ b/drivers/hid/i2c-hid/i2c-hid-acpi.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _I2C_HID_ACPI_H +#define _I2C_HID_ACPI_H + +#include +#include + +static inline int i2c_hid_acpi_get_descriptor(struct acpi_device *adev) +{ + /* HID I²C Device: 3cdff6f7-4267-4555-ad05-b30a3d8938de */ + static const guid_t i2c_hid_guid = + GUID_INIT(0x3CDFF6F7, 0x4267, 0x4555, + 0xAD, 0x05, 0xB3, 0x0A, 0x3D, 0x89, 0x38, 0xDE); + + acpi_handle handle = acpi_device_handle(adev); + union acpi_object *obj; + u16 addr; + + obj = acpi_evaluate_dsm_typed(handle, &i2c_hid_guid, + 1, 1, NULL, ACPI_TYPE_INTEGER); + if (!obj) { + acpi_handle_err(handle, + "Error _DSM call to get HID descriptor address failed\n"); + return -ENODEV; + } + + addr = obj->integer.value; + ACPI_FREE(obj); + return addr; +} + +#endif diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c index 3adb16366e939..0e725a0f0abee 100644 --- a/drivers/hid/i2c-hid/i2c-hid-core.c +++ b/drivers/hid/i2c-hid/i2c-hid-core.c @@ -792,7 +792,7 @@ static int i2c_hid_parse(struct hid_device *hid) ihid->hdesc.wReportDescRegister, rdesc, rsize); if (ret) { - hid_err(hid, "reading report descriptor failed\n"); + dev_err(&client->dev, "reading report descriptor failed\n"); goto out; } } diff --git a/drivers/hid/i2c-hid/i2c-hid-of-goodix.c b/drivers/hid/i2c-hid/i2c-hid-of-goodix.c index f1597ad67e7c8..f4dbcd1d1d472 100644 --- a/drivers/hid/i2c-hid/i2c-hid-of-goodix.c +++ b/drivers/hid/i2c-hid/i2c-hid-of-goodix.c @@ -51,8 +51,10 @@ static int goodix_i2c_hid_power_up(struct i2chid_ops *ops) return ret; ret = regulator_enable(ihid_goodix->vddio); - if (ret) + if (ret) { + regulator_disable(ihid_goodix->vdd); return ret; + } if (ihid_goodix->timings->post_power_delay_ms) msleep(ihid_goodix->timings->post_power_delay_ms); diff --git a/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c b/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c index 46d3e9a019995..4126c2408d986 100644 --- a/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c +++ b/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c @@ -774,6 +774,7 @@ static void quicki2c_remove(struct pci_dev *pdev) quicki2c_hid_remove(qcdev); quicki2c_dma_deinit(qcdev); + pm_runtime_dont_use_autosuspend(qcdev->dev); pm_runtime_get_noresume(qcdev->dev); quicki2c_dev_deinit(qcdev); diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c index 4ae2e1718b30c..5d35edf2b6d86 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c +++ b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c @@ -555,7 +555,14 @@ static int quickspi_alloc_report_buf(struct quickspi_device *qsdev) max_report_len = max(le16_to_cpu(qsdev->dev_desc.max_output_len), le16_to_cpu(qsdev->dev_desc.max_input_len)); - qsdev->report_buf = devm_kzalloc(qsdev->dev, max_report_len, GFP_KERNEL); + /* + * write_cmd_to_txdma() writes the output report header ahead of the + * content in this buffer, so it has to hold both. + */ + qsdev->report_buf_size = HIDSPI_OUTPUT_REPORT_SIZE(max_report_len); + + qsdev->report_buf = devm_kzalloc(qsdev->dev, qsdev->report_buf_size, + GFP_KERNEL); if (!qsdev->report_buf) return -ENOMEM; @@ -713,6 +720,7 @@ static void quickspi_remove(struct pci_dev *pdev) quickspi_hid_remove(qsdev); quickspi_dma_deinit(qsdev); + pm_runtime_dont_use_autosuspend(qsdev->dev); pm_runtime_get_noresume(qsdev->dev); quickspi_dev_deinit(qsdev); diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h index bf5e18f5a5f42..0ed964bfe3dd2 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h +++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h @@ -157,6 +157,7 @@ struct quickspi_device { u8 *report_descriptor; u8 *input_buf; u8 *report_buf; + u32 report_buf_size; u32 report_len; wait_queue_head_t reset_ack_wq; diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c index 91d5807b4a830..a60a0a7f16aad 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c +++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c @@ -61,7 +61,7 @@ static int quickspi_hid_raw_request(struct hid_device *hid, switch (reqtype) { case HID_REQ_GET_REPORT: - ret = quickspi_get_report(qsdev, rtype, reportnum, buf); + ret = quickspi_get_report(qsdev, rtype, reportnum, buf, len); break; case HID_REQ_SET_REPORT: ret = quickspi_set_report(qsdev, rtype, reportnum, buf, len); diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c index cb19057f1191b..847c5ec55569a 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c +++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c @@ -30,6 +30,9 @@ static int write_cmd_to_txdma(struct quickspi_device *qsdev, write_buf = (struct output_report *)qsdev->report_buf; + if (HIDSPI_OUTPUT_REPORT_SIZE(report_buf_len) > qsdev->report_buf_size) + return -EINVAL; + write_buf->output_hdr.report_type = report_type; write_buf->output_hdr.content_len = cpu_to_le16(report_buf_len); write_buf->output_hdr.content_id = report_id; @@ -342,10 +345,12 @@ int reset_tic(struct quickspi_device *qsdev) } int quickspi_get_report(struct quickspi_device *qsdev, - u8 report_type, unsigned int report_id, void *buf) + u8 report_type, unsigned int report_id, void *buf, + u32 buf_len) { int rep_type; int ret; + u32 report_len; if (report_type == HID_INPUT_REPORT) { rep_type = GET_INPUT_REPORT; @@ -372,9 +377,17 @@ int quickspi_get_report(struct quickspi_device *qsdev, } qsdev->get_report_cmpl = false; - memcpy(buf, qsdev->report_buf, qsdev->report_len); + /* quickspi_handle_input_data() updates this from IRQ context. */ + report_len = READ_ONCE(qsdev->report_len); + if (report_len > buf_len) { + dev_err_once(qsdev->dev, "Get report response too big, %u vs %u\n", + report_len, buf_len); + return -EINVAL; + } + + memcpy(buf, qsdev->report_buf, report_len); - return qsdev->report_len; + return report_len; } int quickspi_set_report(struct quickspi_device *qsdev, diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.h b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.h index 775e29c1ed136..8a2338bee8085 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.h +++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.h @@ -12,7 +12,7 @@ struct quickspi_device; void quickspi_handle_input_data(struct quickspi_device *qsdev, u32 buf_len); int quickspi_get_report(struct quickspi_device *qsdev, u8 report_type, - unsigned int report_id, void *buf); + unsigned int report_id, void *buf, u32 buf_len); int quickspi_set_report(struct quickspi_device *qsdev, u8 report_type, unsigned int report_id, void *buf, u32 buf_len); int quickspi_get_report_descriptor(struct quickspi_device *qsdev); diff --git a/drivers/hid/usbhid/hid-pidff.c b/drivers/hid/usbhid/hid-pidff.c index 5f4395f7c6458..22951b7ecd174 100644 --- a/drivers/hid/usbhid/hid-pidff.c +++ b/drivers/hid/usbhid/hid-pidff.c @@ -1539,13 +1539,20 @@ static int pidff_check_autocenter(struct pidff_device *pidff, int hid_pidff_init_with_quirks(struct hid_device *hid, u32 initial_quirks) { struct pidff_device *pidff; - struct hid_input *hidinput = - list_entry(hid->inputs.next, struct hid_input, list); - struct input_dev *dev = hidinput->input; + struct hid_input *hidinput; + struct input_dev *dev; struct ff_device *ff; int max_effects; int error; + if (list_empty(&hid->inputs)) { + hid_err(hid, "no inputs found\n"); + return -ENODEV; + } + + hidinput = list_first_entry(&hid->inputs, struct hid_input, list); + dev = hidinput->input; + hid_dbg(hid, "starting pid init\n"); if (list_empty(&hid->report_enum[HID_OUTPUT_REPORT].report_list)) { diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c index 0eafa483b7f75..40770affdbdee 100644 --- a/drivers/hid/wacom_sys.c +++ b/drivers/hid/wacom_sys.c @@ -113,8 +113,9 @@ static int wacom_wac_pen_serial_enforce(struct hid_device *hdev, /* Queue events which have invalid tool type or serial number */ for (i = 0; i < report->maxfield; i++) { - for (j = 0; j < report->field[i]->maxusage; j++) { - struct hid_field *field = report->field[i]; + struct hid_field *field = report->field[i]; + + for (j = 0; j < field->report_count; j++) { struct hid_usage *usage = &field->usage[j]; unsigned int equivalent_usage = wacom_equivalent_usage(usage->hid); unsigned int offset; diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c index da1f0ea85625d..af76e49fd0f25 100644 --- a/drivers/hid/wacom_wac.c +++ b/drivers/hid/wacom_wac.c @@ -1548,6 +1548,19 @@ static int wacom_intuos_pro2_bt_irq(struct wacom_wac *wacom, size_t len) return 0; } + if (wacom->features.type == INTUOSP2_BT || + wacom->features.type == INTUOSP2S_BT) { + if (len < 286) { + dev_warn(wacom->pen_input->dev.parent, + "Pro2 BT report too short: %zu bytes\n", len); + return 0; + } + } else if (len < 46) { + dev_warn(wacom->pen_input->dev.parent, + "Pro2 BT report too short: %zu bytes\n", len); + return 0; + } + wacom_intuos_pro2_bt_pen(wacom); if (wacom->features.type == INTUOSP2_BT || wacom->features.type == INTUOSP2S_BT) { diff --git a/drivers/hsi/controllers/omap_ssi_core.c b/drivers/hsi/controllers/omap_ssi_core.c index eeacc427fd659..47826315d7d62 100644 --- a/drivers/hsi/controllers/omap_ssi_core.c +++ b/drivers/hsi/controllers/omap_ssi_core.c @@ -502,6 +502,12 @@ static int ssi_probe(struct platform_device *pd) pm_runtime_enable(&pd->dev); + ssi->device.dma_mask = &ssi->device.coherent_dma_mask; + + err = dma_set_mask_and_coherent(&ssi->device, DMA_BIT_MASK(32)); + if (err) + goto out2; + err = ssi_hw_init(ssi); if (err < 0) goto out2; diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 6824bd7cb3c4f..18ee549d98809 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -2982,6 +2982,13 @@ static int __init hv_acpi_init(void) return -ENODEV; if (hv_root_partition() && !hv_nested) + /* + * A non-nested root partition does not need VMBus client + * functionality. However, the mshv_root module may have + * a dependency on the VMBus module as described in + * commit 840b740a35bf. Return success so the module + * loads even though no VMBus initialization is done. + */ return 0; /* @@ -3030,6 +3037,14 @@ static void __exit vmbus_exit(void) { int cpu; + if (hv_root_partition() && !hv_nested) + /* + * If a non-nested root partition loaded the VMBus module, + * hv_acpi_init() did not do any VMBus initialization. + * There's nothing to clean up, so just return. + */ + return; + unregister_syscore(&hv_synic_syscore); hv_remove_kexec_handler(); diff --git a/drivers/hwmon/applesmc.c b/drivers/hwmon/applesmc.c index 90a14a7f2c4c0..bfe9c5b0c4f14 100644 --- a/drivers/hwmon/applesmc.c +++ b/drivers/hwmon/applesmc.c @@ -1249,12 +1249,17 @@ static void applesmc_release_light_sensor(void) static int applesmc_create_key_backlight(void) { + int ret; + if (!smcreg.has_key_backlight) return 0; applesmc_led_wq = create_singlethread_workqueue("applesmc-led"); if (!applesmc_led_wq) return -ENOMEM; - return led_classdev_register(&pdev->dev, &applesmc_backlight); + ret = led_classdev_register(&pdev->dev, &applesmc_backlight); + if (ret) + destroy_workqueue(applesmc_led_wq); + return ret; } static void applesmc_release_key_backlight(void) diff --git a/drivers/hwmon/aspeed-pwm-tacho.c b/drivers/hwmon/aspeed-pwm-tacho.c index 1c5945d4ba377..bfce589c3fb1f 100644 --- a/drivers/hwmon/aspeed-pwm-tacho.c +++ b/drivers/hwmon/aspeed-pwm-tacho.c @@ -934,7 +934,9 @@ static int aspeed_pwm_tacho_probe(struct platform_device *pdev) "missing or invalid reset controller device tree entry"); return PTR_ERR(priv->rst); } - reset_control_deassert(priv->rst); + ret = reset_control_deassert(priv->rst); + if (ret) + return ret; ret = devm_add_action_or_reset(dev, aspeed_pwm_tacho_remove, priv); if (ret) diff --git a/drivers/hwmon/asus_rog_ryujin.c b/drivers/hwmon/asus_rog_ryujin.c index 10a1f5aca988d..f352036f0e52e 100644 --- a/drivers/hwmon/asus_rog_ryujin.c +++ b/drivers/hwmon/asus_rog_ryujin.c @@ -18,15 +18,25 @@ #define USB_VENDOR_ID_ASUS_ROG 0x0b05 #define USB_PRODUCT_ID_RYUJIN_AIO 0x1988 /* ASUS ROG RYUJIN II 360 */ +struct rog_ryujin_device_info { + u8 temp_offset; + u8 pump_speed_offset; + u8 fan_speed_offset; + u8 duty_channel; + bool has_controller; +}; + +static const struct rog_ryujin_device_info rog_ryujin_ii_360_info = { + .temp_offset = 3, + .pump_speed_offset = 5, + .fan_speed_offset = 7, + .duty_channel = 0, + .has_controller = true, +}; + #define STATUS_VALIDITY 1500 /* ms */ #define MAX_REPORT_LENGTH 65 -/* Cooler status report offsets */ -#define RYUJIN_TEMP_SENSOR_1 3 -#define RYUJIN_TEMP_SENSOR_2 4 -#define RYUJIN_PUMP_SPEED 5 -#define RYUJIN_INTERNAL_FAN_SPEED 7 - /* Cooler duty report offsets */ #define RYUJIN_PUMP_DUTY 4 #define RYUJIN_INTERNAL_FAN_DUTY 5 @@ -81,6 +91,7 @@ static const char *const rog_ryujin_speed_label[] = { struct rog_ryujin_data { struct hid_device *hdev; struct device *hwmon_dev; + const struct rog_ryujin_device_info *info; /* For reinitializing the completions below */ spinlock_t status_report_request_lock; struct completion cooler_status_received; @@ -112,6 +123,8 @@ static int rog_ryujin_pwm_to_percent(long val) static umode_t rog_ryujin_is_visible(const void *data, enum hwmon_sensor_types type, u32 attr, int channel) { + const struct rog_ryujin_data *priv = data; + switch (type) { case hwmon_temp: switch (attr) { @@ -123,6 +136,8 @@ static umode_t rog_ryujin_is_visible(const void *data, } break; case hwmon_fan: + if (channel >= 2 && !priv->info->has_controller) + return 0; switch (attr) { case hwmon_fan_label: case hwmon_fan_input: @@ -132,6 +147,8 @@ static umode_t rog_ryujin_is_visible(const void *data, } break; case hwmon_pwm: + if (channel >= 2 && !priv->info->has_controller) + return 0; switch (attr) { case hwmon_pwm_input: return 0644; @@ -156,6 +173,7 @@ static int rog_ryujin_write_expanded(struct rog_ryujin_data *priv, const u8 *cmd static int rog_ryujin_execute_cmd(struct rog_ryujin_data *priv, const u8 *cmd, int cmd_length, struct completion *status_completion) { + unsigned long flags; int ret; /* @@ -163,9 +181,9 @@ static int rog_ryujin_execute_cmd(struct rog_ryujin_data *priv, const u8 *cmd, i * completion. Reinit is done because hidraw could have triggered * the raw event parsing and marked the passed in completion as done. */ - spin_lock_bh(&priv->status_report_request_lock); + spin_lock_irqsave(&priv->status_report_request_lock, flags); reinit_completion(status_completion); - spin_unlock_bh(&priv->status_report_request_lock); + spin_unlock_irqrestore(&priv->status_report_request_lock, flags); /* Send command for getting data */ ret = rog_ryujin_write_expanded(priv, cmd, cmd_length); @@ -198,12 +216,14 @@ static int rog_ryujin_get_status(struct rog_ryujin_data *priv) if (ret < 0) return ret; - /* Retrieve controller status (speeds) */ - ret = - rog_ryujin_execute_cmd(priv, get_controller_speed_cmd, GET_CMD_LENGTH, - &priv->controller_status_received); - if (ret < 0) - return ret; + if (priv->info->has_controller) { + /* Retrieve controller status (speeds) */ + ret = rog_ryujin_execute_cmd(priv, get_controller_speed_cmd, + GET_CMD_LENGTH, + &priv->controller_status_received); + if (ret < 0) + return ret; + } /* Retrieve cooler duty */ ret = @@ -212,12 +232,14 @@ static int rog_ryujin_get_status(struct rog_ryujin_data *priv) if (ret < 0) return ret; - /* Retrieve controller duty */ - ret = - rog_ryujin_execute_cmd(priv, get_controller_duty_cmd, GET_CMD_LENGTH, - &priv->controller_duty_received); - if (ret < 0) - return ret; + if (priv->info->has_controller) { + /* Retrieve controller duty */ + ret = rog_ryujin_execute_cmd(priv, get_controller_duty_cmd, + GET_CMD_LENGTH, + &priv->controller_duty_received); + if (ret < 0) + return ret; + } priv->updated = jiffies; return 0; @@ -289,6 +311,7 @@ static int rog_ryujin_write_fixed_duty(struct rog_ryujin_data *priv, int channel return ret; memcpy(set_cmd, set_cooler_duty_cmd, SET_CMD_LENGTH); + set_cmd[2] = priv->info->duty_channel; /* Cooler duties are set as 0-100% */ val = rog_ryujin_pwm_to_percent(val); @@ -388,20 +411,33 @@ static int rog_ryujin_raw_event(struct hid_device *hdev, struct hid_report *repo int size) { struct rog_ryujin_data *priv = hid_get_drvdata(hdev); + unsigned long flags; - if (data[0] != RYUJIN_CMD_PREFIX) + if (size < 2 || data[0] != RYUJIN_CMD_PREFIX) return 0; + spin_lock_irqsave(&priv->status_report_request_lock, flags); + if (data[1] == RYUJIN_GET_COOLER_STATUS_CMD_RESPONSE) { + if (size <= priv->info->temp_offset + 1 || + size <= priv->info->pump_speed_offset + 1 || + size <= priv->info->fan_speed_offset + 1) + goto unlock; + /* Received coolant temp and speeds of pump and internal fan */ - priv->temp_input[0] = - data[RYUJIN_TEMP_SENSOR_1] * 1000 + data[RYUJIN_TEMP_SENSOR_2] * 100; - priv->speed_input[0] = get_unaligned_le16(data + RYUJIN_PUMP_SPEED); - priv->speed_input[1] = get_unaligned_le16(data + RYUJIN_INTERNAL_FAN_SPEED); + priv->temp_input[0] = data[priv->info->temp_offset] * 1000 + + data[priv->info->temp_offset + 1] * 100; + priv->speed_input[0] = + get_unaligned_le16(data + priv->info->pump_speed_offset); + priv->speed_input[1] = + get_unaligned_le16(data + priv->info->fan_speed_offset); if (!completion_done(&priv->cooler_status_received)) complete_all(&priv->cooler_status_received); } else if (data[1] == RYUJIN_GET_CONTROLLER_SPEED_CMD_RESPONSE) { + if (size <= RYUJIN_CONTROLLER_SPEED_3 + 1) + goto unlock; + /* Received speeds of four fans attached to the controller */ priv->speed_input[2] = get_unaligned_le16(data + RYUJIN_CONTROLLER_SPEED_1); priv->speed_input[3] = get_unaligned_le16(data + RYUJIN_CONTROLLER_SPEED_2); @@ -411,6 +447,9 @@ static int rog_ryujin_raw_event(struct hid_device *hdev, struct hid_report *repo if (!completion_done(&priv->controller_status_received)) complete_all(&priv->controller_status_received); } else if (data[1] == RYUJIN_GET_COOLER_DUTY_CMD_RESPONSE) { + if (size <= RYUJIN_INTERNAL_FAN_DUTY) + goto unlock; + /* Received report for pump and internal fan duties (in %) */ if (data[RYUJIN_PUMP_DUTY] == 0 && data[RYUJIN_INTERNAL_FAN_DUTY] == 0) { /* @@ -428,7 +467,7 @@ static int rog_ryujin_raw_event(struct hid_device *hdev, struct hid_report *repo * We're expecting a report, so parse it. */ goto read_cooler_duty; - return 0; + goto unlock; } read_cooler_duty: priv->duty_input[0] = rog_ryujin_percent_to_pwm(data[RYUJIN_PUMP_DUTY]); @@ -437,6 +476,9 @@ static int rog_ryujin_raw_event(struct hid_device *hdev, struct hid_report *repo if (!completion_done(&priv->cooler_duty_received)) complete_all(&priv->cooler_duty_received); } else if (data[1] == RYUJIN_GET_CONTROLLER_DUTY_CMD_RESPONSE) { + if (size <= RYUJIN_CONTROLLER_DUTY) + goto unlock; + /* Received report for controller duty for fans (in PWM) */ if (data[RYUJIN_CONTROLLER_DUTY] == 0) { /* @@ -454,7 +496,7 @@ static int rog_ryujin_raw_event(struct hid_device *hdev, struct hid_report *repo * We're expecting a report, so parse it. */ goto read_controller_duty; - return 0; + goto unlock; } read_controller_duty: priv->duty_input[2] = data[RYUJIN_CONTROLLER_DUTY]; @@ -463,6 +505,8 @@ static int rog_ryujin_raw_event(struct hid_device *hdev, struct hid_report *repo complete_all(&priv->controller_duty_received); } +unlock: + spin_unlock_irqrestore(&priv->status_report_request_lock, flags); return 0; } @@ -471,11 +515,15 @@ static int rog_ryujin_probe(struct hid_device *hdev, const struct hid_device_id struct rog_ryujin_data *priv; int ret; + if (!id->driver_data) + return -EINVAL; + priv = devm_kzalloc(&hdev->dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; priv->hdev = hdev; + priv->info = (const struct rog_ryujin_device_info *)id->driver_data; hid_set_drvdata(hdev, priv); /* @@ -546,7 +594,8 @@ static void rog_ryujin_remove(struct hid_device *hdev) } static const struct hid_device_id rog_ryujin_table[] = { - { HID_USB_DEVICE(USB_VENDOR_ID_ASUS_ROG, USB_PRODUCT_ID_RYUJIN_AIO) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ASUS_ROG, USB_PRODUCT_ID_RYUJIN_AIO), + .driver_data = (kernel_ulong_t)&rog_ryujin_ii_360_info }, { } }; diff --git a/drivers/hwmon/cgbc-hwmon.c b/drivers/hwmon/cgbc-hwmon.c index 3aff4e092132f..230062a469073 100644 --- a/drivers/hwmon/cgbc-hwmon.c +++ b/drivers/hwmon/cgbc-hwmon.c @@ -41,45 +41,60 @@ enum cgbc_sensor_types { static const char * const cgbc_hwmon_labels_temp[] = { "CPU Temperature", - "Box Temperature", + "Case Temperature", "Ambient Temperature", - "Board Temperature", - "Carrier Temperature", - "Chipset Temperature", - "Video Temperature", + "CPU Board Temperature", + "Carrier Board Temperature", + "System Chipset Temperature", + "Video Controller/Board Temperature", "Other Temperature", - "TOPDIM Temperature", - "BOTTOMDIM Temperature", + "Top DIMM 0 Temperature", + "Bottom DIMM 0 Temperature", + "Alternate Board Temperature", + "Top DIMM 1 Temperature", + "Top DIMM 2 Temperature", + "Top DIMM 3 Temperature", + "Top DIMM 4 Temperature", + "Top DIMM 5 Temperature", + "Top DIMM 6 Temperature", + "Top DIMM 7 Temperature", + "Bottom DIMM 1 Temperature", }; +static const char * const cgbc_hwmon_labels_in[] = { + "CPU Core Voltage", + "DC Runtime Voltage", + "DC Standby Voltage", + "CMOS Battery Voltage", + "Battery Supply Voltage", + "AC Voltage", + "Other Voltage", + "5V Runtime Voltage", + "5V Standby Voltage", + "3V3 Runtime Voltage", + "3V3 Standby Voltage", + "VCore A Voltage", + "VCore B Voltage", + "12V Runtime Voltage", + "12V Standby Voltage", +}; + +/* + * Current sensors are a bit special, they don't have consecutive IDs like + * other types of sensors. So they need to be defined explicitly. + */ static const struct { - enum hwmon_sensor_types type; const char *label; -} cgbc_hwmon_labels_in[] = { - { hwmon_in, "CPU Voltage" }, - { hwmon_in, "DC Runtime Voltage" }, - { hwmon_in, "DC Standby Voltage" }, - { hwmon_in, "CMOS Battery Voltage" }, - { hwmon_in, "Battery Voltage" }, - { hwmon_in, "AC Voltage" }, - { hwmon_in, "Other Voltage" }, - { hwmon_in, "5V Voltage" }, - { hwmon_in, "5V Standby Voltage" }, - { hwmon_in, "3V3 Voltage" }, - { hwmon_in, "3V3 Standby Voltage" }, - { hwmon_in, "VCore A Voltage" }, - { hwmon_in, "VCore B Voltage" }, - { hwmon_in, "12V Voltage" }, - { hwmon_curr, "DC Current" }, - { hwmon_curr, "5V Current" }, - { hwmon_curr, "12V Current" }, + int id; +} cgbc_hwmon_labels_curr[] = { + { "DC Runtime Current", 0x12 }, + { "5V Runtime Current", 0x18 }, + { "12V Runtime Current", 0x1E }, }; -#define CGBC_HWMON_NB_IN_SENSORS 14 - static const char * const cgbc_hwmon_labels_fan[] = { "CPU Fan", - "Box Fan", + "Case Fan", "Ambient Fan", "Chipset Fan", "Video Fan", @@ -114,7 +129,8 @@ static int cgbc_hwmon_probe_sensors(struct device *dev, struct cgbc_hwmon_data * for (i = 0; i < nb_sensors; i++) { enum cgbc_sensor_types type; - unsigned int channel; + unsigned int channel, id; + int j; /* * No need to request data for the first sensor. @@ -128,32 +144,49 @@ static int cgbc_hwmon_probe_sensors(struct device *dev, struct cgbc_hwmon_data * } type = FIELD_GET(CGBC_HWMON_TYPE_MASK, data[1]); - channel = FIELD_GET(CGBC_HWMON_ID_MASK, data[1]) - 1; + id = FIELD_GET(CGBC_HWMON_ID_MASK, data[1]); + channel = id - 1; if (type == CGBC_HWMON_TYPE_TEMP && channel < ARRAY_SIZE(cgbc_hwmon_labels_temp)) { sensor->type = hwmon_temp; sensor->label = cgbc_hwmon_labels_temp[channel]; - } else if (type == CGBC_HWMON_TYPE_IN && - channel < ARRAY_SIZE(cgbc_hwmon_labels_in)) { + } else if (type == CGBC_HWMON_TYPE_IN) { /* * The Board Controller doesn't differentiate current and voltage sensors. - * Get the sensor type from cgbc_hwmon_labels_in[channel].type instead. + * First check if it is a current sensor. */ - sensor->type = cgbc_hwmon_labels_in[channel].type; - sensor->label = cgbc_hwmon_labels_in[channel].label; + for (j = 0; j < ARRAY_SIZE(cgbc_hwmon_labels_curr); j++) { + if (id == cgbc_hwmon_labels_curr[j].id) { + sensor->type = hwmon_curr; + sensor->label = cgbc_hwmon_labels_curr[j].label; + channel = j; + } + } + + /* If it's not a current sensor, it may be a voltage sensor. */ + if (!sensor->label && channel < ARRAY_SIZE(cgbc_hwmon_labels_in)) { + sensor->type = hwmon_in; + sensor->label = cgbc_hwmon_labels_in[channel]; + } } else if (type == CGBC_HWMON_TYPE_FAN && channel < ARRAY_SIZE(cgbc_hwmon_labels_fan)) { sensor->type = hwmon_fan; sensor->label = cgbc_hwmon_labels_fan[channel]; - } else { - dev_warn(dev, "Board Controller returned an unknown sensor (type=%d, channel=%d), ignore it", - type, channel); + } + + if (!sensor->label) { + dev_warn(dev, "Board Controller returned an unknown sensor (bc_type=%d, bc_id=%d), ignore it", + type, id); continue; } sensor->active = FIELD_GET(CGBC_HWMON_ACTIVE_BIT, data[1]); sensor->channel = channel; sensor->index = i; + + dev_dbg(dev, "Found sensor: bc_type=%d, bc_id=%d, hwmon_type=%d, hwmon_channel=%d, hwmon_label='%s', active=%d\n", + type, id, sensor->type, sensor->channel, sensor->label, sensor->active); + sensor++; hwmon->nb_sensors++; } @@ -167,14 +200,6 @@ static struct cgbc_hwmon_sensor *cgbc_hwmon_find_sensor(struct cgbc_hwmon_data * struct cgbc_hwmon_sensor *sensor = NULL; int i; - /* - * The Board Controller doesn't differentiate current and voltage sensors. - * The channel value (from the Board Controller point of view) shall be computed for current - * sensors. - */ - if (type == hwmon_curr) - channel += CGBC_HWMON_NB_IN_SENSORS; - for (i = 0; i < hwmon->nb_sensors; i++) { if (hwmon->sensors[i].type == type && hwmon->sensors[i].channel == channel) { sensor = &hwmon->sensors[i]; @@ -240,7 +265,12 @@ static const struct hwmon_channel_info * const cgbc_hwmon_info[] = { HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL), + HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL), HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL, @@ -248,7 +278,8 @@ static const struct hwmon_channel_info * const cgbc_hwmon_info[] = { HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL, - HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL), + HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL, + HWMON_I_INPUT | HWMON_I_LABEL), HWMON_CHANNEL_INFO(curr, HWMON_C_INPUT | HWMON_C_LABEL, HWMON_C_INPUT | HWMON_C_LABEL, HWMON_C_INPUT | HWMON_C_LABEL), diff --git a/drivers/hwmon/chipcap2.c b/drivers/hwmon/chipcap2.c index 4aecf463180fd..4b6244e13e9e2 100644 --- a/drivers/hwmon/chipcap2.c +++ b/drivers/hwmon/chipcap2.c @@ -89,11 +89,6 @@ struct cc2_data { bool process_irqs; }; -enum cc2_chan_addr { - CC2_CHAN_TEMP = 0, - CC2_CHAN_HUMIDITY, -}; - /* %RH as a per cent mille from a register value */ static long cc2_rh_convert(u16 data) { @@ -492,7 +487,7 @@ static irqreturn_t cc2_low_interrupt(int irq, void *data) if (cc2->process_irqs) { hwmon_notify_event(cc2->hwmon, hwmon_humidity, - hwmon_humidity_min_alarm, CC2_CHAN_HUMIDITY); + hwmon_humidity_min_alarm, 0); cc2->rh_alarm.low_alarm = true; } @@ -505,7 +500,7 @@ static irqreturn_t cc2_high_interrupt(int irq, void *data) if (cc2->process_irqs) { hwmon_notify_event(cc2->hwmon, hwmon_humidity, - hwmon_humidity_max_alarm, CC2_CHAN_HUMIDITY); + hwmon_humidity_max_alarm, 0); cc2->rh_alarm.high_alarm = true; } diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 6215ea49faaa9..ab9c8cbf887aa 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -650,6 +650,7 @@ static void coretemp_device_remove(int zoneid) struct platform_data *pdata = platform_get_drvdata(pdev); ida_destroy(&pdata->ida); + kfree(pdata->core_data); kfree(pdata); platform_device_unregister(pdev); } diff --git a/drivers/hwmon/corsair-cpro.c b/drivers/hwmon/corsair-cpro.c index 8354a002f4c5e..c096451526132 100644 --- a/drivers/hwmon/corsair-cpro.c +++ b/drivers/hwmon/corsair-cpro.c @@ -566,21 +566,18 @@ static int bootloader_show(struct seq_file *seqf, void *unused) } DEFINE_SHOW_ATTRIBUTE(bootloader); -static void ccp_debugfs_init(struct ccp_device *ccp) +static void ccp_debugfs_init(struct ccp_device *ccp, bool fw_valid, bool bl_valid) { char name[32]; - int ret; scnprintf(name, sizeof(name), "corsaircpro-%s", dev_name(&ccp->hdev->dev)); ccp->debugfs = debugfs_create_dir(name, NULL); - ret = get_fw_version(ccp); - if (!ret) + if (fw_valid) debugfs_create_file("firmware_version", 0444, ccp->debugfs, ccp, &firmware_fops); - ret = get_bl_version(ccp); - if (!ret) + if (bl_valid) debugfs_create_file("bootloader_version", 0444, ccp->debugfs, ccp, &bootloader_fops); } @@ -588,6 +585,7 @@ static void ccp_debugfs_init(struct ccp_device *ccp) static int ccp_probe(struct hid_device *hdev, const struct hid_device_id *id) { struct ccp_device *ccp; + bool fw_valid, bl_valid; int ret; ccp = devm_kzalloc(&hdev->dev, sizeof(*ccp), GFP_KERNEL); @@ -632,17 +630,27 @@ static int ccp_probe(struct hid_device *hdev, const struct hid_device_id *id) if (ret) goto out_hw_close; - ccp_debugfs_init(ccp); + /* + * Query the versions before registering the hwmon device: they send + * USB commands without holding ccp->mutex, which is only safe while + * nothing else can call send_usb_cmd(). + */ + fw_valid = !get_fw_version(ccp); + bl_valid = !get_bl_version(ccp); ccp->hwmon_dev = hwmon_device_register_with_info(&hdev->dev, "corsaircpro", ccp, &ccp_chip_info, NULL); if (IS_ERR(ccp->hwmon_dev)) { ret = PTR_ERR(ccp->hwmon_dev); - goto out_hw_close; + goto out_debugfs_remove; } + ccp_debugfs_init(ccp, fw_valid, bl_valid); + return 0; +out_debugfs_remove: + debugfs_remove_recursive(ccp->debugfs); out_hw_close: hid_hw_close(hdev); hid_device_io_stop(hdev); diff --git a/drivers/hwmon/cros_ec_hwmon.c b/drivers/hwmon/cros_ec_hwmon.c index 03bfcc40bb7c3..b65518fa7c9cc 100644 --- a/drivers/hwmon/cros_ec_hwmon.c +++ b/drivers/hwmon/cros_ec_hwmon.c @@ -5,6 +5,7 @@ * Copyright (C) 2024 Thomas Weißschuh */ +#include #include #include #include @@ -24,6 +25,7 @@ struct cros_ec_hwmon_priv { struct cros_ec_device *cros_ec; + struct device *hwmon_dev; const char *temp_sensor_names[EC_TEMP_SENSOR_ENTRIES + EC_TEMP_SENSOR_B_ENTRIES]; u8 usable_fans; bool fan_control_supported; @@ -401,6 +403,8 @@ static int cros_ec_hwmon_cooling_get_cur_state(struct thermal_cooling_device *cd u8 read_val; int ret; + guard(hwmon_lock)(priv->hwmon_priv->hwmon_dev); + ret = cros_ec_hwmon_read_pwm_value(priv->hwmon_priv->cros_ec, priv->index, &read_val); if (ret) return ret; @@ -414,6 +418,8 @@ static int cros_ec_hwmon_cooling_set_cur_state(struct thermal_cooling_device *cd { const struct cros_ec_hwmon_cooling_priv *priv = cdev->devdata; + guard(hwmon_lock)(priv->hwmon_priv->hwmon_dev); + return cros_ec_hwmon_write_pwm_input(priv->hwmon_priv->cros_ec, priv->index, val); } @@ -547,7 +553,6 @@ static int cros_ec_hwmon_probe(struct platform_device *pdev) struct cros_ec_dev *ec_dev = dev_get_drvdata(dev->parent); struct cros_ec_device *cros_ec = ec_dev->ec_dev; struct cros_ec_hwmon_priv *priv; - struct device *hwmon_dev; u8 thermal_version; int ret; @@ -570,13 +575,17 @@ static int cros_ec_hwmon_probe(struct platform_device *pdev) priv->fan_control_supported = cros_ec_hwmon_probe_fan_control_supported(priv->cros_ec); priv->temp_threshold_supported = is_cros_ec_cmd_available(priv->cros_ec, EC_CMD_THERMAL_GET_THRESHOLD, 1); + + priv->hwmon_dev = devm_hwmon_device_register_with_info(dev, "cros_ec", priv, + &cros_ec_hwmon_chip_info, NULL); + if (IS_ERR(priv->hwmon_dev)) + return PTR_ERR(priv->hwmon_dev); + cros_ec_hwmon_register_fan_cooling_devices(dev, priv); - hwmon_dev = devm_hwmon_device_register_with_info(dev, "cros_ec", priv, - &cros_ec_hwmon_chip_info, NULL); platform_set_drvdata(pdev, priv); - return PTR_ERR_OR_ZERO(hwmon_dev); + return 0; } static int cros_ec_hwmon_suspend(struct platform_device *pdev, pm_message_t state) diff --git a/drivers/hwmon/emc1403.c b/drivers/hwmon/emc1403.c index cd753b38709f2..2f6dadbce68a5 100644 --- a/drivers/hwmon/emc1403.c +++ b/drivers/hwmon/emc1403.c @@ -305,10 +305,9 @@ static int emc1403_get_hyst(struct thermal_data *data, int channel, ret = regmap_read(data->regmap, 0x21, &hyst); if (ret < 0) return ret; - if (map == temp_min) - *val = limit + hyst * 1000; - else - *val = limit - hyst * 1000; + + *val = limit - hyst * 1000; + return 0; } @@ -324,9 +323,6 @@ static int emc1403_temp_read(struct thermal_data *data, u32 attr, int channel, l case hwmon_temp_input: ret = emc1403_get_temp(data, channel, ema1403_temp_map[attr], val); break; - case hwmon_temp_min_hyst: - ret = emc1403_get_hyst(data, channel, temp_min, val); - break; case hwmon_temp_max_hyst: ret = emc1403_get_hyst(data, channel, temp_max, val); break; @@ -548,7 +544,6 @@ static umode_t emc1403_temp_is_visible(const void *_data, u32 attr, int channel) case hwmon_temp_max_alarm: case hwmon_temp_crit_alarm: case hwmon_temp_fault: - case hwmon_temp_min_hyst: case hwmon_temp_max_hyst: return 0444; case hwmon_temp_min: @@ -591,35 +586,35 @@ static const struct hwmon_channel_info * const emc1403_info[] = { HWMON_CHANNEL_INFO(chip, HWMON_C_UPDATE_INTERVAL), HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT ), diff --git a/drivers/hwmon/gpio-fan.c b/drivers/hwmon/gpio-fan.c index 084828e1e2817..3f78375eeb448 100644 --- a/drivers/hwmon/gpio-fan.c +++ b/drivers/hwmon/gpio-fan.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -67,7 +68,7 @@ static irqreturn_t fan_alarm_irq_handler(int irq, void *dev_id) schedule_work(&fan_data->alarm_work); - return IRQ_NONE; + return IRQ_HANDLED; } static ssize_t fan1_alarm_show(struct device *dev, @@ -84,6 +85,7 @@ static DEVICE_ATTR_RO(fan1_alarm); static int fan_alarm_init(struct gpio_fan_data *fan_data) { int alarm_irq; + int err; struct device *dev = fan_data->dev; /* @@ -94,10 +96,14 @@ static int fan_alarm_init(struct gpio_fan_data *fan_data) if (alarm_irq <= 0) return 0; - INIT_WORK(&fan_data->alarm_work, fan_alarm_notify); + err = devm_work_autocancel(dev, &fan_data->alarm_work, + fan_alarm_notify); + if (err) + return err; + irq_set_irq_type(alarm_irq, IRQ_TYPE_EDGE_BOTH); return devm_request_irq(dev, alarm_irq, fan_alarm_irq_handler, - IRQF_SHARED, "GPIO fan alarm", fan_data); + 0, "GPIO fan alarm", fan_data); } /* @@ -606,8 +612,11 @@ static void gpio_fan_shutdown(struct platform_device *pdev) { struct gpio_fan_data *fan_data = platform_get_drvdata(pdev); - if (fan_data->gpios) + if (fan_data->gpios) { + mutex_lock(&fan_data->lock); set_fan_speed(fan_data, 0); + mutex_unlock(&fan_data->lock); + } } static int gpio_fan_runtime_suspend(struct device *dev) diff --git a/drivers/hwmon/hp-wmi-sensors.c b/drivers/hwmon/hp-wmi-sensors.c index 03c684ba83bd6..cfa0d6ad0fb3f 100644 --- a/drivers/hwmon/hp-wmi-sensors.c +++ b/drivers/hwmon/hp-wmi-sensors.c @@ -526,14 +526,12 @@ static int check_wobj(const union acpi_object *wobj, for (prop = 0; prop <= last_prop; prop++) { type = elements[prop].type; valid_type = property_map[prop]; - if (type != valid_type) { - if (type == ACPI_TYPE_BUFFER && - valid_type == ACPI_TYPE_STRING && - is_raw_wmi_string(elements[prop].buffer.pointer, - elements[prop].buffer.length)) - continue; + if (type == ACPI_TYPE_BUFFER && + is_raw_wmi_string(elements[prop].buffer.pointer, + elements[prop].buffer.length)) + type = ACPI_TYPE_STRING; + if (type != valid_type) return -EINVAL; - } } return 0; @@ -579,6 +577,7 @@ static int check_numeric_sensor_wobj(const union acpi_object *wobj, int prop = HP_WMI_PROPERTY_NAME; acpi_object_type valid_type; union acpi_object *elements; + union acpi_object *element; u32 elem_count; int last_prop; bool is_new; @@ -602,13 +601,20 @@ static int check_numeric_sensor_wobj(const union acpi_object *wobj, elem_count > HP_WMI_MAX_PROPERTIES) return -EINVAL; - type = elements[HP_WMI_PROPERTY_SIZE].type; + element = &elements[HP_WMI_PROPERTY_SIZE]; + type = element->type; switch (type) { case ACPI_TYPE_INTEGER: is_new = true; last_prop = HP_WMI_PROPERTY_RATE_UNITS; break; + case ACPI_TYPE_BUFFER: + if (!is_raw_wmi_string(element->buffer.pointer, + element->buffer.length)) + return -EINVAL; + fallthrough; + case ACPI_TYPE_STRING: is_new = false; last_prop = HP_WMI_PROPERTY_CURRENT_READING; @@ -631,6 +637,10 @@ static int check_numeric_sensor_wobj(const union acpi_object *wobj, for (i = 0; i < elem_count && prop <= last_prop; i++, prop++) { type = elements[i].type; valid_type = hp_wmi_property_map[prop]; + if (type == ACPI_TYPE_BUFFER && + is_raw_wmi_string(elements[i].buffer.pointer, + elements[i].buffer.length)) + type = ACPI_TYPE_STRING; if (type != valid_type) return -EINVAL; @@ -651,6 +661,10 @@ static int check_numeric_sensor_wobj(const union acpi_object *wobj, /* PossibleStates[0] has already been type-checked. */ for (j = 0; i + 1 < elem_count && j + 1 < count; j++) { type = elements[++i].type; + if (type == ACPI_TYPE_BUFFER && + is_raw_wmi_string(elements[i].buffer.pointer, + elements[i].buffer.length)) + type = ACPI_TYPE_STRING; if (type != valid_type) return -EINVAL; } @@ -1247,7 +1261,9 @@ static int fungible_show(struct seq_file *seqf, enum hp_wmi_property prop) break; case HP_WMI_PROPERTY_CURRENT_STATE: + mutex_lock(&state->lock); seq_printf(seqf, "%s\n", nsensor->current_state); + mutex_unlock(&state->lock); break; case HP_WMI_PROPERTY_UNIT_MODIFIER: diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index 29dc90a2c3fe7..8cf717ae8b1dd 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -371,18 +371,17 @@ static ssize_t pec_store(struct device *dev, struct device_attribute *devattr, * handling is not required. */ hwdev = to_hwmon_device(hdev); - guard(mutex)(&hwdev->lock); - if (hwdev->chip->ops->write) { - err = hwdev->chip->ops->write(hdev, hwmon_chip, hwmon_chip_pec, 0, val); - if (err && err != -EOPNOTSUPP) - goto put; + scoped_guard(mutex, &hwdev->lock) { + if (hwdev->chip->ops->write) { + err = hwdev->chip->ops->write(hdev, hwmon_chip, hwmon_chip_pec, 0, val); + if (err && err != -EOPNOTSUPP) + goto put; + } + if (!val) + client->flags &= ~I2C_CLIENT_PEC; + else + client->flags |= I2C_CLIENT_PEC; } - - if (!val) - client->flags &= ~I2C_CLIENT_PEC; - else - client->flags |= I2C_CLIENT_PEC; - err = count; put: put_device(hdev); diff --git a/drivers/hwmon/ina2xx.c b/drivers/hwmon/ina2xx.c index 449a72c6b40bd..958a1e3ec728d 100644 --- a/drivers/hwmon/ina2xx.c +++ b/drivers/hwmon/ina2xx.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -128,6 +129,17 @@ enum ina2xx_ids { sy24655 }; +enum ina2xx_alert_type { + INA2XX_ALERT_NONE, + INA2XX_ALERT_CURRENT_LOW, + INA2XX_ALERT_CURRENT_HIGH, + INA2XX_ALERT_POWER_HIGH, + INA2XX_ALERT_BUS_VOLTAGE_LOW, + INA2XX_ALERT_BUS_VOLTAGE_HIGH, + INA2XX_ALERT_SHUNT_VOLTAGE_LOW, + INA2XX_ALERT_SHUNT_VOLTAGE_HIGH, +}; + struct ina2xx_config { u16 config_default; bool has_alerts; /* chip supports alerts and limits */ @@ -147,6 +159,7 @@ struct ina2xx_data { const struct ina2xx_config *config; enum ina2xx_ids chip; + enum ina2xx_alert_type active_alert; long rshunt; long current_lsb_uA; long power_lsb_uW; @@ -413,16 +426,78 @@ static u16 ina226_alert_to_reg(struct ina2xx_data *data, int reg, long val) } } -static int ina226_alert_limit_read(struct ina2xx_data *data, u32 mask, int reg, long *val) +static u32 ina2xx_alert_type_to_mask(enum ina2xx_alert_type alert) +{ + switch (alert) { + case INA2XX_ALERT_CURRENT_LOW: + case INA2XX_ALERT_SHUNT_VOLTAGE_LOW: + return INA226_SHUNT_UNDER_VOLTAGE_MASK; + case INA2XX_ALERT_CURRENT_HIGH: + case INA2XX_ALERT_SHUNT_VOLTAGE_HIGH: + return INA226_SHUNT_OVER_VOLTAGE_MASK; + case INA2XX_ALERT_BUS_VOLTAGE_LOW: + return INA226_BUS_UNDER_VOLTAGE_MASK; + case INA2XX_ALERT_BUS_VOLTAGE_HIGH: + return INA226_BUS_OVER_VOLTAGE_MASK; + case INA2XX_ALERT_POWER_HIGH: + return INA226_POWER_OVER_LIMIT_MASK; + case INA2XX_ALERT_NONE: + return 0; + default: + /* programmer error */ + WARN_ON_ONCE(1); + return 0; + } +} + +static enum ina2xx_alert_type ina2xx_mask_to_alert_type(u32 mask) +{ + int top_bit = fls(mask & INA226_ALERT_CONFIG_MASK); + + if (!top_bit) + return INA2XX_ALERT_NONE; + + /* + * Multiple bits may be set, with the highest-set function taking + * precedence according to the datasheet. Shunt voltage masks are + * assumed to map to voltage monitoring rather than current monitoring, + * since the latter isn't directly implemented in the hardware. + */ + switch (BIT(top_bit - 1)) { + case INA226_SHUNT_OVER_VOLTAGE_MASK: + return INA2XX_ALERT_SHUNT_VOLTAGE_HIGH; + case INA226_SHUNT_UNDER_VOLTAGE_MASK: + return INA2XX_ALERT_SHUNT_VOLTAGE_LOW; + case INA226_BUS_OVER_VOLTAGE_MASK: + return INA2XX_ALERT_BUS_VOLTAGE_HIGH; + case INA226_BUS_UNDER_VOLTAGE_MASK: + return INA2XX_ALERT_BUS_VOLTAGE_LOW; + case INA226_POWER_OVER_LIMIT_MASK: + return INA2XX_ALERT_POWER_HIGH; + default: + return INA2XX_ALERT_NONE; + } +} + +static int ina226_alert_limit_read(struct ina2xx_data *data, enum ina2xx_alert_type alert, + int reg, long *val) { struct regmap *regmap = data->regmap; int regval; + u32 mask; int ret; + /* Avoid nonzero reads from inactive alerts caused by shared limit register */ + if (data->active_alert != alert) { + *val = 0; + return 0; + } + ret = regmap_read(regmap, INA226_MASK_ENABLE, ®val); if (ret) return ret; + mask = ina2xx_alert_type_to_mask(alert); if (regval & mask) { ret = regmap_read(regmap, INA226_ALERT_LIMIT, ®val); if (ret) @@ -434,9 +509,11 @@ static int ina226_alert_limit_read(struct ina2xx_data *data, u32 mask, int reg, return 0; } -static int ina226_alert_limit_write(struct ina2xx_data *data, u32 mask, int reg, long val) +static int ina226_alert_limit_write(struct ina2xx_data *data, enum ina2xx_alert_type alert, + int reg, long val) { struct regmap *regmap = data->regmap; + u32 mask; int ret; if (val < 0) @@ -451,15 +528,22 @@ static int ina226_alert_limit_write(struct ina2xx_data *data, u32 mask, int reg, INA226_ALERT_CONFIG_MASK, 0); if (ret < 0) return ret; + data->active_alert = INA2XX_ALERT_NONE; ret = regmap_write(regmap, INA226_ALERT_LIMIT, ina226_alert_to_reg(data, reg, val)); if (ret < 0) return ret; - if (val) - return regmap_update_bits(regmap, INA226_MASK_ENABLE, - INA226_ALERT_CONFIG_MASK, mask); + if (val) { + mask = ina2xx_alert_type_to_mask(alert); + ret = regmap_update_bits(regmap, INA226_MASK_ENABLE, + INA226_ALERT_CONFIG_MASK, mask); + if (ret < 0) + return ret; + data->active_alert = alert; + } + return 0; } @@ -483,15 +567,26 @@ static int ina2xx_chip_read(struct device *dev, u32 attr, long *val) return 0; } -static int ina226_alert_read(struct regmap *regmap, u32 mask, long *val) +static int ina226_alert_read(struct ina2xx_data *data, enum ina2xx_alert_type alert, long *val) { unsigned int regval; + u32 mask; int ret; - ret = regmap_read_bypassed(regmap, INA226_MASK_ENABLE, ®val); + /* + * With alert latching, reading alerts from hardware also clears the + * alert, so return early if the alert is inactive. + */ + if (data->active_alert != alert) { + *val = 0; + return 0; + } + + ret = regmap_read_bypassed(data->regmap, INA226_MASK_ENABLE, ®val); if (ret) return ret; + mask = ina2xx_alert_type_to_mask(alert); *val = (regval & mask) && (regval & INA226_ALERT_FUNCTION_FLAG); return 0; @@ -500,10 +595,10 @@ static int ina226_alert_read(struct regmap *regmap, u32 mask, long *val) static int ina2xx_in_read(struct device *dev, u32 attr, int channel, long *val) { int voltage_reg = channel ? INA2XX_BUS_VOLTAGE : INA2XX_SHUNT_VOLTAGE; - u32 under_voltage_mask = channel ? INA226_BUS_UNDER_VOLTAGE_MASK - : INA226_SHUNT_UNDER_VOLTAGE_MASK; - u32 over_voltage_mask = channel ? INA226_BUS_OVER_VOLTAGE_MASK - : INA226_SHUNT_OVER_VOLTAGE_MASK; + enum ina2xx_alert_type under_voltage_alert = channel ? INA2XX_ALERT_BUS_VOLTAGE_LOW + : INA2XX_ALERT_SHUNT_VOLTAGE_LOW; + enum ina2xx_alert_type over_voltage_alert = channel ? INA2XX_ALERT_BUS_VOLTAGE_HIGH + : INA2XX_ALERT_SHUNT_VOLTAGE_HIGH; struct ina2xx_data *data = dev_get_drvdata(dev); struct regmap *regmap = data->regmap; unsigned int regval; @@ -517,15 +612,15 @@ static int ina2xx_in_read(struct device *dev, u32 attr, int channel, long *val) *val = ina2xx_get_value(data, voltage_reg, regval); break; case hwmon_in_lcrit: - return ina226_alert_limit_read(data, under_voltage_mask, + return ina226_alert_limit_read(data, under_voltage_alert, voltage_reg, val); case hwmon_in_crit: - return ina226_alert_limit_read(data, over_voltage_mask, + return ina226_alert_limit_read(data, over_voltage_alert, voltage_reg, val); case hwmon_in_lcrit_alarm: - return ina226_alert_read(regmap, under_voltage_mask, val); + return ina226_alert_read(data, under_voltage_alert, val); case hwmon_in_crit_alarm: - return ina226_alert_read(regmap, over_voltage_mask, val); + return ina226_alert_read(data, over_voltage_alert, val); default: return -EOPNOTSUPP; } @@ -579,10 +674,10 @@ static int ina2xx_power_read(struct device *dev, u32 attr, long *val) case hwmon_power_average: return sy24655_average_power_read(data, SY24655_EIN, val); case hwmon_power_crit: - return ina226_alert_limit_read(data, INA226_POWER_OVER_LIMIT_MASK, + return ina226_alert_limit_read(data, INA2XX_ALERT_POWER_HIGH, INA2XX_POWER, val); case hwmon_power_crit_alarm: - return ina226_alert_read(data->regmap, INA226_POWER_OVER_LIMIT_MASK, val); + return ina226_alert_read(data, INA2XX_ALERT_POWER_HIGH, val); default: return -EOPNOTSUPP; } @@ -618,15 +713,15 @@ static int ina2xx_curr_read(struct device *dev, u32 attr, long *val) *val = ina2xx_get_value(data, INA2XX_CURRENT, regval); return 0; case hwmon_curr_lcrit: - return ina226_alert_limit_read(data, INA226_SHUNT_UNDER_VOLTAGE_MASK, + return ina226_alert_limit_read(data, INA2XX_ALERT_CURRENT_LOW, INA2XX_CURRENT, val); case hwmon_curr_crit: - return ina226_alert_limit_read(data, INA226_SHUNT_OVER_VOLTAGE_MASK, + return ina226_alert_limit_read(data, INA2XX_ALERT_CURRENT_HIGH, INA2XX_CURRENT, val); case hwmon_curr_lcrit_alarm: - return ina226_alert_read(regmap, INA226_SHUNT_UNDER_VOLTAGE_MASK, val); + return ina226_alert_read(data, INA2XX_ALERT_CURRENT_LOW, val); case hwmon_curr_crit_alarm: - return ina226_alert_read(regmap, INA226_SHUNT_OVER_VOLTAGE_MASK, val); + return ina226_alert_read(data, INA2XX_ALERT_CURRENT_HIGH, val); default: return -EOPNOTSUPP; } @@ -670,12 +765,12 @@ static int ina2xx_in_write(struct device *dev, u32 attr, int channel, long val) switch (attr) { case hwmon_in_lcrit: return ina226_alert_limit_write(data, - channel ? INA226_BUS_UNDER_VOLTAGE_MASK : INA226_SHUNT_UNDER_VOLTAGE_MASK, + channel ? INA2XX_ALERT_BUS_VOLTAGE_LOW : INA2XX_ALERT_SHUNT_VOLTAGE_LOW, channel ? INA2XX_BUS_VOLTAGE : INA2XX_SHUNT_VOLTAGE, val); case hwmon_in_crit: return ina226_alert_limit_write(data, - channel ? INA226_BUS_OVER_VOLTAGE_MASK : INA226_SHUNT_OVER_VOLTAGE_MASK, + channel ? INA2XX_ALERT_BUS_VOLTAGE_HIGH : INA2XX_ALERT_SHUNT_VOLTAGE_HIGH, channel ? INA2XX_BUS_VOLTAGE : INA2XX_SHUNT_VOLTAGE, val); default: @@ -690,7 +785,7 @@ static int ina2xx_power_write(struct device *dev, u32 attr, long val) switch (attr) { case hwmon_power_crit: - return ina226_alert_limit_write(data, INA226_POWER_OVER_LIMIT_MASK, + return ina226_alert_limit_write(data, INA2XX_ALERT_POWER_HIGH, INA2XX_POWER, val); default: return -EOPNOTSUPP; @@ -704,10 +799,10 @@ static int ina2xx_curr_write(struct device *dev, u32 attr, long val) switch (attr) { case hwmon_curr_lcrit: - return ina226_alert_limit_write(data, INA226_SHUNT_UNDER_VOLTAGE_MASK, + return ina226_alert_limit_write(data, INA2XX_ALERT_CURRENT_LOW, INA2XX_CURRENT, val); case hwmon_curr_crit: - return ina226_alert_limit_write(data, INA226_SHUNT_OVER_VOLTAGE_MASK, + return ina226_alert_limit_write(data, INA2XX_ALERT_CURRENT_HIGH, INA2XX_CURRENT, val); default: return -EOPNOTSUPP; @@ -868,8 +963,12 @@ static ssize_t shunt_resistor_show(struct device *dev, struct device_attribute *da, char *buf) { struct ina2xx_data *data = dev_get_drvdata(dev); + long rshunt; - return sysfs_emit(buf, "%li\n", data->rshunt); + scoped_guard(hwmon_lock, dev) { + rshunt = data->rshunt; + } + return sysfs_emit(buf, "%li\n", rshunt); } static ssize_t shunt_resistor_store(struct device *dev, @@ -925,6 +1024,16 @@ static int ina2xx_init(struct device *dev, struct ina2xx_data *data) if (data->config->has_alerts) { bool active_high = device_property_read_bool(dev, "ti,alert-polarity-active-high"); + unsigned int mask_enable; + + /* + * Infer active alert from MASK_ENABLE in case it's already + * configured (e.g., by a past probe or firmware) + */ + ret = regmap_read(regmap, INA226_MASK_ENABLE, &mask_enable); + if (ret < 0) + return ret; + data->active_alert = ina2xx_mask_to_alert_type(mask_enable); regmap_update_bits(regmap, INA226_MASK_ENABLE, INA226_ALERT_LATCH_ENABLE | INA226_ALERT_POLARITY, diff --git a/drivers/hwmon/ltc4282.c b/drivers/hwmon/ltc4282.c index b1675dc5b3c7f..54ba4b8542e94 100644 --- a/drivers/hwmon/ltc4282.c +++ b/drivers/hwmon/ltc4282.c @@ -1106,7 +1106,7 @@ static const struct clk_ops ltc4282_ops = { static int ltc428_clk_provider_setup(struct ltc4282_state *st, struct device *dev) { - struct clk_init_data init; + struct clk_init_data init = {}; int ret; if (!IS_ENABLED(CONFIG_COMMON_CLK)) diff --git a/drivers/hwmon/ltc4283.c b/drivers/hwmon/ltc4283.c index 9b85293ea6649..9ec409bc26482 100644 --- a/drivers/hwmon/ltc4283.c +++ b/drivers/hwmon/ltc4283.c @@ -1772,6 +1772,7 @@ static const struct of_device_id ltc4283_of_match[] = { { .compatible = "adi,ltc4283" }, { } }; +MODULE_DEVICE_TABLE(of, ltc4283_of_match); static const struct i2c_device_id ltc4283_i2c_id[] = { { "ltc4283" }, diff --git a/drivers/hwmon/max6621.c b/drivers/hwmon/max6621.c index e86ec6d237ca6..1a644d7871431 100644 --- a/drivers/hwmon/max6621.c +++ b/drivers/hwmon/max6621.c @@ -17,8 +17,8 @@ #define MAX6621_DRV_NAME "max6621" #define MAX6621_TEMP_INPUT_REG_NUM 9 -#define MAX6621_TEMP_INPUT_MIN -127000 -#define MAX6621_TEMP_INPUT_MAX 128000 +#define MAX6621_TEMP_INPUT_MIN -128000 +#define MAX6621_TEMP_INPUT_MAX 127000 #define MAX6621_TEMP_ALERT_CHAN_SHIFT 1 #define MAX6621_TEMP_S0D0_REG 0x00 @@ -239,7 +239,7 @@ max6621_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, if (ret) return ret; - *val = (regval >> MAX6621_REG_TEMP_SHIFT) * + *val = ((s16)regval >> MAX6621_REG_TEMP_SHIFT) * 1000L; break; @@ -254,7 +254,7 @@ max6621_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, if (ret) return ret; - *val = regval * 1000L; + *val = (s16)regval * 1000L; break; case hwmon_temp_crit_alarm: diff --git a/drivers/hwmon/mcp9982.c b/drivers/hwmon/mcp9982.c index 9e19e2697e25f..3918dc36c9463 100644 --- a/drivers/hwmon/mcp9982.c +++ b/drivers/hwmon/mcp9982.c @@ -395,6 +395,8 @@ static int mcp9982_read(struct device *dev, enum hwmon_sensor_types type, u32 at reg_status, !(reg_status & MCP9982_STATUS_BUSY), MCP9982_WAKE_UP_TIME_US, MCP9982_WAKE_UP_TIME_US * 10); + if (ret) + return ret; break; } break; diff --git a/drivers/hwmon/nct6694-hwmon.c b/drivers/hwmon/nct6694-hwmon.c index 6dcf22ca5018a..9a9a4db434c4f 100644 --- a/drivers/hwmon/nct6694-hwmon.c +++ b/drivers/hwmon/nct6694-hwmon.c @@ -159,6 +159,9 @@ static inline s8 temp_to_reg(long val) #define NCT6694_HWMON_TEMP_CONFIG (HWMON_T_INPUT | HWMON_T_ENABLE | \ HWMON_T_MAX | HWMON_T_MAX_HYST | \ HWMON_T_MAX_ALARM) +#define NCT6694_HWMON_DTIN_CONFIG (HWMON_T_INPUT | \ + HWMON_T_MAX | HWMON_T_MAX_HYST | \ + HWMON_T_MAX_ALARM) #define NCT6694_HWMON_FAN_CONFIG (HWMON_F_INPUT | HWMON_F_ENABLE | \ HWMON_F_MIN | HWMON_F_MIN_ALARM) #define NCT6694_HWMON_PWM_CONFIG (HWMON_PWM_INPUT | HWMON_PWM_ENABLE | \ @@ -193,22 +196,22 @@ static const struct hwmon_channel_info *nct6694_info[] = { NCT6694_HWMON_TEMP_CONFIG, /* TDP2 */ NCT6694_HWMON_TEMP_CONFIG, /* TDP3 */ NCT6694_HWMON_TEMP_CONFIG, /* TDP4 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN0 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN1 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN2 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN3 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN4 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN5 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN6 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN7 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN8 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN9 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN10 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN11 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN12 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN13 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN14 */ - NCT6694_HWMON_TEMP_CONFIG), /* DTIN15 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN0 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN1 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN2 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN3 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN4 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN5 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN6 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN7 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN8 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN9 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN10 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN11 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN12 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN13 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN14 */ + NCT6694_HWMON_DTIN_CONFIG), /* DTIN15 */ HWMON_CHANNEL_INFO(fan, NCT6694_HWMON_FAN_CONFIG, /* FIN0 */ diff --git a/drivers/hwmon/pmbus/pmbus.h b/drivers/hwmon/pmbus/pmbus.h index 23e3eda588702..3d5586c67f840 100644 --- a/drivers/hwmon/pmbus/pmbus.h +++ b/drivers/hwmon/pmbus/pmbus.h @@ -242,6 +242,7 @@ enum pmbus_regs { /* * OPERATION */ +#define PB_OPERATION_CONTROL_V_SRC GENMASK(5, 4) #define PB_OPERATION_CONTROL_ON BIT(7) /* @@ -386,7 +387,7 @@ enum pmbus_sensor_classes { }; #define PMBUS_PAGES 32 /* Per PMBus specification */ -#define PMBUS_PHASES 10 /* Maximum number of phases per page */ +#define PMBUS_PHASES 16 /* Maximum number of phases per page */ /* Functionality bit mask */ #define PMBUS_HAVE_VIN BIT(0) diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c index 3b219ed5386fb..a73a118fca82c 100644 --- a/drivers/hwmon/pmbus/pmbus_core.c +++ b/drivers/hwmon/pmbus/pmbus_core.c @@ -1192,7 +1192,9 @@ static int pmbus_get_boolean(struct i2c_client *client, struct pmbus_boolean *b, regval = status & mask; if (regval) { - if (data->revision >= PMBUS_REV_12) { + /* Generic STATUS_WORD alarms are not individually clearable. */ + if (data->revision >= PMBUS_REV_12 && + reg != PMBUS_STATUS_WORD) { ret = _pmbus_write_byte_data(client, page, reg, regval); if (ret) return ret; diff --git a/drivers/hwmon/pmbus/tps53679.c b/drivers/hwmon/pmbus/tps53679.c index 31e54608b3c9c..6c25701b36fce 100644 --- a/drivers/hwmon/pmbus/tps53679.c +++ b/drivers/hwmon/pmbus/tps53679.c @@ -187,7 +187,7 @@ static int tps53676_identify(struct i2c_client *client, return -EIO; for (i = 0; i < 2 * TPS53676_MAX_PHASES; i += 2) { if (buf[i + 1] & 0x80) { - if (buf[i] & 0x08) + if (buf[i] & BIT(4)) phases_b++; else phases_a++; @@ -200,6 +200,15 @@ static int tps53676_identify(struct i2c_client *client, if (phases_b > 0) { info->pages = 2; info->phases[1] = phases_b; + } else { + /* + * pmbus_set_page() does not update the PAGE register on + * single-page devices, so select page 0 explicitly in case + * the boot firmware left the device on another page. + */ + ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, 0); + if (ret < 0) + return ret; } return 0; } diff --git a/drivers/hwmon/pwm-fan.c b/drivers/hwmon/pwm-fan.c index 37f37813ea511..2983f9fad8d2f 100644 --- a/drivers/hwmon/pwm-fan.c +++ b/drivers/hwmon/pwm-fan.c @@ -483,7 +483,6 @@ static void pwm_fan_cleanup(void *__ctx) { struct pwm_fan_ctx *ctx = __ctx; - timer_delete_sync(&ctx->rpm_timer); if (ctx->pwm_shutdown) { ctx->enable_mode = pwm_enable_reg_enable; __set_pwm(ctx, ctx->pwm_shutdown); @@ -494,6 +493,13 @@ static void pwm_fan_cleanup(void *__ctx) } } +static void pwm_fan_timer_cleanup(void *__ctx) +{ + struct pwm_fan_ctx *ctx = __ctx; + + timer_shutdown_sync(&ctx->rpm_timer); +} + static int pwm_fan_probe(struct platform_device *pdev) { struct thermal_cooling_device *cdev; @@ -648,6 +654,10 @@ static int pwm_fan_probe(struct platform_device *pdev) } if (ctx->tach_count > 0) { + ret = devm_add_action_or_reset(dev, pwm_fan_timer_cleanup, ctx); + if (ret) + return ret; + ctx->sample_start = ktime_get(); mod_timer(&ctx->rpm_timer, jiffies + HZ); @@ -704,6 +714,7 @@ static void pwm_fan_shutdown(struct platform_device *pdev) { struct pwm_fan_ctx *ctx = platform_get_drvdata(pdev); + pwm_fan_timer_cleanup(ctx); pwm_fan_cleanup(ctx); } diff --git a/drivers/hwmon/sht4x.c b/drivers/hwmon/sht4x.c index 9cace0e8acdab..a97dda9e92dc5 100644 --- a/drivers/hwmon/sht4x.c +++ b/drivers/hwmon/sht4x.c @@ -277,6 +277,8 @@ static ssize_t heater_enable_store(struct device *dev, heating_time_bound = 1100; } + guard(hwmon_lock)(dev); + if (time_before(jiffies, data->heating_complete)) return -EBUSY; @@ -286,7 +288,7 @@ static ssize_t heater_enable_store(struct device *dev, data->heating_complete = jiffies + msecs_to_jiffies(heating_time_bound); data->data_pending = true; - return 0; + return count; } static ssize_t heater_power_show(struct device *dev, @@ -314,6 +316,8 @@ static ssize_t heater_power_store(struct device *dev, if (power != 20 && power != 110 && power != 200) return -EINVAL; + guard(hwmon_lock)(dev); + data->heater_power = power; return count; @@ -344,6 +348,8 @@ static ssize_t heater_time_store(struct device *dev, if (time != 100 && time != 1000) return -EINVAL; + guard(hwmon_lock)(dev); + data->heater_time = time; return count; diff --git a/drivers/hwmon/w83791d.c b/drivers/hwmon/w83791d.c index 4a777430af5cd..4b07a25ae59e2 100644 --- a/drivers/hwmon/w83791d.c +++ b/drivers/hwmon/w83791d.c @@ -1415,6 +1415,7 @@ static void w83791d_remove(struct i2c_client *client) struct w83791d_data *data = i2c_get_clientdata(client); hwmon_device_unregister(data->hwmon_dev); + sysfs_remove_group(&client->dev.kobj, &w83791d_group_fanpwm45); sysfs_remove_group(&client->dev.kobj, &w83791d_group); } diff --git a/drivers/hwmon/w83793.c b/drivers/hwmon/w83793.c index a548586369e1d..c6ef04c69856e 100644 --- a/drivers/hwmon/w83793.c +++ b/drivers/hwmon/w83793.c @@ -1928,7 +1928,9 @@ static int w83793_probe(struct i2c_client *client) for (i = 0; i < ARRAY_SIZE(w83793_temp); i++) device_remove_file(dev, &w83793_temp[i].dev_attr); free_mem: - kfree(data); + mutex_lock(&watchdog_data_mutex); + kref_put(&data->kref, w83793_release_resources); + mutex_unlock(&watchdog_data_mutex); exit: return err; } diff --git a/drivers/hwmon/yogafan.c b/drivers/hwmon/yogafan.c index 605cc928f21f3..314ee61955dd8 100644 --- a/drivers/hwmon/yogafan.c +++ b/drivers/hwmon/yogafan.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/** +/* * yoga_fan.c - Lenovo Yoga/Legion Fan Hardware Monitoring Driver * * Provides fan speed monitoring for Lenovo Yoga, Legion, and IdeaPad diff --git a/drivers/hwspinlock/hwspinlock_core.c b/drivers/hwspinlock/hwspinlock_core.c index cc8e952a67727..a509b73da190d 100644 --- a/drivers/hwspinlock/hwspinlock_core.c +++ b/drivers/hwspinlock/hwspinlock_core.c @@ -472,7 +472,7 @@ static int hwspin_lock_register_single(struct hwspinlock *hwlock, int id) out: mutex_unlock(&hwspinlock_tree_lock); - return 0; + return ret; } static struct hwspinlock *hwspin_lock_unregister_single(unsigned int id) diff --git a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c index 762109307b869..b3c67e96a82a4 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c @@ -717,26 +717,19 @@ static DEVICE_ATTR_RW(cntr_rld_event); static ssize_t cntr_val_show(struct device *dev, struct device_attribute *attr, char *buf) { - int i, ret = 0; u32 val; struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etm_config *config = &drvdata->config; if (!coresight_get_mode(drvdata->csdev)) { spin_lock(&drvdata->spinlock); - for (i = 0; i < drvdata->nr_cntr; i++) - ret += sprintf(buf, "counter %d: %x\n", - i, config->cntr_val[i]); + val = config->cntr_val[config->cntr_idx]; spin_unlock(&drvdata->spinlock); - return ret; - } - - for (i = 0; i < drvdata->nr_cntr; i++) { - val = etm_readl(drvdata, ETMCNTVRn(i)); - ret += sprintf(buf, "counter %d: %x\n", i, val); + } else { + val = etm_readl(drvdata, ETMCNTVRn(config->cntr_idx)); } - return ret; + return sysfs_emit(buf, "%#x\n", val); } static ssize_t cntr_val_store(struct device *dev, diff --git a/drivers/hwtracing/coresight/coresight-etm4x-cfg.c b/drivers/hwtracing/coresight/coresight-etm4x-cfg.c index c302072b293a3..e1a59b4345052 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-cfg.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-cfg.c @@ -76,7 +76,7 @@ static int etm4_cfg_map_reg_offset(struct etmv4_drvdata *drvdata, } else if ((offset & GENMASK(11, 4)) == TRCSEQEVRn(0)) { /* sequencer state control registers */ idx = (offset & GENMASK(3, 0)) / 4; - if (idx < ETM_MAX_SEQ_STATES) { + if (idx < ETM_MAX_SEQ_TRANSITIONS) { reg_csdev->driver_regval = &drvcfg->seq_ctrl[idx]; err = 0; } diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 14bb31bd6a0b9..2247ad55d4442 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -93,7 +93,7 @@ static int etm4_probe_cpu(unsigned int cpu); static bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) { return (n < drvdata->nr_ss_cmp) && - drvdata->nr_pe && + drvdata->nr_pe_cmp && (drvdata->config.ss_status[n] & TRCSSCSRn_PC); } @@ -542,7 +542,8 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR); if (drvdata->nr_pe_cmp) etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR); - for (i = 0; i < drvdata->nrseqstate - 1; i++) + + for (i = 0; i < drvdata->nr_seq_ctrls; i++) etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i)); if (drvdata->nrseqstate) { etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); @@ -756,8 +757,7 @@ static int etm4_parse_event_config(struct coresight_device *csdev, .ATTR_CFG_FLD_timestamp_CFG = U64_MAX, }; struct perf_event_attr *attr = &event->attr; - unsigned long cfg_hash; - int preset, cc_threshold; + int cc_threshold; u8 ts_level; /* Clear configuration from previous run */ @@ -843,16 +843,6 @@ static int etm4_parse_event_config(struct coresight_device *csdev, /* bit[12], Return stack enable bit */ config->cfg |= TRCCONFIGR_RS; - /* - * Set any selected configuration and preset. A zero configid means no - * configuration active, preset = 0 means no preset selected. - */ - cfg_hash = ATTR_CFG_GET_FLD(attr, configid); - if (cfg_hash) { - preset = ATTR_CFG_GET_FLD(attr, preset); - ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); - } - /* branch broadcast - enable if selected and supported */ if (ATTR_CFG_GET_FLD(attr, branch_broadcast)) { if (!drvdata->trcbb) { @@ -876,7 +866,9 @@ static int etm4_enable_perf(struct coresight_device *csdev, struct coresight_path *path) { struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - int ret; + struct perf_event_attr *attr = &event->attr; + unsigned long cfg_hash; + int ret, preset; if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) return -EINVAL; @@ -887,7 +879,19 @@ static int etm4_enable_perf(struct coresight_device *csdev, /* Configure the tracer based on the session's specifics */ ret = etm4_parse_event_config(csdev, event); if (ret) - goto out; + goto err; + + /* + * Set any selected configuration and preset. A zero configid means no + * configuration active, preset = 0 means no preset selected. + */ + cfg_hash = ATTR_CFG_GET_FLD(attr, configid); + if (cfg_hash) { + preset = ATTR_CFG_GET_FLD(attr, preset); + ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); + if (ret) + goto err; + } drvdata->trcid = path->trace_id; @@ -896,16 +900,19 @@ static int etm4_enable_perf(struct coresight_device *csdev, /* And enable it */ ret = etm4_enable_hw(drvdata); - -out: - /* Failed to start tracer; roll back to DISABLED mode */ if (ret) { - coresight_set_mode(csdev, CS_MODE_DISABLED); - return ret; + if (cfg_hash) + cscfg_csdev_disable_active_config(csdev); + goto err; } csdev->path = path; return 0; + +err: + /* Failed to start tracer; roll back to DISABLED mode */ + coresight_set_mode(csdev, CS_MODE_DISABLED); + return ret; } static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_path *path) @@ -919,8 +926,10 @@ static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_pa cscfg_config_sysfs_get_active_cfg(&cfg_hash, &preset); if (cfg_hash) { ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); - if (ret) + if (ret) { + etm4_release_trace_id(drvdata); return ret; + } } raw_spin_lock(&drvdata->spinlock); @@ -1508,6 +1517,8 @@ static void etm4_init_arch_data(void *info) drvdata->lpoverride = (etmidr5 & TRCIDR5_LPOVERRIDE) && (!drvdata->skip_power_up); /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */ drvdata->nrseqstate = FIELD_GET(TRCIDR5_NUMSEQSTATE_MASK, etmidr5); + if (drvdata->nrseqstate) + drvdata->nr_seq_ctrls = ETM_MAX_SEQ_TRANSITIONS; /* NUMCNTR, bits[30:28] number of counters available for tracing */ drvdata->nr_cntr = FIELD_GET(TRCIDR5_NUMCNTR_MASK, etmidr5); @@ -1896,7 +1907,7 @@ static int etm4_cpu_save(struct coresight_device *csdev) if (drvdata->nr_pe_cmp) state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); - for (i = 0; i < drvdata->nrseqstate - 1; i++) + for (i = 0; i < drvdata->nr_seq_ctrls; i++) state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); if (drvdata->nrseqstate) { @@ -2009,7 +2020,7 @@ static void etm4_cpu_restore(struct coresight_device *csdev) if (drvdata->nr_pe_cmp) etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); - for (i = 0; i < drvdata->nrseqstate - 1; i++) + for (i = 0; i < drvdata->nr_seq_ctrls; i++) etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); if (drvdata->nrseqstate) { diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index e9eeea6240d55..cc6cdd3ae29d5 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -223,7 +223,7 @@ static ssize_t reset_store(struct device *dev, config->vipcssctlr = 0x0; /* Disable seq events */ - for (i = 0; i < drvdata->nrseqstate-1; i++) + for (i = 0; i < drvdata->nr_seq_ctrls; i++) config->seq_ctrl[i] = 0x0; config->seq_rst = 0x0; config->seq_state = 0x0; @@ -1395,9 +1395,11 @@ static ssize_t seq_idx_store(struct device *dev, struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etmv4_config *config = &drvdata->config; + if (!drvdata->nr_seq_ctrls) + return -ENOTSUPP; if (kstrtoul(buf, 16, &val)) return -EINVAL; - if (val >= drvdata->nrseqstate - 1) + if (val >= drvdata->nr_seq_ctrls) return -EINVAL; /* diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 89d81ce4e04e2..84db8b97c98af 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -614,6 +614,7 @@ static inline u32 etm4_res_sel_pair(u8 res_sel_idx) #define ETM_MAX_NR_PE 8 #define ETMv4_MAX_CNTR 4 #define ETM_MAX_SEQ_STATES 4 +#define ETM_MAX_SEQ_TRANSITIONS 3 #define ETM_MAX_EXT_INP_SEL 4 #define ETM_MAX_EXT_INP 256 #define ETM_MAX_EXT_OUT 4 @@ -877,7 +878,7 @@ struct etmv4_config { u32 vipcssctlr; u8 seq_idx; u8 syncfreq; - u32 seq_ctrl[ETM_MAX_SEQ_STATES]; + u32 seq_ctrl[ETM_MAX_SEQ_TRANSITIONS]; u32 seq_rst; u32 seq_state; u8 cntr_idx; @@ -928,7 +929,7 @@ struct etmv4_save_state { u32 trcvissctlr; u32 trcvipcssctlr; - u32 trcseqevr[ETM_MAX_SEQ_STATES]; + u32 trcseqevr[ETM_MAX_SEQ_TRANSITIONS]; u32 trcseqrstevr; u32 trcseqstr; u32 trcextinselr; @@ -981,6 +982,7 @@ struct etmv4_save_state { * @numcidc: Number of contextID comparators. * @numvmidc: Number of VMID comparators. * @nrseqstate: The number of sequencer states that are implemented. + * @nr_seq_ctrls: The number of sequence state transition control registers. * @nr_event: Indicates how many events the trace unit support. * @nr_resource:The number of resource selection pairs available for tracing. * @nr_ss_cmp: Number of single-shot comparator controls that are available. @@ -1046,6 +1048,7 @@ struct etmv4_drvdata { u8 numextinsel; u8 numvmidc; u8 nrseqstate; + u8 nr_seq_ctrls; u8 nr_event; u8 nr_resource; u8 nr_ss_cmp; diff --git a/drivers/hwtracing/ptt/hisi_ptt.c b/drivers/hwtracing/ptt/hisi_ptt.c index 94c371c491357..b5d851281fbf0 100644 --- a/drivers/hwtracing/ptt/hisi_ptt.c +++ b/drivers/hwtracing/ptt/hisi_ptt.c @@ -171,13 +171,13 @@ static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt *hisi_ptt) HISI_PTT_WAIT_TRACE_TIMEOUT_US); } -static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt) +static bool hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt) { u32 val; - readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS, - val, !val, HISI_PTT_RESET_POLL_INTERVAL_US, - HISI_PTT_RESET_TIMEOUT_US); + return !readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS, + val, !val, HISI_PTT_RESET_POLL_INTERVAL_US, + HISI_PTT_RESET_TIMEOUT_US); } static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt) @@ -202,14 +202,18 @@ static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt) return -EBUSY; } - ctrl->started = true; - /* Reset the DMA before start tracing */ val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); val |= HISI_PTT_TRACE_CTRL_RST; writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); - hisi_ptt_wait_dma_reset_done(hisi_ptt); + if (!hisi_ptt_wait_dma_reset_done(hisi_ptt)) { + pci_err(hisi_ptt->pdev, "timed out waiting for DMA reset\n"); + val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + val &= ~HISI_PTT_TRACE_CTRL_RST; + writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + return -ETIMEDOUT; + } val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); val &= ~HISI_PTT_TRACE_CTRL_RST; @@ -234,6 +238,8 @@ static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt) if (!hisi_ptt->trace_ctrl.is_port) val |= HISI_PTT_TRACE_CTRL_FILTER_MODE; + ctrl->started = true; + /* Start the Trace */ val |= HISI_PTT_TRACE_CTRL_EN; writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); diff --git a/drivers/i2c/busses/i2c-at91-core.c b/drivers/i2c/busses/i2c-at91-core.c index b64adef778d4b..8ca4556d96649 100644 --- a/drivers/i2c/busses/i2c-at91-core.c +++ b/drivers/i2c/busses/i2c-at91-core.c @@ -255,6 +255,7 @@ static int at91_twi_probe(struct platform_device *pdev) if (rc) { pm_runtime_disable(dev->dev); pm_runtime_set_suspended(dev->dev); + at91_twi_dma_release(dev); return rc; } @@ -270,6 +271,8 @@ static void at91_twi_remove(struct platform_device *pdev) i2c_del_adapter(&dev->adapter); + at91_twi_dma_release(dev); + pm_runtime_disable(dev->dev); pm_runtime_set_suspended(dev->dev); } diff --git a/drivers/i2c/busses/i2c-at91-master.c b/drivers/i2c/busses/i2c-at91-master.c index 894cedbca99f5..68238cc8aee0d 100644 --- a/drivers/i2c/busses/i2c-at91-master.c +++ b/drivers/i2c/busses/i2c-at91-master.c @@ -817,11 +817,21 @@ static int at91_twi_configure_dma(struct at91_twi_dev *dev, u32 phy_addr) error: if (ret != -EPROBE_DEFER) dev_info(dev->dev, "can't get DMA channel, continue without DMA support\n"); + at91_twi_dma_release(dev); + return ret; +} + +void at91_twi_dma_release(struct at91_twi_dev *dev) +{ + struct at91_twi_dma *dma = &dev->dma; + if (dma->chan_rx) dma_release_channel(dma->chan_rx); if (dma->chan_tx) dma_release_channel(dma->chan_tx); - return ret; + dma->chan_rx = NULL; + dma->chan_tx = NULL; + dev->use_dma = false; } static int at91_init_twi_recovery_gpio(struct platform_device *pdev, diff --git a/drivers/i2c/busses/i2c-at91.h b/drivers/i2c/busses/i2c-at91.h index 942e9c3973bb5..d68fcbbc3e0f5 100644 --- a/drivers/i2c/busses/i2c-at91.h +++ b/drivers/i2c/busses/i2c-at91.h @@ -172,6 +172,7 @@ void at91_twi_irq_restore(struct at91_twi_dev *dev); void at91_init_twi_bus(struct at91_twi_dev *dev); void at91_init_twi_bus_master(struct at91_twi_dev *dev); +void at91_twi_dma_release(struct at91_twi_dev *dev); int at91_twi_probe_master(struct platform_device *pdev, u32 phy_addr, struct at91_twi_dev *dev); diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c index e4dfa2ec58bb7..a1eca6cd4b75e 100644 --- a/drivers/i2c/busses/i2c-designware-common.c +++ b/drivers/i2c/busses/i2c-designware-common.c @@ -33,6 +33,8 @@ #include #include +#include + #include "i2c-designware-core.h" #define DW_IC_DEFAULT_BUS_CAPACITANCE_pF 100 diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index c71aa2dd368d5..2c929a6e8da2a 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -18,6 +18,8 @@ #include #include +#include + #define DW_IC_DEFAULT_FUNCTIONALITY (I2C_FUNC_I2C | \ I2C_FUNC_SMBUS_BYTE | \ I2C_FUNC_SMBUS_BYTE_DATA | \ @@ -25,23 +27,6 @@ I2C_FUNC_SMBUS_BLOCK_DATA | \ I2C_FUNC_SMBUS_I2C_BLOCK) -#define DW_IC_CON_MASTER BIT(0) -#define DW_IC_CON_SPEED_STD (1 << 1) -#define DW_IC_CON_SPEED_FAST (2 << 1) -#define DW_IC_CON_SPEED_HIGH (3 << 1) -#define DW_IC_CON_SPEED_MASK GENMASK(2, 1) -#define DW_IC_CON_10BITADDR_SLAVE BIT(3) -#define DW_IC_CON_10BITADDR_MASTER BIT(4) -#define DW_IC_CON_RESTART_EN BIT(5) -#define DW_IC_CON_SLAVE_DISABLE BIT(6) -#define DW_IC_CON_STOP_DET_IFADDRESSED BIT(7) -#define DW_IC_CON_TX_EMPTY_CTRL BIT(8) -#define DW_IC_CON_RX_FIFO_FULL_HLD_CTRL BIT(9) -#define DW_IC_CON_BUS_CLEAR_CTRL BIT(11) - -#define DW_IC_DATA_CMD_DAT GENMASK(7, 0) -#define DW_IC_DATA_CMD_FIRST_DATA_BYTE BIT(11) - /* * Register access parameters */ @@ -55,65 +40,9 @@ #define DW_IC_FIFO_RX_FIELD GENMASK(15, 8) #define DW_IC_FIFO_MIN_DEPTH 2 -/* - * Registers offset - */ -#define DW_IC_CON 0x00 -#define DW_IC_TAR 0x04 -#define DW_IC_SAR 0x08 -#define DW_IC_DATA_CMD 0x10 -#define DW_IC_SS_SCL_HCNT 0x14 -#define DW_IC_SS_SCL_LCNT 0x18 -#define DW_IC_FS_SCL_HCNT 0x1c -#define DW_IC_FS_SCL_LCNT 0x20 -#define DW_IC_HS_SCL_HCNT 0x24 -#define DW_IC_HS_SCL_LCNT 0x28 -#define DW_IC_INTR_STAT 0x2c -#define DW_IC_INTR_MASK 0x30 -#define DW_IC_RAW_INTR_STAT 0x34 -#define DW_IC_RX_TL 0x38 -#define DW_IC_TX_TL 0x3c -#define DW_IC_CLR_INTR 0x40 -#define DW_IC_CLR_RX_UNDER 0x44 -#define DW_IC_CLR_RX_OVER 0x48 -#define DW_IC_CLR_TX_OVER 0x4c -#define DW_IC_CLR_RD_REQ 0x50 -#define DW_IC_CLR_TX_ABRT 0x54 -#define DW_IC_CLR_RX_DONE 0x58 -#define DW_IC_CLR_ACTIVITY 0x5c -#define DW_IC_CLR_STOP_DET 0x60 -#define DW_IC_CLR_START_DET 0x64 -#define DW_IC_CLR_GEN_CALL 0x68 -#define DW_IC_ENABLE 0x6c -#define DW_IC_STATUS 0x70 -#define DW_IC_TXFLR 0x74 -#define DW_IC_RXFLR 0x78 -#define DW_IC_SDA_HOLD 0x7c -#define DW_IC_TX_ABRT_SOURCE 0x80 -#define DW_IC_ENABLE_STATUS 0x9c -#define DW_IC_CLR_RESTART_DET 0xa8 -#define DW_IC_SMBUS_INTR_MASK 0xcc -#define DW_IC_COMP_PARAM_1 0xf4 -#define DW_IC_COMP_VERSION 0xf8 #define DW_IC_SDA_HOLD_MIN_VERS 0x3131312A /* "111*" == v1.11* */ -#define DW_IC_COMP_TYPE 0xfc #define DW_IC_COMP_TYPE_VALUE 0x44570140 /* "DW" + 0x0140 */ -#define DW_IC_INTR_RX_UNDER BIT(0) -#define DW_IC_INTR_RX_OVER BIT(1) -#define DW_IC_INTR_RX_FULL BIT(2) -#define DW_IC_INTR_TX_OVER BIT(3) -#define DW_IC_INTR_TX_EMPTY BIT(4) -#define DW_IC_INTR_RD_REQ BIT(5) -#define DW_IC_INTR_TX_ABRT BIT(6) -#define DW_IC_INTR_RX_DONE BIT(7) -#define DW_IC_INTR_ACTIVITY BIT(8) -#define DW_IC_INTR_STOP_DET BIT(9) -#define DW_IC_INTR_START_DET BIT(10) -#define DW_IC_INTR_GEN_CALL BIT(11) -#define DW_IC_INTR_RESTART_DET BIT(12) -#define DW_IC_INTR_MST_ON_HOLD BIT(13) - #define DW_IC_INTR_DEFAULT_MASK (DW_IC_INTR_RX_FULL | \ DW_IC_INTR_TX_ABRT | \ DW_IC_INTR_STOP_DET) @@ -123,16 +52,6 @@ DW_IC_INTR_RX_UNDER | \ DW_IC_INTR_RD_REQ) -#define DW_IC_ENABLE_ENABLE BIT(0) -#define DW_IC_ENABLE_ABORT BIT(1) - -#define DW_IC_STATUS_ACTIVITY BIT(0) -#define DW_IC_STATUS_TFE BIT(2) -#define DW_IC_STATUS_RFNE BIT(3) -#define DW_IC_STATUS_MASTER_ACTIVITY BIT(5) -#define DW_IC_STATUS_SLAVE_ACTIVITY BIT(6) -#define DW_IC_STATUS_MASTER_HOLD_TX_FIFO_EMPTY BIT(7) - #define DW_IC_SDA_HOLD_RX_SHIFT 16 #define DW_IC_SDA_HOLD_RX_MASK GENMASK(23, 16) diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c index 7a301c8b604ef..a1bcc3797e4ff 100644 --- a/drivers/i2c/busses/i2c-designware-master.c +++ b/drivers/i2c/busses/i2c-designware-master.c @@ -25,6 +25,8 @@ #include #include +#include + #include "i2c-designware-core.h" #define AMD_TIMEOUT_MIN_US 25 diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index 6d6e81242f74e..447af5523c2e4 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -272,7 +272,7 @@ static const struct acpi_device_id dw_i2c_acpi_match[] = { { "HISI02A1", 0 }, { "HISI02A2", 0 }, { "HISI02A3", 0 }, - { "HJMC3001", 0 }, + { "HJMC3001", ACCESS_INTR_MASK }, { "HYGO0010", ACCESS_INTR_MASK }, { "INT33C2", 0 }, { "INT33C3", 0 }, diff --git a/drivers/i2c/busses/i2c-designware-slave.c b/drivers/i2c/busses/i2c-designware-slave.c index ad0d5fbfa6d5e..0abcc7757b231 100644 --- a/drivers/i2c/busses/i2c-designware-slave.c +++ b/drivers/i2c/busses/i2c-designware-slave.c @@ -19,6 +19,8 @@ #include #include +#include + #include "i2c-designware-core.h" int i2c_dw_reg_slave(struct i2c_client *slave) diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index 9477d814fde94..cccff6e63db2a 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -1880,6 +1880,8 @@ static int i2c_imx_probe(struct platform_device *pdev) clk_notifier_unregister: clk_notifier_unregister(i2c_imx->clk, &i2c_imx->clk_change_nb); + if (i2c_imx->dma) + i2c_imx_dma_free(i2c_imx); free_irq(irq, i2c_imx); rpm_disable: pm_runtime_put_noidle(&pdev->dev); @@ -1920,6 +1922,7 @@ static void i2c_imx_remove(struct platform_device *pdev) pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); } static int i2c_imx_runtime_suspend(struct device *dev) diff --git a/drivers/i2c/busses/i2c-mxs.c b/drivers/i2c/busses/i2c-mxs.c index 4e07babea9c3f..eee4fdcd9df31 100644 --- a/drivers/i2c/busses/i2c-mxs.c +++ b/drivers/i2c/busses/i2c-mxs.c @@ -839,7 +839,7 @@ static int mxs_i2c_probe(struct platform_device *pdev) } /* Setup the DMA */ - i2c->dmach = dma_request_chan(dev, "rx-tx"); + i2c->dmach = devm_dma_request_chan(dev, "rx-tx"); if (IS_ERR(i2c->dmach)) { return dev_err_probe(dev, PTR_ERR(i2c->dmach), "Failed to request dma\n"); @@ -877,9 +877,6 @@ static void mxs_i2c_remove(struct platform_device *pdev) i2c_del_adapter(&i2c->adapter); - if (i2c->dmach) - dma_release_channel(i2c->dmach); - writel(MXS_I2C_CTRL0_SFTRST, i2c->regs + MXS_I2C_CTRL0_SET); } diff --git a/drivers/i2c/busses/i2c-ocores.c b/drivers/i2c/busses/i2c-ocores.c index df6ebf32d6e8f..2d18c10358375 100644 --- a/drivers/i2c/busses/i2c-ocores.c +++ b/drivers/i2c/busses/i2c-ocores.c @@ -755,7 +755,11 @@ static int ocores_i2c_resume(struct device *dev) rate = clk_get_rate(i2c->clk) / 1000; if (rate) i2c->ip_clock_khz = rate; - return ocores_init(dev, i2c); + ret = ocores_init(dev, i2c); + if (ret) + clk_disable_unprepare(i2c->clk); + + return ret; } static DEFINE_NOIRQ_DEV_PM_OPS(ocores_i2c_pm, diff --git a/drivers/i2c/busses/i2c-qcom-cci.c b/drivers/i2c/busses/i2c-qcom-cci.c index bdeda3979c481..873e901a23d7a 100644 --- a/drivers/i2c/busses/i2c-qcom-cci.c +++ b/drivers/i2c/busses/i2c-qcom-cci.c @@ -497,10 +497,14 @@ static const struct dev_pm_ops qcom_cci_pm = { SET_RUNTIME_PM_OPS(cci_suspend_runtime, cci_resume_runtime, NULL) }; +static void cci_put_of_node(void *data) +{ + of_node_put(data); +} + static int cci_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *child; struct resource *r; struct cci *cci; int ret, i; @@ -516,7 +520,7 @@ static int cci_probe(struct platform_device *pdev) if (!cci->data) return -ENOENT; - for_each_available_child_of_node(dev->of_node, child) { + for_each_available_child_of_node_scoped(dev->of_node, child) { struct cci_master *master; u32 idx; @@ -537,6 +541,9 @@ static int cci_probe(struct platform_device *pdev) master->adap.algo = &cci_algo; master->adap.dev.parent = dev; master->adap.dev.of_node = of_node_get(child); + ret = devm_add_action_or_reset(dev, cci_put_of_node, child); + if (ret) + return ret; master->master = idx; master->cci = cci; @@ -595,32 +602,28 @@ static int cci_probe(struct platform_device *pdev) goto disable_clocks; pm_runtime_set_autosuspend_delay(dev, MSEC_PER_SEC); + ret = devm_pm_runtime_set_active_enabled(dev); + if (ret) + goto disable_clocks; + pm_runtime_use_autosuspend(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); for (i = 0; i < cci->data->num_masters; i++) { if (!cci->master[i].cci) continue; ret = i2c_add_adapter(&cci->master[i].adap); - if (ret < 0) { - of_node_put(cci->master[i].adap.dev.of_node); + if (ret < 0) goto error_i2c; - } } return 0; error_i2c: - pm_runtime_disable(dev); - pm_runtime_dont_use_autosuspend(dev); for (--i ; i >= 0; i--) { - if (cci->master[i].cci) { + if (cci->master[i].cci) i2c_del_adapter(&cci->master[i].adap); - of_node_put(cci->master[i].adap.dev.of_node); - } } disable_clocks: cci_disable_clocks(cci); @@ -636,13 +639,9 @@ static void cci_remove(struct platform_device *pdev) for (i = 0; i < cci->data->num_masters; i++) { if (cci->master[i].cci) { i2c_del_adapter(&cci->master[i].adap); - of_node_put(cci->master[i].adap.dev.of_node); cci_halt(cci, i); } } - - pm_runtime_disable(&pdev->dev); - pm_runtime_set_suspended(&pdev->dev); } static const struct cci_data cci_v1_data = { diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index d2f5055b0b108..de5bc4b8a3a52 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -77,6 +77,14 @@ enum geni_i2c_err_code { #define XFER_TIMEOUT HZ #define RST_TIMEOUT HZ +#define GENI_SE_CLK_32MHZ (32 * HZ_PER_MHZ) +#define GENI_SE_CLK_19P2MHZ 19200000UL + +struct geni_i2c_desc { + bool no_dma_support; + unsigned int tx_fifo_depth; +}; + #define QCOM_I2C_MIN_NUM_OF_MSGS_MULTI_DESC 2 /** @@ -107,9 +115,9 @@ struct geni_i2c_dev { int cur_wr; int cur_rd; spinlock_t lock; - struct clk *core_clk; u32 clk_freq_out; const struct geni_i2c_clk_fld *clk_fld; + u32 clk_idx; void *dma_buf; size_t xfer_len; dma_addr_t dma_addr; @@ -121,13 +129,7 @@ struct geni_i2c_dev { bool is_tx_multi_desc_xfer; u32 num_msgs; struct geni_i2c_gpi_multi_desc_xfer i2c_multi_desc_config; -}; - -struct geni_i2c_desc { - bool has_core_clk; - char *icc_ddr; - bool no_dma_support; - unsigned int tx_fifo_depth; + const struct geni_i2c_desc *dev_data; }; struct geni_i2c_err_log { @@ -177,8 +179,8 @@ static const struct geni_i2c_clk_fld geni_i2c_clk_map_19p2mhz[] = { /* source_clock = 32 MHz */ static const struct geni_i2c_clk_fld geni_i2c_clk_map_32mhz[] = { - { I2C_MAX_STANDARD_MODE_FREQ, 8, 14, 18, 38 }, - { I2C_MAX_FAST_MODE_FREQ, 4, 3, 9, 19 }, + { I2C_MAX_STANDARD_MODE_FREQ, 12, 9, 10, 26 }, + { I2C_MAX_FAST_MODE_FREQ, 4, 3, 9, 19 }, { I2C_MAX_FAST_MODE_PLUS_FREQ, 2, 3, 5, 15 }, {} }; @@ -186,19 +188,44 @@ static const struct geni_i2c_clk_fld geni_i2c_clk_map_32mhz[] = { static int geni_i2c_clk_map_idx(struct geni_i2c_dev *gi2c) { const struct geni_i2c_clk_fld *itr; + unsigned long res_freq; - if (clk_get_rate(gi2c->se.clk) == 32 * HZ_PER_MHZ) + /* + * Frequency counter tables are calibrated for a specific source + * clock frequency and are not valid for any multiple of it + * (e.g. 64 MHz, 128 MHz). + * Use exact=true and verify res_freq matches req_freq literally + * to reject harmonics: a 64 MHz clock that divides evenly to + * 32 MHz would pass exact matching but produce double the intended + * I2C frequency with these counter values. + */ + if (!geni_se_clk_freq_match(&gi2c->se, GENI_SE_CLK_32MHZ, + &gi2c->clk_idx, &res_freq, true) && + res_freq == GENI_SE_CLK_32MHZ) { itr = geni_i2c_clk_map_32mhz; - else + } else if (!geni_se_clk_freq_match(&gi2c->se, GENI_SE_CLK_19P2MHZ, + &gi2c->clk_idx, &res_freq, true) && + res_freq == GENI_SE_CLK_19P2MHZ) { itr = geni_i2c_clk_map_19p2mhz; + } else { + dev_err(gi2c->se.dev, + "Unsupported SE source clock: must be exactly 32 MHz or 19.2 MHz\n"); + return -EINVAL; + } while (itr->clk_freq_out != 0) { if (itr->clk_freq_out == gi2c->clk_freq_out) { gi2c->clk_fld = itr; + dev_dbg(gi2c->se.dev, + "I2C clk selected: freq: %u Hz, clk_idx: %u\n", + gi2c->clk_freq_out, gi2c->clk_idx); return 0; } itr++; } + + dev_err(gi2c->se.dev, "Unsupported I2C output frequency %u Hz\n", gi2c->clk_freq_out); + return -EINVAL; } @@ -207,7 +234,7 @@ static void qcom_geni_i2c_conf(struct geni_i2c_dev *gi2c) const struct geni_i2c_clk_fld *itr = gi2c->clk_fld; u32 val; - writel_relaxed(0, gi2c->se.base + SE_GENI_CLK_SEL); + writel_relaxed(gi2c->clk_idx, gi2c->se.base + SE_GENI_CLK_SEL); val = (itr->clk_div << CLK_DIV_SHFT) | SER_CLK_EN; writel_relaxed(val, gi2c->se.base + GENI_SER_M_CLK_CFG); @@ -944,15 +971,6 @@ static const struct i2c_algorithm geni_i2c_algo = { .functionality = geni_i2c_func, }; -#ifdef CONFIG_ACPI -static const struct acpi_device_id geni_i2c_acpi_match[] = { - { "QCOM0220"}, - { "QCOM0411" }, - { } -}; -MODULE_DEVICE_TABLE(acpi, geni_i2c_acpi_match); -#endif - static void release_gpi_dma(struct geni_i2c_dev *gi2c) { if (gi2c->rx_c) @@ -990,13 +1008,93 @@ static int setup_gpi_dma(struct geni_i2c_dev *gi2c) return ret; } +static int geni_i2c_init(struct geni_i2c_dev *gi2c) +{ + u32 proto, tx_depth; + bool fifo_disable; + int ret; + + ret = pm_runtime_resume_and_get(gi2c->se.dev); + if (ret < 0) { + dev_err(gi2c->se.dev, "error turning on device :%d\n", ret); + return ret; + } + + proto = geni_se_read_proto(&gi2c->se); + if (proto == GENI_SE_INVALID_PROTO) { + ret = geni_load_se_firmware(&gi2c->se, GENI_SE_I2C); + if (ret) { + dev_err_probe(gi2c->se.dev, ret, "i2c firmware load failed ret: %d\n", ret); + goto err; + } + } else if (proto != GENI_SE_I2C) { + ret = dev_err_probe(gi2c->se.dev, -ENXIO, "Invalid proto %d\n", proto); + goto err; + } + + if (gi2c->dev_data->no_dma_support) { + fifo_disable = false; + gi2c->no_dma = true; + } else { + fifo_disable = readl_relaxed(gi2c->se.base + GENI_IF_DISABLE_RO) & FIFO_IF_DISABLE; + } + + if (fifo_disable) { + /* FIFO is disabled, so we can only use GPI DMA */ + gi2c->gpi_mode = true; + ret = setup_gpi_dma(gi2c); + if (ret) + goto err; + + dev_dbg(gi2c->se.dev, "Using GPI DMA mode for I2C\n"); + } else { + gi2c->gpi_mode = false; + tx_depth = geni_se_get_tx_fifo_depth(&gi2c->se); + + /* I2C Master Hub Serial Elements doesn't have the HW_PARAM_0 register */ + if (!tx_depth && gi2c->se.core_clk) + tx_depth = gi2c->dev_data->tx_fifo_depth; + + if (!tx_depth) { + ret = dev_err_probe(gi2c->se.dev, -EINVAL, + "Invalid TX FIFO depth\n"); + goto err; + } + + gi2c->tx_wm = tx_depth - 1; + geni_se_init(&gi2c->se, gi2c->tx_wm, tx_depth); + geni_se_config_packing(&gi2c->se, BITS_PER_BYTE, + PACKING_BYTES_PW, true, true, true); + + dev_dbg(gi2c->se.dev, "i2c fifo/se-dma mode. fifo depth:%d\n", tx_depth); + } + +err: + pm_runtime_put(gi2c->se.dev); + return ret; +} + +static int geni_i2c_resources_init(struct geni_i2c_dev *gi2c) +{ + int ret; + + ret = geni_se_resources_init(&gi2c->se); + if (ret) + return ret; + + ret = geni_i2c_clk_map_idx(gi2c); + if (ret) + return ret; + + return geni_icc_set_bw_ab(&gi2c->se, GENI_DEFAULT_BW, GENI_DEFAULT_BW, + Bps_to_icc(gi2c->clk_freq_out)); +} + static int geni_i2c_probe(struct platform_device *pdev) { struct geni_i2c_dev *gi2c; - u32 proto, tx_depth, fifo_disable; int ret; struct device *dev = &pdev->dev; - const struct geni_i2c_desc *desc = NULL; gi2c = devm_kzalloc(dev, sizeof(*gi2c), GFP_KERNEL); if (!gi2c) @@ -1008,17 +1106,9 @@ static int geni_i2c_probe(struct platform_device *pdev) if (IS_ERR(gi2c->se.base)) return PTR_ERR(gi2c->se.base); - desc = device_get_match_data(&pdev->dev); - - if (desc && desc->has_core_clk) { - gi2c->core_clk = devm_clk_get(dev, "core"); - if (IS_ERR(gi2c->core_clk)) - return PTR_ERR(gi2c->core_clk); - } - - gi2c->se.clk = devm_clk_get(dev, "se"); - if (IS_ERR(gi2c->se.clk) && !has_acpi_companion(dev)) - return PTR_ERR(gi2c->se.clk); + gi2c->dev_data = device_get_match_data(&pdev->dev); + if (!gi2c->dev_data) + return -EINVAL; ret = device_property_read_u32(dev, "clock-frequency", &gi2c->clk_freq_out); @@ -1034,16 +1124,15 @@ static int geni_i2c_probe(struct platform_device *pdev) if (gi2c->irq < 0) return gi2c->irq; - ret = geni_i2c_clk_map_idx(gi2c); - if (ret) - return dev_err_probe(dev, ret, "Invalid clk frequency %d Hz\n", - gi2c->clk_freq_out); - gi2c->adap.algo = &geni_i2c_algo; init_completion(&gi2c->done); spin_lock_init(&gi2c->lock); platform_set_drvdata(pdev, gi2c); + ret = geni_i2c_resources_init(gi2c); + if (ret) + return ret; + /* Keep interrupts disabled initially to allow for low-power modes */ ret = devm_request_irq(dev, gi2c->irq, geni_i2c_irq, IRQF_NO_AUTOEN, dev_name(dev), gi2c); @@ -1056,118 +1145,27 @@ static int geni_i2c_probe(struct platform_device *pdev) gi2c->adap.dev.of_node = dev->of_node; strscpy(gi2c->adap.name, "Geni-I2C", sizeof(gi2c->adap.name)); - ret = geni_icc_get(&gi2c->se, desc ? desc->icc_ddr : "qup-memory"); - if (ret) - return ret; - /* - * Set the bus quota for core and cpu to a reasonable value for - * register access. - * Set quota for DDR based on bus speed. - */ - gi2c->se.icc_paths[GENI_TO_CORE].avg_bw = GENI_DEFAULT_BW; - gi2c->se.icc_paths[CPU_TO_GENI].avg_bw = GENI_DEFAULT_BW; - if (!desc || desc->icc_ddr) - gi2c->se.icc_paths[GENI_TO_DDR].avg_bw = Bps_to_icc(gi2c->clk_freq_out); - - ret = geni_icc_set_bw(&gi2c->se); - if (ret) - return ret; - - ret = clk_prepare_enable(gi2c->core_clk); - if (ret) - return ret; - - ret = geni_se_resources_on(&gi2c->se); - if (ret) { - dev_err_probe(dev, ret, "Error turning on resources\n"); - goto err_clk; - } - proto = geni_se_read_proto(&gi2c->se); - if (proto == GENI_SE_INVALID_PROTO) { - ret = geni_load_se_firmware(&gi2c->se, GENI_SE_I2C); - if (ret) { - dev_err_probe(dev, ret, "i2c firmware load failed ret: %d\n", ret); - goto err_resources; - } - } else if (proto != GENI_SE_I2C) { - ret = dev_err_probe(dev, -ENXIO, "Invalid proto %d\n", proto); - goto err_resources; - } - - if (desc && desc->no_dma_support) { - fifo_disable = false; - gi2c->no_dma = true; - } else { - fifo_disable = readl_relaxed(gi2c->se.base + GENI_IF_DISABLE_RO) & FIFO_IF_DISABLE; - } - - if (fifo_disable) { - /* FIFO is disabled, so we can only use GPI DMA */ - gi2c->gpi_mode = true; - ret = setup_gpi_dma(gi2c); - if (ret) - goto err_resources; - - dev_dbg(dev, "Using GPI DMA mode for I2C\n"); - } else { - gi2c->gpi_mode = false; - tx_depth = geni_se_get_tx_fifo_depth(&gi2c->se); - - /* I2C Master Hub Serial Elements doesn't have the HW_PARAM_0 register */ - if (!tx_depth && desc) - tx_depth = desc->tx_fifo_depth; - - if (!tx_depth) { - ret = dev_err_probe(dev, -EINVAL, - "Invalid TX FIFO depth\n"); - goto err_resources; - } - - gi2c->tx_wm = tx_depth - 1; - geni_se_init(&gi2c->se, gi2c->tx_wm, tx_depth); - geni_se_config_packing(&gi2c->se, BITS_PER_BYTE, - PACKING_BYTES_PW, true, true, true); - - dev_dbg(dev, "i2c fifo/se-dma mode. fifo depth:%d\n", tx_depth); - } - - clk_disable_unprepare(gi2c->core_clk); - ret = geni_se_resources_off(&gi2c->se); - if (ret) { - dev_err_probe(dev, ret, "Error turning off resources\n"); - goto err_dma; - } - - ret = geni_icc_disable(&gi2c->se); - if (ret) - goto err_dma; - pm_runtime_set_suspended(gi2c->se.dev); pm_runtime_set_autosuspend_delay(gi2c->se.dev, I2C_AUTO_SUSPEND_DELAY); pm_runtime_use_autosuspend(gi2c->se.dev); pm_runtime_enable(gi2c->se.dev); + ret = geni_i2c_init(gi2c); + if (ret < 0) { + pm_runtime_disable(gi2c->se.dev); + return ret; + } + ret = i2c_add_adapter(&gi2c->adap); if (ret) { + release_gpi_dma(gi2c); dev_err_probe(dev, ret, "Error adding i2c adapter\n"); pm_runtime_disable(gi2c->se.dev); - goto err_dma; + return ret; } dev_dbg(dev, "Geni-I2C adaptor successfully added\n"); - return ret; - -err_resources: - geni_se_resources_off(&gi2c->se); -err_clk: - clk_disable_unprepare(gi2c->core_clk); - - return ret; - -err_dma: - release_gpi_dma(gi2c); - return ret; } @@ -1200,7 +1198,7 @@ static int __maybe_unused geni_i2c_runtime_suspend(struct device *dev) return ret; } - clk_disable_unprepare(gi2c->core_clk); + clk_disable_unprepare(gi2c->se.core_clk); return geni_icc_disable(&gi2c->se); } @@ -1214,7 +1212,7 @@ static int __maybe_unused geni_i2c_runtime_resume(struct device *dev) if (ret) return ret; - ret = clk_prepare_enable(gi2c->core_clk); + ret = clk_prepare_enable(gi2c->se.core_clk); if (ret) goto out_icc_disable; @@ -1227,7 +1225,7 @@ static int __maybe_unused geni_i2c_runtime_resume(struct device *dev) return 0; out_clk_disable: - clk_disable_unprepare(gi2c->core_clk); + clk_disable_unprepare(gi2c->se.core_clk); out_icc_disable: geni_icc_disable(&gi2c->se); @@ -1267,15 +1265,24 @@ static const struct dev_pm_ops geni_i2c_pm_ops = { NULL) }; +static const struct geni_i2c_desc geni_i2c = {}; + static const struct geni_i2c_desc i2c_master_hub = { - .has_core_clk = true, - .icc_ddr = NULL, .no_dma_support = true, .tx_fifo_depth = 16, }; +#ifdef CONFIG_ACPI +static const struct acpi_device_id geni_i2c_acpi_match[] = { + { "QCOM0220", (kernel_ulong_t)&geni_i2c}, + { "QCOM0411", (kernel_ulong_t)&geni_i2c}, + { } +}; +MODULE_DEVICE_TABLE(acpi, geni_i2c_acpi_match); +#endif + static const struct of_device_id geni_i2c_dt_match[] = { - { .compatible = "qcom,geni-i2c" }, + { .compatible = "qcom,geni-i2c", .data = &geni_i2c }, { .compatible = "qcom,geni-i2c-master-hub", .data = &i2c_master_hub }, {} }; diff --git a/drivers/i2c/i2c-atr.c b/drivers/i2c/i2c-atr.c index e6d2af659d810..ca29633dcd623 100644 --- a/drivers/i2c/i2c-atr.c +++ b/drivers/i2c/i2c-atr.c @@ -855,6 +855,7 @@ int i2c_atr_add_adapter(struct i2c_atr *atr, struct i2c_atr_adap_desc *desc) ret = i2c_add_adapter(&chan->adap); if (ret) { + atr->adapter[chan_id] = NULL; dev_err(dev, "failed to add atr-adapter %u (error=%d)\n", chan_id, ret); goto err_free_alias_pool; diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index 3ec04787a7373..17cf3d225268f 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -1815,6 +1815,12 @@ void i2c_del_adapter(struct i2c_adapter *adap) return; } + /* + * This drains any in-flight writers, so all + * clients will be caught by i2c_deregister_clients(). + */ + device_remove_file(&adap->dev, &dev_attr_new_device); + i2c_acpi_remove_space_handler(adap); i2c_deregister_clients(adap); diff --git a/drivers/i2c/i2c-mux.c b/drivers/i2c/i2c-mux.c index 681a201c239ba..68a4c34b5987d 100644 --- a/drivers/i2c/i2c-mux.c +++ b/drivers/i2c/i2c-mux.c @@ -408,6 +408,7 @@ int i2c_mux_add_adapter(struct i2c_mux_core *muxc, return 0; err_free_priv: + of_node_put(priv->adap.dev.of_node); kfree(priv); return ret; } diff --git a/drivers/i2c/muxes/i2c-demux-pinctrl.c b/drivers/i2c/muxes/i2c-demux-pinctrl.c index f2a1f47449789..2403c0bf7c431 100644 --- a/drivers/i2c/muxes/i2c-demux-pinctrl.c +++ b/drivers/i2c/muxes/i2c-demux-pinctrl.c @@ -247,6 +247,7 @@ static int i2c_demux_pinctrl_probe(struct platform_device *pdev) props[i].value = devm_kstrdup(&pdev->dev, "ok", GFP_KERNEL); if (!props[i].name || !props[i].value) { err = -ENOMEM; + of_node_put(adap_np); goto err_rollback; } props[i].length = 3; diff --git a/drivers/i3c/device.c b/drivers/i3c/device.c index 101eaa77de686..a3778282e84c2 100644 --- a/drivers/i3c/device.c +++ b/drivers/i3c/device.c @@ -309,7 +309,7 @@ EXPORT_SYMBOL_GPL(i3c_device_match_id); */ u32 i3c_device_get_supported_xfer_mode(struct i3c_device *dev) { - return i3c_dev_get_master(dev->desc)->this->info.hdr_cap | BIT(I3C_SDR); + return i3c_bus_to_i3c_master(dev->bus)->this->info.hdr_cap | BIT(I3C_SDR); } EXPORT_SYMBOL_GPL(i3c_device_get_supported_xfer_mode); diff --git a/drivers/i3c/internals.h b/drivers/i3c/internals.h index 0f1f3f7666237..86a36b951e0d5 100644 --- a/drivers/i3c/internals.h +++ b/drivers/i3c/internals.h @@ -72,4 +72,9 @@ static inline void i3c_readl_fifo(const void __iomem *addr, void *buf, } } +static inline struct i3c_master_controller *i3c_bus_to_i3c_master(struct i3c_bus *i3cbus) +{ + return container_of(i3cbus, struct i3c_master_controller, bus); +} + #endif /* I3C_INTERNAL_H */ diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index f1be38a640ca1..62b7a15629103 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -97,12 +97,6 @@ void i3c_bus_normaluse_unlock(struct i3c_bus *bus) up_read(&bus->lock); } -static struct i3c_master_controller * -i3c_bus_to_i3c_master(struct i3c_bus *i3cbus) -{ - return container_of(i3cbus, struct i3c_master_controller, bus); -} - static struct i3c_master_controller *dev_to_i3cmaster(struct device *dev) { return container_of(dev, struct i3c_master_controller, dev); @@ -826,6 +820,11 @@ static struct attribute *i3c_masterdev_attrs[] = { }; ATTRIBUTE_GROUPS(i3c_masterdev); +static void i3c_master_free_i3c_dev(struct i3c_dev_desc *dev) +{ + kfree(dev); +} + static void i3c_masterdev_release(struct device *dev) { struct i3c_master_controller *master = dev_to_i3cmaster(dev); @@ -838,6 +837,8 @@ static void i3c_masterdev_release(struct device *dev) i3c_bus_cleanup(bus); of_node_put(dev->of_node); + + i3c_master_free_i3c_dev(master->this); } static const struct device_type i3c_masterdev_type = { @@ -1048,11 +1049,6 @@ static void i3c_device_release(struct device *dev) kfree(i3cdev); } -static void i3c_master_free_i3c_dev(struct i3c_dev_desc *dev) -{ - kfree(dev); -} - static struct i3c_dev_desc * i3c_master_alloc_i3c_dev(struct i3c_master_controller *master, const struct i3c_device_info *info) @@ -1904,12 +1900,21 @@ static int i3c_master_early_i3c_dev_add(struct i3c_master_controller *master, static void i3c_master_register_new_i3c_devs(struct i3c_master_controller *master) { + struct i3c_device *i3cdev, *tmp; struct i3c_dev_desc *desc; + LIST_HEAD(i3c_unreg_devs); int ret; if (!master->init_done) return; + i3c_bus_maintenance_lock(&master->bus); + + if (master->shutting_down) { + i3c_bus_maintenance_unlock(&master->bus); + return; + } + i3c_bus_for_each_i3cdev(&master->bus, desc) { if (desc->dev || !desc->info.dyn_addr || desc == master->this) continue; @@ -1930,23 +1935,37 @@ i3c_master_register_new_i3c_devs(struct i3c_master_controller *master) if (desc->boardinfo) desc->dev->dev.of_node = desc->boardinfo->of_node; - ret = device_register(&desc->dev->dev); - if (ret) { - dev_err(&master->dev, - "Failed to add I3C device (err = %d)\n", ret); - put_device(&desc->dev->dev); - } + list_add_tail(&desc->dev->node, &i3c_unreg_devs); } + + i3c_bus_maintenance_unlock(&master->bus); + + list_for_each_entry_safe(i3cdev, tmp, &i3c_unreg_devs, node) { + ret = device_register(&i3cdev->dev); + if (ret) + dev_err(&master->dev, "Failed to add I3C device (err = %d)\n", ret); + else + list_del_init(&i3cdev->node); + } + + i3c_bus_maintenance_lock(&master->bus); + + list_for_each_entry_safe(i3cdev, tmp, &i3c_unreg_devs, node) { + list_del(&i3cdev->node); + desc = i3cdev->desc; + i3cdev->desc = NULL; + put_device(&i3cdev->dev); + desc->dev = NULL; + } + + i3c_bus_maintenance_unlock(&master->bus); } static void i3c_master_reg_work_fn(struct work_struct *work) { struct i3c_master_controller *master = container_of(work, typeof(*master), reg_work); - i3c_bus_normaluse_lock(&master->bus); - if (!master->shutting_down) - i3c_master_register_new_i3c_devs(master); - i3c_bus_normaluse_unlock(&master->bus); + i3c_master_register_new_i3c_devs(master); } /** @@ -2075,6 +2094,8 @@ int i3c_master_set_info(struct i3c_master_controller *master, return 0; err_free_dev: + master->bus.cur_master = NULL; + master->this = NULL; i3c_master_free_i3c_dev(i3cdev); return ret; @@ -2095,7 +2116,8 @@ static void i3c_master_detach_free_devs(struct i3c_master_controller *master) i3cdev->boardinfo->init_dyn_addr, I3C_ADDR_SLOT_FREE); - i3c_master_free_i3c_dev(i3cdev); + if (i3cdev != master->this) + i3c_master_free_i3c_dev(i3cdev); } list_for_each_entry_safe(i2cdev, i2ctmp, &master->bus.devs.i2c, @@ -2327,7 +2349,8 @@ i3c_master_search_i3c_dev_duplicate(struct i3c_dev_desc *refdev) struct i3c_dev_desc *i3cdev; i3c_bus_for_each_i3cdev(&master->bus, i3cdev) { - if (i3cdev != refdev && i3cdev->info.pid == refdev->info.pid) + if (i3cdev != refdev && i3cdev->info.pid == refdev->info.pid && + i3cdev != master->this) return i3cdev; } @@ -2977,11 +3000,12 @@ static void i3c_master_unregister_i3c_devs(struct i3c_master_controller *master) if (!i3cdev->dev) continue; - i3cdev->dev->desc = NULL; - if (device_is_registered(&i3cdev->dev->dev)) + if (device_is_registered(&i3cdev->dev->dev)) { + get_device(&i3cdev->dev->dev); device_unregister(&i3cdev->dev->dev); - else - put_device(&i3cdev->dev->dev); + } + i3cdev->dev->desc = NULL; + put_device(&i3cdev->dev->dev); i3cdev->dev = NULL; } } diff --git a/drivers/i3c/master/adi-i3c-master.c b/drivers/i3c/master/adi-i3c-master.c index 047081c9f0643..fb4cfc9026ccf 100644 --- a/drivers/i3c/master/adi-i3c-master.c +++ b/drivers/i3c/master/adi-i3c-master.c @@ -929,6 +929,7 @@ static const struct of_device_id adi_i3c_master_of_match[] = { { .compatible = "adi,i3c-master-v1" }, {} }; +MODULE_DEVICE_TABLE(of, adi_i3c_master_of_match); static int adi_i3c_master_probe(struct platform_device *pdev) { @@ -964,17 +965,9 @@ static int adi_i3c_master_probe(struct platform_device *pdev) writel(0x00, master->regs + REG_ENABLE); writel(0x00, master->regs + REG_IRQ_MASK); - ret = devm_request_irq(&pdev->dev, irq, adi_i3c_master_irq, 0, - dev_name(&pdev->dev), master); - if (ret) - return ret; - platform_set_drvdata(pdev, master); master->free_rr_slots = GENMASK(ADI_MAX_DEVS, 1); - - writel(REG_IRQ_PENDING_CMDR, master->regs + REG_IRQ_MASK); - spin_lock_init(&master->ibi.lock); master->ibi.num_slots = 15; master->ibi.slots = devm_kcalloc(&pdev->dev, master->ibi.num_slots, @@ -986,6 +979,13 @@ static int adi_i3c_master_probe(struct platform_device *pdev) spin_lock_init(&master->xferqueue.lock); INIT_LIST_HEAD(&master->xferqueue.list); + ret = devm_request_irq(&pdev->dev, irq, adi_i3c_master_irq, 0, + dev_name(&pdev->dev), master); + if (ret) + return ret; + + writel(REG_IRQ_PENDING_CMDR, master->regs + REG_IRQ_MASK); + return i3c_master_register(&master->base, &pdev->dev, &adi_i3c_master_ops, false); } diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index 2f8c0c4683e06..dc3b74822f8ea 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -888,7 +888,15 @@ static int dw_i3c_master_daa(struct i3c_master_controller *m) if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT)) dw_i3c_master_dequeue_xfer(master, xfer); - newdevs = GENMASK(master->maxdevs - cmd->rx_len - 1, 0); + /* + * cmd->rx_len holds the number of addresses ENTDAA left unassigned. + * On an empty bus rx_len == maxdevs, so avoid GENMASK(-1, 0). + */ + if (cmd->rx_len >= master->maxdevs) + newdevs = 0; + else + newdevs = GENMASK(master->maxdevs - cmd->rx_len - 1, 0); + newdevs &= ~olddevs; for (pos = 0; pos < master->maxdevs; pos++) { diff --git a/drivers/i3c/master/mipi-i3c-hci/pio.c b/drivers/i3c/master/mipi-i3c-hci/pio.c index ff2657ee220ba..a1341d66bc656 100644 --- a/drivers/i3c/master/mipi-i3c-hci/pio.c +++ b/drivers/i3c/master/mipi-i3c-hci/pio.c @@ -185,8 +185,11 @@ static void __hci_pio_init(struct i3c_hci *hci, u32 *size_val_ptr) pio_reg_write(INTR_SIGNAL_ENABLE, 0x0); pio_reg_write(INTR_STATUS_ENABLE, 0xffffffff); - /* Always accept error interrupts (will be activated on first xfer) */ - pio->enabled_irqs = STAT_ALL_ERRORS; + /* + * Always accept error interrupts and IBI threshold interrupt + * (will be activated on first xfer). + */ + pio->enabled_irqs = STAT_ALL_ERRORS | STAT_IBI_STATUS_THLD; } static void hci_pio_suspend(struct i3c_hci *hci) diff --git a/drivers/i3c/master/renesas-i3c.c b/drivers/i3c/master/renesas-i3c.c index f39c449922ca2..6b403c3e58e9f 100644 --- a/drivers/i3c/master/renesas-i3c.c +++ b/drivers/i3c/master/renesas-i3c.c @@ -260,11 +260,11 @@ struct renesas_i3c { u32 dyn_addr; u32 i2c_STDBR; u32 i3c_STDBR; + u32 extbr; unsigned long rate; u8 addrs[RENESAS_I3C_MAX_DEVS]; struct renesas_i3c_xferqueue xferqueue; void __iomem *regs; - u32 *DATBASn; struct clk_bulk_data *clks; struct reset_control *presetn; struct reset_control *tresetn; @@ -433,6 +433,21 @@ static void renesas_i3c_enqueue_xfer(struct renesas_i3c *i3c, struct renesas_i3c } } +static void renesas_i3c_irqs_mask_and_clear_locked(struct renesas_i3c *i3c) +{ + /* Disable all the interrupts. */ + renesas_writel(i3c->regs, BIE, 0); + renesas_writel(i3c->regs, NTIE, 0); + + /* Clear normal transfer status flags. */ + renesas_writel(i3c->regs, NTST, 0); + + /* Clear bus status flags. */ + renesas_writel(i3c->regs, BST, 0); + /* Read back registers to confirm writes have fully propagated. */ + renesas_readl(i3c->regs, BST); +} + static void renesas_i3c_wait_xfer(struct renesas_i3c *i3c, struct renesas_i3c_xfer *xfer) { unsigned long time_left; @@ -607,10 +622,9 @@ static int renesas_i3c_bus_init(struct i3c_master_controller *m) renesas_writel(i3c->regs, STDBR, i3c->i3c_STDBR); /* Extended Bit Rate setting */ - renesas_writel(i3c->regs, EXTBR, EXTBR_EBRLO(od_low_ticks) | - EXTBR_EBRHO(od_high_ticks) | - EXTBR_EBRLP(pp_low_ticks) | - EXTBR_EBRHP(pp_high_ticks)); + i3c->extbr = EXTBR_EBRLO(od_low_ticks) | EXTBR_EBRHO(od_high_ticks) | + EXTBR_EBRLP(pp_low_ticks) | EXTBR_EBRHP(pp_high_ticks); + renesas_writel(i3c->regs, EXTBR, i3c->extbr); renesas_writel(i3c->regs, REFCKCTL, REFCKCTL_IREFCKS(cks)); i3c->refclk_div = cks; @@ -649,6 +663,10 @@ static int renesas_i3c_daa(struct i3c_master_controller *m) if (!xfer) return -ENOMEM; + init_completion(&xfer->comp); + cmd = xfer->cmds; + cmd->rx_count = i3c->maxdevs; + /* Enable I3C bus. */ renesas_i3c_bus_enable(m, true); @@ -670,10 +688,6 @@ static int renesas_i3c_daa(struct i3c_master_controller *m) renesas_writel(i3c->regs, DATBAS(pos), datbas_dvdyad_with_parity(ret)); } - init_completion(&xfer->comp); - cmd = xfer->cmds; - cmd->rx_count = 0; - ret = renesas_i3c_get_free_pos(i3c); if (ret < 0) return ret; @@ -689,7 +703,11 @@ static int renesas_i3c_daa(struct i3c_master_controller *m) renesas_i3c_wait_xfer(i3c, xfer); - newdevs = GENMASK(i3c->maxdevs - cmd->rx_count - 1, 0); + if (cmd->rx_count >= i3c->maxdevs) + newdevs = 0; + else + newdevs = GENMASK(i3c->maxdevs - cmd->rx_count - 1, 0); + newdevs &= ~olddevs; for (pos = 0; pos < i3c->maxdevs; pos++) { @@ -761,13 +779,13 @@ static int renesas_i3c_send_ccc_cmd(struct i3c_master_controller *m, if (!xfer) return -ENOMEM; - renesas_i3c_bus_enable(m, true); - init_completion(&xfer->comp); cmd = xfer->cmds; cmd->rnw = ccc->rnw; cmd->cmd0 = 0; + renesas_i3c_bus_enable(m, true); + /* Calculate the command descriptor. */ switch (ccc->id) { case I3C_CCC_SETDASA: @@ -817,15 +835,15 @@ static int renesas_i3c_i3c_xfers(struct i3c_dev_desc *dev, struct i3c_xfer *i3c_ struct renesas_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); int i; - /* Enable I3C bus. */ - renesas_i3c_bus_enable(m, true); - struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); if (!xfer) return -ENOMEM; init_completion(&xfer->comp); + /* Enable I3C bus. */ + renesas_i3c_bus_enable(m, true); + for (i = 0; i < i3c_nxfers; i++) { struct renesas_i3c_cmd *cmd = xfer->cmds; @@ -892,10 +910,26 @@ static int renesas_i3c_reattach_i3c_dev(struct i3c_dev_desc *dev, struct i3c_master_controller *m = i3c_dev_get_master(dev); struct renesas_i3c *i3c = to_renesas_i3c(m); struct renesas_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + int pos; + + pos = renesas_i3c_get_free_pos(i3c); + + if (data->index > pos && pos >= 0) { + renesas_writel(i3c->regs, DATBAS(data->index), 0); + i3c->addrs[data->index] = 0; + i3c->free_pos |= BIT(data->index); + + data->index = pos; + i3c->free_pos &= ~BIT(data->index); + } i3c->addrs[data->index] = dev->info.dyn_addr ? dev->info.dyn_addr : dev->info.static_addr; + renesas_writel(i3c->regs, DATBAS(data->index), + DATBAS_DVSTAD(dev->info.static_addr) | + datbas_dvdyad_with_parity(i3c->addrs[data->index])); + return 0; } @@ -905,6 +939,8 @@ static void renesas_i3c_detach_i3c_dev(struct i3c_dev_desc *dev) struct i3c_master_controller *m = i3c_dev_get_master(dev); struct renesas_i3c *i3c = to_renesas_i3c(m); + renesas_writel(i3c->regs, DATBAS(data->index), 0); + i3c_dev_set_master_data(dev, NULL); i3c->addrs[data->index] = 0; i3c->free_pos |= BIT(data->index); @@ -921,19 +957,19 @@ static int renesas_i3c_i2c_xfers(struct i2c_dev_desc *dev, u8 start_bit = CNDCTL_STCND; int i; - struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); - if (!xfer) - return -ENOMEM; - if (!i2c_nxfers) return 0; - renesas_i3c_bus_enable(m, false); + struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); + if (!xfer) + return -ENOMEM; init_completion(&xfer->comp); xfer->is_i2c_xfer = true; cmd = xfer->cmds; + renesas_i3c_bus_enable(m, false); + if (!(renesas_readl(i3c->regs, BCST) & BCST_BFREF)) { cmd->err = -EBUSY; return cmd->err; @@ -1014,6 +1050,11 @@ static irqreturn_t renesas_i3c_tx_isr(int irq, void *data) scoped_guard(spinlock, &i3c->xferqueue.lock) { xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + cmd = xfer->cmds; if (xfer->is_i2c_xfer) { @@ -1054,6 +1095,11 @@ static irqreturn_t renesas_i3c_resp_isr(int irq, void *data) scoped_guard(spinlock, &i3c->xferqueue.lock) { xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + cmd = xfer->cmds; /* Clear the Respone Queue Full status flag*/ @@ -1138,6 +1184,11 @@ static irqreturn_t renesas_i3c_tend_isr(int irq, void *data) scoped_guard(spinlock, &i3c->xferqueue.lock) { xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + cmd = xfer->cmds; if (xfer->is_i2c_xfer) { @@ -1184,6 +1235,11 @@ static irqreturn_t renesas_i3c_rx_isr(int irq, void *data) scoped_guard(spinlock, &i3c->xferqueue.lock) { xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + cmd = xfer->cmds; if (xfer->is_i2c_xfer) { @@ -1234,15 +1290,13 @@ static irqreturn_t renesas_i3c_stop_isr(int irq, void *data) struct renesas_i3c_xfer *xfer; scoped_guard(spinlock, &i3c->xferqueue.lock) { - xfer = i3c->xferqueue.cur; - - /* read back registers to confirm writes have fully propagated */ - renesas_writel(i3c->regs, BST, 0); - renesas_readl(i3c->regs, BST); - renesas_writel(i3c->regs, BIE, 0); - renesas_clear_bit(i3c->regs, NTST, NTST_TDBEF0 | NTST_RDBFF0); + renesas_i3c_irqs_mask_and_clear_locked(i3c); renesas_clear_bit(i3c->regs, SCSTRCTL, SCSTRCTL_RWE); + xfer = i3c->xferqueue.cur; + if (!xfer) + return IRQ_HANDLED; + xfer->ret = 0; complete(&xfer->comp); } @@ -1259,6 +1313,11 @@ static irqreturn_t renesas_i3c_start_isr(int irq, void *data) scoped_guard(spinlock, &i3c->xferqueue.lock) { xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + cmd = xfer->cmds; if (xfer->is_i2c_xfer) { @@ -1365,12 +1424,6 @@ static int renesas_i3c_probe(struct platform_device *pdev) i3c->maxdevs = RENESAS_I3C_MAX_DEVS; i3c->free_pos = GENMASK(i3c->maxdevs - 1, 0); - /* Allocate dynamic Device Address Table backup. */ - i3c->DATBASn = devm_kzalloc(&pdev->dev, sizeof(u32) * i3c->maxdevs, - GFP_KERNEL); - if (!i3c->DATBASn) - return -ENOMEM; - return i3c_master_register(&i3c->base, &pdev->dev, &renesas_i3c_ops, false); } @@ -1381,17 +1434,13 @@ static void renesas_i3c_remove(struct platform_device *pdev) i3c_master_unregister(&i3c->base); } -static int renesas_i3c_suspend_noirq(struct device *dev) +static int renesas_i3c_suspend(struct device *dev) { struct renesas_i3c *i3c = dev_get_drvdata(dev); - int i, ret; + int ret; i2c_mark_adapter_suspended(&i3c->base.i2c); - /* Store Device Address Table values. */ - for (i = 0; i < i3c->maxdevs; i++) - i3c->DATBASn[i] = renesas_readl(i3c->regs, DATBAS(i)); - ret = reset_control_assert(i3c->presetn); if (ret) goto err_mark_resumed; @@ -1412,50 +1461,61 @@ static int renesas_i3c_suspend_noirq(struct device *dev) return ret; } -static int renesas_i3c_resume_noirq(struct device *dev) +static int renesas_i3c_resume(struct device *dev) { struct renesas_i3c *i3c = dev_get_drvdata(dev); - int i, ret; + int ret; - ret = reset_control_deassert(i3c->presetn); + ret = reset_control_deassert(i3c->tresetn); if (ret) return ret; - ret = reset_control_deassert(i3c->tresetn); + ret = reset_control_deassert(i3c->presetn); if (ret) - goto err_presetn; + goto err_tresetn; ret = clk_bulk_enable(i3c->num_clks, i3c->clks); if (ret) - goto err_tresetn; + goto err_presetn; + + ret = renesas_i3c_reset(i3c); + if (ret) + goto err_clks_disable; /* Re-store I3C registers value. */ + renesas_writel(i3c->regs, STDBR, i3c->i3c_STDBR); + renesas_writel(i3c->regs, EXTBR, i3c->extbr); renesas_writel(i3c->regs, REFCKCTL, REFCKCTL_IREFCKS(i3c->refclk_div)); renesas_writel(i3c->regs, MSDVAD, MSDVAD_MDYADV | MSDVAD_MDYAD(i3c->dyn_addr)); - /* Restore Device Address Table values. */ - for (i = 0; i < i3c->maxdevs; i++) - renesas_writel(i3c->regs, DATBAS(i), i3c->DATBASn[i]); - /* I3C hw init. */ renesas_i3c_hw_init(i3c); + ret = i3c_master_do_daa_ext(&i3c->base, true); + if (ret) + dev_err(dev, "DAA failed on resume, ret=%d", ret); + i2c_mark_adapter_resumed(&i3c->base.i2c); + /* + * I3C devices may have retained their dynamic address anyway. Do not + * fail the resume because of DAA error. + */ return 0; -err_tresetn: - reset_control_assert(i3c->tresetn); +err_clks_disable: + clk_bulk_disable(i3c->num_clks, i3c->clks); err_presetn: reset_control_assert(i3c->presetn); +err_tresetn: + reset_control_assert(i3c->tresetn); return ret; } static const struct dev_pm_ops renesas_i3c_pm_ops = { - NOIRQ_SYSTEM_SLEEP_PM_OPS(renesas_i3c_suspend_noirq, - renesas_i3c_resume_noirq) + SYSTEM_SLEEP_PM_OPS(renesas_i3c_suspend, renesas_i3c_resume) }; static const struct of_device_id renesas_i3c_of_ids[] = { diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c index 93805df8a9406..71f1fc6933233 100644 --- a/drivers/i3c/master/svc-i3c-master.c +++ b/drivers/i3c/master/svc-i3c-master.c @@ -455,14 +455,22 @@ static int svc_i3c_master_handle_ibi(struct svc_i3c_master *master, buf = slot->data; while (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS)) && - slot->len < SVC_I3C_FIFO_SIZE) { + slot->len < dev->ibi->max_payload_len) { mdatactrl = readl(master->regs + SVC_I3C_MDATACTRL); count = SVC_I3C_MDATACTRL_RXCOUNT(mdatactrl); + count = min(count, dev->ibi->max_payload_len - slot->len); readsb(master->regs + SVC_I3C_MRDATAB, buf, count); slot->len += count; buf += count; } + /* + * The device may have sent more than the requested payload. Drop the + * extra bytes so they do not leak into the next transfer. + */ + if (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS))) + writel(SVC_I3C_MDATACTRL_FLUSHRB, master->regs + SVC_I3C_MDATACTRL); + master->ibi.tbq_slot = slot; return 0; diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index d74b478db2804..49c4ac9c7bb6e 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -2697,6 +2698,9 @@ static void __init cmdline_table_adjust(struct cpuidle_driver *drv) pr_info("Failed to adjust C-states with data from 'intel_idle.table'\n"); } +#define INTEL_IDLE_INIT_QOS 20 +static struct pm_qos_request qos_req __initdata; + static int __init intel_idle_init(void) { const struct x86_cpu_id *id; @@ -2766,6 +2770,13 @@ static int __init intel_idle_init(void) if (retval) pr_warn("failed to initialized sysfs"); + /* + * Some platforms, in particular the Intel S1200BTL motherboard, have a + * problem with using package idle states too early, so prevent that + * from taking place until the device_initcall() phase is over. + */ + cpu_latency_qos_add_request(&qos_req, INTEL_IDLE_INIT_QOS); + retval = cpuidle_register_driver(&intel_idle_driver); if (retval) { struct cpuidle_driver *drv = cpuidle_get_driver(); @@ -2790,6 +2801,9 @@ static int __init intel_idle_init(void) intel_idle_cpuidle_devices_uninit(); cpuidle_unregister_driver(&intel_idle_driver); init_driver_fail: + if (cpu_latency_qos_request_active((&qos_req))) + cpu_latency_qos_remove_request(&qos_req); + intel_idle_sysfs_uninit(); free_percpu(intel_idle_cpuidle_devices); return retval; @@ -2797,6 +2811,15 @@ static int __init intel_idle_init(void) } subsys_initcall_sync(intel_idle_init); +static int __init intel_idle_init_complete(void) +{ + if (cpu_latency_qos_request_active((&qos_req))) + cpu_latency_qos_remove_request(&qos_req); + + return 0; +} +device_initcall_sync(intel_idle_init_complete); + /* * We are not really modular, but we used to support that. Meaning we also * support "intel_idle.max_cstate=..." at boot and also a read-only export of diff --git a/drivers/iio/accel/dmard09.c b/drivers/iio/accel/dmard09.c index fe35a1270786b..6f0497ab61335 100644 --- a/drivers/iio/accel/dmard09.c +++ b/drivers/iio/accel/dmard09.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #define DMARD09_DRV_NAME "dmard09" @@ -79,6 +80,12 @@ static int dmard09_read_raw(struct iio_dev *indio_dev, *val = accel; return IIO_VAL_INT; + case IIO_CHAN_INFO_SCALE: + *val = 0; + /* 1 g / 32 LSB, in m/s^2 */ + *val2 = IIO_G_TO_M_S_2(NANO / 32); + + return IIO_VAL_INT_PLUS_NANO; default: return -EINVAL; } diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index 3755a81c1efdb..c6d3cf68b1f71 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -1076,6 +1076,7 @@ config MAX1363 config MAX14001 tristate "Analog Devices MAX14001/MAX14002 ADC driver" depends on SPI + select REGMAP help Say yes here to build support for Analog Devices MAX14001/MAX14002 Configurable, Isolated 10-bit ADCs for Multi-Range Binary Inputs. @@ -1086,6 +1087,7 @@ config MAX14001 config MAX34408 tristate "Maxim max34408/max344089 ADC driver" depends on I2C + select REGMAP_I2C help Say yes here to build support for Maxim max34408/max34409 current sense monitor with 8-bits ADC interface with overcurrent delay/threshold and diff --git a/drivers/iio/adc/ad4080.c b/drivers/iio/adc/ad4080.c index 0797d64aeaeca..967144eee3eb3 100644 --- a/drivers/iio/adc/ad4080.c +++ b/drivers/iio/adc/ad4080.c @@ -697,6 +697,11 @@ static int ad4080_setup_channel(struct ad4080_state *st, unsigned int ch) if (ret) return ret; + ret = iio_backend_data_size_set(st->back[ch], + st->info->channels[0].scan_type.realbits); + if (ret) + return ret; + if (!st->lvds_cnv_en) return 0; diff --git a/drivers/iio/adc/adi-axi-adc.c b/drivers/iio/adc/adi-axi-adc.c index 26b9c75bd4d8e..589f03618ecf3 100644 --- a/drivers/iio/adc/adi-axi-adc.c +++ b/drivers/iio/adc/adi-axi-adc.c @@ -53,6 +53,10 @@ #define AXI_AD485X_PACKET_FORMAT_24BIT 0x1 #define AXI_AD485X_PACKET_FORMAT_32BIT 0x2 #define AXI_AD408X_CNTRL_3_FILTER_EN_MSK BIT(0) +#define AXI_AD408X_CNTRL_3_PACKET_FORMAT_MSK GENMASK(3, 2) +#define AXI_AD408X_PACKET_FORMAT_20BIT 0x0 +#define AXI_AD408X_PACKET_FORMAT_16BIT 0x1 +#define AXI_AD408X_PACKET_FORMAT_14BIT 0x2 #define ADI_AXI_ADC_REG_SYNC_STATUS 0x0068 #define ADI_AXI_ADC_SYNC_STATUS_ADC_SYNC_MSK BIT(0) @@ -436,6 +440,31 @@ static int axi_adc_ad408x_filter_type_set(struct iio_backend *back, AXI_AD408X_CNTRL_3_FILTER_EN_MSK); } +static int axi_adc_ad408x_data_size_set(struct iio_backend *back, + unsigned int size) +{ + struct adi_axi_adc_state *st = iio_backend_get_priv(back); + unsigned int val; + + switch (size) { + case 20: + val = AXI_AD408X_PACKET_FORMAT_20BIT; + break; + case 16: + val = AXI_AD408X_PACKET_FORMAT_16BIT; + break; + case 14: + val = AXI_AD408X_PACKET_FORMAT_14BIT; + break; + default: + return -EINVAL; + } + + return regmap_update_bits(st->regmap, ADI_AXI_ADC_REG_CNTRL_3, + AXI_AD408X_CNTRL_3_PACKET_FORMAT_MSK, + FIELD_PREP(AXI_AD408X_CNTRL_3_PACKET_FORMAT_MSK, val)); +} + static int axi_adc_ad408x_interface_data_align(struct iio_backend *back, u32 timeout_us) { @@ -659,6 +688,7 @@ static const struct iio_backend_ops adi_ad408x_ops = { .free_buffer = axi_adc_free_buffer, .data_sample_trigger = axi_adc_data_sample_trigger, .filter_type_set = axi_adc_ad408x_filter_type_set, + .data_size_set = axi_adc_ad408x_data_size_set, .interface_data_align = axi_adc_ad408x_interface_data_align, .num_lanes_set = axi_adc_num_lanes_set, .debugfs_reg_access = iio_backend_debugfs_ptr(axi_adc_reg_access), diff --git a/drivers/iio/adc/pac1921.c b/drivers/iio/adc/pac1921.c index 68bdd2f30bada..f0bab94462f46 100644 --- a/drivers/iio/adc/pac1921.c +++ b/drivers/iio/adc/pac1921.c @@ -1037,7 +1037,7 @@ static irqreturn_t pac1921_trigger_handler(int irq, void *p) iio_for_each_active_channel(idev, bit) { u16 val; - ret = pac1921_read_res(priv, idev->channels[ch].address, &val); + ret = pac1921_read_res(priv, idev->channels[bit].address, &val); if (ret) goto done; diff --git a/drivers/iio/adc/ti-ads7138.c b/drivers/iio/adc/ti-ads7138.c index af87f5f19a0fa..da82a6947b2b1 100644 --- a/drivers/iio/adc/ti-ads7138.c +++ b/drivers/iio/adc/ti-ads7138.c @@ -227,6 +227,26 @@ static int ads7138_osr_to_bits(int osr) return -EINVAL; } +static int ads7138_read_statistics(const struct i2c_client *client, u8 reg, + u8 *out_values, u8 length) +{ + int ret; + + /* Disable statistics update so the value is not updated mid read */ + ret = ads7138_i2c_clear_bit(client, ADS7138_REG_GENERAL_CFG, + ADS7138_GENERAL_CFG_STATS_EN); + if (ret) + return ret; + + ret = ads7138_i2c_read_block(client, reg, out_values, length); + if (ret) + return ret; + + /* Enable statistics update after read */ + return ads7138_i2c_set_bit(client, ADS7138_REG_GENERAL_CFG, + ADS7138_GENERAL_CFG_STATS_EN); +} + static int ads7138_read_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, int *val, int *val2, long mask) @@ -236,28 +256,32 @@ static int ads7138_read_raw(struct iio_dev *indio_dev, u8 values[2]; switch (mask) { + /* + * Reading the statistics registers reinitializes them. This is + * unfortunate but necessary to prevent data races. + */ case IIO_CHAN_INFO_RAW: - ret = ads7138_i2c_read_block(data->client, - ADS7138_REG_RECENT_LSB_CH(chan->channel), - values, ARRAY_SIZE(values)); + ret = ads7138_read_statistics(data->client, + ADS7138_REG_RECENT_LSB_CH(chan->channel), + values, ARRAY_SIZE(values)); if (ret) return ret; *val = get_unaligned_le16(values); return IIO_VAL_INT; case IIO_CHAN_INFO_PEAK: - ret = ads7138_i2c_read_block(data->client, - ADS7138_REG_MAX_LSB_CH(chan->channel), - values, ARRAY_SIZE(values)); + ret = ads7138_read_statistics(data->client, + ADS7138_REG_MAX_LSB_CH(chan->channel), + values, ARRAY_SIZE(values)); if (ret) return ret; *val = get_unaligned_le16(values); return IIO_VAL_INT; case IIO_CHAN_INFO_TROUGH: - ret = ads7138_i2c_read_block(data->client, - ADS7138_REG_MIN_LSB_CH(chan->channel), - values, ARRAY_SIZE(values)); + ret = ads7138_read_statistics(data->client, + ADS7138_REG_MIN_LSB_CH(chan->channel), + values, ARRAY_SIZE(values)); if (ret) return ret; diff --git a/drivers/iio/chemical/atlas-sensor.c b/drivers/iio/chemical/atlas-sensor.c index 1e8adbe1790d0..1d4a9919df1a0 100644 --- a/drivers/iio/chemical/atlas-sensor.c +++ b/drivers/iio/chemical/atlas-sensor.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -87,7 +86,6 @@ struct atlas_data { struct iio_trigger *trig; const struct atlas_device *chip; struct regmap *regmap; - struct irq_work work; unsigned int interrupt_enabled; /* 96-bit data + 32-bit pad + 64-bit timestamp */ __be32 buffer[6] __aligned(8); @@ -412,7 +410,11 @@ static int atlas_buffer_postenable(struct iio_dev *indio_dev) if (ret) return ret; - return atlas_set_interrupt(data, true); + ret = atlas_set_interrupt(data, true); + if (ret) + pm_runtime_put_autosuspend(&data->client->dev); + + return ret; } static int atlas_buffer_predisable(struct iio_dev *indio_dev) @@ -436,13 +438,6 @@ static const struct iio_buffer_setup_ops atlas_buffer_setup_ops = { .predisable = atlas_buffer_predisable, }; -static void atlas_work_handler(struct irq_work *work) -{ - struct atlas_data *data = container_of(work, struct atlas_data, work); - - iio_trigger_poll(data->trig); -} - static irqreturn_t atlas_trigger_handler(int irq, void *private) { struct iio_poll_func *pf = private; @@ -469,7 +464,7 @@ static irqreturn_t atlas_interrupt_handler(int irq, void *private) struct iio_dev *indio_dev = private; struct atlas_data *data = iio_priv(indio_dev); - irq_work_queue(&data->work); + iio_trigger_poll_nested(data->trig); return IRQ_HANDLED; } @@ -665,8 +660,6 @@ static int atlas_probe(struct i2c_client *client) goto unregister_trigger; } - init_irq_work(&data->work, atlas_work_handler); - if (client->irq > 0) { /* interrupt pin toggles on new conversion */ ret = devm_request_threaded_irq(&client->dev, client->irq, diff --git a/drivers/iio/chemical/sgp30.c b/drivers/iio/chemical/sgp30.c index f10bbebc29e4e..379c1c4af8d88 100644 --- a/drivers/iio/chemical/sgp30.c +++ b/drivers/iio/chemical/sgp30.c @@ -548,6 +548,9 @@ static int sgp_probe(struct i2c_client *client) data->iaq_thread = kthread_run(sgp_iaq_threadfn, data, "%s-iaq", data->client->name); + if (IS_ERR(data->iaq_thread)) + return dev_err_probe(dev, PTR_ERR(data->iaq_thread), + "failed to start IAQ thread\n"); return 0; } diff --git a/drivers/iio/dac/Kconfig b/drivers/iio/dac/Kconfig index 657c68e75542e..14a246729d2bc 100644 --- a/drivers/iio/dac/Kconfig +++ b/drivers/iio/dac/Kconfig @@ -552,6 +552,7 @@ config MCP4728 config MCP47FEB02 tristate "MCP47F(E/V)B01/02/04/08/11/12/14/18/21/22/24/28 DAC driver" depends on I2C + select REGMAP_I2C help Say yes here if you want to build the driver for the Microchip: - 8-bit DAC: diff --git a/drivers/iio/dac/ad3552r-hs.c b/drivers/iio/dac/ad3552r-hs.c index 02a124ac4855d..f865843aa4393 100644 --- a/drivers/iio/dac/ad3552r-hs.c +++ b/drivers/iio/dac/ad3552r-hs.c @@ -590,7 +590,7 @@ static ssize_t ad3552r_hs_show_data_source_avail(struct file *f, int i; for (i = 0; i < ARRAY_SIZE(dbgfs_attr_source); i++) { - len += scnprintf(buf + len, PAGE_SIZE - len, "%s ", + len += scnprintf(buf + len, sizeof(buf) - len, "%s ", dbgfs_attr_source[i]); } buf[len - 1] = '\n'; diff --git a/drivers/iio/dac/ad5446-i2c.c b/drivers/iio/dac/ad5446-i2c.c index 9797fc3e57a9b..286695e240c36 100644 --- a/drivers/iio/dac/ad5446-i2c.c +++ b/drivers/iio/dac/ad5446-i2c.c @@ -83,7 +83,7 @@ static const struct of_device_id ad5446_i2c_of_ids[] = { { .compatible = "adi,ad5622", .data = &ad5622_chip_info }, { } }; -MODULE_DEVICE_TABLE(OF, ad5446_i2c_of_ids); +MODULE_DEVICE_TABLE(of, ad5446_i2c_of_ids); static struct i2c_driver ad5446_i2c_driver = { .driver = { diff --git a/drivers/iio/dac/ad5686-spi.c b/drivers/iio/dac/ad5686-spi.c index 8abfaf8f0c469..859874ab861c5 100644 --- a/drivers/iio/dac/ad5686-spi.c +++ b/drivers/iio/dac/ad5686-spi.c @@ -98,8 +98,13 @@ static const struct ad5686_bus_ops ad5686_spi_ops = { static int ad5686_spi_probe(struct spi_device *spi) { - return ad5686_probe(&spi->dev, spi_get_device_match_data(spi), - spi->modalias, &ad5686_spi_ops); + const struct ad5686_chip_info *info; + + info = spi_get_device_match_data(spi); + if (!info) + return -ENODATA; + + return ad5686_probe(&spi->dev, info, spi->modalias, &ad5686_spi_ops); } static const struct spi_device_id ad5686_spi_id[] = { diff --git a/drivers/iio/dac/ad5696-i2c.c b/drivers/iio/dac/ad5696-i2c.c index d49946adbde3a..d5934405d5552 100644 --- a/drivers/iio/dac/ad5696-i2c.c +++ b/drivers/iio/dac/ad5696-i2c.c @@ -68,8 +68,13 @@ static const struct ad5686_bus_ops ad5686_i2c_ops = { static int ad5686_i2c_probe(struct i2c_client *i2c) { - return ad5686_probe(&i2c->dev, i2c_get_match_data(i2c), - i2c->name, &ad5686_i2c_ops); + const struct ad5686_chip_info *info; + + info = i2c_get_match_data(i2c); + if (!info) + return -ENODATA; + + return ad5686_probe(&i2c->dev, info, i2c->name, &ad5686_i2c_ops); } static const struct i2c_device_id ad5686_i2c_id[] = { diff --git a/drivers/iio/dac/m62332.c b/drivers/iio/dac/m62332.c index 7e80c0eb5cc1a..60bb672b70e20 100644 --- a/drivers/iio/dac/m62332.c +++ b/drivers/iio/dac/m62332.c @@ -32,6 +32,7 @@ static int m62332_set_value(struct iio_dev *indio_dev, u8 val, int channel) { struct m62332_data *data = iio_priv(indio_dev); struct i2c_client *client = data->client; + bool enabling, disabling; u8 outbuf[2]; int res; @@ -43,7 +44,10 @@ static int m62332_set_value(struct iio_dev *indio_dev, u8 val, int channel) mutex_lock(&data->mutex); - if (val) { + enabling = val && !data->raw[channel]; + disabling = !val && data->raw[channel]; + + if (enabling) { res = regulator_enable(data->vcc); if (res) goto out; @@ -52,14 +56,17 @@ static int m62332_set_value(struct iio_dev *indio_dev, u8 val, int channel) res = i2c_master_send(client, outbuf, ARRAY_SIZE(outbuf)); if (res >= 0 && res != ARRAY_SIZE(outbuf)) res = -EIO; - if (res < 0) + if (res < 0) { + if (enabling) + regulator_disable(data->vcc); goto out; + } - data->raw[channel] = val; - - if (!val) + if (disabling) regulator_disable(data->vcc); + data->raw[channel] = val; + mutex_unlock(&data->mutex); return 0; diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c index d84e04e4b4314..07c0f9011c637 100644 --- a/drivers/iio/gyro/mpu3050-core.c +++ b/drivers/iio/gyro/mpu3050-core.c @@ -356,7 +356,7 @@ static int mpu3050_read_raw(struct iio_dev *indio_dev, goto out_read_raw_unlock; } - *val = be16_to_cpu(raw_val); + *val = (s16)be16_to_cpu(raw_val); ret = IIO_VAL_INT; goto out_read_raw_unlock; diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_fusion.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_fusion.c index 2e72c7ba94dd1..5321033daebcb 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_fusion.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_fusion.c @@ -112,6 +112,11 @@ int st_lsm6dsx_fusion_set_enable(struct st_lsm6dsx_sensor *sensor, bool enable) return err; } + if (enable) + hw->enable_mask |= BIT(ST_LSM6DSX_ID_FUSION); + else + hw->enable_mask &= ~BIT(ST_LSM6DSX_ID_FUSION); + return st_lsm6dsx_fusion_page_disable(hw); } diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c index 531fc4ccc15de..2c9ec93dff475 100644 --- a/drivers/iio/industrialio-buffer.c +++ b/drivers/iio/industrialio-buffer.c @@ -47,9 +47,6 @@ struct iio_dmabuf_priv { u64 context; - /* Spinlock used for locking the dma_fence */ - spinlock_t lock; - struct dma_buf_attachment *attach; struct sg_table *sgt; enum dma_data_direction dir; @@ -57,7 +54,12 @@ struct iio_dmabuf_priv { }; struct iio_dma_fence { + /* + * Must remain the first member so the default release callback can pass + * the fence directly to dma_fence_free(). + */ struct dma_fence base; + spinlock_t lock; /* protects base */ struct iio_dmabuf_priv *priv; struct work_struct work; }; @@ -1619,12 +1621,16 @@ static int iio_buffer_chrdev_release(struct inode *inode, struct file *filep) wake_up(&buffer->pollq); - guard(mutex)(&buffer->dmabufs_mutex); - - /* Close all attached DMABUFs */ - list_for_each_entry_safe(priv, tmp, &buffer->dmabufs, entry) { - list_del_init(&priv->entry); - iio_buffer_dmabuf_put(priv->attach); + /* + * The mutex must be unlocked before iio_device_put(), which might drop the + * last reference and free the buffer. + */ + scoped_guard(mutex, &buffer->dmabufs_mutex) { + /* Close all attached DMABUFs */ + list_for_each_entry_safe(priv, tmp, &buffer->dmabufs, entry) { + list_del_init(&priv->entry); + iio_buffer_dmabuf_put(priv->attach); + } } kfree(ib); @@ -1702,7 +1708,6 @@ static int iio_buffer_attach_dmabuf(struct iio_dev_buffer_pair *ib, if (!priv) return -ENOMEM; - spin_lock_init(&priv->lock); priv->context = dma_fence_context_alloc(1); dmabuf = dma_buf_get(fd); @@ -1827,18 +1832,9 @@ iio_buffer_dma_fence_get_driver_name(struct dma_fence *fence) return "iio"; } -static void iio_buffer_dma_fence_release(struct dma_fence *fence) -{ - struct iio_dma_fence *iio_fence = - container_of(fence, struct iio_dma_fence, base); - - kfree(iio_fence); -} - static const struct dma_fence_ops iio_buffer_dma_fence_ops = { .get_driver_name = iio_buffer_dma_fence_get_driver_name, .get_timeline_name = iio_buffer_dma_fence_get_driver_name, - .release = iio_buffer_dma_fence_release, }; static int iio_buffer_enqueue_dmabuf(struct iio_dev_buffer_pair *ib, @@ -1892,6 +1888,8 @@ static int iio_buffer_enqueue_dmabuf(struct iio_dev_buffer_pair *ib, goto err_attachment_put; } + spin_lock_init(&fence->lock); + fence->priv = priv; seqno = atomic_add_return(1, &priv->seqno); @@ -1902,7 +1900,7 @@ static int iio_buffer_enqueue_dmabuf(struct iio_dev_buffer_pair *ib, * the dma_fence. */ dma_fence_init(&fence->base, &iio_buffer_dma_fence_ops, - &priv->lock, priv->context, seqno); + &fence->lock, priv->context, seqno); ret = iio_dma_resv_lock(dmabuf, nonblock); if (ret) diff --git a/drivers/iio/light/apds9306.c b/drivers/iio/light/apds9306.c index 5ca4c87524fe2..d582bda4d8476 100644 --- a/drivers/iio/light/apds9306.c +++ b/drivers/iio/light/apds9306.c @@ -469,9 +469,9 @@ static int apds9306_read_data(struct apds9306_data *data, int *val, int reg) int status = 0; u8 buff[3]; - ret = pm_runtime_resume_and_get(data->dev); - if (ret) - return ret; + PM_RUNTIME_ACQUIRE_AUTOSUSPEND(data->dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) + return PM_RUNTIME_ACQUIRE_ERR(&pm); ret = regmap_field_read(rf->intg_time, &intg_time_idx); if (ret) @@ -535,8 +535,6 @@ static int apds9306_read_data(struct apds9306_data *data, int *val, int reg) *val = get_unaligned_le24(&buff); - pm_runtime_put_autosuspend(data->dev); - return 0; } diff --git a/drivers/iio/light/cm32181.c b/drivers/iio/light/cm32181.c index 2590fc8fd1547..b32a94028f091 100644 --- a/drivers/iio/light/cm32181.c +++ b/drivers/iio/light/cm32181.c @@ -368,7 +368,7 @@ static int cm32181_write_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_CALIBSCALE: cm32181->calibscale = val; - return val; + return 0; case IIO_CHAN_INFO_INT_TIME: ret = cm32181_write_als_it(cm32181, val2); return ret; diff --git a/drivers/iio/light/gp2ap002.c b/drivers/iio/light/gp2ap002.c index a8db514cca5ef..de29081fd3b20 100644 --- a/drivers/iio/light/gp2ap002.c +++ b/drivers/iio/light/gp2ap002.c @@ -342,6 +342,10 @@ static int gp2ap002_write_event_config(struct iio_dev *indio_dev, bool state) { struct gp2ap002 *gp2ap002 = iio_priv(indio_dev); + int ret; + + if (state == gp2ap002->enabled) + return 0; if (state) { /* @@ -349,13 +353,16 @@ static int gp2ap002_write_event_config(struct iio_dev *indio_dev, * already) and reintialize the sensor by using runtime_pm * callbacks. */ - pm_runtime_get_sync(gp2ap002->dev); - gp2ap002->enabled = true; + ret = pm_runtime_resume_and_get(gp2ap002->dev); + if (ret) + return ret; + } else { pm_runtime_put_autosuspend(gp2ap002->dev); - gp2ap002->enabled = false; } + gp2ap002->enabled = state; + return 0; } @@ -642,6 +649,7 @@ static int gp2ap002_runtime_suspend(struct device *dev) /* Disable chip and IRQ, everything off */ ret = regmap_write(gp2ap002->map, GP2AP002_OPMOD, 0x00); if (ret) { + enable_irq(gp2ap002->irq); dev_err(gp2ap002->dev, "error setting up operation mode\n"); return ret; } @@ -669,7 +677,7 @@ static int gp2ap002_runtime_resume(struct device *dev) ret = regulator_enable(gp2ap002->vio); if (ret) { dev_err(dev, "failed to enable VIO regulator in resume path\n"); - return ret; + goto out_disable_vdd; } msleep(20); @@ -677,13 +685,19 @@ static int gp2ap002_runtime_resume(struct device *dev) ret = gp2ap002_init(gp2ap002); if (ret) { dev_err(dev, "re-initialization failed\n"); - return ret; + goto out_disable_vio; } /* Re-activate the IRQ */ enable_irq(gp2ap002->irq); return 0; + +out_disable_vio: + regulator_disable(gp2ap002->vio); +out_disable_vdd: + regulator_disable(gp2ap002->vdd); + return ret; } static DEFINE_RUNTIME_DEV_PM_OPS(gp2ap002_dev_pm_ops, gp2ap002_runtime_suspend, diff --git a/drivers/iio/light/isl29028.c b/drivers/iio/light/isl29028.c index b88e7c4eae3ec..6d5dd17e7a7d0 100644 --- a/drivers/iio/light/isl29028.c +++ b/drivers/iio/light/isl29028.c @@ -409,7 +409,7 @@ static int isl29028_write_raw(struct iio_dev *indio_dev, if (ret < 0) return ret; - return ret; + return 0; } static int isl29028_read_raw(struct iio_dev *indio_dev, diff --git a/drivers/iio/light/ltrf216a.c b/drivers/iio/light/ltrf216a.c index 3f34ddc911b4a..0a8851342039e 100644 --- a/drivers/iio/light/ltrf216a.c +++ b/drivers/iio/light/ltrf216a.c @@ -247,11 +247,10 @@ static int ltrf216a_get_lux(struct ltrf216a_data *data) return ret; greendata = ltrf216a_read_data(data, LTRF216A_ALS_DATA_0); + ltrf216a_set_power_state(data, false); if (greendata < 0) return greendata; - ltrf216a_set_power_state(data, false); - lux = greendata * data->info->lux_multiplier * LTRF216A_WIN_FAC; return lux; diff --git a/drivers/iio/light/opt4001.c b/drivers/iio/light/opt4001.c index dd152d921b48e..aa3d87995b7c6 100644 --- a/drivers/iio/light/opt4001.c +++ b/drivers/iio/light/opt4001.c @@ -39,7 +39,7 @@ #define OPT4001_CTRL_OPER_MODE_MASK GENMASK(5, 4) #define OPT4001_CTRL_LATCH_MASK GENMASK(3, 3) #define OPT4001_CTRL_INT_POL_MASK GENMASK(2, 2) -#define OPT4001_CTRL_FAULT_COUNT GENMASK(0, 1) +#define OPT4001_CTRL_FAULT_COUNT_MASK GENMASK(1, 0) /* OPT4001 constants */ #define OPT4001_DEVICE_ID_VAL 0x121 @@ -173,6 +173,7 @@ static int opt4001_read_lux_value(struct iio_dev *indio_dev, u8 crc; u8 calc_crc; u64 lux_raw; + u32 rem; int ret; ret = regmap_read(chip->regmap, OPT4001_LIGHT1_MSB, &light1); @@ -199,8 +200,8 @@ static int opt4001_read_lux_value(struct iio_dev *indio_dev, lux_raw = lux_raw << exp; lux_raw = lux_raw * chip->chip_info->mul; - *val = div_u64_rem(lux_raw, chip->chip_info->div, val2); - *val2 = *val2 * 100; + *val = div_u64_rem(lux_raw, chip->chip_info->div, &rem); + *val2 = rem * 100; return IIO_VAL_INT_PLUS_NANO; } @@ -222,33 +223,14 @@ static int opt4001_set_conf(struct opt4001_chip *chip) return ret; } -static int opt4001_power_down(struct opt4001_chip *chip) -{ - struct device *dev = &chip->client->dev; - int ret; - unsigned int reg; - - ret = regmap_read(chip->regmap, OPT4001_DEVICE_ID, ®); - if (ret) { - dev_err(dev, "Failed to read configuration\n"); - return ret; - } - - /* MODE_OFF is 0x0 so just set bits to 0 */ - reg &= ~OPT4001_CTRL_OPER_MODE_MASK; - - ret = regmap_write(chip->regmap, OPT4001_CTRL, reg); - if (ret) - dev_err(dev, "Failed to set configuration to power down\n"); - - return ret; -} - static void opt4001_chip_off_action(void *data) { struct opt4001_chip *chip = data; + int ret; - opt4001_power_down(chip); + ret = regmap_clear_bits(chip->regmap, OPT4001_CTRL, OPT4001_CTRL_OPER_MODE_MASK); + if (ret) + dev_err(&chip->client->dev, "Failed to power down\n"); } static const struct iio_chan_spec opt4001_channels[] = { @@ -287,6 +269,9 @@ static int opt4001_write_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_INT_TIME: + if (val) + return -EINVAL; + int_time = opt4001_als_time_to_index(val2); if (int_time < 0) return int_time; diff --git a/drivers/iio/light/opt4060.c b/drivers/iio/light/opt4060.c index c391ad3271c63..f79dd342937d6 100644 --- a/drivers/iio/light/opt4060.c +++ b/drivers/iio/light/opt4060.c @@ -632,6 +632,9 @@ static int opt4060_write_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_INT_TIME: + if (val) + return -EINVAL; + int_time = opt4060_als_time_to_index(val2); if (int_time < 0) return int_time; @@ -710,6 +713,7 @@ static ssize_t opt4060_read_ev_period(struct opt4060_chip *chip, int *val, { int ret, pers, fault_count, int_time; u64 uval; + u32 rem; int_time = opt4060_int_time_reg[chip->int_time][0]; @@ -735,7 +739,8 @@ static ssize_t opt4060_read_ev_period(struct opt4060_chip *chip, int *val, } uval = mul_u32_u32(int_time, pers); - *val = div_u64_rem(uval, MICRO, val2); + *val = div_u64_rem(uval, MICRO, &rem); + *val2 = rem; return IIO_VAL_INT_PLUS_MICRO; } @@ -805,7 +810,7 @@ static int opt4060_get_thresholds(struct opt4060_chip *chip, u32 *th_lo, u32 *th ret = regmap_read(chip->regmap, OPT4060_THRESHOLD_HIGH, ®val); if (ret) { - dev_err(chip->dev, "Failed to read THRESHOLD_LOW.\n"); + dev_err(chip->dev, "Failed to read THRESHOLD_HIGH.\n"); return ret; } *th_hi = opt4060_calc_val_from_th_reg(regval); diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c index a0dd122af2cfb..3b2c74d80e25c 100644 --- a/drivers/iio/light/tsl2583.c +++ b/drivers/iio/light/tsl2583.c @@ -794,7 +794,7 @@ static int tsl2583_write_raw(struct iio_dev *indio_dev, if (ret < 0) return ret; - return ret; + return 0; } static const struct iio_info tsl2583_info = { diff --git a/drivers/iio/light/tsl2772.c b/drivers/iio/light/tsl2772.c index 244f44379c369..2287585711c69 100644 --- a/drivers/iio/light/tsl2772.c +++ b/drivers/iio/light/tsl2772.c @@ -1274,7 +1274,7 @@ static int tsl2772_read_raw(struct iio_dev *indio_dev, } break; case IIO_CHAN_INFO_CALIBSCALE: - if (chan->type == IIO_LIGHT) + if (chan->type == IIO_INTENSITY) *val = tsl2772_als_gain[chip->settings.als_gain]; else *val = tsl2772_prox_gain[chip->settings.prox_gain]; diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c index cc3e66dbb90fc..f761ec70b4c6c 100644 --- a/drivers/iio/orientation/hid-sensor-rotation.c +++ b/drivers/iio/orientation/hid-sensor-rotation.c @@ -367,12 +367,6 @@ static int hid_dev_rot_probe(struct platform_device *pdev) return ret; } - ret = iio_device_register(indio_dev); - if (ret) { - dev_err(&pdev->dev, "device register failed\n"); - goto error_remove_trigger; - } - rot_state->callbacks.send_event = dev_rot_proc_event; rot_state->callbacks.capture_sample = dev_rot_capture_sample; rot_state->callbacks.pdev = pdev; @@ -380,13 +374,19 @@ static int hid_dev_rot_probe(struct platform_device *pdev) &rot_state->callbacks); if (ret) { dev_err(&pdev->dev, "callback reg failed\n"); - goto error_iio_unreg; + goto error_remove_trigger; + } + + ret = iio_device_register(indio_dev); + if (ret) { + dev_err(&pdev->dev, "device register failed\n"); + goto error_remove_callback; } return 0; -error_iio_unreg: - iio_device_unregister(indio_dev); +error_remove_callback: + sensor_hub_remove_callback(hsdev, hsdev->usage); error_remove_trigger: hid_sensor_remove_trigger(indio_dev, &rot_state->common_attributes); return ret; @@ -399,8 +399,8 @@ static void hid_dev_rot_remove(struct platform_device *pdev) struct iio_dev *indio_dev = platform_get_drvdata(pdev); struct dev_rot_state *rot_state = iio_priv(indio_dev); - sensor_hub_remove_callback(hsdev, hsdev->usage); iio_device_unregister(indio_dev); + sensor_hub_remove_callback(hsdev, hsdev->usage); hid_sensor_remove_trigger(indio_dev, &rot_state->common_attributes); } diff --git a/drivers/iio/pressure/dps310.c b/drivers/iio/pressure/dps310.c index f45af72a05543..45bdb8c7670f0 100644 --- a/drivers/iio/pressure/dps310.c +++ b/drivers/iio/pressure/dps310.c @@ -845,7 +845,6 @@ static const struct iio_info dps310_info = { static int dps310_probe(struct i2c_client *client) { - const struct i2c_device_id *id = i2c_client_get_device_id(client); struct dps310_data *data; struct iio_dev *iio; int rc; @@ -858,7 +857,7 @@ static int dps310_probe(struct i2c_client *client) data->client = client; mutex_init(&data->lock); - iio->name = id->name; + iio->name = DPS310_DEV_NAME; iio->channels = dps310_channels; iio->num_channels = ARRAY_SIZE(dps310_channels); iio->info = &dps310_info; diff --git a/drivers/iio/pressure/mpl115.c b/drivers/iio/pressure/mpl115.c index 16e112b796ba9..2929ea8e4e01b 100644 --- a/drivers/iio/pressure/mpl115.c +++ b/drivers/iio/pressure/mpl115.c @@ -203,9 +203,9 @@ int mpl115_probe(struct device *dev, const char *name, if (data->shutdown) { /* Enable runtime PM */ - pm_runtime_get_noresume(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); + ret = pm_runtime_set_active(dev); + if (ret) + return ret; /* * As the device takes 3 ms to come up with a fresh @@ -215,7 +215,10 @@ int mpl115_probe(struct device *dev, const char *name, */ pm_runtime_set_autosuspend_delay(dev, 2000); pm_runtime_use_autosuspend(dev); - pm_runtime_put(dev); + + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; dev_dbg(dev, "low-power mode enabled"); } else diff --git a/drivers/iio/proximity/srf04.c b/drivers/iio/proximity/srf04.c index 7be50bdebfcbc..e03f3f0dee04c 100644 --- a/drivers/iio/proximity/srf04.c +++ b/drivers/iio/proximity/srf04.c @@ -330,6 +330,7 @@ static int srf04_probe(struct platform_device *pdev) if (ret) { dev_err(data->dev, "pm_runtime_set_active: %d\n", ret); iio_device_unregister(indio_dev); + return ret; } pm_runtime_enable(data->dev); diff --git a/drivers/iio/temperature/hid-sensor-temperature.c b/drivers/iio/temperature/hid-sensor-temperature.c index 54a00253de114..58d4c02739323 100644 --- a/drivers/iio/temperature/hid-sensor-temperature.c +++ b/drivers/iio/temperature/hid-sensor-temperature.c @@ -243,7 +243,7 @@ static int hid_temperature_probe(struct platform_device *pdev) if (ret) goto error_remove_trigger; - ret = devm_iio_device_register(indio_dev->dev.parent, indio_dev); + ret = iio_device_register(indio_dev); if (ret) goto error_remove_callback; @@ -263,6 +263,7 @@ static void hid_temperature_remove(struct platform_device *pdev) struct iio_dev *indio_dev = platform_get_drvdata(pdev); struct temperature_state *temp_st = iio_priv(indio_dev); + iio_device_unregister(indio_dev); sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_TEMPERATURE); hid_sensor_remove_trigger(indio_dev, &temp_st->common_attributes); } diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index e88d3efb967b3..407310145e6c9 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -3528,10 +3528,12 @@ static void addr_handler(int status, struct sockaddr *src_addr, memcpy(addr, src_addr, rdma_addr_size(src_addr)); if (!status && !id_priv->cma_dev) { status = cma_acquire_dev_by_src_ip(id_priv); - if (status) + if (status) { pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n", status); - rdma_restrack_add(&id_priv->res); + } else { + rdma_restrack_add(&id_priv->res); + } } else if (status) { pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status); } diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c index a9e189194c130..a2c85840c501a 100644 --- a/drivers/infiniband/core/counters.c +++ b/drivers/infiniband/core/counters.c @@ -234,7 +234,6 @@ static void rdma_counter_free(struct rdma_counter *counter) mutex_unlock(&port_counter->lock); - rdma_restrack_del(&counter->res); rdma_free_hw_stats_struct(counter->stats); kfree(counter); } @@ -329,6 +328,7 @@ static void counter_release(struct kref *kref) counter = container_of(kref, struct rdma_counter, kref); counter_history_stat_update(counter); + rdma_restrack_del(&counter->res); counter->device->ops.counter_dealloc(counter); rdma_counter_free(counter); } @@ -490,7 +490,8 @@ static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev, return NULL; counter = container_of(res, struct rdma_counter, res); - kref_get(&counter->kref); + if (!kref_get_unless_zero(&counter->kref)) + counter = NULL; rdma_restrack_put(res); return counter; diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c index 3d7b6cddd131c..1379808e14040 100644 --- a/drivers/infiniband/core/cq.c +++ b/drivers/infiniband/core/cq.c @@ -327,6 +327,7 @@ void ib_free_cq(struct ib_cq *cq) if (WARN_ON_ONCE(cq->cqe_used)) return; + rdma_restrack_del(&cq->res); if (cq->device->ops.pre_destroy_cq) { ret = cq->device->ops.pre_destroy_cq(cq); WARN_ONCE(ret, "Disable of kernel CQ shouldn't fail"); @@ -353,7 +354,6 @@ void ib_free_cq(struct ib_cq *cq) else ret = cq->device->ops.destroy_cq(cq, NULL); WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail"); - rdma_restrack_del(&cq->res); kfree(cq->wc); kfree(cq); } diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index b8193e077a746..d954eda631349 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -3150,6 +3150,7 @@ static void __exit ib_core_cleanup(void) /* Make sure that any pending umem accounting work is done. */ destroy_workqueue(ib_wq); destroy_workqueue(ib_unreg_wq); + rcu_barrier(); WARN_ON(!xa_empty(&clients)); WARN_ON(!xa_empty(&devices)); } diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index dbbc0875132a3..41c84ffe8c093 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -424,6 +424,9 @@ static uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata) lockdep_assert_held(&bundle->ufile->device->disassociate_srcu); + if (!bundle->method_elm) + return NULL; + return srcu_dereference(bundle->method_elm->handler, &bundle->ufile->device->disassociate_srcu); } diff --git a/drivers/infiniband/core/iwpm_util.c b/drivers/infiniband/core/iwpm_util.c index 990cf928b32a8..51af8c1c49f9e 100644 --- a/drivers/infiniband/core/iwpm_util.c +++ b/drivers/infiniband/core/iwpm_util.c @@ -314,10 +314,6 @@ struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq, if (!nlmsg_request) return NULL; - spin_lock_irqsave(&iwpm_nlmsg_req_lock, flags); - list_add_tail(&nlmsg_request->inprocess_list, &iwpm_nlmsg_req_list); - spin_unlock_irqrestore(&iwpm_nlmsg_req_lock, flags); - kref_init(&nlmsg_request->kref); kref_get(&nlmsg_request->kref); nlmsg_request->nlmsg_seq = nlmsg_seq; @@ -326,6 +322,11 @@ struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq, nlmsg_request->err_code = 0; sema_init(&nlmsg_request->sem, 1); down(&nlmsg_request->sem); + + spin_lock_irqsave(&iwpm_nlmsg_req_lock, flags); + list_add_tail(&nlmsg_request->inprocess_list, &iwpm_nlmsg_req_list); + spin_unlock_irqrestore(&iwpm_nlmsg_req_lock, flags); + return nlmsg_request; } diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c index e0b3b36b8b149..3c91f00d09c6b 100644 --- a/drivers/infiniband/core/mad.c +++ b/drivers/infiniband/core/mad.c @@ -2059,6 +2059,8 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv, int ret; INIT_LIST_HEAD(&mad_recv_wc->rmpp_list); + list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list); + ret = ib_mad_enforce_security(mad_agent_priv, mad_recv_wc->wc->pkey_index); if (ret) { @@ -2067,7 +2069,6 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv, return; } - list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list); if (is_kernel_rmpp_data_response(mad_agent_priv, mad_recv_wc)) { spin_lock_irqsave(&mad_agent_priv->lock, flags); mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc); diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 02a0a9c0a4a6a..f0f09670956d6 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -2133,6 +2133,11 @@ static int nldev_stat_set_counter_dynamic_doit(struct nlattr *tb[], nla_for_each_nested(entry_attr, tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS], rem) { + if (nla_len(entry_attr) != sizeof(u32)) { + ret = -EINVAL; + goto out; + } + index = nla_get_u32(entry_attr); if ((index >= stats->num_counters) || !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) { diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index fd5651c003aed..a7cbe643e33c6 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -69,7 +69,6 @@ void ib_uverbs_release_file(struct kref *ref) if (file->disassociate_page) __free_pages(file->disassociate_page, 0); - mutex_destroy(&file->disassociation_lock); mutex_destroy(&file->umap_lock); mutex_destroy(&file->ucontext_lock); kfree(file); diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index cfee2071586c1..0d7e40f63c8a0 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -129,6 +129,46 @@ static void rdma_restrack_attach_task(struct rdma_restrack_entry *res, res->user = true; } +static struct rdma_restrack_root *res_to_rt(struct rdma_restrack_entry *res) +{ + struct ib_device *dev = res_to_dev(res); + + if (WARN_ON(!dev)) + return NULL; + + return &dev->res[res->type]; +} + +static void restrack_drain_res(struct rdma_restrack_root *rt, + struct rdma_restrack_entry *res) +{ + if (rt) { + struct rdma_restrack_entry *old; + + old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, + GFP_KERNEL); + WARN_ON(old != res); + } + + rdma_restrack_put(res); + wait_for_completion(&res->comp); +} + +static void restrack_restore_res(struct rdma_restrack_root *rt, + struct rdma_restrack_entry *res) +{ + reinit_completion(&res->comp); + kref_init(&res->kref); + + if (rt) { + struct rdma_restrack_entry *old; + + old = xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, + GFP_KERNEL); + WARN_ON(old); + } +} + /** * rdma_restrack_set_name() - set the task for this resource * @res: resource entry @@ -177,22 +217,23 @@ void rdma_restrack_new(struct rdma_restrack_entry *res, EXPORT_SYMBOL(rdma_restrack_new); /** - * rdma_restrack_add() - add object to the resource tracking database + * rdma_restrack_add() - add object to the resource tracking database. + * If this resource reuses an ID of a resource that was already destroyed + * after calling rdma_restrack_begin() but didn't yet call + * rdma_restrack_commit_del() it can result in an untracked QP. * @res: resource entry */ void rdma_restrack_add(struct rdma_restrack_entry *res) { - struct ib_device *dev = res_to_dev(res); struct rdma_restrack_root *rt; int ret = 0; - if (!dev) - return; - if (res->no_track) goto out; - rt = &dev->res[res->type]; + rt = res_to_rt(res); + if (!rt) + return; if (res->type == RDMA_RESTRACK_QP) { /* Special case to ensure that LQPN points to right QP */ @@ -229,6 +270,28 @@ void rdma_restrack_add(struct rdma_restrack_entry *res) } EXPORT_SYMBOL(rdma_restrack_add); +/** + * rdma_restrack_abort_del() - re-add object to the resource tracking database + * it can only be used after rdma_restrack_begin_del(). + * @res: resource entry + */ +void rdma_restrack_abort_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt = NULL; + + if (!res->valid) + return; + + if (!res->no_track) { + rt = res_to_rt(res); + if (!rt) + return; + } + + restrack_restore_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_abort_del); + int __must_check rdma_restrack_get(struct rdma_restrack_entry *res) { return kref_get_unless_zero(&res->kref); @@ -265,7 +328,7 @@ static void restrack_release(struct kref *kref) struct rdma_restrack_entry *res; res = container_of(kref, struct rdma_restrack_entry, kref); - if (res->task) { + if (res->task && !res->valid) { put_task_struct(res->task); res->task = NULL; } @@ -291,37 +354,20 @@ EXPORT_SYMBOL(rdma_restrack_put); */ void rdma_restrack_sync(struct rdma_restrack_entry *res) { - struct rdma_restrack_entry *old; struct rdma_restrack_root *rt; - struct task_struct *task; - struct ib_device *dev; if (!res->valid || res->no_track) return; - dev = res_to_dev(res); - if (WARN_ON(!dev)) + rt = res_to_rt(res); + if (!rt) return; - rt = &dev->res[res->type]; if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD))) return; - old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, GFP_KERNEL); - if (WARN_ON(old != res)) - return; - - task = res->task; - if (task) - get_task_struct(task); - rdma_restrack_put(res); - wait_for_completion(&res->comp); - reinit_completion(&res->comp); - if (task) - res->task = task; - kref_init(&res->kref); - - xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, GFP_KERNEL); + restrack_drain_res(rt, res); + restrack_restore_res(rt, res); } EXPORT_SYMBOL(rdma_restrack_sync); @@ -333,7 +379,6 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) { struct rdma_restrack_entry *old; struct rdma_restrack_root *rt; - struct ib_device *dev; if (!res->valid) { if (res->task) { @@ -346,12 +391,10 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) if (res->no_track) goto out; - dev = res_to_dev(res); - if (WARN_ON(!dev)) + rt = res_to_rt(res); + if (!rt) return; - rt = &dev->res[res->type]; - old = xa_erase(&rt->xa, res->id); WARN_ON(old != res); @@ -359,5 +402,61 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) res->valid = false; rdma_restrack_put(res); wait_for_completion(&res->comp); + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } } EXPORT_SYMBOL(rdma_restrack_del); + +/** + * rdma_restrack_begin_del() - invalidate the object from the resource tracking + * database but preserve its index in the array. + * Since this preserves the index in the array until rdma_restrack_commit_del() + * is called, if rdma_restrack_add() is called in between with an old QP ID it + * can result in an untracked QP. + * @res: resource entry + */ +void rdma_restrack_begin_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt = NULL; + + if (!res->valid) + return; + + if (!res->no_track) { + rt = res_to_rt(res); + if (!rt) + return; + } + + restrack_drain_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_begin_del); + +/** + * rdma_restrack_commit_del() - delete object from the resource tracking + * database and free the task. + * @res: resource entry + */ +void rdma_restrack_commit_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt; + + if (!res->valid || res->no_track) + goto out; + + rt = res_to_rt(res); + if (!rt) + return; + + xa_erase(&rt->xa, res->id); + +out: + res->valid = false; + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } +} +EXPORT_SYMBOL(rdma_restrack_commit_del); diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h index 75b8d1005a984..2df78e084e107 100644 --- a/drivers/infiniband/core/restrack.h +++ b/drivers/infiniband/core/restrack.h @@ -26,8 +26,11 @@ struct rdma_restrack_root { int rdma_restrack_init(struct ib_device *dev); void rdma_restrack_clean(struct ib_device *dev); void rdma_restrack_add(struct rdma_restrack_entry *res); +void rdma_restrack_abort_del(struct rdma_restrack_entry *res); void rdma_restrack_del(struct rdma_restrack_entry *res); void rdma_restrack_sync(struct rdma_restrack_entry *res); +void rdma_restrack_begin_del(struct rdma_restrack_entry *res); +void rdma_restrack_commit_del(struct rdma_restrack_entry *res); void rdma_restrack_new(struct rdma_restrack_entry *res, enum rdma_restrack_type type); void rdma_restrack_set_name(struct rdma_restrack_entry *res, diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c index 878561fa1cb55..a15182f7a7c5b 100644 --- a/drivers/infiniband/core/ucma.c +++ b/drivers/infiniband/core/ucma.c @@ -951,7 +951,7 @@ static ssize_t ucma_query_path(struct ucma_context *ctx, resp->num_paths = ctx->cm_id->route.num_pri_alt_paths; for (i = 0, out_len -= sizeof(*resp); - i < resp->num_paths && out_len > sizeof(struct ib_path_rec_data); + i < resp->num_paths && out_len >= sizeof(struct ib_path_rec_data); i++, out_len -= sizeof(struct ib_path_rec_data)) { struct sa_path_rec *rec = &ctx->cm_id->route.path_rec[i]; @@ -1404,7 +1404,10 @@ static int ucma_set_ib_path(struct ucma_context *ctx, memset(&event, 0, sizeof event); event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; - return ucma_event_handler(ctx->cm_id, &event); + rdma_lock_handler(ctx->cm_id); + ret = ucma_event_handler(ctx->cm_id, &event); + rdma_unlock_handler(ctx->cm_id); + return ret; } static int ucma_set_option_ib(struct ucma_context *ctx, int optname, @@ -1553,9 +1556,10 @@ static ssize_t ucma_process_join(struct ucma_file *file, mutex_lock(&ctx->mutex); ret = rdma_join_multicast(ctx->cm_id, (struct sockaddr *)&mc->addr, join_state, mc); - mutex_unlock(&ctx->mutex); - if (ret) + if (ret) { + mutex_unlock(&ctx->mutex); goto err_xa_erase; + } resp.id = mc->id; if (copy_to_user(u64_to_user_ptr(cmd->response), @@ -1563,6 +1567,7 @@ static ssize_t ucma_process_join(struct ucma_file *file, ret = -EFAULT; goto err_leave_multicast; } + mutex_unlock(&ctx->mutex); xa_store(&multicast_table, mc->id, mc, 0); @@ -1570,7 +1575,6 @@ static ssize_t ucma_process_join(struct ucma_file *file, return 0; err_leave_multicast: - mutex_lock(&ctx->mutex); rdma_leave_multicast(ctx->cm_id, (struct sockaddr *) &mc->addr); mutex_unlock(&ctx->mutex); ucma_cleanup_mc_events(mc); @@ -1776,6 +1780,13 @@ static ssize_t ucma_write_cm_event(struct ucma_file *file, goto out; } + rdma_lock_handler(ctx->cm_id); + if (!ctx->uid) { + kfree(uevent); + ret = -EINVAL; + goto err_unlock; + } + uevent->ctx = ctx; uevent->resp.uid = ctx->uid; uevent->resp.id = ctx->id; @@ -1789,6 +1800,8 @@ static ssize_t ucma_write_cm_event(struct ucma_file *file, mutex_unlock(&ctx->file->mut); wake_up_interruptible(&ctx->file->poll_wait); +err_unlock: + rdma_unlock_handler(ctx->cm_id); out: ucma_put_ctx(ctx); return ret; diff --git a/drivers/infiniband/core/uverbs_flow.c b/drivers/infiniband/core/uverbs_flow.c index 1528a294f7f85..de5a2769f0847 100644 --- a/drivers/infiniband/core/uverbs_flow.c +++ b/drivers/infiniband/core/uverbs_flow.c @@ -26,6 +26,7 @@ struct ib_uflow_resources *flow_resources_alloc(size_t num_specs) return resources; err: + kfree(resources->collection); kfree(resources->counters); kfree(resources); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 3ccf58e96aede..2a046c888dfab 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -556,6 +556,7 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, bundle.ufile = file; bundle.context = NULL; /* only valid if bundle has uobject */ bundle.uobject = NULL; + bundle.method_elm = NULL; if (!method_elm->is_ex) { size_t in_len = hdr.in_words * 4 - sizeof(hdr); size_t out_len = hdr.out_words * 4; @@ -643,12 +644,15 @@ static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma) goto out; } - mutex_lock(&file->disassociation_lock); + if (!down_read_trylock(&file->hw_destroy_rwsem)) { + ret = -EIO; + goto out; + } vma->vm_ops = &rdma_umap_ops; ret = ucontext->device->ops.mmap(ucontext, vma); - mutex_unlock(&file->disassociation_lock); + up_read(&file->hw_destroy_rwsem); out: srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); return ret; @@ -670,7 +674,6 @@ static void rdma_umap_open(struct vm_area_struct *vma) /* We are racing with disassociation */ if (!down_read_trylock(&ufile->hw_destroy_rwsem)) goto out_zap; - mutex_lock(&ufile->disassociation_lock); /* * Disassociation already completed, the VMA should already be zapped. @@ -683,12 +686,10 @@ static void rdma_umap_open(struct vm_area_struct *vma) goto out_unlock; rdma_umap_priv_init(priv, vma, opriv->entry); - mutex_unlock(&ufile->disassociation_lock); up_read(&ufile->hw_destroy_rwsem); return; out_unlock: - mutex_unlock(&ufile->disassociation_lock); up_read(&ufile->hw_destroy_rwsem); out_zap: /* @@ -772,7 +773,7 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile) { struct rdma_umap_priv *priv, *next_priv; - mutex_lock(&ufile->disassociation_lock); + lockdep_assert_held_write(&ufile->hw_destroy_rwsem); while (1) { struct mm_struct *mm = NULL; @@ -798,10 +799,8 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile) break; } mutex_unlock(&ufile->umap_lock); - if (!mm) { - mutex_unlock(&ufile->disassociation_lock); + if (!mm) return; - } /* * The umap_lock is nested under mmap_lock since it used within @@ -831,8 +830,6 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile) mmap_read_unlock(mm); mmput(mm); } - - mutex_unlock(&ufile->disassociation_lock); } /** @@ -850,8 +847,11 @@ void rdma_user_mmap_disassociate(struct ib_device *device) mutex_lock(&uverbs_dev->lists_mutex); list_for_each_entry(ufile, &uverbs_dev->uverbs_file_list, list) { - if (ufile->ucontext) + if (ufile->ucontext) { + down_write(&ufile->hw_destroy_rwsem); uverbs_user_mmap_disassociate(ufile); + up_write(&ufile->hw_destroy_rwsem); + } } mutex_unlock(&uverbs_dev->lists_mutex); } @@ -926,8 +926,6 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp) mutex_init(&file->umap_lock); INIT_LIST_HEAD(&file->umaps); - mutex_init(&file->disassociation_lock); - filp->private_data = file; list_add_tail(&file->list, &dev->uverbs_file_list); mutex_unlock(&dev->lists_mutex); diff --git a/drivers/infiniband/core/uverbs_std_types_dmah.c b/drivers/infiniband/core/uverbs_std_types_dmah.c index 97101e0938263..9873ab49a6013 100644 --- a/drivers/infiniband/core/uverbs_std_types_dmah.c +++ b/drivers/infiniband/core/uverbs_std_types_dmah.c @@ -18,11 +18,14 @@ static int uverbs_free_dmah(struct ib_uobject *uobject, if (atomic_read(&dmah->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&dmah->res); ret = dmah->device->ops.dealloc_dmah(dmah, attrs); - if (ret) + if (ret) { + rdma_restrack_abort_del(&dmah->res); return ret; + } - rdma_restrack_del(&dmah->res); + rdma_restrack_commit_del(&dmah->res); kfree(dmah); return 0; } diff --git a/drivers/infiniband/core/uverbs_std_types_mr.c b/drivers/infiniband/core/uverbs_std_types_mr.c index 570b9656801d7..0c72f801e0d32 100644 --- a/drivers/infiniband/core/uverbs_std_types_mr.c +++ b/drivers/infiniband/core/uverbs_std_types_mr.c @@ -364,7 +364,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_REG_MR)( dmah, attrs); else mr = pd->device->ops.reg_user_mr(pd, addr, length, iova, - access_flags, dmah, NULL); + access_flags, dmah, + &attrs->driver_udata); if (IS_ERR(mr)) return PTR_ERR(mr); @@ -527,7 +528,8 @@ DECLARE_UVERBS_NAMED_METHOD( UA_MANDATORY), UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_MR_RESP_RKEY, UVERBS_ATTR_TYPE(u32), - UA_MANDATORY)); + UA_MANDATORY), + UVERBS_ATTR_UHW()); DECLARE_UVERBS_NAMED_METHOD_DESTROY( UVERBS_METHOD_MR_DESTROY, diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 86811d31092ce..5447d39fa89dd 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -392,6 +392,7 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) { int ret; + rdma_restrack_begin_del(&pd->res); if (pd->__internal_mr) { ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL); WARN_ON(ret); @@ -399,10 +400,12 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) } ret = pd->device->ops.dealloc_pd(pd, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&pd->res); return ret; + } - rdma_restrack_del(&pd->res); + rdma_restrack_commit_del(&pd->res); kfree(pd); return ret; } @@ -1140,16 +1143,20 @@ int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata) if (atomic_read(&srq->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&srq->res); + ret = srq->device->ops.destroy_srq(srq, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&srq->res); return ret; + } atomic_dec(&srq->pd->usecnt); if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd) atomic_dec(&srq->ext.xrc.xrcd->usecnt); if (ib_srq_has_cq(srq->srq_type)) atomic_dec(&srq->ext.cq->usecnt); - rdma_restrack_del(&srq->res); + rdma_restrack_commit_del(&srq->res); kfree(srq); return ret; @@ -2049,11 +2056,13 @@ int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed, u8 *width) return -ENODEV; rtnl_lock(); - rc = __ethtool_get_link_ksettings(netdev, &lksettings); - rtnl_unlock(); - - dev_put(netdev); + if (READ_ONCE(netdev->reg_state) != NETREG_REGISTERED) { + dev_put(netdev); + rtnl_unlock(); + return -ENODEV; + } + rc = __ethtool_get_link_ksettings(netdev, &lksettings); if (!rc && lksettings.base.speed != (u32)SPEED_UNKNOWN) { netdev_speed = lksettings.base.speed; } else { @@ -2062,6 +2071,8 @@ int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed, u8 *width) pr_warn("%s speed is unknown, defaulting to %u\n", netdev->name, netdev_speed); } + dev_put(netdev); + rtnl_unlock(); ib_get_width_and_speed(netdev_speed, lksettings.lanes, speed, width); @@ -2154,6 +2165,8 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (qp->real_qp != qp) return __ib_destroy_shared_qp(qp); + rdma_restrack_begin_del(&qp->res); + sec = qp->qp_sec; if (sec) ib_destroy_qp_security_begin(sec); @@ -2166,6 +2179,7 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (ret) { if (sec) ib_destroy_qp_security_abort(sec); + rdma_restrack_abort_del(&qp->res); return ret; } @@ -2178,7 +2192,7 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (sec) ib_destroy_qp_security_end(sec); - rdma_restrack_del(&qp->res); + rdma_restrack_commit_del(&qp->res); kfree(qp); return ret; } @@ -2244,11 +2258,15 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata) if (atomic_read(&cq->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&cq->res); + ret = cq->device->ops.destroy_cq(cq, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&cq->res); return ret; + } - rdma_restrack_del(&cq->res); + rdma_restrack_commit_del(&cq->res); kfree(cq); return ret; } diff --git a/drivers/infiniband/hw/bng_re/bng_fw.c b/drivers/infiniband/hw/bng_re/bng_fw.c index 50156c300b335..ab6a2d2e95b53 100644 --- a/drivers/infiniband/hw/bng_re/bng_fw.c +++ b/drivers/infiniband/hw/bng_re/bng_fw.c @@ -401,14 +401,15 @@ static int __wait_for_resp(struct bng_re_rcfw *rcfw, u16 cookie) { struct bng_re_cmdq_ctx *cmdq; struct bng_re_crsqe *crsqe; + unsigned long time_left; cmdq = &rcfw->cmdq; crsqe = &rcfw->crsqe_tbl[cookie]; do { - wait_event_timeout(cmdq->waitq, - !crsqe->is_in_used, - secs_to_jiffies(rcfw->max_timeout)); + time_left = wait_event_timeout(cmdq->waitq, + !crsqe->is_in_used, + secs_to_jiffies(rcfw->max_timeout)); if (!crsqe->is_in_used) return 0; @@ -417,6 +418,9 @@ static int __wait_for_resp(struct bng_re_rcfw *rcfw, u16 cookie) if (!crsqe->is_in_used) return 0; + + if (!time_left) + return -ENODEV; } while (true); }; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 5657625290076..3890049e25822 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -695,7 +695,7 @@ int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) struct bnxt_re_dev *rdev = pd->rdev; int ret; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; @@ -712,7 +712,7 @@ int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) &pd->qplib_pd)) atomic_dec(&rdev->stats.res.pd_count); } - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) @@ -844,7 +844,7 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, u8 nw_type; int rc; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -901,7 +901,7 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, if (active_ahs > rdev->stats.res.ah_watermark) rdev->stats.res.ah_watermark = active_ahs; - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) @@ -1015,7 +1015,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) unsigned int flags; int rc; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -1064,7 +1064,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) if (scq_nq != rcq_nq) bnxt_re_synchronize_nq(rcq_nq); - return ib_respond_empty_udata(udata); + return 0; } static u8 __from_ib_qp_type(enum ib_qp_type type) @@ -2148,7 +2148,7 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; int ret; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; @@ -2159,7 +2159,7 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) free_page((unsigned long)srq->uctx_srq_page); ib_umem_release(srq->umem); atomic_dec(&rdev->stats.res.srq_count); - return ib_respond_empty_udata(udata); + return 0; } static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, @@ -2297,34 +2297,25 @@ int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, { struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq); - struct bnxt_re_dev *rdev = srq->rdev; int ret; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; - switch (srq_attr_mask) { - case IB_SRQ_MAX_WR: - /* SRQ resize is not supported */ + if (srq_attr_mask != IB_SRQ_LIMIT) return -EINVAL; - case IB_SRQ_LIMIT: - /* Change the SRQ threshold */ - if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) - return -EINVAL; - - srq->qplib_srq.threshold = srq_attr->srq_limit; - bnxt_qplib_srq_arm_db(&srq->qplib_srq.dbinfo, srq->qplib_srq.threshold); - /* On success, update the shadow */ - srq->srq_limit = srq_attr->srq_limit; - /* No need to Build and send response back to udata */ - return ib_respond_empty_udata(udata); - default: - ibdev_err(&rdev->ibdev, - "Unsupported srq_attr_mask 0x%x", srq_attr_mask); + if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) return -EINVAL; - } + + srq->qplib_srq.threshold = srq_attr->srq_limit; + bnxt_qplib_srq_arm_db(&srq->qplib_srq.dbinfo, srq->qplib_srq.threshold); + + /* On success, update the shadow */ + srq->srq_limit = srq_attr->srq_limit; + /* No need to Build and send response back to udata */ + return 0; } int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) @@ -2437,7 +2428,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, unsigned int flags; u8 nw_type; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -2689,7 +2680,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, if (rc) return rc; } - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, @@ -3471,7 +3462,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) nq = cq->qplib_cq.nq; cctx = rdev->chip_ctx; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; @@ -3486,7 +3477,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) atomic_dec(&rdev->stats.res.cq_count); kfree(cq->cql); ib_umem_release(cq->umem); - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, @@ -3688,6 +3679,10 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, if (rc) goto fail; + rc = ib_respond_empty_udata(udata); + if (rc) + goto fail; + cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va, entries * sizeof(struct cq_base), IB_ACCESS_LOCAL_WRITE); @@ -3717,7 +3712,7 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, cq->ib_cq.cqe = cq->resize_cqe; atomic_inc(&rdev->stats.res.resize_count); - return ib_respond_empty_udata(udata); + return 0; fail: if (cq->resize_umem) { @@ -4449,7 +4444,7 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) struct bnxt_re_dev *rdev = mr->rdev; int rc; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -4472,7 +4467,7 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) atomic_dec(&rdev->stats.res.mr_count); if (rc) return rc; - return ib_respond_empty_udata(udata); + return 0; } static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) @@ -4991,11 +4986,13 @@ int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) case BNXT_RE_MMAP_DBR_PAGE: case BNXT_RE_MMAP_TOGGLE_PAGE: /* Driver doesn't expect write access for user space */ - if (vma->vm_flags & VM_WRITE) + if (vma->vm_flags & VM_WRITE) { ret = -EFAULT; - else + } else { + vm_flags_clear(vma, VM_MAYWRITE); ret = vm_insert_page(vma, vma->vm_start, virt_to_page((void *)bnxt_entry->mem_offset)); + } break; default: ret = -EINVAL; diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index d25fdc458120e..c2fa83eb0da59 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -356,9 +356,13 @@ static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev) return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); } -static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev) +static int bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev) { rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq"); + if (!rdev->dcb_wq) + return -ENOMEM; + + return 0; } static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev) @@ -2343,7 +2347,9 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type) bnxt_re_debugfs_add_pdev(rdev); - bnxt_re_init_dcb_wq(rdev); + rc = bnxt_re_init_dcb_wq(rdev); + if (rc) + goto fail; bnxt_re_net_register_async_event(rdev); if (!rdev->is_virtfn) diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index 263238a6e4cdd..f278c4f70ff7b 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -409,7 +409,6 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a uobj->object = obj; uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_DBR_HANDLE); - dbr.umdbr = dpi->umdbr; dbr.dpi = dpi->dpi; ret = uverbs_copy_to_struct_or_zero(attrs, BNXT_RE_ALLOC_DBR_ATTR, &dbr, sizeof(dbr)); @@ -472,7 +471,6 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_DEFAULT_DBR)(struct uverbs_attr_bun return PTR_ERR(ib_uctx); uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); - dpi.umdbr = uctx->dpi.umdbr; dpi.dpi = uctx->dpi.dpi; ret = uverbs_copy_to_struct_or_zero(attrs, BNXT_RE_DEFAULT_DBR_ATTR, @@ -490,7 +488,7 @@ DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_DBR_ALLOC, UA_MANDATORY), UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_DBR_ATTR, UVERBS_ATTR_STRUCT(struct bnxt_re_db_region, - umdbr), + reserved2), UA_MANDATORY), UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_DBR_OFFSET, UVERBS_ATTR_TYPE(u64), @@ -510,7 +508,7 @@ DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_DBR, DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR, UVERBS_ATTR_PTR_OUT(BNXT_RE_DEFAULT_DBR_ATTR, UVERBS_ATTR_STRUCT(struct bnxt_re_db_region, - umdbr), + reserved2), UA_MANDATORY)); DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR, diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c index 102c5646b9ed6..24ac9871ce67c 100644 --- a/drivers/infiniband/hw/cxgb4/device.c +++ b/drivers/infiniband/hw/cxgb4/device.c @@ -933,6 +933,7 @@ static void c4iw_rdev_close(struct c4iw_rdev *rdev) void c4iw_dealloc(struct uld_ctx *ctx) { + debugfs_remove_recursive(ctx->dev->debugfs_root); c4iw_rdev_close(&ctx->dev->rdev); WARN_ON(!xa_empty(&ctx->dev->cqs)); WARN_ON(!xa_empty(&ctx->dev->qps)); @@ -951,7 +952,12 @@ void c4iw_dealloc(struct uld_ctx *ctx) static void c4iw_remove(struct uld_ctx *ctx) { pr_debug("c4iw_dev %p\n", ctx->dev); - debugfs_remove_recursive(ctx->dev->debugfs_root); + + /* c4iw_register_device() may still be using ctx->dev. */ + cancel_work_sync(&ctx->reg_work); + if (!ctx->dev) + return; + c4iw_unregister_device(ctx->dev); c4iw_dealloc(ctx); } diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index cd1b010141984..ddb24a7fd4e6b 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -199,7 +199,8 @@ static int _c4iw_write_mem_dma(struct c4iw_rdev *rdev, u32 addr, u32 len, daddr = dma_map_single(&rdev->lldi.pdev->dev, data, len, DMA_TO_DEVICE); if (dma_mapping_error(&rdev->lldi.pdev->dev, daddr)) - return -1; + return _c4iw_write_mem_inline(rdev, addr, len, data, skb, + wr_waitp); save = daddr; while (remain > inline_threshold) { @@ -235,30 +236,12 @@ static int write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data, struct sk_buff *skb, struct c4iw_wr_wait *wr_waitp) { - int ret; - - if (!rdev->lldi.ulptx_memwrite_dsgl || !use_dsgl) { - ret = _c4iw_write_mem_inline(rdev, addr, len, data, skb, - wr_waitp); - goto out; - } - - if (len <= inline_threshold) { - ret = _c4iw_write_mem_inline(rdev, addr, len, data, skb, + if (!rdev->lldi.ulptx_memwrite_dsgl || !use_dsgl || + len <= inline_threshold) + return _c4iw_write_mem_inline(rdev, addr, len, data, skb, wr_waitp); - goto out; - } - - ret = _c4iw_write_mem_dma(rdev, addr, len, data, skb, wr_waitp); - if (ret) { - pr_warn_ratelimited("%s: dma map failure (non fatal)\n", - pci_name(rdev->lldi.pdev)); - ret = _c4iw_write_mem_inline(rdev, addr, len, data, skb, - wr_waitp); - } -out: - return ret; + return _c4iw_write_mem_dma(rdev, addr, len, data, skb, wr_waitp); } /* @@ -277,14 +260,19 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, int err; struct fw_ri_tpte *tpt; u32 stag_idx; + bool stag_idx_allocated = false; static atomic_t key; - if (c4iw_fatal_error(rdev)) + if (c4iw_fatal_error(rdev)) { + kfree_skb(skb); return -EIO; + } tpt = kmalloc_obj(*tpt); - if (!tpt) + if (!tpt) { + kfree_skb(skb); return -ENOMEM; + } stag_state = stag_state > 0; stag_idx = (*stag) >> 8; @@ -296,9 +284,11 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, rdev->stats.stag.fail++; mutex_unlock(&rdev->stats.lock); kfree(tpt); + kfree_skb(skb); return -ENOMEM; } mutex_lock(&rdev->stats.lock); + stag_idx_allocated = true; rdev->stats.stag.cur += 32; if (rdev->stats.stag.cur > rdev->stats.stag.max) rdev->stats.stag.max = rdev->stats.stag.cur; @@ -333,7 +323,7 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, (rdev->lldi.vr->stag.start >> 5), sizeof(*tpt), tpt, skb, wr_waitp); - if (reset_tpt_entry) { + if (reset_tpt_entry || (err && stag_idx_allocated)) { c4iw_put_resource(&rdev->resource.tpt_table, stag_idx); mutex_lock(&rdev->stats.lock); rdev->stats.stag.cur -= 32; @@ -469,8 +459,10 @@ struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc) FW_RI_STAG_NSMR, mhp->attr.perms, mhp->attr.mw_bind_enable, 0, 0, ~0ULL, 0, 0, 0, NULL, mhp->wr_waitp); - if (ret) - goto err_free_skb; + if (ret) { + kfree_skb(mhp->dereg_skb); + goto err_free_wr_wait; + } ret = finish_mem_reg(mhp, stag); if (ret) @@ -479,8 +471,6 @@ struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc) err_dereg_mem: dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, mhp->attr.pbl_addr, mhp->dereg_skb, mhp->wr_waitp); -err_free_skb: - kfree_skb(mhp->dereg_skb); err_free_wr_wait: c4iw_put_wr_wait(mhp->wr_waitp); err_free_mhp: diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index 7cc3f4af0bb9d..eed6000fc62df 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -786,7 +786,7 @@ int efa_com_admin_init(struct efa_com_dev *edev, aq->dmadev = edev->dmadev; aq->efa_dev = edev->efa_dev; - set_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state); + efa_com_set_admin_polling_mode(edev, true); sema_init(&aq->avail_cmds, aq->depth); @@ -804,8 +804,6 @@ int efa_com_admin_init(struct efa_com_dev *edev, if (err) goto err_destroy_sq; - efa_com_set_admin_polling_mode(edev, false); - err = efa_com_admin_init_aenq(edev, aenq_handlers); if (err) goto err_destroy_cq; @@ -1194,7 +1192,7 @@ static void efa_com_destroy_eq(struct efa_com_dev *edev, err); } -static void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq) +void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq) { u32 val = 0; @@ -1283,7 +1281,6 @@ int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq, eeq->phase = 1; eeq->depth = params.depth; eeq->cb = cb; - efa_com_arm_eq(edev, eeq); return 0; diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h index f8c692b0e092e..d9b3eb0cdf37c 100644 --- a/drivers/infiniband/hw/efa/efa_com.h +++ b/drivers/infiniband/hw/efa/efa_com.h @@ -159,6 +159,7 @@ int efa_com_admin_init(struct efa_com_dev *edev, void efa_com_admin_destroy(struct efa_com_dev *edev); int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq, efa_eqe_handler cb, u16 depth, u8 msix_vec); +void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq); void efa_com_eq_destroy(struct efa_com_dev *edev, struct efa_com_eq *eeq); int efa_com_dev_reset(struct efa_com_dev *edev, enum efa_regs_reset_reason_types reset_reason); diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 97da8e828e340..c2ecd44476d4f 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -300,28 +300,30 @@ static void efa_set_host_info(struct efa_dev *dev) static void efa_destroy_eq(struct efa_dev *dev, struct efa_eq *eq) { - efa_com_eq_destroy(&dev->edev, &eq->eeq); efa_free_irq(dev, &eq->irq); + efa_com_eq_destroy(&dev->edev, &eq->eeq); } static int efa_create_eq(struct efa_dev *dev, struct efa_eq *eq, u32 msix_vec) { int err; - efa_setup_comp_irq(dev, eq, msix_vec); - err = efa_request_irq(dev, &eq->irq); + err = efa_com_eq_init(&dev->edev, &eq->eeq, efa_process_eqe, + dev->dev_attr.max_eq_depth, msix_vec); if (err) return err; - err = efa_com_eq_init(&dev->edev, &eq->eeq, efa_process_eqe, - dev->dev_attr.max_eq_depth, msix_vec); + efa_setup_comp_irq(dev, eq, msix_vec); + err = efa_request_irq(dev, &eq->irq); if (err) - goto err_free_comp_irq; + goto err_destroy_eq; + + efa_com_arm_eq(&dev->edev, &eq->eeq); return 0; -err_free_comp_irq: - efa_free_irq(dev, &eq->irq); +err_destroy_eq: + efa_com_eq_destroy(&dev->edev, &eq->eeq); return err; } @@ -611,18 +613,21 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev) edev->aq.msix_vector_idx = dev->admin_msix_vector_idx; edev->aenq.msix_vector_idx = dev->admin_msix_vector_idx; - err = efa_set_mgmnt_irq(dev); + err = efa_com_admin_init(edev, &aenq_handlers); if (err) goto err_disable_msix; - err = efa_com_admin_init(edev, &aenq_handlers); + err = efa_set_mgmnt_irq(dev); if (err) - goto err_free_mgmnt_irq; + goto err_destroy_admin; + + efa_com_set_admin_polling_mode(edev, false); return dev; -err_free_mgmnt_irq: - efa_free_irq(dev, &dev->admin_irq); +err_destroy_admin: + efa_com_dev_reset(edev, EFA_REGS_RESET_INIT_ERR); + efa_com_admin_destroy(edev); err_disable_msix: efa_disable_msix(dev); err_reg_read_destroy: @@ -646,8 +651,8 @@ static void efa_remove_device(struct pci_dev *pdev, edev = &dev->edev; efa_com_dev_reset(edev, reset_reason); - efa_com_admin_destroy(edev); efa_free_irq(dev, &dev->admin_irq); + efa_com_admin_destroy(edev); efa_disable_msix(dev); efa_com_mmio_reg_read_destroy(edev); devm_iounmap(&pdev->dev, edev->reg_bar); diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 06d3365aeb569..267a74ac15f3d 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1345,9 +1345,11 @@ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) chunk_list->chunks[i].length = EFA_CHUNK_USED_SIZE; } - chunk_list->chunks[chunk_list_size - 1].length = - ((page_cnt % EFA_PTRS_PER_CHUNK) * EFA_CHUNK_PAYLOAD_PTR_SIZE) + - EFA_CHUNK_PTR_SIZE; + + if (page_cnt % EFA_PTRS_PER_CHUNK != 0) + chunk_list->chunks[chunk_list_size - 1].length = + ((page_cnt % EFA_PTRS_PER_CHUNK) * EFA_CHUNK_PAYLOAD_PTR_SIZE) + + EFA_CHUNK_PTR_SIZE; /* fill the dma addresses of sg list pages to chunks: */ chunk_idx = 0; @@ -1359,9 +1361,12 @@ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) rdma_block_iter_dma_address(&biter); if (payload_idx == EFA_PTRS_PER_CHUNK) { + payload_idx = 0; chunk_idx++; + if (chunk_idx >= chunk_list_size) + break; + cur_chunk_buf = chunk_list->chunks[chunk_idx].buf; - payload_idx = 0; } } diff --git a/drivers/infiniband/hw/erdma/Kconfig b/drivers/infiniband/hw/erdma/Kconfig index 267fc1f3c42af..745e5551773f5 100644 --- a/drivers/infiniband/hw/erdma/Kconfig +++ b/drivers/infiniband/hw/erdma/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config INFINIBAND_ERDMA tristate "Alibaba Elastic RDMA Adapter (ERDMA) support" - depends on PCI_MSI && 64BIT + depends on PCI_MSI && 64BIT && !CPU_BIG_ENDIAN depends on INFINIBAND_ADDR_TRANS depends on INFINIBAND_USER_ACCESS help diff --git a/drivers/infiniband/hw/erdma/erdma_cm.c b/drivers/infiniband/hw/erdma/erdma_cm.c index 5c7d3a8f8038c..70c5df566d9de 100644 --- a/drivers/infiniband/hw/erdma/erdma_cm.c +++ b/drivers/infiniband/hw/erdma/erdma_cm.c @@ -1021,10 +1021,9 @@ int erdma_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params) if (laddr->sa_family != AF_INET || raddr->sa_family != AF_INET) return -EAFNOSUPPORT; - qp = find_qp_by_qpn(dev, params->qpn); + qp = erdma_qp_get_by_qpn(dev, params->qpn); if (!qp) return -ENOENT; - erdma_qp_get(qp); ret = sock_create(AF_INET, SOCK_STREAM, IPPROTO_TCP, &s); if (ret < 0) @@ -1154,10 +1153,9 @@ int erdma_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params) return -ECONNRESET; } - qp = find_qp_by_qpn(dev, params->qpn); + qp = erdma_qp_get_by_qpn(dev, params->qpn); if (!qp) return -ENOENT; - erdma_qp_get(qp); down_write(&qp->state_lock); if (qp->attrs.iwarp.state > ERDMA_QPS_IWARP_RTR) { diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c index d5b9d19882b2a..d5d1704cb57ca 100644 --- a/drivers/infiniband/hw/erdma/erdma_eq.c +++ b/drivers/infiniband/hw/erdma/erdma_eq.c @@ -52,7 +52,7 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) if (FIELD_GET(ERDMA_AEQE_HDR_TYPE_MASK, le32_to_cpu(aeqe->hdr)) == ERDMA_AE_TYPE_CQ_ERR) { cqn = le32_to_cpu(aeqe->event_data0); - cq = find_cq_by_cqn(dev, cqn); + cq = erdma_cq_get_by_cqn(dev, cqn); if (!cq) continue; @@ -62,9 +62,10 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) if (cq->ibcq.event_handler) cq->ibcq.event_handler(&event, cq->ibcq.cq_context); + erdma_cq_put(cq); } else { qpn = le32_to_cpu(aeqe->event_data0); - qp = find_qp_by_qpn(dev, qpn); + qp = erdma_qp_get_by_qpn(dev, qpn); if (!qp) continue; @@ -74,6 +75,7 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) if (qp->ibqp.event_handler) qp->ibqp.event_handler(&event, qp->ibqp.qp_context); + erdma_qp_put(qp); } } @@ -157,7 +159,7 @@ void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) poll_cnt++; cqn = FIELD_GET(ERDMA_CEQE_HDR_CQN_MASK, READ_ONCE(*ceqe)); - cq = find_cq_by_cqn(dev, cqn); + cq = erdma_cq_get_by_cqn(dev, cqn); if (!cq) continue; @@ -166,6 +168,7 @@ void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) if (cq->ibcq.comp_handler) cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); + erdma_cq_put(cq); } notify_eq(&ceq_cb->eq); @@ -220,6 +223,7 @@ static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn) irq_set_affinity_hint(eqc->irq.msix_vector, NULL); free_irq(eqc->irq.msix_vector, eqc); + tasklet_kill(&eqc->tasklet); } static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c index 7e87a815e853b..445182c6bc5d3 100644 --- a/drivers/infiniband/hw/erdma/erdma_main.c +++ b/drivers/infiniband/hw/erdma/erdma_main.c @@ -572,8 +572,8 @@ static int erdma_ib_device_add(struct pci_dev *pdev) INIT_LIST_HEAD(&dev->cep_list); spin_lock_init(&dev->lock); - xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1); - xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1); + xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_IRQ); + xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_IRQ); dev->next_alloc_cqn = 1; dev->next_alloc_qpn = 1; diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 74afe6eb18b0b..5efd3bfb59d65 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -1023,15 +1023,15 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, init_completion(&qp->safe_free); if (qp->ibqp.qp_type == IB_QPT_GSI) { - old_entry = xa_store(&dev->qp_xa, 1, qp, GFP_KERNEL); + old_entry = xa_store_irq(&dev->qp_xa, 1, qp, GFP_KERNEL); if (xa_is_err(old_entry)) ret = xa_err(old_entry); else qp->ibqp.qp_num = 1; } else { - ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp, - XA_LIMIT(1, dev->attrs.max_qp - 1), - &dev->next_alloc_qpn, GFP_KERNEL); + ret = xa_alloc_cyclic_irq(&dev->qp_xa, &qp->ibqp.qp_num, qp, + XA_LIMIT(1, dev->attrs.max_qp - 1), + &dev->next_alloc_qpn, GFP_KERNEL); } if (ret < 0) { @@ -1091,7 +1091,7 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, else free_kernel_qp(qp); err_out_xa: - xa_erase(&dev->qp_xa, QP_ID(qp)); + xa_erase_irq(&dev->qp_xa, QP_ID(qp)); err_out: return ret; } @@ -1304,8 +1304,15 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true); + /* + * A timeout disables the command queue, so retry cannot succeed. Treat + * terminal command failures as diagnostic; propagating them can make + * forced uverbs cleanup discard the last software resource pointers. + */ if (ret) - return ret; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to deregister MR 0x%x: %d\n", + ibmr->lkey, ret); erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8); @@ -1321,6 +1328,7 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct erdma_dev *dev = to_edev(ibcq->device); struct erdma_ucontext *ctx = rdma_udata_to_drv_context( udata, struct erdma_ucontext, ibucontext); + unsigned long flags; int err; struct erdma_cmdq_destroy_cq_req req; @@ -1331,7 +1339,16 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true); if (err) - return err; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to destroy CQ %u: %d\n", + cq->cqn, err); + + xa_lock_irqsave(&dev->cq_xa, flags); + __xa_erase(&dev->cq_xa, cq->cqn); + xa_unlock_irqrestore(&dev->cq_xa, flags); + + erdma_cq_put(cq); + wait_for_completion(&cq->free); if (rdma_is_kernel_res(&cq->ibcq.res)) { dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, @@ -1343,8 +1360,6 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) put_mtt_entries(dev, &cq->user_cq.qbuf_mem); } - xa_erase(&dev->cq_xa, cq->cqn); - return 0; } @@ -1356,6 +1371,7 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) udata, struct erdma_ucontext, ibucontext); struct erdma_cmdq_destroy_qp_req req; union erdma_mod_qp_params params; + unsigned long flags; int err; down_write(&qp->state_lock); @@ -1379,7 +1395,13 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true); if (err) - return err; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to destroy QP %u: %d\n", + QP_ID(qp), err); + + xa_lock_irqsave(&dev->qp_xa, flags); + __xa_erase(&dev->qp_xa, QP_ID(qp)); + xa_unlock_irqrestore(&dev->qp_xa, flags); erdma_qp_put(qp); wait_for_completion(&qp->safe_free); @@ -1394,7 +1416,6 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) if (qp->cep) erdma_cep_put(qp->cep); - xa_erase(&dev->qp_xa, QP_ID(qp)); return 0; } @@ -1971,10 +1992,12 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->ibcq.cqe = depth; cq->depth = depth; cq->assoc_eqn = attr->comp_vector + 1; + refcount_set(&cq->refcount, 1); + init_completion(&cq->free); - ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq, - XA_LIMIT(1, dev->attrs.max_cq - 1), - &dev->next_alloc_cqn, GFP_KERNEL); + ret = xa_alloc_cyclic_irq(&dev->cq_xa, &cq->cqn, cq, + XA_LIMIT(1, dev->attrs.max_cq - 1), + &dev->next_alloc_cqn, GFP_KERNEL); if (ret < 0) return ret; @@ -2020,7 +2043,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, } err_out_xa: - xa_erase(&dev->cq_xa, cq->cqn); + xa_erase_irq(&dev->cq_xa, cq->cqn); return ret; } @@ -2281,7 +2304,9 @@ int erdma_destroy_ah(struct ib_ah *ibah, u32 flags) ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, flags & RDMA_DESTROY_AH_SLEEPABLE); if (ret) - return ret; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to destroy AH %u: %d\n", + ah->ahn, ret); erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn); diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index 7d8d3fe501d5c..c73cecf92f611 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -7,6 +7,9 @@ #ifndef __ERDMA_VERBS_H__ #define __ERDMA_VERBS_H__ +#include +#include + #include "erdma.h" /* RDMA Capability. */ @@ -341,6 +344,8 @@ struct erdma_cq { u32 depth; u32 assoc_eqn; + refcount_t refcount; + struct completion free; union { struct erdma_kcq_info kern_cq; @@ -355,9 +360,40 @@ static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id) return (struct erdma_qp *)xa_load(&dev->qp_xa, id); } -static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id) +static inline struct erdma_qp *erdma_qp_get_by_qpn(struct erdma_dev *dev, + int id) +{ + struct erdma_qp *qp; + unsigned long flags; + + xa_lock_irqsave(&dev->qp_xa, flags); + qp = xa_load(&dev->qp_xa, id); + if (qp && !kref_get_unless_zero(&qp->ref)) + qp = NULL; + xa_unlock_irqrestore(&dev->qp_xa, flags); + + return qp; +} + +static inline struct erdma_cq *erdma_cq_get_by_cqn(struct erdma_dev *dev, + int id) +{ + struct erdma_cq *cq; + unsigned long flags; + + xa_lock_irqsave(&dev->cq_xa, flags); + cq = xa_load(&dev->cq_xa, id); + if (cq && !refcount_inc_not_zero(&cq->refcount)) + cq = NULL; + xa_unlock_irqrestore(&dev->cq_xa, flags); + + return cq; +} + +static inline void erdma_cq_put(struct erdma_cq *cq) { - return (struct erdma_cq *)xa_load(&dev->cq_xa, id); + if (refcount_dec_and_test(&cq->refcount)) + complete(&cq->free); } void erdma_qp_get(struct erdma_qp *qp); diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index 56031becb2731..ad2044d1f12c0 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c @@ -326,6 +326,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) void *memvirt = NULL; dma_addr_t memdma = 0; u8 subctxt, mapio = 0, vmf = 0, type; + size_t memdmalen = 0; ssize_t memlen = 0; int ret = 0; u16 ctxt; @@ -371,7 +372,9 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) mapio = 1; break; case PIO_CRED: { + struct credit_return_base *cr = &dd->cr_base[uctxt->sc->node]; u64 cr_page_offset; + if (flags & VM_WRITE) { ret = -EPERM; goto done; @@ -381,11 +384,18 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) * second or third page allocated for credit returns (if number * of enabled contexts > 64 and 128 respectively). */ - cr_page_offset = ((u64)uctxt->sc->hw_free - - (u64)dd->cr_base[uctxt->numa_id].va) & - PAGE_MASK; - memvirt = dd->cr_base[uctxt->numa_id].va + cr_page_offset; - memdma = dd->cr_base[uctxt->numa_id].dma + cr_page_offset; + cr_page_offset = ((u64)uctxt->sc->hw_free - (u64)cr->va) & + PAGE_MASK; + /* + * dma_mmap_coherent() describes the whole coherent buffer and + * selects the page within it with vma->vm_pgoff, so pass the + * base of the allocation and its length and let vm_pgoff pick + * the page. + */ + vma->vm_pgoff = cr_page_offset >> PAGE_SHIFT; + memvirt = cr->va; + memdma = cr->dma; + memdmalen = TXE_NUM_CONTEXTS * sizeof(struct credit_return); memlen = PAGE_SIZE; flags &= ~VM_MAYWRITE; flags |= VM_DONTCOPY | VM_DONTEXPAND; @@ -567,7 +577,8 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) ret = 0; } else if (memdma) { ret = dma_mmap_coherent(&dd->pcidev->dev, vma, - memvirt, memdma, memlen); + memvirt, memdma, + memdmalen ? memdmalen : memlen); } else if (mapio) { ret = io_remap_pfn_range(vma, vma->vm_start, PFN_DOWN(memaddr), diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index b7fd8b1fbbbde..c5f005e5b812c 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -1225,8 +1225,9 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, GFP_KERNEL); if (ret < 0) { dev_err(&pdev->dev, - "Could not allocate unit ID: error %d\n", -ret); - goto bail; + "Could not allocate unit ID: error %pe\n", ERR_PTR(ret)); + rvt_dealloc_device(&dd->verbs_dev.rdi); + return ERR_PTR(ret); } /* @@ -1655,11 +1656,8 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) * we still create devices, so diags, etc. can be used * to determine cause of problem. */ - if (!initfail && !ret) { + if (!initfail && !ret) dd->flags |= HFI1_INITTED; - /* create debufs files after init and ib register */ - hfi1_dbg_ibdev_init(&dd->verbs_dev); - } j = hfi1_device_create(dd); if (j) @@ -1668,7 +1666,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (initfail || ret) { msix_clean_up_interrupts(dd); stop_timers(dd); - flush_workqueue(ib_wq); for (pidx = 0; pidx < dd->num_pports; ++pidx) { hfi1_quiet_serdes(dd->pport + pidx); ppd = dd->pport + pidx; @@ -1685,6 +1682,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) hfi1_device_remove(dd); if (!ret) hfi1_unregister_ib_device(dd); + hfi1_free_rx(dd); postinit_cleanup(dd); if (initfail) ret = initfail; @@ -1692,6 +1690,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) } sdma_start(dd); + hfi1_dbg_ibdev_init(&dd->verbs_dev); return 0; @@ -1741,9 +1740,6 @@ static void remove_one(struct pci_dev *pdev) stop_timers(dd); - /* wait until all of our (qsfp) queue_work() calls complete */ - flush_workqueue(ib_wq); - postinit_cleanup(dd); } diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c index 8ea5ed918a028..be6b82ba93afc 100644 --- a/drivers/infiniband/hw/hfi1/user_sdma.c +++ b/drivers/infiniband/hw/hfi1/user_sdma.c @@ -1026,6 +1026,7 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, struct user_sdma_txreq *tx, u32 datalen) { u32 ahg[AHG_KDETH_ARRAY_SIZE]; + int ret; int idx = 0; u8 omfactor; /* KDETH.OM */ struct hfi1_user_sdma_pkt_q *pq = req->pq; @@ -1130,11 +1131,13 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, trace_hfi1_sdma_user_header_ahg(pq->dd, pq->ctxt, pq->subctxt, req->info.comp_idx, req->sde->this_idx, req->ahg_idx, ahg, idx, tidval); - sdma_txinit_ahg(&tx->txreq, - SDMA_TXREQ_F_USE_AHG, - datalen, req->ahg_idx, idx, - ahg, sizeof(req->hdr), - user_sdma_txreq_cb); + ret = sdma_txinit_ahg(&tx->txreq, + SDMA_TXREQ_F_USE_AHG, + datalen, req->ahg_idx, idx, + ahg, sizeof(req->hdr), + user_sdma_txreq_cb); + if (ret) + return ret; return idx; } diff --git a/drivers/infiniband/hw/ionic/ionic_hw_stats.c b/drivers/infiniband/hw/ionic/ionic_hw_stats.c index f72c9837e135e..4f0a2dedbdfe7 100644 --- a/drivers/infiniband/hw/ionic/ionic_hw_stats.c +++ b/drivers/infiniband/hw/ionic/ionic_hw_stats.c @@ -235,35 +235,34 @@ static int ionic_get_hw_stats(struct ib_device *ibdev, static struct rdma_hw_stats * ionic_counter_alloc_stats(struct rdma_counter *counter) { + struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); - struct ionic_counter *cntr; - int err; + struct rdma_hw_stats *stats; + int id; - cntr = kzalloc_obj(*cntr); - if (!cntr) - return NULL; - - /* buffer for current values from the device */ cntr->vals = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!cntr->vals) - goto err_vals; + return NULL; - err = xa_alloc(&dev->counter_stats->xa_counters, &counter->id, - cntr, - XA_LIMIT(0, IONIC_MAX_QPID), - GFP_KERNEL); - if (err) - goto err_xa; + id = ida_alloc_max(&dev->counter_stats->counter_ida, + IONIC_MAX_QPID, GFP_KERNEL); + if (id < 0) + goto err_ida; - INIT_LIST_HEAD(&cntr->qp_list); + counter->id = id; + + stats = rdma_alloc_hw_stats_struct(dev->counter_stats->stats_hdrs, + dev->counter_stats->queue_stats_count, + RDMA_HW_STATS_DEFAULT_LIFESPAN); + if (!stats) + goto err_hw_stats; + + return stats; - return rdma_alloc_hw_stats_struct(dev->counter_stats->stats_hdrs, - dev->counter_stats->queue_stats_count, - RDMA_HW_STATS_DEFAULT_LIFESPAN); -err_xa: +err_hw_stats: + ida_free(&dev->counter_stats->counter_ida, id); +err_ida: kfree(cntr->vals); -err_vals: - kfree(cntr); return NULL; } @@ -271,14 +270,10 @@ ionic_counter_alloc_stats(struct rdma_counter *counter) static int ionic_counter_dealloc(struct rdma_counter *counter) { struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); - struct ionic_counter *cntr; - - cntr = xa_erase(&dev->counter_stats->xa_counters, counter->id); - if (!cntr) - return -EINVAL; + struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); + ida_free(&dev->counter_stats->counter_ida, counter->id); kfree(cntr->vals); - kfree(cntr); return 0; } @@ -287,13 +282,8 @@ static int ionic_counter_bind_qp(struct rdma_counter *counter, struct ib_qp *ibqp, u32 port) { - struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); + struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); struct ionic_qp *qp = to_ionic_qp(ibqp); - struct ionic_counter *cntr; - - cntr = xa_load(&dev->counter_stats->xa_counters, counter->id); - if (!cntr) - return -EINVAL; list_add_tail(&qp->qp_list_counter, &cntr->qp_list); ibqp->counter = counter; @@ -313,29 +303,23 @@ static int ionic_counter_unbind_qp(struct ib_qp *ibqp, u32 port) return 0; } -static int ionic_get_qp_stats(struct ib_device *ibdev, - struct rdma_hw_stats *hw_stats, - u32 counter_id) +static int ionic_counter_update_stats(struct rdma_counter *counter) { - struct ionic_ibdev *dev = to_ionic_ibdev(ibdev); - struct ionic_counter_stats *cs; - struct ionic_counter *cntr; + struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); + struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); + struct ionic_counter_stats *cs = dev->counter_stats; dma_addr_t hw_stats_dma; struct ionic_qp *qp; int rc, stat_i = 0; - cs = dev->counter_stats; - cntr = xa_load(&cs->xa_counters, counter_id); - if (!cntr) - return -EINVAL; - hw_stats_dma = dma_map_single(dev->lif_cfg.hwdev, cntr->vals, PAGE_SIZE, DMA_FROM_DEVICE); rc = dma_mapping_error(dev->lif_cfg.hwdev, hw_stats_dma); if (rc) return rc; - memset(hw_stats->value, 0, sizeof(u64) * hw_stats->num_counters); + memset(counter->stats->value, 0, + sizeof(u64) * counter->stats->num_counters); list_for_each_entry(qp, &cntr->qp_list, qp_list_counter) { rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE, @@ -345,7 +329,7 @@ static int ionic_get_qp_stats(struct ib_device *ibdev, goto err_cmd; for (stat_i = 0; stat_i < cs->queue_stats_count; ++stat_i) - hw_stats->value[stat_i] += + counter->stats->value[stat_i] += ionic_v1_stat_val(&cs->hdr[stat_i], cntr->vals, PAGE_SIZE); @@ -360,11 +344,6 @@ static int ionic_get_qp_stats(struct ib_device *ibdev, return rc; } -static int ionic_counter_update_stats(struct rdma_counter *counter) -{ - return ionic_get_qp_stats(counter->device, counter->stats, counter->id); -} - static int ionic_alloc_counters(struct ionic_ibdev *dev) { struct ionic_counter_stats *cs = dev->counter_stats; @@ -424,12 +403,22 @@ static const struct ib_device_ops ionic_hw_stats_ops = { .get_hw_stats = ionic_get_hw_stats, }; +static void ionic_counter_init(struct rdma_counter *counter) +{ + struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); + + INIT_LIST_HEAD(&cntr->qp_list); +} + static const struct ib_device_ops ionic_counter_stats_ops = { .counter_alloc_stats = ionic_counter_alloc_stats, .counter_dealloc = ionic_counter_dealloc, .counter_bind_qp = ionic_counter_bind_qp, .counter_unbind_qp = ionic_counter_unbind_qp, .counter_update_stats = ionic_counter_update_stats, + .counter_init = ionic_counter_init, + + INIT_RDMA_OBJ_SIZE(rdma_counter, ionic_rdma_counter, rdma_counter), }; void ionic_stats_init(struct ionic_ibdev *dev) @@ -458,7 +447,7 @@ void ionic_stats_init(struct ionic_ibdev *dev) return; } - xa_init_flags(&dev->counter_stats->xa_counters, XA_FLAGS_ALLOC); + ida_init(&dev->counter_stats->counter_ida); ib_set_device_ops(&dev->ibdev, &ionic_counter_stats_ops); } @@ -467,7 +456,7 @@ void ionic_stats_init(struct ionic_ibdev *dev) void ionic_stats_cleanup(struct ionic_ibdev *dev) { if (dev->counter_stats) { - xa_destroy(&dev->counter_stats->xa_counters); + ida_destroy(&dev->counter_stats->counter_ida); kfree(dev->counter_stats->hdr); kfree(dev->counter_stats->stats_hdrs); kfree(dev->counter_stats); diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h index 63828240d6590..1c43344ec93ce 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.h +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h @@ -331,14 +331,21 @@ struct ionic_counter_stats { int queue_stats_count; struct ionic_v1_stat *hdr; struct rdma_stat_desc *stats_hdrs; - struct xarray xa_counters; + struct ida counter_ida; }; -struct ionic_counter { +struct ionic_rdma_counter { + struct rdma_counter rdma_counter; void *vals; struct list_head qp_list; }; +static inline struct ionic_rdma_counter * +to_ionic_rdma_counter(struct rdma_counter *counter) +{ + return container_of(counter, struct ionic_rdma_counter, rdma_counter); +} + static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev) { return container_of(ibdev, struct ionic_ibdev, ibdev); diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c index f3cd281c3a2f8..45846dc2e89b2 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c @@ -70,7 +70,7 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg) * eq_count is tunable; see ionic_eq_count */ cfg->aq_count = le32_to_cpu(ident->rdma.aq_qtype.qid_count); - cfg->eq_count = le32_to_cpu(ident->rdma.eq_qtype.qid_count); + cfg->eq_count = lif->ionic->neqs_per_lif; cfg->cq_count = le32_to_cpu(ident->rdma.cq_qtype.qid_count); cfg->qp_count = le32_to_cpu(ident->rdma.sq_qtype.qid_count); cfg->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif); diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index e4037d5ef8993..290ad02ed6575 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -1168,6 +1168,12 @@ void irdma_free_qp_rsrc(struct irdma_qp *iwqp) iwqp->kqp.dma_mem.va = NULL; kfree(iwqp->kqp.sq_wrid_mem); kfree(iwqp->kqp.rq_wrid_mem); + + if (iwqp->user_mode && iwqp->iwpbl) { + struct irdma_mr *iwmr = iwqp->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } } /** diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index b7388b41ed958..e926eb37982c6 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -464,6 +464,9 @@ static struct irdma_pbl *irdma_get_pbl(unsigned long va, list_for_each_entry (iwpbl, pbl_list, list) { if (iwpbl->user_base == va) { + struct irdma_mr *iwmr = iwpbl->iwmr; + + refcount_inc(&iwmr->user_ring_refs); list_del(&iwpbl->list); iwpbl->on_list = false; return iwpbl; @@ -1880,6 +1883,11 @@ static void irdma_srq_free_rsrc(struct irdma_pci_f *rf, struct irdma_srq *iwsrq) dma_free_coherent(rf->sc_dev.hw->device, iwsrq->kmem.size, iwsrq->kmem.va, iwsrq->kmem.pa); iwsrq->kmem.va = NULL; + } else { + /* Not called in any failure path, so iwpbl is valid. */ + struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); } irdma_free_rsrc(rf, rf->allocated_srqs, srq->srq_uk.srq_id); @@ -1902,6 +1910,21 @@ static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) iwcq->kmem_shadow.size, iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); iwcq->kmem_shadow.va = NULL; + } else { + struct irdma_mr *iwmr; + + /* May be called in a failure path before iwpbl is valid. */ + if (iwcq->iwpbl) { + iwmr = iwcq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } + + if (iwcq->iwpbl_shadow) { + iwmr = iwcq->iwpbl_shadow->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } } irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); @@ -2017,7 +2040,7 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, struct irdma_modify_cq_info info = {}; struct irdma_dma_mem kmem_buf; struct irdma_cq_mr *cqmr_buf; - struct irdma_pbl *iwpbl_buf; + struct irdma_pbl *iwpbl_buf = NULL; struct irdma_device *iwdev; struct irdma_pci_f *rf; struct irdma_cq_buf *cq_buf = NULL; @@ -2128,6 +2151,19 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, goto error; spin_lock_irqsave(&iwcq->lock, flags); + if (udata) { + struct irdma_pbl *old_iwpbl = iwcq->iwpbl; + + /* Only update if the resize was successful. Otherwise, HW is + * still pointing to the old PBL. + */ + iwcq->iwpbl = iwpbl_buf; + if (old_iwpbl) { + struct irdma_mr *old_iwmr = old_iwpbl->iwmr; + + refcount_dec(&old_iwmr->user_ring_refs); + } + } if (cq_buf) { cq_buf->kmem_buf = iwcq->kmem; cq_buf->hw = dev->hw; @@ -2143,6 +2179,11 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, return 0; error: + if (iwpbl_buf) { + struct irdma_mr *iwmr = iwpbl_buf->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } if (!udata) { dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, kmem_buf.pa); @@ -2419,6 +2460,11 @@ static int irdma_create_srq(struct ib_srq *ibsrq, dma_free_coherent(rf->hw.device, iwsrq->kmem.size, iwsrq->kmem.va, iwsrq->kmem.pa); free_rsrc: + if (iwsrq->user_mode && iwsrq->iwpbl) { + struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } irdma_free_rsrc(rf, rf->allocated_srqs, iwsrq->srq_num); return err_code; } @@ -2498,6 +2544,8 @@ static int irdma_create_cq(struct ib_cq *ibcq, INIT_LIST_HEAD(&iwcq->resize_list); INIT_LIST_HEAD(&iwcq->cmpl_generated); iwcq->cq_num = cq_num; + iwcq->iwpbl = NULL; + iwcq->iwpbl_shadow = NULL; info.dev = dev; ukinfo->cq_size = max(entries, 4); ukinfo->cq_id = cq_num; @@ -2517,8 +2565,6 @@ static int irdma_create_cq(struct ib_cq *ibcq, struct irdma_ucontext *ucontext; struct irdma_create_cq_req req = {}; struct irdma_cq_mr *cqmr; - struct irdma_pbl *iwpbl; - struct irdma_pbl *iwpbl_shadow; struct irdma_cq_mr *cqmr_shadow; iwcq->user_mode = true; @@ -2532,34 +2578,34 @@ static int irdma_create_cq(struct ib_cq *ibcq, } spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, - &ucontext->cq_reg_mem_list); + iwcq->iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, + &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - if (!iwpbl) { + if (!iwcq->iwpbl) { err_code = -EPROTO; goto cq_free_rsrc; } - cqmr = &iwpbl->cq_mr; + cqmr = &iwcq->iwpbl->cq_mr; if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) { spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - iwpbl_shadow = irdma_get_pbl( + iwcq->iwpbl_shadow = irdma_get_pbl( (unsigned long)req.user_shadow_area, &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - if (!iwpbl_shadow) { + if (!iwcq->iwpbl_shadow) { err_code = -EPROTO; goto cq_free_rsrc; } - cqmr_shadow = &iwpbl_shadow->cq_mr; + cqmr_shadow = &iwcq->iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; } else { info.shadow_area_pa = cqmr->shadow; } - if (iwpbl->pbl_allocated) { + if (iwcq->iwpbl->pbl_allocated) { info.virtual_map = true; info.pbl_chunk_size = 1; info.first_pm_pbl_idx = cqmr->cq_pbl.idx; @@ -3362,6 +3408,7 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, if (!iwmr) return ERR_PTR(-ENOMEM); + refcount_set(&iwmr->user_ring_refs, 1); iwpbl = &iwmr->iwpbl; iwpbl->iwmr = iwmr; iwmr->region = region; @@ -3929,41 +3976,41 @@ static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) * irdma_del_memlist - Deleting pbl list entries for CQ/QP * @iwmr: iwmr for IB's user page addresses * @ucontext: ptr to user context + * + * Return: True if the MR is currently in-use by a QP/CQ/SRQ ring. */ -static void irdma_del_memlist(struct irdma_mr *iwmr, +static bool irdma_del_memlist(struct irdma_mr *iwmr, struct irdma_ucontext *ucontext) { struct irdma_pbl *iwpbl = &iwmr->iwpbl; unsigned long flags; + spinlock_t *lock; + bool in_use = false; switch (iwmr->type) { case IRDMA_MEMREG_TYPE_CQ: - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + lock = &ucontext->cq_reg_mem_list_lock; break; case IRDMA_MEMREG_TYPE_QP: - spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + lock = &ucontext->qp_reg_mem_list_lock; break; case IRDMA_MEMREG_TYPE_SRQ: - spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); + lock = &ucontext->srq_reg_mem_list_lock; break; default: - break; + return false; } + + spin_lock_irqsave(lock, flags); + if (!refcount_dec_if_one(&iwmr->user_ring_refs)) { + in_use = true; + } else if (iwpbl->on_list) { + iwpbl->on_list = false; + list_del(&iwpbl->list); + } + spin_unlock_irqrestore(lock, flags); + + return in_use; } /** @@ -3986,7 +4033,12 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - irdma_del_memlist(iwmr, ucontext); + + /* Do not allow the MR to be unpinned if it is still + * backing a user ring. + */ + if (irdma_del_memlist(iwmr, ucontext)) + return -EBUSY; } goto done; } @@ -4192,7 +4244,7 @@ static int irdma_post_send(struct ib_qp *ibqp, stag_info.total_len = iwmr->ibmr.length; stag_info.reg_addr_pa = *palloc->level1.addr; stag_info.first_pm_pbl_index = palloc->level1.idx; - stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; + stag_info.local_fence = true; if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) stag_info.chunk_size = 1; err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h index 289ebc9b23ca7..a1651641eb714 100644 --- a/drivers/infiniband/hw/irdma/verbs.h +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -120,6 +120,7 @@ struct irdma_mr { u64 len; u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR]; struct irdma_pbl iwpbl; + refcount_t user_ring_refs; }; struct irdma_srq { @@ -152,6 +153,8 @@ struct irdma_cq { struct list_head resize_list; struct irdma_cq_poll_info cur_cqe; struct list_head cmpl_generated; + struct irdma_pbl *iwpbl; + struct irdma_pbl *iwpbl_shadow; }; struct irdma_cmpl_gen { diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 60926f39ab9da..389dad4ae1ff0 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -472,6 +472,12 @@ static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num); } +static void mana_table_drain_qp_ref(struct mana_ib_qp *qp) +{ + mana_put_qp_ref(qp); + wait_for_completion(&qp->free); +} + static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) { u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK; @@ -490,6 +496,7 @@ static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *q remove_sq: xa_erase_irq(&mdev->qp_table_wq, qids); + mana_table_drain_qp_ref(qp); return err; } @@ -537,8 +544,7 @@ static void mana_table_remove_qp(struct mana_ib_dev *mdev, qp->ibqp.qp_type); return; } - mana_put_qp_ref(qp); - wait_for_completion(&qp->free); + mana_table_drain_qp_ref(qp); } static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, diff --git a/drivers/infiniband/hw/mlx4/sysfs.c b/drivers/infiniband/hw/mlx4/sysfs.c index e688ad66a895f..5438224bf325b 100644 --- a/drivers/infiniband/hw/mlx4/sysfs.c +++ b/drivers/infiniband/hw/mlx4/sysfs.c @@ -751,11 +751,13 @@ static int register_one_pkey_tree(struct mlx4_ib_dev *dev, int slave) kobject_put(p); } kobject_put(dev->dev_ports_parent[slave]); + dev->dev_ports_parent[slave] = NULL; err_ports: kobject_put(dev->pkeys.device_parent[slave]); /* extra put for the device_parent create_and_add */ kobject_put(dev->pkeys.device_parent[slave]); + dev->pkeys.device_parent[slave] = NULL; fail_dev: kobject_put(dev->iov_parent); @@ -785,6 +787,8 @@ static void unregister_pkey_tree(struct mlx4_ib_dev *device) return; for (slave = device->dev->persist->num_vfs; slave >= 0; --slave) { + if (!device->pkeys.device_parent[slave]) + continue; list_for_each_entry_safe(p, t, &device->pkeys.pkey_port_list[slave], entry) { diff --git a/drivers/infiniband/hw/mlx5/cong.c b/drivers/infiniband/hw/mlx5/cong.c index d0edf83a2f20f..42d005a5a7df4 100644 --- a/drivers/infiniband/hw/mlx5/cong.c +++ b/drivers/infiniband/hw/mlx5/cong.c @@ -361,7 +361,7 @@ static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u32 port_num, MLX5_SET(field_select_r_roce_rp, field, field_select_r_roce_rp, attr_mask); - err = mlx5_cmd_exec_in(dev->mdev, modify_cong_params, in); + err = mlx5_cmd_exec_in(mdev, modify_cong_params, in); kvfree(in); alloc_err: mlx5_ib_put_native_port_mdev(dev, port_num + 1); @@ -399,15 +399,13 @@ static ssize_t get_param(struct file *filp, char __user *buf, size_t count, int offset = param->offset; u32 var = 0; int ret; - char lbuf[11]; + char lbuf[12]; ret = mlx5_ib_get_cc_params(param->dev, param->port_num, offset, &var); if (ret) return ret; - ret = snprintf(lbuf, sizeof(lbuf), "%d\n", var); - if (ret < 0) - return ret; + ret = scnprintf(lbuf, sizeof(lbuf), "%u\n", var); return simple_read_from_buffer(buf, count, pos, lbuf, ret); } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 02809114fc79a..05985395fca00 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1631,14 +1631,15 @@ static int mlx5_ib_query_port_speed_from_vport(struct mlx5_core_dev *mdev, u32 port_num) { u32 max_tx_speed; + u8 vport_state; int err; err = mlx5_query_vport_max_tx_speed(mdev, op_mod, vport, other_vport, - &max_tx_speed); + &max_tx_speed, &vport_state); if (err) return err; - if (max_tx_speed == 0) + if (vport_state == VPORT_STATE_DOWN || max_tx_speed == 0) /* Value 0 indicates field not supported, fallback */ return mlx5_ib_query_port_speed_from_port(dev, port_num, speed); @@ -1683,11 +1684,8 @@ static int mlx5_ib_query_port_speed_rep(struct mlx5_ib_dev *dev, u32 port_num, struct mlx5_core_dev *mdev; u16 op_mod; - if (!dev->port[port_num - 1].rep) { - mlx5_ib_warn(dev, "Representor doesn't exist for port %u\n", - port_num); - return -EINVAL; - } + if (!dev->port[port_num - 1].rep) + return -ENODEV; rep = dev->port[port_num - 1].rep; mdev = mlx5_eswitch_get_core_dev(rep->esw); @@ -5538,6 +5536,7 @@ static int __init mlx5_ib_init(void) dd_err: mlx5r_rep_cleanup(); rep_err: + rcu_barrier(); mlx5_ib_qp_event_cleanup(); qp_event_err: destroy_workqueue(mlx5_ib_event_wq); @@ -5551,6 +5550,7 @@ static void __exit mlx5_ib_cleanup(void) auxiliary_driver_unregister(&mlx5r_driver); auxiliary_driver_unregister(&mlx5r_mp_driver); mlx5r_rep_cleanup(); + rcu_barrier(); mlx5_ib_qp_event_cleanup(); destroy_workqueue(mlx5_ib_event_wq); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 7ff02d89c31d5..e25ac139e43f0 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -647,6 +647,7 @@ static int set_user_buf_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr) { int desc_sz = 1 << qp->sq.wqe_shift; + int rq_buf_size, sq_buf_size; if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) { mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n", @@ -671,11 +672,21 @@ static int set_user_buf_size(struct mlx5_ib_dev *dev, if (attr->qp_type == IB_QPT_RAW_PACKET || qp->flags & IB_QP_CREATE_SOURCE_QPN) { - base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift; - qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6; + if (check_shl_overflow(qp->rq.wqe_cnt, qp->rq.wqe_shift, + &base->ubuffer.buf_size)) + return -EINVAL; + if (check_shl_overflow(qp->sq.wqe_cnt, 6, + &qp->raw_packet_qp.sq.ubuffer.buf_size)) + return -EINVAL; } else { - base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + - (qp->sq.wqe_cnt << 6); + if (check_shl_overflow(qp->rq.wqe_cnt, qp->rq.wqe_shift, + &rq_buf_size)) + return -EINVAL; + if (check_shl_overflow(qp->sq.wqe_cnt, 6, &sq_buf_size)) + return -EINVAL; + if (check_add_overflow(rq_buf_size, sq_buf_size, + &base->ubuffer.buf_size)) + return -EINVAL; } return 0; @@ -1004,7 +1015,11 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, qp->rq.offset = 0; qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); - qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; + if (check_shl_overflow(qp->rq.wqe_cnt, qp->rq.wqe_shift, + &qp->sq.offset)) { + err = -EINVAL; + goto err_bfreg; + } err = set_user_buf_size(dev, qp, ucmd, base, attr); if (err) diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c index 5fa3a1f333268..f37d6d64adb9d 100644 --- a/drivers/infiniband/sw/rdmavt/vt.c +++ b/drivers/infiniband/sw/rdmavt/vt.c @@ -55,8 +55,10 @@ struct rvt_dev_info *rvt_alloc_device(size_t size, int nports) return rdi; rdi->ports = kzalloc_objs(*rdi->ports, nports); - if (!rdi->ports) + if (!rdi->ports) { ib_dealloc_device(&rdi->ibdev); + return NULL; + } return rdi; } diff --git a/drivers/infiniband/sw/rxe/rxe_mcast.c b/drivers/infiniband/sw/rxe/rxe_mcast.c index acd03bd87794e..5ca9211ab33bc 100644 --- a/drivers/infiniband/sw/rxe/rxe_mcast.c +++ b/drivers/infiniband/sw/rxe/rxe_mcast.c @@ -175,7 +175,9 @@ struct rxe_mcg *rxe_lookup_mcg(struct rxe_dev *rxe, union ib_gid *mgid) * @mgid: multicast address as a gid * @mcg: new mcg object * - * Context: caller should hold rxe->mcg lock + * Initializes the mcg fields. The mcg is private and not yet visible in + * mcg_tree, so this may run without rxe->mcg_lock; __rxe_publish_mcg() + * makes it visible under the lock once it is ready. */ static void __rxe_init_mcg(struct rxe_dev *rxe, union ib_gid *mgid, struct rxe_mcg *mcg) @@ -184,13 +186,22 @@ static void __rxe_init_mcg(struct rxe_dev *rxe, union ib_gid *mgid, memcpy(&mcg->mgid, mgid, sizeof(mcg->mgid)); INIT_LIST_HEAD(&mcg->qp_list); mcg->rxe = rxe; +} - /* caller holds a ref on mcg but that will be - * dropped when mcg goes out of scope. We need to take a ref - * on the pointer that will be saved in the red-black tree - * by __rxe_insert_mcg and used to lookup mcg from mgid later. - * Inserting mcg makes it visible to outside so this should - * be done last after the object is ready. +/** + * __rxe_publish_mcg - make a fully initialized mcg visible in mcg_tree + * @mcg: the mcg object + * + * Context: caller must hold rxe->mcg_lock and a reference on mcg + */ +static void __rxe_publish_mcg(struct rxe_mcg *mcg) +{ + /* caller holds a ref on mcg but that will be dropped when mcg goes + * out of scope. We need to take a ref on the pointer that will be + * saved in the red-black tree by __rxe_insert_mcg and used to lookup + * mcg from mgid later. Inserting mcg makes it visible to outside so + * this is done last after the object is ready and the multicast + * address has been programmed. */ kref_get(&mcg->ref_cnt); __rxe_insert_mcg(mcg); @@ -228,26 +239,37 @@ static struct rxe_mcg *rxe_get_mcg(struct rxe_dev *rxe, union ib_gid *mgid) err = -ENOMEM; goto err_dec; } + __rxe_init_mcg(rxe, mgid, mcg); + + /* program the multicast address while mcg is still private, before + * it is inserted into mcg_tree. dev_mc_add() may sleep so this must + * run outside mcg_lock. On failure mcg was never published, so a + * plain free is correct and the tree is untouched. + */ + err = rxe_mcast_add(rxe, mgid); + if (err) { + kfree(mcg); + goto err_dec; + } spin_lock_bh(&rxe->mcg_lock); - /* re-check to see if someone else just added it */ + /* re-check to see if someone else just added it while we were adding + * the multicast address; if so use theirs and drop ours + */ tmp = __rxe_lookup_mcg(rxe, mgid); if (tmp) { spin_unlock_bh(&rxe->mcg_lock); + rxe_mcast_del(rxe, mgid); atomic_dec(&rxe->mcg_num); kfree(mcg); return tmp; } - __rxe_init_mcg(rxe, mgid, mcg); + __rxe_publish_mcg(mcg); spin_unlock_bh(&rxe->mcg_lock); - /* add mcast address outside of lock */ - err = rxe_mcast_add(rxe, mgid); - if (!err) - return mcg; + return mcg; - kfree(mcg); err_dec: atomic_dec(&rxe->mcg_num); return ERR_PTR(err); diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index 875eceb55fdfa..71d9ea4772890 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -33,7 +33,8 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length) case IB_MR_TYPE_USER: case IB_MR_TYPE_MEM_REG: if (iova < mr->ibmr.iova || - iova + length > mr->ibmr.iova + mr->ibmr.length) { + length > mr->ibmr.length || + iova - mr->ibmr.iova > mr->ibmr.length - length) { rxe_dbg_mr(mr, "iova/length out of range\n"); return -EINVAL; } diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c index ff904d5e54a73..0f24a55c06371 100644 --- a/drivers/infiniband/sw/rxe/rxe_odp.c +++ b/drivers/infiniband/sw/rxe/rxe_odp.c @@ -114,6 +114,7 @@ int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, err = rxe_odp_init_pages(mr); if (err) { ib_umem_odp_release(umem_odp); + mr->umem = NULL; return err; } @@ -124,19 +125,23 @@ int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, } static inline bool rxe_check_pagefault(struct ib_umem_odp *umem_odp, u64 iova, - int length) + int length, bool write) { bool need_fault = false; + u64 access = HMM_PFN_VALID; u64 addr; int idx; + if (write) + access |= HMM_PFN_WRITE; + addr = iova & (~(BIT(umem_odp->page_shift) - 1)); /* Skim through all pages that are to be accessed. */ while (addr < iova + length) { idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; - if (!(umem_odp->map.pfn_list[idx] & HMM_PFN_VALID)) { + if ((umem_odp->map.pfn_list[idx] & access) != access) { need_fault = true; break; } @@ -159,6 +164,7 @@ static unsigned long rxe_odp_iova_to_page_offset(struct ib_umem_odp *umem_odp, u static int rxe_odp_map_range_and_lock(struct rxe_mr *mr, u64 iova, int length, u32 flags) { struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem); + bool write = !(flags & RXE_PAGEFAULT_RDONLY); bool need_fault; int err; @@ -167,7 +173,7 @@ static int rxe_odp_map_range_and_lock(struct rxe_mr *mr, u64 iova, int length, u mutex_lock(&umem_odp->umem_mutex); - need_fault = rxe_check_pagefault(umem_odp, iova, length); + need_fault = rxe_check_pagefault(umem_odp, iova, length, write); if (need_fault) { mutex_unlock(&umem_odp->umem_mutex); @@ -177,7 +183,7 @@ static int rxe_odp_map_range_and_lock(struct rxe_mr *mr, u64 iova, int length, u if (err < 0) return err; - need_fault = rxe_check_pagefault(umem_odp, iova, length); + need_fault = rxe_check_pagefault(umem_odp, iova, length, write); if (need_fault) { mutex_unlock(&umem_odp->umem_mutex); return -EFAULT; @@ -339,8 +345,9 @@ int rxe_odp_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int err; u8 *va; + /* A flush never modifies memory; read-only access suffices. */ err = rxe_odp_map_range_and_lock(mr, iova, length, - RXE_PAGEFAULT_DEFAULT); + RXE_PAGEFAULT_RDONLY); if (err) return err; diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index f3dff1aea96af..370d98385c934 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -172,6 +172,7 @@ static void free_rd_atomic_resources(struct rxe_qp *qp) } kfree(qp->resp.resources); qp->resp.resources = NULL; + qp->resp.res = NULL; } } @@ -707,13 +708,24 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask, int max_dest_rd_atomic = attr->max_dest_rd_atomic ? roundup_pow_of_two(attr->max_dest_rd_atomic) : 0; - qp->attr.max_dest_rd_atomic = max_dest_rd_atomic; - + /* + * Not gated by IB_QP_STATE, so the responder task is live. + * Quiesce recv_task like rxe_qp_reset() before swapping the + * rd_atomic array, so rxe_receiver() cannot race the free/ + * realloc. + */ + rxe_disable_task(&qp->recv_task); free_rd_atomic_resources(qp); - + qp->attr.max_dest_rd_atomic = max_dest_rd_atomic; err = alloc_rd_atomic_resources(qp, max_dest_rd_atomic); + /* + * On ENOMEM leave recv_task quiesced: qp->resp.resources is + * NULL and rxe_prepare_res()/find_resource() would deref it. + * Re-enable only after a fresh array is installed. + */ if (err) return err; + rxe_enable_task(&qp->recv_task); } if (mask & IB_QP_EN_SQD_ASYNC_NOTIFY) diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 12d03f390b097..24f5c044363f7 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -701,6 +701,21 @@ int rxe_requester(struct rxe_qp *qp) if (unlikely(!wqe)) goto exit; + /* + * Don't trust user space data: a user QP's WQE comes from an mmap'd + * ring, so num_sge/cur_sge are attacker-controlled. Bound num_sge like + * get_srq_wqe(); bound cur_sge only when payload exists (dma.resid), + * since copy_data() skips dma->sge[] on a zero-length copy (all a + * max_sge == 0 QP can post). + */ + if (unlikely(wqe->dma.num_sge > qp->sq.max_sge || + (wqe->dma.resid && + wqe->dma.cur_sge >= qp->sq.max_sge))) { + rxe_dbg_qp(qp, "invalid num_sge/cur_sge in send wqe\n"); + wqe->status = IB_WC_LOC_QP_OP_ERR; + goto err; + } + if (rxe_wqe_is_fenced(qp, wqe)) { qp->req.wait_fence = 1; goto exit; diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index d8cbdfa70cdbd..02b16e2b49b8f 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -1217,7 +1217,14 @@ static enum resp_states do_complete(struct rxe_qp *qp, spin_lock_irqsave(&qp->state_lock, flags); if (unlikely(qp_state(qp) == IB_QPS_ERR)) { spin_unlock_irqrestore(&qp->state_lock, flags); - return RESPST_CHK_RESOURCE; + /* The packet was executed and completed before the QP + * moved to ERROR; it must be consumed exactly once. + * Re-entering the request chain with the stale packet + * would copy it into every remaining recv WQE as a new + * completion. Remaining WQEs are flushed by the drain + * path at rxe_receiver() entry. + */ + return pkt ? RESPST_CLEANUP : RESPST_CHK_RESOURCE; } spin_unlock_irqrestore(&qp->state_lock, flags); diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 1ec130fee8ea7..f8c29fc5961d7 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -1327,19 +1327,20 @@ static struct ib_mr *rxe_rereg_user_mr(struct ib_mr *ibmr, int flags, if (err) return ERR_PTR(err); + if ((flags & IB_MR_REREG_ACCESS) && + (access & ~RXE_ACCESS_SUPPORTED_MR)) { + rxe_err_mr(mr, "access = %#x not supported\n", access); + return ERR_PTR(-EOPNOTSUPP); + } + if (flags & IB_MR_REREG_PD) { rxe_put(old_pd); rxe_get(pd); mr->ibmr.pd = ibpd; } - if (flags & IB_MR_REREG_ACCESS) { - if (access & ~RXE_ACCESS_SUPPORTED_MR) { - rxe_err_mr(mr, "access = %#x not supported\n", access); - return ERR_PTR(-EOPNOTSUPP); - } + if (flags & IB_MR_REREG_ACCESS) mr->access = access; - } return NULL; } diff --git a/drivers/infiniband/sw/siw/siw_cm.c b/drivers/infiniband/sw/siw/siw_cm.c index 87c79527ac090..ed49818793dde 100644 --- a/drivers/infiniband/sw/siw/siw_cm.c +++ b/drivers/infiniband/sw/siw/siw_cm.c @@ -1719,9 +1719,12 @@ int siw_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params) SIW_QP_ATTR_STATE | SIW_QP_ATTR_LLP_HANDLE | SIW_QP_ATTR_ORD | SIW_QP_ATTR_IRD | SIW_QP_ATTR_MPA); + if (rv) { + qp->cep = NULL; + siw_cep_put(cep); + goto error_unlock; + } up_write(&qp->state_lock); - if (rv) - goto error; siw_dbg_cep(cep, "[QP %u]: send mpa reply, %d byte pdata\n", qp_id(qp), params->private_data_len); @@ -1751,7 +1754,7 @@ int siw_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params) cep->state = SIW_EPSTATE_CLOSED; siw_free_cm_id(cep); - if (qp->cep) { + if (qp->cep == cep) { siw_cep_put(cep); qp->cep = NULL; } diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c index b566d163c5aab..e5b641c8d694a 100644 --- a/drivers/infiniband/sw/siw/siw_qp_rx.c +++ b/drivers/infiniband/sw/siw/siw_qp_rx.c @@ -1079,7 +1079,7 @@ static int siw_get_hdr(struct siw_rx_stream *srx) if (iwarp_pktinfo[opcode].hdr_len > sizeof(struct iwarp_ctrl_tagged)) { int hdrlen = iwarp_pktinfo[opcode].hdr_len; - bytes = min_t(int, hdrlen - MIN_DDP_HDR, srx->skb_new); + bytes = min_t(int, hdrlen - srx->fpdu_part_rcvd, srx->skb_new); skb_copy_bits(skb, srx->skb_offset, (char *)c_hdr + srx->fpdu_part_rcvd, bytes); diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h index 91f866e3fb8bd..143e03b649021 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib.h +++ b/drivers/infiniband/ulp/ipoib/ipoib.h @@ -87,6 +87,7 @@ enum { IPOIB_FLAG_INITIALIZED = 1, IPOIB_FLAG_ADMIN_UP = 2, IPOIB_PKEY_ASSIGNED = 3, + IPOIB_FLAG_MCAST_FLUSH = 4, IPOIB_FLAG_SUBINTERFACE = 5, IPOIB_STOP_REAPER = 7, IPOIB_FLAG_ADMIN_CM = 9, @@ -414,6 +415,12 @@ struct ipoib_dev_priv { const struct net_device_ops *rn_ops; }; +static inline bool ipoib_mcast_allowed(struct ipoib_dev_priv *priv) +{ + return test_bit(IPOIB_FLAG_OPER_UP, &priv->flags) && + !test_bit(IPOIB_FLAG_MCAST_FLUSH, &priv->flags); +} + struct ipoib_ah { struct net_device *dev; struct ib_ah *ah; diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c index 5061d52a7b12c..81bbb3f7c1132 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c @@ -1227,17 +1227,19 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, } if (level == IPOIB_FLUSH_LIGHT) { - int oper_up; ipoib_mark_paths_invalid(dev); - /* Set IPoIB operation as down to prevent races between: + /* Set MCAST_FLUSH to prevent races between: * the flush flow which leaves MCG and on the fly joins * which can happen during that time. mcast restart task * should deal with join requests we missed. + * + * Do not clear OPER_UP for this; restoring it races with + * ipoib_ib_dev_down() and can leave OPER_UP set after the + * device is down. */ - oper_up = test_and_clear_bit(IPOIB_FLAG_OPER_UP, &priv->flags); + set_bit(IPOIB_FLAG_MCAST_FLUSH, &priv->flags); ipoib_mcast_dev_flush(dev); - if (oper_up) - set_bit(IPOIB_FLAG_OPER_UP, &priv->flags); + clear_bit(IPOIB_FLAG_MCAST_FLUSH, &priv->flags); ipoib_reap_dead_ahs(priv); } diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index 16a015b672063..6c14246befb1f 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -2783,6 +2783,7 @@ static int __init ipoib_init_module(void) err_sa: ib_sa_unregister_client(&ipoib_sa_client); destroy_workqueue(ipoib_workqueue); + rcu_barrier(); err_fs: ipoib_unregister_debugfs(); @@ -2800,6 +2801,7 @@ static void __exit ipoib_cleanup_module(void) ib_sa_unregister_client(&ipoib_sa_client); ipoib_unregister_debugfs(); destroy_workqueue(ipoib_workqueue); + rcu_barrier(); } module_init(ipoib_init_module); diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c index 6401af2fd548f..379b78374e210 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c @@ -74,7 +74,7 @@ static void __ipoib_mcast_schedule_join_thread(struct ipoib_dev_priv *priv, struct ipoib_mcast *mcast, bool delay) { - if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) + if (!ipoib_mcast_allowed(priv)) return; /* @@ -469,7 +469,7 @@ static int ipoib_mcast_join(struct net_device *dev, struct ipoib_mcast *mcast) int ret = 0; if (!priv->broadcast || - !test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) + !ipoib_mcast_allowed(priv)) return -EINVAL; init_completion(&mcast->done); @@ -555,7 +555,7 @@ void ipoib_mcast_join_task(struct work_struct *work) unsigned long delay_until = 0; struct ipoib_mcast *mcast = NULL; - if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) + if (!ipoib_mcast_allowed(priv)) return; if (ib_query_port(priv->ca, priv->port, &port_attr)) { @@ -577,7 +577,7 @@ void ipoib_mcast_join_task(struct work_struct *work) netif_addr_unlock_bh(dev); spin_lock_irq(&priv->lock); - if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) + if (!ipoib_mcast_allowed(priv)) goto out; if (!priv->broadcast) { @@ -749,7 +749,7 @@ void ipoib_mcast_send(struct net_device *dev, u8 *daddr, struct sk_buff *skb) spin_lock_irqsave(&priv->lock, flags); - if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags) || + if (!ipoib_mcast_allowed(priv) || !priv->broadcast || !test_bit(IPOIB_MCAST_FLAG_ATTACHED, &priv->broadcast->flags)) { ++dev->stats.tx_dropped; @@ -871,7 +871,7 @@ void ipoib_mcast_restart_task(struct work_struct *work) LIST_HEAD(remove_list); struct ib_sa_mcmember_rec rec; - if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) + if (!ipoib_mcast_allowed(priv)) /* * shortcut...on shutdown flush is called next, just * let it do all the work @@ -965,9 +965,9 @@ void ipoib_mcast_restart_task(struct work_struct *work) ipoib_mcast_remove_list(&remove_list); /* - * Double check that we are still up + * Double check that we are still up and not flushing */ - if (test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) { + if (ipoib_mcast_allowed(priv)) { spin_lock_irq(&priv->lock); __ipoib_mcast_schedule_join_thread(priv, NULL, 0); spin_unlock_irq(&priv->lock); diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c index 12a2d12fef070..7ea6888b479cf 100644 --- a/drivers/infiniband/ulp/iser/iser_initiator.c +++ b/drivers/infiniband/ulp/iser/iser_initiator.c @@ -598,11 +598,8 @@ static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc, iser_dbg("conn %p: remote invalidation for rkey %#x\n", iser_conn, rkey); - if (unlikely(!iser_conn->snd_w_inv)) { - iser_err("conn %p: unexpected remote invalidation, terminating connection\n", - iser_conn); - return -EPROTO; - } + if (unlikely(!iser_conn->snd_w_inv)) + goto bad_inv; task = iscsi_itt_to_ctask(iser_conn->iscsi_conn, hdr->itt); if (likely(task)) { @@ -611,12 +608,16 @@ static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc, if (iser_task->dir[ISER_DIR_IN]) { desc = iser_task->rdma_reg[ISER_DIR_IN].desc; + if (unlikely(!desc)) + goto bad_inv; if (unlikely(iser_inv_desc(desc, rkey))) return -EINVAL; } if (iser_task->dir[ISER_DIR_OUT]) { desc = iser_task->rdma_reg[ISER_DIR_OUT].desc; + if (unlikely(!desc)) + goto bad_inv; if (unlikely(iser_inv_desc(desc, rkey))) return -EINVAL; } @@ -627,6 +628,11 @@ static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc, } return 0; + +bad_inv: + iser_err("conn %p: unexpected remote invalidation, terminating connection\n", + iser_conn); + return -EPROTO; } diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index 1015a51f750af..6a9e76d60f816 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "ib_isert.h" @@ -59,6 +60,8 @@ static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); +static void isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, + struct ib_device *ib_dev); static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp) { @@ -308,6 +311,7 @@ isert_init_conn(struct isert_conn *isert_conn) init_completion(&isert_conn->login_req_comp); init_waitqueue_head(&isert_conn->rem_wait); kref_init(&isert_conn->kref); + atomic_set(&isert_conn->ctrl_comp_cnt, 0); mutex_init(&isert_conn->mutex); INIT_WORK(&isert_conn->release_work, isert_release_work); } @@ -496,6 +500,8 @@ isert_connect_release(struct isert_conn *isert_conn) if (isert_conn->qp) isert_destroy_qp(isert_conn); + isert_unmap_tx_desc(&isert_conn->login_tx_desc, device->ib_device); + if (isert_conn->login_desc) isert_free_login_buf(isert_conn); @@ -943,27 +949,16 @@ isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login, } if (!login->login_failed) { if (login->login_complete) { - ret = isert_alloc_rx_descriptors(isert_conn); - if (ret) - return ret; - - ret = isert_post_recvm(isert_conn, - ISERT_QP_MAX_RECV_DTOS); - if (ret) - return ret; - - /* Now we are in FULL_FEATURE phase */ - mutex_lock(&isert_conn->mutex); - isert_conn->state = ISER_CONN_FULL_FEATURE; - mutex_unlock(&isert_conn->mutex); - goto post_send; + /* Posted and sent from isert_get_rx_pdu(). */ + isert_conn->login_rsp_pending = true; + return 0; } ret = isert_login_post_recv(isert_conn); if (ret) return ret; } -post_send: + ret = isert_login_post_send(isert_conn, tx_desc); if (ret) return ret; @@ -971,6 +966,21 @@ isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login, return 0; } +static int +isert_check_login_req(struct isert_conn *isert_conn) +{ + struct iscsi_hdr *hdr = isert_get_iscsi_hdr(isert_conn->login_desc); + u32 dlength = ntoh24(hdr->dlength); + + if (unlikely(dlength > (u32)isert_conn->login_req_len)) { + isert_dbg("login PDU declares %u data bytes but only %d were received\n", + dlength, isert_conn->login_req_len); + return -EINVAL; + } + + return 0; +} + static void isert_rx_login_req(struct isert_conn *isert_conn) { @@ -1333,6 +1343,21 @@ isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, ISER_RX_SIZE, DMA_FROM_DEVICE); + /* + * The data segment length declared in the BHS is attacker controlled + * and is used further down to read that many bytes out of the fixed + * size receive descriptor, so it has to be checked against the number + * of bytes that were actually received. Comparing without subtracting + * also rejects PDUs shorter than the iSER and iSCSI headers, which + * would otherwise be parsed out of stale descriptor contents. + */ + if (unlikely(wc->byte_len < ISER_HEADERS_LEN + ntoh24(hdr->dlength))) { + isert_err("PDU declares %u data bytes but only %u bytes were received\n", + ntoh24(hdr->dlength), wc->byte_len); + iscsit_cause_connection_reinstatement(isert_conn->conn, 0); + return; + } + isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, (int)(wc->byte_len - ISER_HEADERS_LEN)); @@ -1394,8 +1419,12 @@ isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) if (isert_conn->conn) { struct iscsi_login *login = isert_conn->conn->conn_login; - if (login && !login->first_request) + if (login && !login->first_request) { + if (isert_check_login_req(isert_conn)) + return; + isert_rx_login_req(isert_conn); + } } mutex_lock(&isert_conn->mutex); @@ -1668,6 +1697,8 @@ isert_do_control_comp(struct work_struct *work) struct isert_conn *isert_conn = isert_cmd->conn; struct ib_device *ib_dev = isert_conn->cm_id->device; struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; + /* The switch below may free isert_cmd. */ + bool counted = isert_cmd->ctrl_counted; isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); @@ -1689,6 +1720,14 @@ isert_do_control_comp(struct work_struct *work) dump_stack(); break; } + + /* + * The count is what keeps isert_conn alive, so drop it last. The wait + * queue lives in the global hash table, not in isert_conn, so this is + * safe even if the waiter has already freed the connection. + */ + if (counted && atomic_dec_and_test(&isert_conn->ctrl_comp_cnt)) + wake_up_var(&isert_conn->ctrl_comp_cnt); } static void @@ -1732,6 +1771,12 @@ isert_send_done(struct ib_cq *cq, struct ib_wc *wc) case ISTATE_SEND_TEXTRSP: isert_unmap_tx_desc(tx_desc, ib_dev); + /* Paired with the wait in isert_wait_conn(). */ + isert_cmd->ctrl_counted = + isert_cmd->iscsit_cmd->i_state != ISTATE_SEND_LOGOUTRSP; + if (isert_cmd->ctrl_counted) + atomic_inc(&isert_conn->ctrl_comp_cnt); + INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); queue_work(isert_comp_wq, &isert_cmd->comp_work); return; @@ -2360,6 +2405,10 @@ isert_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login) if (!login->first_request) return 0; + ret = isert_check_login_req(isert_conn); + if (ret) + return ret; + isert_rx_login_req(isert_conn); isert_info("before login_comp conn: %p\n", conn); @@ -2572,6 +2621,10 @@ static void isert_wait_conn(struct iscsit_conn *conn) isert_wait4cmds(conn); isert_wait4logout(isert_conn); + /* Paired with the count taken in isert_send_done(). */ + wait_var_event(&isert_conn->ctrl_comp_cnt, + !atomic_read(&isert_conn->ctrl_comp_cnt)); + queue_work(isert_release_wq, &isert_conn->release_work); } @@ -2585,8 +2638,30 @@ static void isert_free_conn(struct iscsit_conn *conn) static void isert_get_rx_pdu(struct iscsit_conn *conn) { + struct isert_conn *isert_conn = conn->context; struct completion comp; + /* The login timeout timer can fail the login after isert_put_login_tx(). */ + if (!isert_conn->login_rsp_pending) + return; + + isert_conn->login_rsp_pending = false; + + /* The session is registered by now; see isert_put_login_tx(). */ + if (isert_alloc_rx_descriptors(isert_conn)) + return; + + if (isert_post_recvm(isert_conn, ISERT_QP_MAX_RECV_DTOS)) + return; + + /* Now we are in FULL_FEATURE phase */ + mutex_lock(&isert_conn->mutex); + isert_conn->state = ISER_CONN_FULL_FEATURE; + mutex_unlock(&isert_conn->mutex); + + if (isert_login_post_send(isert_conn, &isert_conn->login_tx_desc)) + return; + init_completion(&comp); wait_for_completion_interruptible(&comp); diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h index 0b2dfd6e7e270..519b17e54bd34 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.h +++ b/drivers/infiniband/ulp/isert/ib_isert.h @@ -153,6 +153,7 @@ struct isert_cmd { struct work_struct comp_work; struct scatterlist sg; bool ctx_init_done; + bool ctrl_counted; }; static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc) @@ -178,6 +179,7 @@ struct isert_conn { struct completion login_comp; struct completion login_req_comp; struct iser_tx_desc login_tx_desc; + bool login_rsp_pending; struct rdma_cm_id *cm_id; struct ib_qp *qp; struct ib_cq *cq; @@ -186,6 +188,7 @@ struct isert_conn { struct mutex mutex; struct kref kref; struct work_struct release_work; + atomic_t ctrl_comp_cnt; bool logout_posted; bool snd_w_inv; wait_queue_head_t rem_wait; diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h index 7738e26768557..29e23404bb7b8 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h @@ -55,7 +55,7 @@ DECLARE_EVENT_CLASS(rtrs_clt_conn_class, __entry->max_reconnect_attempts = clt->max_reconnect_attempts; __entry->fail_cnt = clt_path->stats->reconnects.fail_cnt; __entry->success_cnt = clt_path->stats->reconnects.successful_cnt; - memcpy(__entry->sessname, kobject_name(&clt_path->kobj), NAME_MAX); + strscpy(__entry->sessname, kobject_name(&clt_path->kobj) ?: "", NAME_MAX); ), TP_printk("RTRS-CLT: sess='%s' state=%s attempts='%d' max-attempts='%d' fail='%d' success='%d'", diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index d34d7e5f34d6f..62cc93be7b877 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -1735,6 +1735,8 @@ static void destroy_con_cq_qp(struct rtrs_clt_con *con) /* * Be careful here: destroy_con_cq_qp() can be called even * create_con_cq_qp() failed, see comments there. + * Caller must set con->destroyed under this lock first so a + * racing ADDR_RESOLVED cannot ib_cq_pool_get() after we PUT/SKIP. */ lockdep_assert_held(&con->con_mutex); rtrs_cq_qp_destroy(&con->c); @@ -1769,6 +1771,10 @@ static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con) int err; mutex_lock(&con->con_mutex); + if (con->destroyed) { + mutex_unlock(&con->con_mutex); + return -ECONNABORTED; + } err = create_con_cq_qp(con); mutex_unlock(&con->con_mutex); if (err) { @@ -2224,6 +2230,7 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_path *clt_path) break; con = to_clt_con(clt_path->s.con[cid]); mutex_lock(&con->con_mutex); + con->destroyed = true; destroy_con_cq_qp(con); mutex_unlock(&con->con_mutex); destroy_cm(con); @@ -2390,6 +2397,7 @@ static int init_conns(struct rtrs_clt_path *clt_path) if (con->c.cm_id) { stop_cm(con); mutex_lock(&con->con_mutex); + con->destroyed = true; destroy_con_cq_qp(con); mutex_unlock(&con->con_mutex); destroy_cm(con); diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.h b/drivers/infiniband/ulp/rtrs/rtrs-clt.h index 1305601a6251e..ad64f4517c4b3 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.h @@ -75,6 +75,8 @@ struct rtrs_clt_con { unsigned int cpu; struct mutex con_mutex; int cm_err; + /* Set under con_mutex before CQ/QP teardown. */ + bool destroyed; }; /** diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h index 587d3e0330812..a7d7b971e6c80 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h @@ -61,7 +61,7 @@ TRACE_EVENT(send_io_resp_imm, __entry->msg_id = id->msg_id; __entry->wr_cnt = atomic_read(&con->c.wr_cnt); __entry->signal_interval = s->signal_interval; - memcpy(__entry->sessname, kobject_name(&srv_path->kobj), NAME_MAX); + strscpy(__entry->sessname, kobject_name(&srv_path->kobj) ?: "", NAME_MAX); ), TP_printk("sess='%s' state='%s' dir=%s err='%d' inval='%d' glob-inval='%d' msgid='%u' wrcnt='%d' sig-interval='%u'", diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index acbd787de265b..93dcd15682d04 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -1945,7 +1945,8 @@ static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp, ch->req_lim += be32_to_cpu(rsp->req_lim_delta); if (rsp->tag == ch->tsk_mgmt_tag) { ch->tsk_mgmt_status = -1; - if (be32_to_cpu(rsp->resp_data_len) >= 4) + if (be32_to_cpu(rsp->resp_data_len) >= 4 && + byte_len >= sizeof(*rsp) + 4) ch->tsk_mgmt_status = rsp->data[3]; complete(&ch->tsk_mgmt_done); } else { @@ -2045,13 +2046,20 @@ static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta, } static void srp_process_cred_req(struct srp_rdma_ch *ch, - struct srp_cred_req *req) + struct srp_cred_req *req, u32 byte_len) { - struct srp_cred_rsp rsp = { - .opcode = SRP_CRED_RSP, - .tag = req->tag, - }; - s32 delta = be32_to_cpu(req->req_lim_delta); + struct srp_cred_rsp rsp = { .opcode = SRP_CRED_RSP }; + s32 delta; + + if (byte_len < sizeof(*req)) { + shost_printk(KERN_ERR, ch->target->scsi_host, PFX + "dropping truncated SRP_CRED_REQ (%u bytes received, %zu expected)\n", + byte_len, sizeof(*req)); + return; + } + + rsp.tag = req->tag; + delta = be32_to_cpu(req->req_lim_delta); if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) shost_printk(KERN_ERR, ch->target->scsi_host, PFX @@ -2059,14 +2067,21 @@ static void srp_process_cred_req(struct srp_rdma_ch *ch, } static void srp_process_aer_req(struct srp_rdma_ch *ch, - struct srp_aer_req *req) + struct srp_aer_req *req, u32 byte_len) { struct srp_target_port *target = ch->target; - struct srp_aer_rsp rsp = { - .opcode = SRP_AER_RSP, - .tag = req->tag, - }; - s32 delta = be32_to_cpu(req->req_lim_delta); + struct srp_aer_rsp rsp = { .opcode = SRP_AER_RSP }; + s32 delta; + + if (byte_len < sizeof(*req)) { + shost_printk(KERN_ERR, target->scsi_host, PFX + "dropping truncated SRP_AER_REQ (%u bytes received, %zu expected)\n", + byte_len, sizeof(*req)); + return; + } + + rsp.tag = req->tag; + delta = be32_to_cpu(req->req_lim_delta); shost_printk(KERN_ERR, target->scsi_host, PFX "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun)); @@ -2108,11 +2123,11 @@ static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc) break; case SRP_CRED_REQ: - srp_process_cred_req(ch, iu->buf); + srp_process_cred_req(ch, iu->buf, wc->byte_len); break; case SRP_AER_REQ: - srp_process_aer_req(ch, iu->buf); + srp_process_aer_req(ch, iu->buf, wc->byte_len); break; case SRP_T_LOGOUT: diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index f66cfd70c2636..3ec42da1456b8 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -960,6 +960,7 @@ static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx, struct srpt_rdma_ch *ch = ioctx->ch; struct scatterlist *prev = NULL; unsigned prev_nents; + u8 n_rdma, n_rw_ctx; int ret, i; if (nbufs == 1) { @@ -970,6 +971,9 @@ static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx, return -ENOMEM; } + n_rw_ctx = ioctx->n_rw_ctx; + n_rdma = ioctx->n_rdma; + for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) { struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i]; u64 remote_addr = be64_to_cpu(db->va); @@ -1016,6 +1020,9 @@ static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx, } if (ioctx->rw_ctxs != &ioctx->s_rw_ctx) kfree(ioctx->rw_ctxs); + ioctx->rw_ctxs = NULL; + ioctx->n_rw_ctx = n_rw_ctx; + ioctx->n_rdma = n_rdma; return ret; } @@ -1596,7 +1603,7 @@ static void srpt_handle_cmd(struct srpt_rdma_ch *ch, rc = target_init_cmd(cmd, ch->sess, &send_ioctx->sense_data[0], scsilun_to_int(&srp_cmd->lun), data_len, - TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF); + cmd->sam_task_attr, dir, TARGET_SCF_ACK_KREF); if (rc != 0) { pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc, srp_cmd->tag); diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c index 3a718d6000063..8bfaaa45e0b95 100644 --- a/drivers/input/evdev.c +++ b/drivers/input/evdev.c @@ -1229,6 +1229,8 @@ static long evdev_do_ioctl(struct file *file, unsigned int cmd, t = _IOC_NR(cmd) & ABS_MAX; + memset(&abs, 0, sizeof(abs)); + if (copy_from_user(&abs, p, min_t(size_t, size, sizeof(struct input_absinfo)))) return -EFAULT; diff --git a/drivers/input/input-compat.c b/drivers/input/input-compat.c index a5043193ead85..8860a07302942 100644 --- a/drivers/input/input-compat.c +++ b/drivers/input/input-compat.c @@ -76,6 +76,8 @@ int input_ff_effect_from_user(const char __user *buffer, size_t size, */ compat_effect = (struct ff_effect_compat *)effect; + memset(effect, 0, sizeof(*effect)); + if (copy_from_user(compat_effect, buffer, sizeof(struct ff_effect_compat))) return -EFAULT; diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 2da0b7f1722aa..6406ceab988ad 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -215,7 +215,7 @@ static const struct xpad_device { { 0x0e6f, 0x0139, "Afterglow Prismatic Wired Controller", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x013a, "PDP Xbox One Controller", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x0146, "Rock Candy Wired Controller for Xbox One", 0, XTYPE_XBOXONE }, - { 0x0e6f, 0x0147, "PDP Marvel Xbox One Controller", 0, XTYPE_XBOXONE }, + { 0x0e6f, 0x0147, "PDP Marvel Xbox 360 Controller", 0, XTYPE_XBOX360 }, { 0x0e6f, 0x015c, "PDP Xbox One Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE }, { 0x0e6f, 0x015d, "PDP Mirror's Edge Official Wired Controller for Xbox One", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x0161, "PDP Xbox One Controller", 0, XTYPE_XBOXONE }, @@ -227,6 +227,7 @@ static const struct xpad_device { { 0x0e6f, 0x0213, "Afterglow Gamepad for Xbox 360", 0, XTYPE_XBOX360 }, { 0x0e6f, 0x021f, "Rock Candy Gamepad for Xbox 360", 0, XTYPE_XBOX360 }, { 0x0e6f, 0x0246, "Rock Candy Gamepad for Xbox One 2015", 0, XTYPE_XBOXONE }, + { 0x0e6f, 0x024c, "PDP Victrix Pro BFG Wired Controller for Xbox", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x02a0, "PDP Xbox One Controller", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x02a1, "PDP Xbox One Controller", 0, XTYPE_XBOXONE }, { 0x0e6f, 0x02a2, "PDP Wired Controller for Xbox One - Crimson Red", 0, XTYPE_XBOXONE }, @@ -292,6 +293,12 @@ static const struct xpad_device { { 0x1689, 0xfd00, "Razer Onza Tournament Edition", 0, XTYPE_XBOX360 }, { 0x1689, 0xfd01, "Razer Onza Classic Edition", 0, XTYPE_XBOX360 }, { 0x1689, 0xfe00, "Razer Sabertooth", 0, XTYPE_XBOX360 }, + { 0x16d0, 0x1103, "Azeron Cyro", 0, XTYPE_XBOX360 }, + { 0x16d0, 0x113c, "Azeron Cyborg", 0, XTYPE_XBOX360 }, + { 0x16d0, 0x1192, "Azeron Classic/Compact", 0, XTYPE_XBOX360 }, + { 0x16d0, 0x1212, "Azeron Cyro Lefty", 0, XTYPE_XBOX360 }, + { 0x16d0, 0x12f7, "Azeron Cyborg II", 0, XTYPE_XBOX360 }, + { 0x16d0, 0x13ea, "Azeron Keyzen", 0, XTYPE_XBOX360 }, { 0x17ef, 0x6182, "Lenovo Legion Controller for Windows", 0, XTYPE_XBOX360 }, { 0x1949, 0x041a, "Amazon Game Controller", 0, XTYPE_XBOX360 }, { 0x1a86, 0xe310, "Legion Go S", 0, XTYPE_XBOX360 }, @@ -534,6 +541,7 @@ static const struct usb_device_id xpad_table[] = { XPAD_XBOX360_VENDOR(0x15e4), /* Numark Xbox 360 controllers */ XPAD_XBOX360_VENDOR(0x162e), /* Joytech Xbox 360 controllers */ XPAD_XBOX360_VENDOR(0x1689), /* Razer Onza */ + XPAD_XBOX360_VENDOR(0x16d0), /* Azeron controllers */ XPAD_XBOX360_VENDOR(0x17ef), /* Lenovo */ XPAD_XBOX360_VENDOR(0x1949), /* Amazon controllers */ XPAD_XBOX360_VENDOR(0x1a86), /* Nanjing Qinheng Microelectronics (WCH) */ diff --git a/drivers/input/keyboard/adp5588-keys.c b/drivers/input/keyboard/adp5588-keys.c index 40371f5bd9bac..4f0ddff5babae 100644 --- a/drivers/input/keyboard/adp5588-keys.c +++ b/drivers/input/keyboard/adp5588-keys.c @@ -446,12 +446,6 @@ static int adp5588_gpio_add(struct adp5588_kpad *kpad) mutex_init(&kpad->gpio_lock); - error = devm_gpiochip_add_data(dev, &kpad->gc, kpad); - if (error) { - dev_err(dev, "gpiochip_add failed: %d\n", error); - return error; - } - for (i = 0; i <= ADP5588_BANK(ADP5588_MAXGPIO); i++) { kpad->dat_out[i] = adp5588_read(kpad->client, GPIO_DAT_OUT1 + i); @@ -459,6 +453,12 @@ static int adp5588_gpio_add(struct adp5588_kpad *kpad) kpad->pull_dis[i] = adp5588_read(kpad->client, GPIO_PULL1 + i); } + error = devm_gpiochip_add_data(dev, &kpad->gc, kpad); + if (error) { + dev_err(dev, "gpiochip_add failed: %d\n", error); + return error; + } + return 0; } diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c index 4c82e988260e5..5041443e0efec 100644 --- a/drivers/input/keyboard/atkbd.c +++ b/drivers/input/keyboard/atkbd.c @@ -1954,6 +1954,14 @@ static const struct dmi_system_id atkbd_dmi_quirk_table[] __initconst = { }, .callback = atkbd_deactivate_fixup, }, + { + /* Xiaomi Redmi Book Pro 16 2026 (TM2425) */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "XIAOMI"), + DMI_MATCH(DMI_PRODUCT_NAME, "REDMI Book Pro 16 2026"), + }, + .callback = atkbd_deactivate_fixup, + }, { } }; diff --git a/drivers/input/misc/soc_button_array.c b/drivers/input/misc/soc_button_array.c index b8cad415c62ca..f08e29af531fd 100644 --- a/drivers/input/misc/soc_button_array.c +++ b/drivers/input/misc/soc_button_array.c @@ -16,6 +16,7 @@ #include #include #include +#include #include static bool use_low_level_irq; @@ -160,7 +161,7 @@ soc_button_device_create(struct platform_device *pdev, struct gpio_keys_platform_data *gpio_keys_pdata; const struct dmi_system_id *dmi_id; int invalid_acpi_index = -1; - int error, gpio, irq; + int error, gpio, irq = 0; int n_buttons = 0; for (info = button_info; info->name; info++) @@ -191,8 +192,9 @@ soc_button_device_create(struct platform_device *pdev, error = soc_button_lookup_gpio(&pdev->dev, info->acpi_index, &gpio, &irq); if (error || irq < 0) { /* - * Skip GPIO if not present. Note we deliberately - * ignore -EPROBE_DEFER errors here. On some devices + * Propagate -EPROBE_DEFER, skip button on other errors. + * + * -EPROBE_DEFER is ignored on Bay & Cherry Trail. Here * Intel is using so called virtual GPIOs which are not * GPIOs at all but some way for AML code to check some * random status bits without need a custom opregion. @@ -201,6 +203,12 @@ soc_button_device_create(struct platform_device *pdev, * we do not have a driver for these so they will never * show up, therefore we ignore -EPROBE_DEFER. */ + if ((error == -EPROBE_DEFER || irq == -EPROBE_DEFER) && + !(soc_intel_is_byt() || soc_intel_is_cht())) { + error = -EPROBE_DEFER; + goto err_free_mem; + } + continue; } @@ -369,7 +377,7 @@ static struct soc_button_info *soc_button_get_button_info(struct device *dev) } } - if (!btns_desc) { + if (!btns_desc || !btns_desc->package.count) { dev_err(dev, "ACPI Button Descriptors not found\n"); button_info = ERR_PTR(-ENODEV); goto out; diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c index c70502e24031a..007f90759d06e 100644 --- a/drivers/input/mouse/synaptics.c +++ b/drivers/input/mouse/synaptics.c @@ -1837,6 +1837,14 @@ static int synaptics_setup_intertouch(struct psmouse *psmouse, return -ENXIO; } + + /* Disable intertouch on known-broken board revisions */ + if (info->board_id == 2722) { + psmouse_info(psmouse, + "Disabling intertouch for board id %u\n", + info->board_id); + return -ENXIO; + } } psmouse_info(psmouse, "Trying to set up SMBus access\n"); diff --git a/drivers/input/rmi4/rmi_driver.c b/drivers/input/rmi4/rmi_driver.c index 5d49a9021c7d0..a349dfd17519c 100644 --- a/drivers/input/rmi4/rmi_driver.c +++ b/drivers/input/rmi4/rmi_driver.c @@ -991,6 +991,15 @@ int rmi_driver_suspend(struct rmi_device *rmi_dev, bool enable_wake) { int retval; + /* + * Transport driver will try to suspend RMI device even if physical + * driver did not bind to the RMI device, because transport device + * (I2C, SPI) is fully registered and operational. Exit early if + * there is no driver data attached to the RMI device. + */ + if (!dev_get_drvdata(&rmi_dev->dev)) + return 0; + retval = rmi_suspend_functions(rmi_dev); if (retval) dev_warn(&rmi_dev->dev, "Failed to suspend functions: %d\n", @@ -1005,6 +1014,10 @@ int rmi_driver_resume(struct rmi_device *rmi_dev, bool clear_wake) { int retval; + /* Skip if not fully bound to RMI driver */ + if (!dev_get_drvdata(&rmi_dev->dev)) + return 0; + rmi_enable_irq(rmi_dev, clear_wake); retval = rmi_resume_functions(rmi_dev); diff --git a/drivers/input/rmi4/rmi_smbus.c b/drivers/input/rmi4/rmi_smbus.c index 6de68c602558b..d799405fbd337 100644 --- a/drivers/input/rmi4/rmi_smbus.c +++ b/drivers/input/rmi4/rmi_smbus.c @@ -140,7 +140,7 @@ static int rmi_smb_write_block(struct rmi_transport_dev *xport, u16 rmiaddr, u8 commandcode; struct rmi_smb_xport *rmi_smb = container_of(xport, struct rmi_smb_xport, xport); - int cur_len = (int)len; + size_t cur_len = len; mutex_lock(&rmi_smb->page_mutex); @@ -148,7 +148,7 @@ static int rmi_smb_write_block(struct rmi_transport_dev *xport, u16 rmiaddr, /* * break into 32 bytes chunks to write get command code */ - int block_len = min_t(int, len, SMB_MAX_COUNT); + int block_len = min_t(size_t, cur_len, SMB_MAX_COUNT); retval = rmi_smb_get_command_code(xport, rmiaddr, block_len, false, &commandcode); @@ -161,9 +161,9 @@ static int rmi_smb_write_block(struct rmi_transport_dev *xport, u16 rmiaddr, goto exit; /* prepare to write next block of bytes */ - cur_len -= SMB_MAX_COUNT; - databuff += SMB_MAX_COUNT; - rmiaddr += SMB_MAX_COUNT; + cur_len -= block_len; + databuff += block_len; + rmiaddr += block_len; } exit: mutex_unlock(&rmi_smb->page_mutex); diff --git a/drivers/input/serio/hp_sdc.c b/drivers/input/serio/hp_sdc.c index 1461ef319f92a..ecf5347b63ab7 100644 --- a/drivers/input/serio/hp_sdc.c +++ b/drivers/input/serio/hp_sdc.c @@ -981,7 +981,7 @@ static void hp_sdc_exit(void) free_irq(hp_sdc.irq, &hp_sdc); write_unlock_irq(&hp_sdc.lock); - timer_delete_sync(&hp_sdc.kicker); + timer_shutdown_sync(&hp_sdc.kicker); tasklet_kill(&hp_sdc.task); diff --git a/drivers/input/serio/i8042-acpipnpio.h b/drivers/input/serio/i8042-acpipnpio.h index 412f82d7a303b..67da08af75cb8 100644 --- a/drivers/input/serio/i8042-acpipnpio.h +++ b/drivers/input/serio/i8042-acpipnpio.h @@ -258,6 +258,13 @@ static const struct dmi_system_id i8042_dmi_quirk_table[] __initconst = { }, .driver_data = (void *)(SERIO_QUIRK_RESET_ALWAYS) }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_MATCH(DMI_PRODUCT_NAME, "Aspire AG15-42P"), + }, + .driver_data = (void *)(SERIO_QUIRK_RESET_ALWAYS) + }, { .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Acer"), diff --git a/drivers/input/touchscreen/cyttsp5.c b/drivers/input/touchscreen/cyttsp5.c index 9266c07314bee..e878a02dc9b75 100644 --- a/drivers/input/touchscreen/cyttsp5.c +++ b/drivers/input/touchscreen/cyttsp5.c @@ -710,6 +710,7 @@ static irqreturn_t cyttsp5_handle_irq(int irq, void *handle) size = 2; } else { report_id = ts->input_buf[2]; + size = min(size, CY_MAX_INPUT); } switch (report_id) { diff --git a/drivers/input/touchscreen/eeti_ts.c b/drivers/input/touchscreen/eeti_ts.c index b12602bc368ef..b870a939508bb 100644 --- a/drivers/input/touchscreen/eeti_ts.c +++ b/drivers/input/touchscreen/eeti_ts.c @@ -276,6 +276,7 @@ static const struct of_device_id of_eeti_ts_match[] = { { .compatible = "eeti,exc3000-i2c", }, { } }; +MODULE_DEVICE_TABLE(of, of_eeti_ts_match); #endif static struct i2c_driver eeti_ts_driver = { diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c index 73a9db2c7537f..4aa991a541018 100644 --- a/drivers/interconnect/core.c +++ b/drivers/interconnect/core.c @@ -548,7 +548,7 @@ struct icc_path *of_icc_get_by_index(struct device *dev, int idx) path->name = kasprintf(GFP_KERNEL, "%s-%s", src_data->node->name, dst_data->node->name); if (!path->name) { - kfree(path); + icc_put(path); path = ERR_PTR(-ENOMEM); } @@ -646,8 +646,9 @@ struct icc_path *icc_get(struct device *dev, const char *src, const char *dst) path->name = kasprintf(GFP_KERNEL, "%s-%s", src_node->name, dst_node->name); if (!path->name) { - kfree(path); - path = ERR_PTR(-ENOMEM); + mutex_unlock(&icc_lock); + icc_put(path); + return ERR_PTR(-ENOMEM); } out: mutex_unlock(&icc_lock); diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c index 4c53b63613148..5c573ec8e27a4 100644 --- a/drivers/iommu/amd/debugfs.c +++ b/drivers/iommu/amd/debugfs.c @@ -176,19 +176,13 @@ static ssize_t devid_write(struct file *filp, const char __user *ubuf, kfree(srcid_ptr); return -ENODEV; } - break; - } - - if (pci_seg->id != seg) { + sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid); kfree(srcid_ptr); - return -EINVAL; + return cnt; } - sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid); - kfree(srcid_ptr); - - return cnt; + return -EINVAL; } static int devid_show(struct seq_file *m, void *unused) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index e7d7b4cb9337f..3509be3fc744e 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -906,7 +906,9 @@ static void free_ga_log(struct amd_iommu *iommu) { #ifdef CONFIG_IRQ_REMAP iommu_free_pages(iommu->ga_log); + iommu->ga_log = NULL; iommu_free_pages(iommu->ga_log_tail); + iommu->ga_log_tail = NULL; #endif } @@ -953,6 +955,9 @@ static int iommu_init_ga_log(struct amd_iommu *iommu) if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) return -EINVAL; + if (iommu->ga_log && iommu->ga_log_tail) + return 0; + iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE); if (!iommu->ga_log) goto err_out; @@ -1914,19 +1919,20 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, else iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; + if (h->efr_attr & BIT(IOMMU_IVHD_ATTR_HATDIS_SHIFT)) { + pr_warn_once("Host Address Translation is not supported.\n"); + amd_iommu_hatdis = true; + } + /* XT and GAM require GA mode. */ if ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0) { amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; + break; } else { if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; } - if (h->efr_attr & BIT(IOMMU_IVHD_ATTR_HATDIS_SHIFT)) { - pr_warn_once("Host Address Translation is not supported.\n"); - amd_iommu_hatdis = true; - } - early_iommu_features_init(iommu, h); break; @@ -3091,14 +3097,28 @@ static void __init free_iommu_resources(void) /* SB IOAPIC for Hygon family 18h model 4h is on the device 0xb */ #define IOAPIC_SB_DEVID_FAM18H_M4H ((0x00 << 8) | PCI_DEVFN(0xb, 0)) +/* + * The Southbridge IOAPIC is assigned a GSI Base of 0 (handling interrupts + * 0 through 23). + */ +static int __init get_sb_ioapic_id(void) +{ + int idx = mp_find_ioapic(0); + + if (idx < 0) + return -ENODEV; + + return mpc_ioapic_id(idx); +} + static bool __init check_ioapic_information(void) { const char *fw_bug = FW_BUG; bool ret, has_sb_ioapic; - int idx; + int idx, sb_apicid; has_sb_ioapic = false; - ret = false; + ret = true; /* * If we have map overrides on the kernel command line the @@ -3108,6 +3128,16 @@ static bool __init check_ioapic_information(void) if (cmdline_maps) fw_bug = ""; + sb_apicid = get_sb_ioapic_id(); + if (sb_apicid < 0) { + /* + * Lack of SB IOAPIC registration is not a firmware bug, + * e.g. kernel booted with noapic or noacpi. + */ + fw_bug = ""; + goto out; + } + for (idx = 0; idx < nr_ioapics; idx++) { int devid, id = mpc_ioapic_id(idx); @@ -3116,17 +3146,16 @@ static bool __init check_ioapic_information(void) pr_err("%s: IOAPIC[%d] not in IVRS table\n", fw_bug, id); ret = false; - } else if (devid == IOAPIC_SB_DEVID || + } else if (id == sb_apicid && (devid == IOAPIC_SB_DEVID || (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON && boot_cpu_data.x86 == 0x18 && boot_cpu_data.x86_model >= 0x4 && boot_cpu_data.x86_model <= 0xf && - devid == IOAPIC_SB_DEVID_FAM18H_M4H)) { + devid == IOAPIC_SB_DEVID_FAM18H_M4H))) { has_sb_ioapic = true; - ret = true; } } - +out: if (!has_sb_ioapic) { /* * We expect the SB IOAPIC to be listed in the IVRS @@ -3137,6 +3166,7 @@ static bool __init check_ioapic_information(void) * device id for the IOAPIC in the system. */ pr_err("%s: No southbridge IOAPIC found\n", fw_bug); + ret = false; } if (!ret) diff --git a/drivers/iommu/amd/nested.c b/drivers/iommu/amd/nested.c index 63b53b29e0298..f1c7987fc5859 100644 --- a/drivers/iommu/amd/nested.c +++ b/drivers/iommu/amd/nested.c @@ -96,7 +96,7 @@ struct iommu_domain * amd_iommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags, const struct iommu_user_data *user_data) { - int ret; + int ret, hdom_id; unsigned long irqflags; struct nested_domain *ndom; struct guest_domain_mapping_info *gdom_info; @@ -161,8 +161,8 @@ amd_iommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags, } /* The gDomID does not exist. We allocate new hdom_id */ - gdom_info->hdom_id = amd_iommu_pdom_id_alloc(); - if (gdom_info->hdom_id <= 0) { + hdom_id = amd_iommu_pdom_id_alloc(); + if (hdom_id <= 0) { __xa_cmpxchg(&aviommu->gdomid_array, ndom->gdom_id, gdom_info, NULL, GFP_ATOMIC); xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags); @@ -170,6 +170,7 @@ amd_iommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags, goto out_err_gdom_info; } + gdom_info->hdom_id = hdom_id; ndom->gdom_info = gdom_info; refcount_set(&gdom_info->users, 1); diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c index 1f8d2823bea42..1dde19a79e80f 100644 --- a/drivers/iommu/amd/ppr.c +++ b/drivers/iommu/amd/ppr.c @@ -140,7 +140,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw) if (event.fault.prm.pasid == 0 || event.fault.prm.pasid >= dev_data->max_pasids) { pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n", - event.fault.prm.pasid, pdev->dev.id); + event.fault.prm.pasid, dev_data->devid); goto out; } @@ -151,7 +151,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw) /* Submit event */ iommu_report_device_fault(&pdev->dev, &event); - + pci_dev_put(pdev); return; out: @@ -159,6 +159,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw) amd_iommu_complete_ppr(&pdev->dev, PPR_PASID(raw[0]), IOMMU_PAGE_RESP_FAILURE, PPR_TAG(raw[0]) & 0x1FF); + pci_dev_put(pdev); } void amd_iommu_poll_ppr_log(struct amd_iommu *iommu) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 1ed8a6f29dc44..4cd60d3c450ac 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -92,16 +92,6 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) & CTXDESC_CD_1_TTB0_MASK); - - /* - * Enable Hardware Access and Dirty updates (DBM) if supported. - * This is safe to enable by default, as PTE_WRITE and PTE_DBM - * share the same bit. - */ - if (master->smmu->features & ARM_SMMU_FEAT_HA) - target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA); - if (master->smmu->features & ARM_SMMU_FEAT_HD) - target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HD); } else { target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0); @@ -114,6 +104,17 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, target->data[0] &= cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R)); } + /* + * Enable Hardware Access and Dirty updates (DBM) if supported. This is + * safe to enable by default, as PTE_WRITE and PTE_DBM share the same bit, + * while the EPD0 config can't get as far as fetching any PTEs anyway. + */ + if (master->smmu->features & ARM_SMMU_FEAT_HA) + target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA); + if (master->smmu->features & ARM_SMMU_FEAT_HD) + target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HD); + if (master->smmu->features & ARM_SMMU_FEAT_HAFT && system_supports_haft()) + target->data[1] |= cpu_to_le64(CTXDESC_CD_1_HAFT); /* * MAIR value is pretty much constant and global, so we can just get it @@ -211,6 +212,9 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) if (system_supports_bbml2_noabort()) feat_mask |= ARM_SMMU_FEAT_BBML2; + if (system_supports_haft()) + feat_mask |= ARM_SMMU_FEAT_HAFT; + if ((smmu->features & feat_mask) != feat_mask) return false; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index a10affb483a4f..b52dae11fbbef 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1240,9 +1240,9 @@ VISIBLE_IF_KUNIT void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target, __le64 *safe_bits) { - const __le64 eats_s1chk = + const u64 eats_s1chk = FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK); - const __le64 eats_trans = + const u64 eats_trans = FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS); /* @@ -4398,6 +4398,20 @@ int arm_smmu_cmdq_init(struct arm_smmu_device *smmu, return 0; } +static void arm_smmu_free_iopf_action(void *data) +{ + struct iopf_queue *queue = data; + + iopf_queue_free(queue); +} + +static void arm_smmu_destroy_vmid_map(void *data) +{ + struct ida *ida = data; + + ida_destroy(ida); +} + static int arm_smmu_init_queues(struct arm_smmu_device *smmu) { int ret; @@ -4425,6 +4439,11 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu) smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev)); if (!smmu->evtq.iopf) return -ENOMEM; + ret = devm_add_action_or_reset(smmu->dev, + arm_smmu_free_iopf_action, + smmu->evtq.iopf); + if (ret) + return ret; } /* priq */ @@ -4503,7 +4522,8 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) ida_init(&smmu->vmid_map); - return 0; + return devm_add_action_or_reset(smmu->dev, arm_smmu_destroy_vmid_map, + &smmu->vmid_map); } static int arm_smmu_init_structures(struct arm_smmu_device *smmu) @@ -4533,8 +4553,9 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, u32 reg; writel_relaxed(val, smmu->base + reg_off); - return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, - 1, ARM_SMMU_POLL_TIMEOUT_US); + return readl_relaxed_poll_timeout_atomic(smmu->base + ack_off, reg, + reg == val, 1, + ARM_SMMU_POLL_TIMEOUT_US); } /* GBPA is "special" */ @@ -4716,6 +4737,15 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu) return ret; } +static void arm_smmu_disable_action(void *data) +{ + struct arm_smmu_device *smmu = data; + + if (smmu->impl_ops && smmu->impl_ops->device_disable) + smmu->impl_ops->device_disable(smmu); + arm_smmu_device_disable(smmu); +} + static void arm_smmu_write_strtab(struct arm_smmu_device *smmu) { struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; @@ -4921,10 +4951,14 @@ static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu) static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg) { - u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD); + u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD | + ARM_SMMU_FEAT_HAFT); u32 hw_features = 0; switch (FIELD_GET(IDR0_HTTU, reg)) { + case IDR0_HTTU_ACCESS_DIRTY_HAFT: + hw_features |= ARM_SMMU_FEAT_HAFT; + fallthrough; case IDR0_HTTU_ACCESS_DIRTY: hw_features |= ARM_SMMU_FEAT_HD; fallthrough; @@ -5256,6 +5290,9 @@ static int arm_smmu_device_acpi_probe(struct platform_device *pdev, smmu->features |= ARM_SMMU_FEAT_COHERENCY; switch (FIELD_GET(ACPI_IORT_SMMU_V3_HTTU_OVERRIDE, iort_smmu->flags)) { + case IDR0_HTTU_ACCESS_DIRTY_HAFT: + smmu->features |= ARM_SMMU_FEAT_HAFT; + fallthrough; case IDR0_HTTU_ACCESS_DIRTY: smmu->features |= ARM_SMMU_FEAT_HD; fallthrough; @@ -5472,7 +5509,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) /* Initialise in-memory data structures */ ret = arm_smmu_init_structures(smmu); if (ret) - goto err_free_iopf; + return ret; /* Record our private device structure */ platform_set_drvdata(pdev, smmu); @@ -5482,30 +5519,30 @@ static int arm_smmu_device_probe(struct platform_device *pdev) /* Reset the device */ ret = arm_smmu_device_reset(smmu); + if (ret) { + arm_smmu_device_disable(smmu); + return ret; + } + + /* Register last so it unwinds first, while the CMDQ is still up. */ + ret = devm_add_action_or_reset(smmu->dev, arm_smmu_disable_action, smmu); if (ret) - goto err_disable; + return ret; /* And we're up. Go go go! */ ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, "smmu3.%pa", &ioaddr); if (ret) - goto err_disable; + return ret; ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); if (ret) { dev_err(dev, "Failed to register iommu\n"); - goto err_free_sysfs; + iommu_device_sysfs_remove(&smmu->iommu); + return ret; } return 0; - -err_free_sysfs: - iommu_device_sysfs_remove(&smmu->iommu); -err_disable: - arm_smmu_device_disable(smmu); -err_free_iopf: - iopf_queue_free(smmu->evtq.iopf); - return ret; } static void arm_smmu_device_remove(struct platform_device *pdev) @@ -5514,9 +5551,6 @@ static void arm_smmu_device_remove(struct platform_device *pdev) iommu_device_unregister(&smmu->iommu); iommu_device_sysfs_remove(&smmu->iommu); - arm_smmu_device_disable(smmu); - iopf_queue_free(smmu->evtq.iopf); - ida_destroy(&smmu->vmid_map); } static void arm_smmu_device_shutdown(struct platform_device *pdev) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index c909c9a88538b..6859570248e9e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -40,6 +40,7 @@ struct arm_vsmmu; #define IDR0_HTTU GENMASK(7, 6) #define IDR0_HTTU_ACCESS 1 #define IDR0_HTTU_ACCESS_DIRTY 2 +#define IDR0_HTTU_ACCESS_DIRTY_HAFT 3 #define IDR0_COHACC (1 << 4) #define IDR0_TTF GENMASK(3, 2) #define IDR0_TTF_AARCH64 2 @@ -369,6 +370,7 @@ static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid) #define CTXDESC_CD_0_ASET (1UL << 47) #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48) +#define CTXDESC_CD_1_HAFT (1UL << 3) #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4) /* @@ -871,6 +873,7 @@ struct arm_smmu_strtab_cfg { struct arm_smmu_impl_ops { int (*device_reset)(struct arm_smmu_device *smmu); + void (*device_disable)(struct arm_smmu_device *smmu); void (*device_remove)(struct arm_smmu_device *smmu); int (*init_structures)(struct arm_smmu_device *smmu); struct arm_smmu_cmdq *(*get_secondary_cmdq)( @@ -921,6 +924,7 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_HD (1 << 22) #define ARM_SMMU_FEAT_S2FWB (1 << 23) #define ARM_SMMU_FEAT_BBML2 (1 << 24) +#define ARM_SMMU_FEAT_HAFT (1 << 25) u32 features; #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0) diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c index 67be62a6e7640..ef08367567d39 100644 --- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c @@ -56,6 +56,8 @@ #define VINTF_ENABLED BIT(0) #define TEGRA241_VINTF_SID_MATCH(s) (0x0040 + 0x4*(s)) +#define VINTF_SID_MATCH_VIRT_SID GENMASK(20, 1) +#define VINTF_SID_MATCH_ENABLE BIT(0) #define TEGRA241_VINTF_SID_REPLACE(s) (0x0080 + 0x4*(s)) #define TEGRA241_VINTF_LVCMDQ_ERR_MAP_64(m) \ @@ -318,16 +320,30 @@ static void tegra241_vintf0_handle_error(struct tegra241_vintf *vintf) while (map) { unsigned long lidx = __ffs64(map); - struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx]; - u32 gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)); + struct tegra241_vcmdq *vcmdq; + u32 gerror; + map &= ~BIT_ULL(lidx); + + /* Pairs with smp_store_release() publishing it */ + vcmdq = smp_load_acquire(&vintf->lvcmdqs[lidx]); + if (!vcmdq) + continue; + + gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)); __arm_smmu_cmdq_skip_err(&vintf->cmdqv->smmu, &vcmdq->cmdq); writel(gerror, REG_VCMDQ_PAGE0(vcmdq, GERRORN)); - map &= ~BIT_ULL(lidx); } } } +/* + * The CMDQV error interrupt is edge-triggered, so a pending VINTF error fires + * this ISR once and does not re-assert. An unacked guest therefore cannot + * storm the host. The HW latches and forwards each new error event on its + * own, so an already-set ERR_MAP bit does not suppress the interrupt for a + * new error. + */ static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid) { struct tegra241_cmdqv *cmdqv = (struct tegra241_cmdqv *)devid; @@ -350,16 +366,27 @@ static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid) /* Handle VINTF0 and its LVCMDQs */ if (vintf_map & BIT_ULL(0)) { - tegra241_vintf0_handle_error(cmdqv->vintfs[0]); + struct tegra241_vintf *vintf0; + vintf_map &= ~BIT_ULL(0); + + /* NULL until tegra241_cmdqv_init_structures() publishes it */ + vintf0 = smp_load_acquire(&cmdqv->vintfs[0]); + if (vintf0) + tegra241_vintf0_handle_error(vintf0); } /* Handle other user VINTFs and their LVCMDQs */ while (vintf_map) { unsigned long idx = __ffs64(vintf_map); + struct tegra241_vintf *vintf; - tegra241_vintf_user_handle_error(cmdqv->vintfs[idx]); vintf_map &= ~BIT_ULL(idx); + + /* The slot may be published or torn down (NULL'd) concurrently */ + vintf = smp_load_acquire(&cmdqv->vintfs[idx]); + if (vintf) + tegra241_vintf_user_handle_error(vintf); } return IRQ_HANDLED; @@ -666,7 +693,6 @@ static int tegra241_vintf_init_lvcmdq(struct tegra241_vintf *vintf, u16 lidx, vcmdq->page0 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE0(idx, lidx); vcmdq->page1 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE1(idx, lidx); - vintf->lvcmdqs[lidx] = vcmdq; return 0; } @@ -682,7 +708,7 @@ static void tegra241_vintf_free_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) dev_dbg(vintf->cmdqv->dev, "%sdeallocated\n", lvcmdq_error_header(vcmdq, header, 64)); /* Guest-owned VCMDQ is free-ed with hw_queue by iommufd core */ - if (vcmdq->vintf->hyp_own) + if (!vcmdq->vintf->idx) kfree(vcmdq); } @@ -705,14 +731,15 @@ tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) /* Build an arm_smmu_cmdq for each LVCMDQ */ ret = tegra241_vcmdq_alloc_smmu_cmdq(vcmdq); if (ret) - goto deinit_lvcmdq; + goto free_vcmdq; + + /* Pairs with the smp_load_acquire() in the error ISR */ + smp_store_release(&vintf->lvcmdqs[lidx], vcmdq); dev_dbg(cmdqv->dev, "%sallocated\n", lvcmdq_error_header(vcmdq, header, 64)); return vcmdq; -deinit_lvcmdq: - tegra241_vintf_deinit_lvcmdq(vintf, lidx); free_vcmdq: kfree(vcmdq); return ERR_PTR(ret); @@ -723,8 +750,18 @@ tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) static void tegra241_cmdqv_deinit_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) { kfree(cmdqv->vintfs[idx]->lvcmdqs); + /* + * Clear the slot and drain any in-flight ISR before returning idx to + * the IDA, so a concurrent create that reuses idx cannot have its + * freshly published VINTF erased here. A plain WRITE_ONCE() suffices + * since clearing the slot publishes no data. This also covers the + * init-failure unwind, which reaches deinit_vintf() without the + * destroy callback. + */ + WRITE_ONCE(cmdqv->vintfs[idx], NULL); + if (cmdqv->irq > 0) + synchronize_irq(cmdqv->irq); ida_free(&cmdqv->vintf_ids, idx); - cmdqv->vintfs[idx] = NULL; } static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx, @@ -750,7 +787,8 @@ static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx, return -ENOMEM; } - cmdqv->vintfs[idx] = vintf; + /* Pairs with the smp_load_acquire() in tegra241_cmdqv_isr() */ + smp_store_release(&cmdqv->vintfs[idx], vintf); return ret; } @@ -761,8 +799,6 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) struct tegra241_vintf *vintf = cmdqv->vintfs[idx]; u16 lidx; - tegra241_vintf_hw_deinit(vintf); - /* Remove LVCMDQ resources */ for (lidx = 0; lidx < vintf->cmdqv->num_lvcmdqs_per_vintf; lidx++) if (vintf->lvcmdqs[lidx]) @@ -770,7 +806,7 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) dev_dbg(cmdqv->dev, "VINTF%u: deallocated\n", vintf->idx); tegra241_cmdqv_deinit_vintf(cmdqv, idx); - if (!vintf->hyp_own) { + if (vintf->idx) { mutex_destroy(&vintf->lvcmdq_mutex); ida_destroy(&vintf->sids); /* Guest-owned VINTF is free-ed with viommu by iommufd core */ @@ -779,12 +815,31 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) } } +static void tegra241_cmdqv_hw_disable(struct arm_smmu_device *smmu) +{ + struct tegra241_cmdqv *cmdqv = + container_of(smmu, struct tegra241_cmdqv, smmu); + u16 idx; + + for (idx = 0; idx < cmdqv->num_vintfs; idx++) + if (cmdqv->vintfs[idx]) + tegra241_vintf_hw_deinit(cmdqv->vintfs[idx]); +} + static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu) { struct tegra241_cmdqv *cmdqv = container_of(smmu, struct tegra241_cmdqv, smmu); u16 idx; + /* + * Free the IRQ before tearing down the VINTFs. free_irq() waits for any + * in-flight tegra241_cmdqv_isr() to finish and blocks new ones, so the + * ISR cannot dereference a VINTF that is freed by the loop below. + */ + if (cmdqv->irq > 0) + free_irq(cmdqv->irq, cmdqv); + /* Remove VINTF resources */ for (idx = 0; idx < cmdqv->num_vintfs; idx++) { if (cmdqv->vintfs[idx]) { @@ -797,8 +852,6 @@ static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu) /* Remove cmdqv resources */ ida_destroy(&cmdqv->vintf_ids); - if (cmdqv->irq > 0) - free_irq(cmdqv->irq, cmdqv); iounmap(cmdqv->base); kfree(cmdqv->vintfs); put_device(cmdqv->dev); /* smmu->impl_dev */ @@ -844,6 +897,7 @@ static struct arm_smmu_impl_ops tegra241_cmdqv_impl_ops = { /* For in-kernel use */ .get_secondary_cmdq = tegra241_cmdqv_get_cmdq, .device_reset = tegra241_cmdqv_hw_reset, + .device_disable = tegra241_cmdqv_hw_disable, .device_remove = tegra241_cmdqv_remove, /* For user-space use */ .hw_info = tegra241_cmdqv_hw_info, @@ -869,6 +923,12 @@ static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu) ret = tegra241_cmdqv_init_vintf(cmdqv, 0, vintf); if (ret) { dev_err(cmdqv->dev, "failed to init vintf0: %d\n", ret); + /* + * tegra241_cmdqv_init_vintf() failed to publish the vintf0 to + * cmdqv->vintfs[], so the probe unwind path that goes through + * cmdqv->vintfs[] would miss it. Free it here. + */ + kfree(vintf); return ret; } @@ -890,16 +950,22 @@ static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu) static struct dentry *cmdqv_debugfs_dir; #endif -static struct arm_smmu_device * -__tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, - int irq) +/* + * Probe the CMDQV and reallocate @smmu into the larger cmdqv->smmu. + * + * devm_krealloc() may relocate and free the original @smmu, so update *smmu to + * the new pointer once it succeeds. The error paths after it do the same, so a + * caller falling back keeps a live @smmu instead of the freed original. + */ +static int __tegra241_cmdqv_probe(struct arm_smmu_device **smmu, + struct resource *res, int irq) { static const struct arm_smmu_impl_ops init_ops = { .init_structures = tegra241_cmdqv_init_structures, .device_remove = tegra241_cmdqv_remove, }; - struct tegra241_cmdqv *cmdqv = NULL; - struct arm_smmu_device *new_smmu; + struct device *dev = (*smmu)->dev; + struct tegra241_cmdqv *cmdqv; void __iomem *base; u32 regval; int ret; @@ -908,38 +974,30 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, base = ioremap(res->start, resource_size(res)); if (!base) { - dev_err(smmu->dev, "failed to ioremap\n"); - return NULL; + dev_err(dev, "failed to ioremap\n"); + return -ENOMEM; } regval = readl(base + TEGRA241_CMDQV_CONFIG); if (disable_cmdqv) { - dev_info(smmu->dev, "Detected disable_cmdqv=true\n"); + dev_info(dev, "Detected disable_cmdqv=true\n"); writel(regval & ~CMDQV_EN, base + TEGRA241_CMDQV_CONFIG); + ret = -ENODEV; goto iounmap; } - cmdqv = devm_krealloc(smmu->dev, smmu, sizeof(*cmdqv), GFP_KERNEL); - if (!cmdqv) + cmdqv = devm_krealloc(dev, *smmu, sizeof(*cmdqv), GFP_KERNEL); + if (!cmdqv) { + ret = -ENOMEM; goto iounmap; - new_smmu = &cmdqv->smmu; + } + *smmu = &cmdqv->smmu; cmdqv->irq = irq; cmdqv->base = base; - cmdqv->dev = smmu->impl_dev; + cmdqv->dev = (*smmu)->impl_dev; cmdqv->base_phys = res->start; - if (cmdqv->irq > 0) { - ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr, - IRQF_ONESHOT, "tegra241-cmdqv", - cmdqv); - if (ret) { - dev_err(cmdqv->dev, "failed to request irq (%d): %d\n", - cmdqv->irq, ret); - goto iounmap; - } - } - regval = readl_relaxed(REG_CMDQV(cmdqv, PARAM)); cmdqv->num_vintfs = 1 << FIELD_GET(CMDQV_NUM_VINTF_LOG2, regval); cmdqv->num_vcmdqs = 1 << FIELD_GET(CMDQV_NUM_VCMDQ_LOG2, regval); @@ -949,11 +1007,28 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, cmdqv->vintfs = kzalloc_objs(*cmdqv->vintfs, cmdqv->num_vintfs); - if (!cmdqv->vintfs) - goto free_irq; + if (!cmdqv->vintfs) { + ret = -ENOMEM; + goto iounmap; + } ida_init(&cmdqv->vintf_ids); + /* + * Request the IRQ only after cmdqv->vintfs is allocated and zeroed, so + * the ISR would not walk an uninitialized array. + */ + if (cmdqv->irq > 0) { + ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr, + IRQF_ONESHOT, "tegra241-cmdqv", + cmdqv); + if (ret) { + dev_err(cmdqv->dev, "failed to request irq (%d): %d\n", + cmdqv->irq, ret); + goto free_vintfs; + } + } + #ifdef CONFIG_IOMMU_DEBUGFS if (!cmdqv_debugfs_dir) { cmdqv_debugfs_dir = @@ -964,24 +1039,23 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, #endif /* Provide init-level ops only, until tegra241_cmdqv_init_structures */ - new_smmu->impl_ops = &init_ops; + cmdqv->smmu.impl_ops = &init_ops; - return new_smmu; + return 0; -free_irq: - if (cmdqv->irq > 0) - free_irq(cmdqv->irq, cmdqv); +free_vintfs: + ida_destroy(&cmdqv->vintf_ids); + kfree(cmdqv->vintfs); iounmap: iounmap(base); - return NULL; + return ret; } struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu) { struct platform_device *pdev = to_platform_device(smmu->impl_dev); - struct arm_smmu_device *new_smmu; struct resource *res; - int irq; + int irq, ret; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) { @@ -994,15 +1068,15 @@ struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu) dev_warn(&pdev->dev, "no interrupt. errors will not be reported\n"); - new_smmu = __tegra241_cmdqv_probe(smmu, res, irq); - if (new_smmu) - return new_smmu; + ret = __tegra241_cmdqv_probe(&smmu, res, irq); + if (!ret) + return smmu; out_fallback: dev_info(smmu->impl_dev, "Falling back to standard SMMU CMDQ\n"); smmu->options &= ~ARM_SMMU_OPT_TEGRA241_CMDQV; put_device(smmu->impl_dev); - return ERR_PTR(-ENODEV); + return smmu; } /* User space VINTF and VCMDQ Functions */ @@ -1130,13 +1204,15 @@ static int tegra241_vintf_alloc_lvcmdq_user(struct iommufd_hw_queue *hw_queue, if (ret) goto unmap_lvcmdq; + /* No lockless reader of a user VINTF's lvcmdqs[]; mutex-serialized */ + vintf->lvcmdqs[lidx] = vcmdq; + hw_queue->destroy = &tegra241_vintf_destroy_lvcmdq_user; mutex_unlock(&vintf->lvcmdq_mutex); return 0; unmap_lvcmdq: tegra241_vcmdq_unmap_lvcmdq(vcmdq); - tegra241_vintf_deinit_lvcmdq(vintf, lidx); undepend_vcmdq: if (vcmdq->prev) iommufd_hw_queue_undepend(vcmdq, vcmdq->prev, core); @@ -1152,6 +1228,7 @@ static void tegra241_cmdqv_destroy_vintf_user(struct iommufd_viommu *viommu) if (vintf->mmap_offset) iommufd_viommu_destroy_mmap(&vintf->vsmmu.core, vintf->mmap_offset); + tegra241_vintf_hw_deinit(vintf); tegra241_cmdqv_remove_vintf(vintf->cmdqv, vintf->idx); } @@ -1178,10 +1255,11 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev) u64 virt_sid = vdev->virt_id; int sidx; - if (virt_sid > UINT_MAX) + if (virt_sid > FIELD_MAX(VINTF_SID_MATCH_VIRT_SID)) return -EINVAL; - WARN_ON_ONCE(master->num_streams != 1); + if (master->num_streams != 1) + return -EOPNOTSUPP; /* Find an empty pair of SID_REPLACE and SID_MATCH */ sidx = ida_alloc_max(&vintf->sids, vintf->cmdqv->num_sids_per_vintf - 1, @@ -1190,7 +1268,9 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev) return sidx; writel(stream->id, REG_VINTF(vintf, SID_REPLACE(sidx))); - writel(virt_sid << 1 | 0x1, REG_VINTF(vintf, SID_MATCH(sidx))); + writel(FIELD_PREP(VINTF_SID_MATCH_VIRT_SID, virt_sid) | + VINTF_SID_MATCH_ENABLE, + REG_VINTF(vintf, SID_MATCH(sidx))); dev_dbg(vintf->cmdqv->dev, "VINTF%u: allocated SID_REPLACE%d for pSID=%x, vSID=%x\n", vintf->idx, sidx, stream->id, (u32)virt_sid); diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c index a1e8cf29f5948..09f2ee6be988b 100644 --- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c +++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c @@ -200,7 +200,7 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev) fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0); iova = iommu_readq(ctx, ARM_SMMU_CB_FAR); - if (!report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) { + if (report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) { dev_err_ratelimited(ctx->dev, "Unhandled context fault: fsr=0x%x, " "iova=0x%016llx, fsynr=0x%x, cb=%d\n", @@ -855,7 +855,7 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev); if (ret) { dev_err(dev, "Failed to register iommu\n"); - goto err_pm_disable; + goto err_sysfs_remove; } if (qcom_iommu->local_base) { @@ -866,6 +866,8 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) return 0; +err_sysfs_remove: + iommu_device_sysfs_remove(&qcom_iommu->iommu); err_pm_disable: pm_runtime_disable(dev); return ret; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 9abaec0703efb..ae814d8c99c04 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1671,13 +1671,16 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, } if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && - !gfpflags_allow_blocking(gfp) && !coherent) + !gfpflags_allow_blocking(gfp) && !coherent) { page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr, - gfp, NULL); - else + gfp, NULL); + if (!page) + return NULL; + } else { cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); - if (!cpu_addr) - return NULL; + if (!cpu_addr) + return NULL; + } *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot, dev->coherent_dma_mask); @@ -2204,6 +2207,19 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, dma_addr_t iova; int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; size_t size = cookie_msi_granule(domain); + static DEFINE_MUTEX(msi_prepare_lock); + + /* + * Normally a device's default domain is only ever attached to that + * device's own group, and the group mutex held by + * iommu_group_mutex_assert()'s callers is enough on its own. A VFIO + * type1 container is the one case that breaks that assumption: it + * can merge devices from different groups onto one domain, so two + * devices' group mutexes don't serialize each other here. A static + * lock is sufficient due to the expectation that this is a corner + * case that will never be contended in practice. + */ + guard(mutex)(&msi_prepare_lock); msi_addr &= ~(phys_addr_t)(size - 1); list_for_each_entry(msi_page, msi_page_list, list) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index c2752151c80af..07ec2b3ab986a 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -224,7 +224,7 @@ phys_addr_t DOMAIN_NS(iova_to_phys)(struct iommu_domain *domain, ret = make_range(common_from_iommu(iommu_table), &range, iova, 1); if (ret) - return ret; + return 0; ret = pt_walk_range(&range, __iova_to_phys, &res); /* PHYS_ADDR_MAX would be a better error code */ diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 767ec092accd7..e32685402f74b 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -915,6 +915,18 @@ dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg) return 0; } +static bool dmar_required(void) +{ + /* tboot supersedes any user/platform opt */ + if (!intel_iommu_tboot_noforce && tboot_enabled()) + return true; + + if (!no_iommu && (!dmar_disabled || dmar_platform_optin())) + return true; + + return false; +} + void __init detect_intel_iommu(void) { int ret; @@ -928,8 +940,7 @@ void __init detect_intel_iommu(void) if (!ret) ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl, &validate_drhd_cb); - if (!ret && !no_iommu && !iommu_detected && - (!dmar_disabled || dmar_platform_optin())) { + if (!ret && !iommu_detected && dmar_required()) { iommu_detected = 1; /* Make sure ACS will be enabled */ pci_request_acs(); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 849d06dfe1aec..9a87ac1250d7b 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -57,7 +57,7 @@ static int rwbf_quirk; * (used when kernel is launched w/ TXT) */ static int force_on = 0; -static int intel_iommu_tboot_noforce; +int intel_iommu_tboot_noforce; static int no_platform_optin; #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry)) @@ -1108,7 +1108,8 @@ static void copied_context_tear_down(struct intel_iommu *iommu, assert_spin_locked(&iommu->lock); did_old = context_domain_id(context); - context_clear_entry(context); + context_clear_present(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); if (did_old < cap_ndoms(iommu->cap)) { iommu->flush.flush_context(iommu, did_old, @@ -1119,6 +1120,9 @@ static void copied_context_tear_down(struct intel_iommu *iommu, DMA_TLB_DSI_FLUSH); } + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + clear_context_copied(iommu, bus, devfn); } @@ -1246,7 +1250,7 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); - intel_context_flush_no_pasid(info, context, did); + intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn)); context_clear_entry(context); __iommu_flush_cache(iommu, context, sizeof(*context)); } @@ -1446,7 +1450,7 @@ static int copy_context_table(struct intel_iommu *iommu, struct context_entry **tbl, int bus, bool ext) { - int tbl_idx, pos = 0, idx, devfn, ret = 0, did; + int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0, did; struct context_entry *new_ce = NULL, ce; struct context_entry *old_ce = NULL; struct root_entry re; @@ -1462,10 +1466,9 @@ static int copy_context_table(struct intel_iommu *iommu, if (idx == 0) { /* First save what we may have and clean up */ if (new_ce) { - tbl[tbl_idx] = new_ce; + tbl[tbl_idx + tbl_slot] = new_ce; __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE); - pos = 1; } if (old_ce) @@ -1487,6 +1490,9 @@ static int copy_context_table(struct intel_iommu *iommu, } } + /* Track if saving UCTP or LCTP entries in scalable mode */ + tbl_slot = ext && devfn >= 0x80 ? 1 : 0; + ret = -ENOMEM; old_ce = memremap(old_ce_phys, PAGE_SIZE, MEMREMAP_WB); @@ -1515,7 +1521,7 @@ static int copy_context_table(struct intel_iommu *iommu, new_ce[idx] = ce; } - tbl[tbl_idx + pos] = new_ce; + tbl[tbl_idx + tbl_slot] = new_ce; __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE); @@ -2482,10 +2488,11 @@ static bool has_external_pci(void) static int __init platform_optin_force_iommu(void) { - if (!dmar_platform_optin() || no_platform_optin || !has_external_pci()) + if (no_iommu || !dmar_platform_optin() || no_platform_optin || + !has_external_pci()) return 0; - if (no_iommu || dmar_disabled) + if (dmar_disabled) pr_info("Intel-IOMMU force enabled due to platform opt in\n"); /* @@ -2496,7 +2503,6 @@ static int __init platform_optin_force_iommu(void) iommu_set_default_passthrough(false); dmar_disabled = 0; - no_iommu = 0; return 1; } @@ -3135,13 +3141,13 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, if (ret) return ret; - ret = iopf_for_domain_set(domain, dev); + ret = iopf_for_domain_replace(domain, old, dev); if (ret) return ret; ret = dmar_domain_attach_device(to_dmar_domain(domain), dev); if (ret) - iopf_for_domain_remove(domain, dev); + iopf_for_domain_replace(old, domain, dev); return ret; } @@ -3316,6 +3322,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) return &iommu->iommu; free_table: + intel_pasid_teardown_sm_context(dev); intel_pasid_free_table(dev); clear_rbtree: device_rbtree_remove(info); @@ -3844,10 +3851,13 @@ static int identity_domain_attach_dev(struct iommu_domain *domain, return 0; /* - * No PRI support with the global identity domain. No need to enable or - * disable PRI in this path as the iommu has been put in the blocking - * state. + * The identity domain has no iopf_handler, so no IOPF reference is + * taken for it. The reference held by the old domain must still be + * released here; putting the device in the blocking state above does + * not affect the IOPF reference count. */ + iopf_for_domain_remove(old, dev); + if (sm_supported(iommu)) ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID); else diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 775f1c4ae3463..2df59d999715a 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1240,7 +1240,7 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, unsigned long end); void intel_context_flush_no_pasid(struct device_domain_info *info, - struct context_entry *context, u16 did); + struct context_entry *context, u16 did, u16 sid); int intel_iommu_enable_prq(struct intel_iommu *iommu); int intel_iommu_finish_prq(struct intel_iommu *iommu); @@ -1354,6 +1354,7 @@ static inline bool ecmd_has_pmu_essential(struct intel_iommu *iommu) extern int dmar_disabled; extern int intel_iommu_enabled; +extern int intel_iommu_tboot_noforce; #else static inline int iommu_calculate_agaw(struct intel_iommu *iommu) { @@ -1366,6 +1367,7 @@ static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu) #define dmar_disabled (1) #define intel_iommu_enabled (0) #define intel_iommu_sm (0) +#define intel_iommu_tboot_noforce (0) #endif static inline const char *decode_prq_descriptor(char *str, size_t size, diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 81353fd46b37e..e4f24d3f19a61 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -751,7 +751,7 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn) context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); - intel_context_flush_no_pasid(info, context, did); + intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn)); context_clear_entry(context); __iommu_flush_cache(iommu, context, sizeof(*context)); } @@ -955,9 +955,12 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info) * This helper can only be used when IOMMU is working in the legacy mode or * IOMMU is in scalable mode but all PASID table entries of the device are * non-present. + * + * @sid identifies the context entry that was modified, which may be a DMA + * alias of @info->dev rather than its own requester ID. */ void intel_context_flush_no_pasid(struct device_domain_info *info, - struct context_entry *context, u16 did) + struct context_entry *context, u16 did, u16 sid) { struct intel_iommu *iommu = info->iommu; @@ -967,7 +970,7 @@ void intel_context_flush_no_pasid(struct device_domain_info *info, * when operating in scalable mode. Therefore the @did value doesn't * matter in scalable mode. */ - iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info->devfn), + iommu->flush.flush_context(iommu, did, sid, DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); /* diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index bc7c7232a43e2..9742cb593577d 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -145,8 +145,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm list_add(&domain->next, &iommu_mm->sva_domains); out: refcount_set(&handle->users, 1); - mutex_unlock(&iommu_sva_lock); handle->dev = dev; + mutex_unlock(&iommu_sva_lock); return handle; out_free_domain: diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index e8f13dcebbde5..23a531595835f 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -390,7 +390,7 @@ int iommu_mock_device_add(struct device *dev, struct iommu_device *iommu) rc = device_add(dev); if (rc) - iommu_fwspec_free(dev); + dev_iommu_free(dev); return rc; } EXPORT_SYMBOL_GPL(iommu_mock_device_add); diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index 2895e5370910f..c5d122f33ba2d 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -148,29 +148,22 @@ static void iommufd_device_remove_vdev(struct iommufd_device *idev) if (!idev->vdev) goto out_unlock; - vdev = iommufd_get_vdevice(idev->ictx, idev->vdev->obj.id); + vdev = idev->vdev; + /* * An ongoing vdev destroy ioctl has removed the vdev from the object * xarray, but has not finished iommufd_vdevice_destroy() yet as it * needs the same mutex. We exit the locking then wait on wait_cnt * reference for the vdev destruction. */ - if (IS_ERR(vdev)) - goto out_unlock; - - /* Should never happen */ - if (WARN_ON(vdev != idev->vdev)) { - iommufd_put_object(idev->ictx, &vdev->obj); + if (iommufd_try_inc_users(idev->ictx, &vdev->obj)) goto out_unlock; - } /* * vdev is still alive. Hold a users refcount to prevent racing with * userspace destruction, then use iommufd_object_tombstone_user() to * destroy it and leave a tombstone. */ - refcount_inc(&vdev->obj.users); - iommufd_put_object(idev->ictx, &vdev->obj); mutex_unlock(&idev->igroup->lock); iommufd_object_tombstone_user(idev->ictx, &vdev->obj); return; @@ -1307,8 +1300,8 @@ void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova, xa_lock(&ioas->iopt.access_list); xa_for_each(&ioas->iopt.access_list, index, access) { - if (!iommufd_lock_obj(&access->obj) || - iommufd_access_is_internal(access)) + if (iommufd_access_is_internal(access) || + !iommufd_lock_obj(&access->obj)) continue; xa_unlock(&ioas->iopt.access_list); diff --git a/drivers/iommu/iommufd/ioas.c b/drivers/iommu/iommufd/ioas.c index fed06c2b728e2..3459776871e6c 100644 --- a/drivers/iommu/iommufd/ioas.c +++ b/drivers/iommu/iommufd/ioas.c @@ -427,6 +427,8 @@ static int iommufd_take_all_iova_rwsem(struct iommufd_ctx *ictx, rc = xa_err(xa_store(ioas_list, index, ioas, GFP_KERNEL)); if (rc) { + up_write(&ioas->iopt.iova_rwsem); + refcount_dec(&ioas->obj.users); iommufd_release_all_iova_rwsem(ictx, ioas_list); return rc; } @@ -535,6 +537,10 @@ int iommufd_ioas_change_process(struct iommufd_ucmd *ucmd) return rc; for_each_ioas_area(&ioas_list, index, ioas, area) { + if (!area->pages) { + rc = -EBUSY; + goto out; + } if (area->pages->type != IOPT_ADDRESS_FILE) { rc = -EINVAL; goto out; diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 43fbc5bed8de3..421d0cc7c1bc3 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -182,6 +182,7 @@ static inline bool iommufd_lock_obj(struct iommufd_object *obj) return true; } +int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj); struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, enum iommufd_object_type type); static inline void iommufd_put_object(struct iommufd_ctx *ictx, @@ -698,14 +699,6 @@ void iommufd_vdevice_abort(struct iommufd_object *obj); int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd); void iommufd_hw_queue_destroy(struct iommufd_object *obj); -static inline struct iommufd_vdevice * -iommufd_get_vdevice(struct iommufd_ctx *ictx, u32 id) -{ - return container_of(iommufd_get_object(ictx, id, - IOMMUFD_OBJ_VDEVICE), - struct iommufd_vdevice, obj); -} - #ifdef CONFIG_IOMMUFD_TEST int iommufd_test(struct iommufd_ucmd *ucmd); void iommufd_selftest_destroy(struct iommufd_object *obj); diff --git a/drivers/iommu/iommufd/main.c b/drivers/iommu/iommufd/main.c index 8c6d43601afbe..e1097a1db21ad 100644 --- a/drivers/iommu/iommufd/main.c +++ b/drivers/iommu/iommufd/main.c @@ -180,6 +180,26 @@ struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, return obj; } +/* + * Increment the users count of an object outside the context of an ioctl that + * has already locked it. The users refcount cannot be increased on an already + * created object unless the object is installed in the xarray, otherwise things + * are racing with a parallel destruction. + */ +int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj) +{ + struct iommufd_object *cur; + + xa_lock(&ictx->objects); + cur = xa_load(&ictx->objects, obj->id); + if (cur == obj) + refcount_inc(&obj->users); + xa_unlock(&ictx->objects); + if (cur != obj) + return -EBUSY; + return 0; +} + static int iommufd_object_dec_wait(struct iommufd_ctx *ictx, struct iommufd_object *to_destroy) { diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index af07c642a5260..55f21c799d8e6 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -177,6 +178,7 @@ struct mock_dev { struct device dev; struct mock_viommu *viommu; struct rw_semaphore viommu_rwsem; + struct rw_semaphore iopf_rwsem; unsigned long flags; unsigned long vdev_id; int id; @@ -997,6 +999,7 @@ static struct mock_dev *mock_dev_create(unsigned long dev_flags) return ERR_PTR(-ENOMEM); init_rwsem(&mdev->viommu_rwsem); + init_rwsem(&mdev->iopf_rwsem); device_initialize(&mdev->dev); mdev->flags = dev_flags; mdev->dev.release = mock_dev_release; @@ -1022,7 +1025,9 @@ static struct mock_dev *mock_dev_create(unsigned long dev_flags) goto err_put; } + down_write(&mdev->iopf_rwsem); rc = iommu_mock_device_add(&mdev->dev, &mock_iommu.iommu_dev); + up_write(&mdev->iopf_rwsem); if (rc) goto err_put; return mdev; @@ -1077,7 +1082,9 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, } sobj->idev.idev = idev; + down_write(&sobj->idev.mock_dev->iopf_rwsem); rc = iommufd_device_attach(idev, IOMMU_NO_PASID, &pt_id); + up_write(&sobj->idev.mock_dev->iopf_rwsem); if (rc) goto out_unbind; @@ -1092,7 +1099,9 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, return 0; out_detach: + down_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_device_detach(idev, IOMMU_NO_PASID); + up_write(&sobj->idev.mock_dev->iopf_rwsem); out_unbind: iommufd_device_unbind(idev); out_mdev: @@ -1136,7 +1145,9 @@ static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd, if (IS_ERR(sobj)) return PTR_ERR(sobj); + down_write(&sobj->idev.mock_dev->iopf_rwsem); rc = iommufd_device_replace(sobj->idev.idev, IOMMU_NO_PASID, &pt_id); + up_write(&sobj->idev.mock_dev->iopf_rwsem); if (rc) goto out_sobj; @@ -1695,6 +1706,9 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, if (!page_size || !length || iova % page_size || length % page_size || !uptr) return -EINVAL; + max = length / page_size; + if (max > SZ_16M * BITS_PER_BYTE) + return -EOVERFLOW; hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); if (IS_ERR(hwpt)) @@ -1705,7 +1719,6 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, goto out_put; } - max = length / page_size; tmp = kvzalloc(DIV_ROUND_UP(max, BITS_PER_LONG) * sizeof(unsigned long), GFP_KERNEL_ACCOUNT); if (!tmp) { @@ -1739,10 +1752,16 @@ static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd, { struct iopf_fault event = {}; struct iommufd_device *idev; + struct mock_dev *mdev; idev = iommufd_get_device(ucmd, cmd->trigger_iopf.dev_id); if (IS_ERR(idev)) return PTR_ERR(idev); + if (!iommufd_selftest_is_mock_dev(idev->dev)) { + iommufd_put_object(ucmd->ictx, &idev->obj); + return -EINVAL; + } + mdev = to_mock_dev(idev->dev); event.fault.prm.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; if (cmd->trigger_iopf.pasid != IOMMU_NO_PASID) @@ -1753,7 +1772,9 @@ static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd, event.fault.prm.grpid = cmd->trigger_iopf.grpid; event.fault.prm.perm = cmd->trigger_iopf.perm; + down_read(&mdev->iopf_rwsem); iommu_report_device_fault(idev->dev, &event); + up_read(&mdev->iopf_rwsem); iommufd_put_object(ucmd->ictx, &idev->obj); return 0; @@ -1861,14 +1882,19 @@ static int iommufd_test_pasid_attach(struct iommufd_ucmd *ucmd, if (IS_ERR(sobj)) return PTR_ERR(sobj); + down_write(&sobj->idev.mock_dev->iopf_rwsem); rc = iommufd_device_attach(sobj->idev.idev, cmd->pasid_attach.pasid, &cmd->pasid_attach.pt_id); + up_write(&sobj->idev.mock_dev->iopf_rwsem); if (rc) goto out_sobj; rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); - if (rc) + if (rc) { + down_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_device_detach(sobj->idev.idev, cmd->pasid_attach.pasid); + up_write(&sobj->idev.mock_dev->iopf_rwsem); + } out_sobj: iommufd_put_object(ucmd->ictx, &sobj->obj); @@ -1885,8 +1911,10 @@ static int iommufd_test_pasid_replace(struct iommufd_ucmd *ucmd, if (IS_ERR(sobj)) return PTR_ERR(sobj); + down_write(&sobj->idev.mock_dev->iopf_rwsem); rc = iommufd_device_replace(sobj->idev.idev, cmd->pasid_attach.pasid, &cmd->pasid_attach.pt_id); + up_write(&sobj->idev.mock_dev->iopf_rwsem); if (rc) goto out_sobj; @@ -1906,7 +1934,9 @@ static int iommufd_test_pasid_detach(struct iommufd_ucmd *ucmd, if (IS_ERR(sobj)) return PTR_ERR(sobj); + down_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_device_detach(sobj->idev.idev, cmd->pasid_detach.pasid); + up_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_put_object(ucmd->ictx, &sobj->obj); return 0; } @@ -1917,7 +1947,9 @@ void iommufd_selftest_destroy(struct iommufd_object *obj) switch (sobj->type) { case TYPE_IDEV: + down_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_device_detach(sobj->idev.idev, IOMMU_NO_PASID); + up_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_device_unbind(sobj->idev.idev); mock_dev_destroy(sobj->idev.mock_dev); break; diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 0ad5ff431d5b6..9a3ed70563b1b 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -720,7 +720,7 @@ static int msm_iommu_probe(struct platform_device *pdev) iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); if (!iommu) - return -ENODEV; + return -ENOMEM; iommu->dev = &pdev->dev; INIT_LIST_HEAD(&iommu->ctx_list); @@ -784,19 +784,25 @@ static int msm_iommu_probe(struct platform_device *pdev) "msm-smmu.%pa", &ioaddr); if (ret) { pr_err("Could not add msm-smmu at %pa to sysfs\n", &ioaddr); - return ret; + goto err_remove_list; } ret = iommu_device_register(&iommu->iommu, &msm_iommu_ops, &pdev->dev); if (ret) { pr_err("Could not register msm-smmu at %pa\n", &ioaddr); - return ret; + goto err_remove_sysfs; } pr_info("device mapped at %p, irq %d with %d ctx banks\n", iommu->base, iommu->irq, iommu->ncb); return ret; + +err_remove_sysfs: + iommu_device_sysfs_remove(&iommu->iommu); +err_remove_list: + list_del(&iommu->dev_node); + return ret; } static const struct of_device_id msm_iommu_dt_match[] = { diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index ac97dd2868d4b..e907c99531423 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -88,7 +88,7 @@ struct dma_iommu_mapping { /* MTK generation one iommu HW only support 4K size mapping */ #define MT2701_IOMMU_PAGE_SHIFT 12 #define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT) -#define MT2701_LARB_NR_MAX 3 +#define MT2701_LARB_NR_MAX 4 /* * MTK m4u support 4GB iova address space, and only support 4K page diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index cec3ddd7ab103..fe8e6d0f8a23b 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -382,78 +382,97 @@ static int riscv_iommu_queue_wait(struct riscv_iommu_queue *queue, (int)(cons - index) > 0, 0, timeout_us); } -/* Enqueue an entry and wait to be processed if timeout_us > 0 - * - * Error handling for IOMMU hardware not responding in reasonable time - * will be added as separate patch series along with other RAS features. - * For now, only report hardware failure and continue. - */ -static unsigned int riscv_iommu_queue_send(struct riscv_iommu_queue *queue, - void *entry, size_t entry_size) +static int riscv_iommu_queue_wait_for_space(struct riscv_iommu_queue *queue, + unsigned int last) { - unsigned int prod; unsigned int head; unsigned int tail; + unsigned int hw_head; unsigned long flags; + int ret; - /* Do not preempt submission flow. */ - local_irq_save(flags); + ret = riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), hw_head, + !(hw_head & ~queue->mask) && hw_head != last, + 0, RISCV_IOMMU_QUEUE_TIMEOUT); + if (ret) + return ret; - /* 1. Allocate some space in the queue */ - prod = atomic_inc_return(&queue->prod) - 1; + raw_spin_lock_irqsave(&queue->lock, flags); head = atomic_read(&queue->head); + tail = atomic_read(&queue->tail); + if ((tail - head) >= queue->mask) { + last = Q_ITEM(queue, head); + /* + * Re-read hw_head under the lock so that it is consistent with + * the freshly computed 'last'. Using the pre-lock snapshot + * could produce a stale value that wraps around relative to the + * new 'last', advancing the shadow head past entries that have + * not yet been consumed by the hardware. + */ + hw_head = riscv_iommu_readl(queue->iommu, Q_HEAD(queue)); + if (!(hw_head & ~queue->mask) && hw_head != last) + atomic_add((hw_head - last) & queue->mask, &queue->head); + } + raw_spin_unlock_irqrestore(&queue->lock, flags); - /* 2. Wait for space availability. */ - if ((prod - head) > queue->mask) { - if (readx_poll_timeout(atomic_read, &queue->head, - head, (prod - head) < queue->mask, - 0, RISCV_IOMMU_QUEUE_TIMEOUT)) - goto err_busy; - } else if ((prod - head) == queue->mask) { - const unsigned int last = Q_ITEM(queue, head); + return 0; +} + +/* Enqueue an entry and publish it to the hardware queue. */ +static int riscv_iommu_queue_send(struct riscv_iommu_queue *queue, + void *entry, size_t entry_size, + unsigned int *out_prod) +{ + unsigned int prod; + unsigned int head; + unsigned long flags; + int ret; + + /* 1. Wait for space availability and reserve the next slot. */ + for (;;) { + raw_spin_lock_irqsave(&queue->lock, flags); + + prod = atomic_read(&queue->tail); + head = atomic_read(&queue->head); + + if ((prod - head) < queue->mask) + break; + + head = Q_ITEM(queue, head); + raw_spin_unlock_irqrestore(&queue->lock, flags); - if (riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), head, - !(head & ~queue->mask) && head != last, - 0, RISCV_IOMMU_QUEUE_TIMEOUT)) + ret = riscv_iommu_queue_wait_for_space(queue, head); + if (ret) goto err_busy; - atomic_add((head - last) & queue->mask, &queue->head); } - /* 3. Store entry in the ring buffer */ + /* 2. Store entry in the ring buffer. */ memcpy(queue->base + Q_ITEM(queue, prod) * entry_size, entry, entry_size); - /* 4. Wait for all previous entries to be ready */ - if (readx_poll_timeout(atomic_read, &queue->tail, tail, prod == tail, - 0, RISCV_IOMMU_QUEUE_TIMEOUT)) - goto err_busy; - - /* - * 5. Make sure the ring buffer update (whether in normal or I/O memory) is - * completed and visible before signaling the tail doorbell to fetch - * the next command. 'fence ow, ow' - */ + /* 3. Make sure the entry is visible before updating the queue tail. */ dma_wmb(); riscv_iommu_writel(queue->iommu, Q_TAIL(queue), Q_ITEM(queue, prod + 1)); /* - * 6. Make sure the doorbell write to the device has finished before updating - * the shadow tail index in normal memory. 'fence o, w' + * 4. Make sure the doorbell write to the device has finished before + * updating the shadow tail index in normal memory. 'fence o, w' */ #ifdef CONFIG_MMIOWB mmiowb(); #endif - atomic_inc(&queue->tail); + atomic_set(&queue->tail, prod + 1); + atomic_set(&queue->prod, prod + 1); - /* 7. Complete submission and restore local interrupts */ - local_irq_restore(flags); + if (out_prod) + *out_prod = prod; - return prod; + raw_spin_unlock_irqrestore(&queue->lock, flags); + return 0; err_busy: - local_irq_restore(flags); + /* Report the failure and continue; full RAS recovery is not implemented. */ dev_err_once(queue->iommu->dev, "Hardware error: command enqueue failed\n"); - - return prod; + return ret; } /* @@ -492,7 +511,7 @@ static irqreturn_t riscv_iommu_cmdq_process(int irq, void *data) static void riscv_iommu_cmd_send(struct riscv_iommu_device *iommu, struct riscv_iommu_command *cmd) { - riscv_iommu_queue_send(&iommu->cmdq, cmd, sizeof(*cmd)); + riscv_iommu_queue_send(&iommu->cmdq, cmd, sizeof(*cmd), NULL); } /* Send IOFENCE.C command and wait for all scheduled commands to complete. */ @@ -501,9 +520,12 @@ static void riscv_iommu_cmd_sync(struct riscv_iommu_device *iommu, { struct riscv_iommu_command cmd; unsigned int prod; + int ret; riscv_iommu_cmd_iofence(&cmd); - prod = riscv_iommu_queue_send(&iommu->cmdq, &cmd, sizeof(cmd)); + ret = riscv_iommu_queue_send(&iommu->cmdq, &cmd, sizeof(cmd), &prod); + if (ret) + return; if (!timeout_us) return; @@ -1560,6 +1582,7 @@ int riscv_iommu_init(struct riscv_iommu_device *iommu) int rc; RISCV_IOMMU_QUEUE_INIT(&iommu->cmdq, CQ); + raw_spin_lock_init(&iommu->cmdq.lock); RISCV_IOMMU_QUEUE_INIT(&iommu->fltq, FQ); rc = riscv_iommu_init_check(iommu); diff --git a/drivers/iommu/riscv/iommu.h b/drivers/iommu/riscv/iommu.h index 46df79dd54957..5676001548cc7 100644 --- a/drivers/iommu/riscv/iommu.h +++ b/drivers/iommu/riscv/iommu.h @@ -12,6 +12,7 @@ #define _RISCV_IOMMU_H_ #include +#include #include #include @@ -23,6 +24,7 @@ struct riscv_iommu_queue { atomic_t prod; /* unbounded producer allocation index */ atomic_t head; /* unbounded shadow ring buffer consumer index */ atomic_t tail; /* unbounded shadow ring buffer producer index */ + raw_spinlock_t lock; /* serialize queue publishing */ unsigned int mask; /* index mask, queue length - 1 */ unsigned int irq; /* allocated interrupt number */ struct riscv_iommu_device *iommu; /* iommu device handling the queue when active */ diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index f148f559ac564..58ca7727b7f24 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -974,6 +974,8 @@ static unsigned long *get_rto_from_iova(struct s390_domain *domain, case ZPCI_TABLE_TYPE_RFX: case ZPCI_TABLE_TYPE_RSX: rso = get_rso_from_iova(domain, iova); + if (!rso) + return NULL; rsx = calc_rsx(iova); rse = READ_ONCE(rso[rsx]); if (!reg_entry_isvalid(rse)) diff --git a/drivers/ipack/devices/ipoctal.c b/drivers/ipack/devices/ipoctal.c index 1bbefc6de708b..bf71b8952a7c0 100644 --- a/drivers/ipack/devices/ipoctal.c +++ b/drivers/ipack/devices/ipoctal.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,8 @@ static const struct tty_operations ipoctal_fops; +static void ipoctal_release(struct kref *kref); + struct ipoctal_channel { struct ipoctal_stats stats; unsigned int nb_bytes; @@ -49,6 +52,9 @@ struct ipoctal { struct tty_driver *tty_drv; u8 __iomem *mem8_space; u8 __iomem *int_space; + struct kref kref; + struct module *carrier_owner; + bool removed; }; static inline struct ipoctal *chan_to_ipoctal(struct ipoctal_channel *chan, @@ -70,8 +76,14 @@ static void ipoctal_reset_channel(struct ipoctal_channel *channel) static int ipoctal_port_activate(struct tty_port *port, struct tty_struct *tty) { struct ipoctal_channel *channel; + struct ipoctal *ipoctal; channel = dev_get_drvdata(tty->dev); + ipoctal = chan_to_ipoctal(channel, tty->index); + + + if (ipoctal->removed) + return -ENODEV; /* * Enable RX. TX will be enabled when @@ -95,6 +107,7 @@ static int ipoctal_install(struct tty_driver *driver, struct tty_struct *tty) if (res) goto err_put_carrier; + kref_get(&ipoctal->kref); tty->driver_data = channel; return 0; @@ -460,8 +473,13 @@ static ssize_t ipoctal_write_tty(struct tty_struct *tty, const u8 *buf, size_t count) { struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal = chan_to_ipoctal(channel, tty->index); size_t char_copied; + + if (ipoctal->removed || !channel->tty_port.xmit_buf) + return 0; + char_copied = ipoctal_copy_write_buffer(channel, buf, count); /* As the IP-OCTAL 485 only supports half duplex, do it manually */ @@ -501,8 +519,13 @@ static void ipoctal_set_termios(struct tty_struct *tty, unsigned char mr2 = 0; unsigned char csr = 0; struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal = chan_to_ipoctal(channel, tty->index); speed_t baud; + + if (ipoctal->removed) + return; + cflag = tty->termios.c_cflag; /* Disable and reset everything before change the setup */ @@ -631,10 +654,16 @@ static void ipoctal_hangup(struct tty_struct *tty) { unsigned long flags; struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal; if (channel == NULL) return; + ipoctal = chan_to_ipoctal(channel, tty->index); + + if (ipoctal->removed) + return; + spin_lock_irqsave(&channel->lock, flags); channel->nb_bytes = 0; channel->pointer_read = 0; @@ -651,10 +680,16 @@ static void ipoctal_hangup(struct tty_struct *tty) static void ipoctal_shutdown(struct tty_struct *tty) { struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal; if (channel == NULL) return; + ipoctal = chan_to_ipoctal(channel, tty->index); + + if (ipoctal->removed) + return; + ipoctal_reset_channel(channel); tty_port_set_initialized(&channel->tty_port, false); } @@ -664,8 +699,9 @@ static void ipoctal_cleanup(struct tty_struct *tty) struct ipoctal_channel *channel = tty->driver_data; struct ipoctal *ipoctal = chan_to_ipoctal(channel, tty->index); - /* release the carrier driver */ - ipack_put_carrier(ipoctal->dev); + /* release the carrier driver via cached owner */ + module_put(ipoctal->carrier_owner); + kref_put(&ipoctal->kref, ipoctal_release); } static const struct tty_operations ipoctal_fops = { @@ -683,6 +719,13 @@ static const struct tty_operations ipoctal_fops = { .cleanup = ipoctal_cleanup, }; +static void ipoctal_release(struct kref *kref) +{ + struct ipoctal *ipoctal = container_of(kref, struct ipoctal, kref); + + kfree(ipoctal); +} + static int ipoctal_probe(struct ipack_device *dev) { int res; @@ -692,7 +735,10 @@ static int ipoctal_probe(struct ipack_device *dev) if (ipoctal == NULL) return -ENOMEM; + kref_init(&ipoctal->kref); + ipoctal->dev = dev; + ipoctal->carrier_owner = dev->bus->owner; res = ipoctal_inst_slot(ipoctal, dev->bus->bus_nr, dev->slot); if (res) goto out_uninst; @@ -701,7 +747,7 @@ static int ipoctal_probe(struct ipack_device *dev) return 0; out_uninst: - kfree(ipoctal); + kref_put(&ipoctal->kref, ipoctal_release); return res; } @@ -709,6 +755,8 @@ static void __ipoctal_remove(struct ipoctal *ipoctal) { int i; + ipoctal->removed = true; + ipoctal->dev->bus->ops->free_irq(ipoctal->dev); for (i = 0; i < NR_CHANNELS; i++) { @@ -725,7 +773,7 @@ static void __ipoctal_remove(struct ipoctal *ipoctal) tty_unregister_driver(ipoctal->tty_drv); kfree(ipoctal->tty_drv->name); tty_driver_kref_put(ipoctal->tty_drv); - kfree(ipoctal); + kref_put(&ipoctal->kref, ipoctal_release); } static void ipoctal_remove(struct ipack_device *idev) diff --git a/drivers/irqchip/irq-ast2700-intc0.c b/drivers/irqchip/irq-ast2700-intc0.c index 14b8b88f1179b..1e375e9dffae1 100644 --- a/drivers/irqchip/irq-ast2700-intc0.c +++ b/drivers/irqchip/irq-ast2700-intc0.c @@ -48,9 +48,9 @@ #define INTC0_INTMX_IER 0x1b00 #define INTC0_INTMX_ISR 0x1b04 #define INTC0_INTMX_BANK_SIZE 0x10 -#define INTC0_INTM_BANK_NUM 3 #define INTC0_IRQS_PER_BANK 32 #define INTM_IRQS_PER_BANK 10 +#define INTC0_INTM_BANK_NUM (INTC0_INTM_NUM / INTM_IRQS_PER_BANK) #define INTC0_SEL_BASE 0x200 #define INTC0_SEL_BANK_SIZE 0x4 #define INTC0_SEL_ROUTE_SIZE 0x100 @@ -312,7 +312,7 @@ int aspeed_intc0_resolve_route(const struct irq_domain *c0domain, size_t nc1outs return -ENOENT; if (!IS_ENABLED(CONFIG_ASPEED_AST2700_INTC_TEST) && - !fwnode_device_is_compatible(c0domain->fwnode, "aspeed,ast2700-intc0")) + !of_device_is_compatible(to_of_node(c0domain->fwnode), "aspeed,ast2700-intc0")) return -ENODEV; intc0 = c0domain->host_data; diff --git a/drivers/irqchip/irq-ast2700-intc1.c b/drivers/irqchip/irq-ast2700-intc1.c index 59e8f0d5ddcd2..df6acbafce2d0 100644 --- a/drivers/irqchip/irq-ast2700-intc1.c +++ b/drivers/irqchip/irq-ast2700-intc1.c @@ -157,7 +157,8 @@ static int aspeed_intc1_irq_domain_activate(struct irq_domain *domain, intc1->ranges.nranges, intc1->ranges.ranges, &resolved); if (rc < 0) { - if (!fwnode_device_is_compatible(intc1->upstream->fwnode, "riscv,aplic")) { + if (!of_device_is_compatible(to_of_node(intc1->upstream->fwnode), + "riscv,aplic")) { dev_warn(intc1->dev, "Failed to resolve interrupt route for hwirq %lu in domain %s\n", data->hwirq, domain->name); diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 6f5811aae59c1..274b9761c0eec 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -4592,6 +4592,13 @@ static int its_vpe_init(struct its_vpe *vpe) static void its_vpe_teardown(struct its_vpe *vpe) { + /* + * If vpt_page is NULL, then its_vpe_init() has failed, and + * there is nothing to do as no resource has been allocated. + */ + if (vpe->vpt_page == NULL) + return; + its_vpe_db_proxy_unmap(vpe); its_vpe_id_free(vpe->vpe_id); its_free_pending_table(vpe->vpt_page); @@ -4672,8 +4679,10 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i)); } - if (err) + if (err) { + its_vpe_teardown(vm->vpes[i]); its_vpe_irq_domain_free(domain, virq, i); + } return err; } @@ -5320,7 +5329,7 @@ static int __init its_probe_one(struct its_node *its) err = its_init_domain(its); if (err) - goto out_free_tables; + goto out_free_collection; raw_spin_lock(&its_lock); list_add(&its->entry, &its_nodes); @@ -5328,6 +5337,8 @@ static int __init its_probe_one(struct its_node *its) return 0; +out_free_collection: + kfree(its->collections); out_free_tables: its_free_tables(its); out_free_cmd: @@ -5739,9 +5750,13 @@ static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header, its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; err = its_probe_one(its); - if (!err) - return 0; + if (err) + goto probe_err; + + return 0; +probe_err: + its_node_destroy(its); node_err: iort_deregister_domain_token(its_entry->translation_id); dom_err: diff --git a/drivers/irqchip/irq-gic-v5-irs.c b/drivers/irqchip/irq-gic-v5-irs.c index f3fce0b1e25d9..b3feb6340b59e 100644 --- a/drivers/irqchip/irq-gic-v5-irs.c +++ b/drivers/irqchip/irq-gic-v5-irs.c @@ -199,7 +199,7 @@ int gicv5_irs_iste_alloc(const u32 lpi) if (!gicv5_global_data.ist.l2) return 0; - irs_data = per_cpu(per_cpu_irs_data, smp_processor_id()); + irs_data = per_cpu(per_cpu_irs_data, 0); if (!irs_data) return -ENOENT; @@ -488,6 +488,12 @@ static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data) GICV5_IRS_CR0_IDLE, NULL); } +static void __init gicv5_irs_disable(struct gicv5_irs_chip_data *irs_data) +{ + irs_writel_relaxed(irs_data, 0, GICV5_IRS_CR0); + gicv5_irs_wait_for_idle(irs_data); +} + void gicv5_irs_syncr(void) { struct gicv5_irs_chip_data *irs_data; @@ -651,6 +657,18 @@ static int __init gicv5_irs_of_init_affinity(struct device_node *node, return ret; } +static void __init gicv5_irs_clear_affinity(struct gicv5_irs_chip_data *irs_data) +{ + int cpu; + + for_each_possible_cpu(cpu) { + if (per_cpu(per_cpu_irs_data, cpu) == irs_data) { + per_cpu(cpu_iaffid, cpu).valid = false; + per_cpu(per_cpu_irs_data, cpu) = NULL; + } + } +} + static void irs_setup_pri_bits(u32 idr1) { switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) { @@ -722,6 +740,7 @@ static int __init gicv5_irs_init(struct gicv5_irs_chip_data *irs_data) static int __init gicv5_irs_of_init(struct device_node *node) { struct gicv5_irs_chip_data *irs_data; + struct resource res; void __iomem *irs_base; u8 iaffid_bits; u32 idr; @@ -739,6 +758,11 @@ static int __init gicv5_irs_of_init(struct device_node *node) goto out_err; } + if (of_address_to_resource(node, ret, &res)) { + ret = -EINVAL; + goto out_err; + } + irs_base = of_io_request_and_map(node, ret, of_node_full_name(node)); if (IS_ERR(irs_base)) { pr_err("%pOF: unable to map GICv5 IRS registers\n", node); @@ -747,6 +771,7 @@ static int __init gicv5_irs_of_init(struct device_node *node) } irs_data->fwnode = of_fwnode_handle(node); + irs_data->res = res; gicv5_irs_init_bases(irs_data, irs_base, of_property_read_bool(node, "dma-noncoherent")); idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); @@ -773,7 +798,10 @@ static int __init gicv5_irs_of_init(struct device_node *node) return ret; out_iomem: + gicv5_irs_clear_affinity(irs_data); + gicv5_irs_disable(irs_data); iounmap(irs_base); + release_mem_region(res.start, resource_size(&res)); out_err: kfree(irs_data); return ret; @@ -787,7 +815,10 @@ void __init gicv5_irs_remove(void) gicv5_deinit_lpis(); list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) { + gicv5_irs_clear_affinity(irs_data); + gicv5_irs_disable(irs_data); iounmap(irs_data->irs_base); + release_mem_region(irs_data->res.start, resource_size(&irs_data->res)); list_del(&irs_data->entry); kfree(irs_data); } @@ -860,6 +891,8 @@ static int __init gic_acpi_parse_iaffid(union acpi_subtable_headers *header, return 0; cpu = get_logical_index(gicc->arm_mpidr); + if (cpu < 0) + return 0; if (gicc->iaffid & ~GENMASK(current_iaffid_bits - 1, 0)) { pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n", cpu, gicc->iaffid); @@ -934,6 +967,7 @@ static int __init gic_acpi_parse_madt_irs(union acpi_subtable_headers *header, } gicv5_irs_init_bases(irs_data, irs_base, irs->flags & ACPI_MADT_IRS_NON_COHERENT); + irs_data->res = *r; gicv5_irs_acpi_init_affinity(irs->irs_id, irs_data); @@ -951,6 +985,8 @@ static int __init gic_acpi_parse_madt_irs(union acpi_subtable_headers *header, return 0; out_map: + gicv5_irs_clear_affinity(irs_data); + gicv5_irs_disable(irs_data); iounmap(irs_base); out_release: release_mem_region(r->start, resource_size(r)); diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c index e9d1795235a66..21c34e7587323 100644 --- a/drivers/irqchip/irq-gic-v5.c +++ b/drivers/irqchip/irq-gic-v5.c @@ -87,25 +87,13 @@ static void gicv5_ppi_priority_init(void) static void gicv5_hwirq_init(irq_hw_number_t hwirq, u8 priority, u8 hwirq_type) { - u64 cdpri, cdaff; - u16 iaffid; - int ret; + u64 cdpri; if (hwirq_type == GICV5_HWIRQ_TYPE_LPI || hwirq_type == GICV5_HWIRQ_TYPE_SPI) { cdpri = FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority) | FIELD_PREP(GICV5_GIC_CDPRI_TYPE_MASK, hwirq_type) | FIELD_PREP(GICV5_GIC_CDPRI_ID_MASK, hwirq); gic_insn(cdpri, CDPRI); - - ret = gicv5_irs_cpu_to_iaffid(smp_processor_id(), &iaffid); - - if (WARN_ON_ONCE(ret)) - return; - - cdaff = FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, iaffid) | - FIELD_PREP(GICV5_GIC_CDAFF_TYPE_MASK, hwirq_type) | - FIELD_PREP(GICV5_GIC_CDAFF_ID_MASK, hwirq); - gic_insn(cdaff, CDAFF); } } @@ -548,6 +536,7 @@ static const struct irq_chip gicv5_spi_irq_chip = { .irq_get_irqchip_state = gicv5_spi_irq_get_irqchip_state, .irq_set_irqchip_state = gicv5_spi_irq_set_irqchip_state, .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_AFFINITY_PRE_STARTUP | IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, }; @@ -561,7 +550,8 @@ static const struct irq_chip gicv5_lpi_irq_chip = { .irq_retrigger = gicv5_lpi_irq_retrigger, .irq_get_irqchip_state = gicv5_lpi_irq_get_irqchip_state, .irq_set_irqchip_state = gicv5_lpi_irq_set_irqchip_state, - .flags = IRQCHIP_SKIP_SET_WAKE | + .flags = IRQCHIP_AFFINITY_PRE_STARTUP | + IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, }; @@ -862,6 +852,9 @@ void __init gicv5_init_lpi_domain(void) void __init gicv5_free_lpi_domain(void) { + if (!gicv5_global_data.lpi_domain) + return; + irq_domain_remove(gicv5_global_data.lpi_domain); gicv5_global_data.lpi_domain = NULL; } @@ -981,8 +974,10 @@ static void gicv5_cpu_disable_interrupts(void) { u64 cr0; - cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 0); + cr0 = read_sysreg_s(SYS_ICC_CR0_EL1); + cr0 &= ~ICC_CR0_EL1_EN_MASK; write_sysreg_s(cr0, SYS_ICC_CR0_EL1); + isb(); } static void gicv5_cpu_enable_interrupts(void) @@ -997,7 +992,8 @@ static void gicv5_cpu_enable_interrupts(void) pcr = FIELD_PREP(ICC_PCR_EL1_PRIORITY, GICV5_IRQ_PRI_MI); write_sysreg_s(pcr, SYS_ICC_PCR_EL1); - cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 1); + cr0 = read_sysreg_s(SYS_ICC_CR0_EL1); + cr0 |= ICC_CR0_EL1_EN_MASK; write_sysreg_s(cr0, SYS_ICC_CR0_EL1); } @@ -1170,7 +1166,7 @@ static int __init gicv5_init_common(struct fwnode_handle *parent_domain) ret = gicv5_starting_cpu(smp_processor_id()); if (ret) - goto out_dom; + goto out_int; ret = set_handle_irq(gicv5_handle_irq); if (ret) @@ -1178,16 +1174,17 @@ static int __init gicv5_init_common(struct fwnode_handle *parent_domain) ret = gicv5_irs_enable(); if (ret) - goto out_int; + goto out_handle; gicv5_smp_init(); gicv5_irs_its_probe(); return 0; +out_handle: + set_handle_irq(NULL); out_int: gicv5_cpu_disable_interrupts(); -out_dom: gicv5_free_domains(); return ret; } diff --git a/drivers/irqchip/irq-mbigen.c b/drivers/irqchip/irq-mbigen.c index 6f69f4e5dbacd..12919836dadba 100644 --- a/drivers/irqchip/irq-mbigen.c +++ b/drivers/irqchip/irq-mbigen.c @@ -64,20 +64,6 @@ struct mbigen_device { void __iomem *base; }; -static inline unsigned int get_mbigen_node_offset(unsigned int nid) -{ - unsigned int offset = nid * MBIGEN_NODE_OFFSET; - - /* - * To avoid touched clear register in unexpected way, we need to directly - * skip clear register when access to more than 10 mbigen nodes. - */ - if (nid >= (REG_MBIGEN_CLEAR_OFFSET / MBIGEN_NODE_OFFSET)) - offset += MBIGEN_NODE_OFFSET; - - return offset; -} - static inline unsigned int get_mbigen_vec_reg(irq_hw_number_t hwirq) { unsigned int nid, pin; @@ -86,7 +72,8 @@ static inline unsigned int get_mbigen_vec_reg(irq_hw_number_t hwirq) nid = hwirq / IRQS_PER_MBIGEN_NODE + 1; pin = hwirq % IRQS_PER_MBIGEN_NODE; - return pin * 4 + get_mbigen_node_offset(nid) + REG_MBIGEN_VEC_OFFSET; + return pin * 4 + nid * MBIGEN_NODE_OFFSET + + REG_MBIGEN_VEC_OFFSET; } static inline void get_mbigen_type_reg(irq_hw_number_t hwirq, @@ -101,7 +88,8 @@ static inline void get_mbigen_type_reg(irq_hw_number_t hwirq, *mask = 1 << (irq_ofst % 32); ofst = irq_ofst / 32 * 4; - *addr = ofst + get_mbigen_node_offset(nid) + REG_MBIGEN_TYPE_OFFSET; + *addr = ofst + nid * MBIGEN_NODE_OFFSET + + REG_MBIGEN_TYPE_OFFSET; } static inline void get_mbigen_clear_reg(irq_hw_number_t hwirq, diff --git a/drivers/irqchip/irq-realtek-rtl.c b/drivers/irqchip/irq-realtek-rtl.c index 2ae3be7fa6338..90fb1a14c1f13 100644 --- a/drivers/irqchip/irq-realtek-rtl.c +++ b/drivers/irqchip/irq-realtek-rtl.c @@ -1,5 +1,10 @@ // SPDX-License-Identifier: GPL-2.0-only /* + * Realtek Interrupt controller. + * + * The Realtek Interrupt controller is a big endian device found in the + * Realtek MIPS SoCs. + * * Copyright (C) 2020 Birger Koblitz * Copyright (C) 2020 Bert Vermeulen * Copyright (C) 2020 John Crispin @@ -25,6 +30,11 @@ #define REG(cpu, x) (realtek_ictl_base[cpu] + x) +struct realtek_ictl_output { + struct irq_domain *domain; + u32 mask; +}; + static DEFINE_RAW_SPINLOCK(irq_lock); static void __iomem *realtek_ictl_base[NR_CPUS]; @@ -41,18 +51,18 @@ static inline void enable_gimr(unsigned int cpu, unsigned int hw_irq) { u32 gimr; - gimr = readl(REG(cpu, RTL_ICTL_GIMR)); + gimr = readl_be(REG(cpu, RTL_ICTL_GIMR)); gimr |= BIT(hw_irq); - writel(gimr, REG(cpu, RTL_ICTL_GIMR)); + writel_be(gimr, REG(cpu, RTL_ICTL_GIMR)); } static inline void disable_gimr(unsigned int cpu, unsigned int hw_irq) { u32 gimr; - gimr = readl(REG(cpu, RTL_ICTL_GIMR)); + gimr = readl_be(REG(cpu, RTL_ICTL_GIMR)); gimr &= ~BIT(hw_irq); - writel(gimr, REG(cpu, RTL_ICTL_GIMR)); + writel_be(gimr, REG(cpu, RTL_ICTL_GIMR)); } static void write_irr(unsigned int cpu, int hw_irq, u32 value) @@ -62,9 +72,9 @@ static void write_irr(unsigned int cpu, int hw_irq, u32 value) unsigned int shift = IRR_SHIFT(hw_irq); u32 irr; - irr = readl(irr0 + offset) & ~(0xf << shift); + irr = readl_be(irr0 + offset) & ~(0xf << shift); irr |= (value & 0xf) << shift; - writel(irr, irr0 + offset); + writel_be(irr, irr0 + offset); } static void realtek_ictl_unmask_irq(struct irq_data *i) @@ -105,15 +115,17 @@ static struct irq_chip realtek_ictl_irq = { .irq_set_affinity = realtek_ictl_irq_affinity, }; -static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) +static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw_irq) { + struct realtek_ictl_output *output = d->host_data; unsigned int cpu; irq_set_chip_and_handler(irq, &realtek_ictl_irq, handle_level_irq); guard(raw_spinlock_irqsave)(&irq_lock); + output->mask |= BIT(hw_irq); for_each_present_cpu(cpu) - write_irr(cpu, hw, 1); + write_irr(cpu, hw_irq, 1); return 0; } @@ -125,76 +137,99 @@ static const struct irq_domain_ops irq_domain_ops = { static void realtek_irq_dispatch(struct irq_desc *desc) { + struct realtek_ictl_output *output = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); unsigned int cpu = smp_processor_id(); - struct irq_domain *domain; unsigned long pending; - unsigned int soc_int; + unsigned int hw_irq; chained_irq_enter(chip, desc); - pending = readl(REG(cpu, RTL_ICTL_GIMR)) & readl(REG(cpu, RTL_ICTL_GISR)); + pending = readl_be(REG(cpu, RTL_ICTL_GIMR)) & + readl_be(REG(cpu, RTL_ICTL_GISR)) & output->mask; if (unlikely(!pending)) { spurious_interrupt(); goto out; } - domain = irq_desc_get_handler_data(desc); - for_each_set_bit(soc_int, &pending, RTL_ICTL_NUM_INPUTS) - generic_handle_domain_irq(domain, soc_int); + for_each_set_bit(hw_irq, &pending, RTL_ICTL_NUM_INPUTS) + generic_handle_domain_irq(output->domain, hw_irq); out: chained_irq_exit(chip, desc); } -static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent) +static int __init realtek_setup_parents(struct device_node *node) { + int err, parent_irq, num_parents = of_irq_count(node); + struct realtek_ictl_output *output; struct of_phandle_args oirq; struct irq_domain *domain; - int cpu, parent_irq; - for_each_present_cpu(cpu) { - realtek_ictl_base[cpu] = of_iomap(node, cpu); - if (!realtek_ictl_base[cpu]) - return -ENXIO; - - /* Disable all cascaded interrupts and clear routing */ - for (unsigned int hw_irq = 0; hw_irq < RTL_ICTL_NUM_INPUTS; hw_irq++) { - disable_gimr(cpu, hw_irq); - write_irr(cpu, hw_irq, 0); - } - } + output = kcalloc(1, sizeof(*output), GFP_KERNEL); + if (!output) + return -ENOMEM; - if (WARN_ON(!of_irq_count(node))) { + if (WARN_ON(!num_parents)) { /* - * If DT contains no parent interrupts, assume MIPS CPU IRQ 2 - * (HW0) is connected to the first output. This is the case for - * all known hardware anyway. "interrupt-map" is deprecated, so - * don't bother trying to parse that. + * If DT contains no parent interrupts, assume MIPS IRQ 2 (HW0) is + * connected to the first output. This is the case for all known hardware. */ - oirq.np = of_find_compatible_node(NULL, NULL, "mti,cpu-interrupt-controller"); + oirq.np = of_find_compatible_node(NULL, NULL, + "mti,cpu-interrupt-controller"); + if (!oirq.np) { + err = -EINVAL; + goto err_out; + } + oirq.args_count = 1; oirq.args[0] = 2; - parent_irq = irq_create_of_mapping(&oirq); - of_node_put(oirq.np); } else { parent_irq = of_irq_get(node, 0); } - if (parent_irq < 0) - return parent_irq; - else if (!parent_irq) - return -ENODEV; + if (parent_irq <= 0) { + err = parent_irq ? parent_irq : -ENODEV; + goto err_out; + } - domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS, &irq_domain_ops, NULL); - if (!domain) - return -ENOMEM; + domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS, + &irq_domain_ops, output); + if (!domain) { + err = -ENOMEM; + goto err_out; + } - irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, domain); + output->domain = domain; + irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, output); return 0; + +err_out: + kfree(output); + + return err; +} + +static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent) +{ + unsigned int cpu; + + for_each_present_cpu(cpu) { + realtek_ictl_base[cpu] = of_iomap(node, cpu); + if (!realtek_ictl_base[cpu]) + return -ENXIO; + + /* Disable all cascaded interrupts and clear routing */ + for (unsigned int hw_irq = 0; hw_irq < RTL_ICTL_NUM_INPUTS; hw_irq++) { + disable_gimr(cpu, hw_irq); + write_irr(cpu, hw_irq, 0); + } + } + + return realtek_setup_parents(node); } IRQCHIP_DECLARE(realtek_rtl_intc, "realtek,rtl-intc", realtek_rtl_of_init); diff --git a/drivers/irqchip/irq-renesas-irqc.c b/drivers/irqchip/irq-renesas-irqc.c index a20a6471b0e48..1ff3535a4617f 100644 --- a/drivers/irqchip/irq-renesas-irqc.c +++ b/drivers/irqchip/irq-renesas-irqc.c @@ -176,6 +176,7 @@ static int irqc_probe(struct platform_device *pdev) goto err_runtime_pm_disable; } + p->irq_domain->flags |= IRQ_DOMAIN_FLAG_DESTROY_GC; ret = irq_alloc_domain_generic_chips(p->irq_domain, p->number_of_irqs, 1, "irqc", handle_level_irq, 0, 0, IRQ_GC_INIT_NESTED_LOCK); diff --git a/drivers/irqchip/irq-renesas-rzg2l.c b/drivers/irqchip/irq-renesas-rzg2l.c index f6b2e69a2f4e0..b3774f82855ab 100644 --- a/drivers/irqchip/irq-renesas-rzg2l.c +++ b/drivers/irqchip/irq-renesas-rzg2l.c @@ -161,7 +161,7 @@ static void rzg2l_clear_irq_int(struct rzg2l_irqc_priv *priv, unsigned int hwirq * falling/rising-edge. */ if ((iscr & bit) && (iitsr & IITSR_IITSEL_MASK(hw_irq))) { - writel_relaxed(iscr & ~bit, priv->base + ISCR); + writel_relaxed(~bit, priv->base + ISCR); /* * Enforce that the posted write is flushed to prevent that the * just handled interrupt is raised again. @@ -177,7 +177,7 @@ static void rzg2l_clear_tint_int(struct rzg2l_irqc_priv *priv, unsigned int hwir reg = readl_relaxed(priv->base + TSCR); if (reg & bit) { - writel_relaxed(reg & ~bit, priv->base + TSCR); + writel_relaxed(~bit, priv->base + TSCR); /* * Enforce that the posted write is flushed to prevent that the * just handled interrupt is raised again. diff --git a/drivers/irqchip/irq-stm32mp-exti.c b/drivers/irqchip/irq-stm32mp-exti.c index bf3a2def69cad..a19e91fbd0100 100644 --- a/drivers/irqchip/irq-stm32mp-exti.c +++ b/drivers/irqchip/irq-stm32mp-exti.c @@ -22,7 +22,7 @@ #define IRQS_PER_BANK 32 -#define HWSPNLCK_TIMEOUT 1000 /* usec */ +#define HWSPNLCK_TIMEOUT_MS 1 #define EXTI_EnCIDCFGR(n) (0x180 + (n) * 4) #define EXTI_HWCFGR1 0x3f0 @@ -376,7 +376,7 @@ static int stm32mp_exti_set_type(struct irq_data *d, unsigned int type) raw_spin_lock(&chip_data->rlock); if (hwlock) { - err = hwspin_lock_timeout_in_atomic(hwlock, HWSPNLCK_TIMEOUT); + err = hwspin_lock_timeout_in_atomic(hwlock, HWSPNLCK_TIMEOUT_MS); if (err) { pr_err("%s can't get hwspinlock (%d)\n", __func__, err); goto unlock; diff --git a/drivers/leds/leds-gpio.c b/drivers/leds/leds-gpio.c index 8ae71c2e91e01..8810fdcf2d774 100644 --- a/drivers/leds/leds-gpio.c +++ b/drivers/leds/leds-gpio.c @@ -277,6 +277,7 @@ static int gpio_led_probe(struct platform_device *pdev) if (IS_ERR(led_dat->gpiod)) { dev_info(dev, "Skipping unavailable LED gpio %d (%s)\n", template->gpio, template->name); + led_dat->gpiod = NULL; continue; } diff --git a/drivers/leds/leds-pca9532.c b/drivers/leds/leds-pca9532.c index f3bf59495b68d..b2e081c8f1393 100644 --- a/drivers/leds/leds-pca9532.c +++ b/drivers/leds/leds-pca9532.c @@ -327,9 +327,9 @@ static int pca9532_gpio_set_value(struct gpio_chip *gc, unsigned int offset, struct pca9532_led *led = &data->leds[offset]; if (val) - led->state = PCA9532_ON; - else led->state = PCA9532_OFF; + else + led->state = PCA9532_ON; pca9532_setled(led); @@ -349,7 +349,7 @@ static int pca9532_gpio_get_value(struct gpio_chip *gc, unsigned offset) static int pca9532_gpio_direction_input(struct gpio_chip *gc, unsigned offset) { /* To use as input ensure pin is not driven */ - pca9532_gpio_set_value(gc, offset, 0); + pca9532_gpio_set_value(gc, offset, 1); return 0; } @@ -397,10 +397,14 @@ static int pca9532_configure(struct i2c_client *client, for (i = 0; i < 2; i++) { data->pwm[i] = pdata->pwm[i]; data->psc[i] = pdata->psc[i]; - i2c_smbus_write_byte_data(client, PCA9532_REG_PWM(maxleds, i), - data->pwm[i]); - i2c_smbus_write_byte_data(client, PCA9532_REG_PSC(maxleds, i), - data->psc[i]); + err = i2c_smbus_write_byte_data(client, PCA9532_REG_PWM(maxleds, i), + data->pwm[i]); + if (err < 0) + return err; + err = i2c_smbus_write_byte_data(client, PCA9532_REG_PSC(maxleds, i), + data->psc[i]); + if (err < 0) + return err; } data->hw_blink = true; diff --git a/drivers/leds/leds-st1202.c b/drivers/leds/leds-st1202.c index 7f68d956f6948..168df5ecf27b7 100644 --- a/drivers/leds/leds-st1202.c +++ b/drivers/leds/leds-st1202.c @@ -31,10 +31,11 @@ #define ST1202_ILED_REG0 0x09 #define ST1202_MAX_LEDS 12 #define ST1202_MAX_PATTERNS 8 -#define ST1202_MILLIS_PATTERN_DUR_MAX 5660 +#define ST1202_MILLIS_PATTERN_DUR_MAX (ST1202_MILLIS_PATTERN_DUR_MIN * U8_MAX) #define ST1202_MILLIS_PATTERN_DUR_MIN 22 #define ST1202_PATTERN_DUR 0x16 #define ST1202_PATTERN_PWM 0x1E +#define ST1202_PATTERN_PWM_FULL 0x0FFF #define ST1202_PATTERN_REP 0x15 struct st1202_led { @@ -85,7 +86,7 @@ static int st1202_write_reg(struct st1202_chip *chip, int reg, uint8_t val) static uint8_t st1202_prescalar_to_miliseconds(unsigned int value) { - return value / ST1202_MILLIS_PATTERN_DUR_MIN - 1; + return value / ST1202_MILLIS_PATTERN_DUR_MIN; } static int st1202_pwm_pattern_write(struct st1202_chip *chip, int led_num, @@ -127,37 +128,11 @@ static int st1202_duration_pattern_write(struct st1202_chip *chip, int pattern, st1202_prescalar_to_miliseconds(value)); } -static void st1202_brightness_set(struct led_classdev *led_cdev, - enum led_brightness value) -{ - struct st1202_led *led = cdev_to_st1202_led(led_cdev); - struct st1202_chip *chip = led->chip; - - guard(mutex)(&chip->lock); - - st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, value); -} - -static enum led_brightness st1202_brightness_get(struct led_classdev *led_cdev) -{ - struct st1202_led *led = cdev_to_st1202_led(led_cdev); - struct st1202_chip *chip = led->chip; - u8 value = 0; - - guard(mutex)(&chip->lock); - - st1202_read_reg(chip, ST1202_ILED_REG0 + led->led_num, &value); - - return value; -} - -static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active) +static int __st1202_channel_set(struct st1202_chip *chip, int led_num, bool active) { u8 chan_low, chan_high; int ret; - guard(mutex)(&chip->lock); - if (led_num <= 7) { ret = st1202_read_reg(chip, ST1202_CHAN_ENABLE_LOW, &chan_low); if (ret < 0) @@ -185,6 +160,40 @@ static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active return 0; } +static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active) +{ + guard(mutex)(&chip->lock); + + return __st1202_channel_set(chip, led_num, active); +} + +static void st1202_brightness_set(struct led_classdev *led_cdev, + enum led_brightness value) +{ + struct st1202_led *led = cdev_to_st1202_led(led_cdev); + struct st1202_chip *chip = led->chip; + + guard(mutex)(&chip->lock); + + for (int patt = 0; patt < ST1202_MAX_PATTERNS; patt++) + st1202_pwm_pattern_write(chip, led->led_num, patt, ST1202_PATTERN_PWM_FULL); + st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, value); + __st1202_channel_set(chip, led->led_num, !!value); +} + +static enum led_brightness st1202_brightness_get(struct led_classdev *led_cdev) +{ + struct st1202_led *led = cdev_to_st1202_led(led_cdev); + struct st1202_chip *chip = led->chip; + u8 value = 0; + + guard(mutex)(&chip->lock); + + st1202_read_reg(chip, ST1202_ILED_REG0 + led->led_num, &value); + + return value; +} + static int st1202_led_set(struct led_classdev *ldev, enum led_brightness value) { struct st1202_led *led = cdev_to_st1202_led(ldev); @@ -200,12 +209,16 @@ static int st1202_led_pattern_clear(struct led_classdev *ldev) guard(mutex)(&chip->lock); + ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT); + if (ret != 0) + return ret; + for (int patt = 0; patt < ST1202_MAX_PATTERNS; patt++) { - ret = st1202_pwm_pattern_write(chip, led->led_num, patt, LED_OFF); + ret = st1202_pwm_pattern_write(chip, led->led_num, patt, ST1202_PATTERN_PWM_FULL); if (ret != 0) return ret; - ret = st1202_duration_pattern_write(chip, patt, ST1202_MILLIS_PATTERN_DUR_MIN); + ret = st1202_write_reg(chip, ST1202_PATTERN_DUR + patt, 0); if (ret != 0) return ret; } @@ -224,13 +237,19 @@ static int st1202_led_pattern_set(struct led_classdev *ldev, if (len > ST1202_MAX_PATTERNS) return -EINVAL; - guard(mutex)(&chip->lock); - for (int patt = 0; patt < len; patt++) { if (pattern[patt].delta_t < ST1202_MILLIS_PATTERN_DUR_MIN || pattern[patt].delta_t > ST1202_MILLIS_PATTERN_DUR_MAX) return -EINVAL; + } + guard(mutex)(&chip->lock); + + ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT); + if (ret != 0) + return ret; + + for (int patt = 0; patt < len; patt++) { ret = st1202_pwm_pattern_write(chip, led->led_num, patt, pattern[patt].brightness); if (ret != 0) return ret; @@ -244,6 +263,10 @@ static int st1202_led_pattern_set(struct led_classdev *ldev, if (ret != 0) return ret; + ret = __st1202_channel_set(chip, led->led_num, true); + if (ret != 0) + return ret; + ret = st1202_write_reg(chip, ST1202_CONFIG_REG, (ST1202_CONFIG_REG_PATSR | ST1202_CONFIG_REG_PATS | ST1202_CONFIG_REG_SHFT)); if (ret != 0) @@ -256,13 +279,19 @@ static int st1202_dt_init(struct st1202_chip *chip) { struct device *dev = &chip->client->dev; struct st1202_led *led; - int err, reg; + int err; + u32 reg; for_each_available_child_of_node_scoped(dev_of_node(dev), child) { err = of_property_read_u32(child, "reg", ®); if (err) return dev_err_probe(dev, err, "Invalid register\n"); + if (reg >= ST1202_MAX_LEDS) + return dev_err_probe(dev, -EINVAL, + "LED reg %u out of range [0, %d]\n", + reg, ST1202_MAX_LEDS - 1); + led = &chip->leds[reg]; led->is_active = true; led->fwnode = of_fwnode_handle(child); @@ -322,11 +351,6 @@ static int st1202_setup(struct st1202_chip *chip) if (ret < 0) return ret; - ret = st1202_write_reg(chip, ST1202_CONFIG_REG, - ST1202_CONFIG_REG_PATS | ST1202_CONFIG_REG_PATSR); - if (ret < 0) - return ret; - return 0; } diff --git a/drivers/leds/rgb/leds-lp5860-core.c b/drivers/leds/rgb/leds-lp5860-core.c index fd0e2f6e6e0f4..e21d5f2302be0 100644 --- a/drivers/leds/rgb/leds-lp5860-core.c +++ b/drivers/leds/rgb/leds-lp5860-core.c @@ -204,9 +204,9 @@ int lp5860_device_init(struct device *dev) mutex_lock(&lp->lock); ret = regmap_update_bits(lp->regmap, LP5860_REG_DEV_INITIAL, LP5860_MODE_MASK, LP5860_MODE_1 << LP5860_MODE_SHIFT); + mutex_unlock(&lp->lock); if (ret) goto err_disable; - mutex_unlock(&lp->lock); ret = lp5860_init_dt(lp); if (ret) @@ -215,7 +215,6 @@ int lp5860_device_init(struct device *dev) return 0; err_disable: - mutex_unlock(&lp->lock); lp5860_chip_enable(lp, LP5860_CHIP_DISABLE); return ret; } diff --git a/drivers/leds/rgb/leds-lp5860-spi.c b/drivers/leds/rgb/leds-lp5860-spi.c index 5e0c44854a680..6bf6a625c28af 100644 --- a/drivers/leds/rgb/leds-lp5860-spi.c +++ b/drivers/leds/rgb/leds-lp5860-spi.c @@ -38,6 +38,7 @@ static int lp5860_probe(struct spi_device *spi) struct device *dev = &spi->dev; struct lp5860 *lp5860; unsigned int multi_leds; + int ret; multi_leds = device_get_child_node_count(dev); if (!multi_leds) { @@ -61,7 +62,10 @@ static int lp5860_probe(struct spi_device *spi) "Failed to initialise Regmap.\n"); lp5860->dev = dev; - mutex_init(&lp5860->lock); + + ret = devm_mutex_init(dev, &lp5860->lock); + if (ret) + return ret; spi_set_drvdata(spi, lp5860); @@ -70,10 +74,6 @@ static int lp5860_probe(struct spi_device *spi) static void lp5860_remove(struct spi_device *spi) { - struct lp5860 *lp5860 = spi_get_drvdata(spi); - - mutex_destroy(&lp5860->lock); - lp5860_device_remove(&spi->dev); } diff --git a/drivers/mailbox/mailbox-mchp-ipc-sbi.c b/drivers/mailbox/mailbox-mchp-ipc-sbi.c index b87bf2fb4b9b9..f081f8a9bcf8e 100644 --- a/drivers/mailbox/mailbox-mchp-ipc-sbi.c +++ b/drivers/mailbox/mailbox-mchp-ipc-sbi.c @@ -378,6 +378,8 @@ static int mchp_ipc_get_cluster_aggr_irq(struct mchp_ipc_sbi_mbox *ipc) for_each_online_cpu(cpuid) { hartid = cpuid_to_hartid_map(cpuid); irq_name = devm_kasprintf(ipc->dev, GFP_KERNEL, "hart-%lu", hartid); + if (!irq_name) + return -ENOMEM; ret = platform_get_irq_byname_optional(pdev, irq_name); if (ret <= 0) continue; diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index 636879ae1db76..7a8e749374b5c 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -91,12 +91,11 @@ struct pcc_chan_reg { * @plat_irq: platform interrupt * @type: PCC subspace type * @plat_irq_flags: platform interrupt flags - * @chan_in_use: this flag is used just to check if the interrupt needs - * handling when it is shared. Since only one transfer can occur - * at a time and mailbox takes care of locking, this flag can be - * accessed without a lock. Note: the type only support the - * communication from OSPM to Platform, like type3, use it, and - * other types completely ignore it. + * @chan_in_use: lockless flag used by type 3 initiator subspaces to filter + * platform interrupts. Only one transfer can occur at a time, but + * the interrupt handler may sample the flag on another CPU, so all + * accesses must use READ_ONCE() or WRITE_ONCE(). Other subspace + * types do not test it. */ struct pcc_chan_info { struct pcc_mbox_chan chan; @@ -320,8 +319,13 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) if (pcc_chan_reg_read_modify_write(&pchan->plat_irq_ack)) return IRQ_NONE; + /* + * Initiator subspaces use this flag to filter shared interrupts. Use + * READ_ONCE() to sample the lockless flag written by pcc_send_data() + * on another CPU. + */ if (pchan->type == ACPI_PCCT_TYPE_EXT_PCC_MASTER_SUBSPACE && - !pchan->chan_in_use) + !READ_ONCE(pchan->chan_in_use)) return IRQ_NONE; if (!pcc_mbox_cmd_complete_check(pchan)) @@ -331,12 +335,12 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) return IRQ_NONE; /* - * Clear this flag after updating interrupt ack register and just - * before mbox_chan_received_data() which might call pcc_send_data() - * where the flag is set again to start new transfer. This is - * required to avoid any possible race in updatation of this flag. + * Clear this flag after updating the interrupt ack register and before + * notifying the client and mailbox core. mbox_chan_txdone() may submit + * the next queued transfer and set the flag again. Use WRITE_ONCE() for + * the lockless update observed by the send and interrupt paths. */ - pchan->chan_in_use = false; + WRITE_ONCE(pchan->chan_in_use, false); mbox_chan_received_data(chan, NULL); mbox_chan_txdone(chan, 0); @@ -438,9 +442,18 @@ static int pcc_send_data(struct mbox_chan *chan, void *data) if (ret) return ret; + /* + * Set chan_in_use before ringing the doorbell so a fast completion + * interrupt is not mistaken for a shared interrupt from another + * subspace. Use WRITE_ONCE() for the lockless flag update. The + * ordered I/O accessor used to ring the doorbell orders this store + * before the platform is notified. + */ + if (pchan->plat_irq > 0) + WRITE_ONCE(pchan->chan_in_use, true); ret = pcc_chan_reg_read_modify_write(&pchan->db); - if (!ret && pchan->plat_irq > 0) - pchan->chan_in_use = true; + if (ret && pchan->plat_irq > 0) + WRITE_ONCE(pchan->chan_in_use, false); return ret; } diff --git a/drivers/mailbox/qcom-cpucp-mbox.c b/drivers/mailbox/qcom-cpucp-mbox.c index 862e45e8fbd58..298b357c0f9a6 100644 --- a/drivers/mailbox/qcom-cpucp-mbox.c +++ b/drivers/mailbox/qcom-cpucp-mbox.c @@ -63,14 +63,25 @@ static irqreturn_t qcom_cpucp_mbox_irq_fn(int irq, void *data) for_each_set_bit(i, (unsigned long *)&status, cpucp->mbox.num_chans) { u32 val = readl(cpucp->rx_base + APSS_CPUCP_RX_MBOX_CMD(i) + APSS_CPUCP_MBOX_CMD_OFF); struct mbox_chan *chan = &cpucp->chans[i]; + struct mbox_client *cl; unsigned long flags; - /* Provide mutual exclusion with changes to chan->cl */ + /* + * Provide mutual exclusion with changes to chan->cl. + * Save cl locally and clear the HW interrupt inside the lock, + * then invoke mbox_chan_received_data() outside the lock to + * avoid a PREEMPT_RT self-deadlock: mbox_chan_received_data() + * can call back into mbox_send_message() via scmi_rx_callback() + * -> mailbox_clear_channel(), which re-acquires chan->lock + * (converted to an rt_spinlock under PREEMPT_RT). + */ spin_lock_irqsave(&chan->lock, flags); - if (chan->cl) - mbox_chan_received_data(chan, &val); + cl = chan->cl; writeq(BIT(i), cpucp->rx_base + APSS_CPUCP_RX_MBOX_CLEAR); spin_unlock_irqrestore(&chan->lock, flags); + + if (cl) + mbox_chan_received_data(chan, &val); } return IRQ_HANDLED; @@ -106,6 +117,14 @@ static int qcom_cpucp_mbox_send_data(struct mbox_chan *chan, void *data) unsigned long chan_id = channel_number(chan); u32 *val = data; + /* + * mailbox_clear_channel() calls mbox_send_message() with NULL data to + * signal the remote side that the channel has been cleared. Nothing + * needs to be written to the TX register in that case, so just return. + */ + if (!val) + return 0; + writel(*val, cpucp->tx_base + APSS_CPUCP_TX_MBOX_CMD(chan_id) + APSS_CPUCP_MBOX_CMD_OFF); return 0; diff --git a/drivers/mailbox/qcom-ipcc.c b/drivers/mailbox/qcom-ipcc.c index d957d989c0cee..dead907aa7a28 100644 --- a/drivers/mailbox/qcom-ipcc.c +++ b/drivers/mailbox/qcom-ipcc.c @@ -167,7 +167,7 @@ static struct mbox_chan *qcom_ipcc_mbox_xlate(struct mbox_controller *mbox, { struct qcom_ipcc *ipcc = to_qcom_ipcc(mbox); struct qcom_ipcc_chan_info *mchan; - struct mbox_chan *chan; + struct mbox_chan *chan, *free_chan = NULL; struct device *dev; int chan_id; @@ -180,16 +180,21 @@ static struct mbox_chan *qcom_ipcc_mbox_xlate(struct mbox_controller *mbox, chan = &ipcc->chans[chan_id]; mchan = chan->con_priv; - if (!mchan) - break; - else if (mchan->client_id == ph->args[0] && - mchan->signal_id == ph->args[1]) + if (!mchan) { + /* Keep scanning past holes to reject duplicate channel requests. */ + if (!free_chan) + free_chan = chan; + } else if (mchan->client_id == ph->args[0] && + mchan->signal_id == ph->args[1]) { return ERR_PTR(-EBUSY); + } } - if (chan_id >= mbox->num_chans) + if (!free_chan) return ERR_PTR(-EBUSY); + chan = free_chan; + mchan = devm_kzalloc(dev, sizeof(*mchan), GFP_KERNEL); if (!mchan) return ERR_PTR(-ENOMEM); diff --git a/drivers/mailbox/riscv-sbi-mpxy-mbox.c b/drivers/mailbox/riscv-sbi-mpxy-mbox.c index 7c9c006b7244a..714f7fb97a2fd 100644 --- a/drivers/mailbox/riscv-sbi-mpxy-mbox.c +++ b/drivers/mailbox/riscv-sbi-mpxy-mbox.c @@ -314,8 +314,13 @@ static int mpxy_get_notifications(u32 channel_id, channel_id, 0, 0, 0, 0, 0); if (sret.error) goto err_put_cpu; + if (sret.value < 0 || mpxy_shmem_size < sizeof(*notif_data) || + sret.value > mpxy_shmem_size - sizeof(*notif_data)) { + put_cpu(); + return -EOVERFLOW; + } - memcpy(notif_data, mpxy->shmem, sret.value + 16); + memcpy(notif_data, mpxy->shmem, sret.value + sizeof(*notif_data)); *events_data_len = sret.value; err_put_cpu: @@ -480,11 +485,14 @@ static void mpxy_mbox_peek_rpmi_data(struct mbox_chan *chan, struct rpmi_mbox_message msg; unsigned long pos = 0; - while (pos < events_data_len && (events_data_len - pos) <= sizeof(*event)) { + while (events_data_len - pos >= sizeof(*event)) { event = (struct rpmi_notification_event *)(notif->events_data + pos); msg.type = RPMI_MBOX_MSG_TYPE_NOTIFICATION_EVENT; msg.notif.event_datalen = le16_to_cpu(event->event_datalen); + if (msg.notif.event_datalen > + events_data_len - pos - sizeof(*event)) + break; msg.notif.event_id = event->event_id; msg.notif.event_data = event->event_data; msg.error = 0; diff --git a/drivers/mailbox/rockchip-mailbox.c b/drivers/mailbox/rockchip-mailbox.c index a1a7dee643567..d55cbf7f2cb6d 100644 --- a/drivers/mailbox/rockchip-mailbox.c +++ b/drivers/mailbox/rockchip-mailbox.c @@ -196,19 +196,10 @@ static int rockchip_mbox_probe(struct platform_device *pdev) /* Each channel has two buffers for A2B and B2A */ mb->buf_size = (size_t)resource_size(res) / (drv_data->num_chans * 2); - mb->pclk = devm_clk_get(&pdev->dev, "pclk_mailbox"); - if (IS_ERR(mb->pclk)) { - ret = PTR_ERR(mb->pclk); - dev_err(&pdev->dev, "failed to get pclk_mailbox clock: %d\n", - ret); - return ret; - } - - ret = clk_prepare_enable(mb->pclk); - if (ret) { - dev_err(&pdev->dev, "failed to enable pclk: %d\n", ret); - return ret; - } + mb->pclk = devm_clk_get_enabled(&pdev->dev, "pclk_mailbox"); + if (IS_ERR(mb->pclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(mb->pclk), + "failed to get and enable pclk_mailbox clock\n"); for (i = 0; i < mb->mbox.num_chans; i++) { irq = platform_get_irq(pdev, i); diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index a458b9fd2fcd6..d58eed96a8ff5 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1287,11 +1287,11 @@ static void free_buffer(struct dm_buffer *b) * dm-io completion routine. It just calls b->bio.bi_end_io, pretending * that the request was handled directly with bio interface. */ -static void dmio_complete(unsigned long error, void *context) +static void dmio_complete(unsigned long error, unsigned long unsup, void *context) { struct dm_buffer *b = context; - b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0); + b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : unlikely(unsup != 0) ? BLK_STS_NOTSUPP : 0); } static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, @@ -1319,7 +1319,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, io_req.mem.ptr.vma = (char *)b->data + offset; } - r = dm_io(&io_req, 1, ®ion, NULL, ioprio); + r = dm_io(&io_req, 1, ®ion, NULL, NULL, ioprio); if (unlikely(r)) b->end_io(b, errno_to_blk_status(r)); } @@ -2220,7 +2220,7 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c) if (WARN_ON_ONCE(dm_bufio_in_request())) return -EINVAL; - return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); + return dm_io(&io_req, 1, &io_reg, NULL, NULL, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_issue_flush); @@ -2246,7 +2246,7 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c if (WARN_ON_ONCE(dm_bufio_in_request())) return -EINVAL; /* discards are optional */ - return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); + return dm_io(&io_req, 1, &io_reg, NULL, NULL, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_issue_discard); diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c index 7fe4d19ade4f0..ea499adca4ce2 100644 --- a/drivers/md/dm-era-target.c +++ b/drivers/md/dm-era-target.c @@ -1034,6 +1034,7 @@ static int metadata_checkpoint(struct era_metadata *md) static int metadata_take_snap(struct era_metadata *md) { int r, inc; + dm_block_t location; struct dm_block *clone; if (md->metadata_snap != SUPERBLOCK_LOCATION) { @@ -1071,7 +1072,9 @@ static int metadata_take_snap(struct era_metadata *md) r = dm_sm_inc_block(md->sm, md->writeset_tree_root); if (r) { DMERR("%s: couldn't inc writeset tree root", __func__); + location = dm_block_location(clone); dm_tm_unlock(md->tm, clone); + dm_sm_dec_block(md->sm, location); return r; } @@ -1079,7 +1082,9 @@ static int metadata_take_snap(struct era_metadata *md) if (r) { DMERR("%s: couldn't inc era tree root", __func__); dm_sm_dec_block(md->sm, md->writeset_tree_root); + location = dm_block_location(clone); dm_tm_unlock(md->tm, clone); + dm_sm_dec_block(md->sm, location); return r; } diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 1f2593f113f64..86fc35e76a0f7 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -66,6 +66,7 @@ #define SB_VERSION_4 4 #define SB_VERSION_5 5 #define SB_VERSION_6 6 +#define SB_VERSION_7 7 #define SB_SECTORS 8 #define MAX_SECTORS_PER_BLOCK 8 @@ -91,6 +92,7 @@ struct superblock { #define SB_FLAG_FIXED_PADDING 0x8 #define SB_FLAG_FIXED_HMAC 0x10 #define SB_FLAG_INLINE 0x20 +#define SB_FLAG_DISCARD_KEYED 0x40 #define JOURNAL_ENTRY_ROUNDUP 8 @@ -277,6 +279,7 @@ struct dm_integrity_c { bool recalculate_flag; bool reset_recalculate_flag; bool discard; + bool discard_keyed; bool fix_padding; bool fix_hmac; bool legacy_recalculate; @@ -483,7 +486,9 @@ static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr) static void sb_set_version(struct dm_integrity_c *ic) { - if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) + if (ic->sb->flags & cpu_to_le32(SB_FLAG_DISCARD_KEYED)) + ic->sb->version = SB_VERSION_7; + else if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) ic->sb->version = SB_VERSION_6; else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) ic->sb->version = SB_VERSION_5; @@ -562,7 +567,7 @@ static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf) } } - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) return r; @@ -1035,12 +1040,14 @@ static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned in return crypt_journal(ic, encrypt, section, n_sections, comp); } -static void complete_journal_io(unsigned long error, void *context) +static void complete_journal_io(unsigned long error, unsigned long unsup, void *context) { struct journal_completion *comp = context; if (unlikely(error != 0)) dm_integrity_io_error(comp->ic, "writing journal", -EIO); + else if (unlikely(unsup != 0)) + dm_integrity_io_error(comp->ic, "writing journal", -EOPNOTSUPP); complete_journal_op(comp); } @@ -1055,7 +1062,7 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, if (unlikely(dm_integrity_failed(ic))) { if (comp) - complete_journal_io(-1UL, comp); + complete_journal_io(-1UL, -1UL, comp); return; } @@ -1080,13 +1087,13 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, io_loc.sector = ic->start + SB_SECTORS + sector; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ? "reading journal" : "writing journal", r); if (comp) { WARN_ONCE(1, "asynchronous dm_io failed: %d", r); - complete_journal_io(-1UL, comp); + complete_journal_io(-1UL, -1UL, comp); } } } @@ -1177,7 +1184,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u BUG_ON((target | n_sectors | offset) & (unsigned int)(ic->sectors_per_block - 1)); if (unlikely(dm_integrity_failed(ic))) { - fn(-1UL, data); + fn(-1UL, -1UL, data); return; } @@ -1197,10 +1204,10 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u io_loc.sector = target; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { WARN_ONCE(1, "asynchronous dm_io failed: %d", r); - fn(-1UL, data); + fn(-1UL, -1UL, data); } } @@ -1414,7 +1421,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se { unsigned int hash_offset = 0; unsigned char mismatch_hash = 0; - unsigned char mismatch_filler = !ic->discard; + unsigned char mismatch_filler = !ic->discard || ic->discard_keyed; do { unsigned char *data, *dp; @@ -1466,7 +1473,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se } hash_offset = 0; mismatch_hash = 0; - mismatch_filler = !ic->discard; + mismatch_filler = !ic->discard || ic->discard_keyed; } } } @@ -1493,12 +1500,14 @@ struct flush_request { struct completion comp; }; -static void flush_notify(unsigned long error, void *fr_) +static void flush_notify(unsigned long error, unsigned long unsup, void *fr_) { struct flush_request *fr = fr_; if (unlikely(error != 0)) dm_integrity_io_error(fr->ic, "flushing disk cache", -EIO); + else if (unlikely(unsup != 0)) + dm_integrity_io_error(fr->ic, "flushing disk cache", -EOPNOTSUPP); complete(&fr->comp); } @@ -1521,7 +1530,7 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat fr.io_reg.count = 0; fr.ic = ic; init_completion(&fr.comp); - r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT); + r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, NULL, IOPRIO_DEFAULT); BUG_ON(r); } @@ -1642,7 +1651,8 @@ static void integrity_end_io(struct bio *bio) } static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t sector, - const char *data, unsigned offset, char *result) + const char *data, unsigned offset, + unsigned int len, char *result) { __le64 sector_le = cpu_to_le64(sector); SHASH_DESC_ON_STACK(req, ic->internal_shash); @@ -1671,10 +1681,12 @@ static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t goto failed; } - r = crypto_shash_update(req, data + offset, ic->sectors_per_block << SECTOR_SHIFT); - if (unlikely(r < 0)) { - dm_integrity_io_error(ic, "crypto_shash_update", r); - goto failed; + if (likely(len)) { + r = crypto_shash_update(req, data + offset, len); + if (unlikely(r < 0)) { + dm_integrity_io_error(ic, "crypto_shash_update", r); + goto failed; + } } r = crypto_shash_final(req, result); @@ -1695,7 +1707,8 @@ static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t } static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ahash_request **ahash_req, - sector_t sector, struct page *page, unsigned offset, char *result) + sector_t sector, struct page *page, unsigned offset, + unsigned int len, char *result) { __le64 sector_le = cpu_to_le64(sector); struct ahash_request *req; @@ -1704,6 +1717,7 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah int r; unsigned int digest_size; unsigned int nbytes = 0; + unsigned int nents = 1 + (len ? 1 : 0); might_sleep(); @@ -1717,12 +1731,12 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &wait); if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { - sg_init_table(sg, 3); + sg_init_table(sg, nents + 1); sg_set_buf(s, (const __u8 *)&ic->sb->salt, SALT_SIZE); nbytes += SALT_SIZE; s++; } else { - sg_init_table(sg, 2); + sg_init_table(sg, nents); } if (likely(!is_vmalloc_addr(§or_le))) { @@ -1735,8 +1749,10 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah nbytes += sizeof(sector_le); s++; - sg_set_page(s, page, ic->sectors_per_block << SECTOR_SHIFT, offset); - nbytes += ic->sectors_per_block << SECTOR_SHIFT; + if (likely(len)) { + sg_set_page(s, page, len, offset); + nbytes += len; + } ahash_request_set_crypt(req, sg, result, nbytes); @@ -1759,11 +1775,41 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah static void integrity_sector_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req, sector_t sector, const char *data, unsigned offset, char *result) +{ + unsigned int len = ic->sectors_per_block << SECTOR_SHIFT; + + if (likely(ic->internal_shash != NULL)) + integrity_sector_checksum_shash(ic, sector, data, offset, len, result); + else + integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, + offset, len, result); +} + +/* + * Authenticated marker for a discarded block: HMAC_key(salt || sector), with + * no data payload. Because a real data tag's input always covers a full + * block, its length differs from this marker's, so the two can never + * collide structurally, regardless of block content. + */ +static void integrity_discard_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req, + sector_t sector, char *result) { if (likely(ic->internal_shash != NULL)) - integrity_sector_checksum_shash(ic, sector, data, offset, result); + integrity_sector_checksum_shash(ic, sector, NULL, 0, 0, result); else - integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, offset, result); + integrity_sector_checksum_ahash(ic, ahash_req, sector, NULL, 0, 0, result); +} + +static void integrity_discard_fill_tags(struct dm_integrity_c *ic, struct ahash_request **ahash_req, + unsigned char *checksums, sector_t *sector, + unsigned int blocks) +{ + unsigned int i; + + for (i = 0; i < blocks; i++) { + integrity_discard_checksum(ic, ahash_req, *sector, checksums + i * ic->tag_size); + *sector += ic->sectors_per_block; + } } static void *integrity_kmap(struct dm_integrity_c *ic, struct page *p) @@ -1792,6 +1838,29 @@ static void *integrity_identity(struct dm_integrity_c *ic, void *data) return virt_to_page(data); } +static int integrity_recheck_verify_tag(struct dm_integrity_io *dio, char *checksum, + char *on_disk_tag, sector_t logical_sector) +{ + struct dm_integrity_c *ic = dio->ic; + int r; + + if (!ic->discard_keyed) + return dm_integrity_rw_tag(ic, checksum, &dio->metadata_block, + &dio->metadata_offset, ic->tag_size, TAG_CMP); + + r = dm_integrity_rw_tag(ic, on_disk_tag, &dio->metadata_block, + &dio->metadata_offset, ic->tag_size, TAG_READ); + if (unlikely(r)) + return r; + + r = crypto_memneq(on_disk_tag, checksum, ic->tag_size); + if (unlikely(r)) { + integrity_discard_checksum(ic, &dio->ahash_req, logical_sector, checksum); + r = crypto_memneq(on_disk_tag, checksum, ic->tag_size); + } + return r; +} + static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) { struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); @@ -1817,6 +1886,7 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks char *mem; char *buffer = page_to_virt(page); unsigned int buffer_offset; + char on_disk_tag[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; int r; struct dm_io_request io_req; struct dm_io_region io_loc; @@ -1837,15 +1907,15 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks buffer_offset = (sector - io_loc.sector) << SECTOR_SHIFT; io_loc.count = round_up(io_loc.count, alignment); - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dio->bi_status = errno_to_blk_status(r); goto free_ret; } integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum); - r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block, - &dio->metadata_offset, ic->tag_size, TAG_CMP); + r = integrity_recheck_verify_tag(dio, checksum, on_disk_tag, + logical_sector); if (r) { if (r > 0) { DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx", @@ -1910,14 +1980,19 @@ static void integrity_metadata(struct work_struct *w) if (unlikely(dio->op == REQ_OP_DISCARD)) { unsigned int bi_size = dio->bio_details.bi_iter.bi_size; unsigned int max_size = likely(checksums != checksums_onstack) ? PAGE_SIZE : HASH_MAX_DIGESTSIZE; - unsigned int max_blocks = max_size / ic->tag_size; + unsigned int max_blocks = (max_size - extra_space) / ic->tag_size; + sector_t sector = dio->range.logical_sector; - memset(checksums, DISCARD_FILLER, max_size); + if (!ic->discard_keyed) + memset(checksums, DISCARD_FILLER, max_size); while (bi_size) { unsigned int this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); this_step_blocks = min(this_step_blocks, max_blocks); + if (ic->discard_keyed) + integrity_discard_fill_tags(ic, &dio->ahash_req, checksums, + §or, this_step_blocks); r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, this_step_blocks * ic->tag_size, TAG_WRITE); if (unlikely(r)) { @@ -2890,7 +2965,7 @@ static void integrity_commit(struct work_struct *w) } } -static void complete_copy_from_journal(unsigned long error, void *context) +static void complete_copy_from_journal(unsigned long error, unsigned long unsup, void *context) { struct journal_io *io = context; struct journal_completion *comp = io->comp; @@ -2900,6 +2975,8 @@ static void complete_copy_from_journal(unsigned long error, void *context) mempool_free(io, &ic->journal_io_mempool); if (unlikely(error != 0)) dm_integrity_io_error(ic, "copying from journal", -EIO); + else if (unlikely(unsup != 0)) + dm_integrity_io_error(ic, "copying from journal", -EOPNOTSUPP); complete_journal_op(comp); } @@ -3215,7 +3292,7 @@ static void integrity_recalc(struct work_struct *w) io_loc.sector = get_data_sector(ic, area, offset); io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, "reading data", r); goto err; @@ -3792,6 +3869,8 @@ static void dm_integrity_resume(struct dm_target *ti) ic->wrote_to_journal = false; flags = ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING); + if (ic->discard_keyed) + flags |= cpu_to_le32(SB_FLAG_DISCARD_KEYED); r = sync_rw_sb(ic, REQ_OP_READ); if (r) dm_integrity_io_error(ic, "reading superblock", r); @@ -3939,7 +4018,8 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, arg_count += ic->sectors_per_block != 1; arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)); arg_count += ic->reset_recalculate_flag; - arg_count += ic->discard; + arg_count += ic->discard && !ic->discard_keyed; + arg_count += ic->discard_keyed; arg_count += ic->mode != 'I'; /* interleave_sectors */ arg_count += ic->mode == 'J'; /* journal_sectors */ arg_count += ic->mode == 'J'; /* journal_watermark */ @@ -3962,8 +4042,10 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, DMEMIT(" recalculate"); if (ic->reset_recalculate_flag) DMEMIT(" reset_recalculate"); - if (ic->discard) + if (ic->discard && !ic->discard_keyed) DMEMIT(" allow_discards"); + if (ic->discard_keyed) + DMEMIT(" allow_discards_keyed"); if (ic->mode != 'I') DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors); DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors); @@ -4013,6 +4095,7 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ? 'y' : 'n'); DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n'); + DMEMIT(",allow_discards_keyed=%c", ic->discard_keyed ? 'y' : 'n'); DMEMIT(",fix_padding=%c", ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n'); DMEMIT(",fix_hmac=%c", @@ -4170,6 +4253,9 @@ static int initialize_superblock(struct dm_integrity_c *ic, get_random_bytes(ic->sb->salt, SALT_SIZE); } + if (ic->discard_keyed) + ic->sb->flags |= cpu_to_le32(SB_FLAG_DISCARD_KEYED); + if (!ic->meta_dev) { if (ic->fix_padding) ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING); @@ -4827,6 +4913,9 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv ic->reset_recalculate_flag = true; } else if (!strcmp(opt_string, "allow_discards")) { ic->discard = true; + } else if (!strcmp(opt_string, "allow_discards_keyed")) { + ic->discard = true; + ic->discard_keyed = true; } else if (!strcmp(opt_string, "fix_padding")) { ic->fix_padding = true; } else if (!strcmp(opt_string, "fix_hmac")) { @@ -4955,6 +5044,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv ti->error = "Discard can be only used with internal hash"; goto bad; } + if (ic->discard_keyed && !ic->internal_hash_alg.key) { + r = -EINVAL; + ti->error = "Keyed discard can only be used with keyed internal hash"; + goto bad; + } ic->autocommit_jiffies = msecs_to_jiffies(sync_msec); ic->autocommit_msec = sync_msec; @@ -5073,7 +5167,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv should_write_sb = true; } - if (!ic->sb->version || ic->sb->version > SB_VERSION_6) { + if (!ic->sb->version || ic->sb->version > SB_VERSION_7) { r = -EINVAL; ti->error = "Unknown version"; goto bad; @@ -5121,6 +5215,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv goto bad; } } + if (!ic->discard_keyed && (ic->sb->flags & cpu_to_le32(SB_FLAG_DISCARD_KEYED))) { + r = -EINVAL; + ti->error = "Keyed discard cannot be disabled once enabled"; + goto bad; + } if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) { r = -EINVAL; ti->error = "Journal mac mismatch"; @@ -5436,7 +5535,7 @@ static void dm_integrity_dtr(struct dm_target *ti) static struct target_type integrity_target = { .name = "integrity", - .version = {1, 14, 0}, + .version = {1, 15, 0}, .module = THIS_MODULE, .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY, .ctr = dm_integrity_ctr, diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index 1db565b376200..777b917be4002 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c @@ -33,6 +33,7 @@ struct dm_io_client { */ struct io { unsigned long error_bits; + unsigned long unsup_bits; atomic_t count; struct dm_io_client *client; io_notify_fn callback; @@ -119,6 +120,7 @@ static void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io, static void complete_io(struct io *io) { unsigned long error_bits = io->error_bits; + unsigned long unsup_bits = io->unsup_bits; io_notify_fn fn = io->callback; void *context = io->context; @@ -127,13 +129,17 @@ static void complete_io(struct io *io) io->vma_invalidate_size); mempool_free(io, &io->client->pool); - fn(error_bits, context); + fn(error_bits, unsup_bits, context); } static void dec_count(struct io *io, unsigned int region, blk_status_t error) { - if (error) - set_bit(region, &io->error_bits); + if (unlikely(error)) { + if (error == BLK_STS_NOTSUPP || error == BLK_STS_INVAL) + set_bit(region, &io->unsup_bits); + else + set_bit(region, &io->error_bits); + } if (atomic_dec_and_test(&io->count)) complete_io(io); @@ -170,12 +176,11 @@ struct dpages { struct page **p, unsigned long *len, unsigned int *offset); void (*next_page)(struct dpages *dp); - union { - unsigned int context_u; - struct bvec_iter context_bi; - }; + unsigned int context_u; void *context_ptr; + struct bio *orig_bio; + void *vma_invalidate_address; unsigned long vma_invalidate_size; }; @@ -210,44 +215,6 @@ static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned int o dp->context_ptr = pl; } -/* - * Functions for getting the pages from a bvec. - */ -static void bio_get_page(struct dpages *dp, struct page **p, - unsigned long *len, unsigned int *offset) -{ - struct bio_vec bvec = bvec_iter_bvec((struct bio_vec *)dp->context_ptr, - dp->context_bi); - - *p = bvec.bv_page; - *len = bvec.bv_len; - *offset = bvec.bv_offset; - - /* avoid figuring it out again in bio_next_page() */ - dp->context_bi.bi_sector = (sector_t)bvec.bv_len; -} - -static void bio_next_page(struct dpages *dp) -{ - unsigned int len = (unsigned int)dp->context_bi.bi_sector; - - bvec_iter_advance((struct bio_vec *)dp->context_ptr, - &dp->context_bi, len); -} - -static void bio_dp_init(struct dpages *dp, struct bio *bio) -{ - dp->get_page = bio_get_page; - dp->next_page = bio_next_page; - - /* - * We just use bvec iterator to retrieve pages, so it is ok to - * access the bvec table directly here - */ - dp->context_ptr = bio->bi_io_vec; - dp->context_bi = bio->bi_iter; -} - /* * Functions for getting the pages from a VMA. */ @@ -332,6 +299,21 @@ static void do_region(const blk_opf_t opf, unsigned int region, return; } + if (dp->orig_bio) { + bio = bio_alloc_clone(where->bdev, dp->orig_bio, GFP_NOIO, + &io->client->bios); + bio->bi_iter.bi_sector = where->sector; + bio->bi_iter.bi_size = where->count << SECTOR_SHIFT; + bio->bi_opf = opf; + bio->bi_end_io = endio; + bio->bi_ioprio = ioprio; + store_io_and_region_in_bio(bio, io, region); + + atomic_inc(&io->count); + submit_bio(bio); + return; + } + /* * where->count may be zero if op holds a flush and we need to * send a zero-sized flush. @@ -418,6 +400,7 @@ static void async_io(struct dm_io_client *client, unsigned int num_regions, io = mempool_alloc(&client->pool, GFP_NOIO); io->error_bits = 0; + io->unsup_bits = 0; atomic_set(&io->count, 1); /* see dispatch_io() */ io->client = client; io->callback = fn; @@ -431,20 +414,23 @@ static void async_io(struct dm_io_client *client, unsigned int num_regions, struct sync_io { unsigned long error_bits; + unsigned long unsup_bits; struct completion wait; }; -static void sync_io_complete(unsigned long error, void *context) +static void sync_io_complete(unsigned long error, unsigned long unsup, void *context) { struct sync_io *sio = context; sio->error_bits = error; + sio->unsup_bits = unsup; complete(&sio->wait); } static int sync_io(struct dm_io_client *client, unsigned int num_regions, struct dm_io_region *where, blk_opf_t opf, struct dpages *dp, - unsigned long *error_bits, unsigned short ioprio) + unsigned long *error_bits, unsigned long *unsup_bits, + unsigned short ioprio) { struct sync_io sio; @@ -457,8 +443,10 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, if (error_bits) *error_bits = sio.error_bits; + if (unsup_bits) + *unsup_bits = sio.unsup_bits; - return sio.error_bits ? -EIO : 0; + return sio.error_bits ? -EIO : sio.unsup_bits ? -EOPNOTSUPP : 0; } static int dp_init(struct dm_io_request *io_req, struct dpages *dp, @@ -468,6 +456,7 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, dp->vma_invalidate_address = NULL; dp->vma_invalidate_size = 0; + dp->orig_bio = NULL; switch (io_req->mem.type) { case DM_IO_PAGE_LIST: @@ -475,7 +464,11 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, break; case DM_IO_BIO: - bio_dp_init(dp, io_req->mem.ptr.bio); + /* + * The destination bios clone this bio's biovec directly, so + * there are no per-page accessors to set up here. + */ + dp->orig_bio = io_req->mem.ptr.bio; break; case DM_IO_VMA: @@ -500,7 +493,7 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, int dm_io(struct dm_io_request *io_req, unsigned int num_regions, struct dm_io_region *where, unsigned long *sync_error_bits, - unsigned short ioprio) + unsigned long *sync_unsup_bits, unsigned short ioprio) { int r; struct dpages dp; @@ -516,7 +509,8 @@ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, if (!io_req->notify.fn) return sync_io(io_req->client, num_regions, where, - io_req->bi_opf, &dp, sync_error_bits, ioprio); + io_req->bi_opf, &dp, sync_error_bits, + sync_unsup_bits, ioprio); async_io(io_req->client, num_regions, where, io_req->bi_opf, &dp, io_req->notify.fn, io_req->notify.context, ioprio); diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 96c8b8ff61c20..5c9c24a3dcd99 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -517,16 +517,16 @@ static int run_complete_job(struct kcopyd_job *job) return 0; } -static void complete_io(unsigned long error, void *context) +static void complete_io(unsigned long error, unsigned long unsup, void *context) { struct kcopyd_job *job = context; struct dm_kcopyd_client *kc = job->kc; io_job_finish(kc->throttle); - if (error) { + if (unlikely((error | unsup) != 0)) { if (op_is_write(job->op)) - job->write_err |= error; + job->write_err |= error | unsup; else job->read_err = 1; @@ -578,9 +578,9 @@ static int run_io_job(struct kcopyd_job *job) io_job_start(job->kc->throttle); if (job->op == REQ_OP_READ) - r = dm_io(&io_req, 1, &job->source, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &job->source, NULL, NULL, IOPRIO_DEFAULT); else - r = dm_io(&io_req, job->num_dests, job->dests, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, job->num_dests, job->dests, NULL, NULL, IOPRIO_DEFAULT); return r; } diff --git a/drivers/md/dm-log-userspace-base.c b/drivers/md/dm-log-userspace-base.c index 9d5918eb0a30e..3ca35aa4e057d 100644 --- a/drivers/md/dm-log-userspace-base.c +++ b/drivers/md/dm-log-userspace-base.c @@ -139,6 +139,7 @@ static int build_constructor_string(struct dm_target *ti, str_size += strlen(argv[i]) + 1; /* +1 for space between args */ str_size += 20; /* Max number of chars in a printed u64 number */ + str_size++; /* For NUL-terminator */ str = kzalloc(str_size, GFP_KERNEL); if (!str) { diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index 2ddeb4250c592..4c114fc5ef837 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -300,7 +300,7 @@ static int rw_header(struct log_c *lc, enum req_op op) { lc->io_req.bi_opf = op; - return dm_io(&lc->io_req, 1, &lc->header_location, NULL, IOPRIO_DEFAULT); + return dm_io(&lc->io_req, 1, &lc->header_location, NULL, NULL, IOPRIO_DEFAULT); } static int flush_header(struct log_c *lc) @@ -313,7 +313,7 @@ static int flush_header(struct log_c *lc) lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; - return dm_io(&lc->io_req, 1, &null_location, NULL, IOPRIO_DEFAULT); + return dm_io(&lc->io_req, 1, &null_location, NULL, NULL, IOPRIO_DEFAULT); } static int read_header(struct log_c *log) diff --git a/drivers/md/dm-pcache/cache.c b/drivers/md/dm-pcache/cache.c index bb1ada31e4836..b0b3e21677dee 100644 --- a/drivers/md/dm-pcache/cache.c +++ b/drivers/md/dm-pcache/cache.c @@ -118,7 +118,14 @@ int cache_pos_decode(struct pcache_cache *cache, if (!latest_addr) return -EIO; + if (!cache_seg_id_valid(cache, latest.cache_seg_id)) + return -EIO; + pos->cache_seg = &cache->segments[latest.cache_seg_id]; + + if (latest.seg_off >= pos->cache_seg->segment.data_size) + return -EIO; + pos->seg_off = latest.seg_off; *seq = latest.header.seq; *index = (latest_addr - pos_onmedia); @@ -155,6 +162,7 @@ static int cache_init(struct dm_pcache *pcache) cache->cache_dev = &pcache->cache_dev; cache->n_segs = cache_dev->seg_num; atomic_set(&cache->gc_errors, 0); + atomic_set(&cache->writeback_errors, 0); spin_lock_init(&cache->seg_map_lock); spin_lock_init(&cache->key_head_lock); @@ -194,6 +202,7 @@ static int cache_tail_init(struct pcache_cache *cache) { struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE); + int ret; if (new_cache) { __set_bit(0, cache->seg_map); @@ -210,6 +219,12 @@ static int cache_tail_init(struct pcache_cache *cache) pcache_dev_err(pcache, "Corrupted key tail or dirty tail.\n"); return -EIO; } + + ret = cache_verify_dirty_tail(cache); + if (ret) { + pcache_dev_err(pcache, "dirty tail chain does not terminate (crafted cache image?)\n"); + return ret; + } } return 0; @@ -247,6 +262,13 @@ static int get_seg_id(struct pcache_cache *cache, } else { *seg_id = cache->cache_info.seg_id; } + + if (*seg_id >= cache_dev->seg_num) { + pcache_dev_err(pcache, "invalid segment id %u from cache device (seg_num %u)\n", + *seg_id, cache_dev->seg_num); + ret = -EIO; + goto err; + } } return 0; err: @@ -262,6 +284,13 @@ static int cache_segs_init(struct pcache_cache *cache) int ret; u32 i; + if (cache_info->n_segs > cache->cache_dev->seg_num) { + pcache_dev_err(CACHE_TO_PCACHE(cache), + "cache_info n_segs %u exceeds cache device segments %u\n", + cache_info->n_segs, cache->cache_dev->seg_num); + return -EIO; + } + for (i = 0; i < cache_info->n_segs; i++) { ret = get_seg_id(cache, prev_cache_seg, new_cache, &seg_id); if (ret) diff --git a/drivers/md/dm-pcache/cache.h b/drivers/md/dm-pcache/cache.h index 27613b56be54c..8809ec5ae943c 100644 --- a/drivers/md/dm-pcache/cache.h +++ b/drivers/md/dm-pcache/cache.h @@ -180,6 +180,7 @@ struct pcache_cache { u32 advance; int ret; } writeback_ctx; + atomic_t writeback_errors; char gc_kset_onmedia_buf[PCACHE_KSET_ONMEDIA_SIZE_MAX]; struct delayed_work gc_work; @@ -420,6 +421,20 @@ static inline bool cache_seg_is_ctrl_seg(u32 cache_seg_id) return (cache_seg_id == 0); } +/** + * cache_seg_id_valid - Validate a cache segment id read from the cache device. + * @cache: Pointer to the pcache_cache structure. + * @cache_seg_id: Segment id decoded from on-media metadata. + * + * On-media segment ids are only protected by a CRC, which an attacker who can + * format the cache device computes over their chosen value. Reject any id that + * would index cache->segments[] out of bounds before it is dereferenced. + */ +static inline bool cache_seg_id_valid(struct pcache_cache *cache, u32 cache_seg_id) +{ + return cache_seg_id < cache->cache_info.n_segs; +} + /** * cache_key_cutfront - Cuts a specified length from the front of a cache key. * @key: Pointer to pcache_cache_key structure. @@ -491,6 +506,27 @@ static inline u32 cache_key_data_crc(struct pcache_cache_key *key) return crc32c(PCACHE_CRC_SEED, data, key->len); } +/** + * kset_onmedia_valid - Validate a kset header read from the cache device. + * @kset_onmedia: Pointer to the kset copied from on-media metadata. + * + * The magic and CRC are attacker-computable (fixed public seed). A non-last + * kset stores key_num keys inline, and cache_kset_crc() and the replay loop + * read struct_size(.., data, key_num) bytes from a buffer sized for + * PCACHE_KSET_KEYS_MAX keys, so key_num must be bounded before any such use. + */ +static inline bool kset_onmedia_valid(struct pcache_cache_kset_onmedia *kset_onmedia) +{ + if (kset_onmedia->magic != PCACHE_KSET_MAGIC) + return false; + + if (!(kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) && + kset_onmedia->key_num > PCACHE_KSET_KEYS_MAX) + return false; + + return true; +} + static inline u32 cache_kset_crc(struct pcache_cache_kset_onmedia *kset_onmedia) { u32 crc_size; @@ -630,6 +666,8 @@ static inline int cache_decode_dirty_tail(struct pcache_cache *cache) &cache->dirty_tail_index); } +int cache_verify_dirty_tail(struct pcache_cache *cache); + int pcache_cache_init(void); void pcache_cache_exit(void); #endif /* _PCACHE_CACHE_H */ diff --git a/drivers/md/dm-pcache/cache_dev.c b/drivers/md/dm-pcache/cache_dev.c index ece689e6ce59d..f0259353ee397 100644 --- a/drivers/md/dm-pcache/cache_dev.c +++ b/drivers/md/dm-pcache/cache_dev.c @@ -242,6 +242,8 @@ int cache_dev_start(struct dm_pcache *pcache) struct pcache_cache_dev *cache_dev = &pcache->cache_dev; struct pcache_sb sb; bool format = false; + u32 seg_num; + u64 max_segs; int ret; mutex_init(&cache_dev->seg_lock); @@ -269,7 +271,25 @@ int cache_dev_start(struct dm_pcache *pcache) goto dax_release; cache_dev->sb_flags = le32_to_cpu(sb.flags); - ret = cache_dev_init(cache_dev, le32_to_cpu(sb.seg_num)); + + /* + * seg_num is read from the crc32c-only superblock, so whoever supplies + * the cache device controls it. It is the ceiling every later on-media + * segment id is validated against, so bound it against what the device + * physically holds before it is trusted, or a forged seg_num lets a + * segment id address past the DAX mapping. + */ + seg_num = le32_to_cpu(sb.seg_num); + max_segs = (bdev_nr_bytes(cache_dev->dm_dev->bdev) - PCACHE_SEGMENTS_OFF) / + PCACHE_SEG_SIZE; + if (seg_num == 0 || seg_num > max_segs || seg_num > PCACHE_CACHE_SEGS_MAX) { + pcache_dev_err(pcache, "invalid seg_num %u from cache device (device holds %llu, max %u)\n", + seg_num, max_segs, (u32)PCACHE_CACHE_SEGS_MAX); + ret = -EIO; + goto dax_release; + } + + ret = cache_dev_init(cache_dev, seg_num); if (ret) goto dax_release; diff --git a/drivers/md/dm-pcache/cache_gc.c b/drivers/md/dm-pcache/cache_gc.c index 94f8b276a0212..a167892be7cef 100644 --- a/drivers/md/dm-pcache/cache_gc.c +++ b/drivers/md/dm-pcache/cache_gc.c @@ -37,18 +37,18 @@ static bool need_gc(struct pcache_cache *cache, struct pcache_cache_pos *dirty_t kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->gc_kset_onmedia_buf; - to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, PCACHE_SEG_SIZE - key_tail->seg_off); + to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(key_tail)); ret = copy_mc_to_kernel(kset_onmedia, key_addr, to_copy); if (ret) { pcache_dev_err(pcache, "error to read kset: %d", ret); return false; } - /* Check if kset_onmedia is corrupted */ - if (kset_onmedia->magic != PCACHE_KSET_MAGIC) { - pcache_dev_debug(pcache, "gc error: magic is not as expected. key_tail: %u:%u magic: %llx, expected: %llx\n", + /* Reject a corrupted or out-of-bounds kset before reading its keys */ + if (!kset_onmedia_valid(kset_onmedia)) { + pcache_dev_debug(pcache, "gc error: invalid kset. key_tail: %u:%u magic: %llx, key_num: %u\n", key_tail->cache_seg->cache_seg_id, key_tail->seg_off, - kset_onmedia->magic, PCACHE_KSET_MAGIC); + kset_onmedia->magic, kset_onmedia->key_num); return false; } @@ -74,11 +74,17 @@ static bool need_gc(struct pcache_cache *cache, struct pcache_cache_pos *dirty_t * @cache: Pointer to the pcache_cache structure. * @kset_onmedia: Pointer to the kset_onmedia structure for the last kset. */ -static void last_kset_gc(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *kset_onmedia) +static int last_kset_gc(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *kset_onmedia) { struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); struct pcache_cache_segment *cur_seg, *next_seg; + if (!cache_seg_id_valid(cache, kset_onmedia->next_cache_seg_id)) { + pcache_dev_err(pcache, "invalid next_cache_seg_id %u in gc (n_segs %u)\n", + kset_onmedia->next_cache_seg_id, cache->n_segs); + return -EIO; + } + cur_seg = cache->key_tail.cache_seg; next_seg = &cache->segments[kset_onmedia->next_cache_seg_id]; @@ -94,6 +100,8 @@ static void last_kset_gc(struct pcache_cache *cache, struct pcache_cache_kset_on spin_lock(&cache->seg_map_lock); __clear_bit(cur_seg->cache_seg_id, cache->seg_map); spin_unlock(&cache->seg_map_lock); + + return 0; } void pcache_cache_gc_fn(struct work_struct *work) @@ -130,7 +138,11 @@ void pcache_cache_gc_fn(struct work_struct *work) if (dirty_tail.cache_seg == key_tail.cache_seg) break; - last_kset_gc(cache, kset_onmedia); + ret = last_kset_gc(cache, kset_onmedia); + if (ret) { + atomic_inc(&cache->gc_errors); + return; + } continue; } diff --git a/drivers/md/dm-pcache/cache_key.c b/drivers/md/dm-pcache/cache_key.c index e068e878231b4..a0d109f85504d 100644 --- a/drivers/md/dm-pcache/cache_key.c +++ b/drivers/md/dm-pcache/cache_key.c @@ -94,9 +94,23 @@ int cache_key_decode(struct pcache_cache *cache, key->off = key_onmedia->off; key->len = key_onmedia->len; + if (!cache_seg_id_valid(cache, key_onmedia->cache_seg_id)) { + pcache_dev_err(pcache, "invalid cache_seg_id %u in cache key (n_segs %u)\n", + key_onmedia->cache_seg_id, cache->n_segs); + return -EIO; + } + key->cache_pos.cache_seg = &cache->segments[key_onmedia->cache_seg_id]; key->cache_pos.seg_off = key_onmedia->cache_seg_off; + if ((u64)key->cache_pos.seg_off + key->len > + key->cache_pos.cache_seg->segment.data_size) { + pcache_dev_err(pcache, "key seg_off %u + len %u exceeds segment data size %u\n", + key->cache_pos.seg_off, key->len, + key->cache_pos.cache_seg->segment.data_size); + return -EIO; + } + key->seg_gen = key_onmedia->seg_gen; key->flags = key_onmedia->flags; @@ -728,18 +742,17 @@ static int kset_replay(struct pcache_cache *cache, struct pcache_cache_kset_onme goto err; } - __set_bit(key->cache_pos.cache_seg->cache_seg_id, cache->seg_map); - /* Check if the segment generation is valid for insertion. */ if (key->seg_gen < key->cache_pos.cache_seg->gen) { cache_key_put(key); - } else { - cache_subtree = get_subtree(&cache->req_key_tree, key->off); - spin_lock(&cache_subtree->tree_lock); - cache_key_insert(&cache->req_key_tree, key, true); - spin_unlock(&cache_subtree->tree_lock); + continue; } + __set_bit(key->cache_pos.cache_seg->cache_seg_id, cache->seg_map); + cache_subtree = get_subtree(&cache->req_key_tree, key->off); + spin_lock(&cache_subtree->tree_lock); + cache_key_insert(&cache->req_key_tree, key, true); + spin_unlock(&cache_subtree->tree_lock); cache_seg_get(key->cache_pos.cache_seg); } @@ -754,7 +767,7 @@ int cache_replay(struct pcache_cache *cache) struct pcache_cache_pos pos_tail; struct pcache_cache_pos *pos; struct pcache_cache_kset_onmedia *kset_onmedia; - u32 to_copy, count = 0; + u32 to_copy, count = 0, last_hops = 0; int ret = 0; kset_onmedia = kzalloc(PCACHE_KSET_ONMEDIA_SIZE_MAX, GFP_KERNEL); @@ -771,14 +784,14 @@ int cache_replay(struct pcache_cache *cache) __set_bit(pos->cache_seg->cache_seg_id, cache->seg_map); while (true) { - to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, PCACHE_SEG_SIZE - pos->seg_off); + to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(pos)); ret = copy_mc_to_kernel(kset_onmedia, cache_pos_addr(pos), to_copy); if (ret) { ret = -EIO; goto out; } - if (kset_onmedia->magic != PCACHE_KSET_MAGIC || + if (!kset_onmedia_valid(kset_onmedia) || kset_onmedia->crc != cache_kset_crc(kset_onmedia)) { break; } @@ -789,6 +802,16 @@ int cache_replay(struct pcache_cache *cache) pcache_dev_debug(pcache, "last kset replay, next: %u\n", kset_onmedia->next_cache_seg_id); + if (++last_hops > cache->n_segs) { + ret = -EIO; + goto out; + } + + if (!cache_seg_id_valid(cache, kset_onmedia->next_cache_seg_id)) { + ret = -EIO; + goto out; + } + next_seg = &cache->segments[kset_onmedia->next_cache_seg_id]; pos->cache_seg = next_seg; @@ -820,6 +843,75 @@ int cache_replay(struct pcache_cache *cache) return ret; } +/* + * cache_verify_dirty_tail - reject a persisted dirty_tail whose last-kset + * chain does not terminate. + * + * dirty_tail is decoded independently of the key_tail chain cache_replay() + * walks, so replay's hop cap does not cover it. A crafted chain that loops + * back on itself makes the writeback worker re-arm forever; walk it once here + * with the same cap and fail the load if it does not end within n_segs hops. + */ +int cache_verify_dirty_tail(struct pcache_cache *cache) +{ + struct pcache_cache_pos pos; + struct pcache_cache_kset_onmedia *kset_onmedia; + u32 to_copy, last_hops = 0, count = 0; + int ret = 0; + + kset_onmedia = kzalloc(PCACHE_KSET_ONMEDIA_SIZE_MAX, GFP_KERNEL); + if (!kset_onmedia) + return -ENOMEM; + + cache_pos_copy(&pos, &cache->dirty_tail); + + while (true) { + to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(&pos)); + ret = copy_mc_to_kernel(kset_onmedia, cache_pos_addr(&pos), to_copy); + if (ret) { + ret = -EIO; + goto out; + } + + /* A missing, short or corrupt kset is the normal end of the chain. */ + if (!kset_onmedia_valid(kset_onmedia) || + kset_onmedia->crc != cache_kset_crc(kset_onmedia)) { + ret = 0; + goto out; + } + + if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) { + if (kset_onmedia->next_cache_seg_id >= cache->cache_info.n_segs) { + ret = -EIO; + goto out; + } + + if (++last_hops > cache->n_segs) { + ret = -EIO; + goto out; + } + + pos.cache_seg = &cache->segments[kset_onmedia->next_cache_seg_id]; + pos.seg_off = 0; + continue; + } + + if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&pos)) { + ret = -EIO; + goto out; + } + + cache_pos_advance(&pos, get_kset_onmedia_size(kset_onmedia)); + if (++count > 512) { + cond_resched(); + count = 0; + } + } +out: + kfree(kset_onmedia); + return ret; +} + int cache_tree_init(struct pcache_cache *cache, struct pcache_cache_tree *cache_tree, u32 n_subtrees) { int ret; diff --git a/drivers/md/dm-pcache/cache_segment.c b/drivers/md/dm-pcache/cache_segment.c index 9d92e2b067ed0..c698ebbc626d2 100644 --- a/drivers/md/dm-pcache/cache_segment.c +++ b/drivers/md/dm-pcache/cache_segment.c @@ -243,8 +243,16 @@ struct pcache_cache_segment *get_cache_segment(struct pcache_cache *cache) spin_lock(&cache->seg_map_lock); again: - seg_id = find_next_zero_bit(cache->seg_map, cache->n_segs, cache->last_cache_seg); - if (seg_id == cache->n_segs) { + /* + * Only allocate initialized segments. cache_segs_init() initializes + * cache_info.n_segs of the cache->n_segs device segments; a forged + * smaller cache_info.n_segs leaves the rest as zeroed structs whose data + * pointer is NULL. Bounding the search to cache_info.n_segs keeps such a + * segment from reaching cache_kset_close(), which writes through it. + */ + seg_id = find_next_zero_bit(cache->seg_map, cache->cache_info.n_segs, + cache->last_cache_seg); + if (seg_id == cache->cache_info.n_segs) { /* reset the hint of ->last_cache_seg and retry */ if (cache->last_cache_seg) { cache->last_cache_seg = 0; diff --git a/drivers/md/dm-pcache/cache_writeback.c b/drivers/md/dm-pcache/cache_writeback.c index 87a82b3fe8363..c34db79fcefb4 100644 --- a/drivers/md/dm-pcache/cache_writeback.c +++ b/drivers/md/dm-pcache/cache_writeback.c @@ -48,18 +48,18 @@ static inline bool is_cache_clean(struct pcache_cache *cache, struct pcache_cach addr = cache_pos_addr(dirty_tail); kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf; - to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, PCACHE_SEG_SIZE - dirty_tail->seg_off); + to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(dirty_tail)); ret = copy_mc_to_kernel(kset_onmedia, addr, to_copy); if (ret) { pcache_dev_err(pcache, "error to read kset: %d", ret); return true; } - /* Check if the magic number matches the expected value */ - if (kset_onmedia->magic != PCACHE_KSET_MAGIC) { - pcache_dev_debug(pcache, "dirty_tail: %u:%u magic: %llx, not expected: %llx\n", + /* Reject a corrupted or out-of-bounds kset before reading its keys */ + if (!kset_onmedia_valid(kset_onmedia)) { + pcache_dev_debug(pcache, "dirty_tail: %u:%u invalid kset magic: %llx, key_num: %u\n", dirty_tail->cache_seg->cache_seg_id, dirty_tail->seg_off, - kset_onmedia->magic, PCACHE_KSET_MAGIC); + kset_onmedia->magic, kset_onmedia->key_num); return true; } @@ -196,12 +196,18 @@ static int cache_kset_insert_tree(struct pcache_cache *cache, struct pcache_cach return ret; } -static void last_kset_writeback(struct pcache_cache *cache, +static int last_kset_writeback(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *last_kset_onmedia) { struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); struct pcache_cache_segment *next_seg; + if (!cache_seg_id_valid(cache, last_kset_onmedia->next_cache_seg_id)) { + pcache_dev_err(pcache, "invalid next_cache_seg_id %u in writeback (n_segs %u)\n", + last_kset_onmedia->next_cache_seg_id, cache->n_segs); + return -EIO; + } + pcache_dev_debug(pcache, "last kset, next: %u\n", last_kset_onmedia->next_cache_seg_id); next_seg = &cache->segments[last_kset_onmedia->next_cache_seg_id]; @@ -211,6 +217,8 @@ static void last_kset_writeback(struct pcache_cache *cache, cache->dirty_tail.seg_off = 0; cache_encode_dirty_tail(cache); mutex_unlock(&cache->dirty_tail_lock); + + return 0; } void cache_writeback_fn(struct work_struct *work) @@ -229,6 +237,9 @@ void cache_writeback_fn(struct work_struct *work) if (pcache_is_stopping(pcache)) goto unlock; + if (atomic_read(&cache->writeback_errors)) + goto unlock; + kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf; mutex_lock(&cache->dirty_tail_lock); @@ -241,15 +252,19 @@ void cache_writeback_fn(struct work_struct *work) } if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) { - last_kset_writeback(cache, kset_onmedia); + ret = last_kset_writeback(cache, kset_onmedia); + if (ret) { + atomic_inc(&cache->writeback_errors); + goto unlock; + } delay = 0; goto queue_work; } ret = cache_kset_insert_tree(cache, kset_onmedia); if (ret) { - delay = PCACHE_CACHE_WRITEBACK_INTERVAL; - goto queue_work; + atomic_inc(&cache->writeback_errors); + goto unlock; } cache_wb_tree_writeback(cache, get_kset_onmedia_size(kset_onmedia)); diff --git a/drivers/md/dm-pcache/dm_pcache.c b/drivers/md/dm-pcache/dm_pcache.c index d5cfd162c0633..645fc27d82ba4 100644 --- a/drivers/md/dm-pcache/dm_pcache.c +++ b/drivers/md/dm-pcache/dm_pcache.c @@ -439,13 +439,13 @@ static int dm_pcache_message(struct dm_target *ti, unsigned int argc, char **argv, char *result, unsigned int maxlen) { struct dm_pcache *pcache = ti->private; - unsigned long val; + u8 val; if (argc != 2) goto err; if (!strcasecmp(argv[0], "gc_percent")) { - if (kstrtoul(argv[1], 10, &val)) + if (kstrtou8(argv[1], 10, &val)) goto err; return pcache_cache_set_gc_percent(&pcache->cache, val); diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index de5c00704e69c..da2a5e2002eca 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -258,7 +258,7 @@ static void fail_mirror(struct mirror *m, enum dm_raid1_error error_type) static int mirror_flush(struct dm_target *ti) { struct mirror_set *ms = ti->private; - unsigned long error_bits; + unsigned long error_bits, unsup_bits; unsigned int i; struct dm_io_region io[MAX_NR_MIRRORS]; @@ -277,8 +277,8 @@ static int mirror_flush(struct dm_target *ti) } error_bits = -1; - dm_io(&io_req, ms->nr_mirrors, io, &error_bits, IOPRIO_DEFAULT); - if (unlikely(error_bits != 0)) { + dm_io(&io_req, ms->nr_mirrors, io, &error_bits, &unsup_bits, IOPRIO_DEFAULT); + if (unlikely((error_bits | unsup_bits) != 0)) { for (i = 0; i < ms->nr_mirrors; i++) if (test_bit(i, &error_bits)) fail_mirror(ms->mirror + i, @@ -511,7 +511,7 @@ static void hold_bio(struct mirror_set *ms, struct bio *bio) * Reads *--------------------------------------------------------------- */ -static void read_callback(unsigned long error, void *context) +static void read_callback(unsigned long error, unsigned long unsup, void *context) { struct bio *bio = context; struct mirror *m; @@ -520,6 +520,8 @@ static void read_callback(unsigned long error, void *context) bio_set_m(bio, NULL); if (likely(!error)) { + if (unlikely(unsup != 0)) + bio->bi_status = BLK_STS_INVAL; bio_endio(bio); return; } @@ -553,7 +555,7 @@ static void read_async_bio(struct mirror *m, struct bio *bio) map_region(&io, m, bio); bio_set_m(bio, m); - BUG_ON(dm_io(&io_req, 1, &io, NULL, IOPRIO_DEFAULT)); + BUG_ON(dm_io(&io_req, 1, &io, NULL, NULL, IOPRIO_DEFAULT)); } static inline int region_in_sync(struct mirror_set *ms, region_t region, @@ -600,7 +602,7 @@ static void do_reads(struct mirror_set *ms, struct bio_list *reads) * NOSYNC: increment pending, just write to the default mirror *--------------------------------------------------------------------- */ -static void write_callback(unsigned long error, void *context) +static void write_callback(unsigned long error, unsigned long unsup, void *context) { unsigned int i; struct bio *bio = context; @@ -617,7 +619,7 @@ static void write_callback(unsigned long error, void *context) * This way we handle both writes to SYNC and NOSYNC * regions with the same code. */ - if (likely(!error)) { + if (likely(!(error | unsup))) { bio_endio(bio); return; } @@ -632,6 +634,12 @@ static void write_callback(unsigned long error, void *context) return; } + if (!error && unsup) { + bio->bi_status = BLK_STS_INVAL; + bio_endio(bio); + return; + } + for (i = 0; i < ms->nr_mirrors; i++) if (test_bit(i, &error)) fail_mirror(ms->mirror + i, DM_RAID1_WRITE_ERROR); @@ -680,7 +688,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio) */ bio_set_m(bio, get_default_mirror(ms)); - BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, IOPRIO_DEFAULT)); + BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, NULL, IOPRIO_DEFAULT)); } static void do_writes(struct mirror_set *ms, struct bio_list *writes) @@ -1262,7 +1270,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, return DM_ENDIO_DONE; } - if (*error == BLK_STS_NOTSUPP) + if (*error == BLK_STS_NOTSUPP || *error == BLK_STS_INVAL) goto out; if (bio->bi_opf & REQ_RAHEAD) diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c index aa239ccda270e..e86d929b0e951 100644 --- a/drivers/md/dm-snap-persistent.c +++ b/drivers/md/dm-snap-persistent.c @@ -223,7 +223,7 @@ static void do_metadata(struct work_struct *work) { struct mdata_req *req = container_of(work, struct mdata_req, work); - req->result = dm_io(req->io_req, 1, req->where, NULL, IOPRIO_DEFAULT); + req->result = dm_io(req->io_req, 1, req->where, NULL, NULL, IOPRIO_DEFAULT); } /* @@ -247,7 +247,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, blk_opf_t opf, struct mdata_req req; if (!metadata) - return dm_io(&io_req, 1, &where, NULL, IOPRIO_DEFAULT); + return dm_io(&io_req, 1, &where, NULL, NULL, IOPRIO_DEFAULT); req.where = &where; req.io_req = &io_req; diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c index 5df710061a116..beabbe3b39f3b 100644 --- a/drivers/md/dm-stats.c +++ b/drivers/md/dm-stats.c @@ -178,8 +178,10 @@ static void dm_stat_free(struct rcu_head *head) kfree(s->program_id); kfree(s->aux_data); for_each_possible_cpu(cpu) { - dm_kvfree(s->stat_percpu[cpu][0].histogram, s->histogram_alloc_size); - dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size); + if (s->stat_percpu[cpu]) { + dm_kvfree(s->stat_percpu[cpu][0].histogram, s->histogram_alloc_size); + dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size); + } } dm_kvfree(s->stat_shared[0].tmp.histogram, s->histogram_alloc_size); dm_kvfree(s, s->shared_alloc_size); diff --git a/drivers/md/dm-switch.c b/drivers/md/dm-switch.c index 5952f02de1e62..e5b507b4fa7bf 100644 --- a/drivers/md/dm-switch.c +++ b/drivers/md/dm-switch.c @@ -184,7 +184,7 @@ static void switch_region_table_write(struct switch_ctx *sctx, unsigned long reg pte = sctx->region_table[region_index]; pte &= ~((((region_table_slot_t)1 << sctx->region_table_entry_bits) - 1) << bit); pte |= (region_table_slot_t)value << bit; - sctx->region_table[region_index] = pte; + WRITE_ONCE(sctx->region_table[region_index], pte); } /* diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 1b07630912540..c67dea40c9cd3 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -395,7 +395,7 @@ static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io, io_loc.bdev = v->data_dev->bdev; io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT); io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT); - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) goto free_ret; diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index 493f5202ad042..f02305de1c882 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -474,12 +474,14 @@ struct io_notify { atomic_t count; }; -static void writecache_notify_io(unsigned long error, void *context) +static void writecache_notify_io(unsigned long error, unsigned long unsup, void *context) { struct io_notify *endio = context; if (unlikely(error != 0)) writecache_error(endio->wc, -EIO, "error writing metadata"); + else if (unlikely(unsup != 0)) + writecache_error(endio->wc, -EOPNOTSUPP, "error writing metadata"); BUG_ON(atomic_read(&endio->count) <= 0); if (atomic_dec_and_test(&endio->count)) complete(&endio->c); @@ -530,11 +532,11 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) req.notify.context = &endio; /* writing via async dm-io (implied by notify.fn above) won't return an error */ - (void) dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); + (void) dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); i = j; } - writecache_notify_io(0, &endio); + writecache_notify_io(0, 0, &endio); wait_for_completion_io(&endio.c); if (wait_for_ios) @@ -567,7 +569,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc) req.notify.fn = NULL; req.notify.context = NULL; - r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); + r = dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) writecache_error(wc, r, "error writing superblock"); } @@ -595,7 +597,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev) req.client = wc->dm_io; req.notify.fn = NULL; - r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); + r = dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) writecache_error(wc, r, "error flushing metadata: %d", r); } @@ -989,7 +991,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors req.client = wc->dm_io; req.notify.fn = NULL; - return dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); + return dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); } static void writecache_resume(struct dm_target *ti) diff --git a/drivers/md/dm.c b/drivers/md/dm.c index d413bfaf35278..eb9e32995b79c 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -2630,9 +2630,10 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) */ mutex_lock(&md->table_devices_lock); r = add_disk(md->disk); - mutex_unlock(&md->table_devices_lock); - if (r) + if (r) { + mutex_unlock(&md->table_devices_lock); return r; + } /* * Register the holder relationship for devices added before the disk @@ -2643,18 +2644,21 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) if (r) goto out_undo_holders; } + mutex_unlock(&md->table_devices_lock); r = dm_sysfs_init(md); if (r) - goto out_undo_holders; + goto lock_out_undo_holders; md->type = type; + return 0; +lock_out_undo_holders: + mutex_lock(&md->table_devices_lock); out_undo_holders: list_for_each_entry_continue_reverse(td, &md->table_devices, list) bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); - mutex_lock(&md->table_devices_lock); del_gendisk(md->disk); mutex_unlock(&md->table_devices_lock); return r; @@ -3140,7 +3144,7 @@ int dm_resume(struct mapped_device *md) r = -EINVAL; mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); - if (!dm_suspended_md(md)) + if (!dm_suspended_md(md) || test_bit(DMF_FREEING, &md->flags)) goto out; if (dm_suspended_internally_md(md)) { diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 0f02e2956398d..7e4fbca93ccb2 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2064,23 +2064,18 @@ static void bitmap_end_behind_write(struct mddev *mddev) bitmap->mddev->bitmap_info.max_write_behind); } -static bool bitmap_wait_behind_writes(struct mddev *mddev, bool nowait) +static void bitmap_wait_behind_writes(struct mddev *mddev) { struct bitmap *bitmap = mddev->bitmap; /* wait for behind writes to complete */ if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { - if (nowait) - return false; - pr_debug("md:%s: behind writes in progress - waiting to stop.\n", mdname(mddev)); /* need to kick something here to make sure I/O goes? */ wait_event(bitmap->behind_wait, atomic_read(&bitmap->behind_writes) == 0); } - - return true; } static void bitmap_destroy(struct mddev *mddev) @@ -2090,7 +2085,7 @@ static void bitmap_destroy(struct mddev *mddev) if (!bitmap) /* there was no bitmap */ return; - bitmap_wait_behind_writes(mddev, false); + bitmap_wait_behind_writes(mddev); if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags)) mddev_destroy_serial_pool(mddev, NULL); @@ -2629,10 +2624,12 @@ static ssize_t location_store(struct mddev *mddev, const char *buf, size_t len) { int rv; + unsigned int noio_flags; rv = mddev_suspend_and_lock(mddev); if (rv) return rv; + noio_flags = memalloc_noio_save(); if (mddev->pers) { if (mddev->recovery || mddev->sync_thread) { @@ -2719,6 +2716,7 @@ location_store(struct mddev *mddev, const char *buf, size_t len) } rv = 0; out: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); if (rv) return rv; @@ -2862,7 +2860,7 @@ backlog_store(struct mddev *mddev, const char *buf, size_t len) if (!has_write_mostly) { pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n", mdname(mddev)); - mddev_unlock(mddev); + mddev_unlock_and_resume(mddev); return -EINVAL; } diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index f46674bdfeb91..890276d9c66ef 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -29,6 +29,7 @@ enum bitmap_state { BITMAP_FIRST_USE = 3, /* llbitmap is just created */ BITMAP_CLEAN = 4, /* llbitmap is created with assume_clean */ BITMAP_DAEMON_BUSY = 5, /* llbitmap daemon is not finished after daemon_sleep */ + BITMAP_SHUTDOWN = 6, /* llbitmap is being destroyed */ BITMAP_HOSTENDIAN =15, }; @@ -98,7 +99,7 @@ struct bitmap_operations { void (*start_behind_write)(struct mddev *mddev); void (*end_behind_write)(struct mddev *mddev); - bool (*wait_behind_writes)(struct mddev *mddev, bool nowait); + void (*wait_behind_writes)(struct mddev *mddev); md_bitmap_fn *start_write; md_bitmap_fn *end_write; diff --git a/drivers/md/md-linear.c b/drivers/md/md-linear.c index fdff250d0d513..73b367b61b873 100644 --- a/drivers/md/md-linear.c +++ b/drivers/md/md-linear.c @@ -71,6 +71,7 @@ static int linear_set_limits(struct mddev *mddev) int err; md_init_stacking_limits(&lim); + lim.features |= BLK_FEAT_NOWAIT; lim.max_hw_sectors = mddev->chunk_sectors; lim.logical_block_size = mddev->logical_block_size; lim.max_write_zeroes_sectors = mddev->chunk_sectors; diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 5a4e2abaa7577..e0bc6c8c94592 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -789,6 +789,7 @@ static enum llbitmap_state llbitmap_state_machine(struct llbitmap *llbitmap, if (state == BitNeedSync || state == BitNeedSyncUnwritten) need_resync = !mddev->degraded; else if (state == BitDirty && + !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags) && !timer_pending(&llbitmap->pending_timer)) mod_timer(&llbitmap->pending_timer, jiffies + mddev->bitmap_info.daemon_sleep * HZ); @@ -981,7 +982,7 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) else mddev->bitmap_info.space = mddev->bitmap_info.default_space; } - llbitmap->flags = le32_to_cpu(sb->state); + llbitmap->flags = le32_to_cpu(sb->state) & ~BIT(BITMAP_SHUTDOWN); if (test_and_clear_bit(BITMAP_FIRST_USE, &llbitmap->flags)) { ret = llbitmap_init(llbitmap); goto out_put_page; @@ -1037,6 +1038,9 @@ static void llbitmap_pending_timer_fn(struct timer_list *pending_timer) struct llbitmap *llbitmap = container_of(pending_timer, struct llbitmap, pending_timer); + if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) + return; + if (work_busy(&llbitmap->daemon_work)) { pr_warn("md/llbitmap: %s daemon_work not finished in %lu seconds\n", mdname(llbitmap->mddev), @@ -1057,6 +1061,9 @@ static void md_llbitmap_daemon_fn(struct work_struct *work) bool restart; int idx; + if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) + return; + if (llbitmap->mddev->degraded) return; retry: @@ -1096,7 +1103,7 @@ static void md_llbitmap_daemon_fn(struct work_struct *work) goto retry; /* If some page is dirty but not expired, setup timer again */ - if (restart) + if (restart && !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) mod_timer(&llbitmap->pending_timer, jiffies + llbitmap->mddev->bitmap_info.daemon_sleep * HZ); } @@ -1126,10 +1133,11 @@ static int llbitmap_create(struct mddev *mddev) mutex_lock(&mddev->bitmap_info.mutex); mddev->bitmap = llbitmap; ret = llbitmap_read_sb(llbitmap); + if (ret) + mddev->bitmap = NULL; mutex_unlock(&mddev->bitmap_info.mutex); if (ret) { kfree(llbitmap); - mddev->bitmap = NULL; } return ret; @@ -1178,7 +1186,9 @@ static void llbitmap_destroy(struct mddev *mddev) mutex_lock(&mddev->bitmap_info.mutex); - timer_delete_sync(&llbitmap->pending_timer); + set_bit(BITMAP_SHUTDOWN, &llbitmap->flags); + timer_shutdown_sync(&llbitmap->pending_timer); + cancel_work_sync(&llbitmap->daemon_work); flush_workqueue(md_llbitmap_io_wq); flush_workqueue(md_llbitmap_unplug_wq); @@ -1517,7 +1527,7 @@ static void llbitmap_update_sb(void *data) sb = kmap_local_page(sb_page); sb->events = cpu_to_le64(mddev->events); - sb->state = cpu_to_le32(llbitmap->flags); + sb->state = cpu_to_le32(llbitmap->flags & ~BIT(BITMAP_SHUTDOWN)); sb->chunksize = cpu_to_le32(llbitmap->chunksize); sb->sync_size = cpu_to_le64(mddev->resync_max_sectors); sb->events_cleared = cpu_to_le64(llbitmap->events_cleared); @@ -1574,19 +1584,13 @@ static void llbitmap_end_behind_write(struct mddev *mddev) wake_up(&llbitmap->behind_wait); } -static bool llbitmap_wait_behind_writes(struct mddev *mddev, bool nowait) +static void llbitmap_wait_behind_writes(struct mddev *mddev) { struct llbitmap *llbitmap = mddev->bitmap; - if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) { - if (nowait) - return false; - + if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) wait_event(llbitmap->behind_wait, atomic_read(&llbitmap->behind_writes) == 0); - } - - return true; } static ssize_t bits_show(struct mddev *mddev, char *page) diff --git a/drivers/md/md.c b/drivers/md/md.c index d1465bcd86c81..b9aef7ba11515 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -233,23 +233,22 @@ static int rdev_need_serial(struct md_rdev *rdev) void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) { int ret = 0; + unsigned int noio_flags; - if (rdev && !rdev_need_serial(rdev) && + if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags) && + rdev && !rdev_need_serial(rdev) && !test_bit(CollisionCheck, &rdev->flags)) return; + noio_flags = memalloc_noio_save(); if (!rdev) ret = rdevs_init_serial(mddev); else ret = rdev_init_serial(rdev); if (ret) - return; + goto out; if (mddev->serial_info_pool == NULL) { - /* - * already in memalloc noio context by - * mddev_suspend() - */ mddev->serial_info_pool = mempool_create_kmalloc_pool(NR_SERIAL_INFOS, sizeof(struct serial_info)); @@ -258,6 +257,8 @@ void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) pr_err("can't alloc memory pool for serialization\n"); } } +out: + memalloc_noio_restore(noio_flags); } /* @@ -516,9 +517,6 @@ int mddev_suspend(struct mddev *mddev, bool interruptible) */ WRITE_ONCE(mddev->suspended, mddev->suspended + 1); - /* restrict memory reclaim I/O during raid array is suspend */ - mddev->noio_flag = memalloc_noio_save(); - mutex_unlock(&mddev->suspend_mutex); return 0; } @@ -535,9 +533,6 @@ static void __mddev_resume(struct mddev *mddev, bool recovery_needed) return; } - /* entred the memalloc scope from mddev_suspend() */ - memalloc_noio_restore(mddev->noio_flag); - percpu_ref_resurrect(&mddev->active_io); wake_up(&mddev->sb_wait); @@ -1920,6 +1915,13 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_ rdev->bb_page, REQ_OP_READ, true)) return -EIO; bbp = (__le64 *)page_address(rdev->bb_page); + + /* check for badblocks api. */ + if (sb->bblog_shift >= BITS_PER_TYPE(sector_t)) { + pr_err("md: %pg: bogus bblog_shift %u for badblocks.\n", + rdev->bdev, sb->bblog_shift); + return -EINVAL; + } rdev->badblocks.shift = sb->bblog_shift; for (i = 0 ; i < (sectors << (9-3)) ; i++, bbp++) { u64 bb = le64_to_cpu(*bbp); @@ -4047,6 +4049,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) char clevel[16]; ssize_t rv; size_t slen = len; + unsigned int noio_flags; struct md_personality *pers, *oldpers; long level; void *priv, *oldpriv; @@ -4058,6 +4061,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) rv = mddev_suspend_and_lock(mddev); if (rv) return rv; + noio_flags = memalloc_noio_save(); if (mddev->pers == NULL) { memcpy(mddev->clevel, buf, slen); @@ -4233,6 +4237,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) md_new_event(); rv = len; out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return rv; } @@ -4412,6 +4417,7 @@ static ssize_t raid_disks_store(struct mddev *mddev, const char *buf, size_t len) { unsigned int n; + unsigned int noio_flags; int err; err = kstrtouint(buf, 10, &n); @@ -4421,6 +4427,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; + noio_flags = memalloc_noio_save(); if (mddev->pers) { if (n != mddev->raid_disks) err = update_raid_disks(mddev, n); @@ -4444,6 +4451,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) } else mddev->raid_disks = n; out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err ? err : len; } @@ -4824,6 +4832,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) int minor; dev_t dev; struct md_rdev *rdev; + unsigned int noio_flags; int err; if (!*buf || *e != ':' || !e[1] || e[1] == '\n') @@ -4839,6 +4848,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; + noio_flags = memalloc_noio_save(); if (mddev->persistent) { rdev = md_import_device(dev, mddev->major_version, mddev->minor_version); @@ -4857,6 +4867,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) rdev = md_import_device(dev, -1, -1); if (IS_ERR(rdev)) { + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return PTR_ERR(rdev); } @@ -4864,6 +4875,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) out: if (err) export_rdev(rdev); + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); if (!err) md_new_event(); @@ -6283,7 +6295,7 @@ void md_init_stacking_limits(struct queue_limits *lim) { blk_set_stacking_limits(lim); lim->features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | - BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT; + BLK_FEAT_IO_STAT; } EXPORT_SYMBOL_GPL(md_init_stacking_limits); @@ -6631,7 +6643,6 @@ int md_run(struct mddev *mddev) int err; struct md_rdev *rdev; struct md_personality *pers; - bool nowait = true; if (list_empty(&mddev->disks)) /* cannot run an array with no devices.. */ @@ -6702,7 +6713,6 @@ int md_run(struct mddev *mddev) } } sysfs_notify_dirent_safe(rdev->sysfs_state); - nowait = nowait && bdev_nowait(rdev->bdev); } pers = get_pers(mddev->level, mddev->clevel); @@ -7050,7 +7060,7 @@ EXPORT_SYMBOL_GPL(md_stop_writes); static void mddev_detach(struct mddev *mddev) { if (md_bitmap_enabled(mddev, false)) - mddev->bitmap_ops->wait_behind_writes(mddev, false); + mddev->bitmap_ops->wait_behind_writes(mddev); if (mddev->pers && mddev->pers->quiesce && !is_md_suspended(mddev)) { mddev->pers->quiesce(mddev, 1); mddev->pers->quiesce(mddev, 0); @@ -7066,8 +7076,8 @@ static void __md_stop(struct mddev *mddev) { struct md_personality *pers = mddev->pers; - md_bitmap_destroy(mddev); mddev_detach(mddev); + md_bitmap_destroy(mddev); spin_lock(&mddev->lock); mddev->pers = NULL; spin_unlock(&mddev->lock); @@ -8324,8 +8334,10 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, unsigned int cmd, unsigned long arg) { int err = 0; + unsigned int noio_flags = 0; void __user *argp = (void __user *)arg; struct mddev *mddev = NULL; + bool suspend; err = md_ioctl_valid(cmd); if (err) @@ -8375,13 +8387,15 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, if (!md_is_rdwr(mddev)) flush_work(&mddev->sync_work); - err = md_ioctl_need_suspend(cmd) ? mddev_suspend_and_lock(mddev) : - mddev_lock(mddev); + suspend = md_ioctl_need_suspend(cmd); + err = suspend ? mddev_suspend_and_lock(mddev) : mddev_lock(mddev); if (err) { pr_debug("md: ioctl lock interrupted, reason %d, cmd %d\n", err, cmd); goto out; } + if (suspend) + noio_flags = memalloc_noio_save(); if (cmd == SET_ARRAY_INFO) { err = __md_set_array_info(mddev, argp); @@ -8506,8 +8520,12 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, err != -EINVAL) mddev->hold_active = 0; - md_ioctl_need_suspend(cmd) ? mddev_unlock_and_resume(mddev) : - mddev_unlock(mddev); + if (suspend) { + memalloc_noio_restore(noio_flags); + mddev_unlock_and_resume(mddev); + } else { + mddev_unlock(mddev); + } out: if (cmd == STOP_ARRAY_RO || (err && cmd == STOP_ARRAY)) @@ -9421,6 +9439,8 @@ static void md_clone_bio(struct mddev *mddev, struct bio **bio) md_io_clone->mddev = mddev; if (blk_queue_io_stat(bdev->bd_disk->queue)) md_io_clone->start_time = bio_start_io_acct(*bio); + else + md_io_clone->start_time = 0; if (bio_data_dir(*bio) == WRITE && md_bitmap_enabled(mddev, false)) { md_io_clone->offset = (*bio)->bi_iter.bi_sector; @@ -10175,19 +10195,34 @@ static void md_start_sync(struct work_struct *ws) struct mddev *mddev = container_of(ws, struct mddev, sync_work); int spares = 0; bool suspend = false; + unsigned int noio_flags = 0; char *name; /* * If reshape is still in progress, spares won't be added or removed * from conf until reshape is done. */ - if (mddev->reshape_position == MaxSector && + if ((mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) && md_spares_need_change(mddev)) { suspend = true; mddev_suspend(mddev, false); + noio_flags = memalloc_noio_save(); } mddev_lock_nointr(mddev); + + /* + * The spare configuration can change before reconfig_mutex is acquired. + * Recheck while holding the lock and suspend if needed. + */ + if (!suspend && (mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) && + md_spares_need_change(mddev)) { + mddev_unlock(mddev); + mddev_suspend_and_lock_nointr(mddev); + suspend = true; + noio_flags = memalloc_noio_save(); + } + if (!md_is_rdwr(mddev)) { /* * On a read-only array we can: @@ -10231,8 +10266,10 @@ static void md_start_sync(struct work_struct *ws) * https://bugzilla.kernel.org/show_bug.cgi?id=218200 * Therefore, use __mddev_resume(mddev, false). */ - if (suspend) + if (suspend) { + memalloc_noio_restore(noio_flags); __mddev_resume(mddev, false); + } md_wakeup_thread(mddev->sync_thread); sysfs_notify_dirent_safe(mddev->sysfs_action); md_new_event(); @@ -10251,8 +10288,10 @@ static void md_start_sync(struct work_struct *ws) * https://bugzilla.kernel.org/show_bug.cgi?id=218200 * Therefore, use __mddev_resume(mddev, false). */ - if (suspend) + if (suspend) { + memalloc_noio_restore(noio_flags); __mddev_resume(mddev, false); + } wake_up(&resync_wait); if (test_and_clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery) && diff --git a/drivers/md/md.h b/drivers/md/md.h index d8daf0f75cbbe..76488cd9e81ee 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -621,7 +621,6 @@ struct mddev { struct md_cluster_info *cluster_info; struct md_cluster_operations *cluster_ops; unsigned int good_device_nr; /* good device num within cluster raid */ - unsigned int noio_flag; /* for memalloc scope API */ /* * Temporarily store rdev that will be finally removed when diff --git a/drivers/md/persistent-data/dm-array.c b/drivers/md/persistent-data/dm-array.c index 8f8792e558063..961fa3c1439a5 100644 --- a/drivers/md/persistent-data/dm-array.c +++ b/drivers/md/persistent-data/dm-array.c @@ -38,6 +38,14 @@ struct array_block { */ #define CSUM_XOR 595846735 +/* + * Each array block can hold this many values. + */ +static uint32_t calc_max_entries(size_t value_size, size_t size_of_block) +{ + return (size_of_block - sizeof(struct array_block)) / value_size; +} + static void array_block_prepare_for_write(const struct dm_block_validator *v, struct dm_block *b, size_t size_of_block) @@ -55,6 +63,7 @@ static int array_block_check(const struct dm_block_validator *v, size_t size_of_block) { struct array_block *bh_le = dm_block_data(b); + uint32_t nr_entries, max_entries, value_size; __le32 csum_disk; if (dm_block_location(b) != le64_to_cpu(bh_le->blocknr)) { @@ -74,6 +83,26 @@ static int array_block_check(const struct dm_block_validator *v, return -EILSEQ; } + nr_entries = le32_to_cpu(bh_le->nr_entries); + max_entries = le32_to_cpu(bh_le->max_entries); + value_size = le32_to_cpu(bh_le->value_size); + + if (!value_size) { + DMERR_LIMIT("%s failed: value_size is zero", __func__); + return -EILSEQ; + } + + if (max_entries != calc_max_entries(value_size, size_of_block)) { + DMERR_LIMIT("%s failed: max_entries %u invalid for value_size %u", + __func__, max_entries, value_size); + return -EILSEQ; + } + + if (nr_entries > max_entries) { + DMERR_LIMIT("%s failed: too many entries", __func__); + return -EILSEQ; + } + return 0; } @@ -138,14 +167,6 @@ static void dec_ablock_entries(struct dm_array_info *info, struct array_block *a on_entries(info, ab, vt->dec); } -/* - * Each array block can hold this many values. - */ -static uint32_t calc_max_entries(size_t value_size, size_t size_of_block) -{ - return (size_of_block - sizeof(struct array_block)) / value_size; -} - /* * Allocate a new array block. The caller will need to unlock block. */ @@ -225,6 +246,14 @@ static int get_ablock(struct dm_array_info *info, dm_block_t b, return r; *ab = dm_block_data(*block); + if (le32_to_cpu((*ab)->value_size) != info->value_type.size) { + DMERR_LIMIT("%s failed: value_size %u != wanted %u", __func__, + le32_to_cpu((*ab)->value_size), + info->value_type.size); + dm_tm_unlock(info->btree_info.tm, *block); + return -EILSEQ; + } + return 0; } @@ -287,6 +316,14 @@ static int __shadow_ablock(struct dm_array_info *info, dm_block_t b, return r; *ab = dm_block_data(*block); + if (le32_to_cpu((*ab)->value_size) != info->value_type.size) { + DMERR_LIMIT("%s failed: value_size %u != wanted %u", __func__, + le32_to_cpu((*ab)->value_size), + info->value_type.size); + dm_tm_unlock(info->btree_info.tm, *block); + return -EILSEQ; + } + if (inc) inc_ablock_entries(info, *ab); diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index 2c000b3a5f491..35e103f0c2c3e 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -385,6 +385,7 @@ static int raid0_set_limits(struct mddev *mddev) int err; md_init_stacking_limits(&lim); + lim.features |= BLK_FEAT_NOWAIT; lim.max_hw_sectors = mddev->chunk_sectors; lim.max_write_zeroes_sectors = mddev->chunk_sectors; lim.max_hw_wzeroes_unmap_sectors = mddev->chunk_sectors; diff --git a/drivers/md/raid1-10.c b/drivers/md/raid1-10.c index 56a56a4da4f83..3b0e230692ba9 100644 --- a/drivers/md/raid1-10.c +++ b/drivers/md/raid1-10.c @@ -290,9 +290,8 @@ static inline bool raid1_should_read_first(struct mddev *mddev, } /* - * bio with REQ_RAHEAD or REQ_NOWAIT can fail at anytime, before such IO is - * submitted to the underlying disks, hence don't record badblocks or retry - * in this case. + * bio with REQ_RAHEAD can fail at anytime, before such IO is submitted to the + * underlying disks, hence don't record badblocks or retry in this case. * * BLK_STS_INVAL means the bio was not valid for the underlying device. This * is a user error, not a device failure, so retrying or recording bad blocks @@ -300,6 +299,5 @@ static inline bool raid1_should_read_first(struct mddev *mddev, */ static inline bool raid1_should_handle_error(struct bio *bio) { - return !(bio->bi_opf & (REQ_RAHEAD | REQ_NOWAIT)) && - bio->bi_status != BLK_STS_INVAL; + return !(bio->bi_opf & REQ_RAHEAD) && bio->bi_status != BLK_STS_INVAL; } diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index afe2ca96ad8c2..1c50f363efc08 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1051,10 +1051,8 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr) wake_up(&conf->wait_barrier); } -static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) +static void _wait_barrier(struct r1conf *conf, int idx) { - bool ret = true; - /* * We need to increase conf->nr_pending[idx] very early here, * then raise_barrier() can be blocked when it waits for @@ -1085,7 +1083,7 @@ static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) */ if (!READ_ONCE(conf->array_frozen) && !atomic_read(&conf->barrier[idx])) - return ret; + return; /* * After holding conf->resync_lock, conf->nr_pending[idx] @@ -1104,26 +1102,18 @@ static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) wake_up_barrier(conf); /* Wait for the barrier in same barrier unit bucket to drop. */ - /* Return false when nowait flag is set */ - if (nowait) { - ret = false; - } else { - wait_event_lock_irq(conf->wait_barrier, - !conf->array_frozen && - !atomic_read(&conf->barrier[idx]), - conf->resync_lock); - atomic_inc(&conf->nr_pending[idx]); - } + wait_event_lock_irq(conf->wait_barrier, !conf->array_frozen && + !atomic_read(&conf->barrier[idx]), + conf->resync_lock); + atomic_inc(&conf->nr_pending[idx]); atomic_dec(&conf->nr_waiting[idx]); spin_unlock_irq(&conf->resync_lock); - return ret; } -static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) +static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr) { int idx = sector_to_idx(sector_nr); - bool ret = true; /* * Very similar to _wait_barrier(). The difference is, for read @@ -1135,7 +1125,7 @@ static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowa atomic_inc(&conf->nr_pending[idx]); if (!READ_ONCE(conf->array_frozen)) - return ret; + return; spin_lock_irq(&conf->resync_lock); atomic_inc(&conf->nr_waiting[idx]); @@ -1147,27 +1137,19 @@ static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowa wake_up_barrier(conf); /* Wait for array to be unfrozen */ - /* Return false when nowait flag is set */ - if (nowait) { - /* Return false when nowait flag is set */ - ret = false; - } else { - wait_event_lock_irq(conf->wait_barrier, - !conf->array_frozen, - conf->resync_lock); - atomic_inc(&conf->nr_pending[idx]); - } + wait_event_lock_irq(conf->wait_barrier, !conf->array_frozen, + conf->resync_lock); + atomic_inc(&conf->nr_pending[idx]); atomic_dec(&conf->nr_waiting[idx]); spin_unlock_irq(&conf->resync_lock); - return ret; } -static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) +static void wait_barrier(struct r1conf *conf, sector_t sector_nr) { int idx = sector_to_idx(sector_nr); - return _wait_barrier(conf, idx, nowait); + _wait_barrier(conf, idx); } static void _allow_barrier(struct r1conf *conf, int idx) @@ -1342,7 +1324,6 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, int max_sectors; int rdisk; bool r1bio_existed = !!r1_bio; - bool nowait = bio->bi_opf & REQ_NOWAIT; /* * An md cloned bio indicates we are in the error path. @@ -1362,16 +1343,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, * Still need barrier for READ in case that whole * array is frozen. */ - if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, nowait)) { - bio_wouldblock_error(bio); - - if (r1bio_existed) { - set_bit(R1BIO_Returned, &r1_bio->state); - raid_end_bio_io(r1_bio); - } - - return; - } + wait_read_barrier(conf, bio->bi_iter.bi_sector); if (!r1_bio) r1_bio = alloc_r1bio(mddev, bio); @@ -1406,14 +1378,10 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, md_bitmap_enabled(mddev, false)) { /* * Reading from a write-mostly device must take care not to - * over-take any writes that are 'behind' - */ - mddev_add_trace_msg(mddev, "raid1 wait behind writes"); - if (!mddev->bitmap_ops->wait_behind_writes(mddev, nowait)) { - bio_wouldblock_error(bio); - set_bit(R1BIO_Returned, &r1_bio->state); - goto err_handle; - } + * over-take any writes that are 'behind' + */ + mddev_add_trace_msg(mddev, "raid1 wait behind writes"); + mddev->bitmap_ops->wait_behind_writes(mddev); } if (max_sectors < bio_sectors(bio)) { @@ -1435,7 +1403,6 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, } read_bio = bio_alloc_clone(mirror->rdev->bdev, bio, gfp, &mddev->bio_set); - read_bio->bi_opf &= ~REQ_NOWAIT; r1_bio->bios[rdisk] = read_bio; read_bio->bi_iter.bi_sector = r1_bio->sector + @@ -1454,7 +1421,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, raid_end_bio_io(r1_bio); } -static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio) +static void wait_blocked_rdev(struct mddev *mddev, struct bio *bio) { struct r1conf *conf = mddev->private; int disks = conf->raid_disks * 2; @@ -1474,9 +1441,6 @@ static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio) set_bit(BlockedBadBlocks, &rdev->flags); if (rdev_blocked(rdev)) { - if (bio->bi_opf & REQ_NOWAIT) - return false; - mddev_add_trace_msg(rdev->mddev, "raid1 wait rdev %d blocked", rdev->raid_disk); atomic_inc(&rdev->nr_pending); @@ -1484,8 +1448,6 @@ static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio) goto retry; } } - - return true; } static void raid1_start_write_behind(struct mddev *mddev, struct r1bio *r1_bio, @@ -1521,18 +1483,13 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, unsigned long flags; int first_clone; bool write_behind = false; - bool nowait = bio->bi_opf & REQ_NOWAIT; + bool atomic = bio->bi_opf & REQ_ATOMIC; bool is_discard = op_is_discard(bio->bi_opf); sector_t sector = bio->bi_iter.bi_sector; if (mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, sector, bio_end_sector(bio))) { - - if (nowait) { - bio_wouldblock_error(bio); - return false; - } wait_event_idle(conf->wait_barrier, !mddev->cluster_ops->area_resyncing(mddev, WRITE, sector, @@ -1544,15 +1501,9 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, * thread has put up a bar for new requests. * Continue immediately if no resync is active currently. */ - if (!wait_barrier(conf, sector, nowait)) { - bio_wouldblock_error(bio); - return false; - } + wait_barrier(conf, sector); - if (!wait_blocked_rdev(mddev, bio)) { - bio_wouldblock_error(bio); - goto err_allow_barrier; - } + wait_blocked_rdev(mddev, bio); r1_bio = alloc_r1bio(mddev, bio); r1_bio->sectors = max_sectors; @@ -1579,6 +1530,8 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, */ if (!is_discard && rdev && test_bit(WriteMostly, &rdev->flags)) write_behind = true; + if (atomic && max_sectors > BIO_MAX_VECS * (PAGE_SIZE >> 9)) + write_behind = false; r1_bio->bios[i] = NULL; if (!rdev || test_bit(Faulty, &rdev->flags)) @@ -1604,19 +1557,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, if (is_bad) { int good_sectors; - /* - * We cannot atomically write this, so just - * error in that case. It could be possible to - * atomically write other mirrors, but the - * complexity of supporting that is not worth - * the benefit. - */ - if (bio->bi_opf & REQ_ATOMIC) { - bio->bi_status = BLK_STS_NOTSUPP; - bio_endio(bio); - goto err_dec_pending; - } - good_sectors = first_bad - sector; if (good_sectors < max_sectors) max_sectors = good_sectors; @@ -1637,6 +1577,11 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, max_sectors = min_t(int, max_sectors, BIO_MAX_VECS * (PAGE_SIZE >> 9)); if (max_sectors < bio_sectors(bio)) { + if (atomic) { + bio_io_error(bio); + goto err_dec_pending; + } + bio = bio_submit_split_bioset(bio, max_sectors, &conf->bio_split); if (!bio) @@ -1681,7 +1626,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, wait_for_serialization(rdev, r1_bio); } - mbio->bi_opf &= ~REQ_NOWAIT; r1_bio->bios[i] = mbio; mbio->bi_iter.bi_sector = sector + rdev->data_offset; @@ -1720,8 +1664,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, } free_r1bio(r1_bio); - -err_allow_barrier: allow_barrier(conf, sector); return false; @@ -1850,7 +1792,7 @@ static void close_sync(struct r1conf *conf) int idx; for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) { - _wait_barrier(conf, idx, false); + _wait_barrier(conf, idx); _allow_barrier(conf, idx); } @@ -3228,6 +3170,7 @@ static int raid1_set_limits(struct mddev *mddev) md_init_stacking_limits(&lim); lim.max_write_zeroes_sectors = 0; lim.max_hw_wzeroes_unmap_sectors = 0; + lim.chunk_sectors = BARRIER_UNIT_SECTOR_SIZE; lim.logical_block_size = mddev->logical_block_size; lim.features |= BLK_FEAT_ATOMIC_WRITES; lim.features |= BLK_FEAT_PCI_P2PDMA; @@ -3485,8 +3428,6 @@ static void *raid1_takeover(struct mddev *mddev) mddev->new_chunk_sectors = 0; conf = setup_conf(mddev); if (!IS_ERR(conf)) { - /* Array must appear to be quiesced */ - conf->array_frozen = 1; mddev_clear_unsupported_flags(mddev, UNSUPPORTED_MDDEV_FLAGS); } diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 0a3cfdd3f5df8..8c419033a9bf3 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1002,32 +1002,22 @@ static bool wait_barrier_nolock(struct r10conf *conf) return false; } -static bool wait_barrier(struct r10conf *conf, bool nowait) +static void wait_barrier(struct r10conf *conf) { - bool ret = true; - if (wait_barrier_nolock(conf)) - return true; + return; write_seqlock_irq(&conf->resync_lock); if (conf->barrier) { - /* Return false when nowait flag is set */ - if (nowait) { - ret = false; - } else { - conf->nr_waiting++; - mddev_add_trace_msg(conf->mddev, "raid10 wait barrier"); - wait_event_barrier(conf, stop_waiting_barrier(conf)); - conf->nr_waiting--; - } + conf->nr_waiting++; + mddev_add_trace_msg(conf->mddev, "raid10 wait barrier"); + wait_event_barrier(conf, stop_waiting_barrier(conf)); + conf->nr_waiting--; if (!conf->nr_waiting) wake_up(&conf->wait_barrier); } - /* Only increment nr_pending when we wait */ - if (ret) - atomic_inc(&conf->nr_pending); + atomic_inc(&conf->nr_pending); write_sequnlock_irq(&conf->resync_lock); - return ret; } static void allow_barrier(struct r10conf *conf) @@ -1119,30 +1109,22 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) * currently. * 2. If IO spans the reshape position. Need to wait for reshape to pass. */ -static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf, +static void regular_request_wait(struct mddev *mddev, struct r10conf *conf, struct bio *bio, sector_t sectors) { - /* Bail out if REQ_NOWAIT is set for the bio */ - if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { - bio_wouldblock_error(bio); - return false; - } + wait_barrier(conf); + while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && bio->bi_iter.bi_sector < conf->reshape_progress && bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { allow_barrier(conf); - if (bio->bi_opf & REQ_NOWAIT) { - bio_wouldblock_error(bio); - return false; - } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape"); wait_event(conf->wait_barrier, conf->reshape_progress <= bio->bi_iter.bi_sector || conf->reshape_progress >= bio->bi_iter.bi_sector + sectors); - wait_barrier(conf, false); + wait_barrier(conf); } - return true; } static void raid10_read_request(struct mddev *mddev, struct bio *bio, @@ -1191,10 +1173,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, } } - if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) { - free_r10bio(r10_bio); - return; - } + regular_request_wait(mddev, conf, bio, r10_bio->sectors); rdev = read_balance(conf, r10_bio, &max_sectors); if (!rdev) { @@ -1215,7 +1194,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, allow_barrier(conf); bio = bio_submit_split_bioset(bio, max_sectors, &conf->bio_split); - wait_barrier(conf, false); + wait_barrier(conf); if (!bio) { set_bit(R10BIO_Returned, &r10_bio->state); goto err_handle; @@ -1231,7 +1210,6 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, r10_bio->master_bio = bio; } read_bio = bio_alloc_clone(rdev->bdev, bio, gfp, &mddev->bio_set); - read_bio->bi_opf &= ~REQ_NOWAIT; r10_bio->devs[slot].bio = read_bio; r10_bio->devs[slot].rdev = rdev; @@ -1265,7 +1243,6 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, conf->mirrors[devnum].rdev; mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, &mddev->bio_set); - mbio->bi_opf &= ~REQ_NOWAIT; if (replacement) r10_bio->devs[n_copy].repl_bio = mbio; else @@ -1344,7 +1321,7 @@ static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio) "raid10 %s wait rdev %d blocked", __func__, blocked_rdev->raid_disk); md_wait_for_blocked_rdev(blocked_rdev, mddev); - wait_barrier(conf, false); + wait_barrier(conf); goto retry_wait; } } @@ -1356,33 +1333,20 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, int i, k; sector_t sectors; int max_sectors; + bool atomic = bio->bi_opf & REQ_ATOMIC; if ((mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, bio->bi_iter.bi_sector, bio_end_sector(bio)))) { - DEFINE_WAIT(w); - /* Bail out if REQ_NOWAIT is set for the bio */ - if (bio->bi_opf & REQ_NOWAIT) { - bio_wouldblock_error(bio); - return false; - } - for (;;) { - prepare_to_wait(&conf->wait_barrier, - &w, TASK_IDLE); - if (!mddev->cluster_ops->area_resyncing(mddev, WRITE, - bio->bi_iter.bi_sector, bio_end_sector(bio))) - break; - schedule(); - } - finish_wait(&conf->wait_barrier, &w); + wait_event_idle(conf->wait_barrier, + !mddev->cluster_ops->area_resyncing(mddev, WRITE, + bio->bi_iter.bi_sector, + bio_end_sector(bio))); } sectors = r10_bio->sectors; - if (!regular_request_wait(mddev, conf, bio, sectors)) { - free_r10bio(r10_bio); - return false; - } + regular_request_wait(mddev, conf, bio, sectors); if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && (mddev->reshape_backwards @@ -1395,11 +1359,6 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, set_mask_bits(&mddev->sb_flags, 0, BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); md_wakeup_thread(mddev->thread); - if (bio->bi_opf & REQ_NOWAIT) { - allow_barrier(conf); - bio_wouldblock_error(bio); - return false; - } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape metadata"); wait_event(mddev->sb_wait, @@ -1462,16 +1421,6 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, if (is_bad) { int good_sectors; - /* - * We cannot atomically write this, so just - * error in that case. It could be possible to - * atomically write other mirrors, but the - * complexity of supporting that is not worth - * the benefit. - */ - if (bio->bi_opf & REQ_ATOMIC) - goto err_handle; - good_sectors = first_bad - dev_sector; if (good_sectors < max_sectors) max_sectors = good_sectors; @@ -1491,10 +1440,13 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, r10_bio->sectors = max_sectors; if (r10_bio->sectors < bio_sectors(bio)) { + if (atomic) + goto err_handle; + allow_barrier(conf); bio = bio_submit_split_bioset(bio, r10_bio->sectors, &conf->bio_split); - wait_barrier(conf, false); + wait_barrier(conf); if (!bio) { set_bit(R10BIO_Returned, &r10_bio->state); goto err_handle; @@ -1637,11 +1589,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) return -EAGAIN; - if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { - bio_wouldblock_error(bio); - md_write_end(mddev); - return 0; - } + wait_barrier(conf); /* * Check reshape again to avoid reshape happens after checking @@ -1692,7 +1640,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) allow_barrier(conf); /* Resend the fist split part */ submit_bio_noacct(split); - wait_barrier(conf, false); + wait_barrier(conf); } div_u64_rem(bio_end, stripe_size, &remainder); if (remainder) { @@ -1712,7 +1660,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) /* Resend the second split part */ submit_bio_noacct(bio); bio = split; - wait_barrier(conf, false); + wait_barrier(conf); } bio_start = bio->bi_iter.bi_sector; @@ -1870,7 +1818,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) end_disk_offset += geo->stride; atomic_inc(&first_r10bio->remaining); raid_end_discard_bio(r10_bio); - wait_barrier(conf, false); + wait_barrier(conf); goto retry_discard; } @@ -2069,7 +2017,7 @@ static void print_conf(struct r10conf *conf) static void close_sync(struct r10conf *conf) { - wait_barrier(conf, false); + wait_barrier(conf); allow_barrier(conf); mempool_exit(&conf->r10buf_pool); @@ -3365,7 +3313,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, struct md_rdev *rdev = conf->mirrors[j].rdev; if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { - still_degraded = false; + still_degraded = true; break; } } @@ -4702,7 +4650,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, if (need_flush || time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { /* Need to update reshape_position in metadata */ - wait_barrier(conf, false); + wait_barrier(conf); mddev->reshape_position = conf->reshape_progress; if (mddev->reshape_backwards) mddev->curr_resync_completed = raid10_size(mddev, 0, 0) diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index 7be1648c4e4f0..7f8a9d3fd578d 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -643,8 +643,10 @@ static void ppl_do_flush(struct ppl_io_unit *io) log->disk_flush_bitmap = 0; for (i = flushed_disks ; i < raid_disks; i++) { - if (atomic_dec_and_test(&io->pending_flushes)) + if (atomic_dec_and_test(&io->pending_flushes)) { ppl_io_unit_finished(io); + break; + } } } diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index ffb5fcde54a98..d236a0d772d14 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -801,7 +801,7 @@ static bool is_inactive_blocked(struct r5conf *conf, int hash) return true; return (atomic_read(&conf->active_stripes) < - (conf->max_nr_stripes * 3 / 4)); + (READ_ONCE(conf->max_nr_stripes) * 3 / 4)); } struct stripe_head *raid5_get_active_stripe(struct r5conf *conf, @@ -2471,11 +2471,6 @@ static int scribble_alloc(struct raid5_percpu *percpu, sizeof(unsigned int) * (num + 2); void *scribble; - /* - * If here is in raid array suspend context, it is in memalloc noio - * context as well, there is no potential recursive memory reclaim - * I/Os with the GFP_KERNEL flag. - */ scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL); if (!scribble) return -ENOMEM; @@ -2490,6 +2485,7 @@ static int scribble_alloc(struct raid5_percpu *percpu, static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) { unsigned long cpu; + unsigned int noio_flags; int err = 0; /* Never shrink. */ @@ -2498,6 +2494,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) return 0; raid5_quiesce(conf->mddev, true); + noio_flags = memalloc_noio_save(); cpus_read_lock(); for_each_present_cpu(cpu) { @@ -2511,6 +2508,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) } cpus_read_unlock(); + memalloc_noio_restore(noio_flags); raid5_quiesce(conf->mddev, false); if (!err) { @@ -2785,6 +2783,7 @@ static void raid5_end_read_request(struct bio * bi) } else { int retry = 0; int set_bad = 0; + int max_nr_stripes = READ_ONCE(conf->max_nr_stripes); clear_bit(R5_UPTODATE, &sh->dev[i].flags); if (!(bi->bi_status == BLK_STS_PROTECTION)) @@ -2810,13 +2809,12 @@ static void raid5_end_read_request(struct bio * bi) mdname(conf->mddev), (unsigned long long)s, rdev->bdev); - } else if (atomic_read(&rdev->read_errors) - > conf->max_nr_stripes) { + } else if (atomic_read(&rdev->read_errors) > max_nr_stripes) { if (!test_bit(Faulty, &rdev->flags)) { pr_warn("md/raid:%s: %d read_errors > %d stripes\n", mdname(conf->mddev), atomic_read(&rdev->read_errors), - conf->max_nr_stripes); + max_nr_stripes); pr_warn("md/raid:%s: Too many read errors, failing device %pg.\n", mdname(conf->mddev), rdev->bdev); } @@ -3553,7 +3551,7 @@ static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi, sh->dev[dd_idx].sector); if (conf->mddev->bitmap && firstwrite && !sh->batch_head) { - sh->bm_seq = conf->seq_flush+1; + sh->bm_seq = READ_ONCE(conf->seq_flush) + 1; set_bit(STRIPE_BIT_DELAY, &sh->state); } } @@ -3751,11 +3749,10 @@ static int want_replace(struct stripe_head *sh, int disk_idx) int rv = 0; rdev = sh->raid_conf->disks[disk_idx].replacement; - if (rdev - && !test_bit(Faulty, &rdev->flags) - && !test_bit(In_sync, &rdev->flags) - && (rdev->recovery_offset <= sh->sector - || rdev->mddev->resync_offset <= sh->sector)) + if (rdev && !test_bit(Faulty, &rdev->flags) && + !test_bit(In_sync, &rdev->flags) && + (READ_ONCE(rdev->recovery_offset) <= sh->sector || + rdev->mddev->resync_offset <= sh->sector)) rv = 1; return rv; } @@ -4672,7 +4669,8 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) */ rdev = conf->disks[i].replacement; if (rdev && !test_bit(Faulty, &rdev->flags) && - rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) && + READ_ONCE(rdev->recovery_offset) >= + sh->sector + RAID5_STRIPE_SECTORS(conf) && !rdev_has_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf))) set_bit(R5_ReadRepl, &dev->flags); @@ -4714,7 +4712,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) } else if (test_bit(In_sync, &rdev->flags)) set_bit(R5_Insync, &dev->flags); else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= - rdev->recovery_offset) { + READ_ONCE(rdev->recovery_offset)) { /* * in sync if: * - normal IO, or @@ -5458,13 +5456,13 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio) rdev = conf->disks[dd_idx].replacement; if (!rdev || test_bit(Faulty, &rdev->flags) || - rdev->recovery_offset < end_sector) { + READ_ONCE(rdev->recovery_offset) < end_sector) { rdev = conf->disks[dd_idx].rdev; if (!rdev) return 0; if (test_bit(Faulty, &rdev->flags) || !(test_bit(In_sync, &rdev->flags) || - rdev->recovery_offset >= end_sector)) + READ_ONCE(rdev->recovery_offset) >= end_sector)) return 0; } @@ -5718,10 +5716,6 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) struct bio *orig_bi = bi; int stripe_sectors; - /* We need to handle this when io_uring supports discard/trim */ - if (WARN_ON_ONCE(bi->bi_opf & REQ_NOWAIT)) - return; - if (mddev->reshape_position != MaxSector) /* Skip discard while reshape is happening */ return; @@ -5799,7 +5793,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) } spin_unlock_irq(&sh->stripe_lock); if (conf->mddev->bitmap) { - sh->bm_seq = conf->seq_flush + 1; + sh->bm_seq = READ_ONCE(conf->seq_flush) + 1; set_bit(STRIPE_BIT_DELAY, &sh->state); } @@ -5956,8 +5950,11 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, sectors_per_chunk = conf->chunk_sectors * (conf->raid_disks - conf->max_degraded); - start = round_down(start, sectors_per_chunk); - end = round_up(end, sectors_per_chunk); + sector_div(start, sectors_per_chunk); + start *= sectors_per_chunk; + if (sector_div(end, sectors_per_chunk)) + end++; + end *= sectors_per_chunk; start = raid5_compute_sector(conf, start, 0, &dd_idx, NULL); end = raid5_compute_sector(conf, end, 0, &dd_idx, NULL); @@ -5975,8 +5972,10 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, sectors_per_chunk = conf->prev_chunk_sectors * (conf->previous_raid_disks - conf->max_degraded); - prev_start = round_down(prev_start, sectors_per_chunk); - prev_end = round_down(prev_end, sectors_per_chunk); + sector_div(prev_start, sectors_per_chunk); + prev_start *= sectors_per_chunk; + sector_div(prev_end, sectors_per_chunk); + prev_end *= sectors_per_chunk; prev_start = raid5_compute_sector(conf, prev_start, 1, &dd_idx, NULL); prev_end = raid5_compute_sector(conf, prev_end, 1, &dd_idx, NULL); @@ -6191,15 +6190,6 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) pr_debug("raid456: %s, logical %llu to %llu\n", __func__, bi->bi_iter.bi_sector, ctx->last_sector); - /* Bail out if conflicts with reshape and REQ_NOWAIT is set */ - if ((bi->bi_opf & REQ_NOWAIT) && - get_reshape_loc(mddev, conf, logical_sector) == LOC_INSIDE_RESHAPE) { - bio_wouldblock_error(bi); - if (rw == WRITE) - md_write_end(mddev); - mempool_free(ctx, conf->ctx_pool); - return true; - } md_account_bio(mddev, &bi); /* @@ -6427,8 +6417,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && - rdev->recovery_offset < sector_nr) - rdev->recovery_offset = sector_nr; + READ_ONCE(rdev->recovery_offset) < sector_nr) + WRITE_ONCE(rdev->recovery_offset, sector_nr); conf->reshape_checkpoint = jiffies; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); @@ -6536,8 +6526,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && - rdev->recovery_offset < sector_nr) - rdev->recovery_offset = sector_nr; + READ_ONCE(rdev->recovery_offset) < sector_nr) + WRITE_ONCE(rdev->recovery_offset, sector_nr); conf->reshape_checkpoint = jiffies; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); md_wakeup_thread(mddev->thread); @@ -6849,12 +6839,14 @@ static void raid5d(struct md_thread *thread) if ( !list_empty(&conf->bitmap_list)) { /* Now is a good time to flush some bitmap updates */ - conf->seq_flush++; + int seq = conf->seq_flush + 1; + + WRITE_ONCE(conf->seq_flush, seq); spin_unlock_irq(&conf->device_lock); if (md_bitmap_enabled(mddev, true)) mddev->bitmap_ops->unplug(mddev, true); spin_lock_irq(&conf->device_lock); - conf->seq_write = conf->seq_flush; + conf->seq_write = seq; activate_bit_delay(conf, conf->temp_inactive_list); } raid5_activate_delayed(conf); @@ -7039,6 +7031,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) { struct r5conf *conf; unsigned long new; + unsigned int noio_flags = 0; int err; int size; @@ -7079,6 +7072,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) goto out_unlock; } + noio_flags = memalloc_noio_save(); mutex_lock(&conf->cache_size_mutex); size = conf->max_nr_stripes; @@ -7095,6 +7089,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) mutex_unlock(&conf->cache_size_mutex); out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err ?: len; } @@ -8056,9 +8051,9 @@ static int raid5_run(struct mddev *mddev) /* Hack because v0.91 doesn't store recovery_offset properly. */ if (mddev->major_version == 0 && mddev->minor_version > 90) - rdev->recovery_offset = reshape_offset; + WRITE_ONCE(rdev->recovery_offset, reshape_offset); - if (rdev->recovery_offset < reshape_offset) { + if (READ_ONCE(rdev->recovery_offset) < reshape_offset) { /* We need to check old and new layout */ if (!only_parity(rdev->raid_disk, conf->algorithm, @@ -8213,10 +8208,10 @@ static int raid5_spare_active(struct mddev *mddev) for (i = 0; i < conf->raid_disks; i++) { rdev = conf->disks[i].rdev; replacement = conf->disks[i].replacement; - if (replacement - && replacement->recovery_offset == MaxSector - && !test_bit(Faulty, &replacement->flags) - && !test_and_set_bit(In_sync, &replacement->flags)) { + if (replacement && + READ_ONCE(replacement->recovery_offset) == MaxSector && + !test_bit(Faulty, &replacement->flags) && + !test_and_set_bit(In_sync, &replacement->flags)) { /* Replacement has just become active. */ if (!rdev || !test_and_clear_bit(In_sync, &rdev->flags)) @@ -8231,10 +8226,10 @@ static int raid5_spare_active(struct mddev *mddev) rdev->sysfs_state); } sysfs_notify_dirent_safe(replacement->sysfs_state); - } else if (rdev - && rdev->recovery_offset == MaxSector - && !test_bit(Faulty, &rdev->flags) - && !test_and_set_bit(In_sync, &rdev->flags)) { + } else if (rdev && + READ_ONCE(rdev->recovery_offset) == MaxSector && + !test_bit(Faulty, &rdev->flags) && + !test_and_set_bit(In_sync, &rdev->flags)) { count++; sysfs_notify_dirent_safe(rdev->sysfs_state); } @@ -8603,7 +8598,7 @@ static int raid5_start_reshape(struct mddev *mddev) >= conf->previous_raid_disks) set_bit(In_sync, &rdev->flags); else - rdev->recovery_offset = 0; + WRITE_ONCE(rdev->recovery_offset, 0); /* Failure here is OK */ sysfs_link_rdev(mddev, rdev); @@ -8655,7 +8650,7 @@ static void end_reshape(struct r5conf *conf) if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags)) - rdev->recovery_offset = MaxSector; + WRITE_ONCE(rdev->recovery_offset, MaxSector); spin_unlock_irq(&conf->device_lock); wake_up(&conf->wait_for_reshape); @@ -8975,6 +8970,7 @@ static void *raid6_takeover(struct mddev *mddev) static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) { struct r5conf *conf; + unsigned int noio_flags; int err; err = mddev_suspend_and_lock(mddev); @@ -8986,6 +8982,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) return -ENODEV; } + noio_flags = memalloc_noio_save(); if (strncmp(buf, "ppl", 3) == 0) { /* ppl only works with RAID 5 */ if (!raid5_has_ppl(conf) && conf->level == 5) { @@ -9025,6 +9022,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) if (!err) md_update_sb(mddev, 1); + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err; diff --git a/drivers/media/cec/core/cec-adap.c b/drivers/media/cec/core/cec-adap.c index a90cb84a4b4d0..de0c4fcd8dfeb 100644 --- a/drivers/media/cec/core/cec-adap.c +++ b/drivers/media/cec/core/cec-adap.c @@ -965,7 +965,7 @@ int cec_transmit_msg_fh(struct cec_adapter *adap, struct cec_msg *msg, */ mutex_unlock(&adap->lock); err = wait_for_completion_killable(&data->c); - cancel_delayed_work_sync(&data->work); + disable_delayed_work_sync(&data->work); mutex_lock(&adap->lock); if (err) @@ -2219,9 +2219,13 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg, * Unprocessed messages are aborted if userspace isn't doing * any processing either. */ + mutex_lock(&adap->lock); if (!is_broadcast && !is_reply && !adap->follower_cnt && - !adap->cec_follower && msg->msg[1] != CEC_MSG_FEATURE_ABORT) + !adap->cec_follower && msg->msg[1] != CEC_MSG_FEATURE_ABORT) { + mutex_unlock(&adap->lock); return cec_feature_abort(adap, msg); + } + mutex_unlock(&adap->lock); break; } @@ -2234,10 +2238,12 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg, * Send to the exclusive follower if there is one, otherwise send * to all followers. */ + mutex_lock(&adap->lock); if (adap->cec_follower) cec_queue_msg_fh(adap->cec_follower, msg); else cec_queue_msg_followers(adap, msg); + mutex_unlock(&adap->lock); return 0; } diff --git a/drivers/media/cec/core/cec-core.c b/drivers/media/cec/core/cec-core.c index 0fcd3b5e60c8d..90a98f322f5ee 100644 --- a/drivers/media/cec/core/cec-core.c +++ b/drivers/media/cec/core/cec-core.c @@ -371,9 +371,7 @@ void cec_unregister_adapter(struct cec_adapter *adap) return; #ifdef CONFIG_MEDIA_CEC_RC - /* Note: rc_unregister also calls rc_free */ rc_unregister_device(adap->rc); - adap->rc = NULL; #endif debugfs_remove_recursive(adap->cec_dir); #ifdef CONFIG_CEC_NOTIFIER diff --git a/drivers/media/cec/core/cec-pin.c b/drivers/media/cec/core/cec-pin.c index 6e1c391028322..085fc12067afa 100644 --- a/drivers/media/cec/core/cec-pin.c +++ b/drivers/media/cec/core/cec-pin.c @@ -115,7 +115,7 @@ static void cec_pin_update(struct cec_pin *pin, bool v, bool force) return; pin->adap->cec_pin_is_high = v; - if (atomic_read(&pin->work_pin_num_events) < CEC_NUM_PIN_EVENTS) { + if (atomic_read_acquire(&pin->work_pin_num_events) < CEC_NUM_PIN_EVENTS) { u8 ev = v; if (pin->work_pin_events_dropped) { @@ -126,7 +126,7 @@ static void cec_pin_update(struct cec_pin *pin, bool v, bool force) pin->work_pin_ts[pin->work_pin_events_wr] = ktime_get(); pin->work_pin_events_wr = (pin->work_pin_events_wr + 1) % CEC_NUM_PIN_EVENTS; - atomic_inc(&pin->work_pin_num_events); + atomic_inc_return_release(&pin->work_pin_num_events); } else { pin->work_pin_events_dropped = true; pin->work_pin_events_dropped_cnt++; @@ -1101,7 +1101,7 @@ static int cec_pin_thread_func(void *_adap) pin->work_tx_ts); } - while (atomic_read(&pin->work_pin_num_events)) { + while (atomic_read_acquire(&pin->work_pin_num_events)) { unsigned int idx = pin->work_pin_events_rd; u8 v = pin->work_pin_events[idx]; @@ -1110,7 +1110,7 @@ static int cec_pin_thread_func(void *_adap) v & CEC_PIN_EVENT_FL_DROPPED, pin->work_pin_ts[idx]); pin->work_pin_events_rd = (idx + 1) % CEC_NUM_PIN_EVENTS; - atomic_dec(&pin->work_pin_num_events); + atomic_dec_return_release(&pin->work_pin_num_events); } switch (atomic_xchg(&pin->work_irq_change, diff --git a/drivers/media/cec/platform/meson/ao-cec-g12a.c b/drivers/media/cec/platform/meson/ao-cec-g12a.c index 41f5b8669cb01..2c914f0008b50 100644 --- a/drivers/media/cec/platform/meson/ao-cec-g12a.c +++ b/drivers/media/cec/platform/meson/ao-cec-g12a.c @@ -405,6 +405,7 @@ static int meson_ao_cec_g12a_write(void *context, unsigned int addr, } static const struct regmap_config meson_ao_cec_g12a_cec_regmap_conf = { + .name = "core", .reg_bits = 8, .val_bits = 8, .reg_read = meson_ao_cec_g12a_read, diff --git a/drivers/media/cec/platform/stm32/stm32-cec.c b/drivers/media/cec/platform/stm32/stm32-cec.c index 1ec0cece0a5b7..8c2fc232202de 100644 --- a/drivers/media/cec/platform/stm32/stm32-cec.c +++ b/drivers/media/cec/platform/stm32/stm32-cec.c @@ -132,7 +132,8 @@ static void stm32_rx_done(struct stm32_cec *cec, u32 status) u32 val; regmap_read(cec->regmap, CEC_RXDR, &val); - cec->rx_msg.msg[cec->rx_msg.len++] = val & 0xFF; + if (cec->rx_msg.len < CEC_MAX_MSG_SIZE) + cec->rx_msg.msg[cec->rx_msg.len++] = val & 0xFF; } if (cec->irq_status & RXEND) { diff --git a/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c b/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c index 3381d86096a1f..3c6ce6f3d93e7 100644 --- a/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c +++ b/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c @@ -657,7 +657,8 @@ static void extron_process_received(struct extron_port *port, const char *data) if (!port || port->disconnected) return; - if (len < 5 || (len - 2) % 3 || data[len - 2] != '*') + if (len < 5 || ((len - 2) / 3 > sizeof(msg.msg)) || + (len - 2) % 3 || data[len - 2] != '*') goto malformed; while (*data != '*') { diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.c b/drivers/media/dvb-frontends/rtl2832_sdr.c index c564485e3bbb5..0330e7f0881a7 100644 --- a/drivers/media/dvb-frontends/rtl2832_sdr.c +++ b/drivers/media/dvb-frontends/rtl2832_sdr.c @@ -906,9 +906,12 @@ static int rtl2832_sdr_start_streaming(struct vb2_queue *vq, unsigned int count) goto err; mutex_unlock(&dev->v4l2_lock); + return 0; err: + rtl2832_sdr_free_urbs(dev); + rtl2832_sdr_free_stream_bufs(dev); rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED); mutex_unlock(&dev->v4l2_lock); @@ -1477,14 +1480,22 @@ static void rtl2832_sdr_remove(struct platform_device *pdev) dev_dbg(&pdev->dev, "\n"); - mutex_lock(&dev->vb_queue_lock); + /* + * vb2_video_unregister_device() releases the vb2 queue, which + * triggers rtl2832_sdr_stop_streaming() if streaming is active. + * stop_streaming() uses dev->udev to free URBs and coherent DMA + * stream buffers via usb_free_coherent(), so it must run before + * dev->udev is cleared. vb2_video_unregister_device() locks + * vb_queue_lock internally and stop_streaming() locks v4l2_lock, + * so neither may be held by the caller. + */ + v4l2_device_disconnect(&dev->v4l2_dev); + vb2_video_unregister_device(&dev->vdev); + mutex_lock(&dev->v4l2_lock); - /* No need to keep the urbs around after disconnection */ dev->udev = NULL; - v4l2_device_disconnect(&dev->v4l2_dev); - video_unregister_device(&dev->vdev); mutex_unlock(&dev->v4l2_lock); - mutex_unlock(&dev->vb_queue_lock); + v4l2_device_put(&dev->v4l2_dev); module_put(pdev->dev.parent->driver->owner); } diff --git a/drivers/media/i2c/alvium-csi2.c b/drivers/media/i2c/alvium-csi2.c index dd991c2ee7002..f51f9b9877592 100644 --- a/drivers/media/i2c/alvium-csi2.c +++ b/drivers/media/i2c/alvium-csi2.c @@ -1290,7 +1290,7 @@ static int alvium_set_ctrl_auto_exposure(struct alvium_dev *alvium, bool on) struct device *dev = &alvium->i2c_client->dev; int ret; - ret = alvium_write_hshake(alvium, REG_BCRM_WHITE_BALANCE_AUTO_RW, + ret = alvium_write_hshake(alvium, REG_BCRM_EXPOSURE_AUTO_RW, on ? 0x02 : 0x00); if (ret) { dev_err(dev, "Fail to set autoexposure reg\n"); diff --git a/drivers/media/i2c/imx355.c b/drivers/media/i2c/imx355.c index 27a5c212a527f..ac59908f57aae 100644 --- a/drivers/media/i2c/imx355.c +++ b/drivers/media/i2c/imx355.c @@ -1822,12 +1822,13 @@ static int imx355_probe(struct i2c_client *client) */ pm_runtime_set_active(imx355->dev); pm_runtime_enable(imx355->dev); - pm_runtime_idle(imx355->dev); ret = v4l2_async_register_subdev_sensor(&imx355->sd); if (ret < 0) goto error_media_entity_runtime_pm; + pm_runtime_idle(imx355->dev); + return 0; error_media_entity_runtime_pm: diff --git a/drivers/media/i2c/imx415.c b/drivers/media/i2c/imx415.c index 0b424c17e8804..c3b22b238ea0b 100644 --- a/drivers/media/i2c/imx415.c +++ b/drivers/media/i2c/imx415.c @@ -686,7 +686,7 @@ static int imx415_set_testpattern(struct imx415 *sensor, int val) cci_write(sensor->regmap, IMX415_DIG_CLP_MODE, 0x01, &ret); cci_write(sensor->regmap, IMX415_WRJ_OPEN, 0x01, &ret); } - return 0; + return ret; } static int imx415_s_ctrl(struct v4l2_ctrl *ctrl) @@ -720,7 +720,7 @@ static int imx415_s_ctrl(struct v4l2_ctrl *ctrl) ret = cci_write(sensor->regmap, IMX415_VMAX, format->height + ctrl->val, NULL); if (ret) - return ret; + break; /* * Exposure is set based on VMAX which has just changed, so * program exposure register as well diff --git a/drivers/media/i2c/mt9p031.c b/drivers/media/i2c/mt9p031.c index d21510caf45a8..2b09e8315c8ee 100644 --- a/drivers/media/i2c/mt9p031.c +++ b/drivers/media/i2c/mt9p031.c @@ -452,11 +452,7 @@ static int mt9p031_set_params(struct mt9p031 *mt9p031) ret = mt9p031_write(client, MT9P031_HORIZONTAL_BLANK, hblank - 1); if (ret < 0) return ret; - ret = mt9p031_write(client, MT9P031_VERTICAL_BLANK, vblank - 1); - if (ret < 0) - return ret; - - return ret; + return mt9p031_write(client, MT9P031_VERTICAL_BLANK, vblank - 1); } static int mt9p031_s_stream(struct v4l2_subdev *subdev, int enable) diff --git a/drivers/media/i2c/ov02a10.c b/drivers/media/i2c/ov02a10.c index 143dcfe104456..0150e4d296af6 100644 --- a/drivers/media/i2c/ov02a10.c +++ b/drivers/media/i2c/ov02a10.c @@ -820,18 +820,16 @@ static int ov02a10_check_hwcfg(struct device *dev, struct ov02a10 *ov02a10) if (!ep) return -ENXIO; + /* Optional indication of MIPI clock voltage unit */ + if (!fwnode_property_read_u32(ep, "ovti,mipi-clock-voltage", + &clk_volt)) + ov02a10->mipi_clock_voltage = clk_volt; + ret = v4l2_fwnode_endpoint_alloc_parse(ep, &bus_cfg); fwnode_handle_put(ep); if (ret) return ret; - /* Optional indication of MIPI clock voltage unit */ - ret = fwnode_property_read_u32(ep, "ovti,mipi-clock-voltage", - &clk_volt); - - if (!ret) - ov02a10->mipi_clock_voltage = clk_volt; - for (i = 0; i < ARRAY_SIZE(link_freq_menu_items); i++) { for (j = 0; j < bus_cfg.nr_of_link_frequencies; j++) { if (link_freq_menu_items[i] == diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c index c2e02f1918169..c09ad663f9de1 100644 --- a/drivers/media/i2c/ov7740.c +++ b/drivers/media/i2c/ov7740.c @@ -1116,10 +1116,8 @@ static void ov7740_remove(struct i2c_client *client) struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev); - mutex_destroy(&ov7740->mutex); - v4l2_ctrl_handler_free(ov7740->subdev.ctrl_handler); - media_entity_cleanup(&ov7740->subdev.entity); v4l2_async_unregister_subdev(sd); + media_entity_cleanup(&ov7740->subdev.entity); ov7740_free_controls(ov7740); pm_runtime_get_sync(&client->dev); diff --git a/drivers/media/i2c/ov9282.c b/drivers/media/i2c/ov9282.c index 5b6f897a74fcd..5abe51d21f89c 100644 --- a/drivers/media/i2c/ov9282.c +++ b/drivers/media/i2c/ov9282.c @@ -648,7 +648,8 @@ static int ov9282_set_ctrl(struct v4l2_ctrl *ctrl) ctrl->val ? OV9282_OUTPUT_ENABLE6_STROBE : 0, NULL); break; case V4L2_CID_FLASH_DURATION: - ret = cci_write(ov9282->regmap, OV9282_REG_STROBE_FRAME_SPAN, ctrl->val, NULL); + ret = cci_write(ov9282->regmap, OV9282_REG_STROBE_FRAME_SPAN, + ov9282_us_to_flash_duration(ov9282, ctrl->val), NULL); break; default: dev_err(ov9282->dev, "Invalid control %d", ctrl->id); diff --git a/drivers/media/i2c/rdacm21.c b/drivers/media/i2c/rdacm21.c index bcab462708c70..ece8a410e7ced 100644 --- a/drivers/media/i2c/rdacm21.c +++ b/drivers/media/i2c/rdacm21.c @@ -588,10 +588,12 @@ static int rdacm21_probe(struct i2c_client *client) ret = v4l2_async_register_subdev(&dev->sd); if (ret) - goto error_free_ctrls; + goto error_entity_cleanup; return 0; +error_entity_cleanup: + media_entity_cleanup(&dev->sd.entity); error_free_ctrls: v4l2_ctrl_handler_free(&dev->ctrls); error: @@ -606,6 +608,7 @@ static void rdacm21_remove(struct i2c_client *client) v4l2_async_unregister_subdev(&dev->sd); v4l2_ctrl_handler_free(&dev->ctrls); + media_entity_cleanup(&dev->sd.entity); i2c_unregister_device(dev->isp); } diff --git a/drivers/media/i2c/vd55g1.c b/drivers/media/i2c/vd55g1.c index 78d18c0281541..6c0c821babc7d 100644 --- a/drivers/media/i2c/vd55g1.c +++ b/drivers/media/i2c/vd55g1.c @@ -60,7 +60,10 @@ #define VD55G1_PATGEN_ENABLE BIT(0) #define VD55G1_REG_MANUAL_ANALOG_GAIN CCI_REG8(0x0501) #define VD55G1_REG_MANUAL_COARSE_EXPOSURE CCI_REG16_LE(0x0502) -#define VD55G1_REG_MANUAL_DIGITAL_GAIN CCI_REG16_LE(0x0504) +#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH0 CCI_REG16_LE(0x0504) +#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH1 CCI_REG16_LE(0x0506) +#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH2 CCI_REG16_LE(0x0508) +#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH3 CCI_REG16_LE(0x050a) #define VD55G1_REG_APPLIED_COARSE_EXPOSURE CCI_REG16_LE(0x00e8) #define VD55G1_REG_APPLIED_ANALOG_GAIN CCI_REG16_LE(0x00ea) #define VD55G1_REG_APPLIED_DIGITAL_GAIN CCI_REG16_LE(0x00ec) @@ -114,9 +117,8 @@ #define VD55G1_WIDTH 804 #define VD55G1_HEIGHT 704 -#define VD55G1_MODE_DEF 0 +#define VD55G1_MODE_IDX_DEF 0 #define VD55G1_NB_GPIOS 4 -#define VD55G1_MBUS_CODE_DEF 0 #define VD55G1_DGAIN_DEF 256 #define VD55G1_AGAIN_DEF 19 #define VD55G1_EXPO_MAX_TERM 64 @@ -634,7 +636,7 @@ static u32 vd55g1_get_fmt_code(struct vd55g1 *sensor, u32 code) goto adapt_bayer_pattern; } } - dev_warn(sensor->dev, "Unsupported mbus format\n"); + dev_warn(sensor->dev, "Unsupported mbus format: 0x%x\n", code); return code; @@ -850,9 +852,16 @@ static int vd55g1_update_expo_cluster(struct vd55g1 *sensor, bool is_auto) vd55g1_write(sensor, VD55G1_REG_MANUAL_ANALOG_GAIN, sensor->again_ctrl->val, &ret); - if (!is_auto && sensor->dgain_ctrl->is_new) - vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN, + if (!is_auto && sensor->dgain_ctrl->is_new) { + vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH0, sensor->dgain_ctrl->val, &ret); + vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH1, + sensor->dgain_ctrl->val, &ret); + vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH2, + sensor->dgain_ctrl->val, &ret); + vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH3, + sensor->dgain_ctrl->val, &ret); + } return ret; } @@ -1347,6 +1356,7 @@ static int vd55g1_init_state(struct v4l2_subdev *sd, { struct vd55g1 *sensor = to_vd55g1(sd); struct v4l2_subdev_format fmt = { 0 }; + int code; struct v4l2_subdev_route routes[] = { { .flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE } }; @@ -1361,9 +1371,13 @@ static int vd55g1_init_state(struct v4l2_subdev *sd, if (ret) return ret; - vd55g1_update_pad_fmt(sensor, &vd55g1_supported_modes[VD55G1_MODE_DEF], - vd55g1_get_fmt_code(sensor, VD55G1_MBUS_CODE_DEF), - &fmt.format); + if (sensor->id == VD55G1_MODEL_ID_VD55G1) + code = vd55g1_mbus_formats_mono[0]; + else + code = vd55g1_mbus_formats_bayer[0][0]; + vd55g1_update_pad_fmt(sensor, + &vd55g1_supported_modes[VD55G1_MODE_IDX_DEF], + vd55g1_get_fmt_code(sensor, code), &fmt.format); return vd55g1_set_pad_fmt(sd, sd_state, &fmt); } diff --git a/drivers/media/i2c/video-i2c.c b/drivers/media/i2c/video-i2c.c index 6b50fb422a614..4811a04895246 100644 --- a/drivers/media/i2c/video-i2c.c +++ b/drivers/media/i2c/video-i2c.c @@ -522,8 +522,12 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count) data->kthread_vid_cap = kthread_run(video_i2c_thread_vid_cap, data, "%s-vid-cap", data->v4l2_dev.name); ret = PTR_ERR_OR_ZERO(data->kthread_vid_cap); - if (!ret) - return 0; + if (ret) { + data->kthread_vid_cap = NULL; + goto error_rpm_put; + } + + return 0; error_rpm_put: pm_runtime_put_autosuspend(dev); diff --git a/drivers/media/pci/cobalt/cobalt-alsa-main.c b/drivers/media/pci/cobalt/cobalt-alsa-main.c index 7bb7f13c70c0d..9ed547cd40af5 100644 --- a/drivers/media/pci/cobalt/cobalt-alsa-main.c +++ b/drivers/media/pci/cobalt/cobalt-alsa-main.c @@ -135,7 +135,6 @@ int cobalt_alsa_init(struct cobalt_stream *s) err_exit_free: if (sc != NULL) snd_card_free(sc); - kfree(cobsc); err_exit: return ret; } diff --git a/drivers/media/pci/cx23885/cimax2.c b/drivers/media/pci/cx23885/cimax2.c index 1549189845348..723509fa1e065 100644 --- a/drivers/media/pci/cx23885/cimax2.c +++ b/drivers/media/pci/cx23885/cimax2.c @@ -528,6 +528,7 @@ void netup_ci_exit(struct cx23885_tsport *port) if (NULL == state->ca.data) return; + cancel_work_sync(&state->work); dvb_ca_en50221_release(&state->ca); kfree(state); } diff --git a/drivers/media/pci/intel/ipu-bridge.c b/drivers/media/pci/intel/ipu-bridge.c index 88581a4c081d6..510c0eca44841 100644 --- a/drivers/media/pci/intel/ipu-bridge.c +++ b/drivers/media/pci/intel/ipu-bridge.c @@ -173,6 +173,19 @@ static const struct acpi_device_id ivsc_acpi_ids[] = { { "INTC10E1" }, /* PTL */ }; +/* + * The subset of ivsc_acpi_ids[] which are IVSC, rather than CVS, devices. The + * CVS IDs are deliberately not listed here: new ones keep being added, whereas + * this list is complete. + */ +static const struct acpi_device_id ivsc_only_acpi_ids[] = { + { "INTC1059" }, + { "INTC1095" }, + { "INTC100A" }, + { "INTC10CF" }, + { } +}; + static struct acpi_device *ipu_bridge_get_ivsc_acpi_dev(struct acpi_device *adev) { unsigned int i; @@ -224,6 +237,17 @@ static struct device *ipu_bridge_get_ivsc_csi_dev(struct acpi_device *adev) return csi_dev; } + /* + * The lookups below match on the ACPI companion alone. That is fine for + * CVS, which binds a driver to that very device, but not for IVSC: there + * the ACPI device also has a driverless platform device, which would be + * returned instead of the mei-csi client. Return NULL for IVSC so that + * the caller fails and the probe is retried once the IVSC device shows + * up. + */ + if (!acpi_match_device_ids(adev, ivsc_only_acpi_ids)) + return NULL; + /* Try to locate CVS device on the I2C bus */ csi_dev = bus_find_device_by_acpi_dev(&i2c_bus_type, adev); if (csi_dev) @@ -298,9 +322,11 @@ static u32 ipu_bridge_parse_rotation(struct acpi_device *adev, { const struct dmi_system_id *dmi_id; - dmi_id = dmi_first_match(upside_down_sensor_dmi_ids); - if (dmi_id && acpi_dev_hid_match(adev, dmi_id->driver_data)) - return 180; + /* A machine may have one entry per sensor, so check all matches. */ + for (dmi_id = dmi_first_match(upside_down_sensor_dmi_ids); dmi_id; + dmi_id = dmi_first_match(dmi_id + 1)) + if (acpi_dev_hid_match(adev, dmi_id->driver_data)) + return 180; switch (ssdb->degree) { case IPU_SENSOR_ROTATION_NORMAL: diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.c b/drivers/media/pci/intel/ipu6/ipu6-isys.c index c9cdeb7054d75..24db2763de544 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-isys.c +++ b/drivers/media/pci/intel/ipu6/ipu6-isys.c @@ -761,6 +761,7 @@ static int isys_notifier_init(struct ipu6_isys *isys) err_parse: fwnode_handle_put(ep); + v4l2_async_nf_cleanup(&isys->notifier); return ret; } diff --git a/drivers/media/pci/intel/ipu6/ipu6.c b/drivers/media/pci/intel/ipu6/ipu6.c index d033d46181692..5449a2006bcce 100644 --- a/drivers/media/pci/intel/ipu6/ipu6.c +++ b/drivers/media/pci/intel/ipu6/ipu6.c @@ -400,7 +400,6 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent, &ipdata->hw_variant); if (IS_ERR(isys_adev->mmu)) { put_device(&isys_adev->auxdev.dev); - kfree(pdata); return dev_err_cast_probe(dev, isys_adev->mmu, "ipu6_mmu_init(isys_adev->mmu) failed\n"); } @@ -408,10 +407,8 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent, isys_adev->mmu->dev = &isys_adev->auxdev.dev; ret = ipu6_bus_add_device(isys_adev); - if (ret) { - kfree(pdata); + if (ret) return ERR_PTR(ret); - } return isys_adev; } @@ -444,7 +441,6 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent, &ipdata->hw_variant); if (IS_ERR(psys_adev->mmu)) { put_device(&psys_adev->auxdev.dev); - kfree(pdata); return dev_err_cast_probe(&pdev->dev, psys_adev->mmu, "ipu6_mmu_init(psys_adev->mmu) failed\n"); } @@ -452,10 +448,8 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent, psys_adev->mmu->dev = &psys_adev->auxdev.dev; ret = ipu6_bus_add_device(psys_adev); - if (ret) { - kfree(pdata); + if (ret) return ERR_PTR(ret); - } return psys_adev; } diff --git a/drivers/media/pci/saa7164/saa7164-core.c b/drivers/media/pci/saa7164/saa7164-core.c index 6bcde506adf50..db65506ea885b 100644 --- a/drivers/media/pci/saa7164/saa7164-core.c +++ b/drivers/media/pci/saa7164/saa7164-core.c @@ -878,6 +878,9 @@ static int get_resources(struct saa7164_dev *dev) if (request_mem_region(pci_resource_start(dev->pci, 2), pci_resource_len(dev->pci, 2), dev->name)) return 0; + + release_mem_region(pci_resource_start(dev->pci, 0), + pci_resource_len(dev->pci, 0)); } printk(KERN_ERR "%s: can't get MMIO memory @ 0x%llx or 0x%llx\n", @@ -1000,8 +1003,7 @@ static int saa7164_dev_setup(struct saa7164_dev *dev) dev->name, dev->pci->subsystem_vendor, dev->pci->subsystem_device); - saa7164_devcount--; - return -ENODEV; + goto err_devlist; } /* PCI/e allocations */ @@ -1039,7 +1041,7 @@ static int saa7164_dev_setup(struct saa7164_dev *dev) iounmap(dev->lmmio); err_ioremap_bar0: release_resources(dev); - +err_devlist: scoped_guard(mutex, &devlist) { list_del(&dev->devlist); } diff --git a/drivers/media/pci/zoran/zoran_card.c b/drivers/media/pci/zoran/zoran_card.c index f707bdc1fb0f1..b8285df644738 100644 --- a/drivers/media/pci/zoran/zoran_card.c +++ b/drivers/media/pci/zoran/zoran_card.c @@ -885,7 +885,7 @@ static int zoran_init_video_device(struct zoran *zr, struct video_device *video_ static void zoran_exit_video_devices(struct zoran *zr) { video_unregister_device(zr->video_dev); - kfree(zr->video_dev); + zr->video_dev = NULL; } static int zoran_init_video_devices(struct zoran *zr) diff --git a/drivers/media/platform/amd/isp4/Kconfig b/drivers/media/platform/amd/isp4/Kconfig index 9d1927af1cb8c..77b61fae82bab 100644 --- a/drivers/media/platform/amd/isp4/Kconfig +++ b/drivers/media/platform/amd/isp4/Kconfig @@ -5,6 +5,7 @@ config VIDEO_AMD_ISP4_CAPTURE depends on DRM_AMDGPU && DRM_AMD_ISP depends on HAS_DMA depends on VIDEO_DEV + select MEDIA_CONTROLLER select VIDEOBUF2_CORE select VIDEOBUF2_MEMOPS select VIDEOBUF2_V4L2 diff --git a/drivers/media/platform/amd/isp4/isp4_interface.c b/drivers/media/platform/amd/isp4/isp4_interface.c index 8d73f66bb42cb..4801617f95599 100644 --- a/drivers/media/platform/amd/isp4/isp4_interface.c +++ b/drivers/media/platform/amd/isp4/isp4_interface.c @@ -148,12 +148,9 @@ static void isp4if_gpu_mem_free(struct isp4_interface *ispif, struct isp4if_gpu_mem_info **mem_info_ptr) { struct isp4if_gpu_mem_info *mem_info = *mem_info_ptr; - struct device *dev = ispif->dev; - if (!mem_info) { - dev_err(dev, "invalid mem_info\n"); + if (!mem_info) return; - } *mem_info_ptr = NULL; isp_kernel_buffer_free(&mem_info->mem_handle, &mem_info->gpu_mc_addr, @@ -201,6 +198,7 @@ static int isp4if_alloc_fw_gpumem(struct isp4_interface *ispif) error_no_memory: dev_err(dev, "failed to allocate gpu memory\n"); + isp4if_dealloc_fw_gpumem(ispif); return -ENOMEM; } @@ -375,7 +373,7 @@ static int isp4if_send_fw_cmd(struct isp4_interface *ispif, u32 cmd_id, return -ENOMEM; /* Get two references: one for the resp thread, one for us */ - atomic_set(&ele->refcnt, 2); + refcount_set(&ele->refcnt, 2); init_completion(&ele->cmd_done); } @@ -455,7 +453,7 @@ static int isp4if_send_fw_cmd(struct isp4_interface *ispif, u32 cmd_id, put_ele_ref: /* Don't free the command if we didn't put the last reference */ - if (ele && atomic_dec_return(&ele->refcnt)) + if (ele && !refcount_dec_and_test(&ele->refcnt)) ele = NULL; free_ele: diff --git a/drivers/media/platform/amd/isp4/isp4_interface.h b/drivers/media/platform/amd/isp4/isp4_interface.h index ce3ac9b9e5cda..04db71cd54e6c 100644 --- a/drivers/media/platform/amd/isp4/isp4_interface.h +++ b/drivers/media/platform/amd/isp4/isp4_interface.h @@ -68,7 +68,7 @@ struct isp4if_cmd_element { u32 seq_num; u32 cmd_id; struct completion cmd_done; - atomic_t refcnt; + refcount_t refcnt; }; struct isp4_interface { diff --git a/drivers/media/platform/amd/isp4/isp4_subdev.c b/drivers/media/platform/amd/isp4/isp4_subdev.c index 48deea79ce6c2..6716ab9c128a0 100644 --- a/drivers/media/platform/amd/isp4/isp4_subdev.c +++ b/drivers/media/platform/amd/isp4/isp4_subdev.c @@ -391,7 +391,7 @@ static void isp4sd_fw_resp_cmd_done(struct isp4_subdev *isp_subdev, if (ele) { complete(&ele->cmd_done); - if (atomic_dec_and_test(&ele->refcnt)) + if (refcount_dec_and_test(&ele->refcnt)) kfree(ele); } } @@ -687,7 +687,7 @@ int isp4sd_pwron_and_init(struct v4l2_subdev *sd) if (ret) { dev_err(dev, "fail to power on isp_subdev ret %d\n", ret); - goto err_deinit; + goto err_module_disable; } /* ISPPG ISP Power Status */ @@ -697,7 +697,7 @@ int isp4sd_pwron_and_init(struct v4l2_subdev *sd) dev_err(dev, "fail to set performance state %u, ret %d\n", perf_state, ret); - goto err_deinit; + goto err_power_off; } ispif->status = ISP4IF_STATUS_PWR_ON; @@ -709,12 +709,12 @@ int isp4sd_pwron_and_init(struct v4l2_subdev *sd) ret = isp4if_start(ispif); if (ret) { dev_err(dev, "fail to start isp_subdev interface\n"); - goto err_deinit; + goto err_perf_restore; } if (isp4sd_start_resp_proc_threads(isp_subdev)) { dev_err(dev, "isp_start_resp_proc_threads fail\n"); - goto err_deinit; + goto err_stop_interface; } dev_dbg(dev, "create resp threads ok\n"); @@ -724,8 +724,24 @@ int isp4sd_pwron_and_init(struct v4l2_subdev *sd) isp_subdev->irq_enabled = true; return 0; -err_deinit: - isp4sd_pwroff_and_deinit(sd); + +err_stop_interface: + isp4if_stop(ispif); +err_perf_restore: + ret = dev_pm_genpd_set_performance_state(dev, ISP4SD_PERFORMANCE_STATE_LOW); + if (ret) + dev_err(dev, "fail to set performance state %u, ret %d\n", + ISP4SD_PERFORMANCE_STATE_LOW, ret); +err_power_off: + isp4hw_wreg(isp_subdev->mmio, ISP_SOFT_RESET, 0); + isp4hw_wreg(isp_subdev->mmio, ISP_POWER_STATUS, 0); + ret = pm_runtime_put_sync(dev); + if (ret) + dev_err(dev, "power off isp_subdev fail %d\n", ret); + ispif->status = ISP4IF_STATUS_PWR_OFF; +err_module_disable: + isp4sd_module_enable(isp_subdev, false); + msleep(20); return -EINVAL; } diff --git a/drivers/media/platform/amphion/venc.c b/drivers/media/platform/amphion/venc.c index 0b3d58b9f2f7a..2ba4ae20f8292 100644 --- a/drivers/media/platform/amphion/venc.c +++ b/drivers/media/platform/amphion/venc.c @@ -973,8 +973,6 @@ static int venc_start_session(struct vpu_inst *inst, u32 type) venc->ready_count = 0; venc->stopped = false; vpu_process_output_buffer(inst); - if (venc->frame_count == 0) - dev_err(inst->dev, "[%d] there is no input when starting\n", inst->id); return 0; error: diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-core.c b/drivers/media/platform/arm/mali-c55/mali-c55-core.c index 94a389b3f8333..f28e9f4354aca 100644 --- a/drivers/media/platform/arm/mali-c55/mali-c55-core.c +++ b/drivers/media/platform/arm/mali-c55/mali-c55-core.c @@ -698,6 +698,8 @@ static int __mali_c55_power_on(struct mali_c55 *mali_c55) mali_c55->resets); if (ret) { dev_err(mali_c55->dev, "failed to deassert resets\n"); + clk_bulk_disable_unprepare(ARRAY_SIZE(mali_c55->clks), + mali_c55->clks); return ret; } diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-params.c b/drivers/media/platform/arm/mali-c55/mali-c55-params.c index de0e9d898db7b..33e2232ec8f5b 100644 --- a/drivers/media/platform/arm/mali-c55/mali-c55-params.c +++ b/drivers/media/platform/arm/mali-c55/mali-c55-params.c @@ -212,6 +212,7 @@ mali_c55_params_aexp_hist_weights(struct mali_c55 *mali_c55, val = params->zone_weights[MALI_C55_MAX_ZONES - 1]; addr = base + MALI_C55_AEXP_HIST_ZONE_WEIGHTS_OFFSET + (4 * 56); + mali_c55_ctx_write(mali_c55, addr, val & MALI_C55_AEXP_HIST_ZONE_WEIGHT_MASK); } static void mali_c55_params_digital_gain(struct mali_c55 *mali_c55, diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-registers.h b/drivers/media/platform/arm/mali-c55/mali-c55-registers.h index f098effde7b49..4cd13b702d9f0 100644 --- a/drivers/media/platform/arm/mali-c55/mali-c55-registers.h +++ b/drivers/media/platform/arm/mali-c55/mali-c55-registers.h @@ -173,7 +173,7 @@ enum mali_c55_interrupts { #define MALI_C55_AEXP_HIST_SWITCH_MASK GENMASK(14, 13) #define MALI_C55_AEXP_HIST_SWITCH(x) ((x) << 13) #define MALI_C55_AEXP_IHIST_DISABLE_MASK BIT(16) -#define MALI_C55_AEXP_IHIST_DISABLE (0x01 << 12) +#define MALI_C55_AEXP_IHIST_DISABLE (0x01 << 16) #define MALI_C55_AEXP_SRC_MASK BIT(24) #define MALI_C55_REG_TPG_CH0 0x18ed8 diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c b/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c index c4f46651dcee3..6706939b4a902 100644 --- a/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c +++ b/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c @@ -15,7 +15,7 @@ #include "mali-c55-registers.h" /* Scaling factor in Q4.20 format. */ -#define MALI_C55_RSZ_SCALER_FACTOR (1U << 20) +#define MALI_C55_RSZ_SCALER_FACTOR BIT_ULL(20) #define MALI_C55_RSZ_COEFS_BANKS 8 #define MALI_C55_RSZ_COEFS_ENTRIES 64 diff --git a/drivers/media/platform/broadcom/bcm2835-unicam.c b/drivers/media/platform/broadcom/bcm2835-unicam.c index 8d28ba0b59a39..14bb916dd7b1b 100644 --- a/drivers/media/platform/broadcom/bcm2835-unicam.c +++ b/drivers/media/platform/broadcom/bcm2835-unicam.c @@ -2158,7 +2158,8 @@ static int unicam_video_link_validate(struct media_link *link) * In order to allow the applications using the old behaviour to * run, let's accept the old combination, but warn about it. */ - if (fmtinfo->fourcc != fmt->pixelformat) { + if (fmt->pixelformat != fmtinfo->fourcc && + fmt->pixelformat != fmtinfo->unpacked_fourcc) { if ((fmt->pixelformat == V4L2_PIX_FMT_BGR24 && format->code == MEDIA_BUS_FMT_BGR888_1X24) || (fmt->pixelformat == V4L2_PIX_FMT_RGB24 && @@ -2613,6 +2614,7 @@ static int unicam_async_nf_init(struct unicam_device *unicam) return 0; error: + v4l2_async_nf_cleanup(&unicam->notifier); fwnode_handle_put(ep_handle); return ret; } @@ -2745,6 +2747,7 @@ static void unicam_remove(struct platform_device *pdev) v4l2_device_unregister(&unicam->v4l2_dev); media_device_unregister(&unicam->mdev); v4l2_async_nf_unregister(&unicam->notifier); + v4l2_async_nf_cleanup(&unicam->notifier); unicam_subdev_cleanup(unicam); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c index bb2ba9204a836..6564cf3ec739b 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c @@ -5,6 +5,7 @@ * Copyright (C) 2021-2023 CHIPS&MEDIA INC */ +#include #include #include "wave5-helper.h" @@ -474,7 +475,10 @@ static void wave5_vpu_dec_finish_decode(struct vpu_instance *inst) v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx); } - inst->queuing_fail = false; + if (inst->queuing_fail) { + inst->queuing_fail = false; + v4l2_m2m_try_schedule(m2m_ctx); + } } static int wave5_vpu_dec_querycap(struct file *file, void *fh, struct v4l2_capability *cap) @@ -819,7 +823,15 @@ static int wave5_vpu_dec_stop(struct vpu_instance *inst) * calls do not block on a mutex while inside this spinlock. */ spin_unlock_irqrestore(&inst->state_spinlock, flags); + /* + * V4L2_DEC_CMD_STOP can arrive while the device is runtime + * suspended (e.g. on pipeline teardown). Setting the EOS flag + * accesses VPU registers via send_firmware_command(), so the + * device must be resumed first to avoid an asynchronous SError. + */ + pm_runtime_resume_and_get(inst->dev->dev); ret = wave5_vpu_dec_set_eos_on_firmware(inst); + pm_runtime_put_autosuspend(inst->dev->dev); if (ret) return ret; @@ -1403,6 +1415,7 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count } else if (q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { struct dec_initial_info *initial_info = &inst->codec_info->dec_info.initial_info; + struct dec_info *p_dec_info = &inst->codec_info->dec_info; if (inst->state == VPU_INST_STATE_STOP) ret = switch_state(inst, VPU_INST_STATE_INIT_SEQ); @@ -1410,6 +1423,7 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count goto return_buffers; if (inst->state == VPU_INST_STATE_INIT_SEQ && + p_dec_info->initial_info_obtained && inst->dev->product_code == WAVE521C_CODE) { if (initial_info->luma_bitdepth != 8) { dev_info(inst->dev->dev, "%s: no support for %d bit depth", @@ -1418,7 +1432,6 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count goto return_buffers; } } - } pm_runtime_put_autosuspend(inst->dev->dev); return ret; @@ -1536,15 +1549,15 @@ static void wave5_vpu_dec_stop_streaming(struct vb2_queue *q) { struct vpu_instance *inst = vb2_get_drv_priv(q); struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx; - - bool check_cmd = TRUE; + unsigned long timeout; dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type); pm_runtime_resume_and_get(inst->dev->dev); inst->empty_queue = true; - while (check_cmd) { + + timeout = jiffies + msecs_to_jiffies(VPU_DEC_STOP_TIMEOUT); + while (true) { struct queue_status_info q_status; - struct dec_output_info dec_output_info; wave5_vpu_dec_give_command(inst, DEC_GET_QUEUE_STATUS, &q_status); if ((inst->state == VPU_INST_STATE_STOP || @@ -1553,8 +1566,10 @@ static void wave5_vpu_dec_stop_streaming(struct vb2_queue *q) q_status.report_queue_count == 0) break; - if (wave5_vpu_dec_get_output_info(inst, &dec_output_info)) - dev_dbg(inst->dev->dev, "there is no output info\n"); + if (time_after(jiffies, timeout)) + break; + + usleep_range(1000, 2000); } v4l2_m2m_update_stop_streaming_state(m2m_ctx, q); @@ -1651,6 +1666,7 @@ static void wave5_vpu_dec_device_run(void *priv) struct queue_status_info q_status; u32 fail_res = 0; int ret = 0; + bool cmd_issued = false; dev_dbg(inst->dev->dev, "%s: Fill the ring buffer with new bitstream data", __func__); pm_runtime_resume_and_get(inst->dev->dev); @@ -1662,9 +1678,13 @@ static void wave5_vpu_dec_device_run(void *priv) } else if (!inst->eos && inst->queuing_num == 0 && inst->state == VPU_INST_STATE_PIC_RUN) { - dev_dbg(inst->dev->dev, "%s: no bitstream for feeding, so skip ", __func__); - inst->empty_queue = true; - goto finish_job_and_return; + wave5_vpu_dec_give_command(inst, DEC_GET_QUEUE_STATUS, &q_status); + if (q_status.instance_queue_count == v4l2_m2m_num_src_bufs_ready(m2m_ctx)) { + dev_dbg(inst->dev->dev, "%s: no bitstream, skip\n", + __func__); + inst->empty_queue = true; + goto finish_job_and_return; + } } } @@ -1744,6 +1764,7 @@ static void wave5_vpu_dec_device_run(void *priv) inst->retry = false; if (!inst->eos) inst->queuing_num--; + cmd_issued = true; } break; default: @@ -1761,8 +1782,16 @@ static void wave5_vpu_dec_device_run(void *priv) * in power and CPU time. * If EOS is passed, device_run will not call job_finish no more, it is called * only if HW is idle status in order to reduce overhead. + * + * Deferring job_finish() is only safe when this run actually queued a + * DEC_PIC command (cmd_issued): that guarantees a completion IRQ, and + * thus a later finish_decode(), will release the shared job slot. When + * device_run() is entered with no command to issue (e.g. a job that was + * queued while draining but reached the STOP state by the time it ran), + * no IRQ follows, so finish the job here to avoid leaking the slot and + * stalling every instance sharing the VPU. */ - if (!inst->sent_eos) + if (!inst->sent_eos || !cmd_issued) v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx); } @@ -1776,11 +1805,22 @@ static void wave5_vpu_dec_job_abort(void *priv) if (ret) return; + /* + * job_abort() runs from the STREAMOFF path and may be called while the + * device is runtime suspended. Setting the EOS flag talks to the + * firmware (send_firmware_command() accesses VPU registers), so the + * device must be resumed first; otherwise the register access faults + * with an asynchronous SError. + */ + pm_runtime_resume_and_get(inst->dev->dev); + ret = wave5_vpu_dec_set_eos_on_firmware(inst); if (ret) dev_warn(inst->dev->dev, "Setting EOS for the bitstream, fail: %d\n", ret); + pm_runtime_put_autosuspend(inst->dev->dev); + v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx); } diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c index e6c94b6f26711..f9fcdf4c224b3 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c @@ -1494,7 +1494,8 @@ static const struct vb2_ops wave5_vpu_enc_vb2_ops = { .stop_streaming = wave5_vpu_enc_stop_streaming, }; -static void wave5_set_default_format(struct v4l2_pix_format_mplane *src_fmt, +static void wave5_set_default_format(struct vpu_instance *inst, + struct v4l2_pix_format_mplane *src_fmt, struct v4l2_pix_format_mplane *dst_fmt) { src_fmt->pixelformat = enc_fmt_list[VPU_FMT_TYPE_RAW][0].v4l2_pix_fmt; @@ -1506,6 +1507,7 @@ static void wave5_set_default_format(struct v4l2_pix_format_mplane *src_fmt, wave5_update_pix_fmt(dst_fmt, VPU_FMT_TYPE_CODEC, W5_DEF_ENC_PIC_WIDTH, W5_DEF_ENC_PIC_HEIGHT, &enc_frmsize[VPU_FMT_TYPE_CODEC]); + inst->std = wave5_to_vpu_std(dst_fmt->pixelformat, inst->type); } static int wave5_vpu_enc_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq) @@ -1770,7 +1772,7 @@ static int wave5_vpu_open_enc(struct file *filp) inst->v4l2_fh.ctrl_handler = v4l2_ctrl_hdl; v4l2_ctrl_handler_setup(v4l2_ctrl_hdl); - wave5_set_default_format(&inst->src_fmt, &inst->dst_fmt); + wave5_set_default_format(inst, &inst->src_fmt, &inst->dst_fmt); inst->conf_win.width = inst->dst_fmt.width; inst->conf_win.height = inst->dst_fmt.height; inst->colorspace = V4L2_COLORSPACE_REC709; diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h b/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h index 4ebd48d5550ea..e04f2dbf3b655 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h @@ -59,7 +59,7 @@ // application specific configuration #define VPU_ENC_TIMEOUT 60000 #define VPU_DEC_TIMEOUT 60000 -#define VPU_DEC_STOP_TIMEOUT 10 +#define VPU_DEC_STOP_TIMEOUT 300 // for WAVE encoder #define USE_SRC_PRP_AXI 0 diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c index 8f4da4cf55d26..f412aadb5996b 100644 --- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c @@ -294,7 +294,11 @@ static int mdp_probe(struct platform_device *pdev) goto err_destroy_clock_wq; } mdp->scp = platform_get_drvdata(mm_pdev); - put_device(&mm_pdev->dev); + } + + if (!mdp->scp) { + ret = -EPROBE_DEFER; + goto err_destroy_clock_wq; } mdp->rproc_handle = scp_get_rproc(mdp->scp); diff --git a/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_av1_req_lat_if.c b/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_av1_req_lat_if.c index 2d622e85f8271..49d9b4a72387e 100644 --- a/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_av1_req_lat_if.c +++ b/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_av1_req_lat_if.c @@ -1299,11 +1299,12 @@ static void vdec_av1_slice_setup_tile(struct vdec_av1_slice_frame *frame, tile->uniform_tile_spacing_flag = BIT_FLAG(ctrl_tile, V4L2_AV1_TILE_INFO_FLAG_UNIFORM_TILE_SPACING); - for (i = 0; i < tile->tile_cols + 1; i++) + /* Bound the copy to the mi_col_starts[]/mi_row_starts[] capacity. */ + for (i = 0; i < tile->tile_cols + 1 && i < V4L2_AV1_MAX_TILE_COLS + 1; i++) tile->mi_col_starts[i] = ALIGN(ctrl_tile->mi_col_starts[i], BIT(mib_size_log2)) >> mib_size_log2; - for (i = 0; i < tile->tile_rows + 1; i++) + for (i = 0; i < tile->tile_rows + 1 && i < V4L2_AV1_MAX_TILE_ROWS + 1; i++) tile->mi_row_starts[i] = ALIGN(ctrl_tile->mi_row_starts[i], BIT(mib_size_log2)) >> mib_size_log2; } diff --git a/drivers/media/platform/microchip/microchip-sama7g5-isc.c b/drivers/media/platform/microchip/microchip-sama7g5-isc.c index b0302dfc3278d..7383341ec51d6 100644 --- a/drivers/media/platform/microchip/microchip-sama7g5-isc.c +++ b/drivers/media/platform/microchip/microchip-sama7g5-isc.c @@ -598,13 +598,8 @@ static int __maybe_unused xisc_runtime_suspend(struct device *dev) static int __maybe_unused xisc_runtime_resume(struct device *dev) { struct isc_device *isc = dev_get_drvdata(dev); - int ret; - - ret = clk_prepare_enable(isc->hclock); - if (ret) - return ret; - return ret; + return clk_prepare_enable(isc->hclock); } static const struct dev_pm_ops microchip_xisc_dev_pm_ops = { diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c index c580c831972ec..01374b7af8438 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c @@ -107,7 +107,7 @@ static int __mxc_isi_crossbar_set_routing(struct v4l2_subdev *sd, return ret; /* The memory input can be routed to the first pipeline only. */ - for_each_active_route(&state->routing, route) { + for_each_active_route(routing, route) { if (route->sink_pad == xbar->num_sinks - 1 && route->source_pad != xbar->num_sinks) { dev_dbg(xbar->isi->dev, @@ -145,10 +145,10 @@ mxc_isi_crossbar_xlate_streams(struct mxc_isi_crossbar *xbar, */ for_each_active_route(&state->routing, route) { if (route->source_pad != source_pad || - !(source_streams & BIT(route->source_stream))) + !(source_streams & BIT_ULL(route->source_stream))) continue; - sink_streams |= BIT(route->sink_stream); + sink_streams |= BIT_ULL(route->sink_stream); sink_pad = route->sink_pad; } diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c index fe4adfa3a1f01..056603055c816 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c @@ -151,7 +151,7 @@ static const struct mxc_isi_format_info mxc_isi_formats[] = { .fourcc = V4L2_PIX_FMT_XBGR32, .type = MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_OUT | MXC_ISI_VIDEO_M2M_CAP, - .isi_in_format = CHNL_MEM_RD_CTRL_IMG_TYPE_XBGR8, + .isi_in_format = CHNL_MEM_RD_CTRL_IMG_TYPE_XRGB8, .isi_out_format = CHNL_IMG_CTRL_FORMAT_XRGB888, .mem_planes = 1, .color_planes = 1, diff --git a/drivers/media/platform/qcom/iris/iris_core.c b/drivers/media/platform/qcom/iris/iris_core.c index 52bf56e517f91..8c335dbfce166 100644 --- a/drivers/media/platform/qcom/iris/iris_core.c +++ b/drivers/media/platform/qcom/iris/iris_core.c @@ -12,18 +12,24 @@ void iris_core_deinit(struct iris_core *core) { - pm_runtime_resume_and_get(core->dev); + int ret; + + ret = pm_runtime_resume_and_get(core->dev); mutex_lock(&core->lock); if (core->state != IRIS_CORE_DEINIT) { iris_fw_unload(core); - iris_vpu_power_off(core); + + if (!ret) + iris_vpu_power_off(core); + iris_hfi_queues_deinit(core); core->state = IRIS_CORE_DEINIT; } mutex_unlock(&core->lock); - pm_runtime_put_sync(core->dev); + if (!ret) + pm_runtime_put_sync(core->dev); } static int iris_wait_for_system_response(struct iris_core *core) diff --git a/drivers/media/platform/qcom/iris/iris_ctrls.c b/drivers/media/platform/qcom/iris/iris_ctrls.c index 10e33b8a73f60..391e1fc5f6e2e 100644 --- a/drivers/media/platform/qcom/iris/iris_ctrls.c +++ b/drivers/media/platform/qcom/iris/iris_ctrls.c @@ -1293,7 +1293,7 @@ int iris_set_use_and_mark_ltr(struct iris_inst *inst, enum platform_inst_fw_cap_ int iris_set_intra_period(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id) { const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops; - u32 gop_size = inst->fw_caps[GOP_SIZE].value; + u32 gop_size = inst->fw_caps[cap_id].value; u32 b_frame = inst->fw_caps[B_FRAME].value; u32 hfi_id = inst->fw_caps[cap_id].hfi_id; struct hfi_intra_period intra_period; diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen1.c b/drivers/media/platform/qcom/iris/iris_hfi_gen1.c index ca1545d28b531..eff9216bb5014 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen1.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen1.c @@ -171,7 +171,9 @@ static const struct platform_inst_fw_cap inst_fw_cap_sm8250_enc[] = { .max = (1 << 16) - 1, .step_or_mask = 1, .value = 30, - .set = iris_set_u32 + .hfi_id = HFI_PROPERTY_CONFIG_VENC_INTRA_PERIOD, + .flags = CAP_FLAG_OUTPUT_PORT, + .set = iris_set_intra_period, }, { .cap_id = ENTROPY_MODE, @@ -240,7 +242,7 @@ static const struct platform_inst_fw_cap inst_fw_cap_sm8250_enc[] = { .step_or_mask = 1, .value = 0, .hfi_id = HFI_PROPERTY_PARAM_VENC_INTRA_REFRESH, - .flags = CAP_FLAG_OUTPUT_PORT, + .flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_DYNAMIC_ALLOWED, .set = iris_set_ir_period_gen1, }, { @@ -281,16 +283,6 @@ static const struct platform_inst_fw_cap inst_fw_cap_sm8250_enc[] = { .value = 0, .flags = CAP_FLAG_OUTPUT_PORT, }, - { - .cap_id = INTRA_PERIOD, - .min = 0, - .max = 1, - .step_or_mask = 1, - .value = 0, - .hfi_id = HFI_PROPERTY_CONFIG_VENC_INTRA_PERIOD, - .flags = CAP_FLAG_OUTPUT_PORT, - .set = iris_set_intra_period, - }, { .cap_id = LAYER_ENABLE, .min = 0, diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c index ca2954f8bd3ad..ee2730c293bcd 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c @@ -692,7 +692,6 @@ static int iris_hfi_gen2_session_set_config_params(struct iris_inst *inst, u32 p {HFI_PROP_FRAME_RATE, iris_hfi_gen2_set_frame_rate }, {HFI_PROP_AV1_FILM_GRAIN_PRESENT, iris_hfi_gen2_set_film_grain }, {HFI_PROP_AV1_SUPER_BLOCK_ENABLED, iris_hfi_gen2_set_super_block }, - {HFI_PROP_OPB_ENABLE, iris_hfi_gen2_set_opb_enable }, }; if (inst->domain == DECODER) { diff --git a/drivers/media/platform/qcom/iris/iris_platform_common.h b/drivers/media/platform/qcom/iris/iris_platform_common.h index c9256f2323dc4..ff48333ad0898 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_common.h +++ b/drivers/media/platform/qcom/iris/iris_platform_common.h @@ -165,7 +165,6 @@ enum platform_inst_fw_cap_type { USE_LTR, MARK_LTR, B_FRAME, - INTRA_PERIOD, LAYER_ENABLE, LAYER_TYPE_H264, LAYER_TYPE_HEVC, diff --git a/drivers/media/platform/qcom/iris/iris_resources.c b/drivers/media/platform/qcom/iris/iris_resources.c index 773f6548370a2..6d3339423eec0 100644 --- a/drivers/media/platform/qcom/iris/iris_resources.c +++ b/drivers/media/platform/qcom/iris/iris_resources.c @@ -78,24 +78,21 @@ int iris_enable_power_domains(struct iris_core *core, struct device *pd_dev) if (ret) return ret; - ret = pm_runtime_get_sync(pd_dev); - if (ret < 0) - return ret; - - return ret; + return pm_runtime_resume_and_get(pd_dev); } int iris_disable_power_domains(struct iris_core *core, struct device *pd_dev) { int ret; + int pm_ret; ret = iris_opp_set_rate(core->dev, 0); - if (ret) - return ret; - pm_runtime_put_sync(pd_dev); + pm_ret = pm_runtime_put_sync(pd_dev); + if (!ret) + ret = pm_ret; - return 0; + return ret; } static struct clk *iris_get_clk_by_type(struct iris_core *core, enum platform_clk_type clk_type) diff --git a/drivers/media/platform/qcom/iris/iris_state.c b/drivers/media/platform/qcom/iris/iris_state.c index d14472414750d..5552725c614ea 100644 --- a/drivers/media/platform/qcom/iris/iris_state.c +++ b/drivers/media/platform/qcom/iris/iris_state.c @@ -60,9 +60,9 @@ int iris_inst_change_state(struct iris_inst *inst, return -EINVAL; change_state: - inst->state = request_state; dev_dbg(inst->core->dev, "state changed from %x to %x\n", inst->state, request_state); + inst->state = request_state; return 0; } @@ -269,7 +269,7 @@ bool iris_allow_cmd(struct iris_inst *inst, u32 cmd) return true; } else if (cmd == V4L2_DEC_CMD_STOP || cmd == V4L2_ENC_CMD_STOP) { if (vb2_is_streaming(src_q)) - if (inst->sub_state != IRIS_INST_SUB_DRAIN) + if (!(inst->sub_state & IRIS_INST_SUB_DRAIN)) return true; } diff --git a/drivers/media/platform/qcom/iris/iris_vidc.c b/drivers/media/platform/qcom/iris/iris_vidc.c index 14d63dc76c9ba..1c052ca235bec 100644 --- a/drivers/media/platform/qcom/iris/iris_vidc.c +++ b/drivers/media/platform/qcom/iris/iris_vidc.c @@ -449,14 +449,21 @@ static int iris_enum_frameintervals(struct file *filp, void *fh, static int iris_querycap(struct file *filp, void *fh, struct v4l2_capability *cap) { + struct iris_core *core = video_drvdata(filp); struct iris_inst *inst = iris_get_inst(filp); + char *info; strscpy(cap->driver, IRIS_DRV_NAME, sizeof(cap->driver)); - if (inst->domain == DECODER) + if (inst->domain == DECODER) { strscpy(cap->card, "Iris Decoder", sizeof(cap->card)); - else + info = "dec"; + } else { strscpy(cap->card, "Iris Encoder", sizeof(cap->card)); + info = "enc"; + } + snprintf(cap->bus_info, sizeof(cap->bus_info), + "plat:%s:%s", dev_name(core->dev), info); return 0; } diff --git a/drivers/media/platform/qcom/iris/iris_vpu_common.c b/drivers/media/platform/qcom/iris/iris_vpu_common.c index ab41da1f47c82..e4847c1077097 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu_common.c +++ b/drivers/media/platform/qcom/iris/iris_vpu_common.c @@ -237,7 +237,7 @@ void iris_vpu_power_off(struct iris_core *core) iris_unset_icc_bw(core); if (!iris_vpu_watchdog(core, core->intr_status)) - disable_irq_nosync(core->irq); + disable_irq(core->irq); } int iris_vpu_power_on_controller(struct iris_core *core) diff --git a/drivers/media/platform/qcom/venus/hfi_parser.c b/drivers/media/platform/qcom/venus/hfi_parser.c index b1657443f23f6..d07845f37c559 100644 --- a/drivers/media/platform/qcom/venus/hfi_parser.c +++ b/drivers/media/platform/qcom/venus/hfi_parser.c @@ -85,7 +85,7 @@ parse_alloc_mode(struct venus_core *core, u32 codecs, u32 domain, void *data) type++; } - return sizeof(*mode); + return mode->num_entries * sizeof(u32) + sizeof(*mode); } static void fill_profile_level(struct hfi_plat_caps *cap, const void *data, @@ -146,7 +146,7 @@ parse_caps(struct venus_core *core, u32 codecs, u32 domain, void *data) for_each_codec(core->caps, ARRAY_SIZE(core->caps), codecs, domain, fill_caps, caps_arr, num_caps); - return sizeof(*caps); + return num_caps * sizeof(*cap) + sizeof(u32); } static void fill_raw_fmts(struct hfi_plat_caps *cap, const void *fmts, @@ -171,7 +171,7 @@ parse_raw_formats(struct venus_core *core, u32 codecs, u32 domain, void *data) u32 entries = fmt->format_entries; unsigned int i = 0; u32 num_planes = 0; - u32 size; + u32 size = 2 * sizeof(u32); while (entries) { num_planes = pinfo->num_planes; @@ -186,6 +186,7 @@ parse_raw_formats(struct venus_core *core, u32 codecs, u32 domain, void *data) if (pinfo->num_planes > MAX_PLANES) break; + size += sizeof(*constr) * num_planes + 2 * sizeof(u32); pinfo = (void *)pinfo + sizeof(*constr) * num_planes + 2 * sizeof(u32); entries--; @@ -193,8 +194,6 @@ parse_raw_formats(struct venus_core *core, u32 codecs, u32 domain, void *data) for_each_codec(core->caps, ARRAY_SIZE(core->caps), codecs, domain, fill_raw_fmts, rawfmts, i); - size = fmt->format_entries * (sizeof(*constr) * num_planes + 2 * sizeof(u32)) - + 2 * sizeof(u32); return size; } diff --git a/drivers/media/platform/qcom/venus/pm_helpers.c b/drivers/media/platform/qcom/venus/pm_helpers.c index be1cbd5cfe842..e88e66be4f6df 100644 --- a/drivers/media/platform/qcom/venus/pm_helpers.c +++ b/drivers/media/platform/qcom/venus/pm_helpers.c @@ -781,7 +781,6 @@ static int decide_core(struct venus_inst *inst) unsigned long max_freq = ULONG_MAX; struct device *dev = core->dev; struct dev_pm_opp *opp; - int ret = 0; if (legacy_binding) { if (inst->session_type == VIDC_SESSION_TYPE_DEC) @@ -829,11 +828,7 @@ static int decide_core(struct venus_inst *inst) } done: - ret = hfi_session_set_property(inst, ptype, &cu); - if (ret) - return ret; - - return ret; + return hfi_session_set_property(inst, ptype, &cu); } static int acquire_core(struct venus_inst *inst) diff --git a/drivers/media/platform/renesas/rcar-csi2.c b/drivers/media/platform/renesas/rcar-csi2.c index 7305cc4a04cb1..6635f57821755 100644 --- a/drivers/media/platform/renesas/rcar-csi2.c +++ b/drivers/media/platform/renesas/rcar-csi2.c @@ -2273,11 +2273,7 @@ static int rcsi2_init_phtw_v3u(struct rcar_csi2 *priv, return ret; } - ret = rcsi2_phtw_write_array(priv, step4, ARRAY_SIZE(step4)); - if (ret) - return ret; - - return ret; + return rcsi2_phtw_write_array(priv, step4, ARRAY_SIZE(step4)); } /* ----------------------------------------------------------------------------- diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c index 5185a547461d7..91eda50342483 100644 --- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c +++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c @@ -851,6 +851,11 @@ static void rzg2l_cru_format_align(struct rzg2l_cru_dev *cru, v4l2_fill_pixfmt(pix, pix->pixelformat, pix->width, pix->height); + if (info->has_stride) { + pix->bytesperline = ALIGN(pix->bytesperline, RZG2L_CRU_STRIDE_ALIGN); + pix->sizeimage = pix->bytesperline * pix->height; + } + dev_dbg(cru->dev, "Format %ux%u bpl: %u size: %u\n", pix->width, pix->height, pix->bytesperline, pix->sizeimage); } diff --git a/drivers/media/platform/rockchip/rga/rga3-hw.c b/drivers/media/platform/rockchip/rga/rga3-hw.c index ca1c268303dd4..c4a71306278ba 100644 --- a/drivers/media/platform/rockchip/rga/rga3-hw.c +++ b/drivers/media/platform/rockchip/rga/rga3-hw.c @@ -450,10 +450,14 @@ static void *rga3_adjust_and_map_format(struct rga_ctx *ctx, other_format = is_output ? &ctx->out.pix : &ctx->in.pix; other_format_info = v4l2_format_info(other_format->pixelformat); - if ((v4l2_is_format_rgb(format_info) && - v4l2_is_format_yuv(other_format_info)) || - (v4l2_is_format_yuv(format_info) && - v4l2_is_format_rgb(other_format_info))) { + /* + * Only apply the quantization restrictions when we need to + * convert between RGB and YUV. Otherwise there is no point + * to limit the quantization for operations like scaling or + * rotations. + */ + if (v4l2_is_format_yuv(format_info) && + v4l2_is_format_rgb(other_format_info)) { /* * The RGA3 only supports BT601, BT709 and BT2020 RGB<->YUV conversions * Additionally BT709 and BT2020 only support limited range YUV. diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c index f89602075121b..2b8e04dd15727 100644 --- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c +++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c @@ -16,6 +16,7 @@ */ #include +#include #include #include "rkvdec.h" @@ -37,15 +38,17 @@ void compute_tiles_uniform(struct rkvdec_hevc_run *run, u16 log2_min_cb_size, s32 pic_in_cts_height, u16 *column_width, u16 *row_height) { const struct v4l2_ctrl_hevc_pps *pps = run->pps; + unsigned int num_cols = v4l2_hevc_pps_num_tile_columns(pps); + unsigned int num_rows = v4l2_hevc_pps_num_tile_rows(pps); int i; - for (i = 0; i < pps->num_tile_columns_minus1 + 1; i++) + for (i = 0; i < num_cols; i++) column_width[i] = ((i + 1) * pic_in_cts_width) / (pps->num_tile_columns_minus1 + 1) - (i * pic_in_cts_width) / (pps->num_tile_columns_minus1 + 1); - for (i = 0; i < pps->num_tile_rows_minus1 + 1; i++) + for (i = 0; i < num_rows; i++) row_height[i] = ((i + 1) * pic_in_cts_height) / (pps->num_tile_rows_minus1 + 1) - (i * pic_in_cts_height) / @@ -57,17 +60,20 @@ void compute_tiles_non_uniform(struct rkvdec_hevc_run *run, u16 log2_min_cb_size s32 pic_in_cts_height, u16 *column_width, u16 *row_height) { const struct v4l2_ctrl_hevc_pps *pps = run->pps; + unsigned int num_cols = v4l2_hevc_pps_num_tile_columns(pps); + unsigned int num_rows = v4l2_hevc_pps_num_tile_rows(pps); s32 sum = 0; int i; - for (i = 0; i < pps->num_tile_columns_minus1; i++) { + /* The last tile entry is written after the loop, so iterate one less. */ + for (i = 0; i < num_cols - 1; i++) { column_width[i] = pps->column_width_minus1[i] + 1; sum += column_width[i]; } column_width[i] = pic_in_cts_width - sum; sum = 0; - for (i = 0; i < pps->num_tile_rows_minus1; i++) { + for (i = 0; i < num_rows - 1; i++) { row_height[i] = pps->row_height_minus1[i] + 1; sum += row_height[i]; } @@ -209,6 +215,9 @@ static void st_ref_pic_set_prediction(struct rkvdec_hevc_run *run, int idx, int i, j; int dPoc; + if ((unsigned int)rps_data->delta_idx_minus1 + 1 > idx) + return; + ref_rps_idx = st_rps_idx - (rps_data->delta_idx_minus1 + 1); /* 7-59 */ delta_rps = (1 - 2 * rps_data->delta_rps_sign) * (rps_data->abs_delta_rps_minus1 + 1); /* 7-60 */ diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c index 87abf93dfd5e2..92373db50a466 100644 --- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c +++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c @@ -12,6 +12,7 @@ * Jeffy Chen */ +#include #include #include "rkvdec.h" @@ -135,6 +136,8 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx, * packet unit). so the driver copy SPS/PPS information to the exact PPS * packet unit for HW accessing. */ + if (pps->pic_parameter_set_id >= ARRAY_SIZE(priv_tbl->param_set)) + return; hw_ps = &priv_tbl->param_set[pps->pic_parameter_set_id]; memset(hw_ps, 0, sizeof(*hw_ps)); @@ -253,9 +256,9 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx, if (pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) { /* Userspace also provide column width and row height for uniform spacing */ - for (i = 0; i <= pps->num_tile_columns_minus1; i++) + for (i = 0; i < v4l2_hevc_pps_num_tile_columns(pps); i++) WRITE_PPS(pps->column_width_minus1[i], COLUMN_WIDTH(i)); - for (i = 0; i <= pps->num_tile_rows_minus1; i++) + for (i = 0; i < v4l2_hevc_pps_num_tile_rows(pps); i++) WRITE_PPS(pps->row_height_minus1[i], ROW_HEIGHT(i)); } else { WRITE_PPS(((sps->pic_width_in_luma_samples + ctb_size_y - 1) / ctb_size_y) - 1, diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c index fe6414a175510..6dafa1dd28507 100644 --- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c +++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c @@ -145,6 +145,8 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx, * packet unit). so the driver copy SPS/PPS information to the exact PPS * packet unit for HW accessing. */ + if (pps->pic_parameter_set_id >= ARRAY_SIZE(priv_tbl->param_set)) + return; hw_ps = &priv_tbl->param_set[pps->pic_parameter_set_id]; memset(hw_ps, 0, sizeof(*hw_ps)); diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec.c b/drivers/media/platform/rockchip/rkvdec/rkvdec.c index 1d1e9bfef8e96..061281f903f3c 100644 --- a/drivers/media/platform/rockchip/rkvdec/rkvdec.c +++ b/drivers/media/platform/rockchip/rkvdec/rkvdec.c @@ -278,12 +278,12 @@ static const struct rkvdec_ctrl_desc vdpu38x_hevc_ctrl_descs[] = { { .cfg.id = V4L2_CID_STATELESS_HEVC_EXT_SPS_ST_RPS, .cfg.ops = &rkvdec_ctrl_ops, - .cfg.dims = { 65 }, + .cfg.dims = { 64 }, }, { .cfg.id = V4L2_CID_STATELESS_HEVC_EXT_SPS_LT_RPS, .cfg.ops = &rkvdec_ctrl_ops, - .cfg.dims = { 65 }, + .cfg.dims = { 32 }, }, }; @@ -1818,8 +1818,8 @@ static int rkvdec_probe(struct platform_device *pdev) vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32)); irq = platform_get_irq(pdev, 0); - if (irq <= 0) - return -ENXIO; + if (irq < 0) + return irq; ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, rkvdec_irq_handler, IRQF_ONESHOT, diff --git a/drivers/media/platform/samsung/s3c-camif/camif-core.c b/drivers/media/platform/samsung/s3c-camif/camif-core.c index 221e3c447f361..a3c8b0a10eb3c 100644 --- a/drivers/media/platform/samsung/s3c-camif/camif-core.c +++ b/drivers/media/platform/samsung/s3c-camif/camif-core.c @@ -302,7 +302,6 @@ static int camif_media_dev_init(struct camif_dev *camif) struct media_device *md = &camif->media_dev; struct v4l2_device *v4l2_dev = &camif->v4l2_dev; unsigned int ip_rev = camif->variant->ip_revision; - int ret; memset(md, 0, sizeof(*md)); snprintf(md->model, sizeof(md->model), "Samsung S3C%s CAMIF", @@ -317,11 +316,7 @@ static int camif_media_dev_init(struct camif_dev *camif) media_device_init(md); - ret = v4l2_device_register(camif->dev, v4l2_dev); - if (ret < 0) - return ret; - - return ret; + return v4l2_device_register(camif->dev, v4l2_dev); } static void camif_clk_put(struct camif_dev *camif) diff --git a/drivers/media/platform/st/stm32/stm32-dcmi.c b/drivers/media/platform/st/stm32/stm32-dcmi.c index eeb0199864dd1..fc5acd5dbe7aa 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmi.c +++ b/drivers/media/platform/st/stm32/stm32-dcmi.c @@ -2024,8 +2024,10 @@ static int dcmi_probe(struct platform_device *pdev) mdma_chan = dma_request_chan(&pdev->dev, "mdma_tx"); if (IS_ERR(mdma_chan)) { ret = PTR_ERR(mdma_chan); - if (ret != -ENODEV) - return dev_err_probe(&pdev->dev, ret, "Failed to request MDMA channel\n"); + if (ret != -ENODEV) { + dev_err_probe(&pdev->dev, ret, "Failed to request MDMA channel\n"); + goto err_release_chan; + } mdma_chan = NULL; } @@ -2050,6 +2052,7 @@ static int dcmi_probe(struct platform_device *pdev) dcmi->sram_pool = of_gen_pool_get(pdev->dev.of_node, "sram", 0); if (!dcmi->sram_pool) { dev_info(&pdev->dev, "No SRAM pool, can't use MDMA chaining\n"); + ret = -ENOMEM; goto err_dma_slave_config; } @@ -2061,6 +2064,7 @@ static int dcmi_probe(struct platform_device *pdev) &dcmi->sram_dma_buf); if (!dcmi->sram_buf) { dev_err(dcmi->dev, "Failed to allocate from SRAM\n"); + ret = -ENOMEM; goto err_dma_slave_config; } @@ -2206,12 +2210,13 @@ static int dcmi_probe(struct platform_device *pdev) err_media_device_cleanup: media_device_cleanup(&dcmi->mdev); err_mdma_slave_config: - if (dcmi->mdma_chan) + if (mdma_chan) gen_pool_free(dcmi->sram_pool, (unsigned long)dcmi->sram_buf, dcmi->sram_buf_size); err_dma_slave_config: - dma_release_channel(dcmi->dma_chan); - if (dcmi->mdma_chan) + if (mdma_chan) dma_release_channel(mdma_chan); +err_release_chan: + dma_release_channel(chan); return ret; } diff --git a/drivers/media/platform/ti/Kconfig b/drivers/media/platform/ti/Kconfig index d0cb05481bd85..1a020b2bbb4f2 100644 --- a/drivers/media/platform/ti/Kconfig +++ b/drivers/media/platform/ti/Kconfig @@ -50,6 +50,7 @@ config VIDEO_TI_VIP select VIDEO_TI_VPDMA select VIDEO_TI_SC select VIDEO_TI_CSC + select V4L2_FWNODE help Driver support for VIP module on certain TI SoC's VIP = Video Input Port. diff --git a/drivers/media/platform/ti/vpe/vip.c b/drivers/media/platform/ti/vpe/vip.c index e56a95f53ea95..ccb688f4d7afd 100644 --- a/drivers/media/platform/ti/vpe/vip.c +++ b/drivers/media/platform/ti/vpe/vip.c @@ -815,6 +815,22 @@ static void clear_irqs(struct vip_dev *dev, int irq_num, int list_num) vpdma_clear_list_stat(dev->shared->vpdma, irq_num, dev->slice_id); } +/* + * Quiesce recovery work and per-list IRQs before releasing stream resources. + * disable_work_sync() prevents the overflow handler from requeueing recovery + * work. Mask and synchronize IRQs afterwards because a running worker may + * have re-enabled them before exiting. + */ +static void vip_quiesce_stream(struct vip_stream *stream) +{ + struct vip_dev *dev = stream->port->dev; + + disable_work_sync(&stream->recovery_work); + disable_irqs(dev, dev->slice_id, stream->list_num); + clear_irqs(dev, dev->slice_id, stream->list_num); + synchronize_irq(dev->irq); +} + static void populate_desc_list(struct vip_stream *stream) { struct vip_port *port = stream->port; @@ -2429,6 +2445,7 @@ static int vip_start_streaming(struct vb2_queue *vq, unsigned int count) goto err; stream->num_recovery = 0; + enable_work(&stream->recovery_work); clear_irqs(dev, dev->slice_id, stream->list_num); enable_irqs(dev, dev->slice_id, stream->list_num); @@ -2453,13 +2470,17 @@ static void vip_stop_streaming(struct vb2_queue *vq) struct vip_dev *dev = port->dev; int ret; + /* + * A running recovery worker may re-enable the parser, so quiesce it + * and its IRQ handler before stopping the parser or releasing the + * descriptor list. + */ + vip_quiesce_stream(stream); + vip_parser_stop_imm(port, true); vip_enable_parser(port, false); unset_fmt_params(stream); - disable_irqs(dev, dev->slice_id, stream->list_num); - clear_irqs(dev, dev->slice_id, stream->list_num); - if (port->subdev) { ret = v4l2_subdev_call(port->subdev, video, s_stream, 0); if (ret) @@ -3075,6 +3096,8 @@ static int alloc_stream(struct vip_port *port, int stream_id, int vfl_type) goto do_free_hwlist; INIT_WORK(&stream->recovery_work, vip_overflow_recovery_work); + /* Start disabled; vip_start_streaming() enables it before IRQs. */ + disable_work(&stream->recovery_work); INIT_LIST_HEAD(&stream->vidq); @@ -3140,6 +3163,13 @@ static void free_stream(struct vip_stream *stream) return; dev = stream->port->dev; + /* + * Quiesce the IRQ handler and recovery worker, then drop the stream + * from cap_streams[], before releasing stream-owned resources. + */ + vip_quiesce_stream(stream); + stream->port->cap_streams[stream->stream_id] = NULL; + /* Free up the Drop queue */ list_for_each_safe(pos, q, &stream->dropq) { buf = list_entry(pos, @@ -3151,7 +3181,6 @@ static void free_stream(struct vip_stream *stream) video_unregister_device(stream->vfd); vpdma_hwlist_release(dev->shared->vpdma, stream->list_num); - stream->port->cap_streams[stream->stream_id] = NULL; kfree(stream); } diff --git a/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c b/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c index e8c2e83379def..e7a7c7a42467a 100644 --- a/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c +++ b/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c @@ -5,6 +5,8 @@ * Copyright (C) 2020 Safran Passenger Innovations LLC */ +#include + #include "hantro_hw.h" #include "hantro_g2_regs.h" @@ -15,8 +17,8 @@ static void prepare_tile_info_buffer(struct hantro_ctx *ctx) const struct v4l2_ctrl_hevc_pps *pps = ctrls->pps; const struct v4l2_ctrl_hevc_sps *sps = ctrls->sps; u16 *p = (u16 *)((u8 *)ctx->hevc_dec.tile_sizes.cpu); - unsigned int num_tile_rows = pps->num_tile_rows_minus1 + 1; - unsigned int num_tile_cols = pps->num_tile_columns_minus1 + 1; + unsigned int num_tile_rows = v4l2_hevc_pps_num_tile_rows(pps); + unsigned int num_tile_cols = v4l2_hevc_pps_num_tile_columns(pps); unsigned int pic_width_in_ctbs, pic_height_in_ctbs; unsigned int max_log2_ctb_size, ctb_size; bool tiles_enabled, uniform_spacing; diff --git a/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c b/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c index e4e21ad373233..00aa566a4ccdb 100644 --- a/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c +++ b/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c @@ -431,20 +431,39 @@ static int rockchip_vpu981_av1_dec_prepare_run(struct hantro_ctx *ctx) { struct hantro_av1_dec_hw_ctx *av1_dec = &ctx->av1_dec; struct hantro_av1_dec_ctrls *ctrls = &av1_dec->ctrls; + const struct v4l2_av1_tile_info *tile_info; + struct v4l2_ctrl *tge; + u32 num_tiles; ctrls->sequence = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_AV1_SEQUENCE); if (WARN_ON(!ctrls->sequence)) return -EINVAL; - ctrls->tile_group_entry = - hantro_get_ctrl(ctx, V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY); - if (WARN_ON(!ctrls->tile_group_entry)) + tge = v4l2_ctrl_find(&ctx->ctrl_handler, + V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY); + if (WARN_ON(!tge)) return -EINVAL; + ctrls->tile_group_entry = tge->p_cur.p; ctrls->frame = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_AV1_FRAME); if (WARN_ON(!ctrls->frame)) return -EINVAL; + /* + * rockchip_vpu981_av1_dec_set_tile_info() indexes the tile group + * entry array by tile1 * tile_cols + tile0, so it reads up to + * tile_cols * tile_rows entries, and lays out one descriptor per tile + * in the AV1_MAX_TILES tile_info buffer while programming the real + * tile geometry into the hardware. Reject a frame that claims more + * tiles than userspace submitted, or more than the hardware tile + * buffer holds, so the read stays in bounds and the programmed + * geometry matches the descriptors written. + */ + tile_info = &ctrls->frame->tile_info; + num_tiles = (u32)tile_info->tile_cols * tile_info->tile_rows; + if (num_tiles > tge->elems || num_tiles > AV1_MAX_TILES) + return -EINVAL; + ctrls->film_grain = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_AV1_FILM_GRAIN); @@ -578,16 +597,30 @@ static void rockchip_vpu981_av1_dec_set_tile_info(struct hantro_ctx *ctx) const struct v4l2_av1_tile_info *tile_info = &ctrls->frame->tile_info; const struct v4l2_ctrl_av1_tile_group_entry *group_entry = ctrls->tile_group_entry; - int context_update_y = - tile_info->context_update_tile_id / tile_info->tile_cols; - int context_update_x = - tile_info->context_update_tile_id % tile_info->tile_cols; - int context_update_tile_id = - context_update_x * tile_info->tile_rows + context_update_y; + int context_update_y = 0; + int context_update_x = 0; + int context_update_tile_id = 0; u8 *dst = av1_dec->tile_info.cpu; + u8 *dst_end = dst + av1_dec->tile_info.size; struct hantro_dev *vpu = ctx->dev; int tile0, tile1; + /* + * tile_cols and tile_rows are bounded by the V4L2 control validation + * (V4L2_AV1_MAX_TILE_{COLS,ROWS} and V4L2_AV1_MAX_TILE_COUNT). Guard + * the divisor here, and keep the descriptor writes within the + * AV1_MAX_TILES tile_info buffer below; the register values use the + * unmodified tile geometry. + */ + if (tile_info->tile_cols) { + context_update_y = + tile_info->context_update_tile_id / tile_info->tile_cols; + context_update_x = + tile_info->context_update_tile_id % tile_info->tile_cols; + context_update_tile_id = + context_update_x * tile_info->tile_rows + context_update_y; + } + memset(dst, 0, av1_dec->tile_info.size); for (tile0 = 0; tile0 < tile_info->tile_cols; tile0++) { @@ -598,6 +631,10 @@ static void rockchip_vpu981_av1_dec_set_tile_info(struct hantro_ctx *ctx) tile_info->height_in_sbs_minus_1[tile1] + 1; u32 x0 = tile_info->width_in_sbs_minus_1[tile0] + 1; + /* Stop once the tile_info descriptor buffer is full. */ + if (dst + 16 > dst_end) + break; + /* tile size in SB units (width,height) */ *dst++ = x0; *dst++ = 0; @@ -622,6 +659,8 @@ static void rockchip_vpu981_av1_dec_set_tile_info(struct hantro_ctx *ctx) *dst++ = (end >> 16) & 255; *dst++ = (end >> 24) & 255; } + if (dst + 16 > dst_end) + break; } hantro_reg_write(vpu, &av1_multicore_expect_context_update, !!(context_update_x == 0)); diff --git a/drivers/media/rc/sunxi-cir.c b/drivers/media/rc/sunxi-cir.c index cb4c56bf0752a..28e840a7e5b8c 100644 --- a/drivers/media/rc/sunxi-cir.c +++ b/drivers/media/rc/sunxi-cir.c @@ -344,22 +344,25 @@ static int sunxi_ir_probe(struct platform_device *pdev) ir->irq = platform_get_irq(pdev, 0); if (ir->irq < 0) { ret = ir->irq; - goto exit_free_dev; + goto exit_unregister_dev; } ret = devm_request_irq(dev, ir->irq, sunxi_ir_irq, 0, SUNXI_IR_DEV, ir); if (ret) { dev_err(dev, "failed request irq\n"); - goto exit_free_dev; + goto exit_unregister_dev; } ret = sunxi_ir_hw_init(dev); if (ret) - goto exit_free_dev; + goto exit_unregister_dev; dev_info(dev, "initialized sunXi IR driver\n"); return 0; +exit_unregister_dev: + rc_unregister_device(ir->rc); + exit_free_dev: rc_free_device(ir->rc); diff --git a/drivers/media/test-drivers/vicodec/vicodec-core.c b/drivers/media/test-drivers/vicodec/vicodec-core.c index 318e8330f16ae..ff9d50fb05fd1 100644 --- a/drivers/media/test-drivers/vicodec/vicodec-core.c +++ b/drivers/media/test-drivers/vicodec/vicodec-core.c @@ -63,11 +63,11 @@ struct pixfmt_info { }; static const struct v4l2_fwht_pixfmt_info pixfmt_fwht = { - V4L2_PIX_FMT_FWHT, 0, 3, 1, 1, 1, 1, 1, 0, 1 + V4L2_PIX_FMT_FWHT, 0, 4, 1, 1, 1, 1, 1, 0, 1 }; static const struct v4l2_fwht_pixfmt_info pixfmt_stateless_fwht = { - V4L2_PIX_FMT_FWHT_STATELESS, 0, 3, 1, 1, 1, 1, 1, 0, 1 + V4L2_PIX_FMT_FWHT_STATELESS, 0, 4, 1, 1, 1, 1, 1, 0, 1 }; static void vicodec_dev_release(struct device *dev) diff --git a/drivers/media/test-drivers/vimc/vimc-capture.c b/drivers/media/test-drivers/vimc/vimc-capture.c index e0c6bc9f8f20c..ac60c9b5418e9 100644 --- a/drivers/media/test-drivers/vimc/vimc-capture.c +++ b/drivers/media/test-drivers/vimc/vimc-capture.c @@ -175,8 +175,8 @@ static int vimc_capture_enum_framesizes(struct file *file, void *priv, if (fsize->index) return -EINVAL; - /* Only accept code in the pix map table */ - vpix = vimc_pix_map_by_code(fsize->pixel_format); + /* Only accept pixel_format in the pix map table */ + vpix = vimc_pix_map_by_pixelformat(fsize->pixel_format); if (!vpix) return -EINVAL; diff --git a/drivers/media/tuners/tda18250.c b/drivers/media/tuners/tda18250.c index 7bb945ba09898..bd7ed50fb6e85 100644 --- a/drivers/media/tuners/tda18250.c +++ b/drivers/media/tuners/tda18250.c @@ -440,8 +440,8 @@ static int tda18250_pll_calc(struct dvb_frontend *fe, u8 *rdiv, goto err; exp = (uval & 0x70) >> 4; - if (exp > 5) - exp = 0; + if (exp == 0 || exp > 5) + exp = 1; lopd = 1 << (exp - 1); scale = uval & 0x0f; fvco = lopd * scale * ((c->frequency / 1000) + dev->if_frequency); diff --git a/drivers/media/usb/airspy/airspy.c b/drivers/media/usb/airspy/airspy.c index 57edb42463e81..358a66ab8e48a 100644 --- a/drivers/media/usb/airspy/airspy.c +++ b/drivers/media/usb/airspy/airspy.c @@ -464,14 +464,21 @@ static void airspy_disconnect(struct usb_interface *intf) dev_dbg(s->dev, "\n"); - mutex_lock(&s->vb_queue_lock); + /* + * vb2_video_unregister_device() releases the vb2 queue, which + * triggers airspy_stop_streaming() if streaming is active. + * stop_streaming() dereferences s->udev via airspy_ctrl_msg() and + * airspy_free_stream_bufs(), so it must run before s->udev is + * cleared. vb2_video_unregister_device() locks vb_queue_lock + * internally and stop_streaming() locks v4l2_lock, so neither may + * be held by the caller. + */ + v4l2_device_disconnect(&s->v4l2_dev); + vb2_video_unregister_device(&s->vdev); + mutex_lock(&s->v4l2_lock); - /* No need to keep the urbs around after disconnection */ s->udev = NULL; - v4l2_device_disconnect(&s->v4l2_dev); - video_unregister_device(&s->vdev); mutex_unlock(&s->v4l2_lock); - mutex_unlock(&s->vb_queue_lock); v4l2_device_put(&s->v4l2_dev); } diff --git a/drivers/media/usb/as102/as102_usb_drv.c b/drivers/media/usb/as102/as102_usb_drv.c index a11024451cebd..be2f8be560fae 100644 --- a/drivers/media/usb/as102/as102_usb_drv.c +++ b/drivers/media/usb/as102/as102_usb_drv.c @@ -24,35 +24,33 @@ static void as102_usb_stop_stream(struct as102_dev_t *dev); static int as102_open(struct inode *inode, struct file *file); static int as102_release(struct inode *inode, struct file *file); -static const struct usb_device_id as102_usb_id_table[] = { - { USB_DEVICE(AS102_USB_DEVICE_VENDOR_ID, AS102_USB_DEVICE_PID_0001) }, - { USB_DEVICE(PCTV_74E_USB_VID, PCTV_74E_USB_PID) }, - { USB_DEVICE(ELGATO_EYETV_DTT_USB_VID, ELGATO_EYETV_DTT_USB_PID) }, - { USB_DEVICE(NBOX_DVBT_DONGLE_USB_VID, NBOX_DVBT_DONGLE_USB_PID) }, - { USB_DEVICE(SKY_IT_DIGITAL_KEY_USB_VID, SKY_IT_DIGITAL_KEY_USB_PID) }, - { } /* Terminating entry */ +struct as102_dev_info { + const char *name; + /* + * eLNA configuration: devices built on the reference design work best + * with 0xA0, while custom designs seem to require 0xC0 + */ + uint8_t elna_cfg; }; -/* Note that this table must always have the same number of entries as the - as102_usb_id_table struct */ -static const char * const as102_device_names[] = { - AS102_REFERENCE_DESIGN, - AS102_PCTV_74E, - AS102_ELGATO_EYETV_DTT_NAME, - AS102_NBOX_DVBT_DONGLE_NAME, - AS102_SKY_IT_DIGITAL_KEY_NAME, - NULL /* Terminating entry */ -}; +#define DRIVER_INFO(dev_name, dev_elna_cfg) \ + .driver_info = (kernel_ulong_t)&(const struct as102_dev_info){ \ + .name = (dev_name), \ + .elna_cfg = (dev_elna_cfg), \ + } -/* eLNA configuration: devices built on the reference design work best - with 0xA0, while custom designs seem to require 0xC0 */ -static uint8_t const as102_elna_cfg[] = { - 0xA0, - 0xC0, - 0xC0, - 0xA0, - 0xA0, - 0x00 /* Terminating entry */ +static const struct usb_device_id as102_usb_id_table[] = { + { USB_DEVICE(AS102_USB_DEVICE_VENDOR_ID, AS102_USB_DEVICE_PID_0001), + DRIVER_INFO(AS102_REFERENCE_DESIGN, 0xA0) }, + { USB_DEVICE(PCTV_74E_USB_VID, PCTV_74E_USB_PID), + DRIVER_INFO(AS102_PCTV_74E, 0xC0) }, + { USB_DEVICE(ELGATO_EYETV_DTT_USB_VID, ELGATO_EYETV_DTT_USB_PID), + DRIVER_INFO(AS102_ELGATO_EYETV_DTT_NAME, 0xC0) }, + { USB_DEVICE(NBOX_DVBT_DONGLE_USB_VID, NBOX_DVBT_DONGLE_USB_PID), + DRIVER_INFO(AS102_NBOX_DVBT_DONGLE_NAME, 0xA0) }, + { USB_DEVICE(SKY_IT_DIGITAL_KEY_USB_VID, SKY_IT_DIGITAL_KEY_USB_PID), + DRIVER_INFO(AS102_SKY_IT_DIGITAL_KEY_NAME, 0xA0) }, + { } /* Terminating entry */ }; struct usb_driver as102_usb_driver = { @@ -336,29 +334,18 @@ static int as102_usb_probe(struct usb_interface *intf, { int ret; struct as102_dev_t *as102_dev; - int i; - - /* This should never actually happen */ - if (ARRAY_SIZE(as102_usb_id_table) != - (sizeof(as102_device_names) / sizeof(const char *))) { - pr_err("Device names table invalid size"); - return -EINVAL; - } + const struct as102_dev_info *info = (const struct as102_dev_info *)id->driver_info; as102_dev = kzalloc_obj(struct as102_dev_t); if (as102_dev == NULL) return -ENOMEM; - /* Assign the user-friendly device name */ - for (i = 0; i < ARRAY_SIZE(as102_usb_id_table); i++) { - if (id == &as102_usb_id_table[i]) { - as102_dev->name = as102_device_names[i]; - as102_dev->elna_cfg = as102_elna_cfg[i]; - } - } - - if (as102_dev->name == NULL) + if (info) { + as102_dev->name = info->name; + as102_dev->elna_cfg = info->elna_cfg; + } else { as102_dev->name = "Unknown AS102 device"; + } /* set private callback functions */ as102_dev->bus_adap.ops = &as102_priv_ops; diff --git a/drivers/media/usb/cx231xx/cx231xx-video.c b/drivers/media/usb/cx231xx/cx231xx-video.c index 2cd4e333bc4b0..70aa99fead278 100644 --- a/drivers/media/usb/cx231xx/cx231xx-video.c +++ b/drivers/media/usb/cx231xx/cx231xx-video.c @@ -898,7 +898,7 @@ static int vidioc_s_fmt_vid_cap(struct file *file, void *priv, if (rc) return rc; - if (vb2_is_busy(&dev->vidq)) { + if (vb2_is_busy(&dev->vidq) || vb2_is_busy(&dev->vbiq)) { dev_err(dev->dev, "%s: queue busy\n", __func__); return -EBUSY; } @@ -933,7 +933,7 @@ static int vidioc_s_std(struct file *file, void *priv, v4l2_std_id norm) if (dev->norm == norm) return 0; - if (vb2_is_busy(&dev->vidq)) + if (vb2_is_busy(&dev->vidq) || vb2_is_busy(&dev->vbiq)) return -EBUSY; dev->norm = norm; diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf.c b/drivers/media/usb/dvb-usb-v2/mxl111sf.c index 870ac3c8b085d..6404eb74db32f 100644 --- a/drivers/media/usb/dvb-usb-v2/mxl111sf.c +++ b/drivers/media/usb/dvb-usb-v2/mxl111sf.c @@ -987,11 +987,7 @@ static int mxl111sf_frontend_attach_atsc_mh(struct dvb_usb_adapter *adap) if (ret < 0) return ret; - ret = mxl111sf_lg2160_frontend_attach(adap, 2); - if (ret < 0) - return ret; - - return ret; + return mxl111sf_lg2160_frontend_attach(adap, 2); } static int mxl111sf_frontend_attach_mercury(struct dvb_usb_adapter *adap) @@ -1007,11 +1003,7 @@ static int mxl111sf_frontend_attach_mercury(struct dvb_usb_adapter *adap) if (ret < 0) return ret; - ret = mxl111sf_lg2161_ep6_frontend_attach(adap, 2); - if (ret < 0) - return ret; - - return ret; + return mxl111sf_lg2161_ep6_frontend_attach(adap, 2); } static int mxl111sf_frontend_attach_mercury_mh(struct dvb_usb_adapter *adap) diff --git a/drivers/media/usb/em28xx/em28xx-cards.c b/drivers/media/usb/em28xx/em28xx-cards.c index fbfb74eab4759..a6881f7f8e1be 100644 --- a/drivers/media/usb/em28xx/em28xx-cards.c +++ b/drivers/media/usb/em28xx/em28xx-cards.c @@ -3651,6 +3651,7 @@ static void request_module_async(struct work_struct *work) * intf. Don't register extensions twice on those devices. */ if (dev->is_audio_only) { + em28xx_init_extension(dev); #if defined(CONFIG_MODULES) && defined(MODULE) request_module("em28xx-alsa"); #endif @@ -3884,8 +3885,6 @@ static int em28xx_init_dev(struct em28xx *dev, struct usb_device *udev, retval = -ENODEV; goto err_deinit_media; } - em28xx_init_extension(dev); - return 0; } diff --git a/drivers/media/usb/em28xx/em28xx-core.c b/drivers/media/usb/em28xx/em28xx-core.c index 5bbb082dbed9b..3d6b196b84830 100644 --- a/drivers/media/usb/em28xx/em28xx-core.c +++ b/drivers/media/usb/em28xx/em28xx-core.c @@ -1265,6 +1265,8 @@ void em28xx_close_extension(struct em28xx *dev) ops->fini(dev); } } + if (dev->dev_next) + list_del(&dev->dev_next->devlist); list_del(&dev->devlist); mutex_unlock(&em28xx_devlist_mutex); } diff --git a/drivers/media/usb/go7007/snd-go7007.c b/drivers/media/usb/go7007/snd-go7007.c index e4b9f37be77b4..01d547ad678d4 100644 --- a/drivers/media/usb/go7007/snd-go7007.c +++ b/drivers/media/usb/go7007/snd-go7007.c @@ -195,6 +195,13 @@ static const struct snd_device_ops go7007_snd_device_ops = { .dev_free = go7007_snd_free, }; +static void go7007_snd_card_free(struct snd_card *card) +{ + struct go7007 *go = card->private_data; + + v4l2_device_put(&go->v4l2_dev); +} + int go7007_snd_init(struct go7007 *go) { static int dev; @@ -245,6 +252,8 @@ int go7007_snd_init(struct go7007 *go) gosnd->substream = NULL; go->snd_context = gosnd; v4l2_device_get(&go->v4l2_dev); + gosnd->card->private_data = go; + gosnd->card->private_free = go7007_snd_card_free; ++dev; return 0; @@ -263,7 +272,6 @@ int go7007_snd_remove(struct go7007 *go) snd_card_disconnect(gosnd->card); snd_card_free_when_closed(gosnd->card); - v4l2_device_put(&go->v4l2_dev); return 0; } EXPORT_SYMBOL(go7007_snd_remove); diff --git a/drivers/media/usb/gspca/jl2005bcd.c b/drivers/media/usb/gspca/jl2005bcd.c index a408fcc3a0608..4988fbf5005e6 100644 --- a/drivers/media/usb/gspca/jl2005bcd.c +++ b/drivers/media/usb/gspca/jl2005bcd.c @@ -148,17 +148,12 @@ static int jl2005c_start_new_frame(struct gspca_dev *gspca_dev) static int jl2005c_write_reg(struct gspca_dev *gspca_dev, unsigned char reg, unsigned char value) { - int retval; u8 instruction[2]; instruction[0] = reg; instruction[1] = value; - retval = jl2005c_write2(gspca_dev, instruction); - if (retval < 0) - return retval; - - return retval; + return jl2005c_write2(gspca_dev, instruction); } static int jl2005c_get_firmware_id(struct gspca_dev *gspca_dev) diff --git a/drivers/media/usb/s2255/s2255drv.c b/drivers/media/usb/s2255/s2255drv.c index 0b8182edf8e4b..58658dd3e457a 100644 --- a/drivers/media/usb/s2255/s2255drv.c +++ b/drivers/media/usb/s2255/s2255drv.c @@ -617,6 +617,12 @@ static void s2255_fillbuff(struct s2255_vc *vc, break; case V4L2_PIX_FMT_JPEG: case V4L2_PIX_FMT_MJPEG: + if (jpgsize < 0 || + jpgsize > vb2_plane_size(&buf->vb.vb2_buf, 0)) { + dprintk(dev, 1, "bad JPEG frame size %d\n", + jpgsize); + break; + } vb2_set_plane_payload(&buf->vb.vb2_buf, 0, jpgsize); memcpy(vbuf, tmpbuf, jpgsize); break; @@ -2277,6 +2283,11 @@ static int s2255_probe(struct usb_interface *interface, } /* check the firmware is valid */ fw_size = dev->fw_data->fw->size; + if (fw_size < 8) { + dev_err(&interface->dev, "Firmware invalid: too small.\n"); + retval = -ENODEV; + goto errorFWMARKER; + } pdata = (__le32 *) &dev->fw_data->fw->data[fw_size - 8]; if (*pdata != S2255_FW_MARKER) { diff --git a/drivers/media/usb/usbtv/usbtv-audio.c b/drivers/media/usb/usbtv/usbtv-audio.c index 333bd305a4f9f..ae0a14e5ed2ab 100644 --- a/drivers/media/usb/usbtv/usbtv-audio.c +++ b/drivers/media/usb/usbtv/usbtv-audio.c @@ -317,6 +317,13 @@ static const struct snd_pcm_ops snd_usbtv_pcm_ops = { .pointer = snd_usbtv_pointer, }; +static void usbtv_audio_card_free(struct snd_card *card) +{ + struct usbtv *usbtv = card->private_data; + + v4l2_device_put(&usbtv->v4l2_dev); +} + int usbtv_audio_init(struct usbtv *usbtv) { int rv; @@ -331,6 +338,10 @@ int usbtv_audio_init(struct usbtv *usbtv) if (rv < 0) return rv; + v4l2_device_get(&usbtv->v4l2_dev); + card->private_data = usbtv; + card->private_free = usbtv_audio_card_free; + strscpy(card->driver, usbtv->dev->driver->name, sizeof(card->driver)); strscpy(card->shortname, "usbtv", sizeof(card->shortname)); snprintf(card->longname, sizeof(card->longname), diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c index 888a2e213b088..460bf3dbbb882 100644 --- a/drivers/media/v4l2-core/v4l2-async.c +++ b/drivers/media/v4l2-core/v4l2-async.c @@ -392,7 +392,6 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, err_call_unbind: v4l2_async_nf_call_unbind(notifier, sd, asc); - list_del(&asc->asc_subdev_entry); err_unregister_subdev: if (registered) @@ -898,9 +897,18 @@ void v4l2_async_unregister_subdev(struct v4l2_subdev *sd) sd->subdev_notifier = NULL; if (sd->asc_list.next) { - list_for_each_entry_safe(asc, asc_tmp, &sd->asc_list, - asc_subdev_entry) { - v4l2_async_unbind_subdev_one(asc->notifier, asc); + if (list_empty(&sd->asc_list)) { + /* + * If the sub-device was registered through other means + * than v4l2-async, there are no async connections but + * the sub-device may still well be registered. + * Unregister it now. + */ + v4l2_device_unregister_subdev(sd); + } else { + list_for_each_entry_safe(asc, asc_tmp, &sd->asc_list, + asc_subdev_entry) + v4l2_async_unbind_subdev_one(asc->notifier, asc); } } diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c index ba047d7d86010..648b88c868bc6 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls-core.c +++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c @@ -16,6 +16,9 @@ static const union v4l2_ctrl_ptr ptr_null; +#define V4L2_HEVC_MAX_SHORT_TERM_REF_PIC_SETS 64 +#define V4L2_HEVC_MAX_LONG_TERM_REF_PICS_SPS 32 + static void fill_event(struct v4l2_event *ev, struct v4l2_ctrl *ctrl, u32 changes) { @@ -790,10 +793,30 @@ static int validate_av1_film_grain(struct v4l2_ctrl_av1_film_grain *fg) return 0; } +static int validate_av1_tile_info(struct v4l2_av1_tile_info *t) +{ + /* + * tile_cols and tile_rows index the per-tile descriptor arrays and + * bound the tile loops in the stateless AV1 drivers; the product + * bounds the total tile descriptor count. + */ + if (t->tile_cols > V4L2_AV1_MAX_TILE_COLS || + t->tile_rows > V4L2_AV1_MAX_TILE_ROWS) + return -EINVAL; + + if ((u32)t->tile_cols * t->tile_rows > V4L2_AV1_MAX_TILE_COUNT) + return -EINVAL; + + return 0; +} + static int validate_av1_frame(struct v4l2_ctrl_av1_frame *f) { int ret = 0; + ret = validate_av1_tile_info(&f->tile_info); + if (ret) + return ret; ret = validate_av1_quantization(&f->quantization); if (ret) return ret; @@ -1214,6 +1237,10 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, case V4L2_CTRL_TYPE_HEVC_SPS: p_hevc_sps = p; + if (p_hevc_sps->num_short_term_ref_pic_sets > + V4L2_HEVC_MAX_SHORT_TERM_REF_PIC_SETS) + return -EINVAL; + if (!(p_hevc_sps->flags & V4L2_HEVC_SPS_FLAG_PCM_ENABLED)) { p_hevc_sps->pcm_sample_bit_depth_luma_minus1 = 0; p_hevc_sps->pcm_sample_bit_depth_chroma_minus1 = 0; @@ -1224,6 +1251,9 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, if (!(p_hevc_sps->flags & V4L2_HEVC_SPS_FLAG_LONG_TERM_REF_PICS_PRESENT)) p_hevc_sps->num_long_term_ref_pics_sps = 0; + else if (p_hevc_sps->num_long_term_ref_pics_sps > + V4L2_HEVC_MAX_LONG_TERM_REF_PICS_SPS) + return -EINVAL; break; case V4L2_CTRL_TYPE_HEVC_PPS: @@ -1243,6 +1273,18 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, p_hevc_pps->flags &= ~V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED; + } else { + /* + * These count the entries the stateless HEVC drivers + * read from column_width_minus1[] / row_height_minus1[] + * and use as tile-loop bounds. + */ + if (p_hevc_pps->num_tile_columns_minus1 >= + ARRAY_SIZE(p_hevc_pps->column_width_minus1)) + return -EINVAL; + if (p_hevc_pps->num_tile_rows_minus1 >= + ARRAY_SIZE(p_hevc_pps->row_height_minus1)) + return -EINVAL; } if (p_hevc_pps->flags & @@ -1280,6 +1322,11 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, if (p_hevc_st_rps->flags & ~V4L2_HEVC_EXT_SPS_ST_RPS_FLAG_INTER_REF_PIC_SET_PRED) return -EINVAL; + if (p_hevc_st_rps->num_negative_pics > 16 || + p_hevc_st_rps->num_positive_pics > 16 || + p_hevc_st_rps->num_negative_pics + + p_hevc_st_rps->num_positive_pics > 16) + return -EINVAL; break; case V4L2_CTRL_TYPE_HEVC_EXT_SPS_LT_RPS: @@ -1307,24 +1354,41 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, return -EINVAL; } - if (p_hdr10_mastering->white_point_x < - V4L2_HDR10_MASTERING_WHITE_POINT_X_LOW || - p_hdr10_mastering->white_point_x > - V4L2_HDR10_MASTERING_WHITE_POINT_X_HIGH || - p_hdr10_mastering->white_point_y < - V4L2_HDR10_MASTERING_WHITE_POINT_Y_LOW || - p_hdr10_mastering->white_point_y > - V4L2_HDR10_MASTERING_WHITE_POINT_Y_HIGH) - return -EINVAL; - - if (p_hdr10_mastering->max_display_mastering_luminance < - V4L2_HDR10_MASTERING_MAX_LUMA_LOW || - p_hdr10_mastering->max_display_mastering_luminance > - V4L2_HDR10_MASTERING_MAX_LUMA_HIGH || - p_hdr10_mastering->min_display_mastering_luminance < - V4L2_HDR10_MASTERING_MIN_LUMA_LOW || - p_hdr10_mastering->min_display_mastering_luminance > - V4L2_HDR10_MASTERING_MIN_LUMA_HIGH) + /* + * SMPTE ST 2086 Annex A documents that CTA 861-G uses + * (0, 0) to indicate that the white point chromaticity + * is unknown. + */ + if (p_hdr10_mastering->white_point_x || + p_hdr10_mastering->white_point_y) { + if (p_hdr10_mastering->white_point_x < + V4L2_HDR10_MASTERING_WHITE_POINT_X_LOW || + p_hdr10_mastering->white_point_x > + V4L2_HDR10_MASTERING_WHITE_POINT_X_HIGH || + p_hdr10_mastering->white_point_y < + V4L2_HDR10_MASTERING_WHITE_POINT_Y_LOW || + p_hdr10_mastering->white_point_y > + V4L2_HDR10_MASTERING_WHITE_POINT_Y_HIGH) + return -EINVAL; + } + + /* + * SMPTE ST 2086 Annex A documents that CTA 861-G uses zero + * maximum and minimum luminance values to indicate that + * the corresponding values are unknown. + */ + if (p_hdr10_mastering->max_display_mastering_luminance && + (p_hdr10_mastering->max_display_mastering_luminance < + V4L2_HDR10_MASTERING_MAX_LUMA_LOW || + p_hdr10_mastering->max_display_mastering_luminance > + V4L2_HDR10_MASTERING_MAX_LUMA_HIGH)) + return -EINVAL; + + if (p_hdr10_mastering->min_display_mastering_luminance && + (p_hdr10_mastering->min_display_mastering_luminance < + V4L2_HDR10_MASTERING_MIN_LUMA_LOW || + p_hdr10_mastering->min_display_mastering_luminance > + V4L2_HDR10_MASTERING_MIN_LUMA_HIGH)) return -EINVAL; /* The following restriction comes from ITU-T Rec. H.265 spec */ diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c index 5516b2bbb08f0..d750bf10febec 100644 --- a/drivers/media/v4l2-core/v4l2-dev.c +++ b/drivers/media/v4l2-core/v4l2-dev.c @@ -1071,25 +1071,25 @@ int __video_register_device(struct video_device *vdev, vdev->dev.class = &video_class; vdev->dev.devt = MKDEV(VIDEO_MAJOR, vdev->minor); vdev->dev.parent = vdev->dev_parent; - vdev->dev.release = v4l2_device_release; dev_set_name(&vdev->dev, "%s%d", name_base, vdev->num); - - /* Increase v4l2_device refcount */ - v4l2_device_get(vdev->v4l2_dev); - mutex_lock(&videodev_lock); ret = device_register(&vdev->dev); if (ret < 0) { mutex_unlock(&videodev_lock); pr_err("%s: device_register failed\n", __func__); - put_device(&vdev->dev); - return ret; + goto cleanup; } + /* Register the release callback that will be called when the last + reference to the device goes away. */ + vdev->dev.release = v4l2_device_release; if (nr != -1 && nr != vdev->num && warn_if_nr_in_use) pr_warn("%s: requested %s%d, got %s\n", __func__, name_base, nr, video_device_node_name(vdev)); + /* Increase v4l2_device refcount */ + v4l2_device_get(vdev->v4l2_dev); + /* Part 5: Register the entity. */ ret = video_register_media_controller(vdev); diff --git a/drivers/media/v4l2-core/v4l2-fwnode.c b/drivers/media/v4l2-core/v4l2-fwnode.c index 62a3a452f7884..570df3c415b0a 100644 --- a/drivers/media/v4l2-core/v4l2-fwnode.c +++ b/drivers/media/v4l2-core/v4l2-fwnode.c @@ -633,6 +633,8 @@ int v4l2_fwnode_parse_link(struct fwnode_handle *fwnode, if (!link->remote_node) goto err_put_remote_endpoint; + fwnode_handle_put(fwnode); + return 0; err_put_remote_endpoint: diff --git a/drivers/media/v4l2-core/v4l2-h264.c b/drivers/media/v4l2-core/v4l2-h264.c index c00197d095e75..2323f559c6a3b 100644 --- a/drivers/media/v4l2-core/v4l2-h264.c +++ b/drivers/media/v4l2-core/v4l2-h264.c @@ -440,7 +440,8 @@ v4l2_h264_build_b_ref_lists(const struct v4l2_h264_reflist_builder *builder, } if (builder->num_valid > 1 && - !memcmp(b1_reflist, b0_reflist, builder->num_valid)) + !memcmp(b1_reflist, b0_reflist, + sizeof(b1_reflist[0]) * builder->num_valid)) swap(b1_reflist[0], b1_reflist[1]); print_ref_list_b(builder, b0_reflist, 0); diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c index ce33907bfc242..65154e569a9e7 100644 --- a/drivers/memstick/core/ms_block.c +++ b/drivers/memstick/core/ms_block.c @@ -2204,6 +2204,8 @@ static void msb_remove(struct memstick_dev *card) msb_data_clear(msb); mutex_unlock(&msb_disk_lock); + destroy_workqueue(msb->io_queue); + put_disk(msb->disk); memstick_set_drvdata(card, NULL); } diff --git a/drivers/mfd/ab8500-core.c b/drivers/mfd/ab8500-core.c index f0bc0b5a6f4ad..86fa99022cb35 100644 --- a/drivers/mfd/ab8500-core.c +++ b/drivers/mfd/ab8500-core.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include /* diff --git a/drivers/mfd/cgbc-core.c b/drivers/mfd/cgbc-core.c index 10bb4b414c34d..2becaf797646d 100644 --- a/drivers/mfd/cgbc-core.c +++ b/drivers/mfd/cgbc-core.c @@ -364,9 +364,9 @@ static void cgbc_remove(struct platform_device *pdev) { struct cgbc_device_data *cgbc = platform_get_drvdata(pdev); - cgbc_session_release(cgbc); - mfd_remove_devices(&pdev->dev); + + cgbc_session_release(cgbc); } static struct platform_driver cgbc_driver = { diff --git a/drivers/mfd/cs42l43.c b/drivers/mfd/cs42l43.c index ed6d93893de04..af7fee9ee2cf7 100644 --- a/drivers/mfd/cs42l43.c +++ b/drivers/mfd/cs42l43.c @@ -115,14 +115,14 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = { { CS42L43_DECIM_HPF_WNF_CTRL2, 0x00000001 }, { CS42L43_DECIM_HPF_WNF_CTRL3, 0x00000001 }, { CS42L43_DECIM_HPF_WNF_CTRL4, 0x00000001 }, - { CS42L43B_DECIM_HPF_WNF_CTRL5, 0x00000001 }, - { CS42L43B_DECIM_HPF_WNF_CTRL6, 0x00000001 }, { CS42L43_DMIC_PDM_CTRL, 0x00000000 }, { CS42L43_DECIM_VOL_CTRL_CH1_CH2, 0x20122012 }, { CS42L43_DECIM_VOL_CTRL_CH3_CH4, 0x20122012 }, { CS42L43B_DECIM_VOL_CTRL_CH1_CH2, 0x20122012 }, { CS42L43B_DECIM_VOL_CTRL_CH3_CH4, 0x20122012 }, { CS42L43B_DECIM_VOL_CTRL_CH5_CH6, 0x20122012 }, + { CS42L43B_DECIM_HPF_WNF_CTRL5, 0x00000001 }, + { CS42L43B_DECIM_HPF_WNF_CTRL6, 0x00000001 }, { CS42L43_INTP_VOLUME_CTRL1, 0x00000180 }, { CS42L43_INTP_VOLUME_CTRL2, 0x00000180 }, { CS42L43_AMP1_2_VOL_RAMP, 0x00000022 }, @@ -160,10 +160,10 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = { { CS42L43_SWIRE_DP2_CH2_INPUT, 0x00000000 }, { CS42L43_SWIRE_DP3_CH1_INPUT, 0x00000000 }, { CS42L43_SWIRE_DP3_CH2_INPUT, 0x00000000 }, - { CS42L43B_SWIRE_DP3_CH3_INPUT, 0x00000000 }, - { CS42L43B_SWIRE_DP3_CH4_INPUT, 0x00000000 }, { CS42L43_SWIRE_DP4_CH1_INPUT, 0x00000000 }, { CS42L43_SWIRE_DP4_CH2_INPUT, 0x00000000 }, + { CS42L43B_SWIRE_DP3_CH3_INPUT, 0x00000000 }, + { CS42L43B_SWIRE_DP3_CH4_INPUT, 0x00000000 }, { CS42L43B_SWIRE_DP4_CH3_INPUT, 0x00000000 }, { CS42L43B_SWIRE_DP4_CH4_INPUT, 0x00000000 }, { CS42L43_ASRC_INT1_INPUT1, 0x00000000 }, @@ -178,14 +178,10 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = { { CS42L43_ISRC1INT2_INPUT1, 0x00000000 }, { CS42L43_ISRC1DEC1_INPUT1, 0x00000000 }, { CS42L43_ISRC1DEC2_INPUT1, 0x00000000 }, - { CS42L43B_ISRC1DEC3_INPUT1, 0x00000000 }, - { CS42L43B_ISRC1DEC4_INPUT1, 0x00000000 }, { CS42L43_ISRC2INT1_INPUT1, 0x00000000 }, { CS42L43_ISRC2INT2_INPUT1, 0x00000000 }, { CS42L43_ISRC2DEC1_INPUT1, 0x00000000 }, { CS42L43_ISRC2DEC2_INPUT1, 0x00000000 }, - { CS42L43B_ISRC2DEC3_INPUT1, 0x00000000 }, - { CS42L43B_ISRC2DEC4_INPUT1, 0x00000000 }, { CS42L43_EQ1MIX_INPUT1, 0x00800000 }, { CS42L43_EQ1MIX_INPUT2, 0x00800000 }, { CS42L43_EQ1MIX_INPUT3, 0x00800000 }, @@ -212,6 +208,10 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = { { CS42L43_AMP4MIX_INPUT2, 0x00800000 }, { CS42L43_AMP4MIX_INPUT3, 0x00800000 }, { CS42L43_AMP4MIX_INPUT4, 0x00800000 }, + { CS42L43B_ISRC1DEC3_INPUT1, 0x00000000 }, + { CS42L43B_ISRC1DEC4_INPUT1, 0x00000000 }, + { CS42L43B_ISRC2DEC3_INPUT1, 0x00000000 }, + { CS42L43B_ISRC2DEC4_INPUT1, 0x00000000 }, { CS42L43_ASRC_INT_ENABLES, 0x00000100 }, { CS42L43_ASRC_DEC_ENABLES, 0x00000100 }, { CS42L43_PDNCNTL, 0x00000000 }, diff --git a/drivers/mfd/db8500-prcmu.c b/drivers/mfd/db8500-prcmu.c index 21e68a382b114..6672c55f2ebcc 100644 --- a/drivers/mfd/db8500-prcmu.c +++ b/drivers/mfd/db8500-prcmu.c @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include #include @@ -2285,7 +2285,7 @@ void db8500_prcmu_system_reset(u16 reset_code) /** * db8500_prcmu_get_reset_code - Retrieve SW reset reason code * - * Retrieves the reset reason code stored by prcmu_system_reset() before + * Retrieves the reset reason code stored by db8500_prcmu_system_reset() before * last restart. */ u16 db8500_prcmu_get_reset_code(void) @@ -3041,7 +3041,7 @@ static int db8500_prcmu_probe(struct platform_device *pdev) db8500_irq_init(np); - prcmu_config_esram0_deep_sleep(ESRAM0_DEEP_SLEEP_STATE_RET); + db8500_prcmu_config_esram0_deep_sleep(ESRAM0_DEEP_SLEEP_STATE_RET); err = mfd_add_devices(&pdev->dev, 0, common_prcmu_devs, ARRAY_SIZE(common_prcmu_devs), NULL, 0, db8500_irq_domain); diff --git a/drivers/mfd/iqs62x.c b/drivers/mfd/iqs62x.c index ee017617d1d1b..412ae7777f729 100644 --- a/drivers/mfd/iqs62x.c +++ b/drivers/mfd/iqs62x.c @@ -237,7 +237,7 @@ static int iqs62x_firmware_parse(struct iqs62x_core *iqs62x, fw_rec = (struct iqs62x_fw_rec *)(fw->data + pos); pos += sizeof(*fw_rec); - if (pos + fw_rec->len - 1 > fw->size) { + if (!fw_rec->len || fw_rec->len - 1 > fw->size - pos) { ret = -EINVAL; break; } diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 358feec2d088f..514cba7dc897a 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -410,7 +410,7 @@ static int apple_smc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_smc *smc; - u32 count; + __be32 count; int ret; smc = devm_kzalloc(dev, sizeof(*smc), GFP_KERNEL); @@ -461,8 +461,10 @@ static int apple_smc_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, smc); BLOCKING_INIT_NOTIFIER_HEAD(&smc->event_handlers); - ret = apple_smc_read_u32(smc, SMC_KEY(#KEY), &count); - if (ret) + ret = apple_smc_read(smc, SMC_KEY(#KEY), &count, sizeof(count)); + if (ret >= 0 && ret != sizeof(count)) + ret = -EINVAL; + if (ret < 0) return dev_err_probe(smc->dev, ret, "Failed to get key count"); smc->key_count = be32_to_cpu(count); diff --git a/drivers/mfd/qnap-mcu.c b/drivers/mfd/qnap-mcu.c index 8de974ddac3e7..93a3dd93404dd 100644 --- a/drivers/mfd/qnap-mcu.c +++ b/drivers/mfd/qnap-mcu.c @@ -56,6 +56,7 @@ struct qnap_mcu_reply { * @reply: Reply data structure * @variant: Device variant specific information * @version: MCU firmware version + * @rx: Receive buffer the reply is assembled in */ struct qnap_mcu { struct serdev_device *serdev; @@ -63,6 +64,7 @@ struct qnap_mcu { struct qnap_mcu_reply reply; const struct qnap_mcu_variant *variant; u8 version[QNAP_MCU_VERSION_LEN]; + u8 rx[QNAP_MCU_RX_BUFFER_SIZE]; }; /* @@ -214,19 +216,18 @@ int qnap_mcu_exec(struct qnap_mcu *mcu, const u8 *cmd_data, size_t cmd_data_size, u8 *reply_data, size_t reply_data_size) { - unsigned char rx[QNAP_MCU_RX_BUFFER_SIZE]; size_t length = reply_data_size + QNAP_MCU_CHECKSUM_SIZE; struct qnap_mcu_reply *reply = &mcu->reply; int ret = 0; - if (length > sizeof(rx)) { + if (length > sizeof(mcu->rx)) { dev_err(&mcu->serdev->dev, "expected data too big for receive buffer"); return -EINVAL; } guard(mutex)(&mcu->bus_lock); - reply->data = rx; + reply->data = mcu->rx; reply->length = length; reply->received = 0; reinit_completion(&reply->done); @@ -242,15 +243,15 @@ int qnap_mcu_exec(struct qnap_mcu *mcu, return -ETIMEDOUT; } - if (!qnap_mcu_verify_checksum(rx, reply->received)) { + if (!qnap_mcu_verify_checksum(mcu->rx, reply->received)) { dev_err(&mcu->serdev->dev, "Invalid Checksum received from controller\n"); return -EPROTO; } - if (qnap_mcu_reply_is_any_error(mcu, rx, reply->received)) + if (qnap_mcu_reply_is_any_error(mcu, mcu->rx, reply->received)) return -EPROTO; - memcpy(reply_data, rx, reply_data_size); + memcpy(reply_data, mcu->rx, reply_data_size); return 0; } diff --git a/drivers/mfd/rave-sp.c b/drivers/mfd/rave-sp.c index c1b78d127a261..05d26d92df03e 100644 --- a/drivers/mfd/rave-sp.c +++ b/drivers/mfd/rave-sp.c @@ -63,6 +63,12 @@ #define RAVE_SP_TX_BUFFER_SIZE \ (RAVE_SP_STX_ETX_SIZE + 2 * RAVE_SP_RX_BUFFER_SIZE) +enum rave_sp_frame_offset { + RAVE_SP_FRAME_CODE_OFFSET, + RAVE_SP_FRAME_ACK_ID_OFFSET, + RAVE_SP_FRAME_DATA_OFFSET, +}; + /** * enum rave_sp_deframer_state - Possible state for de-framer * @@ -352,7 +358,7 @@ int rave_sp_exec(struct rave_sp *sp, int command, ret = 0; u8 ackid; - command = sp->variant->cmd.translate(data[0]); + command = sp->variant->cmd.translate(data[RAVE_SP_FRAME_CODE_OFFSET]); if (command < 0) return command; @@ -366,8 +372,8 @@ int rave_sp_exec(struct rave_sp *sp, sp->reply = &reply; mutex_unlock(&sp->reply_lock); - data[0] = command; - data[1] = ackid; + data[RAVE_SP_FRAME_CODE_OFFSET] = command; + data[RAVE_SP_FRAME_ACK_ID_OFFSET] = ackid; rave_sp_write(sp, data, data_size); @@ -388,16 +394,23 @@ EXPORT_SYMBOL_GPL(rave_sp_exec); static void rave_sp_receive_event(struct rave_sp *sp, const unsigned char *data, size_t length) { - u8 cmd[] = { - [0] = rave_sp_reply_code(data[0]), - [1] = data[1], - }; + unsigned long action; + u8 cmd[RAVE_SP_FRAME_DATA_OFFSET]; + + if (length < RAVE_SP_FRAME_DATA_OFFSET + 1) { + dev_warn(&sp->serdev->dev, "Dropping short event frame\n"); + return; + } + + cmd[RAVE_SP_FRAME_CODE_OFFSET] = + rave_sp_reply_code(data[RAVE_SP_FRAME_CODE_OFFSET]); + cmd[RAVE_SP_FRAME_ACK_ID_OFFSET] = data[RAVE_SP_FRAME_ACK_ID_OFFSET]; rave_sp_write(sp, cmd, sizeof(cmd)); - blocking_notifier_call_chain(&sp->event_notifier_list, - rave_sp_action_pack(data[0], data[2]), - NULL); + action = rave_sp_action_pack(data[RAVE_SP_FRAME_CODE_OFFSET], + data[RAVE_SP_FRAME_DATA_OFFSET]); + blocking_notifier_call_chain(&sp->event_notifier_list, action, NULL); } static void rave_sp_receive_reply(struct rave_sp *sp, @@ -405,27 +418,35 @@ static void rave_sp_receive_reply(struct rave_sp *sp, { struct device *dev = &sp->serdev->dev; struct rave_sp_reply *reply; - const size_t payload_length = length - 2; + size_t payload_length; + + if (length < RAVE_SP_FRAME_DATA_OFFSET) { + dev_warn(dev, "Dropping short reply frame\n"); + return; + } + payload_length = length - RAVE_SP_FRAME_DATA_OFFSET; mutex_lock(&sp->reply_lock); reply = sp->reply; if (reply) { - if (reply->code == data[0] && reply->ackid == data[1] && + if (reply->code == data[RAVE_SP_FRAME_CODE_OFFSET] && + reply->ackid == data[RAVE_SP_FRAME_ACK_ID_OFFSET] && payload_length >= reply->length) { /* * We are relying on memcpy(dst, src, 0) to be a no-op * when handling commands that have a no-payload reply */ - memcpy(reply->data, &data[2], reply->length); + memcpy(reply->data, &data[RAVE_SP_FRAME_DATA_OFFSET], + reply->length); complete(&reply->received); sp->reply = NULL; } else { dev_err(dev, "Ignoring incorrect reply\n"); dev_dbg(dev, "Code: expected = 0x%08x received = 0x%08x\n", - reply->code, data[0]); + reply->code, data[RAVE_SP_FRAME_CODE_OFFSET]); dev_dbg(dev, "ACK ID: expected = 0x%08x received = 0x%08x\n", - reply->ackid, data[1]); + reply->ackid, data[RAVE_SP_FRAME_ACK_ID_OFFSET]); dev_dbg(dev, "Length: expected = %zu received = %zu\n", reply->length, payload_length); } @@ -439,10 +460,10 @@ static void rave_sp_receive_frame(struct rave_sp *sp, size_t length) { const size_t checksum_length = sp->variant->checksum->length; - const size_t payload_length = length - checksum_length; - const u8 *crc_reported = &data[payload_length]; struct device *dev = &sp->serdev->dev; u8 crc_calculated[RAVE_SP_CHECKSUM_SIZE]; + const u8 *crc_reported; + size_t payload_length; if (unlikely(checksum_length > sizeof(crc_calculated))) { dev_warn(dev, "Checksum too long, dropping\n"); @@ -457,6 +478,9 @@ static void rave_sp_receive_frame(struct rave_sp *sp, return; } + payload_length = length - checksum_length; + crc_reported = &data[payload_length]; + sp->variant->checksum->subroutine(data, payload_length, crc_calculated); @@ -465,10 +489,10 @@ static void rave_sp_receive_frame(struct rave_sp *sp, return; } - if (rave_sp_id_is_event(data[0])) - rave_sp_receive_event(sp, data, length); + if (rave_sp_id_is_event(data[RAVE_SP_FRAME_CODE_OFFSET])) + rave_sp_receive_event(sp, data, payload_length); else - rave_sp_receive_reply(sp, data, length); + rave_sp_receive_reply(sp, data, payload_length); } static size_t rave_sp_receive_buf(struct serdev_device *serdev, diff --git a/drivers/mfd/sm501.c b/drivers/mfd/sm501.c index b5bda477ebfc2..77888965d91ed 100644 --- a/drivers/mfd/sm501.c +++ b/drivers/mfd/sm501.c @@ -1627,6 +1627,7 @@ static void sm501_pci_remove(struct pci_dev *dev) release_mem_region(sm->io_res->start, 0x100); pci_disable_device(dev); + kfree(sm); } static void sm501_plat_remove(struct platform_device *dev) @@ -1637,6 +1638,7 @@ static void sm501_plat_remove(struct platform_device *dev) iounmap(sm->regs); release_mem_region(sm->io_res->start, 0x100); + kfree(sm); } static const struct pci_device_id sm501_pci_tbl[] = { diff --git a/drivers/misc/ad525x_dpot-i2c.c b/drivers/misc/ad525x_dpot-i2c.c index 469478f7a1d33..896ad61bb9e17 100644 --- a/drivers/misc/ad525x_dpot-i2c.c +++ b/drivers/misc/ad525x_dpot-i2c.c @@ -105,6 +105,7 @@ MODULE_DEVICE_TABLE(i2c, ad_dpot_id); static struct i2c_driver ad_dpot_i2c_driver = { .driver = { .name = "ad_dpot", + .dev_groups = ad_dpot_groups, }, .probe = ad_dpot_i2c_probe, .remove = ad_dpot_i2c_remove, diff --git a/drivers/misc/ad525x_dpot-spi.c b/drivers/misc/ad525x_dpot-spi.c index 263055bda48b7..1ebe629715a84 100644 --- a/drivers/misc/ad525x_dpot-spi.c +++ b/drivers/misc/ad525x_dpot-spi.c @@ -131,6 +131,7 @@ MODULE_DEVICE_TABLE(spi, ad_dpot_spi_id); static struct spi_driver ad_dpot_spi_driver = { .driver = { .name = "ad_dpot", + .dev_groups = ad_dpot_groups, }, .probe = ad_dpot_spi_probe, .remove = ad_dpot_spi_remove, diff --git a/drivers/misc/ad525x_dpot.c b/drivers/misc/ad525x_dpot.c index 57bead9fba1be..a4e22fd4a1072 100644 --- a/drivers/misc/ad525x_dpot.c +++ b/drivers/misc/ad525x_dpot.c @@ -630,66 +630,132 @@ static struct attribute *ad525x_attributes_commands[] = { NULL }; -static const struct attribute_group ad525x_group_commands = { - .attrs = ad525x_attributes_commands, +static struct attribute *ad525x_attributes[] = { + &dev_attr_rdac0.attr, + &dev_attr_rdac1.attr, + &dev_attr_rdac2.attr, + &dev_attr_rdac3.attr, + &dev_attr_rdac4.attr, + &dev_attr_rdac5.attr, + &dev_attr_eeprom0.attr, + &dev_attr_eeprom1.attr, + &dev_attr_eeprom2.attr, + &dev_attr_eeprom3.attr, + &dev_attr_eeprom4.attr, + &dev_attr_eeprom5.attr, + &dev_attr_tolerance0.attr, + &dev_attr_tolerance1.attr, + &dev_attr_tolerance2.attr, + &dev_attr_tolerance3.attr, + &dev_attr_tolerance4.attr, + &dev_attr_tolerance5.attr, + &dev_attr_otp0.attr, + &dev_attr_otp1.attr, + &dev_attr_otp2.attr, + &dev_attr_otp3.attr, + &dev_attr_otp4.attr, + &dev_attr_otp5.attr, + &dev_attr_otp0en.attr, + &dev_attr_otp1en.attr, + &dev_attr_otp2en.attr, + &dev_attr_otp3en.attr, + &dev_attr_otp4en.attr, + &dev_attr_otp5en.attr, + &dev_attr_inc_all.attr, + &dev_attr_dec_all.attr, + &dev_attr_inc_all_6db.attr, + &dev_attr_dec_all_6db.attr, + NULL }; -static int ad_dpot_add_files(struct device *dev, - unsigned int features, unsigned int rdac) +static int ad525x_attr_index(struct attribute *attr, + const struct attribute * const *attrs) { - int err = sysfs_create_file(&dev->kobj, - dpot_attrib_wipers[rdac]); - if (features & F_CMD_EEP) - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_eeprom[rdac]); - if (features & F_CMD_TOL) - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_tolerance[rdac]); - if (features & F_CMD_OTP) { - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_otp_en[rdac]); - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_otp[rdac]); - } + int i; - if (err) - dev_err(dev, "failed to register sysfs hooks for RDAC%d\n", - rdac); + for (i = 0; attrs[i]; i++) + if (attr == attrs[i]) + return i; - return err; + return -ENOENT; } -static inline void ad_dpot_remove_files(struct device *dev, - unsigned int features, unsigned int rdac) +static bool ad525x_is_command_attr(struct attribute *attr) { - sysfs_remove_file(&dev->kobj, - dpot_attrib_wipers[rdac]); - if (features & F_CMD_EEP) - sysfs_remove_file(&dev->kobj, - dpot_attrib_eeprom[rdac]); - if (features & F_CMD_TOL) - sysfs_remove_file(&dev->kobj, - dpot_attrib_tolerance[rdac]); - if (features & F_CMD_OTP) { - sysfs_remove_file(&dev->kobj, - dpot_attrib_otp_en[rdac]); - sysfs_remove_file(&dev->kobj, - dpot_attrib_otp[rdac]); + int i; + + for (i = 0; ad525x_attributes_commands[i]; i++) { + if (attr == ad525x_attributes_commands[i]) + return true; } + + return false; +} + +static umode_t ad525x_is_visible(struct kobject *kobj, struct attribute *attr, + int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct dpot_data *data = dev_get_drvdata(dev); + int rdac; + + if (!data) + return 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_wipers); + if (rdac >= 0) + return data->wipers & BIT(rdac) ? attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_eeprom); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_EEP) ? + attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_tolerance); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_TOL) ? + attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_otp); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_OTP) ? + attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_otp_en); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_OTP) ? + attr->mode : 0; + + if (ad525x_is_command_attr(attr)) + return data->feat & F_CMD_INC ? attr->mode : 0; + + return attr->mode; } +static const struct attribute_group ad525x_group = { + .attrs = ad525x_attributes, + .is_visible = ad525x_is_visible, +}; + +const struct attribute_group *ad_dpot_groups[] = { + &ad525x_group, + NULL +}; +EXPORT_SYMBOL(ad_dpot_groups); + int ad_dpot_probe(struct device *dev, struct ad_dpot_bus_data *bdata, unsigned long devid, const char *name) { struct dpot_data *data; - int i, err = 0; + int i; data = kzalloc_obj(struct dpot_data); if (!data) { - err = -ENOMEM; - goto exit; + dev_err(dev, "failed to create client for %s ID 0x%lX\n", + name, devid); + return -ENOMEM; } dev_set_drvdata(dev, data); @@ -705,51 +771,22 @@ int ad_dpot_probe(struct device *dev, data->wipers = DPOT_WIPERS(devid); for (i = DPOT_RDAC0; i < MAX_RDACS; i++) - if (data->wipers & (1 << i)) { - err = ad_dpot_add_files(dev, data->feat, i); - if (err) - goto exit_remove_files; + if (data->wipers & BIT(i)) { /* power-up midscale */ if (data->feat & F_RDACS_WONLY) data->rdac_cache[i] = data->max_pos / 2; } - if (data->feat & F_CMD_INC) - err = sysfs_create_group(&dev->kobj, &ad525x_group_commands); - - if (err) { - dev_err(dev, "failed to register sysfs hooks\n"); - goto exit_free; - } - dev_info(dev, "%s %d-Position Digital Potentiometer registered\n", name, data->max_pos); return 0; - -exit_remove_files: - for (i = DPOT_RDAC0; i < MAX_RDACS; i++) - if (data->wipers & (1 << i)) - ad_dpot_remove_files(dev, data->feat, i); - -exit_free: - kfree(data); - dev_set_drvdata(dev, NULL); -exit: - dev_err(dev, "failed to create client for %s ID 0x%lX\n", - name, devid); - return err; } EXPORT_SYMBOL(ad_dpot_probe); void ad_dpot_remove(struct device *dev) { struct dpot_data *data = dev_get_drvdata(dev); - int i; - - for (i = DPOT_RDAC0; i < MAX_RDACS; i++) - if (data->wipers & (1 << i)) - ad_dpot_remove_files(dev, data->feat, i); kfree(data); } diff --git a/drivers/misc/ad525x_dpot.h b/drivers/misc/ad525x_dpot.h index 72a9d6801937c..2e877c89523b5 100644 --- a/drivers/misc/ad525x_dpot.h +++ b/drivers/misc/ad525x_dpot.h @@ -10,6 +10,8 @@ #include +struct attribute_group; + #define DPOT_CONF(features, wipers, max_pos, uid) \ (((features) << 18) | (((wipers) & 0xFF) << 10) | \ ((max_pos & 0xF) << 6) | (uid & 0x3F)) @@ -210,5 +212,6 @@ struct ad_dpot_bus_data { int ad_dpot_probe(struct device *dev, struct ad_dpot_bus_data *bdata, unsigned long devid, const char *name); void ad_dpot_remove(struct device *dev); +extern const struct attribute_group *ad_dpot_groups[]; #endif diff --git a/drivers/misc/amd-sbi/rmi-core.c b/drivers/misc/amd-sbi/rmi-core.c index d4238ebad3c6c..95c9109101b53 100644 --- a/drivers/misc/amd-sbi/rmi-core.c +++ b/drivers/misc/amd-sbi/rmi-core.c @@ -581,6 +581,8 @@ int create_misc_rmi_device(struct sbrmi_data *data, GFP_KERNEL, "sbrmi-%x", data->dev_static_addr); + if (!data->sbrmi_misc_dev.name) + return -ENOMEM; data->sbrmi_misc_dev.minor = MISC_DYNAMIC_MINOR; data->sbrmi_misc_dev.fops = &sbrmi_fops; data->sbrmi_misc_dev.parent = dev; @@ -588,6 +590,8 @@ int create_misc_rmi_device(struct sbrmi_data *data, GFP_KERNEL, "sbrmi-%x", data->dev_static_addr); + if (!data->sbrmi_misc_dev.nodename) + return -ENOMEM; data->sbrmi_misc_dev.mode = 0600; return misc_register(&data->sbrmi_misc_dev); diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index 3916ec07ecad1..2c084a6b3a929 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -108,7 +108,8 @@ u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq, bool bcm_vk_drv_access_ok(struct bcm_vk *vk) { - return (!!atomic_read(&vk->msgq_inited)); + /* Pair with the release store after message queue initialization. */ + return !!atomic_read_acquire(&vk->msgq_inited); } void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask) @@ -501,7 +502,8 @@ int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync) msgq++; } } - atomic_set(&vk->msgq_inited, 1); + /* Publish message queue info before allowing driver access. */ + atomic_set_release(&vk->msgq_inited, 1); return ret; } diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c index c4d54ca2fa804..c6e602523538e 100644 --- a/drivers/misc/cardreader/rtsx_pcr.c +++ b/drivers/misc/cardreader/rtsx_pcr.c @@ -1196,6 +1196,8 @@ static int rtsx_pci_init_hw(struct rtsx_pcr *pcr) /* Gating real mcu clock */ err = rtsx_pci_write_register(pcr, RTS5261_FW_CFG1, RTS5261_MCU_CLOCK_GATING, 0); + if (err < 0) + return err; err = rtsx_pci_write_register(pcr, RTS5261_REG_FPDCTL, SSC_POWER_DOWN, 0); } else { diff --git a/drivers/misc/cardreader/rtsx_usb.c b/drivers/misc/cardreader/rtsx_usb.c index 1830e9ed25216..a127744918f42 100644 --- a/drivers/misc/cardreader/rtsx_usb.c +++ b/drivers/misc/cardreader/rtsx_usb.c @@ -312,6 +312,9 @@ int rtsx_usb_get_card_status(struct rtsx_ucr *ucr, u16 *status) if (ret < 0) return ret; + ucr->card_status_cache = *status; + ucr->card_status_valid = true; + return 0; } EXPORT_SYMBOL_GPL(rtsx_usb_get_card_status); @@ -623,6 +626,7 @@ static int rtsx_usb_probe(struct usb_interface *intf, { struct usb_device *usb_dev = interface_to_usbdev(intf); struct rtsx_ucr *ucr; + u16 status; int ret; dev_dbg(&intf->dev, @@ -659,6 +663,9 @@ static int rtsx_usb_probe(struct usb_interface *intf, if (ret) goto out_init_fail; + /* Prime cached status for runtime autosuspend decisions. */ + rtsx_usb_get_card_status(ucr, &status); + /* initialize USB SG transfer timer */ timer_setup(&ucr->sg_timer, rtsx_usb_sg_timed_out, 0); @@ -713,22 +720,29 @@ static int rtsx_usb_suspend(struct usb_interface *intf, pm_message_t message) struct rtsx_ucr *ucr = (struct rtsx_ucr *)usb_get_intfdata(intf); u16 val = 0; + bool valid = false; dev_dbg(&intf->dev, "%s called with pm message 0x%04x\n", __func__, message.event); if (PMSG_IS_AUTO(message)) { if (mutex_trylock(&ucr->dev_mutex)) { - rtsx_usb_get_card_status(ucr, &val); + valid = ucr->card_status_valid; + if (valid) + val = ucr->card_status_cache; mutex_unlock(&ucr->dev_mutex); - /* Defer the autosuspend if card exists */ - if (val & (SD_CD | MS_CD)) { + /* + * Do not issue USB commands from runtime autosuspend. + * Raw SD_CD is not authoritative on tray-based readers, + * while a real SD card is protected by the SD/MMC child + * runtime-PM reference once the card is powered. Keep + * the historical Memory Stick autosuspend deferral when + * the cached status says MS media is present. + */ + if (valid && (val & MS_CD)) { device_for_each_child(&intf->dev, NULL, rtsx_usb_resume_child); return -EAGAIN; - } else { - /* if the card does not exists, clear OCP status */ - rtsx_usb_write_register(ucr, OCPCTL, MS_OCP_CLEAR, MS_OCP_CLEAR); } } else { /* There is an ongoing operation*/ diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index eb6c2a78d3c7b..d7a9e12f55758 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -1712,24 +1712,20 @@ static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp) return err; } - bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE); + bp.fd = get_unused_fd_flags(O_ACCMODE); if (bp.fd < 0) { dma_buf_put(buf->dmabuf); - return -EINVAL; + return bp.fd; } if (copy_to_user(argp, &bp, sizeof(bp))) { - /* - * The usercopy failed, but we can't do much about it, as - * dma_buf_fd() already called fd_install() and made the - * file descriptor accessible for the current process. It - * might already be closed and dmabuf no longer valid when - * we reach this point. Therefore "leak" the fd and rely on - * the process exit path to do any required cleanup. - */ + put_unused_fd(bp.fd); + dma_buf_put(buf->dmabuf); return -EFAULT; } + dma_buf_fd_install(buf->dmabuf, bp.fd); + return 0; } diff --git a/drivers/misc/lan966x_pci.c b/drivers/misc/lan966x_pci.c index 0bb90c0943bf7..b0949c653e5ba 100644 --- a/drivers/misc/lan966x_pci.c +++ b/drivers/misc/lan966x_pci.c @@ -183,6 +183,7 @@ static int lan966x_pci_probe(struct pci_dev *pdev, const struct pci_device_id *i return 0; err_unload_overlay: + of_platform_depopulate(dev); lan966x_pci_unload_overlay(data); return ret; } diff --git a/drivers/misc/mei/mei_lb.c b/drivers/misc/mei/mei_lb.c index f6a258c2b838b..9fa69acf28d5b 100644 --- a/drivers/misc/mei/mei_lb.c +++ b/drivers/misc/mei/mei_lb.c @@ -281,7 +281,7 @@ static int mei_lb_check_response_v2(const struct device *dev, ssize_t bytes, bytes, sizeof(rsp->rheader)); return -ENOMSG; } - if (rsp->rheader.header.command_id != MEI_LB2_CMD) { + if (rsp->rheader.header.command_id != cpu_to_le32(MEI_LB2_CMD)) { dev_err(dev, "Mismatch command: 0x%x instead of 0x%x\n", rsp->rheader.header.command_id, MEI_LB2_CMD); return -EPROTO; @@ -327,7 +327,7 @@ static int mei_lb_push_payload_v2(struct device *dev, struct mei_cl_device *clde if (sent_data + chunk_size == payload_size) last_chunk = MEI_LB2_FLAG_LST_CHUNK; - req->header.command_id = MEI_LB2_CMD; + req->header.command_id = cpu_to_le32(MEI_LB2_CMD); req->type = cpu_to_le32(type); req->flags = cpu_to_le32(flags | first_chunk | last_chunk); req->reserved = 0; diff --git a/drivers/misc/nsm.c b/drivers/misc/nsm.c index 3960506eb7ab3..c3b474450d159 100644 --- a/drivers/misc/nsm.c +++ b/drivers/misc/nsm.c @@ -243,7 +243,7 @@ static int nsm_sendrecv_msg_locked(struct nsm *nsm) goto cleanup; } - msg->resp.len = len; + msg->resp.len = min_t(unsigned int, len, sizeof(msg->resp.data)); rc = 0; diff --git a/drivers/misc/ntsync.c b/drivers/misc/ntsync.c index 02c9d1192812a..4a805919bb0c5 100644 --- a/drivers/misc/ntsync.c +++ b/drivers/misc/ntsync.c @@ -875,6 +875,9 @@ static int setup_wait(struct ntsync_device *dev, if (args->pad || (args->flags & ~NTSYNC_WAIT_REALTIME)) return -EINVAL; + if (!args->owner) + return -EINVAL; + if (size >= sizeof(fds)) return -EINVAL; diff --git a/drivers/misc/pch_phub.c b/drivers/misc/pch_phub.c index 19c4fa017f243..0097611b97af6 100644 --- a/drivers/misc/pch_phub.c +++ b/drivers/misc/pch_phub.c @@ -83,6 +83,14 @@ #define PCH_PHUB_OROM_SIZE 15360 +enum pch_phub_type { + PCH_EG20T, + PCH_ML7213, + PCH_ML7223M, + PCH_ML7223N, + PCH_ML7831, +}; + /** * struct pch_phub_reg - PHUB register structure * @phub_id_reg: PHUB_ID register val @@ -125,7 +133,7 @@ struct pch_phub_reg { void __iomem *pch_phub_extrom_base_address; u32 pch_mac_start_address; u32 pch_opt_rom_start_address; - int ioh_type; + enum pch_phub_type ioh_type; struct pci_dev *pdev; }; @@ -344,7 +352,7 @@ static int pch_phub_write_gbe_mac_addr(struct pch_phub_reg *chip, u8 *data) int retval; int i; - if ((chip->ioh_type == 1) || (chip->ioh_type == 5)) /* EG20T or ML7831*/ + if (chip->ioh_type == PCH_EG20T || chip->ioh_type == PCH_ML7831) retval = pch_phub_gbe_serial_rom_conf(chip); else /* ML7223 */ retval = pch_phub_gbe_serial_rom_conf_mp(chip); @@ -537,14 +545,6 @@ static const struct bin_attribute pch_bin_attr = { .write = pch_phub_bin_write, }; -enum { - PCH_EG20T, - PCH_ML7213, - PCH_ML7223M, - PCH_ML7223N, - PCH_ML7831, -}; - static int pch_phub_probe(struct pci_dev *pdev, const struct pci_device_id *id) { diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c index 3635741c3e7a7..26ab9252251f1 100644 --- a/drivers/misc/pci_endpoint_test.c +++ b/drivers/misc/pci_endpoint_test.c @@ -1071,6 +1071,7 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test) struct pci_dev *pdev = test->pdev; struct device *dev = &pdev->dev; int irq_type = test->irq_type; + int ret = 0; enum pci_barno bar; u32 data, status; u32 addr; @@ -1119,8 +1120,11 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test) status = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS); - if (!left || !(status & STATUS_DOORBELL_SUCCESS)) + if (!left || !(status & STATUS_DOORBELL_SUCCESS)) { dev_err(dev, "Failed to trigger doorbell in endpoint\n"); + /* Store error code, but continue to disable doorbell. */ + ret = -EINVAL; + } pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, COMMAND_DISABLE_DOORBELL); @@ -1134,10 +1138,7 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test) return -EINVAL; } - if (!(status & STATUS_DOORBELL_SUCCESS)) - return -EINVAL; - - return 0; + return ret; } static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd, diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 3557d78ee47a2..5a87c12f444a3 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -166,13 +166,8 @@ static void get_clear_fault_map(struct gru_state *gru, } /* - * Atomic (interrupt context) & non-atomic (user context) functions to - * convert a vaddr into a physical address. The size of the page - * is returned in pageshift. - * returns: - * 0 - successful - * < 0 - error code - * 1 - (atomic only) try again in non-atomic context + * Convert a user virtual address to a physical address in process context. + * The size of the page is returned in pageshift. */ static int non_atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, int write, @@ -192,87 +187,25 @@ static int non_atomic_pte_lookup(struct vm_area_struct *vma, return 0; } -/* - * atomic_pte_lookup - * - * Convert a user virtual address to a physical address - * Only supports Intel large pages (2MB only) on x86_64. - * ZZZ - hugepage support is incomplete - * - * NOTE: mmap_lock is already held on entry to this function. This - * guarantees existence of the page tables. - */ -static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, - int write, unsigned long *paddr, int *pageshift) -{ - pgd_t *pgdp; - p4d_t *p4dp; - pud_t *pudp; - pmd_t *pmdp; - pte_t pte; - - pgdp = pgd_offset(vma->vm_mm, vaddr); - if (unlikely(pgd_none(*pgdp))) - goto err; - - p4dp = p4d_offset(pgdp, vaddr); - if (unlikely(p4d_none(*p4dp))) - goto err; - - pudp = pud_offset(p4dp, vaddr); - if (unlikely(pud_none(*pudp))) - goto err; - - pmdp = pmd_offset(pudp, vaddr); - if (unlikely(pmd_none(*pmdp))) - goto err; -#ifdef CONFIG_X86_64 - if (unlikely(pmd_leaf(*pmdp))) - pte = ptep_get((pte_t *)pmdp); - else -#endif - pte = *pte_offset_kernel(pmdp, vaddr); - - if (unlikely(!pte_present(pte) || - (write && (!pte_write(pte) || !pte_dirty(pte))))) - return 1; - - *paddr = pte_pfn(pte) << PAGE_SHIFT; -#ifdef CONFIG_HUGETLB_PAGE - *pageshift = is_vm_hugetlb_page(vma) ? HPAGE_SHIFT : PAGE_SHIFT; -#else - *pageshift = PAGE_SHIFT; -#endif - return 0; - -err: - return 1; -} - static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr, int write, int atomic, unsigned long *gpa, int *pageshift) { struct mm_struct *mm = gts->ts_mm; struct vm_area_struct *vma; unsigned long paddr; - int ret, ps; + int ps; vma = find_vma(mm, vaddr); if (!vma) goto inval; - /* - * Atomic lookup is faster & usually works even if called in non-atomic - * context. - */ - rmb(); /* Must/check ms_range_active before loading PTEs */ - ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &ps); - if (ret) { - if (atomic) - goto upm; - if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps)) - goto inval; - } + if (atomic) + goto upm; + + /* Order the caller's ms_range_active check before loading PTEs. */ + rmb(); + if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps)) + goto inval; if (is_gru_paddr(paddr)) goto inval; paddr = paddr & ~((1UL << ps) - 1); @@ -569,19 +502,9 @@ static irqreturn_t gru_intr(int chiplet, int blade) continue; } - /* - * This is running in interrupt context. Trylock the mmap_lock. - * If it fails, retry the fault in user context. - */ + /* Address translation may sleep, so retry the fault in user context. */ gts->ustats.fmm_tlbmiss++; - if (!gts->ts_force_cch_reload && - mmap_read_trylock(gts->ts_mm)) { - gru_try_dropin(gru, gts, tfh, NULL); - mmap_read_unlock(gts->ts_mm); - } else { - tfh_user_polling_mode(tfh); - STAT(intr_mm_lock_failed); - } + tfh_user_polling_mode(tfh); } return IRQ_HANDLED; } diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c index 97b8b38ab47df..b8139c27bc7f8 100644 --- a/drivers/misc/sgi-gru/gruprocfs.c +++ b/drivers/misc/sgi-gru/gruprocfs.c @@ -54,7 +54,6 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, intr_cbr); printstat(s, intr_tfh); printstat(s, intr_spurious); - printstat(s, intr_mm_lock_failed); printstat(s, call_os); printstat(s, call_os_wait_queue); printstat(s, user_flush_tlb); diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index 640daf1994df7..3348552925c61 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -182,7 +182,6 @@ struct gru_stats_s { atomic_long_t intr_cbr; atomic_long_t intr_tfh; atomic_long_t intr_spurious; - atomic_long_t intr_mm_lock_failed; atomic_long_t call_os; atomic_long_t call_os_wait_queue; atomic_long_t user_flush_tlb; diff --git a/drivers/misc/vmw_vmci/vmci_event.c b/drivers/misc/vmw_vmci/vmci_event.c index fffe068a26eb4..c3ef3b98c432e 100644 --- a/drivers/misc/vmw_vmci/vmci_event.c +++ b/drivers/misc/vmw_vmci/vmci_event.c @@ -179,16 +179,16 @@ int vmci_event_subscribe(u32 event, } } + *new_subscription_id = sub->id; if (have_new_id) { list_add_rcu(&sub->node, &subscriber_array[event]); retval = VMCI_SUCCESS; } else { + kfree(sub); retval = VMCI_ERROR_NO_RESOURCES; } mutex_unlock(&subscriber_mutex); - - *new_subscription_id = sub->id; return retval; } EXPORT_SYMBOL_GPL(vmci_event_subscribe); diff --git a/drivers/mmc/core/bus.c b/drivers/mmc/core/bus.c index be5cf338bdeb9..92ad39c0cabcd 100644 --- a/drivers/mmc/core/bus.c +++ b/drivers/mmc/core/bus.c @@ -417,8 +417,8 @@ void mmc_remove_card(struct mmc_card *card) mmc_hostname(card->host), card->rca); } device_del(&card->dev); - of_node_put(card->dev.of_node); } + of_node_put(card->dev.of_node); if (host->cqe_enabled) { host->cqe_ops->cqe_disable(host); diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c index b7ce3137d4529..828078b3f9622 100644 --- a/drivers/mmc/core/host.c +++ b/drivers/mmc/core/host.c @@ -698,6 +698,7 @@ EXPORT_SYMBOL(mmc_remove_host); void mmc_free_host(struct mmc_host *host) { cancel_delayed_work_sync(&host->detect); + cancel_work_sync(&host->sdio_irq_work); mmc_pwrseq_free(host); put_device(&host->class_dev); } diff --git a/drivers/mmc/core/sdio_uart.c b/drivers/mmc/core/sdio_uart.c index 7fd5dedf3ac03..705163c429724 100644 --- a/drivers/mmc/core/sdio_uart.c +++ b/drivers/mmc/core/sdio_uart.c @@ -104,6 +104,9 @@ static int sdio_uart_add_port(struct sdio_uart_port *port) } spin_unlock(&sdio_uart_table_lock); + if (ret) + kfifo_free(&port->xmit_fifo); + return ret; } diff --git a/drivers/mmc/host/mmc_hsq.c b/drivers/mmc/host/mmc_hsq.c index 79836705c1762..57e172bd34871 100644 --- a/drivers/mmc/host/mmc_hsq.c +++ b/drivers/mmc/host/mmc_hsq.c @@ -7,6 +7,7 @@ * Author: Baolin Wang */ +#include #include #include #include @@ -345,6 +346,7 @@ static const struct mmc_cqe_ops mmc_hsq_ops = { int mmc_hsq_init(struct mmc_hsq *hsq, struct mmc_host *mmc) { + int ret; int i; hsq->num_slots = HSQ_NUM_SLOTS; hsq->next_tag = HSQ_INVALID_TAG; @@ -363,7 +365,11 @@ int mmc_hsq_init(struct mmc_hsq *hsq, struct mmc_host *mmc) for (i = 0; i < HSQ_NUM_SLOTS; i++) hsq->tag_slot[i] = HSQ_INVALID_TAG; - INIT_WORK(&hsq->retry_work, mmc_hsq_retry_handler); + ret = devm_work_autocancel(mmc_dev(mmc), &hsq->retry_work, + mmc_hsq_retry_handler); + if (ret) + return ret; + spin_lock_init(&hsq->lock); init_waitqueue_head(&hsq->wait_queue); diff --git a/drivers/mmc/host/mmc_spi.c b/drivers/mmc/host/mmc_spi.c index b471a7795b4d0..5217d713c8263 100644 --- a/drivers/mmc/host/mmc_spi.c +++ b/drivers/mmc/host/mmc_spi.c @@ -952,6 +952,7 @@ static void mmc_spi_request(struct mmc_host *mmc, struct mmc_request *mrq) status = mmc_spi_command_send(host, mrq, &stop, 0); crc_retry--; mrq->data->error = 0; + mrq->data->bytes_xfered = 0; goto crc_recover; } diff --git a/drivers/mmc/host/mmci.c b/drivers/mmc/host/mmci.c index e500051bd572f..416bdb184ed4c 100644 --- a/drivers/mmc/host/mmci.c +++ b/drivers/mmc/host/mmci.c @@ -2511,6 +2511,9 @@ static void mmci_remove(struct amba_device *dev) writel(0, host->base + MMCICOMMAND); writel(0, host->base + MMCIDATACTRL); + if (variant->busy_detect) + disable_delayed_work_sync(&host->ux500_busy_timeout_work); + mmci_dma_release(host); clk_disable_unprepare(host->clk); } diff --git a/drivers/mmc/host/mxcmmc.c b/drivers/mmc/host/mxcmmc.c index c405cfb8b269b..097498a3f8ff4 100644 --- a/drivers/mmc/host/mxcmmc.c +++ b/drivers/mmc/host/mxcmmc.c @@ -1173,6 +1173,10 @@ static void mxcmci_remove(struct platform_device *pdev) mmc_remove_host(mmc); + devm_free_irq(&pdev->dev, platform_get_irq(pdev, 0), host); + cancel_work_sync(&host->datawork); + timer_delete_sync(&host->watchdog); + if (host->pdata && host->pdata->exit) host->pdata->exit(&pdev->dev, mmc); diff --git a/drivers/mmc/host/rtsx_pci_sdmmc.c b/drivers/mmc/host/rtsx_pci_sdmmc.c index 8dfbc62f165bc..d25d4cb59bc97 100644 --- a/drivers/mmc/host/rtsx_pci_sdmmc.c +++ b/drivers/mmc/host/rtsx_pci_sdmmc.c @@ -1425,6 +1425,11 @@ static void realtek_init_host(struct realtek_pci_sdmmc *host) mmc->caps = mmc->caps | MMC_CAP_AGGRESSIVE_PM; mmc->caps2 = MMC_CAP2_NO_PRESCAN_POWERUP | MMC_CAP2_FULL_PWR_CYCLE | MMC_CAP2_NO_SDIO; + + if (pcr->pci->device == 0x522a && + pcr->pci->subsystem_vendor == PCI_VENDOR_ID_LENOVO && + pcr->pci->subsystem_device == 0x504a) + mmc->caps2 |= MMC_CAP2_NO_WRITE_PROTECT; mmc->max_current_330 = 400; mmc->max_current_180 = 800; mmc->ops = &realtek_pci_sdmmc_ops; diff --git a/drivers/mmc/host/sdhci-of-aspeed.c b/drivers/mmc/host/sdhci-of-aspeed.c index f5d973783cbe1..d317626feb4aa 100644 --- a/drivers/mmc/host/sdhci-of-aspeed.c +++ b/drivers/mmc/host/sdhci-of-aspeed.c @@ -560,12 +560,14 @@ static int aspeed_sdc_probe(struct platform_device *pdev) cpdev = of_platform_device_create(child, NULL, &pdev->dev); if (!cpdev) { ret = -ENODEV; - goto err_clk; + goto err_children; } } return 0; +err_children: + device_for_each_child_reverse(&pdev->dev, NULL, of_platform_device_destroy); err_clk: clk_disable_unprepare(sdc->clk); return ret; @@ -575,6 +577,7 @@ static void aspeed_sdc_remove(struct platform_device *pdev) { struct aspeed_sdc *sdc = dev_get_drvdata(&pdev->dev); + device_for_each_child_reverse(&pdev->dev, NULL, of_platform_device_destroy); clk_disable_unprepare(sdc->clk); } diff --git a/drivers/mmc/host/sdhci_am654.c b/drivers/mmc/host/sdhci_am654.c index d235b0aecfdb1..8edfab5012edc 100644 --- a/drivers/mmc/host/sdhci_am654.c +++ b/drivers/mmc/host/sdhci_am654.c @@ -126,7 +126,7 @@ static const struct timing_data td[] = { NULL, MMC_CAP_UHS_SDR104}, [MMC_TIMING_UHS_DDR50] = {"ti,otap-del-sel-ddr50", - NULL, + "ti,itap-del-sel-ddr50", MMC_CAP_UHS_DDR50}, [MMC_TIMING_MMC_DDR52] = {"ti,otap-del-sel-ddr52", "ti,itap-del-sel-ddr52", @@ -144,6 +144,8 @@ struct sdhci_am654_data { u32 otap_del_sel[ARRAY_SIZE(td)]; u32 itap_del_sel[ARRAY_SIZE(td)]; u32 itap_del_ena[ARRAY_SIZE(td)]; + u32 itap_del_sel_dt_ddr50; + u32 itap_del_ena_dt_ddr50; int clkbuf_sel; int trm_icp; int drv_strength; @@ -151,7 +153,6 @@ struct sdhci_am654_data { u32 flags; u32 quirks; bool dll_enable; - u32 tuning_loop; #define SDHCI_AM654_QUIRK_FORCE_CDTEST BIT(0) #define SDHCI_AM654_QUIRK_SUPPRESS_V1P8_ENA BIT(1) @@ -443,15 +444,13 @@ static int sdhci_am654_execute_tuning(struct mmc_host *mmc, u32 opcode) struct sdhci_host *host = mmc_priv(mmc); int err = sdhci_execute_tuning(mmc, opcode); - if (err) - return err; /* * Tuning data remains in the buffer after tuning. * Do a command and data reset to get rid of it */ sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); - return 0; + return err; } static u32 sdhci_am654_cqhci_irq(struct sdhci_host *host, u32 intmask) @@ -530,7 +529,6 @@ static int sdhci_am654_do_tuning(struct sdhci_host *host, { struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_am654_data *sdhci_am654 = sdhci_pltfm_priv(pltfm_host); - unsigned char timing = host->mmc->ios.timing; struct window fail_window[ITAPDLY_LENGTH]; struct device *dev = mmc_dev(host->mmc); u8 curr_pass, itap; @@ -539,11 +537,8 @@ static int sdhci_am654_do_tuning(struct sdhci_host *host, memset(fail_window, 0, sizeof(fail_window)); - /* Enable ITAPDLY */ - sdhci_am654->itap_del_ena[timing] = 0x1; - for (itap = 0; itap < ITAPDLY_LENGTH; itap++) { - sdhci_am654_write_itapdly(sdhci_am654, itap, sdhci_am654->itap_del_ena[timing]); + sdhci_am654_write_itapdly(sdhci_am654, itap, 0x1); curr_pass = !mmc_send_tuning(host->mmc, opcode, NULL); @@ -576,20 +571,36 @@ static int sdhci_am654_platform_execute_tuning(struct sdhci_host *host, struct sdhci_am654_data *sdhci_am654 = sdhci_pltfm_priv(pltfm_host); unsigned char timing = host->mmc->ios.timing; struct device *dev = mmc_dev(host->mmc); + unsigned int tuning_loop = 0; int itapdly; do { itapdly = sdhci_am654_do_tuning(host, opcode); if (itapdly >= 0) break; - } while (++sdhci_am654->tuning_loop < RETRY_TUNING_MAX); + } while (++tuning_loop < RETRY_TUNING_MAX); if (itapdly < 0) { - dev_err(dev, "Failed to find itapdly, fail tuning\n"); + if (timing == MMC_TIMING_UHS_DDR50) { + dev_dbg(dev, "Failed DDR50 tuning, fallback to DT ITAP\n"); + sdhci_am654->itap_del_sel[timing] = sdhci_am654->itap_del_sel_dt_ddr50; + sdhci_am654->itap_del_ena[timing] = sdhci_am654->itap_del_ena_dt_ddr50; + } else { + dev_err(dev, "Failed to find itapdly, fail tuning\n"); + sdhci_am654->itap_del_ena[timing] = 0; + sdhci_am654->itap_del_sel[timing] = 0; + } + + sdhci_am654_write_itapdly(sdhci_am654, + sdhci_am654->itap_del_sel[timing], + sdhci_am654->itap_del_ena[timing]); return -1; } dev_dbg(dev, "Passed tuning, final itapdly=%d\n", itapdly); + + /* Enable ITAPDLY */ + sdhci_am654->itap_del_ena[timing] = 0x1; sdhci_am654_write_itapdly(sdhci_am654, itapdly, sdhci_am654->itap_del_ena[timing]); /* Save ITAPDLY */ sdhci_am654->itap_del_sel[timing] = itapdly; @@ -758,6 +769,11 @@ static int sdhci_am654_get_otap_delay(struct sdhci_host *host, } } + sdhci_am654->itap_del_sel_dt_ddr50 = + sdhci_am654->itap_del_sel[MMC_TIMING_UHS_DDR50]; + sdhci_am654->itap_del_ena_dt_ddr50 = + sdhci_am654->itap_del_ena[MMC_TIMING_UHS_DDR50]; + return 0; } @@ -806,9 +822,6 @@ static int sdhci_am654_init(struct sdhci_host *host) regmap_update_bits(sdhci_am654->base, CTL_CFG_3, TUNINGFORSDR50_MASK, TUNINGFORSDR50_MASK); - /* Use to re-execute tuning */ - sdhci_am654->tuning_loop = 0; - ret = sdhci_setup_host(host); if (ret) return ret; diff --git a/drivers/mmc/host/sh_mmcif.c b/drivers/mmc/host/sh_mmcif.c index 9831956de1c8a..3e1d46cfbf703 100644 --- a/drivers/mmc/host/sh_mmcif.c +++ b/drivers/mmc/host/sh_mmcif.c @@ -1461,6 +1461,7 @@ static int sh_mmcif_probe(struct platform_device *pdev) host->pd = pdev; spin_lock_init(&host->lock); + mutex_init(&host->thread_lock); mmc->ops = &sh_mmcif_ops; sh_mmcif_init_ocr(host); @@ -1519,8 +1520,6 @@ static int sh_mmcif_probe(struct platform_device *pdev) } } - mutex_init(&host->thread_lock); - ret = mmc_add_host(mmc); if (ret < 0) goto err_clk; diff --git a/drivers/mmc/host/via-sdmmc.c b/drivers/mmc/host/via-sdmmc.c index 8c049f8355cdf..4910d0f444bab 100644 --- a/drivers/mmc/host/via-sdmmc.c +++ b/drivers/mmc/host/via-sdmmc.c @@ -1153,10 +1153,16 @@ static int via_sd_probe(struct pci_dev *pcidev, ret = mmc_add_host(mmc); if (ret) - goto unmap; + goto free_irq; return 0; +free_irq: + writeb(0x0, sdhost->pcictrl_mmiobase + VIA_CRDR_PCIINTCTRL); + free_irq(pcidev->irq, sdhost); + cancel_work_sync(&sdhost->carddet_work); + /* carddet_work may re-enable the interrupt via via_reset_pcictrl(). */ + writeb(0x0, sdhost->pcictrl_mmiobase + VIA_CRDR_PCIINTCTRL); unmap: iounmap(sdhost->mmiobase); release: @@ -1199,6 +1205,10 @@ static void via_sd_remove(struct pci_dev *pcidev) free_irq(pcidev->irq, sdhost); + cancel_work_sync(&sdhost->carddet_work); + /* carddet_work may re-enable the interrupt via via_reset_pcictrl(). */ + writeb(0x0, sdhost->pcictrl_mmiobase + VIA_CRDR_PCIINTCTRL); + timer_delete_sync(&sdhost->timer); cancel_work_sync(&sdhost->finish_bh_work); diff --git a/drivers/mtd/devices/mtd_intel_dg.c b/drivers/mtd/devices/mtd_intel_dg.c index f2fa8f68d1905..a566e86eb5e3a 100644 --- a/drivers/mtd/devices/mtd_intel_dg.c +++ b/drivers/mtd/devices/mtd_intel_dg.c @@ -780,7 +780,7 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, dev_name(&aux_dev->dev), invm->regions[i].name); if (!name) { ret = -ENOMEM; - goto err; + goto err_norpm; } nvm->regions[n].name = name; diff --git a/drivers/mtd/mtdoops.c b/drivers/mtd/mtdoops.c index 39df7ce8f55f7..1da7ef06a7a97 100644 --- a/drivers/mtd/mtdoops.c +++ b/drivers/mtd/mtdoops.c @@ -392,6 +392,9 @@ static void mtdoops_notify_remove(struct mtd_info *mtd) cxt->mtd = NULL; flush_work(&cxt->work_erase); flush_work(&cxt->work_write); + vfree(cxt->oops_page_used); + cxt->oops_page_used = NULL; + cxt->oops_pages = 0; } diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c index 4b41550fd374e..ddded0dbe77e0 100644 --- a/drivers/mtd/mtdpart.c +++ b/drivers/mtd/mtdpart.c @@ -258,7 +258,8 @@ int mtd_add_partition(struct mtd_info *parent, const char *name, /* the direct offset is expected */ if (offset == MTDPART_OFS_APPEND || - offset == MTDPART_OFS_NXTBLK) + offset == MTDPART_OFS_NXTBLK || + offset == MTDPART_OFS_RETAIN) return -EINVAL; if (length == MTDPART_SIZ_FULL) diff --git a/drivers/mtd/mtdswap.c b/drivers/mtd/mtdswap.c index f33f753f0a9fd..92e38ece89317 100644 --- a/drivers/mtd/mtdswap.c +++ b/drivers/mtd/mtdswap.c @@ -1452,6 +1452,7 @@ static void mtdswap_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd) debugfs_failed: del_mtd_blktrans_dev(mbd_dev); + mbd_dev = NULL; cleanup: mtdswap_cleanup(d); diff --git a/drivers/mtd/nand/ecc-realtek.c b/drivers/mtd/nand/ecc-realtek.c index 7d003fd72027a..76d75e2107df4 100644 --- a/drivers/mtd/nand/ecc-realtek.c +++ b/drivers/mtd/nand/ecc-realtek.c @@ -450,6 +450,7 @@ static const struct of_device_id rtl_ecc_of_ids[] = { }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, rtl_ecc_of_ids); static struct platform_driver rtl_ecc_driver = { .driver = { diff --git a/drivers/mtd/nand/raw/nand_onfi.c b/drivers/mtd/nand/raw/nand_onfi.c index cd3ad373883e9..b5e304c357382 100644 --- a/drivers/mtd/nand/raw/nand_onfi.c +++ b/drivers/mtd/nand/raw/nand_onfi.c @@ -35,16 +35,21 @@ static int nand_flash_detect_ext_param_page(struct nand_chip *chip, struct nand_onfi_params *p) { struct nand_device *base = &chip->base; + struct mtd_info *mtd = nand_to_mtd(chip); struct nand_ecc_props requirements; struct onfi_ext_param_page *ep; struct onfi_ext_section *s; struct onfi_ext_ecc_info *ecc; + size_t remaining, section_len; uint8_t *cursor; int ret; int len; int i; len = le16_to_cpu(p->ext_param_page_length) * 16; + if (len < sizeof(*ep)) + return -EINVAL; + ep = kmalloc(len, GFP_KERNEL); if (!ep) return -ENOMEM; @@ -77,11 +82,29 @@ static int nand_flash_detect_ext_param_page(struct nand_chip *chip, /* find the ECC section. */ cursor = (uint8_t *)(ep + 1); + remaining = len - sizeof(*ep); for (i = 0; i < ONFI_EXT_SECTION_MAX; i++) { s = ep->sections + i; - if (s->type == ONFI_SECTION_TYPE_2) + section_len = s->length * 16; + if (section_len > remaining) { + dev_dbg(&mtd->dev, + "ONFI extended parameter section %d exceeds page\n", + i); + goto ext_out; + } + + if (s->type == ONFI_SECTION_TYPE_2) { + if (section_len < sizeof(*ecc)) { + dev_dbg(&mtd->dev, + "ONFI extended parameter ECC section %d is too short\n", + i); + goto ext_out; + } break; - cursor += s->length * 16; + } + + cursor += section_len; + remaining -= section_len; } if (i == ONFI_EXT_SECTION_MAX) { pr_debug("We can not find the ECC section.\n"); diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c index 06f8f1e14b9c2..12b9e0936c8f1 100644 --- a/drivers/mtd/nand/raw/pl35x-nand-controller.c +++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c @@ -914,7 +914,6 @@ static int pl35x_nand_init_hw_ecc_controller(struct pl35x_nandc *nfc, chip->ecc.steps = mtd->writesize / chip->ecc.size; chip->ecc.read_page = pl35x_nand_read_page_hwecc; chip->ecc.write_page = pl35x_nand_write_page_hwecc; - chip->ecc.write_page_raw = nand_monolithic_write_page_raw; pl35x_smc_set_ecc_pg_size(nfc, chip, mtd->writesize); nfc->ecc_buf = devm_kmalloc(nfc->dev, chip->ecc.bytes * chip->ecc.steps, @@ -973,18 +972,19 @@ static int pl35x_nand_attach_chip(struct nand_chip *chip) switch (chip->ecc.engine_type) { case NAND_ECC_ENGINE_TYPE_ON_DIE: - dev_dbg(nfc->dev, "Using on-die ECC\n"); + dev_dbg(nfc->dev, "Using on-die hardware ECC\n"); /* Keep these legacy BBT descriptors for ON_DIE situations */ chip->bbt_td = &bbt_main_descr; chip->bbt_md = &bbt_mirror_descr; fallthrough; case NAND_ECC_ENGINE_TYPE_NONE: + dev_dbg(nfc->dev, "Using no ECC engine\n"); + break; case NAND_ECC_ENGINE_TYPE_SOFT: - dev_dbg(nfc->dev, "Using software ECC (Hamming 1-bit/512B)\n"); - chip->ecc.write_page_raw = nand_monolithic_write_page_raw; + dev_dbg(nfc->dev, "Using software ECC\n"); break; case NAND_ECC_ENGINE_TYPE_ON_HOST: - dev_dbg(nfc->dev, "Using hardware ECC\n"); + dev_dbg(nfc->dev, "Using on-host hardware ECC\n"); ret = pl35x_nand_init_hw_ecc_controller(nfc, chip); if (ret) return ret; @@ -995,6 +995,9 @@ static int pl35x_nand_attach_chip(struct nand_chip *chip) return -EINVAL; } + chip->ecc.read_page_raw = nand_monolithic_read_page_raw; + chip->ecc.write_page_raw = nand_monolithic_write_page_raw; + return 0; } diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c index 02647565c8ba9..ca74c9c333e7d 100644 --- a/drivers/mtd/nand/raw/sunxi_nand.c +++ b/drivers/mtd/nand/raw/sunxi_nand.c @@ -237,6 +237,18 @@ struct sunxi_nand_hw_ecc { u32 ecc_ctl; }; +#define SUNXI_NFC_TIMING_STEPS 4 + +/* Delay arrays contain internal NDFC clock cycles for field values 0 to 3. */ +struct sunxi_nfc_timings { + /* Internal clock cycles used by T1-T4, T7 and T11. */ + u8 setup_cycles; + s32 tWB[SUNXI_NFC_TIMING_STEPS]; + s32 tADL[SUNXI_NFC_TIMING_STEPS]; + s32 tWHR[SUNXI_NFC_TIMING_STEPS]; + s32 tRHW[SUNXI_NFC_TIMING_STEPS]; +}; + /** * struct sunxi_nand_chip - stores NAND chip device related information * @@ -301,6 +313,7 @@ static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand) * bytes to write * @nuser_data_tab: Size of @user_data_len_tab * @sram_size: Size of the NAND controller SRAM + * @timings: Controller timing characteristics */ struct sunxi_nfc_caps { bool has_mdma; @@ -327,6 +340,7 @@ struct sunxi_nfc_caps { unsigned int nuser_data_tab; unsigned int max_ecc_steps; int sram_size; + const struct sunxi_nfc_timings *timings; }; /** @@ -1667,8 +1681,21 @@ static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page) return nand_prog_page_end_op(nand); } -static const s32 tWB_lut[] = {6, 12, 16, 20}; -static const s32 tRHW_lut[] = {4, 8, 12, 20}; +static const struct sunxi_nfc_timings sun4i_a10_nfc_timings = { + .setup_cycles = 1, + .tWB = { 6, 12, 16, 20 }, + .tADL = { 7, 15, 23, 31 }, + .tWHR = { 7, 15, 23, 31 }, + .tRHW = { 4, 8, 12, 20 }, +}; + +static const struct sunxi_nfc_timings sun50i_h616_nfc_timings = { + .setup_cycles = 2, + .tWB = { 28, 44, 60, 76 }, + .tADL = { 0, 12, 28, 44 }, + .tWHR = { 0, 12, 28, 44 }, + .tRHW = { 8, 24, 40, 56 }, +}; static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration, u32 clk_period) @@ -1693,6 +1720,7 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline, { struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); + const struct sunxi_nfc_timings *nfc_timings = nfc->caps->timings; const struct nand_sdr_timings *timings; u32 min_clk_period = 0; s32 tWB, tADL, tWHR, tRHW, tCAD; @@ -1703,20 +1731,28 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline, return -ENOTSUPP; /* T1 <=> tCLS */ - if (timings->tCLS_min > min_clk_period) - min_clk_period = timings->tCLS_min; + if (timings->tCLS_min > + min_clk_period * nfc_timings->setup_cycles) + min_clk_period = DIV_ROUND_UP(timings->tCLS_min, + nfc_timings->setup_cycles); /* T2 <=> tCLH */ - if (timings->tCLH_min > min_clk_period) - min_clk_period = timings->tCLH_min; + if (timings->tCLH_min > + min_clk_period * nfc_timings->setup_cycles) + min_clk_period = DIV_ROUND_UP(timings->tCLH_min, + nfc_timings->setup_cycles); /* T3 <=> tCS */ - if (timings->tCS_min > min_clk_period) - min_clk_period = timings->tCS_min; + if (timings->tCS_min > + min_clk_period * nfc_timings->setup_cycles) + min_clk_period = DIV_ROUND_UP(timings->tCS_min, + nfc_timings->setup_cycles); /* T4 <=> tCH */ - if (timings->tCH_min > min_clk_period) - min_clk_period = timings->tCH_min; + if (timings->tCH_min > + min_clk_period * nfc_timings->setup_cycles) + min_clk_period = DIV_ROUND_UP(timings->tCH_min, + nfc_timings->setup_cycles); /* T5 <=> tWP */ if (timings->tWP_min > min_clk_period) @@ -1727,8 +1763,10 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline, min_clk_period = timings->tWH_min; /* T7 <=> tALS */ - if (timings->tALS_min > min_clk_period) - min_clk_period = timings->tALS_min; + if (timings->tALS_min > + min_clk_period * nfc_timings->setup_cycles) + min_clk_period = DIV_ROUND_UP(timings->tALS_min, + nfc_timings->setup_cycles); /* T8 <=> tDS */ if (timings->tDS_min > min_clk_period) @@ -1743,8 +1781,10 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline, min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3); /* T11 <=> tALH */ - if (timings->tALH_min > min_clk_period) - min_clk_period = timings->tALH_min; + if (timings->tALH_min > + min_clk_period * nfc_timings->setup_cycles) + min_clk_period = DIV_ROUND_UP(timings->tALH_min, + nfc_timings->setup_cycles); /* T12 <=> tRP */ if (timings->tRP_min > min_clk_period) @@ -1763,17 +1803,25 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline, min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2); /* T16 - T19 + tCAD */ - if (timings->tWB_max > (min_clk_period * 20)) - min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20); - - if (timings->tADL_min > (min_clk_period * 32)) - min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32); - - if (timings->tWHR_min > (min_clk_period * 32)) - min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32); - - if (timings->tRHW_min > (min_clk_period * 20)) - min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20); + if (timings->tWB_max > + (min_clk_period * nfc_timings->tWB[SUNXI_NFC_TIMING_STEPS - 1])) + min_clk_period = DIV_ROUND_UP(timings->tWB_max, + nfc_timings->tWB[SUNXI_NFC_TIMING_STEPS - 1]); + + if (timings->tADL_min > + (min_clk_period * nfc_timings->tADL[SUNXI_NFC_TIMING_STEPS - 1])) + min_clk_period = DIV_ROUND_UP(timings->tADL_min, + nfc_timings->tADL[SUNXI_NFC_TIMING_STEPS - 1]); + + if (timings->tWHR_min > + (min_clk_period * nfc_timings->tWHR[SUNXI_NFC_TIMING_STEPS - 1])) + min_clk_period = DIV_ROUND_UP(timings->tWHR_min, + nfc_timings->tWHR[SUNXI_NFC_TIMING_STEPS - 1]); + + if (timings->tRHW_min > + (min_clk_period * nfc_timings->tRHW[SUNXI_NFC_TIMING_STEPS - 1])) + min_clk_period = DIV_ROUND_UP(timings->tRHW_min, + nfc_timings->tRHW[SUNXI_NFC_TIMING_STEPS - 1]); /* * In non-EDO, tREA should be less than tRP to guarantee that the @@ -1789,26 +1837,28 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline, if (timings->tREA_max > min_clk_period && !timings->tRLOH_min) min_clk_period = timings->tREA_max; - tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max, + tWB = sunxi_nand_lookup_timing(nfc_timings->tWB, timings->tWB_max, min_clk_period); if (tWB < 0) { dev_err(nfc->dev, "unsupported tWB\n"); return tWB; } - tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3; - if (tADL > 3) { + tADL = sunxi_nand_lookup_timing(nfc_timings->tADL, + timings->tADL_min, min_clk_period); + if (tADL < 0) { dev_err(nfc->dev, "unsupported tADL\n"); - return -EINVAL; + return tADL; } - tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3; - if (tWHR > 3) { + tWHR = sunxi_nand_lookup_timing(nfc_timings->tWHR, + timings->tWHR_min, min_clk_period); + if (tWHR < 0) { dev_err(nfc->dev, "unsupported tWHR\n"); - return -EINVAL; + return tWHR; } - tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min, + tRHW = sunxi_nand_lookup_timing(nfc_timings->tRHW, timings->tRHW_min, min_clk_period); if (tRHW < 0) { dev_err(nfc->dev, "unsupported tRHW\n"); @@ -2595,6 +2645,7 @@ static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = { .nstrengths = ARRAY_SIZE(sunxi_ecc_strengths_a10), .max_ecc_steps = 16, .sram_size = 1024, + .timings = &sun4i_a10_nfc_timings, }; static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = { @@ -2617,6 +2668,7 @@ static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = { .nstrengths = ARRAY_SIZE(sunxi_ecc_strengths_a10), .max_ecc_steps = 16, .sram_size = 1024, + .timings = &sun4i_a10_nfc_timings, }; static const struct sunxi_nfc_caps sunxi_nfc_h616_caps = { @@ -2641,6 +2693,7 @@ static const struct sunxi_nfc_caps sunxi_nfc_h616_caps = { .nuser_data_tab = ARRAY_SIZE(sunxi_user_data_len_h6), .max_ecc_steps = 32, .sram_size = 8192, + .timings = &sun50i_h616_nfc_timings, }; static const struct of_device_id sunxi_nfc_ids[] = { diff --git a/drivers/mtd/parsers/afs.c b/drivers/mtd/parsers/afs.c index 26116694c821b..7ab3d50f565ec 100644 --- a/drivers/mtd/parsers/afs.c +++ b/drivers/mtd/parsers/afs.c @@ -235,6 +235,9 @@ static int afs_parse_v2_partition(struct mtd_info *mtd, pr_debug("Parsing v2 partition @%08x-%08x\n", off, off + mtd->erasesize); + if (mtd->erasesize < sizeof(footer)) + return -EINVAL; + /* First read the footer */ ptr = off + mtd->erasesize - sizeof(footer); ret = mtd_read(mtd, ptr, sizeof(footer), &sz, (u_char *)footer); @@ -245,6 +248,8 @@ static int afs_parse_v2_partition(struct mtd_info *mtd, } name = (char *) &footer[0]; version = footer[9]; + if (footer[8] > mtd->erasesize - sizeof(footer)) + return -EINVAL; ptr = off + mtd->erasesize - sizeof(footer) - footer[8]; pr_debug("found image \"%s\", version %08x, info @%08x\n", @@ -278,6 +283,8 @@ static int afs_parse_v2_partition(struct mtd_info *mtd, entrypoint = imginfo[pad]; attributes = imginfo[pad+1]; region_count = imginfo[pad+2]; + if (region_count > (ARRAY_SIZE(imginfo) - pad - 3) / 4) + return -EINVAL; block_start = imginfo[20]; block_end = imginfo[21]; diff --git a/drivers/mtd/ubi/attach.c b/drivers/mtd/ubi/attach.c index 0fa115cbf3ad4..0ce7ff7400d37 100644 --- a/drivers/mtd/ubi/attach.c +++ b/drivers/mtd/ubi/attach.c @@ -771,7 +771,7 @@ void ubi_remove_av(struct ubi_attach_info *ai, struct ubi_ainf_volume *av) static int early_erase_peb(struct ubi_device *ubi, const struct ubi_attach_info *ai, int pnum, int ec) { - int err; + int err, torture = 0; struct ubi_ec_hdr *ec_hdr; if ((long long)ec >= UBI_MAX_ERASECOUNTER) { @@ -790,7 +790,7 @@ static int early_erase_peb(struct ubi_device *ubi, ec_hdr->ec = cpu_to_be64(ec); - err = ubi_io_sync_erase(ubi, pnum, 0); + err = ubi_io_sync_erase(ubi, pnum, &torture); if (err < 0) goto out_free; diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c index 674ad87809df0..456090a12f52b 100644 --- a/drivers/mtd/ubi/build.c +++ b/drivers/mtd/ubi/build.c @@ -1105,6 +1105,7 @@ int ubi_detach_mtd_dev(int ubi_num, int anyway) ubi->ref_count -= 1; if (ubi->ref_count) { if (!anyway) { + put_device(&ubi->dev); spin_unlock(&ubi_devices_lock); return -EBUSY; } @@ -1317,10 +1318,10 @@ static int __init ubi_init_attach(void) return 0; out_detach: - for (k = 0; k < i; k++) + for (k = 0; k < UBI_MAX_DEVICES; k++) if (ubi_devices[k]) { mutex_lock(&ubi_devices_mutex); - ubi_detach_mtd_dev(ubi_devices[k]->ubi_num, 1); + ubi_detach_mtd_dev(k, 1); mutex_unlock(&ubi_devices_mutex); } return err; @@ -1574,7 +1575,7 @@ static int ubi_mtd_param_parse(const char *val, const struct kernel_param *kp) } module_param_call(mtd, ubi_mtd_param_parse, NULL, NULL, 0400); -MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: mtd=[,[,max_beb_per1024[,ubi_num]]].\n" +MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: mtd=[,[,max_beb_per1024[,ubi_num[,enable_fm[,need_resv_pool]]]]].\n" "Multiple \"mtd\" parameters may be specified.\n" "MTD devices may be specified by their number, name, or path to the MTD character device node.\n" "Optional \"vid_hdr_offs\" parameter specifies UBI VID header position to be used by UBI. (default value if 0)\n" @@ -1587,9 +1588,9 @@ MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: mtd=e[i]) { - ret = ubi_sync_erase(ubi, old_fm->e[i], 0); + ret = ubi_sync_erase(ubi, old_fm->e[i], + &old_fm->to_be_tortured[i]); if (ret < 0) { ubi_err(ubi, "could not erase old fastmap PEB"); @@ -1578,7 +1579,8 @@ int ubi_update_fastmap(struct ubi_device *ubi) if (old_fm) { /* no fresh anchor PEB was found, reuse the old one */ if (!tmp_e) { - ret = ubi_sync_erase(ubi, old_fm->e[0], 0); + ret = ubi_sync_erase(ubi, old_fm->e[0], + &old_fm->to_be_tortured[0]); if (ret < 0) { ubi_err(ubi, "could not erase old anchor PEB"); diff --git a/drivers/mtd/ubi/io.c b/drivers/mtd/ubi/io.c index 915eb64cb0011..d70d1313ad4e8 100644 --- a/drivers/mtd/ubi/io.c +++ b/drivers/mtd/ubi/io.c @@ -519,8 +519,10 @@ static int nor_erase_prepare(struct ubi_device *ubi, int pnum) /** * ubi_io_sync_erase - synchronously erase a physical eraseblock. * @ubi: UBI device description object - * @pnum: physical eraseblock number to erase - * @torture: if this physical eraseblock has to be tortured + * @pnum: the physical eraseblock number to erase + * @torture: if this physical eraseblock has to be tortured; cleared to zero + * once the torture test has completed successfully so that a retry + * of the erase does not torture the physical eraseblock again * * This function synchronously erases physical eraseblock @pnum. If @torture * flag is not zero, the physical eraseblock is checked by means of writing @@ -532,7 +534,7 @@ static int nor_erase_prepare(struct ubi_device *ubi, int pnum) * codes in case of other errors. Note, %-EIO means that the physical * eraseblock is bad. */ -int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int torture) +int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int *torture) { int err, ret = 0; @@ -560,10 +562,11 @@ int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int torture) return err; } - if (torture) { + if (*torture) { ret = torture_peb(ubi, pnum); if (ret < 0) return ret; + *torture = 0; } err = do_sync_erase(ubi, pnum); diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h index af466cd83ae07..80b7628929228 100644 --- a/drivers/mtd/ubi/ubi.h +++ b/drivers/mtd/ubi/ubi.h @@ -901,7 +901,7 @@ int self_check_eba(struct ubi_device *ubi, struct ubi_attach_info *ai_fastmap, struct ubi_attach_info *ai_scan); /* wl.c */ -int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int torture); +int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int *torture); int ubi_wl_get_peb(struct ubi_device *ubi); int ubi_wl_put_peb(struct ubi_device *ubi, int vol_id, int lnum, int pnum, int torture); @@ -923,7 +923,7 @@ int ubi_io_read(const struct ubi_device *ubi, void *buf, int pnum, int offset, int len); int ubi_io_write(struct ubi_device *ubi, const void *buf, int pnum, int offset, int len); -int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int torture); +int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int *torture); int ubi_io_is_bad(const struct ubi_device *ubi, int pnum); int ubi_io_mark_bad(const struct ubi_device *ubi, int pnum); int ubi_io_read_ec_hdr(struct ubi_device *ubi, int pnum, diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c index e3705db8e5709..edfab98f7064f 100644 --- a/drivers/mtd/ubi/wl.c +++ b/drivers/mtd/ubi/wl.c @@ -443,12 +443,14 @@ static int prot_queue_del(struct ubi_device *ubi, int pnum) * ubi_sync_erase - synchronously erase a physical eraseblock. * @ubi: UBI device description object * @e: the physical eraseblock to erase - * @torture: if the physical eraseblock has to be tortured + * @torture: if the physical eraseblock has to be tortured; cleared to zero + * once the torture test has completed successfully so that a retry + * of the erase does not torture the physical eraseblock again * * This function returns zero in case of success and a negative error code in * case of failure. */ -int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int torture) +int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int *torture) { int err; struct ubi_ec_hdr *ec_hdr; @@ -1113,7 +1115,7 @@ static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk) dbg_wl("erase PEB %d EC %d LEB %d:%d", pnum, e->ec, wl_wrk->vol_id, wl_wrk->lnum); - err = ubi_sync_erase(ubi, e, wl_wrk->torture); + err = ubi_sync_erase(ubi, e, &wl_wrk->torture); if (!err) { spin_lock(&ubi->wl_lock); @@ -1150,7 +1152,8 @@ static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk) int err1; /* Re-schedule the LEB for erasure */ - err1 = schedule_erase(ubi, e, vol_id, lnum, 0, true); + err1 = schedule_erase(ubi, e, vol_id, lnum, wl_wrk->torture, + true); if (err1) { spin_lock(&ubi->wl_lock); wl_entry_destroy(ubi, e); @@ -1757,7 +1760,7 @@ static void shutdown_work(struct ubi_device *ubi) static int erase_aeb(struct ubi_device *ubi, struct ubi_ainf_peb *aeb, bool sync) { struct ubi_wl_entry *e; - int err; + int err, torture = 0; e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL); if (!e) @@ -1768,7 +1771,7 @@ static int erase_aeb(struct ubi_device *ubi, struct ubi_ainf_peb *aeb, bool sync ubi->lookuptbl[e->pnum] = e; if (sync) { - err = ubi_sync_erase(ubi, e, false); + err = ubi_sync_erase(ubi, e, &torture); if (err) goto out_free; diff --git a/drivers/net/amt.c b/drivers/net/amt.c index 182a41d59a75d..7645ce3a29ad9 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -3221,6 +3221,9 @@ static int amt_newlink(struct net_device *dev, struct nlattr **tb = params->tb; int err = -EINVAL; + if (!net_eq(link_net, dev_net(dev))) + return err; + amt->net = link_net; amt->mode = nla_get_u32(data[IFLA_AMT_MODE]); diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c index 839f7482dc182..43ac8e28e4182 100644 --- a/drivers/net/bonding/bond_alb.c +++ b/drivers/net/bonding/bond_alb.c @@ -678,9 +678,15 @@ static struct slave *rlb_arp_xmit(struct sk_buff *skb, struct bonding *bond) if (arp->op_code == htons(ARPOP_REPLY)) { /* the arp must be sent on the selected rx channel */ tx_slave = rlb_choose_channel(skb, bond, arp); - if (tx_slave) + if (tx_slave && + !ether_addr_equal_64bits(arp->mac_src, + tx_slave->dev->dev_addr)) { + if (unlikely(skb_cow_head(skb, 0))) + return NULL; + arp = (struct arp_pkt *)skb_network_header(skb); bond_hw_addr_copy(arp->mac_src, tx_slave->dev->dev_addr, tx_slave->dev->addr_len); + } netdev_dbg(bond->dev, "(slave %s): Server sent ARP Reply packet\n", tx_slave ? tx_slave->dev->name : "NULL"); } else if (arp->op_code == htons(ARPOP_REQUEST)) { @@ -875,7 +881,7 @@ static int rlb_initialize(struct bonding *bond) spin_unlock_bh(&bond->mode_lock); /* register to receive ARPs */ - bond->recv_probe = rlb_arp_recv; + WRITE_ONCE(bond->recv_probe, rlb_arp_recv); return 0; } @@ -1281,10 +1287,10 @@ static int alb_set_mac_address(struct bonding *bond, void *addr) } /* determine if the packet is NA or NS */ -static bool alb_determine_nd(struct sk_buff *skb, struct bonding *bond) +static bool alb_determine_nd(struct sk_buff *skb) { - struct ipv6hdr *ip6hdr; - struct icmp6hdr *hdr; + const struct ipv6hdr *ip6hdr; + const struct icmp6hdr *hdr; if (!pskb_network_may_pull(skb, sizeof(*ip6hdr))) return true; @@ -1296,7 +1302,8 @@ static bool alb_determine_nd(struct sk_buff *skb, struct bonding *bond) if (!pskb_network_may_pull(skb, sizeof(*ip6hdr) + sizeof(*hdr))) return true; - hdr = icmp6_hdr(skb); + ip6hdr = ipv6_hdr(skb); + hdr = (const struct icmp6hdr *)(ip6hdr + 1); return hdr->icmp6_type == NDISC_NEIGHBOUR_ADVERTISEMENT || hdr->icmp6_type == NDISC_NEIGHBOUR_SOLICITATION; } @@ -1339,7 +1346,6 @@ static netdev_tx_t bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond, struct slave *tx_slave) { struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond)); - struct ethhdr *eth_data = eth_hdr(skb); if (!tx_slave) { /* unbalanced or unassigned, send through primary */ @@ -1350,7 +1356,9 @@ static netdev_tx_t bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond, if (tx_slave && bond_slave_can_tx(tx_slave)) { if (tx_slave != rcu_access_pointer(bond->curr_active_slave)) { - ether_addr_copy(eth_data->h_source, + if (unlikely(skb_cow_head(skb, 0))) + return bond_tx_drop(bond->dev, skb); + ether_addr_copy(skb_eth_hdr(skb)->h_source, tx_slave->dev->dev_addr); } @@ -1374,14 +1382,13 @@ struct slave *bond_xmit_tlb_slave_get(struct bonding *bond, struct ethhdr *eth_data; u32 hash_index; - skb_reset_mac_header(skb); - eth_data = eth_hdr(skb); + eth_data = skb_eth_hdr(skb); /* Do not TX balance any multicast or broadcast */ if (!is_multicast_ether_addr(eth_data->h_dest)) { switch (skb->protocol) { case htons(ETH_P_IPV6): - if (alb_determine_nd(skb, bond)) + if (alb_determine_nd(skb)) break; fallthrough; case htons(ETH_P_IP): @@ -1427,8 +1434,7 @@ struct slave *bond_xmit_alb_slave_get(struct bonding *bond, u32 hash_index = 0; int hash_size = 0; - skb_reset_mac_header(skb); - eth_data = eth_hdr(skb); + eth_data = skb_eth_hdr(skb); switch (ntohs(skb->protocol)) { case ETH_P_IP: { @@ -1467,7 +1473,7 @@ struct slave *bond_xmit_alb_slave_get(struct bonding *bond, break; } - if (alb_determine_nd(skb, bond)) { + if (alb_determine_nd(skb)) { do_tx_balance = false; break; } diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 522eab060f9ed..34c5bbbed9aee 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -490,7 +490,7 @@ static int bond_ipsec_add_sa(struct net_device *bond_dev, !real_dev->xfrmdev_ops->xdo_dev_state_add || netif_is_bond_master(real_dev)) { NL_SET_ERR_MSG_MOD(extack, "Slave does not support ipsec offload"); - err = -EINVAL; + err = -EOPNOTSUPP; goto out; } @@ -1245,7 +1245,7 @@ static void bond_peer_notify_may_events(struct bonding *bond, bool force) } if (notified || force) - bond->send_peer_notif--; + WRITE_ONCE(bond->send_peer_notif, bond->send_peer_notif - 1); } /** @@ -2284,7 +2284,7 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, } } - bond->slave_cnt++; + WRITE_ONCE(bond->slave_cnt, bond->slave_cnt + 1); netdev_compute_master_upper_features(bond->dev, true); bond_set_carrier(bond); @@ -2517,9 +2517,7 @@ static int __bond_release_one(struct net_device *bond_dev, bond_alb_deinit_slave(bond, slave); } - if (all) { - RCU_INIT_POINTER(bond->curr_active_slave, NULL); - } else if (oldcurrent == slave) { + if (!all && oldcurrent == slave) { /* Note that we hold RTNL over this sequence, so there * is no concern that another slave add/remove event * will interfere. @@ -2533,7 +2531,7 @@ static int __bond_release_one(struct net_device *bond_dev, unblock_netpoll_tx(); synchronize_rcu(); - bond->slave_cnt--; + WRITE_ONCE(bond->slave_cnt, bond->slave_cnt - 1); if (!bond_has_slaves(bond)) { call_netdevice_notifiers(NETDEV_CHANGEADDR, bond->dev); @@ -4385,13 +4383,13 @@ static int bond_open(struct net_device *bond_dev) if (bond->params.arp_interval) { /* arp interval, in milliseconds. */ queue_delayed_work(bond->wq, &bond->arp_work, 0); - bond->recv_probe = bond_rcv_validate; + WRITE_ONCE(bond->recv_probe, bond_rcv_validate); } if (BOND_MODE(bond) == BOND_MODE_8023AD) { queue_delayed_work(bond->wq, &bond->ad_work, 0); /* register to receive LACPDUs */ - bond->recv_probe = bond_3ad_lacpdu_recv; + WRITE_ONCE(bond->recv_probe, bond_3ad_lacpdu_recv); bond_3ad_initiate_agg_selection(bond, 1); bond_for_each_slave(bond, slave, iter) @@ -4413,7 +4411,7 @@ static int bond_close(struct net_device *bond_dev) struct slave *slave; bond_work_cancel_all(bond); - bond->send_peer_notif = 0; + WRITE_ONCE(bond->send_peer_notif, 0); WRITE_ONCE(bond->recv_probe, NULL); /* Wait for any in-flight RX handlers */ @@ -5124,7 +5122,7 @@ static void bond_skip_slave(struct bond_up_slave *slaves, if (skipslave == slaves->arr[idx]) { slaves->arr[idx] = slaves->arr[slaves->count - 1]; - slaves->count--; + WRITE_ONCE(slaves->count, slaves->count - 1); break; } } diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c index 4a11572f663d3..87d92d3cce4a8 100644 --- a/drivers/net/bonding/bond_netlink.c +++ b/drivers/net/bonding/bond_netlink.c @@ -220,7 +220,7 @@ static int bond_changelink(struct net_device *bond_dev, struct nlattr *tb[], struct bonding *bond = netdev_priv(bond_dev); struct bond_opt_value newval; int miimon = 0; - int err; + int err = 0; if (!data) return 0; diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c index e590c8dee86e1..d8cdbfca1bb16 100644 --- a/drivers/net/bonding/bond_options.c +++ b/drivers/net/bonding/bond_options.c @@ -1147,11 +1147,11 @@ static int bond_option_arp_interval_set(struct bonding *bond, */ if (!newval->value) { if (bond->params.arp_validate) - bond->recv_probe = NULL; + WRITE_ONCE(bond->recv_probe, NULL); cancel_delayed_work_sync(&bond->arp_work); } else { /* arp_validate can be set only in active-backup mode */ - bond->recv_probe = bond_rcv_validate; + WRITE_ONCE(bond->recv_probe, bond_rcv_validate); cancel_delayed_work_sync(&bond->mii_work); queue_delayed_work(bond->wq, &bond->arp_work, 0); } diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index eb856547ae7df..16f80607e150e 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -2464,7 +2464,7 @@ struct m_can_classdev *m_can_class_allocate_dev(struct device *dev, return ERR_PTR(ret); } - if (dev->of_node && of_property_read_bool(dev->of_node, "wakeup-source")) + if (dev->of_node && of_property_present(dev->of_node, "wakeup-source")) device_set_wakeup_capable(dev, true); /* Get TX FIFO size diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index 3f5b9592794da..0880310c9ce3f 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -2219,7 +2219,7 @@ int b53_fdb_dump(struct dsa_switch *ds, int port, mutex_unlock(&priv->arl_mutex); - return 0; + return ret; } EXPORT_SYMBOL(b53_fdb_dump); diff --git a/drivers/net/dsa/bcm_sf2_cfp.c b/drivers/net/dsa/bcm_sf2_cfp.c index 50d3a818eb1b7..84a086c3e99b4 100644 --- a/drivers/net/dsa/bcm_sf2_cfp.c +++ b/drivers/net/dsa/bcm_sf2_cfp.c @@ -1088,6 +1088,8 @@ static int bcm_sf2_cfp_rule_get_all(struct bcm_sf2_priv *priv, unsigned int index = 1, rules_cnt = 0; for_each_set_bit_from(index, priv->cfp.unique, priv->num_cfp_rules) { + if (rules_cnt == nfc->rule_cnt) + return -EMSGSIZE; rule_locs[rules_cnt] = index; rules_cnt++; } diff --git a/drivers/net/dsa/lantiq/lantiq_gswip.h b/drivers/net/dsa/lantiq/lantiq_gswip.h index bc3686faad0d6..0b75be14dc109 100644 --- a/drivers/net/dsa/lantiq/lantiq_gswip.h +++ b/drivers/net/dsa/lantiq/lantiq_gswip.h @@ -42,7 +42,7 @@ #define GSWIP_MDIO_PHY_FDUP_DIS 0x0600 #define GSWIP_MDIO_PHY_FCONTX_MASK 0x0180 #define GSWIP_MDIO_PHY_FCONTX_AUTO 0x0000 -#define GSWIP_MDIO_PHY_FCONTX_EN 0x0100 +#define GSWIP_MDIO_PHY_FCONTX_EN 0x0080 #define GSWIP_MDIO_PHY_FCONTX_DIS 0x0180 #define GSWIP_MDIO_PHY_FCONRX_MASK 0x0060 #define GSWIP_MDIO_PHY_FCONRX_AUTO 0x0000 diff --git a/drivers/net/dsa/mt7530-mdio.c b/drivers/net/dsa/mt7530-mdio.c index 784dd58a71589..de42f70afcfa1 100644 --- a/drivers/net/dsa/mt7530-mdio.c +++ b/drivers/net/dsa/mt7530-mdio.c @@ -227,15 +227,17 @@ mt7530_remove(struct mdio_device *mdiodev) if (!priv) return; - ret = regulator_disable(priv->core_pwr); - if (ret < 0) - dev_err(priv->dev, - "Failed to disable core power: %d\n", ret); - - ret = regulator_disable(priv->io_pwr); - if (ret < 0) - dev_err(priv->dev, "Failed to disable io pwr: %d\n", - ret); + if (priv->id == ID_MT7530) { + ret = regulator_disable(priv->core_pwr); + if (ret < 0) + dev_err(priv->dev, + "Failed to disable core power: %d\n", ret); + + ret = regulator_disable(priv->io_pwr); + if (ret < 0) + dev_err(priv->dev, "Failed to disable io pwr: %d\n", + ret); + } mt7530_remove_common(priv); diff --git a/drivers/net/dsa/mt7530-mmio.c b/drivers/net/dsa/mt7530-mmio.c index 119fdd863d917..fd68b1cd06306 100644 --- a/drivers/net/dsa/mt7530-mmio.c +++ b/drivers/net/dsa/mt7530-mmio.c @@ -12,6 +12,7 @@ static const struct of_device_id mt7988_of_match[] = { { .compatible = "airoha,an7583-switch", .data = &mt753x_table[ID_AN7583], }, { .compatible = "airoha,en7581-switch", .data = &mt753x_table[ID_EN7581], }, + { .compatible = "econet,en7528-switch", .data = &mt753x_table[ID_EN7528], }, { .compatible = "mediatek,mt7988-switch", .data = &mt753x_table[ID_MT7988], }, { /* sentinel */ }, }; diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c index aa33d94e11b5f..93dba4498c420 100644 --- a/drivers/net/dsa/mt7530.c +++ b/drivers/net/dsa/mt7530.c @@ -2902,6 +2902,30 @@ static void en7581_mac_port_get_caps(struct dsa_switch *ds, int port, } } +static void en7528_mac_port_get_caps(struct dsa_switch *ds, int port, + struct phylink_config *config) +{ + switch (port) { + /* Ports which are connected to switch PHYs. There is no MII pinout. */ + case 1 ... 4: + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + config->supported_interfaces); + + config->mac_capabilities |= MAC_10 | MAC_100 | MAC_1000FD; + break; + + /* Port 6 is connected to SoC's GMAC at 1000 Mbps full duplex. There + * is no MII pinout. + */ + case 6: + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + config->supported_interfaces); + + config->mac_capabilities |= MAC_1000FD; + break; + } +} + static void mt7530_mac_config(struct dsa_switch *ds, int port, unsigned int mode, phy_interface_t interface) @@ -3091,19 +3115,34 @@ static void mt753x_phylink_get_caps(struct dsa_switch *ds, int port, struct phylink_config *config) { struct mt7530_priv *priv = ds->priv; - u32 eeecr; config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE; - config->lpi_capabilities = MAC_100FD | MAC_1000FD | MAC_2500FD; + priv->info->mac_port_get_caps(ds, port, config); - eeecr = mt7530_read(priv, MT753X_PMEEECR_P(port)); - /* tx_lpi_timer should be in microseconds. The time units for - * LPI threshold are unspecified. + /* The EN7528 GPHYs report EEE capability, but negotiating EEE with + * common link partners (e.g. Realtek GbE NICs) results in an unstable + * link with dropped frames. Leave the LPI capabilities empty so that + * phylink disables EEE on these PHYs and refuses to enable it from + * userspace. */ - config->lpi_timer_default = FIELD_GET(LPI_THRESH_MASK, eeecr); - - priv->info->mac_port_get_caps(ds, port, config); + if (priv->id != ID_EN7528 && + config->mac_capabilities & (MAC_100FD | MAC_1000FD)) { + u32 eeecr = mt7530_read(priv, MT753X_PMEEECR_P(port)); + + /* LPI above 1 Gbps is not supported */ + config->lpi_capabilities = config->mac_capabilities & + (MAC_100FD | MAC_1000FD); + phy_interface_copy(config->lpi_interfaces, + config->supported_interfaces); + __clear_bit(PHY_INTERFACE_MODE_2500BASEX, + config->lpi_interfaces); + + /* tx_lpi_timer should be in microseconds. The time units for + * LPI threshold are unspecified. + */ + config->lpi_timer_default = FIELD_GET(LPI_THRESH_MASK, eeecr); + } } static int mt753x_pcs_validate(struct phylink_pcs *pcs, @@ -3244,7 +3283,8 @@ mt753x_conduit_state_change(struct dsa_switch *ds, * forwarded to the numerically smallest CPU port whose conduit * interface is up. */ - if (priv->id != ID_MT7530 && priv->id != ID_MT7621) + if (priv->id != ID_MT7530 && priv->id != ID_MT7621 && + priv->id != ID_EN7528) return; mask = BIT(cpu_dp->index); @@ -3449,6 +3489,16 @@ const struct mt753x_info mt753x_table[] = { .phy_write_c45 = mt7531_ind_c45_phy_write, .mac_port_get_caps = en7581_mac_port_get_caps, }, + [ID_EN7528] = { + .id = ID_EN7528, + .pcs_ops = &mt7530_pcs_ops, + .sw_setup = mt7988_setup, + .phy_read_c22 = mt7531_ind_c22_phy_read, + .phy_write_c22 = mt7531_ind_c22_phy_write, + .phy_read_c45 = mt7531_ind_c45_phy_read, + .phy_write_c45 = mt7531_ind_c45_phy_write, + .mac_port_get_caps = en7528_mac_port_get_caps, + }, }; EXPORT_SYMBOL_GPL(mt753x_table); @@ -3489,9 +3539,6 @@ EXPORT_SYMBOL_GPL(mt7530_probe_common); void mt7530_remove_common(struct mt7530_priv *priv) { - if (priv->irq_domain) - mt7530_free_mdio_irq(priv); - dsa_unregister_switch(priv->ds); mutex_destroy(&priv->reg_mutex); diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h index dd33b0df3419e..5f1e841f42c0e 100644 --- a/drivers/net/dsa/mt7530.h +++ b/drivers/net/dsa/mt7530.h @@ -21,6 +21,7 @@ enum mt753x_id { ID_MT7988 = 3, ID_EN7581 = 4, ID_AN7583 = 5, + ID_EN7528 = 6, }; #define NUM_TRGMII_CTRL 5 diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 80b877c74513d..a4a8c7e11bf4f 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -2438,6 +2438,7 @@ static int mv88e6xxx_get_rxnfc(struct dsa_switch *ds, int port, struct ethtool_rx_flow_spec *fs = &rxnfc->fs; struct mv88e6xxx_chip *chip = ds->priv; struct mv88e6xxx_policy *policy; + u32 cnt = 0; int err; int id; @@ -2463,11 +2464,18 @@ static int mv88e6xxx_get_rxnfc(struct dsa_switch *ds, int port, break; case ETHTOOL_GRXCLSRLALL: rxnfc->data = 0; - rxnfc->rule_cnt = 0; - idr_for_each_entry(&chip->policies, policy, id) - if (policy->port == port) - rule_locs[rxnfc->rule_cnt++] = id; err = 0; + idr_for_each_entry(&chip->policies, policy, id) { + if (policy->port != port) + continue; + if (cnt == rxnfc->rule_cnt) { + err = -EMSGSIZE; + break; + } + rule_locs[cnt++] = id; + } + if (!err) + rxnfc->rule_cnt = cnt; break; default: err = -EOPNOTSUPP; @@ -5631,6 +5639,68 @@ static const struct mv88e6xxx_ops mv88e6390x_ops = { .pcs_ops = &mv88e6390_pcs_ops, }; +static const struct mv88e6xxx_ops mv88e6191x_ops = { + /* MV88E6XXX_FAMILY_6393 without AVB and PTP: 6191X and 6193X */ + .irl_init_all = mv88e6390_g2_irl_init_all, + .get_eeprom = mv88e6xxx_g2_get_eeprom8, + .set_eeprom = mv88e6xxx_g2_set_eeprom8, + .set_switch_mac = mv88e6xxx_g2_set_switch_mac, + .phy_read = mv88e6xxx_g2_smi_phy_read_c22, + .phy_write = mv88e6xxx_g2_smi_phy_write_c22, + .phy_read_c45 = mv88e6xxx_g2_smi_phy_read_c45, + .phy_write_c45 = mv88e6xxx_g2_smi_phy_write_c45, + .port_set_link = mv88e6xxx_port_set_link, + .port_sync_link = mv88e6xxx_port_sync_link, + .port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay, + .port_set_speed_duplex = mv88e6393x_port_set_speed_duplex, + .port_tag_remap = mv88e6390_port_tag_remap, + .port_set_policy = mv88e6393x_port_set_policy, + .port_set_frame_mode = mv88e6351_port_set_frame_mode, + .port_set_ucast_flood = mv88e6352_port_set_ucast_flood, + .port_set_mcast_flood = mv88e6352_port_set_mcast_flood, + .port_set_ether_type = mv88e6393x_port_set_ether_type, + .port_set_jumbo_size = mv88e6165_port_set_jumbo_size, + .port_egress_rate_limiting = mv88e6097_port_egress_rate_limiting, + .port_pause_limit = mv88e6390_port_pause_limit, + .port_disable_learn_limit = mv88e6xxx_port_disable_learn_limit, + .port_disable_pri_override = mv88e6xxx_port_disable_pri_override, + .port_get_cmode = mv88e6352_port_get_cmode, + .port_set_cmode = mv88e6393x_port_set_cmode, + .port_setup_message_port = mv88e6xxx_setup_message_port, + .port_set_upstream_port = mv88e6393x_port_set_upstream_port, + .port_enable_tcam = mv88e6xxx_port_enable_tcam, + .stats_snapshot = mv88e6390_g1_stats_snapshot, + .stats_set_histogram = mv88e6390_g1_stats_set_histogram, + .stats_get_sset_count = mv88e6320_stats_get_sset_count, + .stats_get_strings = mv88e6320_stats_get_strings, + .stats_get_stat = mv88e6390_stats_get_stat, + /* .set_cpu_port is missing because this family does not support a global + * CPU port, only per port CPU port which is set via + * .port_set_upstream_port method. + */ + .set_egress_port = mv88e6393x_set_egress_port, + .watchdog_ops = &mv88e6393x_watchdog_ops, + .mgmt_rsvd2cpu = mv88e6393x_port_mgmt_rsvd2cpu, + .pot_clear = mv88e6xxx_g2_pot_clear, + .hardware_reset_pre = mv88e6xxx_g2_eeprom_wait, + .hardware_reset_post = mv88e6xxx_g2_eeprom_wait, + .reset = mv88e6352_g1_reset, + .rmu_disable = mv88e6390_g1_rmu_disable, + .atu_get_hash = mv88e6165_g1_atu_get_hash, + .atu_set_hash = mv88e6165_g1_atu_set_hash, + .vtu_getnext = mv88e6390_g1_vtu_getnext, + .vtu_loadpurge = mv88e6390_g1_vtu_loadpurge, + .stu_getnext = mv88e6390_g1_stu_getnext, + .stu_loadpurge = mv88e6390_g1_stu_loadpurge, + .serdes_get_lane = mv88e6393x_serdes_get_lane, + .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, + /* TODO: serdes stats */ + .gpio_ops = &mv88e6352_gpio_ops, + .phylink_get_caps = mv88e6393x_phylink_get_caps, + .pcs_ops = &mv88e6393x_pcs_ops, + .tcam_ops = &mv88e6393_tcam_ops, +}; + static const struct mv88e6xxx_ops mv88e6393x_ops = { /* MV88E6XXX_FAMILY_6393 */ .irl_init_all = mv88e6390_g2_irl_init_all, @@ -6155,8 +6225,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = { .atu_move_port_mask = 0x1f, .pvt = true, .multi_chip = true, - .ptp_support = true, - .ops = &mv88e6393x_ops, + .ops = &mv88e6191x_ops, }, [MV88E6193X] = { @@ -6182,8 +6251,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = { .atu_move_port_mask = 0x1f, .pvt = true, .multi_chip = true, - .ptp_support = true, - .ops = &mv88e6393x_ops, + .ops = &mv88e6191x_ops, }, [MV88E6220] = { diff --git a/drivers/net/dsa/mv88e6xxx/pcs-6352.c b/drivers/net/dsa/mv88e6xxx/pcs-6352.c index 4228ae5bb9db0..437054711a2d8 100644 --- a/drivers/net/dsa/mv88e6xxx/pcs-6352.c +++ b/drivers/net/dsa/mv88e6xxx/pcs-6352.c @@ -305,13 +305,16 @@ static bool mv88e6352_pcs_link_check(struct marvell_c22_pcs *mpcs) struct mv88e6xxx_port *port = mpcs->port; struct mv88e6xxx_chip *chip = port->chip; u8 cmode; + int err; /* Port 4 can be in auto-media mode. Check that the port is * associated with the mpcs. */ mv88e6xxx_reg_lock(chip); - chip->info->ops->port_get_cmode(chip, port->port, &cmode); + err = chip->info->ops->port_get_cmode(chip, port->port, &cmode); mv88e6xxx_reg_unlock(chip); + if (err) + return false; return cmode == MV88E6XXX_PORT_STS_CMODE_100BASEX || cmode == MV88E6XXX_PORT_STS_CMODE_1000BASEX || diff --git a/drivers/net/dsa/mxl862xx/mxl862xx.c b/drivers/net/dsa/mxl862xx/mxl862xx.c index 45d237b3a40f1..e05ad52cd297e 100644 --- a/drivers/net/dsa/mxl862xx/mxl862xx.c +++ b/drivers/net/dsa/mxl862xx/mxl862xx.c @@ -685,10 +685,22 @@ static int mxl862xx_setup(struct dsa_switch *ds) if (ret) return ret; + ret = mxl862xx_setup_mdio(ds); + if (ret) + return ret; + schedule_delayed_work(&priv->stats_work, MXL862XX_STATS_POLL_INTERVAL); - return mxl862xx_setup_mdio(ds); + return 0; +} + +static void mxl862xx_teardown(struct dsa_switch *ds) +{ + struct mxl862xx_priv *priv = ds->priv; + + set_bit(MXL862XX_FLAG_WORK_STOPPED, &priv->flags); + disable_delayed_work_sync(&priv->stats_work); } static int mxl862xx_port_state(struct dsa_switch *ds, int port, bool enable) @@ -2047,9 +2059,7 @@ static void mxl862xx_get_stats64(struct dsa_switch *ds, int port, spin_unlock_bh(&priv->ports[port].stats_lock); - /* Trigger a fresh poll so the next read sees up-to-date counters. - * No-op if the work is already pending, running, or teardown started. - */ + /* Trigger a fresh poll so the next read sees up-to-date counters. */ if (!test_bit(MXL862XX_FLAG_WORK_STOPPED, &priv->flags)) schedule_delayed_work(&priv->stats_work, 0); } @@ -2057,6 +2067,7 @@ static void mxl862xx_get_stats64(struct dsa_switch *ds, int port, static const struct dsa_switch_ops mxl862xx_switch_ops = { .get_tag_protocol = mxl862xx_get_tag_protocol, .setup = mxl862xx_setup, + .teardown = mxl862xx_teardown, .port_setup = mxl862xx_port_setup, .port_teardown = mxl862xx_port_teardown, .phylink_get_caps = mxl862xx_phylink_get_caps, @@ -2111,6 +2122,7 @@ static int mxl862xx_probe(struct mdio_device *mdiodev) ds->ops = &mxl862xx_switch_ops; ds->phylink_mac_ops = &mxl862xx_phylink_mac_ops; ds->num_ports = MXL862XX_MAX_PORTS; + ds->assisted_learning_on_cpu_port = true; ds->fdb_isolation = true; ds->max_num_bridges = MXL862XX_MAX_BRIDGES; @@ -2130,7 +2142,6 @@ static int mxl862xx_probe(struct mdio_device *mdiodev) err = dsa_register_switch(ds); if (err) { set_bit(MXL862XX_FLAG_WORK_STOPPED, &priv->flags); - cancel_delayed_work_sync(&priv->stats_work); mxl862xx_host_shutdown(priv); for (i = 0; i < MXL862XX_MAX_PORTS; i++) cancel_work_sync(&priv->ports[i].host_flood_work); @@ -2151,7 +2162,6 @@ static void mxl862xx_remove(struct mdio_device *mdiodev) priv = ds->priv; set_bit(MXL862XX_FLAG_WORK_STOPPED, &priv->flags); - cancel_delayed_work_sync(&priv->stats_work); dsa_unregister_switch(ds); @@ -2180,7 +2190,7 @@ static void mxl862xx_shutdown(struct mdio_device *mdiodev) dsa_switch_shutdown(ds); set_bit(MXL862XX_FLAG_WORK_STOPPED, &priv->flags); - cancel_delayed_work_sync(&priv->stats_work); + disable_delayed_work_sync(&priv->stats_work); mxl862xx_host_shutdown(priv); diff --git a/drivers/net/dsa/realtek/rtl83xx.c b/drivers/net/dsa/realtek/rtl83xx.c index 9dd50b20c000a..d2e92bab55d29 100644 --- a/drivers/net/dsa/realtek/rtl83xx.c +++ b/drivers/net/dsa/realtek/rtl83xx.c @@ -321,7 +321,7 @@ void rtl83xx_reset_assert(struct realtek_priv *priv) "Failed to assert the switch reset control: %pe\n", ERR_PTR(ret)); - gpiod_set_value(priv->reset, true); + gpiod_set_value_cansleep(priv->reset, true); } void rtl83xx_reset_deassert(struct realtek_priv *priv) @@ -334,7 +334,7 @@ void rtl83xx_reset_deassert(struct realtek_priv *priv) "Failed to deassert the switch reset control: %pe\n", ERR_PTR(ret)); - gpiod_set_value(priv->reset, false); + gpiod_set_value_cansleep(priv->reset, false); } /** diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c index 79418e682f71f..2fc8c91f210c6 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.c +++ b/drivers/net/ethernet/airoha/airoha_eth.c @@ -2521,16 +2521,20 @@ static int airoha_qdma_get_tx_ets_stats(struct net_device *netdev, int channel, { struct airoha_gdm_dev *dev = netdev_priv(netdev); struct airoha_qdma *qdma = dev->qdma; + u32 cpu_tx_packets, fwd_tx_packets; + u64 tx_packets; - u64 cpu_tx_packets = airoha_qdma_rr(qdma, REG_CNTR_VAL(channel << 1)); - u64 fwd_tx_packets = airoha_qdma_rr(qdma, - REG_CNTR_VAL((channel << 1) + 1)); - u64 tx_packets = (cpu_tx_packets - dev->cpu_tx_packets) + - (fwd_tx_packets - dev->fwd_tx_packets); + cpu_tx_packets = airoha_qdma_rr(qdma, REG_CNTR_VAL(channel << 1)); + fwd_tx_packets = airoha_qdma_rr(qdma, + REG_CNTR_VAL((channel << 1) + 1)); + tx_packets = (u32)(cpu_tx_packets - + dev->qos_stats[channel].cpu_tx_packets); + tx_packets += (u32)(fwd_tx_packets - + dev->qos_stats[channel].fwd_tx_packets); _bstats_update(opt->stats.bstats, 0, tx_packets); - dev->cpu_tx_packets = cpu_tx_packets; - dev->fwd_tx_packets = fwd_tx_packets; + dev->qos_stats[channel].cpu_tx_packets = cpu_tx_packets; + dev->qos_stats[channel].fwd_tx_packets = fwd_tx_packets; return 0; } diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h index fe934f9ffe8a1..70f16189b2582 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.h +++ b/drivers/net/ethernet/airoha/airoha_eth.h @@ -538,7 +538,7 @@ struct airoha_wdma_info { /* RX queue to IRQ mapping: BIT(q) in IRQ(n) */ #define RX_IRQ0_BANK_PIN_MASK 0x839f -#define RX_IRQ1_BANK_PIN_MASK 0x7fe00000 +#define RX_IRQ1_BANK_PIN_MASK 0xffe00000 #define RX_IRQ2_BANK_PIN_MASK 0x20 #define RX_IRQ3_BANK_PIN_MASK 0x40 #define RX_IRQ_BANK_PIN_MASK(_n) \ @@ -580,9 +580,10 @@ struct airoha_gdm_dev { struct airoha_eth *eth; DECLARE_BITMAP(qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS); - /* qos stats counters */ - u64 cpu_tx_packets; - u64 fwd_tx_packets; + struct { + u32 cpu_tx_packets; + u32 fwd_tx_packets; + } qos_stats[AIROHA_NUM_QOS_CHANNELS]; u32 flags; int nbq; diff --git a/drivers/net/ethernet/airoha/airoha_npu.c b/drivers/net/ethernet/airoha/airoha_npu.c index b679bed952de3..3862f829f1835 100644 --- a/drivers/net/ethernet/airoha/airoha_npu.c +++ b/drivers/net/ethernet/airoha/airoha_npu.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -749,12 +750,15 @@ static int airoha_npu_probe(struct platform_device *pdev) if (irq < 0) return irq; + err = devm_work_autocancel(dev, &core->wdt_work, + airoha_npu_wdt_work); + if (err) + return err; + err = devm_request_irq(dev, irq, airoha_npu_wdt_handler, IRQF_SHARED, "airoha-npu-wdt", core); if (err) return err; - - INIT_WORK(&core->wdt_work, airoha_npu_wdt_work); } /* wlan IRQ lines */ @@ -766,7 +770,7 @@ static int airoha_npu_probe(struct platform_device *pdev) npu->irqs[i] = irq; } - err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); if (err) return err; @@ -801,18 +805,8 @@ static int airoha_npu_probe(struct platform_device *pdev) return 0; } -static void airoha_npu_remove(struct platform_device *pdev) -{ - struct airoha_npu *npu = platform_get_drvdata(pdev); - int i; - - for (i = 0; i < ARRAY_SIZE(npu->cores); i++) - cancel_work_sync(&npu->cores[i].wdt_work); -} - static struct platform_driver airoha_npu_driver = { .probe = airoha_npu_probe, - .remove = airoha_npu_remove, .driver = { .name = "airoha-npu", .of_match_table = of_airoha_npu_match, diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c index f6396925722d0..bc2b4deafd83a 100644 --- a/drivers/net/ethernet/airoha/airoha_ppe.c +++ b/drivers/net/ethernet/airoha/airoha_ppe.c @@ -280,27 +280,36 @@ static int airoha_ppe_get_wdma_info(struct net_device *dev, const u8 *addr, struct airoha_wdma_info *info) { struct net_device_path_stack stack; + struct net_device_path_ctx ctx = { + .dev = dev, + }; struct net_device_path *path; int err; if (!dev) return -ENODEV; + ether_addr_copy(ctx.daddr, addr); + rcu_read_lock(); - err = dev_fill_forward_path(dev, addr, &stack); + err = dev_fill_forward_path(&ctx, &stack); rcu_read_unlock(); if (err) return err; path = &stack.path[stack.num_paths - 1]; - if (path->type != DEV_PATH_MTK_WDMA) - return -EINVAL; + if (path->type != DEV_PATH_MTK_WDMA) { + err = -EINVAL; + goto err_out; + } info->idx = path->mtk_wdma.wdma_idx; info->bss = path->mtk_wdma.bss; info->wcid = path->mtk_wdma.wcid; +err_out: + dev_fill_forward_path_release(&stack); - return 0; + return err; } static int airoha_get_dsa_port(struct net_device **dev) diff --git a/drivers/net/ethernet/allwinner/sun4i-emac.c b/drivers/net/ethernet/allwinner/sun4i-emac.c index fc7341a5cbb7d..9382b4b329bdf 100644 --- a/drivers/net/ethernet/allwinner/sun4i-emac.c +++ b/drivers/net/ethernet/allwinner/sun4i-emac.c @@ -996,9 +996,9 @@ static int emac_probe(struct platform_device *pdev) /* fill in parameters for net-dev structure */ ndev->base_addr = (unsigned long)db->membase; ndev->irq = irq_of_parse_and_map(np, 0); - if (ndev->irq == -ENXIO) { + if (!ndev->irq) { netdev_err(ndev, "No irq resource\n"); - ret = ndev->irq; + ret = -ENXIO; goto out_iounmap; } @@ -1067,6 +1067,7 @@ static int emac_probe(struct platform_device *pdev) return 0; out_release_sram: + of_node_put(db->phy_node); sunxi_sram_release(&pdev->dev); out_clk_disable_unprepare: clk_disable_unprepare(db->clk); @@ -1094,6 +1095,7 @@ static void emac_remove(struct platform_device *pdev) } unregister_netdev(ndev); + of_node_put(db->phy_node); sunxi_sram_release(&pdev->dev); clk_disable_unprepare(db->clk); irq_dispose_mapping(ndev->irq); diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index 5d05020a6d058..e35eef9f6ce15 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -4126,6 +4126,8 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err_device_destroy: ena_com_delete_host_info(ena_dev); ena_com_admin_destroy(ena_dev); + ena_phc_destroy(adapter); + ena_com_mmio_reg_read_request_destroy(ena_dev); ena_devlink_destroy: ena_devlink_free(devlink); err_metrics_destroy: diff --git a/drivers/net/ethernet/amd/pds_core/fw.c b/drivers/net/ethernet/amd/pds_core/fw.c index fa626719e68d1..59d262b0d29c5 100644 --- a/drivers/net/ethernet/amd/pds_core/fw.c +++ b/drivers/net/ethernet/amd/pds_core/fw.c @@ -107,8 +107,10 @@ int pdsc_firmware_update(struct pdsc *pdsc, const struct firmware *fw, dev_info(pdsc->dev, "Installing firmware\n"); - if (!pdsc->cmd_regs) + if (!pdsc->cmd_regs) { + NL_SET_ERR_MSG_MOD(extack, "BARs not mapped"); return -ENXIO; + } dl = priv_to_devlink(pdsc); devlink_flash_update_status_notify(dl, "Preparing to flash", @@ -134,6 +136,12 @@ int pdsc_firmware_update(struct pdsc *pdsc, const struct firmware *fw, copy_sz = min_t(unsigned int, buf_sz, fw->size - offset); mutex_lock(&pdsc->devcmd_lock); + if (!pdsc->cmd_regs) { + mutex_unlock(&pdsc->devcmd_lock); + err = -ENXIO; + NL_SET_ERR_MSG_MOD(extack, "Device reset during flash"); + goto err_out; + } memcpy_toio(&pdsc->cmd_regs->data, fw->data + offset, copy_sz); err = pdsc_devcmd_fw_download_locked(pdsc, data_addr, offset, copy_sz); diff --git a/drivers/net/ethernet/amd/pds_core/main.c b/drivers/net/ethernet/amd/pds_core/main.c index 9a2c64198d03b..a19269f6b66c7 100644 --- a/drivers/net/ethernet/amd/pds_core/main.c +++ b/drivers/net/ethernet/amd/pds_core/main.c @@ -507,8 +507,12 @@ static void pdsc_reset_prepare(struct pci_dev *pdev) pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev); } - pdsc_unmap_bars(pdsc); - pci_release_regions(pdev); + if (!pdev->is_virtfn) { + mutex_lock(&pdsc->devcmd_lock); + pdsc_unmap_bars(pdsc); + mutex_unlock(&pdsc->devcmd_lock); + pci_release_regions(pdev); + } if (pci_is_enabled(pdev)) pci_disable_device(pdev); } @@ -536,7 +540,9 @@ static void pdsc_reset_done(struct pci_dev *pdev) return; } + mutex_lock(&pdsc->devcmd_lock); err = pdsc_map_bars(pdsc); + mutex_unlock(&pdsc->devcmd_lock); if (err) return; } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c index 2de974213090c..e2e850c1b90b1 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c @@ -3400,7 +3400,7 @@ static void xgbe_enable_rx(struct xgbe_prv_data *pdata) XGMAC_IOWRITE(pdata, MAC_RQC0R, reg_val); /* Enable MAC Rx */ - XGMAC_IOWRITE_BITS(pdata, MAC_RCR, DCRCC, 1); + XGMAC_IOWRITE_BITS(pdata, MAC_RCR, DCRCC, 0); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, CST, 1); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, ACS, 1); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, RE, 1); @@ -3411,7 +3411,6 @@ static void xgbe_disable_rx(struct xgbe_prv_data *pdata) unsigned int i; /* Disable MAC Rx */ - XGMAC_IOWRITE_BITS(pdata, MAC_RCR, DCRCC, 0); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, CST, 0); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, ACS, 0); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, RE, 0); diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c index 7efa3fc257b39..e58f1d2c26bdf 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c +++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c @@ -1602,6 +1602,9 @@ static int atl1c_clean_tx(struct napi_struct *napi, int budget) AT_READ_REGW(&adapter->hw, atl1c_qregs[tpd_ring->num].tpd_cons, &hw_next_to_clean); + if (unlikely(hw_next_to_clean >= tpd_ring->count)) + hw_next_to_clean = next_to_clean; + while (next_to_clean != hw_next_to_clean) { buffer_info = &tpd_ring->buffer_info[next_to_clean]; if (buffer_info->skb) { diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c index 40290028580ba..437989edb7802 100644 --- a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c +++ b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c @@ -1234,6 +1234,9 @@ static bool atl1e_clean_tx_irq(struct atl1e_adapter *adapter) u16 hw_next_to_clean = AT_READ_REGW(&adapter->hw, REG_TPD_CONS_IDX); u16 next_to_clean = atomic_read(&tx_ring->next_to_clean); + if (unlikely(hw_next_to_clean >= tx_ring->count)) + hw_next_to_clean = next_to_clean; + while (next_to_clean != hw_next_to_clean) { tx_buffer = &tx_ring->tx_buffer[next_to_clean]; if (tx_buffer->dma) { diff --git a/drivers/net/ethernet/atheros/atlx/atl1.c b/drivers/net/ethernet/atheros/atlx/atl1.c index 98a4d089270e4..957d5598dda5e 100644 --- a/drivers/net/ethernet/atheros/atlx/atl1.c +++ b/drivers/net/ethernet/atheros/atlx/atl1.c @@ -2066,6 +2066,9 @@ static int atl1_intr_tx(struct atl1_adapter *adapter) sw_tpd_next_to_clean = atomic_read(&tpd_ring->next_to_clean); cmb_tpd_next_to_clean = le16_to_cpu(adapter->cmb.cmb->tpd_cons_idx); + if (unlikely(cmb_tpd_next_to_clean >= tpd_ring->count)) + cmb_tpd_next_to_clean = sw_tpd_next_to_clean; + while (cmb_tpd_next_to_clean != sw_tpd_next_to_clean) { buffer_info = &tpd_ring->buffer_info[sw_tpd_next_to_clean]; if (buffer_info->dma) { diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c index ed0977832ce4f..f2176ef3a127b 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c @@ -148,8 +148,9 @@ static int tx_spb_ring_full(struct bcmasp_intf *intf, int cnt) int next_index, i; /* Check if we have enough room for cnt descriptors */ + next_index = intf->tx_spb_index; for (i = 0; i < cnt; i++) { - next_index = incr_ring(intf->tx_spb_index, DESC_RING_COUNT); + next_index = incr_ring(next_index, DESC_RING_COUNT); if (next_index == intf->tx_spb_clean_index) return 1; } @@ -301,6 +302,7 @@ static netdev_tx_t bcmasp_xmit(struct sk_buff *skb, struct net_device *dev) txcb->bytes_sent = total_bytes; dma_unmap_addr_set(txcb, dma_addr, mapping); dma_unmap_len_set(txcb, dma_len, size); + txcb->last = false; if (!i) { desc->flags |= DESC_SOF; if (csum_hw) diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index 208a894d6190c..39eb6ab5f8058 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -13473,10 +13473,13 @@ static int bnx2x_init_firmware(struct bnx2x *bp) iro_alloc_err: kfree(bp->init_ops_offsets); + bp->init_ops_offsets = NULL; init_offsets_alloc_err: kfree(bp->init_ops); + bp->init_ops = NULL; init_ops_alloc_err: kfree(bp->init_data); + bp->init_data = NULL; request_firmware_exit: release_firmware(bp->firmware); bp->firmware = NULL; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index bc7b37cb74a71..ced998399efc3 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -462,6 +462,16 @@ u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr, u16 prod) { + /* If the most recent BD has its completion suppressed, unset the bit + * so that a completion is generated, otherwise nothing is left to + * clean the ring and wake the queue. + */ + if (txr->kick_txbd0) { + txr->kick_txbd0->tx_bd_len_flags_type &= + cpu_to_le32(~TX_BD_FLAGS_NO_CMPL); + txr->kick_txbd0 = NULL; + } + /* Sync BD data before updating doorbell */ wmb(); bnxt_db_write(bp, &txr->tx_db, prod); @@ -501,17 +511,30 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) if (skb_shinfo(skb)->nr_frags > TX_MAX_FRAGS) { netdev_warn_once(dev, "SKB has too many (%d) fragments, max supported is %d. SKB will be linearized.\n", skb_shinfo(skb)->nr_frags, TX_MAX_FRAGS); - if (skb_linearize(skb)) { - dev_kfree_skb_any(skb); - dev_core_stats_tx_dropped_inc(dev); - return NETDEV_TX_OK; - } + if (skb_linearize(skb)) + goto tx_free; } #endif if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) && - !(bp->flags & BNXT_FLAG_UDP_GSO_CAP)) - return bnxt_sw_udp_gso_xmit(bp, txr, txq, skb); + !(bp->flags & BNXT_FLAG_UDP_GSO_CAP)) { + int rc = bnxt_sw_udp_gso_xmit(bp, txr, txq, skb); + + /* if SW USO queued a packet, the doorbell will be written + * below and there is no reason to track the last BD with + * suppressed completions + */ + if (rc > 0) + txr->kick_txbd0 = NULL; + + /* if a packet was queued by SW USO or a doorbell was pending + * from a previous xmit that was deferred, write the doorbell. + */ + if (rc > 0 || txr->kick_pending) + bnxt_txr_db_kick(bp, txr, txr->tx_prod); + + return rc < 0 ? NETDEV_TX_BUSY : NETDEV_TX_OK; + } free_size = bnxt_tx_avail(bp, txr); if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { @@ -748,23 +771,23 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) prod = NEXT_TX(prod); WRITE_ONCE(txr->tx_prod, prod); + txr->kick_txbd0 = NULL; if (!netdev_xmit_more() || netif_xmit_stopped(txq)) { bnxt_txr_db_kick(bp, txr, prod); } else { - if (free_size >= bp->tx_wake_thresh) + if (free_size >= bp->tx_wake_thresh) { txbd0->tx_bd_len_flags_type |= cpu_to_le32(TX_BD_FLAGS_NO_CMPL); + txr->kick_txbd0 = txbd0; + } txr->kick_pending = 1; } tx_done: if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { - if (netdev_xmit_more() && !tx_buf->is_push) { - txbd0->tx_bd_len_flags_type &= - cpu_to_le32(~TX_BD_FLAGS_NO_CMPL); + if (txr->kick_pending) bnxt_txr_db_kick(bp, txr, prod); - } netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), bp->tx_wake_thresh); @@ -1511,14 +1534,16 @@ static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, return 0; } -static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) +static u16 bnxt_alloc_agg_idx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, + u16 agg_id) { struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; - u16 idx = agg_id & MAX_TPA_P5_MASK; + u16 idx = agg_id & (bp->max_tpa_roundup_size - 1); if (test_bit(idx, map->agg_idx_bmap)) { - idx = find_first_zero_bit(map->agg_idx_bmap, MAX_TPA_P5); - if (idx >= MAX_TPA_P5) + idx = find_first_zero_bit(map->agg_idx_bmap, + bp->max_tpa_roundup_size); + if (idx >= bp->max_tpa_roundup_size) return INVALID_HW_RING_ID; } __set_bit(idx, map->agg_idx_bmap); @@ -1583,7 +1608,7 @@ static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { agg_id = TPA_START_AGG_ID_P5(tpa_start); - agg_id = bnxt_alloc_agg_idx(rxr, agg_id); + agg_id = bnxt_alloc_agg_idx(bp, rxr, agg_id); if (unlikely(agg_id == INVALID_HW_RING_ID)) { netdev_warn(bp->dev, "Unable to allocate agg ID for ring %d, agg 0x%x\n", rxr->bnapi->index, @@ -3581,7 +3606,7 @@ static void bnxt_free_one_tpa_info_data(struct bnxt *bp, { int i; - for (i = 0; i < bp->max_tpa; i++) { + for (i = 0; i < bp->max_tpa_roundup_size; i++) { struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; u8 *data = tpa_info->data; @@ -3778,7 +3803,7 @@ static void bnxt_free_one_tpa_info(struct bnxt *bp, kfree(rxr->rx_tpa_idx_map); rxr->rx_tpa_idx_map = NULL; if (rxr->rx_tpa) { - for (i = 0; i < bp->max_tpa; i++) { + for (i = 0; i < bp->max_tpa_roundup_size; i++) { kfree(rxr->rx_tpa[i].agg_arr); rxr->rx_tpa[i].agg_arr = NULL; } @@ -3804,13 +3829,14 @@ static int bnxt_alloc_one_tpa_info(struct bnxt *bp, struct rx_agg_cmp *agg; int i; - rxr->rx_tpa = kzalloc_objs(struct bnxt_tpa_info, bp->max_tpa); + rxr->rx_tpa = kzalloc_objs(struct bnxt_tpa_info, + bp->max_tpa_roundup_size); if (!rxr->rx_tpa) return -ENOMEM; if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) return 0; - for (i = 0; i < bp->max_tpa; i++) { + for (i = 0; i < bp->max_tpa_roundup_size; i++) { agg = kzalloc_objs(*agg, MAX_SKB_FRAGS); if (!agg) return -ENOMEM; @@ -3829,6 +3855,9 @@ static int bnxt_alloc_tpa_info(struct bnxt *bp) bp->max_tpa = MAX_TPA; if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { + /* TPA is not supported at all, so there is nothing to + * allocate. + */ if (!bp->max_tpa_v2) return 0; bp->max_tpa = min_t(u16, bp->max_tpa_v2, MAX_TPA_P5); @@ -3836,6 +3865,7 @@ static int bnxt_alloc_tpa_info(struct bnxt *bp) if (bp->max_tpa <= 32 && BNXT_CHIP_P5(bp) && !BNXT_NPAR(bp)) bp->max_tpa = MAX_TPA_P5; } + bp->max_tpa_roundup_size = roundup_pow_of_two(bp->max_tpa); for (i = 0; i < bp->rx_nr_rings; i++) { struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; @@ -4548,7 +4578,7 @@ static int bnxt_alloc_one_tpa_info_data(struct bnxt *bp, u8 *data; int i; - for (i = 0; i < bp->max_tpa; i++) { + for (i = 0; i < bp->max_tpa_roundup_size; i++) { data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, GFP_KERNEL); if (!data) @@ -5003,7 +5033,8 @@ void bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) bnxt_get_max_rings(bp, &rx, &tx, true); if (rx > 1) { bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS; - bp->dev->hw_features |= NETIF_F_LRO; + if (BNXT_SUPPORTS_TPA(bp)) + bp->dev->hw_features |= NETIF_F_LRO; } } @@ -5424,6 +5455,8 @@ static void bnxt_clear_ring_indices(struct bnxt *bp) txr->tx_prod = 0; txr->tx_cons = 0; txr->tx_hw_cons = 0; + txr->kick_pending = 0; + txr->kick_txbd0 = NULL; } rxr = bnapi->rx_ring; @@ -11337,8 +11370,13 @@ static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) { + int rc; + bnxt_init_cp_rings(bp); - bnxt_init_rx_rings(bp); + rc = bnxt_init_rx_rings(bp); + if (rc) + return rc; + bnxt_init_tx_rings(bp); bnxt_init_ring_grps(bp, irq_re_init); bnxt_init_vnics(bp); @@ -11769,6 +11807,8 @@ static int bnxt_tx_queue_start(struct bnxt *bp, int idx) txr->tx_prod = 0; txr->tx_cons = 0; txr->tx_hw_cons = 0; + txr->kick_pending = 0; + txr->kick_txbd0 = NULL; start_tx: WRITE_ONCE(txr->dev_state, 0); synchronize_net(); @@ -11920,9 +11960,11 @@ static int bnxt_request_irq(struct bnxt *bp) #endif /* Enable TPH support as part of IRQ request */ - rc = pcie_enable_tph(bp->pdev, PCI_TPH_ST_IV_MODE); - if (!rc) - bp->tph_mode = PCI_TPH_ST_IV_MODE; + if (BNXT_SUPPORTS_QUEUE_API(bp)) { + rc = pcie_enable_tph(bp->pdev, PCI_TPH_ST_IV_MODE); + if (!rc) + bp->tph_mode = PCI_TPH_ST_IV_MODE; + } for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { int map_idx = bnxt_cp_num_to_irq_num(bp, i); @@ -14558,7 +14600,14 @@ static void bnxt_rx_ring_reset(struct bnxt *bp) rxr->rx_sw_agg_prod = 0; rxr->rx_next_cons = 0; rxr->bnapi->in_reset = false; - bnxt_alloc_one_rx_ring(bp, i); + rc = bnxt_alloc_one_rx_ring(bp, i); + if (rc) { + netdev_warn(bp->dev, "RX ring reset failed to allocate buffers, rc = %d, falling back to global reset\n", + rc); + bnxt_reset_task(bp, true); + bnxt_rtnl_unlock_sp(bp); + return; + } cpr = &rxr->bnapi->cp_ring; cpr->sw_stats->rx.rx_resets++; if (bp->flags & BNXT_FLAG_AGG_RINGS) @@ -16284,6 +16333,8 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, clone->need_head_pool = false; clone->rx_page_size = qcfg->rx_page_size; clone->rx_agg_bmap = NULL; + clone->rx_tpa = NULL; + clone->rx_tpa_idx_map = NULL; rc = bnxt_alloc_rx_page_pool(bp, clone, rxr->page_pool->p.nid); if (rc) @@ -16327,11 +16378,16 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, bnxt_alloc_one_rx_ring_skb(bp, clone, idx); if (bp->flags & BNXT_FLAG_AGG_RINGS) bnxt_alloc_one_rx_ring_netmem(bp, clone, idx); - if (bp->flags & BNXT_FLAG_TPA) - bnxt_alloc_one_tpa_info_data(bp, clone); + if (bp->flags & BNXT_FLAG_TPA) { + rc = bnxt_alloc_one_tpa_info_data(bp, clone); + if (rc) + goto err_free_rx_ring_skbs; + } return 0; +err_free_rx_ring_skbs: + bnxt_free_one_rx_ring_skbs(bp, clone); err_free_tpa_info: bnxt_free_one_tpa_info(bp, clone); err_free_rx_agg_ring: diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index dc8ec5e5733e9..4dbcd08ef7c32 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -789,7 +789,6 @@ struct nqe_cn { #define MAX_TPA 64 #define MAX_TPA_P5 256 -#define MAX_TPA_P5_MASK (MAX_TPA_P5 - 1) #define MAX_TPA_SEGS_P5 0x3f #if (BNXT_PAGE_SHIFT == 16) @@ -993,6 +992,7 @@ struct bnxt_tx_ring_info { u16 txq_index; u8 tx_napi_idx; u8 kick_pending; + struct tx_bd *kick_txbd0; struct bnxt_db_info tx_db; struct tx_bd *tx_desc_ring[MAX_TX_PAGES]; @@ -2382,6 +2382,7 @@ struct bnxt { u16 max_tpa_v2; u16 max_tpa; + u16 max_tpa_roundup_size; u32 rx_buf_size; u32 rx_buf_use_size; /* useable size */ u16 rx_offset; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 62bc9cae613c3..622e89587e5db 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -5733,7 +5733,8 @@ const struct ethtool_ops bnxt_ethtool_ops = { .op_needs_rtnl = ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | ETHTOOL_OP_NEEDS_RTNL_SCOALESCE | - ETHTOOL_OP_NEEDS_RTNL_RSS, + ETHTOOL_OP_NEEDS_RTNL_RSS | + ETHTOOL_OP_NEEDS_RTNL_TEST, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | ETHTOOL_COALESCE_USECS_IRQ | diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.c index f317f60414e81..6c1060fa2ea5c 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.c @@ -31,10 +31,14 @@ static u32 bnxt_sw_gso_lhint(unsigned int len) return TX_BD_FLAGS_LHINT_2048_AND_LARGER; } -netdev_tx_t bnxt_sw_udp_gso_xmit(struct bnxt *bp, - struct bnxt_tx_ring_info *txr, - struct netdev_queue *txq, - struct sk_buff *skb) +/* Transmit an skb requiring software UDP segmentation. + * + * Returns 1 if the skb was queued and new BDs were produced, 0 if the skb + * was dropped, or -1 if the ring is full and the skb should be retried. + * The caller owns the doorbell for all three cases. + */ +int bnxt_sw_udp_gso_xmit(struct bnxt *bp, struct bnxt_tx_ring_info *txr, + struct netdev_queue *txq, struct sk_buff *skb) { unsigned int last_unmap_len __maybe_unused = 0; dma_addr_t last_unmap_addr __maybe_unused = 0; @@ -69,7 +73,7 @@ netdev_tx_t bnxt_sw_udp_gso_xmit(struct bnxt *bp, if (unlikely(bnxt_tx_avail(bp, txr) < bds_needed)) { netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), bp->tx_wake_thresh); - return NETDEV_TX_BUSY; + return -1; } /* BD backpressure alone cannot prevent overwriting in-flight @@ -77,7 +81,7 @@ netdev_tx_t bnxt_sw_udp_gso_xmit(struct bnxt *bp, */ if (!netif_txq_maybe_stop(txq, bnxt_inline_avail(txr), num_segs, num_segs)) - return NETDEV_TX_BUSY; + return -1; if (unlikely(tso_dma_map_init(&map, &pdev->dev, skb, hdr_len))) goto drop; @@ -223,18 +227,15 @@ netdev_tx_t bnxt_sw_udp_gso_xmit(struct bnxt *bp, netdev_tx_sent_queue(txq, skb->len); WRITE_ONCE(txr->tx_prod, prod); - /* Sync BDs before doorbell */ - wmb(); - bnxt_db_write(bp, &txr->tx_db, prod); if (unlikely(bnxt_tx_avail(bp, txr) <= bp->tx_wake_thresh)) netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), bp->tx_wake_thresh); - return NETDEV_TX_OK; + return 1; drop: dev_kfree_skb_any(skb); dev_core_stats_tx_dropped_inc(bp->dev); - return NETDEV_TX_OK; + return 0; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.h index 47528c20f311b..77d9af97cc22a 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.h @@ -38,9 +38,7 @@ static inline int bnxt_min_tx_desc_cnt(struct bnxt *bp, return BNXT_MIN_TX_DESC_CNT; } -netdev_tx_t bnxt_sw_udp_gso_xmit(struct bnxt *bp, - struct bnxt_tx_ring_info *txr, - struct netdev_queue *txq, - struct sk_buff *skb); +int bnxt_sw_udp_gso_xmit(struct bnxt *bp, struct bnxt_tx_ring_info *txr, + struct netdev_queue *txq, struct sk_buff *skb); #endif diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c index de3427c6c6aaf..faed0cf726e1e 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c @@ -40,7 +40,7 @@ void bnxt_hwmon_notify_event(struct bnxt *bp) return; } - hwmon_notify_event(&bp->pdev->dev, hwmon_temp, attr, 0); + hwmon_notify_event(bp->hwmon_dev, hwmon_temp, attr, 0); } static int bnxt_hwrm_temp_query(struct bnxt *bp, u8 *temp) diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index a2305e6428d1f..21668e41b6964 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -749,8 +749,17 @@ static void bcmgenet_hfb_init(struct bcmgenet_priv *priv) INIT_LIST_HEAD(&priv->rxnfc_rules[i].list); priv->rxnfc_rules[i].state = BCMGENET_RXNFC_STATE_UNUSED; } +} + +static void bcmgenet_hfb_restore(struct bcmgenet_priv *priv) +{ + struct bcmgenet_rxnfc_rule *rule; bcmgenet_hfb_clear(priv); + + list_for_each_entry(rule, &priv->rxnfc_list, list) + if (rule->state != BCMGENET_RXNFC_STATE_UNUSED) + bcmgenet_hfb_create_rxnfc_filter(priv, rule); } static int bcmgenet_begin(struct net_device *dev) @@ -843,7 +852,8 @@ static int bcmgenet_get_coalesce(struct net_device *dev, ec->rx_max_coalesced_frames = bcmgenet_rdma_ring_readl(priv, 0, DMA_MBUF_DONE_THRESH); ec->rx_coalesce_usecs = - bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT) * 8192 / 1000; + (bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT) & + DMA_TIMEOUT_MASK) * 8192 / 1000; for (i = 0; i <= priv->hw_params->rx_queues; i++) { ring = &priv->rx_rings[i]; @@ -1337,9 +1347,8 @@ static void bcmgenet_get_ethtool_stats(struct net_device *dev, p = (char *)&stats64; p += s->stat_offset; - if (sizeof(unsigned long) != sizeof(u32) && - s->stat_sizeof == sizeof(unsigned long)) - data[i] = *(unsigned long *)p; + if (s->stat_sizeof == sizeof(u64)) + data[i] = *(u64 *)p; else data[i] = *(u32 *)p; } @@ -1754,13 +1763,12 @@ static int bcmgenet_power_up(struct bcmgenet_priv *priv, int ret = 0; u32 reg; - if (!bcmgenet_has_ext(priv)) - return ret; - - reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT); - switch (mode) { case GENET_POWER_PASSIVE: + if (!bcmgenet_has_ext(priv)) + break; + + reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT); reg &= ~(EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS | EXT_ENERGY_DET_MASK); if (GENET_IS_V5(priv) && !bcmgenet_has_ephy_16nm(priv)) { @@ -1784,8 +1792,12 @@ static int bcmgenet_power_up(struct bcmgenet_priv *priv, break; case GENET_POWER_CABLE_SENSE: + if (!bcmgenet_has_ext(priv)) + break; + /* enable APD */ if (!GENET_IS_V5(priv)) { + reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT); reg |= EXT_PWR_DN_EN_LD; bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT); } @@ -3376,8 +3388,8 @@ static int bcmgenet_open(struct net_device *dev) bcmgenet_set_hw_addr(priv, dev->dev_addr); - /* HFB init */ - bcmgenet_hfb_init(priv); + /* Restore the filters, the MAC was reset above */ + bcmgenet_hfb_restore(priv); /* Reinitialize TDMA and RDMA and SW housekeeping */ ret = bcmgenet_init_dma(priv, true); @@ -3432,6 +3444,8 @@ static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy) { struct bcmgenet_priv *priv = netdev_priv(dev); + /* Stop completion polling before it can wake a stopped queue */ + bcmgenet_disable_tx_napi(priv); netif_tx_disable(dev); /* Disable MAC receive */ @@ -3446,7 +3460,6 @@ static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy) /* Disable MAC transmit. TX DMA disabled must be done before this */ umac_enable_set(priv, CMD_TX_EN, false); - bcmgenet_disable_tx_napi(priv); bcmgenet_disable_rx_napi(priv); bcmgenet_intr_disable(priv); @@ -3623,6 +3636,9 @@ static int bcmgenet_set_mac_addr(struct net_device *dev, void *p) if (netif_running(dev)) return -EBUSY; + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + eth_hw_addr_set(dev, addr->sa_data); return 0; @@ -4075,6 +4091,7 @@ static int bcmgenet_probe(struct platform_device *pdev) /* Mii wait queue */ init_waitqueue_head(&priv->wq); + bcmgenet_hfb_init(priv); INIT_WORK(&priv->bcmgenet_irq_work, bcmgenet_irq_task); priv->clk_wol = devm_clk_get_optional(&priv->pdev->dev, "enet-wol"); @@ -4125,10 +4142,10 @@ static int bcmgenet_probe(struct platform_device *pdev) priv->rx_rings[i].rx_max_coalesced_frames = 1; /* Initialize u64 stats seq counter for 32bit machines */ - for (i = 0; i <= priv->hw_params->rx_queues; i++) + for (i = 0; i <= GENET_MAX_MQ_CNT; i++) { u64_stats_init(&priv->rx_rings[i].stats64.syncp); - for (i = 0; i <= priv->hw_params->tx_queues; i++) u64_stats_init(&priv->tx_rings[i].stats64.syncp); + } /* libphy will determine the link state */ netif_carrier_off(dev); @@ -4272,10 +4289,7 @@ static int bcmgenet_resume(struct device *d) bcmgenet_set_hw_addr(priv, dev->dev_addr); /* Restore hardware filters */ - bcmgenet_hfb_clear(priv); - list_for_each_entry(rule, &priv->rxnfc_list, list) - if (rule->state != BCMGENET_RXNFC_STATE_UNUSED) - bcmgenet_hfb_create_rxnfc_filter(priv, rule); + bcmgenet_hfb_restore(priv); /* Reinitialize TDMA and RDMA and SW housekeeping */ ret = bcmgenet_init_dma(priv, false); @@ -4313,6 +4327,8 @@ static int bcmgenet_suspend(struct device *d) netif_device_detach(dev); if (device_may_wakeup(d) && priv->wolopts) { + /* Stop completion polling before it can wake a stopped queue */ + bcmgenet_disable_tx_napi(priv); netif_tx_disable(dev); /* Suspend non-wake Rx data flows */ @@ -4341,7 +4357,6 @@ static int bcmgenet_suspend(struct device *d) netdev_warn(priv->dev, "Timed out while disabling TX DMA\n"); - bcmgenet_disable_tx_napi(priv); bcmgenet_disable_rx_napi(priv); disable_irq(priv->irq1); bcmgenet_tx_reclaim_all(dev); diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 73a4b569b03e3..8b6806a79edff 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -17915,11 +17915,14 @@ static int tg3_init_one(struct pci_dev *pdev, err = tg3_get_device_address(tp, addr); if (err) { - dev_err(&pdev->dev, - "Could not obtain valid ethernet address, aborting\n"); - goto err_out_apeunmap; + dev_warn_probe(&pdev->dev, err, + "Could not obtain a valid ethernet address\n"); + if (err == -EPROBE_DEFER) + goto err_out_apeunmap; + eth_hw_addr_random(dev); + } else { + eth_hw_addr_set(dev, addr); } - eth_hw_addr_set(dev, addr); intmbx = MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW; rcvmbx = MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW; @@ -18047,6 +18050,10 @@ static int tg3_init_one(struct pci_dev *pdev, return 0; err_out_apeunmap: + if (tg3_flag(tp, USE_PHYLIB)) + tg3_phy_fini(tp); + tg3_mdio_fini(tp); + if (tp->aperegs) { iounmap(tp->aperegs); tp->aperegs = NULL; diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c index 8e19add764db2..3fa805117dde8 100644 --- a/drivers/net/ethernet/brocade/bna/bnad.c +++ b/drivers/net/ethernet/brocade/bna/bnad.c @@ -2571,6 +2571,22 @@ bnad_ioceth_disable(struct bnad *bnad) return err; } +/* + * The IOC timers rearm one another, so deleting one cannot stop a + * sibling callback from arming it again. Shut them down so a later + * mod_timer() is ignored. + */ +static void +bnad_ioc_timers_shutdown(struct bnad *bnad) +{ + struct bfa_ioc *ioc = &bnad->bna.ioceth.ioc; + + timer_shutdown_sync(&ioc->ioc_timer); + timer_shutdown_sync(&ioc->sem_timer); + timer_shutdown_sync(&ioc->hb_timer); + timer_shutdown_sync(&ioc->iocpf_timer); +} + static int bnad_ioceth_enable(struct bnad *bnad) { @@ -3727,9 +3743,7 @@ bnad_pci_probe(struct pci_dev *pdev, bnad_res_free(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX); disable_ioceth: bnad_ioceth_disable(bnad); - timer_delete_sync(&bnad->bna.ioceth.ioc.ioc_timer); - timer_delete_sync(&bnad->bna.ioceth.ioc.sem_timer); - timer_delete_sync(&bnad->bna.ioceth.ioc.hb_timer); + bnad_ioc_timers_shutdown(bnad); spin_lock_irqsave(&bnad->bna_lock, flags); bna_uninit(bna); spin_unlock_irqrestore(&bnad->bna_lock, flags); @@ -3770,9 +3784,7 @@ bnad_pci_remove(struct pci_dev *pdev) mutex_lock(&bnad->conf_mutex); bnad_ioceth_disable(bnad); - timer_delete_sync(&bnad->bna.ioceth.ioc.ioc_timer); - timer_delete_sync(&bnad->bna.ioceth.ioc.sem_timer); - timer_delete_sync(&bnad->bna.ioceth.ioc.hb_timer); + bnad_ioc_timers_shutdown(bnad); spin_lock_irqsave(&bnad->bna_lock, flags); bna_uninit(bna); spin_unlock_irqrestore(&bnad->bna_lock, flags); diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index 2de56017ee0d6..f8bff3e71b682 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -974,6 +974,8 @@ struct macb_dma_desc_ptp { * of the frame * @mapping: DMA address of the skb's fragment buffer * @size: size of the DMA mapped buffer + * @fcs_len: FCS bytes appended in software, 0 or ETH_FCS_LEN, only + * set for the last buffer of the frame * @mapped_as_page: true when buffer was mapped with skb_frag_dma_map(), * false when buffer was mapped with dma_map_single() */ @@ -981,6 +983,7 @@ struct macb_tx_skb { struct sk_buff *skb; dma_addr_t mapping; size_t size; + u8 fcs_len; bool mapped_as_page; }; @@ -1207,11 +1210,11 @@ struct macb_or_gem_ops { /* MACB-PTP interface: adapt to platform needs. */ struct macb_ptp_info { - void (*ptp_init)(struct net_device *ndev); - void (*ptp_remove)(struct net_device *ndev); + void (*ptp_init)(struct net_device *netdev); + void (*ptp_remove)(struct net_device *netdev); s32 (*get_ptp_max_adj)(void); unsigned int (*get_tsu_rate)(struct macb *bp); - int (*get_ts_info)(struct net_device *dev, + int (*get_ts_info)(struct net_device *netdev, struct kernel_ethtool_ts_info *info); int (*get_hwtst)(struct net_device *netdev, struct kernel_hwtstamp_config *tstamp_config); @@ -1326,7 +1329,7 @@ struct macb { struct clk *tx_clk; struct clk *rx_clk; struct clk *tsu_clk; - struct net_device *dev; + struct net_device *netdev; /* Protects hw_stats and ethtool_stats */ spinlock_t stats_lock; union { @@ -1406,8 +1409,8 @@ enum macb_bd_control { TSTAMP_ALL_FRAMES, }; -void gem_ptp_init(struct net_device *ndev); -void gem_ptp_remove(struct net_device *ndev); +void gem_ptp_init(struct net_device *netdev); +void gem_ptp_remove(struct net_device *netdev); void gem_ptp_txstamp(struct macb *bp, struct sk_buff *skb, struct macb_dma_desc *desc); void gem_ptp_rxstamp(struct macb *bp, struct sk_buff *skb, struct macb_dma_desc *desc); static inline void gem_ptp_do_txstamp(struct macb *bp, struct sk_buff *skb, struct macb_dma_desc *desc) @@ -1426,14 +1429,14 @@ static inline void gem_ptp_do_rxstamp(struct macb *bp, struct sk_buff *skb, stru gem_ptp_rxstamp(bp, skb, desc); } -int gem_get_hwtst(struct net_device *dev, +int gem_get_hwtst(struct net_device *netdev, struct kernel_hwtstamp_config *tstamp_config); -int gem_set_hwtst(struct net_device *dev, +int gem_set_hwtst(struct net_device *netdev, struct kernel_hwtstamp_config *tstamp_config, struct netlink_ext_ack *extack); #else -static inline void gem_ptp_init(struct net_device *ndev) { } -static inline void gem_ptp_remove(struct net_device *ndev) { } +static inline void gem_ptp_init(struct net_device *netdev) { } +static inline void gem_ptp_remove(struct net_device *netdev) { } static inline void gem_ptp_do_txstamp(struct macb *bp, struct sk_buff *skb, struct macb_dma_desc *desc) { } static inline void gem_ptp_do_rxstamp(struct macb *bp, struct sk_buff *skb, struct macb_dma_desc *desc) { } diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index d394f1f43b685..a8d830eeb8ea3 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -252,9 +252,9 @@ static void macb_set_hwaddr(struct macb *bp) u32 bottom; u16 top; - bottom = get_unaligned_le32(bp->dev->dev_addr); + bottom = get_unaligned_le32(bp->netdev->dev_addr); macb_or_gem_writel(bp, SA1B, bottom); - top = get_unaligned_le16(bp->dev->dev_addr + 4); + top = get_unaligned_le16(bp->netdev->dev_addr + 4); macb_or_gem_writel(bp, SA1T, top); if (gem_has_ptp(bp)) { @@ -291,13 +291,13 @@ static void macb_get_hwaddr(struct macb *bp) addr[5] = (top >> 8) & 0xff; if (is_valid_ether_addr(addr)) { - eth_hw_addr_set(bp->dev, addr); + eth_hw_addr_set(bp->netdev, addr); return; } } dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); - eth_hw_addr_random(bp->dev); + eth_hw_addr_random(bp->netdev); } static int macb_mdio_wait_for_idle(struct macb *bp) @@ -509,12 +509,12 @@ static void macb_set_tx_clk(struct macb *bp, int speed) ferr = abs(rate_rounded - rate); ferr = DIV_ROUND_UP(ferr, rate / 100000); if (ferr > 5) - netdev_warn(bp->dev, + netdev_warn(bp->netdev, "unable to generate target frequency: %ld Hz\n", rate); if (clk_set_rate(bp->tx_clk, rate_rounded)) - netdev_err(bp->dev, "adjusting tx_clk failed.\n"); + netdev_err(bp->netdev, "adjusting tx_clk failed.\n"); } static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int neg_mode, @@ -697,8 +697,8 @@ static void macb_tx_lpi_wake(struct macb *bp) static void macb_mac_disable_tx_lpi(struct phylink_config *config) { - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); + struct net_device *netdev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(netdev); unsigned long flags; cancel_delayed_work_sync(&bp->tx_lpi_work); @@ -712,8 +712,8 @@ static void macb_mac_disable_tx_lpi(struct phylink_config *config) static int macb_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, bool tx_clk_stop) { - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); + struct net_device *netdev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(netdev); unsigned long flags; spin_lock_irqsave(&bp->lock, flags); @@ -732,8 +732,8 @@ static int macb_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, static void macb_mac_config(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state) { - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); + struct net_device *netdev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(netdev); unsigned long flags; u32 old_ctrl, ctrl; u32 old_ncr, ncr; @@ -774,8 +774,8 @@ static void macb_mac_config(struct phylink_config *config, unsigned int mode, static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, phy_interface_t interface) { - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); + struct net_device *netdev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(netdev); struct macb_queue *queue; unsigned int q; u32 ctrl; @@ -789,7 +789,7 @@ static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE)); macb_writel(bp, NCR, ctrl); - netif_tx_stop_all_queues(ndev); + netif_tx_stop_all_queues(netdev); } /* Use juggling algorithm to left rotate tx ring and tx skb array */ @@ -884,13 +884,13 @@ static void gem_shuffle_tx_rings(struct macb *bp) } static void macb_mac_link_up(struct phylink_config *config, - struct phy_device *phy, + struct phy_device *phydev, unsigned int mode, phy_interface_t interface, int speed, int duplex, bool tx_pause, bool rx_pause) { - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); + struct net_device *netdev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(netdev); struct macb_queue *queue; unsigned long flags; unsigned int q; @@ -946,14 +946,14 @@ static void macb_mac_link_up(struct phylink_config *config, macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE)); - netif_tx_wake_all_queues(ndev); + netif_tx_wake_all_queues(netdev); } static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config, phy_interface_t interface) { - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); + struct net_device *netdev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(netdev); if (interface == PHY_INTERFACE_MODE_10GBASER) return &bp->phylink_usx_pcs; @@ -982,7 +982,7 @@ static bool macb_phy_handle_exists(struct device_node *dn) static int macb_phylink_connect(struct macb *bp) { struct device_node *dn = bp->pdev->dev.of_node; - struct net_device *dev = bp->dev; + struct net_device *netdev = bp->netdev; struct phy_device *phydev; int ret; @@ -992,7 +992,7 @@ static int macb_phylink_connect(struct macb *bp) if (!dn || (ret && !macb_phy_handle_exists(dn))) { phydev = phy_find_first(bp->mii_bus); if (!phydev) { - netdev_err(dev, "no PHY found\n"); + netdev_err(netdev, "no PHY found\n"); return -ENXIO; } @@ -1001,7 +1001,7 @@ static int macb_phylink_connect(struct macb *bp) } if (ret) { - netdev_err(dev, "Could not attach PHY (%d)\n", ret); + netdev_err(netdev, "Could not attach PHY (%d)\n", ret); return ret; } @@ -1013,21 +1013,21 @@ static int macb_phylink_connect(struct macb *bp) static void macb_get_pcs_fixed_state(struct phylink_config *config, struct phylink_link_state *state) { - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); + struct net_device *netdev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(netdev); state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0; } /* based on au1000_eth. c*/ -static int macb_mii_probe(struct net_device *dev) +static int macb_mii_probe(struct net_device *netdev) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops; bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops; - bp->phylink_config.dev = &dev->dev; + bp->phylink_config.dev = &netdev->dev; bp->phylink_config.type = PHYLINK_NETDEV; bp->phylink_config.mac_managed_pm = true; @@ -1086,7 +1086,7 @@ static int macb_mii_probe(struct net_device *dev) bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, bp->phy_interface, &macb_phylink_ops); if (IS_ERR(bp->phylink)) { - netdev_err(dev, "Could not create a phylink instance (%ld)\n", + netdev_err(netdev, "Could not create a phylink instance (%ld)\n", PTR_ERR(bp->phylink)); return PTR_ERR(bp->phylink); } @@ -1133,7 +1133,7 @@ static int macb_mii_init(struct macb *bp) */ mdio_np = of_get_child_by_name(np, "mdio"); if (!mdio_np && of_phy_is_fixed_link(np)) - return macb_mii_probe(bp->dev); + return macb_mii_probe(bp->netdev); /* Enable management port */ macb_writel(bp, NCR, MACB_BIT(MPE)); @@ -1154,16 +1154,18 @@ static int macb_mii_init(struct macb *bp) bp->mii_bus->priv = bp; bp->mii_bus->parent = &bp->pdev->dev; - dev_set_drvdata(&bp->dev->dev, bp->mii_bus); + dev_set_drvdata(&bp->netdev->dev, bp->mii_bus); err = macb_mdiobus_register(bp, mdio_np); if (err) goto err_out_free_mdiobus; - err = macb_mii_probe(bp->dev); + err = macb_mii_probe(bp->netdev); if (err) goto err_out_unregister_bus; + of_node_put(mdio_np); + return 0; err_out_unregister_bus: @@ -1268,8 +1270,8 @@ static void macb_tx_error_task(struct work_struct *work) unsigned long flags; queue_index = queue - bp->queues; - netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", - queue_index, queue->tx_tail, queue->tx_head); + netdev_vdbg(bp->netdev, "%s: q = %u, t = %u, h = %u\n", + __func__, queue_index, queue->tx_tail, queue->tx_head); /* Prevent the queue NAPI TX poll from running, as it calls * macb_tx_complete(), which in turn may call netif_wake_subqueue(). @@ -1281,14 +1283,14 @@ static void macb_tx_error_task(struct work_struct *work) spin_lock_irqsave(&bp->lock, flags); /* Make sure nobody is trying to queue up new packets */ - netif_tx_stop_all_queues(bp->dev); + netif_tx_stop_all_queues(bp->netdev); /* Stop transmission now * (in case we have just queued new packets) * macb/gem must be halted to write TBQP register */ if (macb_halt_tx(bp)) { - netdev_err(bp->dev, "BUG: halt tx timed out\n"); + netdev_err(bp->netdev, "BUG: halt tx timed out\n"); macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE))); halt_timeout = true; } @@ -1317,14 +1319,14 @@ static void macb_tx_error_task(struct work_struct *work) * since it's the only one written back by the hardware */ if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { - netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", + netdev_vdbg(bp->netdev, "txerr skb %u (data %p) TX complete\n", macb_tx_ring_wrap(bp, tail), skb->data); - bp->dev->stats.tx_packets++; + bp->netdev->stats.tx_packets++; queue->stats.tx_packets++; packets++; - bp->dev->stats.tx_bytes += skb->len; - queue->stats.tx_bytes += skb->len; + bp->netdev->stats.tx_bytes += skb->len - tx_skb->fcs_len; + queue->stats.tx_bytes += skb->len - tx_skb->fcs_len; bytes += skb->len; } } else { @@ -1333,7 +1335,7 @@ static void macb_tx_error_task(struct work_struct *work) * those. Statistics are updated by hardware. */ if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) - netdev_err(bp->dev, + netdev_err(bp->netdev, "BUG: TX buffers exhausted mid-frame\n"); desc->ctrl = ctrl | MACB_BIT(TX_USED); @@ -1342,7 +1344,7 @@ static void macb_tx_error_task(struct work_struct *work) macb_tx_unmap(bp, tx_skb, 0); } - netdev_tx_completed_queue(netdev_get_tx_queue(bp->dev, queue_index), + netdev_tx_completed_queue(netdev_get_tx_queue(bp->netdev, queue_index), packets, bytes); /* Set end of TX queue */ @@ -1367,7 +1369,7 @@ static void macb_tx_error_task(struct work_struct *work) macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE)); /* Now we are ready to start transmission again */ - netif_tx_start_all_queues(bp->dev); + netif_tx_start_all_queues(bp->netdev); macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); spin_unlock_irqrestore(&bp->lock, flags); @@ -1446,13 +1448,13 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) !ptp_one_step_sync(skb)) gem_ptp_do_txstamp(bp, skb, desc); - netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", + netdev_vdbg(bp->netdev, "skb %u (data %p) TX complete\n", macb_tx_ring_wrap(bp, tail), skb->data); - bp->dev->stats.tx_packets++; + bp->netdev->stats.tx_packets++; queue->stats.tx_packets++; - bp->dev->stats.tx_bytes += skb->len; - queue->stats.tx_bytes += skb->len; + bp->netdev->stats.tx_bytes += skb->len - tx_skb->fcs_len; + queue->stats.tx_bytes += skb->len - tx_skb->fcs_len; packets++; bytes += skb->len; } @@ -1469,14 +1471,14 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) } } - netdev_tx_completed_queue(netdev_get_tx_queue(bp->dev, queue_index), + netdev_tx_completed_queue(netdev_get_tx_queue(bp->netdev, queue_index), packets, bytes); queue->tx_tail = tail; - if (__netif_subqueue_stopped(bp->dev, queue_index) && + if (__netif_subqueue_stopped(bp->netdev, queue_index) && CIRC_CNT(queue->tx_head, queue->tx_tail, bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) - netif_wake_subqueue(bp->dev, queue_index); + netif_wake_subqueue(bp->netdev, queue_index); spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); if (packets) @@ -1504,9 +1506,9 @@ static void gem_rx_refill(struct macb_queue *queue) if (!queue->rx_skbuff[entry]) { /* allocate sk_buff for this free entry in ring */ - skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); + skb = netdev_alloc_skb(bp->netdev, bp->rx_buffer_size); if (unlikely(!skb)) { - netdev_err(bp->dev, + netdev_err(bp->netdev, "Unable to allocate sk_buff\n"); break; } @@ -1555,8 +1557,8 @@ static void gem_rx_refill(struct macb_queue *queue) /* Make descriptor updates visible to hardware */ wmb(); - netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", - queue, queue->rx_prepared_head, queue->rx_tail); + netdev_vdbg(bp->netdev, "rx ring: queue: %p, prepared head %d, tail %d\n", + queue, queue->rx_prepared_head, queue->rx_tail); } /* Mark DMA descriptors from begin up to and not including end as unused */ @@ -1616,17 +1618,17 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, count++; if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { - netdev_err(bp->dev, + netdev_err(bp->netdev, "not whole frame pointed by descriptor\n"); - bp->dev->stats.rx_dropped++; + bp->netdev->stats.rx_dropped++; queue->stats.rx_dropped++; break; } skb = queue->rx_skbuff[entry]; if (unlikely(!skb)) { - netdev_err(bp->dev, + netdev_err(bp->netdev, "inconsistent Rx descriptor chain\n"); - bp->dev->stats.rx_dropped++; + bp->netdev->stats.rx_dropped++; queue->stats.rx_dropped++; break; } @@ -1634,28 +1636,29 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, queue->rx_skbuff[entry] = NULL; len = ctrl & bp->rx_frm_len_mask; - netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); + netdev_vdbg(bp->netdev, "%s %u (len %u)\n", + __func__, entry, len); skb_put(skb, len); dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, DMA_FROM_DEVICE); - skb->protocol = eth_type_trans(skb, bp->dev); + skb->protocol = eth_type_trans(skb, bp->netdev); skb_checksum_none_assert(skb); - if (bp->dev->features & NETIF_F_RXCSUM && - !(bp->dev->flags & IFF_PROMISC) && + if (bp->netdev->features & NETIF_F_RXCSUM && + !(bp->netdev->flags & IFF_PROMISC) && GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) skb->ip_summed = CHECKSUM_UNNECESSARY; - bp->dev->stats.rx_packets++; + bp->netdev->stats.rx_packets++; queue->stats.rx_packets++; - bp->dev->stats.rx_bytes += skb->len; + bp->netdev->stats.rx_bytes += skb->len; queue->stats.rx_bytes += skb->len; gem_ptp_do_rxstamp(bp, skb, desc); #if defined(DEBUG) && defined(VERBOSE_DEBUG) - netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", + netdev_vdbg(bp->netdev, "received skb of length %u, csum: %08x\n", skb->len, skb->csum); print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, skb_mac_header(skb), 16, true); @@ -1684,9 +1687,10 @@ static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, desc = macb_rx_desc(queue, last_frag); len = desc->ctrl & bp->rx_frm_len_mask; - netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", - macb_rx_ring_wrap(bp, first_frag), - macb_rx_ring_wrap(bp, last_frag), len); + netdev_vdbg(bp->netdev, "%s frags %u - %u (len %u)\n", + __func__, + macb_rx_ring_wrap(bp, first_frag), + macb_rx_ring_wrap(bp, last_frag), len); /* The ethernet header starts NET_IP_ALIGN bytes into the * first buffer. Since the header is 14 bytes, this makes the @@ -1696,9 +1700,9 @@ static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, * the two padding bytes into the skb so that we avoid hitting * the slowpath in memcpy(), and pull them off afterwards. */ - skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); + skb = netdev_alloc_skb(bp->netdev, len + NET_IP_ALIGN); if (!skb) { - bp->dev->stats.rx_dropped++; + bp->netdev->stats.rx_dropped++; for (frag = first_frag; ; frag++) { desc = macb_rx_desc(queue, frag); desc->addr &= ~MACB_BIT(RX_USED); @@ -1742,11 +1746,11 @@ static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, wmb(); __skb_pull(skb, NET_IP_ALIGN); - skb->protocol = eth_type_trans(skb, bp->dev); + skb->protocol = eth_type_trans(skb, bp->netdev); - bp->dev->stats.rx_packets++; - bp->dev->stats.rx_bytes += skb->len; - netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", + bp->netdev->stats.rx_packets++; + bp->netdev->stats.rx_bytes += skb->len; + netdev_vdbg(bp->netdev, "received skb of length %u, csum: %08x\n", skb->len, skb->csum); napi_gro_receive(napi, skb); @@ -1826,7 +1830,7 @@ static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, unsigned long flags; u32 ctrl; - netdev_err(bp->dev, "RX queue corruption: reset it\n"); + netdev_err(bp->netdev, "RX queue corruption: reset it\n"); spin_lock_irqsave(&bp->lock, flags); @@ -1873,7 +1877,7 @@ static int macb_rx_poll(struct napi_struct *napi, int budget) work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); - netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n", + netdev_vdbg(bp->netdev, "RX poll: queue = %u, work_done = %d, budget = %d\n", (unsigned int)(queue - bp->queues), work_done, budget); if (work_done < budget && napi_complete_done(napi, work_done)) { @@ -1892,7 +1896,7 @@ static int macb_rx_poll(struct napi_struct *napi, int budget) if (macb_rx_pending(queue)) { queue_writel(queue, IDR, bp->rx_intr_mask); macb_queue_isr_clear(bp, queue, MACB_BIT(RCOMP)); - netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n"); + netdev_vdbg(bp->netdev, "poll: packets pending, reschedule\n"); napi_schedule(napi); } } @@ -1956,11 +1960,11 @@ static int macb_tx_poll(struct napi_struct *napi, int budget) rmb(); // ensure txubr_pending is up to date if (queue->txubr_pending) { queue->txubr_pending = false; - netdev_vdbg(bp->dev, "poll: tx restart\n"); + netdev_vdbg(bp->netdev, "poll: tx restart\n"); macb_tx_restart(queue); } - netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n", + netdev_vdbg(bp->netdev, "TX poll: queue = %u, work_done = %d, budget = %d\n", (unsigned int)(queue - bp->queues), work_done, budget); if (work_done < budget && napi_complete_done(napi, work_done)) { @@ -1979,7 +1983,7 @@ static int macb_tx_poll(struct napi_struct *napi, int budget) if (macb_tx_complete_pending(queue)) { queue_writel(queue, IDR, MACB_BIT(TCOMP)); macb_queue_isr_clear(bp, queue, MACB_BIT(TCOMP)); - netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n"); + netdev_vdbg(bp->netdev, "TX poll: packets pending, reschedule\n"); napi_schedule(napi); } } @@ -1990,7 +1994,7 @@ static int macb_tx_poll(struct napi_struct *napi, int budget) static void macb_hresp_error_task(struct work_struct *work) { struct macb *bp = from_work(bp, work, hresp_err_bh_work); - struct net_device *dev = bp->dev; + struct net_device *netdev = bp->netdev; struct macb_queue *queue; unsigned int q; u32 ctrl; @@ -2004,8 +2008,8 @@ static void macb_hresp_error_task(struct work_struct *work) ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); macb_writel(bp, NCR, ctrl); - netif_tx_stop_all_queues(dev); - netif_carrier_off(dev); + netif_tx_stop_all_queues(netdev); + netif_carrier_off(netdev); bp->macbgem_ops.mog_init_rings(bp); @@ -2022,8 +2026,8 @@ static void macb_hresp_error_task(struct work_struct *work) ctrl |= MACB_BIT(RE) | MACB_BIT(TE); macb_writel(bp, NCR, ctrl); - netif_carrier_on(dev); - netif_tx_start_all_queues(dev); + netif_carrier_on(netdev); + netif_tx_start_all_queues(netdev); } static void macb_wol_interrupt(struct macb_queue *queue, u32 status) @@ -2032,7 +2036,7 @@ static void macb_wol_interrupt(struct macb_queue *queue, u32 status) queue_writel(queue, IDR, MACB_BIT(WOL)); macb_writel(bp, WOL, 0); - netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", + netdev_vdbg(bp->netdev, "MACB WoL: queue = %u, isr = 0x%08lx\n", (unsigned int)(queue - bp->queues), (unsigned long)status); macb_queue_isr_clear(bp, queue, MACB_BIT(WOL)); @@ -2045,7 +2049,7 @@ static void gem_wol_interrupt(struct macb_queue *queue, u32 status) queue_writel(queue, IDR, GEM_BIT(WOL)); gem_writel(bp, WOL, 0); - netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", + netdev_vdbg(bp->netdev, "GEM WoL: queue = %u, isr = 0x%08lx\n", (unsigned int)(queue - bp->queues), (unsigned long)status); macb_queue_isr_clear(bp, queue, GEM_BIT(WOL)); @@ -2055,10 +2059,10 @@ static void gem_wol_interrupt(struct macb_queue *queue, u32 status) static int macb_interrupt_misc(struct macb_queue *queue, u32 status) { struct macb *bp = queue->bp; - struct net_device *dev; + struct net_device *netdev; u32 ctrl; - dev = bp->dev; + netdev = bp->netdev; if (unlikely(status & (MACB_TX_ERR_FLAGS))) { queue_writel(queue, IDR, MACB_TX_INT_FLAGS); @@ -2099,7 +2103,7 @@ static int macb_interrupt_misc(struct macb_queue *queue, u32 status) if (status & MACB_BIT(HRESP)) { queue_work(system_bh_wq, &bp->hresp_err_bh_work); - netdev_err(dev, "DMA bus error: HRESP not OK\n"); + netdev_err(netdev, "DMA bus error: HRESP not OK\n"); macb_queue_isr_clear(bp, queue, MACB_BIT(HRESP)); } @@ -2118,7 +2122,7 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) { struct macb_queue *queue = dev_id; struct macb *bp = queue->bp; - struct net_device *dev = bp->dev; + struct net_device *netdev = bp->netdev; u32 status; status = queue_readl(queue, ISR); @@ -2130,13 +2134,13 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) while (status) { /* close possible race with dev_close */ - if (unlikely(!netif_running(dev))) { + if (unlikely(!netif_running(netdev))) { queue_writel(queue, IDR, -1); macb_queue_isr_clear(bp, queue, -1); break; } - netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", + netdev_vdbg(netdev, "queue = %u, isr = 0x%08lx\n", (unsigned int)(queue - bp->queues), (unsigned long)status); @@ -2181,16 +2185,16 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) /* Polling receive - used by netconsole and other diagnostic tools * to allow network i/o with interrupts disabled. */ -static void macb_poll_controller(struct net_device *dev) +static void macb_poll_controller(struct net_device *netdev) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct macb_queue *queue; unsigned long flags; unsigned int q; local_irq_save(flags); for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) - macb_interrupt(dev->irq, queue); + macb_interrupt(netdev->irq, queue); local_irq_restore(flags); } #endif @@ -2198,7 +2202,8 @@ static void macb_poll_controller(struct net_device *dev) static unsigned int macb_tx_map(struct macb *bp, struct macb_queue *queue, struct sk_buff *skb, - unsigned int hdrlen) + unsigned int hdrlen, + u8 fcs_len) { unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; unsigned int len, i, tx_head = queue->tx_head; @@ -2277,12 +2282,13 @@ static unsigned int macb_tx_map(struct macb *bp, /* Should never happen */ if (unlikely(!tx_skb)) { - netdev_err(bp->dev, "BUG! empty skb!\n"); + netdev_err(bp->netdev, "BUG! empty skb!\n"); return 0; } /* This is the last buffer of the frame: save socket buffer */ tx_skb->skb = skb; + tx_skb->fcs_len = fcs_len; /* Update TX ring: update buffer descriptors in reverse order * to avoid race condition @@ -2328,7 +2334,7 @@ static unsigned int macb_tx_map(struct macb *bp, if (i == queue->tx_head) { ctrl |= MACB_BF(TX_LSO, lso_ctrl); ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); - if ((bp->dev->features & NETIF_F_HW_CSUM) && + if ((bp->netdev->features & NETIF_F_HW_CSUM) && skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl && !ptp_one_step_sync(skb)) ctrl |= MACB_BIT(TX_NOCRC); @@ -2352,7 +2358,7 @@ static unsigned int macb_tx_map(struct macb *bp, return 0; dma_error: - netdev_err(bp->dev, "TX DMA map failed\n"); + netdev_err(bp->netdev, "TX DMA map failed\n"); for (i = queue->tx_head; i != tx_head; i++) { tx_skb = macb_tx_skb(queue, i); @@ -2364,7 +2370,7 @@ static unsigned int macb_tx_map(struct macb *bp, } static netdev_features_t macb_features_check(struct sk_buff *skb, - struct net_device *dev, + struct net_device *netdev, netdev_features_t features) { unsigned int nr_frags, f; @@ -2416,7 +2422,8 @@ static inline int macb_clear_csum(struct sk_buff *skb) return 0; } -static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) +/* Returns a negative errno, or the FCS bytes appended (0 or ETH_FCS_LEN). */ +static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *netdev) { bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) || skb_is_nonlinear(*skb); @@ -2425,7 +2432,7 @@ static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) struct sk_buff *nskb; u32 fcs; - if (!(ndev->features & NETIF_F_HW_CSUM) || + if (!(netdev->features & NETIF_F_HW_CSUM) || !((*skb)->ip_summed != CHECKSUM_PARTIAL) || skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb)) return 0; @@ -2464,17 +2471,19 @@ static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) skb_put_u8(*skb, (fcs >> 16) & 0xff); skb_put_u8(*skb, (fcs >> 24) & 0xff); - return 0; + return ETH_FCS_LEN; } -static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) +static netdev_tx_t macb_start_xmit(struct sk_buff *skb, + struct net_device *netdev) { u16 queue_index = skb_get_queue_mapping(skb); - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct macb_queue *queue = &bp->queues[queue_index]; unsigned int desc_cnt, nr_frags, frag_size, f; unsigned int hdrlen; unsigned long flags; + int fcs_len; bool is_lso; netdev_tx_t ret = NETDEV_TX_OK; @@ -2483,7 +2492,8 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) return ret; } - if (macb_pad_and_fcs(&skb, dev)) { + fcs_len = macb_pad_and_fcs(&skb, netdev); + if (fcs_len < 0) { dev_kfree_skb_any(skb); return ret; } @@ -2502,7 +2512,7 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) else hdrlen = skb_tcp_all_headers(skb); if (skb_headlen(skb) < hdrlen) { - netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); + netdev_err(bp->netdev, "Error - LSO headers fragmented!!!\n"); /* if this is required, would need to copy to single buffer */ return NETDEV_TX_BUSY; } @@ -2510,7 +2520,7 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) hdrlen = umin(skb_headlen(skb), bp->max_tx_length); #if defined(DEBUG) && defined(VERBOSE_DEBUG) - netdev_vdbg(bp->dev, + netdev_vdbg(bp->netdev, "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", queue_index, skb->len, skb->head, skb->data, skb_tail_pointer(skb), skb_end_pointer(skb)); @@ -2538,15 +2548,15 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) /* This is a hard error, log it. */ if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < desc_cnt) { - netif_stop_subqueue(dev, queue_index); - netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", + netif_stop_subqueue(netdev, queue_index); + netdev_dbg(netdev, "tx_head = %u, tx_tail = %u\n", queue->tx_head, queue->tx_tail); ret = NETDEV_TX_BUSY; goto unlock; } /* Map socket buffer for DMA transfer */ - if (macb_tx_map(bp, queue, skb, hdrlen)) { + if (macb_tx_map(bp, queue, skb, hdrlen, fcs_len)) { dev_kfree_skb_any(skb); goto unlock; } @@ -2554,7 +2564,7 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) /* Make newly initialized descriptor visible to hardware */ wmb(); skb_tx_timestamp(skb); - netdev_tx_sent_queue(netdev_get_tx_queue(bp->dev, queue_index), + netdev_tx_sent_queue(netdev_get_tx_queue(bp->netdev, queue_index), skb->len); spin_lock(&bp->lock); @@ -2563,7 +2573,7 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) spin_unlock(&bp->lock); if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) - netif_stop_subqueue(dev, queue_index); + netif_stop_subqueue(netdev, queue_index); unlock: spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); @@ -2579,7 +2589,7 @@ static void macb_init_rx_buffer_size(struct macb *bp, size_t size) bp->rx_buffer_size = MIN(size, RX_BUFFER_MAX); if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { - netdev_dbg(bp->dev, + netdev_dbg(bp->netdev, "RX buffer must be multiple of %d bytes, expanding\n", RX_BUFFER_MULTIPLE); bp->rx_buffer_size = @@ -2587,8 +2597,8 @@ static void macb_init_rx_buffer_size(struct macb *bp, size_t size) } } - netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", - bp->dev->mtu, bp->rx_buffer_size); + netdev_dbg(bp->netdev, "mtu [%u] rx_buffer_size [%zu]\n", + bp->netdev->mtu, bp->rx_buffer_size); } static void gem_free_rx_buffers(struct macb *bp) @@ -2679,7 +2689,7 @@ static void macb_free_consistent(struct macb *bp) } queue->stats.tx_dropped += dropped; - bp->dev->stats.tx_dropped += dropped; + bp->netdev->stats.tx_dropped += dropped; kfree(queue->tx_skb); queue->tx_skb = NULL; @@ -2704,7 +2714,7 @@ static int gem_alloc_rx_buffers(struct macb *bp) if (!queue->rx_skbuff) return -ENOMEM; else - netdev_dbg(bp->dev, + netdev_dbg(bp->netdev, "Allocated %d RX struct sk_buff entries at %p\n", bp->rx_ring_size, queue->rx_skbuff); } @@ -2722,7 +2732,7 @@ static int macb_alloc_rx_buffers(struct macb *bp) if (!queue->rx_buffers) return -ENOMEM; - netdev_dbg(bp->dev, + netdev_dbg(bp->netdev, "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); return 0; @@ -2746,16 +2756,26 @@ static int macb_alloc_consistent(struct macb *bp) size = bp->num_queues * macb_tx_ring_size_per_queue(bp); tx = dma_alloc_coherent(dev, size, &tx_dma, GFP_KERNEL); - if (!tx || upper_32_bits(tx_dma) != upper_32_bits(tx_dma + size - 1)) + if (!tx) + goto out_err; + /* Record the buffer so that the error path frees it. */ + bp->queues[0].tx_ring = tx; + bp->queues[0].tx_ring_dma = tx_dma; + if (upper_32_bits(tx_dma) != upper_32_bits(tx_dma + size - 1)) goto out_err; - netdev_dbg(bp->dev, "Allocated %zu bytes for %u TX rings at %08lx (mapped %p)\n", + netdev_dbg(bp->netdev, "Allocated %zu bytes for %u TX rings at %08lx (mapped %p)\n", size, bp->num_queues, (unsigned long)tx_dma, tx); size = bp->num_queues * macb_rx_ring_size_per_queue(bp); rx = dma_alloc_coherent(dev, size, &rx_dma, GFP_KERNEL); - if (!rx || upper_32_bits(rx_dma) != upper_32_bits(rx_dma + size - 1)) + if (!rx) goto out_err; - netdev_dbg(bp->dev, "Allocated %zu bytes for %u RX rings at %08lx (mapped %p)\n", + /* Record the buffer so that the error path frees it. */ + bp->queues[0].rx_ring = rx; + bp->queues[0].rx_ring_dma = rx_dma; + if (upper_32_bits(rx_dma) != upper_32_bits(rx_dma + size - 1)) + goto out_err; + netdev_dbg(bp->netdev, "Allocated %zu bytes for %u RX rings at %08lx (mapped %p)\n", size, bp->num_queues, (unsigned long)rx_dma, rx); for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { @@ -2983,7 +3003,7 @@ static void macb_configure_dma(struct macb *bp) else dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ - if (bp->dev->features & NETIF_F_HW_CSUM) + if (bp->netdev->features & NETIF_F_HW_CSUM) dmacfg |= GEM_BIT(TXCOEN); else dmacfg &= ~GEM_BIT(TXCOEN); @@ -2993,7 +3013,7 @@ static void macb_configure_dma(struct macb *bp) dmacfg |= GEM_BIT(ADDR64); if (macb_dma_ptp(bp)) dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); - netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", + netdev_dbg(bp->netdev, "Cadence configure DMA with 0x%08x\n", dmacfg); gem_writel(bp, DMACFG, dmacfg); } @@ -3017,11 +3037,11 @@ static void macb_init_hw(struct macb *bp) config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ else config |= MACB_BIT(BIG); /* Receive oversized frames */ - if (bp->dev->flags & IFF_PROMISC) + if (bp->netdev->flags & IFF_PROMISC) config |= MACB_BIT(CAF); /* Copy All Frames */ - else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) + else if (macb_is_gem(bp) && bp->netdev->features & NETIF_F_RXCSUM) config |= GEM_BIT(RXCOEN); - if (!(bp->dev->flags & IFF_BROADCAST)) + if (!(bp->netdev->flags & IFF_BROADCAST)) config |= MACB_BIT(NBC); /* No BroadCast */ config |= macb_dbw(bp); macb_writel(bp, NCFGR, config); @@ -3095,17 +3115,17 @@ static int hash_get_index(__u8 *addr) } /* Add multicast addresses to the internal multicast-hash table. */ -static void macb_sethashtable(struct net_device *dev) +static void macb_sethashtable(struct net_device *netdev) { struct netdev_hw_addr *ha; unsigned long mc_filter[2]; unsigned int bitnr; - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); mc_filter[0] = 0; mc_filter[1] = 0; - netdev_for_each_mc_addr(ha, dev) { + netdev_for_each_mc_addr(ha, netdev) { bitnr = hash_get_index(ha->addr); mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); } @@ -3115,14 +3135,14 @@ static void macb_sethashtable(struct net_device *dev) } /* Enable/Disable promiscuous and multicast modes. */ -static void macb_set_rx_mode(struct net_device *dev) +static void macb_set_rx_mode(struct net_device *netdev) { unsigned long cfg; - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); cfg = macb_readl(bp, NCFGR); - if (dev->flags & IFF_PROMISC) { + if (netdev->flags & IFF_PROMISC) { /* Enable promiscuous mode */ cfg |= MACB_BIT(CAF); @@ -3134,20 +3154,20 @@ static void macb_set_rx_mode(struct net_device *dev) cfg &= ~MACB_BIT(CAF); /* Enable RX checksum offload only if requested */ - if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) + if (macb_is_gem(bp) && netdev->features & NETIF_F_RXCSUM) cfg |= GEM_BIT(RXCOEN); } - if (dev->flags & IFF_ALLMULTI) { + if (netdev->flags & IFF_ALLMULTI) { /* Enable all multicast mode */ macb_or_gem_writel(bp, HRB, -1); macb_or_gem_writel(bp, HRT, -1); cfg |= MACB_BIT(NCFGR_MTI); - } else if (!netdev_mc_empty(dev)) { + } else if (!netdev_mc_empty(netdev)) { /* Enable specific multicasts */ - macb_sethashtable(dev); + macb_sethashtable(netdev); cfg |= MACB_BIT(NCFGR_MTI); - } else if (dev->flags & (~IFF_ALLMULTI)) { + } else if (netdev->flags & (~IFF_ALLMULTI)) { /* Disable all multicast mode */ macb_or_gem_writel(bp, HRB, 0); macb_or_gem_writel(bp, HRT, 0); @@ -3157,15 +3177,15 @@ static void macb_set_rx_mode(struct net_device *dev) macb_writel(bp, NCFGR, cfg); } -static int macb_open(struct net_device *dev) +static int macb_open(struct net_device *netdev) { - size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; - struct macb *bp = netdev_priv(dev); + size_t bufsz = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; + struct macb *bp = netdev_priv(netdev); struct macb_queue *queue; unsigned int q; int err; - netdev_dbg(bp->dev, "open\n"); + netdev_dbg(bp->netdev, "open\n"); err = pm_runtime_resume_and_get(&bp->pdev->dev); if (err < 0) @@ -3176,7 +3196,7 @@ static int macb_open(struct net_device *dev) err = macb_alloc_consistent(bp); if (err) { - netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", + netdev_err(netdev, "Unable to allocate DMA memory (error %d)\n", err); goto pm_exit; } @@ -3203,10 +3223,10 @@ static int macb_open(struct net_device *dev) if (err) goto phy_off; - netif_tx_start_all_queues(dev); + netif_tx_start_all_queues(netdev); if (bp->ptp_info) - bp->ptp_info->ptp_init(dev); + bp->ptp_info->ptp_init(netdev); return 0; @@ -3225,19 +3245,19 @@ static int macb_open(struct net_device *dev) return err; } -static int macb_close(struct net_device *dev) +static int macb_close(struct net_device *netdev) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct macb_queue *queue; unsigned long flags; unsigned int q; - netif_tx_stop_all_queues(dev); + netif_tx_stop_all_queues(netdev); for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { napi_disable(&queue->napi_rx); napi_disable(&queue->napi_tx); - netdev_tx_reset_queue(netdev_get_tx_queue(dev, q)); + netdev_tx_reset_queue(netdev_get_tx_queue(netdev, q)); } cancel_delayed_work_sync(&bp->tx_lpi_work); @@ -3249,38 +3269,38 @@ static int macb_close(struct net_device *dev) spin_lock_irqsave(&bp->lock, flags); macb_reset_hw(bp); - netif_carrier_off(dev); + netif_carrier_off(netdev); spin_unlock_irqrestore(&bp->lock, flags); macb_free_consistent(bp); if (bp->ptp_info) - bp->ptp_info->ptp_remove(dev); + bp->ptp_info->ptp_remove(netdev); pm_runtime_put(&bp->pdev->dev); return 0; } -static int macb_change_mtu(struct net_device *dev, int new_mtu) +static int macb_change_mtu(struct net_device *netdev, int new_mtu) { - if (netif_running(dev)) + if (netif_running(netdev)) return -EBUSY; - WRITE_ONCE(dev->mtu, new_mtu); + WRITE_ONCE(netdev->mtu, new_mtu); return 0; } -static int macb_set_mac_addr(struct net_device *dev, void *addr) +static int macb_set_mac_addr(struct net_device *netdev, void *addr) { int err; - err = eth_mac_addr(dev, addr); + err = eth_mac_addr(netdev, addr); if (err < 0) return err; - macb_set_hwaddr(netdev_priv(dev)); + macb_set_hwaddr(netdev_priv(netdev)); return 0; } @@ -3318,7 +3338,7 @@ static void gem_get_stats(struct macb *bp, struct rtnl_link_stats64 *nstat) struct gem_stats *hwstat = &bp->hw_stats.gem; spin_lock_irq(&bp->stats_lock); - if (netif_running(bp->dev)) + if (netif_running(bp->netdev)) gem_update_stats(bp); nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + @@ -3351,10 +3371,10 @@ static void gem_get_stats(struct macb *bp, struct rtnl_link_stats64 *nstat) spin_unlock_irq(&bp->stats_lock); } -static void gem_get_ethtool_stats(struct net_device *dev, +static void gem_get_ethtool_stats(struct net_device *netdev, struct ethtool_stats *stats, u64 *data) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); spin_lock_irq(&bp->stats_lock); gem_update_stats(bp); @@ -3363,9 +3383,9 @@ static void gem_get_ethtool_stats(struct net_device *dev, spin_unlock_irq(&bp->stats_lock); } -static int gem_get_sset_count(struct net_device *dev, int sset) +static int gem_get_sset_count(struct net_device *netdev, int sset) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); switch (sset) { case ETH_SS_STATS: @@ -3375,9 +3395,9 @@ static int gem_get_sset_count(struct net_device *dev, int sset) } } -static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) +static void gem_get_ethtool_strings(struct net_device *netdev, u32 sset, u8 *p) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct macb_queue *queue; unsigned int i; unsigned int q; @@ -3396,13 +3416,13 @@ static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) } } -static void macb_get_stats(struct net_device *dev, +static void macb_get_stats(struct net_device *netdev, struct rtnl_link_stats64 *nstat) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct macb_stats *hwstat = &bp->hw_stats.macb; - netdev_stats_to_stats64(nstat, &bp->dev->stats); + netdev_stats_to_stats64(nstat, &bp->netdev->stats); if (macb_is_gem(bp)) { gem_get_stats(bp, nstat); return; @@ -3446,10 +3466,10 @@ static void macb_get_stats(struct net_device *dev, spin_unlock_irq(&bp->stats_lock); } -static void macb_get_pause_stats(struct net_device *dev, +static void macb_get_pause_stats(struct net_device *netdev, struct ethtool_pause_stats *pause_stats) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct macb_stats *hwstat = &bp->hw_stats.macb; spin_lock_irq(&bp->stats_lock); @@ -3459,10 +3479,10 @@ static void macb_get_pause_stats(struct net_device *dev, spin_unlock_irq(&bp->stats_lock); } -static void gem_get_pause_stats(struct net_device *dev, +static void gem_get_pause_stats(struct net_device *netdev, struct ethtool_pause_stats *pause_stats) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct gem_stats *hwstat = &bp->hw_stats.gem; spin_lock_irq(&bp->stats_lock); @@ -3472,10 +3492,10 @@ static void gem_get_pause_stats(struct net_device *dev, spin_unlock_irq(&bp->stats_lock); } -static void macb_get_eth_mac_stats(struct net_device *dev, +static void macb_get_eth_mac_stats(struct net_device *netdev, struct ethtool_eth_mac_stats *mac_stats) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct macb_stats *hwstat = &bp->hw_stats.macb; spin_lock_irq(&bp->stats_lock); @@ -3497,10 +3517,10 @@ static void macb_get_eth_mac_stats(struct net_device *dev, spin_unlock_irq(&bp->stats_lock); } -static void gem_get_eth_mac_stats(struct net_device *dev, +static void gem_get_eth_mac_stats(struct net_device *netdev, struct ethtool_eth_mac_stats *mac_stats) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct gem_stats *hwstat = &bp->hw_stats.gem; spin_lock_irq(&bp->stats_lock); @@ -3530,10 +3550,10 @@ static void gem_get_eth_mac_stats(struct net_device *dev, } /* TODO: Report SQE test errors when added to phy_stats */ -static void macb_get_eth_phy_stats(struct net_device *dev, +static void macb_get_eth_phy_stats(struct net_device *netdev, struct ethtool_eth_phy_stats *phy_stats) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct macb_stats *hwstat = &bp->hw_stats.macb; spin_lock_irq(&bp->stats_lock); @@ -3542,10 +3562,10 @@ static void macb_get_eth_phy_stats(struct net_device *dev, spin_unlock_irq(&bp->stats_lock); } -static void gem_get_eth_phy_stats(struct net_device *dev, +static void gem_get_eth_phy_stats(struct net_device *netdev, struct ethtool_eth_phy_stats *phy_stats) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct gem_stats *hwstat = &bp->hw_stats.gem; spin_lock_irq(&bp->stats_lock); @@ -3554,11 +3574,11 @@ static void gem_get_eth_phy_stats(struct net_device *dev, spin_unlock_irq(&bp->stats_lock); } -static void macb_get_rmon_stats(struct net_device *dev, +static void macb_get_rmon_stats(struct net_device *netdev, struct ethtool_rmon_stats *rmon_stats, const struct ethtool_rmon_hist_range **ranges) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct macb_stats *hwstat = &bp->hw_stats.macb; spin_lock_irq(&bp->stats_lock); @@ -3580,11 +3600,11 @@ static const struct ethtool_rmon_hist_range gem_rmon_ranges[] = { { }, }; -static void gem_get_rmon_stats(struct net_device *dev, +static void gem_get_rmon_stats(struct net_device *netdev, struct ethtool_rmon_stats *rmon_stats, const struct ethtool_rmon_hist_range **ranges) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct gem_stats *hwstat = &bp->hw_stats.gem; spin_lock_irq(&bp->stats_lock); @@ -3615,10 +3635,10 @@ static int macb_get_regs_len(struct net_device *netdev) return MACB_GREGS_NBR * sizeof(u32); } -static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, +static void macb_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); unsigned int tail, head; u32 *regs_buff = p; @@ -3735,16 +3755,16 @@ static int macb_set_ringparam(struct net_device *netdev, return 0; } - if (netif_running(bp->dev)) { + if (netif_running(bp->netdev)) { reset = 1; - macb_close(bp->dev); + macb_close(bp->netdev); } bp->rx_ring_size = new_rx_size; bp->tx_ring_size = new_tx_size; if (reset) - macb_open(bp->dev); + macb_open(bp->netdev); return 0; } @@ -3771,13 +3791,13 @@ static s32 gem_get_ptp_max_adj(void) return 64000000; } -static int gem_get_ts_info(struct net_device *dev, +static int gem_get_ts_info(struct net_device *netdev, struct kernel_ethtool_ts_info *info) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); if (!macb_dma_ptp(bp)) { - ethtool_op_get_ts_info(dev, info); + ethtool_op_get_ts_info(netdev, info); return 0; } @@ -3824,7 +3844,7 @@ static int macb_get_ts_info(struct net_device *netdev, static void gem_enable_flow_filters(struct macb *bp, bool enable) { - struct net_device *netdev = bp->dev; + struct net_device *netdev = bp->netdev; struct ethtool_rx_fs_item *item; u32 t2_scr; int num_t2_scr; @@ -4154,16 +4174,16 @@ static const struct ethtool_ops macb_ethtool_ops = { .set_ringparam = macb_set_ringparam, }; -static int macb_get_eee(struct net_device *dev, struct ethtool_keee *eee) +static int macb_get_eee(struct net_device *netdev, struct ethtool_keee *eee) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); return phylink_ethtool_get_eee(bp->phylink, eee); } -static int macb_set_eee(struct net_device *dev, struct ethtool_keee *eee) +static int macb_set_eee(struct net_device *netdev, struct ethtool_keee *eee) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); return phylink_ethtool_set_eee(bp->phylink, eee); } @@ -4194,43 +4214,43 @@ static const struct ethtool_ops gem_ethtool_ops = { .set_eee = macb_set_eee, }; -static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) +static int macb_ioctl(struct net_device *netdev, struct ifreq *rq, int cmd) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); - if (!netif_running(dev)) + if (!netif_running(netdev)) return -EINVAL; return phylink_mii_ioctl(bp->phylink, rq, cmd); } -static int macb_hwtstamp_get(struct net_device *dev, +static int macb_hwtstamp_get(struct net_device *netdev, struct kernel_hwtstamp_config *cfg) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); - if (!netif_running(dev)) + if (!netif_running(netdev)) return -EINVAL; if (!bp->ptp_info) return -EOPNOTSUPP; - return bp->ptp_info->get_hwtst(dev, cfg); + return bp->ptp_info->get_hwtst(netdev, cfg); } -static int macb_hwtstamp_set(struct net_device *dev, +static int macb_hwtstamp_set(struct net_device *netdev, struct kernel_hwtstamp_config *cfg, struct netlink_ext_ack *extack) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); - if (!netif_running(dev)) + if (!netif_running(netdev)) return -EINVAL; if (!bp->ptp_info) return -EOPNOTSUPP; - return bp->ptp_info->set_hwtst(dev, cfg, extack); + return bp->ptp_info->set_hwtst(netdev, cfg, extack); } static inline void macb_set_txcsum_feature(struct macb *bp, @@ -4253,7 +4273,7 @@ static inline void macb_set_txcsum_feature(struct macb *bp, static inline void macb_set_rxcsum_feature(struct macb *bp, netdev_features_t features) { - struct net_device *netdev = bp->dev; + struct net_device *netdev = bp->netdev; u32 val; if (!macb_is_gem(bp)) @@ -4300,7 +4320,7 @@ static int macb_set_features(struct net_device *netdev, static void macb_restore_features(struct macb *bp) { - struct net_device *netdev = bp->dev; + struct net_device *netdev = bp->netdev; netdev_features_t features = netdev->features; struct ethtool_rx_fs_item *item; @@ -4317,15 +4337,15 @@ static void macb_restore_features(struct macb *bp) macb_set_rxflow_feature(bp, features); } -static int macb_taprio_setup_replace(struct net_device *ndev, +static int macb_taprio_setup_replace(struct net_device *netdev, struct tc_taprio_qopt_offload *conf) { u64 total_on_time = 0, start_time_sec = 0, start_time = conf->base_time; u32 configured_queues = 0, speed = 0, start_time_nsec; struct macb_queue_enst_config *enst_queue; + struct ethtool_link_ksettings kset = {}; + struct macb *bp = netdev_priv(netdev); struct tc_taprio_sched_entry *entry; - struct macb *bp = netdev_priv(ndev); - struct ethtool_link_ksettings kset; struct macb_queue *queue; u32 queue_mask; u8 queue_id; @@ -4333,13 +4353,13 @@ static int macb_taprio_setup_replace(struct net_device *ndev, int err; if (conf->num_entries > bp->num_queues) { - netdev_err(ndev, "Too many TAPRIO entries: %zu > %d queues\n", + netdev_err(netdev, "Too many TAPRIO entries: %zu > %d queues\n", conf->num_entries, bp->num_queues); return -EINVAL; } if (conf->base_time < 0) { - netdev_err(ndev, "Invalid base_time: must be 0 or positive, got %lld\n", + netdev_err(netdev, "Invalid base_time: must be 0 or positive, got %lld\n", conf->base_time); return -ERANGE; } @@ -4347,13 +4367,13 @@ static int macb_taprio_setup_replace(struct net_device *ndev, /* Get the current link speed */ err = phylink_ethtool_ksettings_get(bp->phylink, &kset); if (unlikely(err)) { - netdev_err(ndev, "Failed to get link settings: %d\n", err); + netdev_err(netdev, "Failed to get link settings: %d\n", err); return err; } speed = kset.base.speed; - if (unlikely(speed <= 0)) { - netdev_err(ndev, "Invalid speed: %d\n", speed); + if (unlikely(speed == SPEED_UNKNOWN || !speed)) { + netdev_err(netdev, "Invalid speed %d, link-down?\n", speed); return -EINVAL; } @@ -4366,7 +4386,7 @@ static int macb_taprio_setup_replace(struct net_device *ndev, entry = &conf->entries[i]; if (entry->command != TC_TAPRIO_CMD_SET_GATES) { - netdev_err(ndev, "Entry %zu: unsupported command %d\n", + netdev_err(netdev, "Entry %zu: unsupported command %d\n", i, entry->command); err = -EOPNOTSUPP; goto cleanup; @@ -4374,7 +4394,7 @@ static int macb_taprio_setup_replace(struct net_device *ndev, /* Validate gate_mask: must be nonzero, single queue, and within range */ if (!is_power_of_2(entry->gate_mask)) { - netdev_err(ndev, "Entry %zu: gate_mask 0x%x is not a power of 2 (only one queue per entry allowed)\n", + netdev_err(netdev, "Entry %zu: gate_mask 0x%x is not a power of 2 (only one queue per entry allowed)\n", i, entry->gate_mask); err = -EINVAL; goto cleanup; @@ -4383,7 +4403,7 @@ static int macb_taprio_setup_replace(struct net_device *ndev, /* gate_mask must not select queues outside the valid queues */ queue_id = order_base_2(entry->gate_mask); if (queue_id >= bp->num_queues) { - netdev_err(ndev, "Entry %zu: gate_mask 0x%x exceeds queue range (max_queues=%d)\n", + netdev_err(netdev, "Entry %zu: gate_mask 0x%x exceeds queue range (max_queues=%d)\n", i, entry->gate_mask, bp->num_queues); err = -EINVAL; goto cleanup; @@ -4393,7 +4413,7 @@ static int macb_taprio_setup_replace(struct net_device *ndev, start_time_sec = start_time; start_time_nsec = do_div(start_time_sec, NSEC_PER_SEC); if (start_time_sec > GENMASK(GEM_START_TIME_SEC_SIZE - 1, 0)) { - netdev_err(ndev, "Entry %zu: Start time %llu s exceeds hardware limit\n", + netdev_err(netdev, "Entry %zu: Start time %llu s exceeds hardware limit\n", i, start_time_sec); err = -ERANGE; goto cleanup; @@ -4401,7 +4421,7 @@ static int macb_taprio_setup_replace(struct net_device *ndev, /* Check for on time limit */ if (entry->interval > enst_max_hw_interval(speed)) { - netdev_err(ndev, "Entry %zu: interval %u ns exceeds hardware limit %llu ns\n", + netdev_err(netdev, "Entry %zu: interval %u ns exceeds hardware limit %llu ns\n", i, entry->interval, enst_max_hw_interval(speed)); err = -ERANGE; goto cleanup; @@ -4409,7 +4429,7 @@ static int macb_taprio_setup_replace(struct net_device *ndev, /* Check for off time limit*/ if ((conf->cycle_time - entry->interval) > enst_max_hw_interval(speed)) { - netdev_err(ndev, "Entry %zu: off_time %llu ns exceeds hardware limit %llu ns\n", + netdev_err(netdev, "Entry %zu: off_time %llu ns exceeds hardware limit %llu ns\n", i, conf->cycle_time - entry->interval, enst_max_hw_interval(speed)); err = -ERANGE; @@ -4432,13 +4452,13 @@ static int macb_taprio_setup_replace(struct net_device *ndev, /* Check total interval doesn't exceed cycle time */ if (total_on_time > conf->cycle_time) { - netdev_err(ndev, "Total ON %llu ns exceeds cycle time %llu ns\n", + netdev_err(netdev, "Total ON %llu ns exceeds cycle time %llu ns\n", total_on_time, conf->cycle_time); err = -EINVAL; goto cleanup; } - netdev_dbg(ndev, "TAPRIO setup: %zu entries, base_time=%lld ns, cycle_time=%llu ns\n", + netdev_dbg(netdev, "TAPRIO setup: %zu entries, base_time=%lld ns, cycle_time=%llu ns\n", conf->num_entries, conf->base_time, conf->cycle_time); /* All validations passed - proceed with hardware configuration */ @@ -4463,7 +4483,7 @@ static int macb_taprio_setup_replace(struct net_device *ndev, gem_writel(bp, ENST_CONTROL, configured_queues); } - netdev_info(ndev, "TAPRIO configuration completed successfully: %zu entries, %d queues configured\n", + netdev_info(netdev, "TAPRIO configuration completed successfully: %zu entries, %d queues configured\n", conf->num_entries, hweight32(configured_queues)); cleanup: @@ -4471,14 +4491,14 @@ static int macb_taprio_setup_replace(struct net_device *ndev, return err; } -static void macb_taprio_destroy(struct net_device *ndev) +static void macb_taprio_destroy(struct net_device *netdev) { - struct macb *bp = netdev_priv(ndev); + struct macb *bp = netdev_priv(netdev); struct macb_queue *queue; u32 queue_mask; unsigned int q; - netdev_reset_tc(ndev); + netdev_reset_tc(netdev); queue_mask = BIT_U32(bp->num_queues) - 1; scoped_guard(spinlock_irqsave, &bp->lock) { @@ -4493,30 +4513,30 @@ static void macb_taprio_destroy(struct net_device *ndev) queue_writel(queue, ENST_OFF_TIME, 0); } } - netdev_info(ndev, "TAPRIO destroy: All gates disabled\n"); + netdev_info(netdev, "TAPRIO destroy: All gates disabled\n"); } -static int macb_setup_taprio(struct net_device *ndev, +static int macb_setup_taprio(struct net_device *netdev, struct tc_taprio_qopt_offload *taprio) { - struct macb *bp = netdev_priv(ndev); + struct macb *bp = netdev_priv(netdev); int err = 0; - if (unlikely(!(ndev->hw_features & NETIF_F_HW_TC))) + if (unlikely(!(netdev->hw_features & NETIF_F_HW_TC))) return -EOPNOTSUPP; /* Check if Device is in runtime suspend */ if (unlikely(pm_runtime_suspended(&bp->pdev->dev))) { - netdev_err(ndev, "Device is in runtime suspend\n"); + netdev_err(netdev, "Device is in runtime suspend\n"); return -EOPNOTSUPP; } switch (taprio->cmd) { case TAPRIO_CMD_REPLACE: - err = macb_taprio_setup_replace(ndev, taprio); + err = macb_taprio_setup_replace(netdev, taprio); break; case TAPRIO_CMD_DESTROY: - macb_taprio_destroy(ndev); + macb_taprio_destroy(netdev); break; default: err = -EOPNOTSUPP; @@ -4525,23 +4545,23 @@ static int macb_setup_taprio(struct net_device *ndev, return err; } -static int macb_setup_tc(struct net_device *dev, enum tc_setup_type type, +static int macb_setup_tc(struct net_device *netdev, enum tc_setup_type type, void *type_data) { - if (!dev || !type_data) + if (!netdev || !type_data) return -EINVAL; switch (type) { case TC_SETUP_QDISC_TAPRIO: - return macb_setup_taprio(dev, type_data); + return macb_setup_taprio(netdev, type_data); default: return -EOPNOTSUPP; } } -static void macb_tx_timeout(struct net_device *dev, unsigned int q) +static void macb_tx_timeout(struct net_device *netdev, unsigned int q) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); macb_tx_restart(&bp->queues[q]); } @@ -4749,9 +4769,9 @@ static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, static int macb_init_dflt(struct platform_device *pdev) { - struct net_device *dev = platform_get_drvdata(pdev); + struct net_device *netdev = platform_get_drvdata(pdev); unsigned int hw_q, q; - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); struct macb_queue *queue; int err; u32 val, reg; @@ -4767,8 +4787,8 @@ static int macb_init_dflt(struct platform_device *pdev) queue = &bp->queues[q]; queue->bp = bp; spin_lock_init(&queue->tx_ptr_lock); - netif_napi_add(dev, &queue->napi_rx, macb_rx_poll); - netif_napi_add_tx(dev, &queue->napi_tx, macb_tx_poll); + netif_napi_add(netdev, &queue->napi_rx, macb_rx_poll); + netif_napi_add_tx(netdev, &queue->napi_tx, macb_tx_poll); if (hw_q) { queue->ISR = GEM_ISR(hw_q - 1); queue->IER = GEM_IER(hw_q - 1); @@ -4798,7 +4818,7 @@ static int macb_init_dflt(struct platform_device *pdev) */ queue->irq = platform_get_irq(pdev, q); err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, - IRQF_SHARED, dev->name, queue); + IRQF_SHARED, netdev->name, queue); if (err) { dev_err(&pdev->dev, "Unable to request IRQ %d (error %d)\n", @@ -4810,7 +4830,7 @@ static int macb_init_dflt(struct platform_device *pdev) q++; } - dev->netdev_ops = &macb_netdev_ops; + netdev->netdev_ops = &macb_netdev_ops; /* setup appropriated routines according to adapter type */ if (macb_is_gem(bp)) { @@ -4818,39 +4838,39 @@ static int macb_init_dflt(struct platform_device *pdev) bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; bp->macbgem_ops.mog_init_rings = gem_init_rings; bp->macbgem_ops.mog_rx = gem_rx; - dev->ethtool_ops = &gem_ethtool_ops; + netdev->ethtool_ops = &gem_ethtool_ops; } else { bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; bp->macbgem_ops.mog_init_rings = macb_init_rings; bp->macbgem_ops.mog_rx = macb_rx; - dev->ethtool_ops = &macb_ethtool_ops; + netdev->ethtool_ops = &macb_ethtool_ops; } - netdev_sw_irq_coalesce_default_on(dev); + netdev_sw_irq_coalesce_default_on(netdev); - dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; + netdev->priv_flags |= IFF_LIVE_ADDR_CHANGE; /* Set features */ - dev->hw_features = NETIF_F_SG; + netdev->hw_features = NETIF_F_SG; /* Check LSO capability; runtime detection can be overridden by a cap * flag if the hardware is known to be buggy */ if (!(bp->caps & MACB_CAPS_NO_LSO) && GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) - dev->hw_features |= MACB_NETIF_LSO; + netdev->hw_features |= MACB_NETIF_LSO; /* Checksum offload is only available on gem with packet buffer */ if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) - dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; + netdev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; if (bp->caps & MACB_CAPS_SG_DISABLED) - dev->hw_features &= ~NETIF_F_SG; + netdev->hw_features &= ~NETIF_F_SG; /* Enable HW_TC if hardware supports QBV */ if (bp->caps & MACB_CAPS_QBV) - dev->hw_features |= NETIF_F_HW_TC; + netdev->hw_features |= NETIF_F_HW_TC; - dev->features = dev->hw_features; + netdev->features = netdev->hw_features; /* Check RX Flow Filters support. * Max Rx flows set by availability of screeners & compare regs: @@ -4868,7 +4888,7 @@ static int macb_init_dflt(struct platform_device *pdev) reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); /* Filtering is supported in hw but don't enable it in kernel now */ - dev->hw_features |= NETIF_F_NTUPLE; + netdev->hw_features |= NETIF_F_NTUPLE; /* init Rx flow definitions */ bp->rx_fs_list.count = 0; spin_lock_init(&bp->rx_fs_lock); @@ -5078,9 +5098,9 @@ static void at91ether_stop(struct macb *lp) } /* Open the ethernet interface */ -static int at91ether_open(struct net_device *dev) +static int at91ether_open(struct net_device *netdev) { - struct macb *lp = netdev_priv(dev); + struct macb *lp = netdev_priv(netdev); u32 ctl; int ret; @@ -5102,7 +5122,7 @@ static int at91ether_open(struct net_device *dev) if (ret) goto stop; - netif_start_queue(dev); + netif_start_queue(netdev); return 0; @@ -5114,11 +5134,11 @@ static int at91ether_open(struct net_device *dev) } /* Close the interface */ -static int at91ether_close(struct net_device *dev) +static int at91ether_close(struct net_device *netdev) { - struct macb *lp = netdev_priv(dev); + struct macb *lp = netdev_priv(netdev); - netif_stop_queue(dev); + netif_stop_queue(netdev); phylink_stop(lp->phylink); phylink_disconnect_phy(lp->phylink); @@ -5132,14 +5152,14 @@ static int at91ether_close(struct net_device *dev) /* Transmit packet */ static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, - struct net_device *dev) + struct net_device *netdev) { - struct macb *lp = netdev_priv(dev); + struct macb *lp = netdev_priv(netdev); if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { int desc = 0; - netif_stop_queue(dev); + netif_stop_queue(netdev); /* Store packet information (to free when Tx completed) */ lp->rm9200_txq[desc].skb = skb; @@ -5148,8 +5168,8 @@ static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, skb->len, DMA_TO_DEVICE); if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) { dev_kfree_skb_any(skb); - dev->stats.tx_dropped++; - netdev_err(dev, "%s: DMA mapping error\n", __func__); + netdev->stats.tx_dropped++; + netdev_err(netdev, "%s: DMA mapping error\n", __func__); return NETDEV_TX_OK; } @@ -5159,7 +5179,8 @@ static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, macb_writel(lp, TCR, skb->len); } else { - netdev_err(dev, "%s called, but device is busy!\n", __func__); + netdev_err(netdev, "%s called, but device is busy!\n", + __func__); return NETDEV_TX_BUSY; } @@ -5169,9 +5190,9 @@ static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, /* Extract received frame from buffer descriptors and sent to upper layers. * (Called from interrupt context) */ -static void at91ether_rx(struct net_device *dev) +static void at91ether_rx(struct net_device *netdev) { - struct macb *lp = netdev_priv(dev); + struct macb *lp = netdev_priv(netdev); struct macb_queue *q = &lp->queues[0]; struct macb_dma_desc *desc; unsigned char *p_recv; @@ -5182,21 +5203,21 @@ static void at91ether_rx(struct net_device *dev) while (desc->addr & MACB_BIT(RX_USED)) { p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); - skb = netdev_alloc_skb(dev, pktlen + 2); + skb = netdev_alloc_skb(netdev, pktlen + 2); if (skb) { skb_reserve(skb, 2); skb_put_data(skb, p_recv, pktlen); - skb->protocol = eth_type_trans(skb, dev); - dev->stats.rx_packets++; - dev->stats.rx_bytes += pktlen; + skb->protocol = eth_type_trans(skb, netdev); + netdev->stats.rx_packets++; + netdev->stats.rx_bytes += pktlen; netif_rx(skb); } else { - dev->stats.rx_dropped++; + netdev->stats.rx_dropped++; } if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) - dev->stats.multicast++; + netdev->stats.multicast++; /* reset ownership bit */ desc->addr &= ~MACB_BIT(RX_USED); @@ -5214,8 +5235,8 @@ static void at91ether_rx(struct net_device *dev) /* MAC interrupt handler */ static irqreturn_t at91ether_interrupt(int irq, void *dev_id) { - struct net_device *dev = dev_id; - struct macb *lp = netdev_priv(dev); + struct net_device *netdev = dev_id; + struct macb *lp = netdev_priv(netdev); u32 intstatus, ctl; unsigned int desc; @@ -5226,13 +5247,13 @@ static irqreturn_t at91ether_interrupt(int irq, void *dev_id) /* Receive complete */ if (intstatus & MACB_BIT(RCOMP)) - at91ether_rx(dev); + at91ether_rx(netdev); /* Transmit complete */ if (intstatus & MACB_BIT(TCOMP)) { /* The TCOM bit is set even if the transmission failed */ if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) - dev->stats.tx_errors++; + netdev->stats.tx_errors++; desc = 0; if (lp->rm9200_txq[desc].skb) { @@ -5240,10 +5261,10 @@ static irqreturn_t at91ether_interrupt(int irq, void *dev_id) lp->rm9200_txq[desc].skb = NULL; dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping, lp->rm9200_txq[desc].size, DMA_TO_DEVICE); - dev->stats.tx_packets++; - dev->stats.tx_bytes += lp->rm9200_txq[desc].size; + netdev->stats.tx_packets++; + netdev->stats.tx_bytes += lp->rm9200_txq[desc].size; } - netif_wake_queue(dev); + netif_wake_queue(netdev); } /* Work-around for EMAC Errata section 41.3.1 */ @@ -5255,18 +5276,18 @@ static irqreturn_t at91ether_interrupt(int irq, void *dev_id) } if (intstatus & MACB_BIT(ISR_ROVR)) - netdev_err(dev, "ROVR error\n"); + netdev_err(netdev, "ROVR error\n"); return IRQ_HANDLED; } #ifdef CONFIG_NET_POLL_CONTROLLER -static void at91ether_poll_controller(struct net_device *dev) +static void at91ether_poll_controller(struct net_device *netdev) { unsigned long flags; local_irq_save(flags); - at91ether_interrupt(dev->irq, dev); + at91ether_interrupt(netdev->irq, netdev); local_irq_restore(flags); } #endif @@ -5313,17 +5334,17 @@ static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, static int at91ether_init(struct platform_device *pdev) { - struct net_device *dev = platform_get_drvdata(pdev); - struct macb *bp = netdev_priv(dev); + struct net_device *netdev = platform_get_drvdata(pdev); + struct macb *bp = netdev_priv(netdev); int err; bp->queues[0].bp = bp; - dev->netdev_ops = &at91ether_netdev_ops; - dev->ethtool_ops = &macb_ethtool_ops; + netdev->netdev_ops = &at91ether_netdev_ops; + netdev->ethtool_ops = &macb_ethtool_ops; - err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, - 0, dev->name, dev); + err = devm_request_irq(&pdev->dev, netdev->irq, at91ether_interrupt, + 0, netdev->name, netdev); if (err) return err; @@ -5452,8 +5473,8 @@ static int fu540_c000_init(struct platform_device *pdev) static int init_reset_optional(struct platform_device *pdev) { - struct net_device *dev = platform_get_drvdata(pdev); - struct macb *bp = netdev_priv(dev); + struct net_device *netdev = platform_get_drvdata(pdev); + struct macb *bp = netdev_priv(netdev); int ret; if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { @@ -5761,7 +5782,7 @@ static int macb_probe(struct platform_device *pdev) const struct macb_config *macb_config; struct clk *tsu_clk = NULL; phy_interface_t interface; - struct net_device *dev; + struct net_device *netdev; struct resource *regs; u32 wtrmrk_rst_val; void __iomem *mem; @@ -5796,19 +5817,19 @@ static int macb_probe(struct platform_device *pdev) goto err_disable_clocks; } - dev = alloc_etherdev_mq(sizeof(*bp), num_queues); - if (!dev) { + netdev = alloc_etherdev_mq(sizeof(*bp), num_queues); + if (!netdev) { err = -ENOMEM; goto err_disable_clocks; } - dev->base_addr = regs->start; + netdev->base_addr = regs->start; - SET_NETDEV_DEV(dev, &pdev->dev); + SET_NETDEV_DEV(netdev, &pdev->dev); - bp = netdev_priv(dev); + bp = netdev_priv(netdev); bp->pdev = pdev; - bp->dev = dev; + bp->netdev = netdev; bp->regs = mem; bp->native_io = native_io; if (native_io) { @@ -5867,6 +5888,7 @@ static int macb_probe(struct platform_device *pdev) } spin_lock_init(&bp->lock); spin_lock_init(&bp->stats_lock); + spin_lock_init(&bp->tsu_clk_lock); /* setup capabilities */ macb_configure_caps(bp, macb_config); @@ -5881,21 +5903,21 @@ static int macb_probe(struct platform_device *pdev) bp->caps |= MACB_CAPS_DMA_64B; } #endif - platform_set_drvdata(pdev, dev); + platform_set_drvdata(pdev, netdev); - dev->irq = platform_get_irq(pdev, 0); - if (dev->irq < 0) { - err = dev->irq; + netdev->irq = platform_get_irq(pdev, 0); + if (netdev->irq < 0) { + err = netdev->irq; goto err_out_free_netdev; } /* MTU range: 68 - 1518 or 10240 */ - dev->min_mtu = GEM_MTU_MIN_SIZE; + netdev->min_mtu = GEM_MTU_MIN_SIZE; if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) - dev->max_mtu = MIN(bp->jumbo_max_len, RX_BUFFER_MAX) - + netdev->max_mtu = MIN(bp->jumbo_max_len, RX_BUFFER_MAX) - ETH_HLEN - ETH_FCS_LEN; else - dev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN; + netdev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN; if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); @@ -5913,7 +5935,7 @@ static int macb_probe(struct platform_device *pdev) if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) bp->rx_intr_mask |= MACB_BIT(RXUBR); - err = of_get_ethdev_address(np, bp->dev); + err = of_get_ethdev_address(np, bp->netdev); if (err == -EPROBE_DEFER) goto err_out_free_netdev; else if (err) @@ -5935,9 +5957,9 @@ static int macb_probe(struct platform_device *pdev) if (err) goto err_out_phy_exit; - netif_carrier_off(dev); + netif_carrier_off(netdev); - err = register_netdev(dev); + err = register_netdev(netdev); if (err) { dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); goto err_out_unregister_mdio; @@ -5946,23 +5968,26 @@ static int macb_probe(struct platform_device *pdev) INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task); INIT_DELAYED_WORK(&bp->tx_lpi_work, macb_tx_lpi_work_fn); - netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", + netdev_info(netdev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), - dev->base_addr, dev->irq, dev->dev_addr); + netdev->base_addr, netdev->irq, netdev->dev_addr); pm_runtime_put_autosuspend(&bp->pdev->dev); return 0; err_out_unregister_mdio: - mdiobus_unregister(bp->mii_bus); - mdiobus_free(bp->mii_bus); + if (bp->mii_bus) { + mdiobus_unregister(bp->mii_bus); + mdiobus_free(bp->mii_bus); + } + phylink_destroy(bp->phylink); err_out_phy_exit: phy_exit(bp->phy); err_out_free_netdev: - free_netdev(dev); + free_netdev(netdev); err_disable_clocks: macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk); @@ -5975,17 +6000,19 @@ static int macb_probe(struct platform_device *pdev) static void macb_remove(struct platform_device *pdev) { - struct net_device *dev; + struct net_device *netdev; struct macb *bp; - dev = platform_get_drvdata(pdev); + netdev = platform_get_drvdata(pdev); - if (dev) { - bp = netdev_priv(dev); - unregister_netdev(dev); + if (netdev) { + bp = netdev_priv(netdev); + unregister_netdev(netdev); phy_exit(bp->phy); - mdiobus_unregister(bp->mii_bus); - mdiobus_free(bp->mii_bus); + if (bp->mii_bus) { + mdiobus_unregister(bp->mii_bus); + mdiobus_free(bp->mii_bus); + } device_set_wakeup_enable(&bp->pdev->dev, 0); cancel_delayed_work_sync(&bp->tx_lpi_work); @@ -5994,7 +6021,7 @@ static void macb_remove(struct platform_device *pdev) pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); phylink_destroy(bp->phylink); - free_netdev(dev); + free_netdev(netdev); } } @@ -6009,7 +6036,7 @@ static int __maybe_unused macb_suspend(struct device *dev) u32 tmp, ifa_local; unsigned int q; - if (!device_may_wakeup(&bp->dev->dev)) + if (!device_may_wakeup(&bp->netdev->dev)) phy_exit(bp->phy); if (!netif_running(netdev)) @@ -6019,7 +6046,7 @@ static int __maybe_unused macb_suspend(struct device *dev) if (bp->wolopts & WAKE_ARP) { /* Check for IP address in WOL ARP mode */ rcu_read_lock(); - idev = __in_dev_get_rcu(bp->dev); + idev = __in_dev_get_rcu(bp->netdev); if (idev) ifa = rcu_dereference(idev->ifa_list); if (!ifa) { @@ -6121,7 +6148,7 @@ static int __maybe_unused macb_resume(struct device *dev) unsigned long flags; unsigned int q; - if (!device_may_wakeup(&bp->dev->dev)) + if (!device_may_wakeup(&bp->netdev->dev)) phy_init(bp->phy); if (!netif_running(netdev)) diff --git a/drivers/net/ethernet/cadence/macb_pci.c b/drivers/net/ethernet/cadence/macb_pci.c index b79dec17e6b09..ac009007118f3 100644 --- a/drivers/net/ethernet/cadence/macb_pci.c +++ b/drivers/net/ethernet/cadence/macb_pci.c @@ -24,48 +24,48 @@ #define GEM_PCLK_RATE 50000000 #define GEM_HCLK_RATE 50000000 -static int macb_probe(struct pci_dev *pdev, const struct pci_device_id *id) +static int macb_probe(struct pci_dev *pci, const struct pci_device_id *id) { int err; - struct platform_device *plat_dev; + struct platform_device *pdev; struct platform_device_info plat_info; struct macb_platform_data plat_data; struct resource res[2]; /* enable pci device */ - err = pcim_enable_device(pdev); + err = pcim_enable_device(pci); if (err < 0) { - dev_err(&pdev->dev, "Enabling PCI device has failed: %d", err); + dev_err(&pci->dev, "Enabling PCI device has failed: %d", err); return err; } - pci_set_master(pdev); + pci_set_master(pci); /* set up resources */ memset(res, 0x00, sizeof(struct resource) * ARRAY_SIZE(res)); - res[0].start = pci_resource_start(pdev, 0); - res[0].end = pci_resource_end(pdev, 0); + res[0].start = pci_resource_start(pci, 0); + res[0].end = pci_resource_end(pci, 0); res[0].name = PCI_DRIVER_NAME; res[0].flags = IORESOURCE_MEM; - res[1].start = pci_irq_vector(pdev, 0); + res[1].start = pci_irq_vector(pci, 0); res[1].name = PCI_DRIVER_NAME; res[1].flags = IORESOURCE_IRQ; - dev_info(&pdev->dev, "EMAC physical base addr: %pa\n", + dev_info(&pci->dev, "EMAC physical base addr: %pa\n", &res[0].start); /* set up macb platform data */ memset(&plat_data, 0, sizeof(plat_data)); /* initialize clocks */ - plat_data.pclk = clk_register_fixed_rate(&pdev->dev, "pclk", NULL, 0, + plat_data.pclk = clk_register_fixed_rate(&pci->dev, "pclk", NULL, 0, GEM_PCLK_RATE); if (IS_ERR(plat_data.pclk)) { err = PTR_ERR(plat_data.pclk); goto err_pclk_register; } - plat_data.hclk = clk_register_fixed_rate(&pdev->dev, "hclk", NULL, 0, + plat_data.hclk = clk_register_fixed_rate(&pci->dev, "hclk", NULL, 0, GEM_HCLK_RATE); if (IS_ERR(plat_data.hclk)) { err = PTR_ERR(plat_data.hclk); @@ -74,24 +74,24 @@ static int macb_probe(struct pci_dev *pdev, const struct pci_device_id *id) /* set up platform device info */ memset(&plat_info, 0, sizeof(plat_info)); - plat_info.parent = &pdev->dev; - plat_info.fwnode = pdev->dev.fwnode; + plat_info.parent = &pci->dev; + plat_info.fwnode = pci->dev.fwnode; plat_info.name = PLAT_DRIVER_NAME; - plat_info.id = pdev->devfn; + plat_info.id = pci->devfn; plat_info.res = res; plat_info.num_res = ARRAY_SIZE(res); plat_info.data = &plat_data; plat_info.size_data = sizeof(plat_data); - plat_info.dma_mask = pdev->dma_mask; + plat_info.dma_mask = pci->dma_mask; /* register platform device */ - plat_dev = platform_device_register_full(&plat_info); - if (IS_ERR(plat_dev)) { - err = PTR_ERR(plat_dev); + pdev = platform_device_register_full(&plat_info); + if (IS_ERR(pdev)) { + err = PTR_ERR(pdev); goto err_plat_dev_register; } - pci_set_drvdata(pdev, plat_dev); + pci_set_drvdata(pci, pdev); return 0; @@ -105,14 +105,14 @@ static int macb_probe(struct pci_dev *pdev, const struct pci_device_id *id) return err; } -static void macb_remove(struct pci_dev *pdev) +static void macb_remove(struct pci_dev *pci) { - struct platform_device *plat_dev = pci_get_drvdata(pdev); - struct macb_platform_data *plat_data = dev_get_platdata(&plat_dev->dev); + struct platform_device *pdev = pci_get_drvdata(pci); + struct macb_platform_data *plat_data = dev_get_platdata(&pdev->dev); struct clk *pclk = plat_data->pclk; struct clk *hclk = plat_data->hclk; - platform_device_unregister(plat_dev); + platform_device_unregister(pdev); clk_unregister_fixed_rate(pclk); clk_unregister_fixed_rate(hclk); } diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c index d91f7b1aa39ca..14ae57fa00cba 100644 --- a/drivers/net/ethernet/cadence/macb_ptp.c +++ b/drivers/net/ethernet/cadence/macb_ptp.c @@ -50,7 +50,12 @@ static int gem_tsu_get_time(struct ptp_clock_info *ptp, struct timespec64 *ts, spin_lock_irqsave(&bp->tsu_clk_lock, flags); ptp_read_system_prets(sts); + /* explicit barriers are needed because gem_readl() is relaxed */ + if (sts) + rmb(); first = gem_readl(bp, TN); + if (sts) + rmb(); ptp_read_system_postts(sts); secl = gem_readl(bp, TSL); sech = gem_readl(bp, TSH); @@ -62,7 +67,11 @@ static int gem_tsu_get_time(struct ptp_clock_info *ptp, struct timespec64 *ts, * (assume all done within 1s) */ ptp_read_system_prets(sts); + if (sts) + rmb(); ts->tv_nsec = gem_readl(bp, TN); + if (sts) + rmb(); ptp_read_system_postts(sts); secl = gem_readl(bp, TSL); sech = gem_readl(bp, TSH); @@ -324,9 +333,9 @@ void gem_ptp_txstamp(struct macb *bp, struct sk_buff *skb, skb_tstamp_tx(skb, &shhwtstamps); } -void gem_ptp_init(struct net_device *dev) +void gem_ptp_init(struct net_device *netdev) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); bp->ptp_clock_info = gem_ptp_caps_template; @@ -334,7 +343,8 @@ void gem_ptp_init(struct net_device *dev) bp->tsu_rate = bp->ptp_info->get_tsu_rate(bp); bp->ptp_clock_info.max_adj = bp->ptp_info->get_ptp_max_adj(); gem_ptp_init_timer(bp); - bp->ptp_clock = ptp_clock_register(&bp->ptp_clock_info, &dev->dev); + gem_ptp_init_tsu(bp); + bp->ptp_clock = ptp_clock_register(&bp->ptp_clock_info, &netdev->dev); if (IS_ERR(bp->ptp_clock)) { pr_err("ptp clock register failed: %ld\n", PTR_ERR(bp->ptp_clock)); @@ -345,17 +355,13 @@ void gem_ptp_init(struct net_device *dev) return; } - spin_lock_init(&bp->tsu_clk_lock); - - gem_ptp_init_tsu(bp); - dev_info(&bp->pdev->dev, "%s ptp clock registered.\n", GEM_PTP_TIMER_NAME); } -void gem_ptp_remove(struct net_device *ndev) +void gem_ptp_remove(struct net_device *netdev) { - struct macb *bp = netdev_priv(ndev); + struct macb *bp = netdev_priv(netdev); if (bp->ptp_clock) { ptp_clock_unregister(bp->ptp_clock); @@ -378,10 +384,10 @@ static int gem_ptp_set_ts_mode(struct macb *bp, return 0; } -int gem_get_hwtst(struct net_device *dev, +int gem_get_hwtst(struct net_device *netdev, struct kernel_hwtstamp_config *tstamp_config) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); *tstamp_config = bp->tstamp_config; if (!macb_dma_ptp(bp)) @@ -402,13 +408,13 @@ static void gem_ptp_set_one_step_sync(struct macb *bp, u8 enable) macb_writel(bp, NCR, reg_val & ~MACB_BIT(OSSMODE)); } -int gem_set_hwtst(struct net_device *dev, +int gem_set_hwtst(struct net_device *netdev, struct kernel_hwtstamp_config *tstamp_config, struct netlink_ext_ack *extack) { enum macb_bd_control tx_bd_control = TSTAMP_DISABLED; enum macb_bd_control rx_bd_control = TSTAMP_DISABLED; - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); u32 regval; if (!macb_dma_ptp(bp)) diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c index 4c762229ce420..2dd2fa801829c 100644 --- a/drivers/net/ethernet/cortina/gemini.c +++ b/drivers/net/ethernet/cortina/gemini.c @@ -124,6 +124,7 @@ struct gemini_ethernet_port { unsigned int rx_coalesce_nsecs; struct sk_buff *rx_skb; unsigned int rx_frag_nr; + bool rx_dropping; unsigned int freeq_refill; struct gmac_txq txq[TX_QUEUE_NUM]; @@ -1439,7 +1440,8 @@ static struct sk_buff *gmac_skb_if_good_frame(struct gemini_ethernet_port *port, return skb; } -static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) +static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget, + unsigned int *freeq_consumed) { struct gemini_ethernet_port *port = netdev_priv(netdev); unsigned short m = (1 << port->rxq_order) - 1; @@ -1447,9 +1449,12 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) void __iomem *ptr_reg = port->rxq_rwptr; unsigned int frag_nr = port->rx_frag_nr; struct sk_buff *skb = port->rx_skb; + unsigned int consumed = 0; unsigned int frame_len, frag_len; struct gmac_rxdesc *rx = NULL; struct gmac_queue_page *gpage; + unsigned int received = 0; + bool dropping = port->rx_dropping; union gmac_rxdesc_0 word0; union gmac_rxdesc_1 word1; union gmac_rxdesc_3 word3; @@ -1471,6 +1476,7 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) w = rw.bits.wptr; while (budget && w != r) { + page = NULL; rx = port->rxq_ring + r; word0 = rx->word0; word1 = rx->word1; @@ -1479,11 +1485,22 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) r++; r &= m; + consumed++; frag_len = word0.bits.buffer_size; frame_len = word1.bits.byte_count; page_offs = mapping & ~PAGE_MASK; + if (word3.bits32 & SOF_BIT) { + if (skb) { + napi_free_frags(&port->napi); + port->stats.rx_dropped++; + skb = NULL; + frag_nr = 0; + } + dropping = false; + } + if (!mapping) { netdev_err(netdev, "rxq[%u]: HW BUG: zero DMA desc\n", r); @@ -1494,24 +1511,11 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE); if (!gpage) { dev_err(geth->dev, "could not find mapping\n"); - port->stats.rx_dropped++; - if (skb) { - napi_free_frags(&port->napi); - skb = NULL; - frag_nr = 0; - } - continue; + goto err_drop; } page = gpage->page; if (word3.bits32 & SOF_BIT) { - if (skb) { - napi_free_frags(&port->napi); - port->stats.rx_dropped++; - skb = NULL; - frag_nr = 0; - } - skb = gmac_skb_if_good_frame(port, word0, frame_len); if (!skb) goto err_drop; @@ -1521,8 +1525,7 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) frag_nr = 0; } else if (!skb) { - put_page(page); - continue; + goto err_drop; } if (word3.bits32 & EOF_BIT) @@ -1545,9 +1548,8 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) napi_gro_frags(&port->napi); skb = NULL; frag_nr = 0; - --budget; } - continue; + goto next_desc; err_drop: if (skb) { @@ -1556,16 +1558,29 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) frag_nr = 0; } - if (mapping) + if (page) put_page(page); - port->stats.rx_dropped++; + if (!dropping) { + port->stats.rx_dropped++; + dropping = true; + } + +next_desc: + /* Final or single-descriptor fragment, advance things */ + if (word3.bits32 & EOF_BIT) { + budget--; + received++; + dropping = false; + } } port->rx_skb = skb; port->rx_frag_nr = frag_nr; + port->rx_dropping = dropping; + *freeq_consumed = consumed; writew(r, ptr_reg); - return budget; + return received; } static int gmac_napi_poll(struct napi_struct *napi, int budget) @@ -1573,26 +1588,27 @@ static int gmac_napi_poll(struct napi_struct *napi, int budget) struct gemini_ethernet_port *port = netdev_priv(napi->dev); struct gemini_ethernet *geth = port->geth; unsigned int freeq_threshold; + unsigned int freeq_consumed; unsigned int received; freeq_threshold = 1 << (geth->freeq_order - 1); u64_stats_update_begin(&port->rx_stats_syncp); - received = gmac_rx(napi->dev, budget); - if (received < budget) { - napi_gro_flush(napi, false); - napi_complete_done(napi, received); - gmac_enable_rx_irq(napi->dev, 1); + received = gmac_rx(napi->dev, budget, &freeq_consumed); + if (received < budget) ++port->rx_napi_exits; - } - port->freeq_refill += (budget - received); + u64_stats_update_end(&port->rx_stats_syncp); + + port->freeq_refill += freeq_consumed; if (port->freeq_refill > freeq_threshold) { port->freeq_refill -= freeq_threshold; geth_fill_freeq(geth, true); } - u64_stats_update_end(&port->rx_stats_syncp); + if (received < budget && napi_complete_done(napi, received)) + gmac_enable_rx_irq(napi->dev, 1); + return received; } @@ -1782,7 +1798,7 @@ static irqreturn_t gmac_irq(int irq, void *data) if (val & (GMAC0_RX_OVERRUN_INT_BIT << (netdev->dev_id * 8))) { spin_lock(&geth->irq_lock); - writel(GMAC0_RXDERR_INT_BIT << (netdev->dev_id * 8), + writel(GMAC0_RX_OVERRUN_INT_BIT << (netdev->dev_id * 8), geth->base + GLOBAL_INTERRUPT_STATUS_4_REG); u64_stats_update_begin(&port->ir_stats_syncp); ++port->stats.rx_fifo_errors; @@ -1893,6 +1909,7 @@ static int gmac_stop(struct net_device *netdev) napi_disable(&port->napi); port->rx_skb = NULL; port->rx_frag_nr = 0; + port->rx_dropping = false; gmac_enable_irq(netdev, 0); gmac_cleanup_rxq(netdev); diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index 8e3f345dd9aac..80f0082f6c633 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -81,6 +81,33 @@ void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter) } EXPORT_SYMBOL_GPL(enetc_reset_mac_addr_filter); +void enetc_set_congestion_mode(struct enetc_ndev_priv *priv, bool enable) +{ + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; + + spin_lock(&si->gen_lock); + + if (enable) + set_bit(ENETC_RXBDR_CM, &priv->flags); + else + clear_bit(ENETC_RXBDR_CM, &priv->flags); + + for (int i = 0; i < priv->num_rx_rings; i++) { + u32 old_rbmr = enetc_rxbdr_rd(hw, i, ENETC_RBMR); + u32 rbmr; + + rbmr = u32_replace_bits(old_rbmr, enable, ENETC_RBMR_CM); + if (rbmr == old_rbmr) + continue; + + enetc_rxbdr_wr(hw, i, ENETC_RBMR, rbmr); + } + + spin_unlock(&si->gen_lock); +} +EXPORT_SYMBOL_GPL(enetc_set_congestion_mode); + static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) { int num_tx_rings = priv->num_tx_rings; @@ -2632,7 +2659,6 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, bool extended) { int idx = rx_ring->index; - u32 rbmr = 0; enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, lower_32_bits(rx_ring->bd_dma_base)); @@ -2660,12 +2686,6 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); rx_ring->ext_en = extended; - if (rx_ring->ext_en) - rbmr |= ENETC_RBMR_BDS; - - if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) - rbmr |= ENETC_RBMR_VTE; - rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); rx_ring->idr = hw->reg + ENETC_SIRXIDR; @@ -2676,8 +2696,6 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, enetc_lock_mdio(); enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); enetc_unlock_mdio(); - - enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); } static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) @@ -2704,21 +2722,34 @@ static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) { + struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); int idx = rx_ring->index; - u32 rbmr; + u32 rbmr = ENETC_RBMR_EN; + + if (rx_ring->ext_en) + rbmr |= ENETC_RBMR_BDS; + + if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) + rbmr |= ENETC_RBMR_VTE; + + if (test_bit(ENETC_RXBDR_CM, &priv->flags)) + rbmr |= ENETC_RBMR_CM; - rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); - rbmr |= ENETC_RBMR_EN; enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); } static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) { - struct enetc_hw *hw = &priv->si->hw; + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; int i; + spin_lock(&si->gen_lock); + for (i = 0; i < priv->num_rx_rings; i++) enetc_enable_rxbdr(hw, priv->rx_ring[i]); + + spin_unlock(&si->gen_lock); } static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) @@ -2748,11 +2779,16 @@ static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) { - struct enetc_hw *hw = &priv->si->hw; + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; int i; + spin_lock(&si->gen_lock); + for (i = 0; i < priv->num_rx_rings; i++) enetc_disable_rxbdr(hw, priv->rx_ring[i]); + + spin_unlock(&si->gen_lock); } static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) @@ -3344,11 +3380,16 @@ EXPORT_SYMBOL_GPL(enetc_get_stats); static void enetc_enable_rxvlan(struct net_device *ndev, bool en) { struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct enetc_hw *hw = &priv->si->hw; + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; int i; + spin_lock(&si->gen_lock); + for (i = 0; i < priv->num_rx_rings; i++) enetc_bdr_enable_rxvlan(hw, i, en); + + spin_unlock(&si->gen_lock); } static void enetc_enable_txvlan(struct net_device *ndev, bool en) @@ -3679,6 +3720,7 @@ int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) si = PTR_ALIGN(p, ENETC_SI_ALIGN); si->pad = (char *)si - (char *)p; + spin_lock_init(&si->gen_lock); pci_set_drvdata(pdev, si); si->pdev = pdev; diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h index 04a5dd5ea6c74..2145563ab2ac1 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.h +++ b/drivers/net/ethernet/freescale/enetc/enetc.h @@ -309,6 +309,13 @@ struct enetc_si { struct net_device *ndev; /* back ref. */ + /* General-purpose lock serializing updates that must not race, + * e.g. read-modify-write of shared hardware registers and of + * selected priv->flags bits between the phylink link callbacks + * and the ring (re)configuration path. + */ + spinlock_t gen_lock; + union { struct enetc_cbdr cbd_ring; /* Only ENETC 1.0 */ struct ntmp_user ntmp_user; /* ENETC 4.1 and later */ @@ -324,8 +331,6 @@ struct enetc_si { const struct enetc_drvdata *drvdata; const struct enetc_si_ops *ops; - struct workqueue_struct *workqueue; - struct work_struct rx_mode_task; struct dentry *debugfs_root; struct enetc_msg_swbd msg; /* Only valid for VSI */ }; @@ -419,6 +424,7 @@ enum enetc_active_offloads { enum enetc_flags_bit { ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS = 0, ENETC_TX_DOWN, + ENETC_RXBDR_CM, }; /* interrupt coalescing modes */ @@ -507,6 +513,7 @@ int enetc_get_driver_data(struct enetc_si *si); void enetc_add_mac_addr_ht_filter(struct enetc_mac_filter *filter, const unsigned char *addr); void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter); +void enetc_set_congestion_mode(struct enetc_ndev_priv *priv, bool enable); int enetc_open(struct net_device *ndev); int enetc_close(struct net_device *ndev); diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c b/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c index 1b1591dce73d4..4a769d9e5679c 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c +++ b/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c @@ -31,9 +31,11 @@ static int enetc_mac_filter_show(struct seq_file *s, void *data) struct enetc_si *si = s->private; struct enetc_hw *hw = &si->hw; struct maft_entry_data maft; + struct ntmp_user *user; struct enetc_pf *pf; - int i, err, num_si; - u32 val; + u32 val, entry_id; + int i, num_si; + int err = 0; pf = enetc_si_priv(si); num_si = pf->caps.num_vsi + 1; @@ -50,22 +52,30 @@ static int enetc_mac_filter_show(struct seq_file *s, void *data) for (i = 0; i < num_si; i++) enetc_show_si_mac_hash_filter(s, i); - if (!pf->num_mfe) - return 0; + user = &si->ntmp_user; + rtnl_lock(); + + if (bitmap_empty(user->maft_eid_bitmap, user->maft_num_entries)) + goto unlock_rtnl; /* MAC address filter table */ seq_puts(s, "MAC address filter table\n"); - for (i = 0; i < pf->num_mfe; i++) { + for_each_set_bit(entry_id, user->maft_eid_bitmap, + user->maft_num_entries) { memset(&maft, 0, sizeof(maft)); - err = ntmp_maft_query_entry(&si->ntmp_user, i, &maft); + err = ntmp_maft_query_entry(user, entry_id, &maft); if (err) - return err; + goto unlock_rtnl; - seq_printf(s, "Entry %d, MAC: %pM, SI bitmap: 0x%04x\n", i, - maft.keye.mac_addr, le16_to_cpu(maft.cfge.si_bitmap)); + seq_printf(s, "Entry %d, MAC: %pM, SI bitmap: 0x%04x\n", + entry_id, maft.keye.mac_addr, + le16_to_cpu(maft.cfge.si_bitmap)); } - return 0; +unlock_rtnl: + rtnl_unlock(); + + return err; } DEFINE_SHOW_ATTRIBUTE(enetc_mac_filter); diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_hw.h b/drivers/net/ethernet/freescale/enetc/enetc4_hw.h index f18437556a0ee..56240cd6f0f8b 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_hw.h +++ b/drivers/net/ethernet/freescale/enetc/enetc4_hw.h @@ -69,8 +69,6 @@ /* Port Station interface promiscuous MAC mode register */ #define ENETC4_PSIPMMR 0x200 -#define PSIPMMR_SI_MAC_UP(a) BIT(a) /* a = SI index */ -#define PSIPMMR_SI_MAC_MP(a) BIT((a) + 16) /* Port Station interface promiscuous VLAN mode register */ #define ENETC4_PSIPVMR 0x204 @@ -137,7 +135,6 @@ #define ENETC4_PSIVHFR1(a) ((a) * 0x80 + 0x2064) #define ENETC4_PMCAPR 0x4004 -#define PMCAPR_HD BIT(8) #define PMCAPR_FP GENMASK(10, 9) /* Port capability register */ @@ -200,7 +197,6 @@ #define PM_CMD_CFG_CNT_FRM_EN BIT(13) #define PM_CMD_CFG_TXP BIT(15) #define PM_CMD_CFG_SEND_IDLE BIT(16) -#define PM_CMD_CFG_HD_FCEN BIT(18) #define PM_CMD_CFG_SFD BIT(21) #define PM_CMD_CFG_TX_FLUSH BIT(22) #define PM_CMD_CFG_TX_LOWP_EN BIT(23) diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c index 437a15bbb47ba..65d3c24510e87 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c @@ -29,12 +29,6 @@ static void enetc4_get_port_caps(struct enetc_pf *pf) val = enetc_port_rd(hw, ENETC4_ECAPR2); pf->caps.num_rx_bdr = (val & ECAPR2_NUM_RX_BDR) >> 16; pf->caps.num_tx_bdr = val & ECAPR2_NUM_TX_BDR; - - val = enetc_port_rd(hw, ENETC4_PMCAPR); - pf->caps.half_duplex = (val & PMCAPR_HD) ? 1 : 0; - - val = enetc_port_rd(hw, ENETC4_PSIMAFCAPR); - pf->caps.mac_filter_num = val & PSIMAFCAPR_NUM_MAC_AFTE; } static void enetc4_get_psi_hw_features(struct enetc_si *si) @@ -75,38 +69,6 @@ static void enetc4_pf_get_si_primary_mac(struct enetc_hw *hw, int si, put_unaligned_le16(lower, addr + 4); } -static void enetc4_pf_set_si_mac_promisc(struct enetc_hw *hw, int si, - bool uc_promisc, bool mc_promisc) -{ - u32 val = enetc_port_rd(hw, ENETC4_PSIPMMR); - - if (uc_promisc) - val |= PSIPMMR_SI_MAC_UP(si); - else - val &= ~PSIPMMR_SI_MAC_UP(si); - - if (mc_promisc) - val |= PSIPMMR_SI_MAC_MP(si); - else - val &= ~PSIPMMR_SI_MAC_MP(si); - - enetc_port_wr(hw, ENETC4_PSIPMMR, val); -} - -static void enetc4_pf_set_si_uc_hash_filter(struct enetc_hw *hw, int si, - u64 hash) -{ - enetc_port_wr(hw, ENETC4_PSIUMHFR0(si), lower_32_bits(hash)); - enetc_port_wr(hw, ENETC4_PSIUMHFR1(si), upper_32_bits(hash)); -} - -static void enetc4_pf_set_si_mc_hash_filter(struct enetc_hw *hw, int si, - u64 hash) -{ - enetc_port_wr(hw, ENETC4_PSIMMHFR0(si), lower_32_bits(hash)); - enetc_port_wr(hw, ENETC4_PSIMMHFR1(si), upper_32_bits(hash)); -} - static void enetc4_pf_set_loopback(struct net_device *ndev, bool en) { struct enetc_ndev_priv *priv = netdev_priv(ndev); @@ -124,34 +86,47 @@ static void enetc4_pf_set_loopback(struct net_device *ndev, bool en) static void enetc4_pf_clear_maft_entries(struct enetc_pf *pf) { - int i; - - for (i = 0; i < pf->num_mfe; i++) - ntmp_maft_delete_entry(&pf->si->ntmp_user, i); + struct ntmp_user *user = &pf->si->ntmp_user; + u32 entry_id; - pf->num_mfe = 0; + for_each_set_bit(entry_id, user->maft_eid_bitmap, + user->maft_num_entries) { + if (!ntmp_maft_delete_entry(user, entry_id)) + ntmp_clear_eid_bitmap(user->maft_eid_bitmap, entry_id); + } } static int enetc4_pf_add_maft_entries(struct enetc_pf *pf, - struct enetc_mac_addr *mac, - int mac_cnt) + struct netdev_hw_addr_list *uc) { + struct ntmp_user *user = &pf->si->ntmp_user; + int mac_cnt = netdev_hw_addr_list_count(uc); struct maft_entry_data maft = {}; + struct netdev_hw_addr *ha; + u32 available_entries; u16 si_bit = BIT(0); - int i, err; + u32 entry_id; + int err; + + available_entries = user->maft_num_entries - + bitmap_weight(user->maft_eid_bitmap, + user->maft_num_entries); + + if (mac_cnt > available_entries) + return -ENOSPC; maft.cfge.si_bitmap = cpu_to_le16(si_bit); - for (i = 0; i < mac_cnt; i++) { - ether_addr_copy(maft.keye.mac_addr, mac[i].addr); - err = ntmp_maft_add_entry(&pf->si->ntmp_user, i, &maft); + netdev_hw_addr_list_for_each(ha, uc) { + entry_id = ntmp_lookup_free_eid(user->maft_eid_bitmap, + user->maft_num_entries); + ether_addr_copy(maft.keye.mac_addr, ha->addr); + err = ntmp_maft_add_entry(user, entry_id, &maft); if (unlikely(err)) { - pf->num_mfe = i; + ntmp_clear_eid_bitmap(user->maft_eid_bitmap, entry_id); goto clear_maft_entries; } } - pf->num_mfe = mac_cnt; - return 0; clear_maft_entries: @@ -160,97 +135,70 @@ static int enetc4_pf_add_maft_entries(struct enetc_pf *pf, return err; } -static int enetc4_pf_set_uc_exact_filter(struct enetc_pf *pf) +static void enetc4_pf_set_uc_hash_filter(struct enetc_pf *pf, + struct netdev_hw_addr_list *uc) { - int max_num_mfe = pf->caps.mac_filter_num; - struct enetc_mac_filter mac_filter = {}; - struct net_device *ndev = pf->si->ndev; - struct enetc_hw *hw = &pf->si->hw; - struct enetc_mac_addr *mac_tbl; + struct enetc_mac_filter *mac_filter = &pf->mac_filter[UC]; struct netdev_hw_addr *ha; - int i = 0, err; - int mac_cnt; + u64 hash; - netif_addr_lock_bh(ndev); + enetc_reset_mac_addr_filter(mac_filter); + netdev_hw_addr_list_for_each(ha, uc) + enetc_add_mac_addr_ht_filter(mac_filter, ha->addr); - mac_cnt = netdev_uc_count(ndev); - if (!mac_cnt) { - netif_addr_unlock_bh(ndev); + bitmap_to_arr64(&hash, mac_filter->mac_hash_table, + ENETC_MADDR_HASH_TBL_SZ); + enetc_set_si_uc_hash_filter(pf->si, 0, hash); +} + +static int enetc4_pf_set_uc_exact_filter(struct enetc_pf *pf, + struct netdev_hw_addr_list *uc) +{ + struct enetc_si *si = pf->si; + int err; + + if (netdev_hw_addr_list_empty(uc)) { /* clear both MAC hash and exact filters */ - enetc4_pf_set_si_uc_hash_filter(hw, 0, 0); + enetc_set_si_uc_hash_filter(si, 0, 0); enetc4_pf_clear_maft_entries(pf); return 0; } - if (mac_cnt > max_num_mfe) { - err = -ENOSPC; - goto unlock_netif_addr; - } - - mac_tbl = kzalloc_objs(*mac_tbl, mac_cnt, GFP_ATOMIC); - if (!mac_tbl) { - err = -ENOMEM; - goto unlock_netif_addr; - } - - netdev_for_each_uc_addr(ha, ndev) { - enetc_add_mac_addr_ht_filter(&mac_filter, ha->addr); - ether_addr_copy(mac_tbl[i++].addr, ha->addr); - } - - netif_addr_unlock_bh(ndev); - - /* Set temporary unicast hash filters in case of Rx loss when + /* Set temporary unicast hash filter in case of Rx loss when * updating MAC address filter table */ - enetc4_pf_set_si_uc_hash_filter(hw, 0, *mac_filter.mac_hash_table); + enetc4_pf_set_uc_hash_filter(pf, uc); enetc4_pf_clear_maft_entries(pf); - if (!enetc4_pf_add_maft_entries(pf, mac_tbl, i)) - enetc4_pf_set_si_uc_hash_filter(hw, 0, 0); - - kfree(mac_tbl); - - return 0; - -unlock_netif_addr: - netif_addr_unlock_bh(ndev); + err = enetc4_pf_add_maft_entries(pf, uc); + if (!err) { + enetc_reset_mac_addr_filter(&pf->mac_filter[UC]); + enetc_set_si_uc_hash_filter(si, 0, 0); + } return err; } -static void enetc4_pf_set_mac_hash_filter(struct enetc_pf *pf, int type) +static void enetc4_pf_set_mc_hash_filter(struct enetc_pf *pf, + struct netdev_hw_addr_list *mc) { - struct net_device *ndev = pf->si->ndev; - struct enetc_mac_filter *mac_filter; - struct enetc_hw *hw = &pf->si->hw; + struct enetc_mac_filter *mac_filter = &pf->mac_filter[MC]; struct netdev_hw_addr *ha; + u64 hash; - netif_addr_lock_bh(ndev); - if (type & ENETC_MAC_FILTER_TYPE_UC) { - mac_filter = &pf->mac_filter[UC]; - enetc_reset_mac_addr_filter(mac_filter); - netdev_for_each_uc_addr(ha, ndev) - enetc_add_mac_addr_ht_filter(mac_filter, ha->addr); + enetc_reset_mac_addr_filter(mac_filter); + netdev_hw_addr_list_for_each(ha, mc) + enetc_add_mac_addr_ht_filter(mac_filter, ha->addr); - enetc4_pf_set_si_uc_hash_filter(hw, 0, - *mac_filter->mac_hash_table); - } - - if (type & ENETC_MAC_FILTER_TYPE_MC) { - mac_filter = &pf->mac_filter[MC]; - enetc_reset_mac_addr_filter(mac_filter); - netdev_for_each_mc_addr(ha, ndev) - enetc_add_mac_addr_ht_filter(mac_filter, ha->addr); - - enetc4_pf_set_si_mc_hash_filter(hw, 0, - *mac_filter->mac_hash_table); - } - netif_addr_unlock_bh(ndev); + bitmap_to_arr64(&hash, mac_filter->mac_hash_table, + ENETC_MADDR_HASH_TBL_SZ); + enetc_set_si_mc_hash_filter(pf->si, 0, hash); } -static void enetc4_pf_set_mac_filter(struct enetc_pf *pf, int type) +static void enetc4_pf_set_mac_filter(struct enetc_pf *pf, int type, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc) { /* Currently, the MAC address filter table (MAFT) only has 4 entries, * and multiple multicast addresses for filtering will be configured @@ -258,15 +206,16 @@ static void enetc4_pf_set_mac_filter(struct enetc_pf *pf, int type) * unicast filtering. If the number of unicast addresses exceeds the * table capacity, the MAC hash filter will be used. */ - if (type & ENETC_MAC_FILTER_TYPE_UC && enetc4_pf_set_uc_exact_filter(pf)) { + if (type & ENETC_MAC_FILTER_TYPE_UC && + enetc4_pf_set_uc_exact_filter(pf, uc)) { /* Fall back to the MAC hash filter */ - enetc4_pf_set_mac_hash_filter(pf, ENETC_MAC_FILTER_TYPE_UC); + enetc4_pf_set_uc_hash_filter(pf, uc); /* Clear the old MAC exact filter */ enetc4_pf_clear_maft_entries(pf); } if (type & ENETC_MAC_FILTER_TYPE_MC) - enetc4_pf_set_mac_hash_filter(pf, ENETC_MAC_FILTER_TYPE_MC); + enetc4_pf_set_mc_hash_filter(pf, mc); } static const struct enetc_pf_ops enetc4_pf_ops = { @@ -450,18 +399,60 @@ static void enetc4_configure_port(struct enetc_pf *pf) enetc_set_default_rss_key(pf); } +static void enetc4_get_ntmp_caps(struct enetc_si *si) +{ + struct ntmp_user *user = &si->ntmp_user; + struct enetc_hw *hw = &si->hw; + u32 val; + + val = enetc_port_rd(hw, ENETC4_PSIMAFCAPR); + user->maft_num_entries = FIELD_GET(PSIMAFCAPR_NUM_MAC_AFTE, val); +} + +static int enetc4_ntmp_bitmap_init(struct ntmp_user *user) +{ + user->maft_eid_bitmap = bitmap_zalloc(user->maft_num_entries, + GFP_KERNEL); + if (!user->maft_eid_bitmap) + return -ENOMEM; + + return 0; +} + +static void enetc4_ntmp_bitmap_free(struct ntmp_user *user) +{ + bitmap_free(user->maft_eid_bitmap); + user->maft_eid_bitmap = NULL; +} + static int enetc4_init_ntmp_user(struct enetc_si *si) { struct ntmp_user *user = &si->ntmp_user; + int err; /* For ENETC 4.1, all table versions are 0 */ memset(&user->tbl, 0, sizeof(user->tbl)); - return enetc4_setup_cbdr(si); + err = enetc4_setup_cbdr(si); + if (err) + return err; + + enetc4_get_ntmp_caps(si); + err = enetc4_ntmp_bitmap_init(user); + if (err) + goto teardown_cbdr; + + return 0; + +teardown_cbdr: + enetc4_teardown_cbdr(si); + + return err; } static void enetc4_free_ntmp_user(struct enetc_si *si) { + enetc4_ntmp_bitmap_free(&si->ntmp_user); enetc4_teardown_cbdr(si); } @@ -479,7 +470,7 @@ static int enetc4_pf_init(struct enetc_pf *pf) err = enetc4_init_ntmp_user(pf->si); if (err) { - dev_err(dev, "Failed to init CBDR\n"); + dev_err(dev, "Failed to init NTMP user\n"); return err; } @@ -493,18 +484,17 @@ static void enetc4_pf_free(struct enetc_pf *pf) enetc4_free_ntmp_user(pf->si); } -static void enetc4_psi_do_set_rx_mode(struct work_struct *work) +static int enetc4_pf_set_rx_mode(struct net_device *ndev, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc) { - struct enetc_si *si = container_of(work, struct enetc_si, rx_mode_task); - struct enetc_pf *pf = enetc_si_priv(si); - struct net_device *ndev = si->ndev; - struct enetc_hw *hw = &si->hw; + struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_pf *pf = enetc_si_priv(priv->si); + struct enetc_si *si = priv->si; bool uc_promisc = false; bool mc_promisc = false; int type = 0; - rtnl_lock(); - if (ndev->flags & IFF_PROMISC) { uc_promisc = true; mc_promisc = true; @@ -515,28 +505,21 @@ static void enetc4_psi_do_set_rx_mode(struct work_struct *work) type = ENETC_MAC_FILTER_TYPE_ALL; } - enetc4_pf_set_si_mac_promisc(hw, 0, uc_promisc, mc_promisc); + enetc_set_si_uc_promisc(si, 0, uc_promisc); + enetc_set_si_mc_promisc(si, 0, mc_promisc); if (uc_promisc) { - enetc4_pf_set_si_uc_hash_filter(hw, 0, 0); + enetc_set_si_uc_hash_filter(si, 0, 0); enetc4_pf_clear_maft_entries(pf); } if (mc_promisc) - enetc4_pf_set_si_mc_hash_filter(hw, 0, 0); + enetc_set_si_mc_hash_filter(si, 0, 0); /* Set new MAC filter */ - enetc4_pf_set_mac_filter(pf, type); + enetc4_pf_set_mac_filter(pf, type, uc, mc); - rtnl_unlock(); -} - -static void enetc4_pf_set_rx_mode(struct net_device *ndev) -{ - struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct enetc_si *si = priv->si; - - queue_work(si->workqueue, &si->rx_mode_task); + return 0; } static int enetc4_pf_set_features(struct net_device *ndev, @@ -566,7 +549,7 @@ static const struct net_device_ops enetc4_ndev_ops = { .ndo_start_xmit = enetc_xmit, .ndo_get_stats = enetc_get_stats, .ndo_set_mac_address = enetc_pf_set_mac_addr, - .ndo_set_rx_mode = enetc4_pf_set_rx_mode, + .ndo_set_rx_mode_async = enetc4_pf_set_rx_mode, .ndo_set_features = enetc4_pf_set_features, .ndo_vlan_rx_add_vid = enetc_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = enetc_vlan_rx_del_vid, @@ -602,11 +585,6 @@ static void enetc4_mac_config(struct enetc_pf *pf, unsigned int mode, case PHY_INTERFACE_MODE_RGMII_RXID: case PHY_INTERFACE_MODE_RGMII_TXID: val |= IFMODE_RGMII; - /* We need to enable auto-negotiation for the MAC - * if its RGMII interface support In-Band status. - */ - if (phylink_autoneg_inband(mode)) - val |= PM_IF_MODE_ENA; break; case PHY_INTERFACE_MODE_RMII: val |= IFMODE_RMII; @@ -717,22 +695,6 @@ static void enetc4_set_rmii_mac(struct enetc_pf *pf, int speed, int duplex) enetc_port_mac_wr(si, ENETC4_PM_IF_MODE(0), val); } -static void enetc4_set_hd_flow_control(struct enetc_pf *pf, bool enable) -{ - struct enetc_si *si = pf->si; - u32 old_val, val; - - if (!pf->caps.half_duplex) - return; - - old_val = enetc_port_mac_rd(si, ENETC4_PM_CMD_CFG(0)); - val = u32_replace_bits(old_val, enable ? 1 : 0, PM_CMD_CFG_HD_FCEN); - if (val == old_val) - return; - - enetc_port_mac_wr(si, ENETC4_PM_CMD_CFG(0), val); -} - static void enetc4_set_rx_pause(struct enetc_pf *pf, bool rx_pause) { struct enetc_si *si = pf->si; @@ -746,22 +708,14 @@ static void enetc4_set_rx_pause(struct enetc_pf *pf, bool rx_pause) enetc_port_mac_wr(si, ENETC4_PM_CMD_CFG(0), val); } -static void enetc4_set_tx_pause(struct enetc_pf *pf, int num_rxbdr, bool tx_pause) +static void enetc4_set_tx_pause(struct enetc_pf *pf, bool tx_pause) { + struct enetc_ndev_priv *priv = netdev_priv(pf->si->ndev); u32 pause_off_thresh = 0, pause_on_thresh = 0; u32 init_quanta = 0, refresh_quanta = 0; struct enetc_hw *hw = &pf->si->hw; - u32 rbmr, old_rbmr; - int i; - - for (i = 0; i < num_rxbdr; i++) { - old_rbmr = enetc_rxbdr_rd(hw, i, ENETC_RBMR); - rbmr = u32_replace_bits(old_rbmr, tx_pause ? 1 : 0, ENETC_RBMR_CM); - if (rbmr == old_rbmr) - continue; - enetc_rxbdr_wr(hw, i, ENETC_RBMR, rbmr); - } + enetc_set_congestion_mode(priv, tx_pause); if (tx_pause) { /* When the port first enters congestion, send a PAUSE request @@ -908,13 +862,11 @@ static void enetc4_pl_mac_link_up(struct phylink_config *config, struct enetc_pf *pf = phylink_to_enetc_pf(config); struct enetc_si *si = pf->si; struct enetc_ndev_priv *priv; - bool hd_fc = false; priv = netdev_priv(si->ndev); enetc4_set_port_speed(priv, speed); - if (!phylink_autoneg_inband(mode) && - phy_interface_mode_is_rgmii(interface)) + if (phy_interface_mode_is_rgmii(interface)) enetc4_set_rgmii_mac(pf, speed, duplex); if (interface == PHY_INTERFACE_MODE_RMII) @@ -926,19 +878,9 @@ static void enetc4_pl_mac_link_up(struct phylink_config *config, */ if (priv->active_offloads & ENETC_F_QBU) tx_pause = false; - } else { /* DUPLEX_HALF */ - if (tx_pause || rx_pause) - hd_fc = true; - - /* As per 802.3 annex 31B, PAUSE frames are only supported - * when the link is configured for full duplex operation. - */ - tx_pause = false; - rx_pause = false; } - enetc4_set_hd_flow_control(pf, hd_fc); - enetc4_set_tx_pause(pf, priv->num_rx_rings, tx_pause); + enetc4_set_tx_pause(pf, tx_pause); enetc4_set_rx_pause(pf, rx_pause); enetc4_mac_tx_enable(pf); enetc4_mac_rx_enable(pf); @@ -1009,19 +951,6 @@ static void enetc4_link_deinit(struct enetc_ndev_priv *priv) enetc_mdiobus_destroy(pf); } -static int enetc4_psi_wq_task_init(struct enetc_si *si) -{ - char wq_name[24]; - - INIT_WORK(&si->rx_mode_task, enetc4_psi_do_set_rx_mode); - snprintf(wq_name, sizeof(wq_name), "enetc-%s", pci_name(si->pdev)); - si->workqueue = create_singlethread_workqueue(wq_name); - if (!si->workqueue) - return -ENOMEM; - - return 0; -} - static int enetc4_pf_netdev_create(struct enetc_si *si) { struct device *dev = &si->pdev->dev; @@ -1062,12 +991,6 @@ static int enetc4_pf_netdev_create(struct enetc_si *si) if (err) goto err_link_init; - err = enetc4_psi_wq_task_init(si); - if (err) { - dev_err(dev, "Failed to init workqueue\n"); - goto err_wq_init; - } - err = register_netdev(ndev); if (err) { dev_err(dev, "Failed to register netdev\n"); @@ -1077,8 +1000,6 @@ static int enetc4_pf_netdev_create(struct enetc_si *si) return 0; err_reg_netdev: - destroy_workqueue(si->workqueue); -err_wq_init: enetc4_link_deinit(priv); err_link_init: enetc_free_msix(priv); @@ -1096,8 +1017,6 @@ static void enetc4_pf_netdev_destroy(struct enetc_si *si) struct net_device *ndev = si->ndev; unregister_netdev(ndev); - cancel_work(&si->rx_mode_task); - destroy_workqueue(si->workqueue); enetc4_link_deinit(priv); enetc_free_msix(priv); free_netdev(ndev); diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c index 71f376ef1be1c..07b7832f24276 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c @@ -29,7 +29,7 @@ static const u32 enetc_rxbdr_regs[] = { }; static const u32 enetc_port_regs[] = { - ENETC_PMR, ENETC_PSR, ENETC_PSIPMR, ENETC_PSIPMAR0(0), + ENETC_PMR, ENETC_PSR, ENETC_PSIPMMR, ENETC_PSIPMAR0(0), ENETC_PSIPMAR1(0), ENETC_PTXMBAR, ENETC_PCAPR0, ENETC_PCAPR1, ENETC_PSICFGR0(0), ENETC_PRFSCAPR, ENETC_PTCMSDUR(0), ENETC_PM0_CMD_CFG, ENETC_PM0_MAXFRM, ENETC_PM0_IF_MODE diff --git a/drivers/net/ethernet/freescale/enetc/enetc_hw.h b/drivers/net/ethernet/freescale/enetc/enetc_hw.h index bf99b65d7598a..66bfda60da9cb 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h @@ -180,9 +180,10 @@ enum enetc_bdr_type {TX, RX}; #define ENETC_PMR_PSPEED_1000M BIT(9) #define ENETC_PMR_PSPEED_2500M BIT(10) #define ENETC_PSR 0x0004 /* RO */ -#define ENETC_PSIPMR 0x0018 -#define ENETC_PSIPMR_SET_UP(n) BIT(n) /* n = SI index */ -#define ENETC_PSIPMR_SET_MP(n) BIT((n) + 16) +#define ENETC_PSIPMMR 0x0018 +#define PSIPMMR_SI_MAC_UP(n) BIT(n) /* n = SI index */ +#define PSIPMMR_SI_MAC_MP(n) BIT((n) + 16) + #define ENETC_PSIPVMR 0x001c #define ENETC_VLAN_PROMISC_MAP_ALL 0x7 #define ENETC_PSIPVMR_SET_VP(simap) ((simap) & 0x7) diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c index 2d687bb8c3a06..bdfe6bc056b52 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c @@ -80,37 +80,6 @@ static void enetc_add_mac_addr_em_filter(struct enetc_mac_filter *filter, filter->mac_addr_cnt++; } -static void enetc_clear_mac_ht_flt(struct enetc_si *si, int si_idx, int type) -{ - bool err = si->errata & ENETC_ERR_UCMCSWP; - - if (type == UC) { - enetc_port_wr(&si->hw, ENETC_PSIUMHFR0(si_idx, err), 0); - enetc_port_wr(&si->hw, ENETC_PSIUMHFR1(si_idx), 0); - } else { /* MC */ - enetc_port_wr(&si->hw, ENETC_PSIMMHFR0(si_idx, err), 0); - enetc_port_wr(&si->hw, ENETC_PSIMMHFR1(si_idx), 0); - } -} - -static void enetc_set_mac_ht_flt(struct enetc_si *si, int si_idx, int type, - unsigned long hash) -{ - bool err = si->errata & ENETC_ERR_UCMCSWP; - - if (type == UC) { - enetc_port_wr(&si->hw, ENETC_PSIUMHFR0(si_idx, err), - lower_32_bits(hash)); - enetc_port_wr(&si->hw, ENETC_PSIUMHFR1(si_idx), - upper_32_bits(hash)); - } else { /* MC */ - enetc_port_wr(&si->hw, ENETC_PSIMMHFR0(si_idx, err), - lower_32_bits(hash)); - enetc_port_wr(&si->hw, ENETC_PSIMMHFR1(si_idx), - upper_32_bits(hash)); - } -} - static void enetc_sync_mac_filters(struct enetc_pf *pf) { struct enetc_mac_filter *f = pf->mac_filter; @@ -122,12 +91,16 @@ static void enetc_sync_mac_filters(struct enetc_pf *pf) for (i = 0; i < MADDR_TYPE; i++, f++) { bool em = (f->mac_addr_cnt == 1) && (i == UC); bool clear = !f->mac_addr_cnt; + u64 hash; if (clear) { - if (i == UC) + if (i == UC) { enetc_clear_mac_flt_entry(si, pos); + enetc_set_si_uc_hash_filter(si, 0, 0); + } else { + enetc_set_si_mc_hash_filter(si, 0, 0); + } - enetc_clear_mac_ht_flt(si, 0, i); continue; } @@ -135,7 +108,7 @@ static void enetc_sync_mac_filters(struct enetc_pf *pf) if (em) { int err; - enetc_clear_mac_ht_flt(si, 0, UC); + enetc_set_si_uc_hash_filter(si, 0, 0); err = enetc_set_mac_flt_entry(si, pos, f->mac_addr, BIT(0)); @@ -147,11 +120,15 @@ static void enetc_sync_mac_filters(struct enetc_pf *pf) err); } + bitmap_to_arr64(&hash, f->mac_hash_table, + ENETC_MADDR_HASH_TBL_SZ); /* hash table filter, clear EM filter for UC entries */ - if (i == UC) + if (i == UC) { enetc_clear_mac_flt_entry(si, pos); - - enetc_set_mac_ht_flt(si, 0, i, *f->mac_hash_table); + enetc_set_si_uc_hash_filter(si, 0, hash); + } else { + enetc_set_si_mc_hash_filter(si, 0, hash); + } } } @@ -159,21 +136,17 @@ static void enetc_pf_set_rx_mode(struct net_device *ndev) { struct enetc_ndev_priv *priv = netdev_priv(ndev); struct enetc_pf *pf = enetc_si_priv(priv->si); - struct enetc_hw *hw = &priv->si->hw; bool uprom = false, mprom = false; struct enetc_mac_filter *filter; struct netdev_hw_addr *ha; - u32 psipmr = 0; bool em; if (ndev->flags & IFF_PROMISC) { /* enable promisc mode for SI0 (PF) */ - psipmr = ENETC_PSIPMR_SET_UP(0) | ENETC_PSIPMR_SET_MP(0); uprom = true; mprom = true; } else if (ndev->flags & IFF_ALLMULTI) { /* enable multi cast promisc mode for SI0 (PF) */ - psipmr = ENETC_PSIPMR_SET_MP(0); mprom = true; } @@ -211,9 +184,8 @@ static void enetc_pf_set_rx_mode(struct net_device *ndev) /* update PF entries */ enetc_sync_mac_filters(pf); - psipmr |= enetc_port_rd(hw, ENETC_PSIPMR) & - ~(ENETC_PSIPMR_SET_UP(0) | ENETC_PSIPMR_SET_MP(0)); - enetc_port_wr(hw, ENETC_PSIPMR, psipmr); + enetc_set_si_uc_promisc(priv->si, 0, uprom); + enetc_set_si_mc_promisc(priv->si, 0, mprom); } static void enetc_set_loopback(struct net_device *ndev, bool en) @@ -474,7 +446,7 @@ static void enetc_configure_port(struct enetc_pf *pf) pf->vlan_promisc_simap = ENETC_VLAN_PROMISC_MAP_ALL; enetc_set_vlan_promisc(hw, pf->vlan_promisc_simap); - enetc_port_wr(hw, ENETC_PSIPMR, 0); + enetc_port_wr(hw, ENETC_PSIPMMR, 0); /* enable port */ enetc_port_wr(hw, ENETC_PMR, ENETC_PMR_EN); @@ -608,8 +580,7 @@ static void enetc_pl_mac_link_up(struct phylink_config *config, struct enetc_hw *hw = &pf->si->hw; struct enetc_si *si = pf->si; struct enetc_ndev_priv *priv; - u32 rbmr, cmd_cfg; - int idx; + u32 cmd_cfg; priv = netdev_priv(pf->si->ndev); @@ -621,16 +592,7 @@ static void enetc_pl_mac_link_up(struct phylink_config *config, enetc_force_rgmii_mac(si, speed, duplex); /* Flow control */ - for (idx = 0; idx < priv->num_rx_rings; idx++) { - rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); - - if (tx_pause) - rbmr |= ENETC_RBMR_CM; - else - rbmr &= ~ENETC_RBMR_CM; - - enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); - } + enetc_set_congestion_mode(priv, tx_pause); if (tx_pause) { /* When the port first enters congestion, send a PAUSE request diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.h b/drivers/net/ethernet/freescale/enetc/enetc_pf.h index 285b7e5c48fdc..7e886dc49997f 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.h @@ -17,12 +17,10 @@ struct enetc_vf_state { }; struct enetc_port_caps { - u32 half_duplex:1; int num_vsi; int num_msix; int num_rx_bdr; int num_tx_bdr; - int mac_filter_num; }; struct enetc_pf; @@ -60,8 +58,6 @@ struct enetc_pf { struct enetc_port_caps caps; const struct enetc_pf_ops *ops; - - int num_mfe; /* number of mac address filter table entries */ }; #define phylink_to_enetc_pf(config) \ diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c index 6e5d2f8699150..3597cb81a7ccb 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c @@ -87,6 +87,90 @@ int enetc_setup_mac_addresses(struct device_node *np, struct enetc_pf *pf) } EXPORT_SYMBOL_GPL(enetc_setup_mac_addresses); +void enetc_set_si_uc_promisc(struct enetc_si *si, int si_id, bool promisc) +{ + struct enetc_hw *hw = &si->hw; + int psipmmr_off; + u32 val; + + if (is_enetc_rev1(si)) + psipmmr_off = ENETC_PSIPMMR; + else + psipmmr_off = ENETC4_PSIPMMR; + + val = enetc_port_rd(hw, psipmmr_off); + + if (promisc) + val |= PSIPMMR_SI_MAC_UP(si_id); + else + val &= ~PSIPMMR_SI_MAC_UP(si_id); + + enetc_port_wr(hw, psipmmr_off, val); +} +EXPORT_SYMBOL_GPL(enetc_set_si_uc_promisc); + +void enetc_set_si_mc_promisc(struct enetc_si *si, int si_id, bool promisc) +{ + struct enetc_hw *hw = &si->hw; + int psipmmr_off; + u32 val; + + if (is_enetc_rev1(si)) + psipmmr_off = ENETC_PSIPMMR; + else + psipmmr_off = ENETC4_PSIPMMR; + + val = enetc_port_rd(hw, psipmmr_off); + + if (promisc) + val |= PSIPMMR_SI_MAC_MP(si_id); + else + val &= ~PSIPMMR_SI_MAC_MP(si_id); + + enetc_port_wr(hw, psipmmr_off, val); +} +EXPORT_SYMBOL_GPL(enetc_set_si_mc_promisc); + +void enetc_set_si_uc_hash_filter(struct enetc_si *si, int si_id, u64 hash) +{ + int psiumhfr0_off, psiumhfr1_off; + struct enetc_hw *hw = &si->hw; + + if (is_enetc_rev1(si)) { + bool err = si->errata & ENETC_ERR_UCMCSWP; + + psiumhfr0_off = ENETC_PSIUMHFR0(si_id, err); + psiumhfr1_off = ENETC_PSIUMHFR1(si_id); + } else { + psiumhfr0_off = ENETC4_PSIUMHFR0(si_id); + psiumhfr1_off = ENETC4_PSIUMHFR1(si_id); + } + + enetc_port_wr(hw, psiumhfr0_off, lower_32_bits(hash)); + enetc_port_wr(hw, psiumhfr1_off, upper_32_bits(hash)); +} +EXPORT_SYMBOL_GPL(enetc_set_si_uc_hash_filter); + +void enetc_set_si_mc_hash_filter(struct enetc_si *si, int si_id, u64 hash) +{ + int psimmhfr0_off, psimmhfr1_off; + struct enetc_hw *hw = &si->hw; + + if (is_enetc_rev1(si)) { + bool err = si->errata & ENETC_ERR_UCMCSWP; + + psimmhfr0_off = ENETC_PSIMMHFR0(si_id, err); + psimmhfr1_off = ENETC_PSIMMHFR1(si_id); + } else { + psimmhfr0_off = ENETC4_PSIMMHFR0(si_id); + psimmhfr1_off = ENETC4_PSIMMHFR1(si_id); + } + + enetc_port_wr(hw, psimmhfr0_off, lower_32_bits(hash)); + enetc_port_wr(hw, psimmhfr1_off, upper_32_bits(hash)); +} +EXPORT_SYMBOL_GPL(enetc_set_si_mc_hash_filter); + void enetc_pf_netdev_setup(struct enetc_si *si, struct net_device *ndev, const struct net_device_ops *ndev_ops) { diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h index 57d2e0ebd2b0b..bf9029b0a0173 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h @@ -17,6 +17,10 @@ void enetc_set_default_rss_key(struct enetc_pf *pf); int enetc_vlan_rx_add_vid(struct net_device *ndev, __be16 prot, u16 vid); int enetc_vlan_rx_del_vid(struct net_device *ndev, __be16 prot, u16 vid); int enetc_init_sriov_resources(struct enetc_pf *pf); +void enetc_set_si_uc_promisc(struct enetc_si *si, int si_id, bool promisc); +void enetc_set_si_mc_promisc(struct enetc_si *si, int si_id, bool promisc); +void enetc_set_si_uc_hash_filter(struct enetc_si *si, int si_id, u64 hash); +void enetc_set_si_mc_hash_filter(struct enetc_si *si, int si_id, u64 hash); static inline u16 enetc_get_ip_revision(struct enetc_hw *hw) { diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index ced4dbf8cd90f..794ec427b0ee6 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -5457,7 +5457,8 @@ fec_probe(struct platform_device *pdev) failed_irq: fec_enet_deinit(ndev); failed_init: - fec_ptp_stop(pdev); + if (fep->bufdesc_ex) + fec_ptp_stop(pdev); failed_reset: pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); @@ -5499,7 +5500,8 @@ fec_drv_remove(struct platform_device *pdev) ERR_PTR(ret)); cancel_work_sync(&fep->tx_timeout_work); - fec_ptp_stop(pdev); + if (fep->bufdesc_ex) + fec_ptp_stop(pdev); unregister_netdev(ndev); fec_enet_mii_remove(fep); if (fep->reg_phy) diff --git a/drivers/net/ethernet/freescale/fman/fman.c b/drivers/net/ethernet/freescale/fman/fman.c index 299bab043175b..46cc28895e565 100644 --- a/drivers/net/ethernet/freescale/fman/fman.c +++ b/drivers/net/ethernet/freescale/fman/fman.c @@ -2734,6 +2734,7 @@ static struct fman *read_dts_node(struct platform_device *of_dev) } clk_rate = clk_get_rate(clk); + clk_put(clk); if (!clk_rate) { err = -EINVAL; dev_err(&of_dev->dev, "%s: Failed to determine FM%d clock rate\n", diff --git a/drivers/net/ethernet/google/gve/gve_desc_dqo.h b/drivers/net/ethernet/google/gve/gve_desc_dqo.h index f7786b03c7444..d2c86c8eeae2a 100644 --- a/drivers/net/ethernet/google/gve/gve_desc_dqo.h +++ b/drivers/net/ethernet/google/gve/gve_desc_dqo.h @@ -14,6 +14,11 @@ #define GVE_TX_MAX_HDR_SIZE_DQO 255 #define GVE_TX_MIN_TSO_MSS_DQO 88 +/* HW limit. This also has to fit in the 14 bits of the mss field of + * struct gve_tx_tso_context_desc_dqo. + */ +#define GVE_TX_MAX_TSO_MSS_DQO 9728 + #ifndef __LITTLE_ENDIAN_BITFIELD #error "Only little endian supported" #endif diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c index 80ab0a449ff54..616c1921aebea 100644 --- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c @@ -577,15 +577,18 @@ static int gve_prep_tso(struct sk_buff *skb) int header_len; int err; - /* Note: HW requires MSS (gso_size) to be <= 9728 and the total length - * of the TSO to be <= 262143. + /* Note: HW requires the total length of the TSO to be <= 262143, + * this is enforced by netif_set_tso_max_size(). * - * However, we don't validate these because: - * - Hypervisor enforces a limit of 9K MTU - * - Kernel will not produce a TSO larger than 64k + * MSS (gso_size) can not be trusted: packets forwarded from a tap or + * injected by a packet socket can carry an arbitrary value, while the + * mss field of the TSO context descriptor is only 14 bits wide. + * + * A too big MSS is dropped here instead of being rejected from + * gve_features_check_dqo(), because software segmentation would + * produce packets larger than the device can send. */ - - if (unlikely(shinfo->gso_size < GVE_TX_MIN_TSO_MSS_DQO)) + if (unlikely(shinfo->gso_size > GVE_TX_MAX_TSO_MSS_DQO)) return -1; /* Needed because we will modify header. */ @@ -918,13 +921,22 @@ static bool gve_can_send_tso(const struct sk_buff *skb) { const int max_bufs_per_seg = GVE_TX_MAX_DATA_DESCS - 1; const struct skb_shared_info *shinfo = skb_shinfo(skb); - const int header_len = skb_tcp_all_headers(skb); const int gso_size = shinfo->gso_size; int cur_seg_num_bufs; int prev_frag_size; int cur_seg_size; + int header_len; int i; + if (unlikely(gso_size < GVE_TX_MIN_TSO_MSS_DQO)) + return false; + + /* Must match the header length programmed by gve_prep_tso(). */ + if (skb_is_gso_tcp(skb)) + header_len = skb_tcp_all_headers(skb); + else + header_len = skb_transport_offset(skb) + sizeof(struct udphdr); + cur_seg_size = skb_headlen(skb) - header_len; prev_frag_size = skb_headlen(skb); cur_seg_num_bufs = cur_seg_size > 0; @@ -966,7 +978,17 @@ netdev_features_t gve_features_check_dqo(struct sk_buff *skb, struct net_device *dev, netdev_features_t features) { - if (skb_is_gso(skb) && !gve_can_send_tso(skb)) + if (!skb_is_gso(skb)) + return features; + + /* Keep the GSO bits for a too big MSS, so that gve_prep_tso() drops + * the packet: software segmentation would give packets larger than + * the device can send. + */ + if (skb_shinfo(skb)->gso_size > GVE_TX_MAX_TSO_MSS_DQO) + return features; + + if (!gve_can_send_tso(skb)) return features & ~NETIF_F_GSO_MASK; return features; diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c b/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c index 0ae3149946769..3e6ae2aa0d4b4 100644 --- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c +++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c @@ -553,7 +553,7 @@ static int hbg_ring_page_pool_init(struct hbg_priv *priv, struct hbg_ring *ring) .nid = dev_to_node(&priv->pdev->dev), .dev = &priv->pdev->dev, .napi = &ring->napi, - .dma_dir = DMA_FROM_DEVICE, + .dma_dir = DMA_BIDIRECTIONAL, .offset = 0, .max_len = hbg_get_page_size(ring), }; diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c index bc6b269be2995..f1cb6d56e4b9f 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c @@ -793,6 +793,7 @@ static int hns_mac_register_phy(struct hns_mac_cb *mac_cb) dev_err(mac_cb->dev, "mac%d mdio is NULL, dsaf will probe again later\n", mac_cb->mac_id); + put_device(&pdev->dev); return -EPROBE_DEFER; } @@ -801,6 +802,8 @@ static int hns_mac_register_phy(struct hns_mac_cb *mac_cb) dev_dbg(mac_cb->dev, "mac%d register phy addr:%d\n", mac_cb->mac_id, addr); + put_device(&pdev->dev); + return rc; } diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c index 2784127327e64..6e67a6c9578e1 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c @@ -128,6 +128,7 @@ enum hinic_mbox_tx_status { #define SEQ_ID_START_VAL 0 #define SEQ_ID_MAX_VAL 42 +#define MBOX_LAST_SEG_MAX_LEN (MBOX_MAX_BUF_SZ - SEQ_ID_MAX_VAL * MBOX_SEG_LEN) #define NO_DMA_ATTRIBUTE_VAL 0 @@ -372,7 +373,8 @@ recv_pf_from_vf_mbox_handler(struct hinic_mbox_func_to_func *func_to_func, static bool check_mbox_seq_id_and_seg_len(struct hinic_recv_mbox *recv_mbox, u8 seq_id, u8 seg_len) { - if (seq_id > SEQ_ID_MAX_VAL || seg_len > MBOX_SEG_LEN) + if (seq_id > SEQ_ID_MAX_VAL || seg_len > MBOX_SEG_LEN || + (seq_id == SEQ_ID_MAX_VAL && seg_len > MBOX_LAST_SEG_MAX_LEN)) return false; if (seq_id == 0) { diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c b/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c index 9306bf0020caf..cc541e7a23186 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c @@ -261,8 +261,7 @@ static int hinic3_tx_csum(struct hinic3_txq *txq, struct hinic3_sq_task *task, ((struct udphdr *)skb_transport_header(skb))->dest != VXLAN_OFFLOAD_PORT_LE) { /* Unsupported tunnel packet, disable csum offload */ - skb_checksum_help(skb); - return 0; + return skb_checksum_help(skb); } } @@ -412,6 +411,10 @@ static u32 hinic3_tx_offload(struct sk_buff *skb, struct hinic3_sq_task *task, offload |= HINIC3_TX_OFFLOAD_TSO; } else { tso_cs_en = hinic3_tx_csum(txq, task, skb); + if (tso_cs_en < 0) { + offload = HINIC3_TX_OFFLOAD_INVALID; + return offload; + } if (tso_cs_en) offload |= HINIC3_TX_OFFLOAD_CSUM; } @@ -545,6 +548,7 @@ static netdev_tx_t hinic3_send_one_skb(struct sk_buff *skb, skb->len = MIN_SKB_LEN; } + offload = hinic3_tx_offload(skb, &task, &queue_info, txq); num_sge = skb_shinfo(skb)->nr_frags + 1; /* assume normal wqe format + 1 wqebb for task info */ wqebb_cnt = num_sge + 1; @@ -560,7 +564,6 @@ static netdev_tx_t hinic3_send_one_skb(struct sk_buff *skb, return NETDEV_TX_BUSY; } - offload = hinic3_tx_offload(skb, &task, &queue_info, txq); if (unlikely(offload == HINIC3_TX_OFFLOAD_INVALID)) { goto err_drop_pkt; } else if (!offload) { diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c index 1d46cf6c2c127..e7043523457c1 100644 --- a/drivers/net/ethernet/ibm/emac/core.c +++ b/drivers/net/ethernet/ibm/emac/core.c @@ -3044,6 +3044,15 @@ static int emac_probe(struct platform_device *ofdev) if (err) goto err_gone; + if (emac_phy_supports_gige(dev->phy_mode)) { + ndev->netdev_ops = &emac_gige_netdev_ops; + dev->commac.ops = &emac_commac_sg_ops; + } else { + ndev->netdev_ops = &emac_netdev_ops; + dev->commac.ops = &emac_commac_ops; + } + ndev->ethtool_ops = &emac_ethtool_ops; + dev->emacp = devm_platform_ioremap_resource(ofdev, 0); if (IS_ERR(dev->emacp)) { err = PTR_ERR(dev->emacp); @@ -3076,7 +3085,6 @@ static int emac_probe(struct platform_device *ofdev) dev->mdio_instance = platform_get_drvdata(dev->mdio_dev); /* Register with MAL */ - dev->commac.ops = &emac_commac_ops; dev->commac.dev = dev; dev->commac.tx_chan_mask = MAL_CHAN_MASK(dev->mal_tx_chan); dev->commac.rx_chan_mask = MAL_CHAN_MASK(dev->mal_rx_chan); @@ -3144,12 +3152,6 @@ static int emac_probe(struct platform_device *ofdev) ndev->features |= ndev->hw_features | NETIF_F_RXCSUM; } ndev->watchdog_timeo = 5 * HZ; - if (emac_phy_supports_gige(dev->phy_mode)) { - ndev->netdev_ops = &emac_gige_netdev_ops; - dev->commac.ops = &emac_commac_sg_ops; - } else - ndev->netdev_ops = &emac_netdev_ops; - ndev->ethtool_ops = &emac_ethtool_ops; /* MTU range: 46 - 1500 or whatever is in OF */ ndev->min_mtu = EMAC_MIN_MTU; diff --git a/drivers/net/ethernet/ibm/emac/mal.c b/drivers/net/ethernet/ibm/emac/mal.c index 74526002d52bd..42027665f2a93 100644 --- a/drivers/net/ethernet/ibm/emac/mal.c +++ b/drivers/net/ethernet/ibm/emac/mal.c @@ -35,6 +35,7 @@ int mal_register_commac(struct mal_instance *mal, struct mal_commac *commac) { unsigned long flags; + netdev_lock(mal->napi.dev); spin_lock_irqsave(&mal->lock, flags); MAL_DBG(mal, "reg(%08x, %08x)" NL, @@ -44,18 +45,20 @@ int mal_register_commac(struct mal_instance *mal, struct mal_commac *commac) if ((mal->tx_chan_mask & commac->tx_chan_mask) || (mal->rx_chan_mask & commac->rx_chan_mask)) { spin_unlock_irqrestore(&mal->lock, flags); + netdev_unlock(mal->napi.dev); printk(KERN_WARNING "mal%d: COMMAC channels conflict!\n", mal->index); return -EBUSY; } if (list_empty(&mal->list)) - napi_enable(&mal->napi); + napi_enable_locked(&mal->napi); mal->tx_chan_mask |= commac->tx_chan_mask; mal->rx_chan_mask |= commac->rx_chan_mask; list_add(&commac->list, &mal->list); spin_unlock_irqrestore(&mal->lock, flags); + netdev_unlock(mal->napi.dev); return 0; } @@ -64,7 +67,9 @@ void mal_unregister_commac(struct mal_instance *mal, struct mal_commac *commac) { unsigned long flags; + bool disable_napi; + netdev_lock(mal->napi.dev); spin_lock_irqsave(&mal->lock, flags); MAL_DBG(mal, "unreg(%08x, %08x)" NL, @@ -73,10 +78,12 @@ void mal_unregister_commac(struct mal_instance *mal, mal->tx_chan_mask &= ~commac->tx_chan_mask; mal->rx_chan_mask &= ~commac->rx_chan_mask; list_del_init(&commac->list); - if (list_empty(&mal->list)) - napi_disable(&mal->napi); + disable_napi = list_empty(&mal->list); spin_unlock_irqrestore(&mal->lock, flags); + if (disable_napi) + napi_disable_locked(&mal->napi); + netdev_unlock(mal->napi.dev); } int mal_set_rcbs(struct mal_instance *mal, int channel, unsigned long size) diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink.c b/drivers/net/ethernet/intel/ice/devlink/devlink.c index 22b7d8e6bd9e8..8c2b63eef82bd 100644 --- a/drivers/net/ethernet/intel/ice/devlink/devlink.c +++ b/drivers/net/ethernet/intel/ice/devlink/devlink.c @@ -1890,27 +1890,18 @@ static int ice_devlink_nvm_snapshot(struct devlink *devlink, */ for (i = 0; i < num_blks; i++) { u32 read_sz = min_t(u32, ICE_DEVLINK_READ_BLK_SIZE, left); - - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) { - dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n", - status, hw->adminq.sq_last_status); - NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore"); - vfree(nvm_data); - return -EIO; - } + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; status = ice_read_flat_nvm(hw, i * ICE_DEVLINK_READ_BLK_SIZE, - &read_sz, tmp, read_shadow_ram); + &read_sz, tmp, read_shadow_ram, + &read_aq_err); if (status) { dev_dbg(dev, "ice_read_flat_nvm failed after reading %u bytes, err %d aq_err %d\n", - read_sz, status, hw->adminq.sq_last_status); + read_sz, status, read_aq_err); NL_SET_ERR_MSG_MOD(extack, "Failed to read NVM contents"); - ice_release_nvm(hw); vfree(nvm_data); return -EIO; } - ice_release_nvm(hw); tmp += read_sz; left -= read_sz; @@ -1943,6 +1934,7 @@ static int ice_devlink_nvm_read(struct devlink *devlink, struct netlink_ext_ack *extack, u64 offset, u32 size, u8 *data) { + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; struct ice_pf *pf = devlink_priv(devlink); struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; @@ -1966,24 +1958,14 @@ static int ice_devlink_nvm_read(struct devlink *devlink, return -ERANGE; } - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) { - dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n", - status, hw->adminq.sq_last_status); - NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore"); - return -EIO; - } - status = ice_read_flat_nvm(hw, (u32)offset, &size, data, - read_shadow_ram); + read_shadow_ram, &read_aq_err); if (status) { dev_dbg(dev, "ice_read_flat_nvm failed after reading %u bytes, err %d aq_err %d\n", - size, status, hw->adminq.sq_last_status); + size, status, read_aq_err); NL_SET_ERR_MSG_MOD(extack, "Failed to read NVM contents"); - ice_release_nvm(hw); return -EIO; } - ice_release_nvm(hw); return 0; } diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index ef1ce106f81b5..04633103e3e61 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -1051,14 +1051,13 @@ int ice_init_hw(struct ice_hw *hw) hw->evb_veb = true; - /* init xarray for identifying scheduling nodes uniquely */ - xa_init_flags(&hw->port_info->sched_node_ids, XA_FLAGS_ALLOC); + xa_init_flags(&hw->sched_node_ids, XA_FLAGS_ALLOC); /* Query the allocated resources for Tx scheduler */ status = ice_sched_query_res_alloc(hw); if (status) { ice_debug(hw, ICE_DBG_SCHED, "Failed to get scheduler allocated resources\n"); - goto err_unroll_alloc; + goto err_unroll_xarray; } ice_sched_get_psm_clk_freq(hw); @@ -1146,6 +1145,8 @@ int ice_init_hw(struct ice_hw *hw) ice_cleanup_fltr_mgmt_struct(hw); err_unroll_sched: ice_sched_cleanup_all(hw); +err_unroll_xarray: + xa_destroy(&hw->sched_node_ids); err_unroll_alloc: devm_kfree(ice_hw_to_dev(hw), hw->port_info); err_unroll_cqinit: @@ -1186,6 +1187,8 @@ void ice_deinit_hw(struct ice_hw *hw) /* Clear VSI contexts if not already cleared */ ice_clear_all_vsi_ctx(hw); + + xa_destroy(&hw->sched_node_ids); } /** diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 7eb380be7ed27..bf9a821c543b7 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -853,6 +853,7 @@ static int ice_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom, u8 *bytes) { + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; struct ice_pf *pf = ice_netdev_to_pf(netdev); struct ice_hw *hw = &pf->hw; struct device *dev; @@ -869,24 +870,15 @@ ice_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom, if (!buf) return -ENOMEM; - ret = ice_acquire_nvm(hw, ICE_RES_READ); - if (ret) { - dev_err(dev, "ice_acquire_nvm failed, err %d aq_err %s\n", - ret, libie_aq_str(hw->adminq.sq_last_status)); - goto out; - } - ret = ice_read_flat_nvm(hw, eeprom->offset, &eeprom->len, buf, - false); + false, &read_aq_err); if (ret) { dev_err(dev, "ice_read_flat_nvm failed, err %d aq_err %s\n", - ret, libie_aq_str(hw->adminq.sq_last_status)); - goto release; + ret, libie_aq_str(read_aq_err)); + goto out; } memcpy(bytes, buf, eeprom->len); -release: - ice_release_nvm(hw); out: kfree(buf); return ret; diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 8cdc4fda89e97..9e08db376d3d0 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -2871,6 +2871,9 @@ int ice_vsi_release(struct ice_vsi *vsi) return -ENODEV; pf = vsi->back; + if (ice_is_vsi_dflt_vsi(vsi)) + ice_clear_dflt_vsi(vsi); + if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) ice_rss_clean(vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.c b/drivers/net/ethernet/intel/ice/ice_nvm.c index 7e187a804dfa1..21f3b615dbbf1 100644 --- a/drivers/net/ethernet/intel/ice/ice_nvm.c +++ b/drivers/net/ethernet/intel/ice/ice_nvm.c @@ -53,17 +53,27 @@ int ice_aq_read_nvm(struct ice_hw *hw, u16 module_typeid, u32 offset, * @length: (in) number of bytes to read; (out) number of bytes actually read * @data: buffer to return data in (sized to fit the specified length) * @read_shadow_ram: if true, read from shadow RAM instead of NVM + * @read_aq_err: if non-NULL, receives the AQ error status of the failing read * * Reads a portion of the NVM, as a flat memory space. This function correctly * breaks read requests across Shadow RAM sectors and ensures that no single * read request exceeds the maximum 4KB read for a single AdminQ command. * + * FW caps the read lock at a maximum of 3000ms, so a read spanning multiple + * 4KB sectors cannot be done under a single lock without FW reclaiming it + * mid-read. The NVM lock is therefore acquired and released around each AQ + * read, so this function must be called without the lock held. + * + * Since ice_release_nvm() issues an AQ command that overwrites + * hw->adminq.sq_last_status, callers that need the failing read's AQ error + * must use @read_aq_err rather than inspecting sq_last_status afterwards. + * * Returns a status code on failure. Note that the data pointer may be * partially updated if some reads succeed before a failure. */ int ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, - bool read_shadow_ram) + bool read_shadow_ram, enum libie_aq_err *read_aq_err) { u32 inlen = *length; u32 bytes_read = 0; @@ -92,12 +102,30 @@ ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, last_cmd = !(bytes_read + read_size < inlen); + status = ice_acquire_nvm(hw, ICE_RES_READ); + if (status) { + ice_debug(hw, ICE_DBG_NVM, "Failed to acquire NVM lock, err %d aq_err %s\n", + status, libie_aq_str(hw->adminq.sq_last_status)); + break; + } + status = ice_aq_read_nvm(hw, ICE_AQC_NVM_START_POINT, offset, read_size, data + bytes_read, last_cmd, read_shadow_ram, NULL); - if (status) + if (status) { + /* Capture the read's AQ error before ice_release_nvm() + * issues its own AQ command and overwrites + * sq_last_status. + */ + if (read_aq_err) + *read_aq_err = hw->adminq.sq_last_status; + + ice_release_nvm(hw); break; + } + + ice_release_nvm(hw); bytes_read += read_size; offset += read_size; @@ -177,14 +205,19 @@ int ice_aq_erase_nvm(struct ice_hw *hw, u16 module_typeid, struct ice_sq_cd *cd) } /** - * ice_read_sr_word_aq - Reads Shadow RAM via AQ + * ice_read_sr_word - Reads Shadow RAM word * @hw: pointer to the HW structure * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF) * @data: word read from the Shadow RAM * * Reads one 16 bit word from the Shadow RAM using ice_read_flat_nvm. + * + * The NVM lock is acquired and released internally by ice_read_flat_nvm() + * around the FW read, so this function must be called without the lock held. + * + * Return: zero on success, or a negative error code on failure. */ -static int ice_read_sr_word_aq(struct ice_hw *hw, u16 offset, u16 *data) +int ice_read_sr_word(struct ice_hw *hw, u16 offset, u16 *data) { u32 bytes = sizeof(u16); __le16 data_local; @@ -194,7 +227,7 @@ static int ice_read_sr_word_aq(struct ice_hw *hw, u16 offset, u16 *data) * Shadow RAM sector restrictions necessary when reading from the NVM. */ status = ice_read_flat_nvm(hw, offset * sizeof(u16), &bytes, - (__force u8 *)&data_local, true); + (__force u8 *)&data_local, true, NULL); if (status) return status; @@ -330,13 +363,8 @@ ice_read_flash_module(struct ice_hw *hw, enum ice_bank_select bank, u16 module, return -EINVAL; } - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) - return status; - - status = ice_read_flat_nvm(hw, start + offset, &length, data, false); - - ice_release_nvm(hw); + status = ice_read_flat_nvm(hw, start + offset, &length, data, false, + NULL); return status; } @@ -418,27 +446,6 @@ ice_read_netlist_module(struct ice_hw *hw, enum ice_bank_select bank, u32 offset return status; } -/** - * ice_read_sr_word - Reads Shadow RAM word and acquire NVM if necessary - * @hw: pointer to the HW structure - * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF) - * @data: word read from the Shadow RAM - * - * Reads one 16 bit word from the Shadow RAM using the ice_read_sr_word_aq. - */ -int ice_read_sr_word(struct ice_hw *hw, u16 offset, u16 *data) -{ - int status; - - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (!status) { - status = ice_read_sr_word_aq(hw, offset, data); - ice_release_nvm(hw); - } - - return status; -} - /** * ice_get_pfa_module_tlv - Reads sub module TLV from NVM PFA * @hw: pointer to hardware structure @@ -856,20 +863,18 @@ int ice_get_inactive_netlist_ver(struct ice_hw *hw, struct ice_netlist_info *net static int ice_discover_flash_size(struct ice_hw *hw) { u32 min_size = 0, max_size = ICE_AQC_NVM_MAX_OFFSET + 1; - int status; - - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) - return status; + int status = 0; while ((max_size - min_size) > 1) { + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; u32 offset = (max_size + min_size) / 2; u32 len = 1; u8 data; - status = ice_read_flat_nvm(hw, offset, &len, &data, false); + status = ice_read_flat_nvm(hw, offset, &len, &data, false, + &read_aq_err); if (status == -EIO && - hw->adminq.sq_last_status == LIBIE_AQ_RC_EINVAL) { + read_aq_err == LIBIE_AQ_RC_EINVAL) { ice_debug(hw, ICE_DBG_NVM, "%s: New upper bound of %u bytes\n", __func__, offset); status = 0; @@ -880,7 +885,7 @@ static int ice_discover_flash_size(struct ice_hw *hw) min_size = offset; } else { /* an unexpected error occurred */ - goto err_read_flat_nvm; + return status; } } @@ -888,9 +893,6 @@ static int ice_discover_flash_size(struct ice_hw *hw) hw->flash.flash_size = max_size; -err_read_flat_nvm: - ice_release_nvm(hw); - return status; } diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.h b/drivers/net/ethernet/intel/ice/ice_nvm.h index 63cdc6bdac589..e1d1a11f5ca41 100644 --- a/drivers/net/ethernet/intel/ice/ice_nvm.h +++ b/drivers/net/ethernet/intel/ice/ice_nvm.h @@ -19,7 +19,7 @@ int ice_aq_read_nvm(struct ice_hw *hw, u16 module_typeid, u32 offset, bool read_shadow_ram, struct ice_sq_cd *cd); int ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, - bool read_shadow_ram); + bool read_shadow_ram, enum libie_aq_err *read_aq_err); int ice_get_pfa_module_tlv(struct ice_hw *hw, u16 *module_tlv, u16 *module_tlv_len, u16 module_type); diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c index 8e5f97835954c..3a41c711e751b 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c @@ -4808,15 +4808,12 @@ static int ice_ptp_prep_phy_adj_ll_e810(struct ice_hw *hw, s32 adj) !FIELD_GET(REG_LL_PROXY_H_EXEC, val), 10, REG_LL_PROXY_H_TIMEOUT_US, false, hw, REG_LL_PROXY_H); - if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer adjustment using low latency interface\n"); - spin_unlock_irq(¶ms->atqbal_wq.lock); - return err; - } - spin_unlock_irq(¶ms->atqbal_wq.lock); - return 0; + if (err) + ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer adjustment using low latency interface\n"); + + return err; } /** @@ -4837,8 +4834,12 @@ static int ice_ptp_prep_phy_adj_e810(struct ice_hw *hw, s32 adj) u8 tmr_idx; int err; - if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) - return ice_ptp_prep_phy_adj_ll_e810(hw, adj); + if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) { + err = ice_ptp_prep_phy_adj_ll_e810(hw, adj); + if (err != -ETIMEDOUT) + return err; + ice_debug(hw, ICE_DBG_PTP, "LL adj timed out, falling back to SBQ\n"); + } tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; @@ -4901,15 +4902,12 @@ static int ice_ptp_prep_phy_incval_ll_e810(struct ice_hw *hw, u64 incval) !FIELD_GET(REG_LL_PROXY_H_EXEC, val), 10, REG_LL_PROXY_H_TIMEOUT_US, false, hw, REG_LL_PROXY_H); - if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer increment using low latency interface\n"); - spin_unlock_irq(¶ms->atqbal_wq.lock); - return err; - } - spin_unlock_irq(¶ms->atqbal_wq.lock); - return 0; + if (err) + ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer increment using low latency interface\n"); + + return err; } /** @@ -4927,8 +4925,12 @@ static int ice_ptp_prep_phy_incval_e810(struct ice_hw *hw, u64 incval) u8 tmr_idx; int err; - if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) - return ice_ptp_prep_phy_incval_ll_e810(hw, incval); + if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) { + err = ice_ptp_prep_phy_incval_ll_e810(hw, incval); + if (err != -ETIMEDOUT) + return err; + ice_debug(hw, ICE_DBG_PTP, "LL incval timed out, falling back to SBQ\n"); + } tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; low = lower_32_bits(incval); diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c index fff0c1afdb414..ffa18d86729a0 100644 --- a/drivers/net/ethernet/intel/ice/ice_sched.c +++ b/drivers/net/ethernet/intel/ice/ice_sched.c @@ -371,7 +371,7 @@ void ice_free_sched_node(struct ice_port_info *pi, struct ice_sched_node *node) devm_kfree(ice_hw_to_dev(hw), node->children); kfree(node->name); - xa_erase(&pi->sched_node_ids, node->id); + xa_erase(&hw->sched_node_ids, node->id); devm_kfree(ice_hw_to_dev(hw), node); } @@ -977,7 +977,7 @@ ice_sched_add_elems(struct ice_port_info *pi, struct ice_sched_node *tc_node, if (!new_node->name) return -ENOMEM; - status = xa_alloc(&pi->sched_node_ids, &new_node->id, NULL, XA_LIMIT(0, UINT_MAX), + status = xa_alloc(&hw->sched_node_ids, &new_node->id, NULL, XA_LIMIT(0, UINT_MAX), GFP_KERNEL); if (status) { ice_debug(hw, ICE_DBG_SCHED, "xa_alloc failed for sched node status =%d\n", diff --git a/drivers/net/ethernet/intel/ice/ice_trace.h b/drivers/net/ethernet/intel/ice/ice_trace.h index 4f35ef8d6b299..7568c917cdbe1 100644 --- a/drivers/net/ethernet/intel/ice/ice_trace.h +++ b/drivers/net/ethernet/intel/ice/ice_trace.h @@ -63,23 +63,33 @@ DECLARE_EVENT_CLASS(ice_rx_dim_template, TP_PROTO(struct ice_q_vector *q_vector, struct dim *dim), TP_ARGS(q_vector, dim), - TP_STRUCT__entry(__field(struct ice_q_vector *, q_vector) - __field(struct dim *, dim) + TP_STRUCT__entry(__field(u16, q_index) + __field(u8, state) + __field(u8, profile_ix) + __field(u8, tune_state) + __field(u8, steps_right) + __field(u8, steps_left) + __field(u8, tired) __string(devname, q_vector->rx.rx_ring->netdev->name)), - TP_fast_assign(__entry->q_vector = q_vector; - __entry->dim = dim; + TP_fast_assign(__entry->q_index = q_vector->rx.rx_ring->q_index; + __entry->state = dim->state; + __entry->profile_ix = dim->profile_ix; + __entry->tune_state = dim->tune_state; + __entry->steps_right = dim->steps_right; + __entry->steps_left = dim->steps_left; + __entry->tired = dim->tired; __assign_str(devname);), TP_printk("netdev: %s Rx-Q: %d dim-state: %d dim-profile: %d dim-tune: %d dim-st-right: %d dim-st-left: %d dim-tired: %d", __get_str(devname), - __entry->q_vector->rx.rx_ring->q_index, - __entry->dim->state, - __entry->dim->profile_ix, - __entry->dim->tune_state, - __entry->dim->steps_right, - __entry->dim->steps_left, - __entry->dim->tired) + __entry->q_index, + __entry->state, + __entry->profile_ix, + __entry->tune_state, + __entry->steps_right, + __entry->steps_left, + __entry->tired) ); DEFINE_EVENT(ice_rx_dim_template, ice_rx_dim_work, @@ -90,23 +100,33 @@ DEFINE_EVENT(ice_rx_dim_template, ice_rx_dim_work, DECLARE_EVENT_CLASS(ice_tx_dim_template, TP_PROTO(struct ice_q_vector *q_vector, struct dim *dim), TP_ARGS(q_vector, dim), - TP_STRUCT__entry(__field(struct ice_q_vector *, q_vector) - __field(struct dim *, dim) + TP_STRUCT__entry(__field(u16, q_index) + __field(u8, state) + __field(u8, profile_ix) + __field(u8, tune_state) + __field(u8, steps_right) + __field(u8, steps_left) + __field(u8, tired) __string(devname, q_vector->tx.tx_ring->netdev->name)), - TP_fast_assign(__entry->q_vector = q_vector; - __entry->dim = dim; + TP_fast_assign(__entry->q_index = q_vector->tx.tx_ring->q_index; + __entry->state = dim->state; + __entry->profile_ix = dim->profile_ix; + __entry->tune_state = dim->tune_state; + __entry->steps_right = dim->steps_right; + __entry->steps_left = dim->steps_left; + __entry->tired = dim->tired; __assign_str(devname);), TP_printk("netdev: %s Tx-Q: %d dim-state: %d dim-profile: %d dim-tune: %d dim-st-right: %d dim-st-left: %d dim-tired: %d", __get_str(devname), - __entry->q_vector->tx.tx_ring->q_index, - __entry->dim->state, - __entry->dim->profile_ix, - __entry->dim->tune_state, - __entry->dim->steps_right, - __entry->dim->steps_left, - __entry->dim->tired) + __entry->q_index, + __entry->state, + __entry->profile_ix, + __entry->tune_state, + __entry->steps_right, + __entry->steps_left, + __entry->tired) ); DEFINE_EVENT(ice_tx_dim_template, ice_tx_dim_work, diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index d9a5c1aae7c23..cf147a2127071 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -765,7 +765,6 @@ struct ice_port_info { /* List contain profile ID(s) and other params per layer */ struct list_head rl_prof_list[ICE_AQC_TOPO_MAX_LEVEL_NUM]; struct ice_qos_cfg qos_cfg; - struct xarray sched_node_ids; u8 is_vf:1; u8 is_custom_tx_enabled:1; }; @@ -930,6 +929,7 @@ struct ice_hw { u8 sw_entry_point_layer; u16 max_children[ICE_AQC_TOPO_MAX_LEVEL_NUM]; struct list_head agg_list; /* lists all aggregator */ + struct xarray sched_node_ids; struct ice_vsi_ctx *vsi_ctx[ICE_MAX_VSI]; u8 evb_veb; /* true for VEB, false for VEPA */ diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index c724d429a7aa9..81e83a98a8e2c 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -2408,7 +2408,7 @@ void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, struct idpf_tx_splitq_params *params, u16 td_cmd, u16 size) { - *(u32 *)&desc->flow.qw1.cmd_dtype = (u8)(params->dtype | td_cmd); + *(__le32 *)&desc->flow.qw1.cmd_dtype = cpu_to_le32((u8)(params->dtype | td_cmd)); desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size); desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag); } @@ -3299,6 +3299,7 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, struct libeth_rx_pt decoded) { u16 rsc_segments, rsc_seg_len; + u16 l3_start = 0; bool ipv4, ipv6; int len; @@ -3321,7 +3322,10 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, NAPI_GRO_CB(skb)->count = rsc_segments; skb_shinfo(skb)->gso_size = rsc_seg_len; - skb_reset_network_header(skb); + if (unlikely(eth_type_vlan(skb->protocol))) + l3_start = VLAN_HLEN; + + skb_set_network_header(skb, l3_start); if (ipv4) { struct iphdr *ipv4h = ip_hdr(skb); @@ -3329,7 +3333,7 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; /* Reset and set transport header offset in skb */ - skb_set_transport_header(skb, sizeof(struct iphdr)); + skb_set_transport_header(skb, l3_start + sizeof(struct iphdr)); len = skb->len - skb_transport_offset(skb); /* Compute the TCP pseudo header checksum*/ @@ -3339,7 +3343,7 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, struct ipv6hdr *ipv6h = ipv6_hdr(skb); skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; - skb_set_transport_header(skb, sizeof(struct ipv6hdr)); + skb_set_transport_header(skb, l3_start + sizeof(struct ipv6hdr)); len = skb->len - skb_transport_offset(skb); tcp_hdr(skb)->check = ~tcp_v6_check(len, &ipv6h->saddr, &ipv6h->daddr, 0); @@ -4145,6 +4149,26 @@ static void idpf_vport_intr_ena_irq_all(struct idpf_vport *vport, writel(rsrc->noirq_dyn_ctl_ena, rsrc->noirq_dyn_ctl); } +/** + * idpf_vport_intr_dis_dim_all - Disable DIM work for all q_vectors + * @rsrc: pointer to queue and vector resources + * + * The DIM works are embedded in the q_vector array that + * idpf_vport_intr_rel() frees, and the poll arms them after + * napi_complete_done() has already cleared NAPI_STATE_SCHED. Disable + * rather than just cancel, so that a poll tail still running past + * napi_disable() cannot queue them again behind the drain. + */ +static void idpf_vport_intr_dis_dim_all(struct idpf_q_vec_rsrc *rsrc) +{ + for (u16 v_idx = 0; v_idx < rsrc->num_q_vectors; v_idx++) { + struct idpf_q_vector *q_vector = &rsrc->q_vectors[v_idx]; + + disable_work_sync(&q_vector->tx_dim.work); + disable_work_sync(&q_vector->rx_dim.work); + } +} + /** * idpf_vport_intr_deinit - Release all vector associations for the vport * @vport: main vport structure @@ -4155,6 +4179,7 @@ void idpf_vport_intr_deinit(struct idpf_vport *vport, { idpf_vport_intr_dis_irq_all(rsrc); idpf_vport_intr_napi_dis_all(rsrc); + idpf_vport_intr_dis_dim_all(rsrc); idpf_vport_intr_napi_del_all(rsrc); idpf_vport_intr_rel_irq(vport, rsrc); } @@ -4235,7 +4260,6 @@ static void idpf_vport_intr_napi_ena_all(struct idpf_q_vec_rsrc *rsrc) for (u16 q_idx = 0; q_idx < rsrc->num_q_vectors; q_idx++) { struct idpf_q_vector *q_vector = &rsrc->q_vectors[q_idx]; - idpf_init_dim(q_vector); napi_enable(&q_vector->napi); } } @@ -4578,6 +4602,8 @@ int idpf_vport_intr_alloc(struct idpf_vport *vport, q_coal = &user_config->q_coalesce[v_idx]; q_vector->vport = vport; + idpf_init_dim(q_vector); + q_vector->tx_itr_value = q_coal->tx_coalesce_usecs; q_vector->tx_intr_mode = q_coal->tx_intr_mode; q_vector->tx_itr_idx = VIRTCHNL2_ITR_IDX_1; diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 39043d8ca1bf4..59067939a586d 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -3074,7 +3074,8 @@ static void igc_xdp_xmit_zc(struct igc_ring *ring) olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT; dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr); - meta = xsk_buff_get_metadata(pool, xdp_desc.addr); + meta = xsk_buff_get_metadata(pool, xdp_desc.addr, + xdp_desc.options); xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len); bi = &ring->tx_buffer_info[ntu]; diff --git a/drivers/net/ethernet/intel/igc/igc_tsn.c b/drivers/net/ethernet/intel/igc/igc_tsn.c index 52de2bcbadbec..0c08650d3bb2d 100644 --- a/drivers/net/ethernet/intel/igc/igc_tsn.c +++ b/drivers/net/ethernet/intel/igc/igc_tsn.c @@ -183,13 +183,15 @@ static u32 igc_fpe_map_preempt_tc_to_queue(const struct igc_adapter *adapter, u32 i, queue = 0; for (i = 0; i < dev->num_tc; i++) { + struct netdev_tc_txq res; u32 offset, count; if (!(preemptible_tcs & BIT(i))) continue; - offset = dev->tc_to_txq[i].offset; - count = dev->tc_to_txq[i].count; + res.combined = READ_ONCE(dev->tc_to_txq[i].combined); + offset = res.offset; + count = res.count; queue |= GENMASK(offset + count - 1, offset); } diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 8873a8cc4a185..f91856498eb2d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -9273,10 +9273,11 @@ static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb, if (sb_dev) { u8 tc = netdev_get_prio_tc_map(dev, skb->priority); struct net_device *vdev = sb_dev; + struct netdev_tc_txq res; - txq = vdev->tc_to_txq[tc].offset; - txq += reciprocal_scale(skb_get_hash(skb), - vdev->tc_to_txq[tc].count); + res.combined = READ_ONCE(vdev->tc_to_txq[tc].combined); + txq = res.offset; + txq += reciprocal_scale(skb_get_hash(skb), res.count); return txq; } diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index ccc24a1301f22..848ee655c6ea6 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -5086,7 +5086,8 @@ static int mvpp2_change_mtu(struct net_device *dev, int mtu) netdev_warn(dev, "mtu %d too high, switching to shared buffers", mtu); mvpp2_bm_switch_buffers(priv, false); } - } else { + } else if (priv->hw_version >= MVPP22 && + mvpp2_get_nrxqs(priv) * 2 <= MVPP2_BM_MAX_POOLS) { bool jumbo = false; int i; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/api.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/api.h index 4285b5d6a6a29..f36a1d5f236f6 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/api.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/api.h @@ -21,6 +21,7 @@ int cn20k_rvu_mbox_init(struct rvu *rvu, int type, int num); int cn20k_rvu_get_mbox_regions(struct rvu *rvu, void **mbox_addr, int num, int type, unsigned long *pf_bmap); void cn20k_free_mbox_memory(struct rvu *rvu); +void cn20k_free_mbox_memory_type(struct rvu *rvu, int type); int cn20k_register_afpf_mbox_intr(struct rvu *rvu); int cn20k_register_afvf_mbox_intr(struct rvu *rvu, int pf_vec_start); void cn20k_rvu_enable_mbox_intr(struct rvu *rvu); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c index 71401dec0d776..01f32adac5992 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c @@ -335,13 +335,30 @@ int cn20k_rvu_mbox_init(struct rvu *rvu, int type, int ndevs) return rvu_alloc_mbox_memory(rvu, type, ndevs, MBOX_SIZE); } +void cn20k_free_mbox_memory_type(struct rvu *rvu, int type) +{ + if (!is_cn20k(rvu->pdev) || !rvu->ng_rvu) + return; + + switch (type) { + case TYPE_AFPF: + qmem_free(rvu->dev, rvu->ng_rvu->pf_mbox_addr); + rvu->ng_rvu->pf_mbox_addr = NULL; + break; + case TYPE_AFVF: + qmem_free(rvu->dev, rvu->ng_rvu->vf_mbox_addr); + rvu->ng_rvu->vf_mbox_addr = NULL; + break; + } +} + void cn20k_free_mbox_memory(struct rvu *rvu) { if (!is_cn20k(rvu->pdev)) return; - qmem_free(rvu->dev, rvu->ng_rvu->pf_mbox_addr); - qmem_free(rvu->dev, rvu->ng_rvu->vf_mbox_addr); + cn20k_free_mbox_memory_type(rvu, TYPE_AFPF); + cn20k_free_mbox_memory_type(rvu, TYPE_AFVF); } void cn20k_rvu_disable_afvf_intr(struct rvu *rvu, int vfs) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/common.h b/drivers/net/ethernet/marvell/octeontx2/af/common.h index 779413a383b74..78e42549d9908 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/common.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/common.h @@ -7,6 +7,10 @@ #ifndef COMMON_H #define COMMON_H +#include +#include +#include + #include "rvu_struct.h" #define OTX2_ALIGN 128 /* Align to cacheline */ @@ -44,6 +48,33 @@ struct qmem { u32 qsize; }; +static inline void *otx2_dma_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle) +{ + dma_addr_t dma_addr; + void *vaddr; + + vaddr = kzalloc(size, GFP_KERNEL); + if (!vaddr) + return NULL; + + dma_addr = dma_map_single(dev, vaddr, size, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) { + kfree(vaddr); + return NULL; + } + + *dma_handle = dma_addr; + return vaddr; +} + +static inline void otx2_dma_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle) +{ + dma_unmap_single(dev, dma_handle, size, DMA_BIDIRECTIONAL); + kfree(vaddr); +} + static inline int qmem_alloc(struct device *dev, struct qmem **q, int qsize, int entry_sz) { @@ -60,8 +91,11 @@ static inline int qmem_alloc(struct device *dev, struct qmem **q, qmem->entry_sz = entry_sz; qmem->alloc_sz = (qsize * entry_sz) + OTX2_ALIGN; - qmem->base = dma_alloc_attrs(dev, qmem->alloc_sz, &qmem->iova, - GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS); + + if (get_order(PAGE_ALIGN(qmem->alloc_sz)) > MAX_PAGE_ORDER) + return -ENOMEM; + + qmem->base = otx2_dma_alloc_coherent(dev, qmem->alloc_sz, &qmem->iova); if (!qmem->base) return -ENOMEM; @@ -80,10 +114,9 @@ static inline void qmem_free(struct device *dev, struct qmem *qmem) return; if (qmem->base) - dma_free_attrs(dev, qmem->alloc_sz, - qmem->base - qmem->align, - qmem->iova - qmem->align, - DMA_ATTR_FORCE_CONTIGUOUS); + otx2_dma_free_coherent(dev, qmem->alloc_sz, + qmem->base - qmem->align, + qmem->iova - qmem->align); devm_kfree(dev, qmem); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c index a07e0b3d8d000..211c10aa5880f 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c @@ -1417,6 +1417,16 @@ static int mcs_x2p_calibration(struct mcs *mcs) int i, err = 0; u64 val; + /* Clear any stale calibration state left by firmware/bootloader. + * Some firmware versions may leave MCSX_MIL_GLOBAL bit 5 set, + * preventing the hardware from detecting the rising edge needed to + * trigger X2P calibration. + */ + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + val &= ~BIT_ULL(5); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + usleep_range(100, 200); + /* set X2P calibration */ val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); val |= BIT_ULL(5); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c index d98b49f47970b..fce22e314cac0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c @@ -856,7 +856,7 @@ int rvu_mbox_handler_mcs_ctrl_pkt_rule_write(struct rvu *rvu, static void rvu_mcs_set_lmac_bmap(struct rvu *rvu) { struct mcs *mcs = mcs_get_pdata(0); - unsigned long lmac_bmap; + unsigned long lmac_bmap = 0; int cgx, lmac, port; for (port = 0; port < mcs->hw->lmac_cnt; port++) { diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index ffba56ee8a60c..937b085582b5f 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -2585,12 +2585,6 @@ static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw, if (!pf_bmap) return -ENOMEM; - ng_rvu_mbox = kzalloc_obj(*ng_rvu_mbox); - if (!ng_rvu_mbox) { - err = -ENOMEM; - goto free_bitmap; - } - /* RVU VFs */ if (type == TYPE_AFVF) bitmap_set(pf_bmap, 0, num); @@ -2604,15 +2598,22 @@ static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw, } } - rvu->ng_rvu = ng_rvu_mbox; + if (!rvu->ng_rvu) { + ng_rvu_mbox = devm_kzalloc(rvu->dev, sizeof(*ng_rvu_mbox), GFP_KERNEL); + if (!ng_rvu_mbox) { + err = -ENOMEM; + goto free_bitmap; + } + + rvu->ng_rvu = ng_rvu_mbox; - rvu->ng_rvu->rvu_mbox_ops = &rvu_mbox_ops; + rvu->ng_rvu->rvu_mbox_ops = &rvu_mbox_ops; + mutex_init(&rvu->mbox_lock); + } err = cn20k_rvu_mbox_init(rvu, type, num); if (err) - goto free_mem; - - mutex_init(&rvu->mbox_lock); + goto free_bitmap; mbox_regions = kcalloc(num, sizeof(void __iomem *), GFP_KERNEL); if (!mbox_regions) { @@ -2702,14 +2703,18 @@ static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw, free_regions: kfree(mbox_regions); free_qmem: - cn20k_free_mbox_memory(rvu); -free_mem: - kfree(rvu->ng_rvu); + cn20k_free_mbox_memory_type(rvu, type); free_bitmap: bitmap_free(pf_bmap); return err; } +static void rvu_free_cn20k_mbox_memory(struct rvu *rvu) +{ + if (is_cn20k(rvu->pdev)) + cn20k_free_mbox_memory(rvu); +} + static void rvu_mbox_destroy(struct mbox_wq_info *mw) { struct otx2_mbox *mbox = &mw->mbox; @@ -3331,8 +3336,8 @@ static int rvu_register_interrupts(struct rvu *rvu) goto fail; for (i = 0; i < rvu->num_vec; i++) { - if (strstr(&rvu->irq_name[i * NAME_SIZE], "Mbox") || - strstr(&rvu->irq_name[i * NAME_SIZE], "FLR")) + if (strnstr(&rvu->irq_name[i * NAME_SIZE], "Mbox", NAME_SIZE) || + strnstr(&rvu->irq_name[i * NAME_SIZE], "FLR", NAME_SIZE)) irq_set_affinity(pci_irq_vector(rvu->pdev, i), cpumask_of(0)); } @@ -3463,6 +3468,8 @@ int rvu_get_num_lbk_chans(void) return ret; } +#define PCI_DEVID_OCTEONTX2_RVU_AFVF 0xA0F8 + static int rvu_enable_sriov(struct rvu *rvu) { struct pci_dev *pdev = rvu->pdev; @@ -3481,24 +3488,27 @@ static int rvu_enable_sriov(struct rvu *rvu) return 0; pci_read_config_word(pdev, pos + PCI_SRIOV_VF_DID, &rvu->vf_devid); - chans = rvu_get_num_lbk_chans(); - if (chans < 0) - return chans; - vfs = pci_sriov_get_totalvfs(pdev); - - /* Limit VFs in case we have more VFs than LBK channels available. */ - if (vfs > chans) - vfs = chans; - if (!vfs) return 0; - /* LBK channel number 63 is used for switching packets between - * CGX mapped VFs. Hence limit LBK pairs till 62 only. - */ - if (vfs > 62) - vfs = 62; + if (rvu->vf_devid == PCI_DEVID_OCTEONTX2_RVU_AFVF) { + chans = rvu_get_num_lbk_chans(); + if (chans < 0) + return chans; + + /* The last LBK channel is reserved for switching packets between + * CGX mapped VFs. Also, since LBK VFs work in pairs, limit VF + * count to available LBK channels minus 2. + */ + vfs = min(vfs, chans - 2); + + if (vfs <= 0) { + dev_warn(&pdev->dev, + "Skipping SRIOV enablement, not enough LBK channels available\n"); + return 0; + } + } /* Save VFs number for reference in VF interrupts handlers. * Since interrupts might start arriving during SRIOV enablement @@ -3519,6 +3529,7 @@ static int rvu_enable_sriov(struct rvu *rvu) if (err) { rvu_disable_afvf_intr(rvu); rvu_mbox_destroy(&rvu->afvf_wq_info); + cn20k_free_mbox_memory_type(rvu, TYPE_AFVF); return err; } @@ -3681,6 +3692,7 @@ static int rvu_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_mbox: rvu_mbox_destroy(&rvu->afpf_wq_info); err_hwsetup: + rvu_free_cn20k_mbox_memory(rvu); rvu_cgx_exit(rvu); rvu_fwdata_exit(rvu); rvu_mcs_exit(rvu); @@ -3723,9 +3735,7 @@ static void rvu_remove(struct pci_dev *pdev) pci_set_drvdata(pdev, NULL); devm_kfree(&pdev->dev, rvu->hw); - if (is_cn20k(rvu->pdev)) - cn20k_free_mbox_memory(rvu); - kfree(rvu->ng_rvu); + rvu_free_cn20k_mbox_memory(rvu); devm_kfree(&pdev->dev, rvu); atomic_set(&device_bound, 0); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index 3456313d3b3c0..2927633465d96 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -714,110 +714,72 @@ static int get_max_column_width(struct rvu *rvu) } /* Dumps current provisioning status of all RVU block LFs */ -static ssize_t rvu_dbg_rsrc_attach_status(struct file *filp, - char __user *buffer, - size_t count, loff_t *ppos) +static int rvu_dbg_rsrc_attach_status(struct seq_file *filp, void *unused) { - int index, off = 0, flag = 0, len = 0, i = 0; - struct rvu *rvu = filp->private_data; - int bytes_not_copied = 0; + struct rvu *rvu = filp->private; + int index, pf, vf, pcifunc; struct rvu_block block; - int pf, vf, pcifunc; - int buf_size = 2048; int lf_str_size; char *lfs; - char *buf; - - /* don't allow partial reads */ - if (*ppos != 0) - return 0; - - buf = kzalloc(buf_size, GFP_KERNEL); - if (!buf) - return -ENOMEM; - /* Get the maximum width of a column */ lf_str_size = get_max_column_width(rvu); + if (lf_str_size < 0) + return lf_str_size; lfs = kzalloc(lf_str_size, GFP_KERNEL); - if (!lfs) { - kfree(buf); + if (!lfs) return -ENOMEM; - } - off += scnprintf(&buf[off], buf_size - 1 - off, "%-*s", lf_str_size, - "pcifunc"); - for (index = 0; index < BLK_COUNT; index++) - if (strlen(rvu->hw->block[index].name)) { - off += scnprintf(&buf[off], buf_size - 1 - off, - "%-*s", lf_str_size, - rvu->hw->block[index].name); - } - off += scnprintf(&buf[off], buf_size - 1 - off, "\n"); - bytes_not_copied = copy_to_user(buffer + (i * off), buf, off); - if (bytes_not_copied) - goto out; + seq_printf(filp, "%-*s", lf_str_size, "pcifunc"); + for (index = 0; index < BLK_COUNT; index++) + if (strlen(rvu->hw->block[index].name)) + seq_printf(filp, "%-*s", lf_str_size, + rvu->hw->block[index].name); - i++; - *ppos += off; + seq_putc(filp, '\n'); for (pf = 0; pf < rvu->hw->total_pfs; pf++) { for (vf = 0; vf <= rvu->hw->total_vfs; vf++) { - off = 0; - flag = 0; pcifunc = rvu_make_pcifunc(rvu->pdev, pf, vf); if (!pcifunc) continue; - if (vf) { + for (index = 0; index < BLK_COUNT; index++) { + block = rvu->hw->block[index]; + if (!strlen(block.name)) + continue; + lfs[0] = '\0'; + get_lf_str_list(&block, pcifunc, lfs); + if (strlen(lfs)) + break; + } + if (index == BLK_COUNT) + continue; + + if (vf) sprintf(lfs, "PF%d:VF%d", pf, vf - 1); - off = scnprintf(&buf[off], - buf_size - 1 - off, - "%-*s", lf_str_size, lfs); - } else { + else sprintf(lfs, "PF%d", pf); - off = scnprintf(&buf[off], - buf_size - 1 - off, - "%-*s", lf_str_size, lfs); - } + seq_printf(filp, "%-*s", lf_str_size, lfs); for (index = 0; index < BLK_COUNT; index++) { block = rvu->hw->block[index]; if (!strlen(block.name)) continue; - len = 0; - lfs[len] = '\0'; - get_lf_str_list(&block, pcifunc, lfs); - if (strlen(lfs)) - flag = 1; - off += scnprintf(&buf[off], buf_size - 1 - off, - "%-*s", lf_str_size, lfs); - } - if (flag) { - off += scnprintf(&buf[off], - buf_size - 1 - off, "\n"); - bytes_not_copied = copy_to_user(buffer + - (i * off), - buf, off); - if (bytes_not_copied) - goto out; - - i++; - *ppos += off; + lfs[0] = '\0'; + get_lf_str_list(&block, pcifunc, lfs); + seq_printf(filp, "%-*s", lf_str_size, lfs); } + seq_putc(filp, '\n'); } } -out: kfree(lfs); - kfree(buf); - if (bytes_not_copied) - return -EFAULT; - return *ppos; + return 0; } -RVU_DEBUG_FOPS(rsrc_status, rsrc_attach_status, NULL); +RVU_DEBUG_SEQ_FOPS(rsrc_status, rsrc_attach_status, NULL); static int rvu_dbg_rvu_pf_cgx_map_display(struct seq_file *filp, void *unused) { @@ -829,19 +791,25 @@ static int rvu_dbg_rvu_pf_cgx_map_display(struct seq_file *filp, void *unused) int pf, domain, blkid; u8 cgx_id, lmac_id; u16 pcifunc; + u8 start; - domain = 2; + domain = pci_domain_nr(rvu->pdev->bus); mac_ops = get_mac_ops(rvu_first_cgx_pdata(rvu)); /* There can be no CGX devices at all */ if (!mac_ops) return 0; seq_printf(filp, "PCI dev\t\tRVU PF Func\tNIX block\t%s\tLMAC\tCHAN\n", mac_ops->name); + + /* All the PF devices are on contiguous PCI bus numbers, but the PF0(AF) + * may not start from 1 always. Hence get domain and bus from PCI device. + */ + start = rvu->pdev->bus->number; for (pf = 0; pf < rvu->hw->total_pfs; pf++) { if (!is_pf_cgxmapped(rvu, pf)) continue; - pdev = pci_get_domain_bus_and_slot(domain, pf + 1, 0); + pdev = pci_get_domain_bus_and_slot(domain, pf + start, 0); if (!pdev) continue; @@ -1697,6 +1665,12 @@ static int rvu_dbg_nix_tm_tree_display(struct seq_file *m, void *unused) return -EINVAL; pfvf = rvu_get_pfvf(rvu, pcifunc); + + if (!pfvf->sq_ctx) { + seq_printf(m, "SQ context is not initialized for pcifunc 0x%x\n", pcifunc); + return -EINVAL; + } + max_id = pfvf->sq_ctx->qsize; memset(&aq_req, 0, sizeof(struct nix_aq_enq_req)); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 964bcaae098e2..cee30d5f1e1e5 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -2483,8 +2483,8 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, int pf = rvu_get_pf(rvu->pdev, pcifunc); u8 cgx_id = 0, lmac_id = 0; u16 tl2_tl3_link_schq; - u8 link, link_level; u64 cfg, bmap = 0; + u8 link_level; if (!is_rvu_otx2(rvu)) { /* Skip SMQ flush if pkt count is zero */ @@ -2516,7 +2516,6 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, link_level = rvu_read64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; tl2_tl3_link_schq = smq_flush_ctx->smq_tree_ctx[link_level].schq; - link = smq_flush_ctx->smq_tree_ctx[NIX_TXSCH_LVL_TL1].schq; /* SMQ set enqueue xoff */ cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq)); @@ -2526,13 +2525,13 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, /* Clear all NIX_AF_TL3_TL2_LINK_CFG[ENA] for the TL3/TL2 queue */ for (i = 0; i < (rvu->hw->cgx_links + rvu->hw->lbk_links); i++) { cfg = rvu_read64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link)); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i)); if (!(cfg & BIT_ULL(12))) continue; bmap |= BIT_ULL(i); cfg &= ~BIT_ULL(12); rvu_write64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link), cfg); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i), cfg); } /* Do SMQ flush and set enqueue xoff */ @@ -2553,10 +2552,10 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, if (!(bmap & BIT_ULL(i))) continue; cfg = rvu_read64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link)); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i)); cfg |= BIT_ULL(12); rvu_write64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link), cfg); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i), cfg); } /* clear XOFF on TL2s */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index 38554d51164e1..6f2e6bce164de 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -2957,10 +2957,9 @@ static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, } } - /* Disable the entry */ - npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); - if (!cn20k_dft_rl) { + /* Disable the entry */ + npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; /* Free the entry in bitmap */ npc_mcam_clear_bit(mcam, index); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 3d253132a17f0..52980c4afb54b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -333,7 +333,8 @@ int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id, const u32 *ind_tbl) /* Get memory to put this msg */ for (idx = 0; idx < rss->rss_size; idx++) { /* Ignore the queue if AF_XDP zero copy is enabled */ - if (test_bit(ind_tbl[idx], pfvf->af_xdp_zc_qidx)) + if (pfvf->af_xdp_zc_qidx && + test_bit(ind_tbl[idx], pfvf->af_xdp_zc_qidx)) continue; aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); @@ -1510,7 +1511,8 @@ int otx2_pool_aq_init(struct otx2_nic *pfvf, u16 pool_id, if (type != AURA_NIX_RQ) return 0; - if (!test_bit(pool_id, pfvf->af_xdp_zc_qidx)) { + if (!pfvf->af_xdp_zc_qidx || + !test_bit(pool_id, pfvf->af_xdp_zc_qidx)) { pp_params.order = get_order(buf_size); pp_params.flags = PP_FLAG_DMA_MAP; pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index a0340f3422bf9..9bee1b91eeaa6 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -939,7 +939,8 @@ static int otx2_get_rxfh(struct net_device *dev, for (idx = 0; idx < rss->rss_size; idx++) { /* Ignore if the rx queue is AF_XDP zero copy enabled */ - if (test_bit(rss->ind_tbl[idx], pfvf->af_xdp_zc_qidx)) + if (pfvf->af_xdp_zc_qidx && + test_bit(rss->ind_tbl[idx], pfvf->af_xdp_zc_qidx)) continue; indir[idx] = rss->ind_tbl[idx]; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c index b022f52c68450..fcdf891f90b58 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c @@ -737,16 +737,16 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_ptp_destroy; + err = otx2_vf_wq_init(vf); + if (err) + goto err_ipsec_clean; + err = register_netdev(netdev); if (err) { dev_err(dev, "Failed to register netdevice\n"); - goto err_ipsec_clean; + goto err_wq_destroy; } - err = otx2_vf_wq_init(vf); - if (err) - goto err_unreg_netdev; - otx2vf_set_ethtool_ops(netdev); err = otx2vf_mcam_flow_init(vf); @@ -789,6 +789,10 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) otx2_shutdown_tc(vf); err_unreg_netdev: unregister_netdev(netdev); +err_wq_destroy: + cancel_work_sync(&vf->reset_task); + cancel_work_sync(&vf->rx_mode_work); + destroy_workqueue(vf->otx2_wq); err_ipsec_clean: cn10k_ipsec_clean(vf); err_ptp_destroy: @@ -836,11 +840,13 @@ static void otx2vf_remove(struct pci_dev *pdev) } #endif - cancel_work_sync(&vf->reset_task); otx2_unregister_dl(vf); unregister_netdev(netdev); - if (vf->otx2_wq) + if (vf->otx2_wq) { + cancel_work_sync(&vf->reset_task); + cancel_work_sync(&vf->rx_mode_work); destroy_workqueue(vf->otx2_wq); + } cn10k_ipsec_clean(vf); otx2_ptp_destroy(vf); otx2_mcam_flow_del(vf); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c index 7d67b4cbaf716..0e8a6a6486c4a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c @@ -193,7 +193,8 @@ int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx) { - if (test_bit(qidx, pfvf->af_xdp_zc_qidx)) + if (pfvf->af_xdp_zc_qidx && + test_bit(qidx, pfvf->af_xdp_zc_qidx)) sq->xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, qidx); } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/qos.c b/drivers/net/ethernet/marvell/octeontx2/nic/qos.c index 69c0911e28e91..f160b1618efa2 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/qos.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/qos.c @@ -235,13 +235,63 @@ static int otx2_qos_txschq_set_parent_topology(struct otx2_nic *pfvf, return rc; } +static int otx2_qos_reset_schq_topology(struct otx2_nic *pfvf, u16 lvl, + u16 schq) +{ + struct mbox *mbox = &pfvf->mbox; + struct nix_txschq_config *cfg; + int rc; + + if (lvl < NIX_TXSCH_LVL_TL4 || lvl >= NIX_TXSCH_LVL_TL1) + return 0; + + mutex_lock(&mbox->lock); + + cfg = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox); + if (!cfg) { + mutex_unlock(&mbox->lock); + return -ENOMEM; + } + + cfg->lvl = lvl; + cfg->num_regs = 1; + + if (lvl == NIX_TXSCH_LVL_TL4) + cfg->reg[0] = NIX_AF_TL4X_TOPOLOGY(schq); + else if (lvl == NIX_TXSCH_LVL_TL3) + cfg->reg[0] = NIX_AF_TL3X_TOPOLOGY(schq); + else if (lvl == NIX_TXSCH_LVL_TL2) + cfg->reg[0] = NIX_AF_TL2X_TOPOLOGY(schq); + + cfg->regval[0] = 0; + + rc = otx2_sync_mbox_msg(mbox); + + mutex_unlock(&mbox->lock); + + return rc; +} + +static void otx2_qos_free_hw_schq(struct otx2_nic *pfvf, u16 lvl, u16 schq) +{ + int err; + + err = otx2_qos_reset_schq_topology(pfvf, lvl, schq); + if (err) + netdev_warn(pfvf->netdev, + "QoS: failed to reset topology for schq %u at level %u: %d\n", + schq, lvl, err); + + otx2_txschq_free_one(pfvf, lvl, schq); +} + static void otx2_qos_free_hw_node_schq(struct otx2_nic *pfvf, struct otx2_qos_node *parent) { struct otx2_qos_node *node; list_for_each_entry_reverse(node, &parent->child_schq_list, list) - otx2_txschq_free_one(pfvf, node->level, node->schq); + otx2_qos_free_hw_schq(pfvf, node->level, node->schq); } static void otx2_qos_free_hw_node(struct otx2_nic *pfvf, @@ -252,7 +302,7 @@ static void otx2_qos_free_hw_node(struct otx2_nic *pfvf, list_for_each_entry_safe(node, tmp, &parent->child_list, list) { otx2_qos_free_hw_node(pfvf, node); otx2_qos_free_hw_node_schq(pfvf, node); - otx2_txschq_free_one(pfvf, node->level, node->schq); + otx2_qos_free_hw_schq(pfvf, node->level, node->schq); } } @@ -266,7 +316,7 @@ static void otx2_qos_free_hw_cfg(struct otx2_nic *pfvf, otx2_qos_free_hw_node_schq(pfvf, node); /* free node hw mappings */ - otx2_txschq_free_one(pfvf, node->level, node->schq); + otx2_qos_free_hw_schq(pfvf, node->level, node->schq); mutex_unlock(&pfvf->qos.qos_lock); } @@ -913,7 +963,7 @@ static void otx2_qos_free_cfg(struct otx2_nic *pfvf, struct otx2_qos_cfg *cfg) for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { for (idx = 0; idx < cfg->schq[lvl]; idx++) { schq = cfg->schq_list[lvl][idx]; - otx2_txschq_free_one(pfvf, lvl, schq); + otx2_qos_free_hw_schq(pfvf, lvl, schq); } } @@ -921,7 +971,7 @@ static void otx2_qos_free_cfg(struct otx2_nic *pfvf, struct otx2_qos_cfg *cfg) for (idx = 0; idx < cfg->schq_contig[lvl]; idx++) { if (cfg->schq_index_used[lvl][idx]) { schq = cfg->schq_contig_list[lvl][idx]; - otx2_txschq_free_one(pfvf, lvl, schq); + otx2_qos_free_hw_schq(pfvf, lvl, schq); } } } diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 351444fb48716..efc238abb67b3 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -4508,6 +4508,10 @@ static int mtk_unreg_dev(struct mtk_eth *eth) mac = netdev_priv(eth->netdev[i]); if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) unregister_netdevice_notifier(&mac->device_notifier); + + if (eth->netdev[i]->reg_state != NETREG_REGISTERED) + continue; + unregister_netdev(eth->netdev[i]); } @@ -4827,7 +4831,7 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) phy_interface_t phy_mode; struct phylink *phylink; struct mtk_mac *mac; - int id, err; + int id, err, i; int txqs = 1; u32 val; @@ -4906,8 +4910,8 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) mac->phylink_config.type = PHYLINK_NETDEV; mac->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000 | MAC_2500FD; - mac->phylink_config.lpi_capabilities = MAC_100FD | MAC_1000FD | - MAC_2500FD; + /* LPI above 1 Gbps is not supported */ + mac->phylink_config.lpi_capabilities = MAC_100FD | MAC_1000FD; mac->phylink_config.lpi_timer_default = 1000; /* MT7623 gmac0 is now missing its speed-specific PLL configuration @@ -4965,6 +4969,18 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) __set_bit(PHY_INTERFACE_MODE_INTERNAL, mac->phylink_config.supported_interfaces); + /* LPI wake-up timing is only verified on MTK_GMAC_EEE SoCs */ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_GMAC_EEE)) { + phy_interface_copy(mac->phylink_config.lpi_interfaces, + mac->phylink_config.supported_interfaces); + __clear_bit(PHY_INTERFACE_MODE_2500BASEX, + mac->phylink_config.lpi_interfaces); + for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++) + if (mtk_interface_mode_is_xgmii(eth, i)) + __clear_bit(i, + mac->phylink_config.lpi_interfaces); + } + phylink = phylink_create(&mac->phylink_config, of_fwnode_handle(mac->of_node), phy_mode, mac_ops); @@ -5327,7 +5343,7 @@ static int mtk_probe(struct platform_device *pdev) err = register_netdev(eth->netdev[i]); if (err) { dev_err(eth->dev, "error bringing up device\n"); - goto err_deinit_ppe; + goto err_unreg_netdev; } else netif_info(eth, probe, eth->netdev[i], "mediatek frame engine at 0x%08lx, irq %d\n", diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h index 0168e2fbc6197..88a9b3b23bea5 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h @@ -994,6 +994,7 @@ enum mkt_eth_capabilities { MTK_U3_COPHY_V2_BIT, MTK_SRAM_BIT, MTK_36BIT_DMA_BIT, + MTK_GMAC_EEE_BIT, /* MUX BITS*/ MTK_ETH_MUX_GDM1_TO_GMAC1_ESW_BIT, @@ -1034,6 +1035,7 @@ enum mkt_eth_capabilities { #define MTK_U3_COPHY_V2 BIT_ULL(MTK_U3_COPHY_V2_BIT) #define MTK_SRAM BIT_ULL(MTK_SRAM_BIT) #define MTK_36BIT_DMA BIT_ULL(MTK_36BIT_DMA_BIT) +#define MTK_GMAC_EEE BIT_ULL(MTK_GMAC_EEE_BIT) #define MTK_ETH_MUX_GDM1_TO_GMAC1_ESW \ BIT_ULL(MTK_ETH_MUX_GDM1_TO_GMAC1_ESW_BIT) @@ -1117,7 +1119,7 @@ enum mkt_eth_capabilities { #define MT7981_CAPS (MTK_GMAC1_SGMII | MTK_GMAC2_SGMII | MTK_GMAC2_GEPHY | \ MTK_MUX_GMAC12_TO_GEPHY_SGMII | MTK_QDMA | \ MTK_MUX_U3_GMAC2_TO_QPHY | MTK_U3_COPHY_V2 | \ - MTK_RSTCTRL_PPE1 | MTK_SRAM) + MTK_RSTCTRL_PPE1 | MTK_SRAM | MTK_GMAC_EEE) #define MT7986_CAPS (MTK_GMAC1_SGMII | MTK_GMAC2_SGMII | \ MTK_MUX_GMAC12_TO_GEPHY_SGMII | MTK_QDMA | \ diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c index cc8c4ef8038f3..99b28aaa7cc47 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c @@ -92,6 +92,9 @@ static int mtk_flow_get_wdma_info(struct net_device *dev, const u8 *addr, struct mtk_wdma_info *info) { struct net_device_path_stack stack; + struct net_device_path_ctx ctx = { + .dev = dev, + }; struct net_device_path *path; int err; @@ -101,23 +104,29 @@ mtk_flow_get_wdma_info(struct net_device *dev, const u8 *addr, struct mtk_wdma_i if (!IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED)) return -1; + ether_addr_copy(ctx.daddr, addr); + rcu_read_lock(); - err = dev_fill_forward_path(dev, addr, &stack); + err = dev_fill_forward_path(&ctx, &stack); rcu_read_unlock(); if (err) return err; path = &stack.path[stack.num_paths - 1]; - if (path->type != DEV_PATH_MTK_WDMA) - return -1; + if (path->type != DEV_PATH_MTK_WDMA) { + err = -EINVAL; + goto err_out; + } info->wdma_idx = path->mtk_wdma.wdma_idx; info->queue = path->mtk_wdma.queue; info->bss = path->mtk_wdma.bss; info->wcid = path->mtk_wdma.wcid; info->amsdu = path->mtk_wdma.amsdu; +err_out: + dev_fill_forward_path_release(&stack); - return 0; + return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port.c b/drivers/net/ethernet/mellanox/mlx5/core/en/port.c index 6049ccf475bc4..a4c096a4fed25 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port.c @@ -557,6 +557,7 @@ int mlx5e_set_fec_mode(struct mlx5_core_dev *dev, u16 fec_policy) u32 in[MLX5_ST_SZ_DW(pplm_reg)] = {}; int sz = MLX5_ST_SZ_BYTES(pplm_reg); u16 fec_policy_auto = 0; + bool fec_set = false; int err; int i; @@ -569,9 +570,6 @@ int mlx5e_set_fec_mode(struct mlx5_core_dev *dev, u16 fec_policy) if (fec_policy >= (1 << MLX5E_FEC_LLRS_272_257_1) && !fec_50g_per_lane) return -EOPNOTSUPP; - if (fec_policy && !mlx5e_fec_in_caps(dev, fec_policy)) - return -EOPNOTSUPP; - MLX5_SET(pplm_reg, in, local_port, 1); err = mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PPLM, 0, 0); if (err) @@ -591,12 +589,17 @@ int mlx5e_set_fec_mode(struct mlx5_core_dev *dev, u16 fec_policy) mlx5e_get_fec_cap_field(out, &fec_caps, i); /* policy supported for link speed */ - if (fec_caps & conf_fec) + if (fec_caps & conf_fec) { mlx5e_fec_admin_field(out, &conf_fec, 1, i); - else - /* set FEC to auto*/ + fec_set = true; + } else { + /* set FEC to auto */ mlx5e_fec_admin_field(out, &fec_policy_auto, 1, i); + } } + if (fec_policy && !fec_set) + return -EOPNOTSUPP; + return mlx5_core_access_reg(dev, out, sz, out, sz, MLX5_REG_PPLM, 0, 1); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port.h b/drivers/net/ethernet/mellanox/mlx5/core/en/port.h index fa2283dd383b0..53dbdf77bccea 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/port.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port.h @@ -66,4 +66,8 @@ enum { MLX5E_FEC_LLRS_272_257_1 = 9, }; +#define MLX5E_FEC_RS_MASK (BIT(MLX5E_FEC_RS_528_514) | \ + BIT(MLX5E_FEC_RS_544_514_INTERLEAVED_QUAD) | \ + BIT(MLX5E_FEC_RS_544_514)) + #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c index baac38bece14a..4b7b0a0fc2b2f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c @@ -85,9 +85,16 @@ mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(struct net_device *dev, struct m struct net_device *lower_dev; struct list_head *iter; - if (netif_is_lag_master(dev) || mlx5e_eswitch_rep(dev)) - return mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, esw, vport_num, - esw_owner_vhca_id); + if (netif_is_lag_master(dev) || mlx5e_eswitch_rep(dev)) { + struct net_device *rep; + + rep = mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, esw, vport_num, + esw_owner_vhca_id); + if (rep && !mlx5_esw_bridge_port_exists(*vport_num, *esw_owner_vhca_id, + esw->br_offloads)) + return NULL; + return rep; + } netdev_for_each_lower_dev(dev, lower_dev, iter) { struct net_device *rep; @@ -104,6 +111,28 @@ mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(struct net_device *dev, struct m return NULL; } +static bool mlx5_esw_bridge_rep_port_lookup(struct net_device *dev, + struct mlx5_esw_bridge_offloads *br_offloads, + u16 *vport_num, u16 *esw_owner_vhca_id) +{ + if (!mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, br_offloads->esw, vport_num, + esw_owner_vhca_id)) + return false; + + return mlx5_esw_bridge_port_exists(*vport_num, *esw_owner_vhca_id, br_offloads); +} + +static bool mlx5_esw_bridge_lower_rep_port_lookup(struct net_device *dev, + struct mlx5_esw_bridge_offloads *br_offloads, + u16 *vport_num, u16 *esw_owner_vhca_id) +{ + if (!mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(dev, br_offloads->esw, vport_num, + esw_owner_vhca_id)) + return false; + + return mlx5_esw_bridge_port_exists(*vport_num, *esw_owner_vhca_id, br_offloads); +} + static bool mlx5_esw_bridge_is_local(struct net_device *dev, struct net_device *rep, struct mlx5_eswitch *esw) { @@ -218,8 +247,7 @@ mlx5_esw_bridge_port_obj_add(struct net_device *dev, u16 vport_num, esw_owner_vhca_id; int err; - if (!mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, br_offloads->esw, &vport_num, - &esw_owner_vhca_id)) + if (!mlx5_esw_bridge_rep_port_lookup(dev, br_offloads, &vport_num, &esw_owner_vhca_id)) return 0; port_obj_info->handled = true; @@ -251,8 +279,7 @@ mlx5_esw_bridge_port_obj_del(struct net_device *dev, const struct switchdev_obj_port_mdb *mdb; u16 vport_num, esw_owner_vhca_id; - if (!mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, br_offloads->esw, &vport_num, - &esw_owner_vhca_id)) + if (!mlx5_esw_bridge_rep_port_lookup(dev, br_offloads, &vport_num, &esw_owner_vhca_id)) return 0; port_obj_info->handled = true; @@ -283,8 +310,8 @@ mlx5_esw_bridge_port_obj_attr_set(struct net_device *dev, u16 vport_num, esw_owner_vhca_id; int err = 0; - if (!mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(dev, br_offloads->esw, &vport_num, - &esw_owner_vhca_id)) + if (!mlx5_esw_bridge_lower_rep_port_lookup(dev, br_offloads, &vport_num, + &esw_owner_vhca_id)) return 0; port_attr_info->handled = true; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c index 89490f687a9c6..93c62d3f3e5bd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c @@ -311,12 +311,15 @@ sample_restore_get(struct mlx5e_tc_psample *tc_psample, u32 obj_id, static void sample_restore_put(struct mlx5e_tc_psample *tc_psample, struct mlx5e_sample_restore *restore) { + bool last; + mutex_lock(&tc_psample->restore_lock); - if (--restore->count == 0) + last = --restore->count == 0; + if (last) hash_del(&restore->hlist); mutex_unlock(&tc_psample->restore_lock); - if (!restore->count) { + if (last) { mlx5_del_flow_rules(restore->rule); mlx5_modify_header_dealloc(tc_psample->esw->dev, restore->modify_hdr); kfree(restore); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index 6c87a1c7db095..c1841ce74d9ac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -1082,6 +1082,9 @@ mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv, spin_unlock_bh(&ct_priv->ht_lock); + if (rev_entry) + mlx5_tc_ct_entry_put(rev_entry); + create_counter: shared_counter = mlx5_tc_ct_counter_create(ct_priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c index 3d19dad8f8688..d194eeddd7b55 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c @@ -99,7 +99,7 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) xdptxd.dma_addr = xsk_buff_raw_get_dma(pool, desc.addr); xdptxd.data = xsk_buff_raw_get_data(pool, desc.addr); xdptxd.len = desc.len; - meta = xsk_buff_get_metadata(pool, desc.addr); + meta = xsk_buff_get_metadata(pool, desc.addr, desc.options); xsk_buff_raw_dma_sync_for_device(pool, xdptxd.dma_addr, xdptxd.len); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 329608c59313b..8ffa8068e90ac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -1564,14 +1564,14 @@ static void setup_fte_addr6(struct mlx5_flow_spec *spec, memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16); memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), dmask, 16); + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), smask, 16); } if (!addr6_all_zero(daddr)) { memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16); memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), smask, 16); + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), dmask, 16); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index daff53ba7d09f..38a3415acf7a3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -1724,6 +1724,8 @@ void mlx5e_macsec_build_netdev(struct mlx5e_priv *priv) mlx5_core_dbg(priv->mdev, "mlx5e: MACsec acceleration enabled\n"); netdev->macsec_ops = &macsec_offload_ops; netdev->features |= NETIF_F_HW_MACSEC; + netdev->hw_features |= NETIF_F_HW_MACSEC; + netdev->vlan_features |= NETIF_F_HW_MACSEC; netif_keep_dst(netdev); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c index 00e706e1ede11..741f75b5bfec5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c @@ -148,7 +148,7 @@ static int mlx5e_dcbnl_ieee_getets(struct net_device *netdev, if (err) return err; - if (ets->tc_tx_bw[i] < MLX5E_MAX_BW_ALLOC && + if (priv->dcbx.tc_tsa[i] == IEEE_8021QAZ_TSA_ETS && tc_group[i] == (MLX5E_LOWEST_PRIO_GROUP + 1)) is_zero_bw_ets_tc = true; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 112926d07634d..3ed59ced0407a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -1002,9 +1002,9 @@ static u32 pplm2ethtool_fec(u_long fec_mode, unsigned long size) return 0; } -#define MLX5E_ADVERTISE_SUPPORTED_FEC(mlx5_fec, ethtool_fec) \ +#define MLX5E_ADVERTISE_SUPPORTED_FEC(fec_mask, ethtool_fec) \ do { \ - if (mlx5e_fec_in_caps(dev, 1 << (mlx5_fec))) \ + if (mlx5e_fec_in_caps(dev, fec_mask)) \ __set_bit(ethtool_fec, \ link_ksettings->link_modes.supported);\ } while (0) @@ -1013,6 +1013,7 @@ static const u32 pplm_fec_2_ethtool_linkmodes[] = { [MLX5E_FEC_NOFEC] = ETHTOOL_LINK_MODE_FEC_NONE_BIT, [MLX5E_FEC_FIRECODE] = ETHTOOL_LINK_MODE_FEC_BASER_BIT, [MLX5E_FEC_RS_528_514] = ETHTOOL_LINK_MODE_FEC_RS_BIT, + [MLX5E_FEC_RS_544_514_INTERLEAVED_QUAD] = ETHTOOL_LINK_MODE_FEC_RS_BIT, [MLX5E_FEC_RS_544_514] = ETHTOOL_LINK_MODE_FEC_RS_BIT, [MLX5E_FEC_LLRS_272_257_1] = ETHTOOL_LINK_MODE_FEC_LLRS_BIT, }; @@ -1029,13 +1030,13 @@ static int get_fec_supported_advertised(struct mlx5_core_dev *dev, if (err) return (err == -EOPNOTSUPP) ? 0 : err; - MLX5E_ADVERTISE_SUPPORTED_FEC(MLX5E_FEC_NOFEC, + MLX5E_ADVERTISE_SUPPORTED_FEC(BIT(MLX5E_FEC_NOFEC), ETHTOOL_LINK_MODE_FEC_NONE_BIT); - MLX5E_ADVERTISE_SUPPORTED_FEC(MLX5E_FEC_FIRECODE, + MLX5E_ADVERTISE_SUPPORTED_FEC(BIT(MLX5E_FEC_FIRECODE), ETHTOOL_LINK_MODE_FEC_BASER_BIT); - MLX5E_ADVERTISE_SUPPORTED_FEC(MLX5E_FEC_RS_528_514, + MLX5E_ADVERTISE_SUPPORTED_FEC(MLX5E_FEC_RS_MASK, ETHTOOL_LINK_MODE_FEC_RS_BIT); - MLX5E_ADVERTISE_SUPPORTED_FEC(MLX5E_FEC_LLRS_272_257_1, + MLX5E_ADVERTISE_SUPPORTED_FEC(BIT(MLX5E_FEC_LLRS_272_257_1), ETHTOOL_LINK_MODE_FEC_LLRS_BIT); active_fec_long = active_fec; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index f0407a850ea82..da907248dc6c2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -3247,7 +3247,7 @@ static int mlx5e_update_tc_and_tx_queues(struct mlx5e_priv *priv) old_num_txqs = netdev->real_num_tx_queues; old_ntc = netdev->num_tc ? : 1; for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++) - old_tc_to_txq[i] = netdev->tc_to_txq[i]; + old_tc_to_txq[i].combined = READ_ONCE(netdev->tc_to_txq[i].combined); nch = priv->channels.params.num_channels; ntc = priv->channels.params.mqprio.num_tc; @@ -5851,7 +5851,6 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) netdev->vlan_features |= NETIF_F_SG; netdev->vlan_features |= NETIF_F_HW_CSUM; - netdev->vlan_features |= NETIF_F_HW_MACSEC; netdev->vlan_features |= NETIF_F_GRO; netdev->vlan_features |= NETIF_F_TSO; netdev->vlan_features |= NETIF_F_TSO6; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index c8b76d301c92c..eb3160bfbb29d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -56,7 +56,6 @@ #include "lib/vxlan.h" #define CREATE_TRACE_POINTS #include "diag/en_rep_tracepoint.h" -#include "diag/reporter_vnic.h" #include "en_accel/ipsec.h" #include "en/tc/int_port.h" #include "en/ptp.h" @@ -689,11 +688,13 @@ static int mlx5e_rep_open(struct net_device *dev) if (err) goto unlock; + mutex_lock(&rep->esw->state_lock); if (!mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_UP)) netif_carrier_on(dev); + mutex_unlock(&rep->esw->state_lock); unlock: mutex_unlock(&priv->state_lock); @@ -708,10 +709,12 @@ static int mlx5e_rep_close(struct net_device *dev) int ret; mutex_lock(&priv->state_lock); + mutex_lock(&rep->esw->state_lock); mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_DOWN); + mutex_unlock(&rep->esw->state_lock); ret = mlx5e_close_locked(dev); mutex_unlock(&priv->state_lock); return ret; @@ -783,22 +786,25 @@ static int mlx5e_rep_change_carrier(struct net_device *dev, bool new_carrier) struct mlx5e_priv *priv = netdev_priv(dev); struct mlx5e_rep_priv *rpriv = priv->ppriv; struct mlx5_eswitch_rep *rep = rpriv->rep; - int err; + int err = 0; + mutex_lock(&rep->esw->state_lock); if (new_carrier) { err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_UP); if (err) - return err; + goto unlock; netif_carrier_on(dev); } else { err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_DOWN); if (err) - return err; + goto unlock; netif_carrier_off(dev); } - return 0; +unlock: + mutex_unlock(&rep->esw->state_lock); + return err; } static const struct net_device_ops mlx5e_netdev_ops_rep = { @@ -1337,9 +1343,12 @@ static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) mlx5e_rep_tc_enable(priv); - if (MLX5_CAP_GEN(mdev, uplink_follow)) + if (MLX5_CAP_GEN(mdev, uplink_follow)) { + mutex_lock(&mdev->priv.eswitch->state_lock); mlx5_modify_vport_admin_state(mdev, MLX5_VPORT_STATE_OP_MOD_UPLINK, 0, 0, MLX5_VPORT_ADMIN_STATE_AUTO); + mutex_unlock(&mdev->priv.eswitch->state_lock); + } mlx5_lag_add_netdev(mdev, netdev); priv->events_nb.notifier_call = uplink_rep_async_event; mlx5_notifier_register(mdev, &priv->events_nb); @@ -1425,51 +1434,6 @@ static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv *priv) return ARRAY_SIZE(mlx5e_ul_rep_stats_grps); } -static int -mlx5e_rep_vnic_reporter_diagnose(struct devlink_health_reporter *reporter, - struct devlink_fmsg *fmsg, - struct netlink_ext_ack *extack) -{ - struct mlx5e_rep_priv *rpriv = devlink_health_reporter_priv(reporter); - struct mlx5_eswitch_rep *rep = rpriv->rep; - - mlx5_reporter_vnic_diagnose_counters(rep->esw->dev, fmsg, rep->vport, - true); - return 0; -} - -static const struct devlink_health_reporter_ops mlx5_rep_vnic_reporter_ops = { - .name = "vnic", - .diagnose = mlx5e_rep_vnic_reporter_diagnose, -}; - -static void mlx5e_rep_vnic_reporter_create(struct mlx5e_priv *priv, - struct devlink_port *dl_port) -{ - struct mlx5e_rep_priv *rpriv = priv->ppriv; - struct devlink_health_reporter *reporter; - - reporter = devl_port_health_reporter_create(dl_port, - &mlx5_rep_vnic_reporter_ops, - rpriv); - if (IS_ERR(reporter)) { - mlx5_core_err(priv->mdev, - "Failed to create representor vnic reporter, err = %pe\n", - reporter); - return; - } - - rpriv->rep_vnic_reporter = reporter; -} - -static void mlx5e_rep_vnic_reporter_destroy(struct mlx5e_priv *priv) -{ - struct mlx5e_rep_priv *rpriv = priv->ppriv; - - if (!IS_ERR_OR_NULL(rpriv->rep_vnic_reporter)) - devl_health_reporter_destroy(rpriv->rep_vnic_reporter); -} - static const struct mlx5e_profile mlx5e_rep_profile = { .init = mlx5e_init_rep, .cleanup = mlx5e_cleanup_rep, @@ -1593,10 +1557,8 @@ mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) dl_port = mlx5_esw_offloads_devlink_port(dev->priv.eswitch, rpriv->rep->vport); - if (!IS_ERR(dl_port)) { + if (!IS_ERR(dl_port)) SET_NETDEV_DEVLINK_PORT(netdev, dl_port); - mlx5e_rep_vnic_reporter_create(priv, dl_port); - } err = register_netdev(netdev); if (err) { @@ -1609,7 +1571,6 @@ mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) return 0; err_detach_netdev: - mlx5e_rep_vnic_reporter_destroy(priv); mlx5e_detach_netdev(netdev_priv(netdev)); err_cleanup_profile: priv->profile->cleanup(priv); @@ -1667,7 +1628,6 @@ mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep) } unregister_netdev(netdev); - mlx5e_rep_vnic_reporter_destroy(priv); mlx5e_detach_netdev(priv); priv->profile->cleanup(priv); mlx5e_destroy_netdev(netdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h index 70640fa1ad7bc..bcd7b4e814d06 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h @@ -118,7 +118,6 @@ struct mlx5e_rep_priv { struct rtnl_link_stats64 prev_vf_vport_stats; struct mlx5_flow_handle *send_to_vport_meta_rule; struct rhashtable tc_ht; - struct devlink_health_reporter *rep_vnic_reporter; }; static inline diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 6fbc0441c4b84..6fc6605d20549 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -2263,6 +2263,11 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq data_offset = wqe_offset & (page_size - 1); page_idx = wqe_offset >> rq->mpwqe.page_shift; + if (unlikely(cqe_bcnt <= ETH_ZLEN + 2 * VLAN_HLEN)) { + match = false; + flush = true; + } + if (*skb && !(match && mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt, page_size))) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index e7e6db7f6bf17..cd94bb44f6ab0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -1199,50 +1199,39 @@ void mlx5e_stats_rmon_get(struct mlx5e_priv *priv, void mlx5e_stats_ts_get(struct mlx5e_priv *priv, struct ethtool_ts_stats *ts_stats) { - int i, j; + u16 nch = mlx5e_stats_nch_read(priv); + int i, tc; - mutex_lock(&priv->state_lock); + ts_stats->pkts = 0; - if (priv->tx_ptp_opened) { - struct mlx5e_ptp *ptp = priv->channels.ptp; + for (i = 0; i < nch; i++) { + struct mlx5e_channel_stats *channel_stats = + priv->channel_stats[i]; - ts_stats->pkts = 0; + for (tc = 0; tc < priv->max_opened_tc; tc++) + ts_stats->pkts += channel_stats->sq[tc].timestamps; + } + + /* Accumulate DMA and port timestamp counters so values stay monotonic + * across channel teardown and mode switches. + */ + if (priv->tx_ptp_opened) { + /* Err and Lost stats are only relevant for port timestamping, + * as the DMA layer will always successfully timestamp packets. + */ ts_stats->err = 0; ts_stats->lost = 0; - if (!ptp) - goto out; - - /* Aggregate stats across all TCs */ - for (i = 0; i < ptp->num_tc; i++) { + for (tc = 0; tc < priv->max_opened_tc; tc++) { struct mlx5e_ptp_cq_stats *stats = - ptp->ptpsq[i].cq_stats; + &priv->ptp_stats.cq[tc]; ts_stats->pkts += stats->cqe; ts_stats->err += stats->abort + stats->err_cqe + - stats->late_cqe; + stats->late_cqe; ts_stats->lost += stats->lost_cqe; } - } else { - /* DMA layer will always successfully timestamp packets. Other - * counters do not make sense for this layer. - */ - ts_stats->pkts = 0; - - /* Aggregate stats across all SQs */ - for (j = 0; j < priv->channels.num; j++) { - struct mlx5e_channel *c = priv->channels.c[j]; - - for (i = 0; i < c->num_tc; i++) { - struct mlx5e_sq_stats *stats = c->sq[i].stats; - - ts_stats->pkts += stats->timestamps; - } - } } - -out: - mutex_unlock(&priv->state_lock); } #define PPORT_PHY_LAYER_OFF(c) \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c index ca249b50f830a..7950b82d8b8ae 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c @@ -9,6 +9,28 @@ int mlx5_esw_adj_vport_modify(struct mlx5_core_dev *dev, u16 vport, { u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)] = {}; + lockdep_assert_held(&dev->priv.eswitch->state_lock); + + if (MLX5_CAP_ESW(dev, esw_vport_state_max_tx_speed)) { + u8 op_mod = MLX5_VPORT_STATE_OP_MOD_ESW_VPORT; + struct mlx5_vport *esw_vport; + u32 speed = 0; + int err; + + err = mlx5_query_vport_max_tx_speed(dev, op_mod, vport, + true, &speed, NULL); + if (err) { + esw_vport = mlx5_eswitch_get_vport(dev->priv.eswitch, + vport); + speed = IS_ERR(esw_vport) ? 0 : + esw_vport->agg_max_tx_speed; + mlx5_core_dbg(dev, + "Failed to query vport %d max tx speed, err=%d, using cached %u\n", + vport, err, speed); + } + MLX5_SET(modify_vport_state_in, in, max_tx_speed, speed); + } + MLX5_SET(modify_vport_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VPORT_STATE); MLX5_SET(modify_vport_state_in, in, op_mod, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c index 87b5fd3495945..b4cf3c5ac0dde 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c @@ -1649,10 +1649,8 @@ int mlx5_esw_bridge_vport_unlink(struct net_device *br_netdev, u16 vport_num, int err; port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads); - if (!port) { - NL_SET_ERR_MSG_MOD(extack, "Port is not attached to any bridge"); - return -EINVAL; - } + if (!port) + return 0; if (port->bridge->ifindex != br_netdev->ifindex) { NL_SET_ERR_MSG_MOD(extack, "Port is attached to another bridge"); return -EINVAL; @@ -1682,10 +1680,19 @@ int mlx5_esw_bridge_vport_peer_unlink(struct net_device *br_netdev, u16 vport_nu struct mlx5_esw_bridge_offloads *br_offloads, struct netlink_ext_ack *extack) { + if (!MLX5_CAP_ESW(br_offloads->esw->dev, merged_eswitch)) + return 0; + return mlx5_esw_bridge_vport_unlink(br_netdev, vport_num, esw_owner_vhca_id, br_offloads, extack); } +bool mlx5_esw_bridge_port_exists(u16 vport_num, u16 esw_owner_vhca_id, + struct mlx5_esw_bridge_offloads *br_offloads) +{ + return mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads); +} + int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, u16 flags, struct mlx5_esw_bridge_offloads *br_offloads, struct netlink_ext_ack *extack) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h index d6f5391619930..a4e59cc210894 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h @@ -80,6 +80,8 @@ int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 pro struct mlx5_esw_bridge_offloads *br_offloads); int mlx5_esw_bridge_mcast_set(u16 vport_num, u16 esw_owner_vhca_id, bool enable, struct mlx5_esw_bridge_offloads *br_offloads); +bool mlx5_esw_bridge_port_exists(u16 vport_num, u16 esw_owner_vhca_id, + struct mlx5_esw_bridge_offloads *br_offloads); int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, u16 flags, struct mlx5_esw_bridge_offloads *br_offloads, struct netlink_ext_ack *extack); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c index 6e50311faa27f..2e7eac4bca3dd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c @@ -4,6 +4,26 @@ #include #include "eswitch.h" #include "devlink.h" +#include "diag/reporter_vnic.h" + +static int +mlx5_esw_rep_vnic_reporter_diagnose(struct devlink_health_reporter *reporter, + struct devlink_fmsg *fmsg, + struct netlink_ext_ack *extack) +{ + struct mlx5_vport *vport = devlink_health_reporter_priv(reporter); + + mlx5_reporter_vnic_diagnose_counters(vport->dev, fmsg, vport->vport, + true); + + return 0; +} + +static const +struct devlink_health_reporter_ops mlx5_esw_rep_vnic_reporter_ops = { + .name = "vnic", + .diagnose = mlx5_esw_rep_vnic_reporter_diagnose, +}; static void mlx5_esw_get_port_parent_id(struct mlx5_core_dev *dev, struct netdev_phys_item_id *ppid) @@ -217,6 +237,7 @@ static void mlx5_esw_devlink_port_res_unregister(struct devlink_port *dl_port) int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { + struct devlink_health_reporter *reporter; struct mlx5_core_dev *dev = esw->dev; const struct devlink_port_ops *ops; struct mlx5_devlink_port *dl_port; @@ -252,6 +273,16 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, struct mlx mlx5_core_dbg(dev, "Failed to register port resources: %d\n", err); + reporter = devl_port_health_reporter_create( + &dl_port->dl_port, &mlx5_esw_rep_vnic_reporter_ops, + vport); + if (IS_ERR(reporter)) + mlx5_core_err(dev, + "Failed to create vnic health reporter for vport %d: %pe\n", + vport_num, reporter); + else + dl_port->vnic_reporter = reporter; + return 0; rate_err: @@ -266,6 +297,12 @@ void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_vport *vport) if (!vport->dl_port) return; dl_port = vport->dl_port; + + if (dl_port->vnic_reporter) { + devl_health_reporter_destroy(dl_port->vnic_reporter); + dl_port->vnic_reporter = NULL; + } + mlx5_esw_devlink_port_res_unregister(&dl_port->dl_port); mlx5_esw_qos_vport_update_parent(vport, NULL, NULL); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index a0e2ca87b8d8f..eadc44119b1f4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1040,13 +1040,19 @@ void mlx5_esw_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vport) (vport->info.ipsec_crypto_enabled || vport->info.ipsec_packet_enabled)) esw->enabled_ipsec_vf_count--; + /* Clear rx-mode before esw_vport_change_handle_locked(): on + * MLX5_VPORT_PROMISC_CHANGE it calls esw_update_vport_mc_promisc() + * when vport->allmulti_rule is set, repopulating mc_list with FDB + * rules that dangle once the FDB is destroyed. NULL allmulti_rule + * here skips that path. + */ + esw_apply_vport_rx_mode(esw, vport, false, false); /* We don't assume VFs will cleanup after themselves. * Calling vport change handler while vport is disabled will cleanup * the vport resources. */ esw_vport_change_handle_locked(vport); vport->enabled_events = 0; - esw_apply_vport_rx_mode(esw, vport, false, false); esw_vport_cleanup(esw, vport); esw->enabled_vports--; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index fea72b1dedab9..e8817d45763a6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -189,6 +189,7 @@ struct mlx5_vport; struct mlx5_devlink_port { struct devlink_port dl_port; struct mlx5_vport *vport; + struct devlink_health_reporter *vnic_reporter; }; static inline void mlx5_devlink_port_init(struct mlx5_devlink_port *dl_port, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 247e5d85ec367..d603e294ee0e8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -2567,6 +2567,7 @@ static void mlx5_esw_fdb_active(struct mlx5_eswitch *esw) mlx5_esw_fdb_drop_destroy(esw); mlx5_mpfs_enable(esw->dev); + mutex_lock(&esw->state_lock); mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { if (!vport->adjacent) continue; @@ -2574,6 +2575,7 @@ static void mlx5_esw_fdb_active(struct mlx5_eswitch *esw) vport->vport); mlx5_esw_adj_vport_modify(esw->dev, vport->vport, true); } + mutex_unlock(&esw->state_lock); esw->offloads_inactive = false; esw_warn(esw->dev, "MPFS/FDB active\n"); @@ -2587,6 +2589,7 @@ static void mlx5_esw_fdb_inactive(struct mlx5_eswitch *esw) mlx5_mpfs_disable(esw->dev); mlx5_esw_fdb_drop_create(esw); + mutex_lock(&esw->state_lock); mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { if (!vport->adjacent) continue; @@ -2595,6 +2598,7 @@ static void mlx5_esw_fdb_inactive(struct mlx5_eswitch *esw) mlx5_esw_adj_vport_modify(esw->dev, vport->vport, false); } + mutex_unlock(&esw->state_lock); esw->offloads_inactive = true; esw_warn(esw->dev, "MPFS/FDB inactive\n"); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c index 19f65d4c4def9..d43f073601596 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c @@ -163,12 +163,15 @@ void mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw, struct mlx5_termtbl_handle *tt) { + bool last; + mutex_lock(&esw->offloads.termtbl_mutex); - if (--tt->ref_count == 0) + last = (--tt->ref_count == 0); + if (last) hash_del(&tt->termtbl_hlist); mutex_unlock(&esw->offloads.termtbl_mutex); - if (!tt->ref_count) { + if (last) { mlx5_del_flow_rules(tt->rule); mlx5_destroy_flow_table(tt->termtbl); kfree(tt); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 28d16fdc3f061..dd14cdc378de0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -63,14 +63,15 @@ static int get_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags) return MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY; } -static u8 lag_active_port_bits(struct mlx5_lag *ldev) +static u8 lag_active_port_bits(struct mlx5_lag *ldev, + struct lag_tracker *tracker) { u8 enabled_ports[MLX5_MAX_PORTS] = {}; u8 active_port = 0; int num_enabled; int idx; - mlx5_infer_tx_enabled(&ldev->tracker, ldev, enabled_ports, + mlx5_infer_tx_enabled(tracker, ldev, enabled_ports, &num_enabled); for (idx = 0; idx < num_enabled; idx++) active_port |= BIT_MASK(enabled_ports[idx]); @@ -79,7 +80,8 @@ static u8 lag_active_port_bits(struct mlx5_lag *ldev) } static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, struct mlx5_lag *ldev, - int mode, unsigned long flags) + struct lag_tracker *tracker, int mode, + unsigned long flags) { bool fdb_sel_mode = test_bit(MLX5_LAG_MODE_FLAG_FDB_SEL_MODE_NATIVE, &flags); @@ -108,7 +110,7 @@ static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, struct mlx5_lag *ldev, break; MLX5_SET(lagc, lag_ctx, active_port, - lag_active_port_bits(mlx5_lag_dev(dev))); + lag_active_port_bits(ldev, tracker)); break; default: break; @@ -787,7 +789,8 @@ static int mlx5_cmd_modify_active_port(struct mlx5_core_dev *dev, u8 ports) return mlx5_cmd_exec_in(dev, modify_lag, in); } -static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports) +static int _mlx5_modify_lag(struct mlx5_lag *ldev, + struct lag_tracker *tracker, u8 *ports) { int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); struct mlx5_core_dev *dev0; @@ -804,7 +807,7 @@ static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports) !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table_bypass)) return ret; - active_ports = lag_active_port_bits(ldev); + active_ports = lag_active_port_bits(ldev, tracker); return mlx5_cmd_modify_active_port(dev0, active_ports); } @@ -868,7 +871,7 @@ void mlx5_modify_lag(struct mlx5_lag *ldev, idx = i * ldev->buckets + j; if (ports[idx] == ldev->v2p_map[idx]) continue; - err = _mlx5_modify_lag(ldev, ports); + err = _mlx5_modify_lag(ldev, tracker, ports); if (err) { mlx5_core_err(dev0, "Failed to modify LAG (%d)\n", @@ -976,7 +979,7 @@ static int mlx5_create_lag(struct mlx5_lag *ldev, mlx5_core_info(dev0, "shared_fdb:%d mode:%s\n", shared_fdb, mlx5_get_str_port_sel_mode(mode, flags)); - err = mlx5_cmd_create_lag(dev0, ldev, mode, flags); + err = mlx5_cmd_create_lag(dev0, ldev, tracker, mode, flags); if (err) { mlx5_core_err(dev0, "Failed to create LAG (%d)\n", @@ -1263,25 +1266,45 @@ void mlx5_lag_remove_devices(struct mlx5_lag *ldev) mlx5_lag_remove_devices_filter(ldev, MLX5_LAG_FILTER_PORTS); } +static int mlx5_lag_reload_ib_reps_idx(struct mlx5_lag *ldev, int idx, + u32 flags) +{ + struct lag_func *pf = mlx5_lag_pf(ldev, idx); + struct mlx5_eswitch *esw; + int ret; + + if (pf->dev->priv.flags & flags) + return 0; + + esw = pf->dev->priv.eswitch; + mlx5_esw_reps_block(esw); + ret = mlx5_eswitch_reload_ib_reps(esw); + mlx5_esw_reps_unblock(esw); + + return ret; +} + static int mlx5_lag_reload_ib_reps_unlocked(struct mlx5_lag *ldev, u32 flags, u32 filter, bool cont_on_fail) { - struct lag_func *pf; + int master_idx = mlx5_lag_get_dev_index_by_seq_filter(ldev, MLX5_LAG_P1, + filter); int ret; int i; + if (master_idx < 0) + return -EINVAL; + + ret = mlx5_lag_reload_ib_reps_idx(ldev, master_idx, flags); + if (ret && !cont_on_fail) + return ret; + mlx5_lag_for_each(i, 0, ldev, filter) { - pf = mlx5_lag_pf(ldev, i); - if (!(pf->dev->priv.flags & flags)) { - struct mlx5_eswitch *esw; - - esw = pf->dev->priv.eswitch; - mlx5_esw_reps_block(esw); - ret = mlx5_eswitch_reload_ib_reps(esw); - mlx5_esw_reps_unblock(esw); - if (ret && !cont_on_fail) - return ret; - } + if (i == master_idx) + continue; + ret = mlx5_lag_reload_ib_reps_idx(ldev, i, flags); + if (ret && !cont_on_fail) + return ret; } return 0; @@ -1471,6 +1494,7 @@ static void mlx5_lag_modify_device_vports_speed(struct mlx5_core_dev *mdev, if (!MLX5_CAP_ESW(mdev, esw_vport_state_max_tx_speed)) return; + mutex_lock(&esw->state_lock); mlx5_esw_for_each_vport(esw, i, vport) { if (!vport) continue; @@ -1490,6 +1514,7 @@ static void mlx5_lag_modify_device_vports_speed(struct mlx5_core_dev *mdev, "Failed to set vport %d speed %d, err=%d\n", vport->vport, speed, ret); } + mutex_unlock(&esw->state_lock); } void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c index 6b4ad3c53f2f3..424040918fa37 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c @@ -270,6 +270,7 @@ int mlx5_lag_shared_fdb_create(struct mlx5_lag *ldev, pf->sd_fdb_active = false; } mlx5_lag_destroy_single_fdb_filter(ldev, group_id); + mlx5_lag_unload_reps_from_locked(ldev, filter); } err_add_devices: mlx5_lag_add_devices_filter(ldev, filter); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c index 64f92427602de..75855481522b1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c @@ -37,6 +37,7 @@ struct mlx5_devcom_comp { struct mlx5_devcom_key key; mlx5_devcom_event_handler_t handler; struct kref ref; + int nr_devs; bool ready; struct rw_semaphore sem; struct lock_class_key lock_key; @@ -170,6 +171,7 @@ devcom_alloc_comp_dev(struct mlx5_devcom_dev *devc, down_write(&comp->sem); list_add_tail(&devcom->list, &comp->comp_dev_list_head); + WRITE_ONCE(comp->nr_devs, comp->nr_devs + 1); up_write(&comp->sem); return devcom; @@ -182,6 +184,7 @@ devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom) down_write(&comp->sem); list_del(&devcom->list); + WRITE_ONCE(comp->nr_devs, comp->nr_devs - 1); up_write(&comp->sem); kref_put(&devcom->devc->ref, mlx5_devcom_dev_release); @@ -284,7 +287,7 @@ int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom) { struct mlx5_devcom_comp *comp = devcom->comp; - return kref_read(&comp->ref); + return READ_ONCE(comp->nr_devs); } int mlx5_devcom_locked_send_event(struct mlx5_devcom_comp_dev *devcom, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index 3676e26ac6b09..8aa94ec87a0e3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -89,6 +89,34 @@ int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, { u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)] = {}; +#ifdef CONFIG_MLX5_ESWITCH + lockdep_assert_held(&mdev->priv.eswitch->state_lock); +#endif + + if (MLX5_CAP_ESW(mdev, esw_vport_state_max_tx_speed) && + opmod == MLX5_VPORT_STATE_OP_MOD_ESW_VPORT && + vport != MLX5_VPORT_UPLINK) { + u32 speed = 0; + int err; + + err = mlx5_query_vport_max_tx_speed(mdev, opmod, vport, + other_vport, &speed, NULL); + if (err) { +#ifdef CONFIG_MLX5_ESWITCH + struct mlx5_vport *esw_vport; + + esw_vport = mlx5_eswitch_get_vport(mdev->priv.eswitch, + vport); + speed = IS_ERR(esw_vport) ? 0 : + esw_vport->agg_max_tx_speed; +#endif + mlx5_core_dbg(mdev, + "Failed to query vport %d max tx speed, err=%d, using cached %u\n", + vport, err, speed); + } + MLX5_SET(modify_vport_state_in, in, max_tx_speed, speed); + } + MLX5_SET(modify_vport_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VPORT_STATE); MLX5_SET(modify_vport_state_in, in, op_mod, opmod); @@ -106,6 +134,10 @@ int mlx5_modify_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 opmod, u8 admin_state; int err; +#ifdef CONFIG_MLX5_ESWITCH + lockdep_assert_held(&mdev->priv.eswitch->state_lock); +#endif + err = mlx5_query_vport_admin_state(mdev, opmod, vport, other_vport, &admin_state); if (err) @@ -123,11 +155,11 @@ int mlx5_modify_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 opmod, } int mlx5_query_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 op_mod, - u16 vport, u8 other_vport, u32 *max_tx_speed) + u16 vport, u8 other_vport, + u32 *max_tx_speed, u8 *state) { u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; - u32 state; int err; MLX5_SET(query_vport_state_in, in, opcode, @@ -140,13 +172,9 @@ int mlx5_query_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 op_mod, if (err) return err; - state = MLX5_GET(query_vport_state_out, out, state); - if (state == VPORT_STATE_DOWN) { - *max_tx_speed = 0; - return 0; - } - *max_tx_speed = MLX5_GET(query_vport_state_out, out, max_tx_speed); + if (state) + *state = MLX5_GET(query_vport_state_out, out, state); return 0; } EXPORT_SYMBOL_GPL(mlx5_query_vport_max_tx_speed); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c index 9939749c47bcc..9c5862f4e16a3 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c @@ -572,6 +572,38 @@ mlxsw_sp1_ptp_unmatched_remove(struct mlxsw_sp *mlxsw_sp, mlxsw_sp1_ptp_unmatched_ht_params); } +/* mlxsw_sp1_ptp_packet_finish() is reached both from the NAPI poll context + * (mlxsw_sp1_ptp_got_packet(), mlxsw_sp1_ptp_got_piece() and + * mlxsw_sp1_packet_timestamp()) and from process context, by way of the GC + * workqueue (mlxsw_sp1_ptp_ht_gc_collect() -> + * mlxsw_sp1_ptp_unmatched_finish()). + * + * mlxsw_sp_rx_listener_no_mark_func() ends in napi_gro_receive(), using the + * NAPI pointer that was placed in the SKB control block when the trapped + * packet was received in the NAPI context. That pointer may only be used + * from its own poll context, which this call site cannot guarantee. + * + * netif_receive_skb(), unlike napi_gro_receive(), can be called from outside + * of the NAPI instance's poll context. RX stats accounting and the skb->dev + * assignment are still preserved; the only change is the delivery call. + */ +static void mlxsw_sp1_ptp_rx_finish(struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb) +{ + struct mlxsw_sp_port_pcpu_stats *pcpu_stats; + + skb->dev = mlxsw_sp_port->dev; + + pcpu_stats = this_cpu_ptr(mlxsw_sp_port->pcpu_stats); + u64_stats_update_begin(&pcpu_stats->syncp); + pcpu_stats->rx_packets++; + pcpu_stats->rx_bytes += skb->len; + u64_stats_update_end(&pcpu_stats->syncp); + + skb->protocol = eth_type_trans(skb, skb->dev); + netif_receive_skb(skb); +} + /* This function is called in the following scenarios: * * 1) When a packet is matched with its timestamp. @@ -600,7 +632,7 @@ static void mlxsw_sp1_ptp_packet_finish(struct mlxsw_sp *mlxsw_sp, if (ingress) { if (hwtstamps) *skb_hwtstamps(skb) = *hwtstamps; - mlxsw_sp_rx_listener_no_mark_func(skb, local_port, mlxsw_sp); + mlxsw_sp1_ptp_rx_finish(mlxsw_sp_port, skb); } else { /* skb_tstamp_tx() allows hwtstamps to be NULL. */ skb_tstamp_tx(skb, hwtstamps); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index 64b958df77744..baba3471bf5a4 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -974,6 +974,7 @@ enum { /* PUL User Registers */ #define FBNIC_CSR_START_PUL_USER 0x31000 /* CSR section delimiter */ #define FBNIC_PUL_OB_TLP_HDR_AW_CFG 0x3103d /* 0xc40f4 */ +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH_MODE CSR_BIT(20) #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH CSR_BIT(19) #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_BME CSR_BIT(18) #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_RDE_ATTR CSR_GENMASK(17, 15) @@ -1215,6 +1216,10 @@ enum { #define FBNIC_IPC_MBX_DESC_LEN_MASK DESC_GENMASK(63, 48) #define FBNIC_IPC_MBX_DESC_EOM DESC_BIT(46) #define FBNIC_IPC_MBX_DESC_ADDR_MASK DESC_GENMASK(45, 3) +/* Set with FW_CMPL when the FW completed a descriptor without successfully + * processing it (e.g. a mailbox DMA error); the completion has no valid data. + */ +#define FBNIC_IPC_MBX_DESC_FW_ERR DESC_BIT(2) #define FBNIC_IPC_MBX_DESC_FW_CMPL DESC_BIT(1) #define FBNIC_IPC_MBX_DESC_HOST_CMPL DESC_BIT(0) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c index 3c4563c8f403f..6edfa0aa69f11 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c @@ -539,8 +539,8 @@ static void fbnic_dbg_fw_mbx_display(struct seq_file *s, /* Generate header */ seq_puts(s, mbx_idx == FBNIC_IPC_MBX_RX_IDX ? "Rx\n" : "Tx\n"); - seq_printf(s, "Rdy: %d Head: %d Tail: %d\n", - mbx->ready, mbx->head, mbx->tail); + seq_printf(s, "Rdy: %d Head: %d Tail: %d resp_error: %llu\n", + mbx->ready, mbx->head, mbx->tail, mbx->resp_error); snprintf(hdr, sizeof(hdr), "%3s %-4s %s %-12s %s %-3s %-16s\n", "Idx", "Len", "E", "Addr", "F", "H", "Raw"); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 0e47088ec44ba..76e9a545bb165 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -313,6 +313,11 @@ fbnic_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, kernel_ring->hds_thresh = fbn->hds_thresh; } +static u32 fbnic_ring_size_pow2(u32 size) +{ + return size ? roundup_pow_of_two(size) : 0; +} + static void fbnic_set_rings(struct fbnic_net *fbn, struct ethtool_ringparam *ring, struct kernel_ethtool_ringparam *kernel_ring) @@ -334,10 +339,10 @@ fbnic_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, struct fbnic_net *clone; int err; - ring->rx_pending = roundup_pow_of_two(ring->rx_pending); - ring->rx_mini_pending = roundup_pow_of_two(ring->rx_mini_pending); - ring->rx_jumbo_pending = roundup_pow_of_two(ring->rx_jumbo_pending); - ring->tx_pending = roundup_pow_of_two(ring->tx_pending); + ring->rx_pending = fbnic_ring_size_pow2(ring->rx_pending); + ring->rx_mini_pending = fbnic_ring_size_pow2(ring->rx_mini_pending); + ring->rx_jumbo_pending = fbnic_ring_size_pow2(ring->rx_jumbo_pending); + ring->tx_pending = fbnic_ring_size_pow2(ring->tx_pending); /* These are absolute minimums allowing the device and driver to operate * but not necessarily guarantee reasonable performance. Settings below @@ -2025,7 +2030,8 @@ static const struct ethtool_ops fbnic_ethtool_ops = { ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM | ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | - ETHTOOL_OP_NEEDS_RTNL_GLINK, + ETHTOOL_OP_NEEDS_RTNL_GLINK | + ETHTOOL_OP_NEEDS_RTNL_TEST, .get_drvinfo = fbnic_get_drvinfo, .get_regs_len = fbnic_get_regs_len, .get_regs = fbnic_get_regs, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c index 283d25fae79e7..6d7eb8479edf4 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c @@ -60,8 +60,15 @@ static void fbnic_mbx_reset_desc_ring(struct fbnic_dev *fbd, int mbx_idx) */ switch (mbx_idx) { case FBNIC_IPC_MBX_RX_IDX: + /* Clearing BME blocks the device from writing to the host + * but leaves the requests parked in the write pipeline. The + * write path only clears outstanding requests when both FLUSH + * and FLUSH_MODE are set; FLUSH_MODE lets them drain without + * landing on the host. + */ wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AW_CFG, - FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH); + FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH | + FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH_MODE); break; case FBNIC_IPC_MBX_TX_IDX: wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AR_CFG, @@ -285,6 +292,12 @@ static void fbnic_mbx_process_tx_msgs(struct fbnic_dev *fbd) if (!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL)) break; + if (desc & FBNIC_IPC_MBX_DESC_FW_ERR) { + tx_mbx->resp_error++; + dev_warn_ratelimited(fbd->dev, + "FW completed a Tx mailbox request with an error\n"); + } + fbnic_mbx_unmap_and_free_msg(fbd, FBNIC_IPC_MBX_TX_IDX, head); head++; @@ -1666,6 +1679,13 @@ static void fbnic_mbx_process_rx_msgs(struct fbnic_dev *fbd) if (!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL)) break; + if (desc & FBNIC_IPC_MBX_DESC_FW_ERR) { + rx_mbx->resp_error++; + dev_warn_ratelimited(fbd->dev, + "FW reported an error on an Rx mailbox message; dropping\n"); + goto next_page; + } + dma_sync_single_for_cpu(fbd->dev, rx_mbx->buf_info[head].addr, FBNIC_RX_PAGE_SIZE, DMA_FROM_DEVICE); @@ -1733,7 +1753,9 @@ void fbnic_mbx_poll(struct fbnic_dev *fbd) int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) { struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; + struct fbnic_fw_mbx *rx_mbx = &fbd->mbx[FBNIC_IPC_MBX_RX_IDX]; unsigned long timeout = jiffies + 10 * HZ + 1; + u64 tx_resp_error, rx_resp_error; int err, i; do { @@ -1764,6 +1786,9 @@ int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) * mgmt.version once we get the actual version from the firmware * in the capabilities request message. */ +send_cap_req: + tx_resp_error = tx_mbx->resp_error; + rx_resp_error = rx_mbx->resp_error; err = fbnic_fw_xmit_simple_msg(fbd, FBNIC_TLV_MSG_ID_HOST_CAP_REQ); if (err) goto clean_mbx; @@ -1781,9 +1806,27 @@ int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) msleep(20); fbnic_mbx_poll(fbd); + /* A valid capabilities response ends the poll. Check it + * before the FW_ERR retry below so a response parsed in the + * same poll as an unrelated FW_ERR is not discarded. + */ + if (fbd->fw_cap.running.mgmt.version >= MIN_FW_VER_CODE) + break; + /* set err, but wait till mgmt.version check to report it */ - if (!time_is_after_jiffies(timeout)) + if (!time_is_after_jiffies(timeout)) { err = -ETIMEDOUT; + continue; + } + + /* The FW can flag our capabilities request (Tx) or its + * response (Rx) with FW_ERR, in which case it produced no + * usable response. The ring is not wedged, so re-issue the + * request instead of spinning until the timeout. + */ + if (tx_mbx->resp_error != tx_resp_error || + rx_mbx->resp_error != rx_resp_error) + goto send_cap_req; } return 0; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h index d84723e4cfa36..5f9969247e305 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h @@ -13,6 +13,7 @@ struct fbnic_tlv_msg; struct fbnic_fw_mbx { u8 ready, head, tail; + u64 resp_error; struct { struct fbnic_tlv_msg *msg; dma_addr_t addr; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 8b9bc9e8ea56c..c6698e3002a13 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -434,6 +434,7 @@ static int fbnic_pm_suspend(struct device *dev) { struct fbnic_dev *fbd = dev_get_drvdata(dev); struct net_device *netdev = fbd->netdev; + struct fbnic_net *fbn; if (fbnic_init_failure(fbd)) goto null_uc_addr; @@ -441,11 +442,16 @@ static int fbnic_pm_suspend(struct device *dev) rtnl_lock(); netdev_lock(netdev); + fbn = netdev_priv(netdev); + netif_device_detach(netdev); if (netif_running(netdev)) netdev->netdev_ops->ndo_stop(netdev); + /* The IRQs are about to be freed, so drop the napi vector count */ + fbn->num_napi = 0; + netdev_unlock(netdev); rtnl_unlock(); @@ -508,16 +514,20 @@ static int __fbnic_pm_resume(struct device *dev) if (fbnic_init_failure(fbd)) return 0; + rtnl_lock(); + netdev_lock(netdev); + fbn = netdev_priv(netdev); /* Reset the queues if needed */ fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues); - rtnl_lock(); - netdev_lock(netdev); - - if (netif_running(netdev)) + if (netif_running(netdev)) { err = __fbnic_open(fbn); + /* On failure the vectors are freed, so drop the count */ + if (err) + fbn->num_napi = 0; + } netdev_unlock(netdev); rtnl_unlock(); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index 401f8b8ae1cae..10caacffee0f0 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -311,6 +312,29 @@ fbnic_rx_csum(u64 rcd, struct sk_buff *skb, struct fbnic_ring *rcq, } } +static void fbnic_tx_doorbell(struct fbnic_ring *ring, __le64 *meta) +{ + *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_COMPLETION); + ring->deferred_meta = -1; + + /* Force DMA writes to flush before writing to tail */ + dma_wmb(); + + writel(ring->tail, ring->doorbell); +} + +/* Packets handed to us with xmit_more set are left in the ring without a + * doorbell, and without a completion request, in the expectation that the + * packet ending the burst will ring for all of them. If that packet gets + * dropped instead we have to ring here, otherwise the descriptors sit in + * the ring until the next transmit, which may never come. + */ +static void fbnic_tx_flush_doorbell(struct fbnic_ring *ring) +{ + if (ring->deferred_meta >= 0) + fbnic_tx_doorbell(ring, &ring->desc[ring->deferred_meta]); +} + static bool fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) { @@ -378,14 +402,10 @@ fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) /* Verify there is room for another packet */ fbnic_maybe_stop_tx(skb->dev, ring, FBNIC_MAX_SKB_DESC); - if (fbnic_tx_sent_queue(skb, ring)) { - *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_COMPLETION); - - /* Force DMA writes to flush before writing to tail */ - dma_wmb(); - - writel(tail, ring->doorbell); - } + if (fbnic_tx_sent_queue(skb, ring)) + fbnic_tx_doorbell(ring, meta); + else + ring->deferred_meta = meta - ring->desc; return false; dma_error: @@ -425,8 +445,10 @@ fbnic_xmit_frame_ring(struct sk_buff *skb, struct fbnic_ring *ring) * otherwise try next time */ desc_needed = skb_shinfo(skb)->nr_frags + 10; - if (fbnic_maybe_stop_tx(skb->dev, ring, desc_needed)) + if (fbnic_maybe_stop_tx(skb->dev, ring, desc_needed)) { + fbnic_tx_flush_doorbell(ring); return NETDEV_TX_BUSY; + } *meta = cpu_to_le64(FBNIC_TWD_FLAG_DEST_MAC); @@ -447,6 +469,8 @@ fbnic_xmit_frame_ring(struct sk_buff *skb, struct fbnic_ring *ring) err_free: dev_kfree_skb_any(skb); err_count: + fbnic_tx_flush_doorbell(ring); + u64_stats_update_begin(&ring->stats.syncp); ring->stats.dropped++; u64_stats_update_end(&ring->stats.syncp); @@ -1599,7 +1623,7 @@ fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, return 0; err_destroy_sub0: - page_pool_destroy(pp); + page_pool_destroy(qt->sub0.page_pool); return PTR_ERR(pp); } @@ -1768,7 +1792,7 @@ int fbnic_alloc_napi_vectors(struct fbnic_net *fbn) int err; /* Allocate 1 Tx queue per napi vector */ - if (num_napi < FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) { + if (num_napi <= FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) { while (num_tx) { err = fbnic_alloc_napi_vector(fbd, fbn, num_napi, v_idx, @@ -2491,6 +2515,7 @@ static void fbnic_enable_twq0(struct fbnic_ring *twq) fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_RESET); twq->tail = 0; twq->head = 0; + twq->deferred_meta = -1; /* Store descriptor ring address and size */ fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_BAL, lower_32_bits(twq->dma)); @@ -2829,6 +2854,17 @@ void fbnic_napi_depletion_check(struct net_device *netdev) fbnic_wrfl(fbd); } +/* Returns the napi vector servicing an Rx queue, or NULL if the datapath + * is torn down. The association is published by fbnic_set_netif_napi() + * and cleared by fbnic_reset_netif_napi(), both under the instance lock. + */ +static struct fbnic_napi_vector *fbnic_rxq_nv(struct net_device *dev, int idx) +{ + struct napi_struct *napi = __netif_get_rx_queue(dev, idx)->napi; + + return napi ? container_of(napi, struct fbnic_napi_vector, napi) : NULL; +} + static int fbnic_queue_mem_alloc(struct net_device *dev, struct netdev_queue_config *qcfg, void *qmem, int idx) @@ -2841,8 +2877,16 @@ static int fbnic_queue_mem_alloc(struct net_device *dev, if (!netif_running(dev)) return fbnic_alloc_qt_page_pools(fbn, qt, idx); + /* A failed PCIe recovery or resume can leave the datapath torn down + * while netif_running() is still true. This ndo runs before + * netdev_rx_queue_restart() checks netif_running(), so bail out + * rather than touching rings and vectors that are already freed. + */ + nv = fbnic_rxq_nv(dev, idx); + if (!nv) + return -ENETDOWN; + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); - nv = fbn->napi[idx % fbn->num_napi]; fbnic_ring_init(&qt->sub0, real->sub0.doorbell, real->sub0.q_idx, real->sub0.flags); @@ -2893,7 +2937,7 @@ static int fbnic_queue_start(struct net_device *dev, struct fbnic_q_triad *real; real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); - nv = fbn->napi[idx % fbn->num_napi]; + nv = fbnic_rxq_nv(dev, idx); fbnic_aggregate_ring_bdq_counters(fbn, &real->sub0); fbnic_aggregate_ring_bdq_counters(fbn, &real->sub1); @@ -2915,7 +2959,7 @@ static int fbnic_queue_stop(struct net_device *dev, void *qmem, int idx) int err; real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); - nv = fbn->napi[idx % fbn->num_napi]; + nv = fbnic_rxq_nv(dev, idx); fbnic_dbg_nv_exit(nv); napi_disable_locked(&nv->napi); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h index e03c9d2c38dca..f5899446dcc51 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -128,9 +128,14 @@ struct fbnic_ring { /* Rx BDQs only */ struct page_pool *page_pool; - /* Deferred_head is used to cache the head for TWQ1 if + /* TWQ0 only, index of the meta descriptor of the last packet + * placed in the ring without ringing the doorbell, -1 if the + * doorbell is in sync with the tail. + */ + s32 deferred_meta; + + /* TCQ only, used to cache the head for TWQ1 if * an attempt is made to clean TWQ1 with zero napi_budget. - * We do not use it for any other ring. */ s32 deferred_head; }; diff --git a/drivers/net/ethernet/micrel/ks8851_common.c b/drivers/net/ethernet/micrel/ks8851_common.c index 4afbb40bc0e4a..d49f281c78676 100644 --- a/drivers/net/ethernet/micrel/ks8851_common.c +++ b/drivers/net/ethernet/micrel/ks8851_common.c @@ -143,9 +143,6 @@ static int ks8851_write_mac_addr(struct net_device *dev) ks8851_wrreg16(ks, KS_MAR(i), val); } - if (!netif_running(dev)) - ks8851_set_powermode(ks, PMECR_PM_SOFTDOWN); - ks8851_unlock(ks); return 0; @@ -478,8 +475,7 @@ static int ks8851_net_open(struct net_device *dev) * @dev: The device being closed. * * Called to close down a network device which has been active. Cancel any - * work, shutdown the RX and TX process and then place the chip into a low - * power state whilst it is not being used. + * work and shutdown the RX and TX process. */ static int ks8851_net_stop(struct net_device *dev) { @@ -506,8 +502,6 @@ static int ks8851_net_stop(struct net_device *dev) /* shutdown TX process */ ks8851_wrreg16(ks, KS_TXCR, 0x0000); - /* set powermode to soft power down to save power */ - ks8851_set_powermode(ks, PMECR_PM_SOFTDOWN); ks8851_unlock(ks); /* ensure any queued tx buffers are dumped */ diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index e759171bfd766..2d6e6e8e16f2e 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -2594,7 +2594,7 @@ static int lan743x_rx_process_buffer(struct lan743x_rx *rx) rx->adapter->netdev); if (rx->adapter->netdev->features & NETIF_F_RXCSUM) { if (!is_ice && !is_tce && !is_icsm) - skb->ip_summed = CHECKSUM_UNNECESSARY; + rx->skb_head->ip_summed = CHECKSUM_UNNECESSARY; } netdev_dbg(netdev, "sending %d byte frame to OS", rx->skb_head->len); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c index 1d34af78166a3..4dedf5d926416 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c @@ -162,13 +162,18 @@ static int sparx5_port_stop(struct net_device *ndev) return 0; } -static void sparx5_set_rx_mode(struct net_device *dev) +static int sparx5_set_rx_mode(struct net_device *dev, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc) { struct sparx5_port *port = netdev_priv(dev); struct sparx5 *sparx5 = port->sparx5; if (!test_bit(port->portno, sparx5->bridge_mask)) - __dev_mc_sync(dev, sparx5_mc_sync, sparx5_mc_unsync); + return __hw_addr_sync_dev(mc, dev, sparx5_mc_sync, + sparx5_mc_unsync); + + return 0; } static int sparx5_port_get_phys_port_name(struct net_device *dev, @@ -249,7 +254,7 @@ static const struct net_device_ops sparx5_port_netdev_ops = { .ndo_open = sparx5_port_open, .ndo_stop = sparx5_port_stop, .ndo_start_xmit = sparx5_port_xmit_impl, - .ndo_set_rx_mode = sparx5_set_rx_mode, + .ndo_set_rx_mode_async = sparx5_set_rx_mode, .ndo_get_phys_port_name = sparx5_port_get_phys_port_name, .ndo_set_mac_address = sparx5_set_mac_address, .ndo_validate_addr = eth_validate_addr, diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c index cf332de6bf736..2dee2ce19fce7 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c @@ -2077,8 +2077,9 @@ int sparx5_vcap_init(struct sparx5 *sparx5) dir = vcap_debugfs(sparx5->dev, sparx5->debugfs_root, ctrl); for (idx = 0; idx < consts->n_ports; ++idx) if (sparx5->ports[idx]) - vcap_port_debugfs(sparx5->dev, dir, ctrl, - sparx5->ports[idx]->ndev); + vcap_port_debugfs_portno(sparx5->dev, dir, ctrl, + sparx5->ports[idx]->ndev, + sparx5->ports[idx]->portno); return err; } diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c index e0c65c7ab23e1..476f7496a9d44 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.c @@ -400,6 +400,27 @@ void vcap_port_debugfs(struct device *dev, struct dentry *parent, } EXPORT_SYMBOL_GPL(vcap_port_debugfs); +void vcap_port_debugfs_portno(struct device *dev, + struct dentry *parent, + struct vcap_control *vctrl, + struct net_device *ndev, + unsigned int portno) +{ + struct vcap_port_debugfs_info *info; + char name[16]; + + info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); + if (!info) + return; + + info->vctrl = vctrl; + info->ndev = ndev; + + snprintf(name, sizeof(name), "p%u", portno); + debugfs_create_file(name, 0444, parent, info, &vcap_port_debugfs_fops); +} +EXPORT_SYMBOL_GPL(vcap_port_debugfs_portno); + /* Show the full VCAP instance data (rules with all fields) */ static int vcap_debugfs_show(struct seq_file *m, void *unused) { diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.h b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.h index 9f2c59b5f6f53..7dc6e3411a4d9 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.h +++ b/drivers/net/ethernet/microchip/vcap/vcap_api_debugfs.h @@ -18,6 +18,12 @@ void vcap_port_debugfs(struct device *dev, struct dentry *parent, struct vcap_control *vctrl, struct net_device *ndev); +void vcap_port_debugfs_portno(struct device *dev, + struct dentry *parent, + struct vcap_control *vctrl, + struct net_device *ndev, + unsigned int portno); + /* Create a debugFS entry for a vcap instance */ struct dentry *vcap_debugfs(struct device *dev, struct dentry *parent, struct vcap_control *vctrl); @@ -30,6 +36,14 @@ static inline void vcap_port_debugfs(struct device *dev, struct dentry *parent, { } +static inline void vcap_port_debugfs_portno(struct device *dev, + struct dentry *parent, + struct vcap_control *vctrl, + struct net_device *ndev, + unsigned int portno) +{ +} + static inline struct dentry *vcap_debugfs(struct device *dev, struct dentry *parent, struct vcap_control *vctrl) diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index e8b7ffb47eb98..6a150a8938006 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -180,6 +180,7 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev) struct gdma_query_max_resources_resp resp = {}; struct gdma_general_req req = {}; unsigned int max_num_queues; + unsigned int msix_vec_count; u8 bm_hostmode; u16 num_ports; int err; @@ -216,6 +217,24 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev) gc->num_msix_usable = min(resp.max_msix, num_online_cpus() + 1); } + /* MSI-X vectors are allocated by index into the device MSI-X table, so + * never ask for more than the table holds. It can be smaller than both + * resp.max_msix and the CPU count. + */ + err = pci_msix_vec_count(pdev); + if (err <= 0) { + dev_err(gc->dev, "Failed to query MSI-X table size: %d\n", err); + return err < 0 ? err : -ENOSPC; + } + msix_vec_count = err; + + if (gc->num_msix_usable > msix_vec_count) { + dev_info(gc->dev, + "Limiting MSI-X vectors from %u to table size %u\n", + gc->num_msix_usable, msix_vec_count); + gc->num_msix_usable = msix_vec_count; + } + if (gc->num_msix_usable <= 1) return -ENOSPC; diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c index 53308e139cbe9..5c9961ee9747a 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c +++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c @@ -208,6 +208,7 @@ static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog, if (err) { NL_SET_ERR_MSG_MOD(extack, "XDP: Insufficient memory for tx/rx re-config"); + apc->bpf_prog = old_prog; return err; } diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 92bb55935c1c4..8fb29dcddf8e2 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -2756,6 +2756,10 @@ static int mana_alloc_rx_wqe(struct mana_port_context *apc, *cq_size += COMP_ENTRY_SIZE; } + /* Reserve an extra slot for Fence completion + * event (CQE_RX_OBJECT_FENCE) in case RX CQ is full. + */ + *cq_size += COMP_ENTRY_SIZE; return 0; } @@ -2850,7 +2854,7 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, goto out; rq_size = MANA_PAGE_ALIGN(rq_size); - cq_size = MANA_PAGE_ALIGN(cq_size); + cq_size = MANA_PAGE_ALIGN(roundup_pow_of_two(cq_size)); /* Create RQ */ memset(&spec, 0, sizeof(spec)); @@ -3728,7 +3732,8 @@ static void mana_rdma_service_handle(struct work_struct *work) struct device *dev = gd->gdma_context->dev; int ret; - if (READ_ONCE(gd->rdma_teardown)) + /* Pairs with the smp_store_release() in mana_rdma_probe(). */ + if (smp_load_acquire(&gd->rdma_teardown)) goto out; switch (serv_work->event) { @@ -4024,6 +4029,21 @@ int mana_rdma_probe(struct gdma_dev *gd) if (err) return err; + /* Clear the state left by a previous mana_rdma_remove() so servicing + * events are handled again after a reset cycle. + */ + gd->is_suspended = false; + + /* Publish is_suspended before re-opening the gate, so the handler + * cannot observe an open gate with a stale is_suspended. Pairs + * with the smp_load_acquire() in mana_rdma_service_handle(). This + * matters on the reset path, where mana_rdma_remove() closed the + * gate and drained the workqueue; on the initial probe path the + * gate was never closed and both flags are already clear. It does + * not order gd->adev, which add_adev() publishes below. + */ + smp_store_release(&gd->rdma_teardown, false); + err = add_adev(gd, "rdma"); if (err) mana_gd_deregister_device(gd); diff --git a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c index 9e7c285eaa6bc..960d7513aa8dd 100644 --- a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c +++ b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c @@ -625,11 +625,12 @@ int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb) xa_lock(&nn->xa_ipsec); x = xa_load(&nn->xa_ipsec, saidx); + if (x) + xfrm_state_hold(x); xa_unlock(&nn->xa_ipsec); if (!x) return -EINVAL; - xfrm_state_hold(x); sp->xvec[sp->len++] = x; sp->olen++; xo = xfrm_offload(skb); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index a2a89d48e3cac..4e83637715e03 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -1421,7 +1421,8 @@ static int nfp_net_get_fs_rule(struct nfp_net *nn, struct ethtool_rxnfc *cmd) return -ENOENT; } -static int nfp_net_get_fs_loc(struct nfp_net *nn, u32 *rule_locs) +static int nfp_net_get_fs_loc(struct nfp_net *nn, struct ethtool_rxnfc *cmd, + u32 *rule_locs) { struct nfp_fs_entry *entry; u32 count = 0; @@ -1429,8 +1430,12 @@ static int nfp_net_get_fs_loc(struct nfp_net *nn, u32 *rule_locs) if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_FLOW_STEER)) return -EOPNOTSUPP; - list_for_each_entry(entry, &nn->fs.list, node) + list_for_each_entry(entry, &nn->fs.list, node) { + if (count == cmd->rule_cnt) + return -EMSGSIZE; rule_locs[count++] = entry->loc; + } + cmd->rule_cnt = count; return 0; } @@ -1455,7 +1460,7 @@ static int nfp_net_get_rxnfc(struct net_device *netdev, return nfp_net_get_fs_rule(nn, cmd); case ETHTOOL_GRXCLSRLALL: cmd->data = NFP_FS_MAX_ENTRY; - return nfp_net_get_fs_loc(nn, rule_locs); + return nfp_net_get_fs_loc(nn, cmd, rule_locs); default: return -EOPNOTSUPP; } @@ -1698,8 +1703,14 @@ static int nfp_net_fs_add(struct nfp_net *nn, struct ethtool_rxnfc *cmd) nn->fs.count--; err = nfp_net_fs_add_hw(nn, new); - if (err) + if (err) { + /* mbox broken, adding the old rule back will + * likely also fail. + */ + list_del(&entry->node); + kfree(entry); goto err; + } nn->fs.count++; list_replace(&entry->node, &new->node); diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c index 58d3e55def486..f0218a0eab5ca 100644 --- a/drivers/net/ethernet/nvidia/forcedeth.c +++ b/drivers/net/ethernet/nvidia/forcedeth.c @@ -2740,7 +2740,7 @@ static void nv_tx_timeout(struct net_device *dev, unsigned int txqueue) netdev_info(dev, "Ring at %lx\n", (unsigned long)np->ring_addr); netdev_info(dev, "Dumping tx registers\n"); - for (i = 0; i <= np->register_size; i += 32) { + for (i = 0; i + 32 <= np->register_size; i += 32) { netdev_info(dev, "%3x: %08x %08x %08x %08x " "%08x %08x %08x %08x\n", @@ -6221,7 +6221,7 @@ static int nv_suspend(struct device *device) netif_device_detach(dev); /* save non-pci configuration space */ - for (i = 0; i <= np->register_size/sizeof(u32); i++) + for (i = 0; i < np->register_size/sizeof(u32); i++) np->saved_config_space[i] = readl(base + i*sizeof(u32)); return 0; @@ -6236,7 +6236,7 @@ static int nv_resume(struct device *device) int i, rc = 0; /* restore non-pci configuration space */ - for (i = 0; i <= np->register_size/sizeof(u32); i++) + for (i = 0; i < np->register_size/sizeof(u32); i++) writel(np->saved_config_space[i], base+i*sizeof(u32)); if (np->driver_data & DEV_NEED_MSI_FIX) diff --git a/drivers/net/ethernet/oa_tc6.c b/drivers/net/ethernet/oa_tc6.c index 0727d53345a33..f552165e503e1 100644 --- a/drivers/net/ethernet/oa_tc6.c +++ b/drivers/net/ethernet/oa_tc6.c @@ -12,46 +12,6 @@ #include #include -/* OPEN Alliance TC6 registers */ -/* Standard Capabilities Register */ -#define OA_TC6_REG_STDCAP 0x0002 -#define STDCAP_DIRECT_PHY_REG_ACCESS BIT(8) - -/* Reset Control and Status Register */ -#define OA_TC6_REG_RESET 0x0003 -#define RESET_SWRESET BIT(0) /* Software Reset */ - -/* Configuration Register #0 */ -#define OA_TC6_REG_CONFIG0 0x0004 -#define CONFIG0_SYNC BIT(15) -#define CONFIG0_ZARFE_ENABLE BIT(12) - -/* Status Register #0 */ -#define OA_TC6_REG_STATUS0 0x0008 -#define STATUS0_RESETC BIT(6) /* Reset Complete */ -#define STATUS0_HEADER_ERROR BIT(5) -#define STATUS0_LOSS_OF_FRAME_ERROR BIT(4) -#define STATUS0_RX_BUFFER_OVERFLOW_ERROR BIT(3) -#define STATUS0_TX_PROTOCOL_ERROR BIT(0) - -/* Buffer Status Register */ -#define OA_TC6_REG_BUFFER_STATUS 0x000B -#define BUFFER_STATUS_TX_CREDITS_AVAILABLE GENMASK(15, 8) -#define BUFFER_STATUS_RX_CHUNKS_AVAILABLE GENMASK(7, 0) - -/* Interrupt Mask Register #0 */ -#define OA_TC6_REG_INT_MASK0 0x000C -#define INT_MASK0_HEADER_ERR_MASK BIT(5) -#define INT_MASK0_LOSS_OF_FRAME_ERR_MASK BIT(4) -#define INT_MASK0_RX_BUFFER_OVERFLOW_ERR_MASK BIT(3) -#define INT_MASK0_TX_PROTOCOL_ERR_MASK BIT(0) -#define INT_MASK0_ALL_INTERRUPTS (GENMASK(5, 0) | \ - GENMASK(12, 7)) - -/* PHY Clause 22 registers base address and mask */ -#define OA_TC6_PHY_STD_REG_ADDR_BASE 0xFF00 -#define OA_TC6_PHY_STD_REG_ADDR_MASK 0x1F - /* Control command header */ #define OA_TC6_CTRL_HEADER_DATA_NOT_CTRL BIT(31) #define OA_TC6_CTRL_HEADER_WRITE_NOT_READ BIT(29) @@ -81,23 +41,17 @@ #define OA_TC6_DATA_FOOTER_END_BYTE_OFFSET GENMASK(13, 8) #define OA_TC6_DATA_FOOTER_TX_CREDITS GENMASK(5, 1) -/* PHY – Clause 45 registers memory map selector (MMS) as per table 6 in the - * OPEN Alliance specification. - */ -#define OA_TC6_PHY_C45_PCS_MMS2 2 /* MMD 3 */ -#define OA_TC6_PHY_C45_PMA_PMD_MMS3 3 /* MMD 1 */ -#define OA_TC6_PHY_C45_VS_PLCA_MMS4 4 /* MMD 31 */ -#define OA_TC6_PHY_C45_AUTO_NEG_MMS5 5 /* MMD 7 */ -#define OA_TC6_PHY_C45_POWER_UNIT_MMS6 6 /* MMD 13 */ - +#define OA_TC6_CTRL_PROT_REPLY_SIZE 4 #define OA_TC6_CTRL_HEADER_SIZE 4 #define OA_TC6_CTRL_REG_VALUE_SIZE 4 #define OA_TC6_CTRL_IGNORED_SIZE 4 #define OA_TC6_CTRL_MAX_REGISTERS 128 -#define OA_TC6_CTRL_SPI_BUF_SIZE (OA_TC6_CTRL_HEADER_SIZE +\ - (OA_TC6_CTRL_MAX_REGISTERS *\ - OA_TC6_CTRL_REG_VALUE_SIZE) +\ - OA_TC6_CTRL_IGNORED_SIZE) +#define OA_TC6_CTRL_SPI_BUF_SIZE (OA_TC6_CTRL_HEADER_SIZE +\ + (OA_TC6_CTRL_MAX_REGISTERS *\ + (OA_TC6_CTRL_REG_VALUE_SIZE +\ + OA_TC6_CTRL_PROT_REPLY_SIZE)) +\ + OA_TC6_CTRL_IGNORED_SIZE) + #define OA_TC6_CHUNK_PAYLOAD_SIZE 64 #define OA_TC6_DATA_HEADER_SIZE 4 #define OA_TC6_CHUNK_SIZE (OA_TC6_DATA_HEADER_SIZE +\ @@ -127,9 +81,10 @@ struct oa_tc6 { u16 spi_data_tx_buf_offset; u16 tx_credits; u8 rx_chunks_available; - bool rx_buf_overflow; + bool wait_until_start_valid; bool int_flag; bool disable_traffic; + bool prot_ctrl; }; enum oa_tc6_header_type { @@ -213,25 +168,36 @@ static void oa_tc6_update_ctrl_write_data(struct oa_tc6 *tc6, u32 value[], { __be32 *tx_buf = tc6->spi_ctrl_tx_buf + OA_TC6_CTRL_HEADER_SIZE; - for (int i = 0; i < length; i++) + for (int i = 0; i < length; i++) { *tx_buf++ = cpu_to_be32(value[i]); + if (tc6->prot_ctrl) + *tx_buf++ = cpu_to_be32(~value[i]); + } } -static u16 oa_tc6_calculate_ctrl_buf_size(u8 length) +static u16 oa_tc6_calculate_ctrl_buf_size(u8 length, bool ctrl_prot) { + u32 reply_size = OA_TC6_CTRL_REG_VALUE_SIZE; + + if (ctrl_prot) + reply_size += OA_TC6_CTRL_PROT_REPLY_SIZE; + /* Control command consists 4 bytes header + 4 bytes register value for - * each register + 4 bytes ignored value. + * each register (+ 4 bytes for the register value complement in case + * protected mode is used) + 4 bytes ignored value. */ - return OA_TC6_CTRL_HEADER_SIZE + OA_TC6_CTRL_REG_VALUE_SIZE * length + + return OA_TC6_CTRL_HEADER_SIZE + reply_size * length + OA_TC6_CTRL_IGNORED_SIZE; } static void oa_tc6_prepare_ctrl_spi_buf(struct oa_tc6 *tc6, u32 address, u32 value[], u8 length, - enum oa_tc6_register_op reg_op) + enum oa_tc6_register_op reg_op, + u16 buf_size) { __be32 *tx_buf = tc6->spi_ctrl_tx_buf; + memset(tx_buf, 0, buf_size); *tx_buf = oa_tc6_prepare_ctrl_header(address, length, reg_op); if (reg_op == OA_TC6_CTRL_REG_WRITE) @@ -254,10 +220,12 @@ static int oa_tc6_check_ctrl_write_reply(struct oa_tc6 *tc6, u8 size) return 0; } -static int oa_tc6_check_ctrl_read_reply(struct oa_tc6 *tc6, u8 size) +static int oa_tc6_check_ctrl_read_reply(struct oa_tc6 *tc6, u8 length) { - u32 *rx_buf = tc6->spi_ctrl_rx_buf + OA_TC6_CTRL_IGNORED_SIZE; - u32 *tx_buf = tc6->spi_ctrl_tx_buf; + __be32 *rx_buf = tc6->spi_ctrl_rx_buf + OA_TC6_CTRL_IGNORED_SIZE; + __be32 *tx_buf = tc6->spi_ctrl_tx_buf; + u32 complement; + u32 reply; /* The echoed control read header must match with the one that was * transmitted. @@ -265,6 +233,20 @@ static int oa_tc6_check_ctrl_read_reply(struct oa_tc6 *tc6, u8 size) if (*tx_buf != *rx_buf) return -EPROTO; + if (tc6->prot_ctrl) { + /* Skip past the echoed header to the value/complement pairs */ + rx_buf += 1; + for (int i = 0; i < length; i++) { + reply = be32_to_cpu(rx_buf[0]); + complement = be32_to_cpu(rx_buf[1]); + + if (complement != ~reply) + return -EPROTO; + + rx_buf += 2; + } + } + return 0; } @@ -274,8 +256,13 @@ static void oa_tc6_copy_ctrl_read_data(struct oa_tc6 *tc6, u32 value[], __be32 *rx_buf = tc6->spi_ctrl_rx_buf + OA_TC6_CTRL_IGNORED_SIZE + OA_TC6_CTRL_HEADER_SIZE; - for (int i = 0; i < length; i++) + for (int i = 0; i < length; i++) { value[i] = be32_to_cpu(*rx_buf++); + + /* skip complement word */ + if (tc6->prot_ctrl) + rx_buf++; + } } static int oa_tc6_perform_ctrl(struct oa_tc6 *tc6, u32 address, u32 value[], @@ -284,10 +271,10 @@ static int oa_tc6_perform_ctrl(struct oa_tc6 *tc6, u32 address, u32 value[], u16 size; int ret; - /* Prepare control command and copy to SPI control buffer */ - oa_tc6_prepare_ctrl_spi_buf(tc6, address, value, length, reg_op); + size = oa_tc6_calculate_ctrl_buf_size(length, tc6->prot_ctrl); - size = oa_tc6_calculate_ctrl_buf_size(length); + /* Prepare control command and copy to SPI control buffer */ + oa_tc6_prepare_ctrl_spi_buf(tc6, address, value, length, reg_op, size); /* Perform SPI transfer */ ret = oa_tc6_spi_transfer(tc6, OA_TC6_CTRL_HEADER, size); @@ -302,7 +289,7 @@ static int oa_tc6_perform_ctrl(struct oa_tc6 *tc6, u32 address, u32 value[], return oa_tc6_check_ctrl_write_reply(tc6, size); /* Check echoed/received control read command reply for errors */ - ret = oa_tc6_check_ctrl_read_reply(tc6, size); + ret = oa_tc6_check_ctrl_read_reply(tc6, length); if (ret) return ret; @@ -408,7 +395,7 @@ static int oa_tc6_check_phy_reg_direct_access_capability(struct oa_tc6 *tc6) if (ret) return ret; - if (!(regval & STDCAP_DIRECT_PHY_REG_ACCESS)) + if (!(regval & OA_TC6_STDCAP_DIRECT_PHY_REG_ACCESS)) return -ENODEV; return 0; @@ -423,7 +410,7 @@ static int oa_tc6_mdiobus_read(struct mii_bus *bus, int addr, int regnum) { struct oa_tc6 *tc6 = bus->priv; u32 regval; - bool ret; + int ret; ret = oa_tc6_read_register(tc6, OA_TC6_PHY_STD_REG_ADDR_BASE | (regnum & OA_TC6_PHY_STD_REG_ADDR_MASK), @@ -601,7 +588,7 @@ static int oa_tc6_read_status0(struct oa_tc6 *tc6) static int oa_tc6_sw_reset_macphy(struct oa_tc6 *tc6) { - u32 regval = RESET_SWRESET; + u32 regval = OA_TC6_RESET_SWRESET; int ret; ret = oa_tc6_write_register(tc6, OA_TC6_REG_RESET, regval); @@ -610,7 +597,7 @@ static int oa_tc6_sw_reset_macphy(struct oa_tc6 *tc6) /* Poll for soft reset complete for every 1ms until 1s timeout */ ret = readx_poll_timeout(oa_tc6_read_status0, tc6, regval, - regval & STATUS0_RESETC, + regval & OA_TC6_STATUS0_RESETC, STATUS0_RESETC_POLL_DELAY, STATUS0_RESETC_POLL_TIMEOUT); if (ret) @@ -629,10 +616,10 @@ static int oa_tc6_unmask_macphy_error_interrupts(struct oa_tc6 *tc6) if (ret) return ret; - regval &= ~(INT_MASK0_TX_PROTOCOL_ERR_MASK | - INT_MASK0_RX_BUFFER_OVERFLOW_ERR_MASK | - INT_MASK0_LOSS_OF_FRAME_ERR_MASK | - INT_MASK0_HEADER_ERR_MASK); + regval &= ~(OA_TC6_INT_MASK0_TX_PROTOCOL_ERR_MASK | + OA_TC6_INT_MASK0_RX_BUFFER_OVERFLOW_ERR_MASK | + OA_TC6_INT_MASK0_LOSS_OF_FRAME_ERR_MASK | + OA_TC6_INT_MASK0_HEADER_ERR_MASK); return oa_tc6_write_register(tc6, OA_TC6_REG_INT_MASK0, regval); } @@ -647,11 +634,31 @@ static int oa_tc6_enable_data_transfer(struct oa_tc6 *tc6) return ret; /* Enable configuration synchronization for data transfer */ - value |= CONFIG0_SYNC; + value |= OA_TC6_CONFIG0_SYNC; return oa_tc6_write_register(tc6, OA_TC6_REG_CONFIG0, value); } +/* Called when a frame that is meant to be transmitted, is dropped. */ +static void oa_tc6_drop_tx_skb(struct oa_tc6 *tc6, struct sk_buff *skb) +{ + if (skb) { + tc6->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + } +} + +static struct sk_buff *oa_tc6_detach_waiting_tx_skb(struct oa_tc6 *tc6) +{ + struct sk_buff *skb; + + lockdep_assert_held(&tc6->tx_skb_lock); + skb = tc6->waiting_tx_skb; + tc6->waiting_tx_skb = NULL; + + return skb; +} + static void oa_tc6_cleanup_ongoing_rx_skb(struct oa_tc6 *tc6) { if (tc6->rx_skb) { @@ -663,39 +670,59 @@ static void oa_tc6_cleanup_ongoing_rx_skb(struct oa_tc6 *tc6) static void oa_tc6_cleanup_ongoing_tx_skb(struct oa_tc6 *tc6) { - if (tc6->ongoing_tx_skb) { - tc6->netdev->stats.tx_dropped++; - kfree_skb(tc6->ongoing_tx_skb); - tc6->ongoing_tx_skb = NULL; - } + oa_tc6_drop_tx_skb(tc6, tc6->ongoing_tx_skb); + tc6->ongoing_tx_skb = NULL; } static void oa_tc6_cleanup_waiting_tx_skb(struct oa_tc6 *tc6) { - if (tc6->waiting_tx_skb) { - tc6->netdev->stats.tx_dropped++; - kfree_skb(tc6->waiting_tx_skb); - tc6->waiting_tx_skb = NULL; - } + struct sk_buff *skb; + + spin_lock_bh(&tc6->tx_skb_lock); + skb = oa_tc6_detach_waiting_tx_skb(tc6); + spin_unlock_bh(&tc6->tx_skb_lock); + + oa_tc6_drop_tx_skb(tc6, skb); } -static void oa_tc6_free_pending_skbs(struct oa_tc6 *tc6) +static void oa_tc6_free_ongoing_skbs(struct oa_tc6 *tc6) { oa_tc6_cleanup_ongoing_tx_skb(tc6); oa_tc6_cleanup_ongoing_rx_skb(tc6); +} + +static void oa_tc6_free_pending_skbs(struct oa_tc6 *tc6) +{ + oa_tc6_free_ongoing_skbs(tc6); oa_tc6_cleanup_waiting_tx_skb(tc6); } +static void oa_tc6_look_for_new_frame(struct oa_tc6 *tc6) +{ + tc6->wait_until_start_valid = true; + oa_tc6_cleanup_ongoing_rx_skb(tc6); +} + /* If the failure is at SPI interface level, masking and clearing * the interrupt of the device won't work. Since SPI interrupt is * disabled, it should stop the repeated interrupts. */ static void oa_tc6_disable_traffic(struct oa_tc6 *tc6) { - u32 regval = INT_MASK0_ALL_INTERRUPTS; + u32 regval = OA_TC6_INT_MASK0_ALL_INTERRUPTS; + struct sk_buff *skb; + spin_lock_bh(&tc6->tx_skb_lock); tc6->disable_traffic = true; - oa_tc6_free_pending_skbs(tc6); + skb = oa_tc6_detach_waiting_tx_skb(tc6); + spin_unlock_bh(&tc6->tx_skb_lock); + + /* disable_traffic, when set, is a point of no return to + * working state. Keeping the TX queues disabled. + */ + netif_tx_disable(tc6->netdev); + oa_tc6_drop_tx_skb(tc6, skb); + oa_tc6_free_ongoing_skbs(tc6); oa_tc6_write_register(tc6, OA_TC6_REG_INT_MASK0, regval); oa_tc6_read_register(tc6, OA_TC6_REG_STATUS0, ®val); oa_tc6_write_register(tc6, OA_TC6_REG_STATUS0, regval); @@ -714,6 +741,13 @@ static int oa_tc6_process_extended_status(struct oa_tc6 *tc6) return ret; } + /* This function is called for each chunk received in a given SPI + * transaction. In case, extended status bit is set in more than + * one chunk, skip the write, if status0 is already cleared. + */ + if (!value) + return 0; + /* Clear the error interrupts status */ ret = oa_tc6_write_register(tc6, OA_TC6_REG_STATUS0, value); if (ret) { @@ -722,25 +756,24 @@ static int oa_tc6_process_extended_status(struct oa_tc6 *tc6) return ret; } - if (FIELD_GET(STATUS0_RX_BUFFER_OVERFLOW_ERROR, value)) { - tc6->rx_buf_overflow = true; - oa_tc6_cleanup_ongoing_rx_skb(tc6); + if (FIELD_GET(OA_TC6_STATUS0_RX_BUFFER_OVERFLOW_ERROR, value)) { + oa_tc6_look_for_new_frame(tc6); net_err_ratelimited("%s: Receive buffer overflow error\n", tc6->netdev->name); return -EAGAIN; } - if (FIELD_GET(STATUS0_TX_PROTOCOL_ERROR, value)) { + if (FIELD_GET(OA_TC6_STATUS0_TX_PROTOCOL_ERROR, value)) { netdev_err(tc6->netdev, "Transmit protocol error\n"); return -ENODEV; } /* TODO: Currently loss of frame and header errors are treated as * non-recoverable errors. They will be handled in the next version. */ - if (FIELD_GET(STATUS0_LOSS_OF_FRAME_ERROR, value)) { + if (FIELD_GET(OA_TC6_STATUS0_LOSS_OF_FRAME_ERROR, value)) { netdev_err(tc6->netdev, "Loss of frame error\n"); return -ENODEV; } - if (FIELD_GET(STATUS0_HEADER_ERROR, value)) { + if (FIELD_GET(OA_TC6_STATUS0_HEADER_ERROR, value)) { netdev_err(tc6->netdev, "Header error\n"); return -ENODEV; } @@ -750,6 +783,8 @@ static int oa_tc6_process_extended_status(struct oa_tc6 *tc6) static int oa_tc6_process_rx_chunk_footer(struct oa_tc6 *tc6, u32 footer) { + int ret = 0; + /* Process rx chunk footer for the following, * 1. tx credits * 2. errors if any from MAC-PHY @@ -760,9 +795,11 @@ static int oa_tc6_process_rx_chunk_footer(struct oa_tc6 *tc6, u32 footer) footer); if (FIELD_GET(OA_TC6_DATA_FOOTER_EXTENDED_STS, footer)) { - int ret = oa_tc6_process_extended_status(tc6); - - if (ret) + ret = oa_tc6_process_extended_status(tc6); + /* EAGAIN error is recoverable. Move on to check + * HEADER and SYNC errors before returning. + */ + if (ret && ret != -EAGAIN) return ret; } @@ -780,7 +817,7 @@ static int oa_tc6_process_rx_chunk_footer(struct oa_tc6 *tc6, u32 footer) return -ENODEV; } - return 0; + return ret; } static void oa_tc6_submit_rx_skb(struct oa_tc6 *tc6) @@ -805,13 +842,35 @@ static void oa_tc6_submit_rx_skb(struct oa_tc6 *tc6) tc6->rx_skb = NULL; } -static void oa_tc6_update_rx_skb(struct oa_tc6 *tc6, u8 *payload, u8 length) +/* On oversubscribed traffic condition, particularly with overwhelming rx + * buffer overflow errors, there could be data chunk loss. If tail + length + * goes beyond end pointer, that is an indication that the data chunk with + * end_valid bit is lost. Time to look for a data chunk with start_valid bit. + * + * If rx_skb is NULL, it is time to start looking for data chunk with + * start_bit. + */ +static int oa_tc6_update_rx_skb(struct oa_tc6 *tc6, u8 *payload, u8 length) { + if (!tc6->rx_skb || + skb_tailroom(tc6->rx_skb) < length) { + oa_tc6_look_for_new_frame(tc6); + return -EAGAIN; + } + memcpy(skb_put(tc6->rx_skb, length), payload, length); + return 0; } +/* On overwhelming rx buffer overflow errors, due to data chunk loss, it is + * possible that we get two data chunks with start_valid bit set, without + * end_valid bit set in between. In this case, rx_skb would have a valid + * buffer pointer. We should release, if a valid pointer is found before + * allocating a new one. + */ static int oa_tc6_allocate_rx_skb(struct oa_tc6 *tc6) { + oa_tc6_cleanup_ongoing_rx_skb(tc6); tc6->rx_skb = netdev_alloc_skb_ip_align(tc6->netdev, tc6->netdev->mtu + ETH_HLEN + ETH_FCS_LEN); if (!tc6->rx_skb) { @@ -831,7 +890,9 @@ static int oa_tc6_prcs_complete_rx_frame(struct oa_tc6 *tc6, u8 *payload, if (ret) return ret; - oa_tc6_update_rx_skb(tc6, payload, size); + ret = oa_tc6_update_rx_skb(tc6, payload, size); + if (ret) + return ret; oa_tc6_submit_rx_skb(tc6); @@ -846,22 +907,24 @@ static int oa_tc6_prcs_rx_frame_start(struct oa_tc6 *tc6, u8 *payload, u16 size) if (ret) return ret; - oa_tc6_update_rx_skb(tc6, payload, size); - - return 0; + return oa_tc6_update_rx_skb(tc6, payload, size); } -static void oa_tc6_prcs_rx_frame_end(struct oa_tc6 *tc6, u8 *payload, u16 size) +static int oa_tc6_prcs_rx_frame_end(struct oa_tc6 *tc6, u8 *payload, u16 size) { - oa_tc6_update_rx_skb(tc6, payload, size); + int ret; - oa_tc6_submit_rx_skb(tc6); + ret = oa_tc6_update_rx_skb(tc6, payload, size); + if (!ret) + oa_tc6_submit_rx_skb(tc6); + return ret; } -static void oa_tc6_prcs_ongoing_rx_frame(struct oa_tc6 *tc6, u8 *payload, - u32 footer) +static int oa_tc6_prcs_ongoing_rx_frame(struct oa_tc6 *tc6, u8 *payload, + u32 footer) { - oa_tc6_update_rx_skb(tc6, payload, OA_TC6_CHUNK_PAYLOAD_SIZE); + return oa_tc6_update_rx_skb(tc6, payload, + OA_TC6_CHUNK_PAYLOAD_SIZE); } static int oa_tc6_prcs_rx_chunk_payload(struct oa_tc6 *tc6, u8 *data, @@ -876,10 +939,10 @@ static int oa_tc6_prcs_rx_chunk_payload(struct oa_tc6 *tc6, u8 *data, u16 size; /* Restart the new rx frame after receiving rx buffer overflow error */ - if (start_valid && tc6->rx_buf_overflow) - tc6->rx_buf_overflow = false; + if (start_valid && tc6->wait_until_start_valid) + tc6->wait_until_start_valid = false; - if (tc6->rx_buf_overflow) + if (tc6->wait_until_start_valid) return 0; /* Process the chunk with complete rx frame */ @@ -901,8 +964,7 @@ static int oa_tc6_prcs_rx_chunk_payload(struct oa_tc6 *tc6, u8 *data, /* Process the chunk with only rx frame end */ if (end_valid && !start_valid) { size = end_byte_offset + 1; - oa_tc6_prcs_rx_frame_end(tc6, data, size); - return 0; + return oa_tc6_prcs_rx_frame_end(tc6, data, size); } /* Process the chunk with previous rx frame end and next rx frame @@ -916,6 +978,15 @@ static int oa_tc6_prcs_rx_chunk_payload(struct oa_tc6 *tc6, u8 *data, if (tc6->rx_skb) { size = end_byte_offset + 1; oa_tc6_prcs_rx_frame_end(tc6, data, size); + + /* Return value from oa_tc6_prcs_rx_frame_end is not + * checked. If it returned an error, it is to make + * the code to look for new frame. At this stage, + * code below is going to process a new frame. So, + * error condition is set to false, in case it is + * set before proceeding. + */ + tc6->wait_until_start_valid = false; } size = OA_TC6_CHUNK_PAYLOAD_SIZE - start_byte_offset; return oa_tc6_prcs_rx_frame_start(tc6, @@ -924,9 +995,7 @@ static int oa_tc6_prcs_rx_chunk_payload(struct oa_tc6 *tc6, u8 *data, } /* Process the chunk with ongoing rx frame data */ - oa_tc6_prcs_ongoing_rx_frame(tc6, data, footer); - - return 0; + return oa_tc6_prcs_ongoing_rx_frame(tc6, data, footer); } static u32 oa_tc6_get_rx_chunk_footer(struct oa_tc6 *tc6, u16 footer_offset) @@ -942,8 +1011,9 @@ static u32 oa_tc6_get_rx_chunk_footer(struct oa_tc6 *tc6, u16 footer_offset) static int oa_tc6_process_spi_data_rx_buf(struct oa_tc6 *tc6, u16 length) { u16 no_of_rx_chunks = length / OA_TC6_CHUNK_SIZE; + bool retry = false; + int ret = 0; u32 footer; - int ret; /* All the rx chunks in the receive SPI data buffer are examined here */ for (int i = 0; i < no_of_rx_chunks; i++) { @@ -952,8 +1022,11 @@ static int oa_tc6_process_spi_data_rx_buf(struct oa_tc6 *tc6, u16 length) OA_TC6_CHUNK_PAYLOAD_SIZE); ret = oa_tc6_process_rx_chunk_footer(tc6, footer); - if (ret) - return ret; + if (ret) { + if (ret != -EAGAIN) + return ret; + retry = true; + } /* If there is a data valid chunks then process it for the * information needed to determine the validity and the location @@ -965,12 +1038,35 @@ static int oa_tc6_process_spi_data_rx_buf(struct oa_tc6 *tc6, u16 length) ret = oa_tc6_prcs_rx_chunk_payload(tc6, payload, footer); - if (ret) - return ret; + if (ret) { + if (ret != -ENOMEM && ret != -EAGAIN) + return ret; + retry = true; + } } } - return 0; + /* Not bailing out on recoverable error codes, -EAGAIN and + * -ENOMEM. If subsequent loop iterations, if any, succeeds, + * error code would be overwritten. retry flag helps to + * make the caller to continue and retry. Since recovery + * action for -ENOMEM and -EAGAIN are same, we are returning + * one of the error codes, that is -EAGAIN. + * + * Successful recovery depends on how small the frames are, + * how many chunks, among the received chunks triggered the + * error, whether data is intact even with error conditions. + * As a result, there is no single, best method to recover + * most data when error conditions hit. We do our best by + * processing all the chunks with good "footer header" and + * "data valid" bit set. + */ + if (retry) { + ret = -EAGAIN; + oa_tc6_look_for_new_frame(tc6); + } + + return ret; } static __be32 oa_tc6_prepare_data_header(bool data_valid, bool start_valid, @@ -1132,12 +1228,9 @@ static int oa_tc6_try_spi_transfer(struct oa_tc6 *tc6) } ret = oa_tc6_process_spi_data_rx_buf(tc6, spi_len); - if (ret) { - if (ret == -EAGAIN) - continue; - oa_tc6_cleanup_ongoing_tx_skb(tc6); - oa_tc6_cleanup_ongoing_rx_skb(tc6); + if (ret && ret != -EAGAIN) { + oa_tc6_free_ongoing_skbs(tc6); netdev_err(tc6->netdev, "Device error: %d\n", ret); return ret; } @@ -1159,15 +1252,20 @@ static irqreturn_t oa_tc6_macphy_threaded_irq(int irq, void *data) * no need to attempt spi transfer, once it fails. Pending skbs * are already freed. */ - if (!tc6->disable_traffic) { - while (tc6->int_flag || - (tc6->waiting_tx_skb && tc6->tx_credits)) { - ret = oa_tc6_try_spi_transfer(tc6); - if (ret) { - disable_irq_nosync(tc6->spi->irq); - oa_tc6_disable_traffic(tc6); - break; - } + spin_lock_bh(&tc6->tx_skb_lock); + if (tc6->disable_traffic) { + spin_unlock_bh(&tc6->tx_skb_lock); + return IRQ_HANDLED; + } + spin_unlock_bh(&tc6->tx_skb_lock); + + while (tc6->int_flag || + (tc6->waiting_tx_skb && tc6->tx_credits)) { + ret = oa_tc6_try_spi_transfer(tc6); + if (ret) { + disable_irq_nosync(tc6->spi->irq); + oa_tc6_disable_traffic(tc6); + break; } } @@ -1187,9 +1285,10 @@ static int oa_tc6_update_buffer_status_from_register(struct oa_tc6 *tc6) if (ret) return ret; - tc6->tx_credits = FIELD_GET(BUFFER_STATUS_TX_CREDITS_AVAILABLE, value); - tc6->rx_chunks_available = FIELD_GET(BUFFER_STATUS_RX_CHUNKS_AVAILABLE, - value); + tc6->tx_credits = FIELD_GET(OA_TC6_BUFFER_STATUS_TX_CREDITS_AVAILABLE, + value); + tc6->rx_chunks_available = + FIELD_GET(OA_TC6_BUFFER_STATUS_RX_CHUNKS_AVAILABLE, value); return 0; } @@ -1233,7 +1332,7 @@ int oa_tc6_zero_align_receive_frame_enable(struct oa_tc6 *tc6) return ret; /* Set Zero-Align Receive Frame Enable */ - regval |= CONFIG0_ZARFE_ENABLE; + regval |= OA_TC6_CONFIG0_ZARFE_ENABLE; return oa_tc6_write_register(tc6, OA_TC6_REG_CONFIG0, regval); } @@ -1245,23 +1344,30 @@ EXPORT_SYMBOL_GPL(oa_tc6_zero_align_receive_frame_enable); * @tc6: oa_tc6 struct. * @skb: socket buffer in which the ethernet frame is stored. * - * Return: NETDEV_TX_OK if the transmit ethernet frame skb added in the tx_skb_q - * otherwise returns NETDEV_TX_BUSY. + * Return: NETDEV_TX_OK either on successful queueing of the packet for + * transmission, or on packet getting dropped. Packet can be dropped due to + * failure in linearizing the buffer or disable_traffic is set due to + * earlier fatal error. Returns NETDEV_TX_BUSY when there is no room + * to queue the packet. */ netdev_tx_t oa_tc6_start_xmit(struct oa_tc6 *tc6, struct sk_buff *skb) { - if (tc6->disable_traffic || tc6->waiting_tx_skb) { - netif_stop_queue(tc6->netdev); - return NETDEV_TX_BUSY; - } - if (skb_linearize(skb)) { - dev_kfree_skb_any(skb); - tc6->netdev->stats.tx_dropped++; + oa_tc6_drop_tx_skb(tc6, skb); return NETDEV_TX_OK; } spin_lock_bh(&tc6->tx_skb_lock); + if (tc6->waiting_tx_skb) { + netif_stop_queue(tc6->netdev); + spin_unlock_bh(&tc6->tx_skb_lock); + return NETDEV_TX_BUSY; + } + if (tc6->disable_traffic) { + spin_unlock_bh(&tc6->tx_skb_lock); + oa_tc6_drop_tx_skb(tc6, skb); + return NETDEV_TX_OK; + } tc6->waiting_tx_skb = skb; spin_unlock_bh(&tc6->tx_skb_lock); @@ -1272,6 +1378,20 @@ netdev_tx_t oa_tc6_start_xmit(struct oa_tc6 *tc6, struct sk_buff *skb) } EXPORT_SYMBOL_GPL(oa_tc6_start_xmit); +static int oa_tc6_check_ctrl_protection(struct oa_tc6 *tc6) +{ + u32 regval; + int ret; + + ret = oa_tc6_read_register(tc6, OA_TC6_REG_CONFIG0, ®val); + if (ret) + return ret; + + tc6->prot_ctrl = FIELD_GET(OA_TC6_CONFIG0_PROTE, regval); + + return 0; +} + /** * oa_tc6_init - allocates and initializes oa_tc6 structure. * @spi: device with which data will be exchanged. @@ -1324,6 +1444,14 @@ struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev) if (!tc6->spi_data_rx_buf) return NULL; + /* Check the PROTE bit status so that we can reset the device */ + ret = oa_tc6_check_ctrl_protection(tc6); + if (ret) { + dev_err(&tc6->spi->dev, + "Failed to check the protection mode: %d\n", ret); + return NULL; + } + ret = oa_tc6_sw_reset_macphy(tc6); if (ret) { dev_err(&tc6->spi->dev, @@ -1393,8 +1521,10 @@ EXPORT_SYMBOL_GPL(oa_tc6_init); */ void oa_tc6_exit(struct oa_tc6 *tc6) { - tc6->disable_traffic = true; disable_irq(tc6->spi->irq); + spin_lock_bh(&tc6->tx_skb_lock); + tc6->disable_traffic = true; + spin_unlock_bh(&tc6->tx_skb_lock); oa_tc6_phy_exit(tc6); oa_tc6_free_pending_skbs(tc6); } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index fd3ee98205319..abc8e35304355 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -920,8 +920,21 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq) }; int err; - q->partner = &lif->txqcqs[q->index]->q; - q->partner->partner = q; + q->partner = NULL; + + /* Only normal RX queues have matching TX queue partners. */ + if (q->index < lif->nxqs) { + if (!lif->txqcqs || + q->index >= lif->ionic->ntxqs_per_lif || + !lif->txqcqs[q->index]) { + dev_err(dev, "missing TX queue partner for RX queue %u\n", + q->index); + return -ENXIO; + } + + q->partner = &lif->txqcqs[q->index]->q; + q->partner->partner = q; + } if (!lif->xdp_prog || (lif->xdp_prog->aux && lif->xdp_prog->aux->xdp_has_frags)) diff --git a/drivers/net/ethernet/pensando/ionic/ionic_main.c b/drivers/net/ethernet/pensando/ionic/ionic_main.c index 6e6f3ed07271b..10501be9ef956 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_main.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_main.c @@ -269,6 +269,8 @@ bool ionic_notifyq_service(struct ionic_cq *cq) if ((s64)(eid - lif->last_eid) <= 0) return false; + dma_rmb(); + lif->last_eid = eid; dev_dbg(lif->ionic->dev, "notifyq event:\n"); @@ -314,6 +316,8 @@ bool ionic_adminq_service(struct ionic_cq *cq) if (!color_match(comp->color, cq->done_color)) return false; + dma_rmb(); + /* check for empty queue */ if (q->tail_idx == q->head_idx) return false; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c index 301ebee2fdc50..2543a8ff85476 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c @@ -545,13 +545,18 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, break; case XDP_TX: + txq = rxq->partner; + if (unlikely(!txq)) { + err = -EIO; + break; + } + xdpf = xdp_convert_buff_to_frame(&xdp_buf); if (!xdpf) { err = -ENOSPC; break; } - txq = rxq->partner; nq = netdev_get_tx_queue(netdev, txq->index); __netif_tx_lock(nq, smp_processor_id()); txq_trans_cond_update(nq); @@ -701,11 +706,7 @@ static void ionic_rx_clean(struct ionic_queue *q, __le64 *cq_desc_hwstamp; u64 hwstamp; - cq_desc_hwstamp = - (void *)comp + - qcq->cq.desc_size - - sizeof(struct ionic_rxq_comp) - - IONIC_HWSTAMP_CQ_NEGOFFSET; + cq_desc_hwstamp = (void *)comp - IONIC_HWSTAMP_CQ_NEGOFFSET; hwstamp = le64_to_cpu(*cq_desc_hwstamp); @@ -729,11 +730,18 @@ static bool __ionic_rx_service(struct ionic_cq *cq, struct bpf_prog *xdp_prog) struct ionic_queue *q = cq->bound_q; struct ionic_rxq_comp *comp; - comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx]; + if (likely(cq->desc_size == sizeof(*comp))) + comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx]; + else + comp = cq->base + + cq->desc_size * cq->tail_idx + + cq->desc_size - sizeof(*comp); if (!color_match(comp->pkt_type_color, cq->done_color)) return false; + dma_rmb(); + /* check for empty queue */ if (q->tail_idx == q->head_idx) return false; @@ -1180,7 +1188,6 @@ static void ionic_tx_clean(struct ionic_queue *q, bool in_napi) { struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct ionic_qcq *qcq = q_to_qcq(q); struct sk_buff *skb; if (desc_info->xdpf) { @@ -1205,11 +1212,7 @@ static void ionic_tx_clean(struct ionic_queue *q, __le64 *cq_desc_hwstamp; u64 hwstamp; - cq_desc_hwstamp = - (void *)comp + - qcq->cq.desc_size - - sizeof(struct ionic_txq_comp) - - IONIC_HWSTAMP_CQ_NEGOFFSET; + cq_desc_hwstamp = (void *)comp - IONIC_HWSTAMP_CQ_NEGOFFSET; hwstamp = le64_to_cpu(*cq_desc_hwstamp); @@ -1244,11 +1247,18 @@ static bool ionic_tx_service(struct ionic_cq *cq, unsigned int pkts = 0; u16 index; - comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx]; + if (likely(cq->desc_size == sizeof(*comp))) + comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx]; + else + comp = cq->base + + cq->desc_size * cq->tail_idx + + cq->desc_size - sizeof(*comp); if (!color_match(comp->color, cq->done_color)) return false; + dma_rmb(); + /* clean the related q entries, there could be * several q entries completed for each cq completion */ @@ -1662,15 +1672,22 @@ static int ionic_tx(struct net_device *netdev, struct ionic_queue *q, stats->pkts++; stats->bytes += skb->len; + ionic_txq_post(q, false); + if (likely(!ionic_txq_hwstamp_enabled(q))) { struct netdev_queue *ndq = q_to_ndq(netdev, q); - if (unlikely(!ionic_q_has_space(q, MAX_SKB_FRAGS + 1))) - netif_tx_stop_queue(ndq); + netif_txq_maybe_stop(ndq, ionic_q_space_avail(q), + MAX_SKB_FRAGS + 1, MAX_SKB_FRAGS + 1); ring_dbell = __netdev_tx_sent_queue(ndq, skb->len, netdev_xmit_more()); } - ionic_txq_post(q, ring_dbell); + + if (ring_dbell) { + ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, + q->dbval | q->head_idx); + q->dbell_jiffies = jiffies; + } return 0; } diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h index 042a75f340608..0e7a0c2c17651 100644 --- a/drivers/net/ethernet/qlogic/qede/qede.h +++ b/drivers/net/ethernet/qlogic/qede/qede.h @@ -303,10 +303,10 @@ enum qede_agg_state { }; struct qede_agg_info { - /* rx_buf is a data buffer that can be placed / consumed from rx bd - * chain. It has two purposes: We will preallocate the data buffer - * for each aggregation when we open the interface and will place this - * buffer on the rx-bd-ring when we receive TPA_START. We don't want + /* buffer is used to retain the Rx consumer descriptor when a TPA + * session starts. If the SKB allocation fails during TPA_START, + * we use this saved buffer to safely recycle the physical page + * back into the rx-bd-ring via qede_reuse_page(). We don't want * to be in a state where allocation fails, as we can't reuse the * consumer buffer in the rx-chain since FW may still be writing to it * (since header needs to be modified for TPA). diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index c11e0d8f98aa5..a02624f6d6d59 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -850,6 +850,7 @@ static void qede_tpa_start(struct qede_dev *edev, pad, false); tpa_info->buffer.page_offset = sw_rx_data_cons->page_offset; tpa_info->buffer.mapping = sw_rx_data_cons->mapping; + tpa_info->buffer.data = sw_rx_data_cons->data; if (unlikely(!tpa_info->skb)) { DP_NOTICE(edev, "Failed to allocate SKB for gro\n"); diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c index 9192c5ad5a169..c6bdf1b6ac9d0 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c @@ -6,6 +6,7 @@ #include "qlcnic.h" #include "qlcnic_hw.h" +#include struct crb_addr_pair { u32 addr; @@ -740,159 +741,212 @@ qlcnic_has_mn(struct qlcnic_adapter *adapter) return 0; } -static -struct uni_table_desc *qlcnic_get_table_desc(const u8 *unirom, int section) -{ - u32 i, entries; - struct uni_table_desc *directory = (struct uni_table_desc *) &unirom[0]; - entries = le32_to_cpu(directory->num_entries); - - for (i = 0; i < entries; i++) { +#define FILEHEADER_SIZE (14 * 4) +#define QLCNIC_UNI_DIR_TYPE_OFF (8 * sizeof(__le32)) +#define QLCNIC_UNI_DIR_ENTRY_MIN_SIZE (9 * sizeof(__le32)) +#define QLCNIC_UNI_PRODUCT_ENTRY_MIN_SIZE \ + ((QLCNIC_UNI_FIRMWARE_IDX_OFF + 1) * sizeof(__le32)) +#define QLCNIC_UNI_VERSION_TAIL_SIZE 17 +#define QLCNIC_UNI_BOOTLD_SIZE \ + (QLCNIC_IMAGE_START - QLCNIC_BOOTLD_START) + +struct qlcnic_uni_data { + u32 offset; + u32 size; +}; - u32 offs = le32_to_cpu(directory->findex) + - i * le32_to_cpu(directory->entry_size); - u32 tab_type = le32_to_cpu(*((__le32 *)&unirom[offs] + 8)); +static bool qlcnic_rom_range_valid(size_t size, size_t offset, size_t len) +{ + return offset <= size && len <= size - offset; +} - if (tab_type == section) - return (struct uni_table_desc *) &unirom[offs]; - } +static bool qlcnic_rom_table_valid(size_t size, u32 offset, u32 entries, + u32 entry_size, u32 min_entry_size) +{ + if (entry_size < min_entry_size || offset > size) + return false; - return NULL; + return entries <= (size - offset) / entry_size; } -#define FILEHEADER_SIZE (14 * 4) - -static int -qlcnic_validate_header(struct qlcnic_adapter *adapter) +static int qlcnic_get_directory(struct qlcnic_adapter *adapter, + size_t *offset, u32 *entries, u32 *entry_size) { - const u8 *unirom = adapter->fw->data; - struct uni_table_desc *directory = (struct uni_table_desc *) &unirom[0]; - u32 entries, entry_size, tab_size, fw_file_size; - - fw_file_size = adapter->fw->size; + const struct firmware *fw = adapter->fw; + const u8 *directory = fw->data; - if (fw_file_size < FILEHEADER_SIZE) + if (fw->size < FILEHEADER_SIZE) return -EINVAL; - entries = le32_to_cpu(directory->num_entries); - entry_size = le32_to_cpu(directory->entry_size); - tab_size = le32_to_cpu(directory->findex) + (entries * entry_size); + *offset = get_unaligned_le32(directory + + offsetof(struct uni_table_desc, findex)); + *entries = get_unaligned_le32(directory + + offsetof(struct uni_table_desc, num_entries)); + *entry_size = get_unaligned_le32(directory + + offsetof(struct uni_table_desc, entry_size)); - if (fw_file_size < tab_size) + if (!qlcnic_rom_table_valid(fw->size, *offset, *entries, *entry_size, + QLCNIC_UNI_DIR_ENTRY_MIN_SIZE)) return -EINVAL; return 0; } -static int -qlcnic_validate_bootld(struct qlcnic_adapter *adapter) +static int qlcnic_get_table_desc(struct qlcnic_adapter *adapter, int section, + size_t *desc_offset) { - struct uni_table_desc *tab_desc; - struct uni_data_desc *descr; - u32 offs, tab_size, data_size, idx; const u8 *unirom = adapter->fw->data; - __le32 temp; - - temp = *((__le32 *)&unirom[adapter->file_prd_off] + - QLCNIC_UNI_BOOTLD_IDX_OFF); - idx = le32_to_cpu(temp); - tab_desc = qlcnic_get_table_desc(unirom, QLCNIC_UNI_DIR_SECT_BOOTLD); - - if (!tab_desc) - return -EINVAL; - - tab_size = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * (idx + 1); - - if (adapter->fw->size < tab_size) - return -EINVAL; + size_t directory_offset; + u32 entries, entry_size; + size_t i; + int ret; - offs = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * idx; - descr = (struct uni_data_desc *)&unirom[offs]; + ret = qlcnic_get_directory(adapter, &directory_offset, &entries, + &entry_size); + if (ret) + return ret; - data_size = le32_to_cpu(descr->findex) + le32_to_cpu(descr->size); + for (i = 0; i < entries; i++) { + size_t offset = directory_offset + i * entry_size; + u32 table_type; - if (adapter->fw->size < data_size) - return -EINVAL; + table_type = get_unaligned_le32(unirom + offset + + QLCNIC_UNI_DIR_TYPE_OFF); + if (table_type == section) { + *desc_offset = offset; + return 0; + } + } - return 0; + return -ENOENT; } static int -qlcnic_validate_fw(struct qlcnic_adapter *adapter) +qlcnic_validate_header(struct qlcnic_adapter *adapter) { - struct uni_table_desc *tab_desc; - struct uni_data_desc *descr; - const u8 *unirom = adapter->fw->data; - u32 offs, tab_size, data_size, idx; - __le32 temp; + u32 entries, entry_size; + size_t offset; + + return qlcnic_get_directory(adapter, &offset, &entries, &entry_size); +} - temp = *((__le32 *)&unirom[adapter->file_prd_off] + - QLCNIC_UNI_FIRMWARE_IDX_OFF); - idx = le32_to_cpu(temp); - tab_desc = qlcnic_get_table_desc(unirom, QLCNIC_UNI_DIR_SECT_FW); +static int qlcnic_get_data_desc(struct qlcnic_adapter *adapter, u32 section, + u32 index_offset, struct qlcnic_uni_data *data) +{ + size_t table_desc_offset, table_offset, desc_offset; + const struct firmware *fw = adapter->fw; + const u8 *unirom = fw->data; + size_t product_index_offset; + u32 entries, entry_size, idx; + int ret; - if (!tab_desc) + product_index_offset = adapter->file_prd_off + + (size_t)index_offset * sizeof(__le32); + if (!qlcnic_rom_range_valid(fw->size, product_index_offset, + sizeof(__le32))) return -EINVAL; - tab_size = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * (idx + 1); + idx = get_unaligned_le32(unirom + product_index_offset); + ret = qlcnic_get_table_desc(adapter, section, &table_desc_offset); + if (ret) + return ret; - if (adapter->fw->size < tab_size) + table_offset = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, findex)); + entries = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, num_entries)); + entry_size = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, entry_size)); + if (!qlcnic_rom_table_valid(fw->size, table_offset, entries, + entry_size, sizeof(struct uni_data_desc)) || + idx >= entries) return -EINVAL; - offs = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * idx; - descr = (struct uni_data_desc *)&unirom[offs]; - data_size = le32_to_cpu(descr->findex) + le32_to_cpu(descr->size); + desc_offset = table_offset + (size_t)idx * entry_size; + data->offset = get_unaligned_le32(unirom + desc_offset + + offsetof(struct uni_data_desc, findex)); + data->size = get_unaligned_le32(unirom + desc_offset + + offsetof(struct uni_data_desc, size)); - if (adapter->fw->size < data_size) + if (!qlcnic_rom_range_valid(fw->size, data->offset, data->size)) return -EINVAL; return 0; } +static int qlcnic_validate_bootld(struct qlcnic_adapter *adapter) +{ + struct qlcnic_uni_data data; + int ret; + + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_BOOTLD, + QLCNIC_UNI_BOOTLD_IDX_OFF, &data); + if (ret) + return ret; + + return data.size < QLCNIC_UNI_BOOTLD_SIZE ? -EINVAL : 0; +} + +static int qlcnic_validate_fw(struct qlcnic_adapter *adapter) +{ + struct qlcnic_uni_data data; + int ret; + + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, + QLCNIC_UNI_FIRMWARE_IDX_OFF, &data); + if (ret) + return ret; + + return data.size < QLCNIC_UNI_VERSION_TAIL_SIZE ? -EINVAL : 0; +} + static int qlcnic_validate_product_offs(struct qlcnic_adapter *adapter) { - struct uni_table_desc *ptab_descr; + size_t table_desc_offset, table_offset; const u8 *unirom = adapter->fw->data; int mn_present = qlcnic_has_mn(adapter); - u32 entries, entry_size, tab_size, i; - __le32 temp; - - ptab_descr = qlcnic_get_table_desc(unirom, - QLCNIC_UNI_DIR_SECT_PRODUCT_TBL); - if (!ptab_descr) - return -EINVAL; + u32 entries, entry_size; + size_t i; + int ret; - entries = le32_to_cpu(ptab_descr->num_entries); - entry_size = le32_to_cpu(ptab_descr->entry_size); - tab_size = le32_to_cpu(ptab_descr->findex) + (entries * entry_size); + ret = qlcnic_get_table_desc(adapter, QLCNIC_UNI_DIR_SECT_PRODUCT_TBL, + &table_desc_offset); + if (ret) + return ret; - if (adapter->fw->size < tab_size) + table_offset = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, findex)); + entries = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, num_entries)); + entry_size = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, entry_size)); + if (!qlcnic_rom_table_valid(adapter->fw->size, table_offset, entries, + entry_size, + QLCNIC_UNI_PRODUCT_ENTRY_MIN_SIZE)) return -EINVAL; nomn: for (i = 0; i < entries; i++) { - - u32 flags, file_chiprev, offs; + size_t offset = table_offset + i * entry_size; u8 chiprev = adapter->ahw->revision_id; + u32 flags, file_chiprev; u32 flagbit; - offs = le32_to_cpu(ptab_descr->findex) + - i * le32_to_cpu(ptab_descr->entry_size); - temp = *((__le32 *)&unirom[offs] + QLCNIC_UNI_FLAGS_OFF); - flags = le32_to_cpu(temp); - temp = *((__le32 *)&unirom[offs] + QLCNIC_UNI_CHIP_REV_OFF); - file_chiprev = le32_to_cpu(temp); + flags = get_unaligned_le32(unirom + offset + + QLCNIC_UNI_FLAGS_OFF * sizeof(__le32)); + file_chiprev = get_unaligned_le32(unirom + offset + + QLCNIC_UNI_CHIP_REV_OFF * + sizeof(__le32)); flagbit = mn_present ? 1 : 2; if ((chiprev == file_chiprev) && ((1ULL << flagbit) & flags)) { - adapter->file_prd_off = offs; + if (offset > U32_MAX) + return -EINVAL; + + adapter->file_prd_off = offset; return 0; } } @@ -933,93 +987,81 @@ qlcnic_validate_unified_romimage(struct qlcnic_adapter *adapter) return 0; } -static -struct uni_data_desc *qlcnic_get_data_desc(struct qlcnic_adapter *adapter, - u32 section, u32 idx_offset) -{ - const u8 *unirom = adapter->fw->data; - struct uni_table_desc *tab_desc; - u32 offs, idx; - __le32 temp; - - temp = *((__le32 *)&unirom[adapter->file_prd_off] + idx_offset); - idx = le32_to_cpu(temp); - - tab_desc = qlcnic_get_table_desc(unirom, section); - - if (tab_desc == NULL) - return NULL; - - offs = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * idx; - - return (struct uni_data_desc *)&unirom[offs]; -} - -static u8 * -qlcnic_get_bootld_offs(struct qlcnic_adapter *adapter) +static int qlcnic_get_bootld_data(struct qlcnic_adapter *adapter, + const u8 **bootld) { u32 offs = QLCNIC_BOOTLD_START; - struct uni_data_desc *data_desc; - - data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_BOOTLD, - QLCNIC_UNI_BOOTLD_IDX_OFF); + struct qlcnic_uni_data data; + int ret; - if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) - offs = le32_to_cpu(data_desc->findex); + if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) { + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_BOOTLD, + QLCNIC_UNI_BOOTLD_IDX_OFF, &data); + if (ret || data.size < QLCNIC_UNI_BOOTLD_SIZE) + return -EINVAL; + offs = data.offset; + } else if (!qlcnic_rom_range_valid(adapter->fw->size, offs, + QLCNIC_UNI_BOOTLD_SIZE)) { + return -EINVAL; + } - return (u8 *)&adapter->fw->data[offs]; + *bootld = adapter->fw->data + offs; + return 0; } -static u8 * -qlcnic_get_fw_offs(struct qlcnic_adapter *adapter) +static int qlcnic_get_fw_data(struct qlcnic_adapter *adapter, + const u8 **image, u32 *image_size) { u32 offs = QLCNIC_IMAGE_START; - struct uni_data_desc *data_desc; - - data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, - QLCNIC_UNI_FIRMWARE_IDX_OFF); - if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) - offs = le32_to_cpu(data_desc->findex); - - return (u8 *)&adapter->fw->data[offs]; -} + struct qlcnic_uni_data data; + int ret; -static u32 qlcnic_get_fw_size(struct qlcnic_adapter *adapter) -{ - struct uni_data_desc *data_desc; - const u8 *unirom = adapter->fw->data; + if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) { + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, + QLCNIC_UNI_FIRMWARE_IDX_OFF, &data); + if (ret) + return ret; + offs = data.offset; + *image_size = data.size; + } else { + if (!qlcnic_rom_range_valid(adapter->fw->size, + QLCNIC_FW_SIZE_OFFSET, + sizeof(__le32))) + return -EINVAL; + *image_size = get_unaligned_le32(adapter->fw->data + + QLCNIC_FW_SIZE_OFFSET); + } - data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, - QLCNIC_UNI_FIRMWARE_IDX_OFF); + if (!qlcnic_rom_range_valid(adapter->fw->size, offs, *image_size)) + return -EINVAL; - if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) - return le32_to_cpu(data_desc->size); - else - return le32_to_cpu(*(__le32 *)&unirom[QLCNIC_FW_SIZE_OFFSET]); + *image = adapter->fw->data + offs; + return 0; } static u32 qlcnic_get_fw_version(struct qlcnic_adapter *adapter) { - struct uni_data_desc *fw_data_desc; + char ver_str[QLCNIC_UNI_VERSION_TAIL_SIZE + 1]; const struct firmware *fw = adapter->fw; + struct qlcnic_uni_data data; u32 major, minor, sub; - __le32 version_offset; - const u8 *ver_str; int i, ret; - if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) { - version_offset = *(__le32 *)&fw->data[QLCNIC_FW_VERSION_OFFSET]; - return le32_to_cpu(version_offset); - } + if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) + return get_unaligned_le32(fw->data + QLCNIC_FW_VERSION_OFFSET); + + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, + QLCNIC_UNI_FIRMWARE_IDX_OFF, &data); + if (ret || data.size < QLCNIC_UNI_VERSION_TAIL_SIZE) + return 0; - fw_data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, - QLCNIC_UNI_FIRMWARE_IDX_OFF); - ver_str = fw->data + le32_to_cpu(fw_data_desc->findex) + - le32_to_cpu(fw_data_desc->size) - 17; + memcpy(ver_str, fw->data + data.offset + data.size - + QLCNIC_UNI_VERSION_TAIL_SIZE, + QLCNIC_UNI_VERSION_TAIL_SIZE); + ver_str[QLCNIC_UNI_VERSION_TAIL_SIZE] = '\0'; for (i = 0; i < 12; i++) { - if (!strncmp(&ver_str[i], "REV=", 4)) { + if (!strncmp(ver_str + i, "REV=", 4)) { ret = sscanf(&ver_str[i+4], "%u.%u.%u ", &major, &minor, &sub); if (ret != 3) @@ -1034,18 +1076,15 @@ static u32 qlcnic_get_fw_version(struct qlcnic_adapter *adapter) static u32 qlcnic_get_bios_version(struct qlcnic_adapter *adapter) { - const struct firmware *fw = adapter->fw; u32 bios_ver, prd_off = adapter->file_prd_off; - u8 *version_offset; - __le32 temp; + const struct firmware *fw = adapter->fw; - if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) { - version_offset = (u8 *)&fw->data[QLCNIC_BIOS_VERSION_OFFSET]; - return le32_to_cpu(*(__le32 *)version_offset); - } + if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) + return get_unaligned_le32(fw->data + QLCNIC_BIOS_VERSION_OFFSET); - temp = *((__le32 *)(&fw->data[prd_off]) + QLCNIC_UNI_BIOS_VERSION_OFF); - bios_ver = le32_to_cpu(temp); + bios_ver = get_unaligned_le32(fw->data + prd_off + + QLCNIC_UNI_BIOS_VERSION_OFF * + sizeof(__le32)); return (bios_ver << 16) + ((bios_ver >> 8) & 0xff00) + (bios_ver >> 24); } @@ -1106,24 +1145,26 @@ static const char *fw_name[] = { int qlcnic_load_firmware(struct qlcnic_adapter *adapter) { - __le64 *ptr64; - u32 i, flashaddr, size; const struct firmware *fw = adapter->fw; struct pci_dev *pdev = adapter->pdev; + const u8 *bootld, *image; + u32 i, flashaddr, image_size; + int ret; dev_info(&pdev->dev, "loading firmware from %s\n", fw_name[adapter->ahw->fw_type]); if (fw) { + u32 words, remainder; u64 data; - size = (QLCNIC_IMAGE_START - QLCNIC_BOOTLD_START) / 8; - - ptr64 = (__le64 *)qlcnic_get_bootld_offs(adapter); + ret = qlcnic_get_bootld_data(adapter, &bootld); + if (ret) + return ret; flashaddr = QLCNIC_BOOTLD_START; - for (i = 0; i < size; i++) { - data = le64_to_cpu(ptr64[i]); + for (i = 0; i < QLCNIC_UNI_BOOTLD_SIZE / sizeof(u64); i++) { + data = get_unaligned_le64(bootld + i * sizeof(u64)); if (qlcnic_pci_mem_write_2M(adapter, flashaddr, data)) return -EIO; @@ -1131,13 +1172,15 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) flashaddr += 8; } - size = qlcnic_get_fw_size(adapter) / 8; - - ptr64 = (__le64 *)qlcnic_get_fw_offs(adapter); + ret = qlcnic_get_fw_data(adapter, &image, &image_size); + if (ret) + return ret; + words = image_size / sizeof(u64); + remainder = image_size % sizeof(u64); flashaddr = QLCNIC_IMAGE_START; - for (i = 0; i < size; i++) { - data = le64_to_cpu(ptr64[i]); + for (i = 0; i < words; i++) { + data = get_unaligned_le64(image + i * sizeof(u64)); if (qlcnic_pci_mem_write_2M(adapter, flashaddr, data)) @@ -1146,9 +1189,11 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) flashaddr += 8; } - size = qlcnic_get_fw_size(adapter) % 8; - if (size) { - data = le64_to_cpu(ptr64[i]); + if (remainder) { + __le64 tail = 0; + + memcpy(&tail, image + words * sizeof(u64), remainder); + data = le64_to_cpu(tail); if (qlcnic_pci_mem_write_2M(adapter, flashaddr, data)) @@ -1156,10 +1201,9 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) } } else { - u64 data; - u32 hi, lo; - int ret; struct qlcnic_flt_entry bootld_entry; + u32 hi, lo, size; + u64 data; ret = qlcnic_get_flt_entry(adapter, QLCNIC_BOOTLD_REGION, &bootld_entry); @@ -1200,26 +1244,33 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) static int qlcnic_validate_firmware(struct qlcnic_adapter *adapter) { - u32 val; - u32 ver, bios, min_size; - struct pci_dev *pdev = adapter->pdev; const struct firmware *fw = adapter->fw; + struct pci_dev *pdev = adapter->pdev; u8 fw_type = adapter->ahw->fw_type; + u32 ver, bios, min_size; + const u8 *data; + u32 data_size; + u32 val; + + if (fw_type == QLCNIC_UNIFIED_ROMIMAGE) + min_size = QLCNIC_UNI_FW_MIN_SIZE; + else + min_size = QLCNIC_FW_MIN_SIZE; + + if (fw->size < min_size) + return -EINVAL; if (fw_type == QLCNIC_UNIFIED_ROMIMAGE) { if (qlcnic_validate_unified_romimage(adapter)) return -EINVAL; - - min_size = QLCNIC_UNI_FW_MIN_SIZE; } else { - val = le32_to_cpu(*(__le32 *)&fw->data[QLCNIC_FW_MAGIC_OFFSET]); + val = get_unaligned_le32(fw->data + QLCNIC_FW_MAGIC_OFFSET); if (val != QLCNIC_BDINFO_MAGIC) return -EINVAL; - - min_size = QLCNIC_FW_MIN_SIZE; } - if (fw->size < min_size) + if (qlcnic_get_bootld_data(adapter, &data) || + qlcnic_get_fw_data(adapter, &data, &data_size)) return -EINVAL; val = qlcnic_get_fw_version(adapter); diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c index 305ae15ae8f39..e6f48dc9fe64d 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c @@ -394,6 +394,7 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, if (!skbn) return NULL; + skbn->dev = skb->dev; skb_reserve(skbn, RMNET_MAP_DEAGGR_HEADROOM); skb_put(skbn, packet_len); memcpy(skbn->data, skb->data, packet_len); diff --git a/drivers/net/ethernet/realtek/8139cp.c b/drivers/net/ethernet/realtek/8139cp.c index 5652da8a178c0..9016527e229ac 100644 --- a/drivers/net/ethernet/realtek/8139cp.c +++ b/drivers/net/ethernet/realtek/8139cp.c @@ -2066,7 +2066,7 @@ static int __maybe_unused cp_suspend(struct device *device) /* Disable Rx and Tx */ cpw16 (IntrMask, 0); - cpw8 (Cmd, cpr8 (Cmd) & (~RxOn | ~TxOn)); + cpw8 (Cmd, cpr8 (Cmd) & ~(RxOn | TxOn)); spin_unlock_irqrestore (&cp->lock, flags); diff --git a/drivers/net/ethernet/renesas/ravb.h b/drivers/net/ethernet/renesas/ravb.h index 5e56ec9b1013a..3ee4c61081890 100644 --- a/drivers/net/ethernet/renesas/ravb.h +++ b/drivers/net/ethernet/renesas/ravb.h @@ -1028,6 +1028,7 @@ struct ravb_ptp_perout { struct ravb_ptp { struct ptp_clock *clock; struct ptp_clock_info info; + int phc_index; u32 default_addend; u32 current_addend; int extts[N_EXT_TS]; @@ -1123,6 +1124,8 @@ struct ravb_private { int msg_enable; int speed; int emac_irq; + int err_irq; + int mgmt_irq; unsigned no_avb_link:1; unsigned avb_link_active_low:1; diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index 5f88733094d0f..ea1c7e536791e 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -1779,7 +1779,7 @@ static int ravb_get_ts_info(struct net_device *ndev, (1 << HWTSTAMP_FILTER_NONE) | (1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) | (1 << HWTSTAMP_FILTER_ALL); - info->phc_index = ptp_clock_index(priv->ptp.clock); + info->phc_index = READ_ONCE(priv->ptp.phc_index); } return 0; @@ -2885,11 +2885,13 @@ static int ravb_setup_irqs(struct ravb_private *priv) return error; if (info->err_mgmt_irqs) { - error = ravb_setup_irq(priv, "err_a", "err_a", NULL, ravb_multi_interrupt); + error = ravb_setup_irq(priv, "err_a", "err_a", &priv->err_irq, + ravb_multi_interrupt); if (error) return error; - error = ravb_setup_irq(priv, "mgmt_a", "mgmt_a", NULL, ravb_multi_interrupt); + error = ravb_setup_irq(priv, "mgmt_a", "mgmt_a", &priv->mgmt_irq, + ravb_multi_interrupt); if (error) return error; } @@ -2953,6 +2955,7 @@ static int ravb_probe(struct platform_device *pdev) priv->rstc = rstc; priv->ndev = ndev; priv->pdev = pdev; + priv->ptp.phc_index = -1; priv->num_tx_ring[RAVB_BE] = BE_TX_RING_SIZE; priv->num_rx_ring[RAVB_BE] = BE_RX_RING_SIZE; if (info->nc_queues) { diff --git a/drivers/net/ethernet/renesas/ravb_ptp.c b/drivers/net/ethernet/renesas/ravb_ptp.c index 226c6c0ab945b..43218bc15b151 100644 --- a/drivers/net/ethernet/renesas/ravb_ptp.c +++ b/drivers/net/ethernet/renesas/ravb_ptp.c @@ -289,16 +289,17 @@ static const struct ptp_clock_info ravb_ptp_info = { void ravb_ptp_interrupt(struct net_device *ndev) { struct ravb_private *priv = netdev_priv(ndev); + struct ptp_clock *clock = READ_ONCE(priv->ptp.clock); u32 gis = ravb_read(ndev, GIS); gis &= ravb_read(ndev, GIC); - if (gis & GIS_PTCF) { + if ((gis & GIS_PTCF) && clock) { struct ptp_clock_event event; event.type = PTP_CLOCK_EXTTS; event.index = 0; event.timestamp = ravb_read(ndev, GCPT); - ptp_clock_event(priv->ptp.clock, &event); + ptp_clock_event(clock, &event); } if (gis & GIS_PTMF) { struct ravb_ptp_perout *perout = priv->ptp.perout; @@ -315,6 +316,7 @@ void ravb_ptp_interrupt(struct net_device *ndev) void ravb_ptp_init(struct net_device *ndev, struct platform_device *pdev) { struct ravb_private *priv = netdev_priv(ndev); + struct ptp_clock *clock; unsigned long flags; priv->ptp.info = ravb_ptp_info; @@ -327,15 +329,45 @@ void ravb_ptp_init(struct net_device *ndev, struct platform_device *pdev) ravb_modify(ndev, GCCR, GCCR_TCSS, GCCR_TCSS_ADJGPTP); spin_unlock_irqrestore(&priv->lock, flags); - priv->ptp.clock = ptp_clock_register(&priv->ptp.info, &pdev->dev); + clock = ptp_clock_register(&priv->ptp.info, &pdev->dev); + if (IS_ERR(clock)) { + netdev_err(ndev, "failed to register PTP clock: %pe\n", clock); + clock = NULL; + } + + WRITE_ONCE(priv->ptp.clock, clock); + if (clock) + WRITE_ONCE(priv->ptp.phc_index, ptp_clock_index(clock)); +} + +static void ravb_ptp_disable(struct net_device *ndev) +{ + ravb_write(ndev, 0, GIC); + ravb_write(ndev, 0, GIS); +} + +static void ravb_ptp_sync_irqs(struct net_device *ndev) +{ + struct ravb_private *priv = netdev_priv(ndev); + + synchronize_irq(ndev->irq); + if (priv->info->err_mgmt_irqs) { + synchronize_irq(priv->err_irq); + synchronize_irq(priv->mgmt_irq); + } } void ravb_ptp_stop(struct net_device *ndev) { struct ravb_private *priv = netdev_priv(ndev); + struct ptp_clock *clock; - ravb_write(ndev, 0, GIC); - ravb_write(ndev, 0, GIS); + WRITE_ONCE(priv->ptp.phc_index, -1); + clock = xchg(&priv->ptp.clock, NULL); + + ravb_ptp_disable(ndev); + ravb_ptp_sync_irqs(ndev); - ptp_clock_unregister(priv->ptp.clock); + if (clock) + ptp_clock_unregister(clock); } diff --git a/drivers/net/ethernet/sfc/falcon/tx.c b/drivers/net/ethernet/sfc/falcon/tx.c index 9e18aaf44badd..e4d47d26a87a0 100644 --- a/drivers/net/ethernet/sfc/falcon/tx.c +++ b/drivers/net/ethernet/sfc/falcon/tx.c @@ -439,8 +439,12 @@ int ef4_setup_tc(struct net_device *net_dev, enum tc_setup_type type, return 0; for (tc = 0; tc < num_tc; tc++) { - net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; - net_dev->tc_to_txq[tc].count = efx->n_tx_channels; + struct netdev_tc_txq res = { + .offset = tc * efx->n_tx_channels, + .count = efx->n_tx_channels, + }; + + WRITE_ONCE(net_dev->tc_to_txq[tc].combined, res.combined); } if (num_tc > net_dev->num_tc) { diff --git a/drivers/net/ethernet/sfc/siena/tx.c b/drivers/net/ethernet/sfc/siena/tx.c index 91e87594ed1ea..1ce98f8fdaf81 100644 --- a/drivers/net/ethernet/sfc/siena/tx.c +++ b/drivers/net/ethernet/sfc/siena/tx.c @@ -380,8 +380,12 @@ int efx_siena_setup_tc(struct net_device *net_dev, enum tc_setup_type type, return 0; for (tc = 0; tc < num_tc; tc++) { - net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; - net_dev->tc_to_txq[tc].count = efx->n_tx_channels; + struct netdev_tc_txq res = { + .offset = tc * efx->n_tx_channels, + .count = efx->n_tx_channels, + }; + + WRITE_ONCE(net_dev->tc_to_txq[tc].combined, res.combined); } net_dev->num_tc = num_tc; diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c index d14a6584473c8..79a0a324c921d 100644 --- a/drivers/net/ethernet/socionext/netsec.c +++ b/drivers/net/ethernet/socionext/netsec.c @@ -2149,6 +2149,7 @@ static int netsec_probe(struct platform_device *pdev) pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); free_ndev: + of_node_put(priv->phy_np); free_netdev(ndev); dev_err(&pdev->dev, "init failed\n"); @@ -2166,6 +2167,7 @@ static void netsec_remove(struct platform_device *pdev) netif_napi_del(&priv->napi); pm_runtime_disable(&pdev->dev); + of_node_put(priv->phy_np); free_netdev(priv->ndev); } diff --git a/drivers/net/ethernet/spacemit/k1_emac.c b/drivers/net/ethernet/spacemit/k1_emac.c index f7f16397a2c2a..d641ac26a1e86 100644 --- a/drivers/net/ethernet/spacemit/k1_emac.c +++ b/drivers/net/ethernet/spacemit/k1_emac.c @@ -803,6 +803,9 @@ static void emac_tx_mem_map(struct emac_priv *priv, struct sk_buff *skb) while (i != head) { emac_free_tx_buf(priv, i); + tx_desc_addr = &((struct emac_desc *)tx_ring->desc_addr)[i]; + memset(tx_desc_addr, 0, sizeof(*tx_desc_addr)); + if (++i == tx_ring->total_cnt) i = 0; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index 8d7042e689261..72bdbcb5e863a 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -1162,8 +1162,11 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable) return ret; ret = clk_prepare_enable(bsp_priv->clk_phy); - if (ret) + if (ret) { + clk_bulk_disable_unprepare(bsp_priv->num_clks, + bsp_priv->clks); return ret; + } rk_configure_io_clksel(bsp_priv); rk_ungate_rmii_clock(bsp_priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c index caac85fc08f1b..d4ace39248915 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c @@ -194,7 +194,7 @@ static void dwmac1000_set_filter(struct mac_device_info *hw, dwmac1000_set_mchash(ioaddr, mc_filter, mcbitslog2); /* Handle multiple unicast addresses (perfect filtering) */ - if (netdev_uc_count(dev) > perfect_addr_number) + if (netdev_uc_count(dev) + 1 > perfect_addr_number) /* Switch to promiscuous mode if more than unicast * addresses are requested than supported by hardware. */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index c6fcfae27c3d0..18b357b257cc0 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -521,7 +521,7 @@ static void dwmac4_set_filter(struct mac_device_info *hw, value |= GMAC_PACKET_FILTER_HPF; /* Handle multiple unicast addresses */ - if (netdev_uc_count(dev) > hw->unicast_filter_entries) { + if (netdev_uc_count(dev) + 1 > hw->unicast_filter_entries) { /* Switch to promiscuous mode if more than 128 addrs * are required */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c index 2994df41ec2c4..c6a8f8d735015 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c @@ -474,11 +474,11 @@ static void dwmac4_set_sarc(struct dma_desc *p, u32 sarc_type) sarc_type)); } -static int set_16kib_bfsize(int mtu) +static int set_16kib_bfsize(int len) { int ret = 0; - if (unlikely(mtu >= BUF_SIZE_8KiB)) + if (unlikely(len > BUF_SIZE_8KiB)) ret = BUF_SIZE_16KiB; return ret; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c index 52054f31376d5..fc6ddb51c6820 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c @@ -532,7 +532,7 @@ static void dwxgmac2_set_filter(struct mac_device_info *hw, dwxgmac2_set_mchash(ioaddr, mc_filter, mcbitslog2); /* Handle multiple unicast addresses */ - if (netdev_uc_count(dev) > hw->unicast_filter_entries) { + if (netdev_uc_count(dev) + 1 > hw->unicast_filter_entries) { value |= XGMAC_FILTER_PR; } else { struct netdev_hw_addr *ha; diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h index 04dafec021b4f..857f7562c6c6d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.h +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h @@ -494,7 +494,7 @@ struct stmmac_ops { #define stmmac_set_arp_offload(__priv, __args...) \ stmmac_do_void_callback(__priv, mac, set_arp_offload, __args) #define stmmac_fpe_map_preemption_class(__priv, __args...) \ - stmmac_do_void_callback(__priv, mac, fpe_map_preemption_class, __args) + stmmac_do_callback(__priv, mac, fpe_map_preemption_class, __args) /* PTP and HW Timer helpers */ struct stmmac_hwtimestamp { @@ -540,7 +540,7 @@ struct stmmac_mode_ops { bool (*is_jumbo_frm)(unsigned int len, bool enh_desc); int (*jumbo_frm)(struct stmmac_tx_queue *tx_q, struct sk_buff *skb, int csum); - int (*set_16kib_bfsize)(int mtu); + int (*set_16kib_bfsize)(int len); void (*init_desc3)(struct dma_desc *p); void (*refill_desc3)(struct stmmac_rx_queue *rx_q, struct dma_desc *p); void (*clean_desc3)(struct stmmac_tx_queue *tx_q, struct dma_desc *p); diff --git a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c index 78fc6aa5bbe95..dd796cb74419e 100644 --- a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c +++ b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c @@ -123,10 +123,10 @@ static void clean_desc3(struct stmmac_tx_queue *tx_q, struct dma_desc *p) p->des3 = 0; } -static int set_16kib_bfsize(int mtu) +static int set_16kib_bfsize(int len) { int ret = 0; - if (unlikely(mtu > BUF_SIZE_8KiB)) + if (unlikely(len > BUF_SIZE_8KiB)) ret = BUF_SIZE_16KiB; return ret; } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 8ba8f03e1ce03..7582fca637419 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -265,8 +265,6 @@ struct stmmac_priv { u32 rx_coal_frames[MTL_MAX_RX_QUEUES]; int hwts_tx_en; - /* skb_shinfo(skb)->gso_type types that we handle */ - unsigned int gso_enabled_types; bool tx_path_in_lpi_mode; bool sph_active; bool sph_capable; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index 92585d27ab883..2a4882f2a498d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -1013,8 +1013,6 @@ static int stmmac_get_ts_info(struct net_device *dev, if (priv->ptp_clock) info->phc_index = ptp_clock_index(priv->ptp_clock); - else - info->phc_index = 0; info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_fpe.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_fpe.c index c54c702243517..c889204a7aa5d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_fpe.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_fpe.c @@ -217,8 +217,11 @@ int dwmac5_fpe_map_preemption_class(struct net_device *ndev, * and is direct one-to-one mapping." */ for (u32 tc = 0; tc < num_tc; tc++) { - count = ndev->tc_to_txq[tc].count; - offset = ndev->tc_to_txq[tc].offset; + struct netdev_tc_txq res; + + res.combined = READ_ONCE(ndev->tc_to_txq[tc].combined); + count = res.count; + offset = res.offset; if (pclass & BIT(tc)) preemptible_txqs |= GENMASK(offset + count - 1, offset); @@ -275,8 +278,11 @@ int dwxgmac3_fpe_map_preemption_class(struct net_device *ndev, * any of the scheduling algorithms." */ for (u32 tc = 0; tc < num_tc; tc++) { - count = ndev->tc_to_txq[tc].count; - offset = ndev->tc_to_txq[tc].offset; + struct netdev_tc_txq res; + + res.combined = READ_ONCE(ndev->tc_to_txq[tc].combined); + count = res.count; + offset = res.offset; if (pclass & BIT(tc)) preemptible_txqs |= GENMASK(offset + count - 1, offset); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index a71f0df263785..29025eb416987 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -1330,6 +1330,10 @@ static int stmmac_init_phy(struct net_device *dev) struct phy_device *phydev; if (addr < 0) { + /* If a custom PCS is in use, no PHY is needed */ + if (priv->hw->phylink_pcs) + return 0; + netdev_err(priv->dev, "no phy found\n"); return -ENODEV; } @@ -1527,22 +1531,22 @@ static void stmmac_display_rings(struct stmmac_priv *priv, static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) { if (stmmac_xdp_is_enabled(priv)) - return XDP_PACKET_HEADROOM; + return XDP_PACKET_HEADROOM + NET_IP_ALIGN; - return NET_SKB_PAD; + return NET_SKB_PAD + NET_IP_ALIGN; } -static int stmmac_set_bfsize(int mtu) +static int stmmac_set_bfsize(int len) { int ret; - if (mtu >= BUF_SIZE_8KiB) + if (len > BUF_SIZE_8KiB) ret = BUF_SIZE_16KiB; - else if (mtu >= BUF_SIZE_4KiB) + else if (len > BUF_SIZE_4KiB) ret = BUF_SIZE_8KiB; - else if (mtu >= BUF_SIZE_2KiB) + else if (len > BUF_SIZE_2KiB) ret = BUF_SIZE_4KiB; - else if (mtu > DEFAULT_BUFSIZE) + else if (len > DEFAULT_BUFSIZE) ret = BUF_SIZE_2KiB; else ret = DEFAULT_BUFSIZE; @@ -2719,7 +2723,8 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); - meta = xsk_buff_get_metadata(pool, xdp_desc.addr); + meta = xsk_buff_get_metadata(pool, xdp_desc.addr, + xdp_desc.options); xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); /* To return XDP buffer to XSK pool, we simple call @@ -3671,6 +3676,14 @@ static int stmmac_hw_setup(struct net_device *dev) /* Initialize MTL*/ stmmac_mtl_configuration(priv); + /* Apply the RX packet parser table */ + if (priv->tc_entries) { + ret = stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries, + priv->tc_entries_max); + if (ret) + return ret; + } + /* Initialize Safety Features */ stmmac_safety_feat_configuration(priv); @@ -4050,7 +4063,7 @@ static struct stmmac_dma_conf * stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) { struct stmmac_dma_conf *dma_conf; - int bfsize, ret; + int bfsize, len, ret; u8 chan; dma_conf = kzalloc_obj(*dma_conf); @@ -4060,13 +4073,15 @@ stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) return ERR_PTR(-ENOMEM); } - /* Returns 0 or BUF_SIZE_16KiB if mtu > 8KiB and dwmac4 or ring mode */ - bfsize = stmmac_set_16kib_bfsize(priv, mtu); + len = mtu + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN; + + /* Returns 0 or BUF_SIZE_16KiB if len > 8KiB and dwmac4 or ring mode */ + bfsize = stmmac_set_16kib_bfsize(priv, len); if (bfsize < 0) bfsize = 0; if (bfsize < BUF_SIZE_16KiB) - bfsize = stmmac_set_bfsize(mtu); + bfsize = stmmac_set_bfsize(len); dma_conf->dma_buf_sz = bfsize; /* Chose the tx/rx size from the already defined one in the @@ -4314,6 +4329,7 @@ static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, /** * stmmac_tso_allocator - close entry point of the driver * @priv: driver private structure + * @entry: TX queue buffer index * @des: buffer start address * @total_len: total length to fill in descriptors * @last_segment: condition for the last descriptor @@ -4322,8 +4338,9 @@ static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, * This function fills descriptor and request new descriptors according to * buffer length to fill */ -static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, - int total_len, bool last_segment, u32 queue) +static void stmmac_tso_allocator(struct stmmac_priv *priv, u32 *entry, + dma_addr_t des, int total_len, + bool last_segment, u32 queue) { struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; struct dma_desc *desc; @@ -4335,14 +4352,13 @@ static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, while (tmp_len > 0) { dma_addr_t curr_addr; - tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, - priv->dma_conf.dma_tx_size); - WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); + *entry = STMMAC_NEXT_ENTRY(*entry, priv->dma_conf.dma_tx_size); + WARN_ON(tx_q->tx_skbuff[*entry]); if (tx_q->tbs & STMMAC_TBS_AVAIL) - desc = &tx_q->dma_entx[tx_q->cur_tx].basic; + desc = &tx_q->dma_entx[*entry].basic; else - desc = &tx_q->dma_tx[tx_q->cur_tx]; + desc = &tx_q->dma_tx[*entry]; curr_addr = des + (total_len - tmp_len); stmmac_set_desc_addr(priv, desc, curr_addr); @@ -4371,18 +4387,6 @@ static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, tx_q->cur_tx); } -static void stmmac_set_gso_types(struct stmmac_priv *priv, bool tso) -{ - if (!tso) { - priv->gso_enabled_types = 0; - } else { - /* Manage oversized TCP frames for GMAC4 device */ - priv->gso_enabled_types = SKB_GSO_TCPV4 | SKB_GSO_TCPV6; - if (priv->plat->core_type == DWMAC_CORE_GMAC4) - priv->gso_enabled_types |= SKB_GSO_UDP_L4; - } -} - static void stmmac_set_gso_features(struct net_device *ndev) { struct stmmac_priv *priv = netdev_priv(ndev); @@ -4416,8 +4420,6 @@ static void stmmac_set_gso_features(struct net_device *ndev) if (priv->plat->core_type == DWMAC_CORE_GMAC4) ndev->hw_features |= NETIF_F_GSO_UDP_L4; - stmmac_set_gso_types(priv, true); - dev_info(priv->device, "TSO feature enabled\n"); } @@ -4454,6 +4456,26 @@ static bool stmmac_tso_valid_packet(struct sk_buff *skb) header_len + gso_size < 16383; } +static int stmmac_tso_get_num_desc(struct stmmac_tx_queue *tx_q, + struct sk_buff *skb, u32 pay_len) +{ + int i, ndesc = 1; + + /* head payload */ + ndesc += DIV_ROUND_UP(pay_len, TSO_MAX_BUFF_SIZE); + /* frag payload */ + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + + ndesc += DIV_ROUND_UP(skb_frag_size(frag), + TSO_MAX_BUFF_SIZE); + } + /* MSS update requires a new descriptor */ + ndesc += !!(skb_shinfo(skb)->gso_size != tx_q->mss); + + return ndesc; +} + /** * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) * @skb : the socket buffer @@ -4493,16 +4515,16 @@ static bool stmmac_tso_valid_packet(struct sk_buff *skb) */ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) { + unsigned int first_entry, entry, tx_packets, proto_hdr_len; struct dma_desc *desc, *first, *mss_desc = NULL; struct stmmac_priv *priv = netdev_priv(dev); - unsigned int first_entry, tx_packets; struct stmmac_txq_stats *txq_stats; + int i, first_tx, nfrags, ndesc; struct stmmac_tx_queue *tx_q; bool set_ic, is_last_segment; u32 pay_len, mss, queue; - int i, first_tx, nfrags; - u8 proto_hdr_len, hdr; dma_addr_t des; + u8 hdr; nfrags = skb_shinfo(skb)->nr_frags; queue = skb_get_queue_mapping(skb); @@ -4513,14 +4535,15 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) /* Compute header lengths */ proto_hdr_len = stmmac_tso_header_size(skb); + pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) hdr = sizeof(struct udphdr); else hdr = tcp_hdrlen(skb); - /* Desc availability based on threshold should be enough safe */ - if (unlikely(stmmac_tx_avail(priv, queue) < - (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { + ndesc = stmmac_tso_get_num_desc(tx_q, skb, pay_len); + if (unlikely(stmmac_tx_avail(priv, queue) < ndesc)) { if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); @@ -4532,8 +4555,6 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_BUSY; } - pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ - mss = skb_shinfo(skb)->gso_size; /* set new MSS value if needed */ @@ -4551,29 +4572,31 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) } if (netif_msg_tx_queued(priv)) { - pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", + pr_info("%s: hdrlen %d, hdr_len %u, pay_len %d, mss %d\n", __func__, hdr, proto_hdr_len, pay_len, mss); pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, skb->data_len); } first_entry = tx_q->cur_tx; - WARN_ON(tx_q->tx_skbuff[first_entry]); + entry = first_entry; + + WARN_ON(tx_q->tx_skbuff[entry]); if (tx_q->tbs & STMMAC_TBS_AVAIL) - desc = &tx_q->dma_entx[first_entry].basic; + desc = &tx_q->dma_entx[entry].basic; else - desc = &tx_q->dma_tx[first_entry]; + desc = &tx_q->dma_tx[entry]; first = desc; /* first descriptor: fill Headers on Buf1 */ des = dma_map_single(priv->device, skb->data, skb_headlen(skb), DMA_TO_DEVICE); if (dma_mapping_error(priv->device, des)) - goto dma_map_err; + goto error; stmmac_set_desc_addr(priv, first, des); - stmmac_tso_allocator(priv, des + proto_hdr_len, pay_len, + stmmac_tso_allocator(priv, &entry, des + proto_hdr_len, pay_len, (nfrags == 0), queue); /* In case two or more DMA transmit descriptors are allocated for this @@ -4588,8 +4611,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) * this DMA buffer right after the DMA engine completely finishes the * full buffer transmission. */ - stmmac_set_tx_skb_dma_entry(tx_q, tx_q->cur_tx, des, skb_headlen(skb), - false); + stmmac_set_tx_skb_dma_entry(tx_q, entry, des, skb_headlen(skb), false); /* Prepare fragments */ for (i = 0; i < nfrags; i++) { @@ -4599,14 +4621,15 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) skb_frag_size(frag), DMA_TO_DEVICE); if (dma_mapping_error(priv->device, des)) - goto dma_map_err; + goto error_dma_unmap; - stmmac_tso_allocator(priv, des, skb_frag_size(frag), + stmmac_tso_allocator(priv, &entry, des, skb_frag_size(frag), (i == nfrags - 1), queue); - stmmac_set_tx_skb_dma_entry(tx_q, tx_q->cur_tx, des, + stmmac_set_tx_skb_dma_entry(tx_q, entry, des, skb_frag_size(frag), true); } + tx_q->cur_tx = entry; stmmac_set_tx_dma_last_segment(tx_q, tx_q->cur_tx); @@ -4709,7 +4732,19 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_OK; -dma_map_err: +error_dma_unmap: + for (;;) { + desc = stmmac_get_tx_desc(priv, tx_q, first_entry); + stmmac_release_tx_desc(priv, desc, priv->descriptor_mode); + stmmac_free_tx_buffer(priv, &priv->dma_conf, queue, + first_entry); + if (first_entry == entry) + break; + + first_entry = STMMAC_NEXT_ENTRY(first_entry, + priv->dma_conf.dma_tx_size); + } +error: dev_err(priv->device, "Tx dma map failed\n"); dev_kfree_skb(skb); priv->xstats.tx_dropped++; @@ -4765,8 +4800,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) stmmac_stop_sw_lpi(priv); - if (skb_is_gso(skb) && - skb_shinfo(skb)->gso_type & priv->gso_enabled_types) + if (skb_is_gso(skb)) return stmmac_tso_xmit(skb, dev); if (priv->est && priv->est->enable && @@ -5851,6 +5885,7 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) if (!skb) { page_pool_recycle_direct(rx_q->page_pool, buf->page); + buf->page = NULL; rx_dropped++; count++; goto drain_data; @@ -6198,8 +6233,6 @@ static int stmmac_set_features(struct net_device *netdev, stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); } - stmmac_set_gso_types(priv, features & NETIF_F_TSO); - if (features & NETIF_F_HW_VLAN_CTAG_RX) priv->hw->hw_vlan_en = true; else @@ -8021,6 +8054,7 @@ static int __stmmac_dvr_probe(struct device *device, stmmac_napi_add(ndev); mutex_init(&priv->lock); + rwlock_init(&priv->ptp_lock); stmmac_fpe_init(priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index 960249960004c..3bfcc9760dce7 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -365,7 +365,6 @@ void stmmac_ptp_register(struct stmmac_priv *priv) if (priv->plat->crosststamp) priv->ptp_clock_ops.getcrosststamp = stmmac_getcrosststamp; - rwlock_init(&priv->ptp_lock); mutex_init(&priv->aux_ts_lock); priv->ptp_clock = ptp_clock_register(&priv->ptp_clock_ops, diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c index a0c75886587c3..75647992cbb44 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ struct stmmachdr { sizeof(struct stmmachdr)) #define STMMAC_TEST_PKT_MAGIC 0xdeadcafecafedeadULL #define STMMAC_LB_TIMEOUT msecs_to_jiffies(200) +#define STMMAC_SFT_MAX_LPI (5 * USEC_PER_SEC) struct stmmac_packet_attrs { int vlan; @@ -238,6 +240,10 @@ struct stmmac_test_priv { struct stmmac_packet_attrs *packet; struct packet_type pt; struct completion comp; + __be16 packet_type; + int (*func)(struct sk_buff *skb, struct net_device *ndev, + struct packet_type *pt, struct net_device *orig_ndev); + bool capture_all; int double_vlan; int vlan_id; int ok; @@ -317,6 +323,52 @@ static int stmmac_test_loopback_validate(struct sk_buff *skb, return 0; } +static int stmmac_sft_filter(struct sk_buff *skb, struct net_device *ndev, + struct packet_type *pt, + struct net_device *orig_ndev) +{ + struct stmmac_test_priv *tpriv = pt->af_packet_priv; + struct ethhdr *hdr = eth_hdr(skb); + int ret = 0; + + if (hdr->h_proto == tpriv->packet_type) { + struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); + + if (nskb) + ret = tpriv->func(nskb, ndev, pt, orig_ndev); + } + + kfree_skb(skb); + return ret; +} + +static void stmmac_sft_add_pack(struct packet_type *pt) +{ + struct stmmac_test_priv *tpriv = pt->af_packet_priv; + + if (netdev_uses_dsa(tpriv->pt.dev) || tpriv->capture_all) { + tpriv->packet_type = tpriv->pt.type; + tpriv->func = tpriv->pt.func; + + /* DSA conduit will report ETH_P_XDSA, so our packet handler + * won't match. Let's register a ETH_P_ALL match and filter + * manually in stmmac_sft_filter. This is also useful for + * VLAN tests, to capture packets otherwise marked as + * OTHERHOST. + */ + tpriv->pt.type = htons(ETH_P_ALL); + tpriv->pt.func = stmmac_sft_filter; + tpriv->pt.ignore_outgoing = true; + } + + dev_add_pack(pt); +} + +static void stmmac_sft_remove_pack(struct packet_type *pt) +{ + dev_remove_pack(pt); +} + static int __stmmac_test_loopback(struct stmmac_priv *priv, struct stmmac_packet_attrs *attr) { @@ -338,7 +390,7 @@ static int __stmmac_test_loopback(struct stmmac_priv *priv, tpriv->packet = attr; if (!attr->dont_wait) - dev_add_pack(&tpriv->pt); + stmmac_sft_add_pack(&tpriv->pt); skb = stmmac_test_get_udp_skb(priv, attr); if (!skb) { @@ -361,7 +413,7 @@ static int __stmmac_test_loopback(struct stmmac_priv *priv, cleanup: if (!attr->dont_wait) - dev_remove_pack(&tpriv->pt); + stmmac_sft_remove_pack(&tpriv->pt); kfree(tpriv); return ret; } @@ -415,11 +467,17 @@ static int stmmac_test_mmc(struct stmmac_priv *priv) stmmac_mmc_read(priv, priv->mmcaddr, &final); /* - * The number of MMC counters available depends on HW configuration - * so we just use this one to validate the feature. I hope there is - * not a version without this counter. + * The number of MMC counters available depends on HW configuration, + * and there doesn't seem to be a way to enumerate the implemented + * counters. + * + * Let's check a hand-picked set of counters, knowing that : + * - Starfive JH7110 doesn't implement mmc_tx_framecount_g + * - Amlogic SM1 doesn't implement any mmc_tx_* + * */ - if (final.mmc_tx_framecount_g <= initial.mmc_tx_framecount_g) + if (final.mmc_tx_framecount_g <= initial.mmc_tx_framecount_g && + final.mmc_rx_framecount_gb <= initial.mmc_rx_framecount_gb) return -EINVAL; return 0; @@ -428,12 +486,16 @@ static int stmmac_test_mmc(struct stmmac_priv *priv) static int stmmac_test_eee(struct stmmac_priv *priv) { struct stmmac_extra_stats *initial, *final; - int retries = 10; + unsigned long timeout, max_duration; int ret; if (!priv->dma_cap.eee || !priv->eee_active) return -EOPNOTSUPP; + /* Bail out if the configured LPI timer is too long */ + if (priv->tx_lpi_timer > STMMAC_SFT_MAX_LPI) + return -EOPNOTSUPP; + initial = kzalloc_obj(*initial); if (!initial) return -ENOMEM; @@ -444,14 +506,21 @@ static int stmmac_test_eee(struct stmmac_priv *priv) goto out_free_initial; } + /* Snapshot stats, we want to count the in_lpi events. We may enter + * LPI just after the packet was sent. + */ memcpy(initial, &priv->xstats, sizeof(*initial)); + /* Send a frame, then wait to enter LPI */ ret = stmmac_test_mac_loopback(priv); if (ret) goto out_free_final; + max_duration = usecs_to_jiffies(2 * priv->tx_lpi_timer); + /* We have no traffic in the line so, sooner or later it will go LPI */ - while (--retries) { + timeout = jiffies + max_duration; + while (!time_after(jiffies, timeout)) { memcpy(final, &priv->xstats, sizeof(*final)); if (final->irq_tx_path_in_lpi_mode_n > @@ -460,20 +529,38 @@ static int stmmac_test_eee(struct stmmac_priv *priv) msleep(100); } - if (!retries) { + memcpy(final, &priv->xstats, sizeof(*final)); + if (final->irq_tx_path_in_lpi_mode_n <= + initial->irq_tx_path_in_lpi_mode_n) { ret = -ETIMEDOUT; goto out_free_final; } - if (final->irq_tx_path_in_lpi_mode_n <= - initial->irq_tx_path_in_lpi_mode_n) { - ret = -EINVAL; + /* Re-snapshot, as we want to measure exit_lpi events. We should be + * in LPI right now. + */ + memcpy(initial, &priv->xstats, sizeof(*initial)); + + /* TX something so we go out of LPI */ + ret = stmmac_test_mac_loopback(priv); + if (ret) goto out_free_final; + + /* Wait for the exit LPI interrupt */ + timeout = jiffies + max_duration; + while (!time_after(jiffies, timeout)) { + memcpy(final, &priv->xstats, sizeof(*final)); + + if (final->irq_tx_path_exit_lpi_mode_n > + initial->irq_tx_path_exit_lpi_mode_n) + break; + msleep(100); } + memcpy(final, &priv->xstats, sizeof(*final)); if (final->irq_tx_path_exit_lpi_mode_n <= initial->irq_tx_path_exit_lpi_mode_n) { - ret = -EINVAL; + ret = -ETIMEDOUT; goto out_free_final; } @@ -493,6 +580,21 @@ static int stmmac_filter_check(struct stmmac_priv *priv) return -EOPNOTSUPP; } +static int stmmac_uc_filter_check(struct stmmac_priv *priv) +{ + /* For tests involving the UC filter, we need at least one empty + * slot in the UC filter. The UC filters contains netdev_uc_count() + 1 + * entries: The dev->uc list + one entry for the HW address. + * + * Having an empty slot therefore means netdev_uc_count() + 2 entries + * can fit in the filter + */ + if (netdev_uc_count(priv->dev) + 2 > priv->hw->unicast_filter_entries) + return -EOPNOTSUPP; + + return 0; +} + static bool stmmac_hash_check(struct stmmac_priv *priv, unsigned char *addr) { int mc_offset = 32 - priv->hw->mcast_bits_log2; @@ -584,7 +686,7 @@ static int stmmac_test_pfilt(struct stmmac_priv *priv) if (stmmac_filter_check(priv)) return -EOPNOTSUPP; - if (netdev_uc_count(priv->dev) >= priv->hw->unicast_filter_entries) + if (stmmac_uc_filter_check(priv)) return -EOPNOTSUPP; while (--tries) { @@ -628,7 +730,7 @@ static int stmmac_test_mcfilt(struct stmmac_priv *priv) if (stmmac_filter_check(priv)) return -EOPNOTSUPP; - if (netdev_uc_count(priv->dev) >= priv->hw->unicast_filter_entries) + if (stmmac_uc_filter_check(priv)) return -EOPNOTSUPP; if (netdev_mc_count(priv->dev) >= priv->hw->multicast_filter_bins) return -EOPNOTSUPP; @@ -674,7 +776,7 @@ static int stmmac_test_ucfilt(struct stmmac_priv *priv) if (stmmac_filter_check(priv)) return -EOPNOTSUPP; - if (netdev_uc_count(priv->dev) >= priv->hw->unicast_filter_entries) + if (stmmac_uc_filter_check(priv)) return -EOPNOTSUPP; if (netdev_mc_count(priv->dev) >= priv->hw->multicast_filter_bins) return -EOPNOTSUPP; @@ -738,12 +840,24 @@ static int stmmac_test_flowctrl(struct stmmac_priv *priv) struct phy_device *phydev = priv->dev->phydev; u32 rx_cnt = priv->plat->rx_queues_to_use; struct stmmac_test_priv *tpriv; + unsigned int rx_fifo_size; unsigned int pkt_count; int i, ret = 0; if (!phydev || (!phydev->pause && !phydev->asym_pause)) return -EOPNOTSUPP; + rx_fifo_size = priv->plat->rx_fifo_size; + if (!rx_fifo_size) + rx_fifo_size = priv->dma_cap.rx_fifo_size; + + /* No pause frame is emitted if we don't have at least 4096 bytes per + * queue, except on dwmac100. + */ + if (priv->plat->core_type != DWMAC_CORE_MAC100 && + rx_fifo_size / priv->plat->rx_queues_to_use < 4096) + return -EOPNOTSUPP; + tpriv = kzalloc_obj(*tpriv); if (!tpriv) return -ENOMEM; @@ -754,12 +868,10 @@ static int stmmac_test_flowctrl(struct stmmac_priv *priv) tpriv->pt.func = stmmac_test_flowctrl_validate; tpriv->pt.dev = priv->dev; tpriv->pt.af_packet_priv = tpriv; - dev_add_pack(&tpriv->pt); + stmmac_sft_add_pack(&tpriv->pt); /* Compute minimum number of packets to make FIFO full */ - pkt_count = priv->plat->rx_fifo_size; - if (!pkt_count) - pkt_count = priv->dma_cap.rx_fifo_size; + pkt_count = rx_fifo_size; pkt_count /= 1400; pkt_count *= 2; @@ -812,7 +924,7 @@ static int stmmac_test_flowctrl(struct stmmac_priv *priv) cleanup: dev_mc_del(priv->dev, paddr); dev_set_promiscuity(priv->dev, -1); - dev_remove_pack(&tpriv->pt); + stmmac_sft_remove_pack(&tpriv->pt); kfree(tpriv); return ret; } @@ -854,6 +966,11 @@ static int stmmac_test_vlan_validate(struct sk_buff *skb, goto out; if (skb_headlen(skb) < (STMMAC_TEST_PKT_SIZE - ETH_HLEN)) goto out; + + ehdr = (struct ethhdr *)skb_mac_header(skb); + if (!ether_addr_equal_unaligned(ehdr->h_dest, tpriv->packet->dst)) + goto out; + if (tpriv->vlan_id) { if (skb->vlan_proto != htons(proto)) goto out; @@ -865,10 +982,6 @@ static int stmmac_test_vlan_validate(struct sk_buff *skb, } } - ehdr = (struct ethhdr *)skb_mac_header(skb); - if (!ether_addr_equal_unaligned(ehdr->h_dest, tpriv->packet->dst)) - goto out; - ihdr = ip_hdr(skb); if (tpriv->double_vlan) ihdr = (struct iphdr *)(skb_network_header(skb) + 4); @@ -910,6 +1023,7 @@ static int __stmmac_test_vlanfilt(struct stmmac_priv *priv) tpriv->pt.dev = priv->dev; tpriv->pt.af_packet_priv = tpriv; tpriv->packet = &attr; + tpriv->capture_all = true; /* * As we use HASH filtering, false positives may appear. This is a @@ -917,18 +1031,20 @@ static int __stmmac_test_vlanfilt(struct stmmac_priv *priv) * HASH values. */ tpriv->vlan_id = 0x123; - dev_add_pack(&tpriv->pt); ret = vlan_vid_add(priv->dev, htons(ETH_P_8021Q), tpriv->vlan_id); if (ret) goto cleanup; + attr.vlan = 1; + attr.dst = priv->dev->dev_addr; + attr.sport = 9; + attr.dport = 9; + + stmmac_sft_add_pack(&tpriv->pt); + for (i = 0; i < 4; i++) { - attr.vlan = 1; attr.vlan_id_out = tpriv->vlan_id + i; - attr.dst = priv->dev->dev_addr; - attr.sport = 9; - attr.dport = 9; skb = stmmac_test_get_udp_skb(priv, &attr); if (!skb) { @@ -955,9 +1071,9 @@ static int __stmmac_test_vlanfilt(struct stmmac_priv *priv) } vlan_del: + stmmac_sft_remove_pack(&tpriv->pt); vlan_vid_del(priv->dev, htons(ETH_P_8021Q), tpriv->vlan_id); cleanup: - dev_remove_pack(&tpriv->pt); kfree(tpriv); return ret; } @@ -1004,6 +1120,7 @@ static int __stmmac_test_dvlanfilt(struct stmmac_priv *priv) tpriv->pt.dev = priv->dev; tpriv->pt.af_packet_priv = tpriv; tpriv->packet = &attr; + tpriv->capture_all = true; /* * As we use HASH filtering, false positives may appear. This is a @@ -1011,18 +1128,20 @@ static int __stmmac_test_dvlanfilt(struct stmmac_priv *priv) * HASH values. */ tpriv->vlan_id = 0x123; - dev_add_pack(&tpriv->pt); ret = vlan_vid_add(priv->dev, htons(ETH_P_8021AD), tpriv->vlan_id); if (ret) goto cleanup; + attr.vlan = 2; + attr.dst = priv->dev->dev_addr; + attr.sport = 9; + attr.dport = 9; + + stmmac_sft_add_pack(&tpriv->pt); + for (i = 0; i < 4; i++) { - attr.vlan = 2; attr.vlan_id_out = tpriv->vlan_id + i; - attr.dst = priv->dev->dev_addr; - attr.sport = 9; - attr.dport = 9; skb = stmmac_test_get_udp_skb(priv, &attr); if (!skb) { @@ -1049,9 +1168,9 @@ static int __stmmac_test_dvlanfilt(struct stmmac_priv *priv) } vlan_del: + stmmac_sft_remove_pack(&tpriv->pt); vlan_vid_del(priv->dev, htons(ETH_P_8021AD), tpriv->vlan_id); cleanup: - dev_remove_pack(&tpriv->pt); kfree(tpriv); return ret; } @@ -1282,7 +1401,7 @@ static int stmmac_test_vlanoff_common(struct stmmac_priv *priv, bool svlan) tpriv->pt.af_packet_priv = tpriv; tpriv->packet = &attr; tpriv->vlan_id = 0x123; - dev_add_pack(&tpriv->pt); + tpriv->capture_all = true; ret = vlan_vid_add(priv->dev, htons(proto), tpriv->vlan_id); if (ret) @@ -1290,6 +1409,8 @@ static int stmmac_test_vlanoff_common(struct stmmac_priv *priv, bool svlan) attr.dst = priv->dev->dev_addr; + stmmac_sft_add_pack(&tpriv->pt); + skb = stmmac_test_get_udp_skb(priv, &attr); if (!skb) { ret = -ENOMEM; @@ -1307,9 +1428,9 @@ static int stmmac_test_vlanoff_common(struct stmmac_priv *priv, bool svlan) ret = tpriv->ok ? 0 : -ETIMEDOUT; vlan_del: + stmmac_sft_remove_pack(&tpriv->pt); vlan_vid_del(priv->dev, htons(proto), tpriv->vlan_id); cleanup: - dev_remove_pack(&tpriv->pt); kfree(tpriv); return ret; } @@ -1321,7 +1442,7 @@ static int stmmac_test_vlanoff(struct stmmac_priv *priv) static int stmmac_test_svlanoff(struct stmmac_priv *priv) { - if (!priv->dma_cap.dvlan) + if (!(priv->dev->features & NETIF_F_HW_VLAN_STAG_TX)) return -EOPNOTSUPP; return stmmac_test_vlanoff_common(priv, true); } @@ -1452,11 +1573,11 @@ static int __stmmac_test_l4filt(struct stmmac_priv *priv, u32 dst, u32 src, struct { struct flow_dissector_key_basic bkey; struct flow_dissector_key_ports key; - } __aligned(BITS_PER_LONG / 8) keys; + } __aligned(BITS_PER_LONG / 8) keys = { }; struct { struct flow_dissector_key_basic bmask; struct flow_dissector_key_ports mask; - } __aligned(BITS_PER_LONG / 8) masks; + } __aligned(BITS_PER_LONG / 8) masks = { }; unsigned long dummy_cookie = 0xdeadbeef; struct stmmac_packet_attrs attr = { }; struct flow_dissector *dissector; @@ -1509,6 +1630,8 @@ static int __stmmac_test_l4filt(struct stmmac_priv *priv, u32 dst, u32 src, keys.bkey.ip_proto = udp ? IPPROTO_UDP : IPPROTO_TCP; keys.key.src = htons(src); keys.key.dst = htons(dst); + /* Match the full IP proto field */ + masks.bmask.ip_proto = 0xff; masks.mask.src = src_mask; masks.mask.dst = dst_mask; @@ -1686,6 +1809,9 @@ static int __stmmac_test_jumbo(struct stmmac_priv *priv, u16 queue) struct stmmac_packet_attrs attr = { }; int size = priv->dma_conf.dma_buf_sz; + if (!dwmac_is_xmac(priv->plat->core_type)) + size -= NET_IP_ALIGN; + attr.dst = priv->dev->dev_addr; attr.max_size = size - ETH_FCS_LEN; attr.queue_mapping = queue; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c index 14cabe76e53ec..42a00446e9b41 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c @@ -970,7 +970,7 @@ static int tc_taprio_configure(struct stmmac_priv *priv, struct netlink_ext_ack *extack = qopt->mqprio.extack; struct timespec64 time, current_time, qopt_time; ktime_t current_time_ns; - int i, ret = 0; + int err, i, ret = 0; u64 ctr; if (qopt->base_time < 0) @@ -1120,9 +1120,9 @@ static int tc_taprio_configure(struct stmmac_priv *priv, mutex_unlock(&priv->est_lock); } - stmmac_fpe_map_preemption_class(priv, priv->dev, extack, 0); + err = stmmac_fpe_map_preemption_class(priv, priv->dev, extack, 0); - return ret; + return qopt->cmd == TAPRIO_CMD_DESTROY ? err : ret; } static void tc_taprio_stats(struct stmmac_priv *priv, @@ -1237,58 +1237,99 @@ static int tc_query_caps(struct stmmac_priv *priv, } } -static void stmmac_reset_tc_mqprio(struct net_device *ndev, - struct netlink_ext_ack *extack) +static int stmmac_set_ndev_tcs(struct net_device *ndev, u8 ntc, + struct netdev_tc_txq *tc_to_txq) +{ + int i, err; + + netdev_reset_tc(ndev); + if (!ntc) + return 0; + + err = netdev_set_num_tc(ndev, ntc); + if (err) + return err; + + for (i = 0; i < ntc; i++) { + u16 count, offset; + + count = tc_to_txq[i].count; + offset = tc_to_txq[i].offset; + netdev_set_tc_queue(ndev, i, count, offset); + } + + return 0; +} + +static int stmmac_reset_tc_mqprio(struct net_device *ndev, + struct netlink_ext_ack *extack) { struct stmmac_priv *priv = netdev_priv(ndev); netdev_reset_tc(ndev); netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use); - stmmac_fpe_map_preemption_class(priv, ndev, extack, 0); + + return stmmac_fpe_map_preemption_class(priv, ndev, extack, 0); } static int tc_setup_dwmac510_mqprio(struct stmmac_priv *priv, struct tc_mqprio_qopt_offload *mqprio) { + unsigned int ndev_num_tx_queues, num_tx_queues = 0; + struct netdev_tc_txq ndev_tc_to_txq[TC_MAX_QUEUE]; + struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE] = {}; struct netlink_ext_ack *extack = mqprio->extack; struct tc_mqprio_qopt *qopt = &mqprio->qopt; - u32 offset, count, num_stack_tx_queues = 0; struct net_device *ndev = priv->dev; - u32 num_tc = qopt->num_tc; - int err; - - if (!num_tc) { - stmmac_reset_tc_mqprio(ndev, extack); - return 0; - } + u8 ndev_prio_tc_map[TC_BITMASK + 1]; + int i, err, ndev_ntc; - err = netdev_set_num_tc(ndev, num_tc); - if (err) - return err; + if (!qopt->num_tc) + return stmmac_reset_tc_mqprio(ndev, extack); - for (u32 tc = 0; tc < num_tc; tc++) { - offset = qopt->offset[tc]; - count = qopt->count[tc]; - num_stack_tx_queues += count; + if (qopt->num_tc > ARRAY_SIZE(tc_to_txq)) + return -EINVAL; - err = netdev_set_tc_queue(ndev, tc, count, offset); - if (err) - goto err_reset_tc; + /* save current tc values for reset */ + ndev_ntc = netdev_get_num_tc(ndev); + for (i = 0; i < ARRAY_SIZE(ndev->tc_to_txq); i++) + ndev_tc_to_txq[i].combined = + READ_ONCE(ndev->tc_to_txq[i].combined); + for (i = 0; i < ARRAY_SIZE(ndev_prio_tc_map); i++) + ndev_prio_tc_map[i] = READ_ONCE(ndev->prio_tc_map[i]); + + for (i = 0; i < qopt->num_tc; i++) { + tc_to_txq[i] = (struct netdev_tc_txq) { + .count = qopt->count[i], + .offset = qopt->offset[i], + }; + num_tx_queues += qopt->count[i]; } - err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); + err = stmmac_set_ndev_tcs(ndev, qopt->num_tc, tc_to_txq); + if (err) + goto error_reset_tc; + + ndev_num_tx_queues = ndev->real_num_tx_queues; + err = netif_set_real_num_tx_queues(ndev, num_tx_queues); if (err) - goto err_reset_tc; + goto error_reset_tc; err = stmmac_fpe_map_preemption_class(priv, ndev, extack, mqprio->preemptible_tcs); if (err) - goto err_reset_tc; + goto error_reset_num_tx_queues; return 0; -err_reset_tc: - stmmac_reset_tc_mqprio(ndev, extack); +error_reset_num_tx_queues: + if (netif_set_real_num_tx_queues(ndev, ndev_num_tx_queues)) + netdev_warn(ndev, "Failed to restore %u TX queues\n", + ndev_num_tx_queues); +error_reset_tc: + stmmac_set_ndev_tcs(ndev, ndev_ntc, ndev_tc_to_txq); + for (i = 0; i < ARRAY_SIZE(ndev_prio_tc_map); i++) + netdev_set_prio_tc_map(ndev, i, ndev_prio_tc_map[i]); return err; } diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c index eb8fc2ed05f45..4f9e20468bbbf 100644 --- a/drivers/net/ethernet/ti/netcp_core.c +++ b/drivers/net/ethernet/ti/netcp_core.c @@ -2225,7 +2225,7 @@ static int netcp_probe(struct platform_device *pdev) return -ENOMEM; pm_runtime_enable(&pdev->dev); - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { dev_err(dev, "Failed to enable NETCP power-domain\n"); pm_runtime_disable(&pdev->dev); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c index 260e14d5d5412..9119c931b3a3f 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c @@ -2517,6 +2517,7 @@ int wx_sw_init(struct wx *wx) } spin_lock_init(&wx->hw_stats_lock); + spin_lock_init(&wx->ptp_tx_lock); mutex_init(&wx->reset_lock); bitmap_zero(wx->state, WX_STATE_NBITS); bitmap_zero(wx->flags, WX_PF_FLAGS_NBITS); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c index 5d99e870de5ef..287172348b07e 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c @@ -1163,9 +1163,11 @@ static int wx_tx_map(struct wx_ring *tx_ring, i--; } - dev_kfree_skb_any(first->skb); - first->skb = NULL; - + /* first->skb is released by the caller, which keeps a reference on it + * until the PTP cleanup has compared it against wx->ptp_tx_skb. That + * prevents the address from being reused by a newer request while the + * comparison is pending. + */ tx_ring->next_to_use = i; return -ENOMEM; @@ -1612,9 +1614,11 @@ static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb, if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && wx->ptp_clock) { + unsigned long flags; + + spin_lock_irqsave(&wx->ptp_tx_lock, flags); if (wx->tstamp_config.tx_type == HWTSTAMP_TX_ON && - !test_and_set_bit_lock(WX_STATE_PTP_TX_IN_PROGRESS, - wx->state)) { + !test_and_set_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state)) { skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; tx_flags |= WX_TX_FLAGS_TSTAMP; wx->ptp_tx_skb = skb_get(skb); @@ -1622,6 +1626,7 @@ static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb, } else { wx->tx_hwtstamp_skipped++; } + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); } /* record initial flags and protocol */ @@ -1640,19 +1645,35 @@ static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb, wx->atr(tx_ring, first, ptype); if (wx_tx_map(tx_ring, first, hdr_len)) - goto cleanup_tx_tstamp; + goto out_drop; return NETDEV_TX_OK; out_drop: - dev_kfree_skb_any(first->skb); - first->skb = NULL; -cleanup_tx_tstamp: + /* The frame never reached the hardware, so no timestamp will ever be + * reported for it and the request has to be cancelled. The slot is + * shared, though: wx_ptp_clear_tx_timestamp() or wx_ptp_tx_hang() may + * have dropped our request already, and a transmit on another queue + * can have claimed the slot since. Only cancel it while it is still + * ours, otherwise we would free somebody else's skb and release their + * in-progress bit. + */ if (unlikely(tx_flags & WX_TX_FLAGS_TSTAMP)) { - dev_kfree_skb_any(wx->ptp_tx_skb); - wx->ptp_tx_skb = NULL; - wx->tx_hwtstamp_errors++; - clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); + struct sk_buff *ptp_tx_skb = NULL; + unsigned long flags; + + spin_lock_irqsave(&wx->ptp_tx_lock, flags); + if (wx->ptp_tx_skb == skb) { + ptp_tx_skb = wx->ptp_tx_skb; + wx->ptp_tx_skb = NULL; + clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); + wx->tx_hwtstamp_errors++; + } + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + + dev_kfree_skb_any(ptp_tx_skb); } + dev_kfree_skb_any(first->skb); + first->skb = NULL; return NETDEV_TX_OK; } @@ -3348,5 +3369,23 @@ void wx_service_timer(struct timer_list *t) } EXPORT_SYMBOL(wx_service_timer); +void wx_soft_quiesce(struct wx *wx) +{ + if (!netif_running(wx->netdev) || + test_and_set_bit(WX_STATE_DOWN, wx->state)) + return; + + pci_clear_master(wx->pdev); + netif_tx_stop_all_queues(wx->netdev); + netif_carrier_off(wx->netdev); + netif_tx_disable(wx->netdev); + wx_napi_disable_all(wx); + wx_ptp_quiesce(wx); + + clear_bit(WX_FLAG_NEED_DO_RESET, wx->flags); + timer_delete_sync(&wx->service_timer); +} +EXPORT_SYMBOL(wx_soft_quiesce); + MODULE_DESCRIPTION("Common library for Wangxun(R) Ethernet drivers."); MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.h b/drivers/net/ethernet/wangxun/libwx/wx_lib.h index aed6ea8cf0d64..11bd79985e171 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.h @@ -41,5 +41,6 @@ void wx_set_ring(struct wx *wx, u32 new_tx_count, void wx_service_event_schedule(struct wx *wx); void wx_service_event_complete(struct wx *wx); void wx_service_timer(struct timer_list *t); +void wx_soft_quiesce(struct wx *wx); #endif /* _WX_LIB_H_ */ diff --git a/drivers/net/ethernet/wangxun/libwx/wx_ptp.c b/drivers/net/ethernet/wangxun/libwx/wx_ptp.c index 44f3e65052468..65b8937f6e94c 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_ptp.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_ptp.c @@ -129,6 +129,34 @@ static int wx_ptp_settime64(struct ptp_clock_info *ptp, return 0; } +/** + * __wx_ptp_detach_tx_skb - detach the skb tracking the Tx timestamp request + * @wx: the private board structure + * + * Detach the skb of the outstanding request and release the in-progress bit, + * so that a new request can be submitted. + * + * This performs no register access. Callers that need a timestamp the hardware + * may have left latched must unlatch it themselves, while the device is known + * to be alive. wx_ptp_quiesce() runs during PCIe error recovery, where MMIO is + * not reliable, and therefore deliberately skips the unlatch. + * + * Context: Expects wx->ptp_tx_lock to be held by the caller. + * Return: the detached skb, or NULL if no request was outstanding. The caller + * owns the returned reference and must release it once the lock is dropped. + */ +static struct sk_buff *__wx_ptp_detach_tx_skb(struct wx *wx) +{ + struct sk_buff *skb = wx->ptp_tx_skb; + + lockdep_assert_held(&wx->ptp_tx_lock); + + wx->ptp_tx_skb = NULL; + clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); + + return skb; +} + /** * wx_ptp_clear_tx_timestamp - utility function to clear Tx timestamp state * @wx: the private board structure @@ -139,12 +167,16 @@ static int wx_ptp_settime64(struct ptp_clock_info *ptp, */ static void wx_ptp_clear_tx_timestamp(struct wx *wx) { + struct sk_buff *skb; + unsigned long flags; + + spin_lock_irqsave(&wx->ptp_tx_lock, flags); + /* Unlatch a timestamp the hardware may have left pending. */ rd32ptp(wx, WX_TSC_1588_STMPH); - if (wx->ptp_tx_skb) { - dev_kfree_skb_any(wx->ptp_tx_skb); - wx->ptp_tx_skb = NULL; - } - clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); + skb = __wx_ptp_detach_tx_skb(wx); + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + + dev_kfree_skb_any(skb); } /** @@ -175,49 +207,54 @@ static void wx_ptp_convert_to_hwtstamp(struct wx *wx, } /** - * wx_ptp_tx_hwtstamp - utility function which checks for TX time stamp + * wx_ptp_tx_hwtstamp_work - check for a pending Tx time stamp * @wx: the private board struct * - * if the timestamp is valid, we convert it into the timecounter ns - * value, then store that result into the shhwtstamps structure which - * is passed up the network stack + * If a Tx timestamp request is outstanding and the hardware has latched a + * valid value, we convert it into the timecounter ns value, then store that + * result into the shhwtstamps structure which is passed up the network stack. + * + * Return: 0 when there is nothing left to poll for, -1 when the timestamp is + * not available yet and the caller should poll again. */ -static void wx_ptp_tx_hwtstamp(struct wx *wx) +static int wx_ptp_tx_hwtstamp_work(struct wx *wx) { struct skb_shared_hwtstamps shhwtstamps; - struct sk_buff *skb = wx->ptp_tx_skb; + unsigned long flags; + struct sk_buff *skb; + u32 tsynctxctl; u64 regval = 0; - regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPL); - regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPH) << 32; - - wx_ptp_convert_to_hwtstamp(wx, &shhwtstamps, regval); - - wx->ptp_tx_skb = NULL; - clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); - skb_tstamp_tx(skb, &shhwtstamps); - dev_kfree_skb_any(skb); - wx->tx_hwtstamp_pkts++; -} - -static int wx_ptp_tx_hwtstamp_work(struct wx *wx) -{ - u32 tsynctxctl; + spin_lock_irqsave(&wx->ptp_tx_lock, flags); /* we have to have a valid skb to poll for a timestamp */ if (!wx->ptp_tx_skb) { - wx_ptp_clear_tx_timestamp(wx); + rd32ptp(wx, WX_TSC_1588_STMPH); + __wx_ptp_detach_tx_skb(wx); + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); return 0; } /* stop polling once we have a valid timestamp */ tsynctxctl = rd32ptp(wx, WX_TSC_1588_CTL); - if (tsynctxctl & WX_TSC_1588_CTL_VALID) { - wx_ptp_tx_hwtstamp(wx); - return 0; + if (!(tsynctxctl & WX_TSC_1588_CTL_VALID)) { + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + return -1; } - return -1; + regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPL); + regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPH) << 32; + skb = wx->ptp_tx_skb; + wx->ptp_tx_skb = NULL; + clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + + wx_ptp_convert_to_hwtstamp(wx, &shhwtstamps, regval); + skb_tstamp_tx(skb, &shhwtstamps); + dev_kfree_skb_any(skb); + wx->tx_hwtstamp_pkts++; + + return 0; } /** @@ -296,24 +333,29 @@ static void wx_ptp_rx_hang(struct wx *wx) */ static void wx_ptp_tx_hang(struct wx *wx) { - bool timeout = time_is_before_jiffies(wx->ptp_tx_start + - WX_PTP_TX_TIMEOUT); + struct sk_buff *skb = NULL; + unsigned long flags; - if (!wx->ptp_tx_skb) - return; - - if (!test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state)) - return; + spin_lock_irqsave(&wx->ptp_tx_lock, flags); /* If we haven't received a timestamp within the timeout, it is * reasonable to assume that it will never occur, so we can unlock the * timestamp bit when this occurs. */ - if (timeout) { - wx_ptp_clear_tx_timestamp(wx); - wx->tx_hwtstamp_timeouts++; - dev_warn(&wx->pdev->dev, "clearing Tx timestamp hang\n"); + if (wx->ptp_tx_skb && + test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state) && + time_is_before_jiffies(wx->ptp_tx_start + WX_PTP_TX_TIMEOUT)) { + rd32ptp(wx, WX_TSC_1588_STMPH); + skb = __wx_ptp_detach_tx_skb(wx); } + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + + if (!skb) + return; + + dev_kfree_skb_any(skb); + wx->tx_hwtstamp_timeouts++; + dev_warn(&wx->pdev->dev, "clearing Tx timestamp hang\n"); } static long wx_ptp_do_aux_work(struct ptp_clock_info *ptp) @@ -321,6 +363,9 @@ static long wx_ptp_do_aux_work(struct ptp_clock_info *ptp) struct wx *wx = container_of(ptp, struct wx, ptp_caps); int ts_done; + if (!test_bit(WX_STATE_PTP_RUNNING, wx->state)) + return HZ; + ts_done = wx_ptp_tx_hwtstamp_work(wx); wx_ptp_overflow_check(wx); @@ -555,13 +600,11 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, { u32 tsync_tx_ctl = WX_TSC_1588_CTL_ENABLED; u32 tsync_rx_ctl = WX_PSR_1588_CTL_ENABLED; - DECLARE_BITMAP(flags, WX_PF_FLAGS_NBITS); u32 tsync_rx_mtrl = PTP_EV_PORT << 16; + bool rx_tstamp = false; bool is_l2 = false; u32 regval; - memcpy(flags, wx->flags, sizeof(wx->flags)); - switch (config->tx_type) { case HWTSTAMP_TX_OFF: tsync_tx_ctl = 0; @@ -576,20 +619,16 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, case HWTSTAMP_FILTER_NONE: tsync_rx_ctl = 0; tsync_rx_mtrl = 0; - clear_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - clear_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); break; case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_L4_V1; tsync_rx_mtrl |= WX_PSR_1588_MSG_V1_SYNC; - set_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - set_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); + rx_tstamp = true; break; case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_L4_V1; tsync_rx_mtrl |= WX_PSR_1588_MSG_V1_DELAY_REQ; - set_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - set_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); + rx_tstamp = true; break; case HWTSTAMP_FILTER_PTP_V2_EVENT: case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: @@ -602,9 +641,8 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_EVENT_V2; is_l2 = true; + rx_tstamp = true; config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; - set_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - set_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); break; default: /* register PSR_1588_MSG must be set in order to do V1 packets, @@ -643,7 +681,8 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, WX_WRITE_FLUSH(wx); /* configure adapter flags only when HW is actually configured */ - memcpy(wx->flags, flags, sizeof(wx->flags)); + assign_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, wx->flags, rx_tstamp); + assign_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, wx->flags, rx_tstamp); /* clear TX/RX timestamp state, just to be sure */ wx_ptp_clear_tx_timestamp(wx); @@ -842,6 +881,36 @@ void wx_ptp_stop(struct wx *wx) } EXPORT_SYMBOL(wx_ptp_stop); +void wx_ptp_quiesce(struct wx *wx) +{ + struct sk_buff *skb; + unsigned long flags; + + if (!test_and_clear_bit(WX_STATE_PTP_RUNNING, wx->state)) + return; + + clear_bit(WX_FLAG_PTP_PPS_ENABLED, wx->flags); + + if (wx->ptp_clock) + ptp_cancel_worker_sync(wx->ptp_clock); + + /* Drop a pending Tx timestamp request. Do not touch the registers + * here: quiesce runs during PCIe error recovery, where the device may + * already be gone and MMIO is not reliable. + */ + spin_lock_irqsave(&wx->ptp_tx_lock, flags); + skb = __wx_ptp_detach_tx_skb(wx); + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + dev_kfree_skb_any(skb); + + if (wx->ptp_clock) { + ptp_clock_unregister(wx->ptp_clock); + wx->ptp_clock = NULL; + dev_info(&wx->pdev->dev, "removed PHC on %s\n", wx->netdev->name); + } +} +EXPORT_SYMBOL(wx_ptp_quiesce); + /** * wx_ptp_rx_hwtstamp - utility function which checks for RX time stamp * @wx: pointer to wx struct diff --git a/drivers/net/ethernet/wangxun/libwx/wx_ptp.h b/drivers/net/ethernet/wangxun/libwx/wx_ptp.h index 50db90a6e3ee6..ad2f824875d51 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_ptp.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_ptp.h @@ -10,6 +10,7 @@ void wx_ptp_reset(struct wx *wx); void wx_ptp_init(struct wx *wx); void wx_ptp_suspend(struct wx *wx); void wx_ptp_stop(struct wx *wx); +void wx_ptp_quiesce(struct wx *wx); void wx_ptp_rx_hwtstamp(struct wx *wx, struct sk_buff *skb); int wx_hwtstamp_get(struct net_device *dev, struct kernel_hwtstamp_config *cfg); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h index 65e3e55db1cff..e63a1666d8ed1 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_type.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h @@ -1413,6 +1413,8 @@ struct wx { unsigned long last_overflow_check; unsigned long last_rx_ptp_check; unsigned long ptp_tx_start; + /* protects ptp_tx_skb, ptp_tx_start and the in-progress state bit */ + spinlock_t ptp_tx_lock; seqlock_t hw_tc_lock; /* seqlock for ptp */ struct cyclecounter hw_cc; struct timecounter hw_tc; @@ -1427,7 +1429,7 @@ struct wx { }; #define WX_INTR_ALL (~0ULL) -#define WX_INTR_Q(i) BIT((i)) +#define WX_INTR_Q(i) BIT_ULL((i)) /* register operations */ #define wr32(a, reg, value) writel((value), ((a)->hw_addr + (reg))) diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c index a840108285517..59a47532618c8 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c @@ -591,15 +591,24 @@ static void txgbe_fdir_filter_restore(struct wx *wx) queue = TXGBE_RDB_FDIR_DROP_QUEUE; } else { u32 ring = ethtool_get_flow_spec_ring(filter->action); + u8 vf = ethtool_get_flow_spec_ring_vf(filter->action); - if (ring >= wx->num_rx_queues) { + if (!vf && ring >= wx->num_rx_queues) { wx_err(wx, "FDIR restore failed, ring:%u\n", ring); continue; + } else if (vf && (vf > wx->num_vfs || + ring >= wx->num_rx_queues_per_pool)) { + wx_err(wx, "FDIR restore failed, vf:%u, ring:%u\n", + vf, ring); + continue; } /* Map the ring onto the absolute queue index */ - queue = wx->rx_ring[ring]->reg_idx; + if (!vf) + queue = wx->rx_ring[ring]->reg_idx; + else + queue = ((vf - 1) * wx->num_rx_queues_per_pool) + ring; } ret = txgbe_fdir_write_perfect_filter(wx, diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c index 8746318ad3bcb..5ad1ff7c3ce97 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c @@ -164,6 +164,7 @@ static irqreturn_t txgbe_misc_irq_thread_fn(int irq, void *data) struct wx *wx = txgbe->wx; unsigned int nhandled = 0; unsigned int sub_irq; + u64 misc_mask; u32 eicr; eicr = txgbe->eicr; @@ -183,7 +184,9 @@ static irqreturn_t txgbe_misc_irq_thread_fn(int irq, void *data) nhandled++; } - wx_intr_enable(wx, TXGBE_INTR_MISC(wx)); + misc_mask = wx->pdev->msix_enabled ? TXGBE_INTR_MISC(wx) : BIT(0); + if (!test_bit(WX_STATE_DOWN, wx->state)) + wx_intr_enable(wx, misc_mask); return (nhandled > 0 ? IRQ_HANDLED : IRQ_NONE); } diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c index c277863baf67a..00123c0e3f222 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c @@ -93,12 +93,24 @@ static void txgbe_module_detection_subtask(struct wx *wx) { int err; + if (test_bit(WX_STATE_DOWN, wx->state) || + test_bit(WX_STATE_RESETTING, wx->state)) + return; + if (!test_and_clear_bit(WX_FLAG_NEED_MODULE_RESET, wx->flags)) return; /* wait for SFF module ready */ msleep(200); + /* Re-check state to avoid racing with down/reset paths. + * Module identification is deferred to the next up event, + * so it is safe to bail out here. + */ + if (test_bit(WX_STATE_DOWN, wx->state) || + test_bit(WX_STATE_RESETTING, wx->state)) + return; + err = txgbe_identify_module(wx); if (err == -ENODEV) set_bit(WX_FLAG_NEED_MODULE_RESET, wx->flags); @@ -106,6 +118,10 @@ static void txgbe_module_detection_subtask(struct wx *wx) static void txgbe_link_config_subtask(struct wx *wx) { + if (test_bit(WX_STATE_DOWN, wx->state) || + test_bit(WX_STATE_RESETTING, wx->state)) + return; + if (!test_and_clear_bit(WX_FLAG_NEED_LINK_CONFIG, wx->flags)) return; diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h index 877234e3fdc2b..fddcb011fa2fd 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h @@ -303,7 +303,7 @@ struct txgbe_fdir_filter { #define TXGBE_DEFAULT_RX_WORK 128 #endif -#define TXGBE_INTR_MISC(A) BIT((A)->num_q_vectors) +#define TXGBE_INTR_MISC(A) BIT_ULL((A)->num_q_vectors) #define TXGBE_INTR_QALL(A) (TXGBE_INTR_MISC(A) - 1) #define TXGBE_MAX_EITR GENMASK(11, 3) diff --git a/drivers/net/fddi/skfp/skfddi.c b/drivers/net/fddi/skfp/skfddi.c index a273362c9e703..feea7baa48168 100644 --- a/drivers/net/fddi/skfp/skfddi.c +++ b/drivers/net/fddi/skfp/skfddi.c @@ -928,7 +928,8 @@ static int skfp_ctl_set_mac_address(struct net_device *dev, void *addr) dev_addr_set(dev, p_sockaddr->sa_data); spin_lock_irqsave(&bp->DriverLock, Flags); - ResetAdapter(smc); + if (netif_running(dev)) + ResetAdapter(smc); spin_unlock_irqrestore(&bp->DriverLock, Flags); return 0; /* always return zero */ diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index 9a12cc53da002..69fe5717846b5 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -12,6 +12,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -108,6 +109,7 @@ struct gtp_net { }; static u32 gtp_h_initval; +static DEFINE_MUTEX(gtp_pdp_lock); static struct genl_family gtp_genl_family; @@ -151,7 +153,8 @@ static struct pdp_ctx *gtp0_pdp_find(struct gtp_dev *gtp, u64 tid, u16 family) head = >p->tid_hash[gtp0_hashfn(tid) % gtp->hash_size]; - hlist_for_each_entry_rcu(pdp, head, hlist_tid) { + hlist_for_each_entry_rcu(pdp, head, hlist_tid, + lockdep_is_held(>p_pdp_lock)) { if (pdp->af == family && pdp->gtp_version == GTP_V0 && pdp->u.v0.tid == tid) @@ -168,7 +171,8 @@ static struct pdp_ctx *gtp1_pdp_find(struct gtp_dev *gtp, u32 tid, u16 family) head = >p->tid_hash[gtp1u_hashfn(tid) % gtp->hash_size]; - hlist_for_each_entry_rcu(pdp, head, hlist_tid) { + hlist_for_each_entry_rcu(pdp, head, hlist_tid, + lockdep_is_held(>p_pdp_lock)) { if (pdp->af == family && pdp->gtp_version == GTP_V1 && pdp->u.v1.i_tei == tid) @@ -185,7 +189,8 @@ static struct pdp_ctx *ipv4_pdp_find(struct gtp_dev *gtp, __be32 ms_addr) head = >p->addr_hash[ipv4_hashfn(ms_addr) % gtp->hash_size]; - hlist_for_each_entry_rcu(pdp, head, hlist_addr) { + hlist_for_each_entry_rcu(pdp, head, hlist_addr, + lockdep_is_held(>p_pdp_lock)) { if (pdp->af == AF_INET && pdp->ms.addr.s_addr == ms_addr) return pdp; @@ -220,7 +225,8 @@ static struct pdp_ctx *ipv6_pdp_find(struct gtp_dev *gtp, head = >p->addr_hash[ipv6_hashfn(ms_addr) % gtp->hash_size]; - hlist_for_each_entry_rcu(pdp, head, hlist_addr) { + hlist_for_each_entry_rcu(pdp, head, hlist_addr, + lockdep_is_held(>p_pdp_lock)) { if (pdp->af == AF_INET6 && ipv6_pdp_addr_equal(&pdp->ms.addr6, ms_addr)) return pdp; @@ -312,6 +318,11 @@ static int gtp_inner_proto(struct sk_buff *skb, unsigned int hdrlen, static int gtp_rx(struct pdp_ctx *pctx, struct sk_buff *skb, unsigned int hdrlen, unsigned int role, __u16 inner_proto) { + if (skb_is_gso(skb)) { + netdev_dbg(pctx->dev, "GSO is not supported in GTP\n"); + goto err; + } + if (!gtp_check_ms(skb, pctx, hdrlen, role, inner_proto)) { netdev_dbg(pctx->dev, "No PDP ctx for this MS\n"); return 1; @@ -1543,6 +1554,8 @@ static int gtp_newlink(struct net_device *dev, out_encap: gtp_encap_disable(gtp); out_hashtable: + /* Wait for RCU readers that may still reference this gtp_dev. */ + synchronize_net(); kfree(gtp->addr_hash); kfree(gtp->tid_hash); return err; @@ -1555,9 +1568,11 @@ static void gtp_dellink(struct net_device *dev, struct list_head *head) struct pdp_ctx *pctx; int i; + mutex_lock(>p_pdp_lock); for (i = 0; i < gtp->hash_size; i++) hlist_for_each_entry_safe(pctx, next, >p->tid_hash[i], hlist_tid) pdp_context_delete(pctx); + mutex_unlock(>p_pdp_lock); list_del(>p->list); unregister_netdevice_queue(dev, head); @@ -2053,6 +2068,7 @@ static int gtp_genl_new_pdp(struct sk_buff *skb, struct genl_info *info) goto out_unlock; } + mutex_lock(>p_pdp_lock); pctx = gtp_pdp_add(gtp, sk, info); if (IS_ERR(pctx)) { err = PTR_ERR(pctx); @@ -2060,6 +2076,7 @@ static int gtp_genl_new_pdp(struct sk_buff *skb, struct genl_info *info) gtp_tunnel_notify(pctx, GTP_CMD_NEWPDP, GFP_KERNEL); err = 0; } + mutex_unlock(>p_pdp_lock); out_unlock: rtnl_unlock(); @@ -2134,6 +2151,8 @@ static int gtp_genl_del_pdp(struct sk_buff *skb, struct genl_info *info) if (!info->attrs[GTPA_VERSION]) return -EINVAL; + mutex_lock(>p_pdp_lock); + rcu_read_lock(); pctx = gtp_find_pdp(sock_net(skb->sk), info->attrs); @@ -2154,6 +2173,7 @@ static int gtp_genl_del_pdp(struct sk_buff *skb, struct genl_info *info) out_unlock: rcu_read_unlock(); + mutex_unlock(>p_pdp_lock); return err; } diff --git a/drivers/net/ieee802154/cc2520.c b/drivers/net/ieee802154/cc2520.c index 2b7034193a00b..abfcfe07246a7 100644 --- a/drivers/net/ieee802154/cc2520.c +++ b/drivers/net/ieee802154/cc2520.c @@ -1156,11 +1156,10 @@ static void cc2520_remove(struct spi_device *spi) { struct cc2520_private *priv = spi_get_drvdata(spi); - mutex_destroy(&priv->buffer_mutex); - flush_work(&priv->fifop_irqwork); - + disable_work_sync(&priv->fifop_irqwork); ieee802154_unregister_hw(priv->hw); ieee802154_free_hw(priv->hw); + mutex_destroy(&priv->buffer_mutex); } static const struct spi_device_id cc2520_ids[] = { diff --git a/drivers/net/ieee802154/mac802154_hwsim.c b/drivers/net/ieee802154/mac802154_hwsim.c index 6daa0f198b9fa..a9bd1555d2dc7 100644 --- a/drivers/net/ieee802154/mac802154_hwsim.c +++ b/drivers/net/ieee802154/mac802154_hwsim.c @@ -72,6 +72,8 @@ struct hwsim_phy { struct ieee802154_hw *hw; u32 idx; + /* Serializes phy->pib_updates. */ + spinlock_t pib_lock; struct hwsim_pib __rcu *pib; bool suspended; @@ -102,8 +104,6 @@ static int hwsim_update_pib(struct ieee802154_hw *hw, u8 page, u8 channel, if (!pib) return -ENOMEM; - pib_old = rtnl_dereference(phy->pib); - pib->page = page; pib->channel = channel; pib->filt.short_addr = filt->short_addr; @@ -112,7 +112,10 @@ static int hwsim_update_pib(struct ieee802154_hw *hw, u8 page, u8 channel, pib->filt.pan_coord = filt->pan_coord; pib->filt_level = filt_level; - rcu_assign_pointer(phy->pib, pib); + spin_lock_bh(&phy->pib_lock); + pib_old = rcu_replace_pointer(phy->pib, pib, + lockdep_is_held(&phy->pib_lock)); + spin_unlock_bh(&phy->pib_lock); kfree_rcu(pib_old, rcu); return 0; } @@ -952,6 +955,7 @@ static int hwsim_add_one(struct genl_info *info, struct device *dev, goto err_pib; } + spin_lock_init(&phy->pib_lock); pib->channel = 13; pib->filt.short_addr = cpu_to_le16(IEEE802154_ADDR_BROADCAST); pib->filt.pan_id = cpu_to_le16(IEEE802154_PANID_BROADCAST); diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c index 788dd99af2a4d..8d82fd5bb24b9 100644 --- a/drivers/net/ipa/ipa_main.c +++ b/drivers/net/ipa/ipa_main.c @@ -969,12 +969,12 @@ static void ipa_remove(struct platform_device *pdev) } if (ret) { /* - * Not cleaning up here properly might also yield a - * crash later on. As the device is still unregistered - * in this case, this might even yield a crash later on. + * Continuing teardown after failing to stop the modem + * could crash, so leave the remaining resources allocated. */ dev_err(dev, "Failed to stop modem (%pe), leaking resources\n", ERR_PTR(ret)); + pm_runtime_put_noidle(dev); return; } diff --git a/drivers/net/ipa/ipa_modem.c b/drivers/net/ipa/ipa_modem.c index 9b136f6b8b4ae..d84c1dbd3b1a8 100644 --- a/drivers/net/ipa/ipa_modem.c +++ b/drivers/net/ipa/ipa_modem.c @@ -266,13 +266,29 @@ void ipa_modem_suspend(struct net_device *netdev) * the modem. We can't enable the queue directly in ipa_modem_resume() * because transmits restart the instant the queue is awakened; but the * device power state won't be ACTIVE until *after* ipa_modem_resume() - * returns. + * returns. A transmit restarted before that would stop the queue + * again and get -EINPROGRESS from pm_runtime_get(), and with this + * work having already run, nothing would ever wake the queue again. + * So wait for the resume to complete before waking the queue. */ static void ipa_modem_wake_queue_work(struct work_struct *work) { struct ipa_priv *priv = container_of(work, struct ipa_priv, work); + struct device *dev = priv->ipa->dev; + int ret; + + ret = pm_runtime_get_sync(dev); + /* Wake the queue even if the device could not be resumed, so + * that pending packets are dropped by the transmit path rather + * than stranded behind a stopped queue. + */ netif_wake_queue(priv->tx->netdev); + + if (ret < 0) + pm_runtime_put_noidle(dev); + else + (void)pm_runtime_put_autosuspend(dev); } /** ipa_modem_resume() - resume callback for runtime_pm diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index ee0e2eb7dbc61..e4871cd765790 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -3539,6 +3539,22 @@ static int macsec_dev_init(struct net_device *dev) if (err) return err; + err = -ENOMEM; + macsec->stats = netdev_alloc_pcpu_stats(struct pcpu_secy_stats); + if (!macsec->stats) + goto destroy_gro_cells; + + macsec->secy.tx_sc.stats = + netdev_alloc_pcpu_stats(struct pcpu_tx_sc_stats); + if (!macsec->secy.tx_sc.stats) + goto free_secy_stats; + + macsec->secy.tx_sc.md_dst = metadata_dst_alloc(0, METADATA_MACSEC, + GFP_KERNEL); + if (!macsec->secy.tx_sc.md_dst) + goto free_tx_sc_stats; + macsec->secy.tx_sc.md_dst->u.macsec_info.sci = macsec->secy.sci; + macsec_inherit_tso_max(dev); dev->hw_features = real_dev->hw_features & MACSEC_OFFLOAD_FEATURES; @@ -3551,8 +3567,6 @@ static int macsec_dev_init(struct net_device *dev) macsec_set_head_tail_room(dev); - if (is_zero_ether_addr(dev->dev_addr)) - eth_hw_addr_inherit(dev, real_dev); if (is_zero_ether_addr(dev->broadcast)) memcpy(dev->broadcast, real_dev->broadcast, dev->addr_len); @@ -3560,6 +3574,14 @@ static int macsec_dev_init(struct net_device *dev) netdev_hold(real_dev, &macsec->dev_tracker, GFP_KERNEL); return 0; + +free_tx_sc_stats: + free_percpu(macsec->secy.tx_sc.stats); +free_secy_stats: + free_percpu(macsec->stats); +destroy_gro_cells: + gro_cells_destroy(&macsec->gro_cells); + return err; } static void macsec_dev_uninit(struct net_device *dev) @@ -4114,26 +4136,11 @@ static sci_t dev_to_sci(struct net_device *dev, __be16 port) return make_sci(dev->dev_addr, port); } -static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len) +static void macsec_init_secy(struct net_device *dev, sci_t sci, u8 icv_len) { struct macsec_dev *macsec = macsec_priv(dev); struct macsec_secy *secy = &macsec->secy; - macsec->stats = netdev_alloc_pcpu_stats(struct pcpu_secy_stats); - if (!macsec->stats) - return -ENOMEM; - - secy->tx_sc.stats = netdev_alloc_pcpu_stats(struct pcpu_tx_sc_stats); - if (!secy->tx_sc.stats) - return -ENOMEM; - - secy->tx_sc.md_dst = metadata_dst_alloc(0, METADATA_MACSEC, GFP_KERNEL); - if (!secy->tx_sc.md_dst) - /* macsec and secy percpu stats will be freed when unregistering - * net_device in macsec_free_netdev() - */ - return -ENOMEM; - if (sci == MACSEC_UNDEF_SCI) sci = dev_to_sci(dev, MACSEC_PORT_ES); @@ -4147,15 +4154,12 @@ static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len) secy->xpn = DEFAULT_XPN; secy->sci = sci; - secy->tx_sc.md_dst->u.macsec_info.sci = sci; secy->tx_sc.active = true; secy->tx_sc.encoding_sa = DEFAULT_ENCODING_SA; secy->tx_sc.encrypt = DEFAULT_ENCRYPT; secy->tx_sc.send_sci = DEFAULT_SEND_SCI; secy->tx_sc.end_station = false; secy->tx_sc.scb = false; - - return 0; } static struct lock_class_key macsec_netdev_addr_lock_key; @@ -4218,6 +4222,24 @@ static int macsec_newlink(struct net_device *dev, if (rx_handler && rx_handler != macsec_handle_frame) return -EBUSY; + if (is_zero_ether_addr(dev->dev_addr)) + eth_hw_addr_inherit(dev, real_dev); + + if (data && data[IFLA_MACSEC_SCI]) + sci = nla_get_sci(data[IFLA_MACSEC_SCI]); + else if (data && data[IFLA_MACSEC_PORT]) + sci = dev_to_sci(dev, nla_get_be16(data[IFLA_MACSEC_PORT])); + else + sci = dev_to_sci(dev, MACSEC_PORT_ES); + + /* Registration can notify listeners before returning. */ + macsec_init_secy(dev, sci, icv_len); + if (data) { + err = macsec_changelink_common(dev, data); + if (err) + return err; + } + err = register_netdevice(dev); if (err < 0) return err; @@ -4230,31 +4252,11 @@ static int macsec_newlink(struct net_device *dev, if (err < 0) goto unregister; - /* need to be already registered so that ->init has run and - * the MAC addr is set - */ - if (data && data[IFLA_MACSEC_SCI]) - sci = nla_get_sci(data[IFLA_MACSEC_SCI]); - else if (data && data[IFLA_MACSEC_PORT]) - sci = dev_to_sci(dev, nla_get_be16(data[IFLA_MACSEC_PORT])); - else - sci = dev_to_sci(dev, MACSEC_PORT_ES); - if (rx_handler && sci_exists(real_dev, sci)) { err = -EBUSY; goto unlink; } - err = macsec_add_dev(dev, sci, icv_len); - if (err) - goto unlink; - - if (data) { - err = macsec_changelink_common(dev, data); - if (err) - goto del_dev; - } - /* If h/w offloading is available, propagate to the device */ if (macsec_is_offloaded(macsec)) { const struct macsec_ops *ops; diff --git a/drivers/net/mctp/mctp-i3c.c b/drivers/net/mctp/mctp-i3c.c index 88d9e36cd4a2b..4e857dd5df642 100644 --- a/drivers/net/mctp/mctp-i3c.c +++ b/drivers/net/mctp/mctp-i3c.c @@ -288,6 +288,7 @@ __must_hold(&busdevs_lock) static int mctp_i3c_probe(struct i3c_device *i3c) { struct mctp_i3c_bus *b = NULL, *mbus = NULL; + int rc; /* Look for a known bus */ mutex_lock(&busdevs_lock); @@ -296,14 +297,16 @@ static int mctp_i3c_probe(struct i3c_device *i3c) mbus = b; break; } - mutex_unlock(&busdevs_lock); if (!mbus) { /* probably no "mctp-controller" property on the i3c bus */ - return -ENODEV; + rc = -ENODEV; + } else { + rc = mctp_i3c_add_device(mbus, i3c); } + mutex_unlock(&busdevs_lock); - return mctp_i3c_add_device(mbus, i3c); + return rc; } static void mctp_i3c_remove_device(struct mctp_i3c_device *mi) diff --git a/drivers/net/mdio/mdio-realtek-rtl9300.c b/drivers/net/mdio/mdio-realtek-rtl9300.c index afd52a1cd7f83..9ce2b78075320 100644 --- a/drivers/net/mdio/mdio-realtek-rtl9300.c +++ b/drivers/net/mdio/mdio-realtek-rtl9300.c @@ -88,6 +88,8 @@ #define RTL9310_SMI_INDRT_ACCESS_BC_PHYID_CTRL 0x0c14 #define RTL9310_BC_PORT_ID GENMASK(10, 5) #define RTL9310_SMI_INDRT_ACCESS_CTRL_1 0x0c04 +#define RTL9310_SMI_INDRT_EXT_PAGE GENMASK(8, 0) +#define RTL9310_SMI_INDRT_EXT_PAGE_NO_CHANGE 0x1ff #define RTL9310_SMI_INDRT_ACCESS_CTRL_2_LOW 0x0c08 #define RTL9310_SMI_INDRT_ACCESS_CTRL_2_HIGH 0x0c0c #define RTL9310_SMI_INDRT_ACCESS_CTRL_3 0x0c10 /* I/O fields flipped */ @@ -325,6 +327,8 @@ static int otto_emdio_9310_read_c22(struct mii_bus *bus, int port, int regnum, u .broadcast = FIELD_PREP(RTL9310_BC_PORT_ID, port), .c22_data = FIELD_PREP(RTL9310_PHY_CTRL_REG_ADDR, regnum) | FIELD_PREP(RTL9310_PHY_CTRL_MAIN_PAGE, RAW_PAGE(priv)), + .ext_page = FIELD_PREP(RTL9310_SMI_INDRT_EXT_PAGE, + RTL9310_SMI_INDRT_EXT_PAGE_NO_CHANGE), }; return otto_emdio_read_cmd(bus, RTL9310_PHY_CTRL_TYPE_C22, &cmd_data, @@ -337,6 +341,8 @@ static int otto_emdio_9310_write_c22(struct mii_bus *bus, int port, int regnum, struct otto_emdio_cmd_regs cmd_data = { .c22_data = FIELD_PREP(RTL9310_PHY_CTRL_REG_ADDR, regnum) | FIELD_PREP(RTL9310_PHY_CTRL_MAIN_PAGE, RAW_PAGE(priv)), + .ext_page = FIELD_PREP(RTL9310_SMI_INDRT_EXT_PAGE, + RTL9310_SMI_INDRT_EXT_PAGE_NO_CHANGE), .io_data = FIELD_PREP(RTL9310_PHY_CTRL_INDATA, value), .port_mask_high = (u32)(BIT_ULL(port) >> 32), .port_mask_low = (u32)(BIT_ULL(port)), diff --git a/drivers/net/net_failover.c b/drivers/net/net_failover.c index 3f7d31033bae8..1b5213e870703 100644 --- a/drivers/net/net_failover.c +++ b/drivers/net/net_failover.c @@ -675,7 +675,7 @@ static int net_failover_slave_name_change(struct net_device *slave_dev, /* We need to bring up the slave after the rename by udev in case * open failed with EBUSY when it was registered. */ - dev_open(slave_dev, NULL); + netif_open(slave_dev, NULL); return 0; } diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c index 4e9d7e10b527e..b4a99f3ceac60 100644 --- a/drivers/net/netdevsim/netdev.c +++ b/drivers/net/netdevsim/netdev.c @@ -529,6 +529,7 @@ static void nsim_del_napi(struct netdevsim *ns) for (i = 0; i < dev->num_rx_queues; i++) { struct nsim_rq *rq = ns->rq[i]; + netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, NULL); napi_disable_locked(&rq->napi); __netif_napi_del_locked(&rq->napi); } @@ -826,6 +827,7 @@ nsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg, } ns->rq[idx] = qmem->rq; + netif_queue_set_napi(dev, idx, NETDEV_QUEUE_TYPE_RX, &ns->rq[idx]->napi); napi_enable_locked(&ns->rq[idx]->napi); return 0; diff --git a/drivers/net/ntb_netdev.c b/drivers/net/ntb_netdev.c index 029a4a532a104..9c171697e7620 100644 --- a/drivers/net/ntb_netdev.c +++ b/drivers/net/ntb_netdev.c @@ -144,6 +144,9 @@ static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data, goto enqueue_again; } + ndev->stats.rx_packets++; + ndev->stats.rx_bytes += len; + new_skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN); if (!new_skb) { ndev->stats.rx_dropped++; @@ -155,13 +158,7 @@ static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data, skb->ip_summed = CHECKSUM_NONE; skb_record_rx_queue(skb, q->qid); - if (netif_rx(skb) == NET_RX_DROP) { - ndev->stats.rx_errors++; - ndev->stats.rx_dropped++; - } else { - ndev->stats.rx_packets++; - ndev->stats.rx_bytes += len; - } + netif_rx(skb); skb = new_skb; @@ -199,8 +196,10 @@ static int __ntb_netdev_maybe_stop_tx(struct net_device *netdev, static int ntb_netdev_maybe_stop_tx(struct net_device *ndev, struct ntb_netdev_queue *q, int size) { - if (__netif_subqueue_stopped(ndev, q->qid) || - (ntb_transport_tx_free_entry(q->qp) >= size)) + if (__netif_subqueue_stopped(ndev, q->qid)) + return -EBUSY; + + if (ntb_transport_tx_free_entry(q->qp) >= size) return 0; return __ntb_netdev_maybe_stop_tx(ndev, q, size); @@ -256,21 +255,30 @@ static netdev_tx_t ntb_netdev_start_xmit(struct sk_buff *skb, q = &dev->queues[qid]; - ntb_netdev_maybe_stop_tx(ndev, q, tx_stop); + if (unlikely(ntb_netdev_maybe_stop_tx(ndev, q, tx_stop))) + return NETDEV_TX_BUSY; rc = ntb_transport_tx_enqueue(q->qp, skb, skb->data, skb->len); - if (rc) - goto err; + if (rc) { + if (rc == -EAGAIN || rc == -EBUSY) { + netif_stop_subqueue(ndev, q->qid); + mod_timer(&q->tx_timer, + jiffies + usecs_to_jiffies(tx_time)); + return NETDEV_TX_BUSY; + } + + goto drop; + } /* check for next submit */ ntb_netdev_maybe_stop_tx(ndev, q, tx_stop); return NETDEV_TX_OK; -err: +drop: + dev_kfree_skb_any(skb); ndev->stats.tx_dropped++; - ndev->stats.tx_errors++; - return NETDEV_TX_BUSY; + return NETDEV_TX_OK; } static void ntb_netdev_tx_timer(struct timer_list *t) diff --git a/drivers/net/ovpn/netlink.c b/drivers/net/ovpn/netlink.c index 4dad852941982..5432bc2eb8e80 100644 --- a/drivers/net/ovpn/netlink.c +++ b/drivers/net/ovpn/netlink.c @@ -100,6 +100,8 @@ static bool ovpn_nl_attr_sockaddr_remote(struct nlattr **attrs, struct sockaddr_in6 *sin6; struct sockaddr_in *sin; struct in6_addr *in6; + struct nlattr *scope; + u32 scope_id = 0; __be16 port = 0; __be32 *in; @@ -114,6 +116,9 @@ static bool ovpn_nl_attr_sockaddr_remote(struct nlattr **attrs, } else if (attrs[OVPN_A_PEER_REMOTE_IPV6]) { ss->ss_family = AF_INET6; in6 = nla_data(attrs[OVPN_A_PEER_REMOTE_IPV6]); + scope = attrs[OVPN_A_PEER_REMOTE_IPV6_SCOPE_ID]; + if (scope) + scope_id = nla_get_u32(scope); } else { return false; } @@ -126,6 +131,7 @@ static bool ovpn_nl_attr_sockaddr_remote(struct nlattr **attrs, if (!ipv6_addr_v4mapped(in6)) { sin6 = (struct sockaddr_in6 *)ss; sin6->sin6_port = port; + sin6->sin6_scope_id = scope_id; memcpy(&sin6->sin6_addr, in6, sizeof(*in6)); break; } @@ -179,6 +185,39 @@ static sa_family_t ovpn_nl_family_get(struct nlattr *addr4, return AF_UNSPEC; } +static int ovpn_nl_peer_check_vpn_addrs(const struct in_addr *addr4, + const struct in6_addr *addr6, + struct genl_info *info) +{ + int addr6_type; + + if (addr4->s_addr == htonl(INADDR_ANY) && ipv6_addr_any(addr6)) { + NL_SET_ERR_MSG_MOD(info->extack, + "at least one VPN IP must be configured in MP mode"); + return -EINVAL; + } + + if (ipv4_is_multicast(addr4->s_addr) || ipv4_is_lbcast(addr4->s_addr) || + ipv4_is_loopback(addr4->s_addr)) { + NL_SET_ERR_MSG_MOD(info->extack, + "VPN IPv4 address must be valid unicast or any"); + return -EADDRNOTAVAIL; + } + + if (!ipv6_addr_any(addr6)) { + addr6_type = ipv6_addr_type(addr6); + + if (!(addr6_type & IPV6_ADDR_UNICAST) || + (addr6_type & (IPV6_ADDR_LOOPBACK | IPV6_ADDR_COMPATv4))) { + NL_SET_ERR_MSG_MOD(info->extack, + "VPN IPv6 address must be valid unicast or any"); + return -EADDRNOTAVAIL; + } + } + + return 0; +} + static int ovpn_nl_peer_precheck(struct ovpn_priv *ovpn, struct genl_info *info, struct nlattr **attrs) @@ -346,8 +385,10 @@ static int ovpn_nl_peer_modify(struct ovpn_peer *peer, struct genl_info *info, int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info) { - struct nlattr *attrs[OVPN_A_PEER_MAX + 1]; + struct in_addr vpn_addr4 = { .s_addr = htonl(INADDR_ANY) }; + struct in6_addr vpn_addr6 = IN6ADDR_ANY_INIT; struct ovpn_priv *ovpn = info->user_ptr[0]; + struct nlattr *attrs[OVPN_A_PEER_MAX + 1]; struct ovpn_socket *ovpn_sock; struct socket *sock = NULL; struct ovpn_peer *peer; @@ -371,11 +412,18 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info) return -EINVAL; /* in MP mode VPN IPs are required for selecting the right peer */ - if (ovpn->mode == OVPN_MODE_MP && !attrs[OVPN_A_PEER_VPN_IPV4] && - !attrs[OVPN_A_PEER_VPN_IPV6]) { - NL_SET_ERR_MSG_FMT_MOD(info->extack, - "VPN IP must be provided in MP mode"); - return -EINVAL; + if (ovpn->mode == OVPN_MODE_MP) { + if (attrs[OVPN_A_PEER_VPN_IPV4]) + vpn_addr4.s_addr = + nla_get_in_addr(attrs[OVPN_A_PEER_VPN_IPV4]); + if (attrs[OVPN_A_PEER_VPN_IPV6]) + vpn_addr6 = + nla_get_in6_addr(attrs[OVPN_A_PEER_VPN_IPV6]); + + ret = ovpn_nl_peer_check_vpn_addrs(&vpn_addr4, &vpn_addr6, + info); + if (ret < 0) + return ret; } peer_id = nla_get_u32(attrs[OVPN_A_PEER_ID]); @@ -474,8 +522,10 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info) int ovpn_nl_peer_set_doit(struct sk_buff *skb, struct genl_info *info) { - struct nlattr *attrs[OVPN_A_PEER_MAX + 1]; struct ovpn_priv *ovpn = info->user_ptr[0]; + struct nlattr *attrs[OVPN_A_PEER_MAX + 1]; + struct in6_addr vpn_addr6; + struct in_addr vpn_addr4; struct ovpn_socket *sock; struct ovpn_peer *peer; u32 peer_id; @@ -522,28 +572,58 @@ int ovpn_nl_peer_set_doit(struct sk_buff *skb, struct genl_info *info) rcu_read_unlock(); spin_lock_bh(&ovpn->lock); - ret = ovpn_nl_peer_modify(peer, info, attrs); - if (ret < 0) { - spin_unlock_bh(&ovpn->lock); - ovpn_peer_put(peer); - return ret; + + vpn_addr4 = peer->vpn_addrs.ipv4; + vpn_addr6 = peer->vpn_addrs.ipv6; + + /* reject peer with conflicting VPN address */ + if (attrs[OVPN_A_PEER_VPN_IPV4]) { + vpn_addr4.s_addr = nla_get_in_addr(attrs[OVPN_A_PEER_VPN_IPV4]); + if (ovpn_peer_vpn_addr_conflict4(ovpn, peer, &vpn_addr4)) + goto addr_conflict; + } + if (attrs[OVPN_A_PEER_VPN_IPV6]) { + vpn_addr6 = nla_get_in6_addr(attrs[OVPN_A_PEER_VPN_IPV6]); + if (ovpn_peer_vpn_addr_conflict6(ovpn, peer, &vpn_addr6)) + goto addr_conflict; } + /* in MP mode VPN IPs are required for selecting the right peer */ + if (ovpn->mode == OVPN_MODE_MP) { + ret = ovpn_nl_peer_check_vpn_addrs(&vpn_addr4, &vpn_addr6, + info); + if (ret < 0) + goto unlock; + } + + ret = ovpn_nl_peer_modify(peer, info, attrs); + if (ret < 0) + goto unlock; + /* ret == 1 means that VPN IPv4/6 has been modified and rehashing * is required */ - if (ret > 0) + if (ret > 0) { ovpn_peer_hash_vpn_ip(peer); + ret = 0; + } /* if the remote endpoint was updated, the by_transp_addr hash bucket * also needs to be refreshed, otherwise incoming packets from the new * remote address would fail the lockless lookup */ if (attrs[OVPN_A_PEER_REMOTE_IPV4] || attrs[OVPN_A_PEER_REMOTE_IPV6]) ovpn_peer_hash_transp_addr(peer); + +unlock: spin_unlock_bh(&ovpn->lock); ovpn_peer_put(peer); - return 0; + return ret; +addr_conflict: + NL_SET_ERR_MSG_FMT_MOD(info->extack, + "VPN IP is already assigned to another peer"); + ret = -EADDRINUSE; + goto unlock; } static int ovpn_nl_send_peer(struct sk_buff *skb, const struct genl_info *info, diff --git a/drivers/net/ovpn/peer.c b/drivers/net/ovpn/peer.c index c95656ca7c357..2067825bb5b61 100644 --- a/drivers/net/ovpn/peer.c +++ b/drivers/net/ovpn/peer.c @@ -113,6 +113,7 @@ struct ovpn_peer *ovpn_peer_new(struct ovpn_priv *ovpn, u32 id) RCU_INIT_POINTER(peer->bind, NULL); ovpn_crypto_state_init(&peer->crypto); spin_lock_init(&peer->lock); + seqcount_spinlock_init(&peer->route_key_seq, &peer->lock); kref_init(&peer->refcount); ovpn_peer_stats_init(&peer->vpn_stats); ovpn_peer_stats_init(&peer->link_stats); @@ -199,13 +200,12 @@ static void __ovpn_peer_hash_transp_addr(struct ovpn_peer *peer, */ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) { + const void *local_ip = NULL; struct sockaddr_storage ss; struct sockaddr_in6 *sa6; - bool reset_cache = false; struct sockaddr_in *sa; struct ovpn_bind *bind; - const void *local_ip; - size_t salen = 0; + bool floated = false; spin_lock_bh(&peer->lock); bind = rcu_dereference_protected(peer->bind, @@ -232,8 +232,7 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) .sin_addr.s_addr = ip_hdr(skb)->saddr, .sin_port = udp_hdr(skb)->source, }; - salen = sizeof(*sa); - reset_cache = true; + floated = true; break; } @@ -245,10 +244,12 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) netdev_name(peer->ovpn->dev), peer->id, &bind->local.ipv4.s_addr, &ip_hdr(skb)->daddr); - bind->local.ipv4.s_addr = ip_hdr(skb)->daddr; - reset_cache = true; + local_ip = &ip_hdr(skb)->daddr; + memcpy(&ss, &bind->remote, sizeof(struct sockaddr_in)); + break; } - break; + /* nothing changed */ + goto unlock; case htons(ETH_P_IPV6): /* float check */ if (unlikely(!ovpn_bind_skb_src_match(bind, skb))) { @@ -270,8 +271,7 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, skb->skb_iif), }; - salen = sizeof(*sa6); - reset_cache = true; + floated = true; break; } @@ -284,26 +284,30 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) netdev_name(peer->ovpn->dev), peer->id, &bind->local.ipv6, &ipv6_hdr(skb)->daddr); - bind->local.ipv6 = ipv6_hdr(skb)->daddr; - reset_cache = true; + local_ip = &ipv6_hdr(skb)->daddr; + memcpy(&ss, &bind->remote, sizeof(struct sockaddr_in6)); + break; } - break; + /* nothing changed */ + goto unlock; default: goto unlock; } - if (unlikely(reset_cache)) - dst_cache_reset(&peer->dst_cache); - - /* if the peer did not float, we can bail out now */ - if (likely(!salen)) - goto unlock; - if (unlikely(ovpn_peer_reset_sockaddr(peer, (struct sockaddr_storage *)&ss, local_ip) < 0)) goto unlock; + /* reset the cache only after a successful bind update to avoid useless + * cache misses on concurrent TX + */ + dst_cache_reset(&peer->dst_cache); + + /* if only the local address changed, bail out now */ + if (!floated) + goto unlock; + net_dbg_ratelimited("%s: peer %d floated to %pIScp", netdev_name(peer->ovpn->dev), peer->id, &ss); @@ -484,7 +488,7 @@ static struct ovpn_peer *ovpn_peer_get_by_vpn_addr4(struct ovpn_priv *ovpn, * Return: the peer if found or NULL otherwise */ static struct ovpn_peer *ovpn_peer_get_by_vpn_addr6(struct ovpn_priv *ovpn, - struct in6_addr *addr) + const struct in6_addr *addr) { struct hlist_nulls_head *nhead; struct hlist_nulls_node *ntmp; @@ -509,6 +513,64 @@ static struct ovpn_peer *ovpn_peer_get_by_vpn_addr6(struct ovpn_priv *ovpn, return NULL; } +/** + * ovpn_peer_vpn_addr_conflict4 - check if the VPN v4 address is already in use + * @ovpn: the openvpn instance to search + * @peer: peer being added or updated, or NULL + * @addr: VPN IPv4 address to check + * + * Check whether @addr is already assigned to another peer. @peer is ignored + * when found, allowing peer updates that keep an existing address. + * Unspecified addresses are ignored. + * + * Note: the caller must hold @ovpn->lock. + * + * Return: true on conflict, false otherwise. + */ +bool ovpn_peer_vpn_addr_conflict4(struct ovpn_priv *ovpn, + const struct ovpn_peer *peer, + const struct in_addr *addr) +{ + struct ovpn_peer *tmp = NULL; + + lockdep_assert_held(&ovpn->lock); + + /* we don't hash INADDR_ANY, no conflict in that case */ + if (addr->s_addr != htonl(INADDR_ANY)) + tmp = ovpn_peer_get_by_vpn_addr4(ovpn, addr->s_addr); + + return tmp && tmp != peer; +} + +/** + * ovpn_peer_vpn_addr_conflict6 - check if the VPN v6 address is already in use + * @ovpn: the openvpn instance to search + * @peer: peer being added or updated, or NULL + * @addr: VPN IPv6 address to check + * + * Check whether @addr is already assigned to another peer. @peer is ignored + * when found, allowing peer updates that keep an existing address. + * Unspecified addresses are ignored. + * + * Note: the caller must hold @ovpn->lock. + * + * Return: true on conflict, false otherwise. + */ +bool ovpn_peer_vpn_addr_conflict6(struct ovpn_priv *ovpn, + const struct ovpn_peer *peer, + const struct in6_addr *addr) +{ + struct ovpn_peer *tmp = NULL; + + lockdep_assert_held(&ovpn->lock); + + /* we don't hash ::, no conflict in that case */ + if (!ipv6_addr_any(addr)) + tmp = ovpn_peer_get_by_vpn_addr6(ovpn, addr); + + return tmp && tmp != peer; +} + /** * ovpn_peer_transp_match - check if sockaddr and peer binding match * @peer: the peer to get the binding from @@ -990,10 +1052,11 @@ void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer) if (hlist_unhashed(&peer->hash_entry_id)) return; - if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { - /* remove potential old hashing */ - hlist_nulls_del_init_rcu(&peer->hash_entry_addr4); + /* remove potential old hashing */ + hlist_nulls_del_init_rcu(&peer->hash_entry_addr4); + hlist_nulls_del_init_rcu(&peer->hash_entry_addr6); + if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { nhead = ovpn_get_hash_head(peer->ovpn->peers->by_vpn_addr4, &peer->vpn_addrs.ipv4, sizeof(peer->vpn_addrs.ipv4)); @@ -1001,9 +1064,6 @@ void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer) } if (!ipv6_addr_any(&peer->vpn_addrs.ipv6)) { - /* remove potential old hashing */ - hlist_nulls_del_init_rcu(&peer->hash_entry_addr6); - nhead = ovpn_get_hash_head(peer->ovpn->peers->by_vpn_addr6, &peer->vpn_addrs.ipv6, sizeof(peer->vpn_addrs.ipv6)); @@ -1038,6 +1098,13 @@ static int ovpn_peer_add_mp(struct ovpn_priv *ovpn, struct ovpn_peer *peer) goto out; } + /* reject peer with conflicting VPN address */ + if (ovpn_peer_vpn_addr_conflict4(ovpn, NULL, &peer->vpn_addrs.ipv4) || + ovpn_peer_vpn_addr_conflict6(ovpn, NULL, &peer->vpn_addrs.ipv6)) { + ret = -EADDRINUSE; + goto out; + } + bind = rcu_dereference_protected(peer->bind, true); /* peers connected via TCP have bind == NULL */ if (bind) { diff --git a/drivers/net/ovpn/peer.h b/drivers/net/ovpn/peer.h index dfa5c0037e02b..1879bfb76992d 100644 --- a/drivers/net/ovpn/peer.h +++ b/drivers/net/ovpn/peer.h @@ -10,6 +10,7 @@ #ifndef _NET_OVPN_OVPNPEER_H_ #define _NET_OVPN_OVPNPEER_H_ +#include #include #include @@ -17,6 +18,16 @@ #include "socket.h" #include "stats.h" +/** + * struct ovpn_route_key - route key used for the peer dst cache + * @mark: fwmark used for route lookup + * @sport: UDP source port used for route lookup + */ +struct ovpn_route_key { + u32 mark; + __be16 sport; +}; + /** * struct ovpn_peer - the main remote peer object * @ovpn: main openvpn instance this peer belongs to @@ -45,6 +56,8 @@ * @tcp.sk_cb.ops: pointer to the original prot_ops object (TCP only) * @crypto: the crypto configuration (ciphers, keys, etc..) * @dst_cache: cache for dst_entry used to send to peer + * @route_key: route key matching the current dst cache contents + * @route_key_seq: seqcount protecting lockless route_key reads * @bind: remote peer binding * @keepalive_interval: seconds after which a new keepalive should be sent * @keepalive_xmit_exp: future timestamp when next keepalive should be sent @@ -55,7 +68,7 @@ * @vpn_stats: per-peer in-VPN TX/RX stats * @link_stats: per-peer link/transport TX/RX stats * @delete_reason: why peer was deleted (i.e. timeout, transport error, ..) - * @lock: protects binding to peer (bind) and keepalive* fields + * @lock: protects binding to peer (bind), route_key and keepalive* fields * @refcount: reference counter * @rcu: used to free peer in an RCU safe way * @release_entry: entry for the socket release list @@ -99,6 +112,8 @@ struct ovpn_peer { } tcp; struct ovpn_crypto_state crypto; struct dst_cache dst_cache; + struct ovpn_route_key route_key; + seqcount_spinlock_t route_key_seq; struct ovpn_bind __rcu *bind; unsigned long keepalive_interval; unsigned long keepalive_xmit_exp; @@ -109,7 +124,7 @@ struct ovpn_peer { struct ovpn_peer_stats vpn_stats; struct ovpn_peer_stats link_stats; enum ovpn_del_peer_reason delete_reason; - spinlock_t lock; /* protects bind and keepalive* */ + spinlock_t lock; /* protects bind, route_key and keepalive* */ struct kref refcount; struct rcu_head rcu; struct llist_node release_entry; @@ -149,6 +164,12 @@ struct ovpn_peer *ovpn_peer_get_by_transp_addr(struct ovpn_priv *ovpn, struct ovpn_peer *ovpn_peer_get_by_id(struct ovpn_priv *ovpn, u32 peer_id); struct ovpn_peer *ovpn_peer_get_by_dst(struct ovpn_priv *ovpn, struct sk_buff *skb); +bool ovpn_peer_vpn_addr_conflict4(struct ovpn_priv *ovpn, + const struct ovpn_peer *peer, + const struct in_addr *addr); +bool ovpn_peer_vpn_addr_conflict6(struct ovpn_priv *ovpn, + const struct ovpn_peer *peer, + const struct in6_addr *addr); void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer); void ovpn_peer_hash_transp_addr(struct ovpn_peer *peer); bool ovpn_peer_check_by_src(struct ovpn_priv *ovpn, struct sk_buff *skb, diff --git a/drivers/net/ovpn/udp.c b/drivers/net/ovpn/udp.c index 7f69e8890b5b5..055cdb1bee13b 100644 --- a/drivers/net/ovpn/udp.c +++ b/drivers/net/ovpn/udp.c @@ -131,6 +131,77 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb) return 0; } +static bool ovpn_route_key_equal(const struct ovpn_route_key *a, + const struct ovpn_route_key *b) +{ + return a->mark == b->mark && a->sport == b->sport; +} + +/** + * ovpn_dst_cache_check_key - reset peer dst cache after key changes + * @peer: the peer owning the dst cache + * @cache: the cache that might need to be reset + * @key: the route key for the packet being transmitted + * + * Reset the peer dst cache if it was populated for a different route key. + */ +static void ovpn_dst_cache_check_key(struct ovpn_peer *peer, + struct dst_cache *cache, + const struct ovpn_route_key *key) +{ + struct ovpn_route_key old_key; + unsigned int seq; + + /* snapshot the saved key before deciding whether the cache matches */ + do { + seq = read_seqcount_begin(&peer->route_key_seq); + old_key = peer->route_key; + } while (read_seqcount_retry(&peer->route_key_seq, seq)); + + /* nothing changed: the current cache can be reused */ + if (likely(ovpn_route_key_equal(&old_key, key))) + return; + + /* recheck under lock because another path may have updated the key */ + spin_lock_bh(&peer->lock); + if (!ovpn_route_key_equal(&peer->route_key, key)) { + write_seqcount_begin(&peer->route_key_seq); + peer->route_key = *key; + dst_cache_reset(cache); + write_seqcount_end(&peer->route_key_seq); + } + spin_unlock_bh(&peer->lock); +} + +/** + * ovpn_dst_cache_current - check whether a route lookup matches peer state + * @peer: the peer owning the bind and dst cache + * @bind: the RCU bind used for the route lookup + * @key: the route key used for the route lookup + * + * Check that @bind is still the current peer bind and that @key still matches + * the peer route key. The caller must hold @peer->lock. The TX path keeps + * @bind inside an RCU read-side critical section, so pointer identity is enough + * to detect whether the bind was replaced while the route lookup was running. + * + * Return: true if the lookup result still matches the current peer state and + * may update the dst cache or replace the bind. + */ +static bool ovpn_dst_cache_current(const struct ovpn_peer *peer, + const struct ovpn_bind *bind, + const struct ovpn_route_key *key) +{ + const struct ovpn_bind *curr_bind; + + lockdep_assert_held(&peer->lock); + + curr_bind = rcu_dereference_protected(peer->bind, + lockdep_is_held(&peer->lock)); + + return curr_bind == bind && + ovpn_route_key_equal(key, &peer->route_key); +} + /** * ovpn_udp4_output - send IPv4 packet over udp socket * @peer: the destination peer @@ -138,21 +209,26 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb) * @cache: dst cache * @sk: the socket to send the packet over * @skb: the packet to send + * @key: the route key snapshot used for cache validation and flow lookup * * Return: 0 on success or a negative error code otherwise */ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind, struct dst_cache *cache, struct sock *sk, - struct sk_buff *skb) + struct sk_buff *skb, + const struct ovpn_route_key *key) { + struct sockaddr_storage remote; + struct in_addr local = {}; + bool reset_local = false; struct rtable *rt; struct flowi4 fl = { .saddr = bind->local.ipv4.s_addr, .daddr = bind->remote.in4.sin_addr.s_addr, - .fl4_sport = inet_sk(sk)->inet_sport, + .fl4_sport = key->sport, .fl4_dport = bind->remote.in4.sin_port, .flowi4_proto = sk->sk_protocol, - .flowi4_mark = sk->sk_mark, + .flowi4_mark = key->mark, }; int ret; @@ -161,26 +237,19 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind, if (rt) goto transmit; - if (unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, fl.saddr, - RT_SCOPE_HOST))) { - /* we may end up here when the cached address is not usable - * anymore. In this case we reset address/cache and perform a - * new look up + if (fl.saddr && unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, + fl.saddr, RT_SCOPE_HOST))) { + /* The learned local address is not usable anymore. + * Retry with source address autoselection. */ fl.saddr = 0; - spin_lock_bh(&peer->lock); - bind->local.ipv4.s_addr = 0; - spin_unlock_bh(&peer->lock); - dst_cache_reset(cache); + reset_local = true; } rt = ip_route_output_flow(sock_net(sk), &fl, sk); if (IS_ERR(rt) && PTR_ERR(rt) == -EINVAL) { fl.saddr = 0; - spin_lock_bh(&peer->lock); - bind->local.ipv4.s_addr = 0; - spin_unlock_bh(&peer->lock); - dst_cache_reset(cache); + reset_local = true; rt = ip_route_output_flow(sock_net(sk), &fl, sk); } @@ -193,7 +262,30 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind, ret); goto err; } - dst_cache_set_ip4(cache, &rt->dst, fl.saddr); + + /* avoid storing a stale cache or local address */ + spin_lock_bh(&peer->lock); + if (likely(ovpn_dst_cache_current(peer, bind, key))) { + if (!reset_local) { + dst_cache_set_ip4(cache, &rt->dst, fl.saddr); + spin_unlock_bh(&peer->lock); + goto transmit; + } + + /* invalidate per-CPU dst entries that may still carry + * the stale source + */ + dst_cache_reset(cache); + + /* preserve the current remote */ + memcpy(&remote, &bind->remote, sizeof(struct sockaddr_in)); + /* The current packet already has a valid wildcard-source route. + * If replacing the bind fails, leave the stale local in place; + * a later cache miss will retry the repair. + */ + ovpn_peer_reset_sockaddr(peer, &remote, &local); + } + spin_unlock_bh(&peer->lock); transmit: udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0, @@ -213,23 +305,28 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind, * @cache: dst cache * @sk: the socket to send the packet over * @skb: the packet to send + * @key: the route key snapshot used for cache validation and flow lookup * * Return: 0 on success or a negative error code otherwise */ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind, struct dst_cache *cache, struct sock *sk, - struct sk_buff *skb) + struct sk_buff *skb, + const struct ovpn_route_key *key) { + struct in6_addr local = in6addr_any; + struct sockaddr_storage remote; + bool reset_local = false; struct dst_entry *dst; int ret; struct flowi6 fl = { .saddr = bind->local.ipv6, .daddr = bind->remote.in6.sin6_addr, - .fl6_sport = inet_sk(sk)->inet_sport, + .fl6_sport = key->sport, .fl6_dport = bind->remote.in6.sin6_port, .flowi6_proto = sk->sk_protocol, - .flowi6_mark = sk->sk_mark, + .flowi6_mark = key->mark, .flowi6_oif = bind->remote.in6.sin6_scope_id, }; @@ -238,16 +335,13 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind, if (dst) goto transmit; - if (unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) { - /* we may end up here when the cached address is not usable - * anymore. In this case we reset address/cache and perform a - * new look up + if (!ipv6_addr_any(&fl.saddr) && + unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) { + /* The learned local address is not usable anymore. + * Retry with source address autoselection. */ fl.saddr = in6addr_any; - spin_lock_bh(&peer->lock); - bind->local.ipv6 = in6addr_any; - spin_unlock_bh(&peer->lock); - dst_cache_reset(cache); + reset_local = true; } dst = ip6_dst_lookup_flow(sock_net(sk), sk, &fl, NULL); @@ -258,7 +352,30 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind, &bind->remote.in6, ret); goto err; } - dst_cache_set_ip6(cache, dst, &fl.saddr); + + /* avoid storing a stale cache or local address */ + spin_lock_bh(&peer->lock); + if (likely(ovpn_dst_cache_current(peer, bind, key))) { + if (!reset_local) { + dst_cache_set_ip6(cache, dst, &fl.saddr); + spin_unlock_bh(&peer->lock); + goto transmit; + } + + /* invalidate per-CPU dst entries that may still carry + * the stale source + */ + dst_cache_reset(cache); + + /* preserve the current remote */ + memcpy(&remote, &bind->remote, sizeof(struct sockaddr_in6)); + /* The current packet already has a valid wildcard-source route. + * If replacing the bind fails, leave the stale local in place; + * a later cache miss will retry the repair. + */ + ovpn_peer_reset_sockaddr(peer, &remote, &local); + } + spin_unlock_bh(&peer->lock); transmit: /* user IPv6 packets may be larger than the transport interface @@ -287,6 +404,7 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind, * @cache: dst cache * @sk: the socket to send the packet over * @skb: the packet to send + * @key: route key snapshot used for cache validation and flow lookup * * rcu_read_lock should be held on entry. * On return, the skb is consumed. @@ -294,7 +412,8 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind, * Return: 0 on success or a negative error code otherwise */ static int ovpn_udp_output(struct ovpn_peer *peer, struct dst_cache *cache, - struct sock *sk, struct sk_buff *skb) + struct sock *sk, struct sk_buff *skb, + struct ovpn_route_key *key) { struct ovpn_bind *bind; int ret; @@ -314,11 +433,11 @@ static int ovpn_udp_output(struct ovpn_peer *peer, struct dst_cache *cache, switch (bind->remote.in4.sin_family) { case AF_INET: - ret = ovpn_udp4_output(peer, bind, cache, sk, skb); + ret = ovpn_udp4_output(peer, bind, cache, sk, skb, key); break; #if IS_ENABLED(CONFIG_IPV6) case AF_INET6: - ret = ovpn_udp6_output(peer, bind, cache, sk, skb); + ret = ovpn_udp6_output(peer, bind, cache, sk, skb, key); break; #endif default: @@ -340,15 +459,21 @@ static int ovpn_udp_output(struct ovpn_peer *peer, struct dst_cache *cache, void ovpn_udp_send_skb(struct ovpn_peer *peer, struct sock *sk, struct sk_buff *skb) { + struct ovpn_route_key key = { + .mark = READ_ONCE(sk->sk_mark), + .sport = READ_ONCE(inet_sk(sk)->inet_sport), + }; int ret; skb->dev = peer->ovpn->dev; - skb->mark = READ_ONCE(sk->sk_mark); + skb->mark = key.mark; /* no checksum performed at this layer */ skb->ip_summed = CHECKSUM_NONE; + ovpn_dst_cache_check_key(peer, &peer->dst_cache, &key); + /* crypto layer -> transport (UDP) */ - ret = ovpn_udp_output(peer, &peer->dst_cache, sk, skb); + ret = ovpn_udp_output(peer, &peer->dst_cache, sk, skb, &key); if (unlikely(ret < 0)) kfree_skb(skb); } diff --git a/drivers/net/pcs/pcs-rzn1-miic.c b/drivers/net/pcs/pcs-rzn1-miic.c index 2b72fa98ddf13..cb74861e823c8 100644 --- a/drivers/net/pcs/pcs-rzn1-miic.c +++ b/drivers/net/pcs/pcs-rzn1-miic.c @@ -683,7 +683,8 @@ static int miic_parse_dt(struct miic *miic, u32 *mode_cfg) if (!dt_val) return -ENOMEM; - memset(dt_val, MIIC_MODCTRL_CONF_NONE, sizeof(*dt_val)); + memset(dt_val, MIIC_MODCTRL_CONF_NONE, + sizeof(*dt_val) * miic->of_data->conf_conv_count); if (of_property_read_u32(np, "renesas,miic-switch-portin", &conf) == 0) dt_val[0] = conf; diff --git a/drivers/net/pcs/pcs-xpcs.c b/drivers/net/pcs/pcs-xpcs.c index 0337e2bcc0125..b415b93d77c15 100644 --- a/drivers/net/pcs/pcs-xpcs.c +++ b/drivers/net/pcs/pcs-xpcs.c @@ -1545,8 +1545,10 @@ static int xpcs_init_clks(struct dw_xpcs *xpcs) return dev_err_probe(dev, ret, "Failed to get clocks\n"); ret = clk_bulk_prepare_enable(DW_XPCS_NUM_CLKS, xpcs->clks); - if (ret) + if (ret) { + clk_bulk_put(DW_XPCS_NUM_CLKS, xpcs->clks); return dev_err_probe(dev, ret, "Failed to enable clocks\n"); + } return 0; } diff --git a/drivers/net/phy/air_en8811h.c b/drivers/net/phy/air_en8811h.c index edd49c193e476..0eeb7b9a4e260 100644 --- a/drivers/net/phy/air_en8811h.c +++ b/drivers/net/phy/air_en8811h.c @@ -1173,13 +1173,6 @@ static int en8811h_probe(struct phy_device *phydev) if (ret) return ret; - /* Configure led gpio pins as output */ - ret = air_phy_buckpbus_reg_modify(phydev, EN8811H_GPIO_OUTPUT, - EN8811H_GPIO_OUTPUT_345, - EN8811H_GPIO_OUTPUT_345); - if (ret < 0) - return ret; - return 0; } @@ -1324,6 +1317,17 @@ static int en8811h_config_init(struct phy_device *phydev) return ret; } + /* Configure led gpio pins as output. Must be redone on every + * .config_init(), not just once in .probe(): en8811h_restart_mcu() + * resets buckpbus-mapped MCU state (incl. this register) on every + * call after the first, e.g. on link renegotiation or ifup/ifdown. + */ + ret = air_phy_buckpbus_reg_modify(phydev, EN8811H_GPIO_OUTPUT, + EN8811H_GPIO_OUTPUT_345, + EN8811H_GPIO_OUTPUT_345); + if (ret < 0) + return ret; + return 0; } diff --git a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c index 88255e92b4cdb..61a941aa02d94 100644 --- a/drivers/net/phy/dp83867.c +++ b/drivers/net/phy/dp83867.c @@ -1150,6 +1150,9 @@ static int dp83867_led_polarity_set(struct phy_device *phydev, int index, case PHY_LED_ACTIVE_LOW: polarity = 0; break; + case PHY_LED_ACTIVE_HIGH: + polarity = DP83867_LED_POLARITY(index); + break; default: return -EINVAL; } diff --git a/drivers/net/phy/dp83td510.c b/drivers/net/phy/dp83td510.c index d75dae6071ad6..9e9a41bf64574 100644 --- a/drivers/net/phy/dp83td510.c +++ b/drivers/net/phy/dp83td510.c @@ -439,6 +439,9 @@ static int dp83td510_led_polarity_set(struct phy_device *phydev, int index, case PHY_LED_ACTIVE_LOW: polarity = 0; break; + case PHY_LED_ACTIVE_HIGH: + polarity = DP83TD510E_LED_POLARITY(index); + break; default: return -EINVAL; } diff --git a/drivers/net/phy/intel-xway.c b/drivers/net/phy/intel-xway.c index afbcec7117443..3cee31bb931f2 100644 --- a/drivers/net/phy/intel-xway.c +++ b/drivers/net/phy/intel-xway.c @@ -16,6 +16,11 @@ #define XWAY_MDIO_ISTAT 0x1A /* interrupt status */ #define XWAY_MDIO_LED 0x1B /* led control */ +#define XWAY_MDIO_GCTRL_TM_MASK GENMASK(15, 13) +#define XWAY_MDIO_GCTRL_TM(mode) FIELD_PREP(XWAY_MDIO_GCTRL_TM_MASK, (mode)) +#define XWAY_MDIO_GCTRL_TM_NOP XWAY_MDIO_GCTRL_TM(0) /* Normal operation */ +#define XWAY_MDIO_GCTRL_TM_CDIAG XWAY_MDIO_GCTRL_TM(6) /* Cable diagnostics */ + #define XWAY_MDIO_ERRCNT_SEL GENMASK(11, 8) #define XWAY_MDIO_ERRCNT_COUNT GENMASK(7, 0) #define XWAY_MDIO_ERRCNT_SEL_RXERR 0 @@ -326,6 +331,28 @@ static int xway_gphy_probe(struct phy_device *phydev) return 0; } +static int xway_11g_int_config_init(struct phy_device *phydev) +{ + int err; + + /* An issue has been sporadically observed after device power-on on the + * first link-up attempt in 100BASE-TX mode resulting in either the + * link-up taking a long time, or failing to link-up altogether. + * + * Workaround: + * After power-on, enable Cable Diagnostic Mode for all ports and + * disable it. + */ + err = phy_modify(phydev, MII_CTRL1000, XWAY_MDIO_GCTRL_TM_MASK, XWAY_MDIO_GCTRL_TM_CDIAG); + if (err) + return err; + err = phy_modify(phydev, MII_CTRL1000, XWAY_MDIO_GCTRL_TM_MASK, XWAY_MDIO_GCTRL_TM_NOP); + if (err) + return err; + + return xway_gphy_config_init(phydev); +} + static int xway_gphy14_config_aneg(struct phy_device *phydev) { int reg, err; @@ -735,7 +762,7 @@ static struct phy_driver xway_gphy[] = { .phy_id_mask = 0xffffffff, .name = "Intel XWAY PHY11G (xRX v1.2 integrated)", /* PHY_GBIT_FEATURES */ - .config_init = xway_gphy_config_init, + .config_init = xway_11g_int_config_init, .probe = xway_gphy_probe, .handle_interrupt = xway_gphy_handle_interrupt, .config_intr = xway_gphy_config_intr, diff --git a/drivers/net/phy/mediatek/mtk-ge.c b/drivers/net/phy/mediatek/mtk-ge.c index 73d9b72f9d9e2..96d8ac5154e5e 100644 --- a/drivers/net/phy/mediatek/mtk-ge.c +++ b/drivers/net/phy/mediatek/mtk-ge.c @@ -62,10 +62,38 @@ static void mtk_gephy_config_init(struct phy_device *phydev) FIELD_PREP(MTK_MCC_NEARECHO_OFFSET_MASK, 0x3)); } +static int mt7530_phy_probe(struct phy_device *phydev) +{ + /* The MT7530 internal GE PHY has broken EEE: with EEE advertised, + * some link partners fail to establish a stable link (on a 2-pair + * cable, 1000BASE-T training fails and the port loops instead of + * falling back). MediaTek recommends disabling EEE on this PHY. + * Clear the advertisement as early as possible, before anything + * can negotiate EEE with the link partner. + */ + return phy_write_mmd(phydev, MDIO_MMD_AN, MDIO_AN_EEE_ADV, 0); +} + static int mt7530_phy_config_init(struct phy_device *phydev) { + int ret; + mtk_gephy_config_init(phydev); + /* The probe() clear alone is not durable: phy_init_hw() replays only + * ->config_init after a PHY reset, with the register back at its + * EEE-advertising hardware default, and phy_probe() zeroes + * eee_disabled_modes (of_set_phy_eee_broken()) after ->probe already + * ran. Clear the advertisement again and mark EEE disabled, so that + * neither phylib nor userspace can re-enable it; dp83867 disables + * broken EEE from config_init() the same way. + */ + ret = phy_write_mmd(phydev, MDIO_MMD_AN, MDIO_AN_EEE_ADV, 0); + if (ret) + return ret; + + phy_disable_eee(phydev); + /* Increase post_update_timer */ phy_write_paged(phydev, MTK_PHY_PAGE_EXTENDED_3, MTK_PHY_RG_LPI_PCS_DSP_CTRL_REG11, 0x4b); @@ -100,6 +128,7 @@ static struct phy_driver mtk_gephy_driver[] = { { PHY_ID_MATCH_EXACT(MTK_GPHY_ID_MT7530), .name = "MediaTek MT7530 PHY", + .probe = mt7530_phy_probe, .config_init = mt7530_phy_config_init, /* Interrupts are handled by the switch, not the PHY * itself. diff --git a/drivers/net/phy/mediatek/mtk-phy-lib.c b/drivers/net/phy/mediatek/mtk-phy-lib.c index dfd0f4e439a21..608072fbfde97 100644 --- a/drivers/net/phy/mediatek/mtk-phy-lib.c +++ b/drivers/net/phy/mediatek/mtk-phy-lib.c @@ -156,20 +156,27 @@ int mtk_phy_led_hw_ctrl_get(struct phy_device *phydev, u8 index, if (!rules) return 0; - if (on & on_set) + /* TRIGGER_NETDEV_LINK must not be reported together with any of the + * per-speed rules, the netdev trigger rejects that combination. + * on_set holds every speed this LED can indicate and is what + * mtk_phy_led_hw_ctrl_set() programs for TRIGGER_NETDEV_LINK, so + * report the speed independent rule only when they are all on. + */ + if ((on & on_set) == on_set) { *rules |= BIT(TRIGGER_NETDEV_LINK); + } else { + if (on & MTK_PHY_LED_ON_LINK10) + *rules |= BIT(TRIGGER_NETDEV_LINK_10); - if (on & MTK_PHY_LED_ON_LINK10) - *rules |= BIT(TRIGGER_NETDEV_LINK_10); + if (on & MTK_PHY_LED_ON_LINK100) + *rules |= BIT(TRIGGER_NETDEV_LINK_100); - if (on & MTK_PHY_LED_ON_LINK100) - *rules |= BIT(TRIGGER_NETDEV_LINK_100); + if (on & MTK_PHY_LED_ON_LINK1000) + *rules |= BIT(TRIGGER_NETDEV_LINK_1000); - if (on & MTK_PHY_LED_ON_LINK1000) - *rules |= BIT(TRIGGER_NETDEV_LINK_1000); - - if (on & MTK_PHY_LED_ON_LINK2500) - *rules |= BIT(TRIGGER_NETDEV_LINK_2500); + if (on & MTK_PHY_LED_ON_LINK2500) + *rules |= BIT(TRIGGER_NETDEV_LINK_2500); + } if (on & MTK_PHY_LED_ON_FDX) *rules |= BIT(TRIGGER_NETDEV_FULL_DUPLEX); diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index 55df5efcfc862..cde60498d0544 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -6285,6 +6285,7 @@ static int lanphy_write_reg_data(struct phy_device *phydev, data->val); if (ret) break; + data++; } return ret; diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 18d2ead97aa54..3477e1104a43a 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -965,7 +965,7 @@ static unsigned int phylink_inband_caps(struct phylink *pl, return 0; pcs = pl->mac_ops->mac_select_pcs(pl->config, interface); - if (!pcs) + if (IS_ERR_OR_NULL(pcs)) return 0; return phylink_pcs_inband_caps(pcs, interface); @@ -1630,8 +1630,10 @@ static void phylink_resolve(struct work_struct *w) if (pl->phylink_disable_state) { pl->link_failed = false; + link_state = pl->link_config; link_state.link = false; } else if (pl->link_failed) { + link_state = pl->link_config; link_state.link = false; retrigger = true; } else if (pl->act_link_an_mode == MLO_AN_FIXED) { @@ -2129,7 +2131,6 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy, mutex_lock(&pl->phydev_mutex); mutex_lock(&phy->lock); mutex_lock(&pl->state_mutex); - pl->phydev = phy; pl->phy_state.interface = interface; pl->phy_state.pause = MLO_PAUSE_NONE; pl->phy_state.speed = SPEED_UNKNOWN; @@ -2196,10 +2197,25 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy, ret = 0; } - if (ret == 0 && phy_interrupt_is_valid(phy)) + if (ret) + return ret; + + /* Nothing below can fail, so the PHY can be recorded now. Doing it + * here rather than above keeps a failed bringup from leaving + * pl->phydev pointing at a PHY the caller is about to detach. + */ + mutex_lock(&pl->phydev_mutex); + mutex_lock(&phy->lock); + mutex_lock(&pl->state_mutex); + pl->phydev = phy; + mutex_unlock(&pl->state_mutex); + mutex_unlock(&phy->lock); + mutex_unlock(&pl->phydev_mutex); + + if (phy_interrupt_is_valid(phy)) phy_request_interrupt(phy); - return ret; + return 0; } static int phylink_attach_phy(struct phylink *pl, struct phy_device *phy, diff --git a/drivers/net/ppp/ppp_async.c b/drivers/net/ppp/ppp_async.c index 93a7b0f6c4e7e..6664686b1e041 100644 --- a/drivers/net/ppp/ppp_async.c +++ b/drivers/net/ppp/ppp_async.c @@ -65,8 +65,6 @@ struct asyncppp { struct tasklet_struct tsk; - refcount_t refcnt; - struct completion dead; struct ppp_channel chan; /* interface to generic ppp layer */ unsigned char obuf[OBUFSIZE]; }; @@ -116,38 +114,6 @@ static const struct ppp_channel_ops async_ops = { * Routines implementing the PPP line discipline. */ -/* - * We have a potential race on dereferencing tty->disc_data, - * because the tty layer provides no locking at all - thus one - * cpu could be running ppp_asynctty_receive while another - * calls ppp_asynctty_close, which zeroes tty->disc_data and - * frees the memory that ppp_asynctty_receive is using. The best - * way to fix this is to use a rwlock in the tty struct, but for now - * we use a single global rwlock for all ttys in ppp line discipline. - * - * FIXME: this is no longer true. The _close path for the ldisc is - * now guaranteed to be sane. - */ -static DEFINE_RWLOCK(disc_data_lock); - -static struct asyncppp *ap_get(struct tty_struct *tty) -{ - struct asyncppp *ap; - - read_lock(&disc_data_lock); - ap = tty->disc_data; - if (ap != NULL) - refcount_inc(&ap->refcnt); - read_unlock(&disc_data_lock); - return ap; -} - -static void ap_put(struct asyncppp *ap) -{ - if (refcount_dec_and_test(&ap->refcnt)) - complete(&ap->dead); -} - /* * Called when a tty is put into PPP line discipline. Called in process * context. @@ -182,9 +148,6 @@ ppp_asynctty_open(struct tty_struct *tty) skb_queue_head_init(&ap->rqueue); tasklet_setup(&ap->tsk, ppp_async_process); - refcount_set(&ap->refcnt, 1); - init_completion(&ap->dead); - ap->chan.private = ap; ap->chan.ops = &async_ops; ap->chan.mtu = PPP_MRU; @@ -205,34 +168,18 @@ ppp_asynctty_open(struct tty_struct *tty) } /* - * Called when the tty is put into another line discipline - * or it hangs up. We have to wait for any cpu currently - * executing in any of the other ppp_asynctty_* routines to - * finish before we can call ppp_unregister_channel and free - * the asyncppp struct. This routine must be called from - * process context, not interrupt or softirq context. + * Called when the tty is put into another line discipline or it hangs up. + * This call is serialized against other ldisc functions. */ static void ppp_asynctty_close(struct tty_struct *tty) { - struct asyncppp *ap; + struct asyncppp *ap = tty->disc_data; - write_lock_irq(&disc_data_lock); - ap = tty->disc_data; - tty->disc_data = NULL; - write_unlock_irq(&disc_data_lock); if (!ap) return; - /* - * We have now ensured that nobody can start using ap from now - * on, but we have to wait for all existing users to finish. - * Note that ppp_unregister_channel ensures that no calls to - * our channel ops (i.e. ppp_async_send/ioctl) are in progress - * by the time it returns. - */ - if (!refcount_dec_and_test(&ap->refcnt)) - wait_for_completion(&ap->dead); + tty->disc_data = NULL; tasklet_kill(&ap->tsk); ppp_unregister_channel(&ap->chan); @@ -242,17 +189,6 @@ ppp_asynctty_close(struct tty_struct *tty) kfree(ap); } -/* - * Called on tty hangup in process context. - * - * Wait for I/O to driver to complete and unregister PPP channel. - * This is already done by the close routine, so just call that. - */ -static void ppp_asynctty_hangup(struct tty_struct *tty) -{ - ppp_asynctty_close(tty); -} - /* * Read does nothing - no data is ever available this way. * Pppd reads and writes packets via /dev/ppp instead. @@ -283,7 +219,7 @@ ppp_asynctty_write(struct tty_struct *tty, struct file *file, const u8 *buf, static int ppp_asynctty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { - struct asyncppp *ap = ap_get(tty); + struct asyncppp *ap = tty->disc_data; int err, val; int __user *p = (int __user *)arg; @@ -324,7 +260,6 @@ ppp_asynctty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) err = tty_mode_ioctl(tty, cmd, arg); } - ap_put(ap); return err; } @@ -333,7 +268,7 @@ static void ppp_asynctty_receive(struct tty_struct *tty, const u8 *buf, const u8 *cflags, size_t count) { - struct asyncppp *ap = ap_get(tty); + struct asyncppp *ap = tty->disc_data; unsigned long flags; if (!ap) @@ -343,21 +278,19 @@ ppp_asynctty_receive(struct tty_struct *tty, const u8 *buf, const u8 *cflags, spin_unlock_irqrestore(&ap->recv_lock, flags); if (!skb_queue_empty(&ap->rqueue)) tasklet_schedule(&ap->tsk); - ap_put(ap); tty_unthrottle(tty); } static void ppp_asynctty_wakeup(struct tty_struct *tty) { - struct asyncppp *ap = ap_get(tty); + struct asyncppp *ap = tty->disc_data; clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); if (!ap) return; set_bit(XMIT_WAKEUP, &ap->xmit_flags); tasklet_schedule(&ap->tsk); - ap_put(ap); } @@ -367,7 +300,6 @@ static struct tty_ldisc_ops ppp_ldisc = { .name = "ppp", .open = ppp_asynctty_open, .close = ppp_asynctty_close, - .hangup = ppp_asynctty_hangup, .read = ppp_asynctty_read, .write = ppp_asynctty_write, .ioctl = ppp_asynctty_ioctl, @@ -812,11 +744,8 @@ process_input_packet(struct asyncppp *ap) err: /* frame had an error, remember that, reset SC_TOSS & SC_ESCAPE */ ap->state = SC_PREV_ERROR; - if (skb) { - /* make skb appear as freshly allocated */ - skb_trim(skb, 0); - skb_reserve(skb, - skb_headroom(skb)); - } + kfree_skb(skb); + ap->rpkt = NULL; } /* Called when the tty driver has data for us. Runs parallel with the diff --git a/drivers/net/ppp/ppp_synctty.c b/drivers/net/ppp/ppp_synctty.c index b7f243b416f82..1b64e610f9609 100644 --- a/drivers/net/ppp/ppp_synctty.c +++ b/drivers/net/ppp/ppp_synctty.c @@ -38,11 +38,9 @@ #include #include #include -#include #include #include #include -#include #include #include @@ -69,8 +67,6 @@ struct syncppp { struct tasklet_struct tsk; - refcount_t refcnt; - struct completion dead_cmp; struct ppp_channel chan; /* interface to generic ppp layer */ }; @@ -118,37 +114,6 @@ ppp_print_buffer (const char *name, const __u8 *buf, int count) * Routines implementing the synchronous PPP line discipline. */ -/* - * We have a potential race on dereferencing tty->disc_data, - * because the tty layer provides no locking at all - thus one - * cpu could be running ppp_synctty_receive while another - * calls ppp_synctty_close, which zeroes tty->disc_data and - * frees the memory that ppp_synctty_receive is using. The best - * way to fix this is to use a rwlock in the tty struct, but for now - * we use a single global rwlock for all ttys in ppp line discipline. - * - * FIXME: Fixed in tty_io nowadays. - */ -static DEFINE_RWLOCK(disc_data_lock); - -static struct syncppp *sp_get(struct tty_struct *tty) -{ - struct syncppp *ap; - - read_lock(&disc_data_lock); - ap = tty->disc_data; - if (ap != NULL) - refcount_inc(&ap->refcnt); - read_unlock(&disc_data_lock); - return ap; -} - -static void sp_put(struct syncppp *ap) -{ - if (refcount_dec_and_test(&ap->refcnt)) - complete(&ap->dead_cmp); -} - /* * Called when a tty is put into sync-PPP line discipline. */ @@ -179,9 +144,6 @@ ppp_sync_open(struct tty_struct *tty) skb_queue_head_init(&ap->rqueue); tasklet_setup(&ap->tsk, ppp_sync_process); - refcount_set(&ap->refcnt, 1); - init_completion(&ap->dead_cmp); - ap->chan.private = ap; ap->chan.ops = &sync_ops; ap->chan.mtu = PPP_MRU; @@ -203,34 +165,18 @@ ppp_sync_open(struct tty_struct *tty) } /* - * Called when the tty is put into another line discipline - * or it hangs up. We have to wait for any cpu currently - * executing in any of the other ppp_synctty_* routines to - * finish before we can call ppp_unregister_channel and free - * the syncppp struct. This routine must be called from - * process context, not interrupt or softirq context. + * Called when the tty is put into another line discipline or it hangs up. + * This call is serialized against other ldisc functions. */ static void ppp_sync_close(struct tty_struct *tty) { - struct syncppp *ap; + struct syncppp *ap = tty->disc_data; - write_lock_irq(&disc_data_lock); - ap = tty->disc_data; - tty->disc_data = NULL; - write_unlock_irq(&disc_data_lock); if (!ap) return; - /* - * We have now ensured that nobody can start using ap from now - * on, but we have to wait for all existing users to finish. - * Note that ppp_unregister_channel ensures that no calls to - * our channel ops (i.e. ppp_sync_send/ioctl) are in progress - * by the time it returns. - */ - if (!refcount_dec_and_test(&ap->refcnt)) - wait_for_completion(&ap->dead_cmp); + tty->disc_data = NULL; tasklet_kill(&ap->tsk); ppp_unregister_channel(&ap->chan); @@ -239,17 +185,6 @@ ppp_sync_close(struct tty_struct *tty) kfree(ap); } -/* - * Called on tty hangup in process context. - * - * Wait for I/O to driver to complete and unregister PPP channel. - * This is already done by the close routine, so just call that. - */ -static void ppp_sync_hangup(struct tty_struct *tty) -{ - ppp_sync_close(tty); -} - /* * Read does nothing - no data is ever available this way. * Pppd reads and writes packets via /dev/ppp instead. @@ -275,7 +210,7 @@ ppp_sync_write(struct tty_struct *tty, struct file *file, const u8 *buf, static int ppp_synctty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { - struct syncppp *ap = sp_get(tty); + struct syncppp *ap = tty->disc_data; int __user *p = (int __user *)arg; int err, val; @@ -316,7 +251,6 @@ ppp_synctty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) break; } - sp_put(ap); return err; } @@ -325,7 +259,7 @@ static void ppp_sync_receive(struct tty_struct *tty, const u8 *buf, const u8 *cflags, size_t count) { - struct syncppp *ap = sp_get(tty); + struct syncppp *ap = tty->disc_data; unsigned long flags; if (!ap) @@ -335,21 +269,19 @@ ppp_sync_receive(struct tty_struct *tty, const u8 *buf, const u8 *cflags, spin_unlock_irqrestore(&ap->recv_lock, flags); if (!skb_queue_empty(&ap->rqueue)) tasklet_schedule(&ap->tsk); - sp_put(ap); tty_unthrottle(tty); } static void ppp_sync_wakeup(struct tty_struct *tty) { - struct syncppp *ap = sp_get(tty); + struct syncppp *ap = tty->disc_data; clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); if (!ap) return; set_bit(XMIT_WAKEUP, &ap->xmit_flags); tasklet_schedule(&ap->tsk); - sp_put(ap); } @@ -359,7 +291,6 @@ static struct tty_ldisc_ops ppp_sync_ldisc = { .name = "pppsync", .open = ppp_sync_open, .close = ppp_sync_close, - .hangup = ppp_sync_hangup, .read = ppp_sync_read, .write = ppp_sync_write, .ioctl = ppp_synctty_ioctl, @@ -526,17 +457,9 @@ ppp_sync_txmunge(struct syncppp *ap, struct sk_buff *skb) /* prepend address/control fields if necessary */ if ((ap->flags & SC_COMP_AC) == 0 || islcp) { - if (skb_headroom(skb) < 2) { - struct sk_buff *npkt = dev_alloc_skb(skb->len + 2); - if (npkt == NULL) { - kfree_skb(skb); - return NULL; - } - skb_reserve(npkt,2); - skb_copy_from_linear_data(skb, - skb_put(npkt, skb->len), skb->len); - consume_skb(skb); - skb = npkt; + if (skb_cow_head(skb, 2)) { + kfree_skb(skb); + return NULL; } skb_push(skb,2); skb->data[0] = PPP_ALLSTATIONS; diff --git a/drivers/net/ppp/pppox.c b/drivers/net/ppp/pppox.c index 5861a2f6ce3e6..a6f72c813bef9 100644 --- a/drivers/net/ppp/pppox.c +++ b/drivers/net/ppp/pppox.c @@ -74,7 +74,9 @@ int pppox_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) switch (cmd) { case PPPIOCGCHAN: { + struct sk_buff *skb; int index; + rc = -ENOTCONN; if (!(sk->sk_state & PPPOX_CONNECTED)) break; @@ -85,7 +87,22 @@ int pppox_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) break; rc = 0; + /* PPPIOCGCHAN historically marks the userspace handoff to + * generic PPP; pppd then attaches the returned channel to + * /dev/ppp. + */ sk->sk_state |= PPPOX_BOUND; + /* Let lockless receive paths finish queueing against the old + * state. + */ + synchronize_net(); + /* Drain packets queued before the handoff because a bound + * socket is no longer readable. + */ + while ((skb = skb_dequeue(&sk->sk_receive_queue))) { + skb_orphan(skb); + ppp_input(&po->chan, skb); + } break; } default: diff --git a/drivers/net/slip/slip.c b/drivers/net/slip/slip.c index faae711cf793d..fdd6f63fb849e 100644 --- a/drivers/net/slip/slip.c +++ b/drivers/net/slip/slip.c @@ -628,9 +628,15 @@ static void sl_uninit(struct net_device *dev) struct slip *sl = netdev_priv(dev); sl_free_bufs(sl); + /* Drop the slip_devs[] entry here rather than from the destructor: + * ndo_uninit runs under RTNL, so it cannot race sl_sync(). + */ + slip_devs[dev->base_addr] = NULL; } -/* Hook the destructor so we can free slip devices at the right point in time */ +/* Only for the slip_open() error path: register_netdevice() can fail before + * ndo_init, and then ndo_uninit is not called either. + */ static void sl_free_netdev(struct net_device *dev) { int i = dev->base_addr; @@ -657,7 +663,6 @@ static void sl_setup(struct net_device *dev) { dev->netdev_ops = &sl_netdev_ops; dev->needs_free_netdev = true; - dev->priv_destructor = sl_free_netdev; dev->hard_header_len = 0; dev->addr_len = 0; @@ -881,8 +886,6 @@ static int slip_open(struct tty_struct *tty) * Close down a SLIP channel. * This means flushing out any pending queues, and then returning. This * call is serialized against other ldisc functions. - * - * We also use this method fo a hangup event */ static void slip_close(struct tty_struct *tty) @@ -908,12 +911,7 @@ static void slip_close(struct tty_struct *tty) #endif /* Flush network side */ unregister_netdev(sl->dev); - /* This will complete via sl_free_netdev */ -} - -static void slip_hangup(struct tty_struct *tty) -{ - slip_close(tty); + /* sl_uninit() has dropped the slip_devs[] entry by now */ } /************************************************************************ * STANDARD SLIP ENCAPSULATION * @@ -1275,7 +1273,6 @@ static struct tty_ldisc_ops sl_ldisc = { .name = "slip", .open = slip_open, .close = slip_close, - .hangup = slip_hangup, .ioctl = slip_ioctl, .receive_buf = slip_receive_buf, .write_wakeup = slip_write_wakeup, diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c index 98893732bc6e4..82cef20092b8c 100644 --- a/drivers/net/thunderbolt/main.c +++ b/drivers/net/thunderbolt/main.c @@ -626,6 +626,14 @@ static int tbnet_alloc_tx_buffers(struct tbnet *net) return 0; } +static void tbnet_connect_failed(struct tbnet *net) +{ + /* Leave login_received set: only the peer can make it true again. */ + mutex_lock(&net->connection_lock); + net->login_sent = false; + mutex_unlock(&net->connection_lock); +} + static void tbnet_connected_work(struct work_struct *work) { struct tbnet *net = container_of(work, typeof(*net), connected_work); @@ -647,6 +655,9 @@ static void tbnet_connected_work(struct work_struct *work) ret = tb_xdomain_alloc_in_hopid(net->xd, net->remote_transmit_path); if (ret != net->remote_transmit_path) { netdev_err(net->dev, "failed to allocate Rx HopID\n"); + if (ret >= 0) + tb_xdomain_release_in_hopid(net->xd, ret); + tbnet_connect_failed(net); return; } @@ -691,6 +702,7 @@ static void tbnet_connected_work(struct work_struct *work) tb_ring_stop(net->rx_ring.ring); tb_ring_stop(net->tx_ring.ring); tb_xdomain_release_in_hopid(net->xd, net->remote_transmit_path); + tbnet_connect_failed(net); } static void tbnet_login_work(struct work_struct *work) @@ -892,9 +904,11 @@ static int tbnet_poll(struct napi_struct *napi, int budget) le32_to_cpu(net->rx_hdr.frame_count) - 1; rx_packets++; - net->stats.rx_bytes += frame_size; if (last) { + /* Before eth_type_trans() pulls the Ethernet header. */ + net->stats.rx_packets++; + net->stats.rx_bytes += skb->len; skb->protocol = eth_type_trans(skb, net->dev); trace_tbnet_rx_skb(skb); napi_gro_receive(&net->napi, skb); @@ -902,8 +916,6 @@ static int tbnet_poll(struct napi_struct *napi, int budget) } } - net->stats.rx_packets += rx_packets; - if (cleaned_count) tbnet_alloc_rx_buffers(net, cleaned_count); diff --git a/drivers/net/tun.c b/drivers/net/tun.c index fed9dfdfcc3bc..5bbe3123979e0 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1107,11 +1107,16 @@ static netdev_features_t tun_net_fix_features(struct net_device *dev, static void tun_set_headroom(struct net_device *dev, int new_hr) { struct tun_struct *tun = netdev_priv(dev); + size_t max_headroom; - if (new_hr < NET_SKB_PAD) - new_hr = NET_SKB_PAD; + max_headroom = min_t(size_t, SKB_MAX_HEAD(0), U16_MAX - 1); - tun->align = new_hr; + if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) + max_headroom -= ETH_HLEN + NET_IP_ALIGN; + else + max_headroom -= 1; + + tun->align = clamp_t(int, new_hr, NET_SKB_PAD, max_headroom); } static void @@ -1822,7 +1827,13 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, switch (tun->flags & TUN_TYPE_MASK) { case IFF_TUN: if (tun->flags & IFF_NO_PI) { - u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0; + u8 ip_version; + + if (!pskb_may_pull(skb, 1)) { + err = -EINVAL; + goto drop; + } + ip_version = skb->data[0] >> 4; switch (ip_version) { case 4: @@ -1842,7 +1853,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, skb->dev = tun->dev; break; case IFF_TAP: - if (frags && !pskb_may_pull(skb, ETH_HLEN)) { + if (!pskb_may_pull(skb, ETH_HLEN)) { err = -ENOMEM; drop_reason = SKB_DROP_REASON_HDR_TRUNC; goto drop; diff --git a/drivers/net/usb/catc.c b/drivers/net/usb/catc.c index 96e82f94edcf8..39b678f175dd9 100644 --- a/drivers/net/usb/catc.c +++ b/drivers/net/usb/catc.c @@ -233,17 +233,26 @@ static void catc_rx_done(struct urb *urb) } do { - if(!catc->is_f5u011) { - pkt_len = le16_to_cpup((__le16*)pkt_start); - if (pkt_len > urb->actual_length) { + int remaining = urb->actual_length - + (pkt_start - (u8 *)urb->transfer_buffer); + + if (!catc->is_f5u011) { + if (remaining < pkt_offset) { catc->netdev->stats.rx_length_errors++; catc->netdev->stats.rx_errors++; break; } + pkt_len = le16_to_cpup((__le16 *)pkt_start); } else { pkt_len = urb->actual_length; } + if (pkt_len < ETH_HLEN || pkt_len + pkt_offset > remaining) { + catc->netdev->stats.rx_length_errors++; + catc->netdev->stats.rx_errors++; + break; + } + if (!(skb = dev_alloc_skb(pkt_len))) return; diff --git a/drivers/net/usb/cdc_mbim.c b/drivers/net/usb/cdc_mbim.c index 877fb0ed7d3d7..a7010a0664c7d 100644 --- a/drivers/net/usb/cdc_mbim.c +++ b/drivers/net/usb/cdc_mbim.c @@ -635,6 +635,11 @@ static const struct usb_device_id mbim_devs[] = { .driver_info = (unsigned long)&cdc_mbim_info, }, + /* MeiG Smart SRM821 ZLP conformance */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d53, USB_CLASS_COMM, USB_CDC_SUBCLASS_MBIM, USB_CDC_PROTO_NONE), + .driver_info = (unsigned long)&cdc_mbim_info, + }, + /* Some Huawei devices, ME906s-158 (12d1:15c1) and E3372 * (12d1:157d), are known to fail unless the NDP is placed * after the IP packets. Applying the quirk to all Huawei diff --git a/drivers/net/usb/cx82310_eth.c b/drivers/net/usb/cx82310_eth.c index 068acb052adb0..5df657acf3d55 100644 --- a/drivers/net/usb/cx82310_eth.c +++ b/drivers/net/usb/cx82310_eth.c @@ -282,6 +282,7 @@ static int cx82310_rx_fixup(struct usbnet *dev, struct sk_buff *skb) if (len == 0xffff) { netdev_info(dev->net, "router was rebooted, re-enabling ethernet mode"); schedule_work(&priv->reenable_work); + return 0; } else if (len > CX82310_MTU) { netdev_err(dev->net, "RX packet too long: %d B\n", len); return 0; diff --git a/drivers/net/usb/hso.c b/drivers/net/usb/hso.c index c1aec67688ae7..71caa3764b239 100644 --- a/drivers/net/usb/hso.c +++ b/drivers/net/usb/hso.c @@ -1547,10 +1547,10 @@ hso_wait_modem_status(struct hso_serial *serial, unsigned long arg) spin_unlock_irq(&serial->serial_lock); add_wait_queue(&tiocmget->waitq, &wait); for (;;) { + set_current_state(TASK_INTERRUPTIBLE); spin_lock_irq(&serial->serial_lock); memcpy(&cnow, &tiocmget->icount, sizeof(struct uart_icount)); spin_unlock_irq(&serial->serial_lock); - set_current_state(TASK_INTERRUPTIBLE); if (((arg & TIOCM_RNG) && (cnow.rng != cprev.rng)) || ((arg & TIOCM_DSR) && (cnow.dsr != cprev.dsr)) || ((arg & TIOCM_CD) && (cnow.dcd != cprev.dcd))) { diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index cb782d81d84f2..5655941f14783 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -5239,10 +5239,12 @@ static bool lan78xx_submit_deferred_urbs(struct lan78xx_net *dev) !netif_carrier_ok(dev->net) || pipe_halted) { lan78xx_release_tx_buf(dev, skb); + usb_put_urb(urb); continue; } ret = usb_submit_urb(urb, GFP_ATOMIC); + usb_put_urb(urb); if (ret == 0) { netif_trans_update(dev->net); diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c index 8700eeb8e22d0..aba1a640fc268 100644 --- a/drivers/net/usb/pegasus.c +++ b/drivers/net/usb/pegasus.c @@ -43,21 +43,12 @@ static bool loopback; static bool mii_mode; static char *devid; -static struct usb_eth_dev usb_dev_id[] = { -#define PEGASUS_DEV(pn, vid, pid, flags) \ - {.name = pn, .vendor = vid, .device = pid, .private = flags}, -#define PEGASUS_DEV_CLASS(pn, vid, pid, dclass, flags) \ - PEGASUS_DEV(pn, vid, pid, flags) -#include "pegasus.h" -#undef PEGASUS_DEV -#undef PEGASUS_DEV_CLASS - {NULL, 0, 0, 0}, - {NULL, 0, 0, 0} -}; +static struct usb_eth_dev dynamic_id_info = {}; static struct usb_device_id pegasus_ids[] = { #define PEGASUS_DEV(pn, vid, pid, flags) \ - {.match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = vid, .idProduct = pid}, + {.match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = vid, .idProduct = pid, \ + .driver_info = (kernel_ulong_t)&(const struct usb_eth_dev) {.name = pn, .private = flags}}, /* * The Belkin F8T012xx1 bluetooth adaptor has the same vendor and product * IDs as the Belkin F5D5050, so we need to teach the pegasus driver to @@ -66,7 +57,8 @@ static struct usb_device_id pegasus_ids[] = { */ #define PEGASUS_DEV_CLASS(pn, vid, pid, dclass, flags) \ {.match_flags = (USB_DEVICE_ID_MATCH_DEVICE | USB_DEVICE_ID_MATCH_DEV_CLASS), \ - .idVendor = vid, .idProduct = pid, .bDeviceClass = dclass}, + .idVendor = vid, .idProduct = pid, .bDeviceClass = dclass, \ + .driver_info = (kernel_ulong_t)&(const struct usb_eth_dev) {.name = pn, .private = flags}}, #include "pegasus.h" #undef PEGASUS_DEV #undef PEGASUS_DEV_CLASS @@ -402,12 +394,12 @@ static inline int reset_mac(pegasus_t *pegasus) if (i == REG_TIMEOUT) return -ETIMEDOUT; - if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS || - usb_dev_id[pegasus->dev_index].vendor == VENDOR_DLINK) { + if (le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_LINKSYS || + le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_DLINK) { set_register(pegasus, Gpio0, 0x24); set_register(pegasus, Gpio0, 0x26); } - if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_ELCON) { + if (le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_ELCON) { __u16 auxmode; ret = read_mii_word(pegasus, 3, 0x1b, &auxmode); if (ret < 0) @@ -445,9 +437,9 @@ static int enable_net_traffic(struct net_device *dev, struct usb_device *usb) memcpy(pegasus->eth_regs, data, sizeof(data)); ret = set_registers(pegasus, EthCtrl0, 3, data); - if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS || - usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS2 || - usb_dev_id[pegasus->dev_index].vendor == VENDOR_DLINK) { + if (le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_LINKSYS || + le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_LINKSYS2 || + le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_DLINK) { u16 auxmode; ret = read_mii_word(pegasus, 0, 0x1b, &auxmode); if (ret < 0) @@ -1153,7 +1145,7 @@ static int pegasus_probe(struct usb_interface *intf, struct usb_device *dev = interface_to_usbdev(intf); struct net_device *net; pegasus_t *pegasus; - int dev_index = id - pegasus_ids; + const struct usb_eth_dev *info = (const struct usb_eth_dev *)id->driver_info; int res = -ENOMEM; static const u8 bulk_ep_addr[] = { PEGASUS_USB_EP_BULK_IN | USB_DIR_IN, @@ -1178,7 +1170,6 @@ static int pegasus_probe(struct usb_interface *intf, goto out; pegasus = netdev_priv(net); - pegasus->dev_index = dev_index; pegasus->intf = intf; res = alloc_urbs(pegasus); @@ -1206,7 +1197,7 @@ static int pegasus_probe(struct usb_interface *intf, pegasus->msg_enable = netif_msg_init(msg_level, NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK); - pegasus->features = usb_dev_id[dev_index].private; + pegasus->features = info ? info->private : DEFAULT_GPIO_RESET; res = get_interrupt_interval(pegasus); if (res) goto out2; @@ -1235,7 +1226,7 @@ static int pegasus_probe(struct usb_interface *intf, queue_delayed_work(system_long_wq, &pegasus->carrier_check, CARRIER_CHECK_DELAY); dev_info(&intf->dev, "%s, %s, %pM\n", net->name, - usb_dev_id[dev_index].name, net->dev_addr); + info ? info->name : "(unknown)", net->dev_addr); return 0; out3: @@ -1325,8 +1316,9 @@ static struct usb_driver pegasus_driver = { static void __init parse_id(char *id) { - unsigned int vendor_id = 0, device_id = 0, flags = 0, i = 0; + unsigned int vendor_id = 0, device_id = 0, flags = 0; char *token, *name = NULL; + int dyn_id_index = ARRAY_SIZE(pegasus_ids) - 2; token = strsep(&id, ":"); if (token) @@ -1348,14 +1340,12 @@ static void __init parse_id(char *id) if (device_id > 0x10000 || device_id == 0) return; - for (i = 0; usb_dev_id[i].name; i++); - usb_dev_id[i].name = name; - usb_dev_id[i].vendor = vendor_id; - usb_dev_id[i].device = device_id; - usb_dev_id[i].private = flags; - pegasus_ids[i].match_flags = USB_DEVICE_ID_MATCH_DEVICE; - pegasus_ids[i].idVendor = vendor_id; - pegasus_ids[i].idProduct = device_id; + dynamic_id_info.name = name; + dynamic_id_info.private = flags; + pegasus_ids[dyn_id_index].match_flags = USB_DEVICE_ID_MATCH_DEVICE; + pegasus_ids[dyn_id_index].idVendor = vendor_id; + pegasus_ids[dyn_id_index].idProduct = device_id; + pegasus_ids[dyn_id_index].driver_info = (kernel_ulong_t)&dynamic_id_info; } static int __init pegasus_init(void) diff --git a/drivers/net/usb/pegasus.h b/drivers/net/usb/pegasus.h index a05b143155ba8..ccdedcef52e76 100644 --- a/drivers/net/usb/pegasus.h +++ b/drivers/net/usb/pegasus.h @@ -85,7 +85,6 @@ typedef struct pegasus { unsigned features; u32 msg_enable; u32 wolopts; - int dev_index; int intr_interval; struct tasklet_struct rx_tl; struct delayed_work carrier_check; @@ -102,8 +101,6 @@ typedef struct pegasus { struct usb_eth_dev { char *name; - __u16 vendor; - __u16 device; __u32 private; /* LSB is gpio reset value */ }; diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c index 94cdb61dca83d..8178a8758cd3f 100644 --- a/drivers/net/usb/qmi_wwan.c +++ b/drivers/net/usb/qmi_wwan.c @@ -1360,6 +1360,7 @@ static const struct usb_device_id products[] = { {QMI_FIXED_INTF(0x1bbb, 0x0203, 2)}, /* Alcatel L800MA */ {QMI_FIXED_INTF(0x2357, 0x0201, 4)}, /* TP-LINK HSUPA Modem MA180 */ {QMI_FIXED_INTF(0x2357, 0x9000, 4)}, /* TP-LINK MA260 */ + {QMI_QUIRK_SET_DTR(0x1bc7, 0x0991, 0)}, /* Telit FE990D50 */ {QMI_QUIRK_SET_DTR(0x1bc7, 0x1031, 3)}, /* Telit LE910C1-EUX */ {QMI_QUIRK_SET_DTR(0x1bc7, 0x1034, 2)}, /* Telit LE910C4-WWX */ {QMI_QUIRK_SET_DTR(0x1bc7, 0x1037, 4)}, /* Telit LE910C4-WWX */ diff --git a/drivers/net/usb/rndis_host.c b/drivers/net/usb/rndis_host.c index 5e39d05a2d7b6..37d4865f5c5e6 100644 --- a/drivers/net/usb/rndis_host.c +++ b/drivers/net/usb/rndis_host.c @@ -14,6 +14,7 @@ #include #include #include +#include /* @@ -506,6 +507,7 @@ int rndis_rx_fixup(struct usbnet *dev, struct sk_buff *skb) struct rndis_data_hdr *hdr = (void *)skb->data; struct sk_buff *skb2; u32 msg_type, msg_len, data_offset, data_len; + u32 overflow_check; msg_type = le32_to_cpu(hdr->msg_type); msg_len = le32_to_cpu(hdr->msg_len); @@ -514,7 +516,9 @@ int rndis_rx_fixup(struct usbnet *dev, struct sk_buff *skb) /* don't choke if we see oob, per-packet data, etc */ if (unlikely(msg_type != RNDIS_MSG_PACKET || skb->len < msg_len - || (data_offset + data_len + 8) > msg_len)) { + || (data_offset + data_len + 8) > msg_len + || check_add_overflow(data_offset, data_len, &overflow_check) + || check_add_overflow(overflow_check, 8, &overflow_check))) { dev->net->stats.rx_frame_errors++; netdev_dbg(dev->net, "bad rndis message %d/%d/%d/%d, len %d\n", le32_to_cpu(hdr->msg_type), diff --git a/drivers/net/usb/sr9700.c b/drivers/net/usb/sr9700.c index 937e6fef3ac6d..50981a28376a5 100644 --- a/drivers/net/usb/sr9700.c +++ b/drivers/net/usb/sr9700.c @@ -355,7 +355,8 @@ static int sr9700_rx_fixup(struct usbnet *dev, struct sk_buff *skb) /* ignore the CRC length */ len = (skb->data[1] | (skb->data[2] << 8)) - 4; - if (len > ETH_FRAME_LEN || len > skb->len || len < 0) + if (len > ETH_FRAME_LEN || len < 0 || + len > skb->len - SR_RX_OVERHEAD) return 0; /* the last packet of current skb */ diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 6ab84c837a332..2780b2dd073d1 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -1053,6 +1053,7 @@ static int __veth_napi_enable_range(struct net_device *dev, int start, int end) for (i = start; i < end; i++) { struct veth_rq *rq = &priv->rq[i]; + rcu_assign_pointer(rq->xdp_prog, priv->_xdp_prog); napi_enable(&rq->xdp_napi); rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi); } @@ -1087,6 +1088,7 @@ static void veth_napi_del_range(struct net_device *dev, int start, int end) rcu_assign_pointer(priv->rq[i].napi, NULL); napi_disable(&rq->xdp_napi); + rcu_assign_pointer(rq->xdp_prog, NULL); __netif_napi_del(&rq->xdp_napi); } synchronize_net(); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 3e2a5876c6c8c..bf82ef9874abb 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -3349,6 +3349,14 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) else virtqueue_disable_cb(sq->vq); + if (!use_napi && + unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) { + DEV_STATS_INC(dev, tx_dropped); + dev_kfree_skb_any(skb); + kick = !xmit_more || netif_xmit_stopped(txq); + goto kick_vq; + } + /* timestamp packet in software */ skb_tx_timestamp(skb); @@ -3381,6 +3389,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) kick = use_napi ? __netdev_tx_sent_queue(txq, skb->len, xmit_more) : !xmit_more || netif_xmit_stopped(txq); +kick_vq: if (kick) { if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { u64_stats_update_begin(&sq->stats.syncp); @@ -3444,17 +3453,31 @@ static void virtnet_rx_resume_all(struct virtnet_info *vi) static int virtnet_rx_resize(struct virtnet_info *vi, struct receive_queue *rq, u32 ring_num) { + unsigned int old_ring_num = virtqueue_get_vring_size(rq->vq); + struct xdp_buff **tmp_xsk_buffs = NULL; int err, qindex; qindex = rq - vi->rq; + if (rq->xsk_pool && ring_num > old_ring_num) { + tmp_xsk_buffs = kvzalloc_objs(*tmp_xsk_buffs, ring_num); + if (!tmp_xsk_buffs) + return -ENOMEM; + } + virtnet_rx_pause(vi, rq); err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_unmap_free_buf, NULL); + + /* virtqueue_resize may have changed the size even if err != 0 */ + if (tmp_xsk_buffs && virtqueue_get_vring_size(rq->vq) > old_ring_num) + swap(rq->xsk_buffs, tmp_xsk_buffs); + if (err) netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err); virtnet_rx_resume(vi, rq, true); + kvfree(tmp_xsk_buffs); return err; } diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index 46209917ae4d6..bae0b69cf894c 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -1175,8 +1175,6 @@ static int vrf_prepare_mac_header(struct sk_buff *skb, skb->protocol = eth->h_proto; skb->pkt_type = PACKET_HOST; - skb_postpush_rcsum(skb, skb->data, ETH_HLEN); - skb_pull_inline(skb, ETH_HLEN); return 0; diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index 824144bb7774f..c4bcaadba471d 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -996,6 +996,12 @@ static int vxlan_fdb_update_existing(struct vxlan_dev *vxlan, return -EOPNOTSUPP; } + if (rcu_access_pointer(f->nh) && + !(state & (NUD_PERMANENT | NUD_NOARP))) { + NL_SET_ERR_MSG(extack, "Cannot make a nexthop fdb dynamic"); + return -EOPNOTSUPP; + } + /* Do not allow an externally learned entry to take over an entry added * by the user. */ @@ -1257,6 +1263,11 @@ static int vxlan_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], if (err) return err; + if (nhid && !(ndm->ndm_state & (NUD_PERMANENT | NUD_NOARP))) { + NL_SET_ERR_MSG(extack, "A nexthop fdb cannot be dynamic"); + return -EINVAL; + } + if (vxlan->default_dst.remote_ip.sa.sa_family != ip.sa.sa_family) return -EAFNOSUPPORT; @@ -1881,6 +1892,7 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) if (n) { struct vxlan_rdst *rdst = NULL; + u8 ha[ETH_ALEN] __aligned(2); struct vxlan_fdb *f; struct sk_buff *reply; @@ -1889,8 +1901,10 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) goto out; } + neigh_ha_snapshot(ha, n, n->dev); + rcu_read_lock(); - f = vxlan_find_mac_tx(vxlan, n->ha, vni); + f = vxlan_find_mac_tx(vxlan, ha, vni); if (f) rdst = first_remote_rcu(f); if (rdst && vxlan_addr_any(&rdst->remote_ip)) { @@ -1902,7 +1916,7 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) rcu_read_unlock(); reply = arp_create(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha, - n->ha, sha); + ha, sha); neigh_release(n); @@ -1935,7 +1949,8 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) #if IS_ENABLED(CONFIG_IPV6) static struct sk_buff *vxlan_na_create(struct sk_buff *request, - struct neighbour *n, bool isrouter) + struct neighbour *n, u8 *ha, + bool isrouter) { struct net_device *dev = request->dev; struct sk_buff *reply; @@ -1943,13 +1958,15 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request, struct ipv6hdr *pip6; u8 *daddr; int na_olen = 8; /* opt hdr + ETH_ALEN for target */ + int headroom; int ns_olen; int i, len; if (dev == NULL || !pskb_may_pull(request, request->len)) return NULL; - len = LL_RESERVED_SPACE(dev) + sizeof(struct ipv6hdr) + + headroom = LL_RESERVED_SPACE(dev); + len = headroom + sizeof(struct ipv6hdr) + sizeof(*na) + na_olen + dev->needed_tailroom; reply = alloc_skb(len, GFP_ATOMIC); if (reply == NULL) @@ -1957,7 +1974,7 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request, reply->protocol = htons(ETH_P_IPV6); reply->dev = dev; - skb_reserve(reply, LL_RESERVED_SPACE(request->dev)); + skb_reserve(reply, headroom); skb_push(reply, sizeof(struct ethhdr)); skb_reset_mac_header(reply); @@ -1981,7 +1998,7 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request, /* Ethernet header */ ether_addr_copy(eth_hdr(reply)->h_dest, daddr); - ether_addr_copy(eth_hdr(reply)->h_source, n->ha); + ether_addr_copy(eth_hdr(reply)->h_source, ha); eth_hdr(reply)->h_proto = htons(ETH_P_IPV6); reply->protocol = htons(ETH_P_IPV6); @@ -2010,7 +2027,7 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request, na->icmph.icmp6_override = 1; na->icmph.icmp6_solicited = 1; na->target = ns->target; - ether_addr_copy(&na->opt[2], n->ha); + ether_addr_copy(&na->opt[2], ha); na->opt[0] = ND_OPT_TARGET_LL_ADDR; na->opt[1] = na_olen >> 3; @@ -2051,6 +2068,7 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) if (n) { struct vxlan_rdst *rdst = NULL; + u8 ha[ETH_ALEN] __aligned(2); struct vxlan_fdb *f; struct sk_buff *reply; @@ -2059,7 +2077,8 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) goto out; } - f = vxlan_find_mac_tx(vxlan, n->ha, vni); + neigh_ha_snapshot(ha, n, n->dev); + f = vxlan_find_mac_tx(vxlan, ha, vni); if (f) rdst = first_remote_rcu(f); if (rdst && vxlan_addr_any(&rdst->remote_ip)) { @@ -2068,7 +2087,7 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) goto out; } - reply = vxlan_na_create(skb, n, + reply = vxlan_na_create(skb, n, ha, !!(f ? f->flags & NTF_ROUTER : 0)); neigh_release(n); @@ -2356,7 +2375,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev, struct ip_tunnel_key key; struct vxlan_dev *vxlan = netdev_priv(dev); const struct iphdr *old_iph; - struct vxlan_metadata _md; + struct vxlan_metadata _md = {}; struct vxlan_metadata *md = &_md; unsigned int pkt_len = skb->len; __be16 src_port = 0, dst_port; @@ -3058,18 +3077,19 @@ vxlan_fdb_flush_match_remotes(struct vxlan_fdb *f, struct vxlan_dev *vxlan, const struct vxlan_fdb_flush_desc *desc, bool *p_destroy_fdb) { - bool remotes_flushed = false; struct vxlan_rdst *rd, *tmp; list_for_each_entry_safe(rd, tmp, &f->remotes, list) { if (!vxlan_fdb_flush_remote_matches(desc, rd)) continue; + if (list_is_singular(&f->remotes)) { + *p_destroy_fdb = true; + return; + } + vxlan_fdb_dst_destroy(vxlan, f, rd, true); - remotes_flushed = true; } - - *p_destroy_fdb = remotes_flushed && list_empty(&f->remotes); } /* Purge the forwarding table */ diff --git a/drivers/net/vxlan/vxlan_mdb.c b/drivers/net/vxlan/vxlan_mdb.c index 9a9038ae90c18..841f42ffecb9a 100644 --- a/drivers/net/vxlan/vxlan_mdb.c +++ b/drivers/net/vxlan/vxlan_mdb.c @@ -354,6 +354,10 @@ static bool vxlan_mdb_is_valid_source(const struct nlattr *attr, __be16 proto, NL_SET_ERR_MSG_MOD(extack, "IPv4 multicast source address is not allowed"); return false; } + if (ipv4_is_zeronet(nla_get_in_addr(attr))) { + NL_SET_ERR_MSG_MOD(extack, "IPv4 all-zeros source address is not allowed"); + return false; + } break; #if IS_ENABLED(CONFIG_IPV6) case htons(ETH_P_IPV6): { @@ -368,6 +372,10 @@ static bool vxlan_mdb_is_valid_source(const struct nlattr *attr, __be16 proto, NL_SET_ERR_MSG_MOD(extack, "IPv6 multicast source address is not allowed"); return false; } + if (ipv6_addr_any(&src)) { + NL_SET_ERR_MSG_MOD(extack, "IPv6 all-zeros source address is not allowed"); + return false; + } break; } #endif @@ -1428,14 +1436,17 @@ static void vxlan_mdb_flush(struct vxlan_dev *vxlan, struct vxlan_mdb_entry *mdb_entry; struct hlist_node *tmp; - /* The removal of an entry cannot trigger the removal of another entry - * since entries are always added to the head of the list. - */ hlist_for_each_entry_safe(mdb_entry, tmp, &vxlan->mdb_list, mdb_node) { if (desc->src_vni && desc->src_vni != mdb_entry->key.vni) continue; vxlan_mdb_remotes_flush(vxlan, mdb_entry, desc); + /* The flush can remove the (S, G) entries created for the + * source list of this entry, including the one saved by + * hlist_for_each_entry_safe(), so re-read it while this entry + * is still linked. + */ + tmp = mdb_entry->mdb_node.next; /* Entry will only be removed if its remotes list is empty. */ vxlan_mdb_entry_put(vxlan, mdb_entry); } diff --git a/drivers/net/vxlan/vxlan_vnifilter.c b/drivers/net/vxlan/vxlan_vnifilter.c index 3e76f4e210944..dd94085e08865 100644 --- a/drivers/net/vxlan/vxlan_vnifilter.c +++ b/drivers/net/vxlan/vxlan_vnifilter.c @@ -462,10 +462,8 @@ static int vxlan_vnifilter_dump(struct sk_buff *skb, struct netlink_callback *cb static const struct nla_policy vni_filter_entry_policy[VXLAN_VNIFILTER_ENTRY_MAX + 1] = { [VXLAN_VNIFILTER_ENTRY_START] = { .type = NLA_U32 }, [VXLAN_VNIFILTER_ENTRY_END] = { .type = NLA_U32 }, - [VXLAN_VNIFILTER_ENTRY_GROUP] = { .type = NLA_BINARY, - .len = sizeof_field(struct iphdr, daddr) }, - [VXLAN_VNIFILTER_ENTRY_GROUP6] = { .type = NLA_BINARY, - .len = sizeof(struct in6_addr) }, + [VXLAN_VNIFILTER_ENTRY_GROUP] = NLA_POLICY_EXACT_LEN(sizeof_field(struct iphdr, daddr)), + [VXLAN_VNIFILTER_ENTRY_GROUP6] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), }; static const struct nla_policy vni_filter_policy[VXLAN_VNIFILTER_MAX + 1] = { diff --git a/drivers/net/wireless/ath/ath10k/snoc.c b/drivers/net/wireless/ath/ath10k/snoc.c index 3106502275781..33c98927e8feb 100644 --- a/drivers/net/wireless/ath/ath10k/snoc.c +++ b/drivers/net/wireless/ath/ath10k/snoc.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -1475,11 +1476,15 @@ static void ath10k_msa_dump_memory(struct ath10k *ar, hdr->length = cpu_to_le32(ar->msa.mem_size); if (current_region->len < ar->msa.mem_size) { - memcpy(buf, ar->msa.vaddr, current_region->len); + memcpy_fromio(buf, + (const void __iomem __force *)ar->msa.vaddr, + current_region->len); ath10k_warn(ar, "msa dump length is less than msa size %x, %x\n", current_region->len, ar->msa.mem_size); } else { - memcpy(buf, ar->msa.vaddr, ar->msa.mem_size); + memcpy_fromio(buf, + (const void __iomem __force *)ar->msa.vaddr, + ar->msa.mem_size); } } diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c index 8e2abc7b8383a..33425707c0842 100644 --- a/drivers/net/wireless/ath/ath11k/dp_rx.c +++ b/drivers/net/wireless/ath/ath11k/dp_rx.c @@ -2334,10 +2334,10 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc, case RX_MSDU_START_PKT_TYPE_11N: rx_status->encoding = RX_ENC_HT; if (rate_mcs > ATH11K_HT_MCS_MAX) { - ath11k_warn(ar->ab, - "Received with invalid mcs in HT mode %d\n", - rate_mcs); - break; + ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX, + "Received HT frame with out-of-range mcs %d, capping to %d\n", + rate_mcs, ATH11K_HT_MCS_MAX); + rate_mcs = ATH11K_HT_MCS_MAX; } rx_status->rate_idx = rate_mcs + (8 * (nss - 1)); if (sgi) @@ -2346,13 +2346,13 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc, break; case RX_MSDU_START_PKT_TYPE_11AC: rx_status->encoding = RX_ENC_VHT; - rx_status->rate_idx = rate_mcs; if (rate_mcs > ATH11K_VHT_MCS_MAX) { - ath11k_warn(ar->ab, - "Received with invalid mcs in VHT mode %d\n", - rate_mcs); - break; + ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX, + "Received VHT frame with out-of-range mcs %d, capping to %d\n", + rate_mcs, ATH11K_VHT_MCS_MAX); + rate_mcs = ATH11K_VHT_MCS_MAX; } + rx_status->rate_idx = rate_mcs; rx_status->nss = nss; if (sgi) rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI; @@ -2362,14 +2362,14 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc, rx_status->enc_flags |= RX_ENC_FLAG_LDPC; break; case RX_MSDU_START_PKT_TYPE_11AX: - rx_status->rate_idx = rate_mcs; + rx_status->encoding = RX_ENC_HE; if (rate_mcs > ATH11K_HE_MCS_MAX) { - ath11k_warn(ar->ab, - "Received with invalid mcs in HE mode %d\n", - rate_mcs); - break; + ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX, + "Received HE frame with out-of-range mcs %d, capping to %d\n", + rate_mcs, ATH11K_HE_MCS_MAX); + rate_mcs = ATH11K_HE_MCS_MAX; } - rx_status->encoding = RX_ENC_HE; + rx_status->rate_idx = rate_mcs; rx_status->nss = nss; rx_status->he_gi = ath11k_mac_he_gi_to_nl80211_he_gi(sgi); rx_status->bw = ath11k_mac_bw_to_mac80211_bw(bw); diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c index 2d55cdc4d165d..ae91b57c8422f 100644 --- a/drivers/net/wireless/ath/ath11k/mac.c +++ b/drivers/net/wireless/ath/ath11k/mac.c @@ -873,6 +873,22 @@ static int ath11k_mac_set_kickout(struct ath11k_vif *arvif) return 0; } +static void ath11k_mac_station_cleanup(struct ieee80211_sta *sta) +{ + struct ath11k_sta *arsta; + + if (!sta) + return; + + arsta = ath11k_sta_to_arsta(sta); + + kfree(arsta->tx_stats); + arsta->tx_stats = NULL; + + kfree(arsta->rx_stats); + arsta->rx_stats = NULL; +} + void ath11k_mac_peer_cleanup_all(struct ath11k *ar) { struct ath11k_peer *peer, *tmp; @@ -885,6 +901,7 @@ void ath11k_mac_peer_cleanup_all(struct ath11k *ar) list_for_each_entry_safe(peer, tmp, &ab->peers, list) { ath11k_peer_rx_tid_cleanup(ar, peer); ath11k_peer_rhash_delete(ab, peer); + ath11k_mac_station_cleanup(peer->sta); list_del(&peer->list); kfree(peer); } @@ -9892,7 +9909,6 @@ static int ath11k_mac_station_remove(struct ath11k *ar, { struct ath11k_base *ab = ar->ab; struct ath11k_vif *arvif = ath11k_vif_to_arvif(vif); - struct ath11k_sta *arsta = ath11k_sta_to_arsta(sta); int ret; if (ab->hw_params.vdev_start_delay && @@ -9916,12 +9932,7 @@ static int ath11k_mac_station_remove(struct ath11k *ar, sta->addr, arvif->vdev_id); ath11k_mac_dec_num_stations(arvif, sta); - - kfree(arsta->tx_stats); - arsta->tx_stats = NULL; - - kfree(arsta->rx_stats); - arsta->rx_stats = NULL; + ath11k_mac_station_cleanup(sta); return ret; } diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c index dca6e011cc40e..e46f4d7ddc80a 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.c +++ b/drivers/net/wireless/ath/ath11k/wmi.c @@ -159,6 +159,8 @@ static const struct wmi_tlv_policy wmi_tlv_policies[] = { .min_len = sizeof(struct ath11k_wmi_p2p_noa_info) }, [WMI_TAG_P2P_NOA_EVENT] = { .min_len = sizeof(struct wmi_p2p_noa_event) }, + [WMI_TAG_PDEV_CSA_SWITCH_COUNT_STATUS_EVENT] = { + .min_len = sizeof(struct wmi_pdev_csa_switch_ev) }, }; #define PRIMAP(_hw_mode_) \ @@ -262,6 +264,13 @@ const void **ath11k_wmi_tlv_parse_alloc(struct ath11k_base *ab, return tb; } +static u32 ath11k_wmi_tlv_data_len(const void *data) +{ + const struct wmi_tlv *tlv = (const struct wmi_tlv *)data - 1; + + return FIELD_GET(WMI_TLV_LEN, tlv->header); +} + static int ath11k_wmi_cmd_send_nowait(struct ath11k_pdev_wmi *wmi, struct sk_buff *skb, u32 cmd_id) { @@ -2423,8 +2432,8 @@ int ath11k_wmi_send_scan_start_cmd(struct ath11k *ar, for (i = 0; i < params->num_hint_bssid; ++i) { hint_bssid->freq_flags = params->hint_bssid[i].freq_flags; - ether_addr_copy(¶ms->hint_bssid[i].bssid.addr[0], - &hint_bssid->bssid.addr[0]); + ether_addr_copy(&hint_bssid->bssid.addr[0], + ¶ms->hint_bssid[i].bssid.addr[0]); hint_bssid++; } } @@ -4800,14 +4809,16 @@ static int ath11k_wmi_tlv_mac_phy_caps_parse(struct ath11k_base *soc, if (svc_rdy_ext->n_mac_phy_caps >= svc_rdy_ext->tot_phy_id) return -ENOBUFS; - len = min_t(u16, len, sizeof(struct wmi_mac_phy_capabilities)); if (!svc_rdy_ext->n_mac_phy_caps) { - svc_rdy_ext->mac_phy_caps = kcalloc(svc_rdy_ext->tot_phy_id, - len, GFP_ATOMIC); + svc_rdy_ext->mac_phy_caps = + kzalloc_objs(*svc_rdy_ext->mac_phy_caps, + svc_rdy_ext->tot_phy_id, + GFP_ATOMIC); if (!svc_rdy_ext->mac_phy_caps) return -ENOMEM; } + len = min_t(u16, len, sizeof(struct wmi_mac_phy_capabilities)); memcpy(svc_rdy_ext->mac_phy_caps + svc_rdy_ext->n_mac_phy_caps, ptr, len); svc_rdy_ext->n_mac_phy_caps++; return 0; @@ -5118,6 +5129,7 @@ static int ath11k_service_ready_ext_event(struct ath11k_base *ab, return 0; err: + kfree(svc_rdy_ext.mac_phy_caps); ath11k_wmi_free_dbring_caps(ab); return ret; } @@ -8353,15 +8365,23 @@ ath11k_wmi_process_csa_switch_count_event(struct ath11k_base *ab, const struct wmi_pdev_csa_switch_ev *ev, const u32 *vdev_ids) { - int i; + u32 vdev_ids_len = ath11k_wmi_tlv_data_len(vdev_ids); + u32 num_vdevs = ev->num_vdevs; struct ath11k_vif *arvif; + int i; /* Finish CSA once the switch count becomes NULL */ if (ev->current_switch_count) return; + if (num_vdevs > vdev_ids_len / sizeof(*vdev_ids)) { + ath11k_warn(ab, "csa switch count num_vdevs %u exceeds tlv array length %u\n", + num_vdevs, vdev_ids_len); + return; + } + rcu_read_lock(); - for (i = 0; i < ev->num_vdevs; i++) { + for (i = 0; i < num_vdevs; i++) { arvif = ath11k_mac_get_arvif_by_vdev_id(ab, vdev_ids[i]); if (!arvif) { @@ -8895,13 +8915,15 @@ static void ath11k_wmi_tlv_op_rx(struct ath11k_base *ab, struct sk_buff *skb) struct wmi_cmd_hdr *cmd_hdr; enum wmi_tlv_event_id id; + if (skb->len < sizeof(*cmd_hdr)) + goto out; + cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = FIELD_GET(WMI_CMD_HDR_CMD_ID, (cmd_hdr->cmd_id)); trace_ath11k_wmi_event(ab, id, skb->data, skb->len); - if (skb_pull(skb, sizeof(struct wmi_cmd_hdr)) == NULL) - goto out; + skb_pull(skb, sizeof(*cmd_hdr)); switch (id) { /* Process all the WMI events here */ diff --git a/drivers/net/wireless/ath/ath12k/ahb.c b/drivers/net/wireless/ath/ath12k/ahb.c index 30733a244454e..6df3b775a2148 100644 --- a/drivers/net/wireless/ath/ath12k/ahb.c +++ b/drivers/net/wireless/ath/ath12k/ahb.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "ahb.h" #include "debug.h" #include "hif.h" @@ -338,24 +339,25 @@ static int ath12k_ahb_power_up(struct ath12k_base *ab) char fw2_name[ATH12K_USERPD_FW_NAME_LEN]; struct device *dev = ab->dev; const struct firmware *fw, *fw2; - struct reserved_mem *rmem = NULL; unsigned long time_left; phys_addr_t mem_phys; + struct resource res; void *mem_region; size_t mem_size; u32 pasid; int ret; - rmem = ath12k_core_get_reserved_mem(ab, 0); - if (!rmem) - return -ENODEV; + ret = of_reserved_mem_region_to_resource_byname(dev->of_node, "q6-region", + &res); + if (ret) + return ret; - mem_phys = rmem->base; - mem_size = rmem->size; + mem_phys = res.start; + mem_size = resource_size(&res); mem_region = devm_memremap(dev, mem_phys, mem_size, MEMREMAP_WC); if (IS_ERR(mem_region)) { - ath12k_err(ab, "unable to map memory region: %pa+%pa\n", - &rmem->base, &rmem->size); + ath12k_err(ab, "unable to map memory region: %pa+%zx\n", + &res.start, mem_size); return PTR_ERR(mem_region); } diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c index e87165e4f4b33..5cdf4973d986a 100644 --- a/drivers/net/wireless/ath/ath12k/core.c +++ b/drivers/net/wireless/ath/ath12k/core.c @@ -49,7 +49,7 @@ ath12k_mem_profile_based_param ath12k_mem_profile_based_param[] = { .dp_params = { .tx_comp_ring_size = 32768, .rxdma_monitor_buf_ring_size = 4096, - .rxdma_monitor_dst_ring_size = 8092, + .rxdma_monitor_dst_ring_size = 8192, .num_pool_tx_desc = 32768, .rx_desc_count = 12288, }, @@ -637,31 +637,6 @@ u32 ath12k_core_get_max_peers_per_radio(struct ath12k_base *ab) } EXPORT_SYMBOL(ath12k_core_get_max_peers_per_radio); -struct reserved_mem *ath12k_core_get_reserved_mem(struct ath12k_base *ab, - int index) -{ - struct device *dev = ab->dev; - struct reserved_mem *rmem; - struct device_node *node; - - node = of_parse_phandle(dev->of_node, "memory-region", index); - if (!node) { - ath12k_dbg(ab, ATH12K_DBG_BOOT, - "failed to parse memory-region for index %d\n", index); - return NULL; - } - - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - if (!rmem) { - ath12k_dbg(ab, ATH12K_DBG_BOOT, - "unable to get memory-region for index %d\n", index); - return NULL; - } - - return rmem; -} - static inline void ath12k_core_to_group_ref_get(struct ath12k_base *ab) { @@ -1524,7 +1499,7 @@ static void ath12k_core_pre_reconfigure_recovery(struct ath12k_base *ab) complete_all(&ar->scan.completed); complete(&ar->scan.on_channel); complete(&ar->peer_assoc_done); - complete(&ar->peer_delete_done); + ath12k_peer_delete_wait_flush(ar); complete(&ar->install_key_done); complete(&ar->vdev_setup_done); complete(&ar->vdev_delete_done); diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h index 30726e580833e..f28d2e90b67b2 100644 --- a/drivers/net/wireless/ath/ath12k/core.h +++ b/drivers/net/wireless/ath/ath12k/core.h @@ -666,7 +666,8 @@ struct ath12k { /* protects the radio specific data like debug stats, ppdu_stats_info stats, * vdev_stop_status info, scan data, ath12k_sta info, ath12k_link_vif info, - * channel context data, survey info, test mode data, regd_channel_update_queue. + * channel context data, test mode data, regd_channel_update_queue, + * peer_delete_waits. */ spinlock_t data_lock; @@ -688,7 +689,7 @@ struct ath12k { u8 radio_idx; struct completion peer_assoc_done; - struct completion peer_delete_done; + struct list_head peer_delete_waits; int install_key_status; struct completion install_key_done; @@ -722,7 +723,6 @@ struct ath12k { * avoid reporting garbage data. */ bool ch_info_can_report_survey; - struct survey_info survey[ATH12K_NUM_CHANS]; struct completion bss_survey_done; struct work_struct regd_update_work; @@ -792,6 +792,11 @@ struct ath12k_hw { */ struct mutex hw_mutex; enum ath12k_hw_state state; + + /* protects survey[] shared across radios of this hw. */ + spinlock_t survey_lock; + struct survey_info survey[ATH12K_NUM_CHANS]; + bool regd_updated; bool use_6ghz_regd; bool host_alloc_ml_id; @@ -1297,8 +1302,6 @@ void ath12k_fw_stats_init(struct ath12k *ar); void ath12k_fw_stats_bcn_free(struct list_head *head); void ath12k_fw_stats_free(struct ath12k_fw_stats *stats); void ath12k_fw_stats_reset(struct ath12k *ar); -struct reserved_mem *ath12k_core_get_reserved_mem(struct ath12k_base *ab, - int index); enum ath12k_qmi_mem_mode ath12k_core_get_memory_mode(struct ath12k_base *ab); static inline const char *ath12k_scan_state_str(enum ath12k_scan_state state) diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.c b/drivers/net/wireless/ath/ath12k/dp_tx.c index c10da6195c9c3..9644f9ef2c749 100644 --- a/drivers/net/wireless/ath/ath12k/dp_tx.c +++ b/drivers/net/wireless/ath/ath12k/dp_tx.c @@ -82,6 +82,52 @@ enum hal_encrypt_type ath12k_dp_tx_get_encrypt_type(u32 cipher) } EXPORT_SYMBOL(ath12k_dp_tx_get_encrypt_type); +u8 ath12k_dp_tx_crypto_iv_len(enum hal_encrypt_type enc_type) +{ + switch (enc_type) { + case HAL_ENCRYPT_TYPE_TKIP_NO_MIC: + case HAL_ENCRYPT_TYPE_TKIP_MIC: + return IEEE80211_TKIP_IV_LEN; + case HAL_ENCRYPT_TYPE_CCMP_128: + return IEEE80211_CCMP_HDR_LEN; + case HAL_ENCRYPT_TYPE_CCMP_256: + return IEEE80211_CCMP_256_HDR_LEN; + case HAL_ENCRYPT_TYPE_GCMP_128: + case HAL_ENCRYPT_TYPE_AES_GCMP_256: + return IEEE80211_GCMP_HDR_LEN; + case HAL_ENCRYPT_TYPE_WEP_40: + case HAL_ENCRYPT_TYPE_WEP_104: + case HAL_ENCRYPT_TYPE_WEP_128: + return IEEE80211_WEP_IV_LEN; + default: + return 0; + } +} +EXPORT_SYMBOL(ath12k_dp_tx_crypto_iv_len); + +u8 ath12k_dp_tx_crypto_icv_len(enum hal_encrypt_type enc_type) +{ + switch (enc_type) { + case HAL_ENCRYPT_TYPE_CCMP_128: + return IEEE80211_CCMP_MIC_LEN; + case HAL_ENCRYPT_TYPE_CCMP_256: + return IEEE80211_CCMP_256_MIC_LEN; + case HAL_ENCRYPT_TYPE_GCMP_128: + case HAL_ENCRYPT_TYPE_AES_GCMP_256: + return IEEE80211_GCMP_MIC_LEN; + case HAL_ENCRYPT_TYPE_TKIP_NO_MIC: + case HAL_ENCRYPT_TYPE_TKIP_MIC: + return IEEE80211_TKIP_ICV_LEN; + case HAL_ENCRYPT_TYPE_WEP_40: + case HAL_ENCRYPT_TYPE_WEP_104: + case HAL_ENCRYPT_TYPE_WEP_128: + return IEEE80211_WEP_ICV_LEN; + default: + return 0; + } +} +EXPORT_SYMBOL(ath12k_dp_tx_crypto_icv_len); + void ath12k_dp_tx_release_txbuf(struct ath12k_dp *dp, struct ath12k_tx_desc_info *tx_desc, u8 pool_id) diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.h b/drivers/net/wireless/ath/ath12k/dp_tx.h index 7cef20540179f..1af79af2ada26 100644 --- a/drivers/net/wireless/ath/ath12k/dp_tx.h +++ b/drivers/net/wireless/ath/ath12k/dp_tx.h @@ -19,6 +19,8 @@ enum hal_tcl_encap_type ath12k_dp_tx_get_encap_type(struct ath12k_base *ab, struct sk_buff *skb); void ath12k_dp_tx_encap_nwifi(struct sk_buff *skb); u8 ath12k_dp_tx_get_tid(struct sk_buff *skb); +u8 ath12k_dp_tx_crypto_iv_len(enum hal_encrypt_type enc_type); +u8 ath12k_dp_tx_crypto_icv_len(enum hal_encrypt_type enc_type); void *ath12k_dp_metadata_align_skb(struct sk_buff *skb, u8 tail_len); int ath12k_dp_tx_align_payload(struct ath12k_dp *dp, struct sk_buff **pskb); void ath12k_dp_tx_release_txbuf(struct ath12k_dp *dp, diff --git a/drivers/net/wireless/ath/ath12k/hal.c b/drivers/net/wireless/ath/ath12k/hal.c index a164563fff289..f03817b2fbc52 100644 --- a/drivers/net/wireless/ath/ath12k/hal.c +++ b/drivers/net/wireless/ath/ath12k/hal.c @@ -828,8 +828,8 @@ void *ath12k_hal_encode_tlv64_hdr(void *tlv, u64 tag, u64 len) { struct hal_tlv_64_hdr *tlv64 = tlv; - tlv64->tl = le64_encode_bits(tag, HAL_TLV_HDR_TAG) | - le64_encode_bits(len, HAL_TLV_HDR_LEN); + tlv64->tl = le64_encode_bits(tag, HAL_TLV_64_HDR_TAG) | + le64_encode_bits(len, HAL_TLV_64_HDR_LEN); return tlv64->value; } @@ -851,7 +851,7 @@ u16 ath12k_hal_decode_tlv64_hdr(void *tlv, void **desc) struct hal_tlv_64_hdr *tlv64 = tlv; u16 tag; - tag = le64_get_bits(tlv64->tl, HAL_SRNG_TLV_HDR_TAG); + tag = le64_get_bits(tlv64->tl, HAL_TLV_64_HDR_TAG); *desc = tlv64->value; return tag; @@ -863,7 +863,7 @@ u16 ath12k_hal_decode_tlv32_hdr(void *tlv, void **desc) struct hal_tlv_hdr *tlv32 = tlv; u16 tag; - tag = le32_get_bits(tlv32->tl, HAL_SRNG_TLV_HDR_TAG); + tag = le32_get_bits(tlv32->tl, HAL_TLV_HDR_TAG); *desc = tlv32->value; return tag; diff --git a/drivers/net/wireless/ath/ath12k/hal.h b/drivers/net/wireless/ath/ath12k/hal.h index 21c551d8b2481..3ee49d93e24a0 100644 --- a/drivers/net/wireless/ath/ath12k/hal.h +++ b/drivers/net/wireless/ath/ath12k/hal.h @@ -1444,7 +1444,7 @@ struct hal_ops { }; #define HAL_TLV_HDR_TAG GENMASK(9, 1) -#define HAL_TLV_HDR_LEN GENMASK(25, 10) +#define HAL_TLV_HDR_LEN GENMASK(21, 10) #define HAL_TLV_USR_ID GENMASK(31, 26) #define HAL_TLV_ALIGN 4 @@ -1464,9 +1464,6 @@ struct hal_tlv_64_hdr { u8 value[]; } __packed; -#define HAL_SRNG_TLV_HDR_TAG GENMASK(9, 1) -#define HAL_SRNG_TLV_HDR_LEN GENMASK(25, 10) - dma_addr_t ath12k_hal_srng_get_tp_addr(struct ath12k_base *ab, struct hal_srng *srng); dma_addr_t ath12k_hal_srng_get_hp_addr(struct ath12k_base *ab, diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index a0928890671ac..b24f97786c590 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -10169,16 +10169,16 @@ static void ath12k_mac_update_vif_offload(struct ath12k_link_vif *arvif) if (vif->type != NL80211_IFTYPE_STATION && vif->type != NL80211_IFTYPE_AP) vif->offload_flags &= ~(IEEE80211_OFFLOAD_ENCAP_ENABLED | - IEEE80211_OFFLOAD_DECAP_ENABLED); + IEEE80211_OFFLOAD_DECAP_ENABLED | + IEEE80211_OFFLOAD_ENCAP_MCAST | + IEEE80211_OFFLOAD_ENCAP_4ADDR); - if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) { + if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_ETHERNET; - vif->offload_flags |= IEEE80211_OFFLOAD_ENCAP_4ADDR; - } else if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) { + else if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_RAW; - } else { + else ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_NATIVE_WIFI; - } ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, param_id, ahvif->dp_vif.tx_encap_type); @@ -10188,6 +10188,10 @@ static void ath12k_mac_update_vif_offload(struct ath12k_link_vif *arvif) vif->offload_flags &= ~IEEE80211_OFFLOAD_ENCAP_ENABLED; } + if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) + vif->offload_flags |= (IEEE80211_OFFLOAD_ENCAP_MCAST | + IEEE80211_OFFLOAD_ENCAP_4ADDR); + param_id = WMI_VDEV_PARAM_RX_DECAP_TYPE; if (vif->offload_flags & IEEE80211_OFFLOAD_DECAP_ENABLED) param_value = ATH12K_HW_TXRX_ETHERNET; @@ -10616,22 +10620,8 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif) err_peer_del: if (ahvif->vdev_type == WMI_VDEV_TYPE_AP) { - reinit_completion(&ar->peer_delete_done); - - ret = ath12k_wmi_send_peer_delete_cmd(ar, arvif->bssid, - arvif->vdev_id); - if (ret) { - ath12k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM\n", - arvif->vdev_id, arvif->bssid); - goto err_dp_peer_del; - } - - ret = ath12k_wait_for_peer_delete_done(ar, arvif->vdev_id, - arvif->bssid); - if (ret) - goto err_dp_peer_del; - - ar->num_peers--; + /* ignore return value: propagate the original error */ + ath12k_peer_delete(ar, arvif->vdev_id, arvif->bssid); } err_dp_peer_del: @@ -11305,6 +11295,8 @@ ath12k_mac_mlo_get_vdev_args(struct ath12k_link_vif *arvif, ml_arg->assoc_link = arvif->is_sta_assoc_link; + ml_arg->ieee_link_id = arvif->link_id; + partner_info = ml_arg->partner_info; links = ahvif->links_map; @@ -11328,6 +11320,7 @@ ath12k_mac_mlo_get_vdev_args(struct ath12k_link_vif *arvif, partner_info->vdev_id = arvif_p->vdev_id; partner_info->hw_link_id = arvif_p->ar->pdev->hw_link_id; + partner_info->ieee_link_id = arvif_p->link_id; ether_addr_copy(partner_info->addr, link_conf->addr); ml_arg->num_partner_links++; partner_info++; @@ -13633,52 +13626,54 @@ ath12k_mac_update_bss_chan_survey(struct ath12k *ar, int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx, struct survey_info *survey) { + struct ath12k_hw *ah = hw->priv; struct ath12k *ar; struct ieee80211_supported_band *sband; - struct survey_info *ar_survey; + struct survey_info *ah_survey; + int sband_idx = idx; lockdep_assert_wiphy(hw->wiphy); - if (idx >= ATH12K_NUM_CHANS) + if (sband_idx >= ATH12K_NUM_CHANS) return -ENOENT; sband = hw->wiphy->bands[NL80211_BAND_2GHZ]; - if (sband && idx >= sband->n_channels) { - idx -= sband->n_channels; + if (sband && sband_idx >= sband->n_channels) { + sband_idx -= sband->n_channels; sband = NULL; } if (!sband) sband = hw->wiphy->bands[NL80211_BAND_5GHZ]; - if (sband && idx >= sband->n_channels) { - idx -= sband->n_channels; + if (sband && sband_idx >= sband->n_channels) { + sband_idx -= sband->n_channels; sband = NULL; } if (!sband) sband = hw->wiphy->bands[NL80211_BAND_6GHZ]; - if (!sband || idx >= sband->n_channels) + if (!sband || sband_idx >= sband->n_channels) return -ENOENT; - ar = ath12k_mac_get_ar_by_chan(hw, &sband->channels[idx]); + ar = ath12k_mac_get_ar_by_chan(hw, &sband->channels[sband_idx]); if (!ar) { - if (sband->channels[idx].flags & IEEE80211_CHAN_DISABLED) { + if (sband->channels[sband_idx].flags & IEEE80211_CHAN_DISABLED) { memset(survey, 0, sizeof(*survey)); return 0; } return -ENOENT; } - ar_survey = &ar->survey[idx]; + ah_survey = &ah->survey[idx]; - ath12k_mac_update_bss_chan_survey(ar, &sband->channels[idx]); + ath12k_mac_update_bss_chan_survey(ar, &sband->channels[sband_idx]); - spin_lock_bh(&ar->data_lock); - memcpy(survey, ar_survey, sizeof(*survey)); - spin_unlock_bh(&ar->data_lock); + scoped_guard(spinlock_bh, &ah->survey_lock) { + memcpy(survey, ah_survey, sizeof(*survey)); + } - survey->channel = &sband->channels[idx]; + survey->channel = &sband->channels[sband_idx]; if (ar->rx_channel == survey->channel) survey->filled |= SURVEY_INFO_IN_USE; @@ -15106,11 +15101,11 @@ static void ath12k_mac_setup(struct ath12k *ar) spin_lock_init(&ar->dp.ppdu_list_lock); INIT_LIST_HEAD(&ar->arvifs); INIT_LIST_HEAD(&ar->dp.ppdu_stats_info); + INIT_LIST_HEAD(&ar->peer_delete_waits); init_completion(&ar->vdev_setup_done); init_completion(&ar->vdev_delete_done); init_completion(&ar->peer_assoc_done); - init_completion(&ar->peer_delete_done); init_completion(&ar->install_key_done); init_completion(&ar->bss_survey_done); init_completion(&ar->scan.started); @@ -15360,6 +15355,7 @@ static struct ath12k_hw *ath12k_mac_hw_allocate(struct ath12k_hw_group *ag, mutex_init(&ah->hw_mutex); init_completion(&ah->peer_ml_id_done); + spin_lock_init(&ah->survey_lock); spin_lock_init(&ah->dp_hw.peer_lock); INIT_LIST_HEAD(&ah->dp_hw.dp_peers_list); diff --git a/drivers/net/wireless/ath/ath12k/peer.c b/drivers/net/wireless/ath/ath12k/peer.c index ed0524ddff803..80edebf0e364b 100644 --- a/drivers/net/wireless/ath/ath12k/peer.c +++ b/drivers/net/wireless/ath/ath12k/peer.c @@ -9,6 +9,55 @@ #include "debug.h" #include "debugfs.h" +static void ath12k_peer_delete_wait_register(struct ath12k *ar, + struct ath12k_peer_delete_wait *wait, + u32 vdev_id, const u8 *addr) +{ + wait->vdev_id = vdev_id; + ether_addr_copy(wait->addr, addr); + init_completion(&wait->done); + + spin_lock_bh(&ar->data_lock); + list_add(&wait->list, &ar->peer_delete_waits); + spin_unlock_bh(&ar->data_lock); +} + +static void ath12k_peer_delete_wait_unregister(struct ath12k *ar, + struct ath12k_peer_delete_wait *wait) +{ + spin_lock_bh(&ar->data_lock); + list_del(&wait->list); + spin_unlock_bh(&ar->data_lock); +} + +void ath12k_peer_delete_resp_signal(struct ath12k *ar, u32 vdev_id, const u8 *addr) +{ + struct ath12k_peer_delete_wait *wait; + + guard(spinlock_bh)(&ar->data_lock); + + list_for_each_entry(wait, &ar->peer_delete_waits, list) { + if (wait->vdev_id == vdev_id && + ether_addr_equal(wait->addr, addr)) { + complete(&wait->done); + return; + } + } + + ath12k_warn(ar->ab, "failed to find link peer with vdev id %u addr %pM\n", + vdev_id, addr); +} + +void ath12k_peer_delete_wait_flush(struct ath12k *ar) +{ + struct ath12k_peer_delete_wait *wait; + + spin_lock_bh(&ar->data_lock); + list_for_each_entry(wait, &ar->peer_delete_waits, list) + complete(&wait->done); + spin_unlock_bh(&ar->data_lock); +} + static int ath12k_wait_for_dp_link_peer_common(struct ath12k_base *ab, int vdev_id, const u8 *addr, bool expect_mapped) { @@ -62,20 +111,19 @@ static int ath12k_wait_for_peer_deleted(struct ath12k *ar, int vdev_id, const u8 return ath12k_wait_for_dp_link_peer_common(ar->ab, vdev_id, addr, false); } -int ath12k_wait_for_peer_delete_done(struct ath12k *ar, u32 vdev_id, - const u8 *addr) +int ath12k_wait_for_peer_delete_done(struct ath12k *ar, + struct ath12k_peer_delete_wait *wait) { - int ret; unsigned long time_left; + int ret; - ret = ath12k_wait_for_peer_deleted(ar, vdev_id, addr); + ret = ath12k_wait_for_peer_deleted(ar, wait->vdev_id, wait->addr); if (ret) { - ath12k_warn(ar->ab, "failed wait for peer deleted"); + ath12k_warn(ar->ab, "failed wait for peer deleted\n"); return ret; } - time_left = wait_for_completion_timeout(&ar->peer_delete_done, - 3 * HZ); + time_left = wait_for_completion_timeout(&wait->done, 3 * HZ); if (time_left == 0) { ath12k_warn(ar->ab, "Timeout in receiving peer delete response\n"); return -ETIMEDOUT; @@ -91,8 +139,6 @@ static int ath12k_peer_delete_send(struct ath12k *ar, u32 vdev_id, const u8 *add lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); - reinit_completion(&ar->peer_delete_done); - ret = ath12k_wmi_send_peer_delete_cmd(ar, addr, vdev_id); if (ret) { ath12k_warn(ab, @@ -106,6 +152,7 @@ static int ath12k_peer_delete_send(struct ath12k *ar, u32 vdev_id, const u8 *add int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr) { + struct ath12k_peer_delete_wait wait; int ret; lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); @@ -114,17 +161,25 @@ int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr) &(ath12k_ar_to_ah(ar)->dp_hw), vdev_id, addr, ar->hw_link_id); + /* + * Register the stack waiter before sending so the resp_event for + * this peer cannot arrive while no waiter is queued. + */ + ath12k_peer_delete_wait_register(ar, &wait, vdev_id, addr); + ret = ath12k_peer_delete_send(ar, vdev_id, addr); if (ret) - return ret; + goto out; - ret = ath12k_wait_for_peer_delete_done(ar, vdev_id, addr); + ret = ath12k_wait_for_peer_delete_done(ar, &wait); if (ret) - return ret; + goto out; ar->num_peers--; - return 0; +out: + ath12k_peer_delete_wait_unregister(ar, &wait); + return ret; } static int ath12k_wait_for_peer_created(struct ath12k *ar, int vdev_id, const u8 *addr) @@ -184,22 +239,26 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arg->vdev_id, arg->peer_addr); if (!peer) { + struct ath12k_peer_delete_wait wait; + spin_unlock_bh(&dp->dp_lock); ath12k_warn(ar->ab, "failed to find peer %pM on vdev %i after creation\n", arg->peer_addr, arg->vdev_id); - reinit_completion(&ar->peer_delete_done); + ath12k_peer_delete_wait_register(ar, &wait, arg->vdev_id, + arg->peer_addr); ret = ath12k_wmi_send_peer_delete_cmd(ar, arg->peer_addr, arg->vdev_id); if (ret) { ath12k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM\n", arg->vdev_id, arg->peer_addr); + ath12k_peer_delete_wait_unregister(ar, &wait); return ret; } - ret = ath12k_wait_for_peer_delete_done(ar, arg->vdev_id, - arg->peer_addr); + ret = ath12k_wait_for_peer_delete_done(ar, &wait); + ath12k_peer_delete_wait_unregister(ar, &wait); if (ret) return ret; @@ -308,13 +367,14 @@ void ath12k_peer_ml_free(struct ath12k_hw *ah, struct ath12k_sta *ahsta) int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_sta *ahsta) { + DECLARE_BITMAP(registered, IEEE80211_MLD_MAX_NUM_LINKS); struct ieee80211_sta *sta = ath12k_ahsta_to_sta(ahsta); struct ath12k_hw *ah = ahvif->ah; struct ath12k_link_vif *arvif; struct ath12k_link_sta *arsta; + int ret, err_ret = 0; unsigned long links; struct ath12k *ar; - int ret, err_ret = 0; u8 link_id; lockdep_assert_wiphy(ah->hw->wiphy); @@ -322,8 +382,19 @@ int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_st if (!sta->mlo) return -EINVAL; - /* FW expects delete of all link peers at once before waiting for reception - * of peer unmap or delete responses + struct ath12k_peer_delete_wait *waits __free(kfree) = + kzalloc_objs(*waits, IEEE80211_MLD_MAX_NUM_LINKS); + if (!waits) + return -ENOMEM; + + bitmap_zero(registered, IEEE80211_MLD_MAX_NUM_LINKS); + + /* + * Firmware expects delete of all link peers at once before waiting + * for reception of peer unmap or delete responses. Phase 1 registers + * a per-link stack waiter and sends WMI peer delete for every + * link; the resp_event handler matches each response to its + * (vdev_id, addr) waiter on ar->peer_delete_waits. */ links = ahsta->links_map; for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) { @@ -343,29 +414,36 @@ int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_st arvif->vdev_id, arsta->addr, ar->hw_link_id); + ath12k_peer_delete_wait_register(ar, &waits[link_id], + arvif->vdev_id, arsta->addr); + ret = ath12k_peer_delete_send(ar, arvif->vdev_id, arsta->addr); if (ret) { ath12k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM ret %d\n", arvif->vdev_id, arsta->addr, ret); err_ret = ret; + ath12k_peer_delete_wait_unregister(ar, &waits[link_id]); continue; } + + set_bit(link_id, registered); } - /* Ensure all link peers are deleted and unmapped */ + /* + * Phase 2: wait for unmap + delete_resp on each registered link + * and tear down the waiter. + */ links = ahsta->links_map; for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) { - arvif = wiphy_dereference(ah->hw->wiphy, ahvif->link[link_id]); - arsta = wiphy_dereference(ah->hw->wiphy, ahsta->link[link_id]); - if (!arvif || !arsta) + if (!test_bit(link_id, registered)) continue; + arvif = wiphy_dereference(ah->hw->wiphy, ahvif->link[link_id]); ar = arvif->ar; - if (!ar) - continue; - ret = ath12k_wait_for_peer_delete_done(ar, arvif->vdev_id, arsta->addr); + ret = ath12k_wait_for_peer_delete_done(ar, &waits[link_id]); + ath12k_peer_delete_wait_unregister(ar, &waits[link_id]); if (ret) { err_ret = ret; continue; diff --git a/drivers/net/wireless/ath/ath12k/peer.h b/drivers/net/wireless/ath/ath12k/peer.h index 0f7f25b8e89c0..3f4ac17b9aa65 100644 --- a/drivers/net/wireless/ath/ath12k/peer.h +++ b/drivers/net/wireless/ath/ath12k/peer.h @@ -9,13 +9,23 @@ #include "dp_peer.h" +struct ath12k_peer_delete_wait { + struct list_head list; + u32 vdev_id; + u8 addr[ETH_ALEN]; + struct completion done; +}; + +void ath12k_peer_delete_resp_signal(struct ath12k *ar, u32 vdev_id, const u8 *addr); +void ath12k_peer_delete_wait_flush(struct ath12k *ar); + void ath12k_peer_cleanup(struct ath12k *ar, u32 vdev_id); int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr); int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif, struct ieee80211_sta *sta, struct ath12k_wmi_peer_create_arg *arg); -int ath12k_wait_for_peer_delete_done(struct ath12k *ar, u32 vdev_id, - const u8 *addr); +int ath12k_wait_for_peer_delete_done(struct ath12k *ar, + struct ath12k_peer_delete_wait *wait); int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_sta *ahsta); struct ath12k_ml_peer *ath12k_peer_ml_find(struct ath12k_hw *ah, const u8 *addr); diff --git a/drivers/net/wireless/ath/ath12k/qmi.c b/drivers/net/wireless/ath/ath12k/qmi.c index fd762b5d7bb59..6d50632292214 100644 --- a/drivers/net/wireless/ath/ath12k/qmi.c +++ b/drivers/net/wireless/ath/ath12k/qmi.c @@ -13,6 +13,7 @@ #include #include #include +#include #define SLEEP_CLOCK_SELECT_INTERNAL_BIT 0x02 #define HOST_CSTATE_BIT 0x04 @@ -2725,121 +2726,96 @@ static int ath12k_qmi_alloc_target_mem_chunk(struct ath12k_base *ab) return ret; } +static const char *ath12k_qmi_get_mem_reg_name(int mem_type) +{ + switch (mem_type) { + case HOST_DDR_REGION_TYPE: + case BDF_MEM_REGION_TYPE: + return "q6-region"; + case M3_DUMP_REGION_TYPE: + return "m3-dump"; + case CALDB_MEM_REGION_TYPE: + return "q6-caldb"; + case MLO_GLOBAL_MEM_REGION_TYPE: + return "mlo-global-mem"; + default: + return NULL; + } +} + static int ath12k_qmi_assign_target_mem_chunk(struct ath12k_base *ab) { - struct reserved_mem *rmem; - size_t avail_rmem_size; + struct device_node *np = ab->dev->of_node; + size_t avail_rmem_size, offset = 0; + struct target_mem_chunk *chunk; + struct resource res; + const char *rname; int i, idx, ret; for (i = 0, idx = 0; i < ab->qmi.mem_seg_count; i++) { - switch (ab->qmi.target_mem[i].type) { - case HOST_DDR_REGION_TYPE: - rmem = ath12k_core_get_reserved_mem(ab, 0); - if (!rmem) { - ret = -ENODEV; - goto out; - } - - avail_rmem_size = rmem->size; - if (avail_rmem_size < ab->qmi.target_mem[i].size) { - ath12k_dbg(ab, ATH12K_DBG_QMI, - "failed to assign mem type %u req size %u avail size %zu\n", - ab->qmi.target_mem[i].type, - ab->qmi.target_mem[i].size, - avail_rmem_size); - ret = -EINVAL; - goto out; - } - - ab->qmi.target_mem[idx].paddr = rmem->base; - ab->qmi.target_mem[idx].v.ioaddr = - ioremap(ab->qmi.target_mem[idx].paddr, - ab->qmi.target_mem[i].size); - if (!ab->qmi.target_mem[idx].v.ioaddr) { - ret = -EIO; - goto out; - } - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; - idx++; - break; - case BDF_MEM_REGION_TYPE: - rmem = ath12k_core_get_reserved_mem(ab, 0); - if (!rmem) { - ret = -ENODEV; - goto out; - } - - avail_rmem_size = rmem->size - ab->hw_params->bdf_addr_offset; - if (avail_rmem_size < ab->qmi.target_mem[i].size) { - ath12k_dbg(ab, ATH12K_DBG_QMI, - "failed to assign mem type %u req size %u avail size %zu\n", - ab->qmi.target_mem[i].type, - ab->qmi.target_mem[i].size, - avail_rmem_size); - ret = -EINVAL; - goto out; - } - ab->qmi.target_mem[idx].paddr = - rmem->base + ab->hw_params->bdf_addr_offset; - ab->qmi.target_mem[idx].v.ioaddr = - ioremap(ab->qmi.target_mem[idx].paddr, - ab->qmi.target_mem[i].size); - if (!ab->qmi.target_mem[idx].v.ioaddr) { - ret = -EIO; - goto out; - } - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; - idx++; - break; - case CALDB_MEM_REGION_TYPE: - /* Cold boot calibration is not enabled in Ath12k. Hence, + chunk = &ab->qmi.target_mem[i]; + if (chunk->type == CALDB_MEM_REGION_TYPE) { + /* + * Cold boot calibration is not enabled in Ath12k. Hence, * assign paddr = 0. * Once cold boot calibration is enabled add support to * assign reserved memory from DT. */ ab->qmi.target_mem[idx].paddr = 0; ab->qmi.target_mem[idx].v.ioaddr = NULL; - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; + ab->qmi.target_mem[idx].size = chunk->size; + ab->qmi.target_mem[idx].type = chunk->type; idx++; - break; - case M3_DUMP_REGION_TYPE: - rmem = ath12k_core_get_reserved_mem(ab, 1); - if (!rmem) { - ret = -EINVAL; - goto out; - } + continue; + } - avail_rmem_size = rmem->size; - if (avail_rmem_size < ab->qmi.target_mem[i].size) { - ath12k_dbg(ab, ATH12K_DBG_QMI, - "failed to assign mem type %u req size %u avail size %zu\n", - ab->qmi.target_mem[i].type, - ab->qmi.target_mem[i].size, + rname = ath12k_qmi_get_mem_reg_name(chunk->type); + if (!rname) { + ath12k_warn(ab, "qmi ignore invalid mem req type %u\n", + chunk->type); + continue; + } + + ret = of_reserved_mem_region_to_resource_byname(np, rname, &res); + if (ret) + goto out; + + avail_rmem_size = resource_size(&res); + if (chunk->type == BDF_MEM_REGION_TYPE || + chunk->type == HOST_DDR_REGION_TYPE) { + if (ab->hw_params->bdf_addr_offset > avail_rmem_size || + offset > avail_rmem_size - ab->hw_params->bdf_addr_offset) { + ath12k_err(ab, "qmi mem offset overflow: bdf_offset=%u offset=%zu size=%zu\n", + ab->hw_params->bdf_addr_offset, offset, avail_rmem_size); ret = -EINVAL; goto out; } - ab->qmi.target_mem[idx].paddr = rmem->base; - ab->qmi.target_mem[idx].v.ioaddr = - ioremap(ab->qmi.target_mem[idx].paddr, - ab->qmi.target_mem[i].size); - if (!ab->qmi.target_mem[idx].v.ioaddr) { - ret = -EIO; - goto out; - } - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; - idx++; - break; - default: - ath12k_warn(ab, "qmi ignore invalid mem req type %u\n", - ab->qmi.target_mem[i].type); - break; + avail_rmem_size -= ab->hw_params->bdf_addr_offset + offset; + res.start += ab->hw_params->bdf_addr_offset + offset; + offset += chunk->size; } + + if (avail_rmem_size < chunk->size) { + ath12k_dbg(ab, ATH12K_DBG_QMI, + "failed to assign mem type %u req size %u avail size %zu\n", + chunk->type, chunk->size, avail_rmem_size); + ret = -EINVAL; + goto out; + } + + ab->qmi.target_mem[idx].paddr = res.start; + ab->qmi.target_mem[idx].v.ioaddr = ioremap(ab->qmi.target_mem[idx].paddr, + chunk->size); + if (!ab->qmi.target_mem[idx].v.ioaddr) { + ret = -EIO; + goto out; + } + + ab->qmi.target_mem[idx].size = chunk->size; + ab->qmi.target_mem[idx].type = chunk->type; + idx++; } ab->qmi.mem_seg_count = idx; diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c index d2749de445534..587d58eeccfa5 100644 --- a/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c @@ -13,6 +13,49 @@ #include "hal.h" #include "hal_tx.h" +/* + * Convert an encrypted EAPOL frame from native-WiFi format to + * the layout expected by the firmware RAW encrypt pipeline: + * + * [802.11 hdr][IV (zeroed)][LLC/SNAP][EAPOL payload][ICV (zeroed)] + * + * mac80211 delivers the frame as [802.11 hdr][LLC/SNAP][EAPOL payload]. + * The MAC header length is read from the unmodified skb and is safe because + * ieee80211_hdrlen() only inspects the 2-byte frame_control field. + * pskb_expand_head() is used to grow both head (for the IV) and tail + * (for the ICV) in a single call and allocation. + */ +static int +ath12k_wifi7_dp_tx_encap_eapol(struct sk_buff *skb, + struct hal_tx_info *ti, + struct ath12k_skb_cb *skb_cb) +{ + struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; + enum hal_encrypt_type enc_type = + ath12k_dp_tx_get_encrypt_type(skb_cb->cipher); + u16 mac_hdr_len = ieee80211_hdrlen(hdr->frame_control); + u8 iv_len = ath12k_dp_tx_crypto_iv_len(enc_type); + u8 icv_len = ath12k_dp_tx_crypto_icv_len(enc_type); + + if (pskb_expand_head(skb, iv_len, icv_len, GFP_ATOMIC)) + return -ENOMEM; + + if (iv_len) { + skb_push(skb, iv_len); + memmove(skb->data, skb->data + iv_len, mac_hdr_len); + memset(skb->data + mac_hdr_len, 0, iv_len); + } + + if (icv_len) + memset(skb_put(skb, icv_len), 0, icv_len); + + ti->flags0 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TO_FW); + ti->encap_type = HAL_TCL_ENCAP_TYPE_RAW; + ti->encrypt_type = enc_type; + + return 0; +} + static void ath12k_wifi7_hal_tx_cmd_ext_desc_setup(struct ath12k_base *ab, struct hal_tx_msdu_ext_desc *tcl_ext_cmd, @@ -91,6 +134,7 @@ int ath12k_wifi7_dp_tx(struct ath12k_pdev_dp *dp_pdev, struct ath12k_link_vif *a u32 iova_mask = dp->hw_params->iova_mask; bool is_diff_encap = false; bool is_null_frame = false; + bool eapol_encap_done = false; if (test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags)) return -ESHUTDOWN; @@ -211,9 +255,27 @@ int ath12k_wifi7_dp_tx(struct ath12k_pdev_dp *dp_pdev, struct ath12k_link_vif *a case HAL_TCL_ENCAP_TYPE_NATIVE_WIFI: is_null_frame = ieee80211_is_nullfunc(hdr->frame_control); if (ahvif->vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) { - if (skb->protocol == cpu_to_be16(ETH_P_PAE) || is_null_frame) + if ((skb->protocol == cpu_to_be16(ETH_P_PAE) && + !(skb_cb->flags & ATH12K_SKB_CIPHER_SET)) || is_null_frame) is_diff_encap = true; + if (skb->protocol == cpu_to_be16(ETH_P_PAE) && + (skb_cb->flags & ATH12K_SKB_CIPHER_SET)) { + if (!eapol_encap_done) { + ret = ath12k_wifi7_dp_tx_encap_eapol(skb, &ti, + skb_cb); + if (ret) + goto fail_remove_tx_buf; + hdr = (void *)skb->data; + eapol_encap_done = true; + } else { + ti.flags0 |= u32_encode_bits(1, + HAL_TCL_DATA_CMD_INFO2_TO_FW); + ti.encap_type = HAL_TCL_ENCAP_TYPE_RAW; + ti.encrypt_type = + ath12k_dp_tx_get_encrypt_type(skb_cb->cipher); + } + } /* Firmware expects msdu ext descriptor for nwifi/raw packets * received in ETH mode. Without this, observed tx fail for * Multicast packets in ETH mode. diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h index 0d19a9cbb68ce..6d69851e529d8 100644 --- a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h +++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h @@ -140,6 +140,38 @@ struct rx_mpdu_start_qcn9274 { __le32 res1; } __packed; +struct rx_mpdu_start_qcc2072 { + __le32 info0; + __le32 info2; + __le32 reo_queue_desc_lo; + __le32 info1; + __le32 pn[4]; + __le32 info4; + __le32 peer_meta_data; + __le16 ast_index; + __le16 sw_peer_id; + __le16 info3; + __le16 phy_ppdu_id; + __le32 info5; + __le32 info6; + __le16 frame_ctrl; + __le16 duration; + u8 addr1[ETH_ALEN]; + u8 addr2[ETH_ALEN]; + u8 addr3[ETH_ALEN]; + __le16 seq_ctrl; + u8 addr4[ETH_ALEN]; + __le16 qos_ctrl; + __le32 ht_ctrl; + __le32 info7; + __le32 res0; + __le32 res1; + __le32 res2; + __le32 info8; + __le32 res3; + __le32 res4; +} __packed; + #define QCN9274_MPDU_START_SELECT_MPDU_START_TAG BIT(0) #define QCN9274_MPDU_START_SELECT_INFO0_REO_QUEUE_DESC_LO BIT(1) #define QCN9274_MPDU_START_SELECT_INFO1_PN_31_0 BIT(2) @@ -1492,7 +1524,7 @@ struct hal_rx_desc_qcc2072 { struct rx_msdu_end_qcn9274 msdu_end; u8 rx_padding0[RX_BE_PADDING0_BYTES]; __le32 mpdu_start_tag; - struct rx_mpdu_start_qcn9274 mpdu_start; + struct rx_mpdu_start_qcc2072 mpdu_start; struct rx_pkt_hdr_tlv_qcc2072 pkt_hdr_tlv; u8 msdu_payload[]; }; diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hw.c b/drivers/net/wireless/ath/ath12k/wifi7/hw.c index 03dedfd907fc7..855bdfca34b11 100644 --- a/drivers/net/wireless/ath/ath12k/wifi7/hw.c +++ b/drivers/net/wireless/ath/ath12k/wifi7/hw.c @@ -918,6 +918,7 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, struct ethhdr *eth; bool is_prb_rsp; u16 mcbc_gsn; + u8 cb_flags; u8 link_id; int ret; struct ath12k_dp *tmp_dp; @@ -1011,8 +1012,13 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, ieee80211_has_protected(hdr->frame_control)) is_dvlan = true; + /* + * Add a sta pointer check to differentiate multicast encapsulation + * offload packets, as the ATH12K_SKB_HW_80211_ENCAP flag is also set + * for such packets. + */ if (!vif->valid_links || !is_mcast || is_dvlan || - (skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) || + ((skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) && sta) || test_bit(ATH12K_FLAG_RAW_MODE, &ar->ab->dev_flags)) { ret = ath12k_wifi7_dp_tx(dp_pdev, arvif, arsta, skb, false, 0, is_mcast); if (unlikely(ret)) { @@ -1024,6 +1030,7 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, mcbc_gsn = atomic_inc_return(&ahvif->dp_vif.mcbc_gsn) & 0xfff; links_map = ahvif->links_map; + cb_flags = skb_cb->flags; for_each_set_bit(link_id, &links_map, IEEE80211_MLD_MAX_NUM_LINKS) { tmp_arvif = rcu_dereference(ahvif->link[link_id]); @@ -1031,21 +1038,45 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, continue; tmp_ar = tmp_arvif->ar; - tmp_dp_pdev = ath12k_dp_to_pdev_dp(tmp_ar->ab->dp, + tmp_dp = ath12k_ab_to_dp(tmp_ar->ab); + tmp_dp_pdev = ath12k_dp_to_pdev_dp(tmp_dp, tmp_ar->pdev_idx); if (!tmp_dp_pdev) continue; - msdu_copied = skb_copy(skb, GFP_ATOMIC); - if (!msdu_copied) { - ath12k_err(ar->ab, - "skb copy failure link_id 0x%X vdevid 0x%X\n", - link_id, tmp_arvif->vdev_id); - continue; - } - ath12k_mlo_mcast_update_tx_link_address(vif, link_id, - msdu_copied, - info_flags); + if (cb_flags & ATH12K_SKB_HW_80211_ENCAP) { + /* + * skb->data may be modified for the iova_mask devices. + * It is better to use skb_copy() for such devices + * to avoid any potential skb corruption related issues. + */ + if (tmp_dp->hw_params->iova_mask) + msdu_copied = skb_copy(skb, GFP_ATOMIC); + else + /* + * ath12k_wifi7_dp_tx() should treat cloned HW-encap + * Ethernet multicast frames as read-only. + */ + msdu_copied = skb_clone(skb, GFP_ATOMIC); + if (!msdu_copied) { + ath12k_err(ar->ab, + "skb copy/clone failure link_id 0x%X vdevid 0x%X\n", + link_id, tmp_arvif->vdev_id); + continue; + } + } else { + msdu_copied = skb_copy(skb, GFP_ATOMIC); + if (!msdu_copied) { + ath12k_err(ar->ab, + "skb copy failure link_id 0x%X vdevid 0x%X\n", + link_id, tmp_arvif->vdev_id); + continue; + } + + ath12k_mlo_mcast_update_tx_link_address(vif, link_id, + msdu_copied, + info_flags); + } skb_cb = ATH12K_SKB_CB(msdu_copied); skb_cb->link_id = link_id; @@ -1061,7 +1092,6 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, if (unlikely(!ahvif->dp_vif.key_cipher)) goto skip_peer_find; - tmp_dp = ath12k_ab_to_dp(tmp_ar->ab); spin_lock_bh(&tmp_dp->dp_lock); peer = ath12k_dp_link_peer_find_by_addr(tmp_dp, tmp_arvif->bssid); @@ -1080,11 +1110,16 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw, skb_cb->cipher = key->cipher; skb_cb->flags |= ATH12K_SKB_CIPHER_SET; + if (skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) + goto skip_fctl_protected_check; + hdr = (struct ieee80211_hdr *)msdu_copied->data; if (!ieee80211_has_protected(hdr->frame_control)) hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PROTECTED); } + +skip_fctl_protected_check: spin_unlock_bh(&tmp_dp->dp_lock); skip_peer_find: diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c index 84a31b953db81..90e1549ec43d7 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.c +++ b/drivers/net/wireless/ath/ath12k/wmi.c @@ -207,6 +207,8 @@ static const struct ath12k_wmi_tlv_policy ath12k_wmi_tlv_policies[] = { .min_len = sizeof(struct wmi_per_chain_rssi_stat_params) }, [WMI_TAG_OBSS_COLOR_COLLISION_EVT] = { .min_len = sizeof(struct wmi_obss_color_collision_event) }, + [WMI_TAG_PDEV_CSA_SWITCH_COUNT_STATUS_EVENT] = { + .min_len = sizeof(struct ath12k_wmi_pdev_csa_event) }, }; __le32 ath12k_wmi_tlv_hdr(u32 cmd, u32 len) @@ -374,6 +376,13 @@ ath12k_wmi_tlv_parse(struct ath12k_base *ab, struct sk_buff *skb) return tb; } +static u32 ath12k_wmi_tlv_data_len(const void *data) +{ + const struct wmi_tlv *tlv = (const struct wmi_tlv *)data - 1; + + return le32_get_bits(tlv->header, WMI_TLV_LEN); +} + static int ath12k_wmi_cmd_send_nowait(struct ath12k_wmi_pdev *wmi, struct sk_buff *skb, u32 cmd_id) { @@ -1228,10 +1237,16 @@ int ath12k_wmi_vdev_start(struct ath12k *ar, struct wmi_vdev_start_req_arg *arg, le32_encode_bits(arg->ml.mcast_link, ATH12K_WMI_FLAG_MLO_MCAST_VDEV) | le32_encode_bits(arg->ml.link_add, - ATH12K_WMI_FLAG_MLO_LINK_ADD); + ATH12K_WMI_FLAG_MLO_LINK_ADD) | + le32_encode_bits(arg->ml.assoc_link, + ATH12K_WMI_FLAG_MLO_START_AS_ACTIVE) | + cpu_to_le32(ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID); - ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "vdev %d start ml flags 0x%x\n", - arg->vdev_id, ml_params->flags); + ml_params->ieee_link_id = cpu_to_le32(arg->ml.ieee_link_id); + + ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "vdev %u start link_id %u ml flags 0x%x\n", + arg->vdev_id, arg->ml.ieee_link_id, + le32_to_cpu(ml_params->flags)); ptr += sizeof(*ml_params); @@ -1244,19 +1259,23 @@ int ath12k_wmi_vdev_start(struct ath12k *ar, struct wmi_vdev_start_req_arg *arg, partner_info = ptr; for (i = 0; i < arg->ml.num_partner_links; i++) { + struct wmi_ml_partner_info *pinfo = &arg->ml.partner_info[i]; + partner_info->tlv_header = ath12k_wmi_tlv_cmd_hdr(WMI_TAG_MLO_PARTNER_LINK_PARAMS, sizeof(*partner_info)); - partner_info->vdev_id = - cpu_to_le32(arg->ml.partner_info[i].vdev_id); - partner_info->hw_link_id = - cpu_to_le32(arg->ml.partner_info[i].hw_link_id); + partner_info->vdev_id = cpu_to_le32(pinfo->vdev_id); + partner_info->hw_link_id = cpu_to_le32(pinfo->hw_link_id); ether_addr_copy(partner_info->vdev_addr.addr, - arg->ml.partner_info[i].addr); + pinfo->addr); + partner_info->flags = + cpu_to_le32(ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID_PARTNER); + partner_info->ieee_link_id = cpu_to_le32(pinfo->ieee_link_id); - ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "partner vdev %d hw_link_id %d macaddr%pM\n", - partner_info->vdev_id, partner_info->hw_link_id, - partner_info->vdev_addr.addr); + ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "partner vdev %u hw_link_id %u macaddr %pM link_id %u ml flags 0x%x\n", + pinfo->vdev_id, pinfo->hw_link_id, + pinfo->addr, pinfo->ieee_link_id, + le32_to_cpu(partner_info->flags)); partner_info++; } @@ -2788,8 +2807,8 @@ int ath12k_wmi_send_scan_start_cmd(struct ath12k *ar, for (i = 0; i < arg->num_hint_bssid; ++i) { hint_bssid->freq_flags = arg->hint_bssid[i].freq_flags; - ether_addr_copy(&arg->hint_bssid[i].bssid.addr[0], - &hint_bssid->bssid.addr[0]); + ether_addr_copy(&hint_bssid->bssid.addr[0], + &arg->hint_bssid[i].bssid.addr[0]); hint_bssid++; } } @@ -4751,14 +4770,16 @@ static int ath12k_wmi_mac_phy_caps_parse(struct ath12k_base *soc, if (svc_rdy_ext->n_mac_phy_caps >= svc_rdy_ext->tot_phy_id) return -ENOBUFS; - len = min_t(u16, len, sizeof(struct ath12k_wmi_mac_phy_caps_params)); if (!svc_rdy_ext->n_mac_phy_caps) { - svc_rdy_ext->mac_phy_caps = kzalloc((svc_rdy_ext->tot_phy_id) * len, - GFP_ATOMIC); + svc_rdy_ext->mac_phy_caps = + kzalloc_objs(*svc_rdy_ext->mac_phy_caps, + svc_rdy_ext->tot_phy_id, + GFP_ATOMIC); if (!svc_rdy_ext->mac_phy_caps) return -ENOMEM; } + len = min_t(u16, len, sizeof(struct ath12k_wmi_mac_phy_caps_params)); memcpy(svc_rdy_ext->mac_phy_caps + svc_rdy_ext->n_mac_phy_caps, ptr, len); svc_rdy_ext->n_mac_phy_caps++; return 0; @@ -6713,16 +6734,12 @@ static int ath12k_pull_roam_ev(struct ath12k_base *ab, struct sk_buff *skb, return 0; } -static int freq_to_idx(struct ath12k *ar, int freq) +static int freq_to_idx(struct ieee80211_hw *hw, int freq) { struct ieee80211_supported_band *sband; - struct ieee80211_hw *hw = ath12k_ar_to_hw(ar); int band, ch, idx = 0; for (band = NL80211_BAND_2GHZ; band < NUM_NL80211_BANDS; band++) { - if (!ar->mac.sbands[band].channels) - continue; - sband = hw->wiphy->bands[band]; if (!sband) continue; @@ -7072,25 +7089,29 @@ static void ath12k_peer_delete_resp_event(struct ath12k_base *ab, struct sk_buff { struct wmi_peer_delete_resp_event peer_del_resp; struct ath12k *ar; + u32 vdev_id; if (ath12k_pull_peer_del_resp_ev(ab, skb, &peer_del_resp) != 0) { - ath12k_warn(ab, "failed to extract peer delete resp"); + ath12k_warn(ab, "failed to extract peer delete resp\n"); return; } + vdev_id = le32_to_cpu(peer_del_resp.vdev_id); + rcu_read_lock(); - ar = ath12k_mac_get_ar_by_vdev_id(ab, le32_to_cpu(peer_del_resp.vdev_id)); + ar = ath12k_mac_get_ar_by_vdev_id(ab, vdev_id); if (!ar) { - ath12k_warn(ab, "invalid vdev id in peer delete resp ev %d", - peer_del_resp.vdev_id); + ath12k_warn(ab, "invalid vdev id in peer delete resp ev %d\n", + vdev_id); rcu_read_unlock(); return; } - complete(&ar->peer_delete_done); + ath12k_peer_delete_resp_signal(ar, vdev_id, + peer_del_resp.peer_macaddr.addr); rcu_read_unlock(); ath12k_dbg(ab, ATH12K_DBG_WMI, "peer delete resp for vdev id %d addr %pM\n", - peer_del_resp.vdev_id, peer_del_resp.peer_macaddr.addr); + vdev_id, peer_del_resp.peer_macaddr.addr); } static void ath12k_vdev_delete_resp_event(struct ath12k_base *ab, @@ -7629,6 +7650,7 @@ static void ath12k_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) { struct wmi_chan_info_event ch_info_ev = {}; struct ath12k *ar; + struct ath12k_hw *ah; struct survey_info *survey; int idx; /* HW channel counters frequency value in hertz */ @@ -7660,6 +7682,7 @@ static void ath12k_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) return; } spin_lock_bh(&ar->data_lock); + ah = ath12k_ar_to_ah(ar); switch (ar->scan.state) { case ATH12K_SCAN_IDLE: @@ -7671,8 +7694,8 @@ static void ath12k_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) break; } - idx = freq_to_idx(ar, le32_to_cpu(ch_info_ev.freq)); - if (idx >= ARRAY_SIZE(ar->survey)) { + idx = freq_to_idx(ath12k_ar_to_hw(ar), le32_to_cpu(ch_info_ev.freq)); + if (idx >= ARRAY_SIZE(ah->survey)) { ath12k_warn(ab, "chan info: invalid frequency %d (idx %d out of bounds)\n", ch_info_ev.freq, idx); goto exit; @@ -7685,14 +7708,20 @@ static void ath12k_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) cc_freq_hz = (le32_to_cpu(ch_info_ev.mac_clk_mhz) * 1000); if (ch_info_ev.cmd_flags == WMI_CHAN_INFO_START_RESP) { - survey = &ar->survey[idx]; - memset(survey, 0, sizeof(*survey)); - survey->noise = le32_to_cpu(ch_info_ev.noise_floor); - survey->filled = SURVEY_INFO_NOISE_DBM | SURVEY_INFO_TIME | - SURVEY_INFO_TIME_BUSY; - survey->time = div_u64(le32_to_cpu(ch_info_ev.cycle_count), cc_freq_hz); - survey->time_busy = div_u64(le32_to_cpu(ch_info_ev.rx_clear_count), - cc_freq_hz); + scoped_guard(spinlock_bh, &ah->survey_lock) { + survey = &ah->survey[idx]; + memset(survey, 0, sizeof(*survey)); + survey->noise = le32_to_cpu(ch_info_ev.noise_floor); + survey->time = + div_u64(le32_to_cpu(ch_info_ev.cycle_count), + cc_freq_hz); + survey->time_busy = + div_u64(le32_to_cpu(ch_info_ev.rx_clear_count), + cc_freq_hz); + survey->filled = SURVEY_INFO_NOISE_DBM | + SURVEY_INFO_TIME | + SURVEY_INFO_TIME_BUSY; + } } exit: spin_unlock_bh(&ar->data_lock); @@ -7705,6 +7734,7 @@ ath12k_pdev_bss_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) struct wmi_pdev_bss_chan_info_event bss_ch_info_ev = {}; struct survey_info *survey; struct ath12k *ar; + struct ath12k_hw *ah; u32 cc_freq_hz = ab->cc_freq_hz; u64 busy, total, tx, rx, rx_bss; int idx; @@ -7745,28 +7775,31 @@ ath12k_pdev_bss_chan_info_event(struct ath12k_base *ab, struct sk_buff *skb) return; } - spin_lock_bh(&ar->data_lock); - idx = freq_to_idx(ar, le32_to_cpu(bss_ch_info_ev.freq)); - if (idx >= ARRAY_SIZE(ar->survey)) { + ah = ath12k_ar_to_ah(ar); + + idx = freq_to_idx(ath12k_ar_to_hw(ar), le32_to_cpu(bss_ch_info_ev.freq)); + if (idx >= ARRAY_SIZE(ah->survey)) { ath12k_warn(ab, "bss chan info: invalid frequency %d (idx %d out of bounds)\n", bss_ch_info_ev.freq, idx); goto exit; } - survey = &ar->survey[idx]; + scoped_guard(spinlock_bh, &ah->survey_lock) { + survey = &ah->survey[idx]; + + survey->noise = le32_to_cpu(bss_ch_info_ev.noise_floor); + survey->time = div_u64(total, cc_freq_hz); + survey->time_busy = div_u64(busy, cc_freq_hz); + survey->time_rx = div_u64(rx_bss, cc_freq_hz); + survey->time_tx = div_u64(tx, cc_freq_hz); + survey->filled |= (SURVEY_INFO_NOISE_DBM | + SURVEY_INFO_TIME | + SURVEY_INFO_TIME_BUSY | + SURVEY_INFO_TIME_RX | + SURVEY_INFO_TIME_TX); + } - survey->noise = le32_to_cpu(bss_ch_info_ev.noise_floor); - survey->time = div_u64(total, cc_freq_hz); - survey->time_busy = div_u64(busy, cc_freq_hz); - survey->time_rx = div_u64(rx_bss, cc_freq_hz); - survey->time_tx = div_u64(tx, cc_freq_hz); - survey->filled |= (SURVEY_INFO_NOISE_DBM | - SURVEY_INFO_TIME | - SURVEY_INFO_TIME_BUSY | - SURVEY_INFO_TIME_RX | - SURVEY_INFO_TIME_TX); exit: - spin_unlock_bh(&ar->data_lock); complete(&ar->bss_survey_done); rcu_read_unlock(); @@ -9034,12 +9067,19 @@ ath12k_wmi_process_csa_switch_count_event(struct ath12k_base *ab, const u32 *vdev_ids) { u32 current_switch_count = le32_to_cpu(ev->current_switch_count); + u32 vdev_ids_len = ath12k_wmi_tlv_data_len(vdev_ids); u32 num_vdevs = le32_to_cpu(ev->num_vdevs); struct ieee80211_bss_conf *conf; struct ath12k_link_vif *arvif; struct ath12k_vif *ahvif; int i; + if (num_vdevs > vdev_ids_len / sizeof(*vdev_ids)) { + ath12k_warn(ab, "csa switch count num_vdevs %u exceeds tlv array length %u\n", + num_vdevs, vdev_ids_len); + return; + } + rcu_read_lock(); for (i = 0; i < num_vdevs; i++) { arvif = ath12k_mac_get_arvif_by_vdev_id(ab, vdev_ids[i]); @@ -9929,6 +9969,7 @@ static void ath12k_wmi_process_tpc_stats(struct ath12k_base *ab, void *ptr = skb->data; struct ath12k *ar; u16 tlv_tag; + u16 tlv_len; u32 event_count; int ret; @@ -9944,6 +9985,7 @@ static void ath12k_wmi_process_tpc_stats(struct ath12k_base *ab, tlv = (struct wmi_tlv *)ptr; tlv_tag = le32_get_bits(tlv->header, WMI_TLV_TAG); + tlv_len = le32_get_bits(tlv->header, WMI_TLV_LEN); ptr += sizeof(*tlv); if (tlv_tag != WMI_TAG_HALPHY_CTRL_PATH_EVENT_FIXED_PARAM) { @@ -9951,6 +9993,12 @@ static void ath12k_wmi_process_tpc_stats(struct ath12k_base *ab, return; } + if (tlv_len < sizeof(*fixed_param)) { + ath12k_warn(ab, "TPC stats fixed param tlv len %u too short\n", + tlv_len); + return; + } + fixed_param = (struct ath12k_wmi_pdev_tpc_stats_event_fixed_params *)ptr; rcu_read_lock(); ar = ath12k_mac_get_ar_by_pdev_id(ab, le32_to_cpu(fixed_param->pdev_id) + 1); @@ -10257,12 +10305,12 @@ static void ath12k_wmi_op_rx(struct ath12k_base *ab, struct sk_buff *skb) struct wmi_cmd_hdr *cmd_hdr; enum wmi_tlv_event_id id; - cmd_hdr = (struct wmi_cmd_hdr *)skb->data; - id = le32_get_bits(cmd_hdr->cmd_id, WMI_CMD_HDR_CMD_ID); - - if (!skb_pull(skb, sizeof(struct wmi_cmd_hdr))) + cmd_hdr = skb_pull_data(skb, sizeof(*cmd_hdr)); + if (!cmd_hdr) goto out; + id = le32_get_bits(cmd_hdr->cmd_id, WMI_CMD_HDR_CMD_ID); + switch (id) { /* Process all the WMI events here */ case WMI_SERVICE_READY_EVENTID: diff --git a/drivers/net/wireless/ath/ath12k/wmi.h b/drivers/net/wireless/ath/ath12k/wmi.h index c452e3d57a29a..20e3939e8820b 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.h +++ b/drivers/net/wireless/ath/ath12k/wmi.h @@ -2954,10 +2954,14 @@ struct wmi_vdev_create_mlo_params { #define ATH12K_WMI_FLAG_MLO_EMLSR_SUPPORT BIT(6) #define ATH12K_WMI_FLAG_MLO_FORCED_INACTIVE BIT(7) #define ATH12K_WMI_FLAG_MLO_LINK_ADD BIT(8) +#define ATH12K_WMI_FLAG_MLO_START_AS_ACTIVE BIT(17) +#define ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID BIT(18) +#define ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID_PARTNER BIT(19) struct wmi_vdev_start_mlo_params { __le32 tlv_header; __le32 flags; + __le32 ieee_link_id; } __packed; struct wmi_partner_link_info { @@ -2965,6 +2969,8 @@ struct wmi_partner_link_info { __le32 vdev_id; __le32 hw_link_id; struct ath12k_wmi_mac_addr_params vdev_addr; + __le32 flags; + __le32 ieee_link_id; } __packed; struct wmi_vdev_delete_cmd { @@ -3120,6 +3126,7 @@ struct wmi_ml_partner_info { bool primary_umac; bool logical_link_idx_valid; u32 logical_link_idx; + u32 ieee_link_id; }; struct wmi_ml_arg { @@ -3127,6 +3134,7 @@ struct wmi_ml_arg { bool assoc_link; bool mcast_link; bool link_add; + u32 ieee_link_id; u8 num_partner_links; struct wmi_ml_partner_info partner_info[ATH12K_WMI_MLO_MAX_LINKS]; }; diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c index cc0f2c45fc3a3..62f663c0daa26 100644 --- a/drivers/net/wireless/ath/ath6kl/cfg80211.c +++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c @@ -754,6 +754,11 @@ void ath6kl_cfg80211_connect_event(struct ath6kl_vif *vif, u16 channel, u8 *assoc_resp_ie = assoc_info + beacon_ie_len + assoc_req_len + assoc_resp_ie_offset; + if (assoc_req_len < assoc_req_ie_offset) + assoc_req_len = assoc_req_ie_offset; + if (assoc_resp_len < assoc_resp_ie_offset) + assoc_resp_len = assoc_resp_ie_offset; + assoc_req_len -= assoc_req_ie_offset; assoc_resp_len -= assoc_resp_ie_offset; diff --git a/drivers/net/wireless/ath/ath6kl/wmi.c b/drivers/net/wireless/ath/ath6kl/wmi.c index 2b0c5038ae040..6c29f0bcec9f5 100644 --- a/drivers/net/wireless/ath/ath6kl/wmi.c +++ b/drivers/net/wireless/ath/ath6kl/wmi.c @@ -1296,6 +1296,9 @@ static int ath6kl_wmi_scan_complete_rx(struct wmi *wmi, u8 *datap, int len, { struct wmi_scan_complete_event *ev; + if (len < sizeof(*ev)) + return -EINVAL; + ev = (struct wmi_scan_complete_event *) datap; ath6kl_scan_complete_evt(vif, a_sle32_to_cpu(ev->status)); @@ -3372,7 +3375,12 @@ static int ath6kl_wmi_get_pmkid_list_event_rx(struct wmi *wmi, u8 *datap, static int ath6kl_wmi_addba_req_event_rx(struct wmi *wmi, u8 *datap, int len, struct ath6kl_vif *vif) { - struct wmi_addba_req_event *cmd = (struct wmi_addba_req_event *) datap; + struct wmi_addba_req_event *cmd; + + if (len < sizeof(*cmd)) + return -EINVAL; + + cmd = (struct wmi_addba_req_event *)datap; aggr_recv_addba_req_evt(vif, cmd->tid, le16_to_cpu(cmd->st_seq_no), cmd->win_sz); @@ -3383,7 +3391,12 @@ static int ath6kl_wmi_addba_req_event_rx(struct wmi *wmi, u8 *datap, int len, static int ath6kl_wmi_delba_req_event_rx(struct wmi *wmi, u8 *datap, int len, struct ath6kl_vif *vif) { - struct wmi_delba_event *cmd = (struct wmi_delba_event *) datap; + struct wmi_delba_event *cmd; + + if (len < sizeof(*cmd)) + return -EINVAL; + + cmd = (struct wmi_delba_event *)datap; aggr_recv_delba_req_evt(vif, cmd->tid); diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c index 47f904e7e6529..d3491ff08e6ef 100644 --- a/drivers/net/wireless/ath/ath9k/hif_usb.c +++ b/drivers/net/wireless/ath/ath9k/hif_usb.c @@ -1087,7 +1087,7 @@ static int ath9k_hif_usb_download_fw(struct hif_device_usb *hif_dev) } kfree(buf); - if (IS_AR7010_DEVICE(hif_dev->usb_device_id->driver_info)) + if (IS_AR7010_DEVICE(hif_dev->id_info)) firm_offset = AR7010_FIRMWARE_TEXT; else firm_offset = AR9271_FIRMWARE_TEXT; @@ -1182,7 +1182,7 @@ static int ath9k_hif_request_firmware(struct hif_device_usb *hif_dev, if (MAJOR_VERSION_REQ == 1 && hif_dev->fw_minor_index == 3) { const char *filename; - if (IS_AR7010_DEVICE(hif_dev->usb_device_id->driver_info)) + if (IS_AR7010_DEVICE(hif_dev->id_info)) filename = FIRMWARE_AR7010_1_1; else filename = FIRMWARE_AR9271; @@ -1198,7 +1198,7 @@ static int ath9k_hif_request_firmware(struct hif_device_usb *hif_dev, return -ENOENT; } else { - if (IS_AR7010_DEVICE(hif_dev->usb_device_id->driver_info)) + if (IS_AR7010_DEVICE(hif_dev->id_info)) chip = "7010"; else chip = "9271"; @@ -1255,9 +1255,9 @@ static void ath9k_hif_usb_firmware_cb(const struct firmware *fw, void *context) ret = ath9k_htc_hw_init(hif_dev->htc_handle, &hif_dev->interface->dev, - hif_dev->usb_device_id->idProduct, + le16_to_cpu(hif_dev->udev->descriptor.idProduct), hif_dev->udev->product, - hif_dev->usb_device_id->driver_info); + hif_dev->id_info); if (ret) { ret = -EINVAL; goto err_htc_hw_init; @@ -1369,7 +1369,7 @@ static int ath9k_hif_usb_probe(struct usb_interface *interface, hif_dev->udev = udev; hif_dev->interface = interface; - hif_dev->usb_device_id = id; + hif_dev->id_info = id->driver_info; #ifdef CONFIG_PM udev->reset_resume = 1; #endif diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.h b/drivers/net/wireless/ath/ath9k/hif_usb.h index dc0b0fa5c3257..b3e7b0fb54b84 100644 --- a/drivers/net/wireless/ath/ath9k/hif_usb.h +++ b/drivers/net/wireless/ath/ath9k/hif_usb.h @@ -115,7 +115,7 @@ struct cmd_buf { struct hif_device_usb { struct usb_device *udev; struct usb_interface *interface; - const struct usb_device_id *usb_device_id; + int id_info; const void *fw_data; size_t fw_size; struct completion fw_done; diff --git a/drivers/net/wireless/ath/wcn36xx/dxe.c b/drivers/net/wireless/ath/wcn36xx/dxe.c index 44020ec265fb6..801f1218ef89b 100644 --- a/drivers/net/wireless/ath/wcn36xx/dxe.c +++ b/drivers/net/wireless/ath/wcn36xx/dxe.c @@ -1055,7 +1055,7 @@ void wcn36xx_dxe_deinit(struct wcn36xx *wcn) free_irq(wcn->tx_irq, wcn); free_irq(wcn->rx_irq, wcn); - timer_delete(&wcn->tx_ack_timer); + timer_shutdown_sync(&wcn->tx_ack_timer); if (wcn->tx_ack_skb) { ieee80211_tx_status_irqsafe(wcn->hw, wcn->tx_ack_skb); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index dad6f4563d146..d2ae679856067 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -555,6 +555,8 @@ void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success) if (type == ETH_P_PAE) { atomic_dec(&ifp->pend_8021x_cnt); + /* Order the decrement before waitqueue_active() */ + smp_mb__after_atomic(); if (waitqueue_active(&ifp->pend_8021x_wait)) wake_up(&ifp->pend_8021x_wait); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c index 873754be5174b..c86d0bdde8326 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c @@ -200,7 +200,7 @@ brcmf_cyw_external_auth(struct wiphy *wiphy, struct net_device *dev, { struct brcmf_if *ifp; struct brcmf_pub *drvr; - struct brcmf_auth_req_status_le auth_status; + struct brcmf_auth_req_status_le auth_status = {}; int ret = 0; brcmf_dbg(TRACE, "Enter\n"); @@ -208,6 +208,9 @@ brcmf_cyw_external_auth(struct wiphy *wiphy, struct net_device *dev, ifp = netdev_priv(dev); drvr = ifp->drvr; if (params->status == WLAN_STATUS_SUCCESS) { + if (params->pmkid) + memcpy(auth_status.pmkid, params->pmkid, + WLAN_PMKID_LEN); auth_status.flags = cpu_to_le16(BRCMF_EXTAUTH_SUCCESS); } else { bphy_err(drvr, "External authentication failed: status=%d\n", diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 92c16a3173288..66557be28e7b1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1283,6 +1283,9 @@ static s32 brcmf_p2p_abort_action_frame(struct brcmf_cfg80211_info *cfg) brcmf_dbg(TRACE, "Enter\n"); vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + if (!vif) + vif = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif; + err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe_abort", &int_val, sizeof(s32)); if (err) @@ -1819,6 +1822,7 @@ bool brcmf_p2p_send_action_frame(struct brcmf_if *ifp, /* validate channel and p2p ies */ if (config_af_params.search_channel && IS_P2P_SOCIAL_CHANNEL(le32_to_cpu(af_params->channel)) && + p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif && p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif->saved_ie.probe_req_ie_len) { afx_hdl = &p2p->afx_hdl; afx_hdl->peer_listen_chan = le32_to_cpu(af_params->channel); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 9f7ed1d293a06..381801af3ac98 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -1827,17 +1827,18 @@ brcmf_sdio_read_control(struct brcmf_sdio *bus, u8 *hdr, uint len, uint doff) if (bus->rxctl) { brcmf_err("last control frame is being processed.\n"); spin_unlock_bh(&bus->rxctl_lock); - vfree(buf); goto done; } bus->rxctl = buf + doff; bus->rxctl_orig = buf; bus->rxlen = len - doff; spin_unlock_bh(&bus->rxctl_lock); + buf = NULL; done: /* Awake any waiters */ brcmf_sdio_dcmd_resp_wake(bus); + vfree(buf); } /* Pad read to blocksize for efficiency */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c index 6255d673d2d3e..c1a2318d7ea6f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c @@ -1571,6 +1571,10 @@ void brcms_free_timer(struct brcms_timer *t) /* delete the timer in case it is active */ brcms_del_timer(t); + /* Ensure the callback has finished before freeing the timer + * structure, since brcms_del_timer() uses non-synchronous cancel. + */ + cancel_delayed_work_sync(&t->dly_wrk); if (wl->timers == t) { wl->timers = wl->timers->next; diff --git a/drivers/net/wireless/intel/ipw2x00/libipw_crypto_tkip.c b/drivers/net/wireless/intel/ipw2x00/libipw_crypto_tkip.c index 24bb28ab7a49b..2b0cf0ec496ad 100644 --- a/drivers/net/wireless/intel/ipw2x00/libipw_crypto_tkip.c +++ b/drivers/net/wireless/intel/ipw2x00/libipw_crypto_tkip.c @@ -474,14 +474,16 @@ static int libipw_michael_mic_verify(struct sk_buff *skb, int keyidx, int hdr_len, void *priv) { struct libipw_tkip_data *tkey = priv; - u8 mic[8]; + u8 mic[MICHAEL_MIC_LEN]; - if (!tkey->key_set) + if (!tkey->key_set || skb->len < hdr_len + MICHAEL_MIC_LEN) return -1; michael_mic(&tkey->key[24], (struct ieee80211_hdr *)skb->data, - skb->data + hdr_len, skb->len - 8 - hdr_len, mic); - if (memcmp(mic, skb->data + skb->len - 8, 8) != 0) { + skb->data + hdr_len, + skb->len - MICHAEL_MIC_LEN - hdr_len, mic); + if (memcmp(mic, skb->data + skb->len - MICHAEL_MIC_LEN, + MICHAEL_MIC_LEN) != 0) { struct ieee80211_hdr *hdr; hdr = (struct ieee80211_hdr *)skb->data; printk(KERN_DEBUG "%s: Michael MIC verification failed for " @@ -499,7 +501,7 @@ static int libipw_michael_mic_verify(struct sk_buff *skb, int keyidx, tkey->rx_iv32 = tkey->rx_iv32_new; tkey->rx_iv16 = tkey->rx_iv16_new; - skb_trim(skb, skb->len - 8); + skb_trim(skb, skb->len - MICHAEL_MIC_LEN); return 0; } diff --git a/drivers/net/wireless/intel/ipw2x00/libipw_rx.c b/drivers/net/wireless/intel/ipw2x00/libipw_rx.c index c8841f9b9ad91..424349a6935e4 100644 --- a/drivers/net/wireless/intel/ipw2x00/libipw_rx.c +++ b/drivers/net/wireless/intel/ipw2x00/libipw_rx.c @@ -1209,6 +1209,9 @@ static int libipw_handle_assoc_resp(struct libipw_device *ieee, struct libipw_as struct libipw_network *network = &network_resp; struct net_device *dev = ieee->dev; + if (stats->len < sizeof(*frame)) + return 1; + network->flags = 0; network->qos_data.active = 0; network->qos_data.supported = 0; @@ -1421,6 +1424,9 @@ static void libipw_process_probe_response(struct libipw_device #endif unsigned long flags; + if (stats->len < sizeof(*beacon)) + return; + LIBIPW_DEBUG_SCAN("'%*pE' (%pM): %c%c%c%c %c%c%c%c-%c%c%c%c %c%c%c%c\n", info_element->len, info_element->data, beacon->header.addr3, diff --git a/drivers/net/wireless/intel/iwlegacy/common.c b/drivers/net/wireless/intel/iwlegacy/common.c index 0bb807ff8edff..e5113c6b2d5c5 100644 --- a/drivers/net/wireless/intel/iwlegacy/common.c +++ b/drivers/net/wireless/intel/iwlegacy/common.c @@ -2326,7 +2326,7 @@ il_dealloc_bcast_stations(struct il_priv *il) if (!(il->stations[i].used & IL_STA_BCAST)) continue; - il->stations[i].used &= ~IL_STA_UCODE_ACTIVE; + il->stations[i].used = 0; il->num_stations--; if (WARN_ON(il->num_stations < 0)) il->num_stations = 0; diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/main.c b/drivers/net/wireless/intel/iwlwifi/dvm/main.c index ca5a8140908af..6bd5b6d84b2a9 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/main.c +++ b/drivers/net/wireless/intel/iwlwifi/dvm/main.c @@ -1511,10 +1511,10 @@ static struct iwl_op_mode *iwl_op_mode_dvm_start(struct iwl_trans *trans, priv->workqueue = NULL; out_uninit_drv: iwl_uninit_drv(priv); -out_free_eeprom_blob: - kfree(priv->eeprom_blob); out_free_eeprom: kfree(priv->nvm_data); +out_free_eeprom_blob: + kfree(priv->eeprom_blob); out_leave_trans: iwl_trans_op_mode_leave(priv->trans); out_free_hw: diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dump.c b/drivers/net/wireless/intel/iwlwifi/fw/dump.c index c2af66899a780..bbbf3669a555d 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/dump.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/dump.c @@ -369,7 +369,7 @@ static void iwl_fwrt_dump_fseq_regs(struct iwl_fw_runtime *fwrt) void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt) { struct iwl_pc_data *pc_data; - u8 count; + u32 count; if (!iwl_trans_device_enabled(fwrt->trans)) { IWL_ERR(fwrt, diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c index b1a55909f0d4b..11763dec77eca 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2018-2025 Intel Corporation + * Copyright (C) 2018-2026 Intel Corporation */ #include #include "iwl-drv.h" @@ -611,6 +611,9 @@ static int iwl_dbg_tlv_alloc_fragments(struct iwl_fw_runtime *fwrt, cpu_to_le32(IWL_FW_INI_LOCATION_DRAM_PATH)) return 0; + if (!fw_mon_cfg->req_size) + return -EIO; + num_frags = le32_to_cpu(fw_mon_cfg->max_frags_num); if (fwrt->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) { if (alloc_id != IWL_FW_INI_ALLOCATION_ID_DBGC1) @@ -621,6 +624,9 @@ static int iwl_dbg_tlv_alloc_fragments(struct iwl_fw_runtime *fwrt, return -EIO; } + if (!num_frags) + return -EIO; + remain_pages = DIV_ROUND_UP(le32_to_cpu(fw_mon_cfg->req_size), PAGE_SIZE); num_frags = min_t(u32, num_frags, BUF_ALLOC_MAX_NUM_FRAGS); diff --git a/drivers/net/wireless/intel/iwlwifi/mei/main.c b/drivers/net/wireless/intel/iwlwifi/mei/main.c index c5ff1b1b720f3..b78ab75afc159 100644 --- a/drivers/net/wireless/intel/iwlwifi/mei/main.c +++ b/drivers/net/wireless/intel/iwlwifi/mei/main.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2021-2024 Intel Corporation + * Copyright (C) 2026 Intel Corporation */ #include @@ -457,7 +458,7 @@ static int iwl_mei_send_sap_msg_payload(struct mei_cl_device *cldev, notif_q = &dir->q_ctrl_blk[SAP_QUEUE_IDX_NOTIF]; q_head = mei->shared_mem.q_head[SAP_DIRECTION_HOST_TO_ME][SAP_QUEUE_IDX_NOTIF]; q_sz = mei->shared_mem.q_size[SAP_DIRECTION_HOST_TO_ME][SAP_QUEUE_IDX_NOTIF]; - ret = iwl_mei_write_cyclic_buf(q_head, notif_q, q_head, hdr, q_sz); + ret = iwl_mei_write_cyclic_buf(cldev, notif_q, q_head, hdr, q_sz); if (ret < 0) return ret; @@ -1147,6 +1148,11 @@ static void iwl_mei_handle_sap_rx_cmd(struct mei_cl_device *cldev, iwl_mei_read_from_q(q_head, q_sz, &rd, wr, hdr, sizeof(*hdr)); valid_rx_sz -= sizeof(*hdr); len = le16_to_cpu(hdr->len); + if (len + sizeof(*hdr) > PAGE_SIZE) { + dev_err(&cldev->dev, + "SAP message is too big: %u\n", len); + break; + } if (valid_rx_sz < len) break; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c index 2297392db9558..b07a78524bcc8 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c @@ -954,7 +954,7 @@ static void iwl_mvm_frob_txf_key_iter(struct ieee80211_hw *hw, } match++; if (match == keylen) { - memset(txf->buf + i - keylen, 0xAA, keylen); + memset(txf->buf + i + 1 - keylen, 0xAA, keylen); match = 0; } } diff --git a/drivers/net/wireless/intersil/p54/eeprom.c b/drivers/net/wireless/intersil/p54/eeprom.c index 95580921d9338..0475222d54fc1 100644 --- a/drivers/net/wireless/intersil/p54/eeprom.c +++ b/drivers/net/wireless/intersil/p54/eeprom.c @@ -414,17 +414,22 @@ static int p54_generate_channel_lists(struct ieee80211_hw *dev) } static int p54_convert_rev0(struct ieee80211_hw *dev, - struct pda_pa_curve_data *curve_data) + struct pda_pa_curve_data *curve_data, size_t len) { struct p54_common *priv = dev->priv; struct p54_pa_curve_data_sample *dst; struct pda_pa_curve_data_sample_rev0 *src; + size_t needed = curve_data->channels * + (sizeof(*src) * curve_data->points_per_channel + 2); size_t cd_len = sizeof(*curve_data) + (curve_data->points_per_channel*sizeof(*dst) + 2) * curve_data->channels; unsigned int i, j; void *source, *target; + if (len < sizeof(*curve_data) + needed) + return -EINVAL; + priv->curve_data = kmalloc(sizeof(*priv->curve_data) + cd_len, GFP_KERNEL); if (!priv->curve_data) @@ -466,17 +471,22 @@ static int p54_convert_rev0(struct ieee80211_hw *dev, } static int p54_convert_rev1(struct ieee80211_hw *dev, - struct pda_pa_curve_data *curve_data) + struct pda_pa_curve_data *curve_data, size_t len) { struct p54_common *priv = dev->priv; struct p54_pa_curve_data_sample *dst; struct pda_pa_curve_data_sample_rev1 *src; + size_t needed = curve_data->channels * + (sizeof(*src) * curve_data->points_per_channel + 3); size_t cd_len = sizeof(*curve_data) + (curve_data->points_per_channel*sizeof(*dst) + 2) * curve_data->channels; unsigned int i, j; void *source, *target; + if (len < sizeof(*curve_data) + needed) + return -EINVAL; + priv->curve_data = kzalloc(cd_len + sizeof(*priv->curve_data), GFP_KERNEL); if (!priv->curve_data) @@ -763,6 +773,7 @@ int p54_parse_eeprom(struct ieee80211_hw *dev, void *eeprom, int len) case PDR_PRISM_PA_CAL_CURVE_DATA: { struct pda_pa_curve_data *curve_data = (struct pda_pa_curve_data *)entry->data; + if (data_len < sizeof(*curve_data)) { err = -EINVAL; goto err; @@ -770,10 +781,10 @@ int p54_parse_eeprom(struct ieee80211_hw *dev, void *eeprom, int len) switch (curve_data->cal_method_rev) { case 0: - err = p54_convert_rev0(dev, curve_data); + err = p54_convert_rev0(dev, curve_data, data_len); break; case 1: - err = p54_convert_rev1(dev, curve_data); + err = p54_convert_rev1(dev, curve_data, data_len); break; default: wiphy_err(dev->wiphy, @@ -801,7 +812,8 @@ int p54_parse_eeprom(struct ieee80211_hw *dev, void *eeprom, int len) break; case PDR_INTERFACE_LIST: tmp = entry->data; - while ((u8 *)tmp < entry->data + data_len) { + while ((u8 *)tmp + sizeof(struct exp_if) <= + entry->data + data_len) { struct exp_if *exp_if = tmp; if (exp_if->if_id == cpu_to_le16(IF_ID_ISL39000)) synth = le16_to_cpu(exp_if->variant); diff --git a/drivers/net/wireless/marvell/libertas_tf/main.c b/drivers/net/wireless/marvell/libertas_tf/main.c index 42be6fa22f9c6..411f075b6186a 100644 --- a/drivers/net/wireless/marvell/libertas_tf/main.c +++ b/drivers/net/wireless/marvell/libertas_tf/main.c @@ -173,8 +173,8 @@ static int lbtf_init_adapter(struct lbtf_private *priv) static void lbtf_free_adapter(struct lbtf_private *priv) { lbtf_deb_enter(LBTF_DEB_MAIN); - lbtf_free_cmd_buffer(priv); timer_delete_sync(&priv->command_timer); + lbtf_free_cmd_buffer(priv); lbtf_deb_leave(LBTF_DEB_MAIN); } diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c index abc703441c5d8..e4ad2e29f10e7 100644 --- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c +++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c @@ -4279,6 +4279,7 @@ mwifiex_cfg80211_authenticate(struct wiphy *wiphy, struct mwifiex_adapter *adapter = priv->adapter; struct sk_buff *skb; u16 pkt_len, auth_alg; + size_t frame_len; int ret; struct mwifiex_ieee80211_mgmt *mgmt; struct mwifiex_txinfo *tx_info; @@ -4351,10 +4352,17 @@ mwifiex_cfg80211_authenticate(struct wiphy *wiphy, mwifiex_cancel_scan(adapter); - pkt_len = (u16)req->ie_len + req->auth_data_len + + frame_len = req->ie_len + req->auth_data_len + MWIFIEX_MGMT_HEADER_LEN + MWIFIEX_AUTH_BODY_LEN; if (req->auth_data_len >= 4) - pkt_len -= 4; + frame_len -= 4; + + if (frame_len > U16_MAX) { + mwifiex_dbg(priv->adapter, ERROR, + "auth frame too long: %zu bytes\n", frame_len); + return -EINVAL; + } + pkt_len = frame_len; skb = dev_alloc_skb(MWIFIEX_MIN_DATA_HEADER_LEN + MWIFIEX_MGMT_FRAME_HEADER_SIZE + diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c index a760de191fce7..a9425e9a94f45 100644 --- a/drivers/net/wireless/marvell/mwifiex/pcie.c +++ b/drivers/net/wireless/marvell/mwifiex/pcie.c @@ -3068,7 +3068,7 @@ static int mwifiex_pcie_request_irq(struct mwifiex_adapter *adapter) ret); for (j = 0; j < i; j++) free_irq(card->msix_entries[j].vector, - &card->msix_ctx[i]); + &card->msix_ctx[j]); pci_disable_msix(pdev); } else { mwifiex_dbg(adapter, MSG, "MSIx enabled!"); diff --git a/drivers/net/wireless/marvell/mwifiex/scan.c b/drivers/net/wireless/marvell/mwifiex/scan.c index 97c0ec3b822e7..bdd4b8465863b 100644 --- a/drivers/net/wireless/marvell/mwifiex/scan.c +++ b/drivers/net/wireless/marvell/mwifiex/scan.c @@ -104,12 +104,24 @@ has_vendor_hdr(struct ieee_types_vendor_specific *ie, u8 key) * a given oui in PTK. */ static u8 -mwifiex_search_oui_in_ie(struct ie_body *iebody, u8 *oui) +mwifiex_search_oui_in_ie(struct ie_body *iebody, u8 *oui, int ie_len) { + const size_t ptk_body_offset = offsetof(struct ie_body, ptk_body); u8 count; + /* ie_len is the number of bytes available at iebody. Keep it signed + * and reject a negative (underflowed) length before the unsigned + * comparisons below, so a small or zero IE length cannot wrap. + */ + if (ie_len < 0 || (size_t)ie_len < ptk_body_offset) + return MWIFIEX_OUI_NOT_PRESENT; + count = iebody->ptk_cnt[0]; + /* Reject an OUI count whose list would run past the element. */ + if (ptk_body_offset + count * sizeof(iebody->ptk_body) > (size_t)ie_len) + return MWIFIEX_OUI_NOT_PRESENT; + /* There could be multiple OUIs for PTK hence 1) Take the length. 2) Check all the OUIs for AES. @@ -143,11 +155,14 @@ mwifiex_is_rsn_oui_present(struct mwifiex_bssdescriptor *bss_desc, u32 cipher) u8 ret = MWIFIEX_OUI_NOT_PRESENT; if (has_ieee_hdr(bss_desc->bcn_rsn_ie, WLAN_EID_RSN)) { + int ie_len = (int)bss_desc->bcn_rsn_ie->ieee_hdr.len - + RSN_GTK_OUI_OFFSET; + iebody = (struct ie_body *) (((u8 *) bss_desc->bcn_rsn_ie->data) + RSN_GTK_OUI_OFFSET); oui = &mwifiex_rsn_oui[cipher][0]; - ret = mwifiex_search_oui_in_ie(iebody, oui); + ret = mwifiex_search_oui_in_ie(iebody, oui, ie_len); if (ret) return ret; } @@ -169,10 +184,14 @@ mwifiex_is_wpa_oui_present(struct mwifiex_bssdescriptor *bss_desc, u32 cipher) u8 ret = MWIFIEX_OUI_NOT_PRESENT; if (has_vendor_hdr(bss_desc->bcn_wpa_ie, WLAN_EID_VENDOR_SPECIFIC)) { + int ie_len = (int)bss_desc->bcn_wpa_ie->vend_hdr.len - + (int)sizeof(bss_desc->bcn_wpa_ie->vend_hdr.oui) - + WPA_GTK_OUI_OFFSET; + iebody = (struct ie_body *)((u8 *)bss_desc->bcn_wpa_ie->data + WPA_GTK_OUI_OFFSET); oui = &mwifiex_wpa_oui[cipher][0]; - ret = mwifiex_search_oui_in_ie(iebody, oui); + ret = mwifiex_search_oui_in_ie(iebody, oui, ie_len); if (ret) return ret; } @@ -2096,6 +2115,7 @@ int mwifiex_ret_802_11_scan(struct mwifiex_private *priv, u32 bytes_left; u32 idx; u32 tlv_buf_size; + size_t fixed_size; struct mwifiex_ie_types_chan_band_list_param_set *chan_band_tlv; struct chan_band_param_set *chan_band; u8 is_bgscan_resp; @@ -2111,6 +2131,14 @@ int mwifiex_ret_802_11_scan(struct mwifiex_private *priv, else scan_rsp = &resp->params.scan_resp; + scan_resp_size = le16_to_cpu(resp->size); + fixed_size = scan_rsp->bss_desc_and_tlv_buffer - (u8 *)resp; + if (scan_resp_size < fixed_size) { + mwifiex_dbg(adapter, ERROR, + "SCAN_RESP: response is too short\n"); + ret = -1; + goto check_next_scan; + } if (scan_rsp->number_of_sets > MWIFIEX_MAX_AP) { mwifiex_dbg(adapter, ERROR, @@ -2128,8 +2156,6 @@ int mwifiex_ret_802_11_scan(struct mwifiex_private *priv, "info: SCAN_RESP: bss_descript_size %d\n", bytes_left); - scan_resp_size = le16_to_cpu(resp->size); - mwifiex_dbg(adapter, INFO, "info: SCAN_RESP: returned %d APs before parsing\n", scan_rsp->number_of_sets); @@ -2137,15 +2163,17 @@ int mwifiex_ret_802_11_scan(struct mwifiex_private *priv, bss_info = scan_rsp->bss_desc_and_tlv_buffer; /* - * The size of the TLV buffer is equal to the entire command response - * size (scan_resp_size) minus the fixed fields (sizeof()'s), the - * BSS Descriptions (bss_descript_size as bytesLef) and the command - * response header (S_DS_GEN) + * The TLV buffer follows the command-specific fixed fields and the BSS + * descriptions. Background-scan responses have an additional fixed + * field before scan_rsp, which is included in fixed_size. */ - tlv_buf_size = scan_resp_size - (bytes_left - + sizeof(scan_rsp->bss_descript_size) - + sizeof(scan_rsp->number_of_sets) - + S_DS_GEN); + if (bytes_left > scan_resp_size - fixed_size) { + mwifiex_dbg(adapter, ERROR, + "SCAN_RESP: BSS data exceeds response\n"); + ret = -1; + goto check_next_scan; + } + tlv_buf_size = scan_resp_size - fixed_size - bytes_left; tlv_data = (struct mwifiex_ie_types_data *) (scan_rsp-> bss_desc_and_tlv_buffer + diff --git a/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c b/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c index 9460d5352b234..19196848778c0 100644 --- a/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c +++ b/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c @@ -57,6 +57,18 @@ int mwifiex_wait_queue_complete(struct mwifiex_adapter *adapter, mwifiex_dbg(adapter, ERROR, "cmd_wait_q terminated: %d\n", status); mwifiex_cancel_all_pending_cmd(adapter); + + /* The command response path writes through cmd_node->data_buf. + * On an interrupted wait, the caller can return and release a + * stack-allocated data_buf before a late firmware response is + * processed. Detach the caller-owned buffer from the current + * command so a late response cannot corrupt freed stack memory. + */ + spin_lock_bh(&adapter->mwifiex_cmd_lock); + if (adapter->curr_cmd == cmd_queued) + adapter->curr_cmd->data_buf = NULL; + spin_unlock_bh(&adapter->mwifiex_cmd_lock); + return status; } diff --git a/drivers/net/wireless/marvell/mwifiex/util.c b/drivers/net/wireless/marvell/mwifiex/util.c index 7d3631d212236..71305efb77ac5 100644 --- a/drivers/net/wireless/marvell/mwifiex/util.c +++ b/drivers/net/wireless/marvell/mwifiex/util.c @@ -317,10 +317,16 @@ mwifiex_parse_mgmt_packet(struct mwifiex_private *priv, u8 *payload, u16 len, switch (stype) { case IEEE80211_STYPE_ACTION: - category = *(payload + sizeof(struct ieee80211_hdr)); + if (len < sizeof(*ieee_hdr) + 1) + return -1; + + category = *(payload + sizeof(*ieee_hdr)); switch (category) { case WLAN_CATEGORY_PUBLIC: - action_code = *(payload + sizeof(struct ieee80211_hdr) + if (len < sizeof(*ieee_hdr) + 2) + return -1; + + action_code = *(payload + sizeof(*ieee_hdr) + 1); if (action_code == WLAN_PUB_ACTION_TDLS_DISCOVER_RES) { addr2 = ieee_hdr->addr2; diff --git a/drivers/net/wireless/mediatek/mt76/channel.c b/drivers/net/wireless/mediatek/mt76/channel.c index 6edcb3b8f2798..28ad7bcaffd45 100644 --- a/drivers/net/wireless/mediatek/mt76/channel.c +++ b/drivers/net/wireless/mediatek/mt76/channel.c @@ -186,68 +186,74 @@ int mt76_switch_vif_chanctx(struct ieee80211_hw *hw, int n_vifs, enum ieee80211_chanctx_switch_mode mode) { - struct mt76_chanctx *old_ctx = (struct mt76_chanctx *)vifs->old_ctx->drv_priv; - struct mt76_chanctx *new_ctx = (struct mt76_chanctx *)vifs->new_ctx->drv_priv; - struct ieee80211_chanctx_conf *conf = vifs->new_ctx; - struct mt76_phy *old_phy = old_ctx->phy; - struct mt76_phy *phy = hw->priv; + struct ieee80211_vif_chanctx_switch *v; + struct mt76_chanctx *old_ctx, *new_ctx; + struct mt76_phy *old_phy, *phy = hw->priv; struct mt76_dev *dev = phy->dev; struct mt76_vif_link *mlink; - bool update_chan; + bool need_update[__MT_MAX_BAND] = {}; int i, ret = 0; - if (mode == CHANCTX_SWMODE_SWAP_CONTEXTS) - phy = new_ctx->phy = dev->band_phys[conf->def.chan->band]; - else - phy = new_ctx->phy; - if (!phy) - return -EINVAL; + for (i = 0; i < n_vifs; i++) { + v = &vifs[i]; + new_ctx = (struct mt76_chanctx *)v->new_ctx->drv_priv; + if (mode == CHANCTX_SWMODE_SWAP_CONTEXTS) + phy = new_ctx->phy = dev->band_phys[v->new_ctx->def.chan->band]; + else + phy = new_ctx->phy; - update_chan = phy->chanctx != new_ctx; - if (update_chan) { - if (dev->scan.phy == phy) - mt76_abort_scan(dev); + if (!phy) + return -EINVAL; - cancel_delayed_work_sync(&phy->mac_work); + if (need_update[phy->band_idx]) + continue; + + if (phy->chanctx != new_ctx) { + if (dev->scan.phy == phy) + mt76_abort_scan(dev); + + cancel_delayed_work_sync(&phy->mac_work); + need_update[phy->band_idx] = true; + } } mutex_lock(&dev->mutex); - if (mode == CHANCTX_SWMODE_SWAP_CONTEXTS && - phy != old_phy && old_phy->chanctx == old_ctx) - old_phy->chanctx = NULL; + for (i = 0; i < n_vifs; i++) { + v = &vifs[i]; + old_ctx = (struct mt76_chanctx *)v->old_ctx->drv_priv; + old_phy = old_ctx->phy; + + new_ctx = (struct mt76_chanctx *)v->new_ctx->drv_priv; + phy = new_ctx->phy; - if (update_chan) - ret = mt76_phy_update_channel(phy, vifs->new_ctx); + if (mode == CHANCTX_SWMODE_SWAP_CONTEXTS && old_phy->chanctx && + old_phy->chanctx == old_ctx && phy != old_phy) + old_phy->chanctx = NULL; - if (ret) - goto out; + if (need_update[phy->band_idx]) { + ret = mt76_phy_update_channel(phy, v->new_ctx); + if (ret) + goto out; - if (old_phy == phy) - goto skip_link_replace; + need_update[phy->band_idx] = false; + } - for (i = 0; i < n_vifs; i++) { - mlink = mt76_vif_conf_link(dev, vifs[i].vif, vifs[i].link_conf); + mlink = mt76_vif_conf_link(dev, v->vif, v->link_conf); if (!mlink) continue; - dev->drv->vif_link_remove(old_phy, vifs[i].vif, - vifs[i].link_conf, mlink); - - ret = dev->drv->vif_link_add(phy, vifs[i].vif, - vifs[i].link_conf, mlink); - if (ret) - goto out; - - } + if (old_phy != phy) { + dev->drv->vif_link_remove(old_phy, v->vif, v->link_conf, + mlink); -skip_link_replace: - for (i = 0; i < n_vifs; i++) { - mlink = mt76_vif_conf_link(dev, vifs[i].vif, vifs[i].link_conf); - if (!mlink) - continue; + ret = dev->drv->vif_link_add(phy, v->vif, v->link_conf, + mlink); + if (ret) + goto out; + } - mlink->ctx = vifs->new_ctx; + mlink->ctx = v->new_ctx; if (mlink->beacon_mon_interval) WRITE_ONCE(mlink->beacon_mon_last, jiffies); } diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index f8c2fe5f2f587..a67880a9281c5 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -186,6 +186,18 @@ mt76_dma_queue_magic_cnt_init(struct mt76_dev *dev, struct mt76_queue *q) } } +/* A hung bus (e.g. after a PCIe AER error) reads 0xffffffff from every + * register, so clamp an out-of-range index to the fallback to keep it from + * corrupting q->head/q->tail. + */ +static int +mt76_dma_read_dma_idx(struct mt76_queue *q, int fallback) +{ + u32 idx = Q_READ(q, dma_idx); + + return idx < q->ndesc ? idx : fallback; +} + static void mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) { @@ -201,7 +213,8 @@ mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) } Q_WRITE(q, desc_base, q->desc_dma); - q->head = Q_READ(q, dma_idx); + + q->head = mt76_dma_read_dma_idx(q, 0); q->tail = q->head; } @@ -419,7 +432,7 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) if (flush) last = -1; else - last = Q_READ(q, dma_idx); + last = mt76_dma_read_dma_idx(q, -1); while (q->queued > 0 && q->tail != last) { mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry); @@ -432,7 +445,7 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) } if (!flush && q->tail == last) - last = Q_READ(q, dma_idx); + last = mt76_dma_read_dma_idx(q, -1); } spin_unlock_bh(&q->cleanup_lock); @@ -534,8 +547,13 @@ mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, t->ptr = NULL; mt76_put_rxwi(dev, t); - if (drop) +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + /* the WO MCU owns the RX path only on WED v2, on newer + * versions this buf1 bit carries no drop information + */ + if (drop && dev->mmio.wed.version == 2) *drop |= !!(buf1 & MT_DMA_CTL_WO_DROP); +#endif } else { dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], SKB_WITH_OVERHEAD(q->buf_size), @@ -595,6 +613,15 @@ mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE); else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) return NULL; +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + /* on WED v3 the M_DONE bit signals that WED is done reading + * the txfree descriptor; WED v2 does not set it + */ + else if (dev->mmio.wed.version > 2 && + mt76_queue_is_wed_tx_free(q) && + !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_M_DONE))) + return NULL; +#endif } done: q->tail = (q->tail + 1) % q->ndesc; @@ -625,8 +652,8 @@ mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q, buf.len = skb->len; spin_lock_bh(&q->lock); - mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL); - mt76_dma_kick_queue(dev, q); + if (mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL) >= 0) + mt76_dma_kick_queue(dev, q); spin_unlock_bh(&q->lock); return 0; @@ -977,23 +1004,24 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) struct sk_buff *skb; unsigned char *data; bool check_ddone = false; - bool allow_direct = !mt76_queue_is_wed_rx(q); + bool allow_direct = !mt76_queue_is_wed_rx(q) && + !mt76_queue_is_wed_rro_rxdmad_c(q); bool more; if ((q->flags & MT_QFLAG_WED_RRO_EN) || (IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) && mt76_queue_is_wed_tx_free(q))) { - dma_idx = Q_READ(q, dma_idx); + dma_idx = mt76_dma_read_dma_idx(q, q->tail); check_ddone = true; } while (done < budget) { bool drop = false; - u32 info; + u32 info = 0; if (check_ddone) { if (q->tail == dma_idx) - dma_idx = Q_READ(q, dma_idx); + dma_idx = mt76_dma_read_dma_idx(q, q->tail); if (q->tail == dma_idx) break; diff --git a/drivers/net/wireless/mediatek/mt76/dma.h b/drivers/net/wireless/mediatek/mt76/dma.h index 2a0226c83f3c9..a2cf82cfdaaaa 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.h +++ b/drivers/net/wireless/mediatek/mt76/dma.h @@ -11,7 +11,7 @@ #define MT_DMA_CTL_SD_LEN1 GENMASK(13, 0) #define MT_DMA_CTL_LAST_SEC1 BIT(14) -#define MT_DMA_CTL_BURST BIT(15) +#define MT_DMA_CTL_M_DONE BIT(15) #define MT_DMA_CTL_SD_LEN0 GENMASK(29, 16) #define MT_DMA_CTL_LAST_SEC0 BIT(30) #define MT_DMA_CTL_DMA_DONE BIT(31) diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c index 13c4e8abe2819..f92277770488a 100644 --- a/drivers/net/wireless/mediatek/mt76/mac80211.c +++ b/drivers/net/wireless/mediatek/mt76/mac80211.c @@ -681,6 +681,7 @@ mt76_alloc_device(struct device *pdev, unsigned int size, dev = hw->priv; dev->hw = hw; dev->dev = pdev; + dev->init_wiphy = NULL; dev->drv = drv_ops; dev->dma_dev = pdev; @@ -779,6 +780,12 @@ int mt76_register_device(struct mt76_dev *dev, bool vht, mt76_check_sband(&dev->phy, &phy->sband_5g, NL80211_BAND_5GHZ); mt76_check_sband(&dev->phy, &phy->sband_6g, NL80211_BAND_6GHZ); + if (dev->init_wiphy) { + ret = dev->init_wiphy(dev); + if (ret) + return ret; + } + if (IS_ENABLED(CONFIG_MT76_LEDS)) { ret = mt76_led_init(phy); if (ret) @@ -886,6 +893,7 @@ static void mt76_rx_release_amsdu(struct mt76_phy *phy, enum mt76_rxq_id q) struct sk_buff *skb = phy->rx_amsdu[q].head; struct mt76_rx_status *status = (struct mt76_rx_status *)skb->cb; struct mt76_dev *dev = phy->dev; + struct mt76_queue *rxq = &dev->q_rx[q]; phy->rx_amsdu[q].head = NULL; phy->rx_amsdu[q].tail = NULL; @@ -914,6 +922,13 @@ static void mt76_rx_release_amsdu(struct mt76_phy *phy, enum mt76_rxq_id q) return; } } + + /* RRO 3.0 data queue skbs are processed and completed in the context + * of the indicator queue NAPI, which only polls its own skb list + */ + if (mt76_queue_is_wed_rro_data(rxq) && dev->hwrro_mode == MT76_HWRRO_V3) + q = MT_RXQ_RRO_IND; + __skb_queue_tail(&dev->rx_skb[q], skb); } diff --git a/drivers/net/wireless/mediatek/mt76/mcu.c b/drivers/net/wireless/mediatek/mt76/mcu.c index cbfb3bbec5031..7149b2f7aafde 100644 --- a/drivers/net/wireless/mediatek/mt76/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mcu.c @@ -78,15 +78,19 @@ int mt76_mcu_skb_send_and_get_msg(struct mt76_dev *dev, struct sk_buff *skb, unsigned long expires; int ret, seq; - if (mt76_is_sdio(dev)) - if (test_bit(MT76_RESET, &dev->phy.state) && atomic_read(&dev->bus_hung)) - return -EIO; - if (ret_skb) *ret_skb = NULL; mutex_lock(&dev->mcu.mutex); + if ((mt76_is_mmio(dev) && atomic_read(&dev->bus_hung)) || + (mt76_is_sdio(dev) && test_bit(MT76_RESET, &dev->phy.state) && + atomic_read(&dev->bus_hung))) { + orig_skb = skb; + ret = -EIO; + goto out; + } + if (dev->mcu_ops->mcu_skb_prepare_msg) { orig_skb = skb; ret = dev->mcu_ops->mcu_skb_prepare_msg(dev, skb, cmd, &seq); diff --git a/drivers/net/wireless/mediatek/mt76/mmio.c b/drivers/net/wireless/mediatek/mt76/mmio.c index 05d74cd7248e9..73d47608bf428 100644 --- a/drivers/net/wireless/mediatek/mt76/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mmio.c @@ -35,9 +35,16 @@ static void mt76_mmio_write_copy(struct mt76_dev *dev, u32 offset, { int i; - for (i = 0; i < ALIGN(len, 4); i += 4) + for (i = 0; i + 4 <= len; i += 4) writel(get_unaligned_le32(data + i), dev->mmio.regs + offset + i); + + if (i < len) { + u8 tmp[4] = {}; + + memcpy(tmp, data + i, len - i); + writel(get_unaligned_le32(tmp), dev->mmio.regs + offset + i); + } } static void mt76_mmio_read_copy(struct mt76_dev *dev, u32 offset, @@ -45,9 +52,16 @@ static void mt76_mmio_read_copy(struct mt76_dev *dev, u32 offset, { int i; - for (i = 0; i < ALIGN(len, 4); i += 4) + for (i = 0; i + 4 <= len; i += 4) put_unaligned_le32(readl(dev->mmio.regs + offset + i), data + i); + + if (i < len) { + u8 tmp[4]; + + put_unaligned_le32(readl(dev->mmio.regs + offset + i), tmp); + memcpy(data + i, tmp, len - i); + } } static int mt76_mmio_wr_rp(struct mt76_dev *dev, u32 base, diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index 3822eb8fd88fc..476578e2fbf0c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -672,6 +672,7 @@ struct mt76_usb { u8 out_ep[__MT_EP_OUT_MAX]; u8 in_ep[__MT_EP_IN_MAX]; + void (*ctrl_timeout)(struct mt76_dev *dev, int err); bool sg_en; struct mt76u_mcu { @@ -940,6 +941,9 @@ struct mt76_dev { const struct mt76_bus_ops *bus; const struct mt76_driver_ops *drv; const struct mt76_mcu_ops *mcu_ops; + + /* Optional callback to finalize wiphy state before registration. */ + int (*init_wiphy)(struct mt76_dev *dev); struct device *dev; struct device *dma_dev; @@ -2124,6 +2128,9 @@ mt76_vif_link(struct mt76_dev *dev, struct ieee80211_vif *vif, int link_id) if (!link_id) return mlink; + if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) + return NULL; + return mt76_dereference(mvif->link[link_id], dev); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/main.c b/drivers/net/wireless/mediatek/mt76/mt7615/main.c index fc619acbb40d0..67f56e428d9a1 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/main.c @@ -1239,11 +1239,12 @@ static int mt7615_suspend(struct ieee80211_hw *hw, cancel_delayed_work_sync(&dev->pm.ps_work); mt76_connac_free_pending_tx_skbs(&dev->pm, NULL); + cancel_delayed_work_sync(&phy->scan_work); + cancel_delayed_work_sync(&phy->mt76->mac_work); + mt7615_mutex_acquire(dev); clear_bit(MT76_STATE_RUNNING, &phy->mt76->state); - cancel_delayed_work_sync(&phy->scan_work); - cancel_delayed_work_sync(&phy->mt76->mac_work); set_bit(MT76_STATE_SUSPEND, &phy->mt76->state); ieee80211_iterate_active_interfaces(hw, diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac.h index 2aa6078993e9b..361f29a8b6026 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac.h @@ -48,6 +48,11 @@ enum rx_pkt_type { #define MT_TXD_LEN_MSDU_LAST BIT(14) #define MT_TXD_LEN_AMSDU_LAST BIT(15) +/* PCIE part */ +#define PCIE_AER_UNC_STATUS_OFFSET 0x204 +#define PCIE_AER_UNC_MASK_OFFSET 0x208 +#define PCIE_AER_CO_STATUS_OFFSET 0x210 + enum { CMD_CBW_20MHZ = IEEE80211_STA_RX_BW_20, CMD_CBW_40MHZ = IEEE80211_STA_RX_BW_40, @@ -245,7 +250,8 @@ static inline bool is_mt798x(struct mt76_dev *dev) static inline bool is_mt7996(struct mt76_dev *dev) { - return mt76_chip(dev) == 0x7990; + u16 chip = mt76_chip(dev); + return chip == 0x7990 || chip == 0x7991; } static inline bool is_mt7992(struct mt76_dev *dev) diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c index fc9f782032ef2..976043521f44f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c @@ -1069,6 +1069,10 @@ int mt76_connac2_mac_fill_rx_rate(struct mt76_dev *dev, bw = FIELD_GET(MT_CRXV_FRAME_MODE, v2); } + /* the hardware reports NSTS; report the data NSS for STBC frames */ + if (stbc && nss > 1) + nss >>= 1; + switch (*mode) { case MT_PHY_TYPE_CCK: cck = true; @@ -1114,7 +1118,7 @@ int mt76_connac2_mac_fill_rx_rate(struct mt76_dev *dev, case IEEE80211_STA_RX_BW_20: break; case IEEE80211_STA_RX_BW_40: - if (*mode & MT_PHY_TYPE_HE_EXT_SU && + if (*mode == MT_PHY_TYPE_HE_EXT_SU && (idx & MT_PRXV_TX_ER_SU_106T)) { status->bw = RATE_INFO_BW_HE_RU; status->he_ru = diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c index 58b0b15e4fd6f..b4d81ad02ea92 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c @@ -422,6 +422,10 @@ void mt76_connac_mcu_sta_basic_tlv(struct mt76_dev *dev, struct sk_buff *skb, basic->conn_type = cpu_to_le32(CONNECTION_IBSS_ADHOC); basic->aid = cpu_to_le16(link_sta->sta->aid); break; + case NL80211_IFTYPE_NAN: + case NL80211_IFTYPE_NAN_DATA: + basic->conn_type = cpu_to_le32(CONNECTION_NAN); + break; default: WARN_ON(1); break; @@ -753,7 +757,7 @@ mt76_connac_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta) HE_PHY(CAP3_DCM_MAX_CONST_RX_MASK, elem->phy_cap_info[3]); he->dcm_rx_max_nss = HE_PHY(CAP3_DCM_MAX_RX_NSS_2, elem->phy_cap_info[3]); - he->dcm_rx_max_nss = + he->dcm_max_ru = HE_PHY(CAP8_DCM_MAX_RU_MASK, elem->phy_cap_info[8]); he->pkt_ext = 2; @@ -1217,6 +1221,11 @@ int mt76_connac_mcu_uni_add_dev(struct mt76_phy *phy, case NL80211_IFTYPE_ADHOC: basic_req.basic.conn_type = cpu_to_le32(CONNECTION_IBSS_ADHOC); break; + case NL80211_IFTYPE_NAN: + case NL80211_IFTYPE_NAN_DATA: + basic_req.basic.conn_type = cpu_to_le32(CONNECTION_NAN); + basic_req.basic.conn_state = !enable; + break; default: WARN_ON(1); break; @@ -1627,6 +1636,11 @@ int mt76_connac_mcu_uni_add_bss(struct mt76_phy *phy, case NL80211_IFTYPE_ADHOC: basic_req.basic.conn_type = cpu_to_le32(CONNECTION_IBSS_ADHOC); break; + case NL80211_IFTYPE_NAN: + case NL80211_IFTYPE_NAN_DATA: + basic_req.basic.conn_type = cpu_to_le32(CONNECTION_NAN); + basic_req.basic.active = enable; + break; default: WARN_ON(1); break; diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h index 78f633ad81a07..a9a4a87ae0a76 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h @@ -876,6 +876,7 @@ enum { #define NETWORK_P2P BIT(17) #define NETWORK_IBSS BIT(18) #define NETWORK_WDS BIT(21) +#define NETWORK_NAN BIT(22) #define SCAN_FUNC_RANDOM_MAC BIT(0) #define SCAN_FUNC_RNR_SCAN BIT(3) @@ -888,6 +889,7 @@ enum { #define CONNECTION_IBSS_ADHOC (STA_TYPE_ADHOC | NETWORK_IBSS) #define CONNECTION_WDS (STA_TYPE_WDS | NETWORK_WDS) #define CONNECTION_INFRA_BC (STA_TYPE_BC | NETWORK_INFRA) +#define CONNECTION_NAN (NETWORK_NAN) #define CONN_STATE_DISCONNECT 0 #define CONN_STATE_CONNECT 1 @@ -1074,6 +1076,7 @@ enum { MCU_UNI_EVENT_THERMAL = 0x35, MCU_UNI_EVENT_RSSI_MONITOR = 0x41, MCU_UNI_EVENT_NIC_CAPAB = 0x43, + MCU_UNI_EVENT_NAN = 0x56, MCU_UNI_EVENT_WED_RRO = 0x57, MCU_UNI_EVENT_PER_STA_INFO = 0x6d, MCU_UNI_EVENT_ALL_STA_INFO = 0x6e, @@ -1313,6 +1316,7 @@ enum { MCU_UNI_CMD_FIXED_RATE_TABLE = 0x40, MCU_UNI_CMD_RSSI_MONITOR = 0x41, MCU_UNI_CMD_TESTMODE_CTRL = 0x46, + MCU_UNI_CMD_NAN = 0x56, MCU_UNI_CMD_RRO = 0x57, MCU_UNI_CMD_OFFCH_SCAN_CTRL = 0x58, MCU_UNI_CMD_PER_STA_INFO = 0x6d, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c index 14ee5b3b94d31..aa525adb6743c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c @@ -848,7 +848,7 @@ int mt76x02_mac_process_rx(struct mt76x02_dev *dev, struct sk_buff *skb, } } - if (WARN_ON_ONCE(len > skb->len)) + if (len > skb->len) return -EINVAL; if (pskb_trim(skb, len)) diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c index eb92cbf1a284b..fe7b29ebc0bfb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c @@ -257,7 +257,7 @@ void mt7915_eeprom_parse_hw_cap(struct mt7915_dev *dev, nss = min_t(u8, min_t(u8, nss_max, nss), path); mphy->chainmask = BIT(path) - 1; - if (band) + if (band && dev->dbdc_support) mphy->chainmask <<= dev->chainshift; mphy->antenna_mask = BIT(nss) - 1; dev->chainmask |= mphy->chainmask; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c index 250c2d2479b0c..2ab38f9e52581 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c @@ -1272,14 +1272,19 @@ int mt7915_register_device(struct mt7915_dev *dev) ret = mt7915_init_debugfs(&dev->phy); if (ret) - goto unreg_thermal; + goto unreg_ext_phy; ret = mt7915_coredump_register(dev); if (ret) - goto unreg_thermal; + goto unreg_ext_phy; return 0; +unreg_ext_phy: + if (phy2) { + mt7915_unregister_ext_phy(dev); + phy2 = NULL; + } unreg_thermal: mt7915_unregister_thermal(&dev->phy); unreg_dev: @@ -1295,6 +1300,8 @@ int mt7915_register_device(struct mt7915_dev *dev) void mt7915_unregister_device(struct mt7915_dev *dev) { cancel_work_sync(&dev->dump_work); + cancel_work_sync(&dev->reset_work); + cancel_work_sync(&dev->rc_work); mt7915_unregister_ext_phy(dev); mt7915_coredump_unregister(dev); mt7915_unregister_thermal(&dev->phy); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c index 334c19ab2b22c..0ea0261f2a29d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c @@ -221,7 +221,7 @@ static void mt7915_mac_sta_poll(struct mt7915_dev *dev) rssi[0] = to_rssi(GENMASK(7, 0), val); rssi[1] = to_rssi(GENMASK(15, 8), val); rssi[2] = to_rssi(GENMASK(23, 16), val); - rssi[3] = to_rssi(GENMASK(31, 14), val); + rssi[3] = to_rssi(GENMASK(31, 24), val); msta->ack_signal = mt76_rx_signal(msta->vif->phy->mt76->antenna_mask, rssi); @@ -437,7 +437,7 @@ mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb, if (v0 & MT_PRXV_HT_AD_CODE) status->enc_flags |= RX_ENC_FLAG_LDPC; - status->chains = mphy->antenna_mask; + status->chains = mt7915_band_chainmask(phy); status->chain_signal[0] = to_rssi(MT_PRXV_RCPI0, v1); status->chain_signal[1] = to_rssi(MT_PRXV_RCPI1, v1); status->chain_signal[2] = to_rssi(MT_PRXV_RCPI2, v1); @@ -2345,8 +2345,10 @@ void mt7915_mac_add_twt_setup(struct ieee80211_hw *hw, } flow->tsf = le64_to_cpu(twt_agrt->twt); - if (mt7915_mcu_twt_agrt_update(dev, msta->vif, flow, MCU_TWT_AGRT_ADD)) + if (mt7915_mcu_twt_agrt_update(dev, msta->vif, flow, MCU_TWT_AGRT_ADD)) { + list_del(&flow->list); goto unlock; + } setup_cmd = TWT_SETUP_CMD_ACCEPT; dev->twt.table_mask |= BIT(table_id); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index 51643a48ed151..4783e5f52d229 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -249,7 +249,7 @@ static int mt7915_add_interface(struct ieee80211_hw *hw, idx = mt76_wcid_alloc(dev->mt76.wcid_mask, mt7915_wtbl_size(dev)); if (idx < 0) { ret = -ENOSPC; - goto out; + goto err; } INIT_LIST_HEAD(&mvif->sta.rc_list); @@ -277,7 +277,17 @@ static int mt7915_add_interface(struct ieee80211_hw *hw, mt7915_mcu_add_sta(dev, vif, NULL, CONN_STATE_PORT_SECURE, true); rcu_assign_pointer(dev->mt76.wcid[idx], &mvif->sta.wcid); + mutex_unlock(&dev->mt76.mutex); + + return 0; + +err: + dev->mt76.vif_mask &= ~BIT_ULL(mvif->mt76.idx); + phy->omac_mask &= ~BIT_ULL(mvif->mt76.omac_idx); + mt7915_mcu_add_dev_info(phy, vif, false); out: + if (phy->monitor_vif == vif) + phy->monitor_vif = NULL; mutex_unlock(&dev->mt76.mutex); return ret; @@ -294,7 +304,6 @@ static void mt7915_remove_interface(struct ieee80211_hw *hw, mt7915_mcu_add_bss_info(phy, vif, false); mt7915_mcu_add_sta(dev, vif, NULL, CONN_STATE_DISCONNECT, false); - mt76_wcid_mask_clear(dev->mt76.wcid_mask, mvif->sta.wcid.idx); mutex_lock(&dev->mt76.mutex); mt76_testmode_reset(phy->mt76, true); @@ -310,6 +319,7 @@ static void mt7915_remove_interface(struct ieee80211_hw *hw, mutex_lock(&dev->mt76.mutex); dev->mt76.vif_mask &= ~BIT_ULL(mvif->mt76.idx); phy->omac_mask &= ~BIT_ULL(mvif->mt76.omac_idx); + mt76_wcid_mask_clear(dev->mt76.wcid_mask, mvif->sta.wcid.idx); mutex_unlock(&dev->mt76.mutex); spin_lock_bh(&dev->mt76.sta_poll_lock); @@ -493,7 +503,7 @@ static int mt7915_config(struct ieee80211_hw *hw, int radio_idx, mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_DMA_DCR0_RXD_G5_EN, enabled); - mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_MDP_DCR0_RX_HDR_TRANS_EN, + mt76_rmw_field(dev, MT_MDP_DCR0, MT_MDP_DCR0_RX_HDR_TRANS_EN, !dev->monitor_mask); mt76_testmode_reset(phy->mt76, true); mt76_wr(dev, MT_WF_RFCR(band), rxfilter); @@ -1128,7 +1138,7 @@ mt7915_set_antenna(struct ieee80211_hw *hw, int radio_idx, u32 tx_ant, u32 rx_an struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mt7915_phy *phy = mt7915_hw_phy(hw); int max_nss = hweight8(hw->wiphy->available_antennas_tx); - u8 chainshift = dev->chainshift; + u8 shift = mt7915_band_chainshift(phy); u8 band = phy->mt76->band_idx; if (!tx_ant || tx_ant != rx_ant || ffs(tx_ant) > max_nss) @@ -1141,9 +1151,9 @@ mt7915_set_antenna(struct ieee80211_hw *hw, int radio_idx, u32 tx_ant, u32 rx_an /* handle a variant of mt7916/mt7981 which has 3T3R but nss2 on 5 GHz band */ if ((is_mt7916(&dev->mt76) || is_mt7981(&dev->mt76)) && band && hweight8(tx_ant) == max_nss) - phy->mt76->chainmask = (dev->chainmask >> chainshift) << chainshift; + phy->mt76->chainmask = (dev->chainmask >> shift) << shift; else - phy->mt76->chainmask = tx_ant << (chainshift * band); + phy->mt76->chainmask = tx_ant << shift; mt76_set_stream_caps(phy->mt76, true); mt7915_set_stream_vht_txbf_caps(phy); @@ -1743,7 +1753,7 @@ mt7915_net_fill_forward_path(struct ieee80211_hw *hw, path->mtk_wdma.wdma_idx = wed->wdma_idx; path->mtk_wdma.bss = mvif->mt76.idx; path->mtk_wdma.wcid = is_mt7915(&dev->mt76) ? msta->wcid.idx : 0x3ff; - path->mtk_wdma.queue = phy != &dev->phy; + path->mtk_wdma.queue = phy->mt76->band_idx; ctx->dev = NULL; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c index e8fe86f933097..88955aed62e2b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c @@ -51,7 +51,7 @@ mt7915_mcu_get_sta_nss(u16 mcs_map) break; } - return nss - 1; + return nss ? nss - 1 : 0; } static void @@ -576,9 +576,9 @@ mt7915_mcu_bss_ra_tlv(struct sk_buff *skb, struct ieee80211_vif *vif, ra->rx_streams = max_nss; ra->algo = 4; ra->train_up_rule = 2; - ra->train_up_high_thres = 110; - ra->train_up_rule_rssi = -70; - ra->low_traffic_thres = 2; + ra->train_up_high_thres = cpu_to_le16(110); + ra->train_up_rule_rssi = cpu_to_le16(-70); + ra->low_traffic_thres = cpu_to_le16(2); ra->phy_cap = cpu_to_le32(0xfdf); ra->interval = cpu_to_le32(500); ra->fast_interval = cpu_to_le32(100); @@ -902,7 +902,7 @@ mt7915_mcu_sta_he_tlv(struct sk_buff *skb, struct ieee80211_sta *sta, HE_PHY(CAP3_DCM_MAX_CONST_RX_MASK, elem->phy_cap_info[3]); he->dcm_rx_max_nss = HE_PHY(CAP3_DCM_MAX_RX_NSS_2, elem->phy_cap_info[3]); - he->dcm_rx_max_nss = + he->dcm_max_ru = HE_PHY(CAP8_DCM_MAX_RU_MASK, elem->phy_cap_info[8]); he->pkt_ext = 2; @@ -1625,7 +1625,7 @@ mt7915_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7915_dev *dev, cap |= STA_CAP_VHT_SGI_160; if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_TXSTBC) cap |= STA_CAP_VHT_TX_STBC; - if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_1) + if (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_MASK) cap |= STA_CAP_VHT_RX_STBC; if (mvif->cap.vht_ldpc && (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_RXLDPC)) @@ -2804,7 +2804,7 @@ int mt7915_mcu_set_chan_info(struct mt7915_phy *phy, int cmd) .center_ch = ieee80211_frequency_to_channel(freq1), .bw = mt76_connac_chan_bw(chandef), .tx_path_num = hweight16(phy->mt76->chainmask), - .rx_path = phy->mt76->chainmask >> (dev->chainshift * band), + .rx_path = mt7915_band_chainmask(phy), .band_idx = band, .channel_band = ch_band[chandef->chan->band], }; @@ -2917,8 +2917,15 @@ int mt7915_mcu_get_eeprom(struct mt7915_dev *dev, u32 offset, u8 *read_buf) return ret; res = (struct mt7915_mcu_eeprom_info *)skb->data; - if (!buf) - buf = dev->mt76.eeprom.data + le32_to_cpu(res->addr); + if (!buf) { + u32 addr = le32_to_cpu(res->addr); + + if (addr > dev->mt76.eeprom.size - MT7915_EEPROM_BLOCK_SIZE) { + dev_kfree_skb(skb); + return -EINVAL; + } + buf = dev->mt76.eeprom.data + addr; + } memcpy(buf, res->data, MT7915_EEPROM_BLOCK_SIZE); dev_kfree_skb(skb); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h index 22f73a5ed4259..7c472062a90e1 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h @@ -318,9 +318,9 @@ struct bss_info_ra { u8 antenna_idx; u8 train_up_rule; u8 rsv[3]; - unsigned short train_up_high_thres; - short train_up_rule_rssi; - unsigned short low_traffic_thres; + __le16 train_up_high_thres; + __le16 train_up_rule_rssi; + __le16 low_traffic_thres; __le16 max_phyrate; __le32 phy_cap; __le32 interval; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h index bf1d915a3ca23..43479f6487ede 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h @@ -397,6 +397,24 @@ mt7915_ext_phy(struct mt7915_dev *dev) return phy->priv; } +/* without dbdc, the chainmask is stored unshifted, even if the phy is + * bound to band 1 + */ +static inline u8 mt7915_band_chainshift(struct mt7915_phy *phy) +{ + struct mt7915_dev *dev = phy->dev; + + if (!dev->dbdc_support) + return 0; + + return phy->mt76->band_idx * dev->chainshift; +} + +static inline u16 mt7915_band_chainmask(struct mt7915_phy *phy) +{ + return phy->mt76->chainmask >> mt7915_band_chainshift(phy); +} + static inline u32 mt7915_check_adie(struct mt7915_dev *dev, bool sku) { u32 mask = sku ? MT_CONNINFRA_SKU_MASK : MT_ADIE_TYPE_MASK; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c index f6b03211a879b..8007e620048b3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c @@ -135,7 +135,6 @@ static int mt7915_pci_probe(struct pci_dev *pdev, mdev = &dev->mt76; mt7915_wfsys_reset(dev); - hif2 = mt7915_pci_init_hif2(pdev); ret = mt7915_mmio_wed_init(dev, pdev, true, &irq); if (ret < 0) @@ -145,16 +144,20 @@ static int mt7915_pci_probe(struct pci_dev *pdev, hif2 = mt7915_pci_init_hif2(pdev); ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); - if (ret < 0) + if (ret < 0) { + mt7915_put_hif2(hif2); goto free_device; + } irq = pdev->irq; } ret = devm_request_irq(mdev->dev, irq, mt7915_irq_handler, IRQF_SHARED, KBUILD_MODNAME, dev); - if (ret) + if (ret) { + mt7915_put_hif2(hif2); goto free_wed_or_irq_vector; + } /* master switch of PCIe tnterrupt enable */ mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c index 54ff6de96f3ea..13fba2a061c78 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c @@ -908,7 +908,7 @@ static void mt7986_wmac_clock_enable(struct mt7915_dev *dev, u32 adie_type) read_poll_timeout(mt76_rr, cur, !(cur & MT_SLP_CTRL_BSY_MASK), USEC_PER_MSEC, 50 * USEC_PER_MSEC, false, - dev, MT_ADIE_SLP_CTRL_CK0(0)); + dev, MT_ADIE_SLP_CTRL_CK0(1)); } mt76_wmac_spi_unlock(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c b/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c index 618a5c2bdd29f..7576973f4d4ef 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c @@ -694,9 +694,7 @@ mt7915_tm_set_params(struct mt76_phy *mphy, struct nlattr **tb, { struct mt76_testmode_data *td = &mphy->test; struct mt7915_phy *phy = mphy->priv; - struct mt7915_dev *dev = phy->dev; - u32 chainmask = mphy->chainmask, changed = 0; - bool ext_phy = phy != &dev->phy; + u32 chainmask = mt7915_band_chainmask(phy), changed = 0; int i; BUILD_BUG_ON(NUM_TM_CHANGED >= 32); @@ -705,7 +703,6 @@ mt7915_tm_set_params(struct mt76_phy *mphy, struct nlattr **tb, td->state == MT76_TM_STATE_OFF) return 0; - chainmask = ext_phy ? chainmask >> dev->chainshift : chainmask; if (td->tx_antenna_mask > chainmask) return -EINVAL; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c index f7d54472da1b1..e69978184f68c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c @@ -156,7 +156,7 @@ static void mt7921_mac_sta_poll(struct mt792x_dev *dev) rssi[0] = to_rssi(GENMASK(7, 0), val); rssi[1] = to_rssi(GENMASK(15, 8), val); rssi[2] = to_rssi(GENMASK(23, 16), val); - rssi[3] = to_rssi(GENMASK(31, 14), val); + rssi[3] = to_rssi(GENMASK(31, 24), val); mlink->ack_signal = mt76_rx_signal(msta->vif->phy->mt76->antenna_mask, rssi); @@ -674,6 +674,9 @@ void mt7921_mac_reset_work(struct work_struct *work) cancel_work_sync(&pm->wake_work); for (i = 0; i < 10; i++) { + if (atomic_read(&dev->mt76.bus_hung)) + return; + mutex_lock(&dev->mt76.mutex); ret = mt792x_dev_reset(dev); mutex_unlock(&dev->mt76.mutex); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c index 25b9437250f7b..45439652c847b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c @@ -415,7 +415,8 @@ static int mt7921_load_clc(struct mt792x_dev *dev, const char *fw_name) struct mt76_dev *mdev = &dev->mt76; struct mt792x_phy *phy = &dev->phy; const struct firmware *fw; - int ret, i, len, offset = 0; + size_t clc_len, fw_data_len, len, offset = 0; + int ret, i; u8 *clc_base = NULL, hw_encap = 0; dev->phy.clc_chan_conf = 0xff; @@ -441,13 +442,21 @@ static int mt7921_load_clc(struct mt792x_dev *dev, const char *fw_name) } hdr = (const void *)(fw->data + fw->size - sizeof(*hdr)); + if (hdr->n_region > (fw->size - sizeof(*hdr)) / sizeof(*region)) { + dev_err(mdev->dev, "Invalid firmware region table\n"); + ret = -EINVAL; + goto out; + } + fw_data_len = fw->size - sizeof(*hdr) - + hdr->n_region * sizeof(*region); + for (i = 0; i < hdr->n_region; i++) { region = (const void *)((const u8 *)hdr - (hdr->n_region - i) * sizeof(*region)); len = le32_to_cpu(region->len); /* check if we have valid buffer size */ - if (offset + len > fw->size) { + if (len > fw_data_len - offset) { dev_err(mdev->dev, "Invalid firmware region\n"); ret = -EINVAL; goto out; @@ -464,8 +473,24 @@ static int mt7921_load_clc(struct mt792x_dev *dev, const char *fw_name) if (!clc_base) goto out; - for (offset = 0; offset < len; offset += le32_to_cpu(clc->len)) { + for (offset = 0; offset < len; offset += clc_len) { + if (len - offset < sizeof(*clc)) { + dev_err(mdev->dev, "Invalid CLC record\n"); + ret = -EINVAL; + goto out; + } + clc = (const struct mt7921_clc *)(clc_base + offset); + clc_len = le32_to_cpu(clc->len); + if (clc_len < sizeof(*clc) || clc_len > len - offset) { + dev_err(mdev->dev, "Invalid CLC record\n"); + ret = -EINVAL; + goto out; + } + + /* Newer firmware may add records this driver does not use yet */ + if (clc->idx >= ARRAY_SIZE(phy->clc)) + continue; /* do not init buf again if chip reset triggered */ if (phy->clc[clc->idx]) @@ -477,7 +502,7 @@ static int mt7921_load_clc(struct mt792x_dev *dev, const char *fw_name) continue; phy->clc[clc->idx] = devm_kmemdup(mdev->dev, clc, - le32_to_cpu(clc->len), + clc_len, GFP_KERNEL); if (!phy->clc[clc->idx]) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c index 7728c5ae67914..4617178fb1c46 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c @@ -600,6 +600,108 @@ static int mt7921_pci_resume(struct device *device) return err; } +static u32 mt7921_aer_rr(struct mt76_dev *mdev, u32 offset) +{ + return 0; +} + +static void mt7921_aer_wr(struct mt76_dev *mdev, u32 offset, u32 val) +{ + ; +} + +static u32 mt791_aer_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val) +{ + return 0; +} + +static const struct mt76_bus_ops mt7921_aer_bus_hung_ops = { + .rr = mt7921_aer_rr, + .wr = mt7921_aer_wr, + .rmw = mt791_aer_rmw, + .type = MT76_BUS_MMIO +}; + +static void mt7921_pci_set_aer_bus_hung_ops(struct mt792x_dev *dev) +{ + if (READ_ONCE(dev->mt76.bus) == &mt7921_aer_bus_hung_ops) + return; + + atomic_set(&dev->mt76.bus_hung, true); + WRITE_ONCE(dev->mt76.bus, &mt7921_aer_bus_hung_ops); +} + +static pci_ers_result_t mt7921_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct mt76_dev *mdev = pci_get_drvdata(pdev); + struct mt792x_dev *dev = container_of(mdev, struct mt792x_dev, mt76); + u32 aer_unc_val = 0, aer_co_val = 0; + + dev_err(mdev->dev, "PCIE error detect state: %d\n", state); + + /* Clear SW IRQ tasklet first */ + tasklet_kill(&mdev->irq_tasklet); + + if (state == pci_channel_io_perm_failure) { + mt7921_pci_set_aer_bus_hung_ops(dev); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_read_config_dword(pdev, PCIE_AER_UNC_STATUS_OFFSET, &aer_unc_val); + pci_read_config_dword(pdev, PCIE_AER_CO_STATUS_OFFSET, &aer_co_val); + + dev_warn(mdev->dev, "PCIE_AER_UNC_STATUS_OFFSET: 0x%x\n", aer_unc_val); + dev_warn(mdev->dev, "PCIE_AER_CO_STATUS_OFFSET: 0x%x\n", aer_co_val); + + /** + * Due to this error is from link error and this AER is un-correctable, + * so can't covered by device + **/ + if (aer_unc_val != 0) { + mt7921_pci_set_aer_bus_hung_ops(dev); + return PCI_ERS_RESULT_DISCONNECT; + } + + /** + * Try to recover it when state is pci_channel_io_frozen or + * AER is correctable error + **/ + if (state == pci_channel_io_frozen || aer_co_val != 0) { + /* Disable PCIE activity first. */ + pci_disable_device(pdev); + return PCI_ERS_RESULT_NEED_RESET; + } + + return PCI_ERS_RESULT_NONE; +} + +static pci_ers_result_t mt7921_slot_reset(struct pci_dev *pdev) +{ + struct mt76_dev *mdev = pci_get_drvdata(pdev); + int ret = 0; + + ret = pci_enable_device_mem(pdev); + + if (ret) { + dev_err(mdev->dev, "pci_enable_device_mem failed: %d\n", ret); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_set_master(pdev); + pci_restore_state(pdev); + pci_save_state(pdev); + /* Also try do the vendor reset to let it more clear. */ + mt792x_reset(mdev); + + return PCI_ERS_RESULT_RECOVERED; +} + +static const struct pci_error_handlers mt7921_err_handler = { + .error_detected = mt7921_error_detected, + .slot_reset = mt7921_slot_reset, +}; + static void mt7921_pci_shutdown(struct pci_dev *pdev) { mt7921_pci_remove(pdev); @@ -614,6 +716,7 @@ static struct pci_driver mt7921_pci_driver = { .remove = mt7921_pci_remove, .shutdown = mt7921_pci_shutdown, .driver.pm = pm_sleep_ptr(&mt7921_pm_ops), + .err_handler = &mt7921_err_handler, }; module_pci_driver(mt7921_pci_driver); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/Makefile b/drivers/net/wireless/mediatek/mt76/mt7925/Makefile index 8f1078ce32316..f9dcc0bba393c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/Makefile +++ b/drivers/net/wireless/mediatek/mt76/mt7925/Makefile @@ -4,7 +4,7 @@ obj-$(CONFIG_MT7925_COMMON) += mt7925-common.o obj-$(CONFIG_MT7925E) += mt7925e.o obj-$(CONFIG_MT7925U) += mt7925u.o -mt7925-common-y := mac.o mcu.o regd.o main.o init.o debugfs.o +mt7925-common-y := mac.o mcu.o regd.o main.o init.o debugfs.o nan.o mt7925-common-$(CONFIG_NL80211_TESTMODE) += testmode.o mt7925e-y := pci.o pci_mac.o pci_mcu.o mt7925u-y := usb.o diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index 6b0cd1996ecb3..b52b678330604 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -145,7 +145,7 @@ static void mt7925_mac_sta_poll(struct mt792x_dev *dev) rssi[0] = to_rssi(GENMASK(7, 0), val); rssi[1] = to_rssi(GENMASK(15, 8), val); rssi[2] = to_rssi(GENMASK(23, 16), val); - rssi[3] = to_rssi(GENMASK(31, 14), val); + rssi[3] = to_rssi(GENMASK(31, 24), val); mlink->ack_signal = mt76_rx_signal(msta->vif->phy->mt76->antenna_mask, rssi); @@ -1323,6 +1323,7 @@ void mt7925_mac_reset_work(struct work_struct *work) cancel_delayed_work_sync(&dev->mphy.mac_work); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); for (i = 0; i < 10; i++) { @@ -1434,6 +1435,10 @@ int mt7925_usb_sdio_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, if (!wcid) wcid = &dev->mt76.global_wcid; + err = skb_cow_head(skb, MT_SDIO_TXD_SIZE + MT_SDIO_HDR_SIZE); + if (err) + return err; + if (sta) { struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv; diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 2b6cc8e253c0e..9f080da13341b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -187,19 +187,21 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, eht_cap_elem->phy_cap_info[0] |= IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; + val = (sts > 3) ? sts - 1 : 3; + eht_cap_elem->phy_cap_info[0] |= - u8_encode_bits(u8_get_bits(sts - 1, BIT(0)), + u8_encode_bits(u8_get_bits(val, BIT(0)), IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK); eht_cap_elem->phy_cap_info[1] = - u8_encode_bits(u8_get_bits(sts - 1, GENMASK(2, 1)), + u8_encode_bits(u8_get_bits(val, GENMASK(2, 1)), IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK) | - u8_encode_bits(sts - 1, + u8_encode_bits(val, IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK); if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) eht_cap_elem->phy_cap_info[1] |= - u8_encode_bits(sts - 1, + u8_encode_bits(val, IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK); eht_cap_elem->phy_cap_info[2] = @@ -1640,6 +1642,7 @@ static int mt7925_suspend(struct ieee80211_hw *hw, cancel_delayed_work_sync(&phy->mt76->mac_work); cancel_delayed_work_sync(&dev->pm.ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); mt76_connac_free_pending_tx_skbs(&dev->pm, NULL); mt792x_mutex_acquire(dev); @@ -1827,9 +1830,15 @@ static int mt7925_set_sar_specs(struct ieee80211_hw *hw, int err; mt792x_mutex_acquire(dev); + err = mt7925_mcu_set_clc(dev, dev->mt76.alpha2, + dev->country_ie_env); + if (err < 0) + goto out; + err = mt7925_set_tx_sar_pwr(hw, sar); - mt792x_mutex_release(dev); +out: + mt792x_mutex_release(dev); return err; } @@ -2484,10 +2493,53 @@ static void mt7925_channel_switch_rx_beacon(struct ieee80211_hw *hw, } } +static void mt7925_stop(struct ieee80211_hw *hw, bool suspend) +{ + struct mt792x_dev *dev = mt792x_hw_dev(hw); + + cancel_delayed_work_sync(&dev->mlo_pm_work); + + mt792x_stop(hw, suspend); +} + +static void mt7925_sta_pre_rcu_remove(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt76_phy *phy = hw->priv; + struct mt76_dev *dev = phy->dev; + struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; + + mutex_lock(&dev->mutex); + spin_lock_bh(&dev->status_lock); + + if (ieee80211_vif_is_mld(vif)) { + struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv; + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + unsigned long valid = mvif->valid_links; + struct mt792x_link_sta *mlink; + unsigned int link_id; + + for_each_set_bit(link_id, &valid, IEEE80211_MLD_MAX_NUM_LINKS) { + mlink = mt792x_sta_to_link(msta, link_id); + if (!mlink || !mlink->wcid.sta) + continue; + if (mlink->wcid.idx < ARRAY_SIZE(dev->wcid)) + rcu_assign_pointer(dev->wcid[mlink->wcid.idx], + NULL); + } + } else { + rcu_assign_pointer(dev->wcid[wcid->idx], NULL); + } + + spin_unlock_bh(&dev->status_lock); + mutex_unlock(&dev->mutex); +} + const struct ieee80211_ops mt7925_ops = { .tx = mt792x_tx, .start = mt7925_start, - .stop = mt792x_stop, + .stop = mt7925_stop, .add_interface = mt7925_add_interface, .remove_interface = mt792x_remove_interface, .config = mt7925_config, @@ -2496,7 +2548,7 @@ const struct ieee80211_ops mt7925_ops = { .start_ap = mt7925_start_ap, .stop_ap = mt7925_stop_ap, .sta_state = mt76_sta_state, - .sta_pre_rcu_remove = mt76_sta_pre_rcu_remove, + .sta_pre_rcu_remove = mt7925_sta_pre_rcu_remove, .set_key = mt7925_set_key, .sta_set_decap_offload = mt7925_sta_set_decap_offload, #if IS_ENABLED(CONFIG_IPV6) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index cb265a6fc7adb..53c142345a086 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -2174,6 +2174,8 @@ int mt7925_get_txpwr_info(struct mt792x_dev *dev, u8 band_idx, struct mt7925_txp int mt7925_mcu_set_sniffer(struct mt792x_dev *dev, struct ieee80211_vif *vif, bool enable) { + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + struct ieee80211_chanctx_conf *ctx = mvif->bss_conf.mt76.ctx; struct { struct { u8 band_idx; @@ -2196,6 +2198,15 @@ int mt7925_mcu_set_sniffer(struct mt792x_dev *dev, struct ieee80211_vif *vif, }, }; + if (is_mt7927(&dev->mt76)) { + struct ieee80211_channel *chan; + + chan = ctx ? ctx->def.chan : mvif->phy->mt76->chandef.chan; + + if (chan) + req.hdr.band_idx = mt7927_band_idx(chan->band); + } + return mt76_mcu_send_msg(&dev->mt76, MCU_UNI_CMD(SNIFFER), &req, sizeof(req), true); } @@ -2255,6 +2266,9 @@ int mt7925_mcu_config_sniffer(struct mt792x_vif *vif, }, }; + if (is_mt7927(mphy->dev)) + req.hdr.band_idx = mt7927_band_idx(chandef->chan->band); + if (chandef->chan->band < ARRAY_SIZE(ch_band)) req.tlv.ch_band = ch_band[chandef->chan->band]; if (chandef->width < ARRAY_SIZE(ch_width)) @@ -3829,7 +3843,7 @@ mt7925_mcu_rate_txpower_band(struct mt76_phy *phy, memcpy(tx_power_tlv->alpha2, dev->alpha2, sizeof(dev->alpha2)); tx_power_tlv->n_chan = num_ch; tx_power_tlv->tag = cpu_to_le16(0x1); - tx_power_tlv->len = cpu_to_le16(msg_len); + tx_power_tlv->len = cpu_to_le16(msg_len - 4); switch (band) { case NL80211_BAND_2GHZ: diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/nan.c b/drivers/net/wireless/mediatek/mt76/mt7925/nan.c new file mode 100644 index 0000000000000..74db344a67969 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7925/nan.c @@ -0,0 +1,927 @@ +// SPDX-License-Identifier: BSD-3-Clause-Clear +/* Copyright (C) 2025-2026 MediaTek Inc. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mt7925.h" +#include "mcu.h" +#include "nan.h" +#include "regd.h" + +static void mt7925_nan_set_5g_channel(struct mt792x_dev *dev, + struct mt7925_nan_enable_req_tlv *req, + struct cfg80211_nan_conf *conf) +{ + struct ieee80211_channel *chan; + u32 ch5g = 0; + + chan = conf->band_cfgs[NL80211_BAND_5GHZ].chan; + + if (!chan) + return; + + if (!mt7925_regd_is_valid_channel(dev, NL80211_BAND_5GHZ, chan)) + return; + + req->config_5g_channel = 1; + + if (chan->hw_value == NAN_5G_LOW_DISC_CHANNEL) + ch5g |= BIT(0); + else if (chan->hw_value == NAN_5G_HIGH_DISC_CHANNEL) + ch5g |= BIT(1); + + req->channel_5g_val = cpu_to_le32(ch5g); +} + +static void mt7925_nan_set_cluster_id(struct mt7925_nan_enable_req_tlv *req, + const u8 *cluster_id) +{ + if (!cluster_id) + return; + + req->cluster_high = cpu_to_le16(cluster_id[4] | cluster_id[5] << 8); + req->cluster_low = cpu_to_le16((u16)cluster_id[3]); +} + +static void mt7925_nan_set_dw_interval(struct mt7925_nan_enable_req_tlv *req, + struct cfg80211_nan_conf *conf) +{ + if (conf->band_cfgs[NL80211_BAND_2GHZ].awake_dw_interval > 0) { + req->config_dw.config_2dot4g_dw_band = 1; + req->config_dw.dw_2dot4g_interval_val = + cpu_to_le32(conf->band_cfgs[NL80211_BAND_2GHZ].awake_dw_interval); + } + + if (conf->band_cfgs[NL80211_BAND_5GHZ].awake_dw_interval > 0) { + req->config_dw.config_5g_dw_band = 1; + req->config_dw.dw_5g_interval_val = + cpu_to_le32(conf->band_cfgs[NL80211_BAND_5GHZ].awake_dw_interval); + } +} + +static void mt7925_nan_set_disc_beacon(struct mt7925_nan_enable_req_tlv *req, + struct cfg80211_nan_conf *conf) +{ + if (conf->discovery_beacon_interval > 0) { + req->config_2dot4g_beacons = true; + req->beacon_2dot4g_val = conf->discovery_beacon_interval; + } +} + +static void mt7925_nan_set_rssi_thresholds(struct mt7925_nan_enable_req_tlv *req, + struct cfg80211_nan_conf *conf) +{ + if (conf->band_cfgs[NL80211_BAND_2GHZ].chan) { + req->config_2dot4g_rssi_close = 1; + req->rssi_close_2dot4g_val = + abs(conf->band_cfgs[NL80211_BAND_2GHZ].rssi_close); + req->config_2dot4g_rssi_middle = 1; + req->rssi_middle_2dot4g_val = + abs(conf->band_cfgs[NL80211_BAND_2GHZ].rssi_middle); + } + + if (conf->band_cfgs[NL80211_BAND_5GHZ].chan) { + req->config_5g_rssi_close = 1; + req->rssi_close_5g_val = + abs(conf->band_cfgs[NL80211_BAND_5GHZ].rssi_close); + req->config_5g_rssi_middle = 1; + req->rssi_middle_5g_val = + abs(conf->band_cfgs[NL80211_BAND_5GHZ].rssi_middle); + } +} + +static void mt7925_nan_set_scan_params(struct mt7925_nan_enable_req_tlv *req, + struct cfg80211_nan_conf *conf) +{ + req->scan_params_val.scan_period[0] = + cpu_to_le16(conf->scan_period < 255 ? conf->scan_period : 255); + req->scan_params_val.dwell_time[0] = + conf->scan_dwell_time < 255 ? conf->scan_dwell_time : 255; +} + +static u16 +mt7925_nan_avail_attr_ctrl(const struct ieee80211_nan_sched_cfg *sched) +{ + if (sched->avail_blob_len < NAN_AVAIL_ATTR_CTRL_OFFSET + 2) + return 0; + + return sched->avail_blob[NAN_AVAIL_ATTR_CTRL_OFFSET] | + sched->avail_blob[NAN_AVAIL_ATTR_CTRL_OFFSET + 1] << 8; +} + +static void +mt7925_nan_update_conf(struct mt792x_vif *mvif, + const struct cfg80211_nan_conf *conf) +{ + mvif->nan.conf.master_pref = conf->master_pref; + mvif->nan.conf.bands = conf->bands; + mvif->nan.conf.discovery_beacon_interval = + conf->discovery_beacon_interval; + mvif->nan.conf.enable_dw_notification = + conf->enable_dw_notification; + + memcpy(mvif->nan.conf.cluster_id, conf->cluster_id, ETH_ALEN); +} + +int mt7925_nan_enable(struct ieee80211_vif *vif, + struct mt792x_dev *dev, + struct cfg80211_nan_conf *conf) +{ + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + struct mt76_dev *mdev = &dev->mt76; + struct { + u8 rsv[4]; + struct mt7925_nan_enable_req_tlv nan_req_tlv; + } nan_cmd = { + .rsv = { 0 }, + .nan_req_tlv = { + .tag = cpu_to_le16(NAN_UNI_CMD_ENABLE_REQUEST), + .len = cpu_to_le16(sizeof(struct mt7925_nan_enable_req_tlv)), + .config_random_factor_force = 0, + .random_factor_force_val = 0, + .config_hop_count_force = 0, + .hop_count_force_val = 0, + }, + }; + struct mt7925_nan_enable_req_tlv *p_nan_req_tlv = &nan_cmd.nan_req_tlv; + + if (!vif || !dev || !conf) + return -EINVAL; + + p_nan_req_tlv->master_pref = conf->master_pref; + + mt7925_nan_set_5g_channel(dev, p_nan_req_tlv, conf); + mt7925_nan_set_cluster_id(p_nan_req_tlv, conf->cluster_id); + mt7925_nan_set_dw_interval(p_nan_req_tlv, conf); + mt7925_nan_set_disc_beacon(p_nan_req_tlv, conf); + mt7925_nan_set_rssi_thresholds(p_nan_req_tlv, conf); + mt7925_nan_set_scan_params(p_nan_req_tlv, conf); + + mt7925_nan_update_conf(mvif, conf); + + return mt76_mcu_send_msg(mdev, MCU_UNI_CMD(NAN), &nan_cmd, sizeof(nan_cmd), true); +} + +int mt7925_nan_disable(struct ieee80211_vif *vif, struct mt792x_dev *dev) +{ + struct mt76_dev *mdev = &dev->mt76; + struct { + u8 rsv[4]; + struct tlv nan_dis_tlv; + } nan_cmd = { + .rsv = { 0 }, + .nan_dis_tlv = { + .tag = cpu_to_le16(NAN_UNI_CMD_DISABLE_REQUEST), + .len = cpu_to_le16(sizeof(struct tlv)), + }, + }; + + if (!dev) + return -EINVAL; + + return mt76_mcu_send_msg(mdev, MCU_UNI_CMD(NAN), &nan_cmd, sizeof(nan_cmd), true); +} + +static int +mt7925_nan_mp_tlv(struct sk_buff *skb, u8 master_pref) +{ + struct mt7925_nan_master_preference_tlv *mp_tlv = NULL; + struct tlv *tlv = NULL; + + if (!skb) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_SET_MASTER_PREFERENCE, + sizeof(struct mt7925_nan_master_preference_tlv)); + if (!tlv) + return -ENOMEM; + + mp_tlv = (struct mt7925_nan_master_preference_tlv *)tlv; + + if (master_pref > NAN_MAX_MASTER_PREFERENCE) + return 0; + + mp_tlv->master_preference = master_pref; + + return 0; +} + +static int +mt7925_nan_dw_tlv(struct sk_buff *skb, struct cfg80211_nan_conf *conf) +{ + struct mt7925_nan_dw_interval_tlv *dw_tlv = NULL; + struct tlv *tlv = NULL; + u16 interval; + + if (!skb || !conf) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_SET_DW_INTERVAL, + sizeof(struct mt7925_nan_dw_interval_tlv)); + + if (!tlv) + return -ENOMEM; + + dw_tlv = (struct mt7925_nan_dw_interval_tlv *)tlv; + + /* Set DW interval for 2.4GHz and 5GHz bands if available */ + if (conf->band_cfgs[NL80211_BAND_2GHZ].awake_dw_interval > 0) { + dw_tlv->dw_interval = conf->band_cfgs[NL80211_BAND_2GHZ].awake_dw_interval; + } else if (conf->band_cfgs[NL80211_BAND_5GHZ].awake_dw_interval > 0) { + dw_tlv->dw_interval = conf->band_cfgs[NL80211_BAND_5GHZ].awake_dw_interval; + } else { + /* Fallback to a default value or log a warning */ + dw_tlv->dw_interval = NAN_DEFAULT_DW_INTERVAL; + } + + /* Validate and set NAN Discovery Beacon Interval */ + interval = conf->discovery_beacon_interval > 0 ? + conf->discovery_beacon_interval : + NAN_DEFAULT_DISC_BCN_INTERVAL; + + dw_tlv->disc_bcn_interval = cpu_to_le16(interval); + + return 0; +} + +static int +mt7925_nan_cluster_id_tlv(struct sk_buff *skb, const u8 *cluster_id) +{ + struct mt7925_nan_cluster_id_tlv *cluster_tlv = NULL; + struct tlv *tlv = NULL; + + if (!skb || !cluster_id) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_SET_CLUSTER_ID, + sizeof(struct mt7925_nan_cluster_id_tlv)); + + if (!tlv) + return -ENOMEM; + + cluster_tlv = (struct mt7925_nan_cluster_id_tlv *)tlv; + + memcpy(cluster_tlv->cluster_id, cluster_id, ETH_ALEN); + + return 0; +} + +static int +mt7925_nan_sync_rssi_tlv(struct sk_buff *skb, struct cfg80211_nan_conf *conf) +{ + struct mt7925_nan_sync_rssi_tlv *rssi_tlv = NULL; + struct tlv *tlv = NULL; + + if (!skb || !conf) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_SET_SYNC_RSSI, + sizeof(struct mt7925_nan_sync_rssi_tlv)); + + if (!tlv) + return -ENOMEM; + + rssi_tlv = (struct mt7925_nan_sync_rssi_tlv *)tlv; + + if (conf->band_cfgs[NL80211_BAND_2GHZ].chan) { + rssi_tlv->rssi_close_2g = + conf->band_cfgs[NL80211_BAND_2GHZ].rssi_close; + rssi_tlv->rssi_middle_2g = + conf->band_cfgs[NL80211_BAND_2GHZ].rssi_middle; + } + + if (conf->band_cfgs[NL80211_BAND_5GHZ].chan) { + rssi_tlv->rssi_close_5g = + conf->band_cfgs[NL80211_BAND_5GHZ].rssi_close; + rssi_tlv->rssi_middle_5g = + conf->band_cfgs[NL80211_BAND_5GHZ].rssi_middle; + } + + return 0; +} + +int mt7925_nan_change_configure(struct ieee80211_vif *vif, + struct mt792x_dev *dev, + struct cfg80211_nan_conf *conf) +{ + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + struct mt7925_nan_common_hdr *hdr = NULL; + struct mt76_dev *mdev = &dev->mt76; + struct sk_buff *skb = NULL; + + if (!vif || !dev || !conf) + return -EINVAL; + + skb = mt76_mcu_msg_alloc(mdev, NULL, MT7925_NAN_CONF_MAX_SIZE); + if (!skb) + return -ENOMEM; + + hdr = (struct mt7925_nan_common_hdr *)skb_put(skb, sizeof(*hdr)); + memset(hdr, 0, sizeof(*hdr)); + + if (mt7925_nan_mp_tlv(skb, conf->master_pref) || + mt7925_nan_dw_tlv(skb, conf) || + mt7925_nan_cluster_id_tlv(skb, conf->cluster_id) || + mt7925_nan_sync_rssi_tlv(skb, conf)) { + dev_kfree_skb(skb); + return -ENOMEM; + } + + mt7925_nan_update_conf(mvif, conf); + + return mt76_mcu_skb_send_msg(mdev, skb, + MCU_UNI_CMD(NAN), true); +} + +static void +mt7925_nan_handle_dw_ind(struct mt792x_dev *dev, struct tlv *tlv) +{ + struct ieee80211_channel *chan; + struct nan_rpt_dw_evt *evt; + struct wireless_dev *wdev; + u16 len, channel, dw_num; + struct mt792x_vif *mvif; + enum nl80211_band band; + int freq; + + if (!dev || !tlv) + return; + + len = le16_to_cpu(tlv->len); + if (len < sizeof(*tlv) + sizeof(*evt)) { + dev_warn(dev->mt76.dev, + "nan: short dw event tlv len=%u\n", len); + return; + } + + if (!dev->nan_vif || !ieee80211_vif_nan_started(dev->nan_vif)) + return; + + wdev = ieee80211_vif_to_wdev(dev->nan_vif); + if (!wdev) + return; + + mvif = (struct mt792x_vif *)dev->nan_vif->drv_priv; + if (!mvif->nan.conf.enable_dw_notification) + return; + + evt = (struct nan_rpt_dw_evt *)tlv->data; + channel = le16_to_cpu(evt->channel); + dw_num = le16_to_cpu(evt->dw_num); + + band = channel > 13 ? NL80211_BAND_5GHZ : NL80211_BAND_2GHZ; + freq = ieee80211_channel_to_frequency(channel, band); + chan = ieee80211_get_channel(dev->mt76.hw->wiphy, freq); + if (!chan) { + dev_dbg(dev->mt76.dev, + "nan: no channel for dw end event ch=%u dw=%u\n", + channel, dw_num); + return; + } + + cfg80211_next_nan_dw_notif(wdev, chan, GFP_KERNEL); +} + +static void +mt7925_nan_mcu_handle_de_event(struct mt792x_dev *dev, struct tlv *tlv) +{ + u8 cluster_id[ETH_ALEN] __aligned(2) = {0x50, 0x6f, 0x9a, 0x01, 0x00, 0x00}; + struct mt7925_nan_de_event *de_evt = NULL; + u16 len; + + if (!dev || !tlv) { + if (dev) + dev_warn(dev->mt76.dev, "nan: failed to parse TLV\n"); + return; + } + + len = le16_to_cpu(tlv->len); + if (len < sizeof(*tlv) + sizeof(*de_evt)) { + dev_warn(dev->mt76.dev, + "nan: short de_event tlv len=%u\n", len); + return; + } + + de_evt = (struct mt7925_nan_de_event *)tlv->data; + if (!de_evt) { + dev_warn(dev->mt76.dev, "nan: missing DE event payload\n"); + return; + } + + if (de_evt->event_type == NAN_EVENT_ID_DISC_MAC_ADDR) + return; + + memcpy(cluster_id, de_evt->cluster_id, ETH_ALEN); + + dev_dbg(dev->mt76.dev, "nan: evt=%u cluster=%pM\n", + de_evt->event_type, de_evt->cluster_id); + + if (de_evt->event_type != NAN_EVENT_ID_JOINED_CLUSTER) + return; + + if (!ieee80211_vif_nan_started(dev->nan_vif)) { + dev_warn(dev->mt76.dev, "nan: joined-cluster event but NAN not started\n"); + return; + } + + dev_dbg(dev->mt76.dev, "nan: anchor_master_rank=%*phN\n", + NAN_ANCHOR_MASTER_RANK_NUM, de_evt->anchor_master_rank); + + dev_dbg(dev->mt76.dev, "nan: own_nmi=%pM master_nmi=%pM\n", + de_evt->own_nmi, de_evt->master_nmi); + + ieee80211_nan_cluster_joined(dev->nan_vif, cluster_id, true, GFP_KERNEL); +} + +void mt7925_nan_mcu_event(struct mt792x_dev *dev, struct sk_buff *skb) +{ + struct tlv *tlv; + u32 tlv_len; + + if (!dev || !skb) + return; + + if (skb->len < sizeof(struct mt7925_mcu_rxd) + 4) + return; + + skb_pull(skb, sizeof(struct mt7925_mcu_rxd) + 4); + tlv = (struct tlv *)skb->data; + tlv_len = skb->len; + + while (tlv_len >= sizeof(*tlv)) { + u16 len = le16_to_cpu(tlv->len); + + if (len < sizeof(*tlv) || len > tlv_len) + break; + + switch (le16_to_cpu(tlv->tag)) { + case NAN_UNI_EVENT_ID_DE_EVENT_IND: + mt7925_nan_mcu_handle_de_event(dev, tlv); + break; + case NAN_UNI_EVENT_REPORT_DW_END: + mt7925_nan_handle_dw_ind(dev, tlv); + break; + default: + break; + } + + tlv_len -= len; + tlv = (struct tlv *)((u8 *)tlv + len); + } +} + +static int mt7925_nan_avail_ctrl_tlv(struct sk_buff *skb, + struct ieee80211_vif *vif) +{ + struct mt7925_nan_avail_ctrl_tlv *avail_ctrl_tlv; + struct ieee80211_nan_sched_cfg *sched; + struct tlv *tlv; + u8 seq_id = 0; + u16 ctrl = 0; + + if (!skb || !vif) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_UPDATE_AVAILABILITY_CTRL, + sizeof(struct mt7925_nan_avail_ctrl_tlv)); + + if (!tlv) + return -ENOMEM; + + sched = &vif->cfg.nan_sched; + + ctrl = mt7925_nan_avail_attr_ctrl(sched); + if (sched->avail_blob_len >= NAN_AVAIL_ATTR_CTRL_OFFSET + 2) + seq_id = sched->avail_blob[NAN_AVAIL_SEQ_ID_OFFSET]; + + avail_ctrl_tlv = (struct mt7925_nan_avail_ctrl_tlv *)tlv; + avail_ctrl_tlv->avail_ctrl = + cpu_to_le16(ctrl & NAN_AVAIL_CTRL_CHECK_FOR_CHANGED); + avail_ctrl_tlv->seq_id = seq_id; + + return 0; +} + +static u32 mt7925_nan_slot_to_bitmap(struct ieee80211_vif *vif, + struct mt7925_nan_ch_timeline *ch_list) +{ + struct ieee80211_nan_channel **slots = vif->cfg.nan_sched.schedule; + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + u32 num_channels = 0; + u32 i, j; + + for (i = 0; i < ARRAY_SIZE(mvif->nan.local_sched); i++) { + struct cfg80211_chan_def *slot_chan = &mvif->nan.local_sched[i]; + struct ieee80211_nan_channel *slot = slots[i]; + bool is_found = false; + + if (slot && !IS_ERR(slot) && slot->chanctx_conf) { + *slot_chan = slot->chanctx_conf->def; + } else { + memset(slot_chan, 0, sizeof(*slot_chan)); + continue; + } + + for (j = 0; j < num_channels; j++) { + u32 raw = le32_to_cpu(ch_list[j].ch_info); + + if (FIELD_GET(NAN_CH_CTRL_PRIMARY_CH, raw) == + slot_chan->chan->hw_value) { + u32 map = le32_to_cpu(ch_list[j].avail_map[0]); + + ch_list[j].avail_map[0] = cpu_to_le32(map | BIT(i)); + le32_add_cpu(&ch_list[j].num, 1); + is_found = true; + break; + } + } + + if (!is_found && num_channels < NAN_TIMELINE_MGMT_CHNL_LIST_NUM) { + ch_list[num_channels].ch_info = + cpu_to_le32(FIELD_PREP(NAN_CH_CTRL_OP_CLASS, + slot->channel_entry[0]) | + FIELD_PREP(NAN_CH_CTRL_PRIMARY_CH, + slot_chan->chan->hw_value)); + ch_list[num_channels].avail_map[0] = cpu_to_le32(BIT(i)); + le32_add_cpu(&ch_list[num_channels].num, 1); + ch_list[num_channels].is_valid++; + num_channels++; + } + } + + return num_channels; +} + +static int mt7925_nan_avail_tlv(struct sk_buff *skb, + struct ieee80211_vif *vif) +{ + struct mt7925_nan_avail_entry_tlv *avail_tlv; + struct ieee80211_nan_sched_cfg *sched; + struct tlv *tlv; + u16 ctrl = 0; + + if (!skb || !vif) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_UPDATE_AVAILABILITY, + sizeof(struct mt7925_nan_avail_entry_tlv)); + + if (!tlv) + return -ENOMEM; + + sched = &vif->cfg.nan_sched; + + ctrl = mt7925_nan_avail_attr_ctrl(sched); + + avail_tlv = (struct mt7925_nan_avail_entry_tlv *)tlv; + avail_tlv->map_id = ctrl & NAN_AVAIL_CTRL_MAPID; + avail_tlv->is_cond_avail = false; + avail_tlv->timeline_idx = 0; + + mt7925_nan_slot_to_bitmap(vif, avail_tlv->ch_list); + + avail_tlv->is_multi_map = false; + + return 0; +} + +void mt7925_nan_local_sched_changed(struct mt792x_dev *dev, + struct ieee80211_vif *vif) +{ + struct mt7925_nan_common_hdr *hdr; + struct mt76_dev *mdev; + struct sk_buff *skb; + + if (!dev || !vif) + return; + + mdev = &dev->mt76; + + skb = mt76_mcu_msg_alloc(mdev, NULL, MT7925_NAN_AVAIL_MAX_SIZE); + if (!skb) + return; + + hdr = (struct mt7925_nan_common_hdr *)skb_put(skb, sizeof(*hdr)); + memset(hdr, 0, sizeof(*hdr)); + + if (mt7925_nan_avail_ctrl_tlv(skb, vif) || + mt7925_nan_avail_tlv(skb, vif)) { + dev_kfree_skb(skb); + return; + } + + mt76_mcu_skb_send_msg(mdev, skb, + MCU_UNI_CMD(NAN), true); +} + +static int mt7925_nan_peer_rec_tlv(struct sk_buff *skb, + struct ieee80211_sta *sta, + struct mt792x_sta *msta, + u8 is_activate) +{ + struct mt7925_nan_sched_manage_peer_rec_tlv *peer_rec_tlv; + struct tlv *tlv; + + if (!skb || !sta || !msta) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_MANAGE_PEER_SCH_RECORD, + sizeof(struct mt7925_nan_sched_manage_peer_rec_tlv)); + + if (!tlv) + return -ENOMEM; + + peer_rec_tlv = (struct mt7925_nan_sched_manage_peer_rec_tlv *)tlv; + peer_rec_tlv->sch_idx = cpu_to_le32(msta->nan_sched.sch_idx); + peer_rec_tlv->is_activate = is_activate; + memcpy(peer_rec_tlv->nmi_addr, sta->addr, ETH_ALEN); + + return 0; +} + +static int mt7925_nan_peer_cap_tlv(struct sk_buff *skb, + struct ieee80211_sta *sta, + struct mt792x_sta *msta) +{ + struct mt7925_nan_sched_update_peer_cap_tlv *peer_cap_tlv; + struct ieee80211_nan_peer_sched *sched; + enum nl80211_band band; + struct tlv *tlv; + u16 primary_ch; + u32 i; + + if (!skb || !sta || !msta) + return -EINVAL; + + sched = sta->nan_sched; + if (!sched) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_UPDATE_PEER_CAPABILITY, + sizeof(struct mt7925_nan_sched_update_peer_cap_tlv)); + + if (!tlv) + return -ENOMEM; + + peer_cap_tlv = (struct mt7925_nan_sched_update_peer_cap_tlv *)tlv; + peer_cap_tlv->sch_idx = cpu_to_le32(msta->nan_sched.sch_idx); + peer_cap_tlv->supported_bands = BIT(NAN_SUPPORTED_BAND_ID_2P4G); + peer_cap_tlv->max_chnl_switch_time = cpu_to_le16(sched->max_chan_switch); + + for (i = 0; i < sched->n_channels; i++) { + if (!sched->channels[i].chanctx_conf) + continue; + + band = sched->channels[i].chanctx_conf->def.chan->band; + primary_ch = + sched->channels[i].chanctx_conf->def.chan->hw_value; + + if (band == NL80211_BAND_2GHZ) + peer_cap_tlv->peer_supported_bands |= + BIT(NAN_SUPPORTED_BN_2G); + else if (primary_ch >= UNII1_LOWER_BOUND && + primary_ch <= UNII1_UPPER_BOUND) + peer_cap_tlv->peer_supported_bands |= + BIT(NAN_SUPPORTED_BN_5G_LOW); + else if (primary_ch >= UNII3_LOWER_BOUND && + primary_ch <= UNII3_UPPER_BOUND) + peer_cap_tlv->peer_supported_bands |= + BIT(NAN_SUPPORTED_BN_5G_HIGH); + } + + return 0; +} + +static void +mt7925_nan_fill_crb_committed(struct mt7925_nan_sched_update_crb_tlv *crb_tlv, + struct ieee80211_nan_peer_sched *sched) +{ + u32 m, slot; + + if (!sched) + return; + + for (m = 0; m < CFG80211_NAN_MAX_PEER_MAPS && + m < NAN_TIMELINE_MGMT_SIZE; m++) { + struct mt7925_nan_sched_timeline *tl = + &crb_tlv->comm_faw_timeline[m]; + struct ieee80211_nan_peer_map *map = &sched->maps[m]; + + if (map->map_id == CFG80211_NAN_INVALID_MAP_ID) + continue; + + tl->map_id = map->map_id; + + /* + * Convert peer schedule slots to FW avail_map bitmap. + * Each bit in avail_map[0] represents one time slot where + * the peer has committed availability. + */ + for (slot = 0; slot < CFG80211_NAN_SCHED_NUM_TIME_SLOTS; + slot++) { + struct ieee80211_nan_channel *ch = map->slots[slot]; + + if (!ch || !ch->chanctx_conf) + continue; + + tl->avail_map[0] |= cpu_to_le32(BIT(slot)); + } + } +} + +static int mt7925_nan_update_crb_tlv(struct sk_buff *skb, + struct ieee80211_sta *sta, + struct mt792x_sta *msta) +{ + struct mt7925_nan_sched_update_crb_tlv *crb_tlv; + struct tlv *tlv; + + if (!skb || !sta || !msta) + return -EINVAL; + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_UPDATE_CRB, + sizeof(struct mt7925_nan_sched_update_crb_tlv)); + + if (!tlv) + return -ENOMEM; + + crb_tlv = (struct mt7925_nan_sched_update_crb_tlv *)tlv; + crb_tlv->sch_idx = cpu_to_le32(msta->nan_sched.sch_idx); + crb_tlv->flags = NAN_CRB_USE_DATA_PATH; + crb_tlv->is_use_ranging = false; + crb_tlv->comm_ndc_ctrl.is_valid = false; + + mt7925_nan_fill_crb_committed(crb_tlv, sta->nan_sched); + + return 0; +} + +int mt792x_nan_set_peer_schedule(struct mt792x_dev *dev, + struct ieee80211_sta *sta) +{ + struct mt7925_nan_common_hdr *hdr; + struct mt792x_sta *msta; + struct mt792x_nan *nan; + struct mt76_dev *mdev; + struct sk_buff *skb; + + if (!dev || !sta) + return -EINVAL; + + mdev = &dev->mt76; + + skb = mt76_mcu_msg_alloc(mdev, NULL, MT7925_NAN_PEER_MAX_SIZE); + if (!skb) + return -ENOMEM; + + hdr = (struct mt7925_nan_common_hdr *)skb_put(skb, sizeof(*hdr)); + memset(hdr, 0, sizeof(*hdr)); + + msta = (struct mt792x_sta *)sta->drv_priv; + nan = &msta->vif->nan; + + /* Allocate connection index on first call for this peer */ + if (!msta->nan_sched.idx_assigned) { + int idx = find_first_zero_bit(&nan->conn_bitmap, + NAN_MAX_CONN_CFG); + if (idx >= NAN_MAX_CONN_CFG) { + dev_kfree_skb(skb); + return -ENOSPC; + } + + set_bit(idx, &nan->conn_bitmap); + msta->nan_sched.sch_idx = idx; + msta->nan_sched.idx_assigned = true; + + if (mt7925_nan_peer_rec_tlv(skb, sta, msta, true) || + mt7925_nan_peer_cap_tlv(skb, sta, msta)) { + dev_kfree_skb(skb); + return -ENOMEM; + } + } + + if (mt7925_nan_update_crb_tlv(skb, sta, msta)) { + dev_kfree_skb(skb); + return -ENOMEM; + } + + return mt76_mcu_skb_send_msg(mdev, skb, + MCU_UNI_CMD(NAN), true); +} + +int mt792x_nan_set_peer_rec(struct mt76_dev *mdev, + struct ieee80211_sta *sta) +{ + struct mt7925_nan_common_hdr *hdr; + struct mt792x_sta *msta; + struct mt792x_nan *nan; + struct sk_buff *skb; + + if (!mdev || !sta) + return -EINVAL; + + skb = mt76_mcu_msg_alloc(mdev, NULL, + sizeof(struct mt7925_nan_common_hdr) + + sizeof(struct mt7925_nan_sched_manage_peer_rec_tlv)); + if (!skb) + return -ENOMEM; + + hdr = (struct mt7925_nan_common_hdr *)skb_put(skb, sizeof(*hdr)); + memset(hdr, 0, sizeof(*hdr)); + + msta = (struct mt792x_sta *)sta->drv_priv; + nan = &msta->vif->nan; + + if (!msta->nan_sched.idx_assigned) { + dev_kfree_skb(skb); + return 0; + } + + if (mt7925_nan_peer_rec_tlv(skb, sta, msta, false)) { + dev_kfree_skb(skb); + return -ENOMEM; + } + + clear_bit(msta->nan_sched.sch_idx, &nan->conn_bitmap); + msta->nan_sched.idx_assigned = false; + + return mt76_mcu_skb_send_msg(mdev, skb, + MCU_UNI_CMD(NAN), true); +} + +int mt792x_nan_map_sta_rec(struct mt76_dev *mdev, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta) +{ + struct mt7925_nan_sched_map_sta_rec_tlv *map_tlv; + struct mt7925_nan_common_hdr *hdr; + struct ieee80211_sta *nmi_sta; + struct mt792x_sta *nmi_msta; + struct mt792x_sta *msta; + u8 nmi_addr[ETH_ALEN]; + struct sk_buff *skb; + int ndp_ctx_id = 0; + struct tlv *tlv; + + if (!mdev || !vif || !sta) + return -EINVAL; + + msta = (struct mt792x_sta *)sta->drv_priv; + + rcu_read_lock(); + nmi_sta = rcu_dereference(sta->nmi); + if (!nmi_sta) { + rcu_read_unlock(); + dev_err(mdev->dev, "NAN: NMI sta not found for NDI sta %pM\n", + sta->addr); + return -EINVAL; + } + + memcpy(nmi_addr, nmi_sta->addr, ETH_ALEN); + nmi_msta = (struct mt792x_sta *)nmi_sta->drv_priv; + + ndp_ctx_id = find_first_zero_bit(&nmi_msta->nan_sched.ndp_ctx_bitmap, + NAN_MAX_NDP_CXT); + if (ndp_ctx_id < NAN_MAX_NDP_CXT) + set_bit(ndp_ctx_id, &nmi_msta->nan_sched.ndp_ctx_bitmap); + else + ndp_ctx_id = 0; + rcu_read_unlock(); + + msta->nan_sched.ndp_ctx_id = ndp_ctx_id; + + skb = mt76_mcu_msg_alloc(mdev, NULL, + sizeof(struct mt7925_nan_common_hdr) + + sizeof(struct mt7925_nan_sched_map_sta_rec_tlv)); + if (!skb) + return -ENOMEM; + + hdr = (struct mt7925_nan_common_hdr *)skb_put(skb, sizeof(*hdr)); + memset(hdr, 0, sizeof(*hdr)); + + tlv = mt76_connac_mcu_add_tlv(skb, NAN_UNI_CMD_MAP_STA_RECORD, + sizeof(struct mt7925_nan_sched_map_sta_rec_tlv)); + if (!tlv) { + dev_kfree_skb(skb); + return -ENOMEM; + } + + map_tlv = (struct mt7925_nan_sched_map_sta_rec_tlv *)tlv; + memcpy(map_tlv->nmi_addr, nmi_addr, ETH_ALEN); + map_tlv->sta_rec_idx = msta->deflink.wcid.idx; + map_tlv->ndp_ctx_id = ndp_ctx_id; + map_tlv->role_idx = 0; + memcpy(map_tlv->ndi_addr, vif->addr, ETH_ALEN); + + return mt76_mcu_skb_send_msg(mdev, skb, + MCU_UNI_CMD(NAN), true); +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/nan.h b/drivers/net/wireless/mediatek/mt76/mt7925/nan.h new file mode 100644 index 0000000000000..356d9ef7f6643 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/mt7925/nan.h @@ -0,0 +1,419 @@ +/* SPDX-License-Identifier: BSD-3-Clause-Clear */ +/* Copyright (C) 2025-2026 MediaTek Inc. */ + +#ifndef __MT7925_NAN_H +#define __MT7925_NAN_H + +#include +#include + +#include "../mt76_connac_mcu.h" + +#define NAN_MAX_SOCIAL_CHANNELS 3 +#define NAN_ANCHOR_MASTER_RANK_NUM 8 +#define NAN_5G_LOW_DISC_CHANNEL 44 +#define NAN_5G_HIGH_DISC_CHANNEL 149 +#define NAN_MAX_MASTER_PREFERENCE 255 +#define NAN_DEFAULT_DW_INTERVAL 1 +#define NAN_DEFAULT_DISC_BCN_INTERVAL 100 +#define NAN_TOTAL_DW 16 +#define NAN_SUPPORTED_2G_FAW_CH_NUM 4 +#define NAN_SUPPORTED_5G_FAW_CH_NUM 4 +#define NAN_TIMELINE_MGMT_SIZE 2 +#define NAN_TIMELINE_MGMT_CHNL_LIST_NUM \ + ((NAN_SUPPORTED_2G_FAW_CH_NUM + \ + NAN_SUPPORTED_5G_FAW_CH_NUM) / NAN_TIMELINE_MGMT_SIZE) +#define NAN_NUM_AVAIL_DB 2 +#define NAN_NDC_ATTRIBUTE_ID_LENGTH 6 +#define NAN_MAX_CONN_CFG 8 +#define NAN_MAX_NDP_CXT 4 + +#define MT7925_NAN_CONF_MAX_SIZE \ + (sizeof(struct mt7925_nan_common_hdr) + \ + sizeof(struct mt7925_nan_master_preference_tlv) + \ + sizeof(struct mt7925_nan_dw_interval_tlv) + \ + sizeof(struct mt7925_nan_cluster_id_tlv) + \ + sizeof(struct mt7925_nan_sync_rssi_tlv)) + +#define MT7925_NAN_AVAIL_MAX_SIZE \ + (sizeof(struct mt7925_nan_common_hdr) + \ + sizeof(struct mt7925_nan_avail_ctrl_tlv) + \ + sizeof(struct mt7925_nan_avail_entry_tlv)) + +#define MT7925_NAN_PEER_MAX_SIZE \ + (sizeof(struct mt7925_nan_common_hdr) + \ + sizeof(struct mt7925_nan_sched_manage_peer_rec_tlv) + \ + sizeof(struct mt7925_nan_sched_update_peer_cap_tlv) + \ + sizeof(struct mt7925_nan_sched_update_crb_tlv)) + +/* NAN Availability Attribute */ +#define NAN_AVAIL_ATTR_ID_OFFSET 0 +#define NAN_AVAIL_ATTR_LEN_OFFSET 1 +#define NAN_AVAIL_SEQ_ID_OFFSET 3 +#define NAN_AVAIL_ATTR_CTRL_OFFSET 4 + +/* NAN Availability Attribute - Attribute Control Field */ +#define NAN_AVAIL_CTRL_MAPID GENMASK(3, 0) +#define NAN_AVAIL_CTRL_COMMIT_CHANGED BIT(4) +#define NAN_AVAIL_CTRL_POTN_CHANGED BIT(5) +#define NAN_AVAIL_CTRL_PUBLIC_AVAIL_CHANGED BIT(6) +#define NAN_AVAIL_CTRL_NDC_CHANGED BIT(7) +#define NAN_AVAIL_CTRL_CHECK_FOR_CHANGED GENMASK(7, 4) + +#define UNII1_LOWER_BOUND 36 +#define UNII1_UPPER_BOUND 50 +#define UNII3_LOWER_BOUND 149 +#define UNII3_UPPER_BOUND 165 + +enum nan_uni_cmd_tag { + NAN_UNI_CMD_SET_MASTER_PREFERENCE = 0, + NAN_UNI_CMD_ENABLE_REQUEST = 7, + NAN_UNI_CMD_DISABLE_REQUEST = 8, + NAN_UNI_CMD_UPDATE_AVAILABILITY = 9, + NAN_UNI_CMD_UPDATE_CRB = 10, + NAN_UNI_CMD_MANAGE_PEER_SCH_RECORD = 12, + NAN_UNI_CMD_MAP_STA_RECORD = 13, + NAN_UNI_CMD_UPDATE_AVAILABILITY_CTRL = 20, + NAN_UNI_CMD_UPDATE_PEER_CAPABILITY = 21, + NAN_UNI_CMD_CHANGE_NMI_ADDRESS = 24, + NAN_UNI_CMD_SET_DW_INTERVAL = 26, + NAN_UNI_CMD_SET_SYNC_RSSI = 39, + NAN_UNI_CMD_SET_CLUSTER_ID = 40, + NAN_UNI_CMD_KEY_MANAGEMENT = 53, +}; + +enum nan_uni_event_tag { + NAN_UNI_EVENT_ID_DE_EVENT_IND = 19, + NAN_UNI_EVENT_REPORT_DW_END = 60, +}; + +enum nan_disc_event_type { + NAN_EVENT_ID_DISC_MAC_ADDR = 0, + NAN_EVENT_ID_JOINED_CLUSTER = 2, +}; + +/* NAN 4.0 Table 79. Device Capability attribute format, Supported Bands */ +enum nan_supported_bands { + NAN_SUPPORTED_BAND_ID_2P4G = 2, + NAN_SUPPORTED_BAND_ID_5G = 4, + NAN_PROPRIETARY_BAND_ID_6G = 6, + NAN_SUPPORTED_BAND_ID_6G = 7, +}; + +enum nan_peer_supported_bands { + NAN_SUPPORTED_BN_2G = 0, + NAN_SUPPORTED_BN_5G_LOW, + NAN_SUPPORTED_BN_5G_HIGH, + NAN_SUPPORTED_BN_6G, + NAN_SUPPORTED_BN_NUM +}; + +#define NAN_CH_CTRL_OP_CLASS GENMASK(15, 8) +#define NAN_CH_CTRL_PRIMARY_CH GENMASK(23, 16) + +#define NAN_CRB_USE_DATA_PATH BIT(0) +#define NAN_CRB_AVAIL_6G_FORMAT GENMASK(2, 1) + +struct mt7925_nan_social_ch_scan_params { + u8 dwell_time[NAN_MAX_SOCIAL_CHANNELS]; + __le16 scan_period[NAN_MAX_SOCIAL_CHANNELS]; +} __packed; + +/* Firmware-reported NAN device information */ +struct nan_dev_info_evt { + u8 is_enabled; + u8 my_addr[ETH_ALEN]; + u8 en_fw_election; + __le32 nan_dev_role; + __le32 nan_dev_state; + u8 mst_preference; + u8 random_factor; + u8 cnt_hop; + u8 cluster_id[ETH_ALEN]; + u8 anchor_mst_addr[ETH_ALEN]; + u8 am_preference; + u8 am_random_factor; + u8 parent_mac[ETH_ALEN]; + u8 parent_am_preference; + u8 parent_am_factor; + __le32 ambtt; + __le32 tsf[2]; + u8 pn_igtk[6]; + u8 pn_bigtk[6]; +}; + +/* Firmware NAN discovery window event */ +struct nan_rpt_dw_evt { + struct nan_dev_info_evt device_info; + __le32 expected_tsf_h; + __le32 expected_tsf_l; + __le32 actual_tsf_h; + __le32 actual_tsf_l; + __le16 channel; + __le16 dw_num; +}; + +struct mt7925_nan_conf_dw { + u8 config_2dot4g_dw_band; + __le32 dw_2dot4g_interval_val; + + u8 config_5g_dw_band; + __le32 dw_5g_interval_val; +} __packed; + +struct mt7925_nan_enable_req_tlv { + __le16 tag; + __le16 len; + + u8 master_pref; + __le16 cluster_low; + __le16 cluster_high; + + u8 config_support_5g; + u8 support_5g_val; + + u8 config_sid_beacon; + u8 sid_beacon_val; + + u8 config_2dot4g_rssi_close; + u8 rssi_close_2dot4g_val; + u8 config_2dot4g_rssi_middle; + u8 rssi_middle_2dot4g_val; + + u8 config_2dot4g_rssi_proximity; + u8 rssi_proximity_2dot4g_val; + u8 config_hop_count_limit; + u8 hop_count_limit_val; + + u8 config_2dot4g_support; + u8 support_2dot4g_val; + + u8 config_2dot4g_beacons; + u8 beacon_2dot4g_val; + + u8 config_2dot4g_sdf; + u8 sdf_2dot4g_val; + + u8 config_5g_beacons; + u8 beacon_5g_val; + + u8 config_5g_sdf; + u8 sdf_5g_val; + + u8 config_5g_rssi_close; + u8 rssi_close_5g_val; + + u8 config_5g_rssi_middle; + u8 rssi_middle_5g_val; + + u8 config_5g_rssi_close_proximity; + u8 rssi_close_proximity_5g_val; + + u8 config_rssi_window_size; + u8 rssi_window_size_val; + + u8 config_oui; + __le32 oui_val; + + u8 config_intf_addr; + u8 intf_addr_val[ETH_ALEN]; + + u8 config_cluster_attribute_val; + + u8 config_scan_params; + struct mt7925_nan_social_ch_scan_params scan_params_val; + + u8 config_random_factor_force; + u8 random_factor_force_val; + + u8 config_hop_count_force; + u8 hop_count_force_val; + + u8 config_24g_channel; + __le32 channel_24g_val; + + u8 config_5g_channel; + __le32 channel_5g_val; + + struct mt7925_nan_conf_dw config_dw; + + u8 config_disc_mac_addr_randomization; + __le32 disc_mac_addr_rand_interval_sec; + + u8 discovery_indication_cfg; + + u8 config_subscribe_sid_beacon; + __le32 subscribe_sid_beacon_val; + + u8 enable_log_slot_statistics; +} __packed __aligned(4); + +struct mt7925_nan_common_hdr { + u8 reserved[4]; +}; + +struct mt7925_nan_master_preference_tlv { + __le16 tag; + __le16 len; + u8 master_preference; + u8 reserved[3]; +} __packed __aligned(4); + +struct mt7925_nan_dw_interval_tlv { + __le16 tag; + __le16 len; + u8 dw_interval; + u8 vendor_ioctl; + __le16 disc_bcn_interval; +} __packed __aligned(4); + +struct mt7925_nan_cluster_id_tlv { + __le16 tag; + __le16 len; + u8 cluster_id[ETH_ALEN]; + u8 reserved[2]; +} __packed __aligned(4); + +struct mt7925_nan_sync_rssi_tlv { + __le16 tag; + __le16 len; + s8 rssi_close_2g; + s8 rssi_middle_2g; + s8 rssi_close_5g; + s8 rssi_middle_5g; +} __packed __aligned(4); + +struct mt7925_nan_de_event { + u8 event_type; + u8 cluster_id[ETH_ALEN]; + u8 anchor_master_rank[NAN_ANCHOR_MASTER_RANK_NUM]; + u8 own_nmi[ETH_ALEN]; + u8 master_nmi[ETH_ALEN]; +}; + +struct mt7925_nan_nmi_addr_tlv { + __le16 tag; + __le16 len; + u8 nmi_addr[ETH_ALEN]; +} __packed __aligned(4); + +struct mt7925_nan_avail_ctrl_tlv { + __le16 tag; + __le16 len; + __le16 avail_ctrl; + u8 seq_id; + u8 reserved[1]; +} __packed __aligned(4); + +struct mt7925_nan_ch_timeline { + u8 is_valid; + u8 reserved[3]; + + __le32 ch_info; + + __le32 num; + __le32 avail_map[NAN_TOTAL_DW]; +}; + +struct mt7925_nan_avail_entry_tlv { + __le16 tag; + __le16 len; + u8 map_id; + u8 is_cond_avail; + u8 timeline_idx; + u8 is_multi_map; + + struct mt7925_nan_ch_timeline ch_list[NAN_TIMELINE_MGMT_CHNL_LIST_NUM]; +} __packed __aligned(4); + +struct mt7925_nan_sched_manage_peer_rec_tlv { + __le16 tag; + __le16 len; + __le32 sch_idx; + u8 is_activate; + u8 nmi_addr[ETH_ALEN]; + u8 reserved[1]; +} __packed __aligned(4); + +struct mt7925_nan_sched_update_peer_cap_tlv { + __le16 tag; + __le16 len; + __le32 sch_idx; + u8 supported_bands; + __le16 max_chnl_switch_time; + u8 peer_supported_bands; +} __packed __aligned(4); + +struct mt7925_nan_sched_timeline { + u8 map_id; + u8 local_map_id; + u8 reserved[2]; + union { + __le32 avail_map[NAN_TOTAL_DW]; + u8 avail_block[NAN_TOTAL_DW * 4]; + }; +}; + +struct mt7925_nan_sched_faw_ndc_timeline { + __le32 avail_map[NAN_TOTAL_DW]; +}; + +struct mt7925_nan_sched_ndc_ctrl { + u8 is_valid; + u8 ndc_id[NAN_NDC_ATTRIBUTE_ID_LENGTH]; + u8 ndc_idx; + struct mt7925_nan_sched_timeline timeline[NAN_NUM_AVAIL_DB]; +}; + +struct mt7925_nan_sched_update_crb_tlv { + __le16 tag; + __le16 len; + __le32 sch_idx; + u8 flags; + u8 is_use_ranging; + u8 reserved[2]; + struct mt7925_nan_sched_timeline comm_ranging_timeline[NAN_TIMELINE_MGMT_SIZE]; + struct mt7925_nan_sched_timeline comm_faw_timeline[NAN_TIMELINE_MGMT_SIZE]; + struct mt7925_nan_sched_ndc_ctrl comm_ndc_ctrl; + struct mt7925_nan_sched_faw_ndc_timeline faw_ndc_timeline[NAN_TIMELINE_MGMT_SIZE]; +} __packed __aligned(4); + +struct mt7925_nan_sched_map_sta_rec_tlv { + __le16 tag; + __le16 len; + u8 nmi_addr[ETH_ALEN]; + u8 sta_rec_idx; + u8 ndp_ctx_id; + + __le32 role_idx; + u8 ndi_addr[ETH_ALEN]; + u8 reserved[2]; +} __packed __aligned(4); + +int mt7925_nan_enable(struct ieee80211_vif *vif, + struct mt792x_dev *dev, + struct cfg80211_nan_conf *conf); + +int mt7925_nan_disable(struct ieee80211_vif *vif, + struct mt792x_dev *dev); + +int mt7925_nan_change_configure(struct ieee80211_vif *vif, + struct mt792x_dev *dev, + struct cfg80211_nan_conf *conf); + +void mt7925_nan_mcu_event(struct mt792x_dev *dev, struct sk_buff *skb); + +void mt7925_nan_local_sched_changed(struct mt792x_dev *dev, + struct ieee80211_vif *vif); + +int mt792x_nan_set_peer_schedule(struct mt792x_dev *dev, + struct ieee80211_sta *sta); + +int mt792x_nan_set_peer_rec(struct mt76_dev *mdev, + struct ieee80211_sta *sta); + +int mt792x_nan_map_sta_rec(struct mt76_dev *mdev, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta); + +#endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c index ea64303283edb..084baadfa1833 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c @@ -48,6 +48,7 @@ static void mt7925e_unregister_device(struct mt792x_dev *dev) mt76_for_each_q_rx(&dev->mt76, i) napi_disable(&dev->mt76.napi[i]); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); cancel_work_sync(&dev->reset_work); @@ -517,6 +518,7 @@ static int mt7925_pci_suspend(struct device *device) dev->hif_resumed = false; flush_work(&dev->reset_work); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); mt7925_roc_abort_sync(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/regd.c b/drivers/net/wireless/mediatek/mt76/mt7925/regd.c index 16f56ee879d45..0235437d11d59 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/regd.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/regd.c @@ -217,6 +217,36 @@ mt7925_regd_is_valid_alpha2(const char *alpha2) return false; } +bool +mt7925_regd_is_valid_channel(struct mt792x_dev *dev, + enum nl80211_band band, + struct ieee80211_channel *chan) +{ + struct ieee80211_hw *hw = mt76_hw(dev); + struct wiphy *wiphy = hw->wiphy; + struct ieee80211_supported_band *sband; + struct ieee80211_channel *ch; + int i; + + if (!chan) + return false; + + sband = wiphy->bands[band]; + if (!sband) + return false; + + for (i = 0; i < sband->n_channels; i++) { + ch = &sband->channels[i]; + + if (ch->hw_value == chan->hw_value && + ((ch->flags & IEEE80211_CHAN_DISABLED) == 0)) + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(mt7925_regd_is_valid_channel); + int mt7925_regd_change(struct mt792x_phy *phy, char *alpha2) { struct wiphy *wiphy = phy->mt76->hw->wiphy; diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/regd.h b/drivers/net/wireless/mediatek/mt76/mt7925/regd.h index 0767f078862e7..0b0754cf8ae70 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/regd.h +++ b/drivers/net/wireless/mediatek/mt76/mt7925/regd.h @@ -13,6 +13,9 @@ void mt7925_regd_be_ctrl(struct mt792x_dev *dev, u8 *alpha2); void mt7925_regd_notifier(struct wiphy *wiphy, struct regulatory_request *req); bool mt7925_regd_clc_supported(struct mt792x_dev *dev); int mt7925_regd_change(struct mt792x_phy *phy, char *alpha2); +bool mt7925_regd_is_valid_channel(struct mt792x_dev *dev, + enum nl80211_band band, + struct ieee80211_channel *chan); int mt7925_regd_init(struct mt792x_phy *phy); #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/usb.c b/drivers/net/wireless/mediatek/mt76/mt7925/usb.c index e9f58492bf7df..a798164b9a054 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/usb.c @@ -269,6 +269,7 @@ static int mt7925u_suspend(struct usb_interface *intf, pm_message_t state) pm->suspended = true; dev->hif_resumed = false; flush_work(&dev->reset_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); mt76_connac_mcu_set_hif_suspend(&dev->mt76, true, false); ret = wait_event_timeout(dev->wait, diff --git a/drivers/net/wireless/mediatek/mt76/mt792x.h b/drivers/net/wireless/mediatek/mt76/mt792x.h index 70073b43af543..89c3f84a776a0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x.h +++ b/drivers/net/wireless/mediatek/mt76/mt792x.h @@ -115,6 +115,18 @@ struct mt792x_link_sta { struct ieee80211_link_sta *pri_link; }; +struct mt792x_sta_nan_sched { + u16 committed_dw; + u32 sch_idx; + bool idx_assigned; + unsigned long ndp_ctx_bitmap; + u8 ndp_ctx_id; /* assigned NDP context ID (for NDI sta) */ + struct { + u8 map_id; + struct cfg80211_chan_def chans[CFG80211_NAN_SCHED_NUM_TIME_SLOTS]; + } maps[CFG80211_NAN_MAX_PEER_MAPS]; +}; + struct mt792x_sta { struct mt792x_link_sta deflink; /* must be first */ struct mt792x_link_sta __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS]; @@ -123,6 +135,9 @@ struct mt792x_sta { u16 valid_links; u8 deflink_id; + + /* NAN peer schedule */ + struct mt792x_sta_nan_sched nan_sched; }; DECLARE_EWMA(rssi, 10, 8); @@ -139,6 +154,25 @@ struct mt792x_bss_conf { unsigned int link_id; }; +struct mt792x_nan_conf { + u8 master_pref; + u8 bands; + u8 cluster_id[ETH_ALEN]; + u32 discovery_beacon_interval; + bool enable_dw_notification; +}; + +struct mt792x_nan { + struct mt792x_nan_conf conf; + + /* Scheduler */ + struct cfg80211_chan_def local_sched[CFG80211_NAN_SCHED_NUM_TIME_SLOTS]; + u32 seq_id; + + /* Connection index bitmap, up to NAN_MAX_CONN_CFG peers */ + unsigned long conn_bitmap; +}; + struct mt792x_vif { struct mt792x_bss_conf bss_conf; /* must be first */ struct mt792x_bss_conf __rcu *link_conf[IEEE80211_MLD_MAX_NUM_LINKS]; @@ -153,6 +187,8 @@ struct mt792x_vif { struct work_struct csa_work; struct timer_list csa_timer; + + struct mt792x_nan nan; }; struct mt792x_phy { @@ -283,6 +319,8 @@ struct mt792x_dev { u32 backup_l2; struct ieee80211_chanctx_conf *new_ctx; + + struct ieee80211_vif *nan_vif; }; static inline struct mt792x_bss_conf * diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_usb.c b/drivers/net/wireless/mediatek/mt76/mt792x_usb.c index 910132e94956a..47f80c9ec4e7d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt792x_usb.c @@ -31,10 +31,75 @@ static void mt792xu_reset_work(struct work_struct *work) atomic_set(&dev->usb_reset_pending, 0); } +static void mt792xu_queue_usb_reset(struct mt792x_dev *dev, int err) +{ + if (!atomic_xchg(&dev->usb_reset_pending, 1)) { + dev_warn(dev->mt76.dev, + "USB transport access failed (%d), queueing device reset\n", + err); + + schedule_work(&dev->usb_reset_work); + } +} + +static u32 mt792xu_bus_hung_rr(struct mt76_dev *mdev, u32 offset) +{ + return 0; +} + +static void mt792xu_bus_hung_wr(struct mt76_dev *mdev, u32 offset, u32 val) +{ +} + +static u32 mt792xu_bus_hung_rmw(struct mt76_dev *mdev, u32 offset, + u32 mask, u32 val) +{ + return 0; +} + +static void mt792xu_bus_hung_write_copy(struct mt76_dev *mdev, u32 offset, + const void *data, int len) +{ +} + +static void mt792xu_bus_hung_read_copy(struct mt76_dev *mdev, u32 offset, + void *data, int len) +{ + memset(data, 0, len); +} + +static const struct mt76_bus_ops mt792xu_bus_hung_ops = { + .rr = mt792xu_bus_hung_rr, + .wr = mt792xu_bus_hung_wr, + .rmw = mt792xu_bus_hung_rmw, + .write_copy = mt792xu_bus_hung_write_copy, + .read_copy = mt792xu_bus_hung_read_copy, + .type = MT76_BUS_USB, +}; + +static void mt792xu_set_bus_hung(struct mt792x_dev *dev) +{ + atomic_set(&dev->mt76.bus_hung, true); + + if (READ_ONCE(dev->mt76.bus) == &mt792xu_bus_hung_ops) + return; + + WRITE_ONCE(dev->mt76.bus, &mt792xu_bus_hung_ops); +} + +static void mt792xu_ctrl_timeout(struct mt76_dev *mdev, int err) +{ + struct mt792x_dev *dev = container_of(mdev, struct mt792x_dev, mt76); + + mt792xu_set_bus_hung(dev); + mt792xu_queue_usb_reset(dev, err); +} + void mt792xu_reset_work_init(struct mt792x_dev *dev) { INIT_WORK(&dev->usb_reset_work, mt792xu_reset_work); atomic_set(&dev->usb_reset_pending, 0); + dev->mt76.usb.ctrl_timeout = mt792xu_ctrl_timeout; } EXPORT_SYMBOL_GPL(mt792xu_reset_work_init); @@ -62,26 +127,23 @@ EXPORT_SYMBOL_GPL(mt792xu_check_bus); int mt792xu_reset_on_bus_error(struct mt792x_dev *dev) { - int err = 0; + int err; - if (!atomic_read(&dev->mt76.bus_hung)) - err = mt792xu_check_bus(dev); + /* Once hung, the no-op bus ops stay installed until the queued USB + * reset re-probes the device. Do not clear bus_hung here, or the caller + * would run a full reset over dropped register I/O and report success. + */ + if (atomic_read(&dev->mt76.bus_hung)) + return -EIO; + err = mt792xu_check_bus(dev); if (err) { - atomic_set(&dev->mt76.bus_hung, true); - - if (!atomic_xchg(&dev->usb_reset_pending, 1)) { - dev_warn(dev->mt76.dev, - "USB transport access failed (%d), queueing device reset\n", - err); - - schedule_work(&dev->usb_reset_work); - } + mt792xu_set_bus_hung(dev); + mt792xu_queue_usb_reset(dev, err); return err; } - atomic_set(&dev->mt76.bus_hung, false); return 0; } EXPORT_SYMBOL_GPL(mt792xu_reset_on_bus_error); @@ -344,6 +406,9 @@ int mt792xu_wfsys_reset(struct mt792x_dev *dev) u32 val; int i; + if (atomic_read(&dev->mt76.bus_hung)) + return -EIO; + mt792xu_epctl_rst_opt(dev, false); val = mt792xu_uhw_rr(&dev->mt76, desc->rst_reg); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c index ac05f7d75d63d..ec33521db564b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c @@ -150,6 +150,12 @@ mt7996_eeprom_check_or_use_default(struct mt7996_dev *dev, bool use_default) goto out; } + if (fw->size < MT7996_EEPROM_SIZE) { + dev_err(dev->mt76.dev, "Invalid default bin size\n"); + ret = -EINVAL; + goto out; + } + if (!use_default && mt7996_eeprom_variant_valid(dev, fw->data)) goto out; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/init.c b/drivers/net/wireless/mediatek/mt76/mt7996/init.c index d6f9aa1ab52d1..fa74aba426903 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/init.c @@ -34,7 +34,6 @@ static const struct ieee80211_iface_combination if_comb_global = { BIT(NL80211_CHAN_WIDTH_40) | BIT(NL80211_CHAN_WIDTH_80) | BIT(NL80211_CHAN_WIDTH_160), - .beacon_int_min_gcd = 100, }; static const struct ieee80211_iface_combination if_comb_global_7992 = { @@ -47,7 +46,6 @@ static const struct ieee80211_iface_combination if_comb_global_7992 = { BIT(NL80211_CHAN_WIDTH_40) | BIT(NL80211_CHAN_WIDTH_80) | BIT(NL80211_CHAN_WIDTH_160), - .beacon_int_min_gcd = 100, }; static const struct ieee80211_iface_limit if_limits[] = { @@ -1561,7 +1559,6 @@ mt7996_init_eht_caps(struct mt7996_phy *phy, enum nl80211_band band, struct ieee80211_sta_eht_cap *eht_cap = &data->eht_cap; struct ieee80211_eht_cap_elem_fixed *eht_cap_elem = &eht_cap->eht_cap_elem; struct ieee80211_eht_mcs_nss_supp *eht_nss = &eht_cap->eht_mcs_nss_supp; - enum nl80211_chan_width width = phy->mt76->chandef.width; int nss = hweight8(phy->mt76->antenna_mask); int sts = hweight16(phy->mt76->chainmask); u8 val; @@ -1637,11 +1634,16 @@ mt7996_init_eht_caps(struct mt7996_phy *phy, enum nl80211_band band, u8_encode_bits(u8_get_bits(1, GENMASK(1, 0)), IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK); - val = width == NL80211_CHAN_WIDTH_320 ? 0xf : - width == NL80211_CHAN_WIDTH_160 ? 0x7 : - width == NL80211_CHAN_WIDTH_80 ? 0x3 : 0x1; - eht_cap_elem->phy_cap_info[6] = - u8_encode_bits(val, IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK); + eht_cap_elem->phy_cap_info[6] = IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK; + if (band != NL80211_BAND_6GHZ) { + eht_cap_elem->phy_cap_info[6] &= + ~IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ; + + if (band != NL80211_BAND_5GHZ) + eht_cap_elem->phy_cap_info[6] &= + ~(IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ | + IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ); + } val = u8_encode_bits(nss, IEEE80211_EHT_MCS_NSS_RX) | u8_encode_bits(nss, IEEE80211_EHT_MCS_NSS_TX); @@ -1799,6 +1801,8 @@ void mt7996_unregister_device(struct mt7996_dev *dev) { cancel_work_sync(&dev->dump_work); cancel_work_sync(&dev->wed_rro.work); + cancel_work_sync(&dev->reset_work); + cancel_work_sync(&dev->rc_work); mt7996_unregister_phy(mt7996_phy3(dev)); mt7996_unregister_phy(mt7996_phy2(dev)); mt7996_unregister_thermal(&dev->phy); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 0eebc8182ca9a..85d4adf1c113d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -166,7 +166,7 @@ static void mt7996_mac_sta_poll(struct mt7996_dev *dev) rssi[0] = to_rssi(GENMASK(7, 0), val); rssi[1] = to_rssi(GENMASK(15, 8), val); rssi[2] = to_rssi(GENMASK(23, 16), val); - rssi[3] = to_rssi(GENMASK(31, 14), val); + rssi[3] = to_rssi(GENMASK(31, 24), val); mlink = rcu_dereference(msta->vif->mt76.link[wcid->link_id]); if (mlink) { @@ -294,6 +294,10 @@ mt7996_mac_fill_rx_rate(struct mt7996_dev *dev, dcm = FIELD_GET(MT_PRXV_DCM, v2); bw = FIELD_GET(MT_PRXV_FRAME_MODE, v2); + /* the hardware reports NSTS; report the data NSS for STBC frames */ + if (stbc && nss > 1) + nss >>= 1; + switch (*mode) { case MT_PHY_TYPE_CCK: cck = true; @@ -349,7 +353,7 @@ mt7996_mac_fill_rx_rate(struct mt7996_dev *dev, case IEEE80211_STA_RX_BW_20: break; case IEEE80211_STA_RX_BW_40: - if (*mode & MT_PHY_TYPE_HE_EXT_SU && + if (*mode == MT_PHY_TYPE_HE_EXT_SU && (idx & MT_PRXV_TX_ER_SU_106T)) { status->bw = RATE_INFO_BW_HE_RU; status->he_ru = @@ -441,7 +445,13 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, enum mt76_rxq_id q, memset(status, 0, sizeof(*status)); band_idx = FIELD_GET(MT_RXD1_NORMAL_BAND_IDX, rxd1); + if (!mt7996_band_valid(dev, band_idx)) + return -EINVAL; + mphy = dev->mt76.phys[band_idx]; + if (!mphy) + return -EINVAL; + phy = mphy->priv; status->phy_idx = mphy->band_idx; @@ -854,6 +864,33 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, txwi[6] |= cpu_to_le32(MT_TXD6_DIS_MAT); } +/* The WLAN_IDX in the TXD and TXP must belong to the primary or secondary + * link of an MLD station; any other link id can make the firmware spin when + * that link is in powersave. Completion events carry the same index, so the + * wcid used for status tracking and accounting must match it + */ +struct mt76_wcid *mt7996_get_tx_wcid(struct mt76_wcid *wcid) +{ + struct mt7996_sta_link *msta_link; + struct mt7996_sta *msta; + + if (!wcid->sta) + return wcid; + + msta_link = container_of(wcid, struct mt7996_sta_link, wcid); + msta = msta_link->sta; + + if (!msta || wcid->link_id == msta->seclink_id || + wcid->link_id == msta->deflink_id) + return wcid; + + msta_link = mt7996_sta_link(msta, msta->deflink_id); + if (msta_link) + return &msta_link->wcid; + + return wcid; +} + void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, int pid, @@ -1030,6 +1067,11 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, IEEE80211_TX_CTRL_MLO_LINK); } + /* non-MLD frames are LINK_UNSPECIFIED; use the wcid's own link */ + if (link_id == IEEE80211_LINK_UNSPECIFIED && + wcid != &dev->mt76.global_wcid) + link_id = wcid->link_id; + if (link_id != wcid->link_id && link_id != IEEE80211_LINK_UNSPECIFIED) { if (msta) { struct mt7996_sta_link *msta_link = @@ -1086,6 +1128,8 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, tx_info->buf[1].len, DMA_TO_DEVICE); } + wcid = mt7996_get_tx_wcid(wcid); + pid = mt76_tx_status_skb_add(mdev, wcid, tx_info->skb); memset(txwi_ptr, 0, MT_TXD_SIZE); /* Transmit non qos data by 802.11 header and need to fill txd by host*/ @@ -1239,6 +1283,30 @@ mt7996_tx_check_aggr(struct ieee80211_link_sta *link_sta, clear_bit(tid, &wcid->ampdu_state); } +static void +mt7996_txp_skb_unmap(struct mt76_dev *mdev, struct mt76_txwi_cache *t) +{ + u8 *txwi_ptr = mt76_get_txwi_ptr(mdev, t); + __le32 *txwi = (__le32 *)txwi_ptr; + __le32 *txp; + dma_addr_t addr; + u32 val; + + if (!(le32_to_cpu(txwi[7]) & MT_TXD7_MAC_TXD)) { + mt76_connac_txp_skb_unmap(mdev, t); + return; + } + + txp = (__le32 *)(txwi_ptr + MT_TXD_SIZE); + val = le32_to_cpu(txp[3]); + addr = le32_to_cpu(txp[2]); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + addr |= (dma_addr_t)FIELD_GET(MT_TXP3_DMA_ADDR_H, val) << 32; +#endif + dma_unmap_single(mdev->dma_dev, addr, FIELD_GET(MT_TXP_BUF_LEN, val), + DMA_TO_DEVICE); +} + static void mt7996_txwi_free(struct mt7996_dev *dev, struct mt76_txwi_cache *t, struct ieee80211_link_sta *link_sta, @@ -1248,7 +1316,7 @@ mt7996_txwi_free(struct mt7996_dev *dev, struct mt76_txwi_cache *t, __le32 *txwi; u16 wcid_idx; - mt76_connac_txp_skb_unmap(mdev, t); + mt7996_txp_skb_unmap(mdev, t); if (!t->skb) goto out; @@ -2383,6 +2451,7 @@ mt7996_mac_reset_vif_iter(void *data, u8 *mac, struct ieee80211_vif *vif) rcu_assign_pointer(mvif->link[i], NULL); kfree_rcu(mlink, rcu_head); } + mvif->valid_links = 0; rcu_read_unlock(); } @@ -2396,6 +2465,7 @@ mt7996_mac_full_reset(struct mt7996_dev *dev) dev->recovery.hw_full_reset = true; + set_bit(MT76_MCU_RESET, &dev->mphy.state); wake_up(&dev->mt76.mcu.wait); ieee80211_stop_queues(hw); @@ -2416,12 +2486,14 @@ mt7996_mac_full_reset(struct mt7996_dev *dev) mt7996_for_each_phy(dev, phy) phy->omac_mask = 0; + dev->mld_idx_mask = 0; + dev->mld_remap_idx_mask = 0; ieee80211_iterate_stations_atomic(hw, mt7996_mac_reset_sta_iter, dev); + mt76_reset_device(&dev->mt76); ieee80211_iterate_active_interfaces_atomic(hw, IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER, mt7996_mac_reset_vif_iter, dev); - mt76_reset_device(&dev->mt76); INIT_LIST_HEAD(&dev->sta_rc_list); INIT_LIST_HEAD(&dev->twt_list); @@ -2500,8 +2572,8 @@ void mt7996_mac_reset_work(struct work_struct *work) set_bit(MT76_RESET, &dev->mphy.state); set_bit(MT76_MCU_RESET, &dev->mphy.state); - mt76_abort_scan(&dev->mt76); wake_up(&dev->mt76.mcu.wait); + mt76_abort_scan(&dev->mt76); cancel_work_sync(&dev->wed_rro.work); mt7996_for_each_phy(dev, phy) { @@ -2510,6 +2582,8 @@ void mt7996_mac_reset_work(struct work_struct *work) cancel_delayed_work_sync(&phy->mt76->mac_work); } + mutex_lock(&dev->mt76.mutex); + mt76_worker_disable(&dev->mt76.tx_worker); mt76_for_each_q_rx(&dev->mt76, i) { if (mtk_wed_device_active(&dev->mt76.mmio.wed) && @@ -2527,8 +2601,6 @@ void mt7996_mac_reset_work(struct work_struct *work) } napi_disable(&dev->mt76.tx_napi); - mutex_lock(&dev->mt76.mutex); - mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_DMA_STOPPED); if (mt7996_wait_reset_state(dev, MT_MCU_CMD_RESET_DONE)) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/main.c b/drivers/net/wireless/mediatek/mt76/mt7996/main.c index afbcc8c7b18b2..c6140412ed322 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/main.c @@ -372,7 +372,8 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, CONN_STATE_PORT_SECURE, true); rcu_assign_pointer(dev->mt76.wcid[idx], &msta_link->wcid); - ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); + if (!mlink->wcid->offchannel) + ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); if (!mlink->wcid->offchannel) { if (vif->txq && @@ -1202,15 +1203,9 @@ void mt7996_mac_sta_remove_link(struct mt7996_dev *dev, mt76_wcid_cleanup(&dev->mt76, &msta_link->wcid); if (msta_link->wcid.link_valid) { - struct mt7996_phy *phy; - mt7996_mac_wtbl_update(dev, msta_link->wcid.idx, MT_WTBL_UPDATE_ADM_COUNT_CLEAR); - phy = __mt7996_phy(dev, msta_link->wcid.phy_idx); - if (phy) - phy->mt76->num_sta--; - if (msta->deflink_id == link_id) { msta->deflink_id = IEEE80211_LINK_UNSPECIFIED; if (msta->seclink_id == link_id) { @@ -1236,6 +1231,12 @@ void mt7996_mac_sta_remove_link(struct mt7996_dev *dev, } if (flush) { + struct mt7996_phy *phy = + __mt7996_phy(dev, msta_link->wcid.phy_idx); + + if (phy) + phy->mt76->num_sta--; + rcu_assign_pointer(msta->link[link_id], NULL); rcu_assign_pointer(dev->mt76.wcid[msta_link->wcid.idx], NULL); mt76_wcid_mask_clear(dev->mt76.wcid_mask, msta_link->wcid.idx); @@ -1510,20 +1511,26 @@ static void mt7996_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif = info->control.vif; struct mt7996_vif *mvif = vif ? (void *)vif->drv_priv : NULL; struct mt76_wcid *wcid = &dev->mt76.global_wcid; + u8 deflink_id = IEEE80211_LINK_UNSPECIFIED; u8 link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK); rcu_read_lock(); + if (msta) + deflink_id = msta->deflink_id; + else if (mvif) + deflink_id = mvif->mt76.deflink_id; + + /* the primary link is unset until the first link has been added */ + if (deflink_id >= IEEE80211_MLD_MAX_NUM_LINKS) + deflink_id = 0; + /* Use primary link_id if the value from mac80211 is set to * IEEE80211_LINK_UNSPECIFIED. */ - if (link_id == IEEE80211_LINK_UNSPECIFIED) { - if (msta) - link_id = msta->deflink_id; - else if (mvif) - link_id = mvif->mt76.deflink_id; - } + if (link_id == IEEE80211_LINK_UNSPECIFIED) + link_id = deflink_id; if (vif && ieee80211_vif_is_mld(vif)) { struct ieee80211_bss_conf *link_conf; @@ -1533,7 +1540,7 @@ static void mt7996_tx(struct ieee80211_hw *hw, link_sta = rcu_dereference(sta->link[link_id]); if (!link_sta) - link_sta = rcu_dereference(sta->link[msta->deflink_id]); + link_sta = rcu_dereference(sta->link[deflink_id]); if (link_sta) { memcpy(hdr->addr1, link_sta->addr, ETH_ALEN); @@ -1579,7 +1586,7 @@ static void mt7996_tx(struct ieee80211_hw *hw, if (msta_link) wcid = &msta_link->wcid; } - mt76_tx(mphy, control->sta, wcid, skb); + mt76_tx(mphy, control->sta, mt7996_get_tx_wcid(wcid), skb); unlock: rcu_read_unlock(); } @@ -1883,8 +1890,6 @@ static void mt7996_sta_statistics(struct ieee80211_hw *hw, sinfo->txrate.flags = txrate->flags; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); } - sinfo->txrate.flags = txrate->flags; - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); sinfo->tx_failed = msta_link->wcid.stats.tx_failed; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED); @@ -2460,6 +2465,7 @@ mt7996_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif, idx = get_free_idx(dev->mld_remap_idx_mask, 0, 15) - 1; if (idx < 0) { + dev->mld_idx_mask &= ~BIT_ULL(mvif->mld_group_idx); ret = -ENOSPC; goto out; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c index 2e83f4b79c870..645a8b480871e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c @@ -2483,7 +2483,7 @@ mt7996_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7996_dev *dev, cap |= STA_CAP_VHT_SGI_160; if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_TXSTBC) cap |= STA_CAP_VHT_TX_STBC; - if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_1) + if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_RXSTBC_MASK) cap |= STA_CAP_VHT_RX_STBC; if ((vif->type != NL80211_IFTYPE_AP || link_conf->vht_ldpc) && (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_RXLDPC)) @@ -4023,6 +4023,9 @@ int mt7996_mcu_rdd_background_enable(struct mt7996_phy *phy, struct mt7996_dev *dev = phy->dev; int err, region, rdd_idx = mt7996_get_rdd_idx(phy, true); + if (rdd_idx < 0) + return -EINVAL; + if (!chandef) { /* disable offchain */ err = mt7996_mcu_rdd_cmd(dev, RDD_STOP, rdd_idx, 0); if (err) @@ -4352,11 +4355,18 @@ int mt7996_mcu_get_eeprom(struct mt7996_dev *dev, u32 offset, u8 *buf, u32 buf_l event = (struct mt7996_mcu_eeprom_access_event *)skb->data; if (event->valid) { u32 ret_len = le32_to_cpu(event->eeprom.ext_eeprom.data_len); + u32 block = mode == EEPROM_MODE_EXT ? MT7996_EXT_EEPROM_BLOCK_SIZE : + MT7996_EEPROM_BLOCK_SIZE; addr = le32_to_cpu(event->addr); - if (!buf) + if (!buf) { + if (addr > dev->mt76.eeprom.size - block) { + dev_kfree_skb(skb); + return -EINVAL; + } buf = (u8 *)dev->mt76.eeprom.data + addr; + } switch (mode) { case EEPROM_MODE_EFUSE: @@ -4438,21 +4448,31 @@ int mt7996_mcu_get_chip_config(struct mt7996_dev *dev, u32 *cap) return ret; /* fixed field */ + if (skb->len < 4) { + dev_kfree_skb(skb); + return -EINVAL; + } skb_pull(skb, 4); buf = skb->data; - while (buf - skb->data < skb->len) { + while (buf - skb->data + sizeof(struct tlv) <= skb->len) { struct tlv *tlv = (struct tlv *)buf; + u16 tlv_len = le16_to_cpu(tlv->len); + + if (tlv_len < sizeof(*tlv) || + tlv_len > skb->len - (buf - skb->data)) + break; switch (le16_to_cpu(tlv->tag)) { case UNI_EVENT_CHIP_CONFIG_EFUSE_VERSION: - *cap = le32_to_cpu(*(__le32 *)(buf + sizeof(*tlv))); + if (tlv_len >= sizeof(*tlv) + sizeof(__le32)) + *cap = le32_to_cpu(*(__le32 *)(buf + sizeof(*tlv))); break; default: break; } - buf += le16_to_cpu(tlv->len); + buf += tlv_len; } dev_kfree_skb(skb); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h index 8902e16508b75..c673e986ecb52 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h @@ -917,7 +917,7 @@ enum { #define MT7996_BEACON_UPDATE_SIZE (sizeof(struct bss_req_hdr) + \ sizeof(struct bss_bcn_content_tlv) + \ 4 + MT_TXD_SIZE + \ - sizeof(struct bss_bcn_cntdwn_tlv) + \ + sizeof(struct bss_bcn_cntdwn_tlv) * 2 + \ sizeof(struct bss_bcn_mbss_tlv)) #define MT7996_MAX_BSS_OFFLOAD_SIZE 2048 #define MT7996_MAX_BEACON_SIZE (MT7996_MAX_BSS_OFFLOAD_SIZE - \ diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c index d9780bb425a71..ac81be5fe0230 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c @@ -17,6 +17,8 @@ static bool wed_enable; module_param(wed_enable, bool, 0644); +#define INVALID_REG_ADDR 0xffffffff + static const struct __base mt7996_reg_base[] = { [WF_AGG_BASE] = { { 0x820e2000, 0x820f2000, 0x830e2000 } }, [WF_ARB_BASE] = { { 0x820e3000, 0x820f3000, 0x830e3000 } }, @@ -54,6 +56,14 @@ static const u32 mt7996_offs[] = { [MIB_BSCR7] = 0x9e8, [MIB_BSCR17] = 0xa10, [MIB_TRDR1] = 0xa28, + [MIB_TSCR0] = 0x6b0, + [MIB_TSCR1] = 0x6b4, + [MIB_TSCR2] = 0x6b8, + [MIB_TSCR3] = 0x6bc, + [MIB_TSCR4] = 0x6c0, + [MIB_TSCR5] = 0x6c4, + [MIB_TSCR6] = 0x6c8, + [MIB_TSCR7] = 0x6d0, [HIF_REMAP_L1] = 0x24, [HIF_REMAP_BASE_L1] = 0x130000, [HIF_REMAP_L2] = 0x1b4, @@ -91,6 +101,14 @@ static const u32 mt7992_offs[] = { [MIB_BSCR7] = 0xae4, [MIB_BSCR17] = 0xb0c, [MIB_TRDR1] = 0xb24, + [MIB_TSCR0] = 0x6b0, + [MIB_TSCR1] = 0x6b4, + [MIB_TSCR2] = 0x6b8, + [MIB_TSCR3] = 0x6bc, + [MIB_TSCR4] = 0x6c0, + [MIB_TSCR5] = 0x6c4, + [MIB_TSCR6] = 0x6c8, + [MIB_TSCR7] = 0x6d0, [HIF_REMAP_L1] = 0x8, [HIF_REMAP_BASE_L1] = 0x40000, [HIF_REMAP_L2] = 0x1b4, @@ -128,6 +146,14 @@ static const u32 mt7990_offs[] = { [MIB_BSCR7] = 0xbd4, [MIB_BSCR17] = 0xbfc, [MIB_TRDR1] = 0xc14, + [MIB_TSCR0] = 0x750, + [MIB_TSCR1] = 0x754, + [MIB_TSCR2] = 0x758, + [MIB_TSCR3] = 0x75c, + [MIB_TSCR4] = 0x760, + [MIB_TSCR5] = 0x764, + [MIB_TSCR6] = 0x768, + [MIB_TSCR7] = 0x770, [HIF_REMAP_L1] = 0x8, [HIF_REMAP_BASE_L1] = 0x40000, [HIF_REMAP_L2] = 0x1b8, @@ -334,7 +360,7 @@ static u32 __mt7996_reg_addr(struct mt7996_dev *dev, u32 addr) return dev->reg.map[i].mapped + ofs; } - return 0; + return INVALID_REG_ADDR; } static u32 __mt7996_reg_remap_addr(struct mt7996_dev *dev, u32 addr) @@ -366,7 +392,7 @@ void mt7996_memcpy_fromio(struct mt7996_dev *dev, void *buf, u32 offset, { u32 addr = __mt7996_reg_addr(dev, offset); - if (addr) { + if (addr != INVALID_REG_ADDR) { memcpy_fromio(buf, dev->mt76.mmio.regs + addr, len); return; } @@ -382,7 +408,7 @@ static u32 mt7996_rr(struct mt76_dev *mdev, u32 offset) struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); u32 addr = __mt7996_reg_addr(dev, offset), val; - if (addr) + if (addr != INVALID_REG_ADDR) return dev->bus_ops->rr(mdev, addr); spin_lock_bh(&dev->reg_lock); @@ -397,7 +423,7 @@ static void mt7996_wr(struct mt76_dev *mdev, u32 offset, u32 val) struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); u32 addr = __mt7996_reg_addr(dev, offset); - if (addr) { + if (addr != INVALID_REG_ADDR) { dev->bus_ops->wr(mdev, addr, val); return; } @@ -412,7 +438,7 @@ static u32 mt7996_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val) struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); u32 addr = __mt7996_reg_addr(dev, offset); - if (addr) + if (addr != INVALID_REG_ADDR) return dev->bus_ops->rmw(mdev, addr, mask, val); spin_lock_bh(&dev->reg_lock); @@ -464,8 +490,8 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, if (!wed_enable) return 0; - dev->mt76.hwrro_mode = is_mt7996(&dev->mt76) ? MT76_HWRRO_V3 - : MT76_HWRRO_V3_1; + if (hif2 && !mtk_wed_device_active(&dev->mt76.mmio.wed)) + return 0; hif1_ofs = dev->hif2 ? MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0) : 0; @@ -491,23 +517,16 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.wpdma_tx = wed->wlan.phy_base + hif1_ofs + MT_TXQ_RING_BASE(0) + MT7996_TXQ_BAND2 * MT_RING_SIZE; - if (mt7996_has_hwrro(dev)) { - if (is_mt7996(&dev->mt76)) { - wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_EXT) - 1; - wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + - MT_RXQ_RING_BASE(0) + - MT7996_RXQ_TXFREE2 * MT_RING_SIZE; - } else { - wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_BAND1_EXT) - 1; - wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + - MT_RXQ_RING_BASE(0) + - MT7996_RXQ_MCU_WA_EXT * MT_RING_SIZE; - } + if (is_mt7996(&dev->mt76)) { + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_EXT) - 1; + wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_TXFREE2 * MT_RING_SIZE; } else { + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_BAND1_EXT) - 1; wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + MT_RXQ_RING_BASE(0) + - MT7996_RXQ_MCU_WA_TRI * MT_RING_SIZE; - wed->wlan.txfree_tbit = ffs(MT_INT_RX_DONE_WA_TRI) - 1; + MT7996_RXQ_MCU_WA_EXT * MT_RING_SIZE; } wed->wlan.wpdma_rx_glo = wed->wlan.phy_base + hif1_ofs + MT_WFDMA0_GLO_CFG; @@ -518,7 +537,7 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.id = MT7996_DEVICE_ID_2; wed->wlan.tx_tbit[0] = ffs(MT_INT_TX_DONE_BAND2) - 1; } else { - wed->wlan.hw_rro = mt7996_has_hwrro(dev); + wed->wlan.hw_rro = true; wed->wlan.wpdma_int = wed->wlan.phy_base + MT_INT_SOURCE_CSR; wed->wlan.wpdma_mask = wed->wlan.phy_base + MT_INT_MASK_CSR; wed->wlan.wpdma_tx = wed->wlan.phy_base + MT_TXQ_RING_BASE(0) + @@ -571,23 +590,15 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.tx_tbit[0] = ffs(MT_INT_TX_DONE_BAND0) - 1; wed->wlan.tx_tbit[1] = ffs(MT_INT_TX_DONE_BAND1) - 1; if (is_mt7996(&dev->mt76)) { - if (mt7996_has_hwrro(dev)) { - wed->wlan.wpdma_txfree = wed->wlan.phy_base + - MT_RXQ_RING_BASE(0) + - MT7996_RXQ_TXFREE0 * MT_RING_SIZE; - wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_MAIN) - 1; - } else { - wed->wlan.wpdma_txfree = wed->wlan.phy_base + - MT_RXQ_RING_BASE(0) + - MT7996_RXQ_MCU_WA_MAIN * MT_RING_SIZE; - wed->wlan.txfree_tbit = ffs(MT_INT_RX_DONE_WA_MAIN) - 1; - } + wed->wlan.wpdma_txfree = wed->wlan.phy_base + + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_TXFREE0 * MT_RING_SIZE; + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_MAIN) - 1; } else { wed->wlan.txfree_tbit = ffs(MT_INT_RX_DONE_WA_MAIN) - 1; wed->wlan.wpdma_txfree = wed->wlan.phy_base + MT_RXQ_RING_BASE(0) + MT7996_RXQ_MCU_WA_MAIN * MT_RING_SIZE; } - dev->mt76.rx_token_size = MT7996_TOKEN_SIZE + wed->wlan.rx_npkt; if (dev->hif2 && is_mt7992(&dev->mt76)) wed->wlan.id = 0x7992; @@ -610,9 +621,14 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, wed->wlan.reset_complete = mt76_wed_reset_complete; } - if (mtk_wed_device_attach(wed)) { - dev->mt76.hwrro_mode = MT76_HWRRO_OFF; + if (mtk_wed_device_attach(wed)) return 0; + + if (!hif2) { + dev->mt76.hwrro_mode = is_mt7996(&dev->mt76) ? MT76_HWRRO_V3 + : MT76_HWRRO_V3_1; + dev->mt76.rx_token_size = MT7996_TOKEN_SIZE + + wed->wlan.rx_npkt; } *irq = wed->irq; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h index 0d6488522ba71..2e3edac2b5719 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -871,6 +871,7 @@ bool mt7996_mac_wtbl_update(struct mt7996_dev *dev, int idx, u32 mask); void mt7996_mac_reset_counters(struct mt7996_phy *phy); void mt7996_mac_cca_stats_reset(struct mt7996_phy *phy); void mt7996_mac_enable_nf(struct mt7996_dev *dev, u8 band); +struct mt76_wcid *mt7996_get_tx_wcid(struct mt76_wcid *wcid); void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, int pid, diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h index c6379933b6c36..8ff78cf6eb042 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h @@ -64,6 +64,14 @@ enum offs_rev { MIB_BSCR7, MIB_BSCR17, MIB_TRDR1, + MIB_TSCR0, + MIB_TSCR1, + MIB_TSCR2, + MIB_TSCR3, + MIB_TSCR4, + MIB_TSCR5, + MIB_TSCR6, + MIB_TSCR7, HIF_REMAP_L1, HIF_REMAP_BASE_L1, HIF_REMAP_L2, @@ -250,9 +258,9 @@ enum offs_rev { #define MT_MIB_BSCR7(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR7)) #define MT_MIB_BSCR17(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR17)) -#define MT_MIB_TSCR5(_band) MT_WF_MIB(_band, 0x6c4) -#define MT_MIB_TSCR6(_band) MT_WF_MIB(_band, 0x6c8) -#define MT_MIB_TSCR7(_band) MT_WF_MIB(_band, 0x6d0) +#define MT_MIB_TSCR5(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR5)) +#define MT_MIB_TSCR6(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR6)) +#define MT_MIB_TSCR7(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR7)) #define MT_MIB_RSCR1(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR1)) /* rx mpdu counter, full 32 bits */ @@ -268,14 +276,14 @@ enum offs_rev { #define MT_MIB_RSCR36(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR36)) /* tx ampdu cnt, full 32 bits */ -#define MT_MIB_TSCR0(_band) MT_WF_MIB(_band, 0x6b0) -#define MT_MIB_TSCR2(_band) MT_WF_MIB(_band, 0x6b8) +#define MT_MIB_TSCR0(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR0)) +#define MT_MIB_TSCR2(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR2)) /* counts all mpdus in ampdu, regardless of success */ -#define MT_MIB_TSCR3(_band) MT_WF_MIB(_band, 0x6bc) +#define MT_MIB_TSCR3(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR3)) /* counts all successfully tx'd mpdus in ampdu */ -#define MT_MIB_TSCR4(_band) MT_WF_MIB(_band, 0x6c0) +#define MT_MIB_TSCR4(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR4)) /* rx ampdu count, 32-bit */ #define MT_MIB_RSCR27(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR27)) @@ -299,7 +307,7 @@ enum offs_rev { #define MT_MIB_RVSR1(_band) MT_WF_MIB(_band, __OFFS(MIB_RVSR1)) /* rx blockack count, 32 bits */ -#define MT_MIB_TSCR1(_band) MT_WF_MIB(_band, 0x6b4) +#define MT_MIB_TSCR1(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR1)) #define MT_MIB_BTSCR0(_band) MT_WF_MIB(_band, 0x5e0) #define MT_MIB_BTSCR5(_band) MT_WF_MIB(_band, __OFFS(MIB_BTSCR5)) diff --git a/drivers/net/wireless/mediatek/mt76/scan.c b/drivers/net/wireless/mediatek/mt76/scan.c index 7fe1b1fbb699b..3cb11689d4bf2 100644 --- a/drivers/net/wireless/mediatek/mt76/scan.c +++ b/drivers/net/wireless/mediatek/mt76/scan.c @@ -16,10 +16,17 @@ static void mt76_scan_complete(struct mt76_dev *dev, bool abort) clear_bit(MT76_SCANNING, &phy->state); - if (dev->scan.chan && phy->main_chandef.chan && phy->offchannel && + /* Re-program the operating channel even when the scan never left it: + * any channel set during the scan ran with MT76_SCANNING held, which + * left DFS radar detection disabled + */ + if (phy->main_chandef.chan && !test_bit(MT76_MCU_RESET, &dev->phy.state)) { + bool offchannel = phy->offchannel; + mt76_set_channel(phy, &phy->main_chandef, false); - mt76_offchannel_notify(phy, false); + if (offchannel) + mt76_offchannel_notify(phy, false); } mt76_put_vif_phy_link(phy, dev->scan.vif, dev->scan.mlink); memset(&dev->scan, 0, sizeof(dev->scan)); @@ -48,6 +55,7 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid) struct mt76_phy *phy = dev->scan.phy; struct ieee80211_tx_info *info; struct sk_buff *skb; + u8 link_id; skb = ieee80211_probereq_get(phy->hw, vif->addr, ssid->ssid, ssid->ssid_len, req->ie_len); @@ -77,6 +85,10 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid) info->flags |= IEEE80211_TX_CTL_NO_CCK_RATE; info->control.flags |= IEEE80211_TX_CTRL_DONT_USE_RATE_MASK; + link_id = mvif->wcid ? mvif->wcid->link_id : IEEE80211_LINK_UNSPECIFIED; + info->control.flags &= ~IEEE80211_TX_CTRL_MLO_LINK; + info->control.flags |= u32_encode_bits(link_id, IEEE80211_TX_CTRL_MLO_LINK); + mt76_tx(phy, NULL, mvif->wcid, skb); out: @@ -206,6 +218,7 @@ int mt76_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif, dev->scan.vif = vif; dev->scan.phy = phy; dev->scan.mlink = mlink; + set_bit(MT76_SCANNING, &phy->state); ieee80211_queue_delayed_work(dev->phy.hw, &dev->scan_work, 0); out: diff --git a/drivers/net/wireless/mediatek/mt76/sdio.c b/drivers/net/wireless/mediatek/mt76/sdio.c index 8bae77c761bea..ba5f123f7e39d 100644 --- a/drivers/net/wireless/mediatek/mt76/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/sdio.c @@ -519,6 +519,10 @@ mt76s_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, enum mt76_txq_id qid, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_sta *sta) { + struct ieee80211_tx_status status = { + .sta = sta, + }; + struct mt76_tx_info tx_info = { .skb = skb, }; @@ -531,8 +535,13 @@ mt76s_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, skb->prev = skb->next = NULL; err = dev->drv->tx_prepare_skb(dev, NULL, qid, wcid, sta, &tx_info); - if (err < 0) + if (err < 0) { + status.skb = tx_info.skb; + spin_lock_bh(&dev->rx_lock); + ieee80211_tx_status_ext(dev->hw, &status); + spin_unlock_bh(&dev->rx_lock); return err; + } q->entry[q->head].skb = tx_info.skb; q->entry[q->head].buf_sz = len; diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c index f96d9c4718535..b03be0eb47128 100644 --- a/drivers/net/wireless/mediatek/mt76/tx.c +++ b/drivers/net/wireless/mediatek/mt76/tx.c @@ -324,10 +324,6 @@ __mt76_tx_queue_skb(struct mt76_phy *phy, int qid, struct sk_buff *skb, if (idx < 0 || !sta) return idx; - wcid = (struct mt76_wcid *)sta->drv_priv; - if (!wcid->sta) - return idx; - q->entry[idx].wcid = wcid->idx; if (!non_aql) @@ -635,6 +631,7 @@ mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid, !ieee80211_is_data_present(hdr->frame_control) && (!ieee80211_is_bufferable_mmpdu(skb) || ieee80211_is_deauth(hdr->frame_control) || + ieee80211_is_disassoc(hdr->frame_control) || head == &wcid->tx_offchannel)) qid = MT_TXQ_PSD; @@ -686,8 +683,8 @@ void mt76_txq_schedule_pending(struct mt76_phy *phy) ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_pending); spin_lock(&phy->tx_lock); - if (!skb_queue_empty(&wcid->tx_pending) && - !skb_queue_empty(&wcid->tx_offchannel) && + if ((!skb_queue_empty(&wcid->tx_pending) || + !skb_queue_empty(&wcid->tx_offchannel)) && list_empty(&wcid->tx_list)) list_add_tail(&wcid->tx_list, &phy->tx_list); } diff --git a/drivers/net/wireless/mediatek/mt76/usb.c b/drivers/net/wireless/mediatek/mt76/usb.c index d9638a9b749b6..345f1d9c1947c 100644 --- a/drivers/net/wireless/mediatek/mt76/usb.c +++ b/drivers/net/wireless/mediatek/mt76/usb.c @@ -30,6 +30,8 @@ int __mt76u_vendor_request(struct mt76_dev *dev, u8 req, u8 req_type, for (i = 0; i < MT_VEND_REQ_MAX_RETRY; i++) { if (test_bit(MT76_REMOVED, &dev->phy.state)) return -EIO; + if (dev->usb.ctrl_timeout && atomic_read(&dev->bus_hung)) + return -EIO; ret = usb_control_msg(udev, pipe, req, req_type, val, offset, buf, len, MT_VEND_REQ_TOUT_MS); @@ -42,6 +44,15 @@ int __mt76u_vendor_request(struct mt76_dev *dev, u8 req, u8 req_type, dev_err(dev->dev, "vendor request req:%02x off:%04x failed:%d\n", req, offset, ret); + + if (dev->usb.ctrl_timeout) { + atomic_set(&dev->bus_hung, true); + dev_err(dev->dev, "vendor request req:%02x off:%04x timed out, marking bus hung\n", + req, offset); + dev->usb.ctrl_timeout(dev, ret); + return ret; + } + return ret; } EXPORT_SYMBOL_GPL(__mt76u_vendor_request); diff --git a/drivers/net/wireless/microchip/wilc1000/cfg80211.c b/drivers/net/wireless/microchip/wilc1000/cfg80211.c index 6654fce4ded84..2737f3b726843 100644 --- a/drivers/net/wireless/microchip/wilc1000/cfg80211.c +++ b/drivers/net/wireless/microchip/wilc1000/cfg80211.c @@ -1058,6 +1058,13 @@ void wilc_wfi_p2p_rx(struct wilc_vif *vif, u8 *buff, u32 size) if (!ieee80211_is_public_action((struct ieee80211_hdr *)buff, size)) goto out_rx_mgmt; + /* ieee80211_is_public_action() only validates up to the category + * byte, so reject frames too short for the P2P public action header + * before dereferencing it or computing size - ie_offset. + */ + if (size < ie_offset) + goto out_rx_mgmt; + d = (struct wilc_p2p_pub_act_frame *)(&mgmt->u.action); if (d->oui_subtype != GO_NEG_REQ && d->oui_subtype != GO_NEG_RSP && d->oui_subtype != P2P_INV_REQ && d->oui_subtype != P2P_INV_RSP) @@ -1207,6 +1214,13 @@ static int mgmt_tx(struct wiphy *wiphy, goto out_set_timeout; } + /* ieee80211_is_public_action() only validates up to the category + * byte, so reject frames too short for the P2P public action header + * before dereferencing it or computing len - ie_offset. + */ + if (len < ie_offset) + goto out_set_timeout; + d = (struct wilc_p2p_pub_act_frame *)(&mgmt->u.action); if (d->oui_type != WLAN_OUI_TYPE_WFA_P2P || d->oui_subtype != GO_NEG_CONF) { diff --git a/drivers/net/wireless/microchip/wilc1000/wlan.c b/drivers/net/wireless/microchip/wilc1000/wlan.c index 4b116fe6f9ea9..55a77a2e32887 100644 --- a/drivers/net/wireless/microchip/wilc1000/wlan.c +++ b/drivers/net/wireless/microchip/wilc1000/wlan.c @@ -1197,6 +1197,15 @@ static void wilc_wlan_handle_isr_ext(struct wilc *wilc, u32 int_status) if (size <= 0) return; + /* A size exceeding the RX buffer is bogus; drop the transfer + * instead of overflowing the buffer. + */ + if (size > WILC_RX_BUFF_SIZE) { + wilc->hif_func->hif_clear_int_ext(wilc, + DATA_INT_CLR | ENABLE_RX_VMM); + return; + } + if (WILC_RX_BUFF_SIZE - offset < size) offset = 0; diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c b/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c index 070c0431c4821..4a5989172a5e3 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c +++ b/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c @@ -1652,7 +1652,7 @@ static void rtl8180_eeprom_register_write(struct eeprom_93cx6 *eeprom) static void rtl8180_eeprom_read(struct rtl8180_priv *priv) { - struct eeprom_93cx6 eeprom; + struct eeprom_93cx6 eeprom = {}; int eeprom_cck_table_adr; u16 eeprom_val; int i; diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c b/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c index 1d21c468a2368..a766712187f77 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c +++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c @@ -1445,7 +1445,7 @@ static int rtl8187_probe(struct usb_interface *intf, struct usb_device *udev = interface_to_usbdev(intf); struct ieee80211_hw *dev; struct rtl8187_priv *priv; - struct eeprom_93cx6 eeprom; + struct eeprom_93cx6 eeprom = {}; struct ieee80211_channel *channel; const char *chip_name; u16 txpwr, reg; diff --git a/drivers/net/wireless/realtek/rtl8xxxu/core.c b/drivers/net/wireless/realtek/rtl8xxxu/core.c index 646fe76b086e7..4f4bda3e0ce26 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/core.c @@ -5838,14 +5838,19 @@ static void rtl8xxxu_queue_rx_urb(struct rtl8xxxu_priv *priv, { struct sk_buff *skb; unsigned long flags; - int pending = 0; spin_lock_irqsave(&priv->rx_urb_lock, flags); if (!priv->shutdown) { list_add_tail(&rx_urb->list, &priv->rx_urb_pending_list); priv->rx_urb_pending_count++; - pending = priv->rx_urb_pending_count; + /* + * Arm the worker under rx_urb_lock so this is atomic with the + * shutdown check: moving it out of the lock would let a + * completion arm the work after rtl8xxxu_stop() canceled it. + */ + if (priv->rx_urb_pending_count > RTL8XXXU_RX_URB_PENDING_WATER) + schedule_work(&priv->rx_urb_wq); } else { skb = (struct sk_buff *)rx_urb->urb.context; dev_kfree_skb_irq(skb); @@ -5853,9 +5858,6 @@ static void rtl8xxxu_queue_rx_urb(struct rtl8xxxu_priv *priv, } spin_unlock_irqrestore(&priv->rx_urb_lock, flags); - - if (pending > RTL8XXXU_RX_URB_PENDING_WATER) - schedule_work(&priv->rx_urb_wq); } static void rtl8xxxu_rx_urb_work(struct work_struct *work) @@ -7507,6 +7509,13 @@ static void rtl8xxxu_stop(struct ieee80211_hw *hw, bool suspend) priv->shutdown = true; spin_unlock_irqrestore(&priv->rx_urb_lock, flags); + /* + * Cancel before killing rx_anchor: the worker re-anchors every URB + * it drained via rtl8xxxu_submit_rx_urb(), so a worker still running + * after the kill could submit a URB that escapes it. + */ + cancel_work_sync(&priv->rx_urb_wq); + usb_kill_anchored_urbs(&priv->rx_anchor); usb_kill_anchored_urbs(&priv->tx_anchor); if (priv->usb_interrupts) diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c index 73018a0498b4b..de74ff8f6eee7 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.c +++ b/drivers/net/wireless/realtek/rtlwifi/pci.c @@ -2227,13 +2227,17 @@ int rtl_pci_probe(struct pci_dev *pdev, rtl_dbg(rtlpriv, COMP_INIT, DBG_DMESG, "%s: failed to register IRQ handler\n", wiphy_name(hw->wiphy)); - goto fail3; + goto fail6; } rtlpci->irq_alloc = 1; set_bit(RTL_STATUS_INTERFACE_START, &rtlpriv->status); return 0; +fail6: + rtl_deinit_rfkill(hw); + rtl_debug_remove_one(hw); + ieee80211_unregister_hw(hw); fail5: rtl_pci_deinit(hw); fail4: diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192du/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192du/sw.c index 7bafb051d5ec9..88e8cc590c544 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192du/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192du/sw.c @@ -145,8 +145,10 @@ static int rtl92du_init_sw_vars(struct ieee80211_hw *hw) /* for firmware buf */ rtlpriv->rtlhal.pfirmware = kmalloc(0x8000, GFP_KERNEL); - if (!rtlpriv->rtlhal.pfirmware) - return -ENOMEM; + if (!rtlpriv->rtlhal.pfirmware) { + err = -ENOMEM; + goto error; + } rtlpriv->max_fw_size = 0x8000; pr_info("Driver for Realtek RTL8192DU WLAN interface\n"); @@ -160,10 +162,14 @@ static int rtl92du_init_sw_vars(struct ieee80211_hw *hw) pr_err("Failed to request firmware!\n"); kfree(rtlpriv->rtlhal.pfirmware); rtlpriv->rtlhal.pfirmware = NULL; - return err; + goto error; } return 0; + +error: + rtl92du_deinit_shared_data(hw); + return err; } static void rtl92du_deinit_sw_vars(struct ieee80211_hw *hw) diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192du/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192du/trx.c index 743ce0cfffe6d..c608c51f1b785 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192du/trx.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192du/trx.c @@ -106,7 +106,8 @@ void rtl92du_tx_fill_desc(struct ieee80211_hw *hw, if (sta) { sta_entry = (struct rtl_sta_info *)sta->drv_priv; tid = ieee80211_get_tid(hdr); - agg_state = sta_entry->tids[tid].agg.agg_state; + if (tid < MAX_TID_COUNT) + agg_state = sta_entry->tids[tid].agg.agg_state; ampdu_density = sta->deflink.ht_cap.ampdu_density; } diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c index a304672289124..69f2840fed095 100644 --- a/drivers/net/wireless/realtek/rtw88/pci.c +++ b/drivers/net/wireless/realtek/rtw88/pci.c @@ -1834,7 +1834,7 @@ int rtw_pci_probe(struct pci_dev *pdev, ret = rtw_pci_napi_init(rtwdev); if (ret) { rtw_err(rtwdev, "failed to setup NAPI\n"); - goto err_pci_declaim; + goto err_destroy_rsrc; } ret = rtw_chip_info_setup(rtwdev); @@ -1866,6 +1866,8 @@ int rtw_pci_probe(struct pci_dev *pdev, err_destroy_pci: rtw_pci_napi_deinit(rtwdev); + +err_destroy_rsrc: rtw_pci_destroy(rtwdev, pdev); err_pci_declaim: diff --git a/drivers/net/wireless/realtek/rtw88/tx.c b/drivers/net/wireless/realtek/rtw88/tx.c index 9d747a060b984..8786bbb421c2b 100644 --- a/drivers/net/wireless/realtek/rtw88/tx.c +++ b/drivers/net/wireless/realtek/rtw88/tx.c @@ -619,6 +619,7 @@ static int rtw_txq_push_skb(struct rtw_dev *rtwdev, ret = rtw_hci_tx_write(rtwdev, &pkt_info, skb); if (ret) { rtw_err(rtwdev, "failed to write TX skb to HCI\n"); + ieee80211_free_txskb(rtwdev->hw, skb); return ret; } return 0; diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index 5547888d7e67a..a85b9c39c2570 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -5373,6 +5373,7 @@ enum rtw89_flags { RTW89_FLAG_CHANGING_INTERFACE, RTW89_FLAG_HW_RFKILL_STATE, RTW89_FLAG_UNPLUGGED, + RTW89_FLAG_SHUTDOWN, NUM_OF_RTW89_FLAGS, }; diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index 8f5af873e09f5..5786120602ab0 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -4348,7 +4348,7 @@ static const char *rtw89_ppdu_str(struct rtw89_dev *rtwdev, u8 type, u8 subtype) const struct rtw89_chip_info *chip = rtwdev->chip; const struct rtw89_ppdu_info *ppdu_info; - if (type > ARRAY_SIZE(rtw89_ppdu_infos)) + if (type >= ARRAY_SIZE(rtw89_ppdu_infos)) return "RSVD"; ppdu_info = &rtw89_ppdu_infos[type]; diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index d6a594b75ab20..5edad2d25ae4b 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -3439,7 +3439,7 @@ int rtw89_fw_h2c_lps_ml_cmn_info_v1(struct rtw89_dev *rtwdev, h2c->rfe_type = efuse->rfe_type; h2c->rssi_main = U8_MAX; - memset(h2c->link_id, 0xfe, RTW89_BB_PS_LINK_BUF_MAX); + memset(h2c->link_id, RTW89_BB_PS_LINK_ID_SKIP, RTW89_BB_PS_LINK_BUF_MAX); rtw89_vif_for_each_link(rtwvif, rtwvif_link, link_id) { u8 phy_idx = rtwvif_link->phy_idx; @@ -3447,7 +3447,7 @@ int rtw89_fw_h2c_lps_ml_cmn_info_v1(struct rtw89_dev *rtwdev, bb = rtw89_get_bb_ctx(rtwdev, phy_idx); chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); - h2c->link_id[phy_idx] = phy_idx; + h2c->link_id[phy_idx] = link_id; h2c->central_ch[phy_idx] = chan->channel; h2c->pri_ch[phy_idx] = chan->primary_channel; h2c->band[phy_idx] = chan->band_type; @@ -3729,7 +3729,7 @@ int rtw89_fw_h2c_default_cmac_tbl_be(struct rtw89_dev *rtwdev, le32_encode_bits(4, BE_CCTL_INFO_W1_RTS_RTY_LOWEST_RATE); h2c->m1 = cpu_to_le32(BE_CCTL_INFO_W1_ALL); - h2c->w1 = le32_encode_bits(preld, BE_CCTL_INFO_W2_PRELOAD_ENABLE); + h2c->w2 = le32_encode_bits(preld, BE_CCTL_INFO_W2_PRELOAD_ENABLE); h2c->m2 = cpu_to_le32(BE_CCTL_INFO_W2_ALL); h2c->m3 = cpu_to_le32(BE_CCTL_INFO_W3_ALL); @@ -11780,7 +11780,7 @@ static void rtw89_fw_cmd_ofld_write_rf(struct rtw89_dev *rtwdev, static void rtw89_fw_cmd_ofld_udelay(struct rtw89_dev *rtwdev, u32 us) { struct rtw89_fw_cmd_ofld_arg cmd = { - .src = RTW89_FW_CMD_OFLD_SRC_OTHER, + .src = RTW89_FW_CMD_OFLD_SRC_MAC, .type = RTW89_FW_CMD_OFLD_DELAY, .value = us, }; @@ -11794,7 +11794,7 @@ static void rtw89_fw_cmd_ofld_udelay(struct rtw89_dev *rtwdev, u32 us) static void rtw89_fw_cmd_ofld_mdelay(struct rtw89_dev *rtwdev, u32 ms) { struct rtw89_fw_cmd_ofld_arg cmd = { - .src = RTW89_FW_CMD_OFLD_SRC_OTHER, + .src = RTW89_FW_CMD_OFLD_SRC_MAC, .type = RTW89_FW_CMD_OFLD_DELAY, .value = ms * 1000, }; diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index 20721d5209aa3..8c90865dfa3b5 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -2053,6 +2053,8 @@ enum rtw89_bb_link_rx_gain_table_type { RTW89_BB_PS_LINK_RX_GAIN_TAB_MAX, }; +#define RTW89_BB_PS_LINK_ID_SKIP 0xfe + enum rtw89_bb_ps_link_buf_id { RTW89_BB_PS_LINK_BUF_0 = 0x00, RTW89_BB_PS_LINK_BUF_1 = 0x01, @@ -3142,7 +3144,6 @@ enum rtw89_fw_cmd_ofld_arg_src { RTW89_FW_CMD_OFLD_SRC_RF, RTW89_FW_CMD_OFLD_SRC_MAC, RTW89_FW_CMD_OFLD_SRC_RF_DDIE, - RTW89_FW_CMD_OFLD_SRC_OTHER, }; enum rtw89_fw_cmd_ofld_arg_type { diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index 8c395517bd2fe..99de1b2029768 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -5167,7 +5167,7 @@ static void rtw89_mac_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy, elem = cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY, ies->data, ies->len); - if (!elem || elem->datalen < 10 || + if (!elem || elem->datalen < 11 || !(elem->data[10] & WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT)) *tolerated = false; rcu_read_unlock(); diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c index aade5c5b79e82..c1be69a3c1926 100644 --- a/drivers/net/wireless/realtek/rtw89/mac80211.c +++ b/drivers/net/wireless/realtek/rtw89/mac80211.c @@ -826,11 +826,36 @@ static int rtw89_ops_start_ap(struct ieee80211_hw *hw, ether_addr_copy(rtwvif_link->bssid, link_conf->bssid); rtw89_cam_bssid_changed(rtwdev, rtwvif_link); - rtw89_mac_port_update(rtwdev, rtwvif_link); - rtw89_chip_h2c_assoc_cmac_tbl(rtwdev, rtwvif_link, NULL); - rtw89_fw_h2c_role_maintain(rtwdev, rtwvif_link, NULL, RTW89_ROLE_TYPE_CHANGE); - rtw89_fw_h2c_join_info(rtwdev, rtwvif_link, NULL, true); - rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL, RTW89_ROLE_TYPE_CHANGE); + ret = rtw89_mac_port_update(rtwdev, rtwvif_link); + if (ret) { + rtw89_warn(rtwdev, "failed to update mac port\n"); + return ret; + } + + ret = rtw89_chip_h2c_assoc_cmac_tbl(rtwdev, rtwvif_link, NULL); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c cmac table\n"); + return ret; + } + + ret = rtw89_fw_h2c_role_maintain(rtwdev, rtwvif_link, NULL, RTW89_ROLE_TYPE_CHANGE); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c role info\n"); + return ret; + } + + ret = rtw89_fw_h2c_join_info(rtwdev, rtwvif_link, NULL, true); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c join info\n"); + return ret; + } + + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL, RTW89_ROLE_TYPE_CHANGE); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c cam\n"); + return ret; + } + rtw89_chip_rfk_channel(rtwdev, rtwvif_link); if (RTW89_CHK_FW_FEATURE(NOTIFY_AP_INFO, &rtwdev->fw)) { @@ -1979,7 +2004,8 @@ static void rtw89_ops_rfkill_poll(struct ieee80211_hw *hw) lockdep_assert_wiphy(hw->wiphy); /* wl_disable GPIO get floating when entering LPS */ - if (test_bit(RTW89_FLAG_RUNNING, rtwdev->flags)) + if (test_bit(RTW89_FLAG_RUNNING, rtwdev->flags) || + test_bit(RTW89_FLAG_SHUTDOWN, rtwdev->flags)) return; rtw89_core_rfkill_poll(rtwdev, false); diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index 102bae4881805..61c54b52be1f5 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -4874,6 +4874,19 @@ void rtw89_pci_remove(struct pci_dev *pdev) } EXPORT_SYMBOL(rtw89_pci_remove); +void rtw89_pci_shutdown(struct pci_dev *pdev) +{ + struct ieee80211_hw *hw = pci_get_drvdata(pdev); + struct rtw89_dev *rtwdev; + + if (!hw) + return; + + rtwdev = hw->priv; + set_bit(RTW89_FLAG_SHUTDOWN, rtwdev->flags); +} +EXPORT_SYMBOL(rtw89_pci_shutdown); + MODULE_AUTHOR("Realtek Corporation"); MODULE_DESCRIPTION("Realtek PCI 802.11ax wireless driver"); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/wireless/realtek/rtw89/pci.h b/drivers/net/wireless/realtek/rtw89/pci.h index c3f2d0df5846d..0728120d721ff 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.h +++ b/drivers/net/wireless/realtek/rtw89/pci.h @@ -1751,6 +1751,7 @@ struct pci_device_id; int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id); void rtw89_pci_remove(struct pci_dev *pdev); +void rtw89_pci_shutdown(struct pci_dev *pdev); void rtw89_pci_basic_cfg(struct rtw89_dev *rtwdev, bool resume); void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev); int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8851be.c b/drivers/net/wireless/realtek/rtw89/rtw8851be.c index 640672eb0d268..61e6c997b206f 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8851be.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8851be.c @@ -93,6 +93,7 @@ static struct pci_driver rtw89_8851be_driver = { .id_table = rtw89_8851be_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a.c b/drivers/net/wireless/realtek/rtw89/rtw8852a.c index 2c1f166e687f0..e27a81bd8b77d 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852a.c @@ -2212,7 +2212,7 @@ static void rtw8852a_query_ppdu(struct rtw89_dev *rtwdev, u8 raw; if (!status->signal) { - if (phy_ppdu->to_self) + if (phy_ppdu->to_self && ewma_rssi_read(&bb->bcn_rssi)) raw = ewma_rssi_read(&bb->bcn_rssi); else raw = max(rx_power[RF_PATH_A], rx_power[RF_PATH_B]); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852ae.c b/drivers/net/wireless/realtek/rtw89/rtw8852ae.c index 64306cdc1ee48..e6fbcf75f1ad3 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852ae.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852ae.c @@ -95,6 +95,7 @@ static struct pci_driver rtw89_8852ae_driver = { .id_table = rtw89_8852ae_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852be.c b/drivers/net/wireless/realtek/rtw89/rtw8852be.c index 5bc0a6a99d1d2..25463342ac890 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852be.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852be.c @@ -97,6 +97,7 @@ static struct pci_driver rtw89_8852be_driver = { .id_table = rtw89_8852be_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852bte.c b/drivers/net/wireless/realtek/rtw89/rtw8852bte.c index 49a72ca835ac7..bf22be739f399 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852bte.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852bte.c @@ -99,6 +99,7 @@ static struct pci_driver rtw89_8852bte_driver = { .id_table = rtw89_8852bte_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852ce.c b/drivers/net/wireless/realtek/rtw89/rtw8852ce.c index 3c64c0539205c..b8d4d9f026869 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852ce.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852ce.c @@ -122,6 +122,7 @@ static struct pci_driver rtw89_8852ce_driver = { .id_table = rtw89_8852ce_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8922ae.c b/drivers/net/wireless/realtek/rtw89/rtw8922ae.c index 5527a8db393b5..54bcd3f3d0082 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8922ae.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8922ae.c @@ -111,6 +111,7 @@ static struct pci_driver rtw89_8922ae_driver = { .id_table = rtw89_8922ae_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops_be, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8922de.c b/drivers/net/wireless/realtek/rtw89/rtw8922de.c index a1a81c338be3f..65476a35a323c 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8922de.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8922de.c @@ -111,6 +111,7 @@ static struct pci_driver rtw89_8922de_driver = { .id_table = rtw89_8922de_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops_be, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/rsi/rsi_91x_mgmt.c b/drivers/net/wireless/rsi/rsi_91x_mgmt.c index bb167f03367bf..d9dcbb2553176 100644 --- a/drivers/net/wireless/rsi/rsi_91x_mgmt.c +++ b/drivers/net/wireless/rsi/rsi_91x_mgmt.c @@ -852,8 +852,6 @@ int rsi_hal_load_key(struct rsi_common *common, memcpy(set_key->tx_mic_key, &data[16], 8); memcpy(set_key->rx_mic_key, &data[24], 8); } - } else { - memset(&set_key[FRAME_DESC_SZ], 0, frame_len - FRAME_DESC_SZ); } skb_put(skb, frame_len); diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c index be583ae331c03..ecf55e34caae9 100644 --- a/drivers/net/wireless/ti/wlcore/main.c +++ b/drivers/net/wireless/ti/wlcore/main.c @@ -3724,10 +3724,8 @@ void wlcore_regdomain_config(struct wl1271 *wl) goto out; ret = wlcore_cmd_regdomain_config_locked(wl); - if (ret < 0) { + if (ret < 0) wl12xx_queue_recovery_work(wl); - goto out; - } pm_runtime_put_autosuspend(wl->dev); out: diff --git a/drivers/net/wireless/virtual/mac80211_hwsim_main.c b/drivers/net/wireless/virtual/mac80211_hwsim_main.c index 75caa97becc8d..8c634cada5ddd 100644 --- a/drivers/net/wireless/virtual/mac80211_hwsim_main.c +++ b/drivers/net/wireless/virtual/mac80211_hwsim_main.c @@ -2314,15 +2314,21 @@ static int mac80211_hwsim_start(struct ieee80211_hw *hw) static void mac80211_hwsim_stop(struct ieee80211_hw *hw, bool suspend) { struct mac80211_hwsim_data *data = hw->priv; + struct sk_buff *skb; int i; - data->started = false; + /* + * Serialise against wmediumd userspace, so no more frames + * can be handed to mac80211 after this returns. + */ + scoped_guard(mutex, &data->mutex) + data->started = false; for (i = 0; i < ARRAY_SIZE(data->link_data); i++) hrtimer_cancel(&data->link_data[i].beacon_timer); - while (!skb_queue_empty(&data->pending)) - ieee80211_free_txskb(hw, skb_dequeue(&data->pending)); + while ((skb = skb_dequeue(&data->pending))) + ieee80211_free_txskb(hw, skb); wiphy_dbg(hw->wiphy, "%s\n", __func__); } @@ -6452,12 +6458,12 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2, if (frame_data_len < sizeof(struct ieee80211_hdr_3addr) || frame_data_len > IEEE80211_MAX_DATA_LEN) - goto err; + goto out; /* Allocate new skb here */ skb = alloc_skb(frame_data_len, GFP_KERNEL); if (skb == NULL) - goto err; + goto out; /* Copy the data */ skb_put_data(skb, frame_data, frame_data_len); @@ -6482,10 +6488,17 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2, goto out; } + /* + * Serialise against mac80211_hwsim_stop() - mac80211 doesn't allow + * frames reported while the HW is down, hence the ->started check + * must be under mutex. + */ + mutex_lock(&data2->mutex); + /* check if radio is configured properly */ if ((data2->idle && !data2->tmp_chan) || !data2->started) - goto out; + goto out_unlock; /* A frame is received from user space */ memset(&rx_status, 0, sizeof(rx_status)); @@ -6504,22 +6517,18 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2, iter_data.channel = ieee80211_get_channel(data2->hw->wiphy, rx_status.freq); if (!iter_data.channel) - goto out; + goto out_unlock; rx_status.band = iter_data.channel->band; - mutex_lock(&data2->mutex); if (!hwsim_chans_compat(iter_data.channel, channel)) { ieee80211_iterate_active_interfaces_atomic( data2->hw, IEEE80211_IFACE_ITER_NORMAL, mac80211_hwsim_tx_iter, &iter_data); - if (!iter_data.receive) { - mutex_unlock(&data2->mutex); - goto out; - } + if (!iter_data.receive) + goto out_unlock; } - mutex_unlock(&data2->mutex); } else if (!channel) { - goto out; + goto out_unlock; } else { rx_status.freq = channel->center_freq; rx_status.band = channel->band; @@ -6527,7 +6536,7 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2, rx_status.rate_idx = nla_get_u32(info->attrs[HWSIM_ATTR_RX_RATE]); if (rx_status.rate_idx >= data2->hw->wiphy->bands[rx_status.band]->n_bitrates) - goto out; + goto out_unlock; rx_status.signal = nla_get_u32(info->attrs[HWSIM_ATTR_SIGNAL]); hdr = (void *)skb->data; @@ -6537,10 +6546,11 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2, rx_status.boottime_ns = ktime_get_boottime_ns(); mac80211_hwsim_rx(data2, &rx_status, skb); + mutex_unlock(&data2->mutex); return 0; -err: - pr_debug("mac80211_hwsim: error occurred in %s\n", __func__); +out_unlock: + mutex_unlock(&data2->mutex); out: dev_kfree_skb(skb); return -EINVAL; diff --git a/drivers/net/wireless/virtual/virt_wifi.c b/drivers/net/wireless/virtual/virt_wifi.c index 2335e45db8b85..48afc2432f93a 100644 --- a/drivers/net/wireless/virtual/virt_wifi.c +++ b/drivers/net/wireless/virtual/virt_wifi.c @@ -434,6 +434,7 @@ static netdev_tx_t virt_wifi_start_xmit(struct sk_buff *skb, priv->tx_packets++; if (!priv->is_connected) { priv->tx_failed++; + dev_kfree_skb_any(skb); return NET_XMIT_DROP; } @@ -557,7 +558,6 @@ static int virt_wifi_newlink(struct net_device *dev, } eth_hw_addr_inherit(dev, priv->lowerdev); - netif_stacked_transfer_operstate(priv->lowerdev, dev); dev->ieee80211_ptr = kzalloc_obj(*dev->ieee80211_ptr); @@ -583,6 +583,8 @@ static int virt_wifi_newlink(struct net_device *dev, goto unregister_netdev; } + netif_stacked_transfer_operstate(priv->lowerdev, dev); + dev->priv_destructor = virt_wifi_net_device_destructor; priv->being_deleted = false; priv->is_connected = false; diff --git a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c index 966d8ccb0dbcd..98102c663434b 100644 --- a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c +++ b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c @@ -1353,6 +1353,14 @@ static int probe(struct usb_interface *intf, const struct usb_device_id *id) struct zd_usb *usb; struct ieee80211_hw *hw = NULL; + /* + * ZD1211 devices are single-function. Reject secondary interfaces + * to prevent multiple instances from conflicting on hardcoded endpoints + * and triggering recursive locking warnings. + */ + if (intf->cur_altsetting->desc.bInterfaceNumber != 0) + return -ENODEV; + print_id(udev); if (id->driver_info & DEVICE_INSTALLER) diff --git a/drivers/net/wwan/mhi_wwan_mbim.c b/drivers/net/wwan/mhi_wwan_mbim.c index a949987125976..336f89756f10d 100644 --- a/drivers/net/wwan/mhi_wwan_mbim.c +++ b/drivers/net/wwan/mhi_wwan_mbim.c @@ -251,6 +251,14 @@ static int mbim_rx_verify_ndp16(struct sk_buff *skb, struct usb_cdc_ncm_ndp16 *n return ret; } +static void mhi_mbim_rx_drop(struct mhi_mbim_link *link, struct sk_buff *skb) +{ + dev_kfree_skb_any(skb); + u64_stats_update_begin(&link->rx_syncp); + u64_stats_inc(&link->rx_errors); + u64_stats_update_end(&link->rx_syncp); +} + static void mhi_mbim_rx(struct mhi_mbim_context *mbim, struct sk_buff *skb) { int ndpoffset; @@ -320,7 +328,10 @@ static void mhi_mbim_rx(struct mhi_mbim_context *mbim, struct sk_buff *skb) continue; skb_put(skbn, dgram_len); - skb_copy_bits(skb, dgram_offset, skbn->data, dgram_len); + if (skb_copy_bits(skb, dgram_offset, skbn->data, dgram_len)) { + mhi_mbim_rx_drop(link, skbn); + continue; + } switch (skbn->data[0] & 0xf0) { case 0x40: @@ -332,10 +343,7 @@ static void mhi_mbim_rx(struct mhi_mbim_context *mbim, struct sk_buff *skb) default: net_err_ratelimited("%s: unknown protocol\n", link->ndev->name); - dev_kfree_skb_any(skbn); - u64_stats_update_begin(&link->rx_syncp); - u64_stats_inc(&link->rx_errors); - u64_stats_update_end(&link->rx_syncp); + mhi_mbim_rx_drop(link, skbn); continue; } @@ -349,9 +357,13 @@ static void mhi_mbim_rx(struct mhi_mbim_context *mbim, struct sk_buff *skb) unlock: rcu_read_unlock(); next_ndp: - /* Other NDP to process? */ - ndpoffset = (int)le16_to_cpu(ndp16.wNextNdpIndex); - if (!ndpoffset) + /* Other NDP to process? The offsets must advance, or a + * self-referencing NDP keeps the loop spinning forever. + */ + n = (int)le16_to_cpu(ndp16.wNextNdpIndex); + if (n > ndpoffset) + ndpoffset = n; + else break; } diff --git a/drivers/net/wwan/t7xx/t7xx_netdev.c b/drivers/net/wwan/t7xx/t7xx_netdev.c index fc0a7cb181df2..8f32c2d269312 100644 --- a/drivers/net/wwan/t7xx/t7xx_netdev.c +++ b/drivers/net/wwan/t7xx/t7xx_netdev.c @@ -420,6 +420,10 @@ static void t7xx_ccmni_recv_skb(struct t7xx_ccmni_ctrl *ccmni_ctlb, struct sk_bu skb_cb = T7XX_SKB_CB(skb); netif_id = skb_cb->netif_idx; + if (netif_id >= NIC_DEV_MAX) { + dev_kfree_skb(skb); + return; + } ccmni = READ_ONCE(ccmni_ctlb->ccmni_inst[netif_id]); if (!ccmni) { dev_kfree_skb(skb); diff --git a/drivers/nfc/fdp/i2c.c b/drivers/nfc/fdp/i2c.c index c1896a1d978cd..f292e7f374568 100644 --- a/drivers/nfc/fdp/i2c.c +++ b/drivers/nfc/fdp/i2c.c @@ -166,9 +166,36 @@ static int fdp_nci_i2c_read(struct fdp_i2c_phy *phy, struct sk_buff **skb) /* Packet that contains a length */ if (tmp[0] == 0 && tmp[1] == 0) { phy->next_read_size = (tmp[2] << 8) + tmp[3] + 3; + + /* + * next_read_size is taken from the device and is used + * as the i2c_master_recv() count for the next packet + * and as the data skb size. A value above the receive + * buffer overflows tmp[]; one below the minimum frame + * size runs the header/LRC strip and the length-field + * read past a short receive. Either way the packet is + * corrupt: drop it and force resynchronization. + */ + if (phy->next_read_size < FDP_NCI_I2C_MIN_PAYLOAD || + phy->next_read_size > FDP_NCI_I2C_MAX_PAYLOAD) { + dev_dbg(&client->dev, "%s: corrupted packet\n", + __func__); + phy->next_read_size = FDP_NCI_I2C_MIN_PAYLOAD; + goto flush; + } } else { phy->next_read_size = FDP_NCI_I2C_MIN_PAYLOAD; + /* + * Only one data packet is delivered per call; if the + * device sends another, do not overwrite and leak the + * skb allocated for the previous one. + */ + if (*skb) { + kfree_skb(*skb); + *skb = NULL; + } + *skb = alloc_skb(len, GFP_KERNEL); if (*skb == NULL) { r = -ENOMEM; diff --git a/drivers/nfc/microread/microread.c b/drivers/nfc/microread/microread.c index 4149c5d735bda..2bfafa94e83d6 100644 --- a/drivers/nfc/microread/microread.c +++ b/drivers/nfc/microread/microread.c @@ -251,9 +251,9 @@ static int microread_start_poll(struct nfc_hci_dev *hdev, param[1] |= (1 << 1); if ((im_protocols | tm_protocols) & NFC_PROTO_NFC_DEP_MASK) { - hdev->gb = nfc_get_local_general_bytes(hdev->ndev, - &hdev->gb_len); - if (hdev->gb == NULL || hdev->gb_len == 0) { + nfc_get_local_general_bytes(hdev->ndev, hdev->gb, + sizeof(hdev->gb), &hdev->gb_len); + if (hdev->gb_len == 0) { im_protocols &= ~NFC_PROTO_NFC_DEP_MASK; tm_protocols &= ~NFC_PROTO_NFC_DEP_MASK; } @@ -483,13 +483,19 @@ static void microread_target_discovered(struct nfc_hci_dev *hdev, u8 gate, switch (gate) { case MICROREAD_GATE_ID_MREAD_ISO_A: + if (skb->len <= MICROREAD_EMCF_A_LEN) { + r = -EINVAL; + goto exit_free; + } + targets->supported_protocols = nfc_hci_sak_to_protocol(skb->data[MICROREAD_EMCF_A_SAK]); targets->sens_res = be16_to_cpu(*(u16 *)&skb->data[MICROREAD_EMCF_A_ATQA]); targets->sel_res = skb->data[MICROREAD_EMCF_A_SAK]; targets->nfcid1_len = skb->data[MICROREAD_EMCF_A_LEN]; - if (targets->nfcid1_len > sizeof(targets->nfcid1)) { + if (targets->nfcid1_len > sizeof(targets->nfcid1) || + targets->nfcid1_len > skb->len - MICROREAD_EMCF_A_UID) { r = -EINVAL; goto exit_free; } @@ -497,13 +503,19 @@ static void microread_target_discovered(struct nfc_hci_dev *hdev, u8 gate, targets->nfcid1_len); break; case MICROREAD_GATE_ID_MREAD_ISO_A_3: + if (skb->len <= MICROREAD_EMCF_A3_LEN) { + r = -EINVAL; + goto exit_free; + } + targets->supported_protocols = nfc_hci_sak_to_protocol(skb->data[MICROREAD_EMCF_A3_SAK]); targets->sens_res = be16_to_cpu(*(u16 *)&skb->data[MICROREAD_EMCF_A3_ATQA]); targets->sel_res = skb->data[MICROREAD_EMCF_A3_SAK]; targets->nfcid1_len = skb->data[MICROREAD_EMCF_A3_LEN]; - if (targets->nfcid1_len > sizeof(targets->nfcid1)) { + if (targets->nfcid1_len > sizeof(targets->nfcid1) || + targets->nfcid1_len > skb->len - MICROREAD_EMCF_A3_UID) { r = -EINVAL; goto exit_free; } @@ -511,11 +523,21 @@ static void microread_target_discovered(struct nfc_hci_dev *hdev, u8 gate, targets->nfcid1_len); break; case MICROREAD_GATE_ID_MREAD_ISO_B: + if (skb->len < MICROREAD_EMCF_B_UID + 4) { + r = -EINVAL; + goto exit_free; + } + targets->supported_protocols = NFC_PROTO_ISO14443_B_MASK; memcpy(targets->nfcid1, &skb->data[MICROREAD_EMCF_B_UID], 4); targets->nfcid1_len = 4; break; case MICROREAD_GATE_ID_MREAD_NFC_T1: + if (skb->len < MICROREAD_EMCF_T1_UID + 4) { + r = -EINVAL; + goto exit_free; + } + targets->supported_protocols = NFC_PROTO_JEWEL_MASK; targets->sens_res = le16_to_cpu(*(u16 *)&skb->data[MICROREAD_EMCF_T1_ATQA]); @@ -523,6 +545,11 @@ static void microread_target_discovered(struct nfc_hci_dev *hdev, u8 gate, targets->nfcid1_len = 4; break; case MICROREAD_GATE_ID_MREAD_NFC_T3: + if (skb->len < MICROREAD_EMCF_T3_UID + 8) { + r = -EINVAL; + goto exit_free; + } + targets->supported_protocols = NFC_PROTO_FELICA_MASK; memcpy(targets->nfcid1, &skb->data[MICROREAD_EMCF_T3_UID], 8); targets->nfcid1_len = 8; diff --git a/drivers/nfc/nfcmrvl/fw_dnld.c b/drivers/nfc/nfcmrvl/fw_dnld.c index 2b8f401d8fd7a..8b9d5257320dc 100644 --- a/drivers/nfc/nfcmrvl/fw_dnld.c +++ b/drivers/nfc/nfcmrvl/fw_dnld.c @@ -263,9 +263,14 @@ static int process_state_fw_dnld(struct nfcmrvl_private *priv, * B8..N: payload */ - /* Remove NCI HDR */ - skb_pull(skb, 3); - if (skb->data[0] != HELPER_CMD_PACKET_FORMAT || skb->len != 5) { + if (skb->len != NCI_DATA_HDR_SIZE + 5) { + nfc_err(priv->dev, "bad command"); + return -EINVAL; + } + + /* Remove NCI header */ + skb_pull(skb, NCI_DATA_HDR_SIZE); + if (skb->data[0] != HELPER_CMD_PACKET_FORMAT) { nfc_err(priv->dev, "bad command"); return -EINVAL; } diff --git a/drivers/nfc/pn533/pn533.c b/drivers/nfc/pn533/pn533.c index d7bdbc82e2ba2..572f0f8737c19 100644 --- a/drivers/nfc/pn533/pn533.c +++ b/drivers/nfc/pn533/pn533.c @@ -434,6 +434,18 @@ static int pn533_send_async_complete(struct pn533 *dev) return rc; } +static int pn533_send_cmd_frame(struct pn533 *dev, struct pn533_cmd *cmd) +{ + struct sk_buff *req = cmd->req; + int rc; + + skb_get(req); + dev->cmd = cmd; + rc = dev->phy_ops->send_frame(dev, req); + dev_kfree_skb(req); + return rc; +} + static int __pn533_send_async(struct pn533 *dev, u8 cmd_code, struct sk_buff *req, pn533_send_async_complete_t complete_cb, @@ -458,8 +470,7 @@ static int __pn533_send_async(struct pn533 *dev, u8 cmd_code, mutex_lock(&dev->cmd_lock); if (!dev->cmd_pending) { - dev->cmd = cmd; - rc = dev->phy_ops->send_frame(dev, req); + rc = pn533_send_cmd_frame(dev, cmd); if (rc) { dev->cmd = NULL; goto error; @@ -529,8 +540,7 @@ static int pn533_send_cmd_direct_async(struct pn533 *dev, u8 cmd_code, pn533_build_cmd_frame(dev, cmd_code, req); - dev->cmd = cmd; - rc = dev->phy_ops->send_frame(dev, req); + rc = pn533_send_cmd_frame(dev, cmd); if (rc < 0) { dev->cmd = NULL; kfree(cmd); @@ -569,8 +579,7 @@ static void pn533_wq_cmd(struct work_struct *work) mutex_unlock(&dev->cmd_lock); - dev->cmd = cmd; - rc = dev->phy_ops->send_frame(dev, cmd->req); + rc = pn533_send_cmd_frame(dev, cmd); if (rc < 0) { dev->cmd = NULL; dev_kfree_skb(cmd->req); @@ -1346,10 +1355,11 @@ static int pn533_poll_dep(struct nfc_dev *nfc_dev) u8 *next, nfcid3[NFC_NFCID3_MAXSIZE]; u8 passive_data[PASSIVE_DATA_LEN] = {0x00, 0xff, 0xff, 0x00, 0x3}; - if (!dev->gb) { - dev->gb = nfc_get_local_general_bytes(nfc_dev, &dev->gb_len); - - if (!dev->gb || !dev->gb_len) { + if (!dev->gb_len) { + nfc_get_local_general_bytes(nfc_dev, dev->gb, + sizeof(dev->gb), + &dev->gb_len); + if (!dev->gb_len) { dev->poll_dep = 0; queue_work(dev->wq, &dev->rf_work); } @@ -1647,8 +1657,9 @@ static int pn533_start_poll(struct nfc_dev *nfc_dev, } if (tm_protocols) { - dev->gb = nfc_get_local_general_bytes(nfc_dev, &dev->gb_len); - if (dev->gb == NULL) + nfc_get_local_general_bytes(nfc_dev, dev->gb, + sizeof(dev->gb), &dev->gb_len); + if (dev->gb_len == 0) tm_protocols = 0; } @@ -2799,6 +2810,7 @@ void pn53x_common_clean(struct pn533 *priv) destroy_workqueue(priv->wq); skb_queue_purge(&priv->resp_q); + skb_queue_purge(&priv->fragment_skb); list_for_each_entry_safe(cmd, n, &priv->cmd_queue, queue) { list_del(&cmd->queue); diff --git a/drivers/nfc/pn533/pn533.h b/drivers/nfc/pn533/pn533.h index 09e35b8693f5a..5ab668e05121a 100644 --- a/drivers/nfc/pn533/pn533.h +++ b/drivers/nfc/pn533/pn533.h @@ -6,6 +6,8 @@ * Copyright (C) 2012-2013 Tieto Poland */ +#include + #define PN533_DEVICE_STD 0x1 #define PN533_DEVICE_PASORI 0x2 #define PN533_DEVICE_ACR122U 0x3 @@ -166,7 +168,7 @@ struct pn533 { struct timer_list listen_timer; int cancel_listen; - u8 *gb; + u8 gb[NFC_MAX_GT_LEN]; size_t gb_len; u8 tgt_available_prots; diff --git a/drivers/nfc/pn533/usb.c b/drivers/nfc/pn533/usb.c index efb07f944fce2..972eaac09e592 100644 --- a/drivers/nfc/pn533/usb.c +++ b/drivers/nfc/pn533/usb.c @@ -319,7 +319,9 @@ static bool pn533_acr122_is_rx_frame_valid(void *_frame, struct pn533 *dev) if (frame->ccid.type != 0x83) return false; - if (!frame->ccid.datalen) + if (frame->ccid.datalen < 2 || + frame->ccid.datalen > PN533_ACR122_FRAME_MAX_PAYLOAD_LEN + + PN533_ACR122_RX_FRAME_TAIL_LEN) return false; if (frame->data[frame->ccid.datalen - 2] == 0x63) diff --git a/drivers/nfc/pn544/pn544.c b/drivers/nfc/pn544/pn544.c index 9d0a16ac465e3..c4fa70e45c14f 100644 --- a/drivers/nfc/pn544/pn544.c +++ b/drivers/nfc/pn544/pn544.c @@ -377,10 +377,9 @@ static int pn544_hci_start_poll(struct nfc_hci_dev *hdev, return r; if ((im_protocols | tm_protocols) & NFC_PROTO_NFC_DEP_MASK) { - hdev->gb = nfc_get_local_general_bytes(hdev->ndev, - &hdev->gb_len); - pr_debug("generate local bytes %p\n", hdev->gb); - if (hdev->gb == NULL || hdev->gb_len == 0) { + nfc_get_local_general_bytes(hdev->ndev, hdev->gb, + sizeof(hdev->gb), &hdev->gb_len); + if (hdev->gb_len == 0) { im_protocols &= ~NFC_PROTO_NFC_DEP_MASK; tm_protocols &= ~NFC_PROTO_NFC_DEP_MASK; } diff --git a/drivers/nfc/port100.c b/drivers/nfc/port100.c index 5ae61d7ebcfe1..30a4e09875d35 100644 --- a/drivers/nfc/port100.c +++ b/drivers/nfc/port100.c @@ -636,6 +636,13 @@ static void port100_recv_response(struct urb *urb) in_frame = dev->in_urb->transfer_buffer; + if (urb->actual_length < PORT100_FRAME_HEADER_LEN || + urb->actual_length < port100_rx_frame_size(in_frame)) { + nfc_err(&dev->interface->dev, "Received a truncated frame\n"); + cmd->status = -EIO; + goto sched_wq; + } + if (!port100_rx_frame_is_valid(in_frame)) { nfc_err(&dev->interface->dev, "Received an invalid frame\n"); cmd->status = -EIO; diff --git a/drivers/nfc/st21nfca/core.c b/drivers/nfc/st21nfca/core.c index fd39a05c96222..b5c1ca3acfbe1 100644 --- a/drivers/nfc/st21nfca/core.c +++ b/drivers/nfc/st21nfca/core.c @@ -351,10 +351,10 @@ static int st21nfca_hci_start_poll(struct nfc_hci_dev *hdev, if (r < 0) return r; } else { - hdev->gb = nfc_get_local_general_bytes(hdev->ndev, - &hdev->gb_len); - - if (hdev->gb == NULL || hdev->gb_len == 0) { + nfc_get_local_general_bytes(hdev->ndev, hdev->gb, + sizeof(hdev->gb), + &hdev->gb_len); + if (hdev->gb_len == 0) { im_protocols &= ~NFC_PROTO_NFC_DEP_MASK; tm_protocols &= ~NFC_PROTO_NFC_DEP_MASK; } @@ -577,9 +577,7 @@ static int st21nfca_get_iso15693_inventory(struct nfc_hci_dev *hdev, if (r < 0) goto exit; - skb_pull(inventory_skb, 2); - - if (inventory_skb->len == 0 || + if (!skb_pull(inventory_skb, 2) || inventory_skb->len < 2 || inventory_skb->len > NFC_ISO15693_UID_MAXSIZE) { r = -EPROTO; goto exit; diff --git a/drivers/nfc/st21nfca/dep.c b/drivers/nfc/st21nfca/dep.c index 3425b68f0ddcf..a5fab4fd5129d 100644 --- a/drivers/nfc/st21nfca/dep.c +++ b/drivers/nfc/st21nfca/dep.c @@ -205,6 +205,9 @@ static int st21nfca_tm_recv_atr_req(struct nfc_hci_dev *hdev, if (atr_req->length < sizeof(struct st21nfca_atr_req)) return -EPROTO; + if (atr_req->length > skb->len) + return -EPROTO; + r = st21nfca_tm_send_atr_res(hdev, atr_req); if (r) return r; diff --git a/drivers/nfc/st21nfca/i2c.c b/drivers/nfc/st21nfca/i2c.c index aa5f4922b6b04..11ba4fb498283 100644 --- a/drivers/nfc/st21nfca/i2c.c +++ b/drivers/nfc/st21nfca/i2c.c @@ -289,27 +289,36 @@ static int check_crc(u8 *buf, int buflen) */ static int st21nfca_hci_i2c_repack(struct sk_buff *skb) { - int i, j, r, size; + int read, write, r, size; - if (skb->len < 1 || (skb->len > 1 && skb->data[1] != 0)) + if (skb->len < ST21NFCA_FRAME_HEADROOM || + !IS_START_OF_FRAME(skb->data)) return -EBADMSG; size = get_frame_size(skb->data, skb->len); if (size > 0) { + if (size < ST21NFCA_FRAME_HEADROOM + 2) + return -EBADMSG; + skb_trim(skb, size); /* remove ST21NFCA byte stuffing for upper layer */ - for (i = 1, j = 0; i < skb->len; i++) { - if (skb->data[i + j] == + for (read = 1, write = 1; read < skb->len;) { + if (skb->data[read] == (u8) ST21NFCA_ESCAPE_BYTE_STUFFING) { - skb->data[i] = skb->data[i + j + 1] - | ST21NFCA_BYTE_STUFFING_MASK; - i++; - j++; + if (read + 1 == skb->len) + return -EBADMSG; + + skb->data[write++] = skb->data[read + 1] + | ST21NFCA_BYTE_STUFFING_MASK; + read += 2; + } else { + skb->data[write++] = skb->data[read++]; } - skb->data[i] = skb->data[i + j]; } /* remove byte stuffing useless byte */ - skb_trim(skb, i - j); + skb_trim(skb, write); + if (skb->len < ST21NFCA_FRAME_HEADROOM + 2) + return -EBADMSG; /* remove ST21NFCA_SOF_EOF from head */ skb_pull(skb, 1); diff --git a/drivers/nfc/trf7970a.c b/drivers/nfc/trf7970a.c index f22e091019dee..3bd8c4636055d 100644 --- a/drivers/nfc/trf7970a.c +++ b/drivers/nfc/trf7970a.c @@ -1997,8 +1997,10 @@ static int trf7970a_startup(struct trf7970a *trf) return ret; ret = trf7970a_update_rx_gain_reduction(trf); - if (ret) + if (ret) { + trf7970a_power_down(trf); return ret; + } pm_runtime_set_active(trf->dev); pm_runtime_enable(trf->dev); diff --git a/drivers/nfc/virtual_ncidev.c b/drivers/nfc/virtual_ncidev.c index 8eeb447ac96e9..e51c647b27ebe 100644 --- a/drivers/nfc/virtual_ncidev.c +++ b/drivers/nfc/virtual_ncidev.c @@ -195,7 +195,8 @@ static const struct file_operations virtual_ncidev_fops = { .write = virtual_ncidev_write, .open = virtual_ncidev_open, .release = virtual_ncidev_close, - .unlocked_ioctl = virtual_ncidev_ioctl + .unlocked_ioctl = virtual_ncidev_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static struct miscdevice miscdev = { diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index f59f926d4bfaa..f9caa1a653c5b 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -1719,9 +1719,16 @@ static void ntb_transport_rxc_db(unsigned long data) static void ntb_tx_copy_callback(void *data, const struct dmaengine_result *res) { + struct ntb_payload_header __iomem *hdr; struct ntb_queue_entry *entry = data; - struct ntb_transport_qp *qp = entry->qp; - struct ntb_payload_header __iomem *hdr = entry->tx_hdr; + struct ntb_transport_qp *qp; + unsigned int len; + void *cb_data; + + qp = entry->qp; + hdr = entry->tx_hdr; + cb_data = entry->cb_data; + len = entry->len; /* we need to check DMA results if we are using DMA */ if (res) { @@ -1768,15 +1775,13 @@ static void ntb_tx_copy_callback(void *data, * "link down" or similar. Since no payload is being sent in these * cases, there is nothing to add to the completion queue. */ - if (entry->len > 0) { - qp->tx_bytes += entry->len; - - if (qp->tx_handler) - qp->tx_handler(qp, qp->cb_data, entry->cb_data, - entry->len); - } + if (len > 0) + qp->tx_bytes += len; ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); + + if (len > 0 && qp->tx_handler) + qp->tx_handler(qp, qp->cb_data, cb_data, len); } static void ntb_memcpy_tx_on_stack(struct ntb_queue_entry *entry, void __iomem *offset) @@ -1950,15 +1955,6 @@ static int ntb_process_tx(struct ntb_transport_qp *qp, return -EAGAIN; } - if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { - if (qp->tx_handler) - qp->tx_handler(qp, qp->cb_data, NULL, -EIO); - - ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, - &qp->tx_free_q); - return 0; - } - ntb_async_tx(qp, entry); qp->tx_pkts++; @@ -2348,9 +2344,11 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, if (!qp || !len) return -EINVAL; - /* If the qp link is down already, just ignore. */ if (!qp->link_is_up) - return 0; + return -ENOLINK; + + if (len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) + return -EMSGSIZE; entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); if (!entry) { diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 7e1112960d7f8..380e352dc3ccd 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -883,6 +883,14 @@ static int discover_arenas(struct btt *btt) arena->external_lba_start = cur_nlba; parse_arena_meta(arena, super, cur_off); + if (arena->nfree < btt->nd_region->num_lanes) { + dev_err(to_dev(arena), + "nfree %u smaller than lane count %d\n", + arena->nfree, btt->nd_region->num_lanes); + ret = -ENODEV; + goto out; + } + ret = log_set_indices(arena); if (ret) { dev_err(to_dev(arena), diff --git a/drivers/nvdimm/label.c b/drivers/nvdimm/label.c index 4218e3ac4a2ab..dea2eee86d132 100644 --- a/drivers/nvdimm/label.c +++ b/drivers/nvdimm/label.c @@ -145,10 +145,21 @@ static int __nd_label_validate(struct nvdimm_drvdata *ndd) /* label sizes larger than 128 arrived with v1.2 */ version = __le16_to_cpu(nsindex[i]->major) * 100 + __le16_to_cpu(nsindex[i]->minor); - if (version >= 102) + if (version >= 102) { + /* + * labelsize feeds the shift below; only 0 (128-byte) + * and 1 (256-byte) are valid -- a larger value would + * overflow or exceed the width of int. + */ + if (nsindex[i]->labelsize > 1) { + dev_dbg(dev, "nsindex%d labelsize: %d invalid\n", + i, nsindex[i]->labelsize); + continue; + } labelsize = 1 << (7 + nsindex[i]->labelsize); - else + } else { labelsize = 128; + } if (labelsize != sizeof_namespace_label(ndd)) { dev_dbg(dev, "nsindex%d labelsize %d invalid\n", @@ -202,7 +213,7 @@ static int __nd_label_validate(struct nvdimm_drvdata *ndd) } nslot = __le32_to_cpu(nsindex[i]->nslot); - if (nslot * sizeof_namespace_label(ndd) + if ((u64)nslot * sizeof_namespace_label(ndd) + 2 * sizeof_namespace_index(ndd) > ndd->nsarea.config_size) { dev_dbg(dev, "nsindex%d nslot: %u invalid, config_size: %#x\n", diff --git a/drivers/nvdimm/nd_virtio.c b/drivers/nvdimm/nd_virtio.c index 4176046627beb..cbc0f53398f09 100644 --- a/drivers/nvdimm/nd_virtio.c +++ b/drivers/nvdimm/nd_virtio.c @@ -9,26 +9,48 @@ #include "virtio_pmem.h" #include "nd.h" +struct virtio_pmem_flush_work { + struct work_struct work; + struct nd_region *nd_region; + struct bio *bio; +}; + +static void virtio_pmem_req_release(struct kref *kref) +{ + struct virtio_pmem_request *req; + + req = container_of(kref, struct virtio_pmem_request, kref); + kfree(req); +} + +static void virtio_pmem_wake_one_waiter(struct virtio_pmem *vpmem) +{ + struct virtio_pmem_request *req_buf; + + if (list_empty(&vpmem->req_list)) + return; + + req_buf = list_first_entry(&vpmem->req_list, + struct virtio_pmem_request, list); + list_del_init(&req_buf->list); + WRITE_ONCE(req_buf->wq_buf_avail, true); + wake_up(&req_buf->wq_buf); +} + /* The interrupt handler */ void virtio_pmem_host_ack(struct virtqueue *vq) { struct virtio_pmem *vpmem = vq->vdev->priv; - struct virtio_pmem_request *req_data, *req_buf; + struct virtio_pmem_request *req_data; unsigned long flags; unsigned int len; spin_lock_irqsave(&vpmem->pmem_lock, flags); while ((req_data = virtqueue_get_buf(vq, &len)) != NULL) { - req_data->done = true; + virtio_pmem_wake_one_waiter(vpmem); + WRITE_ONCE(req_data->done, true); wake_up(&req_data->host_acked); - - if (!list_empty(&vpmem->req_list)) { - req_buf = list_first_entry(&vpmem->req_list, - struct virtio_pmem_request, list); - req_buf->wq_buf_avail = true; - wake_up(&req_buf->wq_buf); - list_del(&req_buf->list); - } + kref_put(&req_data->kref, virtio_pmem_req_release); } spin_unlock_irqrestore(&vpmem->pmem_lock, flags); } @@ -59,7 +81,8 @@ static int virtio_pmem_flush(struct nd_region *nd_region) if (!req_data) return -ENOMEM; - req_data->done = false; + kref_init(&req_data->kref); + WRITE_ONCE(req_data->done, false); init_waitqueue_head(&req_data->host_acked); init_waitqueue_head(&req_data->wq_buf); INIT_LIST_HEAD(&req_data->list); @@ -76,18 +99,32 @@ static int virtio_pmem_flush(struct nd_region *nd_region) * to req_list and wait for host_ack to wake us up when free * slots are available. */ - while ((err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req_data, - GFP_ATOMIC)) == -ENOSPC) { + for (;;) { + err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req_data, + GFP_ATOMIC); + if (!err) { + /* + * Take the virtqueue reference while @pmem_lock is + * held so completion cannot run concurrently. + */ + kref_get(&req_data->kref); + break; + } - dev_info(&vdev->dev, "failed to send command to virtio pmem device, no free slots in the virtqueue\n"); - req_data->wq_buf_avail = false; + if (err != -ENOSPC) + break; + + dev_info_ratelimited(&vdev->dev, + "failed to send command to virtio pmem device, no free slots in the virtqueue\n"); + WRITE_ONCE(req_data->wq_buf_avail, false); list_add_tail(&req_data->list, &vpmem->req_list); spin_unlock_irqrestore(&vpmem->pmem_lock, flags); /* A host response results in "host_ack" getting called */ - wait_event(req_data->wq_buf, req_data->wq_buf_avail); + wait_event(req_data->wq_buf, READ_ONCE(req_data->wq_buf_avail)); spin_lock_irqsave(&vpmem->pmem_lock, flags); } + err1 = virtqueue_kick(vpmem->req_vq); spin_unlock_irqrestore(&vpmem->pmem_lock, flags); /* @@ -99,38 +136,54 @@ static int virtio_pmem_flush(struct nd_region *nd_region) err = -EIO; } else { /* A host response results in "host_ack" getting called */ - wait_event(req_data->host_acked, req_data->done); + wait_event(req_data->host_acked, READ_ONCE(req_data->done)); err = le32_to_cpu(req_data->resp.ret); } - kfree(req_data); + kref_put(&req_data->kref, virtio_pmem_req_release); return err; }; +static void virtio_pmem_flush_work(struct work_struct *work) +{ + struct virtio_pmem_flush_work *flush; + int err; + + flush = container_of(work, struct virtio_pmem_flush_work, work); + err = virtio_pmem_flush(flush->nd_region); + if (err > 0) + err = -EIO; + if (err) + flush->bio->bi_status = errno_to_blk_status(err); + bio_endio(flush->bio); + kfree(flush); +} + /* The asynchronous flush callback function */ int async_pmem_flush(struct nd_region *nd_region, struct bio *bio) { - /* - * Create child bio for asynchronous flush and chain with - * parent bio. Otherwise directly call nd_region flush. - */ - if (bio && bio->bi_iter.bi_sector != -1) { - struct bio *child = bio_alloc(bio->bi_bdev, 0, - REQ_OP_WRITE | REQ_PREFLUSH, - GFP_ATOMIC); + struct virtio_device *vdev = nd_region->provider_data; + struct virtio_pmem *vpmem = vdev->priv; + struct virtio_pmem_flush_work *flush; + int err; - if (!child) + if (bio && bio->bi_iter.bi_sector != -1) { + flush = kmalloc_obj(*flush, GFP_NOIO); + if (!flush) return -ENOMEM; - bio_clone_blkg_association(child, bio); - child->bi_iter.bi_sector = -1; - bio_chain(child, bio); - submit_bio(child); - return 0; + + INIT_WORK(&flush->work, virtio_pmem_flush_work); + flush->nd_region = nd_region; + flush->bio = bio; + queue_work(vpmem->flush_wq, &flush->work); + return NVDIMM_FLUSH_ASYNC; } - if (virtio_pmem_flush(nd_region)) + + err = virtio_pmem_flush(nd_region); + if (err > 0) return -EIO; - return 0; + return err; }; EXPORT_SYMBOL_GPL(async_pmem_flush); MODULE_DESCRIPTION("Virtio Persistent Memory Driver"); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 92c67fbbc1c85..ff27e47592164 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -208,8 +208,14 @@ static void pmem_submit_bio(struct bio *bio) struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data; struct nd_region *nd_region = to_region(pmem); - if (bio->bi_opf & REQ_PREFLUSH) - ret = nvdimm_flush(nd_region, bio); + if (bio->bi_opf & REQ_PREFLUSH) { + ret = nvdimm_flush(nd_region, NULL); + if (ret) { + bio->bi_status = errno_to_blk_status(ret); + bio_endio(bio); + return; + } + } do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); if (do_acct) @@ -229,8 +235,11 @@ static void pmem_submit_bio(struct bio *bio) if (do_acct) bio_end_io_acct(bio, start); - if (bio->bi_opf & REQ_FUA) + if ((bio->bi_opf & REQ_FUA) && !bio->bi_status) { ret = nvdimm_flush(nd_region, bio); + if (ret == NVDIMM_FLUSH_ASYNC) + return; + } if (ret) bio->bi_status = errno_to_blk_status(ret); diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 5e079d61cbaa3..24f42b4650ba6 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -1093,7 +1093,10 @@ int nvdimm_flush(struct nd_region *nd_region, struct bio *bio) if (!nd_region->flush) rc = generic_nvdimm_flush(nd_region); else { - if (nd_region->flush(nd_region, bio)) + rc = nd_region->flush(nd_region, bio); + if (rc > 0) + return rc; + if (rc && rc != -ENOMEM) rc = -EIO; } diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c index 77b1966619059..9cf822a6c0c38 100644 --- a/drivers/nvdimm/virtio_pmem.c +++ b/drivers/nvdimm/virtio_pmem.c @@ -67,10 +67,17 @@ static int virtio_pmem_probe(struct virtio_device *vdev) mutex_init(&vpmem->flush_lock); vpmem->vdev = vdev; vdev->priv = vpmem; + vpmem->flush_wq = alloc_ordered_workqueue("virtio-pmem-flush", + WQ_MEM_RECLAIM); + if (!vpmem->flush_wq) { + err = -ENOMEM; + goto out_err; + } + err = init_vq(vpmem); if (err) { dev_err(&vdev->dev, "failed to initialize virtio pmem vq's\n"); - goto out_err; + goto out_wq; } if (virtio_has_feature(vdev, VIRTIO_PMEM_F_SHMEM_REGION)) { @@ -131,6 +138,8 @@ static int virtio_pmem_probe(struct virtio_device *vdev) nvdimm_bus_unregister(vpmem->nvdimm_bus); out_vq: vdev->config->del_vqs(vdev); +out_wq: + destroy_workqueue(vpmem->flush_wq); out_err: return err; } @@ -138,14 +147,20 @@ static int virtio_pmem_probe(struct virtio_device *vdev) static void virtio_pmem_remove(struct virtio_device *vdev) { struct nvdimm_bus *nvdimm_bus = dev_get_drvdata(&vdev->dev); + struct virtio_pmem *vpmem = vdev->priv; nvdimm_bus_unregister(nvdimm_bus); + drain_workqueue(vpmem->flush_wq); vdev->config->del_vqs(vdev); virtio_reset_device(vdev); + destroy_workqueue(vpmem->flush_wq); } static int virtio_pmem_freeze(struct virtio_device *vdev) { + struct virtio_pmem *vpmem = vdev->priv; + + drain_workqueue(vpmem->flush_wq); vdev->config->del_vqs(vdev); virtio_reset_device(vdev); diff --git a/drivers/nvdimm/virtio_pmem.h b/drivers/nvdimm/virtio_pmem.h index f72cf17f9518f..3af92588bd9d1 100644 --- a/drivers/nvdimm/virtio_pmem.h +++ b/drivers/nvdimm/virtio_pmem.h @@ -12,11 +12,14 @@ #include #include +#include #include #include #include +#include struct virtio_pmem_request { + struct kref kref; struct virtio_pmem_req req; struct virtio_pmem_resp resp; @@ -39,6 +42,9 @@ struct virtio_pmem { /* Serialize flush requests to the device. */ struct mutex flush_lock; + /* Complete asynchronous FUA flushes outside the submit path. */ + struct workqueue_struct *flush_wq; + /* nvdimm bus registers virtio pmem device */ struct nvdimm_bus *nvdimm_bus; struct nvdimm_bus_descriptor nd_desc; diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index be3b91b43ea5a..b90812ed74b1b 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -47,9 +47,6 @@ #define APPLE_ANS_BOOT_STATUS 0x1300 #define APPLE_ANS_BOOT_STATUS_OK 0xde71ce55 -#define APPLE_ANS_UNKNOWN_CTRL 0x24008 -#define APPLE_ANS_PRP_NULL_CHECK BIT(11) - #define APPLE_ANS_LINEAR_SQ_CTRL 0x24908 #define APPLE_ANS_LINEAR_SQ_EN BIT(0) @@ -151,6 +148,23 @@ struct apple_nvme_queue { bool enabled; }; +static inline bool apple_nvme_queue_enabled(struct apple_nvme_queue *q) +{ + /* Pair with apple_nvme_enable_queue(). */ + return smp_load_acquire(&q->enabled); +} + +static inline void apple_nvme_enable_queue(struct apple_nvme_queue *q) +{ + /* Publish queue initialization before setting q->enabled. */ + smp_store_release(&q->enabled, true); +} + +static inline void apple_nvme_disable_queue(struct apple_nvme_queue *q) +{ + WRITE_ONCE(q->enabled, false); +} + /* * The apple_nvme_iod describes the data in an I/O. * @@ -318,13 +332,15 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; - tcb->opcode = cmd->common.opcode; + tcb->opcode = 0; tcb->prp1 = cmd->common.dptr.prp1; tcb->prp2 = cmd->common.dptr.prp2; tcb->length = cmd->rw.length; tcb->command_id = tag; - if (nvme_is_write(cmd)) + if (!cmd->common.dptr.prp1) + tcb->dma_flags = 0; + else if (nvme_is_write(cmd)) tcb->dma_flags = APPLE_ANS_TCB_DMA_TO_DEVICE; else tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; @@ -677,7 +693,7 @@ static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force) bool found; DEFINE_IO_COMP_BATCH(iob); - if (!READ_ONCE(q->enabled) && !force) + if (!apple_nvme_queue_enabled(q) && !force) return false; found = apple_nvme_poll_cq(q, &iob); @@ -780,7 +796,7 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, * We should not need to do this, but we're still using this to * ensure we can drain requests on a dying queue. */ - if (unlikely(!READ_ONCE(q->enabled))) + if (unlikely(!apple_nvme_queue_enabled(q))) return BLK_STS_IOERR; if (!nvme_check_ready(&anv->ctrl, req, true)) @@ -863,7 +879,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_quiesce_io_queues(&anv->ctrl); if (!dead) { - if (READ_ONCE(anv->ioq.enabled)) { + if (apple_nvme_queue_enabled(&anv->ioq)) { apple_nvme_remove_sq(anv); apple_nvme_remove_cq(anv); } @@ -887,8 +903,8 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_disable_ctrl(&anv->ctrl, false); } - WRITE_ONCE(anv->ioq.enabled, false); - WRITE_ONCE(anv->adminq.enabled, false); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); mb(); /* ensure that nvme_queue_rq() sees that enabled is cleared */ nvme_quiesce_admin_queue(&anv->ctrl); @@ -1016,8 +1032,7 @@ static void apple_nvme_init_queue(struct apple_nvme_queue *q) memset(q->tcbs, 0, anv->hw->max_queue_depth * sizeof(struct apple_nvmmu_tcb)); memset(q->cqes, 0, depth * sizeof(struct nvme_completion)); - WRITE_ONCE(q->enabled, true); - wmb(); /* ensure the first interrupt sees the initialization */ + apple_nvme_enable_queue(q); } static void apple_nvme_reset_work(struct work_struct *work) @@ -1125,17 +1140,6 @@ static void apple_nvme_reset_work(struct work_struct *work) /* Setup the NVMMU for the maximum admin and IO queue depth */ writel(anv->hw->max_queue_depth - 1, anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); - - /* - * This is probably a chicken bit: without it all commands - * where any PRP is set to zero (including those that don't use - * that field) fail and the co-processor complains about - * "completed with err BAD_CMD-" or a "NULL_PRP_PTR_ERR" in the - * syslog - */ - writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) & - ~APPLE_ANS_PRP_NULL_CHECK, - anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL); } /* Setup the admin queue */ @@ -1581,7 +1585,8 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) } ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, - NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS); + NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | + NVME_QUIRK_ADMIN_PAGE_ALIGN); if (ret) { dev_err_probe(dev, ret, "Failed to initialize nvme_ctrl"); goto put_dev; @@ -1636,6 +1641,15 @@ static void apple_nvme_remove(struct platform_device *pdev) nvme_stop_ctrl(&anv->ctrl); nvme_remove_namespaces(&anv->ctrl); apple_nvme_disable(anv, true); + if (anv->ctrl.admin_q && !blk_queue_dying(anv->ctrl.admin_q)) { + /* + * If the controller was reset during removal, it's possible + * user requests may be waiting on a stopped queue. Start the + * queue to flush these to completion. + */ + nvme_unquiesce_admin_queue(&anv->ctrl); + blk_mq_destroy_queue(anv->ctrl.admin_q); + } nvme_uninit_ctrl(&anv->ctrl); if (apple_rtkit_is_running(anv->rtk)) { diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 453c1f0b2dd09..a73cc6a43f5e6 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -148,8 +148,6 @@ static const struct class nvme_ns_chr_class = { }; static void nvme_put_subsystem(struct nvme_subsystem *subsys); -static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, - unsigned nsid); static void nvme_update_keep_alive(struct nvme_ctrl *ctrl, struct nvme_command *cmd); static int nvme_get_log_lsi(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, @@ -1594,7 +1592,7 @@ static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, } if (nvme_multi_css(ctrl) && !csi_seen) { - dev_warn(ctrl->device, "Command set not reported for nsid:%d\n", + dev_warn(ctrl->device, "Command set not reported for nsid:%u\n", info->nsid); status = -EINVAL; } @@ -2071,7 +2069,10 @@ static void nvme_set_ctrl_limits(struct nvme_ctrl *ctrl, lim->max_integrity_segments = ctrl->max_integrity_segments; lim->virt_boundary_mask = ctrl->ops->get_virt_boundary(ctrl, is_admin); lim->max_segment_size = UINT_MAX; - lim->dma_alignment = 3; + if (is_admin && (ctrl->quirks & NVME_QUIRK_ADMIN_PAGE_ALIGN)) + lim->dma_alignment = NVME_CTRL_PAGE_SIZE - 1; + else + lim->dma_alignment = 3; } static bool nvme_update_disk_info(struct nvme_ns *ns, struct nvme_id_ns *id, @@ -2320,14 +2321,6 @@ static int nvme_query_fdp_info(struct nvme_ns *ns, struct nvme_ns_info *info) size_t size; int i, ret; - /* - * The FDP configuration is static for the lifetime of the namespace, - * so return immediately if we've already registered this namespace's - * streams. - */ - if (head->nr_plids) - return 0; - ret = nvme_get_features(ctrl, NVME_FEAT_FDP, info->endgid, NULL, 0, &fdp); if (ret) { @@ -2374,6 +2367,7 @@ static int nvme_query_fdp_info(struct nvme_ns *ns, struct nvme_ns_info *info) for (i = 0; i < head->nr_plids; i++) head->plids[i] = le16_to_cpu(ruhs->ruhsd[i].pid); + head->write_stream_granularity = min(info->runs, U32_MAX); free: kfree(ruhs); return ret; @@ -2421,12 +2415,6 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, goto out; } - if (ns->ctrl->ctratt & NVME_CTRL_ATTR_FDPS) { - ret = nvme_query_fdp_info(ns, info); - if (ret < 0) - goto out; - } - if (nvme_invalid_lba_sz(le64_to_cpu(id->nsze), id->lbaf[lbaf].ds - SECTOR_SHIFT, &capacity)) { dev_warn_once(ns->ctrl->device, @@ -2447,9 +2435,26 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, if (!nvme_update_disk_info(ns, id, nvm, &lim)) capacity = 0; + /* + * A failed zone info query leaves zi zero-initialized, so skip the + * zoned limits update instead of configuring the queue from it. + * During a revalidation that keeps the zone geometry the queue was + * last validated with; on a first scan the namespace is registered + * without zoned limits, so that it is still available as a handle + * for admin commands. + */ if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && - ns->head->ids.csi == NVME_CSI_ZNS) - nvme_update_zone_info(ns, &lim, &zi); + ns->head->ids.csi == NVME_CSI_ZNS) { + if (zi.zone_size) + nvme_update_zone_info(ns, &lim, &zi); + else + dev_warn(ns->ctrl->device, + "zone info query failed for nsid %u, %s\n", + ns->head->ns_id, + blk_queue_is_zoned(ns->disk->queue) ? + "keeping the previous zone limits" : + "not enabling zoned mode"); + } if ((ns->ctrl->vwc & NVME_CTRL_VWC_PRESENT) && !info->no_vwc) lim.features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA; @@ -2469,10 +2474,7 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, capacity = 0; lim.max_write_streams = ns->head->nr_plids; - if (lim.max_write_streams) - lim.write_stream_granularity = min(info->runs, U32_MAX); - else - lim.write_stream_granularity = 0; + lim.write_stream_granularity = ns->head->write_stream_granularity; /* * Only set the DEAC bit if the device guarantees that reads from @@ -3272,6 +3274,7 @@ static bool nvme_validate_cntlid(struct nvme_subsystem *subsys, } static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) + __context_unsafe(/* initialize unpublished/lock-guarded variables */) { struct nvme_subsystem *subsys, *found; int ret; @@ -3843,6 +3846,7 @@ static const struct file_operations nvme_dev_fops = { static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl, unsigned nsid) + __must_hold(&ctrl->subsys->lock) { struct nvme_ns_head *h; @@ -3865,6 +3869,7 @@ static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl, static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys, struct nvme_ns_ids *ids) + __must_hold(&subsys->lock) { bool has_uuid = !uuid_is_null(&ids->uuid); bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid)); @@ -3964,9 +3969,11 @@ static void nvme_add_ns_cdev(struct nvme_ns *ns) set_bit(NVME_NS_CDEV_LIVE, &ns->flags); } -static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, +static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) + __must_hold(&ns->ctrl->subsys->lock) { + struct nvme_ctrl *ctrl = ns->ctrl; struct nvme_ns_head *head; size_t size = sizeof(*head); int ret = -ENOMEM; @@ -3994,6 +4001,7 @@ static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, ratelimit_state_init(&head->rs_nuse, 5 * HZ, 1); ratelimit_set_flags(&head->rs_nuse, RATELIMIT_MSG_ON_RELEASE); kref_init(&head->ref); + ns->head = head; if (head->ids.csi) { ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects); @@ -4002,21 +4010,30 @@ static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, } else head->effects = ctrl->effects; + if (ctrl->ctratt & NVME_CTRL_ATTR_FDPS) { + ret = nvme_query_fdp_info(ns, info); + if (ret < 0) + goto out_cleanup_srcu; + } + ret = nvme_mpath_alloc_disk(ctrl, head); if (ret) - goto out_cleanup_srcu; + goto out_cleanup_fdp; list_add_tail(&head->entry, &ctrl->subsys->nsheads); kref_get(&ctrl->subsys->ref); return head; +out_cleanup_fdp: + kfree(head->plids); out_cleanup_srcu: cleanup_srcu_struct(&head->srcu); out_ida_remove: ida_free(&ctrl->subsys->ns_ida, head->instance); out_free_head: kfree(head); + ns->head = NULL; out: if (ret > 0) ret = blk_status_to_errno(nvme_error_status(ret)); @@ -4078,13 +4095,13 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) ((ns->ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) && info->is_shared)) { dev_err(ctrl->device, - "ignoring nsid %d because of duplicate IDs\n", + "ignoring nsid %u because of duplicate IDs\n", info->nsid); return ret; } dev_err(ctrl->device, - "clearing duplicate IDs for nsid %d\n", info->nsid); + "clearing duplicate IDs for nsid %u\n", info->nsid); dev_err(ctrl->device, "use of /dev/disk/by-id/ may cause data corruption\n"); memset(&info->ids.nguid, 0, sizeof(info->ids.nguid)); @@ -4099,11 +4116,11 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, &info->ids); if (ret) { dev_err(ctrl->device, - "duplicate IDs in subsystem for nsid %d\n", + "duplicate IDs in subsystem for nsid %u\n", info->nsid); goto out_unlock; } - head = nvme_alloc_ns_head(ctrl, info); + head = nvme_alloc_ns_head(ns, info); if (IS_ERR(head)) { ret = PTR_ERR(head); goto out_unlock; @@ -4113,20 +4130,20 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) if ((!info->is_shared || !head->shared) && !list_empty(&head->list)) { dev_err(ctrl->device, - "Duplicate unshared namespace %d\n", + "Duplicate unshared namespace %u\n", info->nsid); goto out_put_ns_head; } if (!nvme_ns_ids_equal(&head->ids, &info->ids)) { dev_err(ctrl->device, - "IDs don't match for shared namespace %d\n", + "IDs don't match for shared namespace %u\n", info->nsid); goto out_put_ns_head; } if (!multipath) { dev_warn(ctrl->device, - "Found shared namespace %d, but multipathing not supported.\n", + "Found shared namespace %u, but multipathing not supported.\n", info->nsid); dev_warn_once(ctrl->device, "Shared namespace support requires core_nvme.multipath=Y.\n"); @@ -4295,6 +4312,9 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info) last_path = true; } mutex_unlock(&ctrl->subsys->lock); + + /* guarantee not available in head->list */ + synchronize_srcu(&ns->head->srcu); if (last_path) nvme_put_ns_head(ns->head); nvme_put_ns_head(ns->head); @@ -4372,7 +4392,7 @@ static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_info *info) if (!nvme_ns_ids_equal(&ns->head->ids, &info->ids)) { dev_err(ns->ctrl->device, - "identifiers changed for nsid %d\n", ns->head->ns_id); + "identifiers changed for nsid %u\n", ns->head->ns_id); goto out; } @@ -4399,7 +4419,7 @@ static void nvme_scan_ns(struct nvme_ctrl *ctrl, unsigned nsid) if (info.ids.csi != NVME_CSI_NVM && !nvme_multi_css(ctrl)) { dev_warn(ctrl->device, - "command set not reported for nsid: %d\n", nsid); + "command set not reported for nsid: %u\n", nsid); return; } @@ -4463,15 +4483,16 @@ static void nvme_scan_ns_async(void *data, async_cookie_t cookie) nvme_scan_ns(scan_info->ctrl, nsid); } -static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, - unsigned nsid) +static void nvme_remove_nsid_range(struct nvme_ctrl *ctrl, u32 start, u32 end) { struct nvme_ns *ns, *next; LIST_HEAD(rm_list); mutex_lock(&ctrl->namespaces_lock); list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) { - if (ns->head->ns_id > nsid) { + if (ns->head->ns_id >= end) + break; + if (ns->head->ns_id > start) { list_del_rcu(&ns->list); synchronize_srcu(&ctrl->srcu); list_add_tail_rcu(&ns->list, &rm_list); @@ -4521,13 +4542,14 @@ static int nvme_scan_ns_list(struct nvme_ctrl *ctrl) goto out; async_schedule_domain(nvme_scan_ns_async, &scan_info, &domain); - while (++prev < nsid) - nvme_ns_remove_by_nsid(ctrl, prev); + if (prev + 1 < nsid) + nvme_remove_nsid_range(ctrl, prev, nsid); + prev = max(prev + 1, nsid); } async_synchronize_full_domain(&domain); } out: - nvme_remove_invalid_namespaces(ctrl, prev); + nvme_remove_nsid_range(ctrl, prev, UINT_MAX); free: async_synchronize_full_domain(&domain); kfree(ns_list); @@ -4547,7 +4569,7 @@ static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl) for (i = 1; i <= nn; i++) nvme_scan_ns(ctrl, i); - nvme_remove_invalid_namespaces(ctrl, nn); + nvme_remove_nsid_range(ctrl, nn, UINT_MAX); } static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl) @@ -5195,7 +5217,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) > PAGE_SIZE); - ctrl->discard_page = alloc_page(GFP_KERNEL); + ctrl->discard_page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!ctrl->discard_page) { ret = -ENOMEM; goto out; diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index ac3d4f400601f..736570b5fb340 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -1028,6 +1028,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, } if (strlen(p) < 11 || strncmp(p, "DHHC-1:", 7)) { pr_err("Invalid DH-CHAP secret %s\n", p); + kfree_sensitive(p); ret = -EINVAL; goto out; } @@ -1042,6 +1043,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, } if (strlen(p) < 11 || strncmp(p, "DHHC-1:", 7)) { pr_err("Invalid DH-CHAP secret %s\n", p); + kfree_sensitive(p); ret = -EINVAL; goto out; } diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 04363b9c4489e..48454cb7a0fcc 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2100,9 +2100,15 @@ __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, dev_err(ctrl->dev, "FCP Op failed - rspiu dma mapping failed.\n"); ret = -EFAULT; + goto out_unmap; } atomic_set(&op->state, FCPOP_STATE_IDLE); + return 0; + +out_unmap: + fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma, + sizeof(op->cmd_iu), DMA_TO_DEVICE); out_on_error: return ret; } @@ -2318,7 +2324,7 @@ nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize) return 0; delete_queues: - for (; i > 0; i--) + for (--i; i > 0; i--) __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i); return ret; } @@ -2358,9 +2364,15 @@ nvme_fc_ctrl_free(struct kref *ref) struct nvme_fc_ctrl *ctrl = container_of(ref, struct nvme_fc_ctrl, ref); unsigned long flags; + bool owns_opts; - /* remove from rport list */ + /* + * Presence on the rport list means nvme_fc_init_ctrl() completed, + * and with it ownership of the fabrics options passed to it. If it + * failed instead, the options still belong to nvmf_create_ctrl(). + */ spin_lock_irqsave(&ctrl->rport->lock, flags); + owns_opts = !list_empty(&ctrl->ctrl_list); list_del(&ctrl->ctrl_list); spin_unlock_irqrestore(&ctrl->rport->lock, flags); @@ -2370,7 +2382,7 @@ nvme_fc_ctrl_free(struct kref *ref) nvme_fc_rport_put(ctrl->rport); ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum); - if (ctrl->ctrl.opts) + if (owns_opts) nvmf_free_options(ctrl->ctrl.opts); kfree(ctrl); } @@ -3569,14 +3581,14 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { dev_err(ctrl->ctrl.device, "NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum); - goto fail_ctrl; + goto fail_unlist; } if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) { dev_err(ctrl->ctrl.device, "NVME-FC{%d}: failed to schedule initial connect\n", ctrl->cnum); - goto fail_ctrl; + goto fail_unlist; } flush_delayed_work(&ctrl->connect_work); @@ -3587,14 +3599,22 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, return &ctrl->ctrl; +fail_unlist: + /* + * Leaving the list hands the options back to nvmf_create_ctrl(); + * see nvme_fc_ctrl_free(). Re-init so that list_empty() there + * reports the controller as unlisted. + */ + spin_lock_irqsave(&rport->lock, flags); + list_del_init(&ctrl->ctrl_list); + spin_unlock_irqrestore(&rport->lock, flags); + fail_ctrl: nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING); cancel_work_sync(&ctrl->ioerr_work); cancel_work_sync(&ctrl->ctrl.reset_work); cancel_delayed_work_sync(&ctrl->connect_work); - ctrl->ctrl.opts = NULL; - if (ctrl->ctrl.admin_tagset) nvme_remove_admin_tag_set(&ctrl->ctrl); /* initiate nvme ctrl ref counting teardown */ diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index 664216eece4a6..76596cb2ded2d 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -14,45 +14,54 @@ enum { NVME_IOCTL_PARTITION = (1 << 1), }; -static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, - unsigned int flags, bool open_for_write) +static bool nvme_admin_cmd_allowed(struct nvme_ctrl *ctrl, + struct nvme_command *c) { - u32 effects; - - /* - * Do not allow unprivileged passthrough on partitions, as that allows an - * escape from the containment of the partition. - */ - if (flags & NVME_IOCTL_PARTITION) - goto admin; - - /* - * Do not allow unprivileged processes to send vendor specific or fabrics - * commands as we can't be sure about their effects. - */ - if (c->common.opcode >= nvme_cmd_vendor_start || - c->common.opcode == nvme_fabrics_command) - goto admin; - /* * Do not allow unprivileged passthrough of admin commands except * for a subset of identify commands that contain information required * to form proper I/O commands in userspace and do not expose any * potentially sensitive information. */ - if (!ns) { - if (c->common.opcode == nvme_admin_identify) { - switch (c->identify.cns) { - case NVME_ID_CNS_NS: - case NVME_ID_CNS_CS_NS: - case NVME_ID_CNS_NS_CS_INDEP: - case NVME_ID_CNS_CS_CTRL: - case NVME_ID_CNS_CTRL: - return true; - } + switch (c->common.opcode) { + case nvme_admin_identify: + switch (c->identify.cns) { + case NVME_ID_CNS_NS: + case NVME_ID_CNS_CS_NS: + case NVME_ID_CNS_NS_CS_INDEP: + case NVME_ID_CNS_CS_CTRL: + case NVME_ID_CNS_CTRL: + return true; + } + break; + case nvme_admin_set_features: + /* + * Reject Set Features that change controller state the driver + * manages itself; setting them behind the driver's back from + * userspace leaves it unable to react correctly. Keep Alive is + * only armed for fabrics - on other transports it has no + * reserved tag and harms idle power states. + */ + switch (le32_to_cpu(c->features.fid) & 0xff) { + case NVME_FEAT_KATO: + if (ctrl->ops->flags & NVME_F_FABRICS) + break; + fallthrough; + case NVME_FEAT_HOST_BEHAVIOR: + case NVME_FEAT_HOST_MEM_BUF: + case NVME_FEAT_NUM_QUEUES: + case NVME_FEAT_AUTO_PST: + return false; } - goto admin; + break; } + return capable(CAP_SYS_ADMIN); +} + +static bool nvme_ns_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, + bool open_for_write) +{ + u32 effects; /* * Check if the controller provides a Commands Supported and Effects log @@ -61,7 +70,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, */ effects = nvme_command_effects(ns->ctrl, ns, c->common.opcode); if (!(effects & NVME_CMD_EFFECTS_CSUPP)) - goto admin; + return capable(CAP_SYS_ADMIN); /* * Don't allow passthrough for command that have intrusive (or unknown) @@ -70,7 +79,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_UUID_SEL | NVME_CMD_EFFECTS_SCOPE_MASK)) - goto admin; + return capable(CAP_SYS_ADMIN); /* * Only allow I/O commands that transfer data to the controller or that @@ -79,11 +88,34 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, */ if ((nvme_is_write(c) || (effects & NVME_CMD_EFFECTS_LBCC)) && !open_for_write) - goto admin; + return capable(CAP_SYS_ADMIN); return true; -admin: - return capable(CAP_SYS_ADMIN); +} + +static bool nvme_cmd_allowed(struct nvme_ctrl *ctrl, struct nvme_ns *ns, + struct nvme_command *c, unsigned int flags, + bool open_for_write) +{ + /* + * Do not allow unprivileged passthrough on partitions, as that + * allows an escape from the containment of the partition. + */ + if (flags & NVME_IOCTL_PARTITION) + return capable(CAP_SYS_ADMIN); + + /* + * Do not allow unprivileged processes to send vendor specific or + * fabrics commands as we can't be sure about their effects. + */ + if (c->common.opcode >= nvme_cmd_vendor_start || + c->common.opcode == nvme_fabrics_command) + return capable(CAP_SYS_ADMIN); + + if (!ns) + return nvme_admin_cmd_allowed(ctrl, c); + + return nvme_ns_cmd_allowed(ns, c, open_for_write); } /* @@ -202,7 +234,8 @@ static int nvme_submit_user_cmd(struct request_queue *q, return ret; } -static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) +static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio, + unsigned int flags, bool open_for_write) { struct nvme_user_io io; struct nvme_command c; @@ -260,6 +293,9 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) c.rw.lbat = cpu_to_le16(io.apptag); c.rw.lbatm = cpu_to_le16(io.appmask); + if (!nvme_cmd_allowed(ns->ctrl, ns, &c, flags, open_for_write)) + return -EACCES; + return nvme_submit_user_cmd(ns->queue, &c, io.addr, length, metadata, meta_len, NULL, 0, 0); } @@ -307,7 +343,7 @@ static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(cmd.cdw14); c.common.cdw15 = cpu_to_le32(cmd.cdw15); - if (!nvme_cmd_allowed(ns, &c, 0, open_for_write)) + if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) return -EACCES; if (cmd.timeout_ms) @@ -354,7 +390,7 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(cmd.cdw14); c.common.cdw15 = cpu_to_le32(cmd.cdw15); - if (!nvme_cmd_allowed(ns, &c, flags, open_for_write)) + if (!nvme_cmd_allowed(ctrl, ns, &c, flags, open_for_write)) return -EACCES; if (cmd.timeout_ms) @@ -449,6 +485,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe, struct nvme_uring_cmd); struct request_queue *q = ns ? ns->queue : ctrl->admin_q; + bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE; struct nvme_uring_data d; struct nvme_command c; struct iov_iter iter; @@ -479,7 +516,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14)); c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15)); - if (!nvme_cmd_allowed(ns, &c, 0, ioucmd->file->f_mode & FMODE_WRITE)) + if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) return -EACCES; d.metadata = READ_ONCE(cmd->metadata); @@ -595,7 +632,7 @@ static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd, case NVME_IOCTL_SUBMIT_IO32: #endif case NVME_IOCTL_SUBMIT_IO: - return nvme_submit_io(ns, argp); + return nvme_submit_io(ns, argp, flags, open_for_write); case NVME_IOCTL_IO64_CMD_VEC: flags |= NVME_IOCTL_VEC; fallthrough; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 824651cc898db..cfdcf53db5abd 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -178,6 +178,11 @@ enum nvme_quirks { * Align dma pool segment size to 512 bytes */ NVME_QUIRK_DMAPOOL_ALIGN_512 = (1 << 22), + + /* + * Admin queue DMA buffers must be page aligned + */ + NVME_QUIRK_ADMIN_PAGE_ALIGN = (1 << 23), }; static inline char *nvme_quirk_name(enum nvme_quirks q) @@ -229,6 +234,8 @@ static inline char *nvme_quirk_name(enum nvme_quirks q) return "broken_msi"; case NVME_QUIRK_DMAPOOL_ALIGN_512: return "dmapool_align_512"; + case NVME_QUIRK_ADMIN_PAGE_ALIGN: + return "admin_page_align"; } return "unknown"; @@ -496,7 +503,8 @@ struct nvme_subsystem { struct list_head entry; struct mutex lock; struct list_head ctrls; - struct list_head nsheads; + struct list_head nsheads + __guarded_by(&lock); char subnqn[NVMF_NQN_SIZE]; char serial[20]; char model[40]; @@ -560,6 +568,7 @@ struct nvme_ns_head { u16 nr_plids; u16 *plids; + u32 write_stream_granularity; #ifdef CONFIG_NVME_MULTIPATH struct bio_list requeue_list; spinlock_t requeue_lock; @@ -568,7 +577,8 @@ struct nvme_ns_head { struct mutex lock; unsigned long flags; struct delayed_work remove_work; - unsigned int delayed_removal_secs; + unsigned int delayed_removal_secs + __guarded_by(&subsys->lock); atomic_long_t io_requeue_no_usable_path_count; atomic_long_t io_fail_no_available_path_count; #define NVME_NSHEAD_DISK_LIVE 0 @@ -1039,9 +1049,12 @@ static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl) return ctrl->ana_log_buf != NULL; } -void nvme_mpath_unfreeze(struct nvme_subsystem *subsys); -void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys); -void nvme_mpath_start_freeze(struct nvme_subsystem *subsys); +void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) + __must_hold(&subsys->lock); +void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) + __must_hold(&subsys->lock); +void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) + __must_hold(&subsys->lock); void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys); void nvme_failover_req(struct request *req); void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl); diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 69932d640b537..664a36eed23c3 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -213,6 +213,7 @@ static int quirks_param_set(const char *value, const struct kernel_param *kp) if (nvme_parse_quirk_entry(field, &qlist[i])) { pr_err("nvme: failed to parse quirk string %s\n", value); + err = -EINVAL; goto out_free_qlist; } @@ -2400,6 +2401,7 @@ static int nvme_pci_configure_admin_queue(struct nvme_dev *dev) result = queue_request_irq(nvmeq); if (result) { dev->online_queues--; + nvme_disable_ctrl(&dev->ctrl, false); return result; } @@ -3838,6 +3840,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) nvme_dev_remove_admin(dev); nvme_dbbuf_dma_free(dev); nvme_free_queues(dev, 0); + nvme_release_descriptor_pools(dev); out_release_iod_mempool: mempool_destroy(dev->dmavec_mempool); out_dev_unmap: diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 6909e35427942..618ee95444263 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -2003,7 +2003,7 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, struct ib_device *dev; bool queue_ready = test_bit(NVME_RDMA_Q_LIVE, &queue->flags); blk_status_t ret; - int err; + int err = 0; WARN_ON_ONCE(rq->tag < 0); @@ -2059,16 +2059,18 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, err_unmap: nvme_rdma_unmap_data(queue, rq); err: - if (err == -EIO) - ret = nvme_host_path_error(rq); - else if (err == -ENOMEM || err == -EAGAIN) - ret = BLK_STS_RESOURCE; - else - ret = BLK_STS_IOERR; - nvme_cleanup_cmd(rq); + if (err != -EIO) { + nvme_cleanup_cmd(rq); + if (err == -ENOMEM || err == -EAGAIN) + ret = BLK_STS_RESOURCE; + else + ret = BLK_STS_IOERR; + } unmap_qe: ib_dma_unmap_single(dev, req->sqe.dma, sizeof(struct nvme_command), DMA_TO_DEVICE); + if (err == -EIO) + return nvme_host_path_error(rq); return ret; } diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c index 75b2d69b59578..be94739e17dcf 100644 --- a/drivers/nvme/host/sysfs.c +++ b/drivers/nvme/host/sysfs.c @@ -166,7 +166,7 @@ static DEVICE_ATTR_RO(eui); static ssize_t nsid_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sysfs_emit(buf, "%d\n", dev_to_ns_head(dev)->ns_id); + return sysfs_emit(buf, "%u\n", dev_to_ns_head(dev)->ns_id); } static DEVICE_ATTR_RO(nsid); diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index ba5c7b3e2a7c6..52efc6e4f7623 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -80,6 +80,7 @@ struct nvme_tcp_request { struct bio *curr_bio; struct iov_iter iter; + u32 data_recvd; /* send state */ size_t offset; @@ -108,6 +109,7 @@ struct nvme_tcp_queue { struct mutex queue_lock; struct mutex send_mutex; + struct mutex pf_cache_lock; struct llist_head req_list; struct list_head send_list; @@ -409,8 +411,13 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, * if we're the first on the send_list and we can try to send * directly, otherwise queue io_work. Also, only do that if we * are on the same cpu, so we don't introduce contention. + * + * TLS kTLS send takes ctx->tx_lock while blk_mq holds set->srcu. + * lockdep reports circular locking via elevator_lock. Defer TLS + * sends to the io workqueue instead of inline from this path. */ if (queue->io_cpu == raw_smp_processor_id() && + !nvme_tcp_queue_tls(queue) && empty && mutex_trylock(&queue->send_mutex)) { nvme_tcp_send_all(queue); mutex_unlock(&queue->send_mutex); @@ -550,9 +557,11 @@ static int nvme_tcp_init_request(struct blk_mq_tag_set *set, struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; u8 hdgst = nvme_tcp_hdgst_len(queue); + mutex_lock(&queue->pf_cache_lock); req->pdu = page_frag_alloc(&queue->pf_cache, sizeof(struct nvme_tcp_cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); + mutex_unlock(&queue->pf_cache_lock); if (!req->pdu) return -ENOMEM; @@ -612,6 +621,29 @@ static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); } +/* + * NVMe has no short read: a read that completes successfully must + * have transferred everything it asked for. + */ +static bool nvme_tcp_data_in_short(struct nvme_tcp_queue *queue, + struct request *rq) +{ + struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); + + if (le16_to_cpu(req->status) >> 1) + return false; + if (req_op(rq) != REQ_OP_READ || !req->data_len) + return false; + if (likely(req->data_recvd == req->data_len)) + return false; + + dev_err(queue->ctrl->ctrl.device, + "queue %d tag %#x short data-in: got %u of %u\n", + nvme_tcp_queue_id(queue), rq->tag, + req->data_recvd, req->data_len); + return true; +} + static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, struct nvme_completion *cqe) { @@ -631,6 +663,9 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, if (req->status == cpu_to_le16(NVME_SC_SUCCESS)) req->status = cqe->status; + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; + if (!nvme_try_complete_req(rq, req->status, cqe->result)) nvme_complete_rq(rq); queue->nr_cqe++; @@ -641,6 +676,7 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, struct nvme_tcp_data_pdu *pdu) { + struct nvme_tcp_request *req; struct request *rq; rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); @@ -651,7 +687,15 @@ static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, return -ENOENT; } - if (!blk_rq_payload_bytes(rq)) { + if (rq_data_dir(rq) != READ) { + dev_err(queue->ctrl->ctrl.device, + "queue %d tag %#x unexpected data for a write\n", + nvme_tcp_queue_id(queue), rq->tag); + return -EIO; + } + + req = blk_mq_rq_to_pdu(rq); + if (!blk_rq_payload_bytes(rq) || !req->curr_bio || !req->data_len) { dev_err(queue->ctrl->ctrl.device, "queue %d tag %#x unexpected data\n", nvme_tcp_queue_id(queue), rq->tag); @@ -745,6 +789,13 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, } req = blk_mq_rq_to_pdu(rq); + if (unlikely(rq_data_dir(rq) != WRITE)) { + dev_err(queue->ctrl->ctrl.device, + "req %d unexpected r2t for a non-write command\n", + rq->tag); + return -EPROTO; + } + if (unlikely(!r2t_length)) { dev_err(queue->ctrl->ctrl.device, "req %d r2t len is %u, probably a bug...\n", @@ -953,6 +1004,7 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, *len -= recv_len; *offset += recv_len; queue->data_remaining -= recv_len; + req->data_recvd += recv_len; } if (!queue->data_remaining) { @@ -961,6 +1013,8 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; } else { if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; nvme_tcp_end_request(rq, le16_to_cpu(req->status)); queue->nr_cqe++; @@ -1009,6 +1063,9 @@ static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, pdu->command_id); struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; + nvme_tcp_end_request(rq, le16_to_cpu(req->status)); queue->nr_cqe++; } @@ -1417,9 +1474,11 @@ static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) struct nvme_tcp_request *async = &ctrl->async_req; u8 hdgst = nvme_tcp_hdgst_len(queue); + mutex_lock(&queue->pf_cache_lock); async->pdu = page_frag_alloc(&queue->pf_cache, sizeof(struct nvme_tcp_cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); + mutex_unlock(&queue->pf_cache_lock); if (!async->pdu) return -ENOMEM; @@ -1461,6 +1520,7 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) kfree(queue->pdu); mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); + mutex_destroy(&queue->pf_cache_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC lockdep_unregister_key(&queue->nvme_tcp_sk_key); @@ -1788,6 +1848,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, INIT_LIST_HEAD(&queue->send_list); mutex_init(&queue->send_mutex); INIT_WORK(&queue->io_work, nvme_tcp_io_work); + mutex_init(&queue->pf_cache_lock); if (qid > 0) queue->cmnd_capsule_len = nctrl->ioccsz * 16; @@ -1928,6 +1989,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, err_destroy_mutex: mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); + mutex_destroy(&queue->pf_cache_lock); return ret; } @@ -2736,6 +2798,7 @@ static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, req->status = cpu_to_le16(NVME_SC_SUCCESS); req->offset = 0; req->data_sent = 0; + req->data_recvd = 0; req->pdu_len = 0; req->pdu_sent = 0; req->h2cdata_left = 0; diff --git a/drivers/nvme/host/zns.c b/drivers/nvme/host/zns.c index 8ed1b6a33454e..9477cfbd3a513 100644 --- a/drivers/nvme/host/zns.c +++ b/drivers/nvme/host/zns.c @@ -48,12 +48,12 @@ int nvme_query_zone_info(struct nvme_ns *ns, unsigned lbaf, NVME_CMD_EFFECTS_CSUPP)) { if (test_and_clear_bit(NVME_NS_FORCE_RO, &ns->flags)) dev_warn(ns->ctrl->device, - "Zone Append supported for zoned namespace:%d. Remove read-only mode\n", + "Zone Append supported for zoned namespace:%u. Remove read-only mode\n", ns->head->ns_id); } else { set_bit(NVME_NS_FORCE_RO, &ns->flags); dev_warn(ns->ctrl->device, - "Zone Append not supported for zoned namespace:%d. Forcing to read-only mode\n", + "Zone Append not supported for zoned namespace:%u. Forcing to read-only mode\n", ns->head->ns_id); } diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 01b799e92ae67..7764a3c0195c9 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -309,8 +309,10 @@ static void nvmet_execute_get_log_page_rmi(struct nvmet_req *req) } log = kzalloc_obj(*log); - if (!log) + if (!log) { + status = NVME_SC_INTERNAL; goto out; + } log->endgid = req->cmd->get_log_page.lsi; disk = req->ns->bdev->bd_disk; @@ -958,7 +960,7 @@ static void nvmet_execute_identify_nslist(struct nvmet_req *req, bool match_css) nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) { if (ns->nsid <= min_nsid) continue; - if (match_css && req->ns->csi != req->cmd->identify.csi) + if (match_css && ns->csi != req->cmd->identify.csi) continue; list[i++] = cpu_to_le32(ns->nsid); if (i == buf_size / sizeof(__le32)) @@ -1335,7 +1337,7 @@ static u16 nvmet_set_feat_arbitration(struct nvmet_req *req) void nvmet_execute_set_features(struct nvmet_req *req) { - struct nvmet_subsys *subsys = nvmet_req_subsys(req); + struct nvmet_ctrl *ctrl = nvmet_req_ctrl(req); u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11); u16 status = 0; @@ -1357,7 +1359,7 @@ void nvmet_execute_set_features(struct nvmet_req *req) break; } nvmet_set_result(req, - (subsys->max_qid - 1) | ((subsys->max_qid - 1) << 16)); + (ctrl->max_qid - 1) | ((ctrl->max_qid - 1) << 16)); break; case NVME_FEAT_IRQ_COALESCE: status = nvmet_set_feat_irq_coalesce(req); @@ -1494,7 +1496,7 @@ void nvmet_get_feat_async_event(struct nvmet_req *req) void nvmet_execute_get_features(struct nvmet_req *req) { - struct nvmet_subsys *subsys = nvmet_req_subsys(req); + struct nvmet_ctrl *ctrl = nvmet_req_ctrl(req); u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u16 status = 0; @@ -1534,7 +1536,7 @@ void nvmet_execute_get_features(struct nvmet_req *req) break; case NVME_FEAT_NUM_QUEUES: nvmet_set_result(req, - (subsys->max_qid-1) | ((subsys->max_qid-1) << 16)); + (ctrl->max_qid-1) | ((ctrl->max_qid-1) << 16)); break; case NVME_FEAT_KATO: nvmet_get_feat_kato(req); diff --git a/drivers/nvme/target/auth.c b/drivers/nvme/target/auth.c index edb9627d97b09..a55319bcdbd13 100644 --- a/drivers/nvme/target/auth.c +++ b/drivers/nvme/target/auth.c @@ -238,6 +238,12 @@ void nvmet_auth_sq_free(struct nvmet_sq *sq) sq->dhchap_skey = NULL; } +void nvmet_auth_sq_destroy(struct nvmet_sq *sq) +{ + cancel_delayed_work_sync(&sq->auth_expired_work); + nvmet_auth_sq_free(sq); +} + void nvmet_destroy_auth(struct nvmet_ctrl *ctrl) { ctrl->shash_id = 0; diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 2b69ffcfc8dfb..6286e38436ddf 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -312,15 +312,17 @@ static ssize_t nvmet_param_mdts_store(struct config_item *item, const char *page, size_t count) { struct nvmet_port *port = to_nvmet_port(item); - int ret; + int ret, mdts; if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - ret = kstrtoint(page, 0, &port->mdts); - if (ret) { - pr_err("Invalid value '%s' for mdts\n", page); + ret = kstrtoint(page, 0, &mdts); + if (ret || mdts < 0 || mdts > NVMET_MAX_MDTS) { + pr_err("Invalid value '%s' for mdts, should be 0-%d\n", + page, NVMET_MAX_MDTS); return -EINVAL; } + port->mdts = mdts; return count; } @@ -812,7 +814,7 @@ static ssize_t nvmet_ns_resv_enable_store(struct config_item *item, mutex_lock(&ns->subsys->lock); if (ns->enabled) { - pr_err("the ns:%d is already enabled.\n", ns->nsid); + pr_err("the ns:%u is already enabled.\n", ns->nsid); mutex_unlock(&ns->subsys->lock); return -EINVAL; } @@ -878,7 +880,7 @@ static struct config_group *nvmet_ns_make(struct config_group *group, goto out; config_group_init_type_name(&ns->group, name, &nvmet_ns_type); - pr_info("adding nsid %d to subsystem %s\n", nsid, subsys->subsysnqn); + pr_info("adding nsid %u to subsystem %s\n", nsid, subsys->subsysnqn); return &ns->group; out: diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 4477c4d6b1eed..269daefb88854 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -558,7 +558,7 @@ static void nvmet_p2pmem_ns_add_p2p(struct nvmet_ctrl *ctrl, if (ret < 0) pci_dev_put(p2p_dev); - pr_info("using p2pmem on %s for nsid %d\n", pci_name(p2p_dev), + pr_info("using p2pmem on %s for nsid %u\n", pci_name(p2p_dev), ns->nsid); } @@ -610,7 +610,8 @@ int nvmet_ns_enable(struct nvmet_ns *ns) goto out_dev_put; } - if (percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL)) + ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL); + if (ret) goto out_pr_exit; nvmet_ns_changed(subsys, ns->nsid); @@ -875,7 +876,7 @@ u16 nvmet_check_cqid(struct nvmet_ctrl *ctrl, u16 cqid, bool create) if (!ctrl->cqs) return NVME_SC_INTERNAL | NVME_STATUS_DNR; - if (cqid > ctrl->subsys->max_qid) + if (cqid > ctrl->max_qid) return NVME_SC_QID_INVALID | NVME_STATUS_DNR; if ((create && ctrl->cqs[cqid]) || (!create && !ctrl->cqs[cqid])) @@ -923,7 +924,7 @@ u16 nvmet_check_sqid(struct nvmet_ctrl *ctrl, u16 sqid, if (!ctrl->sqs) return NVME_SC_INTERNAL | NVME_STATUS_DNR; - if (sqid > ctrl->subsys->max_qid) + if (sqid > ctrl->max_qid) return NVME_SC_QID_INVALID | NVME_STATUS_DNR; if ((create && ctrl->sqs[sqid]) || @@ -977,7 +978,7 @@ void nvmet_sq_destroy(struct nvmet_sq *sq) wait_for_completion(&sq->confirm_done); wait_for_completion(&sq->free_done); percpu_ref_exit(&sq->ref); - nvmet_auth_sq_free(sq); + nvmet_auth_sq_destroy(sq); nvmet_cq_put(sq->cq); /* @@ -1652,11 +1653,29 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) if (!ctrl->changed_ns_list) goto out_free_ctrl; - ctrl->sqs = kzalloc_objs(struct nvmet_sq *, subsys->max_qid + 1); + /* + * Discovery controllers may use some arbitrary high value + * in order to cleanup stale discovery sessions + */ + if (nvmet_is_disc_subsys(ctrl->subsys) && !kato) + kato = NVMET_DISC_KATO_MS; + + /* keep-alive timeout in seconds */ + ctrl->kato = DIV_ROUND_UP(kato, 1000); + + ctrl->err_counter = 0; + spin_lock_init(&ctrl->error_lock); + + down_read(&nvmet_config_sem); + mutex_lock(&subsys->lock); + + ctrl->max_qid = subsys->max_qid; + + ctrl->sqs = kzalloc_objs(struct nvmet_sq *, ctrl->max_qid + 1); if (!ctrl->sqs) goto out_free_changed_ns_list; - ctrl->cqs = kzalloc_objs(struct nvmet_cq *, subsys->max_qid + 1); + ctrl->cqs = kzalloc_objs(struct nvmet_cq *, ctrl->max_qid + 1); if (!ctrl->cqs) goto out_free_sqs; @@ -1669,22 +1688,6 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) } ctrl->cntlid = ret; - /* - * Discovery controllers may use some arbitrary high value - * in order to cleanup stale discovery sessions - */ - if (nvmet_is_disc_subsys(ctrl->subsys) && !kato) - kato = NVMET_DISC_KATO_MS; - - /* keep-alive timeout in seconds */ - ctrl->kato = DIV_ROUND_UP(kato, 1000); - - ctrl->err_counter = 0; - spin_lock_init(&ctrl->error_lock); - - nvmet_start_keep_alive_timer(ctrl); - - mutex_lock(&subsys->lock); ret = nvmet_ctrl_init_pr(ctrl); if (ret) goto init_pr_fail; @@ -1692,6 +1695,9 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) nvmet_setup_p2p_ns_map(ctrl, args->p2p_client); nvmet_debugfs_ctrl_setup(ctrl); mutex_unlock(&subsys->lock); + up_read(&nvmet_config_sem); + + nvmet_start_keep_alive_timer(ctrl); if (args->hostid) uuid_copy(&ctrl->hostid, args->hostid); @@ -1721,14 +1727,14 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) return ctrl; init_pr_fail: - mutex_unlock(&subsys->lock); - nvmet_stop_keep_alive_timer(ctrl); ida_free(&cntlid_ida, ctrl->cntlid); out_free_cqs: kfree(ctrl->cqs); out_free_sqs: kfree(ctrl->sqs); out_free_changed_ns_list: + mutex_unlock(&subsys->lock); + up_read(&nvmet_config_sem); kfree(ctrl->changed_ns_list); out_free_ctrl: kfree(ctrl); diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index 45820a12750de..0e88e227edac3 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -30,12 +30,16 @@ void nvmet_auth_sq_init(struct nvmet_sq *sq) sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; } -static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) +static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d, u32 tl) { struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmf_auth_dhchap_negotiate_data *data = d; int i, hash_id = 0, fallback_hash_id = 0, dhgid, fallback_dhgid; + if (tl < sizeof(*data) + + sizeof(struct nvmf_auth_dhchap_protocol_descriptor)) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + pr_debug("%s: ctrl %d qid %d: data sc_d %d napd %d authid %d halen %d dhlen %d\n", __func__, ctrl->cntlid, req->sq->qid, data->sc_c, data->napd, data->auth_protocol[0].dhchap.authid, @@ -71,6 +75,10 @@ static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) NVME_AUTH_DHCHAP_AUTH_ID) return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + if (data->auth_protocol[0].dhchap.dhlen > NVME_AUTH_DHCHAP_MAX_DH_IDS || + data->auth_protocol[0].dhchap.halen > NVME_AUTH_DHCHAP_MAX_HASH_IDS) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + for (i = 0; i < data->auth_protocol[0].dhchap.halen; i++) { u8 host_hmac_id = data->auth_protocol[0].dhchap.idlist[i]; @@ -316,7 +324,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req) } else if (data->auth_id != req->sq->dhchap_step) goto done_failure1; /* Validate negotiation parameters */ - dhchap_status = nvmet_auth_negotiate(req, d); + dhchap_status = nvmet_auth_negotiate(req, d, tl); if (dhchap_status == 0) req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE; @@ -557,7 +565,7 @@ void nvmet_execute_auth_receive(struct nvmet_req *req) return; } - d = kmalloc(al, GFP_KERNEL); + d = kzalloc(al, GFP_KERNEL); if (!d) { status = NVME_SC_INTERNAL; goto done; diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c index 7cadd1c9e44cc..42d1d18116715 100644 --- a/drivers/nvme/target/fabrics-cmd.c +++ b/drivers/nvme/target/fabrics-cmd.c @@ -370,7 +370,7 @@ static void nvmet_execute_io_connect(struct nvmet_req *req) goto out; } - if (unlikely(qid > ctrl->subsys->max_qid)) { + if (unlikely(qid > ctrl->max_qid)) { pr_warn("invalid queue id (%d)\n", qid); status = NVME_SC_CONNECT_INVALID_PARAM | NVME_STATUS_DNR; req->cqe->result.u32 = IPO_IATTR_CONNECT_SQE(qid); diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index d161707559ce6..1b557775e033a 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -566,7 +566,7 @@ nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport) list_del(&iod->ls_rcv_list); } - kfree(iod); + kfree(tgtport->iod); return -EFAULT; } diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index aaba745e3c214..7750f5b1edbb4 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -265,6 +265,7 @@ struct nvmet_ctrl { uuid_t hostid; u16 cntlid; + u16 max_qid; u32 kato; struct nvmet_port *port; @@ -753,6 +754,11 @@ static inline struct nvmet_subsys *nvmet_req_subsys(struct nvmet_req *req) return req->sq->ctrl->subsys; } +static inline struct nvmet_ctrl *nvmet_req_ctrl(struct nvmet_req *req) +{ + return req->sq->ctrl; +} + static inline bool nvmet_is_disc_subsys(struct nvmet_subsys *subsys) { return subsys->type != NVME_NQN_NVME; @@ -915,6 +921,7 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset); void nvmet_auth_sq_init(struct nvmet_sq *sq); void nvmet_destroy_auth(struct nvmet_ctrl *ctrl); void nvmet_auth_sq_free(struct nvmet_sq *sq); +void nvmet_auth_sq_destroy(struct nvmet_sq *sq); int nvmet_setup_dhgroup(struct nvmet_ctrl *ctrl, u8 dhgroup_id); bool nvmet_check_auth_status(struct nvmet_req *req); int nvmet_auth_host_hash(struct nvmet_req *req, u8 *response, @@ -941,6 +948,7 @@ static inline void nvmet_auth_sq_init(struct nvmet_sq *sq) } static inline void nvmet_destroy_auth(struct nvmet_ctrl *ctrl) {}; static inline void nvmet_auth_sq_free(struct nvmet_sq *sq) {}; +static inline void nvmet_auth_sq_destroy(struct nvmet_sq *sq) {}; static inline bool nvmet_check_auth_status(struct nvmet_req *req) { return true; diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 4e9db96ebfecd..803e85df50e54 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -1339,6 +1339,7 @@ static u16 nvmet_pci_epf_create_cq(struct nvmet_ctrl *tctrl, nvmet_pci_epf_mem_unmap(ctrl->nvme_epf, &cq->pci_map); err_internal: status = NVME_SC_INTERNAL | NVME_STATUS_DNR; + nvmet_cq_put(&cq->nvme_cq); err: if (test_and_clear_bit(NVMET_PCI_EPF_Q_IRQ_ENABLED, &cq->flags)) nvmet_pci_epf_remove_irq_vector(ctrl, cq->vector); @@ -1594,6 +1595,7 @@ static void nvmet_pci_epf_exec_iod_work(struct work_struct *work) struct nvmet_pci_epf_iod *iod = container_of(work, struct nvmet_pci_epf_iod, work); struct nvmet_req *req = &iod->req; + bool no_wait; int ret; if (!iod->ctrl->link_up) { @@ -1638,14 +1640,16 @@ static void nvmet_pci_epf_exec_iod_work(struct work_struct *work) } } - req->execute(req); - /* * If we do not have data to transfer after the command execution * finishes, nvmet_pci_epf_queue_response() will complete the command * directly. No need to wait for the completion in this case. */ - if (!iod->data_len || iod->dma_dir != DMA_TO_DEVICE) + no_wait = !iod->data_len || iod->dma_dir != DMA_TO_DEVICE; + + req->execute(req); + + if (no_wait) return; wait_for_completion(&iod->done); @@ -2077,7 +2081,7 @@ static int nvmet_pci_epf_create_ctrl(struct nvmet_pci_epf *nvme_epf, } /* Allocate our queues, up to the maximum number. */ - ctrl->nr_queues = min(ctrl->tctrl->subsys->max_qid + 1, max_nr_queues); + ctrl->nr_queues = min(ctrl->tctrl->max_qid + 1, max_nr_queues); ret = nvmet_pci_epf_alloc_queues(ctrl); if (ret) goto out_put_ctrl; diff --git a/drivers/nvme/target/pr.c b/drivers/nvme/target/pr.c index c71ae46244ffc..ad07f55e7f173 100644 --- a/drivers/nvme/target/pr.c +++ b/drivers/nvme/target/pr.c @@ -145,7 +145,7 @@ static void nvmet_pr_add_resv_log(struct nvmet_ctrl *ctrl, u8 log_type, log.nsid = cpu_to_le32(nsid); if (!kfifo_put(&log_mgr->log_queue, log)) { - pr_info("a reservation log lost, cntlid:%d, log_type:%d, nsid:%d\n", + pr_info("a reservation log lost, cntlid:%d, log_type:%d, nsid:%u\n", ctrl->cntlid, log_type, nsid); log_mgr->lost_count++; } @@ -355,9 +355,15 @@ static u16 nvmet_pr_replace(struct nvmet_req *req, u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmet_pr *pr = &req->ns->pr; - struct nvmet_pr_registrant *reg; + struct nvmet_pr_registrant *reg, *new = NULL; u64 nrkey = le64_to_cpu(d->nrkey); + if (ignore_key && nrkey) { + new = kzalloc_obj(*new); + if (!new) + return NVME_SC_INTERNAL; + } + down(&pr->pr_sem); list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { if (uuid_equal(®->hostid, &ctrl->hostid)) { @@ -365,9 +371,26 @@ static u16 nvmet_pr_replace(struct nvmet_req *req, status = nvmet_pr_update_reg_attr(pr, reg, nvmet_pr_update_reg_rkey, &nrkey); - break; + goto free_data; + } + } + + if (ignore_key) { + if (!nrkey) { + status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto free_data; } + INIT_LIST_HEAD(&new->entry); + new->rkey = nrkey; + uuid_copy(&new->hostid, &ctrl->hostid); + list_add_tail_rcu(&new->entry, &pr->registrant_list); + status = NVME_SC_SUCCESS; + goto out; } + +free_data: + kfree(new); +out: up(&pr->pr_sem); return status; } diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index ea1185b8267ef..542138fd669f4 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -657,18 +657,25 @@ static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp *rsp) req->sg, req->sg_cnt, nvmet_data_dir(req)); } -static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) +static void nvmet_rdma_free_rsp_resources(struct nvmet_rdma_rsp *rsp) { struct nvmet_rdma_queue *queue = rsp->queue; - atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); - if (rsp->n_rdma) nvmet_rdma_rw_ctx_destroy(rsp); if (rsp->req.sg < rsp->cmd->inline_sg || rsp->req.sg >= rsp->cmd->inline_sg + queue->dev->inline_page_count) nvmet_req_free_sgls(&rsp->req); +} + +static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) +{ + struct nvmet_rdma_queue *queue = rsp->queue; + + atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); + + nvmet_rdma_free_rsp_resources(rsp); if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list))) nvmet_rdma_process_wr_wait_list(queue); @@ -1338,9 +1345,27 @@ static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue) goto out; } +static bool nvmet_rdma_reclaim_rsp(struct sbitmap *sb, unsigned int bitnr, + void *data) +{ + struct nvmet_rdma_queue *queue = data; + + nvmet_rdma_free_rsp_resources(&queue->rsps[bitnr]); + + return true; +} + static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue *queue) { ib_drain_qp(queue->qp); + + /* + * Reclaim resources of a response that is still in-flight when the + * queue is being torn down. This happens when the connection was + * forcefully disconnected while an I/O is in flight. + */ + sbitmap_for_each_set(&queue->rsp_tags, nvmet_rdma_reclaim_rsp, queue); + if (queue->cm_id) rdma_destroy_id(queue->cm_id); ib_destroy_qp(queue->qp); @@ -1602,19 +1627,13 @@ static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, mutex_unlock(&nvmet_rdma_queue_mutex); if (pending > NVMET_RDMA_BACKLOG) { ret = NVME_SC_CONNECT_CTRL_BUSY; - goto put_device; + goto free_queue; } } ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn); - if (ret) { - /* - * Don't destroy the cm_id in free path, as we implicitly - * destroy the cm_id here with non-zero ret code. - */ - queue->cm_id = NULL; + if (ret) goto free_queue; - } mutex_lock(&nvmet_rdma_queue_mutex); list_add_tail(&queue->queue_list, &nvmet_rdma_queue_list); @@ -1623,6 +1642,11 @@ static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, return 0; free_queue: + /* + * Don't destroy the cm_id in free path, as we implicitly + * destroy the cm_id here with non-zero ret code. + */ + queue->cm_id = NULL; nvmet_rdma_free_queue(queue); put_device: kref_put(&ndev->ref, nvmet_rdma_free_dev); diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 75a276d73be3e..e598101752620 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -103,6 +103,7 @@ enum nvmet_tcp_recv_state { enum { NVMET_TCP_F_INIT_FAILED = (1 << 0), + NVMET_TCP_F_R2T_SENT = (1 << 1), }; struct nvmet_tcp_cmd { @@ -422,6 +423,19 @@ static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) if (!len) return 0; + /* + * inline_data_size only bounds the in-capsule (type 0x01) SGL + * descriptor below. A non-inline transport SGL data-block + * descriptor skips that check entirely and would otherwise reach + * sgl_alloc() with an attacker-controlled len of up to 4 GiB, + * pinning that much kernel memory for a command that may never + * complete. Bound every descriptor type here, before allocating + * anything, using the same ceiling this file already applies to + * per-PDU H2C data. + */ + if (len > NVMET_TCP_MAXH2CDATA) + return NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR; + if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET)) { if (!nvme_is_write(cmd->req.cmd)) @@ -433,13 +447,15 @@ static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) } cmd->req.transfer_len += len; - cmd->req.sg = sgl_alloc(len, GFP_KERNEL, &cmd->req.sg_cnt); + cmd->req.sg = sgl_alloc(len, GFP_KERNEL | __GFP_NOWARN, + &cmd->req.sg_cnt); if (!cmd->req.sg) return NVME_SC_INTERNAL; cmd->cur_sg = cmd->req.sg; if (nvmet_tcp_has_data_in(cmd)) { - cmd->iov = kmalloc_objs(*cmd->iov, cmd->req.sg_cnt); + cmd->iov = kmalloc_objs(*cmd->iov, cmd->req.sg_cnt, + GFP_KERNEL | __GFP_NOWARN); if (!cmd->iov) goto err; } @@ -761,6 +777,7 @@ static int nvmet_try_send_r2t(struct nvmet_tcp_cmd *cmd, bool last_in_batch) return -EAGAIN; cmd->queue->snd_cmd = NULL; + cmd->flags |= NVMET_TCP_F_R2T_SENT; return 1; } @@ -994,6 +1011,12 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) cmd = &queue->connect; } + if (unlikely(!(cmd->flags & NVMET_TCP_F_R2T_SENT))) { + pr_err("queue %d: unsolicited H2CData (ttag %u)\n", + queue->idx, data->ttag); + goto err_proto; + } + if (le32_to_cpu(data->data_offset) != cmd->rbytes_done) { pr_err("ttag %u unexpected data offset %u (expected %u)\n", data->ttag, le32_to_cpu(data->data_offset), @@ -1229,6 +1252,8 @@ static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue) } queue->left = hdr->hlen - queue->offset + hdgst; + if (queue->left > sizeof(queue->pdu) - queue->offset) + return -EPROTO; goto recv; } diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index f00921931eb64..a13befd5f3adf 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -116,7 +116,7 @@ void nvmet_execute_identify_ns_zns(struct nvmet_req *req) mutex_unlock(&req->ns->subsys->lock); } - if (!bdev_is_zoned(req->ns->bdev)) { + if (!req->ns->bdev || !bdev_is_zoned(req->ns->bdev)) { status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; req->error_loc = offsetof(struct nvme_identify, nsid); goto out; diff --git a/drivers/of/base.c b/drivers/of/base.c index 6e7a42dedad3f..477017ed6f49b 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -1967,7 +1967,7 @@ void of_alias_scan(void * (*dt_alloc)(u64 size, u64 align)) /* walk the alias backwards to extract the id and work out * the 'stem' string */ - while (isdigit(*(end-1)) && end > start) + while (end > start && isdigit(*(end - 1))) end--; len = end - start; diff --git a/drivers/of/irq.c b/drivers/of/irq.c index 967c191008797..ec035367c9500 100644 --- a/drivers/of/irq.c +++ b/drivers/of/irq.c @@ -563,7 +563,9 @@ const struct cpumask *of_irq_get_affinity(struct device_node *dev, int index) of_phandle_args_to_fwspec(oirq.np, oirq.args, oirq.args_count, &fwspec); - if (irq_populate_fwspec_info(&fwspec, &info)) + rc = irq_populate_fwspec_info(&fwspec, &info); + of_node_put(oirq.np); + if (rc) return NULL; return info.affinity; diff --git a/drivers/opp/core.c b/drivers/opp/core.c index ab0b0a2f85a17..b6966e509f7dd 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -1412,13 +1412,12 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, */ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) { + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); struct dev_pm_opp *opp __free(put_opp) = NULL; unsigned long freq = 0, temp_freq; bool forced = false; - struct opp_table *opp_table __free(put_opp_table) = - _find_opp_table(dev); - if (IS_ERR(opp_table)) { dev_err(dev, "%s: device's opp table doesn't exist\n", __func__); return PTR_ERR(opp_table); @@ -2870,11 +2869,10 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_add_dynamic); static int _opp_set_availability(struct device *dev, unsigned long freq, bool availability_req) { - struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp; - /* Find the opp_table */ struct opp_table *opp_table __free(put_opp_table) = _find_opp_table(dev); + struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp; if (IS_ERR(opp_table)) { dev_warn(dev, "%s: Device OPP not found (%ld)\n", __func__, @@ -2932,12 +2930,11 @@ int dev_pm_opp_adjust_voltage(struct device *dev, unsigned long freq, unsigned long u_volt_max) { - struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp; - int r; - /* Find the opp_table */ struct opp_table *opp_table __free(put_opp_table) = _find_opp_table(dev); + struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp; + int r; if (IS_ERR(opp_table)) { r = PTR_ERR(opp_table); diff --git a/drivers/opp/of.c b/drivers/opp/of.c index c02e20632fa64..9c4fd1f0e944a 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -1039,7 +1039,7 @@ static int _of_add_opp_table_v1(struct device *dev, struct opp_table *opp_table) val = prop->value; while (nr) { - unsigned long freq = be32_to_cpup(val++) * 1000; + unsigned long freq = (unsigned long)be32_to_cpup(val++) * 1000; unsigned long volt = be32_to_cpup(val++); struct dev_pm_opp_data data = { .freq = freq, diff --git a/drivers/parisc/eisa.c b/drivers/parisc/eisa.c index 9eab974e6baf6..b5b5413bfd1e9 100644 --- a/drivers/parisc/eisa.c +++ b/drivers/parisc/eisa.c @@ -442,6 +442,11 @@ static int __init eisa_irq_setup(char *str) val = (int) simple_strtoul(cur, &pe, 0); if (val > 15 || val < 0) { printk(KERN_ERR "eisa: EISA irq value are 0-15\n"); + cur = strchr(cur, ','); + if (cur) + cur++; + else + break; continue; } if (val == 2) { diff --git a/drivers/pci/controller/cadence/Kconfig b/drivers/pci/controller/cadence/Kconfig index 9e651d5459735..fd704dec47dda 100644 --- a/drivers/pci/controller/cadence/Kconfig +++ b/drivers/pci/controller/cadence/Kconfig @@ -20,11 +20,12 @@ config PCIE_CADENCE_EP config PCIE_CADENCE_PLAT tristate + select PCIE_CADENCE_HOST if PCIE_CADENCE_PLAT_HOST != n + select PCIE_CADENCE_EP if PCIE_CADENCE_PLAT_EP != n config PCIE_CADENCE_PLAT_HOST tristate "Cadence platform PCIe controller (host mode)" depends on OF - select PCIE_CADENCE_HOST select PCIE_CADENCE_PLAT help Say Y here if you want to support the Cadence PCIe platform controller in @@ -35,7 +36,6 @@ config PCIE_CADENCE_PLAT_EP tristate "Cadence platform PCIe controller (endpoint mode)" depends on OF depends on PCI_ENDPOINT - select PCIE_CADENCE_EP select PCIE_CADENCE_PLAT help Say Y here if you want to support the Cadence PCIe platform controller in diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index ae916e7b1927a..822602836b120 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -383,7 +383,7 @@ static const struct j721e_pcie_data j7200_pcie_rc_data = { .quirk_detect_quiet_flag = true, .linkdown_irq_regfield = J7200_LINK_DOWN, .byte_access_allowed = true, - .max_lanes = 2, + .max_lanes = 4, }; static const struct j721e_pcie_data j7200_pcie_ep_data = { @@ -391,7 +391,7 @@ static const struct j721e_pcie_data j7200_pcie_ep_data = { .quirk_detect_quiet_flag = true, .linkdown_irq_regfield = J7200_LINK_DOWN, .quirk_disable_flr = true, - .max_lanes = 2, + .max_lanes = 4, }; static const struct j721e_pcie_data am64_pcie_rc_data = { diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig index aa0b784c85b47..49a7a2c50ca15 100644 --- a/drivers/pci/controller/dwc/Kconfig +++ b/drivers/pci/controller/dwc/Kconfig @@ -126,7 +126,9 @@ config PCI_IMX6_EP depends on ARCH_MXC || COMPILE_TEST depends on PCI_ENDPOINT select PCIE_DW_EP + select PCI_HOST_COMMON select PCI_IMX6 + select PCI_PWRCTRL_GENERIC help Enables support for the PCIe controller in the i.MX SoCs to work in endpoint mode. The PCI controller on i.MX is based diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index f7389b5437dfe..f799e4b62815a 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -1388,12 +1388,6 @@ static int imx_pcie_host_init(struct dw_pcie_rp *pp) goto err_pwrctrl_destroy; } - ret = imx_pcie_clk_enable(imx_pcie); - if (ret) { - dev_err(dev, "unable to enable pcie clocks: %d\n", ret); - goto err_pwrctrl_power_off; - } - if (pp->bridge && imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT)) { pp->bridge->enable_device = imx_pcie_enable_device; pp->bridge->disable_device = imx_pcie_disable_device; @@ -1409,6 +1403,12 @@ static int imx_pcie_host_init(struct dw_pcie_rp *pp) imx_pcie_configure_type(imx_pcie); + ret = imx_pcie_clk_enable(imx_pcie); + if (ret) { + dev_err(dev, "unable to enable pcie clocks: %d\n", ret); + goto err_pwrctrl_power_off; + } + if (imx_pcie->phy) { ret = phy_init(imx_pcie->phy); if (ret) { diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index 278d2dba1db08..f1b27aed488de 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -1389,13 +1389,17 @@ static int ks_pcie_fault(unsigned long addr, unsigned int fsr, static int __init ks_pcie_init(void) { + struct device_node *np; /* * PCIe access errors that result into OCP errors are caught by ARM as * "External aborts" */ - if (of_find_matching_node(NULL, ks_pcie_of_match)) + np = of_find_matching_node(NULL, ks_pcie_of_match); + if (np) { + of_node_put(np); hook_fault_code(17, ks_pcie_fault, SIGBUS, 0, "Asynchronous external abort"); + } return platform_driver_register(&ks_pcie_driver); } diff --git a/drivers/pci/controller/dwc/pci-meson.c b/drivers/pci/controller/dwc/pci-meson.c index 7a4da9fae1eaa..71b0ebc0e4727 100644 --- a/drivers/pci/controller/dwc/pci-meson.c +++ b/drivers/pci/controller/dwc/pci-meson.c @@ -401,7 +401,7 @@ static int meson_pcie_probe(struct platform_device *pdev) return PTR_ERR(mp->phy); } - mp->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); + mp->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(mp->reset_gpio)) { dev_err(dev, "get reset gpio failed\n"); return PTR_ERR(mp->reset_gpio); diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 7d2794945704e..0b915824963af 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -1032,6 +1032,8 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, * there is no unified way to check if we have operations in * flight, thus we don't know if we should WARN() or not. */ + /* flush posted write before unmap */ + readl(ep->msi_mem + ep->msi_iatu_mapped_offset); dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); ep->msi_iatu_mapped = false; } @@ -1044,6 +1046,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, return ret; ep->msi_iatu_mapped = true; + ep->msi_iatu_mapped_offset = offset; ep->msi_msg_addr = msg_addr; ep->msi_map_size = map_size; } @@ -1124,6 +1127,17 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, return -EPERM; } + /* + * ep->msi_iatu_mapped means that an MSI target address is cached, + * unmap it first so that we can reuse ep->msi_mem_phys for MSI-X. + */ + if (ep->msi_iatu_mapped) { + /* flush posted write before unmap */ + readl(ep->msi_mem + ep->msi_iatu_mapped_offset); + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, map_size); @@ -1153,6 +1167,11 @@ void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + if (ep->msi_iatu_mapped) { + dw_pcie_ep_unmap_addr(ep->epc, 0, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + dwc_pcie_debugfs_deinit(pci); dw_pcie_edma_remove(pci); } diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index de4b245b1758c..1d21394741572 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -518,6 +518,7 @@ struct dw_pcie_ep { /* MSI outbound iATU state */ bool msi_iatu_mapped; + size_t msi_iatu_mapped_offset; u64 msi_msg_addr; size_t msi_map_size; }; diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index d8eb52857f69c..d62cf302de2b8 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -1820,6 +1820,23 @@ static const struct pci_ecam_ops pci_qcom_ecam_ops = { } }; +/* Check if @node is a child of @dev in DT */ +static bool qcom_pcie_is_child_node(struct device *dev, + struct device_node *node) +{ + struct device_node *parent; + + for (parent = of_get_parent(node); parent; + parent = of_get_next_parent(parent)) { + if (parent == dev->of_node) { + of_node_put(parent); + return true; + } + } + + return false; +} + /* Parse PERST# from all nodes in depth first manner starting from @np */ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie, struct qcom_pcie_port *port, @@ -1827,6 +1844,7 @@ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie, { struct device *dev = pcie->pci->dev; struct qcom_pcie_perst *perst; + struct device_node *gpio_np; struct gpio_desc *reset; int ret; @@ -1840,6 +1858,25 @@ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie, if (!of_find_property(np, "reset-gpios", NULL)) goto parse_child_node; + /* + * Skip GPIOs provided by a PCIe device which is a child of the Root + * Complex (e.g., a PCIe switch with GPIO controller capability). Such + * controllers won't be available at RC probe time and their PERST# + * should be controlled by the respective PCI client driver + * implementation. + */ + gpio_np = of_parse_phandle(np, "reset-gpios", 0); + if (!gpio_np) { + dev_err(dev, "Failed to parse GPIO provider\n"); + return -EINVAL; + } + + if (qcom_pcie_is_child_node(dev, gpio_np)) { + of_node_put(gpio_np); + goto parse_child_node; + } + of_node_put(gpio_np); + reset = devm_fwnode_gpiod_get(dev, of_fwnode_handle(np), "reset", GPIOD_OUT_HIGH, "PERST#"); if (IS_ERR(reset)) { diff --git a/drivers/pci/controller/dwc/pcie-ultrarisc.c b/drivers/pci/controller/dwc/pcie-ultrarisc.c index 6ee661ceff67a..8e57135453b7c 100644 --- a/drivers/pci/controller/dwc/pcie-ultrarisc.c +++ b/drivers/pci/controller/dwc/pcie-ultrarisc.c @@ -5,6 +5,7 @@ * Copyright (C) 2026 UltraRISC Technology (Shanghai) Co., Ltd. */ +#include #include #include #include @@ -23,6 +24,12 @@ #define ULTRARISC_PCIE_COMP_TIMEOUT_65_210MS 0x6 +struct ultrarisc_pcie { + struct dw_pcie pci; + struct clk_bulk_data *clks; + int num_clks; +}; + static struct pci_ops ultrarisc_pci_ops = { .map_bus = dw_pcie_own_conf_map_bus, .read = pci_generic_config_read32, @@ -98,17 +105,47 @@ static const struct dw_pcie_ops dw_pcie_ops = { .start_link = ultrarisc_pcie_start_link, }; +static int ultrarisc_pcie_enable_clks(struct ultrarisc_pcie *ultra) +{ + return clk_bulk_prepare_enable(ultra->num_clks, ultra->clks); +} + +static void ultrarisc_pcie_disable_clks(void *data) +{ + struct ultrarisc_pcie *ultra = data; + + clk_bulk_disable_unprepare(ultra->num_clks, ultra->clks); +} + +static int ultrarisc_pcie_init_clks(struct ultrarisc_pcie *ultra) +{ + struct device *dev = ultra->pci.dev; + int ret; + + ultra->num_clks = devm_clk_bulk_get_all(dev, &ultra->clks); + if (ultra->num_clks < 0) + return dev_err_probe(dev, ultra->num_clks, "Failed to get clocks\n"); + + ret = ultrarisc_pcie_enable_clks(ultra); + if (ret) + return dev_err_probe(dev, ret, "Failed to enable clocks\n"); + + return devm_add_action_or_reset(dev, ultrarisc_pcie_disable_clks, ultra); +} + static int ultrarisc_pcie_probe(struct platform_device *pdev) { + struct ultrarisc_pcie *ultra; struct device *dev = &pdev->dev; struct dw_pcie_rp *pp; struct dw_pcie *pci; int ret; - pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); - if (!pci) + ultra = devm_kzalloc(dev, sizeof(*ultra), GFP_KERNEL); + if (!ultra) return -ENOMEM; + pci = &ultra->pci; pci->dev = dev; pci->ops = &dw_pcie_ops; @@ -117,7 +154,11 @@ static int ultrarisc_pcie_probe(struct platform_device *pdev) pp = &pci->pp; - platform_set_drvdata(pdev, pci); + platform_set_drvdata(pdev, ultra); + + ret = ultrarisc_pcie_init_clks(ultra); + if (ret) + return ret; pp->num_vectors = MAX_MSI_IRQS; /* No L2/L3 Ready indication is available on this platform */ @@ -135,16 +176,46 @@ static int ultrarisc_pcie_probe(struct platform_device *pdev) static int ultrarisc_pcie_suspend_noirq(struct device *dev) { - struct dw_pcie *pci = dev_get_drvdata(dev); + struct ultrarisc_pcie *ultra = dev_get_drvdata(dev); + struct dw_pcie *pci = &ultra->pci; + int ret; + + /* + * A failed resume leaves the DWC suspended and the clocks disabled. + * A later suspend must not access the controller or disable them again. + */ + if (pci->suspended) + return 0; + + ret = dw_pcie_suspend_noirq(pci); + if (ret) + return ret; + + if (pci->suspended) + ultrarisc_pcie_disable_clks(ultra); - return dw_pcie_suspend_noirq(pci); + return 0; } static int ultrarisc_pcie_resume_noirq(struct device *dev) { - struct dw_pcie *pci = dev_get_drvdata(dev); + struct ultrarisc_pcie *ultra = dev_get_drvdata(dev); + struct dw_pcie *pci = &ultra->pci; + int ret; - return dw_pcie_resume_noirq(pci); + if (pci->suspended) { + ret = ultrarisc_pcie_enable_clks(ultra); + if (ret) + return ret; + + ret = dw_pcie_resume_noirq(pci); + if (ret) { + ultrarisc_pcie_disable_clks(ultra); + return ret; + } + } + + return 0; } static const struct dev_pm_ops ultrarisc_pcie_pm_ops = { diff --git a/drivers/pci/controller/pci-host-generic.c b/drivers/pci/controller/pci-host-generic.c index c1bc0d34348f4..9e85c6e9b4254 100644 --- a/drivers/pci/controller/pci-host-generic.c +++ b/drivers/pci/controller/pci-host-generic.c @@ -16,15 +16,6 @@ #include "pci-host-common.h" -static const struct pci_ecam_ops gen_pci_cfg_cam_bus_ops = { - .bus_shift = 16, - .pci_ops = { - .map_bus = pci_ecam_map_bus, - .read = pci_generic_config_read, - .write = pci_generic_config_write, - } -}; - static bool pci_dw_valid_device(struct pci_bus *bus, unsigned int devfn) { struct pci_config_window *cfg = bus->sysdata; @@ -60,7 +51,7 @@ static const struct pci_ecam_ops pci_dw_ecam_bus_ops = { static const struct of_device_id gen_pci_of_match[] = { { .compatible = "pci-host-cam-generic", - .data = &gen_pci_cfg_cam_bus_ops }, + .data = &pci_generic_cam_ops }, { .compatible = "pci-host-ecam-generic", .data = &pci_generic_ecam_ops }, diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index cfc8fa403dad6..89816a2bd7cd3 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -2119,6 +2119,7 @@ static bool hv_pcie_init_dev_msi_info(struct device *dev, struct irq_domain *dom info->ops->msi_prepare = hv_msi_prepare; chip->irq_set_affinity = irq_chip_set_affinity_parent; + chip->irq_retrigger = irq_chip_retrigger_hierarchy; if (IS_ENABLED(CONFIG_X86)) chip->flags |= IRQCHIP_MOVE_DEFERRED; diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c index b95afa35201d0..83c9a2930eeca 100644 --- a/drivers/pci/controller/pci-xgene.c +++ b/drivers/pci/controller/pci-xgene.c @@ -58,7 +58,6 @@ #define XGENE_PCIE_IP_VER_2 2 struct xgene_pcie { - struct device_node *node; struct device *dev; struct clk *clk; void __iomem *csr_base; @@ -526,7 +525,7 @@ static void xgene_pcie_setup_ib_reg(struct xgene_pcie *port, static int xgene_pcie_parse_map_dma_ranges(struct xgene_pcie *port) { - struct device_node *np = port->node; + struct device_node *np = port->dev->of_node; struct of_pci_range range; struct of_pci_range_parser parser; struct device *dev = port->dev; @@ -612,7 +611,6 @@ static bool xgene_check_pcie_msi_ready(void) static int xgene_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *dn = dev->of_node; struct xgene_pcie *port; struct pci_host_bridge *bridge; int ret; @@ -627,7 +625,6 @@ static int xgene_pcie_probe(struct platform_device *pdev) port = pci_host_bridge_priv(bridge); - port->node = of_node_get(dn); port->dev = dev; port->version = XGENE_PCIE_IP_VER_1; diff --git a/drivers/pci/controller/plda/pcie-plda-host.c b/drivers/pci/controller/plda/pcie-plda-host.c index f9a34f323ad8c..fd1a11b4c8eb8 100644 --- a/drivers/pci/controller/plda/pcie-plda-host.c +++ b/drivers/pci/controller/plda/pcie-plda-host.c @@ -419,6 +419,8 @@ static int plda_pcie_init_irq_domains(struct plda_pcie_rp *port) return plda_allocate_msi_domains(port); } +static void plda_pcie_irq_domain_deinit(struct plda_pcie_rp *pcie); + int plda_init_interrupts(struct platform_device *pdev, struct plda_pcie_rp *port, const struct plda_event *event) @@ -440,14 +442,17 @@ int plda_init_interrupts(struct platform_device *pdev, } port->irq = platform_get_irq(pdev, 0); - if (port->irq < 0) - return -ENODEV; + if (port->irq < 0) { + ret = -ENODEV; + goto err_irq_domain_deinit; + } for_each_set_bit(i, &port->events_bitmap, port->num_events) { event_irq = irq_create_mapping(port->event_domain, i); if (!event_irq) { dev_err(dev, "failed to map hwirq %d\n", i); - return -ENXIO; + ret = -ENXIO; + goto err_irq_domain_deinit; } if (event->request_event_irq) @@ -459,7 +464,7 @@ int plda_init_interrupts(struct platform_device *pdev, if (ret) { dev_err(dev, "failed to request IRQ %d\n", event_irq); - return ret; + goto err_irq_domain_deinit; } } @@ -467,7 +472,8 @@ int plda_init_interrupts(struct platform_device *pdev, event->intx_event); if (!port->intx_irq) { dev_err(dev, "failed to map INTx interrupt\n"); - return -ENXIO; + ret = -ENXIO; + goto err_irq_domain_deinit; } /* Plug the INTx chained handler */ @@ -475,8 +481,11 @@ int plda_init_interrupts(struct platform_device *pdev, port->msi_irq = irq_create_mapping(port->event_domain, event->msi_event); - if (!port->msi_irq) - return -ENXIO; + if (!port->msi_irq) { + dev_err(dev, "failed to map MSI interrupt\n"); + ret = -ENXIO; + goto err_irq_domain_deinit; + } /* Plug the MSI chained handler */ irq_set_chained_handler_and_data(port->msi_irq, plda_handle_msi, port); @@ -485,6 +494,11 @@ int plda_init_interrupts(struct platform_device *pdev, irq_set_chained_handler_and_data(port->irq, plda_handle_event, port); return 0; + +err_irq_domain_deinit: + plda_pcie_irq_domain_deinit(port); + + return ret; } EXPORT_SYMBOL_GPL(plda_init_interrupts); @@ -559,9 +573,27 @@ EXPORT_SYMBOL_GPL(plda_pcie_setup_iomems); static void plda_pcie_irq_domain_deinit(struct plda_pcie_rp *pcie) { - irq_set_chained_handler_and_data(pcie->irq, NULL, NULL); - irq_set_chained_handler_and_data(pcie->msi_irq, NULL, NULL); - irq_set_chained_handler_and_data(pcie->intx_irq, NULL, NULL); + u32 i, event_irq; + + if (pcie->irq > 0) + irq_set_chained_handler_and_data(pcie->irq, NULL, NULL); + if (pcie->msi_irq > 0) + irq_set_chained_handler_and_data(pcie->msi_irq, NULL, NULL); + if (pcie->intx_irq > 0) + irq_set_chained_handler_and_data(pcie->intx_irq, NULL, NULL); + + for_each_set_bit(i, &pcie->events_bitmap, pcie->num_events) { + event_irq = irq_find_mapping(pcie->event_domain, i); + if (event_irq) { + devm_free_irq(pcie->dev, event_irq, pcie); + irq_dispose_mapping(event_irq); + } + } + + if (pcie->intx_irq) + irq_dispose_mapping(pcie->intx_irq); + if (pcie->msi_irq) + irq_dispose_mapping(pcie->msi_irq); irq_domain_remove(pcie->msi.dev_domain); diff --git a/drivers/pci/controller/plda/pcie-starfive.c b/drivers/pci/controller/plda/pcie-starfive.c index 298036c3e7f94..fab44054a5de2 100644 --- a/drivers/pci/controller/plda/pcie-starfive.c +++ b/drivers/pci/controller/plda/pcie-starfive.c @@ -304,12 +304,14 @@ static int starfive_pcie_host_init(struct plda_pcie_rp *plda) ret = starfive_pcie_clk_rst_init(pcie); if (ret) - return ret; + goto err_disable_phy; if (pcie->vpcie3v3) { ret = regulator_enable(pcie->vpcie3v3); - if (ret) + if (ret) { dev_err_probe(dev, ret, "failed to enable vpcie3v3 regulator\n"); + goto err_clk_rst; + } } if (pcie->reset_gpio) @@ -379,6 +381,13 @@ static int starfive_pcie_host_init(struct plda_pcie_rp *plda) dev_info(dev, "port link down\n"); return 0; + +err_clk_rst: + starfive_pcie_clk_rst_deinit(pcie); +err_disable_phy: + starfive_pcie_disable_phy(pcie); + + return ret; } static const struct plda_pcie_host_ops sf_host_ops = { @@ -410,7 +419,11 @@ static int starfive_pcie_probe(struct platform_device *pdev) return ret; pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret < 0) { + pm_runtime_disable(&pdev->dev); + return dev_err_probe(dev, ret, "failed to resume device\n"); + } plda->host_ops = &sf_host_ops; plda->num_events = PLDA_MAX_EVENT_NUM; @@ -436,9 +449,9 @@ static void starfive_pcie_remove(struct platform_device *pdev) { struct starfive_jh7110_pcie *pcie = platform_get_drvdata(pdev); - pm_runtime_put(&pdev->dev); - pm_runtime_disable(&pdev->dev); plda_pcie_host_deinit(&pcie->plda); + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); platform_set_drvdata(pdev, NULL); } diff --git a/drivers/pci/ecam.c b/drivers/pci/ecam.c index 119de32ff07bd..a9b3bce2492f0 100644 --- a/drivers/pci/ecam.c +++ b/drivers/pci/ecam.c @@ -208,6 +208,19 @@ const struct pci_ecam_ops pci_generic_ecam_ops = { }; EXPORT_SYMBOL_GPL(pci_generic_ecam_ops); +/* CAM ops */ +const struct pci_ecam_ops pci_generic_cam_ops = { + .bus_shift = 16, + .pci_ops = { + .add_bus = pci_ecam_add_bus, + .remove_bus = pci_ecam_remove_bus, + .map_bus = pci_ecam_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; +EXPORT_SYMBOL_GPL(pci_generic_cam_ops); + #if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) /* ECAM ops for 32-bit access only (non-compliant) */ const struct pci_ecam_ops pci_32b_ops = { diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c index ff92a5c301b81..37299d59f906f 100644 --- a/drivers/pci/hotplug/pnv_php.c +++ b/drivers/pci/hotplug/pnv_php.c @@ -808,7 +808,7 @@ static struct pnv_php_slot *pnv_php_alloc_slot(struct device_node *dn) if (dn->child && PCI_DN(dn->child)) php_slot->slot_no = PCI_SLOT(PCI_DN(dn->child)->devfn); else - php_slot->slot_no = -1; /* Placeholder slot */ + php_slot->slot_no = PCI_SLOT_PLACEHOLDER; /* Placeholder slot */ kref_init(&php_slot->kref); php_slot->state = PNV_PHP_STATE_INITIALIZED; diff --git a/drivers/pci/hotplug/rpaphp_slot.c b/drivers/pci/hotplug/rpaphp_slot.c index 67362e5b99719..92eabf5f61b9c 100644 --- a/drivers/pci/hotplug/rpaphp_slot.c +++ b/drivers/pci/hotplug/rpaphp_slot.c @@ -84,7 +84,7 @@ int rpaphp_register_slot(struct slot *slot) struct hotplug_slot *php_slot = &slot->hotplug_slot; u32 my_index; int retval; - int slotno = -1; + int slotno = PCI_SLOT_PLACEHOLDER; dbg("%s registering slot:path[%pOF] index[%x], name[%s] pdomain[%x] type[%d]\n", __func__, slot->dn, slot->index, slot->name, diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 209373c92e9e1..80a9db417dc87 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -870,6 +870,7 @@ void __pci_restore_msix_state(struct pci_dev *dev) { struct msi_desc *entry; bool write_msg; + u16 cmd; if (!dev->msix_enabled) return; @@ -879,6 +880,14 @@ void __pci_restore_msix_state(struct pci_dev *dev) pci_msix_clear_and_set_ctrl(dev, 0, PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); + /* + * The restored device state may not have Memory Space enabled. + * Since the MSI-X Table and PBA are in Memory Space, enable it + * while restoring them. + */ + pci_read_config_word(dev, PCI_COMMAND, &cmd); + pci_write_config_word(dev, PCI_COMMAND, cmd | PCI_COMMAND_MEMORY); + write_msg = arch_restore_msi_irqs(dev); scoped_guard (msi_descs_lock, &dev->dev) { @@ -889,6 +898,7 @@ void __pci_restore_msix_state(struct pci_dev *dev) } } + pci_write_config_word(dev, PCI_COMMAND, cmd); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); } diff --git a/drivers/pci/of_property.c b/drivers/pci/of_property.c index 75a358f73e694..1500740cc55d2 100644 --- a/drivers/pci/of_property.c +++ b/drivers/pci/of_property.c @@ -95,9 +95,13 @@ static int of_pci_prop_bus_range(struct pci_dev *pdev, struct of_changeset *ocs, struct device_node *np) { - u32 bus_range[] = { pdev->subordinate->busn_res.start, - pdev->subordinate->busn_res.end }; + u32 bus_range[2]; + if (!pdev->subordinate) + return 0; + + bus_range[0] = pdev->subordinate->busn_res.start; + bus_range[1] = pdev->subordinate->busn_res.end; return of_changeset_add_prop_u32_array(ocs, np, "bus-range", bus_range, ARRAY_SIZE(bus_range)); } @@ -220,6 +224,9 @@ static int of_pci_prop_intr_map(struct pci_dev *pdev, struct of_changeset *ocs, int ret; u8 pin; + if (!pdev->subordinate) + return 0; + pnode = pci_device_to_OF_node(pdev->bus->self); if (!pnode) pnode = pci_bus_to_OF_node(pdev->bus); diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index f36778e62ac1a..e16aa59dd7ac8 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -29,6 +29,47 @@ struct pci_dynid { struct pci_device_id id; }; +/** + * do_pci_add_dynid - Add a new PCI device ID to this driver and re-probe + * @drv: target PCI driver + * @id: ID to be added + * @check_dup: whether to check if matching ID is already present + * + * Add a new dynamic PCI device ID to this driver and causes the driver to + * probe for all devices again. @drv must have been registered prior to calling + * this function. + * + * Context: Does GFP_KERNEL allocation. + * + * Return: 0 on success, -errno on failure. + */ +static int do_pci_add_dynid(struct pci_driver *drv, + const struct pci_device_id *id, + bool check_dup) +{ + struct pci_dynid *dynid, *existing_dynid; + + dynid = kzalloc_obj(*dynid); + if (!dynid) + return -ENOMEM; + + dynid->id = *id; + + scoped_guard(spinlock, &drv->dynids.lock) { + if (check_dup) { + list_for_each_entry(existing_dynid, &drv->dynids.list, node) { + if (pci_match_one_id(&existing_dynid->id, id)) { + kfree(dynid); + return -EEXIST; + } + } + } + list_add_tail(&dynid->node, &drv->dynids.list); + } + + return driver_attach(&drv->driver); +} + /** * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices * @drv: target pci driver @@ -56,25 +97,17 @@ int pci_add_dynid(struct pci_driver *drv, unsigned int class, unsigned int class_mask, unsigned long driver_data) { - struct pci_dynid *dynid; - - dynid = kzalloc_obj(*dynid); - if (!dynid) - return -ENOMEM; - - dynid->id.vendor = vendor; - dynid->id.device = device; - dynid->id.subvendor = subvendor; - dynid->id.subdevice = subdevice; - dynid->id.class = class; - dynid->id.class_mask = class_mask; - dynid->id.driver_data = driver_data; - - spin_lock(&drv->dynids.lock); - list_add_tail(&dynid->node, &drv->dynids.list); - spin_unlock(&drv->dynids.lock); - - return driver_attach(&drv->driver); + struct pci_device_id id = { + .vendor = vendor, + .device = device, + .subvendor = subvendor, + .subdevice = subdevice, + .class = class, + .class_mask = class_mask, + .driver_data = driver_data, + }; + + return do_pci_add_dynid(drv, &id, false); } EXPORT_SYMBOL_GPL(pci_add_dynid); @@ -90,6 +123,31 @@ static void pci_free_dynids(struct pci_driver *drv) spin_unlock(&drv->dynids.lock); } +/** + * do_pci_match_id - See if a PCI ID matches a given pci_id table + * @ids: array of PCI device ID structures to search in + * @dev_id: the actual PCI device ID structure to match against. + * @include_override_only: also match against device ID entries marked as + * override only. + * + * Return: the matching pci_device_id structure or %NULL if there is no match. + */ +static const struct pci_device_id * +do_pci_match_id(const struct pci_device_id *ids, + const struct pci_device_id *dev_id, + bool include_override_only) +{ + if (ids) { + while (ids->vendor || ids->subvendor || ids->class_mask) { + if ((!ids->override_only || include_override_only) && + pci_match_one_id(ids, dev_id)) + return ids; + ids++; + } + } + return NULL; +} + /** * pci_match_id - See if a PCI device matches a given pci_id table * @ids: array of PCI device ID structures to search in @@ -105,14 +163,9 @@ static void pci_free_dynids(struct pci_driver *drv) const struct pci_device_id *pci_match_id(const struct pci_device_id *ids, struct pci_dev *dev) { - if (ids) { - while (ids->vendor || ids->subvendor || ids->class_mask) { - if (pci_match_one_device(ids, dev)) - return ids; - ids++; - } - } - return NULL; + struct pci_device_id dev_id = pci_id_from_device(dev); + + return do_pci_match_id(ids, &dev_id, true); } EXPORT_SYMBOL(pci_match_id); @@ -127,6 +180,7 @@ static const struct pci_device_id pci_device_id_any = { * pci_match_device - See if a device matches a driver's list of IDs * @drv: the PCI driver to match against * @dev: the PCI device structure to match against + * @id_copy: place to store copy of pci_device_id for dynamic ID * * Used by a driver to check whether a PCI device is in its list of * supported devices or in the dynids list, which may have been augmented @@ -134,10 +188,11 @@ static const struct pci_device_id pci_device_id_any = { * structure or %NULL if there is no match. */ static const struct pci_device_id *pci_match_device(struct pci_driver *drv, - struct pci_dev *dev) + struct pci_dev *dev, + struct pci_device_id *id_copy) { - struct pci_dynid *dynid; - const struct pci_device_id *found_id = NULL, *ids; + const struct pci_device_id *found_id = NULL; + struct pci_device_id dev_id; int ret; /* When driver_override is set, only bind to the matching driver */ @@ -145,45 +200,29 @@ static const struct pci_device_id *pci_match_device(struct pci_driver *drv, if (ret == 0) return NULL; + dev_id = pci_id_from_device(dev); /* Look at the dynamic ids first, before the static ones */ - spin_lock(&drv->dynids.lock); - list_for_each_entry(dynid, &drv->dynids.list, node) { - if (pci_match_one_device(&dynid->id, dev)) { - found_id = &dynid->id; - break; + scoped_guard(spinlock, &drv->dynids.lock) { + struct pci_dynid *dynid; + + list_for_each_entry(dynid, &drv->dynids.list, node) { + if (pci_match_one_id(&dynid->id, &dev_id)) { + *id_copy = dynid->id; + return id_copy; + } } } - spin_unlock(&drv->dynids.lock); + found_id = do_pci_match_id(drv->id_table, &dev_id, ret > 0); if (found_id) return found_id; - for (ids = drv->id_table; (found_id = pci_match_id(ids, dev)); - ids = found_id + 1) { - /* - * The match table is split based on driver_override. - * In case override_only was set, enforce driver_override - * matching. - */ - if (found_id->override_only) { - if (ret > 0) - return found_id; - } else { - return found_id; - } - } - /* driver_override will always match, send a dummy id */ if (ret > 0) return &pci_device_id_any; return NULL; } -static void _pci_free_device(struct device *dev) -{ - kfree(to_pci_dev(dev)); -} - /** * new_id_store - sysfs frontend to pci_add_dynid() * @driver: target device driver @@ -197,38 +236,22 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, { struct pci_driver *pdrv = to_pci_driver(driver); const struct pci_device_id *ids = pdrv->id_table; - u32 vendor, device, subvendor = PCI_ANY_ID, - subdevice = PCI_ANY_ID, class = 0, class_mask = 0; - unsigned long driver_data = 0; + struct pci_device_id id = { + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID + }; int fields; int retval = 0; fields = sscanf(buf, "%x %x %x %x %x %x %lx", - &vendor, &device, &subvendor, &subdevice, - &class, &class_mask, &driver_data); + &id.vendor, &id.device, &id.subvendor, &id.subdevice, + &id.class, &id.class_mask, &id.driver_data); if (fields < 2) return -EINVAL; if (fields != 7) { - struct pci_dev *pdev = kzalloc_obj(*pdev); - if (!pdev) - return -ENOMEM; - - pdev->vendor = vendor; - pdev->device = device; - pdev->subsystem_vendor = subvendor; - pdev->subsystem_device = subdevice; - pdev->class = class; - pdev->dev.release = _pci_free_device; - - device_initialize(&pdev->dev); - if (pci_match_device(pdrv, pdev)) - retval = -EEXIST; - - put_device(&pdev->dev); - - if (retval) - return retval; + if (do_pci_match_id(pdrv->id_table, &id, false)) + return -EEXIST; } /* Only accept driver_data values that match an existing id_table @@ -236,7 +259,7 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, if (ids) { retval = -EINVAL; while (ids->vendor || ids->subvendor || ids->class_mask) { - if (driver_data == ids->driver_data) { + if (id.driver_data == ids->driver_data) { retval = 0; break; } @@ -246,8 +269,7 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, return retval; } - retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice, - class, class_mask, driver_data); + retval = do_pci_add_dynid(pdrv, &id, fields != 7); if (retval) return retval; return count; @@ -445,12 +467,13 @@ void pci_probe_flush_workqueue(void) static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev) { const struct pci_device_id *id; + struct pci_device_id id_copy; int error = 0; if (drv->probe) { error = -ENODEV; - id = pci_match_device(drv, pci_dev); + id = pci_match_device(drv, pci_dev, &id_copy); if (id) error = pci_call_probe(drv, pci_dev, id); } @@ -1538,12 +1561,13 @@ static int pci_bus_match(struct device *dev, const struct device_driver *drv) struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *pci_drv; const struct pci_device_id *found_id; + struct pci_device_id id_copy; if (pci_dev_binding_disallowed(pci_dev)) return 0; pci_drv = (struct pci_driver *)to_pci_driver(drv); - found_id = pci_match_device(pci_drv, pci_dev); + found_id = pci_match_device(pci_drv, pci_dev, &id_copy); if (found_id) return 1; diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 5ec0b245a69bd..6b016792e0ad8 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -718,7 +718,7 @@ static ssize_t pci_read_config(struct file *filp, struct kobject *kobj, else if (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS) size = 128; - if (off > size) + if (off >= size) return 0; if (off + count > size) { size -= off; @@ -799,7 +799,7 @@ static ssize_t pci_write_config(struct file *filp, struct kobject *kobj, add_taint(TAINT_USER, LOCKDEP_STILL_OK); } - if (off > dev->cfg_size) + if (off >= dev->cfg_size) return 0; if (off + count > dev->cfg_size) { size = dev->cfg_size - off; @@ -888,12 +888,30 @@ static ssize_t pci_read_legacy_io(struct file *filp, struct kobject *kobj, char *buf, loff_t off, size_t count) { struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj)); + u32 val = 0; + int ret; /* Only support 1, 2 or 4 byte accesses */ if (count != 1 && count != 2 && count != 4) return -EINVAL; - return pci_legacy_read(bus, off, (u32 *)buf, count); + ret = pci_legacy_read(bus, off, &val, count); + if (ret < 0) + return ret; + + switch (count) { + case 1: + buf[0] = *(u8 *)&val; + break; + case 2: + put_unaligned_le16(*(u16 *)&val, buf); + break; + case 4: + put_unaligned_le32(val, buf); + break; + } + + return ret; } /** @@ -913,12 +931,24 @@ static ssize_t pci_write_legacy_io(struct file *filp, struct kobject *kobj, char *buf, loff_t off, size_t count) { struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj)); + u32 val; - /* Only support 1, 2 or 4 byte accesses */ - if (count != 1 && count != 2 && count != 4) + /* Only support 1, 2 or 4 byte accesses. */ + switch (count) { + case 1: + val = *(u8 *)buf; + break; + case 2: + val = get_unaligned_le16(buf); + break; + case 4: + val = get_unaligned_le32(buf); + break; + default: return -EINVAL; + } - return pci_legacy_write(bus, off, *(u32 *)buf, count); + return pci_legacy_write(bus, off, val, count); } /** @@ -937,6 +967,11 @@ static int pci_mmap_legacy_mem(struct file *filp, struct kobject *kobj, struct vm_area_struct *vma) { struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj)); + int ret; + + ret = security_locked_down(LOCKDOWN_PCI_ACCESS); + if (ret) + return ret; return pci_mmap_legacy_page_range(bus, vma, pci_mmap_mem); } @@ -957,6 +992,11 @@ static int pci_mmap_legacy_io(struct file *filp, struct kobject *kobj, struct vm_area_struct *vma) { struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj)); + int ret; + + ret = security_locked_down(LOCKDOWN_PCI_ACCESS); + if (ret) + return ret; return pci_mmap_legacy_page_range(bus, vma, pci_mmap_io); } @@ -973,6 +1013,11 @@ static inline umode_t __pci_legacy_is_visible(struct kobject *kobj, bool sparse) { struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj)); + int ret; + + ret = security_locked_down(LOCKDOWN_PCI_ACCESS); + if (ret) + return ret; if (pci_legacy_has_sparse(bus, type) != sparse) return 0; @@ -1244,7 +1289,16 @@ static loff_t pci_llseek_resource(struct file *filep, .llseek = pci_llseek_resource, \ .mmap = pci_mmap_resource_uc, #else -# define __PCI_RESOURCE_IO_MMAP_ATTRS +static int pci_mmap_resource_io_unsupported(struct file *filp, + struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + return -EINVAL; +} + +# define __PCI_RESOURCE_IO_MMAP_ATTRS \ + .mmap = pci_mmap_resource_io_unsupported, #endif #define pci_dev_resource_io_attr(_bar) \ diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 77b17b13ee615..350bae907ebf8 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4897,8 +4897,9 @@ static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe) static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe) { - if (dev->multifunction || dev->subordinate || !dev->slot || - dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) + if (dev->subordinate || !dev->slot || + dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || + (dev->multifunction && !dev->slot->per_func_slot)) return -ENOTTY; return pci_reset_hotplug_slot(dev->slot->hotplug, probe); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 4469e1a77f3c1..62c1b324a9bd0 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -442,21 +442,40 @@ static inline int pci_setup_cardbus(char *str) { return -ENOENT; } #endif /* CONFIG_CARDBUS */ /** - * pci_match_one_device - Tell if a PCI device structure has a matching - * PCI device id structure - * @id: single PCI device id structure to match - * @dev: the PCI device structure to match against + * pci_id_from_device - Obtain a pci_device_id from a PCI device + * @dev: the PCI device * - * Returns the matching pci_device_id structure or %NULL if there is no match. + * Return: a pci_device_id filled. */ -static inline const struct pci_device_id * -pci_match_one_device(const struct pci_device_id *id, const struct pci_dev *dev) +static inline struct pci_device_id pci_id_from_device(const struct pci_dev *dev) { - if ((id->vendor == PCI_ANY_ID || id->vendor == dev->vendor) && - (id->device == PCI_ANY_ID || id->device == dev->device) && - (id->subvendor == PCI_ANY_ID || id->subvendor == dev->subsystem_vendor) && - (id->subdevice == PCI_ANY_ID || id->subdevice == dev->subsystem_device) && - !((id->class ^ dev->class) & id->class_mask)) + return (struct pci_device_id) { + .vendor = dev->vendor, + .device = dev->device, + .subvendor = dev->subsystem_vendor, + .subdevice = dev->subsystem_device, + .class = dev->class, + }; +} + +/** + * pci_match_one_id - Tell if a PCI device ID matches a needle PCI device ID + * @id: single PCI device id structure to match against (needle) + * @dev_id: the actual ID from the PCI device + * + * ID can be retrieved from device using pci_id_from_device(). + * + * Return: the matching pci_device_id structure or %NULL if there is no match. + */ +static inline const struct pci_device_id * +pci_match_one_id(const struct pci_device_id *id, + const struct pci_device_id *dev_id) +{ + if ((id->vendor == PCI_ANY_ID || id->vendor == dev_id->vendor) && + (id->device == PCI_ANY_ID || id->device == dev_id->device) && + (id->subvendor == PCI_ANY_ID || id->subvendor == dev_id->subvendor) && + (id->subdevice == PCI_ANY_ID || id->subdevice == dev_id->subdevice) && + !((id->class ^ dev_id->class) & id->class_mask)) return id; return NULL; } diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index c4fd9c0b2a548..fbb764ffb916d 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -428,23 +428,32 @@ void pci_aer_exit(struct pci_dev *dev) #define AER_AGENT_REQUESTER 1 #define AER_AGENT_COMPLETER 2 #define AER_AGENT_TRANSMITTER 3 +#define AER_AGENT_COMPONENT 4 #define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \ - 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP)) + 0 : PCI_ERR_UNC_COMP_TIME) #define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \ 0 : PCI_ERR_UNC_COMP_ABORT) #define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \ - (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0) + (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : \ + (PCI_ERR_UNC_POISON_BLK|PCI_ERR_UNC_ATOMEG| \ + PCI_ERR_UNC_DMWR_BLK|PCI_ERR_UNC_XLAT_BLK| \ + PCI_ERR_UNC_TLPPRE)) +#define AER_AGENT_COMPONENT_MASK(t) ((t == AER_CORRECTABLE) ? \ + (PCI_ERR_COR_INTERNAL|PCI_ERR_COR_LOG_OVER) : \ + (PCI_ERR_UNC_INTN|PCI_ERR_UNC_SURPDN)) #define AER_GET_AGENT(t, e) \ ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \ (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \ (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \ + (e & AER_AGENT_COMPONENT_MASK(t)) ? AER_AGENT_COMPONENT : \ AER_AGENT_RECEIVER) #define AER_PHYSICAL_LAYER_ERROR 0 #define AER_DATA_LINK_LAYER_ERROR 1 #define AER_TRANSACTION_LAYER_ERROR 2 +#define AER_GENERAL_ERROR 3 #define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ PCI_ERR_COR_RCVR : 0) @@ -452,11 +461,14 @@ void pci_aer_exit(struct pci_dev *dev) (PCI_ERR_COR_BAD_TLP| \ PCI_ERR_COR_BAD_DLLP| \ PCI_ERR_COR_REP_ROLL| \ - PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP) + PCI_ERR_COR_REP_TIMER) : (PCI_ERR_UNC_DLP|PCI_ERR_UNC_SURPDN)) +#define AER_GENERAL_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ + (PCI_ERR_COR_INTERNAL|PCI_ERR_COR_LOG_OVER) : PCI_ERR_UNC_INTN) #define AER_GET_LAYER_ERROR(t, e) \ ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \ (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \ + (e & AER_GENERAL_ERROR_MASK(t)) ? AER_GENERAL_ERROR : \ AER_TRANSACTION_LAYER_ERROR) /* @@ -471,7 +483,8 @@ static const char * const aer_error_severity_string[] = { static const char *aer_error_layer[] = { "Physical Layer", "Data Link Layer", - "Transaction Layer" + "Transaction Layer", + "General", }; static const char *aer_correctable_error_string[] = { @@ -548,7 +561,8 @@ static const char *aer_agent_string[] = { "Receiver ID", "Requester ID", "Completer ID", - "Transmitter ID" + "Transmitter ID", + "Component ID", }; #define aer_stats_dev_attr(name, stats_array, strings_array, \ @@ -942,7 +956,8 @@ void pci_print_aer(struct pci_dev *dev, int aer_severity, status = aer->uncor_status; mask = aer->uncor_mask; info.level = KERN_ERR; - tlp_header_valid = tlp_header_logged(status, aer->cap_control); + tlp_header_valid = tlp_header_logged(status & ~mask, + aer->cap_control); } info.status = status; @@ -1336,7 +1351,7 @@ int aer_get_device_error_info(struct aer_err_info *info, int i) pci_read_config_dword(dev, aer + PCI_ERR_CAP, &aercc); info->first_error = PCI_ERR_CAP_FEP(aercc); - if (tlp_header_logged(info->status, aercc)) { + if (tlp_header_logged(info->status & ~info->mask, aercc)) { info->tlp_header_valid = 1; pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG, aer + PCI_ERR_PREFIX_LOG, diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 172783e7f519a..9b3548544a85e 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -861,6 +861,7 @@ static void pcie_aspm_override_default_link_state(struct pcie_link_state *link) static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) { struct pci_dev *child = link->downstream, *parent = link->pdev; + struct pci_dev *fn; u16 parent_lnkctl, child_lnkctl; struct pci_bus *linkbus = parent->subordinate; @@ -894,10 +895,11 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) /* Disable L0s/L1 before updating L1SS config */ if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, child_lnkctl) || FIELD_GET(PCI_EXP_LNKCTL_ASPMC, parent_lnkctl)) { - pcie_capability_write_word(child, PCI_EXP_LNKCTL, - child_lnkctl & ~PCI_EXP_LNKCTL_ASPMC); - pcie_capability_write_word(parent, PCI_EXP_LNKCTL, - parent_lnkctl & ~PCI_EXP_LNKCTL_ASPMC); + list_for_each_entry(fn, &linkbus->devices, bus_list) + pcie_capability_clear_and_set_word(fn, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, 0); + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, 0); } /* @@ -927,8 +929,13 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) /* Restore L0s/L1 if they were enabled */ if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, child_lnkctl) || FIELD_GET(PCI_EXP_LNKCTL_ASPMC, parent_lnkctl)) { - pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_lnkctl); - pcie_capability_write_word(child, PCI_EXP_LNKCTL, child_lnkctl); + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, + parent_lnkctl & PCI_EXP_LNKCTL_ASPMC); + list_for_each_entry(fn, &linkbus->devices, bus_list) + pcie_capability_clear_and_set_word(fn, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, + child_lnkctl & PCI_EXP_LNKCTL_ASPMC); } /* Save default state */ diff --git a/drivers/pci/proc.c b/drivers/pci/proc.c index 71ad289fcb8e3..f5fd860a0540b 100644 --- a/drivers/pci/proc.c +++ b/drivers/pci/proc.c @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include "pci.h" @@ -39,13 +41,16 @@ static ssize_t proc_bus_pci_read(struct file *file, char __user *buf, * undefined locations (think of Intel PIIX4 as a typical example). */ - if (capable(CAP_SYS_ADMIN)) + if (file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN)) size = dev->cfg_size; else if (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS) size = 128; else size = 64; + if (!nbytes) + return 0; + if (pos >= size) return 0; if (nbytes >= size) @@ -122,6 +127,15 @@ static ssize_t proc_bus_pci_write(struct file *file, const char __user *buf, if (ret) return ret; + if (!nbytes) + return 0; + + if (resource_is_exclusive(&dev->driver_exclusive_resource, pos, nbytes)) { + pci_warn_once(dev, "%s: Unexpected write to kernel-exclusive config offset %x", + current->comm, pos); + add_taint(TAINT_USER, LOCKDEP_STILL_OK); + } + if (pos >= size) return 0; if (nbytes >= size) diff --git a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c index 1555e8a9b3ca1..b0efe7560c914 100644 --- a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c +++ b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c @@ -240,12 +240,18 @@ static int tc9563_pwrctrl_disable_port(struct tc9563_pwrctrl *tc9563, if (!cfg->disable_port) return 0; - if (port == TC9563_DSP1) { + switch (port) { + case TC9563_DSP1: seq = dsp1_pwroff_seq; len = ARRAY_SIZE(dsp1_pwroff_seq); - } else { + break; + case TC9563_DSP2: seq = dsp2_pwroff_seq; len = ARRAY_SIZE(dsp2_pwroff_seq); + break; + default: + /* Only external downstream ports DSP1/DSP2 can be powered off */ + return 0; } ret = tc9563_pwrctrl_i2c_bulk_write(tc9563->client, seq, len); @@ -595,12 +601,18 @@ static int tc9563_pwrctrl_probe(struct platform_device *pdev) ret = tc9563_pwrctrl_parse_device_dt(tc9563, child, port); if (ret) break; - /* Embedded ethernet device are under DSP3 */ + + /* + * The integrated Ethernet MAC Endpoint under DSP3 is a single + * device whose functions share the same config registers. + */ if (port == TC9563_DSP3) { - for_each_child_of_node_scoped(child, child1) { - port++; + struct device_node *eth __free(device_node) = + of_get_next_available_child(child, NULL); + + if (eth) { ret = tc9563_pwrctrl_parse_device_dt(tc9563, - child1, port); + eth, TC9563_ETHERNET); if (ret) break; } diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index b09f27f7846fc..0500d5fd4f112 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -2507,6 +2507,9 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x10f1, quirk_disable_aspm_l0s); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x10f4, quirk_disable_aspm_l0s); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1508, quirk_disable_aspm_l0s); +/* Realtek RTS525A generates a Replay Timer Timeout storm when L0s is enabled. */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_REALTEK, 0x525a, quirk_disable_aspm_l0s); + static void quirk_disable_aspm_l0s_l1(struct pci_dev *dev) { pcie_aspm_remove_cap(dev, @@ -5361,7 +5364,7 @@ static void pci_quirk_enable_intel_rp_mpc_acs(struct pci_dev *dev) if (!(mpc & INTEL_MPC_REG_IRBNCE)) { pci_info(dev, "Enabling MPC IRBNCE\n"); mpc |= INTEL_MPC_REG_IRBNCE; - pci_write_config_word(dev, INTEL_MPC_REG, mpc); + pci_write_config_dword(dev, INTEL_MPC_REG, mpc); } } @@ -6263,6 +6266,10 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0xb404, pci_fixup_pericom_acs_store_forward); DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0xb404, pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0x2608, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0x2608, + pci_fixup_pericom_acs_store_forward); static void nvidia_ion_ahci_fixup(struct pci_dev *pdev) { diff --git a/drivers/pci/search.c b/drivers/pci/search.c index e3d3177fce549..34f8de551d587 100644 --- a/drivers/pci/search.c +++ b/drivers/pci/search.c @@ -245,8 +245,10 @@ static int match_pci_dev_by_id(struct device *dev, const void *data) { struct pci_dev *pdev = to_pci_dev(dev); const struct pci_device_id *id = data; + struct pci_device_id dev_id; - if (pci_match_one_device(id, pdev)) + dev_id = pci_id_from_device(pdev); + if (pci_match_one_id(id, &dev_id)) return 1; return 0; } @@ -416,9 +418,9 @@ EXPORT_SYMBOL(pci_get_class); * @class: search for a PCI device with this base class code * @from: Previous PCI device found in search, or %NULL for new search. * - * Iterates through the list of known PCI devices. If a PCI device is found + * Iterate through the list of known PCI devices. If a PCI device is found * with a matching base class code, the reference count to the device is - * incremented. See pci_match_one_device() to figure out how does this works. + * incremented. See pci_match_one_id() to figure out how this works. * A new search is initiated by passing %NULL as the @from argument. * Otherwise if @from is not %NULL, searches continue from next device on the * global list. The reference count for @from is always decremented if it is diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index c0a949f2c9956..9e13489cac716 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -2380,6 +2380,7 @@ int pci_do_resource_release_and_resize(struct pci_dev *pdev, int resno, int size struct resource *res = pci_resource_n(pdev, resno); struct pci_dev_resource *dev_res; struct pci_bus *bus = pdev->bus; + struct pci_dev *bridge = pci_upstream_bridge(pdev); struct resource *b_win, *r; LIST_HEAD(saved); unsigned int i; @@ -2397,6 +2398,8 @@ int pci_do_resource_release_and_resize(struct pci_dev *pdev, int resno, int size if (ret) return ret; + down_read(&pci_bus_sem); + pci_dev_for_each_resource(pdev, r, i) { if (i >= PCI_BRIDGE_RESOURCES) break; @@ -2415,13 +2418,21 @@ int pci_do_resource_release_and_resize(struct pci_dev *pdev, int resno, int size pci_resize_resource_set_size(pdev, resno, size); - if (!bus->self) - goto out; + if (bridge) { + ret = pbus_reassign_bridge_resources(bus, res, &saved); + if (ret) + goto restore; + } else { + /* No bridge window to adjust; let the core reassign the bus. */ + pci_bus_assign_resources(bus); - down_read(&pci_bus_sem); - ret = pbus_reassign_bridge_resources(bus, res, &saved); - if (ret) - goto restore; + list_for_each_entry(dev_res, &saved, list) { + if (!resource_assigned(dev_res->res)) { + ret = -ENOSPC; + goto restore; + } + } + } out: up_read(&pci_bus_sem); diff --git a/drivers/pci/slot.c b/drivers/pci/slot.c index 6d5cd37bfb1e7..897223f01f6ac 100644 --- a/drivers/pci/slot.c +++ b/drivers/pci/slot.c @@ -37,7 +37,7 @@ static const struct sysfs_ops pci_slot_sysfs_ops = { static ssize_t address_read_file(struct pci_slot *slot, char *buf) { - if (slot->number == 0xff) + if (slot->number == PCI_SLOT_PLACEHOLDER) return sysfs_emit(buf, "%04x:%02x\n", pci_domain_nr(slot->bus), slot->bus->number); @@ -72,6 +72,23 @@ static ssize_t cur_speed_read_file(struct pci_slot *slot, char *buf) return bus_speed_read(slot->bus->cur_bus_speed, buf); } +static bool pci_dev_matches_slot(struct pci_dev *dev, struct pci_slot *slot) +{ + if (slot->per_func_slot) + return dev->devfn == slot->number; + + return slot->number == PCI_SLOT_ALL_DEVICES || + PCI_SLOT(dev->devfn) == slot->number; +} + +static bool pci_slot_enabled_per_func(void) +{ + if (IS_ENABLED(CONFIG_S390)) + return true; + + return false; +} + static void pci_slot_release(struct kobject *kobj) { struct pci_dev *dev; @@ -82,8 +99,7 @@ static void pci_slot_release(struct kobject *kobj) down_read(&pci_bus_sem); list_for_each_entry(dev, &slot->bus->devices, bus_list) - if (slot->number == PCI_SLOT_ALL_DEVICES || - PCI_SLOT(dev->devfn) == slot->number) + if (pci_dev_matches_slot(dev, slot)) dev->slot = NULL; up_read(&pci_bus_sem); @@ -187,8 +203,7 @@ void pci_dev_assign_slot(struct pci_dev *dev) mutex_lock(&pci_slot_mutex); list_for_each_entry(slot, &dev->bus->slots, list) - if (slot->number == PCI_SLOT_ALL_DEVICES || - PCI_SLOT(dev->devfn) == slot->number) + if (pci_dev_matches_slot(dev, slot)) dev->slot = slot; mutex_unlock(&pci_slot_mutex); } @@ -210,7 +225,7 @@ static struct pci_slot *get_slot(struct pci_bus *parent, int slot_nr) /** * pci_create_slot - create or increment refcount for physical PCI slot * @parent: struct pci_bus of parent bridge - * @slot_nr: PCI_SLOT(pci_dev->devfn), -1 for placeholder, or + * @slot_nr: PCI_SLOT(pci_dev->devfn), PCI_SLOT_PLACEHOLDER for placeholder, or * PCI_SLOT_ALL_DEVICES * @name: user visible string presented in /sys/bus/pci/slots/ * @hotplug: set if caller is hotplug driver, NULL otherwise @@ -236,15 +251,16 @@ static struct pci_slot *get_slot(struct pci_bus *parent, int slot_nr) * In most cases, @pci_bus, @slot_nr will be sufficient to uniquely identify * a slot. There is one notable exception - pSeries (rpaphp), where the * @slot_nr cannot be determined until a device is actually inserted into - * the slot. In this scenario, the caller may pass -1 for @slot_nr. + * the slot. In this scenario, the caller may pass PCI_SLOT_PLACEHOLDER for @slot_nr. * * The following semantics are imposed when the caller passes @slot_nr == - * -1. First, we no longer check for an existing %struct pci_slot, as there - * may be many slots with @slot_nr of -1. The other change in semantics is - * user-visible, which is the 'address' parameter presented in sysfs will - * consist solely of a dddd:bb tuple, where dddd is the PCI domain of the - * %struct pci_bus and bb is the bus number. In other words, the devfn of - * the 'placeholder' slot will not be displayed. + * PCI_SLOT_PLACEHOLDER. First, we no longer check for an existing %struct + * pci_slot, as there may be many slots with @slot_nr of + * PCI_SLOT_PLACEHOLDER. The other change in semantics is user-visible, + * which is the 'address' parameter presented in sysfs will consist solely + * of a dddd:bb tuple, where dddd is the PCI domain of the %struct pci_bus + * and bb is the bus number. In other words, the devfn of the 'placeholder' + * slot will not be displayed. * * Bus-wide slots: * For PCIe hotplug, the physical slot encompasses the entire secondary @@ -267,7 +283,7 @@ struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr, mutex_lock(&pci_slot_mutex); - if (slot_nr == -1) + if (slot_nr == PCI_SLOT_PLACEHOLDER) goto placeholder; /* @@ -298,6 +314,9 @@ struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr, slot->bus = pci_bus_get(parent); slot->number = slot_nr; + if (pci_slot_enabled_per_func()) + slot->per_func_slot = 1; + slot->kobj.kset = pci_slots_kset; slot_name = make_slot_name(name); @@ -318,8 +337,7 @@ struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr, down_read(&pci_bus_sem); list_for_each_entry(dev, &parent->devices, bus_list) - if (slot_nr == PCI_SLOT_ALL_DEVICES || - PCI_SLOT(dev->devfn) == slot_nr) + if (pci_dev_matches_slot(dev, slot)) dev->slot = slot; up_read(&pci_bus_sem); diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 6e5cc4086a9e2..7e75903811710 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -1393,13 +1393,14 @@ static void arm_cmn_claim_wp_idx(struct arm_cmn_dtm *dtm, static u32 arm_cmn_wp_config(struct perf_event *event, int wp_idx) { + struct arm_cmn *cmn = to_cmn(event->pmu); u32 config; u32 dev = CMN_EVENT_WP_DEV_SEL(event); u32 chn = CMN_EVENT_WP_CHN_SEL(event); u32 grp = CMN_EVENT_WP_GRP(event); u32 exc = CMN_EVENT_WP_EXCLUSIVE(event); u32 combine = CMN_EVENT_WP_COMBINE(event); - bool is_cmn600 = to_cmn(event->pmu)->part == PART_CMN600; + bool is_cmn600 = cmn->part == PART_CMN600; /* CMN-600 supports only primary and secondary matching groups */ if (is_cmn600) @@ -1407,8 +1408,11 @@ static u32 arm_cmn_wp_config(struct perf_event *event, int wp_idx) config = FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL, dev) | FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_CHN_SEL, chn) | - FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_GRP, grp) | - FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL2, dev >> 1); + FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_GRP, grp); + + if (!cmn->multi_dtm) + config |= FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL2, dev >> 1); + if (exc) config |= is_cmn600 ? CMN600_WPn_CONFIG_WP_EXCLUSIVE : CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE; diff --git a/drivers/perf/arm_brbe.c b/drivers/perf/arm_brbe.c index ba554e0c846c4..254be4da8ae29 100644 --- a/drivers/perf/arm_brbe.c +++ b/drivers/perf/arm_brbe.c @@ -604,7 +604,7 @@ static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *ent return false; brbinf = bregs.brbinf; - perf_clear_branch_entry_bitfields(entry); + *entry = (struct perf_branch_entry){ }; if (brbe_record_is_complete(brbinf)) { entry->from = bregs.brbsrc; entry->to = bregs.brbtgt; diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c index 8014ff766cff5..b9a8592bf1128 100644 --- a/drivers/perf/arm_pmuv3.c +++ b/drivers/perf/arm_pmuv3.c @@ -1361,7 +1361,7 @@ static int branch_records_alloc(struct arm_pmu *armpmu) struct pmu_hw_events *events_cpu; events_cpu = per_cpu_ptr(armpmu->hw_events, cpu); - events_cpu->branch_stack = kmalloc(size, GFP_KERNEL); + events_cpu->branch_stack = kzalloc(size, GFP_KERNEL); if (!events_cpu->branch_stack) return -ENOMEM; } diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index dbd0da1116390..b64cf2313a20c 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -577,7 +577,7 @@ static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) * the page boundary following it. Keep the tail boundary if * that's lower. */ - if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) + if ((handle->wakeup - handle->head) < handle->size && head <= wakeup) limit = min(limit, round_up(wakeup, PAGE_SIZE)); if (limit > head) diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index 68a54d97d2a8a..39b46550a5109 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -635,7 +635,7 @@ static void cxl_pmu_event_start(struct perf_event *event, int flags) cfg = cxl_pmu_config2_get_hdm_decoder(event); else cfg = GENMASK(31, 0); /* No filtering if 0xFFFF_FFFF */ - writeq(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); + writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); } cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); diff --git a/drivers/perf/riscv_pmu_legacy.c b/drivers/perf/riscv_pmu_legacy.c index 4d6461d6a74f5..1b8e4789cb3c1 100644 --- a/drivers/perf/riscv_pmu_legacy.c +++ b/drivers/perf/riscv_pmu_legacy.c @@ -110,8 +110,9 @@ static void pmu_legacy_init(struct riscv_pmu *pmu) { pr_info("Legacy PMU implementation is available\n"); - pmu->cmask = BIT(RISCV_PMU_LEGACY_CYCLE) | - BIT(RISCV_PMU_LEGACY_INSTRET); + bitmap_zero(pmu->cmask, RISCV_MAX_COUNTERS); + set_bit(RISCV_PMU_LEGACY_CYCLE, pmu->cmask); + set_bit(RISCV_PMU_LEGACY_INSTRET, pmu->cmask); pmu->ctr_start = pmu_legacy_ctr_start; pmu->ctr_stop = NULL; pmu->event_map = pmu_legacy_event_map; diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index dfc886dee5ad0..2e89d2fee53a0 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -97,7 +97,7 @@ static unsigned int riscv_pmu_irq_mask; static unsigned int riscv_pmu_irq; /* Cache the available counters in a bitmask */ -static unsigned long cmask; +static DECLARE_BITMAP(cmask, RISCV_MAX_COUNTERS); static int pmu_event_find_cache(u64 config); struct sbi_pmu_event_data { @@ -359,16 +359,38 @@ static int pmu_sbi_check_event_info(void) return result; } +static struct sbiret pmu_sbi_ctr_cfg_match(unsigned long cbase, + unsigned long ctr_mask, + unsigned long cflags, + unsigned long event_idx, + u64 config) +{ +#if defined(CONFIG_32BIT) + return sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, + ctr_mask, cflags, event_idx, config, config >> 32); +#else + return sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, + ctr_mask, cflags, event_idx, config, 0); +#endif +} + static void pmu_sbi_check_event(struct sbi_pmu_event_data *edata) { - struct sbiret ret; + struct sbiret ret = { .error = SBI_ERR_NOT_SUPPORTED }; + int i; - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, - 0, cmask, 0, edata->event_idx, 0, 0); - if (!ret.error) { - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, - ret.value, 0x1, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); - } else if (ret.error == SBI_ERR_NOT_SUPPORTED) { + for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { + if (!cmask[i]) + continue; + ret = pmu_sbi_ctr_cfg_match(i * BITS_PER_LONG, cmask[i], 0, + edata->event_idx, 0); + if (!ret.error) { + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, + ret.value, 0x1, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); + return; + } + } + if (ret.error == SBI_ERR_NOT_SUPPORTED) { /* This event cannot be monitored by any counter */ edata->event_idx = -ENOENT; } @@ -488,10 +510,10 @@ int riscv_pmu_get_hpm_info(u32 *hw_ctr_width, u32 *num_hw_ctr) union sbi_pmu_ctr_info *info; u32 hpm_width = 0, hpm_count = 0; - if (!cmask) + if (bitmap_empty(cmask, RISCV_MAX_COUNTERS)) return -EINVAL; - for_each_set_bit(i, &cmask, RISCV_MAX_COUNTERS) { + for_each_set_bit(i, cmask, RISCV_MAX_COUNTERS) { info = &pmu_ctr_list[i]; if (!info) continue; @@ -540,8 +562,8 @@ static int pmu_sbi_ctr_get_idx(struct perf_event *event) struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu); struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events); struct sbiret ret; - int idx; - uint64_t cbase = 0, cmask = rvpmu->cmask; + int idx, i; + u64 cbase = 0, cmask = 0; unsigned long cflags = 0; cflags = pmu_sbi_get_filter_flags(event); @@ -562,14 +584,21 @@ static int pmu_sbi_ctr_get_idx(struct perf_event *event) } /* retrieve the available counter index */ -#if defined(CONFIG_32BIT) - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, - cmask, cflags, hwc->event_base, hwc->config, - hwc->config >> 32); -#else - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, - cmask, cflags, hwc->event_base, hwc->config, 0); -#endif + if (cmask) { + ret = pmu_sbi_ctr_cfg_match(cbase, cmask, cflags, hwc->event_base, + hwc->config); + } else { + ret.error = SBI_ERR_NOT_SUPPORTED; + for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { + if (!rvpmu->cmask[i]) + continue; + cbase = i * BITS_PER_LONG; + ret = pmu_sbi_ctr_cfg_match(cbase, rvpmu->cmask[i], cflags, + hwc->event_base, hwc->config); + if (!ret.error) + break; + } + } if (ret.error) { pr_debug("Not able to find a counter for event %lx config %llx\n", hwc->event_base, hwc->config); @@ -577,7 +606,7 @@ static int pmu_sbi_ctr_get_idx(struct perf_event *event) } idx = ret.value; - if (!test_bit(idx, &rvpmu->cmask) || !pmu_ctr_list[idx].value) + if (!test_bit(idx, rvpmu->cmask) || !pmu_ctr_list[idx].value) return -ENOENT; /* Additional sanity check for the counter id */ @@ -881,7 +910,7 @@ static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask) /* The logical counter ids are not expected to be contiguous */ continue; - *mask |= BIT(i); + set_bit(i, mask); cinfo.value = ret.value; if (cinfo.type == SBI_PMU_CTR_TYPE_FW) @@ -898,12 +927,19 @@ static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask) static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu) { + int i; + /* * No need to check the error because we are disabling all the counters * which may include counters that are not enabled yet. */ - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, - 0, pmu->cmask, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); + for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { + if (!pmu->cmask[i]) + continue; + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, + i * BITS_PER_LONG, pmu->cmask[i], + SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); + } } static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu) @@ -1002,7 +1038,7 @@ static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_ struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) { - if (ctr_ovf_mask & BIT(idx)) { + if (ctr_ovf_mask & BIT_ULL(idx)) { event = cpu_hw_evt->events[idx]; hwc = &event->hw; max_period = riscv_pmu_ctr_get_width_mask(event); @@ -1050,11 +1086,13 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) u64 overflowed_ctrs = 0; struct cpu_hw_events *cpu_hw_evt = dev; u64 start_clock = sched_clock(); - struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; + struct riscv_pmu_snapshot_data *sdata; if (WARN_ON_ONCE(!cpu_hw_evt)) return IRQ_NONE; + sdata = cpu_hw_evt->snapshot_addr; + /* Firmware counter don't support overflow yet */ fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS); if (fidx == RISCV_MAX_COUNTERS) { @@ -1109,14 +1147,14 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) hidx = info->csr - CSR_CYCLE; /* check if the corresponding bit is set in scountovf or overflow mask in shmem */ - if (!(overflow & BIT(hidx))) + if (!(overflow & BIT_ULL(hidx))) continue; /* * Keep a track of overflowed counters so that they can be started * with updated initial value. */ - overflowed_ctrs |= BIT(lidx); + overflowed_ctrs |= BIT_ULL(lidx); hw_evt = &event->hw; /* Update the event states here so that we know the state while reading */ hw_evt->state |= PERF_HES_STOPPED; @@ -1442,7 +1480,7 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) } /* cache all the information about counters now */ - if (pmu_sbi_get_ctrinfo(num_counters, &cmask)) + if (pmu_sbi_get_ctrinfo(num_counters, cmask)) goto out_free; ret = pmu_sbi_setup_irqs(pmu, pdev); @@ -1454,7 +1492,7 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) pmu->pmu.attr_groups = riscv_pmu_attr_groups; pmu->pmu.parent = &pdev->dev; - pmu->cmask = cmask; + bitmap_copy(pmu->cmask, cmask, RISCV_MAX_COUNTERS); pmu->ctr_start = pmu_sbi_ctr_start; pmu->ctr_stop = pmu_sbi_ctr_stop; pmu->event_map = pmu_sbi_event_map; diff --git a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c index b05d80e849a14..9a33c06d6fc39 100644 --- a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c +++ b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c @@ -173,9 +173,9 @@ static struct typec_switch_dev *tca_blk_get_typec_switch(struct platform_device return sw; } -static void tca_blk_put_typec_switch(struct typec_switch_dev *sw) +static void tca_blk_put_typec_switch(void *data) { - typec_switch_unregister(sw); + typec_switch_unregister(data); } static void tca_blk_orientation_set(struct tca_blk *tca, @@ -248,6 +248,7 @@ static struct tca_blk *imx95_usb_phy_get_tca(struct platform_device *pdev, struct device *dev = &pdev->dev; struct resource *res; struct tca_blk *tca; + int ret; res = platform_get_resource(pdev, IORESOURCE_MEM, 1); if (!res) @@ -266,17 +267,11 @@ static struct tca_blk *imx95_usb_phy_get_tca(struct platform_device *pdev, tca->orientation = TYPEC_ORIENTATION_NORMAL; tca->sw = tca_blk_get_typec_switch(pdev, imx_phy); - return tca; -} - -static void imx95_usb_phy_put_tca(struct imx8mq_usb_phy *imx_phy) -{ - struct tca_blk *tca = imx_phy->tca; - - if (!tca) - return; + ret = devm_add_action_or_reset(&pdev->dev, tca_blk_put_typec_switch, tca->sw); + if (ret) + return ERR_PTR(ret); - tca_blk_put_typec_switch(tca->sw); + return tca; } static u32 phy_tx_vref_tune_from_property(u32 percent) @@ -741,9 +736,7 @@ static int imx8mq_usb_phy_probe(struct platform_device *pdev) static void imx8mq_usb_phy_remove(struct platform_device *pdev) { - struct imx8mq_usb_phy *imx_phy = platform_get_drvdata(pdev); - imx95_usb_phy_put_tca(imx_phy); } static struct platform_driver imx8mq_usb_phy_driver = { diff --git a/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.c b/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.c index 1426a2db984d5..a4bc1268946d8 100644 --- a/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.c +++ b/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.c @@ -36,7 +36,7 @@ mtk_phy_tmds_clk_ratio(struct mtk_hdmi_phy *hdmi_phy, bool enable) * clock bit ratio 1:40, under 3.4Gbps, clock bit ratio 1:10 */ if (enable) - mtk_phy_update_field(regs + HDMI20_CLK_CFG, REG_TXC_DIV, 3); + mtk_phy_update_field(regs + HDMI20_CLK_CFG, REG_TXC_DIV, VAL_TXC_DIV4); else mtk_phy_clear_bits(regs + HDMI20_CLK_CFG, REG_TXC_DIV); } @@ -290,7 +290,7 @@ static int mtk_hdmi_pll_calc(struct mtk_hdmi_phy *hdmi_phy, struct clk_hw *hw, posdiv2 = 1; /* Digital clk divider, max /32 */ - digital_div = div_u64(ns_hdmipll_ck, posdiv1 * posdiv2 * pixel_clk); + digital_div = div64_u64(ns_hdmipll_ck, posdiv1 * posdiv2 * pixel_clk); if (!(digital_div <= 32 && digital_div >= 1)) return -EINVAL; diff --git a/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.h b/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.h index e26caaf4d104c..58800d7659ca0 100644 --- a/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.h +++ b/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.h @@ -17,6 +17,9 @@ #define HDMI20_CLK_CFG 0x70 #define REG_TXC_DIV GENMASK(31, 30) +#define VAL_TXC_DIV2 1 +#define VAL_TXC_DIV4 2 +#define VAL_TXC_DIV8 3 #define HDMI_1_CFG_0 0x00 #define RG_HDMITX21_DRV_IBIAS_CLK GENMASK(10, 5) diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c index cdcfad2e86b1d..5b278fd54a161 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c @@ -2108,7 +2108,7 @@ static const u8 qmp_dp_v4_pre_emphasis_hbr3_hbr2[4][4] = { static const u8 qmp_dp_v4_pre_emphasis_hbr_rbr[4][4] = { { 0x00, 0x0d, 0x14, 0x1a }, { 0x00, 0x0e, 0x15, 0xff }, - { 0x00, 0x0d, 0xff, 0xff }, + { 0x00, 0x0e, 0xff, 0xff }, { 0x03, 0xff, 0xff, 0xff } }; @@ -2325,7 +2325,6 @@ static int qmp_v3_calibrate_dp_phy(struct qmp_combo *qmp); static void qmp_v4_dp_aux_init(struct qmp_combo *qmp); static void qmp_v4_configure_dp_tx(struct qmp_combo *qmp); static int qmp_v4_configure_dp_phy(struct qmp_combo *qmp); -static int qmp_v4_calibrate_dp_phy(struct qmp_combo *qmp); static void qmp_v8_dp_aux_init(struct qmp_combo *qmp); static int qmp_v8_configure_dp_clocks(struct qmp_combo *qmp); @@ -2466,7 +2465,6 @@ static const struct qmp_phy_cfg sar2130p_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v6_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -2603,7 +2601,6 @@ static const struct qmp_phy_cfg sc8180x_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2650,7 +2647,6 @@ static const struct qmp_phy_cfg sc8280xp_usb43dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2696,7 +2692,6 @@ static const struct qmp_phy_cfg x1e80100_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2785,7 +2780,6 @@ static const struct qmp_phy_cfg sm8250_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2834,7 +2828,6 @@ static const struct qmp_phy_cfg sm8350_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2882,7 +2875,6 @@ static const struct qmp_phy_cfg sm8550_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v6_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -2928,7 +2920,6 @@ static const struct qmp_phy_cfg sm8650_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v6_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -2974,7 +2965,6 @@ static const struct qmp_phy_cfg sm8750_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v8_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -3020,7 +3010,6 @@ static const struct qmp_phy_cfg glymur_usb3dpphy_cfg = { .configure_dp_tx = qmp_v4_configure_dp_tx, .configure_dp_clocks = qmp_v8_configure_dp_clocks, .configure_dp_phy = qmp_v8_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v8_n3_usb43dpphy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -3316,7 +3305,6 @@ static void qmp_v4_dp_aux_init(struct qmp_combo *qmp) writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG7); writel(0xb7, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG8); writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG9); - qmp->dp_aux_cfg = 0; writel(PHY_AUX_STOP_ERR_MASK | PHY_AUX_DEC_ERR_MASK | PHY_AUX_SYNC_ERR_MASK | PHY_AUX_ALIGN_ERR_MASK | @@ -3345,7 +3333,6 @@ static void qmp_v8_dp_aux_init(struct qmp_combo *qmp) writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG7); writel(0xb7, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG8); writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG9); - qmp->dp_aux_cfg = 0; writel(PHY_AUX_STOP_ERR_MASK | PHY_AUX_DEC_ERR_MASK | PHY_AUX_SYNC_ERR_MASK | PHY_AUX_ALIGN_ERR_MASK | @@ -3605,24 +3592,6 @@ static int qmp_v8_configure_dp_phy(struct qmp_combo *qmp) return 0; } -/* - * We need to calibrate the aux setting here as many times - * as the caller tries - */ -static int qmp_v4_calibrate_dp_phy(struct qmp_combo *qmp) -{ - static const u8 cfg1_settings[] = { 0x20, 0x13, 0x23, 0x1d }; - u8 val; - - qmp->dp_aux_cfg++; - qmp->dp_aux_cfg %= ARRAY_SIZE(cfg1_settings); - val = cfg1_settings[qmp->dp_aux_cfg]; - - writel(val, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG1); - - return 0; -} - static int qmp_combo_dp_configure(struct phy *phy, union phy_configure_opts *opts) { const struct phy_configure_opts_dp *dp_opts = &opts->dp; diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c index d3effad7a074b..3618812e84d5f 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c @@ -3554,6 +3554,7 @@ static const struct qmp_pcie_offsets qmp_pcie_offsets_v5 = { .pcs_misc = 0x0600, .tx = 0x0e00, .rx = 0x1000, + .pcs_lane1 = 0x1400, .tx2 = 0x1600, .rx2 = 0x1800, }; diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c b/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c index 8bf951b0490cf..fc490589c8e48 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c @@ -542,6 +542,8 @@ struct qmp_usb { enum phy_mode mode; + bool phy_initialized; + struct phy *phy; struct clk_fixed_rate pipe_clk_fixed; @@ -895,6 +897,7 @@ static int qmp_usb_legacy_power_off(struct phy *phy) static int qmp_usb_legacy_enable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; ret = qmp_usb_legacy_init(phy); @@ -904,14 +907,19 @@ static int qmp_usb_legacy_enable(struct phy *phy) ret = qmp_usb_legacy_power_on(phy); if (ret) qmp_usb_legacy_exit(phy); + else + qmp->phy_initialized = true; return ret; } static int qmp_usb_legacy_disable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; + qmp->phy_initialized = false; + ret = qmp_usb_legacy_power_off(phy); if (ret) return ret; @@ -988,7 +996,7 @@ static int __maybe_unused qmp_usb_legacy_runtime_suspend(struct device *dev) dev_vdbg(dev, "Suspending QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } @@ -1009,7 +1017,7 @@ static int __maybe_unused qmp_usb_legacy_runtime_resume(struct device *dev) dev_vdbg(dev, "Resuming QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c index c5507168e1354..bb905c437a01e 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c @@ -1434,6 +1434,8 @@ struct qmp_usb { enum phy_mode mode; + bool phy_initialized; + struct phy *phy; struct clk_fixed_rate pipe_clk_fixed; @@ -2009,6 +2011,7 @@ static int qmp_usb_power_off(struct phy *phy) static int qmp_usb_enable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; ret = qmp_usb_init(phy); @@ -2018,14 +2021,19 @@ static int qmp_usb_enable(struct phy *phy) ret = qmp_usb_power_on(phy); if (ret) qmp_usb_exit(phy); + else + qmp->phy_initialized = true; return ret; } static int qmp_usb_disable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; + qmp->phy_initialized = false; + ret = qmp_usb_power_off(phy); if (ret) return ret; @@ -2101,7 +2109,7 @@ static int __maybe_unused qmp_usb_runtime_suspend(struct device *dev) dev_vdbg(dev, "Suspending QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } @@ -2121,7 +2129,7 @@ static int __maybe_unused qmp_usb_runtime_resume(struct device *dev) dev_vdbg(dev, "Resuming QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } diff --git a/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c b/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c index f48faa2929a68..7137200c3f310 100644 --- a/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c +++ b/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "phy-qcom-qmp-pcs-sgmii.h" #include "phy-qcom-qmp-qserdes-com-v5.h" @@ -26,7 +27,15 @@ #define QSERDES_PCS_SGMIIPHY_READY BIT(7) #define QSERDES_COM_C_PLL_LOCKED BIT(1) +static const struct regulator_bulk_data qcom_dwmac_sgmii_phy_vregs[] = { + { .supply = "vdda-0p9", .init_load_uA = 46000 }, + { .supply = "vdda-1p2", .init_load_uA = 15000 }, +}; + +#define QCOM_SGMII_NUM_SUPPLIES ARRAY_SIZE(qcom_dwmac_sgmii_phy_vregs) + struct qcom_dwmac_sgmii_phy_data { + struct regulator_bulk_data *vregs; struct regmap *regmap; struct clk *refclk; phy_interface_t interface; @@ -273,13 +282,24 @@ static int qcom_dwmac_sgmii_phy_power_on(struct phy *phy) struct qcom_dwmac_sgmii_phy_data *data = phy_get_drvdata(phy); int ret; - ret = clk_prepare_enable(data->refclk); - if (ret < 0) + ret = regulator_bulk_enable(QCOM_SGMII_NUM_SUPPLIES, data->vregs); + if (ret) return ret; + ret = clk_prepare_enable(data->refclk); + if (ret) + goto err_disable_regulators; + ret = qcom_dwmac_sgmii_phy_calibrate(phy); - if (ret < 0) - clk_disable_unprepare(data->refclk); + if (ret) + goto err_disable_clk; + + return 0; + +err_disable_clk: + clk_disable_unprepare(data->refclk); +err_disable_regulators: + regulator_bulk_disable(QCOM_SGMII_NUM_SUPPLIES, data->vregs); return ret; } @@ -296,6 +316,8 @@ static int qcom_dwmac_sgmii_phy_power_off(struct phy *phy) clk_disable_unprepare(data->refclk); + regulator_bulk_disable(QCOM_SGMII_NUM_SUPPLIES, data->vregs); + return 0; } @@ -357,6 +379,7 @@ static int qcom_dwmac_sgmii_phy_probe(struct platform_device *pdev) struct phy_provider *provider; void __iomem *base; struct phy *phy; + int ret; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -381,6 +404,12 @@ static int qcom_dwmac_sgmii_phy_probe(struct platform_device *pdev) if (IS_ERR(data->refclk)) return PTR_ERR(data->refclk); + ret = devm_regulator_bulk_get_const(dev, QCOM_SGMII_NUM_SUPPLIES, + qcom_dwmac_sgmii_phy_vregs, + &data->vregs); + if (ret) + return ret; + provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); if (IS_ERR(provider)) return PTR_ERR(provider); diff --git a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c index eb0b0f61d98e0..980ad1fb1e2e3 100644 --- a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c +++ b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c @@ -599,8 +599,18 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "failed to get regulator supplies\n"); + /* + * Enable runtime PM before creating the PHY, phy_create() only enables + * it on the PHY device if already enabled on the parent. Hold a usage + * reference so callbacks cannot run before the PHY is ready. + */ + pm_runtime_get_noresume(dev); pm_runtime_set_active(dev); - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) { + pm_runtime_put_noidle(dev); + return ret; + } /* * Prevent runtime pm from being ON by default. Users can enable * it using power/control in sysfs. @@ -611,6 +621,7 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) if (IS_ERR(generic_phy)) { ret = PTR_ERR(generic_phy); dev_err(dev, "failed to create phy, %d\n", ret); + pm_runtime_put_noidle(dev); return ret; } hsphy->phy = generic_phy; @@ -620,12 +631,15 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) qcom_snps_hsphy_read_override_param_seq(dev); phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - if (!IS_ERR(phy_provider)) - dev_dbg(dev, "Registered Qcom-SNPS HS phy\n"); - else - pm_runtime_disable(dev); + if (IS_ERR(phy_provider)) { + pm_runtime_put_noidle(dev); + return PTR_ERR(phy_provider); + } - return PTR_ERR_OR_ZERO(phy_provider); + dev_dbg(dev, "Registered Qcom-SNPS HS phy\n"); + pm_runtime_put(dev); + + return 0; } static struct platform_driver qcom_snps_hsphy_driver = { diff --git a/drivers/phy/renesas/phy-rcar-gen2.c b/drivers/phy/renesas/phy-rcar-gen2.c index 6c671254c6258..5a272e25e051c 100644 --- a/drivers/phy/renesas/phy-rcar-gen2.c +++ b/drivers/phy/renesas/phy-rcar-gen2.c @@ -405,7 +405,6 @@ static int rcar_gen2_phy_probe(struct platform_device *pdev) data->gen2_phy_ops); if (IS_ERR(phy->phy)) { dev_err(dev, "Failed to create PHY\n"); - of_node_put(np); return PTR_ERR(phy->phy); } phy_set_drvdata(phy->phy, phy); diff --git a/drivers/phy/renesas/phy-rcar-gen3-usb2.c b/drivers/phy/renesas/phy-rcar-gen3-usb2.c index 9a45d840efeb2..d0fcc3663e886 100644 --- a/drivers/phy/renesas/phy-rcar-gen3-usb2.c +++ b/drivers/phy/renesas/phy-rcar-gen3-usb2.c @@ -27,6 +27,7 @@ #include #include #include +#include #include /******* USB2.0 Host registers (original offset is +0x200) *******/ @@ -106,6 +107,13 @@ /* RZ/G2L specific */ #define USB2_LINECTRL1_USB2_IDMON BIT(0) +/* + * The OTG initialization is expected to finish in 20ms. Choose a large enough + * timeout to avoid waiters exit prematurely the waiting section under heavy + * CPU load. + */ +#define USB2_OTG_INIT_TIMEOUT msecs_to_jiffies(120) + #define NUM_OF_PHYS 4 enum rcar_gen3_phy_index { PHY_INDEX_BOTH_HC, @@ -138,12 +146,20 @@ struct rcar_gen3_chan { struct rcar_gen3_phy rphys[NUM_OF_PHYS]; struct regulator *vbus; struct work_struct work; + wait_queue_head_t otg_init_done; spinlock_t lock; /* protects access to hardware and driver data structure. */ enum usb_dr_mode dr_mode; bool extcon_host; bool is_otg_channel; bool uses_otg_pins; bool otg_internal_reg; + /* + * The OTG can be initialized only once and needs to release the spinlock + * and wait for 20 ms due to hardware constraints. If a thread executes + * PHY configuration code while the OTG PHY is waiting for the 20 ms, the + * thread will have to wait for the OTG PHY initialization to complete. + */ + bool otg_initializing; }; struct rcar_gen3_phy_drv_data { @@ -392,26 +408,58 @@ static ssize_t role_store(struct device *dev, struct device_attribute *attr, struct rcar_gen3_chan *ch = dev_get_drvdata(dev); bool is_b_device; enum phy_mode cur_mode, new_mode; + int retries = NUM_OF_PHYS; + unsigned long flags; + int ret = -EIO; - guard(spinlock_irqsave)(&ch->lock); + spin_lock_irqsave(&ch->lock, flags); - if (!ch->is_otg_channel || !rcar_gen3_is_any_otg_rphy_initialized(ch)) - return -EIO; + if (!ch->is_otg_channel) + goto unlock; + + while (retries-- && ch->otg_initializing) { + spin_unlock_irqrestore(&ch->lock, flags); - if (sysfs_streq(buf, "host")) + ret = wait_event_timeout(ch->otg_init_done, !ch->otg_initializing, + USB2_OTG_INIT_TIMEOUT); + ret = ret ? 0 : -ETIMEDOUT; + if (ret && !retries) + goto exit; + + spin_lock_irqsave(&ch->lock, flags); + } + + /* If another thread started a new initialization just return -EBUSY. */ + if (ch->otg_initializing) { + ret = -EBUSY; + goto unlock; + } else { + ret = 0; + } + + if (!rcar_gen3_is_any_otg_rphy_initialized(ch)) { + ret = -EIO; + goto unlock; + } + + if (sysfs_streq(buf, "host")) { new_mode = PHY_MODE_USB_HOST; - else if (sysfs_streq(buf, "peripheral")) + } else if (sysfs_streq(buf, "peripheral")) { new_mode = PHY_MODE_USB_DEVICE; - else - return -EINVAL; + } else { + ret = -EINVAL; + goto unlock; + } /* is_b_device: true is B-Device. false is A-Device. */ is_b_device = rcar_gen3_check_id(ch); cur_mode = rcar_gen3_get_phy_mode(ch); /* If current and new mode is the same, this returns the error */ - if (cur_mode == new_mode) - return -EINVAL; + if (cur_mode == new_mode) { + ret = -EINVAL; + goto unlock; + } if (new_mode == PHY_MODE_USB_HOST) { /* And is_host must be false */ if (!is_b_device) /* A-Peripheral */ @@ -425,7 +473,10 @@ static ssize_t role_store(struct device *dev, struct device_attribute *attr, rcar_gen3_init_for_peri(ch); } - return count; +unlock: + spin_unlock_irqrestore(&ch->lock, flags); +exit: + return ret ?: count; } static ssize_t role_show(struct device *dev, struct device_attribute *attr, @@ -441,14 +492,11 @@ static ssize_t role_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RW(role); -static void rcar_gen3_init_otg(struct rcar_gen3_chan *ch) +static void rcar_gen3_init_otg_phase0(struct rcar_gen3_chan *ch) { void __iomem *usb2_base = ch->base; u32 val; - if (!ch->is_otg_channel || rcar_gen3_is_any_otg_rphy_initialized(ch)) - return; - /* Should not use functions of read-modify-write a register */ val = readl(usb2_base + USB2_LINECTRL1); val = (val & ~USB2_LINECTRL1_DP_RPD) | USB2_LINECTRL1_DPRPD_EN | @@ -471,7 +519,11 @@ static void rcar_gen3_init_otg(struct rcar_gen3_chan *ch) writel(val | USB2_ADPCTRL_IDPULLUP, usb2_base + USB2_ADPCTRL); } } - mdelay(20); +} + +static void rcar_gen3_init_otg_phase1(struct rcar_gen3_chan *ch) +{ + void __iomem *usb2_base = ch->base; writel(0xffffffff, usb2_base + USB2_OBINTSTA); writel(ch->phy_data->obint_enable_bits, usb2_base + USB2_OBINTEN); @@ -502,6 +554,7 @@ static irqreturn_t rcar_gen3_phy_usb2_irq(int irq, void *_ch) void __iomem *usb2_base = ch->base; struct device *dev = ch->dev; irqreturn_t ret = IRQ_NONE; + unsigned long flags; u32 status; pm_runtime_get_noresume(dev); @@ -509,33 +562,102 @@ static irqreturn_t rcar_gen3_phy_usb2_irq(int irq, void *_ch) if (pm_runtime_suspended(dev)) goto rpm_put; - scoped_guard(spinlock, &ch->lock) { - status = readl(usb2_base + USB2_OBINTSTA); - if (status & ch->phy_data->obint_enable_bits) { - dev_vdbg(dev, "%s: %08x\n", __func__, status); - if (ch->phy_data->vblvl_ctrl) - writel(USB2_OBINTSTA_CLEAR, usb2_base + USB2_OBINTSTA); - else - writel(ch->phy_data->obint_enable_bits, usb2_base + USB2_OBINTSTA); - rcar_gen3_device_recognition(ch); - rcar_gen3_configure_vblvl_ctrl(ch); - ret = IRQ_HANDLED; - } + spin_lock_irqsave(&ch->lock, flags); + + status = readl(usb2_base + USB2_OBINTSTA); + if (status & ch->phy_data->obint_enable_bits) { + dev_vdbg(dev, "%s: %08x\n", __func__, status); + if (ch->phy_data->vblvl_ctrl) + writel(USB2_OBINTSTA_CLEAR, usb2_base + USB2_OBINTSTA); + else + writel(ch->phy_data->obint_enable_bits, usb2_base + USB2_OBINTSTA); + + ret = IRQ_HANDLED; + + /* This should not happen! */ + if (ch->otg_initializing) + goto unlock; + + rcar_gen3_device_recognition(ch); + rcar_gen3_configure_vblvl_ctrl(ch); } +unlock: + spin_unlock_irqrestore(&ch->lock, flags); rpm_put: pm_runtime_put_noidle(dev); return ret; } +static void rcar_gen3_phy_usb2_irqs_mask_all(struct rcar_gen3_chan *channel, + u32 *masked_irqs_bits) +{ + u32 val, bitmask = USB2_INT_ENABLE_UCOM_INTEN; + void __iomem *usb2_base = channel->base; + + for (unsigned int i = 0; i < NUM_OF_PHYS; i++) + bitmask |= channel->rphys[i].int_enable_bits; + + val = readl(usb2_base + USB2_INT_ENABLE); + *masked_irqs_bits = val & bitmask; + val &= ~bitmask; + writel(val, usb2_base + USB2_INT_ENABLE); + + /* + * Don't report channel->phy_data->obint_enable_bits IRQs. These are + * unmasked anyway in rcar_gen3_init_otg_phase1(). + */ + val = readl(usb2_base + USB2_OBINTEN); + val &= ~channel->phy_data->obint_enable_bits; + writel(val, usb2_base + USB2_OBINTEN); +} + +static void rcar_gen3_phy_usb2_irqs_unmask(struct rcar_gen3_chan *channel, + u32 irqs_bits) +{ + u32 val, bitmask = USB2_INT_ENABLE_UCOM_INTEN; + void __iomem *usb2_base = channel->base; + + for (unsigned int i = 0; i < NUM_OF_PHYS; i++) + bitmask |= channel->rphys[i].int_enable_bits; + + val = readl(usb2_base + USB2_INT_ENABLE); + val &= ~bitmask; + val |= irqs_bits; + writel(val, usb2_base + USB2_INT_ENABLE); +} + static int rcar_gen3_phy_usb2_init(struct phy *p) { struct rcar_gen3_phy *rphy = phy_get_drvdata(p); struct rcar_gen3_chan *channel = rphy->ch; void __iomem *usb2_base = channel->base; + int retries = NUM_OF_PHYS; + unsigned long flags; u32 val; + int ret; + + spin_lock_irqsave(&channel->lock, flags); + + while (retries-- && channel->otg_initializing) { + spin_unlock_irqrestore(&channel->lock, flags); + + ret = wait_event_timeout(channel->otg_init_done, !channel->otg_initializing, + USB2_OTG_INIT_TIMEOUT); + ret = ret ? 0 : -ETIMEDOUT; + if (ret && !retries) + return ret; + + spin_lock_irqsave(&channel->lock, flags); + } - guard(spinlock_irqsave)(&channel->lock); + /* If another thread started a new initialization just return -EBUSY. */ + if (channel->otg_initializing) { + ret = -EBUSY; + goto unlock; + } else { + ret = 0; + } /* Initialize USB2 part */ val = readl(usb2_base + USB2_INT_ENABLE); @@ -548,8 +670,23 @@ static int rcar_gen3_phy_usb2_init(struct phy *p) } /* Initialize otg part (only if we initialize a PHY with IRQs). */ - if (rphy->int_enable_bits) - rcar_gen3_init_otg(channel); + if (rphy->int_enable_bits && channel->is_otg_channel && + !rcar_gen3_is_any_otg_rphy_initialized(channel)) { + u32 masked_irq_bits = 0; + + rcar_gen3_init_otg_phase0(channel); + rcar_gen3_phy_usb2_irqs_mask_all(channel, &masked_irq_bits); + channel->otg_initializing = true; + spin_unlock_irqrestore(&channel->lock, flags); + + fsleep(20000); + + spin_lock_irqsave(&channel->lock, flags); + rcar_gen3_phy_usb2_irqs_unmask(channel, masked_irq_bits); + rcar_gen3_init_otg_phase1(channel); + channel->otg_initializing = false; + wake_up_all(&channel->otg_init_done); + } if (channel->phy_data->vblvl_ctrl) { /* SIDDQ mode release */ @@ -568,7 +705,10 @@ static int rcar_gen3_phy_usb2_init(struct phy *p) rphy->initialized = true; - return 0; +unlock: + spin_unlock_irqrestore(&channel->lock, flags); + + return ret; } static int rcar_gen3_phy_usb2_exit(struct phy *p) @@ -576,9 +716,32 @@ static int rcar_gen3_phy_usb2_exit(struct phy *p) struct rcar_gen3_phy *rphy = phy_get_drvdata(p); struct rcar_gen3_chan *channel = rphy->ch; void __iomem *usb2_base = channel->base; + int retries = NUM_OF_PHYS; + unsigned long flags; u32 val; + int ret; + + spin_lock_irqsave(&channel->lock, flags); + + while (retries-- && channel->otg_initializing) { + spin_unlock_irqrestore(&channel->lock, flags); - guard(spinlock_irqsave)(&channel->lock); + ret = wait_event_timeout(channel->otg_init_done, !channel->otg_initializing, + USB2_OTG_INIT_TIMEOUT); + ret = ret ? 0 : -ETIMEDOUT; + if (ret && !retries) + return ret; + + spin_lock_irqsave(&channel->lock, flags); + } + + /* If another thread started a new initialization just return -EBUSY. */ + if (channel->otg_initializing) { + ret = -EBUSY; + goto unlock; + } else { + ret = 0; + } rphy->initialized = false; @@ -588,7 +751,9 @@ static int rcar_gen3_phy_usb2_exit(struct phy *p) val &= ~USB2_INT_ENABLE_UCOM_INTEN; writel(val, usb2_base + USB2_INT_ENABLE); - return 0; +unlock: + spin_unlock_irqrestore(&channel->lock, flags); + return ret; } static int rcar_gen3_phy_usb2_power_on(struct phy *p) @@ -596,8 +761,10 @@ static int rcar_gen3_phy_usb2_power_on(struct phy *p) struct rcar_gen3_phy *rphy = phy_get_drvdata(p); struct rcar_gen3_chan *channel = rphy->ch; void __iomem *usb2_base = channel->base; + int retries = NUM_OF_PHYS; + unsigned long flags; u32 val; - int ret = 0; + int ret; if (channel->vbus && !channel->otg_internal_reg) { ret = regulator_enable(channel->vbus); @@ -605,7 +772,27 @@ static int rcar_gen3_phy_usb2_power_on(struct phy *p) return ret; } - guard(spinlock_irqsave)(&channel->lock); + spin_lock_irqsave(&channel->lock, flags); + + while (retries-- && channel->otg_initializing) { + spin_unlock_irqrestore(&channel->lock, flags); + + ret = wait_event_timeout(channel->otg_init_done, !channel->otg_initializing, + USB2_OTG_INIT_TIMEOUT); + ret = ret ? 0 : -ETIMEDOUT; + if (ret && !retries) + goto disable_regulator; + + spin_lock_irqsave(&channel->lock, flags); + } + + /* If another thread started a new initialization just return -EBUSY. */ + if (channel->otg_initializing) { + ret = -EBUSY; + goto unlock; + } else { + ret = 0; + } if (!rcar_gen3_are_all_rphys_power_off(channel)) goto out; @@ -620,27 +807,59 @@ static int rcar_gen3_phy_usb2_power_on(struct phy *p) /* The powered flag should be set for any other phys anyway */ rphy->powered = true; - return 0; +unlock: + spin_unlock_irqrestore(&channel->lock, flags); + +disable_regulator: + if (ret && channel->vbus && !channel->otg_internal_reg) + regulator_disable(channel->vbus); + + return ret; } static int rcar_gen3_phy_usb2_power_off(struct phy *p) { struct rcar_gen3_phy *rphy = phy_get_drvdata(p); struct rcar_gen3_chan *channel = rphy->ch; - int ret = 0; + int retries = NUM_OF_PHYS; + unsigned long flags; + int ret; - scoped_guard(spinlock_irqsave, &channel->lock) { - rphy->powered = false; + spin_lock_irqsave(&channel->lock, flags); - if (rcar_gen3_are_all_rphys_power_off(channel)) { - u32 val = readl(channel->base + USB2_USBCTR); + while (retries-- && channel->otg_initializing) { + spin_unlock_irqrestore(&channel->lock, flags); - val |= USB2_USBCTR_PLL_RST; - writel(val, channel->base + USB2_USBCTR); - } + ret = wait_event_timeout(channel->otg_init_done, !channel->otg_initializing, + USB2_OTG_INIT_TIMEOUT); + ret = ret ? 0 : -ETIMEDOUT; + if (ret && !retries) + return ret; + + spin_lock_irqsave(&channel->lock, flags); + } + + /* If another thread started a new initialization just return -EBUSY. */ + if (channel->otg_initializing) { + ret = -EBUSY; + goto unlock; + } else { + ret = 0; + } + + rphy->powered = false; + + if (rcar_gen3_are_all_rphys_power_off(channel)) { + u32 val = readl(channel->base + USB2_USBCTR); + + val |= USB2_USBCTR_PLL_RST; + writel(val, channel->base + USB2_USBCTR); } - if (channel->vbus && !channel->otg_internal_reg) +unlock: + spin_unlock_irqrestore(&channel->lock, flags); + + if (!ret && channel->vbus && !channel->otg_internal_reg) ret = regulator_disable(channel->vbus); return ret; @@ -898,18 +1117,29 @@ static int rcar_gen3_phy_usb2_vbus_regulator_get_exclusive_enable(struct rcar_ge int ret; channel->vbus = devm_regulator_get_exclusive(dev, "vbus"); - if (IS_ERR(channel->vbus)) - return PTR_ERR(channel->vbus); + if (IS_ERR(channel->vbus)) { + ret = PTR_ERR(channel->vbus); + /* If vbus-regulator node was present vbus regulator should be available */ + if (channel->otg_internal_reg) + return ret; + + if (ret == -EPROBE_DEFER) + return ret; - if (!enable) return 0; + } - ret = regulator_enable(channel->vbus); - if (ret) - return ret; + if (enable) { + ret = regulator_enable(channel->vbus); + if (ret) + return ret; + } + + if (regulator_is_enabled(channel->vbus)) + return devm_add_action_or_reset(dev, rcar_gen3_phy_usb2_vbus_disable_action, + channel->vbus); - return devm_add_action_or_reset(dev, rcar_gen3_phy_usb2_vbus_disable_action, - channel->vbus); + return 0; } static int rcar_gen3_phy_usb2_vbus_regulator_register(struct rcar_gen3_chan *channel) @@ -1007,6 +1237,7 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev) return ret; spin_lock_init(&channel->lock); + init_waitqueue_head(&channel->otg_init_done); for (i = 0; i < NUM_OF_PHYS; i++) { channel->rphys[i].phy = devm_phy_create(dev, NULL, channel->phy_data->phy_usb2_ops); diff --git a/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c b/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c index c79fb53d8ee5c..5281f8dea0ad3 100644 --- a/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c +++ b/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c @@ -170,7 +170,7 @@ static const struct hsfreq_range rk1808_mipidphy_hsfreq_ranges[] = { { 299, 0x06}, { 399, 0x08}, { 499, 0x0b}, { 599, 0x0e}, { 699, 0x10}, { 799, 0x12}, { 999, 0x16}, {1199, 0x1e}, {1399, 0x23}, {1599, 0x2d}, {1799, 0x32}, {1999, 0x37}, - {2199, 0x3c}, {2399, 0x41}, {2499, 0x46} + {2199, 0x3c}, {2399, 0x41}, {2500, 0x46} }; static const struct hsfreq_range rk3326_mipidphy_hsfreq_ranges[] = { diff --git a/drivers/phy/rockchip/phy-rockchip-samsung-dcphy.c b/drivers/phy/rockchip/phy-rockchip-samsung-dcphy.c index cbd780556da8f..18f5f582d1d7a 100644 --- a/drivers/phy/rockchip/phy-rockchip-samsung-dcphy.c +++ b/drivers/phy/rockchip/phy-rockchip-samsung-dcphy.c @@ -1526,7 +1526,7 @@ static const struct regmap_config samsung_mipi_dcphy_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, - .max_register = 0x10000, + .max_register = 0xfffc, }; static struct phy *samsung_mipi_dcphy_xlate(struct device *dev, diff --git a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c index 2d973bc37f076..c24e7f2841703 100644 --- a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c +++ b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c @@ -413,6 +413,7 @@ struct rk_hdptx_phy { /* clk provider */ struct clk_hw hw; + bool pll_config_dirty; bool restrict_rate_change; atomic_t usage_count; @@ -1260,13 +1261,19 @@ static int rk_hdptx_tmds_ropll_cmn_config(struct rk_hdptx_phy *hdptx) static int rk_hdptx_pll_cmn_config(struct rk_hdptx_phy *hdptx) { + int ret; + if (hdptx->hdmi_cfg.rate <= HDMI20_MAX_RATE) - return rk_hdptx_tmds_ropll_cmn_config(hdptx); + ret = rk_hdptx_tmds_ropll_cmn_config(hdptx); + else if (hdptx->hdmi_cfg.rate == FRL_8G4L_RATE) + ret = rk_hdptx_frl_lcpll_ropll_cmn_config(hdptx); + else + ret = rk_hdptx_frl_lcpll_cmn_config(hdptx); - if (hdptx->hdmi_cfg.rate == FRL_8G4L_RATE) - return rk_hdptx_frl_lcpll_ropll_cmn_config(hdptx); + if (!ret) + hdptx->pll_config_dirty = false; - return rk_hdptx_frl_lcpll_cmn_config(hdptx); + return ret; } static int rk_hdptx_frl_lcpll_mode_config(struct rk_hdptx_phy *hdptx) @@ -1347,25 +1354,22 @@ static int rk_hdptx_phy_consumer_get(struct rk_hdptx_phy *hdptx) return 0; ret = regmap_read(hdptx->grf, GRF_HDPTX_STATUS, &status); - if (ret) - goto dec_usage; - - if (status & HDPTX_O_PLL_LOCK_DONE) - dev_warn(hdptx->dev, "PLL locked by unknown consumer!\n"); + if (ret) { + atomic_dec(&hdptx->usage_count); + return ret; + } if (mode == PHY_MODE_DP) { rk_hdptx_dp_reset(hdptx); } else { - ret = rk_hdptx_pll_cmn_config(hdptx); - if (ret) - goto dec_usage; + /* + * Ignore PLL config errors at this point as pll_config_dirty + * was not reset and, therefore, operation will be retried. + */ + rk_hdptx_pll_cmn_config(hdptx); } return 0; - -dec_usage: - atomic_dec(&hdptx->usage_count); - return ret; } static int rk_hdptx_phy_consumer_put(struct rk_hdptx_phy *hdptx, bool force) @@ -1700,13 +1704,18 @@ static int rk_hdptx_phy_power_on(struct phy *phy) if (ret) rk_hdptx_phy_consumer_put(hdptx, true); } else { - regmap_write(hdptx->grf, GRF_HDPTX_CON0, - HDPTX_MODE_SEL << 16 | FIELD_PREP(HDPTX_MODE_SEL, 0x0)); + if (hdptx->pll_config_dirty) + ret = rk_hdptx_pll_cmn_config(hdptx); - if (hdptx->hdmi_cfg.mode == PHY_HDMI_MODE_FRL) - ret = rk_hdptx_frl_lcpll_mode_config(hdptx); - else - ret = rk_hdptx_tmds_ropll_mode_config(hdptx); + if (!ret) { + regmap_write(hdptx->grf, GRF_HDPTX_CON0, + HDPTX_MODE_SEL << 16 | FIELD_PREP(HDPTX_MODE_SEL, 0x0)); + + if (hdptx->hdmi_cfg.mode == PHY_HDMI_MODE_FRL) + ret = rk_hdptx_frl_lcpll_mode_config(hdptx); + else + ret = rk_hdptx_tmds_ropll_mode_config(hdptx); + } if (ret) rk_hdptx_phy_consumer_put(hdptx, true); @@ -2081,7 +2090,10 @@ static int rk_hdptx_phy_configure(struct phy *phy, union phy_configure_opts *opt dev_err(hdptx->dev, "invalid hdmi params for phy configure\n"); } else { hdptx->restrict_rate_change = true; - dev_dbg(hdptx->dev, "%s rate=%llu bpc=%u\n", __func__, + hdptx->pll_config_dirty = true; + + dev_dbg(hdptx->dev, "%s %s rate=%llu bpc=%u\n", __func__, + hdptx->hdmi_cfg.mode ? "FRL" : "TMDS", hdptx->hdmi_cfg.rate, hdptx->hdmi_cfg.bpc); } @@ -2168,7 +2180,7 @@ static u64 rk_hdptx_phy_clk_calc_rate_from_pll_cfg(struct rk_hdptx_phy *hdptx) struct lcpll_config lcpll_hw; struct ropll_config ropll_hw; u64 fout, sdm; - u32 mode, val; + u32 mode, bpc, val; int ret, i; ret = regmap_read(hdptx->regmap, CMN_REG(0008), &mode); @@ -2206,16 +2218,31 @@ static u64 rk_hdptx_phy_clk_calc_rate_from_pll_cfg(struct rk_hdptx_phy *hdptx) return 0; lcpll_hw.sdc_n = (val & LCPLL_SDC_N_MASK) >> 1; - for (i = 0; i < ARRAY_SIZE(rk_hdptx_frl_lcpll_cfg); i++) { - const struct lcpll_config *cfg = &rk_hdptx_frl_lcpll_cfg[i]; + ret = regmap_read(hdptx->grf, GRF_HDPTX_CON0, &val); + if (ret) + return 0; - if (cfg->pms_mdiv == lcpll_hw.pms_mdiv && - cfg->pms_sdiv == lcpll_hw.pms_sdiv && - cfg->sdm_num_sign == lcpll_hw.sdm_num_sign && - cfg->sdm_num == lcpll_hw.sdm_num && - cfg->sdm_deno == lcpll_hw.sdm_deno && - cfg->sdc_n == lcpll_hw.sdc_n) - return cfg->rate; + if (val & LC_REF_CLK_SEL) { + if (lcpll_hw.pms_mdiv == 0x6b && + lcpll_hw.sdm_num_sign == 0x01 && + lcpll_hw.sdm_num == 0x02 && + lcpll_hw.sdm_deno == 0x09 && + lcpll_hw.sdc_n == FIELD_GET(LCPLL_SDC_N_MASK, 0x02)) + return FRL_8G4L_RATE; + } else { + const struct lcpll_config *cfg; + + for (i = 0; i < ARRAY_SIZE(rk_hdptx_frl_lcpll_cfg); i++) { + cfg = &rk_hdptx_frl_lcpll_cfg[i]; + + if (cfg->pms_mdiv == lcpll_hw.pms_mdiv && + cfg->pms_sdiv == lcpll_hw.pms_sdiv && + cfg->sdm_num_sign == lcpll_hw.sdm_num_sign && + cfg->sdm_num == lcpll_hw.sdm_num && + cfg->sdm_deno == lcpll_hw.sdm_deno && + cfg->sdc_n == lcpll_hw.sdc_n) + return cfg->rate; + } } dev_dbg(hdptx->dev, "%s no FRL match found\n", __func__); @@ -2266,21 +2293,42 @@ static u64 rk_hdptx_phy_clk_calc_rate_from_pll_cfg(struct rk_hdptx_phy *hdptx) if (ret) return 0; ropll_hw.pms_sdiv = ((val & PLL_PCG_POSTDIV_SEL_MASK) >> 4) + 1; + bpc = (FIELD_GET(PLL_PCG_CLK_SEL_MASK, val) << 1) + 8; fout = PLL_REF_CLK * ropll_hw.pms_mdiv; if (ropll_hw.sdm_en) { + val = 16U * ropll_hw.sdm_deno * + (ropll_hw.sdc_deno * ropll_hw.sdc_n - ropll_hw.sdc_num); + if (!val) { + /* + * The PLL config currently stored in hardware can't be + * translated into a rate. The next .set_rate() should + * program a valid configuration and help with recovery. + */ + dev_dbg(hdptx->dev, "Invalid ROPLL hw state: deno == 0\n"); + return 0; + } + sdm = div_u64(PLL_REF_CLK * ropll_hw.sdc_deno * - ropll_hw.pms_mdiv * ropll_hw.sdm_num, - 16 * ropll_hw.sdm_deno * - (ropll_hw.sdc_deno * ropll_hw.sdc_n - ropll_hw.sdc_num)); + ropll_hw.pms_mdiv * ropll_hw.sdm_num, val); + + if (ropll_hw.sdm_num_sign) { + if (sdm > fout) { + /* + * Similarly to the case above, it is expected + * the next .set_rate() will help with recovery. + */ + dev_dbg(hdptx->dev, "Invalid ROPLL hw state: sdm > fout\n"); + return 0; + } - if (ropll_hw.sdm_num_sign) fout = fout - sdm; - else + } else { fout = fout + sdm; + } } - return div_u64(fout * 2, ropll_hw.pms_sdiv * 10); + return DIV_ROUND_CLOSEST_ULL(fout * 2 * 8, ropll_hw.pms_sdiv * 10 * bpc); } static unsigned long rk_hdptx_phy_clk_recalc_rate(struct clk_hw *hw, @@ -2288,19 +2336,13 @@ static unsigned long rk_hdptx_phy_clk_recalc_rate(struct clk_hw *hw, { struct rk_hdptx_phy *hdptx = to_rk_hdptx_phy(hw); u32 status; - u64 rate; int ret; ret = regmap_read(hdptx->grf, GRF_HDPTX_CON0, &status); if (ret || !(status & HDPTX_I_PLL_EN)) return 0; - rate = rk_hdptx_phy_clk_calc_rate_from_pll_cfg(hdptx); - - if (hdptx->hdmi_cfg.mode == PHY_HDMI_MODE_FRL) - return rate; - - return DIV_ROUND_CLOSEST_ULL(rate * 8, hdptx->hdmi_cfg.bpc); + return rk_hdptx_phy_clk_calc_rate_from_pll_cfg(hdptx); } static int rk_hdptx_phy_clk_determine_rate(struct clk_hw *hw, @@ -2308,8 +2350,19 @@ static int rk_hdptx_phy_clk_determine_rate(struct clk_hw *hw, { struct rk_hdptx_phy *hdptx = to_rk_hdptx_phy(hw); - if (hdptx->hdmi_cfg.mode == PHY_HDMI_MODE_FRL) - return hdptx->hdmi_cfg.rate; + /* + * Invalidate current clock rate to ensure rk_hdptx_phy_clk_set_rate() + * will be invoked to commit PLL configuration. + */ + if (hdptx->pll_config_dirty) { + req->rate = 0; + return 0; + } + + if (hdptx->hdmi_cfg.mode == PHY_HDMI_MODE_FRL) { + req->rate = hdptx->hdmi_cfg.rate; + return 0; + } /* * FIXME: Temporarily allow altering TMDS char rate via CCF. @@ -2341,17 +2394,6 @@ static int rk_hdptx_phy_clk_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { struct rk_hdptx_phy *hdptx = to_rk_hdptx_phy(hw); - unsigned long long link_rate = rate; - - if (hdptx->hdmi_cfg.mode != PHY_HDMI_MODE_FRL) - link_rate = DIV_ROUND_CLOSEST_ULL(rate * hdptx->hdmi_cfg.bpc, 8); - - /* Revert any unlikely link rate change since determine_rate() */ - if (hdptx->hdmi_cfg.rate != link_rate) { - dev_warn(hdptx->dev, "Reverting unexpected rate change from %llu to %llu\n", - link_rate, hdptx->hdmi_cfg.rate); - hdptx->hdmi_cfg.rate = link_rate; - } /* * The link rate would be normally programmed in HW during diff --git a/drivers/phy/starfive/phy-jh7110-dphy-rx.c b/drivers/phy/starfive/phy-jh7110-dphy-rx.c index 0b039e1f71c55..d06f21ad63325 100644 --- a/drivers/phy/starfive/phy-jh7110-dphy-rx.c +++ b/drivers/phy/starfive/phy-jh7110-dphy-rx.c @@ -150,6 +150,7 @@ static int stf_dphy_probe(struct platform_device *pdev) { struct phy_provider *phy_provider; struct stf_dphy *dphy; + int ret; dphy = devm_kzalloc(&pdev->dev, sizeof(*dphy), GFP_KERNEL); if (!dphy) @@ -190,7 +191,9 @@ static int stf_dphy_probe(struct platform_device *pdev) return PTR_ERR(dphy->phy); } - pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; phy_set_drvdata(dphy->phy, dphy); phy_provider = devm_of_phy_provider_register(&pdev->dev, diff --git a/drivers/phy/starfive/phy-jh7110-dphy-tx.c b/drivers/phy/starfive/phy-jh7110-dphy-tx.c index c64d1c91b1307..181491a938079 100644 --- a/drivers/phy/starfive/phy-jh7110-dphy-tx.c +++ b/drivers/phy/starfive/phy-jh7110-dphy-tx.c @@ -392,6 +392,7 @@ static int stf_dphy_probe(struct platform_device *pdev) { struct phy_provider *phy_provider; struct stf_dphy *dphy; + int ret; dphy = devm_kzalloc(&pdev->dev, sizeof(*dphy), GFP_KERNEL); if (!dphy) @@ -406,7 +407,9 @@ static int stf_dphy_probe(struct platform_device *pdev) if (IS_ERR(dphy->topsys)) return PTR_ERR(dphy->topsys); - pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; dphy->txesc_clk = devm_clk_get(&pdev->dev, "txesc"); if (IS_ERR(dphy->txesc_clk)) diff --git a/drivers/phy/sunplus/phy-sunplus-usb2.c b/drivers/phy/sunplus/phy-sunplus-usb2.c index 637a5fbae6d9a..0ad4c7160d17c 100644 --- a/drivers/phy/sunplus/phy-sunplus-usb2.c +++ b/drivers/phy/sunplus/phy-sunplus-usb2.c @@ -116,11 +116,11 @@ static int sp_uphy_init(struct phy *phy) ret = clk_prepare_enable(usbphy->phy_clk); if (ret) - goto err_clk; + return ret; ret = reset_control_deassert(usbphy->rstc); if (ret) - goto err_reset; + goto err_clk; /* Default value modification */ writel(HIGH_MASK_BITS | 0x4002, usbphy->moon4_regs + UPHY_CONTROL0); @@ -129,7 +129,7 @@ static int sp_uphy_init(struct phy *phy) /* disconnect voltage */ ret = update_disc_vol(usbphy); if (ret < 0) - return ret; + goto err_reset; /* board uphy 0 internal register modification for tid certification */ val = readl(usbphy->phy_regs + CONFIG9); diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig index eda54aa5fde6f..bf73bc709c181 100644 --- a/drivers/pinctrl/Kconfig +++ b/drivers/pinctrl/Kconfig @@ -275,8 +275,8 @@ config PINCTRL_GEMINI config PINCTRL_GENERIC_MUX tristate "Generic Pinctrl driver by using multiplexer" - depends on MULTIPLEXER depends on OF + select MULTIPLEXER select PINMUX select GENERIC_PINCTRL help @@ -488,6 +488,7 @@ config PINCTRL_PIC32 depends on MACH_PIC32 || COMPILE_TEST select PINMUX select GENERIC_PINCONF + select GPIOLIB select GPIOLIB_IRQCHIP help This is the pin controller and gpio driver for Microchip PIC32 @@ -564,6 +565,7 @@ config PINCTRL_STMFX depends on I2C depends on HAS_IOMEM select GENERIC_PINCONF + select GPIOLIB select GPIOLIB_IRQCHIP select MFD_STMFX help diff --git a/drivers/pinctrl/airoha/pinctrl-airoha.c b/drivers/pinctrl/airoha/pinctrl-airoha.c index 04b4424c688b2..6a40a776d3158 100644 --- a/drivers/pinctrl/airoha/pinctrl-airoha.c +++ b/drivers/pinctrl/airoha/pinctrl-airoha.c @@ -49,7 +49,7 @@ /* MUX */ #define REG_GPIO_2ND_I2C_MODE 0x0214 -#define GPIO_MDC_IO_MASTER_MODE_MODE BIT(14) +#define GPIO_MDC_IO_MASTER_MODE_MASK BIT(14) #define GPIO_I2C_MASTER_MODE_MODE BIT(13) #define GPIO_I2S_MODE_MASK BIT(12) #define GPIO_I2C_SLAVE_MODE_MODE BIT(11) @@ -66,6 +66,7 @@ #define GPIO_2ND_I2C_MODE_MASK BIT(0) #define REG_GPIO_SPI_CS1_MODE 0x0218 +#define AN7583_GPIO_MDC_IO_MASTER_MODE_MASK BIT(22) #define GPIO_PCM_SPI_CS4_MODE_MASK BIT(21) #define GPIO_PCM_SPI_CS3_MODE_MASK BIT(20) #define GPIO_PCM_SPI_CS2_MODE_P156_MASK BIT(19) @@ -84,6 +85,18 @@ #define GPIO_SPI_CS1_MODE_MASK BIT(0) #define REG_GPIO_PON_MODE 0x021c +#define AN7583_MDIO_0_GPIO_MODE_MASK BIT(26) +#define AN7583_MDC_0_GPIO_MODE_MASK BIT(25) +#define AN7583_UART_RXD_GPIO_MODE_MASK BIT(24) +#define AN7583_UART_TXD_GPIO_MODE_MASK BIT(23) +#define AN7583_SPI_MISO_GPIO_MODE_MASK BIT(22) +#define AN7583_SPI_MOSI_GPIO_MODE_MASK BIT(21) +#define AN7583_SPI_CS_GPIO_MODE_MASK BIT(20) +#define AN7583_SPI_CLK_GPIO_MODE_MASK BIT(19) +#define AN7583_I2C1_SDA_GPIO_MODE_MASK BIT(18) +#define AN7583_I2C1_SCL_GPIO_MODE_MASK BIT(17) +#define AN7583_I2C0_SDA_GPIO_MODE_MASK BIT(16) +#define AN7583_I2C0_SCL_GPIO_MODE_MASK BIT(15) #define GPIO_PARALLEL_NAND_MODE_MASK BIT(14) #define GPIO_SGMII_MDIO_MODE_MASK BIT(13) #define GPIO_PCIE_RESET2_MASK BIT(12) @@ -184,8 +197,8 @@ #define I2C_SDA_PU_MASK BIT(0) #define REG_I2C_SDA_PD 0x0048 -#define AN7583_I2C1_SDA_PD_MASK BIT(16) -#define AN7583_I2C1_SCL_PD_MASK BIT(15) +#define AN7583_I2C1_SCL_PD_MASK BIT(16) +#define AN7583_I2C1_SDA_PD_MASK BIT(15) #define SPI_MISO_PD_MASK BIT(14) #define SPI_MOSI_PD_MASK BIT(13) #define SPI_CLK_PD_MASK BIT(12) @@ -469,9 +482,9 @@ static struct pinctrl_pin_desc en7581_pinctrl_pins[] = { PINCTRL_PIN(57, "gpio44"), PINCTRL_PIN(58, "gpio45"), PINCTRL_PIN(59, "gpio46"), - PINCTRL_PIN(61, "pcie_reset0"), - PINCTRL_PIN(62, "pcie_reset1"), - PINCTRL_PIN(63, "pcie_reset2"), + PINCTRL_PIN(60, "pcie_reset0"), + PINCTRL_PIN(61, "pcie_reset1"), + PINCTRL_PIN(62, "pcie_reset2"), }; static const int en7581_pon_pins[] = { 49, 50, 51, 52, 53, 54 }; @@ -554,9 +567,12 @@ static const int en7581_gpio43_pins[] = { 56 }; static const int en7581_gpio44_pins[] = { 57 }; static const int en7581_gpio45_pins[] = { 58 }; static const int en7581_gpio46_pins[] = { 59 }; -static const int en7581_pcie_reset0_pins[] = { 61 }; -static const int en7581_pcie_reset1_pins[] = { 62 }; -static const int en7581_pcie_reset2_pins[] = { 63 }; +static const int en7581_gpio47_pins[] = { 60 }; +static const int en7581_gpio48_pins[] = { 61 }; +static const int en7581_gpio49_pins[] = { 62 }; +static const int en7581_pcie_reset0_pins[] = { 60 }; +static const int en7581_pcie_reset1_pins[] = { 61 }; +static const int en7581_pcie_reset2_pins[] = { 62 }; static const struct pingroup en7581_pinctrl_groups[] = { PINCTRL_PIN_GROUP("pon", en7581_pon), @@ -639,6 +655,9 @@ static const struct pingroup en7581_pinctrl_groups[] = { PINCTRL_PIN_GROUP("gpio44", en7581_gpio44), PINCTRL_PIN_GROUP("gpio45", en7581_gpio45), PINCTRL_PIN_GROUP("gpio46", en7581_gpio46), + PINCTRL_PIN_GROUP("gpio47", en7581_gpio47), + PINCTRL_PIN_GROUP("gpio48", en7581_gpio48), + PINCTRL_PIN_GROUP("gpio49", en7581_gpio49), PINCTRL_PIN_GROUP("pcie_reset0", en7581_pcie_reset0), PINCTRL_PIN_GROUP("pcie_reset1", en7581_pcie_reset1), PINCTRL_PIN_GROUP("pcie_reset2", en7581_pcie_reset2), @@ -719,7 +738,7 @@ static const int an7583_jtag_udi_pins[] = { 23, 24, 22, 25, 26 }; static const int an7583_jtag_dfd_pins[] = { 23, 24, 22, 25, 26 }; static const int an7583_pcm1_pins[] = { 10, 11, 12, 13, 14 }; static const int an7583_pcm2_pins[] = { 28, 29, 30, 31, 24 }; -static const int an7583_spi_pins[] = { 28, 29, 30, 31 }; +static const int an7583_spi_pins[] = { 45, 46, 47, 48 }; static const int an7583_spi_quad_pins[] = { 25, 26 }; static const int an7583_spi_cs1_pins[] = { 27 }; static const int an7583_pcm_spi_pins[] = { 28, 29, 30, 31, 10, 11, 12, 13 }; @@ -776,6 +795,10 @@ static const int an7583_gpio45_pins[] = { 47 }; static const int an7583_gpio46_pins[] = { 48 }; static const int an7583_gpio47_pins[] = { 49 }; static const int an7583_gpio48_pins[] = { 50 }; +static const int an7583_gpio49_pins[] = { 51 }; +static const int an7583_gpio50_pins[] = { 52 }; +static const int an7583_gpio51_pins[] = { 53 }; +static const int an7583_gpio52_pins[] = { 54 }; static const int an7583_pcie_reset0_pins[] = { 51 }; static const int an7583_pcie_reset1_pins[] = { 52 }; @@ -856,6 +879,10 @@ static const struct pingroup an7583_pinctrl_groups[] = { PINCTRL_PIN_GROUP("gpio46", an7583_gpio46), PINCTRL_PIN_GROUP("gpio47", an7583_gpio47), PINCTRL_PIN_GROUP("gpio48", an7583_gpio48), + PINCTRL_PIN_GROUP("gpio49", an7583_gpio49), + PINCTRL_PIN_GROUP("gpio50", an7583_gpio50), + PINCTRL_PIN_GROUP("gpio51", an7583_gpio51), + PINCTRL_PIN_GROUP("gpio52", an7583_gpio52), PINCTRL_PIN_GROUP("pcie_reset0", an7583_pcie_reset0), PINCTRL_PIN_GROUP("pcie_reset1", an7583_pcie_reset1), }; @@ -882,8 +909,14 @@ static const char *const an7583_pcm_spi_groups[] = { "pcm_spi", static const char *const i2s_groups[] = { "i2s" }; static const char *const emmc_groups[] = { "emmc" }; static const char *const pnand_groups[] = { "pnand" }; +static const char *const gpio_groups[] = { "gpio47", "gpio48", "gpio49" }; static const char *const pcie_reset_groups[] = { "pcie_reset0", "pcie_reset1", "pcie_reset2" }; +static const char *const an7583_gpio_groups[] = { "gpio39", "gpio40", "gpio41", + "gpio42", "gpio43", "gpio44", + "gpio45", "gpio46", "gpio47", + "gpio48", "gpio49", "gpio50", + "gpio51", "gpio52" }; static const char *const an7583_pcie_reset_groups[] = { "pcie_reset0", "pcie_reset1" }; static const char *const pwm_groups[] = { "gpio0", "gpio1", "gpio2", "gpio3", @@ -906,7 +939,8 @@ static const char *const pwm_groups[] = { "gpio0", "gpio1", "gpio40", "gpio41", "gpio42", "gpio43", "gpio44", "gpio45", - "gpio46" }; + "gpio46", "gpio47", + "gpio48", "gpio49" }; static const char *const an7583_pwm_groups[] = { "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", @@ -929,7 +963,8 @@ static const char *const an7583_pwm_groups[] = { "gpio0", "gpio1", "gpio42", "gpio43", "gpio44", "gpio45", "gpio46", "gpio47", - "gpio48" }; + "gpio48", "gpio49", + "gpio50", "gpio51" }; static const char *const phy1_led0_groups[] = { "gpio33", "gpio34", "gpio35", "gpio42" }; static const char *const phy2_led0_groups[] = { "gpio33", "gpio34", @@ -1026,8 +1061,8 @@ static const struct airoha_pinctrl_func_group mdio_func_group[] = { .regmap[0] = { AIROHA_FUNC_MUX, REG_GPIO_2ND_I2C_MODE, - GPIO_MDC_IO_MASTER_MODE_MODE, - GPIO_MDC_IO_MASTER_MODE_MODE + GPIO_MDC_IO_MASTER_MODE_MASK, + GPIO_MDC_IO_MASTER_MODE_MASK }, .regmap[1] = { AIROHA_FUNC_MUX, @@ -1051,8 +1086,8 @@ static const struct airoha_pinctrl_func_group an7583_mdio_func_group[] = { .regmap[1] = { AIROHA_FUNC_MUX, REG_GPIO_SPI_CS1_MODE, - GPIO_MDC_IO_MASTER_MODE_MODE, - GPIO_MDC_IO_MASTER_MODE_MODE + AN7583_GPIO_MDC_IO_MASTER_MODE_MASK, + AN7583_GPIO_MDC_IO_MASTER_MODE_MASK }, .regmap_size = 2, }, @@ -1405,6 +1440,45 @@ static const struct airoha_pinctrl_func_group pnand_func_group[] = { }, }; +#define AIROHA_PINCTRL_GPIO(gpio, mux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_PON_MODE, \ + (mux_val), \ + (mux_val) \ + }, \ + .regmap_size = 1, \ + } + +#define AIROHA_PINCTRL_GPIO_EXT(gpio, mux_val, smux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_PWM_EXT_MUX, \ + REG_GPIO_FLASH_MODE_CFG_EXT, \ + (mux_val), \ + 0 \ + }, \ + .regmap[1] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_PON_MODE, \ + (smux_val), \ + (smux_val) \ + }, \ + .regmap_size = 2, \ + } + +static const struct airoha_pinctrl_func_group gpio_func_group[] = { + AIROHA_PINCTRL_GPIO_EXT("gpio47", GPIO47_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio48", GPIO48_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET2_MASK), +}; + static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = { { .name = "pcie_reset0", @@ -1412,7 +1486,7 @@ static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = { AIROHA_FUNC_MUX, REG_GPIO_PON_MODE, GPIO_PCIE_RESET0_MASK, - GPIO_PCIE_RESET0_MASK + 0 }, .regmap_size = 1, }, { @@ -1421,7 +1495,7 @@ static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = { AIROHA_FUNC_MUX, REG_GPIO_PON_MODE, GPIO_PCIE_RESET1_MASK, - GPIO_PCIE_RESET1_MASK + 0 }, .regmap_size = 1, }, { @@ -1430,12 +1504,42 @@ static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = { AIROHA_FUNC_MUX, REG_GPIO_PON_MODE, GPIO_PCIE_RESET2_MASK, - GPIO_PCIE_RESET2_MASK + 0 }, .regmap_size = 1, }, }; +static const struct airoha_pinctrl_func_group an7583_gpio_func_group[] = { + AIROHA_PINCTRL_GPIO_EXT("gpio39", GPIO39_FLASH_MODE_CFG, + AN7583_I2C0_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio40", GPIO40_FLASH_MODE_CFG, + AN7583_I2C0_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio41", GPIO41_FLASH_MODE_CFG, + AN7583_I2C1_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio42", GPIO42_FLASH_MODE_CFG, + AN7583_I2C1_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio43", GPIO43_FLASH_MODE_CFG, + AN7583_SPI_CLK_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio44", GPIO44_FLASH_MODE_CFG, + AN7583_SPI_CS_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio45", GPIO45_FLASH_MODE_CFG, + AN7583_SPI_MOSI_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio46", GPIO46_FLASH_MODE_CFG, + AN7583_SPI_MISO_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio47", GPIO47_FLASH_MODE_CFG, + AN7583_UART_TXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio48", GPIO48_FLASH_MODE_CFG, + AN7583_UART_RXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio50", GPIO50_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio51", GPIO51_FLASH_MODE_CFG, + AN7583_MDC_0_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO("gpio52", AN7583_MDIO_0_GPIO_MODE_MASK), +}; + static const struct airoha_pinctrl_func_group an7583_pcie_reset_func_group[] = { { .name = "pcie_reset0", @@ -1443,7 +1547,7 @@ static const struct airoha_pinctrl_func_group an7583_pcie_reset_func_group[] = { AIROHA_FUNC_MUX, REG_GPIO_PON_MODE, GPIO_PCIE_RESET0_MASK, - GPIO_PCIE_RESET0_MASK + 0 }, .regmap_size = 1, }, { @@ -1452,7 +1556,7 @@ static const struct airoha_pinctrl_func_group an7583_pcie_reset_func_group[] = { AIROHA_FUNC_MUX, REG_GPIO_PON_MODE, GPIO_PCIE_RESET1_MASK, - GPIO_PCIE_RESET1_MASK + 0 }, .regmap_size = 1, }, @@ -1483,6 +1587,24 @@ static const struct airoha_pinctrl_func_group an7583_pcie_reset_func_group[] = { .regmap_size = 1, \ } \ +#define AIROHA_PINCTRL_PWM_EXT_SEC(gpio, mux_val, smux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_PWM_EXT_MUX, \ + REG_GPIO_FLASH_MODE_CFG_EXT, \ + (mux_val), \ + (mux_val) \ + }, \ + .regmap[1] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_PON_MODE, \ + (smux_val), \ + (smux_val) \ + }, \ + .regmap_size = 2, \ + } + static const struct airoha_pinctrl_func_group pwm_func_group[] = { AIROHA_PINCTRL_PWM("gpio0", GPIO0_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM("gpio1", GPIO1_FLASH_MODE_CFG), @@ -1527,6 +1649,12 @@ static const struct airoha_pinctrl_func_group pwm_func_group[] = { AIROHA_PINCTRL_PWM_EXT("gpio44", GPIO44_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio45", GPIO45_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio46", GPIO46_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio47", GPIO47_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio48", GPIO48_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET2_MASK), }; static const struct airoha_pinctrl_func_group an7583_pwm_func_group[] = { @@ -1565,16 +1693,32 @@ static const struct airoha_pinctrl_func_group an7583_pwm_func_group[] = { AIROHA_PINCTRL_PWM_EXT("gpio36", GPIO36_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio37", GPIO37_FLASH_MODE_CFG), AIROHA_PINCTRL_PWM_EXT("gpio38", GPIO38_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio39", GPIO39_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio40", GPIO40_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio41", GPIO41_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio42", GPIO42_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio43", GPIO43_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio44", GPIO44_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio45", GPIO45_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio46", GPIO46_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio47", GPIO47_FLASH_MODE_CFG), - AIROHA_PINCTRL_PWM_EXT("gpio48", GPIO48_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio39", GPIO39_FLASH_MODE_CFG, + AN7583_I2C0_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio40", GPIO40_FLASH_MODE_CFG, + AN7583_I2C0_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio41", GPIO41_FLASH_MODE_CFG, + AN7583_I2C1_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio42", GPIO42_FLASH_MODE_CFG, + AN7583_I2C1_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio43", GPIO43_FLASH_MODE_CFG, + AN7583_SPI_CLK_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio44", GPIO44_FLASH_MODE_CFG, + AN7583_SPI_CS_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio45", GPIO45_FLASH_MODE_CFG, + AN7583_SPI_MOSI_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio46", GPIO46_FLASH_MODE_CFG, + AN7583_SPI_MISO_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio47", GPIO47_FLASH_MODE_CFG, + AN7583_UART_TXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio48", GPIO48_FLASH_MODE_CFG, + AN7583_UART_RXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio50", GPIO50_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio51", GPIO51_FLASH_MODE_CFG, + AN7583_MDC_0_GPIO_MODE_MASK), }; #define AIROHA_PINCTRL_PHY_LED0(gpio, mux_val, map_mask, map_val) \ @@ -1803,6 +1947,7 @@ static const struct airoha_pinctrl_func en7581_pinctrl_funcs[] = { PINCTRL_FUNC_DESC("i2s", i2s), PINCTRL_FUNC_DESC("emmc", emmc), PINCTRL_FUNC_DESC("pnand", pnand), + PINCTRL_FUNC_DESC("gpio", gpio), PINCTRL_FUNC_DESC("pcie_reset", pcie_reset), PINCTRL_FUNC_DESC("pwm", pwm), PINCTRL_FUNC_DESC("phy1_led0", phy1_led0), @@ -1828,6 +1973,7 @@ static const struct airoha_pinctrl_func an7583_pinctrl_funcs[] = { PINCTRL_FUNC_DESC("pcm_spi", an7583_pcm_spi), PINCTRL_FUNC_DESC("emmc", emmc), PINCTRL_FUNC_DESC("pnand", pnand), + PINCTRL_FUNC_DESC("gpio", an7583_gpio), PINCTRL_FUNC_DESC("pcie_reset", an7583_pcie_reset), PINCTRL_FUNC_DESC("pwm", an7583_pwm), PINCTRL_FUNC_DESC("phy1_led0", an7583_phy1_led0), @@ -1843,8 +1989,8 @@ static const struct airoha_pinctrl_func an7583_pinctrl_funcs[] = { static const struct airoha_pinctrl_conf en7581_pinctrl_pullup_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_PU, UART1_TXD_PU_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_PU, UART1_RXD_PU_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_PU, I2C_SDA_PU_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_PU, I2C_SCL_PU_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_PU, I2C_SCL_PU_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_PU, I2C_SDA_PU_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_PU, SPI_CS0_PU_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_PU, SPI_CLK_PU_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_PU, SPI_MOSI_PU_MASK), @@ -1896,9 +2042,9 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_pullup_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_PU, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_PU, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_PU, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_PU, PCIE0_RESET_PU_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_PU, PCIE1_RESET_PU_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_PU, PCIE2_RESET_PU_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_PU, PCIE0_RESET_PU_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_PU, PCIE1_RESET_PU_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_PU, PCIE2_RESET_PU_MASK), }; static const struct airoha_pinctrl_conf an7583_pinctrl_pullup_conf[] = { @@ -1960,8 +2106,8 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_pullup_conf[] = { static const struct airoha_pinctrl_conf en7581_pinctrl_pulldown_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_PD, UART1_TXD_PD_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_PD, UART1_RXD_PD_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_PD, I2C_SDA_PD_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_PD, I2C_SCL_PD_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_PD, I2C_SCL_PD_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_PD, I2C_SDA_PD_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_PD, SPI_CS0_PD_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_PD, SPI_CLK_PD_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_PD, SPI_MOSI_PD_MASK), @@ -2013,9 +2159,9 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_pulldown_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_PD, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_PD, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_PD, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_PD, PCIE0_RESET_PD_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_PD, PCIE1_RESET_PD_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_PD, PCIE2_RESET_PD_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_PD, PCIE0_RESET_PD_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_PD, PCIE1_RESET_PD_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_PD, PCIE2_RESET_PD_MASK), }; static const struct airoha_pinctrl_conf an7583_pinctrl_pulldown_conf[] = { @@ -2077,8 +2223,8 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_pulldown_conf[] = { static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e2_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_E2, UART1_TXD_E2_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_E2, UART1_RXD_E2_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_E2, I2C_SDA_E2_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_E2, I2C_SCL_E2_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_E2, I2C_SCL_E2_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_E2, I2C_SDA_E2_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_E2, SPI_CS0_E2_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_E2, SPI_CLK_E2_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_E2, SPI_MOSI_E2_MASK), @@ -2130,9 +2276,9 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e2_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_E2, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_E2, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_E2, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_E2, PCIE0_RESET_E2_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_E2, PCIE1_RESET_E2_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_E2, PCIE2_RESET_E2_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_E2, PCIE0_RESET_E2_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_E2, PCIE1_RESET_E2_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_E2, PCIE2_RESET_E2_MASK), }; static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e2_conf[] = { @@ -2194,8 +2340,8 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e2_conf[] = { static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e4_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_E4, UART1_TXD_E4_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_E4, UART1_RXD_E4_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_E4, I2C_SDA_E4_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_E4, I2C_SCL_E4_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_E4, I2C_SCL_E4_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_E4, I2C_SDA_E4_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_E4, SPI_CS0_E4_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_E4, SPI_CLK_E4_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_E4, SPI_MOSI_E4_MASK), @@ -2247,9 +2393,9 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e4_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_E4, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_E4, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_E4, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_E4, PCIE0_RESET_E4_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_E4, PCIE1_RESET_E4_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_E4, PCIE2_RESET_E4_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_E4, PCIE0_RESET_E4_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_E4, PCIE1_RESET_E4_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_E4, PCIE2_RESET_E4_MASK), }; static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e4_conf[] = { @@ -2309,9 +2455,9 @@ static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e4_conf[] = { }; static const struct airoha_pinctrl_conf en7581_pinctrl_pcie_rst_od_conf[] = { - PINCTRL_CONF_DESC(61, REG_PCIE_RESET_OD, PCIE0_RESET_OD_MASK), - PINCTRL_CONF_DESC(62, REG_PCIE_RESET_OD, PCIE1_RESET_OD_MASK), - PINCTRL_CONF_DESC(63, REG_PCIE_RESET_OD, PCIE2_RESET_OD_MASK), + PINCTRL_CONF_DESC(60, REG_PCIE_RESET_OD, PCIE0_RESET_OD_MASK), + PINCTRL_CONF_DESC(61, REG_PCIE_RESET_OD, PCIE1_RESET_OD_MASK), + PINCTRL_CONF_DESC(62, REG_PCIE_RESET_OD, PCIE2_RESET_OD_MASK), }; static const struct airoha_pinctrl_conf an7583_pinctrl_pcie_rst_od_conf[] = { @@ -2358,6 +2504,22 @@ static int airoha_gpio_get(struct gpio_chip *chip, unsigned int gpio) return err ? err : !!(val & BIT(pin)); } +static int airoha_gpio_get_direction(struct gpio_chip *chip, unsigned int gpio) +{ + struct airoha_pinctrl *pinctrl = gpiochip_get_data(chip); + u32 val, mask; + u8 index; + int err; + + index = gpio / AIROHA_REG_GPIOCTRL_NUM_PIN; + err = regmap_read(pinctrl->regmap, pinctrl->gpiochip.dir[index], &val); + if (err) + return err; + + mask = BIT(2 * (gpio % AIROHA_REG_GPIOCTRL_NUM_PIN)); + return val & mask ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN; +} + static int airoha_gpio_direction_output(struct gpio_chip *chip, unsigned int gpio, int value) { @@ -2373,18 +2535,18 @@ static int airoha_gpio_direction_output(struct gpio_chip *chip, /* irq callbacks */ static void airoha_irq_unmask(struct irq_data *data) { + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; u8 offset = data->hwirq % AIROHA_REG_GPIOCTRL_NUM_PIN; u8 index = data->hwirq / AIROHA_REG_GPIOCTRL_NUM_PIN; u32 mask = GENMASK(2 * offset + 1, 2 * offset); - struct airoha_pinctrl_gpiochip *gpiochip; - struct airoha_pinctrl *pinctrl; u32 val = BIT(2 * offset); - gpiochip = irq_data_get_irq_chip_data(data); if (WARN_ON_ONCE(data->hwirq >= ARRAY_SIZE(gpiochip->irq_type))) return; - pinctrl = container_of(gpiochip, struct airoha_pinctrl, gpiochip); + gpiochip_enable_irq(gc, irqd_to_hwirq(data)); switch (gpiochip->irq_type[data->hwirq]) { case IRQ_TYPE_LEVEL_LOW: val = val << 1; @@ -2410,34 +2572,64 @@ static void airoha_irq_unmask(struct irq_data *data) static void airoha_irq_mask(struct irq_data *data) { + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; u8 offset = data->hwirq % AIROHA_REG_GPIOCTRL_NUM_PIN; u8 index = data->hwirq / AIROHA_REG_GPIOCTRL_NUM_PIN; u32 mask = GENMASK(2 * offset + 1, 2 * offset); - struct airoha_pinctrl_gpiochip *gpiochip; - struct airoha_pinctrl *pinctrl; - gpiochip = irq_data_get_irq_chip_data(data); - pinctrl = container_of(gpiochip, struct airoha_pinctrl, gpiochip); + if (data->hwirq >= ARRAY_SIZE(gpiochip->irq_type)) + return; regmap_clear_bits(pinctrl->regmap, gpiochip->level[index], mask); regmap_clear_bits(pinctrl->regmap, gpiochip->edge[index], mask); + gpiochip_disable_irq(gc, irqd_to_hwirq(data)); +} + +static void airoha_irq_ack(struct irq_data *data) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; + u8 offset = data->hwirq % AIROHA_PIN_BANK_SIZE; + u8 index = data->hwirq / AIROHA_PIN_BANK_SIZE; + + if (data->hwirq >= ARRAY_SIZE(gpiochip->irq_type)) + return; + + regmap_write(pinctrl->regmap, gpiochip->status[index], BIT(offset)); } static int airoha_irq_type(struct irq_data *data, unsigned int type) { - struct airoha_pinctrl_gpiochip *gpiochip; + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; - gpiochip = irq_data_get_irq_chip_data(data); if (data->hwirq >= ARRAY_SIZE(gpiochip->irq_type)) return -EINVAL; + if (type == IRQ_TYPE_NONE) { + gpiochip->irq_type[data->hwirq] = IRQ_TYPE_NONE; + irq_set_handler_locked(data, handle_bad_irq); + + return 0; + } + if (type == IRQ_TYPE_PROBE) { if (gpiochip->irq_type[data->hwirq]) return 0; type = IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING; } + gpiochip->irq_type[data->hwirq] = type & IRQ_TYPE_SENSE_MASK; + if (type & IRQ_TYPE_EDGE_BOTH) + irq_set_handler_locked(data, handle_edge_irq); + else + irq_set_handler_locked(data, handle_level_irq); + return 0; } @@ -2462,8 +2654,7 @@ static irqreturn_t airoha_irq_handler(int irq, void *data) for_each_set_bit(irq, &status, AIROHA_PIN_BANK_SIZE) { u32 offset = irq + i * AIROHA_PIN_BANK_SIZE; - generic_handle_irq(irq_find_mapping(girq->domain, - offset)); + generic_handle_domain_irq(girq->domain, offset); regmap_write(pinctrl->regmap, pinctrl->gpiochip.status[i], BIT(irq)); } @@ -2477,9 +2668,10 @@ static const struct irq_chip airoha_gpio_irq_chip = { .name = "airoha-gpio-irq", .irq_unmask = airoha_irq_unmask, .irq_mask = airoha_irq_mask, - .irq_mask_ack = airoha_irq_mask, + .irq_ack = airoha_irq_ack, .irq_set_type = airoha_irq_type, .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static int airoha_pinctrl_add_gpiochip(struct airoha_pinctrl *pinctrl, @@ -2491,26 +2683,20 @@ static int airoha_pinctrl_add_gpiochip(struct airoha_pinctrl *pinctrl, struct device *dev = &pdev->dev; int irq, err; - chip->data = gpio_data_regs; - chip->dir = gpio_dir_regs; - chip->out = gpio_out_regs; - chip->status = irq_status_regs; - chip->level = irq_level_regs; - chip->edge = irq_edge_regs; - gc->parent = dev; gc->label = dev_name(dev); gc->request = gpiochip_generic_request; gc->free = gpiochip_generic_free; gc->direction_input = pinctrl_gpio_direction_input; gc->direction_output = airoha_gpio_direction_output; + gc->get_direction = airoha_gpio_get_direction; gc->set = airoha_gpio_set; gc->get = airoha_gpio_get; gc->base = -1; gc->ngpio = AIROHA_NUM_PINS; girq->default_type = IRQ_TYPE_NONE; - girq->handler = handle_simple_irq; + girq->handler = handle_bad_irq; gpio_irq_chip_set_chip(girq, &airoha_gpio_irq_chip); irq = platform_get_irq(pdev, 0); @@ -2713,21 +2899,18 @@ static int airoha_pinctrl_set_conf(struct airoha_pinctrl *pinctrl, static int airoha_pinconf_get_direction(struct pinctrl_dev *pctrl_dev, u32 p) { struct airoha_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctrl_dev); - u32 val, mask; - int err, pin; - u8 index; + int ret, pin; pin = airoha_convert_pin_to_reg_offset(pctrl_dev, NULL, p); if (pin < 0) return pin; - index = pin / AIROHA_REG_GPIOCTRL_NUM_PIN; - err = regmap_read(pinctrl->regmap, pinctrl->gpiochip.dir[index], &val); - if (err) - return err; + ret = airoha_gpio_get_direction(&pinctrl->gpiochip.chip, pin); + if (ret < 0) + return ret; - mask = BIT(2 * (pin % AIROHA_REG_GPIOCTRL_NUM_PIN)); - return val & mask ? PIN_CONFIG_OUTPUT_ENABLE : PIN_CONFIG_INPUT_ENABLE; + return ret == GPIO_LINE_DIRECTION_OUT ? + PIN_CONFIG_OUTPUT_ENABLE : PIN_CONFIG_INPUT_ENABLE; } static int airoha_pinconf_get(struct pinctrl_dev *pctrl_dev, @@ -2973,6 +3156,17 @@ static int airoha_pinctrl_probe(struct platform_device *pdev) pinctrl->desc.pins = data->pins; pinctrl->desc.npins = data->num_pins; + /* + * some pinctrl operations (ex: get_direction) might use gpio registers + * before gpio chip abstraction will be completely initialized. + */ + pinctrl->gpiochip.data = gpio_data_regs; + pinctrl->gpiochip.dir = gpio_dir_regs; + pinctrl->gpiochip.out = gpio_out_regs; + pinctrl->gpiochip.status = irq_status_regs; + pinctrl->gpiochip.level = irq_level_regs; + pinctrl->gpiochip.edge = irq_edge_regs; + err = devm_pinctrl_register_and_init(dev, &pinctrl->desc, pinctrl, &pinctrl->ctrl); if (err) diff --git a/drivers/pinctrl/bcm/Kconfig b/drivers/pinctrl/bcm/Kconfig index 206f3f1249cf5..19d22e7fd11ed 100644 --- a/drivers/pinctrl/bcm/Kconfig +++ b/drivers/pinctrl/bcm/Kconfig @@ -121,6 +121,7 @@ source "drivers/pinctrl/bcm/Kconfig.stb" config PINCTRL_IPROC_GPIO bool "Broadcom iProc GPIO (with PINCONF) driver" depends on ARCH_BCM_IPROC || COMPILE_TEST + select GPIOLIB select GPIOLIB_IRQCHIP select PINCONF select GENERIC_PINCONF @@ -186,6 +187,7 @@ config PINCTRL_NS config PINCTRL_NSP_GPIO bool "Broadcom NSP GPIO (with PINCONF) driver" depends on ARCH_BCM_NSP || COMPILE_TEST + select GPIOLIB select GPIOLIB_IRQCHIP select PINCONF select GENERIC_PINCONF diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c index e7b35019a5a7d..725e880ae086f 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c @@ -1350,7 +1350,6 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev) pc->pctl_desc = *pdata->pctl_desc; pc->pctl_dev = devm_pinctrl_register(dev, &pc->pctl_desc, pc); if (IS_ERR(pc->pctl_dev)) { - gpiochip_remove(&pc->gpio_chip); return PTR_ERR(pc->pctl_dev); } diff --git a/drivers/pinctrl/freescale/pinctrl-imx1-core.c b/drivers/pinctrl/freescale/pinctrl-imx1-core.c index b7bd4ef9c0db5..4a6bdaefa42f2 100644 --- a/drivers/pinctrl/freescale/pinctrl-imx1-core.c +++ b/drivers/pinctrl/freescale/pinctrl-imx1-core.c @@ -547,14 +547,11 @@ static int imx1_pinctrl_parse_functions(struct device_node *np, */ static bool imx1_pinctrl_dt_is_flat_functions(struct device_node *np) { - struct device_node *function_np; - struct device_node *pinctrl_np; - - for_each_child_of_node(np, function_np) { + for_each_child_of_node_scoped(np, function_np) { if (of_property_present(function_np, "fsl,pins")) return true; - for_each_child_of_node(function_np, pinctrl_np) { + for_each_child_of_node_scoped(function_np, pinctrl_np) { if (of_property_present(pinctrl_np, "fsl,pins")) return false; } diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c index 47ac92ea98c2c..8b022545a3e9f 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.c +++ b/drivers/pinctrl/mediatek/mtk-eint.c @@ -12,8 +12,10 @@ */ #include +#include #include #include +#include #include #include #include @@ -509,6 +511,27 @@ int mtk_eint_find_irq(struct mtk_eint *eint, unsigned long eint_n) } EXPORT_SYMBOL_GPL(mtk_eint_find_irq); +static void mtk_eint_teardown(void *data) +{ + struct mtk_eint *eint = data; + unsigned int i, virq; + + /* Detach the demux handler so it can no longer reference freed data. */ + irq_set_chained_handler_and_data(eint->irq, NULL, NULL); + + /* Wait for any in-flight handler to finish before tearing down. */ + synchronize_irq(eint->irq); + + /* Dispose of all child mappings before the domain is removed. */ + for (i = 0; i < eint->hw->ap_num; i++) { + virq = irq_find_mapping(eint->domain, i); + if (virq) + irq_dispose_mapping(virq); + } + + irq_domain_remove(eint->domain); +} + int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) { unsigned int size, i, port, virq, inst = 0; @@ -601,7 +624,7 @@ int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) irq_set_chained_handler_and_data(eint->irq, mtk_eint_irq_handler, eint); - return 0; + return devm_add_action_or_reset(eint->dev, mtk_eint_teardown, eint); err_eint: for (i = 0; i < eint->nbase; i++) { diff --git a/drivers/pinctrl/mediatek/pinctrl-moore.c b/drivers/pinctrl/mediatek/pinctrl-moore.c index 17e30f83dc197..38f15dbe9a283 100644 --- a/drivers/pinctrl/mediatek/pinctrl-moore.c +++ b/drivers/pinctrl/mediatek/pinctrl-moore.c @@ -594,7 +594,7 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw) chip->base = -1; chip->ngpio = hw->soc->npins; - ret = gpiochip_add_data(chip, hw); + ret = devm_gpiochip_add_data(hw->dev, chip, hw); if (ret < 0) return ret; @@ -608,10 +608,8 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw) if (!of_property_present(hw->dev->of_node, "gpio-ranges")) { ret = gpiochip_add_pin_range(chip, dev_name(hw->dev), 0, 0, chip->ngpio); - if (ret < 0) { - gpiochip_remove(chip); + if (ret < 0) return ret; - } } return 0; diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c index dd2c8aa039385..791eddd7a2c63 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c @@ -1130,30 +1130,24 @@ int mtk_pctrl_init(struct platform_device *pdev, pctl->chip->parent = &pdev->dev; pctl->chip->base = -1; - ret = gpiochip_add_data(pctl->chip, pctl); + ret = devm_gpiochip_add_data(&pdev->dev, pctl->chip, pctl); if (ret) return -EINVAL; /* Register the GPIO to pin mappings. */ ret = gpiochip_add_pin_range(pctl->chip, dev_name(&pdev->dev), 0, 0, pctl->devdata->npins); - if (ret) { - ret = -EINVAL; - goto chip_error; - } + if (ret) + return -EINVAL; /* Only initialize EINT if we have EINT pins */ if (data->eint_hw.ap_num > 0) { ret = mtk_eint_init(pctl, pdev); if (ret) - goto chip_error; + return ret; } return 0; - -chip_error: - gpiochip_remove(pctl->chip); - return ret; } int mtk_pctrl_common_probe(struct platform_device *pdev) diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c index 23f04b24fd65e..09098b68f7257 100644 --- a/drivers/pinctrl/mediatek/pinctrl-paris.c +++ b/drivers/pinctrl/mediatek/pinctrl-paris.c @@ -957,7 +957,7 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw) chip->base = -1; chip->ngpio = hw->soc->npins; - ret = gpiochip_add_data(chip, hw); + ret = devm_gpiochip_add_data(hw->dev, chip, hw); if (ret < 0) return ret; diff --git a/drivers/pinctrl/meson/pinctrl-meson-s4.c b/drivers/pinctrl/meson/pinctrl-meson-s4.c index 872948699e9fe..365dafe457a9f 100644 --- a/drivers/pinctrl/meson/pinctrl-meson-s4.c +++ b/drivers/pinctrl/meson/pinctrl-meson-s4.c @@ -854,7 +854,7 @@ static const char * const i2c1_groups[] = { static const char * const i2c2_groups[] = { "i2c2_sda_d", "i2c2_scl_d", "i2c2_sda_h8", "i2c2_scl_h9", - "i2c2_sda_h0", "i2c2_scl_h1l," + "i2c2_sda_h0", "i2c2_scl_h1", }; static const char * const i2c3_groups[] = { diff --git a/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c b/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c index ea1026a0d22c8..92d38e5336de0 100644 --- a/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c +++ b/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c @@ -86,7 +86,7 @@ static struct mpfs_pinctrl_bank_voltage mpfs_pinctrl_bank_voltages[8] = { { .uv = 1800000, .val = 4 }, { .uv = 2500000, .val = 6 }, { .uv = 3300000, .val = 8 }, - { .uv = 0, .val = 0x3f }, // pin unused + { .uv = 0, .val = 0xf }, // pin unused }; static int mpfs_pinctrl_get_drive_strength_ma(u32 drive_strength) @@ -156,10 +156,10 @@ static void mpfs_pinctrl_set_bank_voltage(struct mpfs_pinctrl *pctrl, unsigned i u32 val = FIELD_PREP(MPFS_PINCTRL_BANK_VOLTAGE_MASK, bank_voltage); if (pin < MPFS_PINCTRL_BANK2_START) - regmap_assign_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK4_CFG_CR, + regmap_update_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK4_CFG_CR, MPFS_PINCTRL_BANK_VOLTAGE_MASK, val); else - regmap_assign_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK2_CFG_CR, + regmap_update_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK2_CFG_CR, MPFS_PINCTRL_BANK_VOLTAGE_MASK, val); } diff --git a/drivers/pinctrl/pinctrl-eic7700.c b/drivers/pinctrl/pinctrl-eic7700.c index d553ec20c6191..09a3b097383cb 100644 --- a/drivers/pinctrl/pinctrl-eic7700.c +++ b/drivers/pinctrl/pinctrl-eic7700.c @@ -422,6 +422,9 @@ static int eic7700_pin_config_set(struct pinctrl_dev *pctldev, unsigned int pin, else value &= ~EIC7700_ST; break; + /* All pins are persistent over suspend */ + case PIN_CONFIG_PERSIST_STATE: + return 0; default: return -EOPNOTSUPP; } diff --git a/drivers/pinctrl/pinctrl-generic.c b/drivers/pinctrl/pinctrl-generic.c index 9759b0186bcc2..4277c87485135 100644 --- a/drivers/pinctrl/pinctrl-generic.c +++ b/drivers/pinctrl/pinctrl-generic.c @@ -3,8 +3,10 @@ #define pr_fmt(fmt) "generic pinconfig core: " fmt #include +#include #include #include +#include #include #include @@ -42,33 +44,44 @@ int pinctrl_generic_to_map(struct pinctrl_dev *pctldev, struct device_node *pare ret = pinctrl_utils_add_map_mux(pctldev, maps, num_reserved_maps, num_maps, group_name, parent->name); if (ret < 0) - return ret; + goto err_free_map; ret = pinctrl_generic_add_group(pctldev, group_name, pins, npins, data); - if (ret < 0) - return dev_err_probe(dev, ret, "failed to add group %s: %d\n", + if (ret < 0) { + dev_err_probe(dev, ret, "failed to add group %s: %d\n", group_name, ret); + goto err_free_map; + } ret = pinconf_generic_parse_dt_config(np, pctldev, &configs, &num_configs); - if (ret) - return dev_err_probe(dev, ret, "failed to parse pin config of group %s\n", + if (ret) { + dev_err_probe(dev, ret, "failed to parse pin config of group %s\n", group_name); + goto err_free_map; + } if (num_configs == 0) return 0; ret = pinctrl_utils_reserve_map(pctldev, maps, num_reserved_maps, num_maps, reserve); if (ret) - return ret; + goto err_free_map; ret = pinctrl_utils_add_map_configs(pctldev, maps, num_reserved_maps, num_maps, group_name, configs, num_configs, PIN_MAP_TYPE_CONFIGS_GROUP); kfree(configs); if (ret) - return ret; + goto err_free_map; return 0; + +err_free_map: + pinctrl_utils_free_map(pctldev, *maps, *num_maps); + *maps = NULL; + *num_maps = 0; + *num_reserved_maps = 0; + return ret; }; EXPORT_SYMBOL_GPL(pinctrl_generic_to_map); @@ -185,6 +198,8 @@ static int pinctrl_generic_dt_node_to_map(struct pinctrl_dev *pctldev, int ngroups = 0; int ret; + guard(mutex)(&pctldev->mutex); + *maps = NULL; *num_maps = 0; diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c index 7e0fcd45fd265..7f7ea3b3638d9 100644 --- a/drivers/pinctrl/pinctrl-rockchip.c +++ b/drivers/pinctrl/pinctrl-rockchip.c @@ -3212,7 +3212,7 @@ static int rockchip_get_drive_perpin(struct rockchip_pin_bank *bank, u8 bit; int drv_type = bank->drv[pin_num / 8].drv_type; - if (ctrl->type == RV1103B && pin_num >= 12) + if (ctrl->type == RV1103B && bank->bank_num == 2 && pin_num >= 12) drv_type = DRV_TYPE_IO_LEVEL_2_BIT; ret = ctrl->drv_calc_reg(bank, pin_num, ®map, ®, &bit); @@ -3267,6 +3267,25 @@ static int rockchip_get_drive_perpin(struct rockchip_pin_bank *bank, case DRV_TYPE_IO_1V8_ONLY: rmask_bits = RK3288_DRV_BITS_PER_PIN; break; + case DRV_TYPE_IO_LEVEL_2_BIT: + ret = regmap_read(regmap, reg, &data); + if (ret) + return ret; + data >>= bit; + + return data & 0x3; + case DRV_TYPE_IO_LEVEL_8_BIT: + ret = regmap_read(regmap, reg, &data); + if (ret) + return ret; + data >>= bit; + data &= (1 << 8) - 1; + + ret = hweight8(data); + if (ret > 0) + return ret - 1; + else + return -EINVAL; default: dev_err(dev, "unsupported pinctrl drive type: %d\n", drv_type); return -EINVAL; @@ -3390,25 +3409,6 @@ static int rockchip_set_drive_perpin(struct rockchip_pin_bank *bank, case DRV_TYPE_IO_1V8_ONLY: rmask_bits = RK3288_DRV_BITS_PER_PIN; break; - case DRV_TYPE_IO_LEVEL_2_BIT: - ret = regmap_read(regmap, reg, &data); - if (ret) - return ret; - data >>= bit; - - return data & 0x3; - case DRV_TYPE_IO_LEVEL_8_BIT: - ret = regmap_read(regmap, reg, &data); - if (ret) - return ret; - data >>= bit; - data &= (1 << 8) - 1; - - ret = hweight8(data); - if (ret > 0) - return ret - 1; - else - return -EINVAL; default: dev_err(dev, "unsupported pinctrl drive type: %d\n", drv_type); return -EINVAL; @@ -4296,6 +4296,16 @@ static struct rockchip_pin_ctrl *rockchip_pinctrl_get_soc_data( pmu_offs = ctrl->pmu_mux_offset; drv_pmu_offs = ctrl->pmu_drv_offset; drv_grf_offs = ctrl->grf_drv_offset; + + /* + * This function mutates the static per-SoC data. Most of it is + * idempotent: recalculated iomux and drv offsets anchor at the + * values calculated by a previous run. The pin count is not, so + * reset it here; otherwise it accumulates when the probe runs + * again after a probe deferral, shifting every bank's pin_base. + */ + ctrl->nr_pins = 0; + bank = ctrl->pin_banks; for (i = 0; i < ctrl->nr_banks; ++i, ++bank) { int bank_pins = 0; diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 4d5f85b7e6bbe..e0eb7240a9859 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -1628,7 +1628,8 @@ static int pcs_irq_init_chained_handler(struct pcs_device *pcs, &pcs_irqdomain_ops, pcs_soc); if (!pcs->domain) { - irq_set_chained_handler(pcs_soc->irq, NULL); + pcs_irq_free(pcs); + pcs_soc->irq = -1; return -EINVAL; } diff --git a/drivers/pinctrl/qcom/pinctrl-ipq5210.c b/drivers/pinctrl/qcom/pinctrl-ipq5210.c index 827a4ad07a6b0..d5b4eb3e73435 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq5210.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq5210.c @@ -867,6 +867,7 @@ static const struct of_device_id ipq5210_tlmm_of_match[] = { { .compatible = "qcom,ipq5210-tlmm", }, { }, }; +MODULE_DEVICE_TABLE(of, ipq5210_tlmm_of_match); static int ipq5210_tlmm_probe(struct platform_device *pdev) { diff --git a/drivers/pinctrl/qcom/pinctrl-nord.c b/drivers/pinctrl/qcom/pinctrl-nord.c index 7c21306e77ff7..7f37f8e819bab 100644 --- a/drivers/pinctrl/qcom/pinctrl-nord.c +++ b/drivers/pinctrl/qcom/pinctrl-nord.c @@ -570,8 +570,10 @@ enum nord_functions { msm_mux_qup0_se5, msm_mux_qup1_se0, msm_mux_qup1_se1, - msm_mux_qup1_se2, - msm_mux_qup1_se3, + msm_mux_qup1_se2_01, + msm_mux_qup1_se2_23, + msm_mux_qup1_se3_01, + msm_mux_qup1_se3_23, msm_mux_qup1_se4, msm_mux_qup1_se5, msm_mux_qup1_se6, @@ -1152,11 +1154,19 @@ static const char *const qup1_se1_groups[] = { "gpio123", "gpio124", "gpio125", "gpio126", }; -static const char *const qup1_se2_groups[] = { - "gpio127", "gpio128", "gpio129", "gpio130", +static const char *const qup1_se2_01_groups[] = { + "gpio127", "gpio128", }; -static const char *const qup1_se3_groups[] = { +static const char *const qup1_se2_23_groups[] = { + "gpio127", "gpio128", +}; + +static const char *const qup1_se3_01_groups[] = { + "gpio129", "gpio130", +}; + +static const char *const qup1_se3_23_groups[] = { "gpio129", "gpio130", }; @@ -1428,8 +1438,10 @@ static const struct pinfunction nord_functions[] = { MSM_PIN_FUNCTION(qup0_se5), MSM_PIN_FUNCTION(qup1_se0), MSM_PIN_FUNCTION(qup1_se1), - MSM_PIN_FUNCTION(qup1_se2), - MSM_PIN_FUNCTION(qup1_se3), + MSM_PIN_FUNCTION(qup1_se2_01), + MSM_PIN_FUNCTION(qup1_se2_23), + MSM_PIN_FUNCTION(qup1_se3_01), + MSM_PIN_FUNCTION(qup1_se3_23), MSM_PIN_FUNCTION(qup1_se4), MSM_PIN_FUNCTION(qup1_se5), MSM_PIN_FUNCTION(qup1_se6), @@ -1633,13 +1645,13 @@ static const struct msm_pingroup nord_groups[] = { _, _, _, _, _, _, _), [126] = PINGROUP(126, qup1_se1, qup1_se0, ccu_i2c_scl, mdp1_vsync_out, _, atest_usb20, ddr_pxi, _, _, _, _), - [127] = PINGROUP(127, qup1_se2, qup1_se2, _, atest_usb21, ddr_pxi, + [127] = PINGROUP(127, qup1_se2_23, qup1_se2_01, _, atest_usb21, ddr_pxi, _, _, _, _, _, _), - [128] = PINGROUP(128, qup1_se2, qup1_se2, _, atest_usb20, ddr_pxi, + [128] = PINGROUP(128, qup1_se2_23, qup1_se2_01, _, atest_usb20, ddr_pxi, _, _, _, _, _, _), - [129] = PINGROUP(129, qup1_se3, qup1_se3, ccu_i2c_sda, mdp1_vsync_out, + [129] = PINGROUP(129, qup1_se3_23, qup1_se3_01, ccu_i2c_sda, mdp1_vsync_out, _, atest_usb21, ddr_pxi, _, _, _, _), - [130] = PINGROUP(130, qup1_se3, qup1_se3, ccu_i2c_scl, mdp1_vsync_out, + [130] = PINGROUP(130, qup1_se3_23, qup1_se3_01, ccu_i2c_scl, mdp1_vsync_out, _, atest_usb20, ddr_pxi, _, _, _, _), [131] = PINGROUP(131, qup1_se4, qup1_se6, ccu_i2c_sda, mdp1_vsync_out, _, atest_usb21, ddr_pxi, _, _, _, _), diff --git a/drivers/pinctrl/qcom/pinctrl-shikra.c b/drivers/pinctrl/qcom/pinctrl-shikra.c index 0fc98369948cb..55aec2f675e6b 100644 --- a/drivers/pinctrl/qcom/pinctrl-shikra.c +++ b/drivers/pinctrl/qcom/pinctrl-shikra.c @@ -44,6 +44,7 @@ .intr_status_bit = 0, \ .intr_wakeup_enable_bit = 7, \ .intr_wakeup_present_bit = 6, \ + .intr_target_width = 4, \ .intr_target_bit = 8, \ .intr_target_kpss_val = 3, \ .intr_raw_status_bit = 4, \ diff --git a/drivers/pinctrl/spacemit/pinctrl-k1.c b/drivers/pinctrl/spacemit/pinctrl-k1.c index f0b5ebd9e223c..c3a7538783b8a 100644 --- a/drivers/pinctrl/spacemit/pinctrl-k1.c +++ b/drivers/pinctrl/spacemit/pinctrl-k1.c @@ -503,13 +503,14 @@ static int spacemit_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin, unsigned long *config) { struct spacemit_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev); + const struct spacemit_pin *spin = spacemit_get_pin(pctrl, pin); int param = pinconf_to_config_param(*config); u32 value, arg = 0; - if (!pin) + if (!spin) return -EINVAL; - value = readl(spacemit_pin_to_reg(pctrl, pin)); + value = readl(spacemit_pin_to_reg(pctrl, spin->pin)); switch (param) { case PIN_CONFIG_SLEW_RATE: @@ -689,6 +690,11 @@ static int spacemit_pinconf_group_set(struct pinctrl_dev *pctldev, if (ret) return ret; + for (i = 0; i < group->grp.npins; i++) { + if (!spacemit_get_pin(pctrl, group->grp.pins[i])) + return -EINVAL; + } + for (i = 0; i < group->grp.npins; i++) spacemit_pin_set_config(pctrl, group->grp.pins[i], value); diff --git a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c index 462aa1c4a5fa6..e27e4945def26 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c +++ b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c @@ -26,7 +26,7 @@ static const u8 a523_r_irq_bank_muxes[SUNXI_PINCTRL_MAX_BANKS] = static struct sunxi_pinctrl_desc a523_r_pinctrl_data = { .irq_banks = ARRAY_SIZE(a523_r_irq_bank_map), .irq_bank_map = a523_r_irq_bank_map, - .io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_SEL, + .io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV, .pin_base = PL_BASE, }; diff --git a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c index b6f78f1f30ac4..88d8acd5bc245 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c +++ b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c @@ -26,7 +26,7 @@ static const u8 a523_irq_bank_muxes[SUNXI_PINCTRL_MAX_BANKS] = static struct sunxi_pinctrl_desc a523_pinctrl_data = { .irq_banks = ARRAY_SIZE(a523_irq_bank_map), .irq_bank_map = a523_irq_bank_map, - .io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_SEL, + .io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV, }; static int a523_pinctrl_probe(struct platform_device *pdev) diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c index 25489beeb3125..31cd142ce0f79 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c +++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c @@ -728,6 +728,7 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl, { unsigned short bank; unsigned long flags; + bool inverted = false; u32 val, reg; int uV; @@ -766,6 +767,9 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl, reg &= ~IO_BIAS_MASK; writel(reg | val, pctl->membase + sunxi_grp_config_reg(pin)); return 0; + case BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV: + inverted = true; + fallthrough; case BIAS_VOLTAGE_PIO_POW_MODE_CTL: val = uV > 1800000 && uV <= 2500000 ? BIT(bank) : 0; @@ -780,6 +784,8 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl, fallthrough; case BIAS_VOLTAGE_PIO_POW_MODE_SEL: val = uV <= 1800000 ? 1 : 0; + if (inverted) + val = !val; raw_spin_lock_irqsave(&pctl->lock, flags); reg = readl(pctl->membase + pctl->pow_mod_sel_offset); @@ -837,6 +843,21 @@ static void sunxi_pmx_set(struct pinctrl_dev *pctldev, writel((readl(pctl->membase + reg) & ~mask) | config << shift, pctl->membase + reg); + /* + * A pin muxed to gpio_out directly through a pinmux node bypasses + * sunxi_pinctrl_gpio_set() and drives whatever its output latch + * holds. Now that the pin is in output mode the data register + * reads back the latch, so refresh the shadow to keep such pins + * driving their pre-existing level. + */ + if (config == SUN4I_FUNC_OUTPUT) { + u32 *shadow = &pctl->dat_shadow[pin / PINS_PER_BANK]; + + sunxi_data_reg(pctl, pin, ®, &shift, &mask); + *shadow = (*shadow & ~mask) | + (readl(pctl->membase + reg) & mask); + } + raw_spin_unlock_irqrestore(&pctl->lock, flags); } @@ -1017,21 +1038,29 @@ static int sunxi_pinctrl_gpio_set(struct gpio_chip *chip, unsigned int offset, int value) { struct sunxi_pinctrl *pctl = gpiochip_get_data(chip); - u32 reg, shift, mask, val; + u32 *shadow = &pctl->dat_shadow[offset / PINS_PER_BANK]; + u32 reg, shift, mask; unsigned long flags; sunxi_data_reg(pctl, offset, ®, &shift, &mask); raw_spin_lock_irqsave(&pctl->lock, flags); - val = readl(pctl->membase + reg); - + /* + * Reading the data register returns the pin level, not the output + * latch, for pins muxed as inputs. A read-modify-write based on + * the register would therefore corrupt the latches of input-muxed + * pins in the same bank (e.g. an emulated open-drain I2C line + * released high), making them drive the wrong level once switched + * to output. Base the read-modify-write on a shadow copy of the + * latches instead. + */ if (value) - val |= mask; + *shadow |= mask; else - val &= ~mask; + *shadow &= ~mask; - writel(val, pctl->membase + reg); + writel(*shadow, pctl->membase + reg); raw_spin_unlock_irqrestore(&pctl->lock, flags); @@ -1572,7 +1601,7 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev, struct pinctrl_pin_desc *pins; struct sunxi_pinctrl *pctl; struct pinmux_ops *pmxops; - int i, ret, last_pin, pin_idx; + int i, ret, last_pin, pin_idx, nbanks; struct clk *clk; pctl = devm_kzalloc(&pdev->dev, sizeof(*pctl), GFP_KERNEL); @@ -1610,6 +1639,37 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev, if (!pctl->irq_array) return -ENOMEM; + /* + * The bus clock has to be enabled before the pinctrl device + * registers, as the pin hogs claimed from there access registers. + */ + ret = of_clk_get_parent_count(node); + clk = devm_clk_get_enabled(&pdev->dev, ret == 1 ? NULL : "apb"); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + /* + * Seed the output latch shadow from the hardware so pins the + * bootloader left in output mode keep their state; see + * sunxi_pinctrl_gpio_set() for why a shadow is needed. This must + * happen before the pinctrl device registers, as pin hogs can mux + * pins to gpio_out and thereby update the shadow. + */ + last_pin = pctl->desc->pins[pctl->desc->npins - 1].pin.number; + nbanks = DIV_ROUND_UP(last_pin + 1 - pctl->desc->pin_base, + PINS_PER_BANK); + pctl->dat_shadow = devm_kcalloc(&pdev->dev, nbanks, + sizeof(*pctl->dat_shadow), GFP_KERNEL); + if (!pctl->dat_shadow) + return -ENOMEM; + + for (i = 0; i < nbanks; i++) { + u32 reg, shift, mask; + + sunxi_data_reg(pctl, i * PINS_PER_BANK, ®, &shift, &mask); + pctl->dat_shadow[i] = readl(pctl->membase + reg); + } + ret = sunxi_pinctrl_build_state(pdev); if (ret) { dev_err(&pdev->dev, "dt probe failed: %d\n", ret); @@ -1665,7 +1725,6 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev, if (!pctl->chip) return -ENOMEM; - last_pin = pctl->desc->pins[pctl->desc->npins - 1].pin.number; pctl->chip->owner = THIS_MODULE; pctl->chip->request = gpiochip_generic_request; pctl->chip->free = gpiochip_generic_free; @@ -1699,13 +1758,6 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev, goto gpiochip_error; } - ret = of_clk_get_parent_count(node); - clk = devm_clk_get_enabled(&pdev->dev, ret == 1 ? NULL : "apb"); - if (IS_ERR(clk)) { - ret = PTR_ERR(clk); - goto gpiochip_error; - } - pctl->irq = devm_kcalloc(&pdev->dev, pctl->desc->irq_banks, sizeof(*pctl->irq), diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.h b/drivers/pinctrl/sunxi/pinctrl-sunxi.h index 0daf7600e2fb0..8bd00c6ff6280 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sunxi.h +++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.h @@ -85,6 +85,7 @@ #define IO_BIAS_MASK GENMASK(3, 0) #define SUN4I_FUNC_INPUT 0 +#define SUN4I_FUNC_OUTPUT 1 #define SUN4I_FUNC_IRQ 6 #define SUN4I_FUNC_DISABLED_OLD 7 #define SUN4I_FUNC_DISABLED_NEW 15 @@ -116,8 +117,10 @@ enum sunxi_desc_bias_voltage { * Bias voltage is set through PIO_POW_MOD_SEL_REG * and PIO_POW_MOD_CTL_REG register, as seen on * A100 and D1 SoC, for example. + * Some SoCs invert the encoding for 1.8V vs. 3.3V. */ BIAS_VOLTAGE_PIO_POW_MODE_CTL, + BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV, }; struct sunxi_desc_function { @@ -175,6 +178,12 @@ struct sunxi_pinctrl { int *irq; unsigned *irq_array; raw_spinlock_t lock; + /* + * Output latch shadow, one word per bank. Seeded lockless at + * probe before the pinctrl device registers, protected by @lock + * afterwards. + */ + u32 *dat_shadow; struct pinctrl_dev *pctl_dev; unsigned long flags; u32 bank_mem_size; diff --git a/drivers/pinctrl/tegra/pinctrl-tegra238.c b/drivers/pinctrl/tegra/pinctrl-tegra238.c index ec482365f14f2..40aba285944eb 100644 --- a/drivers/pinctrl/tegra/pinctrl-tegra238.c +++ b/drivers/pinctrl/tegra/pinctrl-tegra238.c @@ -1744,57 +1744,57 @@ static const char * const tegra238_functions[] = { #define drive_sdmmc1_dat0_pu2 DRV_PINGROUP_ENTRY_Y(0x8034, 28, 2, 30, 2, -1, -1, -1, -1, 0) #define drive_ufs0_rst_n_pv1 DRV_PINGROUP_ENTRY_Y(0x11004, 12, 5, 24, 5, -1, -1, -1, -1, 0) #define drive_ufs0_ref_clk_pv0 DRV_PINGROUP_ENTRY_Y(0x1100c, 12, 5, 24, 5, -1, -1, -1, -1, 0) -#define drive_batt_oc_paa4 DRV_PINGROUP_ENTRY_Y(0x1024, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_bootv_ctl_n_paa0 DRV_PINGROUP_ENTRY_Y(0x102c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_vcomp_alert_paa2 DRV_PINGROUP_ENTRY_Y(0x105c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_hdmi_cec_pbb0 DRV_PINGROUP_ENTRY_Y(0x1064, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_touch_clk_pdd3 DRV_PINGROUP_ENTRY_Y(0x106c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart3_rx_pcc6 DRV_PINGROUP_ENTRY_Y(0x1074, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart3_tx_pcc5 DRV_PINGROUP_ENTRY_Y(0x107c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_gen8_i2c_sda_pdd2 DRV_PINGROUP_ENTRY_Y(0x1084, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_gen8_i2c_scl_pdd1 DRV_PINGROUP_ENTRY_Y(0x108c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_spi2_mosi_pcc2 DRV_PINGROUP_ENTRY_Y(0x1094, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_gen2_i2c_scl_pcc7 DRV_PINGROUP_ENTRY_Y(0x109c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_spi2_cs0_pcc3 DRV_PINGROUP_ENTRY_Y(0x10a4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_gen2_i2c_sda_pdd0 DRV_PINGROUP_ENTRY_Y(0x10ac, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_spi2_sck_pcc0 DRV_PINGROUP_ENTRY_Y(0x10b4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_spi2_miso_pcc1 DRV_PINGROUP_ENTRY_Y(0x10bc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio49_pee2 DRV_PINGROUP_ENTRY_Y(0x10c4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio50_pee4 DRV_PINGROUP_ENTRY_Y(0x10cc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio82_pee3 DRV_PINGROUP_ENTRY_Y(0x10d4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio71_pff2 DRV_PINGROUP_ENTRY_Y(0x10dc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio76_pff7 DRV_PINGROUP_ENTRY_Y(0x10e4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio74_pff5 DRV_PINGROUP_ENTRY_Y(0x10ec, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio00_paa1 DRV_PINGROUP_ENTRY_Y(0x10f4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio19_pdd6 DRV_PINGROUP_ENTRY_Y(0x10fc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio86_phh3 DRV_PINGROUP_ENTRY_Y(0x1104, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio72_pff3 DRV_PINGROUP_ENTRY_Y(0x110c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio77_pgg0 DRV_PINGROUP_ENTRY_Y(0x1114, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio80_pff6 DRV_PINGROUP_ENTRY_Y(0x111c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio84_pgg1 DRV_PINGROUP_ENTRY_Y(0x1124, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio83_pee5 DRV_PINGROUP_ENTRY_Y(0x112c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio73_pff4 DRV_PINGROUP_ENTRY_Y(0x1134, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio70_pff1 DRV_PINGROUP_ENTRY_Y(0x113c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio04_paa5 DRV_PINGROUP_ENTRY_Y(0x1144, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio85_pgg6 DRV_PINGROUP_ENTRY_Y(0x114c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio69_pff0 DRV_PINGROUP_ENTRY_Y(0x1154, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio25_paa6 DRV_PINGROUP_ENTRY_Y(0x115c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio26_paa7 DRV_PINGROUP_ENTRY_Y(0x1164, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart5_tx_pgg7 DRV_PINGROUP_ENTRY_Y(0x116c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart5_rx_phh0 DRV_PINGROUP_ENTRY_Y(0x1174, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart2_tx_pgg2 DRV_PINGROUP_ENTRY_Y(0x117c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart2_rx_pgg3 DRV_PINGROUP_ENTRY_Y(0x1184, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart2_cts_pgg5 DRV_PINGROUP_ENTRY_Y(0x118c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart2_rts_pgg4 DRV_PINGROUP_ENTRY_Y(0x1194, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart5_cts_phh2 DRV_PINGROUP_ENTRY_Y(0x119c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart5_rts_phh1 DRV_PINGROUP_ENTRY_Y(0x11a4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_pwm7_pee1 DRV_PINGROUP_ENTRY_Y(0x11ac, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_pwm2_pdd7 DRV_PINGROUP_ENTRY_Y(0x11b4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_pwm3_pee0 DRV_PINGROUP_ENTRY_Y(0x11bc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_pwm1_paa3 DRV_PINGROUP_ENTRY_Y(0x11c4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_spi2_cs1_pcc4 DRV_PINGROUP_ENTRY_Y(0x11cc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_dmic1_clk_pdd4 DRV_PINGROUP_ENTRY_Y(0x11d4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_dmic1_dat_pdd5 DRV_PINGROUP_ENTRY_Y(0x11dc, 12, 5, 20, 5, -1, -1, -1, -1, 1) +#define drive_batt_oc_paa4 DRV_PINGROUP_ENTRY_Y(0x1024, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_bootv_ctl_n_paa0 DRV_PINGROUP_ENTRY_Y(0x102c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_vcomp_alert_paa2 DRV_PINGROUP_ENTRY_Y(0x105c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_hdmi_cec_pbb0 DRV_PINGROUP_ENTRY_Y(0x1064, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_touch_clk_pdd3 DRV_PINGROUP_ENTRY_Y(0x106c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart3_rx_pcc6 DRV_PINGROUP_ENTRY_Y(0x1074, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart3_tx_pcc5 DRV_PINGROUP_ENTRY_Y(0x107c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_gen8_i2c_sda_pdd2 DRV_PINGROUP_ENTRY_Y(0x1084, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_gen8_i2c_scl_pdd1 DRV_PINGROUP_ENTRY_Y(0x108c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_spi2_mosi_pcc2 DRV_PINGROUP_ENTRY_Y(0x1094, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_gen2_i2c_scl_pcc7 DRV_PINGROUP_ENTRY_Y(0x109c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_spi2_cs0_pcc3 DRV_PINGROUP_ENTRY_Y(0x10a4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_gen2_i2c_sda_pdd0 DRV_PINGROUP_ENTRY_Y(0x10ac, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_spi2_sck_pcc0 DRV_PINGROUP_ENTRY_Y(0x10b4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_spi2_miso_pcc1 DRV_PINGROUP_ENTRY_Y(0x10bc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio49_pee2 DRV_PINGROUP_ENTRY_Y(0x10c4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio50_pee4 DRV_PINGROUP_ENTRY_Y(0x10cc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio82_pee3 DRV_PINGROUP_ENTRY_Y(0x10d4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio71_pff2 DRV_PINGROUP_ENTRY_Y(0x10dc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio76_pff7 DRV_PINGROUP_ENTRY_Y(0x10e4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio74_pff5 DRV_PINGROUP_ENTRY_Y(0x10ec, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio00_paa1 DRV_PINGROUP_ENTRY_Y(0x10f4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio19_pdd6 DRV_PINGROUP_ENTRY_Y(0x10fc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio86_phh3 DRV_PINGROUP_ENTRY_Y(0x1104, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio72_pff3 DRV_PINGROUP_ENTRY_Y(0x110c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio77_pgg0 DRV_PINGROUP_ENTRY_Y(0x1114, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio80_pff6 DRV_PINGROUP_ENTRY_Y(0x111c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio84_pgg1 DRV_PINGROUP_ENTRY_Y(0x1124, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio83_pee5 DRV_PINGROUP_ENTRY_Y(0x112c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio73_pff4 DRV_PINGROUP_ENTRY_Y(0x1134, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio70_pff1 DRV_PINGROUP_ENTRY_Y(0x113c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio04_paa5 DRV_PINGROUP_ENTRY_Y(0x1144, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio85_pgg6 DRV_PINGROUP_ENTRY_Y(0x114c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio69_pff0 DRV_PINGROUP_ENTRY_Y(0x1154, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio25_paa6 DRV_PINGROUP_ENTRY_Y(0x115c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio26_paa7 DRV_PINGROUP_ENTRY_Y(0x1164, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart5_tx_pgg7 DRV_PINGROUP_ENTRY_Y(0x116c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart5_rx_phh0 DRV_PINGROUP_ENTRY_Y(0x1174, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart2_tx_pgg2 DRV_PINGROUP_ENTRY_Y(0x117c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart2_rx_pgg3 DRV_PINGROUP_ENTRY_Y(0x1184, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart2_cts_pgg5 DRV_PINGROUP_ENTRY_Y(0x118c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart2_rts_pgg4 DRV_PINGROUP_ENTRY_Y(0x1194, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart5_cts_phh2 DRV_PINGROUP_ENTRY_Y(0x119c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart5_rts_phh1 DRV_PINGROUP_ENTRY_Y(0x11a4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_pwm7_pee1 DRV_PINGROUP_ENTRY_Y(0x11ac, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_pwm2_pdd7 DRV_PINGROUP_ENTRY_Y(0x11b4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_pwm3_pee0 DRV_PINGROUP_ENTRY_Y(0x11bc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_pwm1_paa3 DRV_PINGROUP_ENTRY_Y(0x11c4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_spi2_cs1_pcc4 DRV_PINGROUP_ENTRY_Y(0x11cc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_dmic1_clk_pdd4 DRV_PINGROUP_ENTRY_Y(0x11d4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_dmic1_dat_pdd5 DRV_PINGROUP_ENTRY_Y(0x11dc, 12, 5, 20, 5, -1, -1, -1, -1, 0) #define drive_sdmmc1_comp DRV_PINGROUP_ENTRY_N @@ -1961,57 +1961,57 @@ static const struct tegra_pingroup tegra238_groups[] = { }; static const struct tegra_pingroup tegra238_aon_groups[] = { - PINGROUP(bootv_ctl_n_paa0, RSVD0, RSVD1, RSVD2, RSVD3, 0x1028, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio00_paa1, RSVD0, RSVD1, RSVD2, RSVD3, 0x10f0, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(vcomp_alert_paa2, SOC_THERM_OC1, RSVD1, RSVD2, RSVD3, 0x1058, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(pwm1_paa3, GP_PWM1, RSVD1, RSVD2, RSVD3, 0x11c0, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(batt_oc_paa4, SOC_THERM_OC2, RSVD1, RSVD2, RSVD3, 0x1020, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio04_paa5, RSVD0, RSVD1, RSVD2, RSVD3, 0x1140, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio25_paa6, RSVD0, RSVD1, RSVD2, RSVD3, 0x1158, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio26_paa7, RSVD0, SOC_THERM_OC3, RSVD2, RSVD3, 0x1160, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(hdmi_cec_pbb0, HDMI_CEC, RSVD1, RSVD2, RSVD3, 0x1060, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(spi2_sck_pcc0, SPI2_SCK, RSVD1, RSVD2, RSVD3, 0x10b0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(spi2_miso_pcc1, SPI2_DIN, RSVD1, RSVD2, RSVD3, 0x10b8, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(spi2_mosi_pcc2, SPI2_DOUT, RSVD1, RSVD2, RSVD3, 0x1090, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(spi2_cs0_pcc3, SPI2_CS0, RSVD1, RSVD2, RSVD3, 0x10a0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(spi2_cs1_pcc4, SPI2_CS1, RSVD1, RSVD2, RSVD3, 0x11c8, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(uart3_tx_pcc5, UARTC_TXD, RSVD1, RSVD2, RSVD3, 0x1078, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart3_rx_pcc6, UARTC_RXD, RSVD1, RSVD2, RSVD3, 0x1070, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(gen2_i2c_scl_pcc7, I2C2_CLK, RSVD1, RSVD2, RSVD3, 0x1098, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(gen2_i2c_sda_pdd0, I2C2_DAT, RSVD1, RSVD2, RSVD3, 0x10a8, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(gen8_i2c_scl_pdd1, I2C8_CLK, RSVD1, RSVD2, RSVD3, 0x1088, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(gen8_i2c_sda_pdd2, I2C8_DAT, RSVD1, RSVD2, RSVD3, 0x1080, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(touch_clk_pdd3, GP_PWM4, TOUCH_CLK, RSVD2, RSVD3, 0x1068, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(dmic1_clk_pdd4, DMIC1_CLK, RSVD1, DMIC5_CLK, RSVD3, 0x11d0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(dmic1_dat_pdd5, DMIC1_DAT, RSVD1, DMIC5_DAT, RSVD3, 0x11d8, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio19_pdd6, RSVD0, WDT_RESET_OUTB, RSVD2, RSVD3, 0x10f8, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio49_pee2, RSVD0, RSVD1, RSVD2, RSVD3, 0x10c0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio50_pee4, RSVD0, RSVD1, RSVD2, RSVD3, 0x10c8, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio82_pee3, RSVD0, RSVD1, RSVD2, RSVD3, 0x10d0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio71_pff2, PPC_MODE_1, RSVD1, RSVD2, RSVD3, 0x10d8, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio76_pff7, RSVD0, RSVD1, TSC_EDGE_OUT0, TSC_EDGE_OUT0A, 0x10e0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio74_pff5, PPC_READY, PPC_I2C_DAT, RSVD2, RSVD3, 0x10e8, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio86_phh3, RSVD0, SPI5_CS1, TSC_EDGE_OUT3, TSC_EDGE_OUT0D, 0x1100, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio72_pff3, PPC_MODE_2, RSVD1, RSVD2, RSVD3, 0x1108, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio77_pgg0, RSVD0, RSVD1, TSC_EDGE_OUT1, TSC_EDGE_OUT0B, 0x1110, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio80_pff6, RSVD0, PPC_RST_N, RSVD2, RSVD3, 0x1118, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio84_pgg1, RSVD0, RSVD1, TSC_EDGE_OUT2, TSC_EDGE_OUT0C, 0x1120, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio83_pee5, RSVD0, RSVD1, RSVD2, RSVD3, 0x1128, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio73_pff4, PPC_CC, PPC_I2C_CLK, RSVD2, RSVD3, 0x1130, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio70_pff1, PPC_MODE_0, RSVD1, RSVD2, RSVD3, 0x1138, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio85_pgg6, RSVD0, SPI4_CS1, RSVD2, RSVD3, 0x1148, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio69_pff0, PPC_INT_N, RSVD1, RSVD2, RSVD3, 0x1150, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart5_tx_pgg7, UARTE_TXD, SPI5_SCK, RSVD2, RSVD3, 0x1168, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart5_rx_phh0, UARTE_RXD, SPI5_MISO, RSVD2, RSVD3, 0x1170, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart2_tx_pgg2, UARTB_TXD, SPI4_SCK, RSVD2, RSVD3, 0x1178, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart2_rx_pgg3, UARTB_RXD, SPI4_MISO, RSVD2, RSVD3, 0x1180, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart2_cts_pgg5, UARTB_CTS, SPI4_CS0, RSVD2, RSVD3, 0x1188, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart2_rts_pgg4, UARTB_RTS, SPI4_MOSI, RSVD2, RSVD3, 0x1190, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart5_cts_phh2, UARTE_CTS, SPI5_CS0, RSVD2, RSVD3, 0x1198, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart5_rts_phh1, UARTE_RTS, SPI5_MOSI, RSVD2, RSVD3, 0x11a0, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(pwm2_pdd7, GP_PWM2, LED_BLINK, RSVD2, RSVD3, 0x11b0, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(pwm3_pee0, GP_PWM3, RSVD1, RSVD2, RSVD3, 0x11b8, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(pwm7_pee1, GP_PWM7, RSVD1, RSVD2, RSVD3, 0x11a8, 1, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(bootv_ctl_n_paa0, RSVD0, RSVD1, RSVD2, RSVD3, 0x1028, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio00_paa1, RSVD0, RSVD1, RSVD2, RSVD3, 0x10f0, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(vcomp_alert_paa2, SOC_THERM_OC1, RSVD1, RSVD2, RSVD3, 0x1058, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(pwm1_paa3, GP_PWM1, RSVD1, RSVD2, RSVD3, 0x11c0, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(batt_oc_paa4, SOC_THERM_OC2, RSVD1, RSVD2, RSVD3, 0x1020, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio04_paa5, RSVD0, RSVD1, RSVD2, RSVD3, 0x1140, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio25_paa6, RSVD0, RSVD1, RSVD2, RSVD3, 0x1158, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio26_paa7, RSVD0, SOC_THERM_OC3, RSVD2, RSVD3, 0x1160, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(hdmi_cec_pbb0, HDMI_CEC, RSVD1, RSVD2, RSVD3, 0x1060, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(spi2_sck_pcc0, SPI2_SCK, RSVD1, RSVD2, RSVD3, 0x10b0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(spi2_miso_pcc1, SPI2_DIN, RSVD1, RSVD2, RSVD3, 0x10b8, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(spi2_mosi_pcc2, SPI2_DOUT, RSVD1, RSVD2, RSVD3, 0x1090, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(spi2_cs0_pcc3, SPI2_CS0, RSVD1, RSVD2, RSVD3, 0x10a0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(spi2_cs1_pcc4, SPI2_CS1, RSVD1, RSVD2, RSVD3, 0x11c8, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(uart3_tx_pcc5, UARTC_TXD, RSVD1, RSVD2, RSVD3, 0x1078, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart3_rx_pcc6, UARTC_RXD, RSVD1, RSVD2, RSVD3, 0x1070, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(gen2_i2c_scl_pcc7, I2C2_CLK, RSVD1, RSVD2, RSVD3, 0x1098, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(gen2_i2c_sda_pdd0, I2C2_DAT, RSVD1, RSVD2, RSVD3, 0x10a8, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(gen8_i2c_scl_pdd1, I2C8_CLK, RSVD1, RSVD2, RSVD3, 0x1088, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(gen8_i2c_sda_pdd2, I2C8_DAT, RSVD1, RSVD2, RSVD3, 0x1080, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(touch_clk_pdd3, GP_PWM4, TOUCH_CLK, RSVD2, RSVD3, 0x1068, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(dmic1_clk_pdd4, DMIC1_CLK, RSVD1, DMIC5_CLK, RSVD3, 0x11d0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(dmic1_dat_pdd5, DMIC1_DAT, RSVD1, DMIC5_DAT, RSVD3, 0x11d8, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio19_pdd6, RSVD0, WDT_RESET_OUTB, RSVD2, RSVD3, 0x10f8, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio49_pee2, RSVD0, RSVD1, RSVD2, RSVD3, 0x10c0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio50_pee4, RSVD0, RSVD1, RSVD2, RSVD3, 0x10c8, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio82_pee3, RSVD0, RSVD1, RSVD2, RSVD3, 0x10d0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio71_pff2, PPC_MODE_1, RSVD1, RSVD2, RSVD3, 0x10d8, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio76_pff7, RSVD0, RSVD1, TSC_EDGE_OUT0, TSC_EDGE_OUT0A, 0x10e0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio74_pff5, PPC_READY, PPC_I2C_DAT, RSVD2, RSVD3, 0x10e8, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio86_phh3, RSVD0, SPI5_CS1, TSC_EDGE_OUT3, TSC_EDGE_OUT0D, 0x1100, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio72_pff3, PPC_MODE_2, RSVD1, RSVD2, RSVD3, 0x1108, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio77_pgg0, RSVD0, RSVD1, TSC_EDGE_OUT1, TSC_EDGE_OUT0B, 0x1110, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio80_pff6, RSVD0, PPC_RST_N, RSVD2, RSVD3, 0x1118, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio84_pgg1, RSVD0, RSVD1, TSC_EDGE_OUT2, TSC_EDGE_OUT0C, 0x1120, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio83_pee5, RSVD0, RSVD1, RSVD2, RSVD3, 0x1128, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio73_pff4, PPC_CC, PPC_I2C_CLK, RSVD2, RSVD3, 0x1130, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio70_pff1, PPC_MODE_0, RSVD1, RSVD2, RSVD3, 0x1138, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio85_pgg6, RSVD0, SPI4_CS1, RSVD2, RSVD3, 0x1148, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio69_pff0, PPC_INT_N, RSVD1, RSVD2, RSVD3, 0x1150, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart5_tx_pgg7, UARTE_TXD, SPI5_SCK, RSVD2, RSVD3, 0x1168, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart5_rx_phh0, UARTE_RXD, SPI5_MISO, RSVD2, RSVD3, 0x1170, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart2_tx_pgg2, UARTB_TXD, SPI4_SCK, RSVD2, RSVD3, 0x1178, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart2_rx_pgg3, UARTB_RXD, SPI4_MISO, RSVD2, RSVD3, 0x1180, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart2_cts_pgg5, UARTB_CTS, SPI4_CS0, RSVD2, RSVD3, 0x1188, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart2_rts_pgg4, UARTB_RTS, SPI4_MOSI, RSVD2, RSVD3, 0x1190, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart5_cts_phh2, UARTE_CTS, SPI5_CS0, RSVD2, RSVD3, 0x1198, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart5_rts_phh1, UARTE_RTS, SPI5_MOSI, RSVD2, RSVD3, 0x11a0, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(pwm2_pdd7, GP_PWM2, LED_BLINK, RSVD2, RSVD3, 0x11b0, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(pwm3_pee0, GP_PWM3, RSVD1, RSVD2, RSVD3, 0x11b8, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(pwm7_pee1, GP_PWM7, RSVD1, RSVD2, RSVD3, 0x11a8, 0, Y, 5, 7, 6, 8, -1, 10, 12), }; static const struct tegra_pinctrl_soc_data tegra238_pinctrl_aon = { diff --git a/drivers/pinctrl/tegra/pinctrl-tegra264.c b/drivers/pinctrl/tegra/pinctrl-tegra264.c index be64fba34dce4..f8afb0bdb68a8 100644 --- a/drivers/pinctrl/tegra/pinctrl-tegra264.c +++ b/drivers/pinctrl/tegra/pinctrl-tegra264.c @@ -96,8 +96,8 @@ enum { TEGRA_PIN_UART4_RTS_N_PV4, TEGRA_PIN_UART4_CTS_N_PV5, TEGRA_PIN_DAP2_CLK_PV6, - TEGRA_PIN_DAP2_DIN_PV7, - TEGRA_PIN_DAP2_DOUT_PW0, + TEGRA_PIN_DAP2_DIN_PW0, + TEGRA_PIN_DAP2_DOUT_PV7, TEGRA_PIN_DAP2_FS_PW1, TEGRA_PIN_GEN1_I2C_SCL_PW2, TEGRA_PIN_GEN1_I2C_SDA_PW3, @@ -329,8 +329,8 @@ static const struct pinctrl_pin_desc tegra264_main_pins[] = { PINCTRL_PIN(TEGRA_PIN_UART4_RTS_N_PV4, "UART4_RTS_N_PV4"), PINCTRL_PIN(TEGRA_PIN_UART4_CTS_N_PV5, "UART4_CTS_N_PV5"), PINCTRL_PIN(TEGRA_PIN_DAP2_CLK_PV6, "DAP2_CLK_PV6"), - PINCTRL_PIN(TEGRA_PIN_DAP2_DIN_PV7, "DAP2_DIN_PV7"), - PINCTRL_PIN(TEGRA_PIN_DAP2_DOUT_PW0, "DAP2_DOUT_PW0"), + PINCTRL_PIN(TEGRA_PIN_DAP2_DIN_PW0, "DAP2_DIN_PW0"), + PINCTRL_PIN(TEGRA_PIN_DAP2_DOUT_PV7, "DAP2_DOUT_PV7"), PINCTRL_PIN(TEGRA_PIN_DAP2_FS_PW1, "DAP2_FS_PW1"), PINCTRL_PIN(TEGRA_PIN_GEN1_I2C_SCL_PW2, "GEN1_I2C_SCL_PW2"), PINCTRL_PIN(TEGRA_PIN_GEN1_I2C_SDA_PW3, "GEN1_I2C_SDA_PW3"), @@ -827,12 +827,12 @@ static const unsigned int dap2_clk_pv6_pins[] = { TEGRA_PIN_DAP2_CLK_PV6, }; -static const unsigned int dap2_din_pv7_pins[] = { - TEGRA_PIN_DAP2_DIN_PV7, +static const unsigned int dap2_din_pw0_pins[] = { + TEGRA_PIN_DAP2_DIN_PW0, }; -static const unsigned int dap2_dout_pw0_pins[] = { - TEGRA_PIN_DAP2_DOUT_PW0, +static const unsigned int dap2_dout_pv7_pins[] = { + TEGRA_PIN_DAP2_DOUT_PV7, }; static const unsigned int dap2_fs_pw1_pins[] = { @@ -1834,8 +1834,8 @@ static const char * const tegra264_functions[] = { #define drive_gen1_i2c_sda_pw3 DRV_PINGROUP_ENTRY_Y(0x601c, 12, 4, 20, 4, -1, -1, -1, -1, 0) #define drive_dap2_fs_pw1 DRV_PINGROUP_ENTRY_Y(0x6044, 12, 4, 20, 4, -1, -1, -1, -1, 0) #define drive_dap2_clk_pv6 DRV_PINGROUP_ENTRY_Y(0x604c, 12, 4, 20, 4, -1, -1, -1, -1, 0) -#define drive_dap2_din_pv7 DRV_PINGROUP_ENTRY_Y(0x6054, 12, 4, 20, 4, -1, -1, -1, -1, 0) -#define drive_dap2_dout_pw0 DRV_PINGROUP_ENTRY_Y(0x605c, 12, 4, 20, 4, -1, -1, -1, -1, 0) +#define drive_dap2_din_pw0 DRV_PINGROUP_ENTRY_Y(0x6054, 12, 4, 20, 4, -1, -1, -1, -1, 0) +#define drive_dap2_dout_pv7 DRV_PINGROUP_ENTRY_Y(0x605c, 12, 4, 20, 4, -1, -1, -1, -1, 0) #define drive_pwm10_pv1 DRV_PINGROUP_ENTRY_Y(0x6064, 12, 4, 20, 4, -1, -1, -1, -1, 0) #define drive_soc_gpio170_pu0 DRV_PINGROUP_ENTRY_Y(0x606c, 12, 4, 20, 4, -1, -1, -1, -1, 0) #define drive_soc_gpio171_pu1 DRV_PINGROUP_ENTRY_Y(0x6074, 12, 4, 20, 4, -1, -1, -1, -1, 0) @@ -2051,8 +2051,8 @@ static const struct tegra_pingroup tegra264_main_groups[] = { PINGROUP(gen1_i2c_sda_pw3, I2C1_DAT, RSVD1, RSVD2, RSVD3, 0x6018, 0, Y, 5, 7, 6, 8, -1, 10, 11), PINGROUP(dap2_fs_pw1, I2S2_LRCK, RSVD1, RSVD2, RSVD3, 0x6040, 0, Y, 5, 7, 6, 8, -1, 10, 11), PINGROUP(dap2_clk_pv6, I2S2_SCLK, RSVD1, RSVD2, RSVD3, 0x6048, 0, Y, 5, 7, 6, 8, -1, 10, 11), - PINGROUP(dap2_din_pv7, I2S2_SDATA_OUT, RSVD1, RSVD2, RSVD3, 0x6050, 0, Y, 5, 7, 6, 8, -1, 10, 11), - PINGROUP(dap2_dout_pw0, I2S2_SDATA_IN, RSVD1, RSVD2, RSVD3, 0x6058, 0, Y, 5, 7, 6, 8, -1, 10, 11), + PINGROUP(dap2_din_pw0, I2S2_SDATA_IN, RSVD1, RSVD2, RSVD3, 0x6050, 0, Y, 5, 7, 6, 8, -1, 10, 11), + PINGROUP(dap2_dout_pv7, I2S2_SDATA_OUT, RSVD1, RSVD2, RSVD3, 0x6058, 0, Y, 5, 7, 6, 8, -1, 10, 11), PINGROUP(pwm10_pv1, GP_PWM10, SDMMC1_CD, I2S7_LRCK, RSVD3, 0x6060, 0, Y, 5, 7, 6, 8, -1, 10, 11), PINGROUP(soc_gpio170_pu0, RSVD0, I2S7_SDATA_IN, CCLA_LA_TRIGGER_MUX, RSVD3, 0x6068, 0, Y, 5, 7, 6, 8, -1, 10, 11), PINGROUP(soc_gpio171_pu1, RSVD0, SPI4_SCK, RSVD2, RSVD3, 0x6070, 0, Y, 5, 7, 6, 8, -1, 10, 11), diff --git a/drivers/platform/arm64/qcom-hamoa-ec.c b/drivers/platform/arm64/qcom-hamoa-ec.c index 5ca7308c60774..4d2ad042a7f84 100644 --- a/drivers/platform/arm64/qcom-hamoa-ec.c +++ b/drivers/platform/arm64/qcom-hamoa-ec.c @@ -92,8 +92,10 @@ static int qcom_ec_read(struct qcom_ec *ec, u8 cmd, u8 resp_len, u8 *resp) return ret; else if (ret == 0 || ret == 0xff) return -EOPNOTSUPP; + else if (ret != resp_len) + return -EIO; - if (resp[0] >= resp_len) + if (resp[0] != resp_len - 1) return -EINVAL; return 0; diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c index 139cab6fcba17..c0cb50bd24413 100644 --- a/drivers/platform/chrome/cros_ec_debugfs.c +++ b/drivers/platform/chrome/cros_ec_debugfs.c @@ -512,7 +512,7 @@ static int cros_ec_debugfs_probe(struct platform_device *pd) ret = blocking_notifier_chain_register(&ec->ec_dev->panic_notifier, &debug_info->notifier_panic); if (ret) - goto remove_debugfs; + goto cleanup_console_log; ec->debug_info = debug_info; @@ -520,6 +520,8 @@ static int cros_ec_debugfs_probe(struct platform_device *pd) return 0; +cleanup_console_log: + cros_ec_cleanup_console_log(debug_info); remove_debugfs: debugfs_remove_recursive(debug_info->dir); return ret; @@ -529,6 +531,8 @@ static void cros_ec_debugfs_remove(struct platform_device *pd) { struct cros_ec_dev *ec = dev_get_drvdata(pd->dev.parent); + blocking_notifier_chain_unregister(&ec->ec_dev->panic_notifier, + &ec->debug_info->notifier_panic); debugfs_remove_recursive(ec->debug_info->dir); cros_ec_cleanup_console_log(ec->debug_info); } diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c index ac919c14c631e..1a89e90957cde 100644 --- a/drivers/platform/chrome/cros_ec_lightbar.c +++ b/drivers/platform/chrome/cros_ec_lightbar.c @@ -504,9 +504,14 @@ static ssize_t program_store(struct device *dev, struct device_attribute *attr, return -EINVAL; } } else { + /* + * Bound the payload strictly by the maximum value the structural + * size field can natively support. + */ extra_bytes = offsetof(typeof(*param), set_program_ex) + sizeof(param->set_program_ex); - max_size = ec->ec_dev->max_request - extra_bytes; + max_size = min_t(size_t, ec->ec_dev->max_request - extra_bytes, + type_max(typeof(param->set_program_ex.size))); } msg = alloc_lightbar_cmd_msg(ec); diff --git a/drivers/platform/chrome/cros_ec_sensorhub_ring.c b/drivers/platform/chrome/cros_ec_sensorhub_ring.c index a10579144c34b..d92b602137207 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub_ring.c +++ b/drivers/platform/chrome/cros_ec_sensorhub_ring.c @@ -475,6 +475,21 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub, fifo_timestamp, *current_timestamp, now); + + /* + * A standalone timestamp event typically has a sensor_num of + * 0xff. Return early here to prevent it from hitting the + * bounds check below and spamming the logs. + */ + return false; + } + + /* Skip event if sensor_num from EC is out of bounds. */ + if (in->sensor_num >= sensorhub->sensor_num) { + dev_warn_ratelimited(sensorhub->dev, + "Invalid sensor number %u from EC\n", + in->sensor_num); + return false; } if (in->flags & MOTIONSENSE_SENSOR_FLAG_ODR) { @@ -502,10 +517,6 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub, return true; } - if (in->flags & MOTIONSENSE_SENSOR_FLAG_TIMESTAMP) - /* If we just have a timestamp, skip this entry. */ - return false; - /* Regular sample */ out->sensor_id = in->sensor_num; trace_cros_ec_sensorhub_data(in->sensor_num, @@ -825,8 +836,15 @@ static void cros_ec_sensorhub_ring_handler(struct cros_ec_sensorhub *sensorhub) sensorhub->msg->outsize = 1; sensorhub->msg->insize = fifo_info_length; - if (cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg) < 0) + ret = cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg); + if (ret < 0) goto error; + if (ret != fifo_info_length) { + dev_warn_ratelimited(sensorhub->dev, + "Mismatch read length: size %d - expected %d\n", + ret, fifo_info_length); + goto error; + } memcpy(fifo_info, &sensorhub->resp->fifo_info, fifo_info_length); diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index c0806c562bb93..50a68819ceb7b 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -1119,6 +1119,12 @@ static void cros_typec_register_partner_pdos(struct cros_typec_data *typec, if (!resp->source_cap_count && !resp->sink_cap_count) return; + if (resp->source_cap_count > PDO_MAX_OBJECTS || + resp->sink_cap_count > PDO_MAX_OBJECTS) { + dev_warn(typec->dev, "Invalid PDO count from EC, port: %d\n", port_num); + return; + } + port->partner_pd = typec_partner_usb_power_delivery_register(port->partner, &desc); if (IS_ERR(port->partner_pd)) { dev_warn(typec->dev, "Failed to register partner PD device, port: %d\n", port_num); diff --git a/drivers/platform/mellanox/mlxbf-bootctl.c b/drivers/platform/mellanox/mlxbf-bootctl.c index f67c7f56ab2b7..9ddc7fa1a9732 100644 --- a/drivers/platform/mellanox/mlxbf-bootctl.c +++ b/drivers/platform/mellanox/mlxbf-bootctl.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include diff --git a/drivers/platform/mellanox/mlxbf-pmc.c b/drivers/platform/mellanox/mlxbf-pmc.c index 5ec1ad4716967..2ad9e2b0493c4 100644 --- a/drivers/platform/mellanox/mlxbf-pmc.c +++ b/drivers/platform/mellanox/mlxbf-pmc.c @@ -2262,13 +2262,19 @@ static int mlxbf_pmc_map_counters(struct device *dev) static int mlxbf_pmc_probe(struct platform_device *pdev) { - struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev); - const char *hid = acpi_device_hid(acpi_dev); struct device *dev = &pdev->dev; + struct acpi_device *acpi_dev; struct arm_smccc_res res; + const char *hid; guid_t guid; int ret; + acpi_dev = ACPI_COMPANION(&pdev->dev); + if (!acpi_dev) + return -ENODEV; + + hid = acpi_device_hid(acpi_dev); + /* Ensure we have the UUID we expect for this service. */ arm_smccc_smc(MLXBF_PMC_SIP_SVC_UID, 0, 0, 0, 0, 0, 0, 0, &res); guid_parse(mlxbf_pmc_svc_uuid_str, &guid); diff --git a/drivers/platform/surface/surface_acpi_notify.c b/drivers/platform/surface/surface_acpi_notify.c index a9dcb0bbe90ee..593a7aba62432 100644 --- a/drivers/platform/surface/surface_acpi_notify.c +++ b/drivers/platform/surface/surface_acpi_notify.c @@ -777,12 +777,16 @@ static int san_consumer_links_setup(struct platform_device *pdev) static int san_probe(struct platform_device *pdev) { - struct acpi_device *san = ACPI_COMPANION(&pdev->dev); struct ssam_controller *ctrl; + struct acpi_device *san; struct san_data *data; acpi_status astatus; int status; + san = ACPI_COMPANION(&pdev->dev); + if (!san) + return -ENODEV; + ctrl = ssam_client_bind(&pdev->dev); if (IS_ERR(ctrl)) return PTR_ERR(ctrl) == -ENODEV ? -EPROBE_DEFER : PTR_ERR(ctrl); diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index b54b5212b2047..96f3b7ad497aa 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -330,6 +330,21 @@ config ASUS_TF103C_DOCK If you have an Asus TF103C tablet say Y or M here, for a generic x86 distro config say M here. +config AYN_EC + tristate "AYN x86 devices EC platform control" + depends on ACPI + depends on HWMON + depends on NEW_LEDS + select LEDS_CLASS + select LEDS_CLASS_MULTICOLOR + help + This is a driver for AYN and Tectoy x86 handheld devices. It provides + temperature monitoring, manual fan speed control, fan curve control, + and chassis RGB settings. + + If you have an x86 AYN or Tectoy handheld device say M here. The module + will be called ayn-platform. + config AYANEO_EC tristate "Ayaneo EC platform control" depends on DMI diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile index 872ac3842391f..31b4fd9083948 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile @@ -40,6 +40,9 @@ obj-$(CONFIG_ASUS_TF103C_DOCK) += asus-tf103c-dock.o obj-$(CONFIG_EEEPC_LAPTOP) += eeepc-laptop.o obj-$(CONFIG_EEEPC_WMI) += eeepc-wmi.o +# Ayn +obj-$(CONFIG_AYN_EC) += ayn-ec.o + # Ayaneo obj-$(CONFIG_AYANEO_EC) += ayaneo-ec.o diff --git a/drivers/platform/x86/acer-wmi.c b/drivers/platform/x86/acer-wmi.c index e0eaaefb13d04..61ae622c93d94 100644 --- a/drivers/platform/x86/acer-wmi.c +++ b/drivers/platform/x86/acer-wmi.c @@ -1581,7 +1581,9 @@ static int WMI_gaming_execute_u32_u64(u32 method_id, u32 in, u64 *out) return -EIO; obj = result.pointer; - if (obj && out) { + if (!obj && out) { + ret = -ENOMSG; + } else if (obj && out) { switch (obj->type) { case ACPI_TYPE_INTEGER: *out = obj->integer.value; diff --git a/drivers/platform/x86/amd/hsmp/hwmon.c b/drivers/platform/x86/amd/hsmp/hwmon.c index 0cc9a742497fc..c8314eee06f4f 100644 --- a/drivers/platform/x86/amd/hsmp/hwmon.c +++ b/drivers/platform/x86/amd/hsmp/hwmon.c @@ -31,6 +31,9 @@ static int hsmp_hwmon_write(struct device *dev, enum hwmon_sensor_types type, if (attr != hwmon_power_cap) return -EOPNOTSUPP; + if (val < 0) + return -EINVAL; + msg.num_args = 1; msg.args[0] = val / MICROWATT_PER_MILLIWATT; msg.msg_id = HSMP_SET_SOCKET_POWER_LIMIT; diff --git a/drivers/platform/x86/amd/pmc/mp1_stb.c b/drivers/platform/x86/amd/pmc/mp1_stb.c index 753d630f3283d..1ec0e599df7f3 100644 --- a/drivers/platform/x86/amd/pmc/mp1_stb.c +++ b/drivers/platform/x86/amd/pmc/mp1_stb.c @@ -157,7 +157,7 @@ static int amd_stb_debugfs_open_v2(struct inode *inode, struct file *filp) struct amd_pmc_dev *dev = filp->f_inode->i_private; u32 fsize, num_samples, val, stb_rdptr_offset = 0; struct amd_stb_v2_data *stb_data_arr; - int ret; + int ret = 0; /* Write dummy postcode while reading the STB buffer */ ret = amd_stb_write(dev, AMD_PMC_STB_DUMMY_PC); @@ -176,22 +176,24 @@ static int amd_stb_debugfs_open_v2(struct inode *inode, struct file *filp) * the enhanced dram size. Note that we land here only for the * platforms that support enhanced dram size reporting. */ - if (dump_custom_stb) - return amd_stb_handle_efr(filp); + if (dump_custom_stb) { + ret = amd_stb_handle_efr(filp); + goto out; + } /* Get the num_samples to calculate the last push location */ ret = amd_pmc_send_cmd(dev, S2D_NUM_SAMPLES, &num_samples, dev->stb_arg.s2d_msg_id, true); - /* Clear msg_port for other SMU operation */ - dev->msg_port = MSG_PORT_PMC; if (ret) { dev_err(dev->dev, "error: S2D_NUM_SAMPLES not supported : %d\n", ret); - return ret; + goto out; } fsize = min(num_samples, S2D_TELEMETRY_BYTES_MAX); stb_data_arr = kmalloc_flex(*stb_data_arr, data, fsize); - if (!stb_data_arr) - return -ENOMEM; + if (!stb_data_arr) { + ret = -ENOMEM; + goto out; + } stb_data_arr->size = fsize; @@ -214,7 +216,10 @@ static int amd_stb_debugfs_open_v2(struct inode *inode, struct file *filp) filp->private_data = stb_data_arr; - return 0; +out: + /* Restore the default message port for subsequent SMU operations */ + dev->msg_port = MSG_PORT_PMC; + return ret; } static ssize_t amd_stb_debugfs_read_v2(struct file *filp, char __user *buf, size_t size, @@ -289,7 +294,7 @@ int amd_stb_s2d_init(struct amd_pmc_dev *dev) u32 phys_addr_low, phys_addr_hi; u64 stb_phys_addr; u32 size = 0; - int ret; + int ret = 0; if (!enable_stb) return 0; @@ -306,27 +311,42 @@ int amd_stb_s2d_init(struct amd_pmc_dev *dev) /* Spill to DRAM feature uses separate SMU message port */ dev->msg_port = MSG_PORT_S2D; - amd_pmc_send_cmd(dev, S2D_TELEMETRY_SIZE, &size, dev->stb_arg.s2d_msg_id, true); - if (size != S2D_TELEMETRY_BYTES_MAX) - return -EIO; + ret = amd_pmc_send_cmd(dev, S2D_TELEMETRY_SIZE, &size, dev->stb_arg.s2d_msg_id, true); + if (ret) + goto out; + if (size != S2D_TELEMETRY_BYTES_MAX) { + ret = -EIO; + goto out; + } - /* Get DRAM size */ - ret = amd_pmc_send_cmd(dev, S2D_DRAM_SIZE, &dev->dram_size, dev->stb_arg.s2d_msg_id, true); - if (ret || !dev->dram_size) + /* Get DRAM size; fall back to the default if the query fails */ + if (amd_pmc_send_cmd(dev, S2D_DRAM_SIZE, &dev->dram_size, dev->stb_arg.s2d_msg_id, true) || + !dev->dram_size) dev->dram_size = S2D_TELEMETRY_DRAMBYTES_MAX; /* Get STB DRAM address */ - amd_pmc_send_cmd(dev, S2D_PHYS_ADDR_LOW, &phys_addr_low, dev->stb_arg.s2d_msg_id, true); - amd_pmc_send_cmd(dev, S2D_PHYS_ADDR_HIGH, &phys_addr_hi, dev->stb_arg.s2d_msg_id, true); + ret = amd_pmc_send_cmd(dev, S2D_PHYS_ADDR_LOW, &phys_addr_low, + dev->stb_arg.s2d_msg_id, true); + if (ret) + goto out; + ret = amd_pmc_send_cmd(dev, S2D_PHYS_ADDR_HIGH, &phys_addr_hi, + dev->stb_arg.s2d_msg_id, true); + if (ret) + goto out; stb_phys_addr = ((u64)phys_addr_hi << 32 | phys_addr_low); - - /* Clear msg_port for other SMU operation */ - dev->msg_port = MSG_PORT_PMC; + if (!stb_phys_addr) { + dev_err(dev->dev, "S2D phys addr query returned invalid address\n"); + ret = -ENXIO; + goto out; + } dev->stb_virt_addr = devm_ioremap(dev->dev, stb_phys_addr, dev->dram_size); if (!dev->stb_virt_addr) - return -ENOMEM; + ret = -ENOMEM; - return 0; +out: + /* Restore the default message port for subsequent SMU operations */ + dev->msg_port = MSG_PORT_PMC; + return ret; } diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c index a37083cdb9083..542b5d04515ae 100644 --- a/drivers/platform/x86/amd/pmc/pmc.c +++ b/drivers/platform/x86/amd/pmc/pmc.c @@ -941,13 +941,17 @@ static int amd_pmc_probe(struct platform_device *pdev) amd_pmc_dbgfs_register(dev); err = amd_stb_s2d_init(dev); if (err) - goto err_pci_dev_put; + goto err_dbgfs_unregister; if (IS_ENABLED(CONFIG_AMD_MP2_STB)) amd_mp2_stb_init(dev); pm_report_max_hw_sleep(U64_MAX); return 0; +err_dbgfs_unregister: + amd_pmc_dbgfs_unregister(dev); + if (IS_ENABLED(CONFIG_SUSPEND)) + acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops); err_pci_dev_put: pci_dev_put(rdev); return err; diff --git a/drivers/platform/x86/asus-laptop.c b/drivers/platform/x86/asus-laptop.c index 140ac8a105372..19024fa3796e6 100644 --- a/drivers/platform/x86/asus-laptop.c +++ b/drivers/platform/x86/asus-laptop.c @@ -1887,7 +1887,7 @@ static int asus_acpi_probe(struct platform_device *pdev) if (result && result != -ENODEV) goto fail_pega_rfkill; - result = acpi_dev_install_notify_handler(device, ACPI_DEVICE_NOTIFY, + result = acpi_dev_install_notify_handler(device, ACPI_ALL_NOTIFY, asus_acpi_notify, asus); if (result) goto fail_pega_rfkill; @@ -1917,7 +1917,7 @@ static void asus_acpi_remove(struct platform_device *pdev) { struct asus_laptop *asus = platform_get_drvdata(pdev); - acpi_dev_remove_notify_handler(asus->device, ACPI_DEVICE_NOTIFY, + acpi_dev_remove_notify_handler(asus->device, ACPI_ALL_NOTIFY, asus_acpi_notify); asus_backlight_exit(asus); asus_rfkill_exit(asus); diff --git a/drivers/platform/x86/asus-wireless.c b/drivers/platform/x86/asus-wireless.c index 2b494bf3cba8c..aab45f0442c5b 100644 --- a/drivers/platform/x86/asus-wireless.c +++ b/drivers/platform/x86/asus-wireless.c @@ -132,6 +132,10 @@ static int asus_wireless_probe(struct platform_device *pdev) const struct acpi_device_id *id; int err; + id = acpi_match_acpi_device(device_ids, adev); + if (!id) + return -ENODEV; + data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -139,6 +143,7 @@ static int asus_wireless_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); data->adev = adev; + data->hswc_params = (const struct hswc_params *)id->driver_data; data->idev = devm_input_allocate_device(&pdev->dev); if (!data->idev) @@ -153,12 +158,6 @@ static int asus_wireless_probe(struct platform_device *pdev) if (err) return err; - id = acpi_match_acpi_device(device_ids, adev); - if (!id) - return 0; - - data->hswc_params = (const struct hswc_params *)id->driver_data; - data->wq = create_singlethread_workqueue("asus_wireless_workqueue"); if (!data->wq) return -ENOMEM; diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index e835779b6f5f1..c162fbdb5b106 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -5244,20 +5244,20 @@ static int asus_wmi_add(struct platform_device *pdev) return 0; fail_wmi_handler: + asus_screenpad_exit(asus); +fail_screenpad: asus_wmi_backlight_exit(asus); fail_backlight: asus_wmi_rfkill_exit(asus); -fail_screenpad: - asus_screenpad_exit(asus); fail_rfkill: asus_wmi_led_exit(asus); fail_leds: +fail_custom_fan_curve: fail_hwmon: asus_wmi_input_exit(asus); fail_input: asus_wmi_sysfs_exit(asus->platform_device); fail_sysfs: -fail_custom_fan_curve: fail_platform_profile_setup: fail_fan_boost_mode: fail_platform: diff --git a/drivers/platform/x86/ayaneo-ec.c b/drivers/platform/x86/ayaneo-ec.c index 41a24e0912486..81dac81262e7e 100644 --- a/drivers/platform/x86/ayaneo-ec.c +++ b/drivers/platform/x86/ayaneo-ec.c @@ -76,6 +76,13 @@ static const struct ayaneo_ec_quirk quirk_ayaneo3 = { }; static const struct dmi_system_id dmi_table[] = { + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "AYANEO"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "AYANEO 2S"), + }, + .driver_data = (void *)&quirk_charge_limit, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "AYANEO"), diff --git a/drivers/platform/x86/ayn-ec.c b/drivers/platform/x86/ayn-ec.c new file mode 100644 index 0000000000000..25f748d7db18e --- /dev/null +++ b/drivers/platform/x86/ayn-ec.c @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Platform driver for AYN x86 Handhelds. + * + * Implements multiple attributes provided by the EC. Fan reading and control, + * as well as temperature sensor readings are exposed via hwmon sysfs. EC RGB + * control is exposed via an led-class-multicolor interface. + * + * Fan control is provided via a pwm interface in the range [0-255]. AYN use + * [0-128] as the range in the EC, the written value is scaled to accommodate. + * The EC also provides a configurable fan curve with five set points that + * associate a temperature in Celcius [0-100] with a fan speed [0-128]. The + * auto_point fan speeds are also scaled from the range [0-255]. Temperature + * readings are scaled from degrees to millidegrees when read. + * + * RGB control is provided using 4 registers. One each for the colors red, + * green, and blue are [0-255]. There is also a effect register that takes + * switches between an EC controlled breathing that cycles through all colors + * and fades in/out, and manual, which enables setting a user defined color. + * + * Copyright (C) 2025 Derek J. Clark + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Fan speed and PWM registers */ +#define AYN_SENSOR_PWM_FAN_ENABLE_REG 0x10 /* PWM operating mode */ +#define AYN_SENSOR_PWM_FAN_SET_REG 0x11 /* PWM duty cycle */ +#define AYN_SENSOR_PWM_FAN_SPEED_REG 0x20 /* Fan speed */ + +/* EC controlled fan curve registers */ +#define AYN_SENSOR_PWM_FAN_SPEED_1_REG 0x12 +#define AYN_SENSOR_PWM_FAN_SPEED_2_REG 0x14 +#define AYN_SENSOR_PWM_FAN_SPEED_3_REG 0x16 +#define AYN_SENSOR_PWM_FAN_SPEED_4_REG 0x18 +#define AYN_SENSOR_PWM_FAN_SPEED_5_REG 0x1A +#define AYN_SENSOR_PWM_FAN_TEMP_1_REG 0x13 +#define AYN_SENSOR_PWM_FAN_TEMP_2_REG 0x15 +#define AYN_SENSOR_PWM_FAN_TEMP_3_REG 0x17 +#define AYN_SENSOR_PWM_FAN_TEMP_4_REG 0x19 +#define AYN_SENSOR_PWM_FAN_TEMP_5_REG 0x1B + +/* AYN EC PWM Fan modes */ +#define AYN_PWM_FAN_MODE_MANUAL 0x00 +#define AYN_PWM_FAN_MODE_AUTO 0x01 +#define AYN_PWM_FAN_MODE_EC_CURVE 0x02 + +/* hwmon fan modes */ +#define HWMON_PWM_FAN_MODE_FULL 0x00 +#define HWMON_PWM_FAN_MODE_MANUAL 0x01 +#define HWMON_PWM_FAN_MODE_AUTO 0x02 +#define HWMON_PWM_FAN_MODE_EC_CURVE 0x03 + +/* EC Temperature Sensors */ +#define AYN_SENSOR_BAT_TEMP_REG 0x04 /* Battery */ +#define AYN_SENSOR_CHARGE_TEMP_REG 0x07 /* Charger IC */ +#define AYN_SENSOR_MB_TEMP_REG 0x05 /* Motherboard */ +#define AYN_SENSOR_PROC_TEMP_REG 0x09 /* CPU Core */ +#define AYN_SENSOR_VCORE_TEMP_REG 0x08 /* vCore */ + +/* EC Controlled RGB registers */ +#define AYN_LED_MC_RED_REG 0xB0 /* Range 0x00-0xFF */ +#define AYN_LED_MC_GREEN_REG 0xB1 /* Range 0x00-0xFF */ +#define AYN_LED_MC_BLUE_REG 0xB2 /* Range 0x00-0xFF */ +#define AYN_RGB_EFFECT_REG 0xB3 + +/* RGB effect modes */ +#define AYN_RGB_EFFECT_BREATHE 0x00 +#define AYN_RGB_EFFECT_MONOCOLOR 0x55 +#define AYN_RGB_EFFECT_WRITE 0xAA + +/* Handle ACPI lock mechanism */ +#define ACPI_LOCK_DELAY_MS 500 + +int ayn_pwm_curve_registers[10] = { + AYN_SENSOR_PWM_FAN_SPEED_1_REG, + AYN_SENSOR_PWM_FAN_SPEED_2_REG, + AYN_SENSOR_PWM_FAN_SPEED_3_REG, + AYN_SENSOR_PWM_FAN_SPEED_4_REG, + AYN_SENSOR_PWM_FAN_SPEED_5_REG, + AYN_SENSOR_PWM_FAN_TEMP_1_REG, + AYN_SENSOR_PWM_FAN_TEMP_2_REG, + AYN_SENSOR_PWM_FAN_TEMP_3_REG, + AYN_SENSOR_PWM_FAN_TEMP_4_REG, + AYN_SENSOR_PWM_FAN_TEMP_5_REG, +}; + +struct ayn_device { + struct led_classdev *led_cdev; + u32 ayn_lock; /* ACPI EC Lock */ + u8 rgb_effect; +} drvdata; + +struct thermal_sensor { + char *name; + int reg; +}; + +static struct thermal_sensor thermal_sensors[] = { + { "Battery", AYN_SENSOR_BAT_TEMP_REG }, + { "Motherboard", AYN_SENSOR_MB_TEMP_REG }, + { "Charger IC", AYN_SENSOR_CHARGE_TEMP_REG }, + { "vCore", AYN_SENSOR_VCORE_TEMP_REG }, + { "CPU Core", AYN_SENSOR_PROC_TEMP_REG }, + {} +}; + +#define DEVICE_ATTR_RW_NAMED(_name, _attrname) \ + struct device_attribute dev_attr_##_name = { \ + .attr = { .name = _attrname, .mode = 0644 }, \ + .show = _name##_show, \ + .store = _name##_store, \ + } + +#define DEVICE_ATTR_RO_NAMED(_name, _attrname) \ + struct device_attribute dev_attr_##_name = { \ + .attr = { .name = _attrname, .mode = 0444 }, \ + .show = _name##_show, \ + } + +/* Handle ACPI lock mechanism */ +#define ACPI_LOCK_DELAY_MS 500 + +/* RGB effect values */ +enum RGB_EFFECT_OPTION { + BREATHE, + MONOCOLOR, +}; + +static const char *const RGB_EFFECT_TEXT[] = { + [BREATHE] = "breathe", + [MONOCOLOR] = "monocolor", +}; + +static bool lock_global_acpi_lock(void) +{ + return ACPI_SUCCESS(acpi_acquire_global_lock(ACPI_LOCK_DELAY_MS, + &drvdata.ayn_lock)); +} + +static bool unlock_global_acpi_lock(void) +{ + return ACPI_SUCCESS(acpi_release_global_lock(drvdata.ayn_lock)); +} + +/** + * read_from_ec() - Reads a value from the embedded controller. + * + * @reg: The register to start the read from. + * @size: The number of sequential registers the data is contained in. + * @val: Pointer to return the data with. + * + * Return: 0, or an error. + */ +static int read_from_ec(u8 reg, int size, long *val) +{ + int ret, i; + u8 buf; + + if (!lock_global_acpi_lock()) + return -EBUSY; + + *val = 0; + for (i = 0; i < size; i++) { + ret = ec_read(reg + i, &buf); + if (ret) + return ret; + *val <<= i * 8; + *val += buf; + } + + if (!unlock_global_acpi_lock()) + return -EBUSY; + + return 0; +} + +/** + * write_to_ec() - Writes a value to the embedded controller. + * + * @reg: The register to write to. + * @val: Value to write + * + * Return: 0, or an error. + */ +static int write_to_ec(u8 reg, u8 val) +{ + int ret; + + if (!lock_global_acpi_lock()) + return -EBUSY; + + pr_info("Writing EC value %d to register %u\n", val, reg); + ret = ec_write(reg, val); + + if (!unlock_global_acpi_lock()) + return -EBUSY; + + return ret; +} + +/** + * ayn_pwm_manual() - Enable manual control of the fan. + */ +static int ayn_pwm_manual(void) +{ + return write_to_ec(AYN_SENSOR_PWM_FAN_ENABLE_REG, 0x00); +} + +/** + * ayn_pwm_full() - Set fan to 100% speed. + */ +static int ayn_pwm_full(void) +{ + int ret; + + ret = write_to_ec(AYN_SENSOR_PWM_FAN_ENABLE_REG, 0x00); + if (ret) + return ret; + + return write_to_ec(AYN_SENSOR_PWM_FAN_SET_REG, 128); +} + +/** + * ayn_pwm_auto() - Enable automatic EC control of the fan. + */ +static int ayn_pwm_auto(void) +{ + return write_to_ec(AYN_SENSOR_PWM_FAN_ENABLE_REG, 0x01); +} + +/** + * ayn_pwm_ec_curve() - Enable manually setting the fan curve for automatic + * EC control of the fan. + */ +static int ayn_pwm_ec_curve(void) +{ + return write_to_ec(AYN_SENSOR_PWM_FAN_ENABLE_REG, 0x02); +} + +/** + * ayn_ec_hwmon_is_visible() - Determines RO or RW for hwmon attribute sysfs. + * + * @drvdata: Unused void pointer to context data. + * @type: The hwmon_sensor_types type. + * @attr: The attribute to set RO/RW on. + * @channel: HWMON subsystem usage flags for the attribute. + * + * Return: Permission level. + */ +static umode_t ayn_ec_hwmon_is_visible(const void *drvdata, + enum hwmon_sensor_types type, u32 attr, + int channel) +{ + switch (type) { + case hwmon_fan: + return 0444; + case hwmon_pwm: + return 0644; + default: + return 0; + } +} + +/** + * ayn_pwm_fan_read() - Read from a hwmon pwm or fan attribute. + * + * @dev: parent device of the given attribute. + * @type: The hwmon_sensor_types type. + * @attr: The attribute to read from. + * @channel: HWMON subsystem usage flags for the attribute. + * @val: Pointer to return the read value from. + * + * Return: 0, or an error. + */ +static int ayn_pwm_fan_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + int ret; + + switch (type) { + case hwmon_fan: + switch (attr) { + case hwmon_fan_input: + return read_from_ec(AYN_SENSOR_PWM_FAN_SPEED_REG, 2, + val); + default: + break; + } + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_enable: + ret = read_from_ec(AYN_SENSOR_PWM_FAN_ENABLE_REG, 1, + val); + if (ret) + return ret; + + /* EC uses 0 for manual, 1 for automatic, 2 for user + * fan curve. Reflect hwmon usage instead. + */ + if (*val == 1) { + *val = 2; + return 0; + } + + if (*val == 2) { + *val = 3; + return 0; + } + + /* Return 0 when fan at max, otherwise 1 for manual. */ + ret = read_from_ec(AYN_SENSOR_PWM_FAN_SET_REG, 1, val); + if (ret) + return ret; + + if (*val == 128) + *val = 0; + else + *val = 1; + + return ret; + case hwmon_pwm_input: + ret = read_from_ec(AYN_SENSOR_PWM_FAN_SET_REG, 1, val); + if (ret) + return ret; + + *val = *val << 1; /* Max value is 128, scale to 255 */ + + return 0; + default: + break; + } + break; + default: + break; + } + return -EOPNOTSUPP; +} + +/** + * ayn_pwm_fan_write() - Write to a hwmon pwm attribute. + * + * @dev: parent device of the given attribute. + * @type: The hwmon_sensor_types type. + * @attr: The attribute to write to. + * @channel: HWMON subsystem usage flags for the attribute. + * @val: Value to write. + * + * Return: 0, or an error. + */ +static int ayn_pwm_fan_write(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long val) +{ + if (type != hwmon_pwm) + return -EOPNOTSUPP; + switch (attr) { + case hwmon_pwm_enable: + switch (val) { + case HWMON_PWM_FAN_MODE_FULL: + return ayn_pwm_full(); + case HWMON_PWM_FAN_MODE_MANUAL: + return ayn_pwm_manual(); + case HWMON_PWM_FAN_MODE_AUTO: + return ayn_pwm_auto(); + case HWMON_PWM_FAN_MODE_EC_CURVE: + return ayn_pwm_ec_curve(); + default: + return -EINVAL; + } + case hwmon_pwm_input: + if (val < 0 || val > 255) + return -EINVAL; + + val = val >> 1; /* Max value is 128, scale from 255 */ + + return write_to_ec(AYN_SENSOR_PWM_FAN_SET_REG, val); + default: + return -EOPNOTSUPP; + } +} + +static const struct hwmon_channel_info *ayn_ec_sensors[] = { + HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT), + HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT | HWMON_PWM_ENABLE), + NULL, +}; + +static const struct hwmon_ops ayn_ec_hwmon_ops = { + .is_visible = ayn_ec_hwmon_is_visible, + .read = ayn_pwm_fan_read, + .write = ayn_pwm_fan_write, +}; + +static const struct hwmon_chip_info ayn_ec_chip_info = { + .ops = &ayn_ec_hwmon_ops, + .info = ayn_ec_sensors, +}; + +/** + * pwm_curve_store() - Write a fan curve speed or temperature value. + * + * @dev: The attribute's parent device. + * @attr: The attribute to read. + * @buf: Input value string from sysfs write. + * + * Return: Number of bytes read, or an error. + */ +static ssize_t pwm_curve_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t count) +{ + int i = to_sensor_dev_attr(attr)->index; + int ret, val; + u8 reg; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (i < 5) { + if (val < 0 || val > 255) + return -EINVAL; + val = val >> 1; /* Max EC value is 128, scale from 255 */ + } else + if (val < 0 || val > 100) + return -EINVAL; + + reg = ayn_pwm_curve_registers[i]; + + ret = write_to_ec(reg, val); + if (ret) + return ret; + + return count; +} + +/** + * pwm_curve_show() - Read a fan curve speed or temperature value. + * + * @dev: The attribute's parent device. + * @attr: The attribute to read. + * @buf: Output buffer. + * + * Return: Number of bytes read, or an error. + */ +static ssize_t pwm_curve_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + int i = to_sensor_dev_attr(attr)->index; + long val; + int ret; + u8 reg; + + reg = ayn_pwm_curve_registers[i]; + + ret = read_from_ec(reg, 1, &val); + if (ret) + return ret; + + if (i < 5) + val = val << 1; /* Max EC value is 128, scale to 255 */ + + return sysfs_emit(buf, "%ld\n", val); +} + +/* Fan curve attributes */ +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point1_pwm, pwm_curve, 0); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point2_pwm, pwm_curve, 1); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point3_pwm, pwm_curve, 2); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point4_pwm, pwm_curve, 3); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point5_pwm, pwm_curve, 4); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point1_temp, pwm_curve, 5); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point2_temp, pwm_curve, 6); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point3_temp, pwm_curve, 7); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point4_temp, pwm_curve, 8); +static SENSOR_DEVICE_ATTR_RW(pwm1_auto_point5_temp, pwm_curve, 9); + +/** + * thermal_sensor_show() - Read a thermal sensor attribute value. + * + * @dev: The attribute's parent device. + * @attr: The attribute to read. + * @buf: Buffer to write the result into. + * + * Return: Number of bytes read, or an error. + */ +static ssize_t thermal_sensor_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + long ret, val; + int i; + + i = to_sensor_dev_attr(attr)->index; + + ret = read_from_ec(thermal_sensors[i].reg, 1, &val); + if (ret) + return ret; + + val = val * 1000L; + + return sysfs_emit(buf, "%ld\n", val); +} + +/** + * thermal_sensor_label_show() - Read a thermal sensor attribute label. + * + * @dev: The attribute's parent device. + * @attr: The attribute to read. + * @buf: Buffer to read to. + * + * Return: Number of bytes read, or an error. + */ +static ssize_t thermal_sensor_label_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int i = to_sensor_dev_attr(attr)->index; + + return sysfs_emit(buf, "%s\n", thermal_sensors[i].name); +} + +static SENSOR_DEVICE_ATTR_RO(temp1_input, thermal_sensor, 0); +static SENSOR_DEVICE_ATTR_RO(temp2_input, thermal_sensor, 1); +static SENSOR_DEVICE_ATTR_RO(temp3_input, thermal_sensor, 2); +static SENSOR_DEVICE_ATTR_RO(temp4_input, thermal_sensor, 3); +static SENSOR_DEVICE_ATTR_RO(temp5_input, thermal_sensor, 4); +static SENSOR_DEVICE_ATTR_RO(temp1_label, thermal_sensor_label, 0); +static SENSOR_DEVICE_ATTR_RO(temp2_label, thermal_sensor_label, 1); +static SENSOR_DEVICE_ATTR_RO(temp3_label, thermal_sensor_label, 2); +static SENSOR_DEVICE_ATTR_RO(temp4_label, thermal_sensor_label, 3); +static SENSOR_DEVICE_ATTR_RO(temp5_label, thermal_sensor_label, 4); + +static struct attribute *ayn_sensors_attrs[] = { + &sensor_dev_attr_pwm1_auto_point1_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point1_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point2_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point2_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point3_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point3_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point4_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point4_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point5_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point5_temp.dev_attr.attr, + &sensor_dev_attr_temp1_input.dev_attr.attr, + &sensor_dev_attr_temp1_label.dev_attr.attr, + &sensor_dev_attr_temp2_input.dev_attr.attr, + &sensor_dev_attr_temp2_label.dev_attr.attr, + &sensor_dev_attr_temp3_input.dev_attr.attr, + &sensor_dev_attr_temp3_label.dev_attr.attr, + &sensor_dev_attr_temp4_input.dev_attr.attr, + &sensor_dev_attr_temp4_label.dev_attr.attr, + &sensor_dev_attr_temp5_input.dev_attr.attr, + &sensor_dev_attr_temp5_label.dev_attr.attr, + NULL, +}; + +ATTRIBUTE_GROUPS(ayn_sensors); + +/** + * rgb_effect_write() - Set the RGB effect stored in drvdata.rgb_effect. + */ +static int rgb_effect_write(void) +{ + return write_to_ec(AYN_RGB_EFFECT_REG, drvdata.rgb_effect); +}; + +/** + * rgb_effect_read() - Read the RGB effect and store it in drvdata.rgb_effect. + */ +static int rgb_effect_read(void) +{ + int ret; + long effect; + + ret = read_from_ec(AYN_RGB_EFFECT_REG, 1, &effect); + if (ret) + return ret; + + switch (effect) { + case AYN_RGB_EFFECT_WRITE: + case AYN_RGB_EFFECT_MONOCOLOR: + drvdata.rgb_effect = AYN_RGB_EFFECT_WRITE; + break; + default: + drvdata.rgb_effect = AYN_RGB_EFFECT_BREATHE; + } + + return 0; +} + +/** + * rgb_effect_store() - Store the given RGB effect and set it. + * + * @dev: parent device of the given attribute. + * @attr: The attribute to write to. + * @buf: Input value string from sysfs write. + * @count: The number of bytes written. + * + * Return: The number of bytes written, or an error. + */ +static ssize_t rgb_effect_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t count) +{ + int ret; + + ret = sysfs_match_string(RGB_EFFECT_TEXT, buf); + if (ret < 0) + return ret; + + if (ret) + drvdata.rgb_effect = AYN_RGB_EFFECT_WRITE; + else + drvdata.rgb_effect = AYN_RGB_EFFECT_BREATHE; + + ret = rgb_effect_write(); + if (ret) + return ret; + + return count; +}; + +/** + * rgb_effect_show() - Read the current RGB effect. + * + * @dev: parent device of the given attribute. + * @attr: The attribute to read. + * @buf: Buffer to read to. + * + * Return: The number of bytes read, or an error. + */ +static ssize_t rgb_effect_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret, i; + + ret = rgb_effect_read(); + if (ret) + return ret; + + switch (drvdata.rgb_effect) { + case AYN_RGB_EFFECT_WRITE: + case AYN_RGB_EFFECT_MONOCOLOR: + i = MONOCOLOR; + break; + default: + i = BREATHE; + break; + } + + return sysfs_emit(buf, "%s\n", RGB_EFFECT_TEXT[i]); +}; + +static DEVICE_ATTR_RW_NAMED(rgb_effect, "effect"); + +/** + * rgb_effect_show() - Display the RGB effects available. + * + * @dev: parent device of the given attribute. + * @attr: The attribute to read. + * @buf: Buffer to read to. + * + * Return: The number of bytes read, or an error. + */ +static ssize_t rgb_effect_index_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + size_t count = 0; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(RGB_EFFECT_TEXT); i++) + count += sysfs_emit_at(buf, count, "%s ", RGB_EFFECT_TEXT[i]); + + buf[count - 1] = '\n'; + + return count; +} + +static DEVICE_ATTR_RO_NAMED(rgb_effect_index, "effect_index"); + +/** + * ayn_led_mc_brightness_set() - Write the brightness for the RGB LED. + * + * @led_cdev: Parent LED device for the led_classdev_mc. + * @brightness: Brightness value to write [0-255]. + */ +static void ayn_led_mc_brightness_set(struct led_classdev *led_cdev, + enum led_brightness brightness) +{ + struct led_classdev_mc *led_cdev_mc = lcdev_to_mccdev(led_cdev); + struct mc_subled s_led; + int i, ret, val; + + switch (drvdata.rgb_effect) { + case AYN_RGB_EFFECT_WRITE: + case AYN_RGB_EFFECT_MONOCOLOR: + break; + case AYN_RGB_EFFECT_BREATHE: + return; + } + + led_cdev->brightness = brightness; + for (i = 0; i < led_cdev_mc->num_colors; i++) { + s_led = led_cdev_mc->subled_info[i]; + val = brightness * s_led.intensity / led_cdev->max_brightness; + ret = write_to_ec(s_led.channel, val); + if (ret) { + dev_err(led_cdev->dev, + "Error setting brightness: %d\n", ret); + return; + } + } + + /* Must write mode again to change to set color */ + write_to_ec(AYN_RGB_EFFECT_REG, AYN_RGB_EFFECT_WRITE); +}; + +/** + * ayn_led_mc_brightness_get() - Get the brightness for the RGB LED. + * + * @led_cdev: Parent LED device for the led_classdev_mc. + * + * Return: Current brightness. + */ +static enum led_brightness ayn_led_mc_brightness_get(struct led_classdev *led_cdev) +{ + return led_cdev->brightness; +}; + +static struct attribute *ayn_led_mc_attrs[] = { + &dev_attr_rgb_effect.attr, + &dev_attr_rgb_effect_index.attr, + NULL, +}; + +static struct attribute_group ayn_led_mc_group = { + .attrs = ayn_led_mc_attrs, +}; + +struct mc_subled ayn_led_mc_subled_info[] = { + { + .color_index = LED_COLOR_ID_RED, + .brightness = 0, + .intensity = 0, + .channel = AYN_LED_MC_RED_REG, + }, + { + .color_index = LED_COLOR_ID_GREEN, + .brightness = 0, + .intensity = 0, + .channel = AYN_LED_MC_GREEN_REG, + }, + { + .color_index = LED_COLOR_ID_BLUE, + .brightness = 0, + .intensity = 0, + .channel = AYN_LED_MC_BLUE_REG, + }, +}; + +struct led_classdev_mc ayn_led_mc = { + .led_cdev = { + .name = "ayn:rgb:joystick_rings", + .brightness = 0, + .max_brightness = 255, + .brightness_set = ayn_led_mc_brightness_set, + .brightness_get = ayn_led_mc_brightness_get, + .color = LED_COLOR_ID_RGB, + }, + .num_colors = ARRAY_SIZE(ayn_led_mc_subled_info), + .subled_info = ayn_led_mc_subled_info, +}; + +static int ayn_ec_resume(struct platform_device *pdev) +{ + struct led_classdev *led_cdev = drvdata.led_cdev; + int ret; + + ret = rgb_effect_write(); + if (ret) + return ret; + + ayn_led_mc_brightness_set(led_cdev, led_cdev->brightness); + + return 0; +} + +static int ayn_ec_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device *hwdev; + int ret; + + ret = devm_led_classdev_multicolor_register(dev, &ayn_led_mc); + if (ret) + return ret; + + ret = devm_device_add_group(ayn_led_mc.led_cdev.dev, &ayn_led_mc_group); + if (ret) + return ret; + + drvdata.led_cdev = &ayn_led_mc.led_cdev; + ret = rgb_effect_read(); + if (ret) + return ret; + + hwdev = devm_hwmon_device_register_with_info(dev, "aynec", NULL, + &ayn_ec_chip_info, + ayn_sensors_groups); + return PTR_ERR_OR_ZERO(hwdev); +} + +static struct platform_driver ayn_ec_driver = { + .driver = { + .name = "ayn-ec", + }, + .probe = ayn_ec_probe, + .resume = ayn_ec_resume, +}; + +static struct platform_device *ayn_ec_device; + +static int __init ayn_ec_init(void) +{ + ayn_ec_device = platform_create_bundle(&ayn_ec_driver, ayn_ec_probe, + NULL, 0, NULL, 0); + + return PTR_ERR_OR_ZERO(ayn_ec_device); +} + +static void __exit ayn_ec_exit(void) +{ + platform_device_unregister(ayn_ec_device); + platform_driver_unregister(&ayn_ec_driver); +} + +static const struct dmi_system_id ayn_dmi_table[] = { + { + .ident = "AYN Loki Max", + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "ayn"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Loki Max"), + }, + }, + { + .ident = "AYN Loki MiniPro", + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "ayn"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Loki MiniPro"), + }, + }, + { + .ident = "AYN Loki Zero", + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "ayn"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Loki Zero"), + }, + }, + { + .ident = "Tectoy Zeenix Lite", + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "Tectoy"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Zeenix Lite"), + }, + }, + {}, +}; + +MODULE_DEVICE_TABLE(dmi, ayn_dmi_table); + +module_init(ayn_ec_init); +module_exit(ayn_ec_exit); + +MODULE_AUTHOR("Derek J. Clark "); +MODULE_DESCRIPTION("Platform driver that handles EC sensors of AYN x86 devices"); +MODULE_LICENSE("GPL"); diff --git a/drivers/platform/x86/dell/dell-wmi-base.c b/drivers/platform/x86/dell/dell-wmi-base.c index 997383ba18465..38a6b3ae2f75c 100644 --- a/drivers/platform/x86/dell/dell-wmi-base.c +++ b/drivers/platform/x86/dell/dell-wmi-base.c @@ -456,7 +456,7 @@ static int dell_wmi_process_key(struct wmi_device *wdev, int type, int code, __l key++; used = 1; } else if (type == 0x0012 && code == 0x000d && remaining > 0) { - value = (le16_to_cpu(buffer[2]) == 2); + value = (le16_to_cpu(buffer[0]) == 2); used = 1; } @@ -843,9 +843,22 @@ static int __init dell_wmi_init(void) err = dell_privacy_register_driver(); if (err) - return err; + goto out_smbios; - return wmi_driver_register(&dell_wmi_driver); + err = wmi_driver_register(&dell_wmi_driver); + if (err) + goto out_privacy; + + return 0; + +out_privacy: + dell_privacy_unregister_driver(); + +out_smbios: + if (wmi_requires_smbios_request) + dell_wmi_events_set_enabled(false); + + return err; } late_initcall(dell_wmi_init); diff --git a/drivers/platform/x86/dell/dell-wmi-privacy.c b/drivers/platform/x86/dell/dell-wmi-privacy.c index f9d275b2f900c..366e5b8dc868f 100644 --- a/drivers/platform/x86/dell/dell-wmi-privacy.c +++ b/drivers/platform/x86/dell/dell-wmi-privacy.c @@ -92,11 +92,11 @@ bool dell_privacy_has_mic_mute(void) { struct privacy_wmi_data *priv; - mutex_lock(&list_mutex); + guard(mutex)(&list_mutex); + priv = list_first_entry_or_null(&wmi_list, struct privacy_wmi_data, list); - mutex_unlock(&list_mutex); return priv && (priv->features_present & BIT(DELL_PRIVACY_TYPE_AUDIO)); } diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c b/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c index db278ff4cc4d4..154e115af4b4c 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c @@ -84,7 +84,6 @@ int set_attribute(const char *a_name, const char *a_value) if (ret < 0) goto out; - print_hex_dump_bytes("set attribute data: ", DUMP_PREFIX_NONE, buffer, buffer_size); ret = call_biosattributes_interface(wmi_priv.bios_attr_wdev, buffer, buffer_size, SETATTRIBUTE_METHOD_ID); diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h b/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h index 3bddedad5eba2..eb48ced558232 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h +++ b/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h @@ -107,7 +107,7 @@ enum { static int get_##type##_instance_id(struct kobject *kobj) \ { \ int i; \ - for (i = 0; i <= wmi_priv.type##_instances_count; i++) { \ + for (i = 0; i < wmi_priv.type##_instances_count; i++) { \ if (!(strcmp(kobj->name, wmi_priv.type##_data[i].attribute_name)))\ return i; \ } \ diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c index 27fd6cd215290..309634c1cc20e 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c +++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c @@ -85,7 +85,7 @@ int hp_get_string_from_buffer(u8 **buffer, u32 *buffer_size, char *dst, u32 dst_ * bytes. */ conv_dst_size = size; - if (size > dst_size) + if (size >= dst_size) conv_dst_size = dst_size - 1; /* @@ -661,12 +661,17 @@ static int hp_init_bios_package_attribute(enum hp_wmi_data_type attr_type, int ret = 0; /* Take action appropriate to each ACPI TYPE */ - if (obj->package.count < min_elements) { - pr_err("ACPI-package does not have enough elements: %d < %d\n", - obj->package.count, min_elements); + if (obj->package.count < COMMON_ELEM_CNT) { + pr_err("ACPI-package is missing common elements: %d < %d\n", + obj->package.count, COMMON_ELEM_CNT); goto pack_attr_exit; } + if (obj->package.count < min_elements) { + pr_warn("ACPI-package has fewer elements than expected: %d < %d, parsing available elements\n", + obj->package.count, min_elements); + } + elements = obj->package.elements; /* sanity checking */ @@ -731,26 +736,31 @@ static int hp_init_bios_package_attribute(enum hp_wmi_data_type attr_type, switch (attr_type) { case HPWMI_STRING_TYPE: ret = hp_populate_string_package_data(elements, + obj->package.count, instance_id, attr_name_kobj); break; case HPWMI_INTEGER_TYPE: ret = hp_populate_integer_package_data(elements, + obj->package.count, instance_id, attr_name_kobj); break; case HPWMI_ENUMERATION_TYPE: ret = hp_populate_enumeration_package_data(elements, + obj->package.count, instance_id, attr_name_kobj); break; case HPWMI_ORDERED_LIST_TYPE: ret = hp_populate_ordered_list_package_data(elements, + obj->package.count, instance_id, attr_name_kobj); break; case HPWMI_PASSWORD_TYPE: ret = hp_populate_password_package_data(elements, + obj->package.count, instance_id, attr_name_kobj); break; diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h index f1eec0e4ba075..ac57d6eab4c35 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h +++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h @@ -279,6 +279,9 @@ enum hp_wmi_data_elements { PSWD_ENCODINGS = 13, PSWD_IS_SET = 14, PSWD_ELEM_CNT = 15, + + /* Minimum elements shared by all attribute types (NAME..SECURITY_LEVEL) */ + COMMON_ELEM_CNT = SECURITY_LEVEL + 1, }; #define GET_INSTANCE_ID(type) \ @@ -401,6 +404,7 @@ int hp_populate_string_buffer_data(u8 *buffer_ptr, u32 *buffer_size, int hp_alloc_string_data(void); void hp_exit_string_attributes(void); int hp_populate_string_package_data(union acpi_object *str_obj, + int str_obj_count, int instance_id, struct kobject *attr_name_kobj); @@ -411,6 +415,7 @@ int hp_populate_integer_buffer_data(u8 *buffer_ptr, u32 *buffer_size, int hp_alloc_integer_data(void); void hp_exit_integer_attributes(void); int hp_populate_integer_package_data(union acpi_object *integer_obj, + int integer_obj_count, int instance_id, struct kobject *attr_name_kobj); @@ -421,6 +426,7 @@ int hp_populate_enumeration_buffer_data(u8 *buffer_ptr, u32 *buffer_size, int hp_alloc_enumeration_data(void); void hp_exit_enumeration_attributes(void); int hp_populate_enumeration_package_data(union acpi_object *enum_obj, + int enum_obj_count, int instance_id, struct kobject *attr_name_kobj); @@ -432,6 +438,7 @@ int hp_populate_ordered_list_buffer_data(u8 *buffer_ptr, int hp_alloc_ordered_list_data(void); void hp_exit_ordered_list_attributes(void); int hp_populate_ordered_list_package_data(union acpi_object *order_obj, + int order_obj_count, int instance_id, struct kobject *attr_name_kobj); @@ -440,6 +447,7 @@ int hp_populate_password_buffer_data(u8 *buffer_ptr, u32 *buffer_size, int instance_id, struct kobject *attr_name_kobj); int hp_populate_password_package_data(union acpi_object *password_obj, + int password_obj_count, int instance_id, struct kobject *attr_name_kobj); int hp_alloc_password_data(void); diff --git a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c index af4d1920d4880..446dd18d2cee4 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c @@ -163,10 +163,11 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum /* Check that both expected and read object type match */ if (expected_enum_types[eloc] != enum_obj[elem].type) { - pr_err("Error expected type %d for elem %d, but got type %d instead\n", - expected_enum_types[eloc], elem, enum_obj[elem].type); + pr_warn("Unexpected element type at elem %d: expected %d, got %d, skipping\n", + elem, expected_enum_types[eloc], enum_obj[elem].type); kfree(str_value); - return -EIO; + str_value = NULL; + continue; } /* Assign appropriate element value to corresponding field */ @@ -227,6 +228,8 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum kfree(str_value); str_value = NULL; } + if (size) + elem += size - 1; break; case SECURITY_LEVEL: @@ -280,6 +283,8 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum kfree(str_value); str_value = NULL; } + if (size) + elem += (size < MAX_VALUES_SIZE ? size : MAX_VALUES_SIZE) - 1; break; default: pr_warn("Invalid element: %d found in Enumeration attribute or data may be malformed\n", elem); @@ -300,10 +305,12 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum * Populate all properties of an instance under enumeration attribute * * @enum_obj: ACPI object with enumeration data + * @enum_obj_count: Number of elements in @enum_obj * @instance_id: The instance to enumerate * @attr_name_kobj: The parent kernel object */ int hp_populate_enumeration_package_data(union acpi_object *enum_obj, + int enum_obj_count, int instance_id, struct kobject *attr_name_kobj) { @@ -312,7 +319,7 @@ int hp_populate_enumeration_package_data(union acpi_object *enum_obj, enum_data->attr_name_kobj = attr_name_kobj; hp_populate_enumeration_elements_from_package(enum_obj, - enum_obj->package.count, + enum_obj_count, instance_id); hp_update_attribute_permissions(enum_data->common.is_readonly, &enumeration_current_val); diff --git a/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c index d96e160953e39..a279070664480 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c @@ -243,6 +243,8 @@ static int hp_populate_integer_elements_from_package(union acpi_object *integer_ kfree(str_value); str_value = NULL; } + if (size) + elem += size - 1; break; case SECURITY_LEVEL: @@ -275,10 +277,12 @@ static int hp_populate_integer_elements_from_package(union acpi_object *integer_ * Populate all properties of an instance under integer attribute * * @integer_obj: ACPI object with integer data + * @integer_obj_count: Number of elements in @integer_obj * @instance_id: The instance to enumerate * @attr_name_kobj: The parent kernel object */ int hp_populate_integer_package_data(union acpi_object *integer_obj, + int integer_obj_count, int instance_id, struct kobject *attr_name_kobj) { @@ -286,7 +290,7 @@ int hp_populate_integer_package_data(union acpi_object *integer_obj, integer_data->attr_name_kobj = attr_name_kobj; hp_populate_integer_elements_from_package(integer_obj, - integer_obj->package.count, + integer_obj_count, instance_id); hp_update_attribute_permissions(integer_data->common.is_readonly, &integer_current_val); diff --git a/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c index f09489a085c86..5bf8d40bdf816 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c @@ -145,7 +145,7 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord if (!order_obj) return -EINVAL; - for (elem = 1, eloc = 1; eloc < ORD_ELEM_CNT; elem++, eloc++) { + for (elem = 1, eloc = 1; eloc < ORD_ELEM_CNT && elem < order_obj_count; elem++, eloc++) { switch (order_obj[elem].type) { case ACPI_TYPE_STRING: @@ -232,6 +232,8 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord kfree(str_value); str_value = NULL; } + if (size) + elem += size - 1; break; case SECURITY_LEVEL: @@ -261,7 +263,9 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord * Ordered list data is stored in hex and comma separated format * Convert the data and split it to show each element */ - ret = hp_convert_hexstr_to_str(str_value, value_len, &tmpstr, &tmp_len); + ret = hp_convert_hexstr_to_str(order_obj[elem].string.pointer, + order_obj[elem].string.length, + &tmpstr, &tmp_len); if (ret) goto exit_list; @@ -298,10 +302,12 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord * Populate all properties of an instance under ordered_list attribute * * @order_obj: ACPI object with ordered_list data + * @order_obj_count: Number of elements in @order_obj * @instance_id: The instance to enumerate * @attr_name_kobj: The parent kernel object */ -int hp_populate_ordered_list_package_data(union acpi_object *order_obj, int instance_id, +int hp_populate_ordered_list_package_data(union acpi_object *order_obj, int order_obj_count, + int instance_id, struct kobject *attr_name_kobj) { struct ordered_list_data *ordered_list_data = &bioscfg_drv.ordered_list_data[instance_id]; @@ -309,7 +315,7 @@ int hp_populate_ordered_list_package_data(union acpi_object *order_obj, int inst ordered_list_data->attr_name_kobj = attr_name_kobj; hp_populate_ordered_list_elements_from_package(order_obj, - order_obj->package.count, + order_obj_count, instance_id); hp_update_attribute_permissions(ordered_list_data->common.is_readonly, &ordered_list_current_val); diff --git a/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c index 4d79eb8056a5d..a9e1786374162 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c @@ -66,7 +66,7 @@ static int validate_password_input(int instance_id, const char *buf) struct password_data *password_data = &bioscfg_drv.password_data[instance_id]; length = strlen(buf); - if (buf[length - 1] == '\n') + if (length > 0 && buf[length - 1] == '\n') length--; if (length > MAX_PASSWD_SIZE) @@ -123,7 +123,7 @@ static ssize_t new_password_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - return store_password_instance(kobj, buf, count, true); + return store_password_instance(kobj, buf, count, false); } static struct kobj_attribute password_new_password = __ATTR_WO(new_password); @@ -321,6 +321,8 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor str_value = NULL; } + if (size) + elem += size - 1; break; case SECURITY_LEVEL: password_data->common.security_level = int_value; @@ -351,6 +353,11 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor case PSWD_ENCODINGS: size = min_t(u32, password_data->encodings_size, MAX_ENCODINGS_SIZE); for (pos_values = 0; pos_values < size; pos_values++) { + if (elem + pos_values >= password_obj_count) { + pr_err("Error elem-objects package is too small\n"); + return -EINVAL; + } + ret = hp_convert_hexstr_to_str(password_obj[elem + pos_values].string.pointer, password_obj[elem + pos_values].string.length, &str_value, &value_len); @@ -362,6 +369,8 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor str_value = NULL; } + if (size) + elem += size - 1; break; case PSWD_IS_SET: password_data->is_enabled = int_value; @@ -385,10 +394,12 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor * Populate all properties for an instance under password attribute * * @password_obj: ACPI object with password data + * @password_obj_count: Number of elements in @password_obj * @instance_id: The instance to enumerate * @attr_name_kobj: The parent kernel object */ -int hp_populate_password_package_data(union acpi_object *password_obj, int instance_id, +int hp_populate_password_package_data(union acpi_object *password_obj, int password_obj_count, + int instance_id, struct kobject *attr_name_kobj) { struct password_data *password_data = &bioscfg_drv.password_data[instance_id]; @@ -396,7 +407,7 @@ int hp_populate_password_package_data(union acpi_object *password_obj, int insta password_data->attr_name_kobj = attr_name_kobj; hp_populate_password_elements_from_package(password_obj, - password_obj->package.count, + password_obj_count, instance_id); hp_friendly_user_name_update(password_data->common.path, diff --git a/drivers/platform/x86/hp/hp-bioscfg/spmobj-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/spmobj-attributes.c index 2b00a14792e92..4d94e48c1a4c8 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/spmobj-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/spmobj-attributes.c @@ -238,7 +238,7 @@ static ssize_t sk_store(struct kobject *kobj, ret = hp_wmi_perform_query(HPWMI_SECUREPLATFORM_SET_SK, HPWMI_SECUREPLATFORM, (void *)bioscfg_drv.spm_data.signing_key, - count, 0); + length, 0); if (!ret) { bioscfg_drv.spm_data.mechanism = SIGNING_KEY; @@ -274,7 +274,7 @@ static ssize_t kek_store(struct kobject *kobj, ret = hp_wmi_perform_query(HPWMI_SECUREPLATFORM_SET_KEK, HPWMI_SECUREPLATFORM, (void *)bioscfg_drv.spm_data.endorsement_key, - count, 0); + length, 0); if (!ret) { bioscfg_drv.spm_data.mechanism = ENDORSEMENT_KEY; diff --git a/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c index fe5a9a3a4ef17..f3dfba270f7e8 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c @@ -233,6 +233,8 @@ static int hp_populate_string_elements_from_package(union acpi_object *string_ob kfree(str_value); str_value = NULL; } + if (size) + elem += size - 1; break; case SECURITY_LEVEL: @@ -263,10 +265,12 @@ static int hp_populate_string_elements_from_package(union acpi_object *string_ob * Populate all properties of an instance under string attribute * * @string_obj: ACPI object with string data + * @string_obj_count: Number of elements in @string_obj * @instance_id: The instance to enumerate * @attr_name_kobj: The parent kernel object */ int hp_populate_string_package_data(union acpi_object *string_obj, + int string_obj_count, int instance_id, struct kobject *attr_name_kobj) { @@ -275,7 +279,7 @@ int hp_populate_string_package_data(union acpi_object *string_obj, string_data->attr_name_kobj = attr_name_kobj; hp_populate_string_elements_from_package(string_obj, - string_obj->package.count, + string_obj_count, instance_id); hp_update_attribute_permissions(string_data->common.is_readonly, diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c index 8ba286ed87216..051676c7ec1ad 100644 --- a/drivers/platform/x86/hp/hp-wmi.c +++ b/drivers/platform/x86/hp/hp-wmi.c @@ -133,11 +133,35 @@ static const struct thermal_profile_params omen_v1_no_ec_thermal_params = { .ec_tp_offset = HP_NO_THERMAL_PROFILE_OFFSET, }; -/* - * A generic pointer for the currently-active board's thermal profile - * parameters. - */ -static struct thermal_profile_params *active_thermal_profile_params; +struct hp_wmi_board_params { + const struct thermal_profile_params *thermal_profile; +}; + +static const struct hp_wmi_board_params victus_s_board_params = { + .thermal_profile = &victus_s_thermal_params, +}; + +static const struct hp_wmi_board_params omen_v1_board_params = { + .thermal_profile = &omen_v1_thermal_params, +}; + +static const struct hp_wmi_board_params omen_v1_legacy_board_params = { + .thermal_profile = &omen_v1_legacy_thermal_params, +}; + +static const struct hp_wmi_board_params omen_v1_no_ec_board_params = { + .thermal_profile = &omen_v1_no_ec_thermal_params, +}; + +static const struct hp_wmi_board_params *active_board_params; + +static const struct thermal_profile_params *hp_wmi_thermal_profile(void) +{ + if (!active_board_params) + return NULL; + + return active_board_params->thermal_profile; +} /* DMI board names of devices that should use the omen specific path for * thermal profiles. @@ -187,87 +211,91 @@ static const char * const victus_thermal_profile_boards[] = { "8A25", }; -/* DMI Board names of Victus 16-r and Victus 16-s laptops */ -static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst = { +/* DMI board-specific feature data for Omen and Victus laptops. */ +static const struct dmi_system_id hp_wmi_feature_boards[] __initconst = { { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8902") }, - .driver_data = (void *)&omen_v1_legacy_thermal_params, + .driver_data = (void *)&omen_v1_legacy_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8A44") }, - .driver_data = (void *)&omen_v1_legacy_thermal_params, + .driver_data = (void *)&omen_v1_legacy_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8A4D") }, - .driver_data = (void *)&omen_v1_legacy_thermal_params, + .driver_data = (void *)&omen_v1_legacy_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BAB") }, - .driver_data = (void *)&omen_v1_thermal_params, + .driver_data = (void *)&omen_v1_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8B2F") }, - .driver_data = (void *)&victus_s_thermal_params, + .driver_data = (void *)&victus_s_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BBE") }, - .driver_data = (void *)&victus_s_thermal_params, + .driver_data = (void *)&victus_s_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BC2") }, - .driver_data = (void *)&omen_v1_thermal_params, + .driver_data = (void *)&omen_v1_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCA") }, - .driver_data = (void *)&omen_v1_thermal_params, + .driver_data = (void *)&omen_v1_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCD") }, - .driver_data = (void *)&omen_v1_thermal_params, + .driver_data = (void *)&omen_v1_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD4") }, - .driver_data = (void *)&victus_s_thermal_params, + .driver_data = (void *)&victus_s_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD5") }, - .driver_data = (void *)&victus_s_thermal_params, + .driver_data = (void *)&victus_s_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C76") }, - .driver_data = (void *)&omen_v1_thermal_params, + .driver_data = (void *)&omen_v1_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C77") }, - .driver_data = (void *)&omen_v1_thermal_params, + .driver_data = (void *)&omen_v1_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C78") }, - .driver_data = (void *)&omen_v1_thermal_params, + .driver_data = (void *)&omen_v1_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C99") }, - .driver_data = (void *)&victus_s_thermal_params, + .driver_data = (void *)&victus_s_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C9C") }, - .driver_data = (void *)&victus_s_thermal_params, + .driver_data = (void *)&victus_s_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D26") }, - .driver_data = (void *)&omen_v1_legacy_thermal_params, + .driver_data = (void *)&omen_v1_legacy_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D41") }, - .driver_data = (void *)&omen_v1_no_ec_thermal_params, + .driver_data = (void *)&omen_v1_no_ec_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D87") }, - .driver_data = (void *)&omen_v1_no_ec_thermal_params, + .driver_data = (void *)&omen_v1_no_ec_board_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8DD6") }, + .driver_data = (void *)&omen_v1_no_ec_board_params, }, { .matches = { DMI_MATCH(DMI_BOARD_NAME, "8E35") }, - .driver_data = (void *)&omen_v1_legacy_thermal_params, + .driver_data = (void *)&omen_v1_legacy_board_params, }, {}, }; @@ -1874,7 +1902,10 @@ static int platform_profile_victus_s_get_ec(enum platform_profile_option *profil u8 current_dstate, current_gpu_slowdown_temp, tp; const struct thermal_profile_params *params; - params = active_thermal_profile_params; + params = hp_wmi_thermal_profile(); + if (!params) + return -ENODEV; + if (params->ec_tp_offset == HP_EC_OFFSET_UNKNOWN || params->ec_tp_offset == HP_NO_THERMAL_PROFILE_OFFSET) { *profile = active_platform_profile; @@ -1886,10 +1917,10 @@ static int platform_profile_victus_s_get_ec(enum platform_profile_option *profil return ret; /* - * We cannot use active_thermal_profile_params here, because boards - * like 8C78 have tp == 0x0 || tp == 0x1 after cold boot, but logically - * it should have tp == 0x30 || tp == 0x31, as corrected by the Omen - * Gaming Hub on windows. Hence accept both of these values. + * Boards like 8C78 have tp == 0x0 || tp == 0x1 after cold boot, + * but logically it should have tp == 0x30 || tp == 0x31, as + * corrected by the Omen Gaming Hub on windows. Hence accept both + * of these values. */ if (tp == victus_s_thermal_params.performance || tp == omen_v1_thermal_params.performance) { @@ -1924,12 +1955,12 @@ static int platform_profile_victus_s_get_ec(enum platform_profile_option *profil static int platform_profile_victus_s_set_ec(enum platform_profile_option profile) { - struct thermal_profile_params *params; + const struct thermal_profile_params *params; bool gpu_ctgp_enable, gpu_ppab_enable; u8 gpu_dstate; /* Test shows 1 = 100%, 2 = 50%, 3 = 25%, 4 = 12.5% */ int err, tp; - params = active_thermal_profile_params; + params = hp_wmi_thermal_profile(); if (!params) return -ENODEV; @@ -2195,6 +2226,7 @@ static const struct platform_profile_ops hp_wmi_platform_profile_ops = { static int thermal_profile_setup(struct platform_device *device) { const struct platform_profile_ops *ops; + const struct thermal_profile_params *params; int err, tp; if (is_omen_thermal_profile()) { @@ -2226,13 +2258,17 @@ static int thermal_profile_setup(struct platform_device *device) ops = &platform_profile_victus_ops; } else if (is_victus_s_thermal_profile()) { + params = hp_wmi_thermal_profile(); + if (!params) + return -ENODEV; + /* * For an unknown EC layout board, platform_profile_victus_s_get_ec(), * behaves like a wrapper around active_platform_profile, to avoid using * uninitialized data, we default to PLATFORM_PROFILE_BALANCED. */ - if (active_thermal_profile_params->ec_tp_offset == HP_EC_OFFSET_UNKNOWN || - active_thermal_profile_params->ec_tp_offset == HP_NO_THERMAL_PROFILE_OFFSET) { + if (params->ec_tp_offset == HP_EC_OFFSET_UNKNOWN || + params->ec_tp_offset == HP_NO_THERMAL_PROFILE_OFFSET) { active_platform_profile = PLATFORM_PROFILE_BALANCED; } else { err = platform_profile_victus_s_get_ec(&active_platform_profile); @@ -2693,24 +2729,25 @@ static int hp_wmi_hwmon_init(void) return 0; } -static void __init setup_active_thermal_profile_params(void) +static void __init setup_active_board_params(void) { const struct dmi_system_id *id; + const struct thermal_profile_params *params; - /* - * Currently only victus_s devices use the - * active_thermal_profile_params - */ - id = dmi_first_match(victus_s_thermal_profile_boards); + id = dmi_first_match(hp_wmi_feature_boards); if (id) { + active_board_params = id->driver_data; + params = hp_wmi_thermal_profile(); + if (!params) + return; + /* * Marking this boolean is required to ensure that * is_victus_s_thermal_profile() behaves like a valid * wrapper. */ is_victus_s_board = true; - active_thermal_profile_params = id->driver_data; - if (active_thermal_profile_params->ec_tp_offset == HP_EC_OFFSET_UNKNOWN) { + if (params->ec_tp_offset == HP_EC_OFFSET_UNKNOWN) { pr_warn("Unknown EC layout for board %s. Thermal profile readback will be disabled. Please report this to platform-driver-x86@vger.kernel.org\n", dmi_get_system_info(DMI_BOARD_NAME)); } @@ -2745,10 +2782,10 @@ static int __init hp_wmi_init(void) } /* - * Setup active board's thermal profile parameters before - * starting platform driver probe. + * Setup active board feature data before starting platform + * driver probe. */ - setup_active_thermal_profile_params(); + setup_active_board_params(); err = platform_driver_probe(&hp_wmi_driver, hp_wmi_bios_setup); if (err) goto err_unregister_device; diff --git a/drivers/platform/x86/intel/int1092/intel_sar.c b/drivers/platform/x86/intel/int1092/intel_sar.c index 849f7b415c1e5..7263114f0b3d0 100644 --- a/drivers/platform/x86/intel/int1092/intel_sar.c +++ b/drivers/platform/x86/intel/int1092/intel_sar.c @@ -91,8 +91,10 @@ static acpi_status parse_package(struct wwan_sar_context *context, union acpi_ob item->package.count <= data->total_dev_mode) return AE_ERROR; - data->device_mode_info = kmalloc_objs(struct wwan_device_mode_info, - data->total_dev_mode); + data->device_mode_info = devm_kmalloc_array(&context->sar_device->dev, + data->total_dev_mode, + sizeof(*data->device_mode_info), + GFP_KERNEL); if (!data->device_mode_info) return AE_ERROR; @@ -253,7 +255,7 @@ static int sar_probe(struct platform_device *device) if (!handle) return -ENODEV; - context = kzalloc_obj(*context); + context = devm_kzalloc(&device->dev, sizeof(*context), GFP_KERNEL); if (!context) return -ENOMEM; @@ -264,7 +266,7 @@ static int sar_probe(struct platform_device *device) result = guid_parse(SAR_DSM_UUID, &context->guid); if (result) { dev_err(&device->dev, "SAR UUID parse error: %d\n", result); - goto r_free; + return result; } for (reg = 0; reg < MAX_REGULATORY; reg++) @@ -272,43 +274,29 @@ static int sar_probe(struct platform_device *device) if (sar_get_device_mode(device) != AE_OK) { dev_err(&device->dev, "Failed to get device mode\n"); - result = -EIO; - goto r_free; + return -EIO; } result = sysfs_create_group(&device->dev.kobj, &intcsar_group); if (result) { dev_err(&device->dev, "sysfs creation failed\n"); - goto r_free; + return result; } if (acpi_install_notify_handler(ACPI_HANDLE(&device->dev), ACPI_DEVICE_NOTIFY, sar_notify, (void *)device) != AE_OK) { dev_err(&device->dev, "Failed acpi_install_notify_handler\n"); - result = -EIO; - goto r_sys; + sysfs_remove_group(&device->dev.kobj, &intcsar_group); + return -EIO; } return 0; - -r_sys: - sysfs_remove_group(&device->dev.kobj, &intcsar_group); -r_free: - kfree(context); - return result; } static void sar_remove(struct platform_device *device) { - struct wwan_sar_context *context = dev_get_drvdata(&device->dev); - int reg; - acpi_remove_notify_handler(ACPI_HANDLE(&device->dev), ACPI_DEVICE_NOTIFY, sar_notify); sysfs_remove_group(&device->dev.kobj, &intcsar_group); - for (reg = 0; reg < MAX_REGULATORY; reg++) - kfree(context->config_data[reg].device_mode_info); - - kfree(context); } static struct platform_driver sar_driver = { diff --git a/drivers/platform/x86/intel/ishtp_eclite.c b/drivers/platform/x86/intel/ishtp_eclite.c index 93ac8b2dbf387..bca7e217878b4 100644 --- a/drivers/platform/x86/intel/ishtp_eclite.c +++ b/drivers/platform/x86/intel/ishtp_eclite.c @@ -600,13 +600,16 @@ static int ecl_ishtp_cl_probe(struct ishtp_cl_device *cl_device) rv = acpi_opregion_init(opr_dev); if (rv) { dev_err(cl_data_to_dev(opr_dev), "ACPI opregion init failed\n"); - goto err_exit; + goto err_put; } /* Reprobe devices depending on ECLite - battery, fan, etc. */ acpi_dev_clear_dependencies(opr_dev->adev); return 0; + +err_put: + acpi_dev_put(opr_dev->adev); err_exit: ishtp_set_connection_state(ecl_ishtp_cl, ISHTP_CL_DISCONNECTING); ishtp_cl_disconnect(ecl_ishtp_cl); diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c index 24334ae70d825..66dd974720bea 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c @@ -336,8 +336,11 @@ static int sst_add_perf_profiles(struct auxiliary_device *auxdev, int i; pd_info->perf_levels = devm_kcalloc(dev, levels, sizeof(struct perf_level), GFP_KERNEL); - if (!pd_info->perf_levels) - return 0; + if (!pd_info->perf_levels) { + pd_info->pp_header.allowed_level_mask = 0; + pd_info->pp_header.level_en_mask = 0; + return -ENOMEM; + } pd_info->ratio_unit = pd_info->pp_header.ratio_unit; pd_info->avx_levels = SST_MAX_AVX_LEVELS; @@ -367,7 +370,7 @@ static int sst_add_perf_profiles(struct auxiliary_device *auxdev, static int sst_main(struct auxiliary_device *auxdev, struct tpmi_per_power_domain_info *pd_info) { struct device *dev = &auxdev->dev; - int i, mask, levels; + int i, ret, mask, levels; *((u64 *)&pd_info->sst_header) = readq(pd_info->sst_base); pd_info->sst_header.cp_offset *= 8; @@ -399,8 +402,12 @@ static int sst_main(struct auxiliary_device *auxdev, struct tpmi_per_power_domai levels = i; mask <<= 1; } + + ret = sst_add_perf_profiles(auxdev, pd_info, levels + 1); + if (ret) + return ret; + pd_info->max_level = levels; - sst_add_perf_profiles(auxdev, pd_info, levels + 1); return 0; } @@ -599,6 +606,9 @@ static bool disable_dynamic_sst_features(void) #define SST_CP_PRIORITY_TYPE_START 1 #define SST_CP_PRIORITY_TYPE_WIDTH 1 +#define SST_CP_MAX_ENABLE 1 +#define SST_CP_MAX_PRIORITY_TYPE 1 + static long isst_if_core_power_state(void __user *argp) { struct tpmi_per_power_domain_info *power_domain_info; @@ -618,6 +628,10 @@ static long isst_if_core_power_state(void __user *argp) if (power_domain_info->write_blocked || !capable(CAP_SYS_ADMIN)) return -EPERM; + if (core_power.enable > SST_CP_MAX_ENABLE || + core_power.priority_type > SST_CP_MAX_PRIORITY_TYPE) + return -EINVAL; + _write_cp_info("cp_enable", core_power.enable, SST_CP_CONTROL_OFFSET, SST_CP_ENABLE_START, SST_CP_ENABLE_WIDTH, SST_MUL_FACTOR_NONE) _write_cp_info("cp_prio_type", core_power.priority_type, SST_CP_CONTROL_OFFSET, @@ -649,6 +663,11 @@ static long isst_if_core_power_state(void __user *argp) #define SST_CLOS_CONFIG_MAX_START 16 #define SST_CLOS_CONFIG_MAX_WIDTH 8 +#define SST_MAX_CLOS 3 + +#define SST_MAX_FREQ 0xff +#define SST_CLOS_MAX_PRIORITY 0x0f + static long isst_if_clos_param(void __user *argp) { struct tpmi_per_power_domain_info *power_domain_info; @@ -657,6 +676,9 @@ static long isst_if_clos_param(void __user *argp) if (copy_from_user(&clos_param, argp, sizeof(clos_param))) return -EFAULT; + if (clos_param.clos > SST_MAX_CLOS) + return -EINVAL; + power_domain_info = get_instance(clos_param.socket_id, clos_param.power_domain_id); if (!power_domain_info) return -EINVAL; @@ -665,6 +687,15 @@ static long isst_if_clos_param(void __user *argp) if (power_domain_info->write_blocked || !capable(CAP_SYS_ADMIN)) return -EPERM; + if (!in_range(clos_param.min_freq_mhz / SST_MUL_FACTOR_FREQ, 0, SST_MAX_FREQ + 1)) + return -EINVAL; + + if (!in_range(clos_param.max_freq_mhz / SST_MUL_FACTOR_FREQ, 0, SST_MAX_FREQ + 1)) + return -EINVAL; + + if (!in_range(clos_param.prop_prio, 0, SST_CLOS_MAX_PRIORITY + 1)) + return -EINVAL; + _write_cp_info("clos.min_freq", clos_param.min_freq_mhz, (SST_CLOS_CONFIG_0_OFFSET + clos_param.clos * SST_REG_SIZE), SST_CLOS_CONFIG_MIN_START, SST_CLOS_CONFIG_MIN_WIDTH, @@ -703,6 +734,8 @@ static long isst_if_clos_param(void __user *argp) #define SST_CLOS_ASSOC_CPUS_PER_REG 16 #define SST_CLOS_ASSOC_BITS_PER_CPU 4 +#define SST_CLOS_ASSOC_MAX_LOGICAL_CPU 63 + static long isst_if_clos_assoc(void __user *argp) { struct isst_if_clos_assoc_cmds assoc_cmds; @@ -729,7 +762,13 @@ static long isst_if_clos_assoc(void __user *argp) if (copy_from_user(&clos_assoc, ptr, sizeof(clos_assoc))) return -EFAULT; - if (clos_assoc.socket_id > topology_max_packages()) + if (clos_assoc.clos > SST_MAX_CLOS) + return -EINVAL; + + if (clos_assoc.socket_id >= topology_max_packages()) + return -EINVAL; + + if (clos_assoc.logical_cpu > SST_CLOS_ASSOC_MAX_LOGICAL_CPU) return -EINVAL; cpu = clos_assoc.logical_cpu; @@ -747,6 +786,8 @@ static long isst_if_clos_assoc(void __user *argp) pkg_id = clos_assoc.socket_id; sst_inst = isst_common.sst_inst[pkg_id]; + if (!sst_inst) + return -EINVAL; punit_id = map_partition_power_domain_id(sst_inst, punit_id, &part); if (punit_id < 0) @@ -845,6 +886,7 @@ static long isst_if_clos_assoc(void __user *argp) #define SST_PP_FEATURE_STATE_START 8 #define SST_PP_FEATURE_STATE_WIDTH 8 +#define SST_PP_FEATURE_STATE_VALID_MASK GENMASK(1, 0) #define SST_BF_FEATURE_SUPPORTED_START 12 #define SST_BF_FEATURE_SUPPORTED_WIDTH 1 @@ -877,7 +919,7 @@ static int isst_if_get_perf_level(void __user *argp) SST_PP_FEATURE_STATE_START, SST_PP_FEATURE_STATE_WIDTH, SST_MUL_FACTOR_NONE) perf_level.enabled = !!(power_domain_info->sst_header.cap_mask & BIT(1)); - level_mask = perf_level.level_mask; + level_mask = perf_level.level_mask & power_domain_info->pp_header.level_en_mask; perf_level.sst_bf_support = 0; for_each_set_bit(level, &level_mask, BITS_PER_BYTE) { /* @@ -932,6 +974,9 @@ static int isst_if_set_perf_level(void __user *argp) if (!power_domain_info) return -EINVAL; + if (perf_level.level > power_domain_info->max_level) + return -EINVAL; + if (power_domain_info->write_blocked || !capable(CAP_SYS_ADMIN)) return -EPERM; @@ -995,6 +1040,9 @@ static int isst_if_set_perf_feature(void __user *argp) if (power_domain_info->write_blocked || !capable(CAP_SYS_ADMIN)) return -EPERM; + if (perf_feature.feature & ~SST_PP_FEATURE_STATE_VALID_MASK) + return -EINVAL; + _write_pp_info("perf_feature", perf_feature.feature, SST_PP_CONTROL_OFFSET, SST_PP_FEATURE_STATE_START, SST_PP_FEATURE_STATE_WIDTH, SST_MUL_FACTOR_NONE) @@ -1261,6 +1309,12 @@ static int isst_if_get_perf_level_mask(void __user *argp) if (!power_domain_info) return -EINVAL; + if (cpumask.level > power_domain_info->max_level) + return -EINVAL; + + if (!(power_domain_info->pp_header.level_en_mask & BIT(cpumask.level))) + return -EINVAL; + _read_pp_level_info("mask", mask, cpumask.level, SST_PP_INFO_2_OFFSET, SST_PP_RSLVD_CORE_MASK_START, SST_PP_RSLVD_CORE_MASK_WIDTH, SST_MUL_FACTOR_NONE) @@ -1306,6 +1360,9 @@ static int isst_if_get_base_freq_info(void __user *argp) if (base_freq.level > power_domain_info->max_level) return -EINVAL; + if (!(power_domain_info->pp_header.level_en_mask & BIT(base_freq.level))) + return -EINVAL; + _read_bf_level_info("p1_high", base_freq.high_base_freq_mhz, base_freq.level, SST_BF_INFO_0_OFFSET, SST_BF_P1_HIGH_START, SST_BF_P1_HIGH_WIDTH, SST_MUL_FACTOR_FREQ) @@ -1342,6 +1399,12 @@ static int isst_if_get_base_freq_mask(void __user *argp) if (!power_domain_info) return -EINVAL; + if (cpumask.level > power_domain_info->max_level) + return -EINVAL; + + if (!(power_domain_info->pp_header.level_en_mask & BIT(cpumask.level))) + return -EINVAL; + _read_bf_level_info("BF-cpumask", mask, cpumask.level, SST_BF_INFO_1_OFFSET, P1_HI_CORE_MASK_START, P1_HI_CORE_MASK_WIDTH, SST_MUL_FACTOR_NONE) @@ -1370,6 +1433,8 @@ static int isst_if_get_tpmi_instance_count(void __user *argp) return -EINVAL; sst_inst = isst_common.sst_inst[tpmi_inst.socket_id]; + if (!sst_inst) + return -EINVAL; tpmi_inst.count = isst_instance_count(sst_inst); @@ -1437,6 +1502,9 @@ static int isst_if_get_turbo_freq_info(void __user *argp) if (turbo_freq.level > power_domain_info->max_level) return -EINVAL; + if (!(power_domain_info->pp_header.level_en_mask & BIT(turbo_freq.level))) + return -EINVAL; + turbo_freq.max_buckets = TRL_MAX_BUCKETS; turbo_freq.max_trl_levels = TRL_MAX_LEVELS; turbo_freq.max_clip_freqs = SST_TF_MAX_LP_CLIP_RATIOS; diff --git a/drivers/platform/x86/lenovo/think-lmi.c b/drivers/platform/x86/lenovo/think-lmi.c index e215e86e3db73..a0e3fa766e37b 100644 --- a/drivers/platform/x86/lenovo/think-lmi.c +++ b/drivers/platform/x86/lenovo/think-lmi.c @@ -438,14 +438,13 @@ static ssize_t current_password_store(struct kobject *kobj, struct tlmi_pwd_setting *setting = to_tlmi_pwd_setting(kobj); size_t pwdlen; - pwdlen = strlen(buf); + /* Strip newline; setting password won't work if one is present. */ + pwdlen = strchrnul(buf, '\n') - buf; /* pwdlen == 0 is allowed to clear the password */ if (pwdlen && ((pwdlen < setting->minlen) || (pwdlen > setting->maxlen))) return -EINVAL; - strscpy(setting->password, buf, setting->maxlen); - /* Strip out CR if one is present, setting password won't work if it is present */ - strreplace(setting->password, '\n', '\0'); + strscpy(setting->password, buf, pwdlen + 1); return count; } @@ -745,6 +744,8 @@ static ssize_t certificate_thumbprint_show(struct kobject *kobj, struct kobj_att return -EOPNOTSUPP; for (i = 0; i < ARRAY_SIZE(thumbtypes); i++) { + ssize_t ret; + if (tlmi_priv.pwdcfg.core.password_mode >= TLMI_PWDCFG_MODE_MULTICERT) { /* Format: 'SVC | SMC, Thumbtype' */ wmistr = kasprintf(GFP_KERNEL, "%s,%s", @@ -756,8 +757,12 @@ static ssize_t certificate_thumbprint_show(struct kobject *kobj, struct kobj_att } if (!wmistr) return -ENOMEM; - count += cert_thumbprint(buf, wmistr, count); + + ret = cert_thumbprint(buf, wmistr, count); kfree(wmistr); + if (ret < 0) + return ret; + count = ret; } return count; @@ -1456,6 +1461,10 @@ static void tlmi_release_attr(void) /* Free up any saved signatures */ kfree(tlmi_priv.pwd_admin->signature); kfree(tlmi_priv.pwd_admin->save_signature); + if (tlmi_priv.pwd_system) { + kfree(tlmi_priv.pwd_system->signature); + kfree(tlmi_priv.pwd_system->save_signature); + } /* Authentication structures */ list_for_each_entry_safe(pos, n, &tlmi_priv.authentication_kset->list, entry) diff --git a/drivers/platform/x86/lenovo/ymc.c b/drivers/platform/x86/lenovo/ymc.c index 1b73a55f1b89c..015e046b0fcea 100644 --- a/drivers/platform/x86/lenovo/ymc.c +++ b/drivers/platform/x86/lenovo/ymc.c @@ -8,6 +8,8 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include +#include #include #include #include @@ -20,6 +22,8 @@ #define LENOVO_YMC_QUERY_INSTANCE 0 #define LENOVO_YMC_QUERY_METHOD 0x01 +#define LENOVO_YMC_STATE_MASK GENMASK(7, 0) + static bool force; module_param(force, bool, 0444); MODULE_PARM_DESC(force, "Force loading on boards without a convertible DMI chassis-type"); @@ -85,7 +89,9 @@ static void lenovo_ymc_notify(struct wmi_device *wdev, union acpi_object *data) "WMI event data is not an integer\n"); goto free_obj; } - code = obj->integer.value; + + /* strip upper bits (e.g. 0x50000) on newer devices */ + code = FIELD_GET(LENOVO_YMC_STATE_MASK, obj->integer.value); if (!sparse_keymap_report_event(priv->input_dev, code, 1, true)) dev_warn(&wdev->dev, "Unknown key %d pressed\n", code); diff --git a/drivers/platform/x86/lg-laptop.c b/drivers/platform/x86/lg-laptop.c index a8f2f465ef3f9..a2db9657027e5 100644 --- a/drivers/platform/x86/lg-laptop.c +++ b/drivers/platform/x86/lg-laptop.c @@ -100,6 +100,7 @@ static u32 inited; #define INIT_SPARSE_KEYMAP 0x80 static int battery_limit_use_wmbb; +static bool kbd_backlight_available; static struct led_classdev kbd_backlight; static enum led_brightness get_kbd_backlight_level(struct device *dev); @@ -214,6 +215,7 @@ static union acpi_object *lg_wmbb(struct device *dev, u32 method_id, u32 arg1, u static void wmi_notify(union acpi_object *obj, void *context) { long data = (long)context; + unsigned int brightness; pr_debug("event guid %li\n", data); if (!obj) @@ -224,8 +226,11 @@ static void wmi_notify(union acpi_object *obj, void *context) struct key_entry *key; if (eventcode == 0x10000000) { - led_classdev_notify_brightness_hw_changed( - &kbd_backlight, get_kbd_backlight_level(kbd_backlight.dev->parent)); + if (kbd_backlight_available) { + brightness = get_kbd_backlight_level(kbd_backlight.dev->parent); + led_classdev_notify_brightness_hw_changed(&kbd_backlight, + brightness); + } } else { key = sparse_keymap_entry_from_scancode( wmi_input_dev, eventcode); @@ -865,8 +870,13 @@ static int acpi_probe(struct platform_device *pdev) goto out_platform_device; /* LEDs are optional */ - led_classdev_register(&pf_device->dev, &kbd_backlight); - led_classdev_register(&pf_device->dev, &tpad_led); + ret = devm_led_classdev_register(&pdev->dev, &kbd_backlight); + if (ret < 0) + kbd_backlight_available = false; + else + kbd_backlight_available = true; + + devm_led_classdev_register(&pdev->dev, &tpad_led); wmi_input_setup(); battery_hook_register(&battery_hook); @@ -884,9 +894,6 @@ static void acpi_remove(struct platform_device *pdev) { sysfs_remove_group(&pf_device->dev.kobj, &dev_attribute_group); - led_classdev_unregister(&tpad_led); - led_classdev_unregister(&kbd_backlight); - battery_hook_unregister(&battery_hook); wmi_input_destroy(); platform_device_unregister(pf_device); diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c index 719add753cb3d..49098d82d6e02 100644 --- a/drivers/platform/x86/panasonic-laptop.c +++ b/drivers/platform/x86/panasonic-laptop.c @@ -360,7 +360,16 @@ static int acpi_pcc_retrieve_biosdata(struct pcc_acpi *pcc) } else pr_err("Invalid HKEY.SINF data\n"); } - pcc->sinf[hkey->package.count] = -1; + /* + * pcc->sinf[] has pcc->num_sifr elements (valid indices + * 0..num_sifr-1). On DSDTs where SINF's package count equals + * num_sifr exactly -- the off-by-one case probe()'s num_sifr++ + * already allocates a spare element for -- there is no room left + * for this trailing sentinel; nothing reads it back, so just skip + * the write rather than running one element past the flex array. + */ + if (hkey->package.count < pcc->num_sifr) + pcc->sinf[hkey->package.count] = -1; end: kfree(buffer.pointer); diff --git a/drivers/platform/x86/x86-android-tablets/core.c b/drivers/platform/x86/x86-android-tablets/core.c index 5db794d65eb5f..7751261f4b7cf 100644 --- a/drivers/platform/x86/x86-android-tablets/core.c +++ b/drivers/platform/x86/x86-android-tablets/core.c @@ -367,6 +367,7 @@ static void gpio_secondary_unset(void *data) struct device *dev = data; set_secondary_fwnode(dev, NULL); + put_device(dev); } static void gpio_secondary_unregister_node_group(void *data) @@ -380,6 +381,7 @@ static int gpio_secondary_fwnode_init(struct device *parent) { const struct software_node *const *swnode; struct fwnode_handle *fwnode; + struct device *phys_dev; int ret; if (!gpiochip_node_group) @@ -407,9 +409,15 @@ static int gpio_secondary_fwnode_init(struct device *parent) if (WARN_ON(!fwnode)) return -ENOENT; - set_secondary_fwnode(dev, fwnode); + phys_dev = acpi_get_first_physical_node(to_acpi_device(dev)); + if (!phys_dev) + return dev_err_probe(parent, -ENODEV, + "No physical device for ACPI GPIO dev: %pfwP\n", + fwnode); - ret = devm_add_action_or_reset(parent, gpio_secondary_unset, dev); + set_secondary_fwnode(phys_dev, fwnode); + + ret = devm_add_action_or_reset(parent, gpio_secondary_unset, get_device(phys_dev)); if (ret) return ret; } diff --git a/drivers/pmdomain/bcm/bcm2835-power.c b/drivers/pmdomain/bcm/bcm2835-power.c index b76d74e3849be..68a0a7a8cee38 100644 --- a/drivers/pmdomain/bcm/bcm2835-power.c +++ b/drivers/pmdomain/bcm/bcm2835-power.c @@ -677,7 +677,12 @@ static int bcm2835_power_probe(struct platform_device *pdev) if (ret) goto fail; - of_genpd_add_provider_onecell(dev->parent->of_node, &power->pd_xlate); + ret = of_genpd_add_provider_onecell(dev->parent->of_node, + &power->pd_xlate); + if (ret) { + dev_err_probe(dev, ret, "failed to add genpd provider\n"); + goto fail; + } dev_info(dev, "Broadcom BCM2835 power domains driver"); return 0; diff --git a/drivers/pmdomain/mediatek/Kconfig b/drivers/pmdomain/mediatek/Kconfig index 8923e6516441c..a2eb704a773c9 100644 --- a/drivers/pmdomain/mediatek/Kconfig +++ b/drivers/pmdomain/mediatek/Kconfig @@ -43,9 +43,12 @@ config MTK_MFG_PM_DOMAIN This driver is required for the Mali GPU to work at all on MT8196 and MT6991. +endmenu + config AIROHA_CPU_PM_DOMAIN tristate "Airoha CPU power domain" default ARCH_AIROHA + depends on ARCH_AIROHA || COMPILE_TEST depends on HAVE_ARM_SMCCC depends on PM select PM_GENERIC_DOMAINS @@ -54,5 +57,3 @@ config AIROHA_CPU_PM_DOMAIN CPU frequency and power is controlled by ATF with SMC command to set performance states. - -endmenu diff --git a/drivers/pmdomain/qcom/rpmhpd.c b/drivers/pmdomain/qcom/rpmhpd.c index ffc46de8d07a1..63120e703923f 100644 --- a/drivers/pmdomain/qcom/rpmhpd.c +++ b/drivers/pmdomain/qcom/rpmhpd.c @@ -245,13 +245,9 @@ static struct rpmhpd *eliza_rpmhpds[] = { [RPMHPD_GFX] = &gfx, [RPMHPD_LCX] = &lcx, [RPMHPD_LMX] = &lmx, - [RPMHPD_MMCX] = &mmcx, - [RPMHPD_MMCX_AO] = &mmcx_ao, [RPMHPD_MSS] = &mss, [RPMHPD_MX] = &mx, [RPMHPD_MX_AO] = &mx_ao, - [RPMHPD_MXC] = &mxc, - [RPMHPD_MXC_AO] = &mxc_ao, [RPMHPD_NSP] = &nsp, }; diff --git a/drivers/power/reset/reboot-mode.c b/drivers/power/reset/reboot-mode.c index d20e44db05325..af00c00eceee7 100644 --- a/drivers/power/reset/reboot-mode.c +++ b/drivers/power/reset/reboot-mode.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -168,10 +169,11 @@ static int reboot_mode_create_device(struct reboot_mode_driver *reboot) */ int reboot_mode_register(struct reboot_mode_driver *reboot) { - struct mode_info *info; + struct mode_info *info = NULL; struct property *prop; struct device_node *np = reboot->dev->of_node; size_t len = strlen(PREFIX); + u32 magic; int ret; INIT_LIST_HEAD(&reboot->head); @@ -180,22 +182,22 @@ int reboot_mode_register(struct reboot_mode_driver *reboot) if (strncmp(prop->name, PREFIX, len)) continue; - info = devm_kzalloc(reboot->dev, sizeof(*info), GFP_KERNEL); + if (device_property_read_u32(reboot->dev, prop->name, &magic)) { + dev_dbg(reboot->dev, "reboot mode %s without magic number\n", + prop->name); + continue; + } + + info = kzalloc_obj(*info, GFP_KERNEL); if (!info) { ret = -ENOMEM; goto error; } - if (of_property_read_u32(np, prop->name, &info->magic)) { - dev_err(reboot->dev, "reboot mode %s without magic number\n", - info->mode); - devm_kfree(reboot->dev, info); - continue; - } - + info->magic = magic; info->mode = kstrdup_const(prop->name + len, GFP_KERNEL); if (!info->mode) { - ret = -ENOMEM; + ret = -ENOMEM; goto error; } else if (info->mode[0] == '\0') { kfree_const(info->mode); @@ -206,6 +208,7 @@ int reboot_mode_register(struct reboot_mode_driver *reboot) } list_add_tail(&info->list, &reboot->head); + info = NULL; } reboot->reboot_notifier.notifier_call = reboot_mode_notify; @@ -218,6 +221,7 @@ int reboot_mode_register(struct reboot_mode_driver *reboot) return 0; error: + kfree(info); reboot_mode_unregister(reboot); return ret; } @@ -261,12 +265,16 @@ static inline void reboot_mode_unregister_device(struct reboot_mode_driver *rebo int reboot_mode_unregister(struct reboot_mode_driver *reboot) { struct mode_info *info; + struct mode_info *next; unregister_reboot_notifier(&reboot->reboot_notifier); reboot_mode_unregister_device(reboot); - list_for_each_entry(info, &reboot->head, list) + list_for_each_entry_safe(info, next, &reboot->head, list) { + list_del(&info->list); kfree_const(info->mode); + kfree(info); + } return 0; } diff --git a/drivers/power/sequencing/Kconfig b/drivers/power/sequencing/Kconfig index 1c5f5820f5b76..226c62704d9bb 100644 --- a/drivers/power/sequencing/Kconfig +++ b/drivers/power/sequencing/Kconfig @@ -29,7 +29,8 @@ config POWER_SEQUENCING_QCOM_WCN config POWER_SEQUENCING_TH1520_GPU tristate "T-HEAD TH1520 GPU power sequencing driver" - depends on (ARCH_THEAD && AUXILIARY_BUS) || COMPILE_TEST + depends on ARCH_THEAD || COMPILE_TEST + select AUXILIARY_BUS help Say Y here to enable the power sequencing driver for the TH1520 SoC GPU. This driver handles the complex clock and reset sequence diff --git a/drivers/power/sequencing/core.c b/drivers/power/sequencing/core.c index 02f42da915985..112f05625aa64 100644 --- a/drivers/power/sequencing/core.c +++ b/drivers/power/sequencing/core.c @@ -101,6 +101,7 @@ static struct pwrseq_unit *pwrseq_unit_new(const struct pwrseq_unit_data *data) } kref_init(&unit->ref); + INIT_LIST_HEAD(&unit->list); INIT_LIST_HEAD(&unit->deps); unit->enable = data->enable; unit->disable = data->disable; @@ -504,10 +505,6 @@ pwrseq_device_register(const struct pwrseq_config *config) */ device_initialize(&pwrseq->dev); - ret = dev_set_name(&pwrseq->dev, "pwrseq.%d", pwrseq->id); - if (ret) - goto err_put_pwrseq; - pwrseq->owner = config->owner ?: THIS_MODULE; pwrseq->match = config->match; @@ -516,6 +513,10 @@ pwrseq_device_register(const struct pwrseq_config *config) INIT_LIST_HEAD(&pwrseq->targets); INIT_LIST_HEAD(&pwrseq->units); + ret = dev_set_name(&pwrseq->dev, "pwrseq.%d", pwrseq->id); + if (ret) + goto err_put_pwrseq; + ret = pwrseq_setup_targets(config->targets, pwrseq); if (ret) goto err_put_pwrseq; @@ -912,6 +913,8 @@ int pwrseq_power_on(struct pwrseq_desc *desc) if (!ret) desc->powered_on = true; } + if (ret) + return ret; if (target->post_enable) { ret = target->post_enable(pwrseq); diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c index eb5c1ae68e440..cfcc05dfe28d6 100644 --- a/drivers/power/supply/ab8500_fg.c +++ b/drivers/power/supply/ab8500_fg.c @@ -3054,6 +3054,20 @@ static void ab8500_fg_unbind(struct device *dev, struct device *master, flush_workqueue(di->fg_wq); } +/* Disable, not cancel: works stay disabled so nothing can re-arm them. */ +static void ab8500_fg_destroy_workqueue(void *data) +{ + struct ab8500_fg *di = data; + + disable_work_sync(&di->fg_acc_cur_work); + disable_work_sync(&di->fg_work); + disable_delayed_work_sync(&di->fg_reinit_work); + disable_delayed_work_sync(&di->fg_low_bat_work); + disable_delayed_work_sync(&di->fg_check_hw_failure_work); + disable_delayed_work_sync(&di->fg_periodic_work); + destroy_workqueue(di->fg_wq); +} + static const struct component_ops ab8500_fg_component_ops = { .bind = ab8500_fg_bind, .unbind = ab8500_fg_unbind, @@ -3155,6 +3169,11 @@ static int ab8500_fg_probe(struct platform_device *pdev) return PTR_ERR(di->fg_psy); } + /* Registered after fg_psy, before the IRQs: devm frees IRQ -> workqueue -> fg_psy. */ + ret = devm_add_action_or_reset(dev, ab8500_fg_destroy_workqueue, di); + if (ret) + return ret; + di->fg_samples = SEC_TO_SAMPLE(di->bm->fg_params->init_timer); /* @@ -3167,22 +3186,16 @@ static int ab8500_fg_probe(struct platform_device *pdev) /* Register primary interrupt handlers */ for (i = 0; i < ARRAY_SIZE(ab8500_fg_irq); i++) { irq = platform_get_irq_byname(pdev, ab8500_fg_irq[i].name); - if (irq < 0) { - destroy_workqueue(di->fg_wq); + if (irq < 0) return irq; - } ret = devm_request_threaded_irq(dev, irq, NULL, ab8500_fg_irq[i].isr, IRQF_SHARED | IRQF_NO_SUSPEND | IRQF_ONESHOT, ab8500_fg_irq[i].name, di); - if (ret != 0) { - dev_err(dev, "failed to request %s IRQ %d: %d\n", - ab8500_fg_irq[i].name, irq, ret); - destroy_workqueue(di->fg_wq); + if (ret != 0) return ret; - } dev_dbg(dev, "Requested %s IRQ %d: %d\n", ab8500_fg_irq[i].name, irq, ret); } @@ -3196,7 +3209,6 @@ static int ab8500_fg_probe(struct platform_device *pdev) ret = ab8500_fg_sysfs_init(di); if (ret) { dev_err(dev, "failed to create sysfs entry\n"); - destroy_workqueue(di->fg_wq); return ret; } @@ -3204,7 +3216,6 @@ static int ab8500_fg_probe(struct platform_device *pdev) if (ret) { dev_err(dev, "failed to create FG psy\n"); ab8500_fg_sysfs_exit(di); - destroy_workqueue(di->fg_wq); return ret; } @@ -3224,7 +3235,6 @@ static void ab8500_fg_remove(struct platform_device *pdev) { struct ab8500_fg *di = platform_get_drvdata(pdev); - destroy_workqueue(di->fg_wq); component_del(&pdev->dev, &ab8500_fg_component_ops); list_del(&di->node); ab8500_fg_sysfs_exit(di); diff --git a/drivers/power/supply/bd71828-power.c b/drivers/power/supply/bd71828-power.c index 19f24f859666a..3be7a75a7d6ca 100644 --- a/drivers/power/supply/bd71828-power.c +++ b/drivers/power/supply/bd71828-power.c @@ -191,12 +191,13 @@ static int bd71828_get_current_ds_adc(struct bd71828_power *pwr, int *curr, int { __be16 tmp_curr; char *tmp = (char *)&tmp_curr; - int dir = 1; int regs[] = { pwr->regs->ibat, pwr->regs->ibat_avg }; int *vals[] = { curr, curr_avg }; int ret, i; - for (dir = 1, i = 0; i < ARRAY_SIZE(regs); i++) { + for (i = 0; i < ARRAY_SIZE(regs); i++) { + int dir = 1; + ret = regmap_bulk_read(pwr->regmap, regs[i], &tmp_curr, sizeof(tmp_curr)); if (ret) @@ -224,13 +225,13 @@ static int bd71815_get_temp(struct bd71828_power *pwr, int *temp) if (ret) return ret; - t = 200 - t; - if (t > 200) { dev_err(pwr->dev, "Failed to read battery temperature\n"); return -ENODATA; } + *temp = 200 - t; + return 0; } @@ -454,8 +455,10 @@ static int bd71828_charger_get_property(struct power_supply *psy, switch (psp) { case POWER_SUPPLY_PROP_ONLINE: ret = get_chg_online(pwr, &online); - if (!ret) - val->intval = online; + if (ret) + return ret; + + val->intval = online; break; case POWER_SUPPLY_PROP_VOLTAGE_NOW: ret = bd7182x_read16_himask(pwr, pwr->regs->vdcin, @@ -638,7 +641,6 @@ static const enum power_supply_property bd71828_charger_props[] = { static const enum power_supply_property bd71828_battery_props[] = { POWER_SUPPLY_PROP_STATUS, - POWER_SUPPLY_PROP_HEALTH, POWER_SUPPLY_PROP_VOLTAGE_NOW, POWER_SUPPLY_PROP_HEALTH, POWER_SUPPLY_PROP_PRESENT, @@ -1217,7 +1219,9 @@ static int bd71828_power_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, ret, "sense resistor missing\n"); dev_set_drvdata(&pdev->dev, pwr); - bd71828_init_hardware(pwr); + ret = bd71828_init_hardware(pwr); + if (ret) + return dev_err_probe(&pdev->dev, ret, "hardware init failed\n"); bat_cfg.drv_data = pwr; bat_cfg.fwnode = dev_fwnode(&pdev->dev); diff --git a/drivers/power/supply/bd99954-charger.h b/drivers/power/supply/bd99954-charger.h index f588979253830..d053ac901274c 100644 --- a/drivers/power/supply/bd99954-charger.h +++ b/drivers/power/supply/bd99954-charger.h @@ -279,17 +279,7 @@ enum bd9995x_fields { F_VBUS_EXTID, F_VBUS_IDRDET, F_VBUS_INDO, - F_VBUS_UCDSWEN, - F_VBUS_RREF_EN, - F_VBUS_DPPU_EN, - F_VBUS_DPREF_EN, - F_VBUS_DMREF_EN, - F_VBUS_DPDET_EN, - F_VBUS_DMDET_EN, - F_VBUS_DPSINK_EN, - F_VBUS_DMSINK_EN, - F_VBUS_DP_BUFF_EN, - F_VBUS_DM_BUFF_EN, + F_VBUS_EXTCLKENBL, F_VBUS_PLSTESTEN, F_VBUS_UCDSWEN_TSTENB, @@ -626,17 +616,6 @@ static const struct reg_field bd9995x_reg_fields[] = { [F_VBUS_EXTID] = REG_FIELD(VBUS_IDD_STATUS, 5, 5), [F_VBUS_IDRDET] = REG_FIELD(VBUS_IDD_STATUS, 4, 4), [F_VBUS_INDO] = REG_FIELD(VBUS_IDD_STATUS, 0, 3), - [F_VBUS_UCDSWEN] = REG_FIELD(VCC_UCD_FCTRL_SET, 10, 10), - [F_VBUS_RREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 9, 9), - [F_VBUS_DPPU_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 8, 8), - [F_VBUS_DPREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 7, 7), - [F_VBUS_DMREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 6, 6), - [F_VBUS_DPDET_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 5, 5), - [F_VBUS_DMDET_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 4, 4), - [F_VBUS_DPSINK_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 3, 3), - [F_VBUS_DMSINK_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 2, 2), - [F_VBUS_DP_BUFF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 1, 1), - [F_VBUS_DM_BUFF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 0, 0), [F_VBUS_EXTCLKENBL] = REG_FIELD(VBUS_UCD_FCTRL_EN, 15, 15), [F_VBUS_PLSTESTEN] = REG_FIELD(VBUS_UCD_FCTRL_EN, 14, 14), diff --git a/drivers/power/supply/bq24257_charger.c b/drivers/power/supply/bq24257_charger.c index 72f1bfea8d541..54dd9d4ec39cd 100644 --- a/drivers/power/supply/bq24257_charger.c +++ b/drivers/power/supply/bq24257_charger.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -1003,10 +1004,6 @@ static int bq24257_probe(struct i2c_client *client) if (bq->info->chip == BQ24250) bq->iilimit_autoset_enable = false; - if (bq->iilimit_autoset_enable) - INIT_DELAYED_WORK(&bq->iilimit_setup_work, - bq24257_iilimit_setup_work); - /* * The BQ24250 doesn't have a dedicated Power Good (PG) pin so let's * not probe for it and instead use a SW-based approach to determine @@ -1047,6 +1044,14 @@ static int bq24257_probe(struct i2c_client *client) return ret; } + if (bq->iilimit_autoset_enable) { + ret = devm_delayed_work_autocancel(dev, + &bq->iilimit_setup_work, + bq24257_iilimit_setup_work); + if (ret) + return ret; + } + ret = devm_request_threaded_irq(dev, client->irq, NULL, bq24257_irq_handler_thread, IRQF_TRIGGER_FALLING | @@ -1064,9 +1069,6 @@ static void bq24257_remove(struct i2c_client *client) { struct bq24257_device *bq = i2c_get_clientdata(client); - if (bq->iilimit_autoset_enable) - cancel_delayed_work_sync(&bq->iilimit_setup_work); - bq24257_field_write(bq, F_RESET, 1); /* reset to defaults */ } diff --git a/drivers/power/supply/bq256xx_charger.c b/drivers/power/supply/bq256xx_charger.c index 5a6634fde8376..43e1da849df42 100644 --- a/drivers/power/supply/bq256xx_charger.c +++ b/drivers/power/supply/bq256xx_charger.c @@ -896,6 +896,8 @@ static void bq256xx_charger_reset(void *data) if (!IS_ERR_OR_NULL(bq->usb3_phy)) usb_unregister_notifier(bq->usb3_phy, &bq->usb_nb); + + cancel_work_sync(&bq->usb_work); } static int bq256xx_set_charger_property(struct power_supply *psy, @@ -1721,24 +1723,12 @@ static int bq256xx_probe(struct i2c_client *client) return ret; } - ret = devm_add_action_or_reset(dev, bq256xx_charger_reset, bq); - if (ret) - return ret; + INIT_WORK(&bq->usb_work, bq256xx_usb_work); + bq->usb_nb.notifier_call = bq256xx_usb_notifier; /* OTG reporting */ bq->usb2_phy = devm_usb_get_phy(dev, USB_PHY_TYPE_USB2); - if (!IS_ERR_OR_NULL(bq->usb2_phy)) { - INIT_WORK(&bq->usb_work, bq256xx_usb_work); - bq->usb_nb.notifier_call = bq256xx_usb_notifier; - usb_register_notifier(bq->usb2_phy, &bq->usb_nb); - } - bq->usb3_phy = devm_usb_get_phy(dev, USB_PHY_TYPE_USB3); - if (!IS_ERR_OR_NULL(bq->usb3_phy)) { - INIT_WORK(&bq->usb_work, bq256xx_usb_work); - bq->usb_nb.notifier_call = bq256xx_usb_notifier; - usb_register_notifier(bq->usb3_phy, &bq->usb_nb); - } ret = bq256xx_power_supply_init(bq, &psy_cfg, dev); if (ret) { @@ -1746,6 +1736,17 @@ static int bq256xx_probe(struct i2c_client *client) return ret; } + /* Register after the power supplies so devm runs it first. */ + ret = devm_add_action_or_reset(dev, bq256xx_charger_reset, bq); + if (ret) + return ret; + + if (!IS_ERR_OR_NULL(bq->usb2_phy)) + usb_register_notifier(bq->usb2_phy, &bq->usb_nb); + + if (!IS_ERR_OR_NULL(bq->usb3_phy)) + usb_register_notifier(bq->usb3_phy, &bq->usb_nb); + if (client->irq) { ret = devm_request_threaded_irq(dev, client->irq, NULL, bq256xx_irq_handler_thread, diff --git a/drivers/power/supply/bq25890_charger.c b/drivers/power/supply/bq25890_charger.c index 180bc137a8632..741a45cc5f35d 100644 --- a/drivers/power/supply/bq25890_charger.c +++ b/drivers/power/supply/bq25890_charger.c @@ -1389,6 +1389,14 @@ static int bq25890_fw_read_u32_props(struct bq25890_device *bq) return 0; } +static void bq25890_release_secondary_chrg(void *data) +{ + struct bq25890_device *bq = data; + + power_supply_put(bq->secondary_chrg); + bq->secondary_chrg = NULL; +} + static int bq25890_fw_probe(struct bq25890_device *bq) { int ret; @@ -1401,6 +1409,10 @@ static int bq25890_fw_probe(struct bq25890_device *bq) bq->secondary_chrg = power_supply_get_by_name(str); if (!bq->secondary_chrg) return -EPROBE_DEFER; + + ret = devm_add_action_or_reset(bq->dev, bq25890_release_secondary_chrg, bq); + if (ret) + return ret; } /* Optional, left at 0 if property is not present */ diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 45f0e39b8c2dd..01c9ec1116fa0 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -330,7 +330,7 @@ static u8 [BQ27XXX_REG_FLAGS] = 0x0a, [BQ27XXX_REG_TTE] = 0x16, [BQ27XXX_REG_TTF] = INVALID_REG_ADDR, - [BQ27XXX_REG_TTES] = 0x1c, + [BQ27XXX_REG_TTES] = 0x1a, [BQ27XXX_REG_TTECP] = INVALID_REG_ADDR, [BQ27XXX_REG_NAC] = 0x0c, [BQ27XXX_REG_RC] = 0x10, @@ -495,7 +495,7 @@ static u8 [BQ27XXX_REG_RC] = 0x10, [BQ27XXX_REG_FCC] = 0x12, [BQ27XXX_REG_CYCT] = 0x2a, - [BQ27XXX_REG_AE] = 0x22, + [BQ27XXX_REG_AE] = INVALID_REG_ADDR, [BQ27XXX_REG_SOC] = 0x2c, [BQ27XXX_REG_DCAP] = 0x3c, [BQ27XXX_REG_AP] = 0x22, @@ -516,7 +516,7 @@ static u8 [BQ27XXX_REG_RC] = 0x10, [BQ27XXX_REG_FCC] = 0x12, [BQ27XXX_REG_CYCT] = 0x2a, - [BQ27XXX_REG_AE] = 0x22, + [BQ27XXX_REG_AE] = INVALID_REG_ADDR, [BQ27XXX_REG_SOC] = 0x2c, [BQ27XXX_REG_DCAP] = 0x3c, [BQ27XXX_REG_AP] = 0x22, diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c index 71b6e49a835f8..fd4cd9cd2ce2a 100644 --- a/drivers/power/supply/charger-manager.c +++ b/drivers/power/supply/charger-manager.c @@ -1014,6 +1014,29 @@ static int charger_extcon_init(struct charger_manager *cm, return 0; } +static int charger_manager_get_regulators(struct charger_manager *cm) +{ + struct charger_desc *desc = cm->desc; + struct charger_regulator *charger; + int i, ret; + + for (i = 0; i < desc->num_charger_regulators; i++) { + charger = &desc->charger_regulators[i]; + charger->consumer = regulator_get(cm->dev, + charger->regulator_name); + if (IS_ERR(charger->consumer)) { + dev_err(cm->dev, "Cannot find charger(%s)\n", + charger->regulator_name); + ret = PTR_ERR(charger->consumer); + while (i-- > 0) + regulator_put(desc->charger_regulators[i].consumer); + return ret; + } + charger->cm = cm; + } + return 0; +} + /** * charger_manager_register_extcon - Register extcon device to receive state * of charger cable. @@ -1036,15 +1059,6 @@ static int charger_manager_register_extcon(struct charger_manager *cm) for (i = 0; i < desc->num_charger_regulators; i++) { charger = &desc->charger_regulators[i]; - charger->consumer = regulator_get(cm->dev, - charger->regulator_name); - if (IS_ERR(charger->consumer)) { - dev_err(cm->dev, "Cannot find charger(%s)\n", - charger->regulator_name); - return PTR_ERR(charger->consumer); - } - charger->cm = cm; - for (j = 0; j < charger->num_cables; j++) { struct charger_cable *cable = &charger->cables[j]; @@ -1580,13 +1594,23 @@ static int charger_manager_probe(struct platform_device *pdev) } psy_cfg.attr_grp = desc->sysfs_groups; + /* + * Acquire charger regulators before exposing the sysfs entries, so + * userspace cannot reach externally_control before the regulators + * (and charger->cm) are available. Mirrors the order in remove(). + */ + ret = charger_manager_get_regulators(cm); + if (ret < 0) + return ret; + cm->charger_psy = power_supply_register(&pdev->dev, &cm->charger_psy_desc, &psy_cfg); if (IS_ERR(cm->charger_psy)) { dev_err(&pdev->dev, "Cannot register charger-manager with name \"%s\"\n", cm->charger_psy_desc.name); - return PTR_ERR(cm->charger_psy); + ret = PTR_ERR(cm->charger_psy); + goto err_regulator; } /* Register extcon device for charger cable */ @@ -1620,11 +1644,11 @@ static int charger_manager_probe(struct platform_device *pdev) return 0; err_reg_extcon: + power_supply_unregister(cm->charger_psy); +err_regulator: for (i = 0; i < desc->num_charger_regulators; i++) regulator_put(desc->charger_regulators[i].consumer); - power_supply_unregister(cm->charger_psy); - return ret; } @@ -1642,12 +1666,12 @@ static void charger_manager_remove(struct platform_device *pdev) cancel_work_sync(&setup_polling); cancel_delayed_work_sync(&cm_monitor_work); - for (i = 0 ; i < desc->num_charger_regulators ; i++) - regulator_put(desc->charger_regulators[i].consumer); + try_charger_enable(cm, false); power_supply_unregister(cm->charger_psy); - try_charger_enable(cm, false); + for (i = 0 ; i < desc->num_charger_regulators ; i++) + regulator_put(desc->charger_regulators[i].consumer); } static const struct platform_device_id charger_manager_id[] = { diff --git a/drivers/power/supply/cros_usbpd-charger.c b/drivers/power/supply/cros_usbpd-charger.c index c1cbe6e5476e9..69d227ceb5c9e 100644 --- a/drivers/power/supply/cros_usbpd-charger.c +++ b/drivers/power/supply/cros_usbpd-charger.c @@ -124,6 +124,11 @@ static int cros_usbpd_charger_get_num_ports(struct charger_data *charger) if (ret < 0) return ret; + if (resp.port_count > EC_USB_PD_MAX_PORTS) { + dev_warn(charger->dev, "Charge port count out of bounds\n"); + return EC_USB_PD_MAX_PORTS; + } + return resp.port_count; } @@ -137,6 +142,11 @@ static int cros_usbpd_charger_get_usbpd_num_ports(struct charger_data *charger) if (ret < 0) return ret; + if (resp.num_ports > EC_USB_PD_MAX_PORTS) { + dev_warn(charger->dev, "USB PD port count out of bounds\n"); + return EC_USB_PD_MAX_PORTS; + } + return resp.num_ports; } @@ -588,10 +598,13 @@ static int cros_usbpd_charger_probe(struct platform_device *pd) /* * Sanity checks on the number of ports: - * there should be at most 1 dedicated port + * there should be at most 1 dedicated port, and the count must + * not exceed the maximum number of supported ports + * (EC_USB_PD_MAX_PORTS). */ if (charger->num_charger_ports < charger->num_usbpd_ports || - charger->num_charger_ports > (charger->num_usbpd_ports + 1)) { + charger->num_charger_ports > (charger->num_usbpd_ports + 1) || + charger->num_charger_ports > EC_USB_PD_MAX_PORTS) { dev_err(dev, "Unexpected number of charge port count\n"); ret = -EPROTO; goto fail_nowarn; diff --git a/drivers/power/supply/isp1704_charger.c b/drivers/power/supply/isp1704_charger.c index 237912a922724..e329321d06dbd 100644 --- a/drivers/power/supply/isp1704_charger.c +++ b/drivers/power/supply/isp1704_charger.c @@ -482,6 +482,7 @@ static void isp1704_charger_remove(struct platform_device *pdev) struct isp1704_charger *isp = platform_get_drvdata(pdev); usb_unregister_notifier(isp->phy, &isp->nb); + cancel_work_sync(&isp->work); power_supply_unregister(isp->psy); isp1704_charger_set_power(isp, 0); } diff --git a/drivers/power/supply/lp8727_charger.c b/drivers/power/supply/lp8727_charger.c index 641e0445a4109..4537b7b413ddc 100644 --- a/drivers/power/supply/lp8727_charger.c +++ b/drivers/power/supply/lp8727_charger.c @@ -280,10 +280,10 @@ static int lp8727_setup_irq(struct lp8727_chg *pchg) static void lp8727_release_irq(struct lp8727_chg *pchg) { - cancel_delayed_work_sync(&pchg->work); - if (pchg->irq) free_irq(pchg->irq, pchg); + + cancel_delayed_work_sync(&pchg->work); } static enum power_supply_property lp8727_charger_prop[] = { diff --git a/drivers/power/supply/lp8788-charger.c b/drivers/power/supply/lp8788-charger.c index f0a680c155c48..1bc2c150fb912 100644 --- a/drivers/power/supply/lp8788-charger.c +++ b/drivers/power/supply/lp8788-charger.c @@ -710,8 +710,8 @@ static void lp8788_charger_remove(struct platform_device *pdev) { struct lp8788_charger *pchg = platform_get_drvdata(pdev); - flush_work(&pchg->charger_work); lp8788_irq_unregister(pdev, pchg); + cancel_work_sync(&pchg->charger_work); } static struct platform_driver lp8788_charger_driver = { diff --git a/drivers/power/supply/max17040_battery.c b/drivers/power/supply/max17040_battery.c index 03ac569e185c5..41744191a3638 100644 --- a/drivers/power/supply/max17040_battery.c +++ b/drivers/power/supply/max17040_battery.c @@ -192,8 +192,11 @@ static int max17040_raw_vcell_to_uvolts(struct max17040_chip *chip, u16 vcell) static int max17040_get_vcell(struct max17040_chip *chip) { u32 vcell; + int ret; - regmap_read(chip->regmap, MAX17040_VCELL, &vcell); + ret = regmap_read(chip->regmap, MAX17040_VCELL, &vcell); + if (ret) + return ret; return max17040_raw_vcell_to_uvolts(chip, vcell); } @@ -201,8 +204,11 @@ static int max17040_get_vcell(struct max17040_chip *chip) static int max17040_get_soc(struct max17040_chip *chip) { u32 soc; + int ret; - regmap_read(chip->regmap, MAX17040_SOC, &soc); + ret = regmap_read(chip->regmap, MAX17040_SOC, &soc); + if (ret) + return ret; return soc >> (chip->quirk_double_soc ? 9 : 8); } @@ -261,7 +267,11 @@ static int max17040_get_of_data(struct max17040_chip *chip) static void max17040_check_changes(struct max17040_chip *chip) { - chip->soc = max17040_get_soc(chip); + int soc; + + soc = max17040_get_soc(chip); + if (soc >= 0) + chip->soc = soc; } static void max17040_queue_work(struct max17040_chip *chip) @@ -396,10 +406,16 @@ static int max17040_get_property(struct power_supply *psy, val->intval = max17040_get_online(chip); break; case POWER_SUPPLY_PROP_VOLTAGE_NOW: - val->intval = max17040_get_vcell(chip); + ret = max17040_get_vcell(chip); + if (ret < 0) + return ret; + val->intval = ret; break; case POWER_SUPPLY_PROP_CAPACITY: - val->intval = max17040_get_soc(chip); + ret = max17040_get_soc(chip); + if (ret < 0) + return ret; + val->intval = ret; break; case POWER_SUPPLY_PROP_CAPACITY_ALERT_MIN: val->intval = chip->low_soc_alert; @@ -458,16 +474,12 @@ static const struct power_supply_desc max17040_battery_desc = { static int max17040_probe(struct i2c_client *client) { const struct i2c_device_id *id = i2c_client_get_device_id(client); - struct i2c_adapter *adapter = client->adapter; struct power_supply_config psy_cfg = {}; struct max17040_chip *chip; enum chip_id chip_id; bool enable_irq = false; int ret; - if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE)) - return -EIO; - chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL); if (!chip) return -ENOMEM; @@ -568,7 +580,7 @@ static int max17040_suspend(struct device *dev) // disable soc alert to prevent wakeup max17040_set_soc_alert(chip, 0); else - cancel_delayed_work(&chip->work); + cancel_delayed_work_sync(&chip->work); if (client->irq && device_may_wakeup(dev)) enable_irq_wake(client->irq); diff --git a/drivers/power/supply/pf1550-charger.c b/drivers/power/supply/pf1550-charger.c index 41036f4cb64a0..38cac156d2e68 100644 --- a/drivers/power/supply/pf1550-charger.c +++ b/drivers/power/supply/pf1550-charger.c @@ -514,7 +514,7 @@ static int pf1550_reg_init(struct pf1550_charger *chg) * a battery. The other supported mode is mode 2, the charger is turned * on to charge a battery when present. */ - if (power_supply_get_battery_info(chg->charger, &info)) { + if (!power_supply_get_battery_info(chg->charger, &info)) { ret = regmap_write(chg->pf1550->regmap, PF1550_CHARG_REG_CHG_OPER, PF1550_CHG_BAT_ON); diff --git a/drivers/power/supply/qcom_battmgr.c b/drivers/power/supply/qcom_battmgr.c index 490137a23d00e..91cf39b0083a3 100644 --- a/drivers/power/supply/qcom_battmgr.c +++ b/drivers/power/supply/qcom_battmgr.c @@ -5,6 +5,7 @@ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include +#include #include #include #include @@ -1231,7 +1232,7 @@ static void qcom_battmgr_sc8280xp_strcpy(char *dest, const char *src) memcpy(dest, src + 1, len); dest[len] = '\0'; } else { - memcpy(dest, src, BATTMGR_STRING_LEN); + strscpy(dest, src, BATTMGR_STRING_LEN); } } @@ -1648,7 +1649,6 @@ static int qcom_battmgr_probe(struct auxiliary_device *adev, psy_cfg_supply.supplied_to = qcom_battmgr_battery; psy_cfg_supply.num_supplicants = 1; - INIT_WORK(&battmgr->enable_work, qcom_battmgr_enable_worker); mutex_init(&battmgr->lock); init_completion(&battmgr->ack); @@ -1711,6 +1711,11 @@ static int qcom_battmgr_probe(struct auxiliary_device *adev, "failed to register wireless charing power supply\n"); } + ret = devm_work_autocancel(dev, &battmgr->enable_work, + qcom_battmgr_enable_worker); + if (ret) + return ret; + battmgr->client = devm_pmic_glink_client_alloc(dev, PMIC_GLINK_OWNER_BATTMGR, qcom_battmgr_callback, qcom_battmgr_pdr_notify, diff --git a/drivers/power/supply/rt9455_charger.c b/drivers/power/supply/rt9455_charger.c index 7045d2908148b..515bb3b2d67a4 100644 --- a/drivers/power/supply/rt9455_charger.c +++ b/drivers/power/supply/rt9455_charger.c @@ -1582,6 +1582,19 @@ static const struct regmap_config rt9455_regmap_config = { .cache_type = REGCACHE_MAPLE, }; +static void rt9455_cancel_all_delayed_works(void *data) +{ + struct rt9455_info *info = data; + + /* + * Both pwr_rdy_work and batt_presence_work can queue + * max_charging_time_work, so cancel them first. + */ + cancel_delayed_work_sync(&info->pwr_rdy_work); + cancel_delayed_work_sync(&info->batt_presence_work); + cancel_delayed_work_sync(&info->max_charging_time_work); +} + static int rt9455_probe(struct i2c_client *client) { struct i2c_adapter *adapter = client->adapter; @@ -1672,6 +1685,10 @@ static int rt9455_probe(struct i2c_client *client) goto put_usb_notifier; } + ret = devm_add_action_or_reset(dev, rt9455_cancel_all_delayed_works, info); + if (ret) + goto put_usb_notifier; + ret = devm_request_threaded_irq(dev, client->irq, NULL, rt9455_irq_handler_thread, IRQF_TRIGGER_LOW | IRQF_ONESHOT, @@ -1712,10 +1729,6 @@ static void rt9455_remove(struct i2c_client *client) if (info->nb.notifier_call) usb_unregister_notifier(info->usb_phy, &info->nb); #endif - - cancel_delayed_work_sync(&info->pwr_rdy_work); - cancel_delayed_work_sync(&info->max_charging_time_work); - cancel_delayed_work_sync(&info->batt_presence_work); } static const struct i2c_device_id rt9455_i2c_id_table[] = { diff --git a/drivers/power/supply/sbs-battery.c b/drivers/power/supply/sbs-battery.c index 017ec06be7666..9bdb6c599c5f6 100644 --- a/drivers/power/supply/sbs-battery.c +++ b/drivers/power/supply/sbs-battery.c @@ -217,6 +217,7 @@ struct sbs_info { u32 flags; int technology; char strings[NR_STRING_BUFFERS][I2C_SMBUS_BLOCK_MAX + 1]; + char serial[5]; }; static char *sbs_get_string_buf(struct sbs_info *chip, @@ -821,18 +822,18 @@ static int sbs_get_battery_capacity(struct i2c_client *client, return 0; } -static char sbs_serial[5]; static int sbs_get_battery_serial_number(struct i2c_client *client, union power_supply_propval *val) { + struct sbs_info *chip = i2c_get_clientdata(client); int ret; ret = sbs_read_word_data(client, sbs_data[REG_SERIAL_NUMBER].addr); if (ret < 0) return ret; - sprintf(sbs_serial, "%04x", ret); - val->strval = sbs_serial; + sprintf(chip->serial, "%04x", ret); + val->strval = chip->serial; return 0; } diff --git a/drivers/power/supply/sc2731_charger.c b/drivers/power/supply/sc2731_charger.c index 58b86fd787713..2b25e44da7978 100644 --- a/drivers/power/supply/sc2731_charger.c +++ b/drivers/power/supply/sc2731_charger.c @@ -466,6 +466,7 @@ static int sc2731_charger_probe(struct platform_device *pdev) mutex_init(&info->lock); info->dev = &pdev->dev; INIT_WORK(&info->work, sc2731_charger_work); + platform_set_drvdata(pdev, info); info->regmap = dev_get_regmap(pdev->dev.parent, NULL); if (!info->regmap) { @@ -516,6 +517,7 @@ static void sc2731_charger_remove(struct platform_device *pdev) struct sc2731_charger_info *info = platform_get_drvdata(pdev); usb_unregister_notifier(info->usb_phy, &info->usb_notify); + cancel_work_sync(&info->work); } static const struct of_device_id sc2731_charger_of_match[] = { diff --git a/drivers/power/supply/twl4030_charger.c b/drivers/power/supply/twl4030_charger.c index 04216b2bfb6c3..e5b4d72720de5 100644 --- a/drivers/power/supply/twl4030_charger.c +++ b/drivers/power/supply/twl4030_charger.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -1002,8 +1003,15 @@ static int twl4030_bci_probe(struct platform_device *pdev) platform_set_drvdata(pdev, bci); - INIT_WORK(&bci->work, twl4030_bci_usb_work); - INIT_DELAYED_WORK(&bci->current_worker, twl4030_current_worker); + ret = devm_delayed_work_autocancel(&pdev->dev, &bci->current_worker, + twl4030_current_worker); + if (ret) + return ret; + + ret = devm_work_autocancel(&pdev->dev, &bci->work, + twl4030_bci_usb_work); + if (ret) + return ret; bci->channel_vac = devm_iio_channel_get(&pdev->dev, "vac"); if (IS_ERR(bci->channel_vac)) { diff --git a/drivers/power/supply/ucs1002_power.c b/drivers/power/supply/ucs1002_power.c index 3f44cc902f84d..ca58c214e93e3 100644 --- a/drivers/power/supply/ucs1002_power.c +++ b/drivers/power/supply/ucs1002_power.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -640,7 +641,10 @@ static int ucs1002_probe(struct i2c_client *client) } info->health = POWER_SUPPLY_HEALTH_GOOD; - INIT_DELAYED_WORK(&info->health_poll, ucs1002_health_poll); + ret = devm_delayed_work_autocancel(dev, &info->health_poll, + ucs1002_health_poll); + if (ret) + return ret; if (irq_a_det > 0) { ret = devm_request_threaded_irq(dev, irq_a_det, NULL, diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 1006d183d5081..6b7d11a0abc5d 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -32,6 +32,9 @@ #define ENERGY_STATUS_MASK GENMASK(31, 0) +/* Width of the RAPL energy counters, see the *_ENERGY_STATUS_MASK defines */ +#define RAPL_CNTR_WIDTH 32 + #define POWER_UNIT_OFFSET 0x00 #define POWER_UNIT_MASK GENMASK(3, 0) @@ -1227,6 +1230,7 @@ static u64 rapl_event_update(struct perf_event *event) struct rapl_package_pmu_data *data = event_to_pmu_data(event); u64 prev_raw_count, new_raw_count; s64 delta, sdelta; + int shift = 64 - RAPL_CNTR_WIDTH; /* * Follow the generic code to drain hwc->prev_count. @@ -1243,8 +1247,13 @@ static u64 rapl_event_update(struct perf_event *event) * Now we have the new raw value and have updated the prev * timestamp already. We can now calculate the elapsed delta * (event-)time and add that to the generic event. + * + * Careful, the counter is narrower than u64 and is not + * sign-extended above its physical width. Shift both values up + * so that the subtraction wraps, then shift the result back down. */ - delta = new_raw_count - prev_raw_count; + delta = (new_raw_count << shift) - (prev_raw_count << shift); + delta >>= shift; /* * Scale delta to smallest unit (2^-32) diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 7f41491d9cd11..73f36d9c09b1b 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -414,7 +414,10 @@ static int intel_rapl_tpmi_probe(struct auxiliary_device *auxdev, goto err; } - rapl_package_add_pmu(trp->rp); + ret = rapl_package_add_pmu(trp->rp); + if (ret) + dev_info(&auxdev->dev, "Failed to add RAPL PMU for Package%d, %d\n", + info->package_id, ret); auxiliary_set_drvdata(auxdev, trp); diff --git a/drivers/pps/clients/pps-gpio.c b/drivers/pps/clients/pps-gpio.c index 402f910f3e25d..73ec2c7335e51 100644 --- a/drivers/pps/clients/pps-gpio.c +++ b/drivers/pps/clients/pps-gpio.c @@ -31,38 +31,37 @@ struct pps_gpio_device_data { struct gpio_desc *echo_pin; struct timer_list echo_timer; /* timer to reset echo active state */ bool assert_falling_edge; - bool capture_clear; unsigned int echo_active_ms; /* PPS echo active duration */ unsigned long echo_timeout; /* timer timeout value in jiffies */ + struct pps_event_time ts; /* timestamp captured in hardirq */ }; /* * Report the PPS event */ -static irqreturn_t pps_gpio_irq_handler(int irq, void *data) +/* + * Primary hardirq handler -- runs in hardirq context even on PREEMPT_RT. + * Only captures the timestamp; all other work is deferred to the thread. + */ +static irqreturn_t pps_gpio_irq_hardirq(int irq, void *data) { - const struct pps_gpio_device_data *info; - struct pps_event_time ts; - int rising_edge; - - /* Get the time stamp first */ - pps_get_ts(&ts); - - info = data; - - /* Small trick to bypass the check on edge's direction when capture_clear is unset */ - rising_edge = info->capture_clear ? - gpiod_get_value(info->gpio_pin) : !info->assert_falling_edge; - if ((rising_edge && !info->assert_falling_edge) || - (!rising_edge && info->assert_falling_edge)) - pps_event(info->pps, &ts, PPS_CAPTUREASSERT, data); - else if (info->capture_clear && - ((rising_edge && info->assert_falling_edge) || - (!rising_edge && !info->assert_falling_edge))) - pps_event(info->pps, &ts, PPS_CAPTURECLEAR, data); - else - dev_warn_ratelimited(&info->pps->dev, "IRQ did not trigger any PPS event\n"); + struct pps_gpio_device_data *info = data; + + pps_get_ts(&info->ts); + + return IRQ_WAKE_THREAD; +} + +/* + * Threaded handler -- processes the PPS event using the timestamp + * captured in hardirq context above. + */ +static irqreturn_t pps_gpio_irq_thread(int irq, void *data) +{ + struct pps_gpio_device_data *info = data; + + pps_event(info->pps, &info->ts, PPS_CAPTUREASSERT, data); return IRQ_HANDLED; } @@ -78,11 +77,6 @@ static void pps_gpio_echo(struct pps_device *pps, int event, void *data) if (pps->params.mode & PPS_ECHOASSERT) gpiod_set_value(info->echo_pin, 1); break; - - case PPS_CAPTURECLEAR: - if (pps->params.mode & PPS_ECHOCLEAR) - gpiod_set_value(info->echo_pin, 1); - break; } /* fire the timer */ @@ -144,15 +138,8 @@ static int pps_gpio_setup(struct device *dev) static unsigned long get_irqf_trigger_flags(const struct pps_gpio_device_data *data) { - unsigned long flags = data->assert_falling_edge ? - IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING; - - if (data->capture_clear) { - flags |= ((flags & IRQF_TRIGGER_RISING) ? - IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING); - } - - return flags; + return data->assert_falling_edge ? IRQF_TRIGGER_FALLING : + IRQF_TRIGGER_RISING; } static int pps_gpio_probe(struct platform_device *pdev) @@ -185,9 +172,6 @@ static int pps_gpio_probe(struct platform_device *pdev) /* initialize PPS specific parts of the bookkeeping data structure. */ data->info.mode = PPS_CAPTUREASSERT | PPS_OFFSETASSERT | PPS_ECHOASSERT | PPS_CANWAIT | PPS_TSFMT_TSPEC; - if (data->capture_clear) - data->info.mode |= PPS_CAPTURECLEAR | PPS_OFFSETCLEAR | - PPS_ECHOCLEAR; data->info.owner = THIS_MODULE; snprintf(data->info.name, PPS_MAX_NAME_LEN - 1, "%s.%d", pdev->name, pdev->id); @@ -199,8 +183,6 @@ static int pps_gpio_probe(struct platform_device *pdev) /* register PPS source */ pps_default_params = PPS_CAPTUREASSERT | PPS_OFFSETASSERT; - if (data->capture_clear) - pps_default_params |= PPS_CAPTURECLEAR | PPS_OFFSETCLEAR; data->pps = pps_register_source(&data->info, pps_default_params); if (IS_ERR(data->pps)) { dev_err(dev, "failed to register IRQ %d as PPS source\n", @@ -209,8 +191,10 @@ static int pps_gpio_probe(struct platform_device *pdev) } /* register IRQ interrupt handler */ - ret = request_irq(data->irq, pps_gpio_irq_handler, - get_irqf_trigger_flags(data), data->info.name, data); + ret = request_threaded_irq(data->irq, + pps_gpio_irq_hardirq, pps_gpio_irq_thread, + get_irqf_trigger_flags(data) | IRQF_ONESHOT, + data->info.name, data); if (ret) { pps_unregister_source(data->pps); dev_err(dev, "failed to acquire IRQ %d\n", data->irq); diff --git a/drivers/pps/pps.c b/drivers/pps/pps.c index de1122bb69ea6..7ecdd774a44b9 100644 --- a/drivers/pps/pps.c +++ b/drivers/pps/pps.c @@ -66,13 +66,19 @@ static int pps_cdev_pps_fetch(struct pps_device *pps, struct pps_fdata *fdata) err = wait_event_interruptible(pps->queue, ev != pps->last_ev); else { + struct timespec64 ts; unsigned long ticks; dev_dbg(&pps->dev, "timeout %lld.%09d\n", (long long) fdata->timeout.sec, fdata->timeout.nsec); - ticks = fdata->timeout.sec * HZ; - ticks += fdata->timeout.nsec / (NSEC_PER_SEC / HZ); + + if (fdata->timeout.sec < 0) + return -ETIMEDOUT; + + ts.tv_sec = fdata->timeout.sec; + ts.tv_nsec = fdata->timeout.nsec; + ticks = timespec64_to_jiffies(&ts); if (ticks != 0) { err = wait_event_interruptible_timeout( diff --git a/drivers/ptp/ptp_netc.c b/drivers/ptp/ptp_netc.c index 1c20d7efab929..19ca99d80e95b 100644 --- a/drivers/ptp/ptp_netc.c +++ b/drivers/ptp/ptp_netc.c @@ -440,7 +440,7 @@ static int net_timer_enable_perout(struct netc_timer *priv, } if (on) { - u64 period_ns, gclk_period, max_period, min_period; + u64 period_ns, gclk_period, min_period; struct timespec64 period, stime; u32 integral_period; int alarm_id; @@ -450,12 +450,12 @@ static int net_timer_enable_perout(struct netc_timer *priv, period_ns = timespec64_to_ns(&period); integral_period = netc_timer_get_integral_period(priv); - max_period = (u64)NETC_TMR_DEFAULT_FIPER + integral_period; gclk_period = netc_timer_get_gclk_period(priv); min_period = gclk_period * 4 + integral_period; - if (period_ns > max_period || period_ns < min_period) { - dev_err(dev, "The period range is %llu ~ %llu\n", - min_period, max_period); + if (period_ns > NETC_TMR_DEFAULT_FIPER || + period_ns < min_period) { + dev_err(dev, "The period range is %llu ~ %lu\n", + min_period, NETC_TMR_DEFAULT_FIPER); err = -EINVAL; goto unlock_spinlock; } @@ -482,6 +482,9 @@ static int net_timer_enable_perout(struct netc_timer *priv, netc_timer_enable_periodic_pulse(priv, channel); } else { + if (!pp->enabled) + goto unlock_spinlock; + netc_timer_disable_periodic_pulse(priv, channel); priv->fs_alarm_bitmap &= ~BIT(pp->alarm_id); memset(pp, 0, sizeof(*pp)); diff --git a/drivers/ptp/ptp_vmclock.c b/drivers/ptp/ptp_vmclock.c index eebdcd5ebc080..8ba14bbec15e4 100644 --- a/drivers/ptp/ptp_vmclock.c +++ b/drivers/ptp/ptp_vmclock.c @@ -372,6 +372,18 @@ static int vmclock_miscdev_mmap(struct file *fp, struct vm_area_struct *vma) if ((vma->vm_flags & (VM_READ|VM_WRITE)) != VM_READ) return -EROFS; + /* + * Restrict the read-only mapping so it cannot be upgraded to + * writable later with mprotect(). + */ + vm_flags_clear(vma, VM_MAYWRITE); + + /* + * Restrict the read-only mapping so it cannot be upgraded to + * writable later with mprotect(). + */ + vm_flags_clear(vma, VM_MAYWRITE); + if (vma->vm_end - vma->vm_start != PAGE_SIZE || vma->vm_pgoff) return -EINVAL; diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c index 951b38ff5f8ec..14fb460a45655 100644 --- a/drivers/pwm/pwm-samsung.c +++ b/drivers/pwm/pwm-samsung.c @@ -102,7 +102,7 @@ struct samsung_pwm_chip { * IP. Should this change, both drivers will need to be modified to * properly synchronize accesses to particular instances. */ -static DEFINE_SPINLOCK(samsung_pwm_lock); +static DEFINE_RAW_SPINLOCK(samsung_pwm_lock); #endif static inline @@ -141,14 +141,14 @@ static void pwm_samsung_set_divisor(struct samsung_pwm_chip *our_chip, bits = (fls(divisor) - 1) - our_chip->variant.div_base; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); reg = readl(our_chip->base + REG_TCFG1); reg &= ~(TCFG1_MUX_MASK << shift); reg |= bits << shift; writel(reg, our_chip->base + REG_TCFG1); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static int pwm_samsung_is_tdiv(struct samsung_pwm_chip *our_chip, unsigned int chan) @@ -249,7 +249,7 @@ static int pwm_samsung_enable(struct pwm_chip *chip, struct pwm_device *pwm) unsigned long flags; u32 tcon; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl(our_chip->base + REG_TCON); @@ -263,7 +263,7 @@ static int pwm_samsung_enable(struct pwm_chip *chip, struct pwm_device *pwm) our_chip->disabled_mask &= ~BIT(pwm->hwpwm); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); return 0; } @@ -275,7 +275,7 @@ static void pwm_samsung_disable(struct pwm_chip *chip, struct pwm_device *pwm) unsigned long flags; u32 tcon; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl(our_chip->base + REG_TCON); tcon &= ~TCON_AUTORELOAD(tcon_chan); @@ -290,7 +290,7 @@ static void pwm_samsung_disable(struct pwm_chip *chip, struct pwm_device *pwm) our_chip->disabled_mask |= BIT(pwm->hwpwm); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static void pwm_samsung_manual_update(struct samsung_pwm_chip *our_chip, @@ -298,11 +298,11 @@ static void pwm_samsung_manual_update(struct samsung_pwm_chip *our_chip, { unsigned long flags; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); __pwm_samsung_manual_update(our_chip, pwm); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, @@ -390,7 +390,7 @@ static void pwm_samsung_set_invert(struct samsung_pwm_chip *our_chip, unsigned long flags; u32 tcon; - spin_lock_irqsave(&samsung_pwm_lock, flags); + raw_spin_lock_irqsave(&samsung_pwm_lock, flags); tcon = readl(our_chip->base + REG_TCON); @@ -404,7 +404,7 @@ static void pwm_samsung_set_invert(struct samsung_pwm_chip *our_chip, writel(tcon, our_chip->base + REG_TCON); - spin_unlock_irqrestore(&samsung_pwm_lock, flags); + raw_spin_unlock_irqrestore(&samsung_pwm_lock, flags); } static int pwm_samsung_set_polarity(struct pwm_chip *chip, diff --git a/drivers/rapidio/devices/rio_mport_cdev.c b/drivers/rapidio/devices/rio_mport_cdev.c index 009b3b595bbf5..ad82c2108a567 100644 --- a/drivers/rapidio/devices/rio_mport_cdev.c +++ b/drivers/rapidio/devices/rio_mport_cdev.c @@ -564,9 +564,13 @@ static void dma_req_free(struct kref *ref) } if (req->map) { - mutex_lock(&req->map->md->buf_mutex); - kref_put(&req->map->ref, mport_release_mapping); - mutex_unlock(&req->map->md->buf_mutex); + struct rio_mport_mapping *map = req->map; + struct mport_dev *md = map->md; + + mutex_lock(&md->buf_mutex); + req->map = NULL; + kref_put(&map->ref, mport_release_mapping); + mutex_unlock(&md->buf_mutex); } kref_put(&priv->dma_ref, mport_release_dma); diff --git a/drivers/rapidio/rio-scan.c b/drivers/rapidio/rio-scan.c index dcd6619a4b027..3cc25d0534513 100644 --- a/drivers/rapidio/rio-scan.c +++ b/drivers/rapidio/rio-scan.c @@ -874,6 +874,7 @@ static struct rio_net *rio_scan_alloc_net(struct rio_mport *mport, net->dev.release = rio_scan_release_dev; if (rio_add_net(net)) { put_device(&net->dev); + mport->net = NULL; net = NULL; } } diff --git a/drivers/ras/amd/atl/core.c b/drivers/ras/amd/atl/core.c index 0f7cd6dab0b0e..d77dacdd4f569 100644 --- a/drivers/ras/amd/atl/core.c +++ b/drivers/ras/amd/atl/core.c @@ -190,7 +190,6 @@ static const struct x86_cpu_id amd_atl_cpuids[] = { X86_MATCH_FEATURE(X86_FEATURE_ZEN, NULL), { } }; -MODULE_DEVICE_TABLE(x86cpu, amd_atl_cpuids); static int __init amd_atl_init(void) { diff --git a/drivers/regulator/adp5055-regulator.c b/drivers/regulator/adp5055-regulator.c index 9ebd52b392357..7eaa7d9dc08cc 100644 --- a/drivers/regulator/adp5055-regulator.c +++ b/drivers/regulator/adp5055-regulator.c @@ -224,7 +224,7 @@ static int adp5055_of_parse_cb(struct device_node *np, adp5055->dvs_limit_upper[id] = pval; if (adp5055->dvs_limit_upper[id] > 192000 || adp5055->dvs_limit_upper[id] < 12000) - return dev_err_probe(config->dev, adp5055->dvs_limit_upper[id], + return dev_err_probe(config->dev, -EINVAL, "Out of range - dvs-limit-upper-microvolt value."); ret = of_property_read_u32(np, "adi,dvs-limit-lower-microvolt", &pval); diff --git a/drivers/regulator/as3722-regulator.c b/drivers/regulator/as3722-regulator.c index da378bfdba406..85e2aaa1a06bd 100644 --- a/drivers/regulator/as3722-regulator.c +++ b/drivers/regulator/as3722-regulator.c @@ -600,7 +600,6 @@ static int as3722_get_regulator_dt_data(struct platform_device *pdev, ret = of_regulator_match(&pdev->dev, np, as3722_regulator_matches, ARRAY_SIZE(as3722_regulator_matches)); - of_node_put(np); if (ret < 0) { dev_err(&pdev->dev, "Parsing of regulator node failed: %d\n", ret); diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index 2e61606fc1d05..6a4008f387b5e 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -27,6 +27,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -6899,8 +6900,9 @@ static int __init regulator_init_complete(void) * we'd only do this on systems that need it, and a kernel * command line option might be useful. */ - schedule_delayed_work(®ulator_init_complete_work, - msecs_to_jiffies(30000)); + queue_delayed_work(system_freezable_wq, + ®ulator_init_complete_work, + msecs_to_jiffies(30000)); return 0; } diff --git a/drivers/regulator/db8500-prcmu.c b/drivers/regulator/db8500-prcmu.c index 1ec2e1348891d..751fe36580faf 100644 --- a/drivers/regulator/db8500-prcmu.c +++ b/drivers/regulator/db8500-prcmu.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -93,13 +93,13 @@ static int enable_epod(u16 epod_id, bool ramret) if (ramret) { if (!epod_on[epod_id]) { - ret = prcmu_set_epod(epod_id, EPOD_STATE_RAMRET); + ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_RAMRET); if (ret < 0) return ret; } epod_ramret[epod_id] = true; } else { - ret = prcmu_set_epod(epod_id, EPOD_STATE_ON); + ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_ON); if (ret < 0) return ret; epod_on[epod_id] = true; @@ -114,18 +114,18 @@ static int disable_epod(u16 epod_id, bool ramret) if (ramret) { if (!epod_on[epod_id]) { - ret = prcmu_set_epod(epod_id, EPOD_STATE_OFF); + ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_OFF); if (ret < 0) return ret; } epod_ramret[epod_id] = false; } else { if (epod_ramret[epod_id]) { - ret = prcmu_set_epod(epod_id, EPOD_STATE_RAMRET); + ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_RAMRET); if (ret < 0) return ret; } else { - ret = prcmu_set_epod(epod_id, EPOD_STATE_OFF); + ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_OFF); if (ret < 0) return ret; } diff --git a/drivers/regulator/max8998.c b/drivers/regulator/max8998.c index cc85fbe8b77c3..90651f4d18281 100644 --- a/drivers/regulator/max8998.c +++ b/drivers/regulator/max8998.c @@ -582,7 +582,6 @@ static int max8998_pmic_dt_parse_pdata(struct max8998_dev *iodev, } pdata->num_regulators = rdata - pdata->regulators; - of_node_put(reg_np); of_node_put(regulators_np); pdata->buck_voltage_lock = of_property_read_bool(pmic_np, "max8998,pmic-buck-voltage-lock"); diff --git a/drivers/regulator/pf1550-regulator.c b/drivers/regulator/pf1550-regulator.c index 610eac9bb9cb2..ceee553a84b23 100644 --- a/drivers/regulator/pf1550-regulator.c +++ b/drivers/regulator/pf1550-regulator.c @@ -283,63 +283,61 @@ static struct pf1550_desc pf1550_regulators[] = { PF_LDO1(PF1550, "ldo3", LDO3, 0x1f, pf1550_ldo13_volts), }; +/* + * The _LS interrupts indicate an over-current event. The _HS + * interrupts, which are more accurate and can detect catastrophic + * faults, issue an error event. The current limit FAULT interrupt is + * similar to the _HS. + */ +static const struct pf1550_regulator_irq { + unsigned int event; + u8 id; +} pf1550_regulator_irqs[] = { + [PF1550_PMIC_IRQ_SW1_LS] = { REGULATOR_EVENT_OVER_CURRENT_WARN, PF1550_SW1 }, + [PF1550_PMIC_IRQ_SW2_LS] = { REGULATOR_EVENT_OVER_CURRENT_WARN, PF1550_SW2 }, + [PF1550_PMIC_IRQ_SW3_LS] = { REGULATOR_EVENT_OVER_CURRENT_WARN, PF1550_SW3 }, + [PF1550_PMIC_IRQ_SW1_HS] = { REGULATOR_EVENT_OVER_CURRENT, PF1550_SW1 }, + [PF1550_PMIC_IRQ_SW2_HS] = { REGULATOR_EVENT_OVER_CURRENT, PF1550_SW2 }, + [PF1550_PMIC_IRQ_SW3_HS] = { REGULATOR_EVENT_OVER_CURRENT, PF1550_SW3 }, + [PF1550_PMIC_IRQ_LDO1_FAULT] = { REGULATOR_EVENT_OVER_CURRENT, PF1550_LDO1 }, + [PF1550_PMIC_IRQ_LDO2_FAULT] = { REGULATOR_EVENT_OVER_CURRENT, PF1550_LDO2 }, + [PF1550_PMIC_IRQ_LDO3_FAULT] = { REGULATOR_EVENT_OVER_CURRENT, PF1550_LDO3 }, +}; + static irqreturn_t pf1550_regulator_irq_handler(int irq, void *data) { + const struct pf1550_regulator_irq *map; struct pf1550_regulator_info *info = data; struct device *dev = info->dev; struct platform_device *pdev = to_platform_device(dev); int i, irq_type = -1; - unsigned int event; for (i = 0; i < PF1550_REGULATOR_IRQ_NR; i++) if (irq == platform_get_irq(pdev, i)) irq_type = i; - switch (irq_type) { - /* The _LS interrupts indicate over-current event. The _HS interrupts - * which are more accurate and can detect catastrophic faults, issue - * an error event. The current limit FAULT interrupt is similar to the - * _HS' - */ - case PF1550_PMIC_IRQ_SW1_LS: - case PF1550_PMIC_IRQ_SW2_LS: - case PF1550_PMIC_IRQ_SW3_LS: - event = REGULATOR_EVENT_OVER_CURRENT_WARN; - for (i = 0; i < PF1550_MAX_REGULATOR; i++) - if (!strcmp(rdev_get_name(info->rdevs[i]), "SW3")) - regulator_notifier_call_chain(info->rdevs[i], - event, NULL); - break; - case PF1550_PMIC_IRQ_SW1_HS: - case PF1550_PMIC_IRQ_SW2_HS: - case PF1550_PMIC_IRQ_SW3_HS: - event = REGULATOR_EVENT_OVER_CURRENT; - for (i = 0; i < PF1550_MAX_REGULATOR; i++) - if (!strcmp(rdev_get_name(info->rdevs[i]), "SW3")) - regulator_notifier_call_chain(info->rdevs[i], - event, NULL); - break; - case PF1550_PMIC_IRQ_LDO1_FAULT: - case PF1550_PMIC_IRQ_LDO2_FAULT: - case PF1550_PMIC_IRQ_LDO3_FAULT: - event = REGULATOR_EVENT_OVER_CURRENT; - for (i = 0; i < PF1550_MAX_REGULATOR; i++) - if (!strcmp(rdev_get_name(info->rdevs[i]), "LDO3")) - regulator_notifier_call_chain(info->rdevs[i], - event, NULL); - break; - case PF1550_PMIC_IRQ_TEMP_110: - case PF1550_PMIC_IRQ_TEMP_125: - event = REGULATOR_EVENT_OVER_TEMP; + /* The die temperature concerns every rail. */ + if (irq_type == PF1550_PMIC_IRQ_TEMP_110 || + irq_type == PF1550_PMIC_IRQ_TEMP_125) { for (i = 0; i < PF1550_MAX_REGULATOR; i++) regulator_notifier_call_chain(info->rdevs[i], - event, NULL); - break; - default: - dev_err(dev, "regulator interrupt: irq %d occurred\n", - irq_type); + REGULATOR_EVENT_OVER_TEMP, + NULL); + return IRQ_HANDLED; + } + + if (irq_type < 0 || irq_type >= (int)ARRAY_SIZE(pf1550_regulator_irqs)) { + dev_err(dev, "regulator interrupt: irq %d occurred\n", irq_type); + return IRQ_HANDLED; } + map = &pf1550_regulator_irqs[irq_type]; + + for (i = 0; i < PF1550_MAX_REGULATOR; i++) + if (rdev_get_id(info->rdevs[i]) == map->id) + regulator_notifier_call_chain(info->rdevs[i], + map->event, NULL); + return IRQ_HANDLED; } diff --git a/drivers/regulator/qcom-refgen-regulator.c b/drivers/regulator/qcom-refgen-regulator.c index 299ac3c8c3bc3..6a3795469927e 100644 --- a/drivers/regulator/qcom-refgen-regulator.c +++ b/drivers/regulator/qcom-refgen-regulator.c @@ -66,7 +66,7 @@ static const struct regulator_desc sdm845_refgen_desc = { .enable_time = 5, .name = "refgen", .owner = THIS_MODULE, - .type = REGULATOR_VOLTAGE, + .type = REGULATOR_CURRENT, .ops = &(const struct regulator_ops) { .enable = qcom_sdm845_refgen_enable, .disable = qcom_sdm845_refgen_disable, @@ -82,7 +82,7 @@ static const struct regulator_desc sm8250_refgen_desc = { .enable_time = 5, .name = "refgen", .owner = THIS_MODULE, - .type = REGULATOR_VOLTAGE, + .type = REGULATOR_CURRENT, .ops = &(const struct regulator_ops) { .enable = regulator_enable_regmap, .disable = regulator_disable_regmap, diff --git a/drivers/regulator/qcom-rpmh-regulator.c b/drivers/regulator/qcom-rpmh-regulator.c index 73856e3df337a..ef5523ff427b8 100644 --- a/drivers/regulator/qcom-rpmh-regulator.c +++ b/drivers/regulator/qcom-rpmh-regulator.c @@ -111,6 +111,7 @@ static const struct resource_name_formats vreg_rsc_name_lookup[NUM_REGULATOR_TYP * @hpm_min_load_uA: Minimum load current in microamps that requires * high power mode (HPM) operation. This is used * for LDO hardware type regulators only. + * @pmic_bypass_mode: The PMIC bypass mode value. * @pmic_mode_map: Array indexed by regulator framework mode * containing PMIC hardware modes. Must be large * enough to index all framework modes supported @@ -125,6 +126,7 @@ struct rpmh_vreg_hw_data { int n_linear_ranges; int n_voltages; int hpm_min_load_uA; + int pmic_bypass_mode; const int *pmic_mode_map; unsigned int (*of_map_mode)(unsigned int mode); }; @@ -311,7 +313,7 @@ static int rpmh_regulator_vrm_set_mode_bypass(struct rpmh_vreg *vreg, return pmic_mode; if (bypassed) - cmd.data = PMIC4_BOB_MODE_PASS; + cmd.data = vreg->hw_data->pmic_bypass_mode; else cmd.data = pmic_mode; @@ -767,6 +769,7 @@ static const struct rpmh_vreg_hw_data pmic4_bob = { }, .n_linear_ranges = 1, .n_voltages = 84, + .pmic_bypass_mode = PMIC4_BOB_MODE_PASS, .pmic_mode_map = pmic_mode_map_pmic4_bob, .of_map_mode = rpmh_regulator_pmic4_bob_of_map_mode, }; @@ -975,6 +978,7 @@ static const struct rpmh_vreg_hw_data pmic5_bob = { }, .n_linear_ranges = 1, .n_voltages = 32, + .pmic_bypass_mode = PMIC5_BOB_MODE_PASS, .pmic_mode_map = pmic_mode_map_pmic5_bob, .of_map_mode = rpmh_regulator_pmic4_bob_of_map_mode, }; diff --git a/drivers/regulator/tps65185.c b/drivers/regulator/tps65185.c index 786622d8d5980..6a3130177e7a3 100644 --- a/drivers/regulator/tps65185.c +++ b/drivers/regulator/tps65185.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -183,7 +184,10 @@ static int tps65185_vposneg_enable(struct regulator_dev *rdev) wait_for_completion_timeout(&data->pgood_completion, msecs_to_jiffies(PGOOD_TIMEOUT_MSECS)); dev_dbg(data->dev, "turned on"); - if (gpiod_get_value_cansleep(data->pgood_gpio) != 1) + ret = gpiod_get_value_cansleep(data->pgood_gpio); + if (ret < 0) + return ret; + if (!ret) return -ETIMEDOUT; return 0; @@ -372,6 +376,11 @@ static int tps65185_probe(struct i2c_client *client) return dev_err_probe(&client->dev, ret, "failed to get vin regulator\n"); + // TPS65185x PMIC for E Ink Vizplex Enabled Electronic Paper Display Chapter 7.6 Figure 2: + // "Minimum delay time between WAKEUP rising edge and IC ready to accept I2C transaction." + // https://www.ti.com/lit/ds/symlink/tps65185.pdf + usleep_range(1800, 3000); + data->dev = &client->dev; i2c_set_clientdata(client, data); diff --git a/drivers/regulator/tps6594-regulator.c b/drivers/regulator/tps6594-regulator.c index 645e83462c645..31a5218d55105 100644 --- a/drivers/regulator/tps6594-regulator.c +++ b/drivers/regulator/tps6594-regulator.c @@ -669,13 +669,20 @@ static int tps6594_regulator_probe(struct platform_device *pdev) * buck_configured to avoid creating bucks for every buck in multiphase */ for (multi = 0; multi < desc->num_multi_phase_regs; multi++) { + struct device_node *np_parent; + multi_regs = &desc->multi_phase_regs[multi]; np = of_find_node_by_name(tps->dev->of_node, multi_regs->supply_name); - npname = of_node_full_name(np); - np_pmic_parent = of_get_parent(of_get_parent(np)); - if (of_node_cmp(of_node_full_name(np_pmic_parent), tps->dev->of_node->full_name)) + if (!np) continue; - if (strcmp(npname, multi_regs->supply_name) == 0) { + + npname = of_node_full_name(np); + np_parent = of_get_parent(np); + np_pmic_parent = of_get_parent(np_parent); + + if (np_pmic_parent && + !of_node_cmp(of_node_full_name(np_pmic_parent), tps->dev->of_node->full_name) && + strcmp(npname, multi_regs->supply_name) == 0) { switch (multi) { case MULTI_BUCK12: buck_multi[0] = true; @@ -706,6 +713,10 @@ static int tps6594_regulator_probe(struct platform_device *pdev) break; } } + + of_node_put(np_pmic_parent); + of_node_put(np_parent); + of_node_put(np); } reg_irq_nb = desc->num_irq_types * (desc->num_buck_regs + desc->num_ldo_regs); diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index 85a74c9ec5216..436656bdfa8be 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -36,6 +36,7 @@ struct mtk_scp *scp_get(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *scp_node; struct platform_device *scp_pdev; + struct mtk_scp *scp; scp_node = of_parse_phandle(dev->of_node, "mediatek,scp", 0); if (!scp_node) { @@ -51,7 +52,13 @@ struct mtk_scp *scp_get(struct platform_device *pdev) return NULL; } - return platform_get_drvdata(scp_pdev); + scp = platform_get_drvdata(scp_pdev); + if (!scp) { + put_device(&scp_pdev->dev); + return NULL; + } + + return scp; } EXPORT_SYMBOL_GPL(scp_get); diff --git a/drivers/remoteproc/qcom_common.c b/drivers/remoteproc/qcom_common.c index e1a955476c9b3..5294e327f1587 100644 --- a/drivers/remoteproc/qcom_common.c +++ b/drivers/remoteproc/qcom_common.c @@ -253,8 +253,10 @@ void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink, return; glink->ssr_name = kstrdup_const(ssr_name, GFP_KERNEL); - if (!glink->ssr_name) + if (!glink->ssr_name) { + of_node_put(glink->node); return; + } glink->dev = dev; glink->subdev.start = glink_subdev_start; diff --git a/drivers/remoteproc/qcom_common.h b/drivers/remoteproc/qcom_common.h index b07fbaa091a06..b0e7e336d363e 100644 --- a/drivers/remoteproc/qcom_common.h +++ b/drivers/remoteproc/qcom_common.h @@ -68,6 +68,7 @@ struct qcom_sysmon *qcom_add_sysmon_subdev(struct rproc *rproc, int ssctl_instance); void qcom_remove_sysmon_subdev(struct qcom_sysmon *sysmon); bool qcom_sysmon_shutdown_acked(struct qcom_sysmon *sysmon); +bool qcom_sysmon_shutdown_irq_state(struct qcom_sysmon *sysmon); #else static inline struct qcom_sysmon *qcom_add_sysmon_subdev(struct rproc *rproc, const char *name, @@ -84,6 +85,11 @@ static inline bool qcom_sysmon_shutdown_acked(struct qcom_sysmon *sysmon) { return false; } + +static inline bool qcom_sysmon_shutdown_irq_state(struct qcom_sysmon *sysmon) +{ + return false; +} #endif #endif diff --git a/drivers/remoteproc/qcom_q6v5.c b/drivers/remoteproc/qcom_q6v5.c index 58d5b85e58cda..241478ea29782 100644 --- a/drivers/remoteproc/qcom_q6v5.c +++ b/drivers/remoteproc/qcom_q6v5.c @@ -197,12 +197,13 @@ static irqreturn_t q6v5_stop_interrupt(int irq, void *data) */ int qcom_q6v5_request_stop(struct qcom_q6v5 *q6v5, struct qcom_sysmon *sysmon) { + bool was_running = q6v5->running; int ret; q6v5->running = false; - /* Don't perform SMP2P dance if remote isn't running */ - if (q6v5->rproc->state != RPROC_RUNNING || qcom_sysmon_shutdown_acked(sysmon)) + /* A watchdog/fatal IRQ clears running; logical crashes still need a stop. */ + if (!was_running || qcom_sysmon_shutdown_acked(sysmon)) return 0; qcom_smem_state_update_bits(q6v5->state, diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index b5c8d6d38c9cb..9c5dedf0aa605 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -104,6 +104,7 @@ struct qcom_adsp { struct completion stop_done; phys_addr_t mem_phys; + unsigned long iova; phys_addr_t mem_reloc; void *mem_region; size_t mem_size; @@ -333,7 +334,7 @@ static void adsp_unmap_carveout(struct rproc *rproc) struct qcom_adsp *adsp = rproc->priv; if (adsp->has_iommu) - iommu_unmap(rproc->domain, adsp->mem_phys, adsp->mem_size); + iommu_unmap(rproc->domain, adsp->iova, adsp->mem_size); } static int adsp_map_carveout(struct rproc *rproc) @@ -341,7 +342,6 @@ static int adsp_map_carveout(struct rproc *rproc) struct qcom_adsp *adsp = rproc->priv; struct of_phandle_args args; long long sid; - unsigned long iova; int ret; if (!adsp->has_iommu) @@ -355,11 +355,12 @@ static int adsp_map_carveout(struct rproc *rproc) return ret; sid = args.args[0] & SID_MASK_DEFAULT; + of_node_put(args.np); /* Add SID configuration for ADSP Firmware to SMMU */ - iova = adsp->mem_phys | (sid << 32); + adsp->iova = adsp->mem_phys | (sid << 32); - ret = iommu_map(rproc->domain, iova, adsp->mem_phys, + ret = iommu_map(rproc->domain, adsp->iova, adsp->mem_phys, adsp->mem_size, IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); if (ret) { diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index ae78f5c7c1b69..eb14308e7aef0 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -162,6 +162,7 @@ struct rproc_hexagon_res { char **active_clk_names; char **proxy_pd_names; int version; + int ssctl_id; bool need_mem_protection; bool need_pas_mem_setup; bool has_alt_reset; @@ -2191,7 +2192,7 @@ static int q6v5_probe(struct platform_device *pdev) qcom_add_smd_subdev(rproc, &qproc->smd_subdev); qcom_add_pdm_subdev(rproc, &qproc->pdm_subdev); qcom_add_ssr_subdev(rproc, &qproc->ssr_subdev, "mpss"); - qproc->sysmon = qcom_add_sysmon_subdev(rproc, "modem", 0x12); + qproc->sysmon = qcom_add_sysmon_subdev(rproc, "modem", desc->ssctl_id); if (IS_ERR(qproc->sysmon)) { ret = PTR_ERR(qproc->sysmon); goto remove_subdevs; @@ -2271,6 +2272,7 @@ static const struct rproc_hexagon_res sc7180_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_SC7180, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res sc7280_mss = { @@ -2301,6 +2303,7 @@ static const struct rproc_hexagon_res sc7280_mss = { .has_ext_cntl_regs = true, .has_vq6 = true, .version = MSS_SC7280, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res sdm660_mss = { @@ -2334,6 +2337,7 @@ static const struct rproc_hexagon_res sdm660_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_SDM660, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res sdm845_mss = { @@ -2371,6 +2375,7 @@ static const struct rproc_hexagon_res sdm845_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_SDM845, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8998_mss = { @@ -2404,6 +2409,7 @@ static const struct rproc_hexagon_res msm8998_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8998, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8996_mss = { @@ -2444,6 +2450,7 @@ static const struct rproc_hexagon_res msm8996_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8996, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res mdm9607_mss = { @@ -2479,6 +2486,7 @@ static const struct rproc_hexagon_res mdm9607_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MDM9607, + .ssctl_id = 0x22, }; static const struct rproc_hexagon_res msm8909_mss = { @@ -2515,6 +2523,7 @@ static const struct rproc_hexagon_res msm8909_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8909, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8916_mss = { @@ -2562,6 +2571,7 @@ static const struct rproc_hexagon_res msm8916_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8916, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8917_mss = { @@ -2606,6 +2616,7 @@ static const struct rproc_hexagon_res msm8917_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8917, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8937_mss = { @@ -2650,6 +2661,7 @@ static const struct rproc_hexagon_res msm8937_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8937, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8940_mss = { @@ -2694,6 +2706,7 @@ static const struct rproc_hexagon_res msm8940_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8940, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8953_mss = { @@ -2731,6 +2744,7 @@ static const struct rproc_hexagon_res msm8953_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8953, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8974_mss = { @@ -2785,6 +2799,7 @@ static const struct rproc_hexagon_res msm8974_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8974, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8226_mss = { @@ -2824,6 +2839,7 @@ static const struct rproc_hexagon_res msm8226_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8226, + .ssctl_id = 0x12, }; static const struct rproc_hexagon_res msm8926_mss = { @@ -2871,6 +2887,7 @@ static const struct rproc_hexagon_res msm8926_mss = { .has_ext_cntl_regs = false, .has_vq6 = false, .version = MSS_MSM8926, + .ssctl_id = 0x12, }; static const struct of_device_id q6v5_of_match[] = { diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index 808e9609988d3..cb3e51d6f7cec 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -60,6 +60,7 @@ struct qcom_pas_data { int region_assign_count; bool region_assign_shared; int region_assign_vmid; + bool early_boot; }; struct qcom_pas { @@ -240,17 +241,14 @@ static int qcom_pas_load(struct rproc *rproc, const struct firmware *fw) ret = qcom_mdt_pas_load(pas->dtb_pas_ctx, pas->dtb_firmware, pas->dtb_firmware_name, pas->dtb_mem_region, &pas->dtb_mem_reloc); - if (ret) - goto release_dtb_metadata; + if (ret) { + qcom_scm_pas_metadata_release(pas->dtb_pas_ctx); + release_firmware(pas->dtb_firmware); + return ret; + } } return 0; - -release_dtb_metadata: - qcom_scm_pas_metadata_release(pas->dtb_pas_ctx); - release_firmware(pas->dtb_firmware); - - return ret; } static void qcom_pas_unmap_carveout(struct rproc *rproc, phys_addr_t mem_phys, size_t size) @@ -507,6 +505,67 @@ static unsigned long qcom_pas_panic(struct rproc *rproc) return qcom_q6v5_panic(&pas->q6v5); } +static int qcom_pas_attach(struct rproc *rproc) +{ + struct qcom_pas *pas = rproc->priv; + bool ready_state; + bool crash_state; + bool stop_state; + int ret; + + pas->q6v5.handover_issued = true; + enable_irq(pas->q6v5.handover_irq); + + pas->q6v5.running = true; + ret = irq_get_irqchip_state(pas->q6v5.fatal_irq, + IRQCHIP_STATE_LINE_LEVEL, &crash_state); + if (ret) + goto disable_running; + + if (crash_state) { + dev_err(pas->dev, "Subsystem has crashed before driver probe\n"); + rproc_report_crash(rproc, RPROC_FATAL_ERROR); + ret = -EINVAL; + goto disable_running; + } + + ret = irq_get_irqchip_state(pas->q6v5.stop_irq, + IRQCHIP_STATE_LINE_LEVEL, &stop_state); + if (ret) + goto disable_running; + + if (stop_state || qcom_sysmon_shutdown_irq_state(pas->sysmon)) { + dev_info(pas->dev, "Subsystem found stop state set. Falling back to start.\n"); + goto unroll_attach; + } + + ret = irq_get_irqchip_state(pas->q6v5.ready_irq, + IRQCHIP_STATE_LINE_LEVEL, &ready_state); + if (ret) + goto disable_running; + + if (unlikely(!ready_state)) { + /* + * The bootloader may not support early boot, mark the state as + * RPROC_OFFLINE so that the PAS driver can load the firmware and + * start the remoteproc. + */ + dev_err(pas->dev, "Failed to get subsystem ready interrupt\n"); + goto unroll_attach; + } + + return 0; + +unroll_attach: + pas->rproc->state = RPROC_OFFLINE; + ret = -EINVAL; +disable_running: + disable_irq(pas->q6v5.handover_irq); + pas->q6v5.running = false; + + return ret; +} + static const struct rproc_ops qcom_pas_ops = { .unprepare = qcom_pas_unprepare, .start = qcom_pas_start, @@ -515,6 +574,7 @@ static const struct rproc_ops qcom_pas_ops = { .parse_fw = qcom_pas_parse_firmware, .load = qcom_pas_load, .panic = qcom_pas_panic, + .attach = qcom_pas_attach, }; static const struct rproc_ops qcom_pas_minidump_ops = { @@ -526,6 +586,7 @@ static const struct rproc_ops qcom_pas_minidump_ops = { .load = qcom_pas_load, .panic = qcom_pas_panic, .coredump = qcom_pas_minidump, + .attach = qcom_pas_attach, }; static int qcom_pas_init_clock(struct qcom_pas *pas) @@ -852,6 +913,10 @@ static int qcom_pas_probe(struct platform_device *pdev) pas->pas_ctx->use_tzmem = rproc->has_iommu; pas->dtb_pas_ctx->use_tzmem = rproc->has_iommu; + + if (desc->early_boot) + pas->rproc->state = RPROC_DETACHED; + ret = rproc_add(rproc); if (ret) goto remove_ssr_sysmon; diff --git a/drivers/remoteproc/qcom_sysmon.c b/drivers/remoteproc/qcom_sysmon.c index 913e3b750a869..a0830a48b1f40 100644 --- a/drivers/remoteproc/qcom_sysmon.c +++ b/drivers/remoteproc/qcom_sysmon.c @@ -736,6 +736,25 @@ bool qcom_sysmon_shutdown_acked(struct qcom_sysmon *sysmon) } EXPORT_SYMBOL_GPL(qcom_sysmon_shutdown_acked); +bool qcom_sysmon_shutdown_irq_state(struct qcom_sysmon *sysmon) +{ + bool shutdown_state; + int ret; + + if (!sysmon) + return false; + + ret = irq_get_irqchip_state(sysmon->shutdown_irq, + IRQCHIP_STATE_LINE_LEVEL, &shutdown_state); + if (ret) { + dev_warn(sysmon->dev, "failed to get shutdown_state: %d\n", ret); + return false; + } + + return shutdown_state; +} +EXPORT_SYMBOL_GPL(qcom_sysmon_shutdown_irq_state); + /** * sysmon_probe() - probe sys_mon channel * @rpdev: rpmsg device handle diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index 4add9037dbd5a..5e4a623b89251 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -441,25 +441,31 @@ static void wcnss_release_pds(struct qcom_wcnss *wcnss) } static int wcnss_init_regulators(struct qcom_wcnss *wcnss, - const struct wcnss_vreg_info *info, - int num_vregs, int num_pd_vregs) + const struct wcnss_data *data) { + const struct wcnss_vreg_info *info = data->vregs; struct regulator_bulk_data *bulk; + size_t i, possible_pds = 0, num_vregs = data->num_vregs; int ret; - int i; + + for (i = 0; i < WCNSS_MAX_PDS; i++) + if (data->pd_names[i]) + possible_pds++; /* * If attaching the power domains suceeded we can skip requesting * the regulators for the power domains. For old device trees we need to * reserve extra space to manage them through the regulator interface. */ - if (wcnss->num_pds) { + if (possible_pds >= num_vregs) { + /* Do nothing if vregs do not include PD regulators (pronto-v3) */ + } else if (wcnss->num_pds) { info += wcnss->num_pds; /* Handle single power domain case */ - if (wcnss->num_pds < num_pd_vregs) - num_vregs += num_pd_vregs - wcnss->num_pds; + if (wcnss->num_pds < data->num_pd_vregs) + num_vregs += data->num_pd_vregs - wcnss->num_pds; } else { - num_vregs += num_pd_vregs; + num_vregs += data->num_pd_vregs; } bulk = devm_kcalloc(wcnss->dev, @@ -607,8 +613,7 @@ static int wcnss_probe(struct platform_device *pdev) if (ret && (ret != -ENODATA || !data->num_pd_vregs)) return ret; - ret = wcnss_init_regulators(wcnss, data->vregs, data->num_vregs, - data->num_pd_vregs); + ret = wcnss_init_regulators(wcnss, data); if (ret) goto detach_pds; diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index f003be006b1bf..527996e8ecdaa 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -1668,18 +1668,21 @@ static void rproc_auto_boot_callback(const struct firmware *fw, void *context) release_firmware(fw); } +static void rproc_attach_work(struct work_struct *work) +{ + struct rproc *rproc = container_of(work, struct rproc, attach_work); + + rproc_boot(rproc); +} + static int rproc_trigger_auto_boot(struct rproc *rproc) { int ret; - /* - * Since the remote processor is in a detached state, it has already - * been booted by another entity. As such there is no point in waiting - * for a firmware image to be loaded, we can simply initiate the process - * of attaching to it immediately. - */ - if (rproc->state == RPROC_DETACHED) - return rproc_boot(rproc); + if (rproc->state == RPROC_DETACHED) { + schedule_work(&rproc->attach_work); + return 0; + } /* * We're initiating an asynchronous firmware loading, so we can @@ -1828,6 +1831,11 @@ int rproc_trigger_recovery(struct rproc *rproc) if (ret) return ret; + if (READ_ONCE(rproc->deleting)) { + ret = -ENODEV; + goto unlock_mutex; + } + /* State could have changed before we got the mutex */ if (rproc->state != RPROC_CRASHED) goto unlock_mutex; @@ -1860,6 +1868,11 @@ static void rproc_crash_handler_work(struct work_struct *work) mutex_lock(&rproc->lock); + if (READ_ONCE(rproc->deleting)) { + mutex_unlock(&rproc->lock); + goto out; + } + if (rproc->state == RPROC_CRASHED) { /* handle only the first crash detected */ mutex_unlock(&rproc->lock); @@ -1915,9 +1928,9 @@ int rproc_boot(struct rproc *rproc) return ret; } - if (rproc->state == RPROC_DELETED) { + if (READ_ONCE(rproc->deleting)) { ret = -ENODEV; - dev_err(dev, "can't boot deleted rproc %s\n", rproc->name); + dev_err(dev, "can't boot deleting rproc %s\n", rproc->name); goto unlock_mutex; } @@ -1979,6 +1992,7 @@ EXPORT_SYMBOL(rproc_boot); int rproc_shutdown(struct rproc *rproc) { struct device *dev = &rproc->dev; + bool crashed; int ret; ret = mutex_lock_interruptible(&rproc->lock); @@ -1988,16 +2002,18 @@ int rproc_shutdown(struct rproc *rproc) } if (rproc->state != RPROC_RUNNING && - rproc->state != RPROC_ATTACHED) { + rproc->state != RPROC_ATTACHED && + rproc->state != RPROC_CRASHED) { ret = -EINVAL; goto out; } + crashed = rproc->state == RPROC_CRASHED; /* if the remote proc is still needed, bail out */ if (!atomic_dec_and_test(&rproc->power)) goto out; - ret = rproc_stop(rproc, false); + ret = rproc_stop(rproc, crashed); if (ret) { atomic_inc(&rproc->power); goto out; @@ -2506,7 +2522,9 @@ struct rproc *rproc_alloc(struct device *dev, const char *name, INIT_LIST_HEAD(&rproc->subdevs); INIT_LIST_HEAD(&rproc->dump_segments); + INIT_WORK(&rproc->attach_work, rproc_attach_work); INIT_WORK(&rproc->crash_handler, rproc_crash_handler_work); + spin_lock_init(&rproc->crash_handler_lock); rproc->state = RPROC_OFFLINE; @@ -2570,16 +2588,21 @@ EXPORT_SYMBOL(rproc_put); */ int rproc_del(struct rproc *rproc) { + unsigned long flags; + if (!rproc) return -EINVAL; + spin_lock_irqsave(&rproc->crash_handler_lock, flags); + WRITE_ONCE(rproc->deleting, true); + spin_unlock_irqrestore(&rproc->crash_handler_lock, flags); + + if (cancel_work_sync(&rproc->crash_handler)) + pm_relax(rproc->dev.parent); + /* TODO: make sure this works with rproc->power > 1 */ rproc_shutdown(rproc); - mutex_lock(&rproc->lock); - rproc->state = RPROC_DELETED; - mutex_unlock(&rproc->lock); - rproc_delete_debug_dir(rproc); /* the rproc is downref'ed as soon as it's removed from the klist */ @@ -2691,18 +2714,26 @@ EXPORT_SYMBOL(rproc_get_by_child); */ void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type) { + unsigned long flags; + if (!rproc) { pr_err("NULL rproc pointer\n"); return; } + spin_lock_irqsave(&rproc->crash_handler_lock, flags); + if (READ_ONCE(rproc->deleting)) { + spin_unlock_irqrestore(&rproc->crash_handler_lock, flags); + return; + } + /* Prevent suspend while the remoteproc is being recovered */ pm_stay_awake(rproc->dev.parent); + queue_work(rproc_recovery_wq, &rproc->crash_handler); + spin_unlock_irqrestore(&rproc->crash_handler_lock, flags); dev_err(&rproc->dev, "crash detected in %s: type %s\n", rproc->name, rproc_crash_to_string(type)); - - queue_work(rproc_recovery_wq, &rproc->crash_handler); } EXPORT_SYMBOL(rproc_report_crash); diff --git a/drivers/remoteproc/remoteproc_sysfs.c b/drivers/remoteproc/remoteproc_sysfs.c index 138e752c5e4e0..925b0cdbe5778 100644 --- a/drivers/remoteproc/remoteproc_sysfs.c +++ b/drivers/remoteproc/remoteproc_sysfs.c @@ -168,7 +168,6 @@ static const char * const rproc_state_string[] = { [RPROC_SUSPENDED] = "suspended", [RPROC_RUNNING] = "running", [RPROC_CRASHED] = "crashed", - [RPROC_DELETED] = "deleted", [RPROC_ATTACHED] = "attached", [RPROC_DETACHED] = "detached", [RPROC_LAST] = "invalid", diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 2f09f4b78bd3b..dd422c56fbb14 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -1196,8 +1196,7 @@ static u64 mpam_msmon_overflow_val(enum mpam_device_features type, { u64 overflow_val = __mpam_msmon_overflow_val(type); - if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && - type != mpam_feat_msmon_mbwu_63counter) + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc)) overflow_val *= 64; return overflow_val; @@ -1293,8 +1292,7 @@ static void __ris_msmon_read(void *arg) now = FIELD_GET(MSMON___VALUE, now); } - if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && - m->type != mpam_feat_msmon_mbwu_63counter) + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc)) now *= 64; if (nrdy) @@ -2025,6 +2023,9 @@ static void mpam_msc_drv_remove(struct platform_device *pdev) { struct mpam_msc *msc = platform_get_drvdata(pdev); + if (!msc) + return; + mutex_lock(&mpam_list_lock); mpam_msc_destroy(msc); mutex_unlock(&mpam_list_lock); @@ -2138,6 +2139,7 @@ static int mpam_msc_drv_probe(struct platform_device *pdev) static struct platform_driver mpam_msc_driver = { .driver = { .name = "mpam_msc", + .suppress_bind_attrs = true, }, .probe = mpam_msc_drv_probe, .remove = mpam_msc_drv_remove, diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c index d9d4468e4cbdf..fda1ddda05016 100644 --- a/drivers/rpmsg/qcom_glink_native.c +++ b/drivers/rpmsg/qcom_glink_native.c @@ -1418,9 +1418,6 @@ static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) channel->ept.cb = NULL; spin_unlock_irqrestore(&channel->recv_lock, flags); - /* Decouple the potential rpdev from the channel */ - qcom_glink_remove_rpmsg_device(glink, channel); - qcom_glink_send_close_req(glink, channel); } diff --git a/drivers/rpmsg/qcom_glink_smem.c b/drivers/rpmsg/qcom_glink_smem.c index 62adc4db23170..35bb03e67ae8a 100644 --- a/drivers/rpmsg/qcom_glink_smem.c +++ b/drivers/rpmsg/qcom_glink_smem.c @@ -103,6 +103,13 @@ static void glink_smem_rx_peek(struct qcom_glink_pipe *np, if (tail >= pipe->native.length) tail -= pipe->native.length; + /* + * Order the availability (head) read in glink_smem_rx_avail() + * against the FIFO payload read below, so APPS never consumes + * stale data the remote has not yet published. + */ + rmb(); + len = min_t(size_t, count, pipe->native.length - tail); if (len) memcpy_fromio(data, pipe->fifo + tail, len); diff --git a/drivers/rtc/rtc-gamecube.c b/drivers/rtc/rtc-gamecube.c index 045d5d45ab4b0..889028cecf4a5 100644 --- a/drivers/rtc/rtc-gamecube.c +++ b/drivers/rtc/rtc-gamecube.c @@ -355,9 +355,7 @@ static int gamecube_rtc_probe(struct platform_device *pdev) rtc->range_max = U32_MAX; rtc->ops = &gamecube_rtc_ops; - devm_rtc_register_device(rtc); - - return 0; + return devm_rtc_register_device(rtc); } static const struct of_device_id gamecube_rtc_of_match[] = { diff --git a/drivers/rtc/rtc-pcf85363.c b/drivers/rtc/rtc-pcf85363.c index 540042b9eec8f..ccc7834e57599 100644 --- a/drivers/rtc/rtc-pcf85363.c +++ b/drivers/rtc/rtc-pcf85363.c @@ -426,8 +426,8 @@ static int pcf85363_probe(struct i2c_client *client) err = pcf85363_load_capacitance(pcf85363, client->dev.of_node); if (err < 0) - dev_warn(&client->dev, "failed to set xtal load capacitance: %d", - err); + return dev_err_probe(&client->dev, err, + "failed to set xtal load capacitance\n"); pcf85363->rtc->ops = &rtc_ops; pcf85363->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; @@ -436,9 +436,16 @@ static int pcf85363_probe(struct i2c_client *client) wakeup_source = device_property_read_bool(&client->dev, "wakeup-source"); if (client->irq > 0 || wakeup_source) { - regmap_write(pcf85363->regmap, CTRL_FLAGS, 0); - regmap_update_bits(pcf85363->regmap, CTRL_PIN_IO, - PIN_IO_INTAPM, PIN_IO_INTA_OUT); + err = regmap_write(pcf85363->regmap, CTRL_FLAGS, 0); + if (err) + return dev_err_probe(&client->dev, err, + "failed to clear flags\n"); + + err = regmap_update_bits(pcf85363->regmap, CTRL_PIN_IO, + PIN_IO_INTAPM, PIN_IO_INTA_OUT); + if (err) + return dev_err_probe(&client->dev, err, + "failed to set interrupt pin mode\n"); } if (client->irq > 0) { diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c index 81d13733b1e93..a90dc940474bf 100644 --- a/drivers/rtc/rtc-pcf8563.c +++ b/drivers/rtc/rtc-pcf8563.c @@ -449,7 +449,9 @@ static struct clk *pcf8563_clkout_register_clk(struct pcf8563 *pcf8563) clk = devm_clk_register(&pcf8563->rtc->dev, &pcf8563->clkout_hw); if (!IS_ERR(clk)) - of_clk_add_provider(node, of_clk_src_simple_get, clk); + devm_of_clk_add_hw_provider(pcf8563->rtc->dev.parent, + of_clk_hw_simple_get, + &pcf8563->clkout_hw); return clk; } diff --git a/drivers/rtc/rtc-rzn1.c b/drivers/rtc/rtc-rzn1.c index 305f10a8a85b6..919ed0fb1e230 100644 --- a/drivers/rtc/rtc-rzn1.c +++ b/drivers/rtc/rtc-rzn1.c @@ -234,13 +234,24 @@ static int rzn1_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) if (ret) return ret; + ctl1 = readl(rtc->base + RZN1_RTC_CTL1); + alrm->enabled = !!(ctl1 & (RZN1_RTC_CTL1_ALME | RZN1_RTC_CTL1_1SE)); + min = readl(rtc->base + RZN1_RTC_ALM); hour = readl(rtc->base + RZN1_RTC_ALH); - wday = readl(rtc->base + RZN1_RTC_ALW); tm->tm_sec = 0; tm->tm_min = bcd2bin(min); tm->tm_hour = bcd2bin(hour); + + /* + * If wday is zero, no bit is set in RZN1_RTC_ALW. This is the + * register's power-on reset value. + */ + wday = readl(rtc->base + RZN1_RTC_ALW); + if (!wday) + return 0; + delta_days = ((fls(wday) - 1) - tm->tm_wday + 7) % 7; tm->tm_wday = fls(wday) - 1; @@ -249,9 +260,6 @@ static int rzn1_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) rtc_time64_to_tm(alarm, tm); } - ctl1 = readl(rtc->base + RZN1_RTC_CTL1); - alrm->enabled = !!(ctl1 & (RZN1_RTC_CTL1_ALME | RZN1_RTC_CTL1_1SE)); - return 0; } @@ -260,7 +268,6 @@ static int rzn1_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) struct rzn1_rtc *rtc = dev_get_drvdata(dev); struct rtc_time *tm = &alrm->time, tm_now; unsigned long alarm, farest; - unsigned int days_ahead, wday; int ret; ret = rzn1_rtc_read_time(dev, &tm_now); @@ -273,13 +280,14 @@ static int rzn1_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) if (time_after(alarm, farest)) return -ERANGE; - /* Convert alarm day into week day */ - days_ahead = tm->tm_mday - tm_now.tm_mday; - wday = (tm_now.tm_wday + days_ahead) % 7; + /* Disable alarm interrupts before reprogramming the alarm. */ + ret = rzn1_rtc_alarm_irq_enable(dev, 0); + if (ret) + return ret; writel(bin2bcd(tm->tm_min), rtc->base + RZN1_RTC_ALM); writel(bin2bcd(tm->tm_hour), rtc->base + RZN1_RTC_ALH); - writel(BIT(wday), rtc->base + RZN1_RTC_ALW); + writel(BIT(tm->tm_wday), rtc->base + RZN1_RTC_ALW); rtc->tm_alarm = alrm->time; @@ -464,6 +472,10 @@ static int rzn1_rtc_probe(struct platform_device *pdev) } irq = platform_get_irq_byname_optional(pdev, "pps"); + if (irq == -EPROBE_DEFER) { + ret = irq; + goto dis_runtime_pm; + } if (irq >= 0) ret = devm_request_irq(&pdev->dev, irq, rzn1_rtc_1s_irq, 0, "RZN1 RTC 1s", rtc); diff --git a/drivers/rtc/rtc-spacemit-p1.c b/drivers/rtc/rtc-spacemit-p1.c index 43ab62494bb4b..1de7bd995d296 100644 --- a/drivers/rtc/rtc-spacemit-p1.c +++ b/drivers/rtc/rtc-spacemit-p1.c @@ -57,8 +57,9 @@ static int p1_rtc_read_time(struct device *dev, struct rtc_time *t) u8 time[6]; int ret; - if (!regmap_test_bits(regmap, RTC_CTRL, RTC_EN)) - return -EINVAL; /* RTC is disabled */ + ret = regmap_test_bits(regmap, RTC_CTRL, RTC_EN); + if (ret <= 0) + return ret ?: -EINVAL; /* RTC is disabled or error */ ret = regmap_bulk_read(regmap, RTC_TIME, time, sizeof(time)); if (ret) diff --git a/drivers/rtc/rtc-zynqmp.c b/drivers/rtc/rtc-zynqmp.c index 2ae54804b87a4..5bcb7536e9734 100644 --- a/drivers/rtc/rtc-zynqmp.c +++ b/drivers/rtc/rtc-zynqmp.c @@ -334,10 +334,9 @@ static int xlnx_rtc_probe(struct platform_device *pdev) /* Getting the rtc info */ xrtcdev->rtc_clk = devm_clk_get_optional(&pdev->dev, "rtc"); - if (IS_ERR(xrtcdev->rtc_clk)) { - if (PTR_ERR(xrtcdev->rtc_clk) != -EPROBE_DEFER) - dev_warn(&pdev->dev, "Device clock not found.\n"); - } + if (IS_ERR(xrtcdev->rtc_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(xrtcdev->rtc_clk), + "Failed to get rtc clock\n"); xrtcdev->freq = clk_get_rate(xrtcdev->rtc_clk); if (!xrtcdev->freq) { ret = of_property_read_u32(pdev->dev.of_node, "calibration", diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index d8d912a3b3fec..56ef38243f823 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -1698,8 +1698,10 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm, return; } if (rq_data_dir(req) == READ) { - device->discipline->ese_read(cqr, irb); - cqr->status = DASD_CQR_SUCCESS; + if (device->discipline->ese_read(cqr, irb)) + cqr->status = DASD_CQR_ERROR; + else + cqr->status = DASD_CQR_SUCCESS; cqr->stopclk = now; dasd_device_clear_timer(device); dasd_schedule_device_bh(device); diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index d356a9f8f016f..b64ca714b53e7 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -1492,6 +1492,8 @@ static void dasd_eckd_reset_path(struct dasd_device *device, __u8 pm) struct dasd_eckd_private *private = device->private; unsigned long flags; + if (!private) + return; if (!private->fcx_max_data) private->fcx_max_data = get_fcx_max_data(device); spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); @@ -1647,6 +1649,9 @@ static int dasd_eckd_is_ese(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->vsq.vol_info.ese; } @@ -1654,6 +1659,9 @@ static int dasd_eckd_ext_pool_id(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->vsq.extent_pool_id; } @@ -1667,6 +1675,9 @@ static int dasd_eckd_space_configured(struct dasd_device *device) struct dasd_eckd_private *private = device->private; int rc; + if (!private) + return 0; + rc = dasd_eckd_read_vol_info(device); return rc ? : private->vsq.space_configured; @@ -1681,6 +1692,9 @@ static int dasd_eckd_space_allocated(struct dasd_device *device) struct dasd_eckd_private *private = device->private; int rc; + if (!private) + return 0; + rc = dasd_eckd_read_vol_info(device); return rc ? : private->vsq.space_allocated; @@ -1690,6 +1704,9 @@ static int dasd_eckd_logical_capacity(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->vsq.logical_capacity; } @@ -1832,7 +1849,11 @@ static int dasd_eckd_read_ext_pool_info(struct dasd_device *device) static int dasd_eckd_ext_size(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; - struct dasd_ext_pool_sum eps = private->eps; + struct dasd_ext_pool_sum eps; + + if (!private) + return 0; + eps = private->eps; if (!eps.flags.extent_size_valid) return 0; @@ -1848,6 +1869,9 @@ static int dasd_eckd_ext_pool_warn_thrshld(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->eps.warn_thrshld; } @@ -1855,6 +1879,9 @@ static int dasd_eckd_ext_pool_cap_at_warnlevel(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->eps.flags.capacity_at_warnlevel; } @@ -1865,6 +1892,9 @@ static int dasd_eckd_ext_pool_oos(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->eps.flags.pool_oos; } @@ -5938,8 +5968,11 @@ static int dasd_eckd_query_host_access(struct dasd_device *device, struct ccw1 *ccw; int rc; + if (!private) + return -ENODEV; + /* not available for HYPER PAV alias devices */ - if (!device->block && private->lcu->pav == HYPER_PAV) + if (!device->block && private->lcu && private->lcu->pav == HYPER_PAV) return -EOPNOTSUPP; /* may not be supported by the storage server */ @@ -6804,6 +6837,9 @@ static int dasd_eckd_hpf_enabled(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->fcx_max_data ? 1 : 0; } diff --git a/drivers/s390/block/dasd_erp.c b/drivers/s390/block/dasd_erp.c index 89d7516b9ec86..468f0b2cc3423 100644 --- a/drivers/s390/block/dasd_erp.c +++ b/drivers/s390/block/dasd_erp.c @@ -123,6 +123,7 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) int success; unsigned long startclk, stopclk; struct dasd_device *startdev; + unsigned int proc_bytes; BUG_ON(cqr->refers == NULL || cqr->function == NULL); @@ -130,6 +131,7 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) startclk = cqr->startclk; stopclk = cqr->stopclk; startdev = cqr->startdev; + proc_bytes = cqr->proc_bytes; /* free all ERPs - but NOT the original cqr */ while (cqr->refers != NULL) { @@ -147,6 +149,7 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) cqr->startclk = startclk; cqr->stopclk = stopclk; cqr->startdev = startdev; + cqr->proc_bytes = proc_bytes; if (success) cqr->status = DASD_CQR_DONE; else { diff --git a/drivers/s390/cio/chp.c b/drivers/s390/cio/chp.c index c890f21a82ce3..eaf0527bff6cc 100644 --- a/drivers/s390/cio/chp.c +++ b/drivers/s390/cio/chp.c @@ -78,6 +78,9 @@ u8 chp_get_sch_opm(struct subchannel *sch) int opm; int i; + if (!sch->schib.pmcw.dnv) + return 0; + opm = 0; chp_id_init(&chpid); for (i = 0; i < 8; i++) { diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c index 70dc8cc765948..e1c62eb60cca7 100644 --- a/drivers/s390/cio/cio.c +++ b/drivers/s390/cio/cio.c @@ -453,7 +453,8 @@ EXPORT_SYMBOL_GPL(cio_commit_config); /** * cio_update_schib - Perform stsch and update schib if subchannel is valid. * @sch: subchannel on which to perform stsch - * Return zero on success, -ENODEV otherwise. + * Return zero on success, -ENODEV if the subchannel is not operational, + * -EACCES if the subchannel has no valid device. */ int cio_update_schib(struct subchannel *sch) { @@ -462,10 +463,12 @@ int cio_update_schib(struct subchannel *sch) if (stsch(sch->schid, &schib)) return -ENODEV; - memcpy(&sch->schib, &schib, sizeof(schib)); - - if (!css_sch_is_valid(&schib)) + if (!css_sch_is_valid(&schib)) { + memset(&sch->schib, 0, sizeof(sch->schib)); return -EACCES; + } + + memcpy(&sch->schib, &schib, sizeof(schib)); return 0; } diff --git a/drivers/s390/cio/cio.h b/drivers/s390/cio/cio.h index bad142c536e1e..6d28a62bc67a3 100644 --- a/drivers/s390/cio/cio.h +++ b/drivers/s390/cio/cio.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -49,7 +50,7 @@ struct pmcw { /* Target SCHIB configuration. */ struct schib_config { - u64 mba; + dma64_t mba; u32 intparm; u16 mbi; u32 isc:3; @@ -66,7 +67,7 @@ struct schib_config { struct schib { struct pmcw pmcw; /* path management control word */ union scsw scsw; /* subchannel status word */ - __u64 mba; /* measurement block address */ + dma64_t mba; /* measurement block address */ __u8 mda[4]; /* model dependent area */ } __attribute__ ((packed,aligned(4))); diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index 92ab3d546fe47..66b14fedbd187 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -183,7 +183,7 @@ static int set_schib(struct ccw_device *cdev, u32 mme, int mbfc, sch->config.mbfc = mbfc; /* address can be either a block address or a block index */ if (mbfc) - sch->config.mba = address; + sch->config.mba = address ? virt_to_dma64((void *)address) : 0; else sch->config.mbi = address; diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index fb591118ecb2a..68dd4a62975d2 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -922,7 +922,7 @@ static int ccw_device_move_to_sch(struct ccw_device *cdev, if (!sch_is_pseudo_sch(old_sch)) { spin_lock_irq(&old_sch->lock); - old_enabled = old_sch->schib.pmcw.ena; + old_enabled = old_sch->schib.pmcw.dnv && old_sch->schib.pmcw.ena; rc = 0; if (old_enabled) rc = cio_disable_subchannel(old_sch); @@ -941,7 +941,7 @@ static int ccw_device_move_to_sch(struct ccw_device *cdev, CIO_MSG_EVENT(0, "device_move(0.%x.%04x,0.%x.%04x)=%d\n", cdev->private->dev_id.ssid, cdev->private->dev_id.devno, sch->schid.ssid, - sch->schib.pmcw.dev, rc); + sch->schid.sch_no, rc); if (old_enabled) { /* Try to re-enable the old subchannel. */ spin_lock_irq(&old_sch->lock); @@ -1207,7 +1207,7 @@ static void io_subchannel_quiesce(struct subchannel *sch) cdev = sch_get_cdev(sch); if (cio_is_console(sch->schid)) goto out_unlock; - if (!sch->schib.pmcw.ena) + if (!sch->schib.pmcw.dnv || !sch->schib.pmcw.ena) goto out_unlock; ret = cio_disable_subchannel(sch); if (ret != -EBUSY) @@ -1254,7 +1254,8 @@ static int recovery_check(struct device *dev, void *data) switch (cdev->private->state) { case DEV_STATE_ONLINE: sch = to_subchannel(cdev->dev.parent); - if ((sch->schib.pmcw.pam & sch->opm) == sch->vpm) + if (sch->schib.pmcw.dnv && + (sch->schib.pmcw.pam & sch->opm) == sch->vpm) break; fallthrough; case DEV_STATE_DISCONNECTED: diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c index ab419d40a8a7a..b5686c25c83c7 100644 --- a/drivers/s390/cio/device_fsm.c +++ b/drivers/s390/cio/device_fsm.c @@ -170,6 +170,9 @@ __recover_lost_chpids(struct subchannel *sch, int old_lpm) int mask, i; struct chp_id chpid; + if (!sch->schib.pmcw.dnv) + return; + chp_id_init(&chpid); for (i = 0; i<8; i++) { mask = 0x80 >> i; diff --git a/drivers/s390/cio/device_ops.c b/drivers/s390/cio/device_ops.c index 61c07b4a0fe89..1f7e83fd50872 100644 --- a/drivers/s390/cio/device_ops.c +++ b/drivers/s390/cio/device_ops.c @@ -142,6 +142,8 @@ int ccw_device_clear(struct ccw_device *cdev, unsigned long intparm) if (!cdev || !cdev->dev.parent) return -ENODEV; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state == DEV_STATE_NOT_OPER) @@ -198,6 +200,8 @@ int ccw_device_start_timeout_key(struct ccw_device *cdev, struct ccw1 *cpa, if (!cdev || !cdev->dev.parent) return -ENODEV; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state == DEV_STATE_NOT_OPER) @@ -379,6 +383,8 @@ int ccw_device_halt(struct ccw_device *cdev, unsigned long intparm) if (!cdev || !cdev->dev.parent) return -ENODEV; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state == DEV_STATE_NOT_OPER) @@ -413,6 +419,8 @@ int ccw_device_resume(struct ccw_device *cdev) if (!cdev || !cdev->dev.parent) return -ENODEV; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state == DEV_STATE_NOT_OPER) @@ -482,6 +490,8 @@ struct channel_path_desc_fmt0 *ccw_device_get_chp_desc(struct ccw_device *cdev, struct chp_id chpid; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return NULL; chp_id_init(&chpid); chpid.id = sch->schib.pmcw.chpid[chp_idx]; return chp_get_chp_desc(chpid); @@ -502,9 +512,13 @@ u8 *ccw_device_get_util_str(struct ccw_device *cdev, int chp_idx) struct chp_id chpid; u8 *util_str; + if (!sch->schib.pmcw.dnv) + return NULL; chp_id_init(&chpid); chpid.id = sch->schib.pmcw.chpid[chp_idx]; chp = chpid_to_chp(chpid); + if (!chp) + return NULL; util_str = kmalloc(sizeof(chp->desc_fmt3.util_str), GFP_KERNEL); if (!util_str) @@ -548,6 +562,8 @@ int ccw_device_tm_start_timeout_key(struct ccw_device *cdev, struct tcw *tcw, int rc; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state == DEV_STATE_VERIFY) { @@ -652,6 +668,9 @@ int ccw_device_get_mdc(struct ccw_device *cdev, u8 mask) struct chp_id chpid; int mdc = 0, i; + if (!sch->schib.pmcw.dnv) + return 0; + /* Adjust requested path mask to excluded varied off paths. */ if (mask) mask &= sch->lpm; @@ -694,6 +713,8 @@ int ccw_device_tm_intrg(struct ccw_device *cdev) { struct subchannel *sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state != DEV_STATE_ONLINE) @@ -786,6 +807,8 @@ int ccw_device_get_chpid(struct ccw_device *cdev, int chp_idx, u8 *chpid) if ((chp_idx < 0) || (chp_idx > 7)) return -EINVAL; + if (!sch->schib.pmcw.dnv) + return -ENODEV; mask = 0x80 >> chp_idx; if (!(sch->schib.pmcw.pim & mask)) return -ENODEV; diff --git a/drivers/s390/cio/vfio_ccw_fsm.c b/drivers/s390/cio/vfio_ccw_fsm.c index 5fd94e9d5c618..9a000b0231d60 100644 --- a/drivers/s390/cio/vfio_ccw_fsm.c +++ b/drivers/s390/cio/vfio_ccw_fsm.c @@ -399,7 +399,7 @@ static void fsm_close(struct vfio_ccw_private *private, spin_lock_irq(&sch->lock); - if (!sch->schib.pmcw.ena) + if (!sch->schib.pmcw.dnv || !sch->schib.pmcw.ena) goto err_unlock; ret = cio_disable_subchannel(sch); diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 44b3a1dcc1b36..4db878c18f41c 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -48,15 +48,19 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q); * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM * guest's APCB. * 2. kvm->lock: required to update a guest's APCB - * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev + * 3. kvm->arch.crypto.pqap_hook_rwsem: required to update pqap_hook and + * serialize against PQAP intercepts + * 4. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev * - * Note: If @kvm is NULL, the KVM lock will not be taken. + * Note: If @kvm is NULL, the KVM lock and pqap_hook_rwsem will not be taken. */ static inline void get_update_locks_for_kvm(struct kvm *kvm) { mutex_lock(&matrix_dev->guests_lock); - if (kvm) + if (kvm) { mutex_lock(&kvm->lock); + down_write(&kvm->arch.crypto.pqap_hook_rwsem); + } mutex_lock(&matrix_dev->mdevs_lock); } @@ -68,16 +72,19 @@ static inline void get_update_locks_for_kvm(struct kvm *kvm) * * The proper unlocking order is: * 1. matrix_dev->mdevs_lock - * 2. kvm->lock - * 3. matrix_dev->guests_lock + * 2. kvm->arch.crypto.pqap_hook_rwsem + * 3. kvm->lock + * 4. matrix_dev->guests_lock * - * Note: If @kvm is NULL, the KVM lock will not be released. + * Note: If @kvm is NULL, the KVM lock and pqap_hook_rwsem will not be released. */ static inline void release_update_locks_for_kvm(struct kvm *kvm) { mutex_unlock(&matrix_dev->mdevs_lock); - if (kvm) + if (kvm) { + up_write(&kvm->arch.crypto.pqap_hook_rwsem); mutex_unlock(&kvm->lock); + } mutex_unlock(&matrix_dev->guests_lock); } @@ -270,9 +277,9 @@ static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q) { if (!q) return; - if (q->saved_isc != VFIO_AP_ISC_INVALID && - !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) { - kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc); + if (q->saved_isc != VFIO_AP_ISC_INVALID) { + if (!WARN_ON(!q->matrix_mdev) && q->matrix_mdev->kvm) + kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc); q->saved_isc = VFIO_AP_ISC_INVALID; } if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) { @@ -800,12 +807,17 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev) ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev); if (ret) goto err_put_vdev; - matrix_mdev->req_trigger = NULL; - matrix_mdev->cfg_chg_trigger = NULL; + + /* + * Take the matrix_dev->guests_lock mutex before adding the matrix_mdev + * to the mdev_list. All functions that traverse the list must also hold + * this lock to guard against additions to or removals from the list + * while it is being traversed. + */ + mutex_lock(&matrix_dev->guests_lock); dev_set_drvdata(&mdev->dev, matrix_mdev); - mutex_lock(&matrix_dev->mdevs_lock); list_add(&matrix_mdev->node, &matrix_dev->mdev_list); - mutex_unlock(&matrix_dev->mdevs_lock); + mutex_unlock(&matrix_dev->guests_lock); return 0; err_put_vdev: @@ -1821,26 +1833,17 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, struct kvm *kvm) { - struct ap_matrix_mdev *m; - if (kvm->arch.crypto.crycbd) { - down_write(&kvm->arch.crypto.pqap_hook_rwsem); - kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; - up_write(&kvm->arch.crypto.pqap_hook_rwsem); - get_update_locks_for_kvm(kvm); - - list_for_each_entry(m, &matrix_dev->mdev_list, node) { - if (m != matrix_mdev && m->kvm == kvm) { - release_update_locks_for_kvm(kvm); - return -EPERM; - } + if (kvm->arch.crypto.pqap_hook) { + release_update_locks_for_kvm(kvm); + return -EPERM; } + kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; kvm_get_kvm(kvm); matrix_mdev->kvm = kvm; vfio_ap_mdev_update_guest_apcb(matrix_mdev); - release_update_locks_for_kvm(kvm); } @@ -1883,18 +1886,15 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) struct kvm *kvm = matrix_mdev->kvm; if (kvm && kvm->arch.crypto.crycbd) { - down_write(&kvm->arch.crypto.pqap_hook_rwsem); - kvm->arch.crypto.pqap_hook = NULL; - up_write(&kvm->arch.crypto.pqap_hook_rwsem); - get_update_locks_for_kvm(kvm); + kvm->arch.crypto.pqap_hook = NULL; kvm_arch_crypto_clear_masks(kvm); vfio_ap_mdev_reset_queues(matrix_mdev); - kvm_put_kvm(kvm); matrix_mdev->kvm = NULL; release_update_locks_for_kvm(kvm); + kvm_put_kvm(kvm); } } @@ -1935,6 +1935,8 @@ static int apq_status_check(int apqn, struct ap_queue_status *status) * a value indicating a reset needs to be performed again. */ return -EAGAIN; + case AP_RESPONSE_Q_NOT_AVAIL: + return -ENODEV; default: WARN(true, "failed to verify reset of queue %02x.%04x: TAPQ rc=%u\n", @@ -1961,6 +1963,10 @@ static void apq_reset_check(struct work_struct *reset_work) ret = apq_status_check(q->apqn, &status); if (ret == -EIO) return; + if (ret == -ENODEV) { + vfio_ap_free_aqic_resources(q); + return; + } if (ret == -EBUSY) { pr_notice_ratelimited(WAIT_MSG, elapsed, AP_QID_CARD(q->apqn), @@ -2004,6 +2010,7 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q) break; case AP_RESPONSE_DECONFIGURED: case AP_RESPONSE_CHECKSTOPPED: + case AP_RESPONSE_Q_NOT_AVAIL: vfio_ap_free_aqic_resources(q); break; default: @@ -2297,6 +2304,8 @@ static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q) unsigned long apid = AP_QID_CARD(q->apqn); unsigned long apqi = AP_QID_QUEUE(q->apqn); + lockdep_assert_held(&matrix_dev->guests_lock); + list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { if (test_bit_inv(apid, matrix_mdev->matrix.apm) && test_bit_inv(apqi, matrix_mdev->matrix.aqm)) @@ -2316,8 +2325,26 @@ static ssize_t status_show(struct device *dev, struct ap_matrix_mdev *matrix_mdev; struct ap_device *apdev = to_ap_dev(dev); + mutex_lock(&matrix_dev->guests_lock); mutex_lock(&matrix_dev->mdevs_lock); q = dev_get_drvdata(&apdev->device); + + /* + * Make sure the drvdata has been set before proceeding. There is a + * possibility that the drvdata was not set if the vfio_ap_queue object + * could not be allocated when the queue device was probed. In that case, + * the locks used in vfio_ap_mdev_probe_queue() are released prior to + * removing the sysfs status attribute to avoid a lockdep + * splat. That opens a very small window where the status attribute is + * still available without the vfio_ap_queue object having been + * stored in the device drvdata. In that case, indicate the queue is not + * assigned. + */ + if (!q) { + nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED); + goto done; + } + matrix_mdev = vfio_ap_mdev_for_queue(q); /* If the queue is assigned to the matrix mediated device, then @@ -2342,7 +2369,9 @@ static ssize_t status_show(struct device *dev, nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED); } +done: mutex_unlock(&matrix_dev->mdevs_lock); + mutex_unlock(&matrix_dev->guests_lock); return nchars; } @@ -2415,14 +2444,17 @@ void vfio_ap_mdev_unregister(void) int vfio_ap_mdev_probe_queue(struct ap_device *apdev) { - int ret; + int ret, apqn; struct vfio_ap_queue *q; DECLARE_BITMAP(apm_filtered, AP_DEVICES); struct ap_matrix_mdev *matrix_mdev; + apqn = to_ap_queue(&apdev->device)->qid; + matrix_mdev = get_update_locks_by_apqn(apqn); + ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group); if (ret) - return ret; + goto err_release_locks; q = kzalloc_obj(*q); if (!q) { @@ -2430,11 +2462,10 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) goto err_remove_group; } - q->apqn = to_ap_queue(&apdev->device)->qid; + q->apqn = apqn; q->saved_isc = VFIO_AP_ISC_INVALID; memset(&q->reset_status, 0, sizeof(q->reset_status)); INIT_WORK(&q->reset_work, apq_reset_check); - matrix_mdev = get_update_locks_by_apqn(q->apqn); if (matrix_mdev) { vfio_ap_mdev_link_queue(matrix_mdev, q); @@ -2463,8 +2494,13 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) return ret; err_remove_group: + release_update_locks_for_mdev(matrix_mdev); sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); return ret; + +err_release_locks: + release_update_locks_for_mdev(matrix_mdev); + return ret; } void vfio_ap_mdev_remove_queue(struct ap_device *apdev) @@ -2499,12 +2535,15 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) /* * If the queue is not in the host's AP configuration, then resetting * it will fail with response code 01, (APQN not valid); so, let's make - * sure it is in the host's config. + * sure it is in the host's config. If it is not, free the KVM GISC + * resources. */ if (test_bit_inv(apid, (unsigned long *)matrix_dev->info.apm) && test_bit_inv(apqi, (unsigned long *)matrix_dev->info.aqm)) { vfio_ap_mdev_reset_queue(q); flush_work(&q->reset_work); + } else { + vfio_ap_free_aqic_resources(q); } done: @@ -2559,24 +2598,28 @@ static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev, unsigned long *aqrem, unsigned long *cdrem) { - int do_hotplug = 0; + bool do_hotplug = false; - if (!bitmap_empty(aprem, AP_DEVICES)) { - do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm, - matrix_mdev->shadow_apcb.apm, - aprem, AP_DEVICES); + if (bitmap_intersects(matrix_mdev->shadow_apcb.apm, aprem, AP_DEVICES)) { + bitmap_andnot(matrix_mdev->shadow_apcb.apm, + matrix_mdev->shadow_apcb.apm, + aprem, AP_DEVICES); + do_hotplug = true; } - if (!bitmap_empty(aqrem, AP_DOMAINS)) { - do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm, - matrix_mdev->shadow_apcb.aqm, - aqrem, AP_DEVICES); + if (bitmap_intersects(matrix_mdev->shadow_apcb.aqm, aqrem, AP_DOMAINS)) { + bitmap_andnot(matrix_mdev->shadow_apcb.aqm, + matrix_mdev->shadow_apcb.aqm, + aqrem, AP_DOMAINS); + do_hotplug = true; } - if (!bitmap_empty(cdrem, AP_DOMAINS)) - do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm, - matrix_mdev->shadow_apcb.adm, - cdrem, AP_DOMAINS); + if (bitmap_intersects(matrix_mdev->shadow_apcb.adm, cdrem, AP_DOMAINS)) { + bitmap_andnot(matrix_mdev->shadow_apcb.adm, + matrix_mdev->shadow_apcb.adm, + cdrem, AP_DOMAINS); + do_hotplug = true; + } if (do_hotplug) vfio_ap_mdev_update_guest_apcb(matrix_mdev); @@ -2603,28 +2646,41 @@ static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove, DECLARE_BITMAP(aprem, AP_DEVICES); DECLARE_BITMAP(aqrem, AP_DOMAINS); DECLARE_BITMAP(cdrem, AP_DOMAINS); - int do_remove = 0; + int do_remove; + /* + * It is safe to traverse this list here because the + * required guard - matrix_dev->guests_lock - is taken in the + * vfio_ap_on_cfg_changed function prior to this function getting + * called. + */ list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { - mutex_lock(&matrix_mdev->kvm->lock); + /* + * The mdevs_lock must be held to access fields within matrix_mdev, + * and kvm->lock must be taken before mdevs_lock to satisfy the lock + * ordering requirement and prevent a lockdep splat. + */ + if (matrix_mdev->kvm) + mutex_lock(&matrix_mdev->kvm->lock); mutex_lock(&matrix_dev->mdevs_lock); - do_remove |= bitmap_and(aprem, ap_remove, - matrix_mdev->matrix.apm, - AP_DEVICES); + do_remove = bitmap_and(aprem, ap_remove, + matrix_mdev->matrix.apm, + AP_DEVICES); do_remove |= bitmap_and(aqrem, aq_remove, matrix_mdev->matrix.aqm, AP_DOMAINS); - do_remove |= bitmap_andnot(cdrem, cd_remove, - matrix_mdev->matrix.adm, - AP_DOMAINS); + do_remove |= bitmap_and(cdrem, cd_remove, + matrix_mdev->matrix.adm, + AP_DOMAINS); if (do_remove) vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem, cdrem); mutex_unlock(&matrix_dev->mdevs_lock); - mutex_unlock(&matrix_mdev->kvm->lock); + if (matrix_mdev->kvm) + mutex_unlock(&matrix_mdev->kvm->lock); } } @@ -2748,13 +2804,27 @@ static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add, vfio_ap_filter_apid_by_qtype(apm_add, aqm_add); + /* + * It is safe to traverse this list here because the + * required guard - matrix_dev->guests_lock - is taken in the + * vfio_ap_on_cfg_changed function prior to this function getting + * called. + */ list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + /* + * The mdevs_lock must be held in order to access fields + * within matrix_mdev + */ + mutex_lock(&matrix_dev->mdevs_lock); + bitmap_and(matrix_mdev->apm_add, matrix_mdev->matrix.apm, apm_add, AP_DEVICES); bitmap_and(matrix_mdev->aqm_add, matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS); bitmap_and(matrix_mdev->adm_add, matrix_mdev->matrix.adm, adm_add, AP_DEVICES); + + mutex_unlock(&matrix_dev->mdevs_lock); } } @@ -2807,6 +2877,10 @@ void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info, if (!cur_cfg_info || !prev_cfg_info) return; + /* + * Take the guests_lock mutex here to guard access to the + * matrix_dev->mdev_list in the two functions called below. + */ mutex_lock(&matrix_dev->guests_lock); vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info); @@ -2821,8 +2895,14 @@ static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) DECLARE_BITMAP(apm_filtered, AP_DEVICES); bool filter_domains, filter_adapters, filter_cdoms, do_hotplug = false; - mutex_lock(&matrix_mdev->kvm->lock); - mutex_lock(&matrix_dev->mdevs_lock); + /* + * Zero out the apm_filtered bitmap in case there are no adapters or + * domains to be added, but only control domains. In that case, + * vfio_ap_mdev_filter_matrix() - which initializes apm_filtered - will + * not get called and the reset_queues_for_apids will crash because it + * will access an uninitialized bitmap. + */ + bitmap_zero(apm_filtered, AP_DEVICES); filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm, matrix_mdev->apm_add, AP_DEVICES); @@ -2841,9 +2921,6 @@ static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) vfio_ap_mdev_update_guest_apcb(matrix_mdev); reset_queues_for_apids(matrix_mdev, apm_filtered); - - mutex_unlock(&matrix_dev->mdevs_lock); - mutex_unlock(&matrix_mdev->kvm->lock); } void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, @@ -2854,15 +2931,29 @@ void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, mutex_lock(&matrix_dev->guests_lock); list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + /* + * The mdevs_lock must be held to access fields within matrix_mdev, + * and kvm->lock must be taken before mdevs_lock to satisfy the lock + * ordering requirement and prevent a lockdep splat. + */ + if (matrix_mdev->kvm) + mutex_lock(&matrix_mdev->kvm->lock); + mutex_lock(&matrix_dev->mdevs_lock); + if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) && bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) && bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS)) - continue; + goto do_unlock; vfio_ap_mdev_hot_plug_cfg(matrix_mdev); bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES); bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS); bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS); + +do_unlock: + mutex_unlock(&matrix_dev->mdevs_lock); + if (matrix_mdev->kvm) + mutex_unlock(&matrix_mdev->kvm->lock); } mutex_unlock(&matrix_dev->guests_lock); diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c index 242da20f27e0e..035b233abb4e9 100644 --- a/drivers/s390/net/ism_drv.c +++ b/drivers/s390/net/ism_drv.c @@ -231,6 +231,7 @@ static void ism_free_dmb(struct ism_dev *ism, struct dibs_dmb *dmb) dma_unmap_page(&ism->pdev->dev, dmb->dma_addr, dmb->dmb_len, DMA_FROM_DEVICE); folio_put(virt_to_folio(dmb->cpu_addr)); + dmb->cpu_addr = NULL; } static int ism_alloc_dmb(struct ism_dev *ism, struct dibs_dmb *dmb) @@ -274,7 +275,8 @@ static int ism_alloc_dmb(struct ism_dev *ism, struct dibs_dmb *dmb) return 0; out_free: - kfree(dmb->cpu_addr); + folio_put(folio); + dmb->cpu_addr = NULL; out_bit: clear_bit(dmb->idx, ism->sba_bitmap); return rc; diff --git a/drivers/s390/net/qeth_l2.h b/drivers/s390/net/qeth_l2.h index 7c646e2fed7e5..f94975e970cad 100644 --- a/drivers/s390/net/qeth_l2.h +++ b/drivers/s390/net/qeth_l2.h @@ -13,7 +13,8 @@ extern const struct attribute_group *qeth_l2_attr_groups[]; int qeth_bridgeport_query_ports(struct qeth_card *card, enum qeth_sbp_roles *role, - enum qeth_sbp_states *state); + enum qeth_sbp_states *state, + bool *os_mismatch); int qeth_bridgeport_setrole(struct qeth_card *card, enum qeth_sbp_roles role); int qeth_bridgeport_an_set(struct qeth_card *card, int enable); diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c index a9e7d1d637a20..2935c2ecc314b 100644 --- a/drivers/s390/net/qeth_l2_main.c +++ b/drivers/s390/net/qeth_l2_main.c @@ -1158,7 +1158,7 @@ static void qeth_l2_setup_bridgeport_attrs(struct qeth_card *card) qeth_bridgeport_setrole(card, card->options.sbp.role); /* Let the callback function refresh the stored role value. */ qeth_bridgeport_query_ports(card, &card->options.sbp.role, - NULL); + NULL, NULL); } if (card->options.sbp.hostnotification) { if (qeth_bridgeport_an_set(card, 1)) @@ -1545,6 +1545,7 @@ struct _qeth_sbp_cbctl { struct { enum qeth_sbp_roles *role; enum qeth_sbp_states *state; + bool *os_mismatch; } qports; } data; }; @@ -1721,10 +1722,19 @@ static int qeth_bridgeport_query_ports_cb(struct qeth_card *card, struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data; struct _qeth_sbp_cbctl *cbctl = (struct _qeth_sbp_cbctl *)reply->param; struct qeth_sbp_port_data *qports; + u16 sbp_rc; int rc; QETH_CARD_TEXT(card, 2, "brqprtcb"); - rc = qeth_bridgeport_makerc(card, cmd); + sbp_rc = cmd->data.sbp.hdr.return_code; + + /* on OS family mismatch, query still returns valid port data; + * treat as success + */ + if (sbp_rc == IPA_RC_SBP_IQD_OS_MISMATCH && !cmd->hdr.return_code) + rc = 0; + else + rc = qeth_bridgeport_makerc(card, cmd); if (rc) return rc; @@ -1740,6 +1750,9 @@ static int qeth_bridgeport_query_ports_cb(struct qeth_card *card, if (cbctl->data.qports.state) *cbctl->data.qports.state = qports->entry[0].state; } + if (cbctl->data.qports.os_mismatch) + *cbctl->data.qports.os_mismatch = + (sbp_rc == IPA_RC_SBP_IQD_OS_MISMATCH); return 0; } @@ -1748,13 +1761,17 @@ static int qeth_bridgeport_query_ports_cb(struct qeth_card *card, * @card: qeth_card structure pointer. * @role: Role of the port: 0-none, 1-primary, 2-secondary. * @state: State of the port: 0-inactive, 1-standby, 2-active. + * @os_mismatch: if non-NULL, set to true when firmware reports + * OS family mismatch. * * Returns negative errno-compatible error indication or 0 on success. * - * 'role' and 'state' are not updated in case of hardware operation failure. + * 'role', 'state' and 'os_mismatch' are not updated in case of + * hardware operation failure. */ int qeth_bridgeport_query_ports(struct qeth_card *card, - enum qeth_sbp_roles *role, enum qeth_sbp_states *state) + enum qeth_sbp_roles *role, enum qeth_sbp_states *state, + bool *os_mismatch) { struct qeth_cmd_buffer *iob; struct _qeth_sbp_cbctl cbctl = { @@ -1762,6 +1779,7 @@ int qeth_bridgeport_query_ports(struct qeth_card *card, .qports = { .role = role, .state = state, + .os_mismatch = os_mismatch, }, }, }; diff --git a/drivers/s390/net/qeth_l2_sys.c b/drivers/s390/net/qeth_l2_sys.c index 7f592f9125172..7101be62eb1d5 100644 --- a/drivers/s390/net/qeth_l2_sys.c +++ b/drivers/s390/net/qeth_l2_sys.c @@ -15,6 +15,7 @@ static ssize_t qeth_bridge_port_role_state_show(struct device *dev, { struct qeth_card *card = dev_get_drvdata(dev); enum qeth_sbp_states state = QETH_SBP_STATE_INACTIVE; + bool os_mismatch = false; int rc = 0; char *word; @@ -25,7 +26,7 @@ static ssize_t qeth_bridge_port_role_state_show(struct device *dev, if (qeth_card_hw_is_reachable(card) && card->options.sbp.supported_funcs) rc = qeth_bridgeport_query_ports(card, - &card->options.sbp.role, &state); + &card->options.sbp.role, &state, &os_mismatch); if (!rc) { if (show_state) switch (state) { @@ -52,6 +53,10 @@ static ssize_t qeth_bridge_port_role_state_show(struct device *dev, if (rc) QETH_CARD_TEXT_(card, 2, "SBP%02x:%02x", card->options.sbp.role, state); + else if (!show_state && + card->options.sbp.role == QETH_SBP_ROLE_NONE && + os_mismatch) + rc = sysfs_emit(buf, "%s (OS family mismatch)\n", word); else rc = sysfs_emit(buf, "%s\n", word); } diff --git a/drivers/scsi/fnic/fdls_disc.c b/drivers/scsi/fnic/fdls_disc.c index 554dea7678854..5880ca28a0ad9 100644 --- a/drivers/scsi/fnic/fdls_disc.c +++ b/drivers/scsi/fnic/fdls_disc.c @@ -104,7 +104,7 @@ uint8_t *fdls_alloc_frame(struct fnic_iport_s *iport) frame = mempool_alloc(fnic->frame_pool, GFP_ATOMIC); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame"); return NULL; } @@ -136,7 +136,7 @@ uint16_t fdls_alloc_oxid(struct fnic_iport_s *iport, int oxid_frame_type, */ idx = find_next_zero_bit(oxid_pool->bitmap, FNIC_OXID_POOL_SZ, oxid_pool->next_idx); if (idx == FNIC_OXID_POOL_SZ) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Alloc oxid: all oxid slots are busy iport state:%d\n", iport->state); return FNIC_UNASSIGNED_OXID; @@ -148,7 +148,7 @@ uint16_t fdls_alloc_oxid(struct fnic_iport_s *iport, int oxid_frame_type, oxid = FNIC_OXID_ENCODE(idx, oxid_frame_type); *active_oxid = oxid; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "alloc oxid: 0x%x, iport state: %d\n", oxid, iport->state); return oxid; @@ -169,7 +169,7 @@ static void fdls_free_oxid_idx(struct fnic_iport_s *iport, uint16_t oxid_idx) lockdep_assert_held(&fnic->fnic_lock); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "free oxid idx: 0x%x\n", oxid_idx); WARN_ON(!test_and_clear_bit(oxid_idx, oxid_pool->bitmap)); @@ -194,7 +194,7 @@ void fdls_reclaim_oxid_handler(struct work_struct *work) struct reclaim_entry_s *reclaim_entry, *next; unsigned long delay_j, cur_jiffies; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Reclaim oxid callback\n"); spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); @@ -223,7 +223,7 @@ void fdls_reclaim_oxid_handler(struct work_struct *work) delay_j = reclaim_entry->expires - cur_jiffies; schedule_delayed_work(&oxid_pool->oxid_reclaim_work, delay_j); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Scheduling next callback at:%ld jiffies\n", delay_j); } @@ -266,7 +266,7 @@ void fdls_schedule_oxid_free(struct fnic_iport_s *iport, uint16_t *active_oxid) lockdep_assert_held(&fnic->fnic_lock); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Schedule oxid free. oxid: 0x%x\n", *active_oxid); *active_oxid = FNIC_UNASSIGNED_OXID; @@ -275,7 +275,7 @@ void fdls_schedule_oxid_free(struct fnic_iport_s *iport, uint16_t *active_oxid) kzalloc_obj(struct reclaim_entry_s, GFP_ATOMIC); if (!reclaim_entry) { - FNIC_FCS_DBG(KERN_WARNING, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_WARNING, fnic, "Failed to allocate memory for reclaim struct for oxid idx: %d\n", oxid_idx); @@ -313,7 +313,7 @@ void fdls_schedule_oxid_free_retry_work(struct work_struct *work) for_each_set_bit(idx, oxid_pool->pending_schedule_free, FNIC_OXID_POOL_SZ) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Schedule oxid free. oxid idx: %d\n", idx); reclaim_entry = kzalloc_obj(*reclaim_entry); @@ -416,7 +416,7 @@ fdls_start_fabric_timer(struct fnic_iport_s *iport, int timeout) struct fnic *fnic = iport->fnic; if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport fcid: 0x%x: Canceling fabric disc timer\n", iport->fcid); fnic_del_fabric_timer_sync(fnic); @@ -429,7 +429,7 @@ fdls_start_fabric_timer(struct fnic_iport_s *iport, int timeout) fabric_tov = jiffies + msecs_to_jiffies(timeout); mod_timer(&iport->fabric.retry_timer, round_jiffies(fabric_tov)); iport->fabric.timer_pending = 1; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "fabric timer is %d ", timeout); } @@ -441,7 +441,7 @@ fdls_start_tport_timer(struct fnic_iport_s *iport, struct fnic *fnic = iport->fnic; if (tport->timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport fcid 0x%x: Canceling disc timer\n", tport->fcid); fnic_del_tport_timer_sync(fnic, tport); @@ -575,7 +575,7 @@ fdls_send_rscn_resp(struct fnic_iport_s *iport, frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send RSCN response"); return; } @@ -588,7 +588,7 @@ fdls_send_rscn_resp(struct fnic_iport_s *iport, oxid = FNIC_STD_GET_OX_ID(rscn_fchdr); FNIC_STD_SET_OX_ID(pels_acc->fchdr, oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send RSCN response with oxid: 0x%x", iport->fcid, oxid); @@ -608,7 +608,7 @@ fdls_send_logo_resp(struct fnic_iport_s *iport, frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send LOGO response"); return; } @@ -621,7 +621,7 @@ fdls_send_logo_resp(struct fnic_iport_s *iport, oxid = FNIC_STD_GET_OX_ID(req_fchdr); FNIC_STD_SET_OX_ID(plogo_resp->fchdr, oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send LOGO response with oxid: 0x%x", iport->fcid, oxid); @@ -642,7 +642,7 @@ fdls_send_tport_abts(struct fnic_iport_s *iport, frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send tport ABTS"); return; } @@ -665,7 +665,7 @@ fdls_send_tport_abts(struct fnic_iport_s *iport, FNIC_STD_SET_OX_ID(*ptport_abts, tport->active_oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send tport abts: tport->state: %d ", iport->fcid, tport->state); @@ -687,7 +687,7 @@ static void fdls_send_fabric_abts(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send fabric ABTS"); return; } @@ -750,7 +750,7 @@ static void fdls_send_fabric_abts(struct fnic_iport_s *iport) oxid = iport->active_oxid_fabric_req; FNIC_STD_SET_OX_ID(*pfabric_abts, oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send fabric abts. iport->fabric.state: %d oxid: 0x%x", iport->fcid, iport->fabric.state, oxid); @@ -773,7 +773,7 @@ static uint8_t *fdls_alloc_init_fdmi_abts_frame(struct fnic_iport_s *iport, frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send FDMI ABTS"); return NULL; } @@ -802,7 +802,7 @@ static void fdls_send_fdmi_abts(struct fnic_iport_s *iport) if (frame == NULL) return; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send FDMI PLOGI abts. iport->fabric.state: %d oxid: 0x%x", iport->fcid, iport->fabric.state, iport->active_oxid_fdmi_plogi); fnic_send_fcoe_frame(iport, frame, frame_size); @@ -813,7 +813,7 @@ static void fdls_send_fdmi_abts(struct fnic_iport_s *iport) if (frame == NULL) return; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send FDMI RHBA abts. iport->fabric.state: %d oxid: 0x%x", iport->fcid, iport->fabric.state, iport->active_oxid_fdmi_rhba); fnic_send_fcoe_frame(iport, frame, frame_size); @@ -828,7 +828,7 @@ static void fdls_send_fdmi_abts(struct fnic_iport_s *iport) return; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send FDMI RPA abts. iport->fabric.state: %d oxid: 0x%x", iport->fcid, iport->fabric.state, iport->active_oxid_fdmi_rpa); fnic_send_fcoe_frame(iport, frame, frame_size); @@ -840,7 +840,7 @@ static void fdls_send_fdmi_abts(struct fnic_iport_s *iport) mod_timer(&iport->fabric.fdmi_timer, round_jiffies(fdmi_tov)); iport->fabric.fdmi_pending |= FDLS_FDMI_ABORT_PENDING; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: iport->fabric.fdmi_pending: 0x%x", iport->fcid, iport->fabric.fdmi_pending); } @@ -856,7 +856,7 @@ static void fdls_send_fabric_flogi(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send FLOGI"); iport->fabric.flags |= FNIC_FDLS_RETRY_FRAME; goto err_out; @@ -885,7 +885,7 @@ static void fdls_send_fabric_flogi(struct fnic_iport_s *iport) &iport->active_oxid_fabric_req); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send FLOGI", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -894,7 +894,7 @@ static void fdls_send_fabric_flogi(struct fnic_iport_s *iport) } FNIC_STD_SET_OX_ID(pflogi->fchdr, oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send fabric FLOGI with oxid: 0x%x", iport->fcid, oxid); @@ -916,7 +916,7 @@ static void fdls_send_fabric_plogi(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send PLOGI"); iport->fabric.flags |= FNIC_FDLS_RETRY_FRAME; goto err_out; @@ -928,7 +928,7 @@ static void fdls_send_fabric_plogi(struct fnic_iport_s *iport) oxid = fdls_alloc_oxid(iport, FNIC_FRAME_TYPE_FABRIC_PLOGI, &iport->active_oxid_fabric_req); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send fabric PLOGI", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -937,7 +937,7 @@ static void fdls_send_fabric_plogi(struct fnic_iport_s *iport) } FNIC_STD_SET_OX_ID(pplogi->fchdr, oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send fabric PLOGI with oxid: 0x%x", iport->fcid, oxid); @@ -962,7 +962,7 @@ static void fdls_send_fdmi_plogi(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send FDMI PLOGI"); goto err_out; } @@ -974,7 +974,7 @@ static void fdls_send_fdmi_plogi(struct fnic_iport_s *iport) &iport->active_oxid_fdmi_plogi); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send FDMI PLOGI", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -985,7 +985,7 @@ static void fdls_send_fdmi_plogi(struct fnic_iport_s *iport) hton24(d_id, FC_FID_MGMT_SERV); FNIC_STD_SET_D_ID(pplogi->fchdr, d_id); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send FDMI PLOGI with oxid: 0x%x", iport->fcid, oxid); @@ -1009,7 +1009,7 @@ static void fdls_send_rpn_id(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send RPN_ID"); iport->fabric.flags |= FNIC_FDLS_RETRY_FRAME; goto err_out; @@ -1036,7 +1036,7 @@ static void fdls_send_rpn_id(struct fnic_iport_s *iport) &iport->active_oxid_fabric_req); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send RPN_ID", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1045,7 +1045,7 @@ static void fdls_send_rpn_id(struct fnic_iport_s *iport) } FNIC_STD_SET_OX_ID(prpn_id->fchdr, oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send RPN ID with oxid: 0x%x", iport->fcid, oxid); @@ -1068,7 +1068,7 @@ static void fdls_send_scr(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send SCR"); iport->fabric.flags |= FNIC_FDLS_RETRY_FRAME; goto err_out; @@ -1090,7 +1090,7 @@ static void fdls_send_scr(struct fnic_iport_s *iport) oxid = fdls_alloc_oxid(iport, FNIC_FRAME_TYPE_FABRIC_SCR, &iport->active_oxid_fabric_req); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send SCR", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1099,7 +1099,7 @@ static void fdls_send_scr(struct fnic_iport_s *iport) } FNIC_STD_SET_OX_ID(pscr->fchdr, oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send SCR with oxid: 0x%x", iport->fcid, oxid); @@ -1123,7 +1123,7 @@ static void fdls_send_gpn_ft(struct fnic_iport_s *iport, int fdls_state) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send GPN FT"); iport->fabric.flags |= FNIC_FDLS_RETRY_FRAME; goto err_out; @@ -1148,7 +1148,7 @@ static void fdls_send_gpn_ft(struct fnic_iport_s *iport, int fdls_state) &iport->active_oxid_fabric_req); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send GPN FT", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1157,7 +1157,7 @@ static void fdls_send_gpn_ft(struct fnic_iport_s *iport, int fdls_state) } FNIC_STD_SET_OX_ID(pgpn_ft->fchdr, oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send GPN FT with oxid: 0x%x", iport->fcid, oxid); @@ -1183,7 +1183,7 @@ fdls_send_tgt_adisc(struct fnic_iport_s *iport, struct fnic_tport_s *tport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send TGT ADISC"); tport->flags |= FNIC_FDLS_RETRY_FRAME; goto err_out; @@ -1203,7 +1203,7 @@ fdls_send_tgt_adisc(struct fnic_iport_s *iport, struct fnic_tport_s *tport) oxid = fdls_alloc_oxid(iport, FNIC_FRAME_TYPE_TGT_ADISC, &tport->active_oxid); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send TGT ADISC", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1222,7 +1222,7 @@ fdls_send_tgt_adisc(struct fnic_iport_s *iport, struct fnic_tport_s *tport) padisc->els.adisc_cmd = ELS_ADISC; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send ADISC to tgt fcid: 0x%x", iport->fcid, tport->fcid); @@ -1242,7 +1242,7 @@ bool fdls_delete_tport(struct fnic_iport_s *iport, struct fnic_tport_s *tport) if ((tport->state == FDLS_TGT_STATE_OFFLINING) || (tport->state == FDLS_TGT_STATE_OFFLINE)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport fcid 0x%x: tport state is offlining/offline\n", tport->fcid); return false; @@ -1257,7 +1257,7 @@ bool fdls_delete_tport(struct fnic_iport_s *iport, struct fnic_tport_s *tport) tport->flags |= FNIC_FDLS_TPORT_TERMINATING; if (tport->timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport fcid 0x%x: Canceling disc timer\n", tport->fcid); fnic_del_tport_timer_sync(fnic, tport); @@ -1272,9 +1272,9 @@ bool fdls_delete_tport(struct fnic_iport_s *iport, struct fnic_tport_s *tport) tport_del_evt = kzalloc_obj(struct fnic_tport_event_s, GFP_ATOMIC); if (!tport_del_evt) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, - "Failed to allocate memory for tport fcid: 0x%0x\n", - tport->fcid); + FNIC_FCS_DBG(KERN_INFO, fnic, + "iport: 0x%x tport 0x%x: Failed to allocate memory\n", + iport->fcid, tport->fcid); return false; } tport_del_evt->event = TGT_EV_RPORT_DEL; @@ -1282,9 +1282,9 @@ bool fdls_delete_tport(struct fnic_iport_s *iport, struct fnic_tport_s *tport) list_add_tail(&tport_del_evt->links, &fnic->tport_event_list); queue_work(fnic_event_queue, &fnic->tport_work); } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, - "tport 0x%x not reg with scsi_transport. Freeing locally", - tport->fcid); + FNIC_FCS_DBG(KERN_INFO, fnic, + "tport 0x%x not registered, freeing locally\n", + tport->fcid); list_del(&tport->links); kfree(tport); } @@ -1305,7 +1305,7 @@ fdls_send_tgt_plogi(struct fnic_iport_s *iport, struct fnic_tport_s *tport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send TGT PLOGI"); tport->flags |= FNIC_FDLS_RETRY_FRAME; goto err_out; @@ -1316,7 +1316,7 @@ fdls_send_tgt_plogi(struct fnic_iport_s *iport, struct fnic_tport_s *tport) oxid = fdls_alloc_oxid(iport, FNIC_FRAME_TYPE_TGT_PLOGI, &tport->active_oxid); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate oxid to send PLOGI to fcid: 0x%x", iport->fcid, tport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1330,7 +1330,7 @@ fdls_send_tgt_plogi(struct fnic_iport_s *iport, struct fnic_tport_s *tport) hton24(d_id, tport->fcid); FNIC_STD_SET_D_ID(pplogi->fchdr, d_id); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send tgt PLOGI to tgt: 0x%x with oxid: 0x%x", iport->fcid, tport->fcid, oxid); @@ -1353,7 +1353,7 @@ fnic_fc_plogi_rsp_rdf(struct fnic_iport_s *iport, be16_to_cpu(plogi_rsp->els.fl_cssp[2].cp_rdfs) & FNIC_FC_C3_RDF; struct fnic *fnic = iport->fnic; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "MFS: b2b_rdf_size: 0x%x spc3_rdf_size: 0x%x", b2b_rdf_size, spc3_rdf_size); @@ -1372,7 +1372,7 @@ static void fdls_send_register_fc4_types(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send RFT"); return; } @@ -1396,7 +1396,7 @@ static void fdls_send_register_fc4_types(struct fnic_iport_s *iport) &iport->active_oxid_fabric_req); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send RFT", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1404,15 +1404,18 @@ static void fdls_send_register_fc4_types(struct fnic_iport_s *iport) } FNIC_STD_SET_OX_ID(prft_id->fchdr, oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, - "0x%x: FDLS send RFT with oxid: 0x%x", iport->fcid, - oxid); prft_id->rft_id.fr_fts.ff_type_map[0] = cpu_to_be32(1 << FC_TYPE_FCP); prft_id->rft_id.fr_fts.ff_type_map[1] = cpu_to_be32(1 << (FC_TYPE_CT % FC_NS_BPW)); + FNIC_FCS_DBG(KERN_INFO, fnic, + "0x%x: FDLS send RFT 0x%08x 0x%08x 0x%08x with oxid: 0x%x", + iport->fcid, prft_id->rft_id.fr_fts.ff_type_map[0], + prft_id->rft_id.fr_fts.ff_type_map[1], + prft_id->rft_id.fr_fts.ff_type_map[2], + oxid); fnic_send_fcoe_frame(iport, frame, frame_size); @@ -1432,7 +1435,7 @@ static void fdls_send_register_fc4_features(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send RFF"); return; } @@ -1458,7 +1461,7 @@ static void fdls_send_register_fc4_features(struct fnic_iport_s *iport) &iport->active_oxid_fabric_req); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send RFF", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1466,12 +1469,13 @@ static void fdls_send_register_fc4_features(struct fnic_iport_s *iport) } FNIC_STD_SET_OX_ID(prff_id->fchdr, oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, - "0x%x: FDLS send RFF with oxid: 0x%x", iport->fcid, - oxid); - prff_id->rff_id.fr_type = FC_TYPE_FCP; + FNIC_FCS_DBG(KERN_INFO, fnic, + "0x%x: FDLS send RFF with oxid: 0x%x type 0%x feat 0%x", + iport->fcid, oxid, prff_id->rff_id.fr_type, + prff_id->rff_id.fr_feat); + fnic_send_fcoe_frame(iport, frame, frame_size); /* Even if fnic_send_fcoe_frame() fails we want to retry after timeout */ @@ -1493,7 +1497,7 @@ fdls_send_tgt_prli(struct fnic_iport_s *iport, struct fnic_tport_s *tport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send TGT PRLI"); tport->flags |= FNIC_FDLS_RETRY_FRAME; goto err_out; @@ -1513,7 +1517,7 @@ fdls_send_tgt_prli(struct fnic_iport_s *iport, struct fnic_tport_s *tport) oxid = fdls_alloc_oxid(iport, FNIC_FRAME_TYPE_TGT_PRLI, &tport->active_oxid); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "0x%x: Failed to allocate OXID to send TGT PRLI to 0x%x", iport->fcid, tport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1530,7 +1534,7 @@ fdls_send_tgt_prli(struct fnic_iport_s *iport, struct fnic_tport_s *tport) FNIC_STD_SET_S_ID(pprli->fchdr, s_id); FNIC_STD_SET_D_ID(pprli->fchdr, d_id); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send PRLI to tgt: 0x%x with oxid: 0x%x", iport->fcid, tport->fcid, oxid); @@ -1564,7 +1568,7 @@ void fdls_send_fabric_logo(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send fabric LOGO"); return; } @@ -1576,7 +1580,7 @@ void fdls_send_fabric_logo(struct fnic_iport_s *iport) &iport->active_oxid_fabric_req); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send fabric LOGO", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1589,7 +1593,7 @@ void fdls_send_fabric_logo(struct fnic_iport_s *iport) iport->fabric.flags &= ~FNIC_FDLS_FABRIC_ABORT_ISSUED; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send fabric LOGO with oxid: 0x%x", iport->fcid, oxid); @@ -1621,7 +1625,7 @@ void fdls_tgt_logout(struct fnic_iport_s *iport, struct fnic_tport_s *tport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send fabric LOGO"); return; } @@ -1631,7 +1635,7 @@ void fdls_tgt_logout(struct fnic_iport_s *iport, struct fnic_tport_s *tport) oxid = fdls_alloc_oxid(iport, FNIC_FRAME_TYPE_TGT_LOGO, &tport->active_oxid); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send tgt LOGO", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1642,7 +1646,7 @@ void fdls_tgt_logout(struct fnic_iport_s *iport, struct fnic_tport_s *tport) hton24(d_id, tport->fcid); FNIC_STD_SET_D_ID(plogo->fchdr, d_id); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send tgt LOGO with oxid: 0x%x", iport->fcid, oxid); @@ -1657,7 +1661,7 @@ static void fdls_tgt_discovery_start(struct fnic_iport_s *iport) u32 old_link_down_cnt = iport->fnic->link_down_cnt; struct fnic *fnic = iport->fnic; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Starting FDLS target discovery", iport->fcid); list_for_each_entry_safe(tport, next, &iport->tport_list, links) { @@ -1711,7 +1715,7 @@ static void fdls_target_restart_nexus(struct fnic_tport_s *tport) struct fnic *fnic = iport->fnic; bool retval = true; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport fcid: 0x%x state: %d restart_count: %d", tport->fcid, tport->state, tport->nexus_restart_count); @@ -1721,13 +1725,13 @@ static void fdls_target_restart_nexus(struct fnic_tport_s *tport) retval = fdls_delete_tport(iport, tport); if (retval != true) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Error deleting tport: 0x%x", fcid); return; } if (nexus_restart_count >= FNIC_TPORT_MAX_NEXUS_RESTART) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Exceeded nexus restart retries tport: 0x%x", fcid); return; @@ -1744,7 +1748,7 @@ static void fdls_target_restart_nexus(struct fnic_tport_s *tport) */ new_tport = fdls_create_tport(iport, fcid, wwpn); if (!new_tport) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Error creating new tport: 0x%x", fcid); return; } @@ -1773,12 +1777,12 @@ static struct fnic_tport_s *fdls_create_tport(struct fnic_iport_s *iport, struct fnic_tport_s *tport; struct fnic *fnic = iport->fnic; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDLS create tport: fcid: 0x%x wwpn: 0x%llx", fcid, wwpn); tport = kzalloc_obj(struct fnic_tport_s, GFP_ATOMIC); if (!tport) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Memory allocation failure while creating tport: 0x%x\n", fcid); return NULL; @@ -1791,12 +1795,12 @@ static struct fnic_tport_s *fdls_create_tport(struct fnic_iport_s *iport, tport->wwpn = wwpn; tport->iport = iport; - FNIC_FCS_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_DEBUG, fnic, "Need to setup tport timer callback"); timer_setup(&tport->retry_timer, fdls_tport_timer_callback, 0); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Added tport 0x%x", tport->fcid); fdls_set_tport_state(tport, FDLS_TGT_STATE_INIT); list_add_tail(&tport->links, &iport->tport_list); @@ -1847,7 +1851,7 @@ static void fdls_fdmi_register_hba(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send FDMI RHBA"); return; } @@ -1875,7 +1879,7 @@ static void fdls_fdmi_register_hba(struct fnic_iport_s *iport) &iport->active_oxid_fdmi_rhba); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send FDMI RHBA", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -1896,14 +1900,14 @@ static void fdls_fdmi_register_hba(struct fnic_iport_s *iport) fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_NODE_NAME, FNIC_FDMI_NN_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "NN set, off=%d", attr_off_bytes); strscpy_pad(data, FNIC_FDMI_MANUFACTURER, FNIC_FDMI_MANU_LEN); fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_MANUFACTURER, FNIC_FDMI_MANU_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "MFG set <%s>, off=%d", data, attr_off_bytes); err = vnic_dev_fw_info(fnic->vdev, &fw_info); @@ -1912,7 +1916,7 @@ static void fdls_fdmi_register_hba(struct fnic_iport_s *iport) FNIC_FDMI_SERIAL_LEN); fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_SERIAL_NUMBER, FNIC_FDMI_SERIAL_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "SERIAL set <%s>, off=%d", data, attr_off_bytes); } @@ -1923,21 +1927,21 @@ static void fdls_fdmi_register_hba(struct fnic_iport_s *iport) fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_MODEL, FNIC_FDMI_MODEL_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "MODEL set <%s>, off=%d", data, attr_off_bytes); strscpy_pad(data, FNIC_FDMI_MODEL_DESCRIPTION, FNIC_FDMI_MODEL_DES_LEN); fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_MODEL_DES, FNIC_FDMI_MODEL_DES_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "MODEL_DESC set <%s>, off=%d", data, attr_off_bytes); if (!err) { strscpy_pad(data, fw_info->hw_version, FNIC_FDMI_HW_VER_LEN); fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_HARDWARE_VERSION, FNIC_FDMI_HW_VER_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "HW_VER set <%s>, off=%d", data, attr_off_bytes); } @@ -1946,14 +1950,14 @@ static void fdls_fdmi_register_hba(struct fnic_iport_s *iport) fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_DRIVER_VERSION, FNIC_FDMI_DR_VER_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "DRV_VER set <%s>, off=%d", data, attr_off_bytes); strscpy_pad(data, "N/A", FNIC_FDMI_ROM_VER_LEN); fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_ROM_VERSION, FNIC_FDMI_ROM_VER_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "ROM_VER set <%s>, off=%d", data, attr_off_bytes); if (!err) { @@ -1961,14 +1965,14 @@ static void fdls_fdmi_register_hba(struct fnic_iport_s *iport) fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_FIRMWARE_VERSION, FNIC_FDMI_FW_VER_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FW_VER set <%s>, off=%d", data, attr_off_bytes); } len = sizeof(struct fc_std_fdmi_rhba) + attr_off_bytes; frame_size += len; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send FDMI RHBA with oxid: 0x%x fs: %d", iport->fcid, oxid, frame_size); @@ -1992,7 +1996,7 @@ static void fdls_fdmi_register_pa(struct fnic_iport_s *iport) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send FDMI RPA"); return; } @@ -2020,7 +2024,7 @@ static void fdls_fdmi_register_pa(struct fnic_iport_s *iport) &iport->active_oxid_fdmi_rpa); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Failed to allocate OXID to send FDMI RPA", iport->fcid); mempool_free(frame, fnic->frame_pool); @@ -2085,7 +2089,7 @@ static void fdls_fdmi_register_pa(struct fnic_iport_s *iport) fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_OS_NAME, FNIC_FDMI_OS_NAME_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "OS name set <%s>, off=%d", data, attr_off_bytes); sprintf(fc_host_system_hostname(fnic->host), "%s", utsname()->nodename); @@ -2094,13 +2098,13 @@ static void fdls_fdmi_register_pa(struct fnic_iport_s *iport) fnic_fdmi_attr_set(fdmi_attr, FNIC_FDMI_TYPE_HOST_NAME, FNIC_FDMI_HN_LEN, data, &attr_off_bytes); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Host name set <%s>, off=%d", data, attr_off_bytes); len = sizeof(struct fc_std_fdmi_rpa) + attr_off_bytes; frame_size += len; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send FDMI RPA with oxid: 0x%x fs: %d", iport->fcid, oxid, frame_size); @@ -2117,7 +2121,7 @@ void fdls_fabric_timer_callback(struct timer_list *t) struct fnic *fnic = iport->fnic; unsigned long flags; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tp: %d fab state: %d fab retry counter: %d max_flogi_retries: %d", iport->fabric.timer_pending, iport->fabric.state, iport->fabric.retry_counter, iport->max_flogi_retries); @@ -2132,7 +2136,7 @@ void fdls_fabric_timer_callback(struct timer_list *t) if (iport->fabric.del_timer_inprogress) { iport->fabric.del_timer_inprogress = 0; spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "fabric_del_timer inprogress(%d). Skip timer cb", iport->fabric.del_timer_inprogress); return; @@ -2160,7 +2164,7 @@ void fdls_fabric_timer_callback(struct timer_list *t) iport->fabric.flags &= ~FNIC_FDLS_FABRIC_ABORT_ISSUED; fdls_send_fabric_flogi(iport); } else - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Exceeded max FLOGI retries"); } break; @@ -2182,7 +2186,7 @@ void fdls_fabric_timer_callback(struct timer_list *t) iport->fabric.flags &= ~FNIC_FDLS_FABRIC_ABORT_ISSUED; fdls_send_fabric_plogi(iport); } else - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Exceeded max PLOGI retries"); } break; @@ -2213,7 +2217,7 @@ void fdls_fabric_timer_callback(struct timer_list *t) else { /* ABTS has timed out */ fdls_schedule_oxid_free(iport, &iport->active_oxid_fabric_req); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "ABTS timed out. Starting PLOGI: %p", iport); fnic_fdls_start_plogi(iport); } @@ -2230,7 +2234,7 @@ void fdls_fabric_timer_callback(struct timer_list *t) } else { /* ABTS has timed out */ fdls_schedule_oxid_free(iport, &iport->active_oxid_fabric_req); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "ABTS timed out. Starting PLOGI: %p", iport); fnic_fdls_start_plogi(iport); /* go back to fabric Plogi */ } @@ -2247,7 +2251,7 @@ void fdls_fabric_timer_callback(struct timer_list *t) else { /* ABTS has timed out */ fdls_schedule_oxid_free(iport, &iport->active_oxid_fabric_req); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "ABTS timed out. Starting PLOGI %p", iport); fnic_fdls_start_plogi(iport); /* go back to fabric Plogi */ } @@ -2269,7 +2273,7 @@ void fdls_fabric_timer_callback(struct timer_list *t) if (iport->fabric.retry_counter < FDLS_RETRY_COUNT) { fdls_send_gpn_ft(iport, iport->fabric.state); } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "ABTS timeout for fabric GPN_FT. Check name server: %p", iport); } @@ -2289,7 +2293,7 @@ void fdls_fdmi_retry_plogi(struct fnic_iport_s *iport) /* If max retries not exhausted, start over from fdmi plogi */ if (iport->fabric.fdmi_retry < FDLS_FDMI_MAX_RETRY) { iport->fabric.fdmi_retry++; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Retry FDMI PLOGI. FDMI retry: %d", iport->fabric.fdmi_retry); fdls_send_fdmi_plogi(iport); @@ -2307,7 +2311,7 @@ void fdls_fdmi_timer_callback(struct timer_list *t) spin_lock_irqsave(&fnic->fnic_lock, flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport->fabric.fdmi_pending: 0x%x\n", iport->fabric.fdmi_pending); if (!iport->fabric.fdmi_pending) { @@ -2315,7 +2319,7 @@ void fdls_fdmi_timer_callback(struct timer_list *t) spin_unlock_irqrestore(&fnic->fnic_lock, flags); return; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport->fabric.fdmi_pending: 0x%x\n", iport->fabric.fdmi_pending); /* if not abort pending, send an abort */ @@ -2324,7 +2328,7 @@ void fdls_fdmi_timer_callback(struct timer_list *t) spin_unlock_irqrestore(&fnic->fnic_lock, flags); return; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport->fabric.fdmi_pending: 0x%x\n", iport->fabric.fdmi_pending); /* ABTS pending for an active fdmi request that is pending. @@ -2332,29 +2336,36 @@ void fdls_fdmi_timer_callback(struct timer_list *t) * Schedule to free the OXID after 2*r_a_tov and proceed */ if (iport->fabric.fdmi_pending & FDLS_FDMI_PLOGI_PENDING) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDMI PLOGI ABTS timed out. Schedule oxid free: 0x%x\n", iport->active_oxid_fdmi_plogi); fdls_schedule_oxid_free(iport, &iport->active_oxid_fdmi_plogi); } else { if (iport->fabric.fdmi_pending & FDLS_FDMI_REG_HBA_PENDING) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDMI RHBA ABTS timed out. Schedule oxid free: 0x%x\n", iport->active_oxid_fdmi_rhba); fdls_schedule_oxid_free(iport, &iport->active_oxid_fdmi_rhba); } if (iport->fabric.fdmi_pending & FDLS_FDMI_RPA_PENDING) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDMI RPA ABTS timed out. Schedule oxid free: 0x%x\n", iport->active_oxid_fdmi_rpa); fdls_schedule_oxid_free(iport, &iport->active_oxid_fdmi_rpa); } } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport->fabric.fdmi_pending: 0x%x\n", iport->fabric.fdmi_pending); - fdls_fdmi_retry_plogi(iport); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + iport->fabric.fdmi_pending = 0; + /* If max retries not exhaused, start over from fdmi plogi */ + if (iport->fabric.fdmi_retry < FDLS_FDMI_MAX_RETRY) { + iport->fabric.fdmi_retry++; + FNIC_FCS_DBG(KERN_INFO, fnic, + "retry fdmi timer %d", iport->fabric.fdmi_retry); + fdls_send_fdmi_plogi(iport); + } + FNIC_FCS_DBG(KERN_INFO, fnic, "iport->fabric.fdmi_pending: 0x%x\n", iport->fabric.fdmi_pending); spin_unlock_irqrestore(&fnic->fnic_lock, flags); } @@ -2367,7 +2378,7 @@ static void fdls_send_delete_tport_msg(struct fnic_tport_s *tport) tport_del_evt = kzalloc_obj(struct fnic_tport_event_s, GFP_ATOMIC); if (!tport_del_evt) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Failed to allocate memory for tport event fcid: 0x%x", tport->fcid); return; @@ -2400,13 +2411,13 @@ static void fdls_tport_timer_callback(struct timer_list *t) if (tport->del_timer_inprogress) { tport->del_timer_inprogress = 0; spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport_del_timer inprogress. Skip timer cb tport fcid: 0x%x\n", tport->fcid); return; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport fcid: 0x%x timer pending: %d state: %d retry counter: %d", tport->fcid, tport->timer_pending, tport->state, tport->retry_counter); @@ -2467,15 +2478,16 @@ static void fdls_tport_timer_callback(struct timer_list *t) } else { /* exceeded retry count */ fdls_schedule_oxid_free(iport, &tport->active_oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "ADISC not responding. Deleting target port: 0x%x", tport->fcid); fdls_send_delete_tport_msg(tport); } break; default: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, - "oxid: 0x%x Unknown tport state: 0x%x", oxid, tport->state); + FNIC_FCS_DBG(KERN_INFO, fnic, + "0x%x timeout tport 0x%x oxid 0x%x state %d\n", + iport->fcid, tport->fcid, oxid, tport->state); break; } spin_unlock_irqrestore(&fnic->fnic_lock, flags); @@ -2524,26 +2536,26 @@ fdls_process_tgt_adisc_rsp(struct fnic_iport_s *iport, tport = fnic_find_tport_by_fcid(iport, tgt_fcid); if (!tport) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Tgt ADISC response tport not found: 0x%x", tgt_fcid); return; } if ((iport->state != FNIC_IPORT_STATE_READY) || (tport->state != FDLS_TGT_STATE_ADISC) || (tport->flags & FNIC_FDLS_TGT_ABORT_ISSUED)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Dropping this ADISC response"); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport state: %d tport state: %d Is abort issued on PRLI? %d", iport->state, tport->state, (tport->flags & FNIC_FDLS_TGT_ABORT_ISSUED)); return; } if (FNIC_STD_GET_OX_ID(fchdr) != tport->active_oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Dropping frame from target: 0x%x", tgt_fcid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Reason: Stale ADISC/Aborted ADISC/OOO frame delivery"); return; } @@ -2555,7 +2567,7 @@ fdls_process_tgt_adisc_rsp(struct fnic_iport_s *iport, case ELS_LS_ACC: atomic64_inc(&iport->iport_stats.tport_adisc_ls_accepts); if (tport->timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport 0x%p Canceling fabric disc timer\n", tport); fnic_del_tport_timer_sync(fnic, tport); @@ -2565,12 +2577,12 @@ fdls_process_tgt_adisc_rsp(struct fnic_iport_s *iport, frame_wwnn = get_unaligned_be64(&adisc_rsp->els.adisc_wwnn); frame_wwpn = get_unaligned_be64(&adisc_rsp->els.adisc_wwpn); if ((frame_wwnn == tport->wwnn) && (frame_wwpn == tport->wwpn)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "ADISC accepted from target: 0x%x. Target logged in", tgt_fcid); fdls_set_tport_state(tport, FDLS_TGT_STATE_READY); } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Error mismatch frame: ADISC"); } break; @@ -2580,14 +2592,14 @@ fdls_process_tgt_adisc_rsp(struct fnic_iport_s *iport, if (((els_rjt->rej.er_reason == ELS_RJT_BUSY) || (els_rjt->rej.er_reason == ELS_RJT_UNAB)) && (tport->retry_counter < FDLS_RETRY_COUNT)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "ADISC ret ELS_LS_RJT BUSY. Retry from timer routine: 0x%x", tgt_fcid); /* Retry ADISC again from the timer routine. */ tport->flags |= FNIC_FDLS_RETRY_FRAME; } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "ADISC returned ELS_LS_RJT from target: 0x%x", tgt_fcid); fdls_delete_tport(iport, tport); @@ -2611,33 +2623,33 @@ fdls_process_tgt_plogi_rsp(struct fnic_iport_s *iport, fcid = FNIC_STD_GET_S_ID(fchdr); tgt_fcid = ntoh24(fcid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDLS processing target PLOGI response: tgt_fcid: 0x%x", tgt_fcid); tport = fnic_find_tport_by_fcid(iport, tgt_fcid); if (!tport) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport not found: 0x%x", tgt_fcid); return; } if ((iport->state != FNIC_IPORT_STATE_READY) || (tport->flags & FNIC_FDLS_TGT_ABORT_ISSUED)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Dropping frame! iport state: %d tport state: %d", iport->state, tport->state); return; } if (tport->state != FDLS_TGT_STATE_PLOGI) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PLOGI rsp recvd in wrong state. Drop the frame and restart nexus"); fdls_target_restart_nexus(tport); return; } if (FNIC_STD_GET_OX_ID(fchdr) != tport->active_oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PLOGI response from target: 0x%x. Dropping frame", tgt_fcid); return; @@ -2649,7 +2661,7 @@ fdls_process_tgt_plogi_rsp(struct fnic_iport_s *iport, switch (plogi_rsp->els.fl_cmd) { case ELS_LS_ACC: atomic64_inc(&iport->iport_stats.tport_plogi_ls_accepts); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PLOGI accepted by target: 0x%x", tgt_fcid); break; @@ -2658,14 +2670,14 @@ fdls_process_tgt_plogi_rsp(struct fnic_iport_s *iport, if (((els_rjt->rej.er_reason == ELS_RJT_BUSY) || (els_rjt->rej.er_reason == ELS_RJT_UNAB)) && (tport->retry_counter < iport->max_plogi_retries)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PLOGI ret ELS_LS_RJT BUSY. Retry from timer routine: 0x%x", tgt_fcid); /* Retry plogi again from the timer routine. */ tport->flags |= FNIC_FDLS_RETRY_FRAME; return; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PLOGI returned ELS_LS_RJT from target: 0x%x", tgt_fcid); fdls_delete_tport(iport, tport); @@ -2673,18 +2685,18 @@ fdls_process_tgt_plogi_rsp(struct fnic_iport_s *iport, default: atomic64_inc(&iport->iport_stats.tport_plogi_misc_rejects); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PLOGI not accepted from target fcid: 0x%x", tgt_fcid); return; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Found the PLOGI target: 0x%x and state: %d", (unsigned int) tgt_fcid, tport->state); if (tport->timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport fcid 0x%x: Canceling disc timer\n", tport->fcid); fnic_del_tport_timer_sync(fnic, tport); @@ -2702,13 +2714,13 @@ fdls_process_tgt_plogi_rsp(struct fnic_iport_s *iport, min(max_payload_size, iport->max_payload_size); if (tport->max_payload_size < FNIC_MIN_DATA_FIELD_SIZE) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "MFS: tport max frame size below spec bounds: %d", tport->max_payload_size); tport->max_payload_size = FNIC_MIN_DATA_FIELD_SIZE; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "MAX frame size: %u iport max_payload_size: %d tport mfs: %d", max_payload_size, iport->max_payload_size, tport->max_payload_size); @@ -2736,12 +2748,12 @@ fdls_process_tgt_prli_rsp(struct fnic_iport_s *iport, fcid = FNIC_STD_GET_S_ID(fchdr); tgt_fcid = ntoh24(fcid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDLS process tgt PRLI response: 0x%x", tgt_fcid); tport = fnic_find_tport_by_fcid(iport, tgt_fcid); if (!tport) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport not found: 0x%x", tgt_fcid); /* Handle or just drop? */ return; @@ -2749,24 +2761,24 @@ fdls_process_tgt_prli_rsp(struct fnic_iport_s *iport, if ((iport->state != FNIC_IPORT_STATE_READY) || (tport->flags & FNIC_FDLS_TGT_ABORT_ISSUED)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Dropping frame! iport st: %d tport st: %d tport fcid: 0x%x", iport->state, tport->state, tport->fcid); return; } if (tport->state != FDLS_TGT_STATE_PRLI) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PRLI rsp recvd in wrong state. Drop frame. Restarting nexus"); fdls_target_restart_nexus(tport); return; } if (FNIC_STD_GET_OX_ID(fchdr) != tport->active_oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Dropping PRLI response from target: 0x%x ", tgt_fcid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Reason: Stale PRLI response/Aborted PDISC/OOO frame delivery"); return; } @@ -2777,11 +2789,11 @@ fdls_process_tgt_prli_rsp(struct fnic_iport_s *iport, switch (prli_rsp->els_prli.prli_cmd) { case ELS_LS_ACC: atomic64_inc(&iport->iport_stats.tport_prli_ls_accepts); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PRLI accepted from target: 0x%x", tgt_fcid); if (prli_rsp->sp.spp_type != FC_FC4_TYPE_SCSI) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "mismatched target zoned with FC SCSI initiator: 0x%x", tgt_fcid); mismatched_tgt = true; @@ -2798,7 +2810,7 @@ fdls_process_tgt_prli_rsp(struct fnic_iport_s *iport, || (els_rjt->rej.er_reason == ELS_RJT_UNAB)) && (tport->retry_counter < FDLS_RETRY_COUNT)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PRLI ret ELS_LS_RJT BUSY. Retry from timer routine: 0x%x", tgt_fcid); @@ -2806,7 +2818,7 @@ fdls_process_tgt_prli_rsp(struct fnic_iport_s *iport, tport->flags |= FNIC_FDLS_RETRY_FRAME; return; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PRLI returned ELS_LS_RJT from target: 0x%x", tgt_fcid); @@ -2815,17 +2827,17 @@ fdls_process_tgt_prli_rsp(struct fnic_iport_s *iport, return; default: atomic64_inc(&iport->iport_stats.tport_prli_misc_rejects); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PRLI not accepted from target: 0x%x", tgt_fcid); return; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Found the PRLI target: 0x%x and state: %d", (unsigned int) tgt_fcid, tport->state); if (tport->timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport fcid 0x%x: Canceling disc timer\n", tport->fcid); fnic_del_tport_timer_sync(fnic, tport); @@ -2841,7 +2853,7 @@ fdls_process_tgt_prli_rsp(struct fnic_iport_s *iport, /* Check if the device plays Target Mode Function */ if (!(tport->fcp_csp & FCP_PRLI_FUNC_TARGET)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Remote port(0x%x): no target support. Deleting it\n", tgt_fcid); fdls_tgt_logout(iport, tport); @@ -2854,16 +2866,16 @@ fdls_process_tgt_prli_rsp(struct fnic_iport_s *iport, /* Inform the driver about new target added */ tport_add_evt = kzalloc_obj(struct fnic_tport_event_s, GFP_ATOMIC); if (!tport_add_evt) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, - "tport event memory allocation failure: 0x%0x\n", - tport->fcid); + FNIC_FCS_DBG(KERN_INFO, fnic, + "iport fcid: 0x%x tport event memory allocation failure: 0x%0x\n", + iport->fcid, tport->fcid); return; } tport_add_evt->event = TGT_EV_RPORT_ADD; tport_add_evt->arg1 = (void *) tport; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, - "iport fcid: 0x%x add tport event fcid: 0x%x\n", - tport->fcid, iport->fcid); + FNIC_FCS_DBG(KERN_INFO, fnic, + "iport fcid: 0x%x add tport event fcid: 0x%x\n", + tport->fcid, iport->fcid); list_add_tail(&tport_add_evt->links, &fnic->tport_event_list); queue_work(fnic_event_queue, &fnic->tport_work); } @@ -2881,21 +2893,21 @@ fdls_process_rff_id_rsp(struct fnic_iport_s *iport, uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr); if (fdls_get_state(fdls) != FDLS_STATE_REGISTER_FC4_FEATURES) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RFF_ID resp recvd in state(%d). Dropping.", fdls_get_state(fdls)); return; } if (iport->active_oxid_fabric_req != oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Incorrect OXID in response. state: %d, oxid recvd: 0x%x, active oxid: 0x%x\n", fdls_get_state(fdls), oxid, iport->active_oxid_fabric_req); return; } rsp = FNIC_STD_GET_FC_CT_CMD((&rff_rsp->fc_std_ct_hdr)); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS process RFF ID response: 0x%04x", iport->fcid, (uint32_t) rsp); @@ -2904,7 +2916,7 @@ fdls_process_rff_id_rsp(struct fnic_iport_s *iport, switch (rsp) { case FC_FS_ACC: if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Canceling fabric disc timer %p\n", iport); fnic_del_fabric_timer_sync(fnic); } @@ -2918,18 +2930,18 @@ fdls_process_rff_id_rsp(struct fnic_iport_s *iport, if (((reason_code == FC_FS_RJT_BSY) || (reason_code == FC_FS_RJT_UNABL)) && (fdls->retry_counter < FDLS_RETRY_COUNT)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RFF_ID ret ELS_LS_RJT BUSY. Retry from timer routine %p", iport); /* Retry again from the timer routine */ fdls->flags |= FNIC_FDLS_RETRY_FRAME; } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RFF_ID returned ELS_LS_RJT. Halting discovery %p", iport); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Canceling fabric disc timer %p\n", iport); fnic_del_fabric_timer_sync(fnic); } @@ -2954,14 +2966,14 @@ fdls_process_rft_id_rsp(struct fnic_iport_s *iport, uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr); if (fdls_get_state(fdls) != FDLS_STATE_REGISTER_FC4_TYPES) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RFT_ID resp recvd in state(%d). Dropping.", fdls_get_state(fdls)); return; } if (iport->active_oxid_fabric_req != oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Incorrect OXID in response. state: %d, oxid recvd: 0x%x, active oxid: 0x%x\n", fdls_get_state(fdls), oxid, iport->active_oxid_fabric_req); return; @@ -2969,7 +2981,7 @@ fdls_process_rft_id_rsp(struct fnic_iport_s *iport, rsp = FNIC_STD_GET_FC_CT_CMD((&rft_rsp->fc_std_ct_hdr)); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS process RFT ID response: 0x%04x", iport->fcid, (uint32_t) rsp); @@ -2978,7 +2990,7 @@ fdls_process_rft_id_rsp(struct fnic_iport_s *iport, switch (rsp) { case FC_FS_ACC: if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Canceling fabric disc timer %p\n", iport); fnic_del_fabric_timer_sync(fnic); } @@ -2992,19 +3004,19 @@ fdls_process_rft_id_rsp(struct fnic_iport_s *iport, if (((reason_code == FC_FS_RJT_BSY) || (reason_code == FC_FS_RJT_UNABL)) && (fdls->retry_counter < FDLS_RETRY_COUNT)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: RFT_ID ret ELS_LS_RJT BUSY. Retry from timer routine", iport->fcid); /* Retry again from the timer routine */ fdls->flags |= FNIC_FDLS_RETRY_FRAME; } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: RFT_ID REJ. Halting discovery reason %d expl %d", iport->fcid, reason_code, rft_rsp->fc_std_ct_hdr.ct_explan); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Canceling fabric disc timer %p\n", iport); fnic_del_fabric_timer_sync(fnic); } @@ -3029,20 +3041,20 @@ fdls_process_rpn_id_rsp(struct fnic_iport_s *iport, uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr); if (fdls_get_state(fdls) != FDLS_STATE_RPN_ID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RPN_ID resp recvd in state(%d). Dropping.", fdls_get_state(fdls)); return; } if (iport->active_oxid_fabric_req != oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Incorrect OXID in response. state: %d, oxid recvd: 0x%x, active oxid: 0x%x\n", fdls_get_state(fdls), oxid, iport->active_oxid_fabric_req); return; } rsp = FNIC_STD_GET_FC_CT_CMD((&rpn_rsp->fc_std_ct_hdr)); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS process RPN ID response: 0x%04x", iport->fcid, (uint32_t) rsp); fdls_free_oxid(iport, oxid, &iport->active_oxid_fabric_req); @@ -3050,7 +3062,7 @@ fdls_process_rpn_id_rsp(struct fnic_iport_s *iport, switch (rsp) { case FC_FS_ACC: if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Canceling fabric disc timer %p\n", iport); fnic_del_fabric_timer_sync(fnic); } @@ -3064,17 +3076,17 @@ fdls_process_rpn_id_rsp(struct fnic_iport_s *iport, if (((reason_code == FC_FS_RJT_BSY) || (reason_code == FC_FS_RJT_UNABL)) && (fdls->retry_counter < FDLS_RETRY_COUNT)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RPN_ID returned REJ BUSY. Retry from timer routine %p", iport); /* Retry again from the timer routine */ fdls->flags |= FNIC_FDLS_RETRY_FRAME; } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RPN_ID ELS_LS_RJT. Halting discovery %p", iport); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Canceling fabric disc timer %p\n", iport); fnic_del_fabric_timer_sync(fnic); } @@ -3097,18 +3109,18 @@ fdls_process_scr_rsp(struct fnic_iport_s *iport, struct fnic *fnic = iport->fnic; uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDLS process SCR response: 0x%04x", (uint32_t) scr_rsp->scr.scr_cmd); if (fdls_get_state(fdls) != FDLS_STATE_SCR) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "SCR resp recvd in state(%d). Dropping.", fdls_get_state(fdls)); return; } if (iport->active_oxid_fabric_req != oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Incorrect OXID in response. state: %d, oxid recvd: 0x%x, active oxid: 0x%x\n", fdls_get_state(fdls), oxid, iport->active_oxid_fabric_req); } @@ -3119,7 +3131,7 @@ fdls_process_scr_rsp(struct fnic_iport_s *iport, case ELS_LS_ACC: atomic64_inc(&iport->iport_stats.fabric_scr_ls_accepts); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Canceling fabric disc timer %p\n", iport); fnic_del_fabric_timer_sync(fnic); } @@ -3133,17 +3145,17 @@ fdls_process_scr_rsp(struct fnic_iport_s *iport, if (((els_rjt->rej.er_reason == ELS_RJT_BUSY) || (els_rjt->rej.er_reason == ELS_RJT_UNAB)) && (fdls->retry_counter < FDLS_RETRY_COUNT)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "SCR ELS_LS_RJT BUSY. Retry from timer routine %p", iport); /* Retry again from the timer routine */ fdls->flags |= FNIC_FDLS_RETRY_FRAME; } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "SCR returned ELS_LS_RJT. Halting discovery %p", iport); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Canceling fabric disc timer %p\n", iport); fnic_del_fabric_timer_sync(fnic); @@ -3171,7 +3183,7 @@ fdls_process_gpn_ft_tgt_list(struct fnic_iport_s *iport, u32 old_link_down_cnt = iport->fnic->link_down_cnt; struct fnic *fnic = iport->fnic; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS process GPN_FT tgt list", iport->fcid); gpn_ft_tgt = @@ -3185,7 +3197,7 @@ fdls_process_gpn_ft_tgt_list(struct fnic_iport_s *iport, fcid = ntoh24(gpn_ft_tgt->fcid); wwpn = be64_to_cpu(gpn_ft_tgt->wwpn); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "tport: 0x%x: ctrl:0x%x", fcid, gpn_ft_tgt->ctrl); if (fcid == iport->fcid) { @@ -3232,7 +3244,7 @@ fdls_process_gpn_ft_tgt_list(struct fnic_iport_s *iport, rem_len -= sizeof(struct fc_gpn_ft_rsp_iu); } if (rem_len <= 0) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "GPN_FT response: malformed/corrupt frame rxlen: %d remlen: %d", len, rem_len); } @@ -3242,7 +3254,7 @@ fdls_process_gpn_ft_tgt_list(struct fnic_iport_s *iport, list_for_each_entry_safe(tport, next, &iport->tport_list, links) { if (!(tport->flags & FNIC_FDLS_TPORT_IN_GPN_FT_LIST)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Remove port: 0x%x not found in GPN_FT list", tport->fcid); fdls_delete_tport(iport, tport); @@ -3271,7 +3283,7 @@ fdls_process_gpn_ft_rsp(struct fnic_iport_s *iport, struct fnic *fnic = iport->fnic; uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDLS process GPN_FT response: iport state: %d len: %d", iport->state, len); @@ -3291,14 +3303,14 @@ fdls_process_gpn_ft_rsp(struct fnic_iport_s *iport, && ((fdls_get_state(fdls) == FDLS_STATE_RSCN_GPN_FT) || (fdls_get_state(fdls) == FDLS_STATE_SEND_GPNFT) || (fdls_get_state(fdls) == FDLS_STATE_TGT_DISCOVERY))))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "GPNFT resp recvd in fab state(%d) iport_state(%d). Dropping.", fdls_get_state(fdls), iport->state); return; } if (iport->active_oxid_fabric_req != oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Incorrect OXID in response. state: %d, oxid recvd: 0x%x, active oxid: 0x%x\n", fdls_get_state(fdls), oxid, iport->active_oxid_fabric_req); } @@ -3311,10 +3323,10 @@ fdls_process_gpn_ft_rsp(struct fnic_iport_s *iport, switch (rsp) { case FC_FS_ACC: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: GPNFT_RSP accept", iport->fcid); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Canceling fabric disc timer\n", iport->fcid); fnic_del_fabric_timer_sync(fnic); @@ -3329,7 +3341,7 @@ fdls_process_gpn_ft_rsp(struct fnic_iport_s *iport, * that will be taken care in next link up event */ if (iport->state != FNIC_IPORT_STATE_READY) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Halting target discovery: fab st: %d iport st: %d ", fdls_get_state(fdls), iport->state); break; @@ -3339,22 +3351,22 @@ fdls_process_gpn_ft_rsp(struct fnic_iport_s *iport, case FC_FS_RJT: reason_code = gpn_ft_rsp->fc_std_ct_hdr.ct_reason; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: GPNFT_RSP Reject reason: %d", iport->fcid, reason_code); if (((reason_code == FC_FS_RJT_BSY) || (reason_code == FC_FS_RJT_UNABL)) && (fdls->retry_counter < FDLS_RETRY_COUNT)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: GPNFT_RSP ret REJ/BSY. Retry from timer routine", iport->fcid); /* Retry again from the timer routine */ fdls->flags |= FNIC_FDLS_RETRY_FRAME; } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: GPNFT_RSP reject", iport->fcid); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Canceling fabric disc timer\n", iport->fcid); fnic_del_fabric_timer_sync(fnic); @@ -3368,7 +3380,7 @@ fdls_process_gpn_ft_rsp(struct fnic_iport_s *iport, count = 0; list_for_each_entry_safe(tport, next, &iport->tport_list, links) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "GPN_FT_REJECT: Remove port: 0x%x", tport->fcid); fdls_delete_tport(iport, tport); @@ -3378,7 +3390,7 @@ fdls_process_gpn_ft_rsp(struct fnic_iport_s *iport, } count++; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "GPN_FT_REJECT: Removed (0x%x) ports", count); } break; @@ -3403,7 +3415,7 @@ fdls_process_fabric_logo_rsp(struct fnic_iport_s *iport, uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr); if (iport->active_oxid_fabric_req != oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Incorrect OXID in response. state: %d, oxid recvd: 0x%x, active oxid: 0x%x\n", fdls_get_state(fdls), oxid, iport->active_oxid_fabric_req); } @@ -3412,7 +3424,7 @@ fdls_process_fabric_logo_rsp(struct fnic_iport_s *iport, switch (flogo_rsp->els.fl_cmd) { case ELS_LS_ACC: if (iport->fabric.state != FDLS_STATE_FABRIC_LOGO) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Flogo response. Fabric not in LOGO state. Dropping! %p", iport); return; @@ -3422,25 +3434,25 @@ fdls_process_fabric_logo_rsp(struct fnic_iport_s *iport, iport->state = FNIC_IPORT_STATE_LINK_WAIT; if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport 0x%p Canceling fabric disc timer\n", iport); fnic_del_fabric_timer_sync(fnic); } iport->fabric.timer_pending = 0; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Flogo response from Fabric for did: 0x%x", ntoh24(fchdr->fh_d_id)); return; case ELS_LS_RJT: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Flogo response from Fabric for did: 0x%x returned ELS_LS_RJT", ntoh24(fchdr->fh_d_id)); return; default: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FLOGO response not accepted or rejected: 0x%x", flogo_rsp->els.fl_cmd); } @@ -3458,17 +3470,17 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, struct fnic *fnic = iport->fnic; uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS processing FLOGI response", iport->fcid); if (fdls_get_state(fabric) != FDLS_STATE_FABRIC_FLOGI) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FLOGI response received in state (%d). Dropping frame", fdls_get_state(fabric)); return; } if (iport->active_oxid_fabric_req != oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Incorrect OXID in response. state: %d, oxid recvd: 0x%x, active oxid: 0x%x\n", fdls_get_state(fabric), oxid, iport->active_oxid_fabric_req); return; @@ -3480,7 +3492,7 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, case ELS_LS_ACC: atomic64_inc(&iport->iport_stats.fabric_flogi_ls_accepts); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport fcid: 0x%x Canceling fabric disc timer\n", iport->fcid); fnic_del_fabric_timer_sync(fnic); @@ -3490,7 +3502,7 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, iport->fabric.retry_counter = 0; fcid = FNIC_STD_GET_D_ID(fchdr); iport->fcid = ntoh24(fcid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FLOGI response accepted", iport->fcid); /* Learn the Service Params */ @@ -3500,7 +3512,7 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, iport->max_payload_size = min(rdf_size, iport->max_payload_size); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "max_payload_size from fabric: %u set: %d", rdf_size, iport->max_payload_size); @@ -3510,7 +3522,7 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, if (FNIC_LOGI_FEATURES(flogi_rsp->els) & FNIC_FC_EDTOV_NSEC) iport->e_d_tov = iport->e_d_tov / FNIC_NSEC_TO_MSEC; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "From fabric: R_A_TOV: %d E_D_TOV: %d", iport->r_a_tov, iport->e_d_tov); @@ -3521,13 +3533,13 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, fnic_fdls_learn_fcoe_macs(iport, rx_frame, fcid); if (fnic_fdls_register_portid(iport, iport->fcid, rx_frame) != 0) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FLOGI registration failed", iport->fcid); break; } memcpy(&fcmac[3], fcid, 3); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Adding vNIC device MAC addr: %02x:%02x:%02x:%02x:%02x:%02x", fcmac[0], fcmac[1], fcmac[2], fcmac[3], fcmac[4], fcmac[5]); @@ -3535,7 +3547,7 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, if (fdls_get_state(fabric) == FDLS_STATE_FABRIC_FLOGI) { fnic_fdls_start_plogi(iport); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FLOGI response received. Starting PLOGI"); } else { /* From FDLS_STATE_FABRIC_FLOGI state fabric can only go to @@ -3543,7 +3555,7 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, * state, hence we don't have to worry about undoing: * the fnic_fdls_register_portid and vnic_dev_add_addr */ - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FLOGI response received in state (%d). Dropping frame", fdls_get_state(fabric)); } @@ -3552,7 +3564,7 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, case ELS_LS_RJT: atomic64_inc(&iport->iport_stats.fabric_flogi_ls_rejects); if (fabric->retry_counter < iport->max_flogi_retries) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FLOGI returned ELS_LS_RJT BUSY. Retry from timer routine %p", iport); @@ -3560,11 +3572,11 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, fabric->flags |= FNIC_FDLS_RETRY_FRAME; } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FLOGI returned ELS_LS_RJT. Halting discovery %p", iport); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport 0x%p Canceling fabric disc timer\n", iport); fnic_del_fabric_timer_sync(fnic); @@ -3575,7 +3587,7 @@ fdls_process_flogi_rsp(struct fnic_iport_s *iport, break; default: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FLOGI response not accepted: 0x%x", flogi_rsp->els.fl_cmd); atomic64_inc(&iport->iport_stats.fabric_flogi_misc_rejects); @@ -3594,13 +3606,13 @@ fdls_process_fabric_plogi_rsp(struct fnic_iport_s *iport, uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr); if (fdls_get_state((&iport->fabric)) != FDLS_STATE_FABRIC_PLOGI) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Fabric PLOGI response received in state (%d). Dropping frame", fdls_get_state(&iport->fabric)); return; } if (iport->active_oxid_fabric_req != oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Incorrect OXID in response. state: %d, oxid recvd: 0x%x, active oxid: 0x%x\n", fdls_get_state(fdls), oxid, iport->active_oxid_fabric_req); return; @@ -3611,7 +3623,7 @@ fdls_process_fabric_plogi_rsp(struct fnic_iport_s *iport, case ELS_LS_ACC: atomic64_inc(&iport->iport_stats.fabric_plogi_ls_accepts); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport fcid: 0x%x fabric PLOGI response: Accepted\n", iport->fcid); fnic_del_fabric_timer_sync(fnic); @@ -3626,15 +3638,15 @@ fdls_process_fabric_plogi_rsp(struct fnic_iport_s *iport, if (((els_rjt->rej.er_reason == ELS_RJT_BUSY) || (els_rjt->rej.er_reason == ELS_RJT_UNAB)) && (iport->fabric.retry_counter < iport->max_plogi_retries)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Fabric PLOGI ELS_LS_RJT BUSY. Retry from timer routine", iport->fcid); } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Fabric PLOGI ELS_LS_RJT. Halting discovery", iport->fcid); if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport fcid: 0x%x Canceling fabric disc timer\n", iport->fcid); fnic_del_fabric_timer_sync(fnic); @@ -3645,7 +3657,7 @@ fdls_process_fabric_plogi_rsp(struct fnic_iport_s *iport, } break; default: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PLOGI response not accepted: 0x%x", plogi_rsp->els.fl_cmd); atomic64_inc(&iport->iport_stats.fabric_plogi_misc_rejects); @@ -3664,7 +3676,7 @@ static void fdls_process_fdmi_plogi_rsp(struct fnic_iport_s *iport, uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr); if (iport->active_oxid_fdmi_plogi != oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Incorrect OXID in response. state: %d, oxid recvd: 0x%x, active oxid: 0x%x\n", fdls_get_state(fdls), oxid, iport->active_oxid_fdmi_plogi); return; @@ -3678,9 +3690,9 @@ static void fdls_process_fdmi_plogi_rsp(struct fnic_iport_s *iport, iport->fabric.fdmi_pending = 0; switch (plogi_rsp->els.fl_cmd) { case ELS_LS_ACC: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDLS process fdmi PLOGI response status: ELS_LS_ACC\n"); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Sending fdmi registration for port 0x%x\n", iport->fcid); @@ -3691,7 +3703,7 @@ static void fdls_process_fdmi_plogi_rsp(struct fnic_iport_s *iport, round_jiffies(fdmi_tov)); break; case ELS_LS_RJT: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Fabric FDMI PLOGI returned ELS_LS_RJT reason: 0x%x", els_rjt->rej.er_reason); @@ -3715,7 +3727,7 @@ static void fdls_process_fdmi_reg_ack(struct fnic_iport_s *iport, uint16_t oxid; if (!iport->fabric.fdmi_pending) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Received FDMI ack while not waiting: 0x%x\n", FNIC_STD_GET_OX_ID(fchdr)); return; @@ -3725,7 +3737,7 @@ static void fdls_process_fdmi_reg_ack(struct fnic_iport_s *iport, if ((iport->active_oxid_fdmi_rhba != oxid) && (iport->active_oxid_fdmi_rpa != oxid)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Incorrect OXID in response. oxid recvd: 0x%x, active oxids(rhba,rpa): 0x%x, 0x%x\n", oxid, iport->active_oxid_fdmi_rhba, iport->active_oxid_fdmi_rpa); return; @@ -3738,13 +3750,13 @@ static void fdls_process_fdmi_reg_ack(struct fnic_iport_s *iport, fdls_free_oxid(iport, oxid, &iport->active_oxid_fdmi_rpa); } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport fcid: 0x%x: Received FDMI registration ack\n", iport->fcid); if (!iport->fabric.fdmi_pending) { timer_delete_sync(&iport->fabric.fdmi_timer); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport fcid: 0x%x: Canceling FDMI timer\n", iport->fcid); } @@ -3760,7 +3772,7 @@ static void fdls_process_fdmi_abts_rsp(struct fnic_iport_s *iport, s_id = ntoh24(FNIC_STD_GET_S_ID(fchdr)); if (!(s_id != FC_FID_MGMT_SERV)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received abts rsp with invalid SID: 0x%x. Dropping frame", s_id); return; @@ -3770,23 +3782,23 @@ static void fdls_process_fdmi_abts_rsp(struct fnic_iport_s *iport, switch (FNIC_FRAME_TYPE(oxid)) { case FNIC_FRAME_TYPE_FDMI_PLOGI: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received FDMI PLOGI ABTS rsp with oxid: 0x%x", oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: iport->fabric.fdmi_pending: 0x%x", iport->fcid, iport->fabric.fdmi_pending); fdls_free_oxid(iport, oxid, &iport->active_oxid_fdmi_plogi); iport->fabric.fdmi_pending &= ~FDLS_FDMI_PLOGI_PENDING; iport->fabric.fdmi_pending &= ~FDLS_FDMI_ABORT_PENDING; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: iport->fabric.fdmi_pending: 0x%x", iport->fcid, iport->fabric.fdmi_pending); break; case FNIC_FRAME_TYPE_FDMI_RHBA: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received FDMI RHBA ABTS rsp with oxid: 0x%x", oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: iport->fabric.fdmi_pending: 0x%x", iport->fcid, iport->fabric.fdmi_pending); @@ -3800,14 +3812,14 @@ static void fdls_process_fdmi_abts_rsp(struct fnic_iport_s *iport, iport->fabric.fdmi_pending &= ~FDLS_FDMI_ABORT_PENDING; fdls_free_oxid(iport, oxid, &iport->active_oxid_fdmi_rhba); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: iport->fabric.fdmi_pending: 0x%x", iport->fcid, iport->fabric.fdmi_pending); break; case FNIC_FRAME_TYPE_FDMI_RPA: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received FDMI RPA ABTS rsp with oxid: 0x%x", oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: iport->fabric.fdmi_pending: 0x%x", iport->fcid, iport->fabric.fdmi_pending); @@ -3821,12 +3833,12 @@ static void fdls_process_fdmi_abts_rsp(struct fnic_iport_s *iport, iport->fabric.fdmi_pending &= ~FDLS_FDMI_ABORT_PENDING; fdls_free_oxid(iport, oxid, &iport->active_oxid_fdmi_rpa); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: iport->fabric.fdmi_pending: 0x%x", iport->fcid, iport->fabric.fdmi_pending); break; default: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received abts rsp with invalid oxid: 0x%x. Dropping frame", oxid); break; @@ -3861,7 +3873,7 @@ fdls_process_fabric_abts_rsp(struct fnic_iport_s *iport, if (!((s_id == FC_FID_DIR_SERV) || (s_id == FC_FID_FLOGI) || (s_id == FC_FID_FCTRL))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received abts rsp with invalid SID: 0x%x. Dropping frame", s_id); return; @@ -3869,14 +3881,14 @@ fdls_process_fabric_abts_rsp(struct fnic_iport_s *iport, oxid = FNIC_STD_GET_OX_ID(fchdr); if (iport->active_oxid_fabric_req != oxid) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received abts rsp with invalid oxid: 0x%x. Dropping frame", oxid); return; } if (iport->fabric.timer_pending) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Canceling fabric disc timer %p\n", iport); fnic_del_fabric_timer_sync(fnic); } @@ -3884,11 +3896,11 @@ fdls_process_fabric_abts_rsp(struct fnic_iport_s *iport, iport->fabric.flags &= ~FNIC_FDLS_FABRIC_ABORT_ISSUED; if (fchdr->fh_r_ctl == FC_RCTL_BA_ACC) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received abts rsp BA_ACC for fabric_state: %d OX_ID: 0x%x", fabric_state, be16_to_cpu(ba_acc->acc.ba_ox_id)); } else if (fchdr->fh_r_ctl == FC_RCTL_BA_RJT) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "BA_RJT fs: %d OX_ID: 0x%x rc: 0x%x rce: 0x%x", fabric_state, FNIC_STD_GET_OX_ID(&ba_rjt->fchdr), ba_rjt->rjt.br_reason, ba_rjt->rjt.br_explan); @@ -3903,7 +3915,7 @@ fdls_process_fabric_abts_rsp(struct fnic_iport_s *iport, if (iport->fabric.retry_counter < iport->max_flogi_retries) fdls_send_fabric_flogi(iport); else - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Exceeded max FLOGI retries"); break; case FNIC_FRAME_TYPE_FABRIC_LOGO: @@ -3914,7 +3926,7 @@ fdls_process_fabric_abts_rsp(struct fnic_iport_s *iport, if (iport->fabric.retry_counter < iport->max_plogi_retries) fdls_send_fabric_plogi(iport); else - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Exceeded max PLOGI retries"); break; case FNIC_FRAME_TYPE_FABRIC_RPN: @@ -3928,7 +3940,7 @@ fdls_process_fabric_abts_rsp(struct fnic_iport_s *iport, if (iport->fabric.retry_counter < FDLS_RETRY_COUNT) fdls_send_scr(iport); else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "SCR exhausted retries. Start fabric PLOGI %p", iport); fnic_fdls_start_plogi(iport); /* go back to fabric Plogi */ @@ -3938,7 +3950,7 @@ fdls_process_fabric_abts_rsp(struct fnic_iport_s *iport, if (iport->fabric.retry_counter < FDLS_RETRY_COUNT) fdls_send_register_fc4_types(iport); else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RFT exhausted retries. Start fabric PLOGI %p", iport); fnic_fdls_start_plogi(iport); /* go back to fabric Plogi */ @@ -3948,7 +3960,7 @@ fdls_process_fabric_abts_rsp(struct fnic_iport_s *iport, if (iport->fabric.retry_counter < FDLS_RETRY_COUNT) fdls_send_register_fc4_features(iport); else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RFF exhausted retries. Start fabric PLOGI %p", iport); fnic_fdls_start_plogi(iport); /* go back to fabric Plogi */ @@ -3958,7 +3970,7 @@ fdls_process_fabric_abts_rsp(struct fnic_iport_s *iport, if (iport->fabric.retry_counter <= FDLS_RETRY_COUNT) fdls_send_gpn_ft(iport, fabric_state); else - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "GPN FT exhausted retries. Start fabric PLOGI %p", iport); break; @@ -3967,7 +3979,7 @@ fdls_process_fabric_abts_rsp(struct fnic_iport_s *iport, * We should not be here since we already validated rx oxid with * our active_oxid_fabric_req */ - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Invalid OXID/active oxid 0x%x\n", oxid); WARN_ON(true); return; @@ -3987,7 +3999,7 @@ fdls_process_abts_req(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) sizeof(struct fc_std_abts_ba_acc); nport_id = ntoh24(fchdr->fh_s_id); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received abort from SID 0x%8x", nport_id); tport = fnic_find_tport_by_fcid(iport, nport_id); @@ -4000,7 +4012,7 @@ fdls_process_abts_req(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "0x%x: Failed to allocate frame to send response for ABTS req", iport->fcid); return; @@ -4021,7 +4033,7 @@ fdls_process_abts_req(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) pba_acc->acc.ba_rx_id = cpu_to_be16(FNIC_STD_GET_RX_ID(fchdr)); pba_acc->acc.ba_ox_id = cpu_to_be16(FNIC_STD_GET_OX_ID(fchdr)); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS send BA ACC with oxid: 0x%x", iport->fcid, oxid); @@ -4041,7 +4053,7 @@ fdls_process_unsupported_els_req(struct fnic_iport_s *iport, sizeof(struct fc_std_els_rjt_rsp); if (iport->fcid != d_id) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Dropping unsupported ELS with illegal frame bits 0x%x\n", d_id); atomic64_inc(&iport->iport_stats.unsupported_frames_dropped); @@ -4050,7 +4062,7 @@ fdls_process_unsupported_els_req(struct fnic_iport_s *iport, if ((iport->state != FNIC_IPORT_STATE_READY) && (iport->state != FNIC_IPORT_STATE_FABRIC_DISC)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Dropping unsupported ELS request in iport state: %d", iport->state); atomic64_inc(&iport->iport_stats.unsupported_frames_dropped); @@ -4059,7 +4071,7 @@ fdls_process_unsupported_els_req(struct fnic_iport_s *iport, frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send response to unsupported ELS request"); return; } @@ -4067,7 +4079,7 @@ fdls_process_unsupported_els_req(struct fnic_iport_s *iport, pls_rsp = (struct fc_std_els_rjt_rsp *) (frame + FNIC_ETH_FCOE_HDRS_OFFSET); fdls_init_els_rjt_frame(frame, iport); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Process unsupported ELS request from SID: 0x%x", iport->fcid, ntoh24(fchdr->fh_s_id)); @@ -4094,12 +4106,12 @@ fdls_process_rls_req(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) uint16_t frame_size = FNIC_ETH_FCOE_HDRS_OFFSET + sizeof(struct fc_std_rls_acc); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Process RLS request %d", iport->fnic->fnic_num); if ((iport->state != FNIC_IPORT_STATE_READY) && (iport->state != FNIC_IPORT_STATE_FABRIC_DISC)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received RLS req in iport state: %d. Dropping the frame.", iport->state); return; @@ -4107,7 +4119,7 @@ fdls_process_rls_req(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send RLS accept"); return; } @@ -4148,33 +4160,33 @@ fdls_process_els_req(struct fnic_iport_s *iport, struct fc_frame_header *fchdr, if ((iport->state != FNIC_IPORT_STATE_READY) && (iport->state != FNIC_IPORT_STATE_FABRIC_DISC)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Dropping ELS frame type: 0x%x in iport state: %d", type, iport->state); return; } switch (type) { case ELS_ECHO: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "sending LS_ACC for ECHO request %d\n", iport->fnic->fnic_num); break; case ELS_RRQ: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "sending LS_ACC for RRQ request %d\n", iport->fnic->fnic_num); break; default: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "sending LS_ACC for 0x%x ELS frame\n", type); break; } frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send ELS response for 0x%x", type); return; @@ -4220,17 +4232,17 @@ fdls_process_tgt_abts_rsp(struct fnic_iport_s *iport, tport = fnic_find_tport_by_fcid(iport, s_id); if (!tport) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Received tgt abts rsp with invalid SID: 0x%x", s_id); return; } if (tport->timer_pending) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "tport 0x%p Canceling fabric disc timer\n", tport); fnic_del_tport_timer_sync(fnic, tport); } if (iport->state != FNIC_IPORT_STATE_READY) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Received tgt abts rsp in iport state(%d). Dropping.", iport->state); return; @@ -4245,15 +4257,15 @@ fdls_process_tgt_abts_rsp(struct fnic_iport_s *iport, switch (frame_type) { case FNIC_FRAME_TYPE_TGT_ADISC: if (fchdr->fh_r_ctl == FC_RCTL_BA_ACC) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "OX_ID: 0x%x tgt_fcid: 0x%x rcvd tgt adisc abts resp BA_ACC", be16_to_cpu(ba_acc->acc.ba_ox_id), tport->fcid); } else if (fchdr->fh_r_ctl == FC_RCTL_BA_RJT) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "ADISC BA_RJT rcvd tport_fcid: 0x%x tport_state: %d ", tport->fcid, tport_state); - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "reason code: 0x%x reason code explanation:0x%x ", ba_rjt->rjt.br_reason, ba_rjt->rjt.br_explan); @@ -4265,7 +4277,7 @@ fdls_process_tgt_abts_rsp(struct fnic_iport_s *iport, return; } fdls_free_oxid(iport, oxid, &tport->active_oxid); - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "ADISC not responding. Deleting target port: 0x%x", tport->fcid); fdls_delete_tport(iport, tport); @@ -4278,14 +4290,14 @@ fdls_process_tgt_abts_rsp(struct fnic_iport_s *iport, break; case FNIC_FRAME_TYPE_TGT_PLOGI: if (fchdr->fh_r_ctl == FC_RCTL_BA_ACC) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Received tgt PLOGI abts response BA_ACC tgt_fcid: 0x%x", tport->fcid); } else if (fchdr->fh_r_ctl == FC_RCTL_BA_RJT) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PLOGI BA_RJT received for tport_fcid: 0x%x OX_ID: 0x%x", tport->fcid, FNIC_STD_GET_OX_ID(fchdr)); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "reason code: 0x%x reason code explanation: 0x%x", ba_rjt->rjt.br_reason, ba_rjt->rjt.br_explan); @@ -4308,14 +4320,14 @@ fdls_process_tgt_abts_rsp(struct fnic_iport_s *iport, break; case FNIC_FRAME_TYPE_TGT_PRLI: if (fchdr->fh_r_ctl == FC_RCTL_BA_ACC) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Received tgt PRLI abts response BA_ACC", tport->fcid); } else if (fchdr->fh_r_ctl == FC_RCTL_BA_RJT) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PRLI BA_RJT received for tport_fcid: 0x%x OX_ID: 0x%x ", tport->fcid, FNIC_STD_GET_OX_ID(fchdr)); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "reason code: 0x%x reason code explanation: 0x%x", ba_rjt->rjt.br_reason, ba_rjt->rjt.br_explan); @@ -4331,7 +4343,7 @@ fdls_process_tgt_abts_rsp(struct fnic_iport_s *iport, fdls_set_tport_state(tport, FDLS_TGT_STATE_PLOGI); break; default: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received ABTS response for unknown frame %p", iport); break; } @@ -4351,14 +4363,14 @@ fdls_process_plogi_req(struct fnic_iport_s *iport, sizeof(struct fc_std_els_rjt_rsp); if (iport->fcid != d_id) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received PLOGI with illegal frame bits. Dropping frame from 0x%x", d_id); return; } if (iport->state != FNIC_IPORT_STATE_READY) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received PLOGI request in iport state: %d Dropping frame", iport->state); return; @@ -4366,7 +4378,7 @@ fdls_process_plogi_req(struct fnic_iport_s *iport, frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send response to PLOGI request"); return; } @@ -4374,7 +4386,7 @@ fdls_process_plogi_req(struct fnic_iport_s *iport, pplogi_rsp = (struct fc_std_els_rjt_rsp *) (frame + FNIC_ETH_FCOE_HDRS_OFFSET); fdls_init_els_rjt_frame(frame, iport); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: Process PLOGI request from SID: 0x%x", iport->fcid, ntoh24(fchdr->fh_s_id)); @@ -4404,11 +4416,11 @@ fdls_process_logo_req(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) nport_id = ntoh24(logo->els.fl_n_port_id); nport_name = be64_to_cpu(logo->els.fl_n_port_wwn); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Process LOGO request from fcid: 0x%x", nport_id); if (iport->state != FNIC_IPORT_STATE_READY) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Dropping LOGO req from 0x%x in iport state: %d", nport_id, iport->state); return; @@ -4418,19 +4430,19 @@ fdls_process_logo_req(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) if (!tport) { /* We are not logged in with the nport, log and drop... */ - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Received LOGO from an nport not logged in: 0x%x(0x%llx)", nport_id, nport_name); return; } if (tport->fcid != nport_id) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Received LOGO with invalid target port fcid: 0x%x(0x%llx)", nport_id, nport_name); return; } if (tport->timer_pending) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "tport fcid 0x%x: Canceling disc timer\n", tport->fcid); fnic_del_tport_timer_sync(fnic, tport); @@ -4447,7 +4459,7 @@ fdls_process_logo_req(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) if ((iport->state == FNIC_IPORT_STATE_READY) && (fdls_get_state(&iport->fabric) != FDLS_STATE_SEND_GPNFT) && (fdls_get_state(&iport->fabric) != FDLS_STATE_RSCN_GPN_FT)) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Sending GPNFT in response to LOGO from Target:0x%x", nport_id); fdls_send_gpn_ft(iport, FDLS_STATE_SEND_GPNFT); @@ -4460,7 +4472,7 @@ fdls_process_logo_req(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) fdls_send_logo_resp(iport, &logo->fchdr); if ((fdls_get_state(&iport->fabric) != FDLS_STATE_SEND_GPNFT) && (fdls_get_state(&iport->fabric) != FDLS_STATE_RSCN_GPN_FT)) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Sending GPNFT in response to LOGO from Target:0x%x", nport_id); fdls_send_gpn_ft(iport, FDLS_STATE_SEND_GPNFT); @@ -4487,11 +4499,11 @@ fdls_process_rscn(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) atomic64_inc(&iport->iport_stats.num_rscns); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDLS process RSCN %p", iport); if (iport->state != FNIC_IPORT_STATE_READY) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDLS RSCN received in state(%d). Dropping", fdls_get_state(fdls)); return; @@ -4508,18 +4520,18 @@ fdls_process_rscn(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) if ((rscn_payload_len == 0xFFFF) && (sid == FC_FID_FCTRL)) { rscn_type = PC_RSCN; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "pcrscn: PCRSCN received. sid: 0x%x payload len: 0x%x", sid, rscn_payload_len); } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RSCN payload_len: 0x%x page_len: 0x%x", rscn_payload_len, rscn->els.rscn_page_len); /* if this happens then we need to send ADISC to all the tports. */ list_for_each_entry_safe(tport, next, &iport->tport_list, links) { if (tport->state == FDLS_TGT_STATE_READY) tport->flags |= FNIC_FDLS_TPORT_SEND_ADISC; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RSCN for port id: 0x%x", tport->fcid); } } /* end else */ @@ -4527,7 +4539,7 @@ fdls_process_rscn(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) num_ports = (rscn_payload_len - 4) / rscn->els.rscn_page_len; rscn_port = (struct fc_els_rscn_page *)(rscn + 1); } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RSCN received for num_ports: %d payload_len: %d page_len: %d ", num_ports, rscn_payload_len, rscn->els.rscn_page_len); @@ -4551,14 +4563,14 @@ fdls_process_rscn(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) if (tport->state == FDLS_TGT_STATE_READY) tport->flags |= FNIC_FDLS_TPORT_SEND_ADISC; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RSCN for port id: 0x%x", tport->fcid); } break; } tport = fnic_find_tport_by_fcid(iport, nport_id); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "RSCN port id list: 0x%x", nport_id); if (!tport) { @@ -4573,13 +4585,13 @@ fdls_process_rscn(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) rscn_type == PC_RSCN && fnic->role == FNIC_ROLE_FCP_INITIATOR) { if (fnic->pc_rscn_handling_status == PC_RSCN_HANDLING_IN_PROGRESS) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PCRSCN handling already in progress. Skip host reset: %d", iport->fnic->fnic_num); return; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Processing PCRSCN. Queuing fnic for host reset: %d", iport->fnic->fnic_num); fnic->pc_rscn_handling_status = PC_RSCN_HANDLING_IN_PROGRESS; @@ -4595,7 +4607,7 @@ fdls_process_rscn(struct fnic_iport_s *iport, struct fc_frame_header *fchdr) queue_work(reset_fnic_work_queue, &reset_fnic_work); spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FDLS process RSCN sending GPN_FT: newports: %d", newports); fdls_send_gpn_ft(iport, FDLS_STATE_RSCN_GPN_FT); fdls_send_rscn_resp(iport, fchdr); @@ -4644,20 +4656,20 @@ fdls_process_adisc_req(struct fnic_iport_s *iport, uint16_t acc_frame_size = FNIC_ETH_FCOE_HDRS_OFFSET + sizeof(struct fc_std_els_adisc); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Process ADISC request %d", iport->fnic->fnic_num); fcid = FNIC_STD_GET_S_ID(fchdr); tgt_fcid = ntoh24(fcid); tport = fnic_find_tport_by_fcid(iport, tgt_fcid); if (!tport) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "tport for fcid: 0x%x not found. Dropping ADISC req.", tgt_fcid); return; } if (iport->state != FNIC_IPORT_STATE_READY) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Dropping ADISC req from fcid: 0x%x in iport state: %d", tgt_fcid, iport->state); return; @@ -4668,16 +4680,16 @@ fdls_process_adisc_req(struct fnic_iport_s *iport, if ((frame_wwnn != tport->wwnn) || (frame_wwpn != tport->wwpn)) { /* send reject */ - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "ADISC req from fcid: 0x%x mismatch wwpn: 0x%llx wwnn: 0x%llx", tgt_fcid, frame_wwpn, frame_wwnn); - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "local tport wwpn: 0x%llx wwnn: 0x%llx. Sending RJT", tport->wwpn, tport->wwnn); rjt_frame = fdls_alloc_frame(iport); if (rjt_frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate rjt_frame to send response to ADISC request"); return; } @@ -4700,7 +4712,7 @@ fdls_process_adisc_req(struct fnic_iport_s *iport, acc_frame = fdls_alloc_frame(iport); if (acc_frame == NULL) { - FNIC_FCS_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_ERR, fnic, "Failed to allocate frame to send ADISC accept"); return; } @@ -4754,7 +4766,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, /* some common validation */ if (fdls_get_state(fabric) > FDLS_STATE_FABRIC_FLOGI) { if (iport->fcid != d_id || (!FNIC_FC_FRAME_CS_CTL(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "invalid frame received. Dropping frame"); return -1; } @@ -4764,14 +4776,14 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, if ((fchdr->fh_r_ctl == FC_RCTL_BA_ACC) || (fchdr->fh_r_ctl == FC_RCTL_BA_RJT)) { if (!(FNIC_FC_FRAME_TYPE_BLS(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received ABTS invalid frame. Dropping frame"); return -1; } if (fdls_is_oxid_fabric_req(oxid)) { if (!(iport->fabric.flags & FNIC_FDLS_FABRIC_ABORT_ISSUED)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received unexpected ABTS RSP(oxid:0x%x) from 0x%x. Dropping frame", oxid, s_id); return -1; @@ -4782,7 +4794,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, } else if (fdls_is_oxid_tgt_req(oxid)) { return FNIC_TPORT_BLS_ABTS_RSP; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received ABTS rsp with unknown oxid(0x%x) from 0x%x. Dropping frame", oxid, s_id); return -1; @@ -4791,7 +4803,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, /* BLS ABTS Req */ if ((fchdr->fh_r_ctl == FC_RCTL_BA_ABTS) && (FNIC_FC_FRAME_TYPE_BLS(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Receiving Abort Request from s_id: 0x%x", s_id); return FNIC_BLS_ABTS_REQ; } @@ -4803,7 +4815,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, if ((!FNIC_FC_FRAME_FCTL_FIRST_LAST_SEQINIT(fchdr)) || (!FNIC_FC_FRAME_UNSOLICITED(fchdr)) || (!FNIC_FC_FRAME_TYPE_ELS(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received LOGO invalid frame. Dropping frame"); return -1; } @@ -4812,12 +4824,12 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, if ((!FNIC_FC_FRAME_FCTL_FIRST_LAST_SEQINIT(fchdr)) || (!FNIC_FC_FRAME_TYPE_ELS(fchdr)) || (!FNIC_FC_FRAME_UNSOLICITED(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received RSCN invalid FCTL. Dropping frame"); return -1; } if (s_id != FC_FID_FCTRL) - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received RSCN from target FCTL: 0x%x type: 0x%x s_id: 0x%x.", fchdr->fh_f_ctl[0], fchdr->fh_type, s_id); return FNIC_ELS_RSCN_REQ; @@ -4832,7 +4844,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, case ELS_RRQ: return FNIC_ELS_RRQ; default: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Unsupported frame (type:0x%02x) from fcid: 0x%x", type, s_id); return FNIC_ELS_UNSUPPORTED_REQ; @@ -4841,14 +4853,14 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, /* solicited response from fabric or target */ oxid_frame_type = FNIC_FRAME_TYPE(oxid); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "oxid frame code: 0x%x, oxid: 0x%x\n", oxid_frame_type, oxid); switch (oxid_frame_type) { case FNIC_FRAME_TYPE_FABRIC_FLOGI: if (type == ELS_LS_ACC) { if ((s_id != FC_FID_FLOGI) || (!FNIC_FC_FRAME_TYPE_ELS(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received unknown frame. Dropping frame"); return -1; } @@ -4859,7 +4871,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, if (type == ELS_LS_ACC) { if ((s_id != FC_FID_DIR_SERV) || (!FNIC_FC_FRAME_TYPE_ELS(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received unknown frame. Dropping frame"); return -1; } @@ -4870,7 +4882,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, if (type == ELS_LS_ACC) { if ((s_id != FC_FID_FCTRL) || (!FNIC_FC_FRAME_TYPE_ELS(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received unknown frame. Dropping frame"); return -1; } @@ -4879,7 +4891,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, case FNIC_FRAME_TYPE_FABRIC_RPN: if ((s_id != FC_FID_DIR_SERV) || (!FNIC_FC_FRAME_TYPE_FC_GS(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received unknown frame. Dropping frame"); return -1; } @@ -4887,7 +4899,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, case FNIC_FRAME_TYPE_FABRIC_RFT: if ((s_id != FC_FID_DIR_SERV) || (!FNIC_FC_FRAME_TYPE_FC_GS(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received unknown frame. Dropping frame"); return -1; } @@ -4895,7 +4907,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, case FNIC_FRAME_TYPE_FABRIC_RFF: if ((s_id != FC_FID_DIR_SERV) || (!FNIC_FC_FRAME_TYPE_FC_GS(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received unknown frame. Dropping frame"); return -1; } @@ -4903,7 +4915,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, case FNIC_FRAME_TYPE_FABRIC_GPN_FT: if ((s_id != FC_FID_DIR_SERV) || (!FNIC_FC_FRAME_TYPE_FC_GS(fchdr))) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received unknown frame. Dropping frame"); return -1; } @@ -4925,7 +4937,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, return FNIC_TPORT_ADISC_RSP; case FNIC_FRAME_TYPE_TGT_LOGO: if (!FNIC_FC_FRAME_TYPE_ELS(fchdr)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Dropping Unknown frame in tport solicited exchange range type: 0x%x.", fchdr->fh_type); return -1; @@ -4933,7 +4945,7 @@ fnic_fdls_validate_and_get_frame_type(struct fnic_iport_s *iport, return FNIC_TPORT_LOGO_RSP; default: /* Drop the Rx frame and log/stats it */ - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Solicited response: unknown OXID: 0x%x", oxid); return -1; } @@ -5003,7 +5015,7 @@ void fnic_fdls_recv_frame(struct fnic_iport_s *iport, void *rx_frame, break; case FNIC_TPORT_LOGO_RSP: /* Logo response from tgt which we have deleted */ - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Logo response from tgt: 0x%x", ntoh24(fchdr->fh_s_id)); break; @@ -5046,9 +5058,9 @@ void fnic_fdls_recv_frame(struct fnic_iport_s *iport, void *rx_frame, fdls_process_fdmi_reg_ack(iport, fchdr, frame_type); break; default: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "s_id: 0x%x d_did: 0x%x", s_id, d_id); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Received unknown FCoE frame of len: %d. Dropping frame", len); break; } @@ -5065,7 +5077,7 @@ void fnic_fdls_link_down(struct fnic_iport_s *iport) struct fnic_tport_s *tport, *next; struct fnic *fnic = iport->fnic; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS processing link down", iport->fcid); fdls_set_state((&iport->fabric), FDLS_STATE_LINKDOWN); @@ -5075,7 +5087,7 @@ void fnic_fdls_link_down(struct fnic_iport_s *iport) fnic_fcpio_reset(iport->fnic); spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); list_for_each_entry_safe(tport, next, &iport->tport_list, links) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "removing rport: 0x%x", tport->fcid); fdls_delete_tport(iport, tport); } @@ -5088,6 +5100,6 @@ void fnic_fdls_link_down(struct fnic_iport_s *iport) iport->flags &= ~FNIC_FDMI_ACTIVE; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "0x%x: FDLS finish processing link down", iport->fcid); } diff --git a/drivers/scsi/fnic/fip.c b/drivers/scsi/fnic/fip.c index 132f00512ee16..231dec180675d 100644 --- a/drivers/scsi/fnic/fip.c +++ b/drivers/scsi/fnic/fip.c @@ -27,7 +27,7 @@ void fnic_fcoe_reset_vlans(struct fnic *fnic) } spin_unlock_irqrestore(&fnic->vlans_lock, flags); - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "Reset vlan complete\n"); } @@ -46,7 +46,7 @@ void fnic_fcoe_send_vlan_req(struct fnic *fnic) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FIP_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_ERR, fnic, "Failed to allocate frame to send VLAN req"); return; } @@ -54,10 +54,10 @@ void fnic_fcoe_send_vlan_req(struct fnic *fnic) fnic_fcoe_reset_vlans(fnic); fnic->set_vlan(fnic, 0); - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "set vlan done\n"); - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "got MAC 0x%x:%x:%x:%x:%x:%x\n", iport->hwmac[0], iport->hwmac[1], iport->hwmac[2], iport->hwmac[3], iport->hwmac[4], iport->hwmac[5]); @@ -81,13 +81,13 @@ void fnic_fcoe_send_vlan_req(struct fnic *fnic) iport->fip.state = FDLS_FIP_VLAN_DISCOVERY_STARTED; - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "Send VLAN req\n"); fnic_send_fip_frame(iport, frame, frame_size); vlan_tov = jiffies + msecs_to_jiffies(FCOE_CTLR_FIPVLAN_TOV); mod_timer(&fnic->retry_fip_timer, round_jiffies(vlan_tov)); - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fip timer set\n"); } @@ -111,11 +111,11 @@ void fnic_fcoe_process_vlan_resp(struct fnic *fnic, struct fip_header *fiph) struct fip_vlan_desc *vlan_desc; unsigned long flags; - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p got vlan resp\n", fnic); desc_len = be16_to_cpu(vlan_notif->fip.fip_dl_len); - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "desc_len %d\n", desc_len); spin_lock_irqsave(&fnic->vlans_lock, flags); @@ -128,23 +128,20 @@ void fnic_fcoe_process_vlan_resp(struct fnic *fnic, struct fip_header *fiph) if (vlan_desc->fd_desc.fip_dtype == FIP_DT_VLAN) { if (vlan_desc->fd_desc.fip_dlen != 1) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "Invalid descriptor length(%x) in VLan response\n", vlan_desc->fd_desc.fip_dlen); } num_vlan++; vid = be16_to_cpu(vlan_desc->fd_vlan); - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "process_vlan_resp: FIP VLAN %d\n", vid); - vlan = kzalloc_obj(*vlan); + vlan = kzalloc_obj(*vlan, GFP_ATOMIC); if (!vlan) { /* retry from timer */ - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "Mem Alloc failure\n"); spin_unlock_irqrestore(&fnic->vlans_lock, flags); @@ -155,14 +152,14 @@ void fnic_fcoe_process_vlan_resp(struct fnic *fnic, struct fip_header *fiph) list_add_tail(&vlan->list, &fnic->vlan_list); break; } - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, - "Invalid descriptor type(%x) in VLan response\n", - vlan_desc->fd_desc.fip_dtype); /* - * Note : received a type=2 descriptor here i.e. FIP - * MAC Address Descriptor + * Note : skip any type=2 descriptor here + * (i.e. FIP MAC Address Descriptor) */ + if (vlan_desc->fd_desc.fip_dtype != FIP_DT_MAC) + FNIC_FIP_DBG(KERN_INFO, fnic, + "Invalid descriptor type(0x%x) in vlan response\n", + vlan_desc->fd_desc.fip_dtype); cur_desc += vlan_desc->fd_desc.fip_dlen; desc_len -= vlan_desc->fd_desc.fip_dlen; } @@ -170,7 +167,7 @@ void fnic_fcoe_process_vlan_resp(struct fnic *fnic, struct fip_header *fiph) /* any VLAN descriptors present ? */ if (num_vlan == 0) { atomic64_inc(&fnic_stats->vlan_stats.resp_withno_vlanID); - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p No VLAN descriptors in FIP VLAN response\n", fnic); } @@ -195,7 +192,7 @@ void fnic_fcoe_start_fcf_discovery(struct fnic *fnic) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FIP_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_ERR, fnic, "Failed to allocate frame to start FCF discovery"); return; } @@ -222,7 +219,7 @@ void fnic_fcoe_start_fcf_discovery(struct fnic *fnic) FNIC_STD_SET_NODE_NAME(&pdisc_sol->name_desc.fd_wwn, iport->wwnn); - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "Start FCF discovery\n"); fnic_send_fip_frame(iport, frame, frame_size); @@ -257,16 +254,14 @@ void fnic_fcoe_fip_discovery_resp(struct fnic *fnic, struct fip_header *fiph) switch (iport->fip.state) { case FDLS_FIP_FCF_DISCOVERY_STARTED: if (be16_to_cpu(disc_adv->fip.fip_flags) & FIP_FL_SOL) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p Solicited adv\n", fnic); if ((disc_adv->prio_desc.fd_pri < iport->selected_fcf.fcf_priority) && (be16_to_cpu(disc_adv->fip.fip_flags) & FIP_FL_AVAIL)) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p FCF Available\n", fnic); memcpy(iport->selected_fcf.fcf_mac, disc_adv->mac_desc.fd_mac, ETH_ALEN); @@ -274,8 +269,8 @@ void fnic_fcoe_fip_discovery_resp(struct fnic *fnic, struct fip_header *fiph) disc_adv->prio_desc.fd_pri; iport->selected_fcf.fka_adv_period = be32_to_cpu(disc_adv->fka_adv_desc.fd_fka_period); - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, "adv time %d", + FNIC_FIP_DBG(KERN_INFO, fnic, + "adv time %d", iport->selected_fcf.fka_adv_period); iport->selected_fcf.ka_disabled = (disc_adv->fka_adv_desc.fd_flags & 1); @@ -294,8 +289,7 @@ void fnic_fcoe_fip_discovery_resp(struct fnic *fnic, struct fip_header *fiph) iport->selected_fcf.fka_adv_period = be32_to_cpu(disc_adv->fka_adv_desc.fd_fka_period); FNIC_FIP_DBG(KERN_INFO, - fnic->host, - fnic->fnic_num, + fnic, "change fka to %d", iport->selected_fcf.fka_adv_period); } @@ -362,7 +356,7 @@ void fnic_fcoe_start_flogi(struct fnic *fnic) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FIP_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_ERR, fnic, "Failed to allocate frame to start FIP FLOGI"); return; } @@ -415,7 +409,7 @@ void fnic_fcoe_start_flogi(struct fnic *fnic) oxid = fdls_alloc_oxid(iport, FNIC_FRAME_TYPE_FABRIC_FLOGI, &iport->active_oxid_fabric_req); if (oxid == FNIC_UNASSIGNED_OXID) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Failed to allocate OXID to send FIP FLOGI"); mempool_free(frame, fnic->frame_pool); return; @@ -427,7 +421,7 @@ void fnic_fcoe_start_flogi(struct fnic *fnic) FNIC_STD_SET_NODE_NAME(&pflogi_req->flogi_desc.flogi.els.fl_wwnn, iport->wwnn); - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "FIP start FLOGI\n"); fnic_send_fip_frame(iport, frame, frame_size); iport->fip.flogi_retry++; @@ -457,11 +451,11 @@ void fnic_fcoe_process_flogi_resp(struct fnic *fnic, struct fip_header *fiph) struct fnic_stats *fnic_stats = &fnic->fnic_stats; struct fc_frame_header *fchdr = &flogi_rsp->rsp_desc.flogi.fchdr; - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p FIP FLOGI rsp\n", fnic); desc_len = be16_to_cpu(flogi_rsp->fip.fip_dl_len); if (desc_len != 38) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "Invalid Descriptor List len (%x). Dropping frame\n", desc_len); return; @@ -471,7 +465,7 @@ void fnic_fcoe_process_flogi_resp(struct fnic *fnic, struct fip_header *fiph) && (flogi_rsp->rsp_desc.fd_desc.fip_dlen == 36)) || !((flogi_rsp->mac_desc.fd_desc.fip_dtype == 2) && (flogi_rsp->mac_desc.fd_desc.fip_dlen == 2))) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "Dropping frame invalid type and len mix\n"); return; } @@ -484,7 +478,7 @@ void fnic_fcoe_process_flogi_resp(struct fnic *fnic, struct fip_header *fiph) || (s_id != FC_FID_FLOGI) || (frame_type != FNIC_FABRIC_FLOGI_RSP) || (fchdr->fh_type != 0x01)) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "Dropping invalid frame: s_id %x F %x R %x t %x OX_ID %x\n", s_id, fchdr->fh_f_ctl[0], fchdr->fh_r_ctl, fchdr->fh_type, FNIC_STD_GET_OX_ID(fchdr)); @@ -492,7 +486,7 @@ void fnic_fcoe_process_flogi_resp(struct fnic *fnic, struct fip_header *fiph) } if (iport->fip.state == FDLS_FIP_FLOGI_STARTED) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p rsp for pending FLOGI\n", fnic); oxid = FNIC_STD_GET_OX_ID(fchdr); @@ -502,8 +496,7 @@ void fnic_fcoe_process_flogi_resp(struct fnic *fnic, struct fip_header *fiph) if ((be16_to_cpu(flogi_rsp->fip.fip_dl_len) == FIP_FLOGI_LEN) && (flogi_rsp->rsp_desc.flogi.els.fl_cmd == ELS_LS_ACC)) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p FLOGI success\n", fnic); memcpy(iport->fpma, flogi_rsp->mac_desc.fd_mac, ETH_ALEN); iport->fcid = @@ -519,8 +512,7 @@ void fnic_fcoe_process_flogi_resp(struct fnic *fnic, struct fip_header *fiph) if (fnic_fdls_register_portid(iport, iport->fcid, NULL) != 0) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p flogi registration failed\n", fnic); return; @@ -528,8 +520,8 @@ void fnic_fcoe_process_flogi_resp(struct fnic *fnic, struct fip_header *fiph) iport->fip.state = FDLS_FIP_FLOGI_COMPLETE; iport->state = FNIC_IPORT_STATE_FABRIC_DISC; - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, "iport->state:%d\n", + FNIC_FIP_DBG(KERN_INFO, fnic, + "iport->state:%d\n", iport->state); fnic_fdls_disc_start(iport); if (!((iport->selected_fcf.ka_disabled) @@ -575,7 +567,7 @@ void fnic_common_fip_cleanup(struct fnic *fnic) if (!iport->usefip) return; - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p fip cleanup\n", fnic); iport->fip.state = FDLS_FIP_INIT; @@ -617,7 +609,7 @@ void fnic_fcoe_process_cvl(struct fnic *fnic, struct fip_header *fiph) int found = false; int max_count = 0; - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p clear virtual link handler\n", fnic); if (!((cvl_msg->fcf_mac_desc.fd_desc.fip_dtype == 2) @@ -625,7 +617,7 @@ void fnic_fcoe_process_cvl(struct fnic *fnic, struct fip_header *fiph) || !((cvl_msg->name_desc.fd_desc.fip_dtype == 4) && (cvl_msg->name_desc.fd_desc.fip_dlen == 3))) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "invalid mix: ft %x fl %x ndt %x ndl %x", cvl_msg->fcf_mac_desc.fd_desc.fip_dtype, cvl_msg->fcf_mac_desc.fd_desc.fip_dlen, @@ -640,8 +632,7 @@ void fnic_fcoe_process_cvl(struct fnic *fnic, struct fip_header *fiph) if (!((cvl_msg->vn_ports_desc[i].fd_desc.fip_dtype == 11) && (cvl_msg->vn_ports_desc[i].fd_desc.fip_dlen == 5))) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "Invalid type and len mix type: %d len: %d\n", cvl_msg->vn_ports_desc[i].fd_desc.fip_dtype, cvl_msg->vn_ports_desc[i].fd_desc.fip_dlen); @@ -664,12 +655,12 @@ void fnic_fcoe_process_cvl(struct fnic *fnic, struct fip_header *fiph) spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); max_count++; if (max_count >= FIP_FNIC_RESET_WAIT_COUNT) { - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "Rthr waited too long. Skipping handle link event %p\n", fnic); return; } - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic reset in progress. Link event needs to wait %p", fnic); } @@ -714,7 +705,7 @@ int fdls_fip_recv_frame(struct fnic *fnic, void *frame) return true; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Not a FIP Frame"); return false; } @@ -724,7 +715,7 @@ void fnic_work_on_fip_timer(struct work_struct *work) struct fnic *fnic = container_of(work, struct fnic, fip_timer_work); struct fnic_iport_s *iport = &fnic->iport; - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "FIP timeout\n"); if (iport->fip.state == FDLS_FIP_VLAN_DISCOVERY_STARTED) { @@ -732,7 +723,7 @@ void fnic_work_on_fip_timer(struct work_struct *work) } else if (iport->fip.state == FDLS_FIP_FCF_DISCOVERY_STARTED) { u8 zmac[ETH_ALEN] = { 0, 0, 0, 0, 0, 0 }; - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "FCF Discovery timeout\n"); if (memcmp(iport->selected_fcf.fcf_mac, zmac, ETH_ALEN) != 0) { @@ -754,13 +745,13 @@ void fnic_work_on_fip_timer(struct work_struct *work) round_jiffies(fcf_tov)); } } else { - FNIC_FIP_DBG(KERN_INFO, fnic->host, - fnic->fnic_num, "FCF Discovery timeout\n"); + FNIC_FIP_DBG(KERN_INFO, fnic, + "FCF Discovery timeout\n"); fnic_vlan_discovery_timeout(fnic); } } else if (iport->fip.state == FDLS_FIP_FLOGI_STARTED) { fdls_schedule_oxid_free(iport, &iport->active_oxid_fabric_req); - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "FLOGI timeout\n"); if (iport->fip.flogi_retry < fnic->config.flogi_retries) fnic_fcoe_start_flogi(fnic); @@ -807,7 +798,7 @@ void fnic_handle_enode_ka_timer(struct timer_list *t) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FIP_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_ERR, fnic, "Failed to allocate frame to send enode ka"); return; } @@ -828,7 +819,7 @@ void fnic_handle_enode_ka_timer(struct timer_list *t) memcpy(penode_ka->eth.h_dest, iport->selected_fcf.fcf_mac, ETH_ALEN); memcpy(penode_ka->mac_desc.fd_mac, iport->hwmac, ETH_ALEN); - FNIC_FIP_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_DEBUG, fnic, "Handle enode KA timer\n"); fnic_send_fip_frame(iport, frame, frame_size); enode_ka_tov = jiffies @@ -861,7 +852,7 @@ void fnic_handle_vn_ka_timer(struct timer_list *t) frame = fdls_alloc_frame(iport); if (frame == NULL) { - FNIC_FIP_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_ERR, fnic, "Failed to allocate frame to send vn ka"); return; } @@ -887,7 +878,7 @@ void fnic_handle_vn_ka_timer(struct timer_list *t) memcpy(pvn_port_ka->vn_port_desc.fd_fc_id, fcid, 3); FNIC_STD_SET_NPORT_NAME(&pvn_port_ka->vn_port_desc.fd_wwpn, iport->wwpn); - FNIC_FIP_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_DEBUG, fnic, "Handle vnport KA timer\n"); fnic_send_fip_frame(iport, frame, frame_size); vn_ka_tov = jiffies + msecs_to_jiffies(FIP_VN_KA_PERIOD); @@ -977,7 +968,7 @@ void fnic_work_on_fcs_ka_timer(struct work_struct *work) *fnic = container_of(work, struct fnic, fip_timer_work); struct fnic_iport_s *iport = &fnic->iport; - FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FIP_DBG(KERN_INFO, fnic, "fnic 0x%p fcs ka timeout\n", fnic); fnic_common_fip_cleanup(fnic); diff --git a/drivers/scsi/fnic/fip.h b/drivers/scsi/fnic/fip.h index 79fee76288705..4866152a32df2 100644 --- a/drivers/scsi/fnic/fip.h +++ b/drivers/scsi/fnic/fip.h @@ -142,7 +142,7 @@ fnic_debug_dump_fip_frame(struct fnic *fnic, struct ethhdr *eth, u16 op = be16_to_cpu(fiph->fip_op); u8 sub = fiph->fip_subcode; - FNIC_FCS_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_DEBUG, fnic, "FIP %s packet contents: op: 0x%x sub: 0x%x (len = %d)", pfx, op, sub, len); diff --git a/drivers/scsi/fnic/fnic.h b/drivers/scsi/fnic/fnic.h index 8724d64f25253..7c685abd5a5aa 100644 --- a/drivers/scsi/fnic/fnic.h +++ b/drivers/scsi/fnic/fnic.h @@ -30,7 +30,7 @@ #define DRV_NAME "fnic" #define DRV_DESCRIPTION "Cisco FCoE HBA Driver" -#define DRV_VERSION "1.8.0.3" +#define DRV_VERSION "1.9.0.1" #define PFX DRV_NAME ": " #define DFX DRV_NAME "%d: " @@ -144,6 +144,9 @@ #define PCI_SUBDEVICE_ID_CISCO_HELSINKI 0x02e4 /* VIC 15235 */ #define PCI_SUBDEVICE_ID_CISCO_GOTHENBURG 0x02f2 /* VIC 15425 */ +#define IS_FNIC_FCP_INITIATOR(fnic) (fnic->role == FNIC_ROLE_FCP_INITIATOR) +#define IS_FNIC_NVME_INITIATOR(fnic) (fnic->role == FNIC_ROLE_NVME_INITIATOR) + struct fnic_pcie_device { u32 device; u8 *desc; @@ -240,6 +243,9 @@ extern struct work_struct reset_fnic_work; #define FNIC_FCS_LOGGING 0x02 #define FNIC_SCSI_LOGGING 0x04 #define FNIC_ISR_LOGGING 0x08 +#define FNIC_FDLS_LOGGING 0x10 +#define FNIC_NVME_LOGGING 0x20 +#define FNIC_FIP_LOGGING 0x40 #define FNIC_CHECK_LOGGING(LEVEL, CMD) \ do { \ @@ -249,38 +255,39 @@ do { \ } while (0); \ } while (0) -#define FNIC_MAIN_DBG(kern_level, host, fnic_num, fmt, args...) \ - FNIC_CHECK_LOGGING(FNIC_MAIN_LOGGING, \ - shost_printk(kern_level, host, \ - "fnic<%d>: %s: %d: " fmt, fnic_num,\ - __func__, __LINE__, ##args);) - -#define FNIC_FCS_DBG(kern_level, host, fnic_num, fmt, args...) \ - FNIC_CHECK_LOGGING(FNIC_FCS_LOGGING, \ - shost_printk(kern_level, host, \ - "fnic<%d>: %s: %d: " fmt, fnic_num,\ - __func__, __LINE__, ##args);) - -#define FNIC_FIP_DBG(kern_level, host, fnic_num, fmt, args...) \ - FNIC_CHECK_LOGGING(FNIC_FCS_LOGGING, \ - shost_printk(kern_level, host, \ - "fnic<%d>: %s: %d: " fmt, fnic_num,\ - __func__, __LINE__, ##args);) - -#define FNIC_SCSI_DBG(kern_level, host, fnic_num, fmt, args...) \ - FNIC_CHECK_LOGGING(FNIC_SCSI_LOGGING, \ - shost_printk(kern_level, host, \ - "fnic<%d>: %s: %d: " fmt, fnic_num,\ - __func__, __LINE__, ##args);) - -#define FNIC_ISR_DBG(kern_level, host, fnic_num, fmt, args...) \ - FNIC_CHECK_LOGGING(FNIC_ISR_LOGGING, \ - shost_printk(kern_level, host, \ - "fnic<%d>: %s: %d: " fmt, fnic_num,\ - __func__, __LINE__, ##args);) - -#define FNIC_MAIN_NOTE(kern_level, host, fmt, args...) \ - shost_printk(kern_level, host, fmt, ##args) +#define fnic_printk(kern_level, fnic, fmt, ...) \ + (IS_FNIC_FCP_INITIATOR(fnic) ? \ + shost_printk(kern_level, fnic->host, "fnic<%d>: %s: %d: " fmt, \ + fnic->fnic_num, __func__, __LINE__, ##__VA_ARGS__) : \ + printk(kern_level "fnic<%d>: %s: %d: " fmt, fnic->fnic_num, \ + __func__, __LINE__, ##__VA_ARGS__)) + +#define FNIC_MAIN_DBG(kern_level, fnic, fmt, args...) \ + FNIC_CHECK_LOGGING(FNIC_MAIN_LOGGING, \ + fnic_printk(kern_level, fnic, fmt, ##args);) + +#define FNIC_FCS_DBG(kern_level, fnic, fmt, args...) \ + FNIC_CHECK_LOGGING(FNIC_FCS_LOGGING, \ + fnic_printk(kern_level, fnic, fmt, ##args);) + +#define FNIC_FIP_DBG(kern_level, fnic, fmt, args...) \ + FNIC_CHECK_LOGGING(FNIC_FIP_LOGGING, \ + fnic_printk(kern_level, fnic, fmt, ##args);) + +#define FNIC_SCSI_DBG(kern_level, fnic, fmt, args...) \ + FNIC_CHECK_LOGGING(FNIC_SCSI_LOGGING, \ + fnic_printk(kern_level, fnic, fmt, ##args);) + +#define FNIC_ISR_DBG(kern_level, fnic, fmt, args...) \ + FNIC_CHECK_LOGGING(FNIC_ISR_LOGGING, \ + fnic_printk(kern_level, fnic, fmt, ##args);) + +#define FNIC_NVME_DBG(kern_level, fnic, fmt, args...) \ + FNIC_CHECK_LOGGING(FNIC_NVME_LOGGING, \ + fnic_printk(kern_level, fnic, fmt, ##args);) + +#define FNIC_MAIN_NOTE(kern_level, fnic, fmt, args...) \ + fnic_printk(kern_level, fnic, fmt, ##args) #define FNIC_WQ_COPY_MAX 64 #define FNIC_WQ_MAX 1 @@ -325,6 +332,7 @@ enum fnic_state { enum fnic_role_e { FNIC_ROLE_FCP_INITIATOR = 0, + FNIC_ROLE_NVME_INITIATOR, }; enum fnic_evt { @@ -598,7 +606,7 @@ fnic_debug_dump(struct fnic *fnic, uint8_t *u8arr, int len) int i; for (i = 0; i < len; i = i+8) { - FNIC_FCS_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "%d: %02x %02x %02x %02x %02x %02x %02x %02x", i / 8, u8arr[i + 0], u8arr[i + 1], u8arr[i + 2], u8arr[i + 3], u8arr[i + 4], u8arr[i + 5], u8arr[i + 6], u8arr[i + 7]); @@ -613,7 +621,7 @@ fnic_debug_dump_fc_frame(struct fnic *fnic, struct fc_frame_header *fchdr, s_id = ntoh24(fchdr->fh_s_id); d_id = ntoh24(fchdr->fh_d_id); - FNIC_FCS_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "%s packet contents: sid/did/type/oxid = 0x%x/0x%x/0x%x/0x%x (len = %d)\n", pfx, s_id, d_id, fchdr->fh_type, FNIC_STD_GET_OX_ID(fchdr), len); diff --git a/drivers/scsi/fnic/fnic_fcs.c b/drivers/scsi/fnic/fnic_fcs.c index 063eb864a5cd0..ca592bc3f6182 100644 --- a/drivers/scsi/fnic/fnic_fcs.c +++ b/drivers/scsi/fnic/fnic_fcs.c @@ -39,7 +39,7 @@ static uint8_t FCOE_ALL_FCF_MAC[6] = FC_FCOE_FLOGI_MAC; static inline void fnic_fdls_set_fcoe_srcmac(struct fnic *fnic, uint8_t *src_mac) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Setting src mac: %02x:%02x:%02x:%02x:%02x:%02x", src_mac[0], src_mac[1], src_mac[2], src_mac[3], src_mac[4], src_mac[5]); @@ -54,7 +54,7 @@ static inline void fnic_fdls_set_fcoe_srcmac(struct fnic *fnic, static inline void fnic_fdls_set_fcoe_dstmac(struct fnic *fnic, uint8_t *dst_mac) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Setting dst mac: %02x:%02x:%02x:%02x:%02x:%02x", dst_mac[0], dst_mac[1], dst_mac[2], dst_mac[3], dst_mac[4], dst_mac[5]); @@ -82,7 +82,7 @@ void fnic_fdls_link_status_change(struct fnic *fnic, int linkup) { struct fnic_iport_s *iport = &fnic->iport; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "link up: %d, usefip: %d", linkup, iport->usefip); spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); @@ -90,12 +90,12 @@ void fnic_fdls_link_status_change(struct fnic *fnic, int linkup) if (linkup) { if (iport->usefip) { iport->state = FNIC_IPORT_STATE_FIP; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "link up: %d, usefip: %d", linkup, iport->usefip); fnic_fcoe_send_vlan_req(fnic); } else { iport->state = FNIC_IPORT_STATE_FABRIC_DISC; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iport->state: %d", iport->state); fnic_fdls_disc_start(iport); } @@ -125,13 +125,13 @@ void fnic_fdls_learn_fcoe_macs(struct fnic_iport_s *iport, void *rx_frame, memcpy(&fcmac[3], fcid, 3); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "learn fcoe: dst_mac: %02x:%02x:%02x:%02x:%02x:%02x", ethhdr->h_dest[0], ethhdr->h_dest[1], ethhdr->h_dest[2], ethhdr->h_dest[3], ethhdr->h_dest[4], ethhdr->h_dest[5]); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "learn fcoe: fc_mac: %02x:%02x:%02x:%02x:%02x:%02x", fcmac[0], fcmac[1], fcmac[2], fcmac[3], fcmac[4], fcmac[5]); @@ -149,7 +149,7 @@ void fnic_fdls_init(struct fnic *fnic, int usefip) iport->fnic = fnic; iport->usefip = usefip; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "iportsrcmac: %02x:%02x:%02x:%02x:%02x:%02x", iport->hwmac[0], iport->hwmac[1], iport->hwmac[2], iport->hwmac[3], iport->hwmac[4], iport->hwmac[5]); @@ -168,14 +168,14 @@ void fnic_handle_link(struct work_struct *work) int max_count = 0; if (vnic_dev_get_intr_mode(fnic->vdev) != VNIC_DEV_INTR_MODE_MSI) - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Interrupt mode is not MSI\n"); spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); if (fnic->stop_rx_link_events) { spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Stop link rx events\n"); return; } @@ -184,10 +184,10 @@ void fnic_handle_link(struct work_struct *work) if ((fnic->state != FNIC_IN_ETH_MODE) && (fnic->state != FNIC_IN_FC_MODE)) { spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "fnic in transitional state: %d. link up: %d ignored", fnic->state, vnic_dev_link_status(fnic->vdev)); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Current link status: %d iport state: %d\n", fnic->link_status, fnic->iport.state); return; @@ -199,36 +199,36 @@ void fnic_handle_link(struct work_struct *work) fnic->link_down_cnt = vnic_dev_link_down_cnt(fnic->vdev); while (fnic->reset_in_progress == IN_PROGRESS) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "fnic reset in progress. Link event needs to wait\n"); spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "waiting for reset completion\n"); wait_for_completion_timeout(&fnic->reset_completion_wait, msecs_to_jiffies(5000)); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "woken up from reset completion wait\n"); spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); max_count++; if (max_count >= MAX_RESET_WAIT_COUNT) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Rstth waited for too long. Skipping handle link event\n"); spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); return; } } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Marking fnic reset in progress\n"); fnic->reset_in_progress = IN_PROGRESS; if ((vnic_dev_get_intr_mode(fnic->vdev) != VNIC_DEV_INTR_MODE_MSI) || (fnic->link_status != old_link_status)) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "old link status: %d link status: %d\n", old_link_status, (int) fnic->link_status); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "old down count %d down count: %d\n", old_link_down_cnt, (int) fnic->link_down_cnt); } @@ -237,36 +237,36 @@ void fnic_handle_link(struct work_struct *work) if (!fnic->link_status) { /* DOWN -> DOWN */ spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "down->down\n"); } else { if (old_link_down_cnt != fnic->link_down_cnt) { /* UP -> DOWN -> UP */ spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "up->down. Link down\n"); fnic_fdls_link_status_change(fnic, 0); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "down->up. Link up\n"); fnic_fdls_link_status_change(fnic, 1); } else { /* UP -> UP */ spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "up->up\n"); } } } else if (fnic->link_status) { /* DOWN -> UP */ spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "down->up. Link up\n"); fnic_fdls_link_status_change(fnic, 1); } else { /* UP -> DOWN */ spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "up->down. Link down\n"); fnic_fdls_link_status_change(fnic, 0); } @@ -275,7 +275,7 @@ void fnic_handle_link(struct work_struct *work) fnic->reset_in_progress = NOT_IN_PROGRESS; complete(&fnic->reset_completion_wait); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Marking fnic reset completion\n"); spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); } @@ -302,7 +302,7 @@ void fnic_handle_frame(struct work_struct *work) */ if (fnic->state != FNIC_IN_FC_MODE && fnic->state != FNIC_IN_ETH_MODE) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Cannot process frame in transitional state\n"); spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); return; @@ -328,7 +328,7 @@ void fnic_handle_fip_frame(struct work_struct *work) struct fnic_frame_list *cur_frame, *next; struct fnic *fnic = container_of(work, struct fnic, fip_frame_work); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Processing FIP frame\n"); spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); @@ -407,7 +407,7 @@ void fnic_update_mac_locked(struct fnic *fnic, u8 *new) if (ether_addr_equal(data, new)) return; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Update MAC: %u\n", *new); if (!is_zero_ether_addr(data) && !ether_addr_equal(data, ctl)) @@ -477,7 +477,7 @@ static void fnic_rq_cmpl_frame_recv(struct vnic_rq *rq, struct cq_desc if (!fcs_ok) { atomic64_inc(&fnic_stats->misc_stats.frame_errors); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "fnic 0x%p fcs error. Dropping packet.\n", fnic); goto drop; } @@ -487,21 +487,21 @@ static void fnic_rq_cmpl_frame_recv(struct vnic_rq *rq, struct cq_desc if (fnic_import_rq_eth_pkt(fnic, fp)) return; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Dropping h_proto 0x%x", be16_to_cpu(eh->h_proto)); goto drop; } } else { /* wrong CQ type */ - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "fnic rq_cmpl wrong cq type x%x\n", type); goto drop; } if (!fcs_ok || packet_error || !fcoe_fnic_crc_ok || fcoe_enc_error) { atomic64_inc(&fnic_stats->misc_stats.frame_errors); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "fcoe %x fcsok %x pkterr %x ffco %x fee %x\n", fcoe, fcs_ok, packet_error, fcoe_fnic_crc_ok, fcoe_enc_error); @@ -511,7 +511,7 @@ static void fnic_rq_cmpl_frame_recv(struct vnic_rq *rq, struct cq_desc spin_lock_irqsave(&fnic->fnic_lock, flags); if (fnic->stop_rx_link_events) { spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "fnic->stop_rx_link_events: %d\n", fnic->stop_rx_link_events); goto drop; @@ -521,7 +521,7 @@ static void fnic_rq_cmpl_frame_recv(struct vnic_rq *rq, struct cq_desc frame_elem = mempool_alloc(fnic->frame_elem_pool, GFP_ATOMIC); if (!frame_elem) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Failed to allocate memory for frame elem"); goto drop; } @@ -567,7 +567,7 @@ int fnic_rq_cmpl_handler(struct fnic *fnic, int rq_work_to_do) if (cur_work_done && fnic->stop_rx_link_events != 1) { err = vnic_rq_fill(&fnic->rq[i], fnic_alloc_rq_frame); if (err) - shost_printk(KERN_ERR, fnic->host, + fnic_printk(KERN_ERR, fnic, "fnic_alloc_rq_frame can't alloc" " frame\n"); } @@ -593,7 +593,7 @@ int fnic_alloc_rq_frame(struct vnic_rq *rq) len = FNIC_FRAME_HT_ROOM; buf = mempool_alloc(fnic->frame_recv_pool, GFP_ATOMIC); if (!buf) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Unable to allocate RQ buffer of size: %d\n", len); return -ENOMEM; } @@ -601,7 +601,7 @@ int fnic_alloc_rq_frame(struct vnic_rq *rq) pa = dma_map_single(&fnic->pdev->dev, buf, len, DMA_FROM_DEVICE); if (dma_mapping_error(&fnic->pdev->dev, pa)) { ret = -ENOMEM; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "PCI mapping failed with error %d\n", ret); goto free_buf; } @@ -642,7 +642,7 @@ static int fnic_send_frame(struct fnic *fnic, void *frame, int frame_len) if ((fnic_fc_trace_set_data(fnic->fnic_num, FNIC_FC_SEND | 0x80, (char *) frame, frame_len)) != 0) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "fnic ctlr frame trace error"); } @@ -650,7 +650,7 @@ static int fnic_send_frame(struct fnic *fnic, void *frame, int frame_len) if (!vnic_wq_desc_avail(wq)) { dma_unmap_single(&fnic->pdev->dev, pa, frame_len, DMA_TO_DEVICE); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "vnic work queue descriptor is not available"); ret = -1; goto fnic_send_frame_end; @@ -706,13 +706,13 @@ fdls_send_fcoe_frame(struct fnic *fnic, void *frame, int frame_size, && (fnic->state != FNIC_IN_ETH_MODE)) { frame_elem = mempool_alloc(fnic->frame_elem_pool, GFP_ATOMIC); if (!frame_elem) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Failed to allocate memory for frame elem"); return -ENOMEM; } memset(frame_elem, 0, sizeof(struct fnic_frame_list)); - FNIC_FCS_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Queueing FC frame: sid/did/type/oxid = 0x%x/0x%x/0x%x/0x%x\n", ntoh24(fchdr->fh_s_id), ntoh24(fchdr->fh_d_id), fchdr->fh_type, FNIC_STD_GET_OX_ID(fchdr)); @@ -778,7 +778,7 @@ void fnic_flush_tx(struct work_struct *work) struct fc_frame *fp; struct fnic_frame_list *cur_frame, *next; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Flush queued frames"); list_for_each_entry_safe(cur_frame, next, &fnic->tx_queue, links) { @@ -797,7 +797,7 @@ fnic_fdls_register_portid(struct fnic_iport_s *iport, u32 port_id, struct ethhdr *ethhdr; int ret; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Setting port id: 0x%x fp: 0x%p fnic state: %d", port_id, fp, fnic->state); @@ -810,7 +810,7 @@ fnic_fdls_register_portid(struct fnic_iport_s *iport, u32 port_id, if (fnic->state == FNIC_IN_ETH_MODE || fnic->state == FNIC_IN_FC_MODE) fnic->state = FNIC_IN_ETH_TRANS_FC_MODE; else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Unexpected fnic state while processing FLOGI response\n"); return -1; } @@ -821,7 +821,7 @@ fnic_fdls_register_portid(struct fnic_iport_s *iport, u32 port_id, */ ret = fnic_flogi_reg_handler(fnic, port_id); if (ret < 0) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FLOGI registration error ret: %d fnic state: %d\n", ret, fnic->state); if (fnic->state == FNIC_IN_ETH_TRANS_FC_MODE) @@ -831,7 +831,7 @@ fnic_fdls_register_portid(struct fnic_iport_s *iport, u32 port_id, } iport->fabric.flags |= FNIC_FDLS_FPMA_LEARNT; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FLOGI registration success\n"); return 0; } @@ -931,7 +931,7 @@ fnic_fdls_add_tport(struct fnic_iport_s *iport, struct fnic_tport_s *tport, struct fc_rport_identifiers ids; struct rport_dd_data_s *rdd_data; - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Adding rport fcid: 0x%x", tport->fcid); ids.node_name = tport->wwnn; @@ -943,12 +943,12 @@ fnic_fdls_add_tport(struct fnic_iport_s *iport, struct fnic_tport_s *tport, rport = fc_remote_port_add(fnic->host, 0, &ids); spin_lock_irqsave(&fnic->fnic_lock, flags); if (!rport) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Failed to add rport for tport: 0x%x", tport->fcid); return; } - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Added rport fcid: 0x%x", tport->fcid); /* Mimic these assignments in queuecommand to avoid timing issues */ @@ -987,7 +987,7 @@ fnic_fdls_remove_tport(struct fnic_iport_s *iport, fc_remote_port_delete(rport); spin_lock_irqsave(&fnic->fnic_lock, flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Deregistered and freed tport fcid: 0x%x from scsi transport fc", tport->fcid); @@ -1014,7 +1014,7 @@ void fnic_delete_fcp_tports(struct fnic *fnic) spin_lock_irqsave(&fnic->fnic_lock, flags); list_for_each_entry_safe(tport, next, &fnic->iport.tport_list, links) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "removing fcp rport fcid: 0x%x", tport->fcid); fdls_set_tport_state(tport, FDLS_TGT_STATE_OFFLINING); fnic_del_tport_timer_sync(fnic, tport); @@ -1041,36 +1041,36 @@ void fnic_tport_event_handler(struct work_struct *work) tport = cur_evt->arg1; switch (cur_evt->event) { case TGT_EV_RPORT_ADD: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Add rport event"); if (tport->state == FDLS_TGT_STATE_READY) { fnic_fdls_add_tport(&fnic->iport, (struct fnic_tport_s *) cur_evt->arg1, flags); } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Target not ready. Add rport event dropped: 0x%x", tport->fcid); } break; case TGT_EV_RPORT_DEL: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Remove rport event"); if (tport->state == FDLS_TGT_STATE_OFFLINING) { fnic_fdls_remove_tport(&fnic->iport, (struct fnic_tport_s *) cur_evt->arg1, flags); } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "remove rport event dropped tport fcid: 0x%x", tport->fcid); } break; case TGT_EV_TPORT_DELETE: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Delete tport event"); fdls_delete_tport(tport->iport, tport); break; default: - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Unknown tport event"); break; } @@ -1142,7 +1142,7 @@ void fnic_fcpio_reset(struct fnic *fnic) if (unlikely(fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)) { /* fw reset is in progress, poll for its completion */ spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "fnic is in unexpected state: %d for fw_reset\n", fnic->state); return; @@ -1155,7 +1155,7 @@ void fnic_fcpio_reset(struct fnic *fnic) fnic->fw_reset_done = &fw_reset_done; spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Issuing fw reset\n"); if (fnic_fw_reset_handler(fnic)) { spin_lock_irqsave(&fnic->fnic_lock, flags); @@ -1163,14 +1163,14 @@ void fnic_fcpio_reset(struct fnic *fnic) fnic->state = old_state; spin_unlock_irqrestore(&fnic->fnic_lock, flags); } else { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Waiting for fw completion\n"); time_remain = wait_for_completion_timeout(&fw_reset_done, msecs_to_jiffies(FNIC_FW_RESET_TIMEOUT)); - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "Woken up after fw completion timeout\n"); if (time_remain == 0) { - FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_FCS_DBG(KERN_INFO, fnic, "FW reset completion timed out after %d ms\n", FNIC_FW_RESET_TIMEOUT); } diff --git a/drivers/scsi/fnic/fnic_isr.c b/drivers/scsi/fnic/fnic_isr.c index 7ed50b11afa6a..43149d8312ec6 100644 --- a/drivers/scsi/fnic/fnic_isr.c +++ b/drivers/scsi/fnic/fnic_isr.c @@ -222,7 +222,7 @@ int fnic_request_intr(struct fnic *fnic) fnic->msix[i].devname, fnic->msix[i].devid); if (err) { - FNIC_ISR_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_ERR, fnic, "request_irq failed with error: %d\n", err); fnic_free_intr(fnic); @@ -245,15 +245,22 @@ int fnic_set_intr_mode_msix(struct fnic *fnic) unsigned int m = ARRAY_SIZE(fnic->wq); unsigned int o = ARRAY_SIZE(fnic->hw_copy_wq); unsigned int min_irqs = n + m + 1 + 1; /*rq, raw wq, wq, err*/ - + /* + * Make driver critical vectors unmanaged, or else it can get tied + * to an offline CPU. This can happen when hyper-threading is off. + */ + struct irq_affinity affd = { + .pre_vectors = n + m + 1, /* rq, raw wq, 1 ioq */ + .post_vectors = 1, /* err */ + }; /* * We need n RQs, m WQs, o Copy WQs, n+m+o CQs, and n+m+o+1 INTRs * (last INTR is used for WQ/RQ errors and notification area) */ - FNIC_ISR_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_INFO, fnic, "rq-array size: %d wq-array size: %d copy-wq array size: %d\n", n, m, o); - FNIC_ISR_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_INFO, fnic, "rq_count: %d raw_wq_count: %d wq_copy_count: %d cq_count: %d\n", fnic->rq_count, fnic->raw_wq_count, fnic->wq_copy_count, fnic->cq_count); @@ -263,19 +270,19 @@ int fnic_set_intr_mode_msix(struct fnic *fnic) int vec_count = 0; int vecs = fnic->rq_count + fnic->raw_wq_count + fnic->wq_copy_count + 1; - vec_count = pci_alloc_irq_vectors(fnic->pdev, min_irqs, vecs, - PCI_IRQ_MSIX | PCI_IRQ_AFFINITY); - FNIC_ISR_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + vec_count = pci_alloc_irq_vectors_affinity(fnic->pdev, min_irqs, + vecs, PCI_IRQ_MSIX|PCI_IRQ_AFFINITY, &affd); + FNIC_ISR_DBG(KERN_INFO, fnic, "allocated %d MSI-X vectors\n", vec_count); if (vec_count > 0) { if (vec_count < vecs) { - FNIC_ISR_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_ERR, fnic, "interrupts number mismatch: vec_count: %d vecs: %d\n", vec_count, vecs); if (vec_count < min_irqs) { - FNIC_ISR_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_ERR, fnic, "no interrupts for copy wq\n"); return 1; } @@ -287,7 +294,7 @@ int fnic_set_intr_mode_msix(struct fnic *fnic) fnic->wq_copy_count = vec_count - n - m - 1; fnic->wq_count = fnic->raw_wq_count + fnic->wq_copy_count; if (fnic->cq_count != vec_count - 1) { - FNIC_ISR_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_ERR, fnic, "CQ count: %d does not match MSI-X vector count: %d\n", fnic->cq_count, vec_count); fnic->cq_count = vec_count - 1; @@ -295,23 +302,23 @@ int fnic_set_intr_mode_msix(struct fnic *fnic) fnic->intr_count = vec_count; fnic->err_intr_offset = fnic->rq_count + fnic->wq_count; - FNIC_ISR_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_INFO, fnic, "rq_count: %d raw_wq_count: %d copy_wq_base: %d\n", fnic->rq_count, fnic->raw_wq_count, fnic->copy_wq_base); - FNIC_ISR_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_INFO, fnic, "wq_copy_count: %d wq_count: %d cq_count: %d\n", fnic->wq_copy_count, fnic->wq_count, fnic->cq_count); - FNIC_ISR_DBG(KERN_INFO, fnic->host, fnic->fnic_num, - "intr_count: %d err_intr_offset: %u", + FNIC_ISR_DBG(KERN_INFO, fnic, + "intr_count: %d err_intr_offset: %u\n", fnic->intr_count, fnic->err_intr_offset); vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_MSIX); - FNIC_ISR_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_INFO, fnic, "fnic using MSI-X\n"); return 0; } @@ -351,7 +358,7 @@ int fnic_set_intr_mode(struct fnic *fnic) fnic->intr_count = 1; fnic->err_intr_offset = 0; - FNIC_ISR_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_DEBUG, fnic, "Using MSI Interrupts\n"); vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_MSI); @@ -377,7 +384,7 @@ int fnic_set_intr_mode(struct fnic *fnic) fnic->cq_count = 3; fnic->intr_count = 3; - FNIC_ISR_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_ISR_DBG(KERN_DEBUG, fnic, "Using Legacy Interrupts\n"); vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_INTX); diff --git a/drivers/scsi/fnic/fnic_main.c b/drivers/scsi/fnic/fnic_main.c index 24d62c0874ac4..de88786a3bb9b 100644 --- a/drivers/scsi/fnic/fnic_main.c +++ b/drivers/scsi/fnic/fnic_main.c @@ -185,7 +185,7 @@ static void fnic_get_host_speed(struct Scsi_Host *shost) u32 port_speed = vnic_dev_port_speed(fnic->vdev); struct fnic_stats *fnic_stats = &fnic->fnic_stats; - FNIC_MAIN_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_MAIN_DBG(KERN_INFO, fnic, "port_speed: %d Mbps", port_speed); atomic64_set(&fnic_stats->misc_stats.port_speed_in_mbps, port_speed); @@ -235,7 +235,7 @@ static void fnic_get_host_speed(struct Scsi_Host *shost) fc_host_speed(shost) = FC_PORTSPEED_128GBIT; break; default: - FNIC_MAIN_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_MAIN_DBG(KERN_INFO, fnic, "Unknown FC speed: %d Mbps", port_speed); fc_host_speed(shost) = FC_PORTSPEED_UNKNOWN; break; @@ -261,7 +261,7 @@ static struct fc_host_statistics *fnic_get_stats(struct Scsi_Host *host) spin_unlock_irqrestore(&fnic->fnic_lock, flags); if (ret) { - FNIC_MAIN_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_MAIN_DBG(KERN_DEBUG, fnic, "fnic: Get vnic stats failed: 0x%x", ret); return stats; } @@ -287,64 +287,66 @@ static struct fc_host_statistics *fnic_get_stats(struct Scsi_Host *host) void fnic_dump_fchost_stats(struct Scsi_Host *host, struct fc_host_statistics *stats) { - FNIC_MAIN_NOTE(KERN_NOTICE, host, + struct fnic *fnic = *((struct fnic **) shost_priv(host)); + + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: seconds since last reset = %llu\n", stats->seconds_since_last_reset); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: tx frames = %llu\n", stats->tx_frames); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: tx words = %llu\n", stats->tx_words); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: rx frames = %llu\n", stats->rx_frames); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: rx words = %llu\n", stats->rx_words); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: lip count = %llu\n", stats->lip_count); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: nos count = %llu\n", stats->nos_count); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: error frames = %llu\n", stats->error_frames); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: dumped frames = %llu\n", stats->dumped_frames); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: link failure count = %llu\n", stats->link_failure_count); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: loss of sync count = %llu\n", stats->loss_of_sync_count); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: loss of signal count = %llu\n", stats->loss_of_signal_count); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: prim seq protocol err count = %llu\n", stats->prim_seq_protocol_err_count); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: invalid tx word count= %llu\n", stats->invalid_tx_word_count); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: invalid crc count = %llu\n", stats->invalid_crc_count); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: fcp input requests = %llu\n", stats->fcp_input_requests); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: fcp output requests = %llu\n", stats->fcp_output_requests); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: fcp control requests = %llu\n", stats->fcp_control_requests); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: fcp input megabytes = %llu\n", stats->fcp_input_megabytes); - FNIC_MAIN_NOTE(KERN_NOTICE, host, + FNIC_MAIN_NOTE(KERN_NOTICE, fnic, "fnic: fcp output megabytes = %llu\n", stats->fcp_output_megabytes); return; @@ -370,7 +372,7 @@ static void fnic_reset_host_stats(struct Scsi_Host *host) spin_unlock_irqrestore(&fnic->fnic_lock, flags); if (ret) { - FNIC_MAIN_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_MAIN_DBG(KERN_DEBUG, fnic, "fnic: Reset vnic stats failed" " 0x%x", ret); return; @@ -678,27 +680,58 @@ static int fnic_scsi_drv_init(struct fnic *fnic) void fnic_mq_map_queues_cpus(struct Scsi_Host *host) { + const struct cpumask *mask; + unsigned int queue, cpu; struct fnic *fnic = *((struct fnic **) shost_priv(host)); struct pci_dev *l_pdev = fnic->pdev; int intr_mode = fnic->config.intr_mode; struct blk_mq_queue_map *qmap = &host->tag_set.map[HCTX_TYPE_DEFAULT]; if (intr_mode == VNIC_DEV_INTR_MODE_MSI || intr_mode == VNIC_DEV_INTR_MODE_INTX) { - FNIC_MAIN_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_MAIN_DBG(KERN_ERR, fnic, "intr_mode is not msix\n"); return; } - FNIC_MAIN_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_MAIN_DBG(KERN_INFO, fnic, "qmap->nr_queues: %d\n", qmap->nr_queues); if (l_pdev == NULL) { - FNIC_MAIN_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_MAIN_DBG(KERN_ERR, fnic, "l_pdev is null\n"); return; } - blk_mq_map_hw_queues(qmap, &l_pdev->dev, FNIC_PCI_OFFSET); + for_each_possible_cpu(cpu) + qmap->mq_map[cpu] = 0; + + /* + * Setup CPU to Queue mapping for all managed MSI-X IRQs. + * Q0 is driver critical and non-managed, hence start from Q1. + */ + for (queue = 1; queue < qmap->nr_queues; queue++) { + int irq_num = pci_irq_vector(fnic->pdev, + queue + FNIC_PCI_OFFSET); + + if (irq_num < 0) + continue; + + mask = pci_irq_get_affinity(fnic->pdev, + queue + FNIC_PCI_OFFSET); + if (!mask) { + shost_printk(KERN_ERR, host, + "failed to get irq_affinity map for queue:%d\n", irq_num); + continue; + } + FNIC_MAIN_DBG(KERN_INFO, fnic, + "got irq_affinity map for %d:\n", irq_num); + for_each_cpu(cpu, mask) { + qmap->mq_map[cpu] = qmap->queue_offset + queue; + FNIC_MAIN_DBG(KERN_INFO, fnic, + "[Q%d] cpu:%d <=> irq:%d\n", + queue, cpu, irq_num); + } + } } static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c index 6ee3c559e129c..0759540f66753 100644 --- a/drivers/scsi/fnic/fnic_scsi.c +++ b/drivers/scsi/fnic/fnic_scsi.c @@ -148,7 +148,7 @@ unsigned int fnic_count_ioreqs(struct fnic *fnic, u32 portid) fnic_scsi_io_iter(fnic, fnic_count_portid_ioreqs_iter, &portid, &count); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "portid = 0x%x count = %u\n", portid, count); return count; } @@ -180,7 +180,7 @@ fnic_count_lun_ioreqs(struct fnic *fnic, struct scsi_device *scsi_device) fnic_scsi_io_iter(fnic, fnic_count_lun_ioreqs_iter, scsi_device, &count); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "lun = %p count = %u\n", scsi_device, count); return count; } @@ -268,7 +268,7 @@ int fnic_fw_reset_handler(struct fnic *fnic) if (!vnic_wq_copy_desc_avail(wq)) ret = -EAGAIN; else { - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "ioreq_count: %u\n", ioreq_count); fnic_queue_wq_copy_desc_fw_reset(wq, SCSI_NO_TAG); atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs); @@ -283,11 +283,11 @@ int fnic_fw_reset_handler(struct fnic *fnic) if (!ret) { atomic64_inc(&fnic->fnic_stats.reset_stats.fw_resets); - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "Issued fw reset\n"); } else { fnic_clear_state_flags(fnic, FNIC_FLAGS_FWRESET); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "Failed to issue fw reset\n"); } @@ -326,13 +326,13 @@ int fnic_flogi_reg_handler(struct fnic *fnic, u32 fc_id) fc_id, gw_mac, fnic->iport.fpma, iport->r_a_tov, iport->e_d_tov); - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "FLOGI FIP reg issued fcid: 0x%x src %p dest %p\n", fc_id, fnic->iport.fpma, gw_mac); } else { fnic_queue_wq_copy_desc_flogi_reg(wq, SCSI_NO_TAG, format, fc_id, gw_mac); - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "FLOGI reg issued fcid 0x%x dest %p\n", fc_id, gw_mac); } @@ -413,7 +413,7 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, free_wq_copy_descs(fnic, wq, hwq); if (unlikely(!vnic_wq_copy_desc_avail(wq))) { - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "fnic_queue_wq_copy_desc failure - no descriptors\n"); atomic64_inc(&misc_stats->io_cpwq_alloc_failures); return SCSI_MLQUEUE_HOST_BUSY; @@ -478,7 +478,7 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost, rport = starget_to_rport(scsi_target(sc->device)); if (!rport) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "returning DID_NO_CONNECT for IO as rport is NULL\n"); sc->result = DID_NO_CONNECT << 16; done(sc); @@ -487,7 +487,7 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost, ret = fc_remote_port_chkready(rport); if (ret) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "rport is not ready\n"); atomic64_inc(&fnic_stats->misc_stats.tport_not_ready); sc->result = ret; @@ -501,7 +501,7 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost, if (iport->state != FNIC_IPORT_STATE_READY) { spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "returning DID_NO_CONNECT for IO as iport state: %d\n", iport->state); sc->result = DID_NO_CONNECT << 16; @@ -515,13 +515,13 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost, rdd_data = rport->dd_data; tport = rdd_data->tport; if (!tport || (rdd_data->iport != iport)) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "dd_data not yet set in SCSI for rport portid: 0x%x\n", rport->port_id); tport = fnic_find_tport_by_fcid(iport, rport->port_id); if (!tport) { spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "returning DID_BUS_BUSY for IO as tport not found for: 0x%x\n", rport->port_id); sc->result = DID_BUS_BUSY << 16; @@ -544,7 +544,7 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost, if ((tport->state != FDLS_TGT_STATE_READY) && (tport->state != FDLS_TGT_STATE_ADISC)) { spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "returning DID_NO_CONNECT for IO as tport state: %d\n", tport->state); sc->result = DID_NO_CONNECT << 16; @@ -564,7 +564,7 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost, if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_FWRESET))) { spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "fnic flags FW reset: 0x%lx. Returning SCSI_MLQUEUE_HOST_BUSY\n", fnic->state_flags); return SCSI_MLQUEUE_HOST_BUSY; @@ -704,7 +704,7 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost, atomic_dec(&tport->in_flight); if (lun0_delay) { - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "LUN0 delay\n"); mdelay(LUN0_DELAY_TIME); } @@ -744,12 +744,12 @@ static int fnic_fcpio_fw_reset_cmpl_handler(struct fnic *fnic, if (fnic->state == FNIC_IN_FC_TRANS_ETH_MODE) { /* Check status of reset completion */ if (!hdr_status) { - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "reset cmpl success\n"); /* Ready to send flogi out */ fnic->state = FNIC_IN_ETH_MODE; } else { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "reset failed with header status: %s\n", fnic_fcpio_status_to_str(hdr_status)); @@ -758,7 +758,7 @@ static int fnic_fcpio_fw_reset_cmpl_handler(struct fnic *fnic, ret = -1; } } else { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "Unexpected state while processing reset completion: %s\n", fnic_state_to_str(fnic->state)); atomic64_inc(&reset_stats->fw_reset_failures); @@ -810,19 +810,18 @@ static int fnic_fcpio_flogi_reg_cmpl_handler(struct fnic *fnic, /* Check flogi registration completion status */ if (!hdr_status) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "FLOGI reg succeeded\n"); fnic->state = FNIC_IN_FC_MODE; } else { - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "fnic flogi reg failed: %s\n", fnic_fcpio_status_to_str(hdr_status)); fnic->state = FNIC_IN_ETH_MODE; ret = -1; } } else { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "Unexpected fnic state %s while" " processing flogi reg completion\n", fnic_state_to_str(fnic->state)); @@ -933,19 +932,19 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, unsigned int cq_ind hwq = blk_mq_unique_tag_to_hwq(mqtag); if (hwq != cq_index) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", hwq, mqtag, tag, cq_index); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hdr status: %s icmnd completion on the wrong queue\n", fnic_fcpio_status_to_str(hdr_status)); } if (tag >= fnic->fnic_max_tag_id) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", hwq, mqtag, tag, cq_index); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hdr status: %s Out of range tag\n", fnic_fcpio_status_to_str(hdr_status)); return; @@ -957,7 +956,7 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, unsigned int cq_ind if (!sc) { atomic64_inc(&fnic_stats->io_stats.sc_null); spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - shost_printk(KERN_ERR, fnic->host, + fnic_printk(KERN_ERR, fnic, "icmnd_cmpl sc is null - " "hdr status = %s tag = 0x%x desc = 0x%p\n", fnic_fcpio_status_to_str(hdr_status), id, desc); @@ -985,7 +984,7 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, unsigned int cq_ind atomic64_inc(&fnic_stats->io_stats.ioreq_null); fnic_priv(sc)->flags |= FNIC_IO_REQ_NULL; spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - shost_printk(KERN_ERR, fnic->host, + fnic_printk(KERN_ERR, fnic, "icmnd_cmpl io_req is null - " "hdr status = %s tag = 0x%x sc 0x%p\n", fnic_fcpio_status_to_str(hdr_status), id, sc); @@ -1012,7 +1011,7 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, unsigned int cq_ind if(FCPIO_ABORTED == hdr_status) fnic_priv(sc)->flags |= FNIC_IO_ABORTED; - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "icmnd_cmpl abts pending " "hdr status = %s tag = 0x%x sc = 0x%p " "scsi_status = %x residual = %d\n", @@ -1044,7 +1043,7 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, unsigned int cq_ind if (icmnd_cmpl->scsi_status == SAM_STAT_TASK_SET_FULL) atomic64_inc(&fnic_stats->misc_stats.queue_fulls); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "xfer_len: %llu", xfer_len); break; @@ -1107,7 +1106,7 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, unsigned int cq_ind if (hdr_status != FCPIO_SUCCESS) { atomic64_inc(&fnic_stats->io_stats.io_failures); - shost_printk(KERN_ERR, fnic->host, "hdr status = %s\n", + fnic_printk(KERN_ERR, fnic, "hdr status = %s\n", fnic_fcpio_status_to_str(hdr_status)); } @@ -1200,27 +1199,27 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde hwq = blk_mq_unique_tag_to_hwq(id & FNIC_TAG_MASK); if (hwq != cq_index) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", hwq, mqtag, tag, cq_index); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hdr status: %s ITMF completion on the wrong queue\n", fnic_fcpio_status_to_str(hdr_status)); } if (tag > fnic->fnic_max_tag_id) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", hwq, mqtag, tag, cq_index); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hdr status: %s Tag out of range\n", fnic_fcpio_status_to_str(hdr_status)); return; } else if ((tag == fnic->fnic_max_tag_id) && !(id & FNIC_TAG_DEV_RST)) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", hwq, mqtag, tag, cq_index); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hdr status: %s Tag out of range\n", fnic_fcpio_status_to_str(hdr_status)); return; @@ -1243,7 +1242,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde if (!sc) { atomic64_inc(&fnic_stats->io_stats.sc_null); spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - shost_printk(KERN_ERR, fnic->host, + fnic_printk(KERN_ERR, fnic, "itmf_cmpl sc is null - hdr status = %s tag = 0x%x\n", fnic_fcpio_status_to_str(hdr_status), tag); return; @@ -1255,7 +1254,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde atomic64_inc(&fnic_stats->io_stats.ioreq_null); spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_REQ_NULL; - shost_printk(KERN_ERR, fnic->host, + fnic_printk(KERN_ERR, fnic, "itmf_cmpl io_req is null - " "hdr status = %s tag = 0x%x sc 0x%p\n", fnic_fcpio_status_to_str(hdr_status), tag, sc); @@ -1266,7 +1265,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde if ((id & FNIC_TAG_ABORT) && (id & FNIC_TAG_DEV_RST)) { /* Abort and terminate completion of device reset req */ /* REVISIT : Add asserts about various flags */ - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s Abt/term completion received\n", hwq, mqtag, tag, fnic_fcpio_status_to_str(hdr_status)); @@ -1278,7 +1277,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } else if (id & FNIC_TAG_ABORT) { /* Completion of abort cmd */ - shost_printk(KERN_DEBUG, fnic->host, + fnic_printk(KERN_DEBUG, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x Abort header status: %s\n", hwq, mqtag, tag, fnic_fcpio_status_to_str(hdr_status)); @@ -1293,7 +1292,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde &term_stats->terminate_fw_timeouts); break; case FCPIO_ITMF_REJECTED: - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "abort reject recd. id %d\n", (int)(id & FNIC_TAG_MASK)); break; @@ -1328,7 +1327,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde if (!(fnic_priv(sc)->flags & (FNIC_IO_ABORTED | FNIC_IO_DONE))) atomic64_inc(&misc_stats->no_icmnd_itmf_cmpls); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "abts cmpl recd. id %d status %s\n", (int)(id & FNIC_TAG_MASK), fnic_fcpio_status_to_str(hdr_status)); @@ -1341,11 +1340,11 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde if (io_req->abts_done) { complete(io_req->abts_done); spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - shost_printk(KERN_INFO, fnic->host, + fnic_printk(KERN_INFO, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x Waking up abort thread\n", hwq, mqtag, tag); } else { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s Completing IO\n", hwq, mqtag, tag, fnic_fcpio_status_to_str(hdr_status)); @@ -1376,7 +1375,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde } } else if (id & FNIC_TAG_DEV_RST) { /* Completion of device reset */ - shost_printk(KERN_INFO, fnic->host, + fnic_printk(KERN_INFO, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x DR hst: %s\n", hwq, mqtag, tag, fnic_fcpio_status_to_str(hdr_status)); @@ -1388,7 +1387,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde sc->device->host->host_no, id, sc, jiffies_to_msecs(jiffies - start_time), desc, 0, fnic_flags_and_state(sc)); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s Terminate pending\n", hwq, mqtag, tag, fnic_fcpio_status_to_str(hdr_status)); @@ -1401,7 +1400,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde sc->device->host->host_no, id, sc, jiffies_to_msecs(jiffies - start_time), desc, 0, fnic_flags_and_state(sc)); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "dev reset cmpl recd after time out. " "id %d status %s\n", (int)(id & FNIC_TAG_MASK), @@ -1410,7 +1409,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde } fnic_priv(sc)->state = FNIC_IOREQ_CMD_COMPLETE; fnic_priv(sc)->flags |= FNIC_DEV_RST_DONE; - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s DR completion received\n", hwq, mqtag, tag, fnic_fcpio_status_to_str(hdr_status)); @@ -1419,7 +1418,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_inde spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } else { - shost_printk(KERN_ERR, fnic->host, + fnic_printk(KERN_ERR, fnic, "%s: Unexpected itmf io state: hwq: %d tag 0x%x %s\n", __func__, hwq, id, fnic_ioreq_state_to_str(fnic_priv(sc)->state)); spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); @@ -1474,7 +1473,7 @@ static int fnic_fcpio_cmpl_handler(struct vnic_dev *vdev, break; default: - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "firmware completion type %d\n", desc->hdr.type); break; @@ -1535,7 +1534,7 @@ static bool fnic_cleanup_io_iter(struct scsi_cmnd *sc, void *data) io_req = fnic_priv(sc)->io_req; if (!io_req) { spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hwq: %d mqtag: 0x%x tag: 0x%x flags: 0x%x No ioreq. Returning\n", hwq, mqtag, tag, fnic_priv(sc)->flags); return true; @@ -1573,7 +1572,7 @@ static bool fnic_cleanup_io_iter(struct scsi_cmnd *sc, void *data) mempool_free(io_req, fnic->io_req_pool); sc->result = DID_TRANSPORT_DISRUPTED << 16; - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "mqtag: 0x%x tag: 0x%x sc: 0x%p duration = %lu DID_TRANSPORT_DISRUPTED\n", mqtag, tag, sc, (jiffies - start_time)); @@ -1605,7 +1604,7 @@ static void fnic_cleanup_io(struct fnic *fnic, int exclude_id) struct scsi_cmnd *sc = NULL; io_count = fnic_count_all_ioreqs(fnic); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "Outstanding ioreq count: %d active io count: %lld Waiting\n", io_count, atomic64_read(&fnic->fnic_stats.io_stats.active_ios)); @@ -1630,7 +1629,7 @@ static void fnic_cleanup_io(struct fnic *fnic, int exclude_id) spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags); while ((io_count = fnic_count_all_ioreqs(fnic))) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "Outstanding ioreq count: %d active io count: %lld Waiting\n", io_count, atomic64_read(&fnic->fnic_stats.io_stats.active_ios)); @@ -1686,7 +1685,7 @@ void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, wq_copy_cleanup_scsi_cmd: sc->result = DID_NO_CONNECT << 16; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "wq_copy_cleanup_handler:" + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "wq_copy_cleanup_handler:" " DID_NO_CONNECT\n"); FNIC_TRACE(fnic_wq_copy_cleanup_handler, @@ -1730,7 +1729,7 @@ static inline int fnic_queue_abort_io_req(struct fnic *fnic, int tag, spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); atomic_dec(&fnic->in_flight); atomic_dec(&tport->in_flight); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "fnic_queue_abort_io_req: failure: no descriptors\n"); atomic64_inc(&misc_stats->abts_cpwq_alloc_failures); return 1; @@ -1775,7 +1774,7 @@ static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data) hwq = blk_mq_unique_tag_to_hwq(abt_tag); if (!sc) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "sc is NULL abt_tag: 0x%x hwq: %d\n", abt_tag, hwq); return true; } @@ -1789,7 +1788,7 @@ static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data) if ((fnic_priv(sc)->flags & FNIC_DEVICE_RESET) && !(fnic_priv(sc)->flags & FNIC_DEV_RST_ISSUED)) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hwq: %d abt_tag: 0x%x flags: 0x%x Device reset is not pending\n", hwq, abt_tag, fnic_priv(sc)->flags); spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); @@ -1806,16 +1805,16 @@ static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data) } if (io_req->abts_done) { - shost_printk(KERN_ERR, fnic->host, + fnic_printk(KERN_ERR, fnic, "fnic_rport_exch_reset: io_req->abts_done is set state is %s\n", fnic_ioreq_state_to_str(fnic_priv(sc)->state)); } if (!(fnic_priv(sc)->flags & FNIC_IO_ISSUED)) { - shost_printk(KERN_ERR, fnic->host, + fnic_printk(KERN_ERR, fnic, "rport_exch_reset IO not yet issued %p abt_tag 0x%x", sc, abt_tag); - shost_printk(KERN_ERR, fnic->host, + fnic_printk(KERN_ERR, fnic, "flags %x state %d\n", fnic_priv(sc)->flags, fnic_priv(sc)->state); } @@ -1826,13 +1825,13 @@ static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data) if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) { atomic64_inc(&reset_stats->device_reset_terminates); abt_tag |= FNIC_TAG_DEV_RST; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "dev reset sc 0x%p\n", sc); } - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "fnic_rport_exch_reset: dev rst sc 0x%p\n", sc); WARN_ON_ONCE(io_req->abts_done); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "fnic_rport_reset_exch: Issuing abts\n"); spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); @@ -1850,7 +1849,7 @@ static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data) * lun reset */ spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hwq: %d abt_tag: 0x%x flags: 0x%x Queuing abort failed\n", hwq, abt_tag, fnic_priv(sc)->flags); if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) @@ -1881,7 +1880,7 @@ void fnic_rport_exch_reset(struct fnic *fnic, u32 port_id) .term_cnt = 0, }; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "fnic rport exchange reset for tport: 0x%06x\n", port_id); @@ -1889,7 +1888,7 @@ void fnic_rport_exch_reset(struct fnic *fnic, u32 port_id) return; io_count = fnic_count_ioreqs(fnic, port_id); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "Starting terminates: rport:0x%x portid-io-count: %d active-io-count: %lld\n", port_id, io_count, atomic64_read(&fnic->fnic_stats.io_stats.active_ios)); @@ -1915,7 +1914,7 @@ void fnic_rport_exch_reset(struct fnic *fnic, u32 port_id) while ((io_count = fnic_count_ioreqs(fnic, port_id))) schedule_timeout(msecs_to_jiffies(1000)); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "rport: 0x%x remaining portid-io-count: %d ", port_id, io_count); } @@ -2040,10 +2039,10 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) tport = rdd_data->tport; if (!tport) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "Abort cmd called after tport delete! rport fcid: 0x%x", rport->port_id); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "lun: %llu hwq: 0x%x mqtag: 0x%x Op: 0x%x flags: 0x%x\n", sc->device->lun, hwq, mqtag, sc->cmnd[0], fnic_priv(sc)->flags); @@ -2052,18 +2051,18 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) goto fnic_abort_cmd_end; } - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "Abort cmd called rport fcid: 0x%x lun: %llu hwq: 0x%x mqtag: 0x%x", rport->port_id, sc->device->lun, hwq, mqtag); - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "Op: 0x%x flags: 0x%x\n", sc->cmnd[0], fnic_priv(sc)->flags); if (iport->state != FNIC_IPORT_STATE_READY) { atomic64_inc(&fnic_stats->misc_stats.iport_not_ready); - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "iport NOT in READY state"); ret = FAILED; spin_unlock_irqrestore(&fnic->fnic_lock, flags); @@ -2072,7 +2071,7 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) if ((tport->state != FDLS_TGT_STATE_READY) && (tport->state != FDLS_TGT_STATE_ADISC)) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "tport state: %d\n", tport->state); ret = FAILED; spin_unlock_irqrestore(&fnic->fnic_lock, flags); @@ -2123,7 +2122,7 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) else atomic64_inc(&abts_stats->abort_issued_greater_than_60_sec); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "CDB Opcode: 0x%02x Abort issued time: %lu msec\n", sc->cmnd[0], abt_issued_time); /* @@ -2214,7 +2213,7 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) if (!(fnic_priv(sc)->flags & (FNIC_IO_ABORTED | FNIC_IO_DONE))) { spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "Issuing host reset due to out of order IO\n"); ret = FAILED; @@ -2262,7 +2261,7 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) (u64)sc->cmnd[4] << 8 | sc->cmnd[5]), fnic_flags_and_state(sc)); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "Returning from abort cmd type %x %s\n", task_req, (ret == SUCCESS) ? "SUCCESS" : "FAILED"); @@ -2303,7 +2302,7 @@ static inline int fnic_queue_dr_io_req(struct fnic *fnic, free_wq_copy_descs(fnic, wq, hwq); if (!vnic_wq_copy_desc_avail(wq)) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "queue_dr_io_req failure - no descriptors\n"); atomic64_inc(&misc_stats->devrst_cpwq_alloc_failures); ret = -EAGAIN; @@ -2371,7 +2370,7 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) * Found IO that is still pending with firmware and * belongs to the LUN that we are resetting */ - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "Found IO in %s on lun\n", fnic_ioreq_state_to_str(fnic_priv(sc)->state)); @@ -2381,14 +2380,14 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) } if ((fnic_priv(sc)->flags & FNIC_DEVICE_RESET) && (!(fnic_priv(sc)->flags & FNIC_DEV_RST_ISSUED))) { - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "dev rst not pending sc 0x%p\n", sc); spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return true; } if (io_req->abts_done) - shost_printk(KERN_ERR, fnic->host, + fnic_printk(KERN_ERR, fnic, "%s: io_req->abts_done is set state is %s\n", __func__, fnic_ioreq_state_to_str(fnic_priv(sc)->state)); old_ioreq_state = fnic_priv(sc)->state; @@ -2404,7 +2403,7 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) BUG_ON(io_req->abts_done); if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) { - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "dev rst sc 0x%p\n", sc); } @@ -2426,7 +2425,7 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) fnic_priv(sc)->state = old_ioreq_state; spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); iter_data->ret = FAILED; - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "hwq: %d abt_tag: 0x%lx Abort could not be queued\n", hwq, abt_tag); return false; @@ -2518,7 +2517,7 @@ static int fnic_clean_pending_aborts(struct fnic *fnic, ret = 1; clean_pending_aborts_end: - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "exit status: %d\n", ret); return ret; } @@ -2568,7 +2567,7 @@ int fnic_device_reset(struct scsi_cmnd *sc) rport = starget_to_rport(scsi_target(sc->device)); spin_lock_irqsave(&fnic->fnic_lock, flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "fcid: 0x%x lun: %llu hwq: %d mqtag: 0x%x flags: 0x%x Device reset\n", rport->port_id, sc->device->lun, hwq, mqtag, fnic_priv(sc)->flags); @@ -2576,7 +2575,7 @@ int fnic_device_reset(struct scsi_cmnd *sc) rdd_data = rport->dd_data; tport = rdd_data->tport; if (!tport) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "Dev rst called after tport delete! rport fcid: 0x%x lun: %llu\n", rport->port_id, sc->device->lun); spin_unlock_irqrestore(&fnic->fnic_lock, flags); @@ -2585,7 +2584,7 @@ int fnic_device_reset(struct scsi_cmnd *sc) if (iport->state != FNIC_IPORT_STATE_READY) { atomic64_inc(&fnic_stats->misc_stats.iport_not_ready); - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "iport NOT in READY state"); spin_unlock_irqrestore(&fnic->fnic_lock, flags); goto fnic_device_reset_end; @@ -2593,7 +2592,7 @@ int fnic_device_reset(struct scsi_cmnd *sc) if ((tport->state != FDLS_TGT_STATE_READY) && (tport->state != FDLS_TGT_STATE_ADISC)) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "tport state: %d\n", tport->state); spin_unlock_irqrestore(&fnic->fnic_lock, flags); goto fnic_device_reset_end; @@ -2656,7 +2655,7 @@ int fnic_device_reset(struct scsi_cmnd *sc) fnic_priv(sc)->lr_status = FCPIO_INVALID_CODE; spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "TAG %x\n", mqtag); + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "TAG %x\n", mqtag); /* * issue the device reset, if enqueue failed, clean up the ioreq @@ -2707,13 +2706,13 @@ int fnic_device_reset(struct scsi_cmnd *sc) io_req = fnic_priv(sc)->io_req; if (!io_req) { spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "io_req is null mqtag 0x%x sc 0x%p\n", mqtag, sc); goto fnic_device_reset_end; } if (exit_dr) { - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "Host reset called for fnic. Exit device reset\n"); io_req->dr_done = NULL; goto fnic_device_reset_clean; @@ -2728,7 +2727,7 @@ int fnic_device_reset(struct scsi_cmnd *sc) */ if (status == FCPIO_INVALID_CODE) { atomic64_inc(&reset_stats->device_reset_timeouts); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "Device reset timed out\n"); fnic_priv(sc)->flags |= FNIC_DEV_RST_TIMED_OUT; int_to_scsilun(sc->device->lun, &fc_lun); @@ -2740,8 +2739,7 @@ int fnic_device_reset(struct scsi_cmnd *sc) /* Completed, but not successful, clean up the io_req, return fail */ if (status != FCPIO_SUCCESS) { spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "Device reset completed - failed\n"); io_req = fnic_priv(sc)->io_req; goto fnic_device_reset_clean; @@ -2757,7 +2755,7 @@ int fnic_device_reset(struct scsi_cmnd *sc) if (fnic_clean_pending_aborts(fnic, sc, new_sc)) { spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "Device reset failed: Cannot abort all IOs\n"); goto fnic_device_reset_clean; } @@ -2811,13 +2809,13 @@ int fnic_device_reset(struct scsi_cmnd *sc) ret = FAILED; break; } - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "Cannot clean up all IOs for the LUN\n"); schedule_timeout(msecs_to_jiffies(1000)); count++; } - FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_DEBUG, fnic, "Returning from device reset %s\n", (ret == SUCCESS) ? "SUCCESS" : "FAILED"); @@ -2852,13 +2850,13 @@ void fnic_reset(struct Scsi_Host *shost) fnic = *((struct fnic **) shost_priv(shost)); reset_stats = &fnic->fnic_stats.reset_stats; - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "Issuing fnic reset\n"); atomic64_inc(&reset_stats->fnic_resets); fnic_post_flogo_linkflap(fnic); - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "Returning from fnic reset"); atomic64_inc(&reset_stats->fnic_reset_completions); @@ -2869,7 +2867,7 @@ int fnic_issue_fc_host_lip(struct Scsi_Host *shost) int ret = 0; struct fnic *fnic = *((struct fnic **) shost_priv(shost)); - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "FC host lip issued"); ret = fnic_host_reset(shost); @@ -2895,7 +2893,7 @@ int fnic_host_reset(struct Scsi_Host *shost) spin_lock_irqsave(&fnic->fnic_lock, flags); if (fnic->reset_in_progress == IN_PROGRESS) { spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_SCSI_DBG(KERN_WARNING, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_WARNING, fnic, "Firmware reset in progress. Skipping another host reset\n"); return SUCCESS; } @@ -2933,7 +2931,7 @@ int fnic_host_reset(struct Scsi_Host *shost) } spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "host reset return status: %d\n", ret); return ret; } @@ -2973,7 +2971,7 @@ static bool fnic_abts_pending_iter(struct scsi_cmnd *sc, void *data) * Found IO that is still pending with firmware and * belongs to the LUN that we are resetting */ - FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_INFO, fnic, "hwq: %d tag: 0x%x Found IO in state: %s on lun\n", hwq, tag, fnic_ioreq_state_to_str(fnic_priv(sc)->state)); @@ -3027,7 +3025,7 @@ int fnic_eh_host_reset_handler(struct scsi_cmnd *sc) struct Scsi_Host *shost = sc->device->host; struct fnic *fnic = *((struct fnic **) shost_priv(shost)); - FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + FNIC_SCSI_DBG(KERN_ERR, fnic, "SCSI error handling: fnic host reset"); ret = fnic_host_reset(shost); diff --git a/drivers/scsi/libiscsi_tcp.c b/drivers/scsi/libiscsi_tcp.c index 7223bb18b0480..d35f93451ee9b 100644 --- a/drivers/scsi/libiscsi_tcp.c +++ b/drivers/scsi/libiscsi_tcp.c @@ -480,6 +480,9 @@ static int iscsi_tcp_data_in(struct iscsi_conn *conn, struct iscsi_task *task) int datasn = be32_to_cpu(rhdr->datasn); unsigned total_in_length = task->sc->sdb.length; + if (task->sc->sc_data_direction != DMA_FROM_DEVICE) + return ISCSI_ERR_PROTO; + /* * lib iscsi will update this in the completion handling if there * is status. diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index ecd365d78ae3c..f25ed22195fa6 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -1973,12 +1973,23 @@ megasas_set_nvme_device_properties(struct scsi_device *sdev, { struct megasas_instance *instance; u32 mr_nvme_pg_size; + u64 max_prp_io; instance = (struct megasas_instance *)sdev->host->hostdata; mr_nvme_pg_size = max_t(u32, instance->nvme_page_size, MR_DEFAULT_NVME_PAGE_SIZE); - lim->max_hw_sectors = max_io_size / 512; + /* + * megasas_make_prp_nvme() builds the PRP list in cmd->sg_frame without + * bounding it against that buffer, and spends one entry per page of + * it on the chain pointer. Cap the transfer at what the buffer holds, + * less one page for lists that start off a page boundary. + */ + max_prp_io = (u64)((instance->max_chain_frame_sz / sizeof(u64)) - + (instance->max_chain_frame_sz / mr_nvme_pg_size) - 1) * + mr_nvme_pg_size; + + lim->max_hw_sectors = min_t(u64, max_io_size, max_prp_io) >> SECTOR_SHIFT; lim->virt_boundary_mask = mr_nvme_pg_size - 1; } @@ -7875,7 +7886,9 @@ megasas_resume(struct device *dev) goto fail_init_mfi; } - if (megasas_get_ctrl_info(instance) != DCMD_SUCCESS) + scoped_guard(mutex, &instance->reset_mutex) + rval = megasas_get_ctrl_info(instance); + if (rval != DCMD_SUCCESS) goto fail_init_mfi; tasklet_init(&instance->isr_tasklet, instance->instancet->tasklet, diff --git a/drivers/scsi/mpi3mr/mpi3mr_transport.c b/drivers/scsi/mpi3mr/mpi3mr_transport.c index 240f67a8e2e3b..232af978d7372 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_transport.c +++ b/drivers/scsi/mpi3mr/mpi3mr_transport.c @@ -1428,9 +1428,15 @@ static struct mpi3mr_sas_port *mpi3mr_sas_port_add(struct mpi3mr_ioc *mrioc, } port = sas_port_alloc_num(mr_sas_node->parent_dev); + if (!port) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out_fail; + } if ((sas_port_add(port))) { ioc_err(mrioc, "failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__); + sas_port_free(port); goto out_fail; } @@ -1501,6 +1507,8 @@ static struct mpi3mr_sas_port *mpi3mr_sas_port_add(struct mpi3mr_ioc *mrioc, list_for_each_entry_safe(mr_sas_phy, next, &mr_sas_port->phy_list, port_siblings) list_del(&mr_sas_phy->port_siblings); + if (tgtdev) + mpi3mr_tgtdev_put(tgtdev); kfree(mr_sas_port); return NULL; } diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index 79052f2accbd3..ae2f7d9c281f9 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -3238,7 +3238,10 @@ _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) * corresponding to high iops queues. */ if (ioc->high_iops_queues) { - mask = cpumask_of_node(dev_to_node(&ioc->pdev->dev)); + int node = dev_to_node(&ioc->pdev->dev); + + mask = (node == NUMA_NO_NODE) ? + cpu_online_mask : cpumask_of_node(node); for (index = 0; index < ioc->high_iops_queues; index++) { irq = pci_irq_vector(ioc->pdev, index); @@ -5870,6 +5873,8 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) if (ioc->pcie_sgl_dma_pool) { for (i = 0; i < ioc->scsiio_depth; i++) { + if (!ioc->pcie_sg_lookup[i].pcie_sgl) + continue; dma_pool_free(ioc->pcie_sgl_dma_pool, ioc->pcie_sg_lookup[i].pcie_sgl, ioc->pcie_sg_lookup[i].pcie_sgl_dma); diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c index e93ea76b565e5..5af81c73a8f87 100644 --- a/drivers/scsi/pm8001/pm8001_init.c +++ b/drivers/scsi/pm8001/pm8001_init.c @@ -58,15 +58,15 @@ MODULE_PARM_DESC(link_rate, "Enable link rate.\n" bool pm8001_use_msix = true; module_param_named(use_msix, pm8001_use_msix, bool, 0444); -MODULE_PARM_DESC(zoned, "Use MSIX interrupts. Default: true"); +MODULE_PARM_DESC(use_msix, "Use MSIX interrupts. Default: true"); static bool pm8001_use_tasklet = true; module_param_named(use_tasklet, pm8001_use_tasklet, bool, 0444); -MODULE_PARM_DESC(zoned, "Use MSIX interrupts. Default: true"); +MODULE_PARM_DESC(use_tasklet, "Use tasklets for interrupt handling. Default: true"); static bool pm8001_read_wwn = true; module_param_named(read_wwn, pm8001_read_wwn, bool, 0444); -MODULE_PARM_DESC(zoned, "Get WWN from the controller. Default: true"); +MODULE_PARM_DESC(read_wwn, "Get WWN from the controller. Default: true"); uint pcs_event_log_severity = 0x03; module_param(pcs_event_log_severity, int, 0644); @@ -1029,8 +1029,8 @@ static u32 pm8001_request_msix(struct pm8001_hba_info *pm8001_ha) &(pm8001_ha->irq_vector[i])); if (rc) { for (j = 0; j < i; j++) { - free_irq(pci_irq_vector(pm8001_ha->pdev, i), - &(pm8001_ha->irq_vector[i])); + free_irq(pci_irq_vector(pm8001_ha->pdev, j), + &pm8001_ha->irq_vector[j]); } pci_free_irq_vectors(pm8001_ha->pdev); break; diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c index 6a05ce195aa05..b3b7cdf0cfa90 100644 --- a/drivers/scsi/qla2xxx/qla_attr.c +++ b/drivers/scsi/qla2xxx/qla_attr.c @@ -580,7 +580,6 @@ qla2x00_sysfs_read_vpd(struct file *filp, struct kobject *kobj, ha->isp_ops->read_optrom(vha, ha->vpd, faddr, ha->vpd_size); mutex_unlock(&ha->optrom_mutex); - ha->isp_ops->read_optrom(vha, ha->vpd, faddr, ha->vpd_size); skip: return memory_read_from_buffer(buf, count, &off, ha->vpd, ha->vpd_size); } @@ -816,7 +815,9 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj, "Unable to allocate memory for VPD information update.\n"); return -ENOMEM; } + mutex_lock(&ha->optrom_mutex); ha->isp_ops->get_flash_version(vha, tmp_data); + mutex_unlock(&ha->optrom_mutex); vfree(tmp_data); break; } @@ -1660,10 +1661,8 @@ qla2x00_fw_state_show(struct device *dev, struct device_attribute *attr, rval = qla2x00_get_firmware_state(vha, state); mutex_unlock(&vha->hw->optrom_mutex); out: - if (rval != QLA_SUCCESS) { + if (rval != QLA_SUCCESS) memset(state, -1, sizeof(state)); - rval = qla2x00_get_firmware_state(vha, state); - } return scnprintf(buf, PAGE_SIZE, "0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", state[0], state[1], state[2], state[3], state[4], state[5]); diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c index 5e910b5ca670d..79144530fa24d 100644 --- a/drivers/scsi/qla2xxx/qla_bsg.c +++ b/drivers/scsi/qla2xxx/qla_bsg.c @@ -160,6 +160,12 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct bsg_job *bsg_job) goto exit_fcp_prio_cfg; } + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) { + ret = -EINVAL; + goto exit_fcp_prio_cfg; + } + /* Get the sub command */ oper = bsg_request->rqst_data.h_vendor.vendor_cmd[1]; @@ -758,6 +764,10 @@ qla2x00_process_loopback(struct bsg_job *bsg_job) return -EIO; } + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 3 * sizeof(uint32_t)) + return -EINVAL; + memset(&elreq, 0, sizeof(elreq)); elreq.req_sg_cnt = dma_map_sg(&ha->pdev->dev, @@ -990,6 +1000,10 @@ qla84xx_reset(struct bsg_job *bsg_job) return -EINVAL; } + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) + return -EINVAL; + flag = bsg_request->rqst_data.h_vendor.vendor_cmd[1]; rval = qla84xx_reset_chip(vha, flag == A84_ISSUE_RESET_DIAG_FW); @@ -1034,6 +1048,10 @@ qla84xx_updatefw(struct bsg_job *bsg_job) return -EINVAL; } + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) + return -EINVAL; + sg_cnt = dma_map_sg(&ha->pdev->dev, bsg_job->request_payload.sg_list, bsg_job->request_payload.sg_cnt, DMA_TO_DEVICE); if (!sg_cnt) { @@ -1484,6 +1502,10 @@ qla2x00_read_optrom(struct bsg_job *bsg_job) struct qla_hw_data *ha = vha->hw; int rval = 0; + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) + return -EINVAL; + if (ha->flags.nic_core_reset_hdlr_active) return -EBUSY; @@ -1521,6 +1543,10 @@ qla2x00_update_optrom(struct bsg_job *bsg_job) struct qla_hw_data *ha = vha->hw; int rval = 0; + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) + return -EINVAL; + mutex_lock(&ha->optrom_mutex); rval = qla2x00_optrom_setup(bsg_job, vha, 1); if (rval) { @@ -1562,12 +1588,12 @@ qla2x00_update_fru_versions(struct bsg_job *bsg_job) scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; - uint8_t bsg[DMA_POOL_SIZE]; + uint8_t bsg[DMA_POOL_SIZE] = {}; struct qla_image_version_list *list = (void *)bsg; struct qla_image_version *image; uint32_t count; dma_addr_t sfp_dma; - void *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); + void *sfp = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = @@ -1580,6 +1606,13 @@ qla2x00_update_fru_versions(struct bsg_job *bsg_job) image = list->version; count = list->count; + + if (struct_size(list, version, count) > sizeof(bsg)) { + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = + EXT_STATUS_INVALID_PARAM; + goto dealloc; + } + while (count--) { memcpy(sfp, &image->field_info, sizeof(image->field_info)); rval = qla2x00_write_sfp(vha, sfp_dma, sfp, @@ -1615,10 +1648,10 @@ qla2x00_read_fru_status(struct bsg_job *bsg_job) scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; - uint8_t bsg[DMA_POOL_SIZE]; + uint8_t bsg[DMA_POOL_SIZE] = {}; struct qla_status_reg *sr = (void *)bsg; dma_addr_t sfp_dma; - uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); + uint8_t *sfp = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = @@ -1666,10 +1699,10 @@ qla2x00_write_fru_status(struct bsg_job *bsg_job) scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; - uint8_t bsg[DMA_POOL_SIZE]; + uint8_t bsg[DMA_POOL_SIZE] = {}; struct qla_status_reg *sr = (void *)bsg; dma_addr_t sfp_dma; - uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); + uint8_t *sfp = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = @@ -1713,10 +1746,10 @@ qla2x00_write_i2c(struct bsg_job *bsg_job) scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; - uint8_t bsg[DMA_POOL_SIZE]; + uint8_t bsg[DMA_POOL_SIZE] = {}; struct qla_i2c_access *i2c = (void *)bsg; dma_addr_t sfp_dma; - uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); + uint8_t *sfp = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = @@ -1727,6 +1760,12 @@ qla2x00_write_i2c(struct bsg_job *bsg_job) sg_copy_to_buffer(bsg_job->request_payload.sg_list, bsg_job->request_payload.sg_cnt, i2c, sizeof(*i2c)); + if (i2c->length > sizeof(i2c->buffer)) { + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = + EXT_STATUS_INVALID_PARAM; + goto dealloc; + } + memcpy(sfp, i2c->buffer, i2c->length); rval = qla2x00_write_sfp(vha, sfp_dma, sfp, i2c->device, i2c->offset, i2c->length, i2c->option); @@ -1759,10 +1798,10 @@ qla2x00_read_i2c(struct bsg_job *bsg_job) scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; - uint8_t bsg[DMA_POOL_SIZE]; + uint8_t bsg[DMA_POOL_SIZE] = {}; struct qla_i2c_access *i2c = (void *)bsg; dma_addr_t sfp_dma; - uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); + uint8_t *sfp = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = @@ -1773,6 +1812,12 @@ qla2x00_read_i2c(struct bsg_job *bsg_job) sg_copy_to_buffer(bsg_job->request_payload.sg_list, bsg_job->request_payload.sg_cnt, i2c, sizeof(*i2c)); + if (i2c->length > sizeof(i2c->buffer)) { + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = + EXT_STATUS_INVALID_PARAM; + goto dealloc; + } + rval = qla2x00_read_sfp(vha, sfp_dma, sfp, i2c->device, i2c->offset, i2c->length, i2c->option); @@ -1993,6 +2038,11 @@ qlafx00_mgmt_cmd(struct bsg_job *bsg_job) struct fc_port *fcport; char *type = "FC_BSG_HST_FX_MGMT"; + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + sizeof(uint32_t) + + sizeof(struct qla_mt_iocb_rqst_fx00)) + return -EINVAL; + /* Copy the IOCB specific information */ piocb_rqst = (struct qla_mt_iocb_rqst_fx00 *) &bsg_request->rqst_data.h_vendor.vendor_cmd[1]; @@ -2395,7 +2445,7 @@ qla2x00_do_dport_diagnostics(struct bsg_job *bsg_job) !IS_QLA28XX(vha->hw)) return -EPERM; - dd = kmalloc_obj(*dd); + dd = kzalloc_obj(*dd); if (!dd) { ql_log(ql_log_warn, vha, 0x70db, "Failed to allocate memory for dport.\n"); @@ -2906,6 +2956,13 @@ qla2x00_process_vendor_specific(struct scsi_qla_host *vha, struct bsg_job *bsg_j { struct fc_bsg_request *bsg_request = bsg_job->request; + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + sizeof(uint32_t)) { + ql_log(ql_log_warn, vha, 0x7000, + "BSG request too small for vendor cmd.\n"); + return -EINVAL; + } + ql_dbg(ql_dbg_edif, vha, 0x911b, "%s FC_BSG_HST_VENDOR cmd[0]=0x%x\n", __func__, bsg_request->rqst_data.h_vendor.vendor_cmd[0]); @@ -3037,8 +3094,11 @@ qla24xx_bsg_request(struct bsg_job *bsg_job) } /* Disable port will bring down the chip, allow enable command */ - if (bsg_request->rqst_data.h_vendor.vendor_cmd[0] == QL_VND_MANAGE_HOST_PORT || - bsg_request->rqst_data.h_vendor.vendor_cmd[0] == QL_VND_GET_HOST_STATS) + if (bsg_request->msgcode == FC_BSG_HST_VENDOR && + bsg_job->request_len >= + sizeof(struct fc_bsg_request) + sizeof(uint32_t) && + (bsg_request->rqst_data.h_vendor.vendor_cmd[0] == QL_VND_MANAGE_HOST_PORT || + bsg_request->rqst_data.h_vendor.vendor_cmd[0] == QL_VND_GET_HOST_STATS)) goto skip_chip_chk; if (vha->hw->flags.port_isolated) { @@ -3347,6 +3407,10 @@ static int qla28xx_validate_flash_image(struct bsg_job *bsg_job) if (!IS_QLA28XX(ha) || vha->vp_idx != 0) return -EPERM; + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) + return -EINVAL; + mutex_lock(&ha->optrom_mutex); rval = qla28xx_do_validate_flash_image(bsg_job, &state); if (rval) @@ -3365,9 +3429,8 @@ static int qla28xx_validate_flash_image(struct bsg_job *bsg_job) bsg_reply->result = DID_OK << 16; bsg_reply->reply_payload_rcv_len = 0; bsg_job->reply_len = sizeof(struct fc_bsg_reply); - if (!rval) - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return QLA_SUCCESS; } diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c index a7e3ec9bba47f..c6b8e6d798c63 100644 --- a/drivers/scsi/qla2xxx/qla_dbg.c +++ b/drivers/scsi/qla2xxx/qla_dbg.c @@ -16,7 +16,7 @@ * | | | 0x2127-0x2128 | * | Queue Command and IO tracing | 0x3074 | 0x300b | * | | | 0x3027-0x3028 | - * | | | 0x303d-0x3041 | + * | | | 0x303e-0x3041 | * | | | 0x302e,0x3033 | * | | | 0x3036,0x3038 | * | | | 0x303a | diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h index 5593ad7fad274..8be3dd40f6d77 100644 --- a/drivers/scsi/qla2xxx/qla_def.h +++ b/drivers/scsi/qla2xxx/qla_def.h @@ -2591,6 +2591,8 @@ typedef struct fc_port { struct list_head list; struct scsi_qla_host *vha; struct list_head unsol_ctx_head; + /* Serializes unsol_ctx_head against ISR, DPC and NVMe transport. */ + spinlock_t unsol_ctx_lock; unsigned int conf_compl_supported:1; unsigned int deleted:2; diff --git a/drivers/scsi/qla2xxx/qla_dfs.c b/drivers/scsi/qla2xxx/qla_dfs.c index 43970caca7b39..cb4278b88e1d0 100644 --- a/drivers/scsi/qla2xxx/qla_dfs.c +++ b/drivers/scsi/qla2xxx/qla_dfs.c @@ -510,7 +510,9 @@ qla2x00_dfs_fce_write(struct file *file, const char __user *buffer, return PTR_ERR(buf); } - enable = kstrtoul(buf, 0, 0); + rc = kstrtoul(buf, 0, &enable); + if (rc) + goto out_free; rc = count; mutex_lock(&ha->fce_mutex); diff --git a/drivers/scsi/qla2xxx/qla_edif.c b/drivers/scsi/qla2xxx/qla_edif.c index eccedb38a5154..6caae48510b38 100644 --- a/drivers/scsi/qla2xxx/qla_edif.c +++ b/drivers/scsi/qla2xxx/qla_edif.c @@ -3492,6 +3492,9 @@ void qla_chk_edif_rx_sa_delete_pending(scsi_qla_host_t *vha, struct scsi_cmnd *cmd = GET_CMD_SP(sp); uint32_t handle; + if (!cmd) + return; + handle = (uint32_t)LSW(sts24->handle); /* find out if this status iosb is for a scsi read */ diff --git a/drivers/scsi/qla2xxx/qla_fw.h b/drivers/scsi/qla2xxx/qla_fw.h index f307beed9d29d..e7ded95afb9a8 100644 --- a/drivers/scsi/qla2xxx/qla_fw.h +++ b/drivers/scsi/qla2xxx/qla_fw.h @@ -1442,6 +1442,10 @@ struct vp_ctrl_entry_24xx { uint8_t reserved_5[24]; }; +/* vp_idx_map is a 128-bit (16-byte) bitmap selecting target VPs. */ +#define VP_CTRL_IDX_MAP_BITS \ + (sizeof_field(struct vp_ctrl_entry_24xx, vp_idx_map) * 8) + /* * Modify Virtual Port Configuration IOCB */ diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c index 880cd73feaca4..ed46217ce26e1 100644 --- a/drivers/scsi/qla2xxx/qla_gs.c +++ b/drivers/scsi/qla2xxx/qla_gs.c @@ -157,8 +157,8 @@ qla2x00_chk_ms_status(scsi_qla_host_t *vha, ms_iocb_entry_t *ms_pkt, break; case CS_PORT_LOGGED_OUT: if (IS_FWI2_CAPABLE(ha)) { - if (le16_to_cpu(ms_pkt->loop_id.extended) == - NPH_SNS) + if (le16_to_cpu(((struct ct_entry_24xx *) + ms_pkt)->nport_handle) == NPH_SNS) lid_is_sns = true; } else { if (le16_to_cpu(ms_pkt->loop_id.extended) == diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c index e746c9274cded..22374f844e081 100644 --- a/drivers/scsi/qla2xxx/qla_init.c +++ b/drivers/scsi/qla2xxx/qla_init.c @@ -228,7 +228,7 @@ qla2x00_async_iocb_timeout(void *data) srb_t *sp = data; fc_port_t *fcport = sp->fcport; struct srb_iocb *lio = &sp->u.iocb_cmd; - int rc, h; + int rc, h, found; unsigned long flags; if (fcport) { @@ -251,6 +251,7 @@ qla2x00_async_iocb_timeout(void *data) lio->u.logio.data[1] = lio->u.logio.flags & SRB_LOGIN_RETRIED ? QLA_LOGIO_LOGIN_RETRIED : 0; + found = 0; spin_lock_irqsave(sp->qpair->qp_lock_ptr, flags); for (h = 1; h < sp->qpair->req->num_outstanding_cmds; h++) { @@ -258,11 +259,19 @@ qla2x00_async_iocb_timeout(void *data) sp) { sp->qpair->req->outstanding_cmds[h] = NULL; + found = 1; break; } } spin_unlock_irqrestore(sp->qpair->qp_lock_ptr, flags); - sp->done(sp, QLA_FUNCTION_TIMEOUT); + /* + * Only complete the command if this path removed it + * from outstanding_cmds. Otherwise the ISR already + * completed it and a second sp->done() would race the + * submitter's freeing of the on-stack completion. + */ + if (found) + sp->done(sp, QLA_FUNCTION_TIMEOUT); } break; case SRB_LOGOUT_CMD: @@ -275,6 +284,7 @@ qla2x00_async_iocb_timeout(void *data) default: rc = qla24xx_async_abort_cmd(sp, false); if (rc) { + found = 0; spin_lock_irqsave(sp->qpair->qp_lock_ptr, flags); for (h = 1; h < sp->qpair->req->num_outstanding_cmds; h++) { @@ -282,11 +292,19 @@ qla2x00_async_iocb_timeout(void *data) sp) { sp->qpair->req->outstanding_cmds[h] = NULL; + found = 1; break; } } spin_unlock_irqrestore(sp->qpair->qp_lock_ptr, flags); - sp->done(sp, QLA_FUNCTION_TIMEOUT); + /* + * Only complete the command if this path removed it + * from outstanding_cmds. Otherwise the ISR already + * completed it and a second sp->done() would race the + * submitter's freeing of the on-stack completion. + */ + if (found) + sp->done(sp, QLA_FUNCTION_TIMEOUT); } break; } @@ -3764,11 +3782,27 @@ int qla2x00_alloc_fce_trace(scsi_qla_host_t *vha) void qla2x00_free_fce_trace(struct qla_hw_data *ha) { - if (!ha->fce) + void *fce; + dma_addr_t fce_dma; + unsigned long flags; + + /* + * Unpublish ha->fce under hardware_lock so a firmware dump in + * progress (which reads ha->fce under the same lock) cannot race + * with the buffer being freed. + */ + spin_lock_irqsave(&ha->hardware_lock, flags); + if (!ha->fce) { + spin_unlock_irqrestore(&ha->hardware_lock, flags); return; - dma_free_coherent(&ha->pdev->dev, FCE_SIZE, ha->fce, ha->fce_dma); + } + fce = ha->fce; + fce_dma = ha->fce_dma; ha->fce = NULL; ha->fce_dma = 0; + spin_unlock_irqrestore(&ha->hardware_lock, flags); + + dma_free_coherent(&ha->pdev->dev, FCE_SIZE, fce, fce_dma); } static void @@ -4387,6 +4421,19 @@ qla2x00_setup_chip(scsi_qla_host_t *vha) MIN_MULTI_ID_FABRIC)) ha->max_npiv_vports = MIN_MULTI_ID_FABRIC - 1; + + /* + * The VP_CTRL IOCB selects target VPs + * through the fixed vp_idx_map bitmap, + * so a vp_index beyond it can be enabled + * via VP_CONFIG but never disabled via + * VP_CTRL, leaking the VP. Cap the count + * to the bitmap capacity. + */ + if (ha->max_npiv_vports >= + VP_CTRL_IDX_MAP_BITS) + ha->max_npiv_vports = + VP_CTRL_IDX_MAP_BITS - 1; } qlt_config_nvram_with_fw_version(vha); qla2x00_get_resource_cnts(vha); @@ -5616,6 +5663,7 @@ qla2x00_alloc_fcport(scsi_qla_host_t *vha, gfp_t flags) INIT_LIST_HEAD(&fcport->gnl_entry); INIT_LIST_HEAD(&fcport->list); INIT_LIST_HEAD(&fcport->unsol_ctx_head); + spin_lock_init(&fcport->unsol_ctx_lock); INIT_LIST_HEAD(&fcport->sess_cmd_list); spin_lock_init(&fcport->sess_cmd_lock); @@ -10026,11 +10074,28 @@ int qla2xxx_delete_qpair(struct scsi_qla_host *vha, struct qla_qpair *qpair) { int ret = QLA_FUNCTION_FAILED; struct qla_hw_data *ha = qpair->hw; + struct rsp_que *rsp = qpair->rsp; qpair->delete_in_progress = 1; qla_free_buf_pool(qpair); + /* + * The response-queue interrupt schedules qla_do_work(), which + * dereferences qpair->rsp->req. Release the interrupt and flush + * any pending work before the request queue is freed below so a + * late completion cannot touch the freed request queue. The + * firmware queue-delete order (request then response) is kept. + */ + if (rsp && rsp->msix && rsp->msix->have_irq) { + free_irq(rsp->msix->vector, rsp->msix->handle); + rsp->msix->have_irq = 0; + rsp->msix->in_use = 0; + rsp->msix->handle = NULL; + } + if (rsp && ha->wq) + cancel_work_sync(&qpair->q_work); + ret = qla25xx_delete_req_que(vha, qpair->req); if (ret != QLA_SUCCESS) goto fail; diff --git a/drivers/scsi/qla2xxx/qla_inline.h b/drivers/scsi/qla2xxx/qla_inline.h index 47fbd830ff163..abf0db7f72d5d 100644 --- a/drivers/scsi/qla2xxx/qla_inline.h +++ b/drivers/scsi/qla2xxx/qla_inline.h @@ -54,6 +54,19 @@ qla2x00_debounce_register(volatile __le16 __iomem *addr) return (first); } +static inline u8 +qla_calc_queue_count(u16 msix_count) +{ + /* + * Request/response queues are bounded by the MSI-X vector count less + * the mailbox vector. These counters are u8, so a board advertising + * e.g. 257 vectors would truncate msix_count - 1 (256) to 0 and hand + * kzalloc_objs() a zero count (ZERO_SIZE_PTR), faulting on the first + * ha->req_q_map[0] store. Clamp into [1, QLA_MAX_QUEUES - 1]. + */ + return clamp_t(u16, msix_count - 1, 1, QLA_MAX_QUEUES - 1); +} + static inline void qla2x00_poll(struct rsp_que *rsp) { diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c index dbe3cd4e274c8..1a58dd4e8ec2f 100644 --- a/drivers/scsi/qla2xxx/qla_iocb.c +++ b/drivers/scsi/qla2xxx/qla_iocb.c @@ -11,6 +11,10 @@ #include +/* All adapters supported by this tree use the 64-byte request ring. */ +#define qla_req_ring_slot(ha, req) ((req)->ring_ptr) +#define qla_req_entry_size(ha) REQUEST_ENTRY_SIZE + static int qla_start_scsi_type6(srb_t *sp); /** * qla2x00_get_cmd_direction() - Determine control_flag data direction. @@ -2286,10 +2290,9 @@ __qla2x00_alloc_iocbs(struct qla_qpair *qpair, srb_t *sp) struct req_que *req = qpair->req; device_reg_t *reg = ISP_QUE_REG(ha, req->id); uint32_t handle; - request_t *pkt; uint16_t cnt, req_cnt; + request_t *pkt = NULL; - pkt = NULL; req_cnt = 1; handle = 0; @@ -2343,14 +2346,18 @@ __qla2x00_alloc_iocbs(struct qla_qpair *qpair, srb_t *sp) sp->handle = handle; } - /* Prep packet */ + /* + * Prep the current request-ring slot. + */ req->cnt -= req_cnt; - pkt = req->ring_ptr; - memset(pkt, 0, REQUEST_ENTRY_SIZE); + pkt = qla_req_ring_slot(ha, req); if (IS_QLAFX00(ha)) { + memset_io((void __iomem __force *)pkt, 0, + qla_req_entry_size(ha)); wrt_reg_byte((u8 __force __iomem *)&pkt->entry_count, req_cnt); wrt_reg_dword((__le32 __force __iomem *)&pkt->handle, handle); } else { + memset(pkt, 0, qla_req_entry_size(ha)); pkt->entry_count = req_cnt; pkt->handle = handle; } @@ -3826,6 +3833,12 @@ qla25xx_ctrlvp_iocb(srb_t *sp, struct vp_ctrl_entry_24xx *vce) */ map = (sp->u.iocb_cmd.u.ctrlvp.vp_index - 1) / 8; pos = (sp->u.iocb_cmd.u.ctrlvp.vp_index - 1) & 7; + if (map >= ARRAY_SIZE(vce->vp_idx_map)) { + ql_log(ql_log_warn, sp->vha, 0x307c, + "ctrlvp: vp_index %u exceeds vp_idx_map capacity\n", + sp->u.iocb_cmd.u.ctrlvp.vp_index); + return; + } vce->vp_idx_map[map] |= 1 << pos; } diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c index 33776330956c9..81997119fd07d 100644 --- a/drivers/scsi/qla2xxx/qla_isr.c +++ b/drivers/scsi/qla2xxx/qla_isr.c @@ -205,6 +205,17 @@ void __qla_consume_iocb(struct scsi_qla_host *vha, struct purex_entry_24xx *purex = *pkt; entry_count_remaining = purex->entry_count; + + /* + * The caller already advanced ring_ptr past the head IOCB, so mark + * the head processed and account for it here, then consume only the + * continuation IOCBs that follow. + */ + ((response_t *)purex)->signature = RESPONSE_PROCESSED; + /* flush signature */ + wmb(); + --entry_count_remaining; + while (entry_count_remaining > 0) { new_pkt = rsp_q->ring_ptr; *pkt = new_pkt; @@ -910,14 +921,6 @@ qla27xx_copy_multiple_pkt(struct scsi_qla_host *vha, void **pkt, do { while ((total_bytes > 0) && (entry_count_remaining > 0)) { - if (rsp_q->ring_ptr->signature == RESPONSE_PROCESSED) { - ql_dbg(ql_dbg_async, vha, 0x5084, - "Ran out of IOCBs, partial data 0x%x\n", - buffer_copy_offset); - cpu_relax(); - continue; - } - new_pkt = (sts_cont_entry_t *)rsp_q->ring_ptr; *pkt = new_pkt; @@ -1194,14 +1197,6 @@ qla27xx_copy_fpin_pkt(struct scsi_qla_host *vha, void **pkt, do { while ((total_bytes > 0) && (entry_count_remaining > 0)) { - if (rsp_q->ring_ptr->signature == RESPONSE_PROCESSED) { - ql_dbg(ql_dbg_async, vha, 0x5084, - "Ran out of IOCBs, partial data 0x%x\n", - buffer_copy_offset); - cpu_relax(); - continue; - } - new_pkt = (sts_cont_entry_t *)rsp_q->ring_ptr; *pkt = new_pkt; @@ -3406,6 +3401,14 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt) return; } + /* Everything below is the SCSI fast path; reject other SRB types. */ + if (sp->type != SRB_SCSI_CMD) { + ql_dbg(ql_dbg_io, vha, 0x303d, + "Unexpected SRB type %x for status IOCB, sp %p.\n", + sp->type, sp); + return; + } + /* Fast path completion. */ qla_chk_edif_rx_sa_delete_pending(vha, sp, sts24); sp->qpair->cmd_completion_cnt++; @@ -3463,6 +3466,18 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt) if (scsi_status & SS_RESPONSE_INFO_LEN_VALID) { /* Sense data lies beyond any FCP RESPONSE data. */ if (IS_FWI2_CAPABLE(ha)) { + /* + * A hostile or buggy target may report an + * rsp_info_len larger than the IOCB data area. + * Clamp it so the par_sense_len subtraction cannot + * underflow and walk sense_data out of bounds. + */ + if (rsp_info_len > par_sense_len) { + ql_log(ql_log_warn, fcport->vha, 0x3107, + "Truncating bogus rsp_info_len 0x%x to 0x%x.\n", + rsp_info_len, par_sense_len); + rsp_info_len = par_sense_len; + } sense_data += rsp_info_len; par_sense_len -= rsp_info_len; } @@ -3778,10 +3793,12 @@ qla2x00_error_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, sts_entry_t *pkt) "iocb type %xh with error status %xh, handle %xh, rspq id %d\n", pkt->entry_type, pkt->entry_status, pkt->handle, rsp->id); - if (que >= ha->max_req_queues || !ha->req_q_map[que]) + if (que >= ha->max_req_queues) goto fatal; req = ha->req_q_map[que]; + if (!req) + goto fatal; if (pkt->entry_status & RF_BUSY) res = DID_BUS_BUSY << 16; @@ -4109,9 +4126,24 @@ void qla24xx_process_response_queue(struct scsi_qla_host *vha, "SCM not active for this port\n"); break; } + if (qla_chk_cont_iocb_avail(vha, rsp, + (response_t *)pkt, rsp_in)) { + /* + * ring_ptr and ring_index were + * pre-incremented above. Reset them + * back to current. Wait for next + * interrupt with all IOCBs to arrive + * and re-process. + */ + rsp->ring_ptr = (response_t *)pkt; + rsp->ring_index = cur_ring_index; + + ql_dbg(ql_dbg_init, vha, 0x5095, + "Defer processing FPIN...\n"); + return; + } pure_item = qla27xx_copy_fpin_pkt(vha, (void **)&pkt, &rsp); - __update_rsp_in(is_shadow_hba, rsp, rsp_in); if (!pure_item) break; qla24xx_queue_purex_item(vha, pure_item, @@ -4548,10 +4580,10 @@ qla24xx_enable_msix(struct qla_hw_data *ha, struct rsp_que *rsp) ha->msix_count = ret; /* Recalculate queue values */ if (ha->mqiobase && (ql2xmqsupport || ql2xnvmeenable)) { - ha->max_req_queues = ha->msix_count - 1; + ha->max_req_queues = qla_calc_queue_count(ha->msix_count); /* ATIOQ needs 1 vector. That's 1 less QPair */ - if (QLA_TGT_MODE_ENABLED()) + if (QLA_TGT_MODE_ENABLED() && ha->max_req_queues > 1) ha->max_req_queues--; ha->max_rsp_queues = ha->max_req_queues; diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c index 44e310f1a3708..aec8e9ff3f3e7 100644 --- a/drivers/scsi/qla2xxx/qla_mbx.c +++ b/drivers/scsi/qla2xxx/qla_mbx.c @@ -2262,6 +2262,8 @@ qla2x00_get_firmware_state(scsi_qla_host_t *vha, uint16_t *states) if (!ha->flags.fw_started) return QLA_FUNCTION_FAILED; + memset(&mc, 0, sizeof(mc)); + mcp->mb[0] = MBC_GET_FIRMWARE_STATE; mcp->out_mb = MBX_0; if (IS_FWI2_CAPABLE(vha->hw)) @@ -4202,7 +4204,10 @@ qla24xx_report_id_acquisition(scsi_qla_host_t *vha, spin_lock_irqsave(&ha->vport_slock, flags); list_for_each_entry(vp, &ha->vp_list, list) { if (rptid_entry->vp_idx == vp->vp_idx) { + if (test_bit(VPORT_DELETE, &vp->dpc_flags)) + break; found = 1; + atomic_inc(&vp->vref_count); break; } } @@ -4211,7 +4216,9 @@ qla24xx_report_id_acquisition(scsi_qla_host_t *vha, if (!found) return; + spin_lock_irqsave(&ha->vport_slock, flags); qla_update_host_map(vp, id); + spin_unlock_irqrestore(&ha->vport_slock, flags); /* * Cannot configure here as we are still sitting on the @@ -4220,6 +4227,10 @@ qla24xx_report_id_acquisition(scsi_qla_host_t *vha, set_bit(VP_IDX_ACQUIRED, &vp->vp_flags); set_bit(REGISTER_FC4_NEEDED, &vp->dpc_flags); set_bit(REGISTER_FDMI_NEEDED, &vp->dpc_flags); + + spin_lock_irqsave(&ha->vport_slock, flags); + atomic_dec(&vp->vref_count); + spin_unlock_irqrestore(&ha->vport_slock, flags); } set_bit(VP_DPC_NEEDED, &vha->dpc_flags); qla2xxx_wake_dpc(vha); @@ -4330,10 +4341,10 @@ qla24xx_modify_vp_config(scsi_qla_host_t *vha) if (rval != QLA_SUCCESS) { ql_dbg(ql_dbg_mbx, vha, 0x10bd, "Failed to issue VP config IOCB (%x).\n", rval); - } else if (vpmod->comp_status != 0) { + } else if (vpmod->entry_status != 0) { ql_dbg(ql_dbg_mbx, vha, 0x10be, "Failed to complete IOCB -- error status (%x).\n", - vpmod->comp_status); + vpmod->entry_status); rval = QLA_FUNCTION_FAILED; } else if (vpmod->comp_status != cpu_to_le16(CS_COMPLETE)) { ql_dbg(ql_dbg_mbx, vha, 0x10bf, @@ -5706,7 +5717,7 @@ qla2x00_get_data_rate(scsi_qla_host_t *vha) ql_dbg(ql_dbg_mbx, vha, 0x1107, "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]); } else { - if (mcp->mb[1] != 0x7) + if (mcp->mb[1] != 0x7 || IS_QLA28XX(ha)) ha->link_data_rate = mcp->mb[1]; if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) { @@ -5717,8 +5728,6 @@ qla2x00_get_data_rate(scsi_qla_host_t *vha) ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1108, "Done %s.\n", __func__); - if (mcp->mb[1] != 0x7) - ha->link_data_rate = mcp->mb[1]; } return rval; @@ -6490,6 +6499,7 @@ qla26xx_dport_diagnostics(scsi_qla_host_t *vha, mbx_cmd_t mc; mbx_cmd_t *mcp = &mc; dma_addr_t dd_dma; + void *dd; if (!IS_QLA83XX(vha->hw) && !IS_QLA27XX(vha->hw) && !IS_QLA28XX(vha->hw)) @@ -6498,15 +6508,12 @@ qla26xx_dport_diagnostics(scsi_qla_host_t *vha, ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x119f, "Entered %s.\n", __func__); - dd_dma = dma_map_single(&vha->hw->pdev->dev, - dd_buf, size, DMA_FROM_DEVICE); - if (dma_mapping_error(&vha->hw->pdev->dev, dd_dma)) { - ql_log(ql_log_warn, vha, 0x1194, "Failed to map dma buffer.\n"); + dd = dma_alloc_coherent(&vha->hw->pdev->dev, size, &dd_dma, GFP_KERNEL); + if (!dd) { + ql_log(ql_log_warn, vha, 0x1194, "Failed to allocate dma buffer.\n"); return QLA_MEMORY_ALLOC_FAILED; } - memset(dd_buf, 0, size); - mcp->mb[0] = MBC_DPORT_DIAGNOSTICS; mcp->mb[1] = options; mcp->mb[2] = MSW(LSD(dd_dma)); @@ -6528,8 +6535,9 @@ qla26xx_dport_diagnostics(scsi_qla_host_t *vha, "Done %s.\n", __func__); } - dma_unmap_single(&vha->hw->pdev->dev, dd_dma, - size, DMA_FROM_DEVICE); + memcpy(dd_buf, dd, size); + + dma_free_coherent(&vha->hw->pdev->dev, size, dd, dd_dma); return rval; } diff --git a/drivers/scsi/qla2xxx/qla_mid.c b/drivers/scsi/qla2xxx/qla_mid.c index c563133f751e8..21e2f9dbe0a9c 100644 --- a/drivers/scsi/qla2xxx/qla_mid.c +++ b/drivers/scsi/qla2xxx/qla_mid.c @@ -574,16 +574,19 @@ qla25xx_free_req_que(struct scsi_qla_host *vha, struct req_que *req) { struct qla_hw_data *ha = vha->hw; uint16_t que_id = req->id; + size_t req_entry_size = sizeof(request_t); - dma_free_coherent(&ha->pdev->dev, (req->length + 1) * - sizeof(request_t), req->ring, req->dma); + if (req->ring) + dma_free_coherent(&ha->pdev->dev, + (req->length + 1) * req_entry_size, + req->ring, req->dma); req->ring = NULL; req->dma = 0; if (que_id) { + mutex_lock(&ha->mq_lock); ha->req_q_map[que_id] = NULL; - mutex_lock(&ha->vport_lock); clear_bit(que_id, ha->req_qid_map); - mutex_unlock(&ha->vport_lock); + mutex_unlock(&ha->mq_lock); } kfree(req->outstanding_cmds); kfree(req); @@ -594,6 +597,7 @@ qla25xx_free_rsp_que(struct scsi_qla_host *vha, struct rsp_que *rsp) { struct qla_hw_data *ha = vha->hw; uint16_t que_id = rsp->id; + size_t rsp_entry_size = sizeof(response_t); if (rsp->msix && rsp->msix->have_irq) { free_irq(rsp->msix->vector, rsp->msix->handle); @@ -601,15 +605,22 @@ qla25xx_free_rsp_que(struct scsi_qla_host *vha, struct rsp_que *rsp) rsp->msix->in_use = 0; rsp->msix->handle = NULL; } - dma_free_coherent(&ha->pdev->dev, (rsp->length + 1) * - sizeof(response_t), rsp->ring, rsp->dma); + + /* Flush any queued response work before freeing the queue/qpair. */ + if (rsp->qpair && ha->wq) + cancel_work_sync(&rsp->qpair->q_work); + + if (rsp->ring) + dma_free_coherent(&ha->pdev->dev, + (rsp->length + 1) * rsp_entry_size, + rsp->ring, rsp->dma); rsp->ring = NULL; rsp->dma = 0; if (que_id) { + mutex_lock(&ha->mq_lock); ha->rsp_q_map[que_id] = NULL; - mutex_lock(&ha->vport_lock); clear_bit(que_id, ha->rsp_qid_map); - mutex_unlock(&ha->vport_lock); + mutex_unlock(&ha->mq_lock); } kfree(rsp); } @@ -794,9 +805,6 @@ qla25xx_create_req_que(struct qla_hw_data *ha, uint16_t options, if (ret != QLA_SUCCESS) { ql_log(ql_log_fatal, base_vha, 0x00df, "%s failed.\n", __func__); - mutex_lock(&ha->mq_lock); - clear_bit(que_id, ha->req_qid_map); - mutex_unlock(&ha->mq_lock); goto que_failed; } vha->flags.qpairs_req_created = 1; @@ -908,9 +916,6 @@ qla25xx_create_rsp_que(struct qla_hw_data *ha, uint16_t options, if (ret != QLA_SUCCESS) { ql_log(ql_log_fatal, base_vha, 0x00e7, "%s failed.\n", __func__); - mutex_lock(&ha->mq_lock); - clear_bit(que_id, ha->rsp_qid_map); - mutex_unlock(&ha->mq_lock); goto que_failed; } vha->flags.qpairs_rsp_created = 1; @@ -957,6 +962,22 @@ int qla24xx_control_vp(scsi_qla_host_t *vha, int cmd) if (vp_index == 0 || vp_index >= ha->max_npiv_vports) return QLA_PARAMETER_ERROR; + /* + * The VP_CTRL IOCB selects the target VP through a fixed 128-bit + * (16-byte) vp_idx_map bitmap, so vp_index must fit within it even + * if firmware advertises more NPIV vports. + */ + if (vp_index > sizeof_field(struct vp_ctrl_entry_24xx, vp_idx_map) * 8) + return QLA_PARAMETER_ERROR; + + /* + * The VP_CTRL IOCB selects the target VP through a fixed 128-bit + * (16-byte) vp_idx_map bitmap, so vp_index must fit within it even + * if firmware advertises more NPIV vports. + */ + if (vp_index > VP_CTRL_IDX_MAP_BITS) + return QLA_PARAMETER_ERROR; + /* ref: INIT */ sp = qla2x00_get_sp(base_vha, NULL, GFP_KERNEL); if (!sp) diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c index 2531e71c39dc8..7177222681962 100644 --- a/drivers/scsi/qla2xxx/qla_nvme.c +++ b/drivers/scsi/qla2xxx/qla_nvme.c @@ -257,7 +257,9 @@ static void qla_nvme_release_lsrsp_cmd_kref(struct kref *kref) fd_rsp = uctx->fd_rsp; + spin_lock_irqsave(&uctx->fcport->unsol_ctx_lock, flags); list_del(&uctx->elem); + spin_unlock_irqrestore(&uctx->fcport->unsol_ctx_lock, flags); fd_rsp->done(fd_rsp); kfree(uctx); @@ -374,6 +376,7 @@ static int qla_nvme_xmt_ls_rsp(struct nvme_fc_local_port *lport, srb_t *sp; int rval = QLA_FUNCTION_FAILED; uint8_t cnt = 0; + unsigned long flags; if (!fcport || fcport->deleted) goto out; @@ -440,7 +443,14 @@ static int qla_nvme_xmt_ls_rsp(struct nvme_fc_local_port *lport, a.vp_idx = vha->vp_idx; a.nport_handle = uctx->nport_handle; a.xchg_address = uctx->exchange_address; - qla_nvme_ls_reject_iocb(vha, ha->base_qpair, &a, true); + if (ha->flags.fw_started) { + spin_lock_irqsave(ha->base_qpair->qp_lock_ptr, flags); + qla_nvme_ls_reject_iocb(vha, ha->base_qpair, &a, true); + spin_unlock_irqrestore(ha->base_qpair->qp_lock_ptr, flags); + } + spin_lock_irqsave(&uctx->fcport->unsol_ctx_lock, flags); + list_del(&uctx->elem); + spin_unlock_irqrestore(&uctx->fcport->unsol_ctx_lock, flags); kfree(uctx); return rval; } @@ -463,8 +473,8 @@ static void qla_nvme_ls_abort(struct nvme_fc_local_port *lport, } spin_unlock_irqrestore(&priv->cmd_lock, flags); - INIT_WORK(&priv->abort_work, qla_nvme_abort_work); - schedule_work(&priv->abort_work); + if (!schedule_work(&priv->abort_work)) + kref_put(&priv->sp->cmd_kref, priv->sp->put_fn); } static int qla_nvme_ls_req(struct nvme_fc_local_port *lport, @@ -501,6 +511,7 @@ static int qla_nvme_ls_req(struct nvme_fc_local_port *lport, priv->sp = sp; kref_init(&sp->cmd_kref); spin_lock_init(&priv->cmd_lock); + INIT_WORK(&priv->abort_work, qla_nvme_abort_work); nvme = &sp->u.iocb_cmd; priv->fd = fd; nvme->u.nvme.desc = fd; @@ -545,8 +556,8 @@ static void qla_nvme_fcp_abort(struct nvme_fc_local_port *lport, } spin_unlock_irqrestore(&priv->cmd_lock, flags); - INIT_WORK(&priv->abort_work, qla_nvme_abort_work); - schedule_work(&priv->abort_work); + if (!schedule_work(&priv->abort_work)) + kref_put(&priv->sp->cmd_kref, priv->sp->put_fn); } static inline int qla2x00_start_nvme_mq(srb_t *sp) @@ -811,6 +822,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport, kref_init(&sp->cmd_kref); spin_lock_init(&priv->cmd_lock); + INIT_WORK(&priv->abort_work, qla_nvme_abort_work); sp->priv = priv; priv->sp = sp; sp->type = SRB_NVME_CMD; @@ -1127,6 +1139,10 @@ static void qla_nvme_lsrjt_pt_iocb(struct scsi_qla_host *vha, lsrjt_iocb->rx_byte_count = 0; } +/* + * Allocates from and advances the request ring, so the caller must hold + * qp->qp_lock_ptr (the response-queue caller already holds it). + */ static int qla_nvme_ls_reject_iocb(struct scsi_qla_host *vha, struct qla_qpair *qp, struct qla_nvme_lsrjt_pt_arg *a, bool is_xchg_terminate) @@ -1183,6 +1199,7 @@ qla2xxx_process_purls_pkt(struct scsi_qla_host *vha, struct purex_item *item) { struct qla_nvme_unsol_ctx *uctx = item->purls_context; struct qla_nvme_lsrjt_pt_arg a; + unsigned long flags; int ret = 1; #if (IS_ENABLED(CONFIG_NVME_FC)) @@ -1195,8 +1212,17 @@ qla2xxx_process_purls_pkt(struct scsi_qla_host *vha, struct purex_item *item) a.vp_idx = vha->vp_idx; a.nport_handle = uctx->nport_handle; a.xchg_address = uctx->exchange_address; - qla_nvme_ls_reject_iocb(vha, vha->hw->base_qpair, &a, true); + if (vha->hw->flags.fw_started) { + spin_lock_irqsave(vha->hw->base_qpair->qp_lock_ptr, + flags); + qla_nvme_ls_reject_iocb(vha, vha->hw->base_qpair, &a, + true); + spin_unlock_irqrestore(vha->hw->base_qpair->qp_lock_ptr, + flags); + } + spin_lock_irqsave(&uctx->fcport->unsol_ctx_lock, flags); list_del(&uctx->elem); + spin_unlock_irqrestore(&uctx->fcport->unsol_ctx_lock, flags); kfree(uctx); } } @@ -1238,6 +1264,7 @@ void qla2xxx_process_purls_iocb(void **pkt, struct rsp_que **rsp) struct purex_item *item; port_id_t d_id = {0}; port_id_t id = {0}; + unsigned long flags; u8 *opcode; bool xmt_reject = false; @@ -1303,7 +1330,9 @@ void qla2xxx_process_purls_iocb(void **pkt, struct rsp_que **rsp) uctx->ox_id = p->ox_id; qla_rport->uctx = uctx; INIT_LIST_HEAD(&uctx->elem); + spin_lock_irqsave(&fcport->unsol_ctx_lock, flags); list_add_tail(&uctx->elem, &fcport->unsol_ctx_head); + spin_unlock_irqrestore(&fcport->unsol_ctx_lock, flags); item->purls_context = (void *)uctx; ql_dbg(ql_dbg_unsol, vha, 0x2121, diff --git a/drivers/scsi/qla2xxx/qla_nx2.c b/drivers/scsi/qla2xxx/qla_nx2.c index 41ff6fbdb9333..04d7ab6ad037a 100644 --- a/drivers/scsi/qla2xxx/qla_nx2.c +++ b/drivers/scsi/qla2xxx/qla_nx2.c @@ -3507,7 +3507,7 @@ qla8044_poll_flash_status_reg(struct scsi_qla_host *vha) msleep(QLA8044_FLASH_STATUS_REG_POLL_DELAY); } - if (!retries) + if (retries == -1) ret_val = QLA_FUNCTION_FAILED; return ret_val; diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c index 72b1c28e4dae0..f2114b386fa0f 100644 --- a/drivers/scsi/qla2xxx/qla_os.c +++ b/drivers/scsi/qla2xxx/qla_os.c @@ -352,7 +352,7 @@ MODULE_PARM_DESC(ql2xnvme_queues, int ql2xfc2target = 1; module_param(ql2xfc2target, int, 0444); -MODULE_PARM_DESC(qla2xfc2target, +MODULE_PARM_DESC(ql2xfc2target, "Enables FC2 Target support. " "0 - FC2 Target support is disabled. " "1 - FC2 Target support is enabled (default)."); @@ -2127,7 +2127,7 @@ qla2x00_iospace_config(struct qla_hw_data *ha) ha->msix_count = msix + 1; /* Max queues are bounded by available msix vectors */ /* MB interrupt uses 1 vector */ - ha->max_req_queues = ha->msix_count - 1; + ha->max_req_queues = qla_calc_queue_count(ha->msix_count); ha->max_rsp_queues = ha->max_req_queues; /* Queue pairs is the max value minus the base queue pair */ ha->max_qpairs = ha->max_rsp_queues - 1; @@ -2213,10 +2213,10 @@ qla83xx_iospace_config(struct qla_hw_data *ha) */ if (ql2xmqsupport || ql2xnvmeenable) { /* MB interrupt uses 1 vector */ - ha->max_req_queues = ha->msix_count - 1; + ha->max_req_queues = qla_calc_queue_count(ha->msix_count); /* ATIOQ needs 1 vector. That's 1 less QPair */ - if (QLA_TGT_MODE_ENABLED()) + if (QLA_TGT_MODE_ENABLED() && ha->max_req_queues > 1) ha->max_req_queues--; ha->max_rsp_queues = ha->max_req_queues; @@ -3933,8 +3933,6 @@ qla2x00_remove_one(struct pci_dev *pdev) qla2x00_dfs_remove(base_vha); - qla84xx_put_chip(base_vha); - /* Disable timer */ if (base_vha->timer_active) qla2x00_stop_timer(base_vha); @@ -3959,6 +3957,8 @@ qla2x00_remove_one(struct pci_dev *pdev) scsi_remove_host(base_vha->host); + qla84xx_put_chip(base_vha); + qla2x00_free_device(base_vha); qla2x00_clear_drv_active(ha); @@ -4484,25 +4484,40 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len, fail_lsrjt: dma_free_coherent(&ha->pdev->dev, ha->elsrej.size, ha->elsrej.c, ha->elsrej.cdma); + ha->elsrej.c = NULL; + ha->elsrej.cdma = 0; fail_elsrej: dma_pool_destroy(ha->purex_dma_pool); + ha->purex_dma_pool = NULL; fail_flt: dma_free_coherent(&ha->pdev->dev, sizeof(struct qla_flt_header) + FLT_REGIONS_SIZE, ha->flt, ha->flt_dma); + ha->flt = NULL; + ha->flt_dma = 0; fail_flt_buffer: dma_free_coherent(&ha->pdev->dev, SFP_DEV_SIZE, ha->sfp_data, ha->sfp_data_dma); + ha->sfp_data = NULL; + ha->sfp_data_dma = 0; fail_sfp_data: kfree(ha->loop_id_map); + ha->loop_id_map = NULL; fail_loop_id_map: dma_pool_free(ha->s_dma_pool, ha->async_pd, ha->async_pd_dma); + ha->async_pd = NULL; + ha->async_pd_dma = 0; fail_async_pd: dma_pool_free(ha->s_dma_pool, ha->sf_init_cb, ha->sf_init_cb_dma); + ha->sf_init_cb = NULL; + ha->sf_init_cb_dma = 0; fail_sf_init_cb: dma_pool_free(ha->s_dma_pool, ha->ex_init_cb, ha->ex_init_cb_dma); + ha->ex_init_cb = NULL; + ha->ex_init_cb_dma = 0; fail_ex_init_cb: kfree(ha->npiv_info); + ha->npiv_info = NULL; fail_npiv_info: dma_free_coherent(&ha->pdev->dev, ((*rsp)->length + 1) * sizeof(response_t), (*rsp)->ring, (*rsp)->dma); @@ -6828,8 +6843,6 @@ qla2x00_disable_board_on_pci_error(struct work_struct *work) qla2x00_dfs_remove(base_vha); - qla84xx_put_chip(base_vha); - if (base_vha->timer_active) qla2x00_stop_timer(base_vha); @@ -6847,6 +6860,8 @@ qla2x00_disable_board_on_pci_error(struct work_struct *work) scsi_remove_host(base_vha->host); + qla84xx_put_chip(base_vha); + base_vha->flags.init_done = 0; qla25xx_delete_queues(base_vha); qla2x00_free_fcports(base_vha); diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c index 76cdad063f7bc..f285521d9de6d 100644 --- a/drivers/scsi/scsi.c +++ b/drivers/scsi/scsi.c @@ -727,6 +727,7 @@ int scsi_cdl_enable(struct scsi_device *sdev, bool enable) struct scsi_mode_data data; struct scsi_sense_hdr sshdr; char *buf_data; + size_t avail, offset; int len; ret = scsi_mode_sense(sdev, 0x08, 0x0a, 0xf2, buf, sizeof(buf), @@ -735,11 +736,24 @@ int scsi_cdl_enable(struct scsi_device *sdev, bool enable) return -EINVAL; /* Enable or disable CDL using the ATA feature page */ - len = min_t(size_t, sizeof(buf), - data.length - data.header_length - - data.block_descriptor_length); - buf_data = buf + data.header_length + - data.block_descriptor_length; + avail = min_t(size_t, data.length, sizeof(buf)); + if (data.header_length > avail) + return -EINVAL; + + offset = data.header_length; + avail -= data.header_length; + + if (data.block_descriptor_length > avail) + return -EINVAL; + + offset += data.block_descriptor_length; + avail -= data.block_descriptor_length; + + if (avail < 5) + return -EINVAL; + + buf_data = buf + offset; + len = avail; /* * If we want to enable CDL and CDL is already enabled on the diff --git a/drivers/scsi/scsi_bsg.c b/drivers/scsi/scsi_bsg.c index e80dec53174ea..5eec248a77a6b 100644 --- a/drivers/scsi/scsi_bsg.c +++ b/drivers/scsi/scsi_bsg.c @@ -18,6 +18,7 @@ struct scsi_bsg_uring_cmd_pdu { struct bio *bio; /* mapped user buffer, unmap in task work */ struct request *req; /* block request, freed in task work */ u64 response_addr; /* user space response buffer address */ + u32 max_response_len; /* user response buffer size */ }; static_assert(sizeof(struct scsi_bsg_uring_cmd_pdu) <= sizeof_field(struct io_uring_cmd, pdu)); @@ -45,8 +46,8 @@ static void scsi_bsg_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw) if (scsi_status_is_check_condition(scmd->result)) { driver_status = DRIVER_SENSE; if (pdu->response_addr) - sense_len_wr = min_t(u8, scmd->sense_len, - SCSI_SENSE_BUFFERSIZE); + sense_len_wr = min_t(unsigned int, pdu->max_response_len, + scmd->sense_len); } if (sense_len_wr) { @@ -76,12 +77,10 @@ static enum rq_end_io_ret scsi_bsg_uring_cmd_done(struct request *req, static int scsi_bsg_map_user_buffer(struct request *req, struct io_uring_cmd *ioucmd, - unsigned int issue_flags, gfp_t gfp_mask) + unsigned int issue_flags, gfp_t gfp_mask, + bool is_write, u64 buf_addr, + unsigned long buf_len) { - const struct bsg_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe, struct bsg_uring_cmd); - bool is_write = cmd->dout_xfer_len > 0; - u64 buf_addr = is_write ? cmd->dout_xferp : cmd->din_xferp; - unsigned long buf_len = is_write ? cmd->dout_xfer_len : cmd->din_xfer_len; struct iov_iter iter; int ret; @@ -104,21 +103,28 @@ static int scsi_bsg_uring_cmd(struct request_queue *q, struct io_uring_cmd *iouc unsigned int issue_flags, bool open_for_write) { struct scsi_bsg_uring_cmd_pdu *pdu = scsi_bsg_uring_cmd_pdu(ioucmd); - const struct bsg_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe, struct bsg_uring_cmd); + const struct bsg_uring_cmd *cmd = + io_uring_sqe128_cmd(ioucmd->sqe, struct bsg_uring_cmd); struct scsi_cmnd *scmd; struct request *req; blk_mq_req_flags_t blk_flags = 0; gfp_t gfp_mask = GFP_KERNEL; + u64 request = READ_ONCE(cmd->request); + u32 request_len = READ_ONCE(cmd->request_len); + u64 dout_xferp = READ_ONCE(cmd->dout_xferp); + u32 dout_xfer_len = READ_ONCE(cmd->dout_xfer_len); + u64 din_xferp = READ_ONCE(cmd->din_xferp); + u32 din_xfer_len = READ_ONCE(cmd->din_xfer_len); int ret; if (cmd->protocol != BSG_PROTOCOL_SCSI || cmd->subprotocol != BSG_SUB_PROTOCOL_SCSI_CMD) return -EINVAL; - if (!cmd->request || cmd->request_len == 0) + if (!request || request_len == 0) return -EINVAL; - if (cmd->dout_xfer_len && cmd->din_xfer_len) { + if (dout_xfer_len && din_xfer_len) { pr_warn_once("BIDI support in bsg has been removed.\n"); return -EOPNOTSUPP; } @@ -131,20 +137,20 @@ static int scsi_bsg_uring_cmd(struct request_queue *q, struct io_uring_cmd *iouc gfp_mask = GFP_NOWAIT; } - req = scsi_alloc_request(q, cmd->dout_xfer_len ? + req = scsi_alloc_request(q, dout_xfer_len ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, blk_flags); if (IS_ERR(req)) return PTR_ERR(req); scmd = blk_mq_rq_to_pdu(req); - if (cmd->request_len > sizeof(scmd->cmnd)) { + if (request_len > sizeof(scmd->cmnd)) { ret = -EINVAL; goto out_free_req; } - scmd->cmd_len = cmd->request_len; + scmd->cmd_len = request_len; scmd->allowed = SG_DEFAULT_RETRIES; - if (copy_from_user(scmd->cmnd, uptr64(cmd->request), cmd->request_len)) { + if (copy_from_user(scmd->cmnd, uptr64(request), request_len)) { ret = -EFAULT; goto out_free_req; } @@ -155,11 +161,16 @@ static int scsi_bsg_uring_cmd(struct request_queue *q, struct io_uring_cmd *iouc } pdu->response_addr = cmd->response; - scmd->sense_len = cmd->max_response_len ? - min(cmd->max_response_len, SCSI_SENSE_BUFFERSIZE) : SCSI_SENSE_BUFFERSIZE; + pdu->max_response_len = cmd->max_response_len; - if (cmd->dout_xfer_len || cmd->din_xfer_len) { - ret = scsi_bsg_map_user_buffer(req, ioucmd, issue_flags, gfp_mask); + if (dout_xfer_len || din_xfer_len) { + bool is_write = dout_xfer_len > 0; + u64 buf_addr = is_write ? dout_xferp : din_xferp; + unsigned long buf_len = is_write ? dout_xfer_len : din_xfer_len; + + ret = scsi_bsg_map_user_buffer(req, ioucmd, issue_flags, + gfp_mask, is_write, buf_addr, + buf_len); if (ret) goto out_free_req; pdu->bio = req->bio; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index daeb3693fe559..480a0bf2aff33 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1187,8 +1187,10 @@ blk_status_t scsi_alloc_sgtables(struct scsi_cmnd *cmd) if (blk_rq_bytes(rq) & rq->q->limits.dma_pad_mask) { unsigned int pad_len = (rq->q->limits.dma_pad_mask & ~blk_rq_bytes(rq)) + 1; + unsigned int data_len = last_sg->length; last_sg->length += pad_len; + sg_zero_buffer(last_sg, 1, pad_len, data_len); cmd->extra_len += pad_len; } diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 599e75f333343..a1b21ea14e549 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -958,6 +958,21 @@ static unsigned char sd_setup_protect_cmnd(struct scsi_cmnd *scmd, return protect; } +static void sd_uninit_command(struct scsi_cmnd *cmd) +{ + struct request *rq = scsi_cmd_to_rq(cmd); + struct scsi_device *sdp = cmd->device; + + if (!(rq->rq_flags & RQF_SPECIAL_PAYLOAD)) + return; + + if (sdp->sector_size > PAGE_SIZE) + mempool_free(rq->special_vec.bv_page, sd_large_page_pool); + else + mempool_free(rq->special_vec.bv_page, sd_page_pool); + rq->rq_flags &= ~RQF_SPECIAL_PAYLOAD; +} + static void *sd_set_special_bvec(struct scsi_cmnd *cmd, unsigned int data_len) { struct page *page; @@ -990,6 +1005,7 @@ static blk_status_t sd_setup_unmap_cmnd(struct scsi_cmnd *cmd) u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); unsigned int data_len = 24; char *buf; + blk_status_t ret; buf = sd_set_special_bvec(cmd, data_len); if (!buf) @@ -1008,7 +1024,10 @@ static blk_status_t sd_setup_unmap_cmnd(struct scsi_cmnd *cmd) cmd->transfersize = data_len; rq->timeout = SD_TIMEOUT; - return scsi_alloc_sgtables(cmd); + ret = scsi_alloc_sgtables(cmd); + if (ret != BLK_STS_OK) + sd_uninit_command(cmd); + return ret; } static void sd_config_atomic(struct scsi_disk *sdkp, struct queue_limits *lim) @@ -1079,6 +1098,7 @@ static blk_status_t sd_setup_write_same16_cmnd(struct scsi_cmnd *cmd, u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); u32 data_len = sdp->sector_size; + blk_status_t ret; if (!sd_set_special_bvec(cmd, data_len)) return BLK_STS_RESOURCE; @@ -1094,7 +1114,10 @@ static blk_status_t sd_setup_write_same16_cmnd(struct scsi_cmnd *cmd, cmd->transfersize = data_len; rq->timeout = unmap ? SD_TIMEOUT : SD_WRITE_SAME_TIMEOUT; - return scsi_alloc_sgtables(cmd); + ret = scsi_alloc_sgtables(cmd); + if (ret != BLK_STS_OK) + sd_uninit_command(cmd); + return ret; } static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, @@ -1106,6 +1129,7 @@ static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); u32 data_len = sdp->sector_size; + blk_status_t ret; if (!sd_set_special_bvec(cmd, data_len)) return BLK_STS_RESOURCE; @@ -1121,7 +1145,10 @@ static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, cmd->transfersize = data_len; rq->timeout = unmap ? SD_TIMEOUT : SD_WRITE_SAME_TIMEOUT; - return scsi_alloc_sgtables(cmd); + ret = scsi_alloc_sgtables(cmd); + if (ret != BLK_STS_OK) + sd_uninit_command(cmd); + return ret; } static blk_status_t sd_setup_write_zeroes_cmnd(struct scsi_cmnd *cmd) @@ -1550,20 +1577,6 @@ static blk_status_t sd_init_command(struct scsi_cmnd *cmd) } } -static void sd_uninit_command(struct scsi_cmnd *SCpnt) -{ - struct request *rq = scsi_cmd_to_rq(SCpnt); - struct scsi_device *sdp = SCpnt->device; - unsigned sector_size = sdp->sector_size; - - if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { - if (sector_size > PAGE_SIZE) - mempool_free(rq->special_vec.bv_page, sd_large_page_pool); - else - mempool_free(rq->special_vec.bv_page, sd_page_pool); - } -} - static bool sd_need_revalidate(struct gendisk *disk, struct scsi_disk *sdkp) { if (sdkp->device->removable || sdkp->write_prot) { @@ -2406,8 +2419,8 @@ static int sd_done(struct scsi_cmnd *SCpnt) } sdkp->medium_access_timed_out = 0; - if (!scsi_status_is_check_condition(result) && - (!sense_valid || sense_deferred)) + if (!scsi_status_is_check_condition(result) || + !sense_valid || sense_deferred) goto out; switch (sshdr.sense_key) { @@ -4089,7 +4102,9 @@ static int sd_probe(struct scsi_device *sdp) if (sdp->sector_size > PAGE_SIZE) { if (sd_large_pool_create()) { error = -ENOMEM; - goto out_free_index; + device_unregister(&sdkp->disk_dev); + put_disk(gd); + goto out; } } diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index 56e455fb5addd..456beaf2e7690 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -589,7 +589,7 @@ int sd_zbc_revalidate_zones(struct scsi_disk *sdkp) int sd_zbc_read_zones(struct scsi_disk *sdkp, struct queue_limits *lim, u8 buf[SD_BUF_SIZE]) { - unsigned int nr_zones; + u64 nr_zones; u32 zone_blocks = 0; int ret; @@ -621,6 +621,12 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, struct queue_limits *lim, goto err; nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks); + if (nr_zones > INT_MAX) { + sd_printk(KERN_ERR, sdkp, "Too many zones (%llu)\n", + nr_zones); + ret = -EINVAL; + goto err; + } sdkp->early_zone_info.nr_zones = nr_zones; sdkp->early_zone_info.zone_blocks = zone_blocks; diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index 5ec583dc2e7df..3a75b9fbedf45 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -66,6 +66,12 @@ static struct pqi_cmd_priv *pqi_cmd_priv(struct scsi_cmnd *cmd) return scsi_cmd_priv(cmd); } +static int pqi_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) +{ + memset(pqi_cmd_priv(cmd), 0, sizeof(struct pqi_cmd_priv)); + return 0; +} + static void pqi_verify_structures(void); static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info, enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason); @@ -5958,6 +5964,17 @@ void pqi_prep_for_scsi_done(struct scsi_cmnd *scmd) struct pqi_scsi_dev *device; struct completion *wait; + /* + * Clear the AIO-retry marker on final completion so the tag + * starts clean on its next dispatch. On DID_IMM_RETRY leave + * it intact: pqi_aio_io_complete() sets DID_IMM_RETRY and + * bumps the marker to steer the requeue onto the RAID path, + * and pqi_process_raid_io_error() consumes the non-zero + * marker to offline a misbehaving drive. + */ + if (host_byte(scmd->result) != DID_IMM_RETRY) + pqi_cmd_priv(scmd)->this_residual = 0; + if (!scmd->device) { set_host_byte(scmd, DID_NO_CONNECT); return; @@ -7612,6 +7629,7 @@ static const struct scsi_host_template pqi_driver_template = { .sdev_groups = pqi_sdev_groups, .shost_groups = pqi_shost_groups, .cmd_size = sizeof(struct pqi_cmd_priv), + .init_cmd_priv = pqi_init_cmd_priv, }; static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info) diff --git a/drivers/soc/fsl/qe/gpio.c b/drivers/soc/fsl/qe/gpio.c index 66828f2a35774..6d8f4d549fe29 100644 --- a/drivers/soc/fsl/qe/gpio.c +++ b/drivers/soc/fsl/qe/gpio.c @@ -135,6 +135,30 @@ static int qe_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) return 0; } +static int qe_gpio_get_direction(struct gpio_chip *gc, unsigned int gpio) +{ + struct qe_gpio_chip *qe_gc = gpiochip_get_data(gc); + struct qe_pio_regs __iomem *regs = qe_gc->regs; + unsigned long flags; + u32 val, mask; + + spin_lock_irqsave(&qe_gc->lock, flags); + + if (gpio < QE_PIO_PINS / 2) + val = ioread32be(®s->cpdir1); + else + val = ioread32be(®s->cpdir2); + + spin_unlock_irqrestore(&qe_gc->lock, flags); + + mask = (u32)QE_PIO_DIR_OUT << (QE_PIO_PINS - 2 - (gpio % (QE_PIO_PINS / 2)) * 2); + + if (val & mask) + return GPIO_LINE_DIRECTION_OUT; + else + return GPIO_LINE_DIRECTION_IN; +} + struct qe_pin { /* * The qe_gpio_chip name is unfortunate, we should change that to @@ -308,6 +332,7 @@ static int qe_gpio_probe(struct platform_device *ofdev) gc->ngpio = QE_PIO_PINS; gc->direction_input = qe_gpio_dir_in; gc->direction_output = qe_gpio_dir_out; + gc->get_direction = qe_gpio_get_direction; gc->get = qe_gpio_get; gc->set = qe_gpio_set; gc->set_multiple = qe_gpio_set_multiple; diff --git a/drivers/soc/fsl/qe/qe_ic.c b/drivers/soc/fsl/qe/qe_ic.c index 943911053af68..bccc7d612009b 100644 --- a/drivers/soc/fsl/qe/qe_ic.c +++ b/drivers/soc/fsl/qe/qe_ic.c @@ -473,7 +473,6 @@ static struct platform_driver qe_ic_driver = static int __init qe_ic_of_init(void) { - platform_driver_register(&qe_ic_driver); - return 0; + return platform_driver_register(&qe_ic_driver); } subsys_initcall(qe_ic_of_init); diff --git a/drivers/soc/fsl/qe/qe_io.c b/drivers/soc/fsl/qe/qe_io.c index a5e2d0e5ab511..150913fce9818 100644 --- a/drivers/soc/fsl/qe/qe_io.c +++ b/drivers/soc/fsl/qe/qe_io.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -184,3 +185,17 @@ int par_io_of_config(struct device_node *np) return 0; } EXPORT_SYMBOL(par_io_of_config); + +static int __init par_io_populate(void) +{ + struct device_node *np = of_find_node_by_type(NULL, "par_io"); + + if (!np) + return 0; + + of_platform_default_populate(np, NULL, NULL); + of_node_put(np); + + return 0; +} +arch_initcall(par_io_populate); diff --git a/drivers/soc/fsl/qe/qe_ports_ic.c b/drivers/soc/fsl/qe/qe_ports_ic.c index 9b0bba64e91e4..73a77763210d7 100644 --- a/drivers/soc/fsl/qe/qe_ports_ic.c +++ b/drivers/soc/fsl/qe/qe_ports_ic.c @@ -6,6 +6,7 @@ */ #include +#include #include #include @@ -79,7 +80,13 @@ static int qepic_get_irq(struct irq_desc *desc) static void qepic_cascade(struct irq_desc *desc) { + struct irq_chip *chip = irq_desc_get_chip(desc); + + chained_irq_enter(chip, desc); + generic_handle_irq(qepic_get_irq(desc)); + + chained_irq_exit(chip, desc); } static int qepic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) diff --git a/drivers/soc/qcom/pmic_glink_altmode.c b/drivers/soc/qcom/pmic_glink_altmode.c index 619bad2c27eeb..13c434f8d03a6 100644 --- a/drivers/soc/qcom/pmic_glink_altmode.c +++ b/drivers/soc/qcom/pmic_glink_altmode.c @@ -53,7 +53,7 @@ struct usbc_sc8280x_tbt_data { /* This field is NOP on USB4, all cables support rounded rates by spec */ u8 rounded_cable : 1; u8 power_limited : 1; - u8 res[11]; + u8 res[7]; }; struct usbc_notify { @@ -74,6 +74,7 @@ struct usbc_notify { struct usbc_sc8280x_dp_data dp; struct usbc_sc8280x_tbt_data tbt; } extended_data; + u32 reserved; }; struct usbc_sc8180x_notify { diff --git a/drivers/soc/qcom/qcom-geni-se.c b/drivers/soc/qcom/qcom-geni-se.c index 15636a8dc907a..107fa2057654a 100644 --- a/drivers/soc/qcom/qcom-geni-se.c +++ b/drivers/soc/qcom/qcom-geni-se.c @@ -154,8 +154,6 @@ struct se_fw_hdr { /*Magic numbers*/ #define SE_MAGIC_NUM 0x57464553 -#define MAX_GENI_CFG_RAMn_CNT 455 - #define MI_PBT_NON_PAGED_SEGMENT 0x0 #define MI_PBT_HASH_SEGMENT 0x2 #define MI_PBT_NOTUSED_SEGMENT 0x3 @@ -1224,24 +1222,27 @@ EXPORT_SYMBOL_GPL(geni_se_resources_init); /** * geni_find_protocol_fw() - Locate and validate SE firmware for a protocol. - * @dev: Pointer to the device structure. + * @se: Pointer to the serial engine structure. * @fw: Pointer to the firmware image. * @protocol: Expected serial engine protocol type. * * Identifies the appropriate firmware image or configuration required for a - * specific communication protocol instance running on a Qualcomm GENI - * controller. + * specific communication protocol instance running on a Qualcomm GENI + * controller. Validates the firmware size against the hardware PROG_RAM_DEPTH + * read from SE_HW_PARAM_2. * * Return: pointer to a valid 'struct se_fw_hdr' if found, or NULL otherwise. */ -static struct se_fw_hdr *geni_find_protocol_fw(struct device *dev, const struct firmware *fw, +static struct se_fw_hdr *geni_find_protocol_fw(struct geni_se *se, const struct firmware *fw, enum geni_se_protocol_type protocol) { + struct device *dev = se->dev; const struct elf32_hdr *ehdr; const struct elf32_phdr *phdrs; const struct elf32_phdr *phdr; struct se_fw_hdr *sefw; u32 fw_end, cfg_idx_end, cfg_val_end; + u32 prog_ram_depth; u16 fw_size; int i; @@ -1300,10 +1301,11 @@ static struct se_fw_hdr *geni_find_protocol_fw(struct device *dev, const struct sefw->fw_size_in_items = cpu_to_le16(fw_size); } - if (fw_size >= MAX_GENI_CFG_RAMn_CNT) { - dev_err(dev, - "Firmware size (%u) exceeds max allowed RAMn count (%u)\n", - fw_size, MAX_GENI_CFG_RAMn_CNT); + prog_ram_depth = FIELD_GET(PROG_RAM_DEPTH_MSK, + readl_relaxed(se->base + SE_HW_PARAM_2)); + if (fw_size >= prog_ram_depth) { + dev_err(dev, "Firmware size (%u) exceeds RAM size (%u)\n", + fw_size, prog_ram_depth); continue; } @@ -1427,7 +1429,7 @@ static int geni_load_se_fw(struct geni_se *se, const struct firmware *fw, int ret; struct se_fw_hdr *hdr; - hdr = geni_find_protocol_fw(se->dev, fw, protocol); + hdr = geni_find_protocol_fw(se, fw, protocol); if (!hdr) return -EINVAL; diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index c6f7d5c9c493d..66928ca40b9aa 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -944,17 +944,30 @@ static int rpmh_rsc_pd_callback(struct notifier_block *nfb, return NOTIFY_OK; } +static void rpmh_rsc_pd_detach(void *data) +{ + dev_pm_genpd_remove_notifier(data); +} + static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev) { int ret; - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback; ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb); if (ret) - pm_runtime_disable(dev); + return ret; - return ret; + return devm_add_action_or_reset(dev, rpmh_rsc_pd_detach, dev); +} + +static void rpmh_rsc_cpu_pm_unregister(void *data) +{ + cpu_pm_unregister_notifier(data); } static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv) @@ -1107,7 +1120,15 @@ static int rpmh_rsc_probe(struct platform_device *pdev) return ret; } else { drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; - cpu_pm_register_notifier(&drv->rsc_pm); + ret = cpu_pm_register_notifier(&drv->rsc_pm); + if (ret) + return ret; + + ret = devm_add_action_or_reset(&pdev->dev, + rpmh_rsc_cpu_pm_unregister, + &drv->rsc_pm); + if (ret) + return ret; } } @@ -1122,13 +1143,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, drv); drv->dev = &pdev->dev; - ret = devm_of_platform_populate(&pdev->dev); - if (ret && pdev->dev.pm_domain) { - dev_pm_genpd_remove_notifier(&pdev->dev); - pm_runtime_disable(&pdev->dev); - } - - return ret; + return devm_of_platform_populate(&pdev->dev); } static const struct of_device_id rpmh_drv_match[] = { diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig index 2ab150d04bb1f..fdf18ed2dfc21 100644 --- a/drivers/soc/renesas/Kconfig +++ b/drivers/soc/renesas/Kconfig @@ -356,7 +356,6 @@ config ARCH_R8A779H0 config ARCH_R8A78000 bool "ARM64 Platform support for R8A78000 (R-Car X5H)" default y if ARCH_RENESAS - default ARCH_RENESAS select ARCH_RCAR_GEN5 help This enables support for the Renesas R-Car X5H SoC. diff --git a/drivers/soc/samsung/exynos-pmu.c b/drivers/soc/samsung/exynos-pmu.c index d58376c38179b..efccdd63e40ea 100644 --- a/drivers/soc/samsung/exynos-pmu.c +++ b/drivers/soc/samsung/exynos-pmu.c @@ -167,8 +167,8 @@ static const struct mfd_cell exynos_pmu_devs[] = { */ struct regmap *exynos_get_pmu_regmap(void) { - struct device_node *np = of_find_matching_node(NULL, - exynos_pmu_of_device_ids); + struct device_node *np __free(device_node) = + of_find_matching_node(NULL, exynos_pmu_of_device_ids); if (np) return exynos_get_pmu_regmap_by_phandle(np, NULL); return ERR_PTR(-ENODEV); @@ -409,13 +409,12 @@ static struct notifier_block exynos_cpupm_reboot_nb = { static int setup_cpuhp_and_cpuidle(struct device *dev) { - struct device_node *intr_gen_node; + struct device_node *intr_gen_node __free(device_node) = + of_parse_phandle(dev->of_node, "google,pmu-intr-gen-syscon", 0); struct resource intrgen_res; void __iomem *virt_addr; int ret, cpu; - intr_gen_node = of_parse_phandle(dev->of_node, - "google,pmu-intr-gen-syscon", 0); if (!intr_gen_node) { /* * To maintain support for older DTs that didn't specify syscon @@ -431,8 +430,6 @@ static int setup_cpuhp_and_cpuidle(struct device *dev) * syscon provided regmap. */ ret = of_address_to_resource(intr_gen_node, 0, &intrgen_res); - of_node_put(intr_gen_node); - virt_addr = devm_ioremap(dev, intrgen_res.start, resource_size(&intrgen_res)); if (!virt_addr) diff --git a/drivers/soc/ti/knav_qmss.h b/drivers/soc/ti/knav_qmss.h index 037dc1b366453..8a624fbda84ab 100644 --- a/drivers/soc/ti/knav_qmss.h +++ b/drivers/soc/ti/knav_qmss.h @@ -304,6 +304,7 @@ struct knav_device { struct list_head pools; struct list_head pdsps; struct list_head qmgrs; + struct dentry *debugfs_file; enum qmss_version version; }; diff --git a/drivers/soc/ti/knav_qmss_queue.c b/drivers/soc/ti/knav_qmss_queue.c index 7410b63af0e62..3e4041454f694 100644 --- a/drivers/soc/ti/knav_qmss_queue.c +++ b/drivers/soc/ti/knav_qmss_queue.c @@ -1849,8 +1849,9 @@ static int knav_queue_probe(struct platform_device *pdev) goto err; } - debugfs_create_file("qmss", S_IFREG | S_IRUGO, NULL, NULL, - &knav_queue_debug_fops); + knav_qdev->debugfs_file = + debugfs_create_file("qmss", 0444, NULL, NULL, + &knav_queue_debug_fops); device_ready = true; return 0; @@ -1868,6 +1869,8 @@ static void knav_queue_remove(struct platform_device *pdev) struct knav_device *kdev = platform_get_drvdata(pdev); device_ready = false; + debugfs_remove(kdev->debugfs_file); + kdev->debugfs_file = NULL; knav_queue_stop_pdsps(kdev); knav_queue_free_regions(kdev); knav_free_queue_ranges(kdev); diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index eb66345a6a42b..cf456d7cf0a8f 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -1701,6 +1701,13 @@ int sdw_cdns_clock_stop(struct sdw_cdns *cdns, bool block_wake) return 0; } + /* + * wait for any in-flight peripheral event handling to complete before stopping the clock. + * No need to disable peripheral interrupts before canceling the work, as the peripheral + * interrupts are already masked before the work is scheduled. + */ + cancel_work_sync(&cdns->work); + /* * Before entering clock stop we mask the Slave * interrupts. This helps avoid having to deal with e.g. a @@ -2358,7 +2365,9 @@ int sdw_cdns_prepare_write_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, i p_data = sec[i].buf; while (section_size >= data_per_frame) { - header[1] = data_per_frame; + header[0] &= ~BIT(0); + header[0] |= (data_per_frame >> 8) & BIT(0); + header[1] = data_per_frame & 0xFF; header[2] = start_register >> 24 & 0xFF; header[3] = start_register >> 16 & 0xFF; header[4] = start_register >> 8 & 0xFF; @@ -2384,7 +2393,9 @@ int sdw_cdns_prepare_write_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, i } if (section_size) { - header[1] = section_size; + header[0] &= ~BIT(0); + header[0] |= (section_size >> 8) & BIT(0); + header[1] = section_size & 0xFF; header[2] = start_register >> 24 & 0xFF; header[3] = start_register >> 16 & 0xFF; header[4] = start_register >> 8 & 0xFF; @@ -2435,7 +2446,9 @@ int sdw_cdns_prepare_read_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, in start_register = sec[i].addr; data_size = sec[i].len; while (data_size >= data_per_frame) { - header[1] = data_per_frame; + header[0] &= ~BIT(0); + header[0] |= (data_per_frame >> 8) & BIT(0); + header[1] = data_per_frame & 0xFF; header[2] = start_register >> 24 & 0xFF; header[3] = start_register >> 16 & 0xFF; header[4] = start_register >> 8 & 0xFF; @@ -2459,7 +2472,9 @@ int sdw_cdns_prepare_read_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, in } if (data_size) { - header[1] = data_size; + header[0] &= ~BIT(0); + header[0] |= (data_size >> 8) & BIT(0); + header[1] = data_size & 0xFF; header[2] = start_register >> 24 & 0xFF; header[3] = start_register >> 16 & 0xFF; header[4] = start_register >> 8 & 0xFF; @@ -2482,7 +2497,9 @@ int sdw_cdns_prepare_read_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, in /* Add fake frame */ header[0] &= ~GENMASK(7, 6); /* Set inactive flag in BPT/BRA frame heade */ while (fake_size >= data_per_frame) { - header[1] = data_per_frame; + header[0] &= ~BIT(0); + header[0] |= (data_per_frame >> 8) & BIT(0); + header[1] = data_per_frame & 0xFF; ret = sdw_cdns_prepare_read_pd0_buffer(header, SDW_CDNS_BRA_HDR, p_dma_buffer, dma_buffer_size, &dma_data_written, counter); @@ -2498,7 +2515,9 @@ int sdw_cdns_prepare_read_dma_buffer(u8 dev_num, struct sdw_bpt_section *sec, in } if (fake_size) { - header[1] = fake_size; + header[0] &= ~BIT(0); + header[0] |= (fake_size >> 8) & BIT(0); + header[1] = fake_size & 0xFF; ret = sdw_cdns_prepare_read_pd0_buffer(header, SDW_CDNS_BRA_HDR, p_dma_buffer, dma_buffer_size, &dma_data_written, counter); diff --git a/drivers/soundwire/dmi-quirks.c b/drivers/soundwire/dmi-quirks.c index 32a46a2d90f7c..ff00c4b2578ed 100644 --- a/drivers/soundwire/dmi-quirks.c +++ b/drivers/soundwire/dmi-quirks.c @@ -192,6 +192,13 @@ static const struct dmi_system_id adr_remap_quirk_table[] = { }, .driver_data = (void *)ghost_realtek, }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUS"), + DMI_MATCH(DMI_BOARD_NAME, "GX651AX"), + }, + .driver_data = (void *)ghost_realtek, + }, { .matches = { DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c index 3d8f5a81eff19..b288218f64b42 100644 --- a/drivers/soundwire/qcom.c +++ b/drivers/soundwire/qcom.c @@ -1271,7 +1271,7 @@ static int qcom_swrm_stream_alloc_ports(struct qcom_swrm_ctrl *ctrl, else pn = find_first_zero_bit(port_mask, maxport); - if (pn > maxport) { + if (pn >= maxport) { dev_err(ctrl->dev, "All ports busy\n"); return -EBUSY; } diff --git a/drivers/spi/spi-amlogic-spisg.c b/drivers/spi/spi-amlogic-spisg.c index afc8af04638d3..9049a87e9d0f2 100644 --- a/drivers/spi/spi-amlogic-spisg.c +++ b/drivers/spi/spi-amlogic-spisg.c @@ -636,7 +636,7 @@ static int aml_spisg_target_abort(struct spi_controller *ctlr) static int aml_spisg_clk_init(struct spisg_device *spisg, void __iomem *base) { struct device *dev = &spisg->pdev->dev; - struct clk_init_data init; + struct clk_init_data init = {}; struct clk_divider *div; struct clk_div_table *tbl; char name[32]; diff --git a/drivers/spi/spi-bcm63xx-hsspi.c b/drivers/spi/spi-bcm63xx-hsspi.c index 58012e1b5ae74..fd7b9d5d96865 100644 --- a/drivers/spi/spi-bcm63xx-hsspi.c +++ b/drivers/spi/spi-bcm63xx-hsspi.c @@ -915,7 +915,13 @@ static int bcm63xx_hsspi_resume(struct device *dev) } } - spi_controller_resume(host); + ret = spi_controller_resume(host); + if (ret) { + if (bs->pll_clk) + clk_disable_unprepare(bs->pll_clk); + clk_disable_unprepare(bs->clk); + return ret; + } return 0; } diff --git a/drivers/spi/spi-bcm63xx.c b/drivers/spi/spi-bcm63xx.c index 43d7b54e3ae85..74c06934cd2dd 100644 --- a/drivers/spi/spi-bcm63xx.c +++ b/drivers/spi/spi-bcm63xx.c @@ -650,7 +650,11 @@ static int bcm63xx_spi_resume(struct device *dev) if (ret) return ret; - spi_controller_resume(host); + ret = spi_controller_resume(host); + if (ret) { + clk_disable_unprepare(bs->clk); + return ret; + } return 0; } diff --git a/drivers/spi/spi-bcmbca-hsspi.c b/drivers/spi/spi-bcmbca-hsspi.c index 09c1472ae4fa1..d4491058b4561 100644 --- a/drivers/spi/spi-bcmbca-hsspi.c +++ b/drivers/spi/spi-bcmbca-hsspi.c @@ -594,7 +594,13 @@ static int bcmbca_hsspi_resume(struct device *dev) } } - spi_controller_resume(host); + ret = spi_controller_resume(host); + if (ret) { + if (bs->pll_clk) + clk_disable_unprepare(bs->pll_clk); + clk_disable_unprepare(bs->clk); + return ret; + } return 0; } diff --git a/drivers/spi/spi-davinci.c b/drivers/spi/spi-davinci.c index da7f2ae3a570a..087735ed9c80c 100644 --- a/drivers/spi/spi-davinci.c +++ b/drivers/spi/spi-davinci.c @@ -928,7 +928,7 @@ static int davinci_spi_probe(struct platform_device *pdev) int ret = 0; u32 spipc0; - host = spi_alloc_host(&pdev->dev, sizeof(struct davinci_spi)); + host = devm_spi_alloc_host(&pdev->dev, sizeof(struct davinci_spi)); if (host == NULL) { ret = -ENOMEM; goto err; @@ -1057,7 +1057,6 @@ static int davinci_spi_probe(struct platform_device *pdev) dma_release_channel(dspi->dma_tx); } free_host: - spi_controller_put(host); err: return ret; } @@ -1081,6 +1080,8 @@ static void davinci_spi_remove(struct platform_device *pdev) spi_bitbang_stop(&dspi->bitbang); + devm_free_irq(&pdev->dev, dspi->irq, dspi); + /* This bit needs to be cleared to disable dpsi->clk */ clear_io_bits(dspi->base + SPIGCR1, SPIGCR1_POWERDOWN_MASK); @@ -1088,8 +1089,6 @@ static void davinci_spi_remove(struct platform_device *pdev) dma_release_channel(dspi->dma_rx); dma_release_channel(dspi->dma_tx); } - - spi_controller_put(host); } static struct platform_driver davinci_spi_driver = { diff --git a/drivers/spi/spi-fsl-qspi.c b/drivers/spi/spi-fsl-qspi.c index 57358851029ba..d2c2090442f84 100644 --- a/drivers/spi/spi-fsl-qspi.c +++ b/drivers/spi/spi-fsl-qspi.c @@ -289,6 +289,7 @@ struct fsl_qspi { struct pm_qos_request pm_qos_req; struct device *dev; int selected; + u32 selected_freq; u32 memmap_phy; }; @@ -551,7 +552,8 @@ static void fsl_qspi_select_mem(struct fsl_qspi *q, struct spi_device *spi, unsigned long rate = op->max_freq; int ret; - if (q->selected == spi_get_chipselect(spi, 0)) + if (q->selected == spi_get_chipselect(spi, 0) && + q->selected_freq == op->max_freq) return; if (needs_4x_clock(q)) @@ -571,6 +573,7 @@ static void fsl_qspi_select_mem(struct fsl_qspi *q, struct spi_device *spi, } q->selected = spi_get_chipselect(spi, 0); + q->selected_freq = op->max_freq; fsl_qspi_invalidate(q); } diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c index 26e723cfea61e..1fd9cf6e30f7a 100644 --- a/drivers/spi/spi-geni-qcom.c +++ b/drivers/spi/spi-geni-qcom.c @@ -613,25 +613,30 @@ static int spi_geni_init(struct spi_geni_master *mas) u32 spi_tx_cfg, fifo_disable; int ret = -ENXIO; - pm_runtime_get_sync(mas->dev); + PM_RUNTIME_ACQUIRE_IF_ENABLED(mas->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret < 0) { + dev_err(mas->dev, "Failed to resume and get %d\n", ret); + return ret; + } proto = geni_se_read_proto(se); if (spi->target) { if (proto != GENI_SE_SPI_SLAVE) { dev_err(mas->dev, "Invalid proto %d\n", proto); - goto out_pm; + return -EINVAL; } spi_slv_setup(mas); } else if (proto == GENI_SE_INVALID_PROTO) { ret = geni_load_se_firmware(se, GENI_SE_SPI); if (ret) { dev_err(mas->dev, "spi master firmware load failed ret: %d\n", ret); - goto out_pm; + return ret; } } else if (proto != GENI_SE_SPI) { dev_err(mas->dev, "Invalid proto %d\n", proto); - goto out_pm; + return -EINVAL; } mas->tx_fifo_depth = geni_se_get_tx_fifo_depth(se); @@ -664,7 +669,7 @@ static int spi_geni_init(struct spi_geni_master *mas) dev_dbg(mas->dev, "Using GPI DMA mode for SPI\n"); break; } else if (ret == -EPROBE_DEFER) { - goto out_pm; + return ret; } /* * in case of failure to get gpi dma channel, we can still do the @@ -693,8 +698,6 @@ static int spi_geni_init(struct spi_geni_master *mas) writel(spi_tx_cfg, se->base + SE_SPI_TRANS_CFG); } -out_pm: - pm_runtime_put(mas->dev); return ret; } diff --git a/drivers/spi/spi-img-spfi.c b/drivers/spi/spi-img-spfi.c index aec724e3f8246..a9310a1b0354f 100644 --- a/drivers/spi/spi-img-spfi.c +++ b/drivers/spi/spi-img-spfi.c @@ -611,7 +611,7 @@ static int img_spfi_probe(struct platform_device *pdev) ret = PTR_ERR(spfi->tx_ch); spfi->tx_ch = NULL; if (ret == -EPROBE_DEFER) - goto disable_pm; + goto free_dma; } spfi->rx_ch = dma_request_chan(spfi->dev, "rx"); @@ -619,7 +619,7 @@ static int img_spfi_probe(struct platform_device *pdev) ret = PTR_ERR(spfi->rx_ch); spfi->rx_ch = NULL; if (ret == -EPROBE_DEFER) - goto disable_pm; + goto free_dma; } if (!spfi->tx_ch || !spfi->rx_ch) { @@ -647,6 +647,7 @@ static int img_spfi_probe(struct platform_device *pdev) disable_pm: pm_runtime_disable(spfi->dev); +free_dma: if (spfi->rx_ch) dma_release_channel(spfi->rx_ch); if (spfi->tx_ch) diff --git a/drivers/spi/spi-oc-tiny.c b/drivers/spi/spi-oc-tiny.c index 29333b1f82d7a..1cd2a934c0329 100644 --- a/drivers/spi/spi-oc-tiny.c +++ b/drivers/spi/spi-oc-tiny.c @@ -210,11 +210,11 @@ static int tiny_spi_probe(struct platform_device *pdev) struct tiny_spi_platform_data *platp = dev_get_platdata(&pdev->dev); struct tiny_spi *hw; struct spi_controller *host; - int err = -ENODEV; + int err; - host = spi_alloc_host(&pdev->dev, sizeof(struct tiny_spi)); + host = devm_spi_alloc_host(&pdev->dev, sizeof(struct tiny_spi)); if (!host) - return err; + return -ENOMEM; /* setup the host state. */ host->bus_num = pdev->id; @@ -232,10 +232,8 @@ static int tiny_spi_probe(struct platform_device *pdev) /* find and map our resources */ hw->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(hw->base)) { - err = PTR_ERR(hw->base); - goto exit; - } + if (IS_ERR(hw->base)) + return PTR_ERR(hw->base); /* irq is optional */ hw->irq = platform_get_irq(pdev, 0); if (hw->irq >= 0) { @@ -243,7 +241,7 @@ static int tiny_spi_probe(struct platform_device *pdev) err = devm_request_irq(&pdev->dev, hw->irq, tiny_spi_irq, 0, pdev->name, hw); if (err) - goto exit; + return err; } /* find platform data */ if (platp) { @@ -252,29 +250,23 @@ static int tiny_spi_probe(struct platform_device *pdev) } else { err = tiny_spi_of_probe(pdev); if (err) - goto exit; + return err; } /* register our spi controller */ err = spi_bitbang_start(&hw->bitbang); if (err) - goto exit; + return err; dev_info(&pdev->dev, "base %p, irq %d\n", hw->base, hw->irq); return 0; - -exit: - spi_controller_put(host); - return err; } static void tiny_spi_remove(struct platform_device *pdev) { struct tiny_spi *hw = platform_get_drvdata(pdev); - struct spi_controller *host = hw->bitbang.ctlr; spi_bitbang_stop(&hw->bitbang); - spi_controller_put(host); } #ifdef CONFIG_OF diff --git a/drivers/spi/spi-qpic-snand.c b/drivers/spi/spi-qpic-snand.c index b6c58d9cfe143..8388c637eee00 100644 --- a/drivers/spi/spi-qpic-snand.c +++ b/drivers/spi/spi-qpic-snand.c @@ -740,8 +740,6 @@ static int qcom_spi_read_cw_raw(struct qcom_nand_controller *snandc, u8 *data_bu qcom_write_reg_dma(snandc, &snandc->regs->addr0, NAND_ADDR0, 2, 0); qcom_write_reg_dma(snandc, &snandc->regs->cfg0, NAND_DEV0_CFG0, 3, 0); - qcom_write_reg_dma(snandc, &snandc->regs->ecc_buf_cfg, NAND_EBI2_ECC_BUF_CFG, 1, 0); - qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_clr, NAND_ERASED_CW_DETECT_CFG, 1, 0); qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_set, @@ -1008,8 +1006,6 @@ static void qcom_spi_config_page_write(struct qcom_nand_controller *snandc) { qcom_write_reg_dma(snandc, &snandc->regs->addr0, NAND_ADDR0, 2, 0); qcom_write_reg_dma(snandc, &snandc->regs->cfg0, NAND_DEV0_CFG0, 3, 0); - qcom_write_reg_dma(snandc, &snandc->regs->ecc_buf_cfg, NAND_EBI2_ECC_BUF_CFG, - 1, NAND_BAM_NEXT_SGL); } static void qcom_spi_config_cw_write(struct qcom_nand_controller *snandc) diff --git a/drivers/spi/spi-sprd-adi.c b/drivers/spi/spi-sprd-adi.c index e7d83c16b46c0..a6baaf02b3c86 100644 --- a/drivers/spi/spi-sprd-adi.c +++ b/drivers/spi/spi-sprd-adi.c @@ -553,6 +553,12 @@ static int sprd_adi_probe(struct platform_device *pdev) return -ENXIO; } else { switch (ret) { + case 0: + /* + * Only reachable with CONFIG_HWSPINLOCK=n, where the + * of_hwspin_lock_get_id() stub returns 0. + */ + fallthrough; case -ENOENT: dev_info(&pdev->dev, "no hardware spinlock supplied\n"); break; diff --git a/drivers/spi/spi-virtio.c b/drivers/spi/spi-virtio.c index 2256dfec5407d..3e181bd8bc94b 100644 --- a/drivers/spi/spi-virtio.c +++ b/drivers/spi/spi-virtio.c @@ -168,7 +168,7 @@ static int virtio_spi_transfer_one(struct spi_controller *ctrl, /* Fill struct spi_transfer_head */ th->chip_select_id = spi_get_chipselect(spi, 0); - th->bits_per_word = spi->bits_per_word; + th->bits_per_word = xfer->bits_per_word; th->cs_change = xfer->cs_change; th->tx_nbits = xfer->tx_nbits; th->rx_nbits = xfer->rx_nbits; diff --git a/drivers/spi/spi-zynqmp-gqspi.c b/drivers/spi/spi-zynqmp-gqspi.c index f9a1427dabade..5e23b10d82a7b 100644 --- a/drivers/spi/spi-zynqmp-gqspi.c +++ b/drivers/spi/spi-zynqmp-gqspi.c @@ -1374,11 +1374,45 @@ static void zynqmp_qspi_remove(struct platform_device *pdev) clk_disable_unprepare(xqspi->pclk); } +static void zynqmp_qspi_shutdown(struct platform_device *pdev) +{ + struct zynqmp_qspi *xqspi = platform_get_drvdata(pdev); + int ret; + + /* + * Stop the queue and reject any later transfer first, so the write + * below cannot cut into a message that is still being executed. + * Unlike ->suspend this cannot abort on error: a controller left + * mastering the bus is worse than a truncated transfer. + */ + ret = spi_controller_suspend(xqspi->ctlr); + if (ret) + dev_warn(&pdev->dev, "could not stop the queue: %d\n", ret); + + /* + * Only a runtime suspended controller can be left alone: its clocks + * are gated, so it cannot be mastering the bus, and its registers + * must not be accessed either. Any other answer means it may be + * running and has to be stopped. In particular, on a kernel built + * without runtime PM this returns -EINVAL, and there the clocks + * enabled in probe() are never gated at all. + */ + ret = pm_runtime_get_if_in_use(&pdev->dev); + if (!ret) + return; + + zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, 0x0); + + if (ret > 0) + pm_runtime_put_noidle(&pdev->dev); +} + MODULE_DEVICE_TABLE(of, zynqmp_qspi_of_match); static struct platform_driver zynqmp_qspi_driver = { .probe = zynqmp_qspi_probe, .remove = zynqmp_qspi_remove, + .shutdown = zynqmp_qspi_shutdown, .driver = { .name = "zynqmp-qspi", .of_match_table = zynqmp_qspi_of_match, diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index d9e6b4b87c899..12b3a3b29a89f 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1231,6 +1231,8 @@ void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev, spi_unmap_buf_attrs(ctlr, dev, sgt, dir, 0); } +static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg); + static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) { struct device *tx_dev, *rx_dev; @@ -1254,7 +1256,13 @@ static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) else rx_dev = ctlr->dev.parent; - ret = -ENOMSG; + /* + * Store the devices before mapping so partial failures can be unwound + * with the device that created each mapping. + */ + ctlr->cur_tx_dma_dev = tx_dev; + ctlr->cur_rx_dma_dev = rx_dev; + list_for_each_entry(xfer, &msg->transfers, transfer_list) { /* The sync is done before each transfer. */ unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC; @@ -1267,8 +1275,8 @@ static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) (void *)xfer->tx_buf, xfer->len, DMA_TO_DEVICE, attrs); - if (ret != 0) - return ret; + if (ret) + goto unwind; xfer->tx_sg_mapped = true; } @@ -1277,25 +1285,19 @@ static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) ret = spi_map_buf_attrs(ctlr, rx_dev, &xfer->rx_sg, xfer->rx_buf, xfer->len, DMA_FROM_DEVICE, attrs); - if (ret != 0) { - spi_unmap_buf_attrs(ctlr, tx_dev, - &xfer->tx_sg, DMA_TO_DEVICE, - attrs); - - return ret; - } + if (ret) + goto unwind; xfer->rx_sg_mapped = true; } } - /* No transfer has been mapped, bail out with success */ - if (ret) - return 0; - - ctlr->cur_rx_dma_dev = rx_dev; - ctlr->cur_tx_dma_dev = tx_dev; return 0; + +unwind: + __spi_unmap_msg(ctlr, msg); + + return ret; } static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg) diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c index ca0c38221c16b..7925d974de801 100644 --- a/drivers/staging/fbtft/fbtft-core.c +++ b/drivers/staging/fbtft/fbtft-core.c @@ -298,14 +298,15 @@ static void fbtft_mkdirty(struct fb_info *info, int y, int height) { struct fbtft_par *par = info->par; struct fb_deferred_io *fbdefio = info->fbdefio; + unsigned long flags; /* Mark display lines/area as dirty */ - spin_lock(&par->dirty_lock); + spin_lock_irqsave(&par->dirty_lock, flags); if (y < par->dirty_lines_start) par->dirty_lines_start = y; if (y + height - 1 > par->dirty_lines_end) par->dirty_lines_end = y + height - 1; - spin_unlock(&par->dirty_lock); + spin_unlock_irqrestore(&par->dirty_lock, flags); /* Schedule deferred_io to update display (no-op if already on queue)*/ schedule_delayed_work(&info->deferred_work, fbdefio->delay); @@ -318,13 +319,13 @@ static void fbtft_deferred_io(struct fb_info *info, struct list_head *pagereflis struct fb_deferred_io_pageref *pageref; unsigned int y_low = 0, y_high = 0; - spin_lock(&par->dirty_lock); + spin_lock_irq(&par->dirty_lock); dirty_lines_start = par->dirty_lines_start; dirty_lines_end = par->dirty_lines_end; /* set display line markers as clean */ par->dirty_lines_start = par->info->var.yres - 1; par->dirty_lines_end = 0; - spin_unlock(&par->dirty_lock); + spin_unlock_irq(&par->dirty_lock); /* Mark display lines as dirty */ list_for_each_entry(pageref, pagereflist, list) { diff --git a/drivers/staging/fbtft/fbtft-sysfs.c b/drivers/staging/fbtft/fbtft-sysfs.c index d05599d80011a..343545e83a377 100644 --- a/drivers/staging/fbtft/fbtft-sysfs.c +++ b/drivers/staging/fbtft/fbtft-sysfs.c @@ -98,7 +98,7 @@ sprintf_gamma(struct fbtft_par *par, u32 *curves, char *buf) mutex_lock(&par->gamma.lock); for (i = 0; i < par->gamma.num_curves; i++) { for (j = 0; j < par->gamma.num_values; j++) - len += scnprintf(&buf[len], PAGE_SIZE, + len += sysfs_emit_at(buf, len, "%04x ", curves[i * par->gamma.num_values + j]); buf[len - 1] = '\n'; } diff --git a/drivers/staging/greybus/audio_gb.c b/drivers/staging/greybus/audio_gb.c index 9d8994fdb41a2..144591f1a5128 100644 --- a/drivers/staging/greybus/audio_gb.c +++ b/drivers/staging/greybus/audio_gb.c @@ -37,6 +37,19 @@ int gb_audio_gb_get_topology(struct gb_connection *connection, return ret; } + /* + * The size_* fields are supplied by the module and are used by + * gbaudio_tplg_parse_data() to compute offsets into the blob; make + * sure the sections fit within the fetched topology, so walking it + * cannot read out of bounds. + */ + if ((u64)le32_to_cpu(topo->size_dais) + le32_to_cpu(topo->size_controls) + + le32_to_cpu(topo->size_widgets) + le32_to_cpu(topo->size_routes) > + size - sizeof(*topo)) { + kfree(topo); + return -EINVAL; + } + *topology = topo; return 0; diff --git a/drivers/staging/greybus/audio_manager_sysfs.c b/drivers/staging/greybus/audio_manager_sysfs.c index fcd518f9540cd..ff323ca8154f3 100644 --- a/drivers/staging/greybus/audio_manager_sysfs.c +++ b/drivers/staging/greybus/audio_manager_sysfs.c @@ -23,7 +23,7 @@ static ssize_t manager_sysfs_add_store(struct kobject *kobj, desc.name, &desc.vid, &desc.pid, &desc.intf_id, &desc.ip_devices, &desc.op_devices); - if (num != 7) + if (num != 6) return -EINVAL; num = gb_audio_manager_add(&desc); diff --git a/drivers/staging/greybus/hid.c b/drivers/staging/greybus/hid.c index f1f9f6fbc00e5..1d7186eecd23e 100644 --- a/drivers/staging/greybus/hid.c +++ b/drivers/staging/greybus/hid.c @@ -256,7 +256,7 @@ static int __gb_hid_output_raw_report(struct hid_device *hid, __u8 *buf, if (report_id && ret >= 0) ret++; /* add report_id to the number of transferred bytes */ - return 0; + return ret; } static int gb_hid_raw_request(struct hid_device *hid, unsigned char reportnum, diff --git a/drivers/staging/media/ipu7/ipu7-isys.c b/drivers/staging/media/ipu7/ipu7-isys.c index cb2f49f3e0fa3..601e5a79ef8ec 100644 --- a/drivers/staging/media/ipu7/ipu7-isys.c +++ b/drivers/staging/media/ipu7/ipu7-isys.c @@ -233,6 +233,7 @@ static int isys_notifier_init(struct ipu7_isys *isys) err_parse: fwnode_handle_put(ep); + v4l2_async_nf_cleanup(&isys->notifier); return ret; } @@ -773,6 +774,7 @@ static int isys_probe(struct auxiliary_device *auxdev, return 0; out_cleanup: + isys_notifier_cleanup(isys); isys_unregister_devices(isys); out_cleanup_fw: ipu7_fw_isys_release(isys); diff --git a/drivers/staging/media/ipu7/ipu7.c b/drivers/staging/media/ipu7/ipu7.c index 310e3f24e571f..48a35bda42370 100644 --- a/drivers/staging/media/ipu7/ipu7.c +++ b/drivers/staging/media/ipu7/ipu7.c @@ -2343,7 +2343,7 @@ static int ipu7_init_fw_code_region_by_sys(struct ipu7_bus_device *sys, return ret; } - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "Failed to get runtime PM\n"); return ret; @@ -2702,7 +2702,7 @@ static int ipu7_resume(struct device *dev) if (ret) dev_err(dev, "IPC reset protocol failed!\n"); - ret = pm_runtime_get_sync(&isp->psys->auxdev.dev); + ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev); if (ret < 0) { dev_err(dev, "Failed to get runtime PM\n"); return 0; diff --git a/drivers/staging/media/meson/vdec/vdec.c b/drivers/staging/media/meson/vdec/vdec.c index a039d925c0fe5..c18bf352b1996 100644 --- a/drivers/staging/media/meson/vdec/vdec.c +++ b/drivers/staging/media/meson/vdec/vdec.c @@ -504,8 +504,8 @@ vdec_try_fmt_common(struct amvdec_session *sess, u32 size, case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: fmt_out = find_format(fmts, size, pixmp->pixelformat); if (!fmt_out) { - pixmp->pixelformat = V4L2_PIX_FMT_MPEG2; - fmt_out = find_format(fmts, size, pixmp->pixelformat); + pixmp->pixelformat = fmts[0].pixfmt; + fmt_out = &fmts[0]; } break; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c index bbd186b8035b2..96acd52e380c8 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus.c @@ -285,8 +285,10 @@ static int cedrus_init_ctrls(struct cedrus_dev *dev, struct cedrus_ctx *ctx) ctrl_size = sizeof(ctrl) * CEDRUS_CONTROLS_COUNT + 1; ctx->ctrls = kzalloc(ctrl_size, GFP_KERNEL); - if (!ctx->ctrls) + if (!ctx->ctrls) { + v4l2_ctrl_handler_free(hdl); return -ENOMEM; + } j = 0; for (i = 0; i < CEDRUS_CONTROLS_COUNT; i++) { diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c index 456134a9e8cf2..f461e117305e0 100644 --- a/drivers/staging/media/tegra-video/vi.c +++ b/drivers/staging/media/tegra-video/vi.c @@ -1257,6 +1257,7 @@ static int tegra_vi_channels_alloc(struct tegra_vi *vi) struct device_node *parent; struct v4l2_fwnode_endpoint v4l2_ep = { .bus_type = 0 }; unsigned int lanes; + int err; int ret = 0; ports = of_get_child_by_name(node, "ports"); @@ -1267,8 +1268,8 @@ static int tegra_vi_channels_alloc(struct tegra_vi *vi) if (!of_node_name_eq(port, "port")) continue; - ret = of_property_read_u32(port, "reg", &port_num); - if (ret < 0) + err = of_property_read_u32(port, "reg", &port_num); + if (err < 0) continue; if (port_num > vi->soc->vi_max_channels) { @@ -1289,10 +1290,10 @@ static int tegra_vi_channels_alloc(struct tegra_vi *vi) ep = of_graph_get_endpoint_by_regs(parent, 0, 0); of_node_put(parent); - ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep), + err = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep), &v4l2_ep); of_node_put(ep); - if (ret) + if (err) continue; lanes = v4l2_ep.bus.mipi_csi2.num_data_lanes; diff --git a/drivers/staging/media/tegra-video/vip.c b/drivers/staging/media/tegra-video/vip.c index 9ff1f1750a154..5fba11e31e1d0 100644 --- a/drivers/staging/media/tegra-video/vip.c +++ b/drivers/staging/media/tegra-video/vip.c @@ -126,7 +126,7 @@ static int tegra_vip_channel_of_parse(struct tegra_vip *vip) if (!ep) { err = -EINVAL; dev_err_probe(dev, err, "%pOF: error getting endpoint node\n", np); - goto err_node_put; + return err; } fwh = of_fwnode_handle(ep); @@ -134,14 +134,14 @@ static int tegra_vip_channel_of_parse(struct tegra_vip *vip) of_node_put(ep); if (err) { dev_err_probe(dev, err, "%pOF: failed to parse v4l2 endpoint\n", np); - goto err_node_put; + return err; } num_pads = of_graph_get_endpoint_count(np); if (num_pads != TEGRA_VIP_PADS_NUM) { err = -EINVAL; dev_err_probe(dev, err, "%pOF: need 2 pads, got %d\n", np, num_pads); - goto err_node_put; + return err; } vip->chan.of_node = of_node_get(np); @@ -149,10 +149,6 @@ static int tegra_vip_channel_of_parse(struct tegra_vip *vip) vip->chan.pads[TEGRA_VIP_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE; return 0; - -err_node_put: - of_node_put(np); - return err; } static int tegra_vip_channel_init(struct tegra_vip *vip) diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c index cd36b5ba6f6c2..3e9d58d321560 100644 --- a/drivers/staging/octeon/ethernet-rx.c +++ b/drivers/staging/octeon/ethernet-rx.c @@ -535,6 +535,8 @@ void cvm_oct_rx_shutdown(struct platform_device *pdev) else cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), 0); + napi_disable(&plat->rx_group[i].napi); + /* Free the interrupt handler */ free_irq(plat->rx_group[i].irq, &plat->rx_group[i].napi); diff --git a/drivers/staging/octeon/ethernet-tx.c b/drivers/staging/octeon/ethernet-tx.c index 14d10659bce71..785c6492f170b 100644 --- a/drivers/staging/octeon/ethernet-tx.c +++ b/drivers/staging/octeon/ethernet-tx.c @@ -668,4 +668,6 @@ void cvm_oct_tx_shutdown(void) { /* Free the interrupt handler */ free_irq(OCTEON_IRQ_TIMER1, cvm_oct_device); + + tasklet_kill(&cvm_oct_tx_cleanup_tasklet); } diff --git a/drivers/staging/rtl8723bs/core/rtw_cmd.c b/drivers/staging/rtl8723bs/core/rtw_cmd.c index b932670f5d63a..a34ee407285be 100644 --- a/drivers/staging/rtl8723bs/core/rtw_cmd.c +++ b/drivers/staging/rtl8723bs/core/rtw_cmd.c @@ -899,7 +899,7 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_ if (enqueue) { ph2c = kzalloc_obj(*ph2c); if (!ph2c) { - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); res = _FAIL; goto exit; } @@ -907,7 +907,7 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_ psetstakey_rsp = kzalloc_obj(*psetstakey_rsp); if (!psetstakey_rsp) { kfree(ph2c); - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); res = _FAIL; goto exit; } @@ -918,7 +918,7 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_ res = rtw_enqueue_cmd(pcmdpriv, ph2c); } else { set_stakey_hdl(padapter, (u8 *)psetstakey_para); - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); } exit: return res; @@ -958,7 +958,7 @@ u8 rtw_clearstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 enqueu psetstakey_rsp = kzalloc_obj(*psetstakey_rsp); if (!psetstakey_rsp) { kfree(ph2c); - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); res = _FAIL; goto exit; } diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c index e60ed33598662..b6e7de2352047 100644 --- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c +++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c @@ -737,6 +737,10 @@ u8 *rtw_get_wps_attr(u8 *wps_ie, uint wps_ielen, u16 target_attr_id, u8 *buf_att u16 attr_data_len = get_unaligned_be16(attr_ptr + 2); u16 attr_len = attr_data_len + 4; + /* Reject attributes whose claimed length runs past the IE */ + if (attr_ptr + attr_len > wps_ie + wps_ielen) + break; + if (attr_id == target_attr_id) { target_attr_ptr = attr_ptr; @@ -1145,6 +1149,9 @@ int rtw_action_frame_parse(const u8 *frame, u32 frame_len, u8 *category, u8 *act u8 c; u8 a = ACT_PUBLIC_MAX; + if (frame_len < sizeof(struct ieee80211_hdr_3addr) + 2) + return false; + fc = le16_to_cpu(((struct ieee80211_hdr_3addr *)frame)->frame_control); if ((fc & (IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) != diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c index 1196ec011455b..7bdc5fe6dc8af 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c @@ -1980,6 +1980,9 @@ int rtw_restruct_wmm_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_ break; } + if (i + 1 >= in_len) + break; + i += (in_ie[i + 1] + 2); /* to the next IE element */ } diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c index a9382dc1294b3..4166b1a8eea76 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c @@ -2181,8 +2181,11 @@ void issue_beacon(struct adapter *padapter, int timeout_ms) spin_unlock_bh(&pmlmepriv->bcn_update_lock); - if ((pattrib->pktlen + TXDESC_SIZE) > 512) + if ((pattrib->pktlen + TXDESC_SIZE) > 512) { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } pattrib->last_txcmdsz = pattrib->pktlen; @@ -2243,8 +2246,11 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p pattrib->pktlen = pattrib->hdrlen; pframe += pattrib->hdrlen; - if (cur_network->ie_length > MAX_IE_SZ) + if (cur_network->ie_length > MAX_IE_SZ) { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE) { pwps_ie = rtw_get_wps_ie(cur_network->ies+_FIXED_IE_LENGTH_, cur_network->ie_length-_FIXED_IE_LENGTH_, NULL, &wps_ielen); @@ -2291,8 +2297,11 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p u8 *ies = pmgntframe->buf_addr+TXDESC_OFFSET+sizeof(struct ieee80211_hdr_3addr); buf = kzalloc(MAX_IE_SZ, GFP_ATOMIC); - if (!buf) + if (!buf) { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } ssid_ie = rtw_get_ie(ies+_FIXED_IE_LENGTH_, WLAN_EID_SSID, &ssid_ielen, (pframe-ies)-_FIXED_IE_LENGTH_); @@ -2670,10 +2679,13 @@ void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_i SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq); pmlmeext->mgnt_seq++; - if ((pkt_type == WIFI_ASSOCRSP) || (pkt_type == WIFI_REASSOCRSP)) + if ((pkt_type == WIFI_ASSOCRSP) || (pkt_type == WIFI_REASSOCRSP)) { SetFrameSubType(pwlanhdr, pkt_type); - else + } else { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr); pattrib->pktlen += pattrib->hdrlen; diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c index c43a0391a5ca7..ee4a9c66aceed 100644 --- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c +++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c @@ -285,8 +285,8 @@ static struct adapter *rtw_sdio_if1_init(struct dvobj_priv *dvobj, const struct status = _SUCCESS; free_hal_data: - if (status != _SUCCESS && padapter->HalData) - kfree(padapter->HalData); + if (status != _SUCCESS) + rtw_hal_data_deinit(padapter); if (status != _SUCCESS) { rtw_wdev_unregister(padapter->rtw_wdev); diff --git a/drivers/staging/sm750fb/Kconfig b/drivers/staging/sm750fb/Kconfig index 08bcccdd0f1c4..25fe422f55f2c 100644 --- a/drivers/staging/sm750fb/Kconfig +++ b/drivers/staging/sm750fb/Kconfig @@ -6,6 +6,7 @@ config FB_SM750 select FB_CFB_FILLRECT select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT + select FB_IOMEM_FOPS help Frame buffer driver for the Silicon Motion SM750 chip with 2D acceleration and dual head support. diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c index 89c811e0806c4..d85ccf5f7a7e5 100644 --- a/drivers/staging/sm750fb/sm750.c +++ b/drivers/staging/sm750fb/sm750.c @@ -261,7 +261,7 @@ static void lynxfb_ops_imageblit(struct fb_info *info, spin_lock(&sm750_dev->slock); sm750_dev->accel.de_imageblit(&sm750_dev->accel, - image->data, image->width >> 3, 0, + image->data, 0, base, pitch, bpp, image->dx, image->dy, image->width, image->height, @@ -591,7 +591,7 @@ static int sm750fb_set_drv(struct lynxfb_par *par) crtc = &par->crtc; crtc->vidmem_size = sm750_dev->vidmem_size; - if (sm750_dev->fb_count > 1) + if (g_dualview) crtc->vidmem_size >>= 1; /* setup crtc and output member */ @@ -896,7 +896,7 @@ static void sm750fb_setup(struct sm750_dev *sm750_dev, char *src) NO_PARAM: if (sm750_dev->revid != SM750LE_REVISION_ID) { - if (sm750_dev->fb_count > 1) { + if (g_dualview) { if (swap) sm750_dev->dataflow = sm750_dual_swap; else diff --git a/drivers/staging/sm750fb/sm750.h b/drivers/staging/sm750fb/sm750.h index d2c522e67f26a..83229fe155ecc 100644 --- a/drivers/staging/sm750fb/sm750.h +++ b/drivers/staging/sm750fb/sm750.h @@ -73,7 +73,7 @@ struct lynx_accel { u32 rop2); int (*de_imageblit)(struct lynx_accel *accel, const char *p_srcbuf, - u32 src_delta, u32 start_bit, u32 d_base, u32 d_pitch, + u32 start_bit, u32 d_base, u32 d_pitch, u32 byte_per_pixel, u32 dx, u32 dy, u32 width, u32 height, u32 f_color, u32 b_color, u32 rop2); diff --git a/drivers/staging/sm750fb/sm750_accel.c b/drivers/staging/sm750fb/sm750_accel.c index 0316ea69d0097..bac9a209899cf 100644 --- a/drivers/staging/sm750fb/sm750_accel.c +++ b/drivers/staging/sm750fb/sm750_accel.c @@ -288,8 +288,6 @@ static unsigned int de_get_transparency(struct lynx_accel *accel) * sm750_hw_imageblit * @accel: Acceleration device data * @src_buf: pointer to start of source buffer in system memory - * @src_delta: Pitch value (in bytes) of the source buffer, +ive means top down - * and -ive mean button up * @start_bit: Mono data can start at any bit in a byte, this value should be * 0 to 7 * @dest_base: Address of destination: offset in frame buffer @@ -304,7 +302,7 @@ static unsigned int de_get_transparency(struct lynx_accel *accel) * @rop2: ROP value */ int sm750_hw_imageblit(struct lynx_accel *accel, const char *src_buf, - u32 src_delta, u32 start_bit, u32 dest_base, u32 dest_pitch, + u32 start_bit, u32 dest_base, u32 dest_pitch, u32 byte_per_pixel, u32 dx, u32 dy, u32 width, u32 height, u32 fg_color, u32 bg_color, u32 rop2) { @@ -395,7 +393,7 @@ int sm750_hw_imageblit(struct lynx_accel *accel, const char *src_buf, write_dp_port(accel, *(unsigned int *)remain); } - src_buf += src_delta; + src_buf += bytes_per_scan; } return 0; diff --git a/drivers/staging/sm750fb/sm750_accel.h b/drivers/staging/sm750fb/sm750_accel.h index d15a40cacb84d..4f609d6af0fa4 100644 --- a/drivers/staging/sm750fb/sm750_accel.h +++ b/drivers/staging/sm750fb/sm750_accel.h @@ -220,8 +220,6 @@ int sm750_hw_copyarea(struct lynx_accel *accel, /** * sm750_hw_imageblit * @pSrcbuf: pointer to start of source buffer in system memory - * @srcDelta: Pitch value (in bytes) of the source buffer, +ive means top down - *>----- and -ive mean button up * @startBit: Mono data can start at any bit in a byte, this value should be *>----- 0 to 7 * @dBase: Address of destination: offset in frame buffer @@ -236,7 +234,7 @@ int sm750_hw_copyarea(struct lynx_accel *accel, * @rop2: ROP value */ int sm750_hw_imageblit(struct lynx_accel *accel, const char *pSrcbuf, - u32 srcDelta, u32 startBit, u32 dBase, u32 dPitch, + u32 startBit, u32 dBase, u32 dPitch, u32 bytePerPixel, u32 dx, u32 dy, u32 width, u32 height, u32 fColor, u32 bColor, u32 rop2); diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c index 62ada3a52210e..124ff269b8e75 100644 --- a/drivers/target/iscsi/iscsi_target.c +++ b/drivers/target/iscsi/iscsi_target.c @@ -1533,8 +1533,10 @@ __iscsit_check_dataout_hdr(struct iscsit_conn *conn, void *buf, */ if (se_cmd->transport_state & CMD_T_ABORTED) { if (hdr->flags & ISCSI_FLAG_CMD_FINAL && - --cmd->outstanding_r2ts < 1) + --cmd->outstanding_r2ts < 1) { iscsit_stop_dataout_timer(cmd); + target_complete_cmd(se_cmd, SAM_STAT_TASK_ABORTED); + } return iscsit_dump_data_payload(conn, payload_length, 1); } diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c index aafc94bcb6350..c282b6a702968 100644 --- a/drivers/target/iscsi/iscsi_target_login.c +++ b/drivers/target/iscsi/iscsi_target_login.c @@ -47,7 +47,7 @@ static struct iscsi_login *iscsi_login_init_conn(struct iscsit_conn *conn) login->conn = conn; login->first_request = 1; - login->req_buf = kzalloc(MAX_KEY_VALUE_PAIRS, GFP_KERNEL); + login->req_buf = kzalloc(MAX_KEY_VALUE_PAIRS + 1, GFP_KERNEL); if (!login->req_buf) { pr_err("Unable to allocate memory for response buffer.\n"); goto out_login; diff --git a/drivers/target/target_core_fabric_configfs.c b/drivers/target/target_core_fabric_configfs.c index 166dbf4c40616..ab8f81650710e 100644 --- a/drivers/target/target_core_fabric_configfs.c +++ b/drivers/target/target_core_fabric_configfs.c @@ -690,6 +690,14 @@ static void target_fabric_port_unlink( } core_dev_del_lun(se_tpg, lun); + + if (tf->tf_ops->fabric_post_unlink) { + /* + * Allow fabrics to release state that must remain valid until + * core_dev_del_lun() has drained all active LUN references. + */ + tf->tf_ops->fabric_post_unlink(se_tpg, lun); + } } static void target_fabric_port_release(struct config_item *item) diff --git a/drivers/thermal/airoha_thermal.c b/drivers/thermal/airoha_thermal.c index b9fd6bfc88e5e..829a7327fc403 100644 --- a/drivers/thermal/airoha_thermal.c +++ b/drivers/thermal/airoha_thermal.c @@ -273,7 +273,7 @@ static int airoha_thermal_set_trips(struct thermal_zone_device *tz, int low, if (low != -INT_MAX) { /* Validate low and clamp it to a supported value */ - low = clamp_t(int, high, RAW_TO_TEMP(priv, 0), + low = clamp_t(int, low, RAW_TO_TEMP(priv, 0), RAW_TO_TEMP(priv, FIELD_MAX(EN7581_DOUT_TADC_MASK))); /* We offset the low temp of 1°C to trigger correct event */ @@ -403,7 +403,7 @@ static void airoha_thermal_setup_monitor(struct airoha_thermal_priv *priv) * sen interval is 379 * 52.715us = 19.97ms */ writel(FIELD_PREP(EN7581_FILT_INTERVAL, 1) | - FIELD_PREP(EN7581_FILT_INTERVAL, 379), + FIELD_PREP(EN7581_SEN_INTERVAL, 379), priv->base + EN7581_TEMPMONCTL2); /* AHB poll is set to 146 * 68.64 = 10.02us */ diff --git a/drivers/thermal/db8500_thermal.c b/drivers/thermal/db8500_thermal.c index 576f88b6a1b35..cf1706569e6d7 100644 --- a/drivers/thermal/db8500_thermal.c +++ b/drivers/thermal/db8500_thermal.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include @@ -82,7 +82,7 @@ static void db8500_thermal_update_config(struct db8500_thermal_zone *th, unsigned long next_low, unsigned long next_high) { - prcmu_stop_temp_sense(); + db8500_prcmu_stop_temp_sense(); th->cur_index = idx; th->interpolated_temp = (next_low + next_high)/2; @@ -91,8 +91,8 @@ static void db8500_thermal_update_config(struct db8500_thermal_zone *th, * The PRCMU accept absolute temperatures in celsius so divide * down the millicelsius with 1000 */ - prcmu_config_hotmon((u8)(next_low/1000), (u8)(next_high/1000)); - prcmu_start_temp_sense(PRCMU_DEFAULT_MEASURE_TIME); + db8500_prcmu_config_hotmon((u8)(next_low / 1000), (u8)(next_high / 1000)); + db8500_prcmu_start_temp_sense(PRCMU_DEFAULT_MEASURE_TIME); } static irqreturn_t prcmu_low_irq_handler(int irq, void *irq_data) @@ -204,7 +204,7 @@ static int db8500_thermal_probe(struct platform_device *pdev) static int db8500_thermal_suspend(struct platform_device *pdev, pm_message_t state) { - prcmu_stop_temp_sense(); + db8500_prcmu_stop_temp_sense(); return 0; } diff --git a/drivers/thermal/gov_step_wise.c b/drivers/thermal/gov_step_wise.c index ea277c466d8d2..4fa4377f0d428 100644 --- a/drivers/thermal/gov_step_wise.c +++ b/drivers/thermal/gov_step_wise.c @@ -65,14 +65,12 @@ static unsigned long get_target_state(struct thermal_instance *instance, min(instance->lower + 1, instance->upper), instance->upper); } else if (trend == THERMAL_TREND_DROPPING) { - if (cur_state <= instance->lower) - return THERMAL_NO_TARGET; - /* - * If 'throttle' is false, no mitigation is necessary, so - * request the lower state for this instance. + * If 'throttle' is false, no mitigation is necessary and + * passive polling is already deactivated, so clear this + * instance state by returning THERMAL_NO_TARGET. */ - return instance->lower; + return THERMAL_NO_TARGET; } return instance->target; diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index 5aaacbc534786..b0e88b6a9352a 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -832,12 +832,12 @@ static int imx_thermal_runtime_resume(struct device *dev) ret = regmap_write(map, socdata->sensor_ctrl + REG_CLR, socdata->power_down_mask); if (ret) - return ret; + goto disable_clk; ret = regmap_write(map, socdata->sensor_ctrl + REG_SET, socdata->measure_temp_mask); if (ret) - return ret; + goto disable_clk; /* * According to the temp sensor designers, it may require up to ~17us @@ -846,6 +846,11 @@ static int imx_thermal_runtime_resume(struct device *dev) usleep_range(20, 50); return 0; + +disable_clk: + clk_disable_unprepare(data->thermal_clk); + + return ret; } static const struct dev_pm_ops imx_thermal_pm_ops = { diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c index d200734625ee8..5d70301d4a3d9 100644 --- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c +++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c @@ -356,8 +356,10 @@ static void cleanup_odvp(struct int3400_thermal_priv *priv) kfree(priv->odvp_attrs[i].attr.attr.name); } kfree(priv->odvp_attrs); + priv->odvp_attrs = NULL; } kfree(priv->odvp); + priv->odvp = NULL; priv->odvp_count = 0; } @@ -635,7 +637,6 @@ static int int3400_thermal_probe(struct platform_device *pdev) acpi_remove_notify_handler(priv->adev->handle, ACPI_DEVICE_NOTIFY, int3400_notify); free_sysfs: - cleanup_odvp(priv); if (!ZERO_OR_NULL_PTR(priv->data_vault)) { device_remove_bin_file(&pdev->dev, &bin_attr_data_vault); kfree(priv->data_vault); @@ -649,6 +650,7 @@ static int int3400_thermal_probe(struct platform_device *pdev) acpi_thermal_rel_misc_device_remove(priv->adev->handle); thermal_zone_device_unregister(priv->thermal); free_art_trt: + cleanup_odvp(priv); kfree(priv->trts); kfree(priv->arts); free_priv: diff --git a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c index d7f2e6ca92c2c..d1b086737bcd2 100644 --- a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c +++ b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c @@ -369,9 +369,6 @@ static int adc_tm5_get_temp(struct thermal_zone_device *tz, int *temp) if (ret < 0) return ret; - if (ret != IIO_VAL_INT) - return -EINVAL; - return 0; } diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c index 35439ec5f8bcb..297724e81593f 100644 --- a/drivers/thermal/qoriq_thermal.c +++ b/drivers/thermal/qoriq_thermal.c @@ -415,11 +415,20 @@ static int qoriq_tmu_resume(struct device *dev) if (data->ver > TMU_VER1) { ret = regmap_clear_bits(data->regmap, REGS_TMR, TMR_CMD); if (ret) - return ret; + goto disable_clk; } /* Enable monitoring */ - return regmap_update_bits(data->regmap, REGS_TMR, TMR_ME, TMR_ME); + ret = regmap_update_bits(data->regmap, REGS_TMR, TMR_ME, TMR_ME); + if (ret) + goto disable_clk; + + return 0; + +disable_clk: + clk_disable_unprepare(data->clk); + + return ret; } static DEFINE_SIMPLE_DEV_PM_OPS(qoriq_tmu_pm_ops, diff --git a/drivers/thermal/renesas/rcar_thermal.c b/drivers/thermal/renesas/rcar_thermal.c index 6e5dcac5d47ae..fd686da9252e1 100644 --- a/drivers/thermal/renesas/rcar_thermal.c +++ b/drivers/thermal/renesas/rcar_thermal.c @@ -492,12 +492,6 @@ static int rcar_thermal_probe(struct platform_device *pdev) "rcar_thermal", trips, ARRAY_SIZE(trips), priv, &rcar_thermal_zone_ops, NULL, 0, idle); - - ret = thermal_zone_device_enable(priv->zone); - if (ret) { - thermal_zone_device_unregister(priv->zone); - priv->zone = ERR_PTR(ret); - } } if (IS_ERR(priv->zone)) { dev_err(dev, "can't register thermal zone\n"); @@ -506,11 +500,12 @@ static int rcar_thermal_probe(struct platform_device *pdev) goto error_unregister; } - if (chip->use_of_thermal) { + if (chip->use_of_thermal) ret = thermal_add_hwmon_sysfs(priv->zone); - if (ret) - goto error_unregister; - } + else + ret = thermal_zone_device_enable(priv->zone); + if (ret) + goto error_unregister; rcar_thermal_irq_enable(priv); diff --git a/drivers/thermal/spacemit/k1_tsensor.c b/drivers/thermal/spacemit/k1_tsensor.c index 79222d233129d..a9eaedefdac3b 100644 --- a/drivers/thermal/spacemit/k1_tsensor.c +++ b/drivers/thermal/spacemit/k1_tsensor.c @@ -156,13 +156,12 @@ static int k1_tsensor_set_trips(struct thermal_zone_device *tz, int low, int hig struct k1_tsensor *ts = ch->ts; u32 val; - if (low >= high) - return -EINVAL; - low = clamp_val(low / 1000 + TEMPERATURE_OFFSET, TEMPERATURE_OFFSET, FIELD_MAX(K1_TSENSOR_THRSH_LOW_MASK)); high = clamp_val(high / 1000 + TEMPERATURE_OFFSET, TEMPERATURE_OFFSET, FIELD_MAX(K1_TSENSOR_THRSH_HIGH_MASK)); + if (low >= high) + return -EINVAL; val = readl(ts->base + K1_TSENSOR_THRSH_REG(ch->id)); diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c index ec73d03a1e60a..45ce21914df86 100644 --- a/drivers/thermal/thermal_hwmon.c +++ b/drivers/thermal/thermal_hwmon.c @@ -95,34 +95,12 @@ thermal_hwmon_lookup_by_type(const struct thermal_zone_device *tz) struct thermal_hwmon_device *hwmon; char type[THERMAL_NAME_LENGTH]; - mutex_lock(&thermal_hwmon_list_lock); list_for_each_entry(hwmon, &thermal_hwmon_list, node) { strscpy(type, tz->type); strreplace(type, '-', '_'); - if (!strcmp(hwmon->type, type)) { - mutex_unlock(&thermal_hwmon_list_lock); + if (!strcmp(hwmon->type, type)) return hwmon; - } } - mutex_unlock(&thermal_hwmon_list_lock); - - return NULL; -} - -/* Find the temperature input matching a given thermal zone */ -static struct thermal_hwmon_temp * -thermal_hwmon_lookup_temp(const struct thermal_hwmon_device *hwmon, - const struct thermal_zone_device *tz) -{ - struct thermal_hwmon_temp *temp; - - mutex_lock(&thermal_hwmon_list_lock); - list_for_each_entry(temp, &hwmon->tz_list, hwmon_node) - if (temp->tz == tz) { - mutex_unlock(&thermal_hwmon_list_lock); - return temp; - } - mutex_unlock(&thermal_hwmon_list_lock); return NULL; } @@ -138,7 +116,9 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) struct thermal_hwmon_device *hwmon; struct thermal_hwmon_temp *temp; int new_hwmon_device = 1; - int result; + int result = 0; + + mutex_lock(&thermal_hwmon_list_lock); hwmon = thermal_hwmon_lookup_by_type(tz); if (hwmon) { @@ -147,8 +127,10 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) } hwmon = kzalloc_obj(*hwmon); - if (!hwmon) - return -ENOMEM; + if (!hwmon) { + result = -ENOMEM; + goto unlock; + } INIT_LIST_HEAD(&hwmon->tz_list); strscpy(hwmon->type, tz->type, THERMAL_NAME_LENGTH); @@ -196,24 +178,24 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) temp->temp_crit_present = true; } - mutex_lock(&thermal_hwmon_list_lock); if (new_hwmon_device) list_add_tail(&hwmon->node, &thermal_hwmon_list); list_add_tail(&temp->hwmon_node, &hwmon->tz_list); - mutex_unlock(&thermal_hwmon_list_lock); - return 0; + goto unlock; - unregister_input: +unregister_input: device_remove_file(hwmon->device, &temp->temp_input.attr); - free_temp_mem: +free_temp_mem: kfree(temp); - unregister_name: +unregister_name: if (new_hwmon_device) hwmon_device_unregister(hwmon->device); - free_mem: +free_mem: if (new_hwmon_device) kfree(hwmon); +unlock: + mutex_unlock(&thermal_hwmon_list_lock); return result; } @@ -221,8 +203,11 @@ EXPORT_SYMBOL_GPL(thermal_add_hwmon_sysfs); void thermal_remove_hwmon_sysfs(struct thermal_zone_device *tz) { + struct thermal_hwmon_temp *temp, *entry; struct thermal_hwmon_device *hwmon; - struct thermal_hwmon_temp *temp; + bool unregister; + + guard(mutex)(&thermal_hwmon_list_lock); hwmon = thermal_hwmon_lookup_by_type(tz); if (unlikely(!hwmon)) { @@ -231,29 +216,25 @@ void thermal_remove_hwmon_sysfs(struct thermal_zone_device *tz) return; } - temp = thermal_hwmon_lookup_temp(hwmon, tz); - if (unlikely(!temp)) { - /* Should never happen... */ - dev_dbg(&tz->device, "temperature input lookup failed!\n"); - return; - } + unregister = hwmon->device->parent == &tz->device; - device_remove_file(hwmon->device, &temp->temp_input.attr); - if (temp->temp_crit_present) - device_remove_file(hwmon->device, &temp->temp_crit.attr); + list_for_each_entry_safe_reverse(temp, entry, &hwmon->tz_list, hwmon_node) { + if (!unregister && temp->tz != tz) + continue; - mutex_lock(&thermal_hwmon_list_lock); - list_del(&temp->hwmon_node); - kfree(temp); - if (!list_empty(&hwmon->tz_list)) { - mutex_unlock(&thermal_hwmon_list_lock); - return; + device_remove_file(hwmon->device, &temp->temp_input.attr); + if (temp->temp_crit_present) + device_remove_file(hwmon->device, &temp->temp_crit.attr); + + list_del(&temp->hwmon_node); + kfree(temp); } - list_del(&hwmon->node); - mutex_unlock(&thermal_hwmon_list_lock); - hwmon_device_unregister(hwmon->device); - kfree(hwmon); + if (unregister) { + list_del(&hwmon->node); + hwmon_device_unregister(hwmon->device); + kfree(hwmon); + } } EXPORT_SYMBOL_GPL(thermal_remove_hwmon_sysfs); diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index b44abfc997edf..44bd7c50e4ac2 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -400,8 +400,8 @@ static int create_trip_attrs(struct thermal_zone_device *tz) struct thermal_trip_attrs *trip_attrs = &td->trip_attrs; /* create trip type attribute */ - snprintf(trip_attrs->type.name, THERMAL_NAME_LENGTH, - "trip_point_%d_type", i); + scnprintf(trip_attrs->type.name, sizeof(trip_attrs->type.name), + "trip_point_%d_type", i); sysfs_attr_init(&trip_attrs->type.attr.attr); trip_attrs->type.attr.attr.name = trip_attrs->type.name; @@ -410,8 +410,8 @@ static int create_trip_attrs(struct thermal_zone_device *tz) attrs[i] = &trip_attrs->type.attr.attr; /* create trip temp attribute */ - snprintf(trip_attrs->temp.name, THERMAL_NAME_LENGTH, - "trip_point_%d_temp", i); + scnprintf(trip_attrs->temp.name, sizeof(trip_attrs->temp.name), + "trip_point_%d_temp", i); sysfs_attr_init(&trip_attrs->temp.attr.attr); trip_attrs->temp.attr.attr.name = trip_attrs->temp.name; @@ -423,8 +423,8 @@ static int create_trip_attrs(struct thermal_zone_device *tz) } attrs[i + tz->num_trips] = &trip_attrs->temp.attr.attr; - snprintf(trip_attrs->hyst.name, THERMAL_NAME_LENGTH, - "trip_point_%d_hyst", i); + scnprintf(trip_attrs->hyst.name, sizeof(trip_attrs->hyst.name), + "trip_point_%d_hyst", i); sysfs_attr_init(&trip_attrs->hyst.attr.attr); trip_attrs->hyst.attr.attr.name = trip_attrs->hyst.name; diff --git a/drivers/thunderbolt/stream.c b/drivers/thunderbolt/stream.c index 4cc86d8d64918..43d29d69dcdbb 100644 --- a/drivers/thunderbolt/stream.c +++ b/drivers/thunderbolt/stream.c @@ -512,8 +512,10 @@ tbstream_dev_alloc_tx(struct tbstream_dev *sdev, enum tbstream_frame_pdf pdf, dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy, size, DMA_TO_DEVICE); if (pdf == TBSTREAM_DATA) { - if (copy_page_from_iter(sf->page, 0, size, from) != size) + if (copy_page_from_iter(sf->page, 0, size, from) != size) { + sdev->tx_ring.cons--; return ERR_PTR(-EFAULT); + } } else { memset(page_address(sf->page), 0, size); } @@ -671,7 +673,7 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to) } nbytes = 0; - while (nbytes < iov_iter_count(to)) { + while (iov_iter_count(to)) { struct tbstream_frame *sf; size_t size, sf_size; @@ -693,7 +695,7 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to) } sf_size = tb_ring_frame_size(&sf->frame); - size = min(iov_iter_count(to) - nbytes, sf_size); + size = min(iov_iter_count(to), sf_size); if (copy_page_to_iter(sf->page, sf->offset, size, to) != size) { ret = -EFAULT; @@ -763,10 +765,10 @@ tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from) } nbytes = 0; - while (nbytes < iov_iter_count(from)) { + while (iov_iter_count(from)) { size_t size; - size = min(iov_iter_count(from) - nbytes, TB_MAX_FRAME_SIZE); + size = min(iov_iter_count(from), TB_MAX_FRAME_SIZE); ret = tbstream_dev_send_data(sdev, from, size); if (ret) { /* diff --git a/drivers/tty/hvc/Kconfig b/drivers/tty/hvc/Kconfig index c2a4e88b328f3..5866195de26a6 100644 --- a/drivers/tty/hvc/Kconfig +++ b/drivers/tty/hvc/Kconfig @@ -79,7 +79,7 @@ config HVC_UDBG config HVC_DCC bool "ARM JTAG DCC console" - depends on ARM || ARM64 + depends on (ARM && (CPU_V6 || CPU_V6K || CPU_V7)) || ARM64 select HVC_DRIVER select SERIAL_CORE_CONSOLE help diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index 9abaeecd05fc8..c4824c201e1c3 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -2543,7 +2543,7 @@ static int pl011_console_setup(struct console *co, char *options) /* Allow pins to be muxed in and configured */ pinctrl_pm_select_default_state(uap->port.dev); - ret = clk_prepare(uap->clk); + ret = clk_prepare_enable(uap->clk); if (ret) return ret; @@ -2572,6 +2572,15 @@ static int pl011_console_setup(struct console *co, char *options) return uart_set_options(&uap->port, co, baud, parity, bits, flow); } +static int pl011_console_exit(struct console *co) +{ + struct uart_amba_port *uap = amba_ports[co->index]; + + clk_disable_unprepare(uap->clk); + + return 0; +} + /** * pl011_console_match - non-standard console matching * @co: registering console @@ -2641,8 +2650,6 @@ pl011_console_write_atomic(struct console *co, struct nbcon_write_context *wctxt if (!nbcon_enter_unsafe(wctxt)) return; - clk_enable(uap->clk); - if (!uap->vendor->always_enabled) { old_cr = pl011_read(uap, REG_CR); pl011_write((old_cr & ~UART011_CR_CTSEN) | (UART01x_CR_UARTEN | UART011_CR_TXE), @@ -2659,8 +2666,6 @@ pl011_console_write_atomic(struct console *co, struct nbcon_write_context *wctxt if (!uap->vendor->always_enabled) pl011_write(old_cr, uap, REG_CR); - clk_disable(uap->clk); - nbcon_exit_unsafe(wctxt); } @@ -2673,8 +2678,6 @@ pl011_console_write_thread(struct console *co, struct nbcon_write_context *wctxt if (!nbcon_enter_unsafe(wctxt)) return; - clk_enable(uap->clk); - if (!uap->vendor->always_enabled) { old_cr = pl011_read(uap, REG_CR); pl011_write((old_cr & ~UART011_CR_CTSEN) | (UART01x_CR_UARTEN | UART011_CR_TXE), @@ -2703,8 +2706,6 @@ pl011_console_write_thread(struct console *co, struct nbcon_write_context *wctxt if (!uap->vendor->always_enabled) pl011_write(old_cr, uap, REG_CR); - clk_disable(uap->clk); - nbcon_exit_unsafe(wctxt); } @@ -2725,6 +2726,7 @@ static struct console amba_console = { .name = "ttyAMA", .device = uart_console_device, .setup = pl011_console_setup, + .exit = pl011_console_exit, .match = pl011_console_match, .write_atomic = pl011_console_write_atomic, .write_thread = pl011_console_write_thread, @@ -3092,21 +3094,45 @@ static void pl011_remove(struct amba_device *dev) static int pl011_suspend(struct device *dev) { struct uart_amba_port *uap = dev_get_drvdata(dev); + int ret; if (!uap) return -EINVAL; - return uart_suspend_port(&amba_reg, &uap->port); + ret = uart_suspend_port(&amba_reg, &uap->port); + if (ret) + return ret; + + if (console_suspend_enabled && uap->port.suspended && + uart_console_registered(&uap->port)) + clk_disable_unprepare(uap->clk); + + return 0; } static int pl011_resume(struct device *dev) { struct uart_amba_port *uap = dev_get_drvdata(dev); + bool resume_console; + int ret; if (!uap) return -EINVAL; - return uart_resume_port(&amba_reg, &uap->port); + resume_console = console_suspend_enabled && + uap->port.suspended && + uart_console_registered(&uap->port); + if (resume_console) { + ret = clk_prepare_enable(uap->clk); + if (ret) + return ret; + } + + ret = uart_resume_port(&amba_reg, &uap->port); + if (ret && resume_console) + clk_disable_unprepare(uap->clk); + + return ret; } #endif diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index 251a50c8aa387..b0f34a6e7d4f8 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -2080,6 +2081,9 @@ static const struct uart_ops imx_uart_pops = { static struct imx_port *imx_uart_ports[UART_NR]; +/* Held across uart_add/remove_one_port(); console callbacks must not take it. */ +static DEFINE_MUTEX(imx_uart_ports_lock); + #if IS_ENABLED(CONFIG_SERIAL_IMX_CONSOLE) static void imx_uart_console_putchar(struct uart_port *port, unsigned char ch) { @@ -2632,11 +2636,19 @@ static int imx_uart_probe(struct platform_device *pdev) } } - imx_uart_ports[sport->port.line] = sport; - platform_set_drvdata(pdev, sport); - ret = uart_add_one_port(&imx_uart_uart_driver, &sport->port); + scoped_guard(mutex, &imx_uart_ports_lock) { + if (imx_uart_ports[sport->port.line]) { + ret = -EBUSY; + } else { + imx_uart_ports[sport->port.line] = sport; + ret = uart_add_one_port(&imx_uart_uart_driver, + &sport->port); + if (ret) + imx_uart_ports[sport->port.line] = NULL; + } + } err_clk: clk_disable_unprepare(sport->clk_ipg); @@ -2648,7 +2660,9 @@ static void imx_uart_remove(struct platform_device *pdev) { struct imx_port *sport = platform_get_drvdata(pdev); + guard(mutex)(&imx_uart_ports_lock); uart_remove_one_port(&imx_uart_uart_driver, &sport->port); + imx_uart_ports[sport->port.line] = NULL; } static void imx_uart_restore_context(struct imx_port *sport) diff --git a/drivers/tty/serial/ma35d1_serial.c b/drivers/tty/serial/ma35d1_serial.c index 285b0fe41a86a..920fe7ff5083b 100644 --- a/drivers/tty/serial/ma35d1_serial.c +++ b/drivers/tty/serial/ma35d1_serial.c @@ -608,8 +608,14 @@ static int __init ma35d1serial_console_setup(struct console *co, char *options) if (!np || !p) return -ENODEV; - if (of_property_read_u32_array(np, "reg", val32, ARRAY_SIZE(val32)) != 0) + if (of_property_read_u32_array(np, "reg", val32, ARRAY_SIZE(val32)) != 0) { + of_node_put(np); + ma35d1serial_uart_nodes[co->index] = NULL; return -EINVAL; + } + + of_node_put(np); + ma35d1serial_uart_nodes[co->index] = NULL; p->port.iobase = val32[1]; p->port.membase = ioremap(p->port.iobase, MA35_UART_REG_SIZE); @@ -648,8 +654,10 @@ static void ma35d1serial_console_init_port(void) of_node_get(np); ma35d1serial_uart_nodes[i] = np; i++; - if (i == MA35_UART_NR) + if (i == MA35_UART_NR) { + of_node_put(np); break; + } } } } diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c index 1e39122ec09f4..1fc869ad84bd8 100644 --- a/drivers/tty/serial/qcom_geni_serial.c +++ b/drivers/tty/serial/qcom_geni_serial.c @@ -143,6 +143,7 @@ struct qcom_geni_serial_port { unsigned int tx_remaining; unsigned int tx_queued; + bool tx_dma_stale; int wakeup_irq; bool rx_tx_swap; bool cts_rts_swap; @@ -697,6 +698,7 @@ static void qcom_geni_serial_start_tx_dma(struct uart_port *uport) } port->tx_remaining = xmit_size; + port->tx_dma_stale = false; } static void qcom_geni_serial_start_tx_fifo(struct uart_port *uport) @@ -1029,6 +1031,7 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport) struct qcom_geni_serial_port *port = to_dev_port(uport); struct tty_port *tport = &uport->state->port; unsigned int fifo_len = kfifo_len(&tport->xmit_fifo); + bool tx_dma_stale = port->tx_dma_stale; /* * Only advance the kfifo if it still contains the bytes that were @@ -1039,12 +1042,13 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport) * kfifo->in, making kfifo_len() wrap to UART_XMIT_SIZE - tx_remaining * and triggering a spurious large DMA transfer of stale data. */ - if (fifo_len >= port->tx_remaining) + if (!tx_dma_stale && fifo_len >= port->tx_remaining) uart_xmit_advance(uport, port->tx_remaining); geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining); port->tx_dma_addr = 0; port->tx_remaining = 0; + port->tx_dma_stale = false; if (!kfifo_is_empty(&tport->xmit_fifo)) qcom_geni_serial_start_tx_dma(uport); @@ -1182,6 +1186,10 @@ static void qcom_geni_serial_shutdown(struct uart_port *uport) static void qcom_geni_serial_flush_buffer_fifo(struct uart_port *uport) { + struct qcom_geni_serial_port *port = to_dev_port(uport); + + if (port->tx_dma_addr) + port->tx_dma_stale = true; qcom_geni_serial_cancel_tx_cmd(uport); } diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index a530ad372b434..234976fb2a87f 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -2777,8 +2777,10 @@ int uart_register_driver(struct uart_driver *drv) for (i = 0; i < drv->nr; i++) tty_port_destroy(&drv->state[i].port); tty_driver_kref_put(normal); + drv->tty_driver = NULL; out_kfree: kfree(drv->state); + drv->state = NULL; out: return retval; } @@ -3056,7 +3058,6 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u struct uart_state *state; struct tty_port *port; struct device *tty_dev; - int num_groups; if (uport->line >= drv->nr) return -EINVAL; @@ -3068,6 +3069,22 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u if (state->uart_port) return -EINVAL; + uport->name = kasprintf(GFP_KERNEL, "%s%u", drv->dev_name, + drv->tty_driver->name_base + uport->line); + if (!uport->name) + return -ENOMEM; + + /* + * uart_configure_port() may set uport->attr_group and register the + * console. Allocate room for both groups and a NULL terminator first. + */ + uport->tty_groups = kzalloc_objs(*uport->tty_groups, 3); + if (!uport->tty_groups) { + kfree(uport->name); + return -ENOMEM; + } + uport->tty_groups[0] = &tty_dev_attr_group; + /* Link the port to the driver state table and vice versa */ atomic_set(&state->refcount, 1); init_waitqueue_head(&state->remove_wait); @@ -3084,10 +3101,6 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u state->pm_state = UART_PM_STATE_UNDEFINED; uart_port_set_cons(uport, drv->cons); uport->minor = drv->tty_driver->minor_start + uport->line; - uport->name = kasprintf(GFP_KERNEL, "%s%u", drv->dev_name, - drv->tty_driver->name_base + uport->line); - if (!uport->name) - return -ENOMEM; if (uport->cons && uport->dev) of_console_check(uport->dev->of_node, uport->cons->name, uport->line); @@ -3102,15 +3115,6 @@ static int serial_core_add_one_port(struct uart_driver *drv, struct uart_port *u port->console = uart_console(uport); - num_groups = 2; - if (uport->attr_group) - num_groups++; - - uport->tty_groups = kzalloc_objs(*uport->tty_groups, num_groups); - if (!uport->tty_groups) - return -ENOMEM; - - uport->tty_groups[0] = &tty_dev_attr_group; if (uport->attr_group) uport->tty_groups[1] = uport->attr_group; diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 6b283fd03ff82..4889076b975f3 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -3167,8 +3167,10 @@ static int tty_cdev_add(struct tty_driver *driver, dev_t dev, driver->cdevs[index]->ops = &tty_fops; driver->cdevs[index]->owner = driver->owner; err = cdev_add(driver->cdevs[index], dev, count); - if (err) + if (err) { kobject_put(&driver->cdevs[index]->kobj); + driver->cdevs[index] = NULL; + } return err; } @@ -3305,7 +3307,7 @@ EXPORT_SYMBOL_GPL(tty_register_device_attr); void tty_unregister_device(struct tty_driver *driver, unsigned index) { device_destroy(&tty_class, MKDEV(driver->major, driver->minor_start) + index); - if (!(driver->flags & TTY_DRIVER_DYNAMIC_ALLOC)) { + if (!(driver->flags & TTY_DRIVER_DYNAMIC_ALLOC) && driver->cdevs[index]) { cdev_del(driver->cdevs[index]); driver->cdevs[index] = NULL; } diff --git a/drivers/ufs/core/ufs-debugfs.c b/drivers/ufs/core/ufs-debugfs.c index e3dd81d6fe828..be527209540d7 100644 --- a/drivers/ufs/core/ufs-debugfs.c +++ b/drivers/ufs/core/ufs-debugfs.c @@ -165,7 +165,7 @@ static ssize_t ufs_saved_err_write(struct file *file, const char __user *buf, char val_str[16] = { }; int val, ret; - if (count > sizeof(val_str)) + if (count >= sizeof(val_str)) return -EINVAL; if (copy_from_user(val_str, buf, count)) return -EFAULT; diff --git a/drivers/ufs/core/ufs-rpmb.c b/drivers/ufs/core/ufs-rpmb.c index 62120dc2e9da7..3a0888eaa2ce7 100644 --- a/drivers/ufs/core/ufs-rpmb.c +++ b/drivers/ufs/core/ufs-rpmb.c @@ -69,6 +69,10 @@ static int ufs_rpmb_route_frames(struct device *dev, u8 *req, unsigned int req_l hba = ufs_rpmb->hba; + /* req_resp is at the end of an RPMB frame. */ + if (req_len < sizeof(*frm_out)) + return -EINVAL; + req_type = be16_to_cpu(frm_out->req_resp); switch (req_type) { diff --git a/drivers/ufs/core/ufs-txeq.c b/drivers/ufs/core/ufs-txeq.c index aa64f2bf4f1ef..7df3d3b18cbec 100644 --- a/drivers/ufs/core/ufs-txeq.c +++ b/drivers/ufs/core/ufs-txeq.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include "ufshcd-priv.h" @@ -1216,14 +1217,25 @@ static int ufshcd_tx_eqtr(struct ufs_hba *hba, struct ufs_pa_layer_attr *pwr_mode) { struct ufs_pa_layer_attr old_pwr_info; + unsigned int noio_flag; int ret; + /* + * ufshcd_tx_eqtr() is called from a power-mode-change context where + * I/O is suspended. Use memalloc_noio_save() to propagate GFP_NOIO + * to all allocations in the call tree instead of tagging each call + * site individually. + */ + noio_flag = memalloc_noio_save(); + if (!params->eqtr_record) { params->eqtr_record = devm_kzalloc(hba->dev, sizeof(*params->eqtr_record), GFP_KERNEL); - if (!params->eqtr_record) - return -ENOMEM; + if (!params->eqtr_record) { + ret = -ENOMEM; + goto out_noio_restore; + } } memcpy(&old_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr)); @@ -1231,23 +1243,26 @@ static int ufshcd_tx_eqtr(struct ufs_hba *hba, ret = ufshcd_tx_eqtr_prepare(hba, pwr_mode); if (ret) { dev_err(hba->dev, "Failed to prepare TX EQTR: %d\n", ret); - goto out; + goto out_unprepare; } ret = ufshcd_vops_tx_eqtr_notify(hba, PRE_CHANGE, pwr_mode); if (ret) - goto out; + goto out_unprepare; ret = __ufshcd_tx_eqtr(hba, params, pwr_mode); if (ret) - goto out; + goto out_unprepare; ret = ufshcd_vops_tx_eqtr_notify(hba, POST_CHANGE, pwr_mode); -out: +out_unprepare: if (ret) ufshcd_tx_eqtr_unprepare(hba, &old_pwr_info); +out_noio_restore: + memalloc_noio_restore(noio_flag); + return ret; } diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index a41b56dbf3b7f..bc33da89af267 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -1317,6 +1317,7 @@ static int ufshcd_wait_for_pending_cmds(struct ufs_hba *hba, break; } + __set_current_state(TASK_UNINTERRUPTIBLE); io_schedule_timeout(msecs_to_jiffies(20)); if (ktime_to_us(ktime_sub(ktime_get(), start)) > wait_timeout_us) { @@ -3865,7 +3866,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum u { struct uc_string_id *uc_str; u8 *str; - int ret; + int ret, uc_len; if (!buf) return -EINVAL; @@ -3890,11 +3891,19 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum u goto out; } + uc_len = uc_str->len - QUERY_DESC_HDR_SIZE; + if (uc_len % sizeof(*uc_str->uc)) { + dev_err(hba->dev, "String Desc has an odd UTF-16 payload length\n"); + str = NULL; + ret = -EINVAL; + goto out; + } + if (fmt == SD_ASCII_STD) { ssize_t ascii_len; int i; - /* remove header and divide by 2 to move from UTF16 to UTF8 */ - ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1; + /* Allow up to three UTF-8 bytes per UTF-16 code unit plus a NUL. */ + ascii_len = uc_len / sizeof(*uc_str->uc) * 3 + 1; str = kzalloc(ascii_len, GFP_KERNEL); if (!str) { ret = -ENOMEM; @@ -3906,7 +3915,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum u * we need to convert to utf-8 so it can be displayed */ ret = utf16s_to_utf8s(uc_str->uc, - uc_str->len - QUERY_DESC_HDR_SIZE, + uc_len / sizeof(*uc_str->uc), UTF16_BIG_ENDIAN, str, ascii_len - 1); /* replace non-printable or non-ASCII characters with spaces */ @@ -3916,11 +3925,17 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum u str[ret++] = '\0'; } else { - str = kmemdup(uc_str->uc, uc_str->len, GFP_KERNEL); + /* + * Keep the bLength-sized raw output for the RPMB device ID ABI. + * The two bytes beyond the UTF-16 payload are explicitly zeroed + * instead of being read past the descriptor buffer. + */ + str = kzalloc(uc_str->len, GFP_KERNEL); if (!str) { ret = -ENOMEM; goto out; } + memcpy(str, uc_str->uc, uc_len); ret = uc_str->len; } out: @@ -4715,7 +4730,9 @@ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), &pwr_info->lane_tx); - if (!pwr_info->lane_rx || !pwr_info->lane_tx) { + if (!pwr_info->lane_rx || !pwr_info->lane_tx || + pwr_info->lane_rx > UFS_MAX_LANES || + pwr_info->lane_tx > UFS_MAX_LANES) { dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n", __func__, pwr_info->lane_rx, @@ -5846,8 +5863,8 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); enum utp_ocs ocs; - if (WARN_ONCE(!cmd, "cqe->command_desc_base_addr = %#llx\n", - le64_to_cpu(cqe->command_desc_base_addr))) + if (WARN_ONCE(!cmd, "invalid completion tag %d, cqe->command_desc_base_addr = %#llx\n", + task_tag, cqe ? le64_to_cpu(cqe->command_desc_base_addr) : 0ULL)) return; if (hba->monitor.enabled) { @@ -6731,11 +6748,17 @@ static void ufshcd_err_handling_prepare(struct ufs_hba *hba) } /* Wait for ongoing ufshcd_queuecommand() calls to finish. */ blk_mq_quiesce_tagset(&hba->host->tag_set); + /* + * Internal commands are submitted on the pseudo SCSI device. Let them + * through so that the error handler can recover the link. + */ + blk_mq_unquiesce_queue(hba->host->pseudo_sdev->request_queue); cancel_work_sync(&hba->eeh_work); } static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) { + blk_mq_quiesce_queue_nowait(hba->host->pseudo_sdev->request_queue); blk_mq_unquiesce_tagset(&hba->host->tag_set); ufshcd_release(hba); if (ufshcd_is_clkscaling_supported(hba)) diff --git a/drivers/ufs/host/ufshcd-pci.c b/drivers/ufs/host/ufshcd-pci.c index f2433879b0eba..21bb11c724bea 100644 --- a/drivers/ufs/host/ufshcd-pci.c +++ b/drivers/ufs/host/ufshcd-pci.c @@ -181,6 +181,25 @@ static int ufs_intel_lkf_pwr_change_notify(struct ufs_hba *hba, return err; } +static int ufs_intel_nvl_pwr_change_notify(struct ufs_hba *hba, + enum ufs_notify_change_status stage, + struct ufs_pa_layer_attr *dev_req_params) +{ + int adapt_val; + + if (stage != PRE_CHANGE || hba->ufs_version < ufshci_version(4, 0)) + return 0; + + if (dev_req_params->pwr_tx == FAST_MODE || dev_req_params->pwr_tx == FASTAUTO_MODE) + adapt_val = PA_INITIAL_ADAPT; + else + adapt_val = PA_NO_ADAPT; + + ufshcd_dme_configure_adapt(hba, dev_req_params->gear_tx, adapt_val); + + return 0; +} + static int ufs_intel_lkf_apply_dev_quirks(struct ufs_hba *hba) { u32 granularity, peer_granularity; @@ -441,6 +460,43 @@ static int ufs_intel_mtl_init(struct ufs_hba *hba) return ufs_intel_common_init(hba); } +static int ufs_intel_mcq_config_resource(struct ufs_hba *hba) +{ + hba->mcq_base = hba->mmio_base + ufshcd_mcq_queue_cfg_addr(hba); + + return 0; +} + +/* + * This Intel UFS4.0 controller maps MCQ doorbell and interrupt-status + * registers into the same PCI BAR as the legacy HCI space, at this + * fixed offset/stride. + */ +#define UFS_INTEL_SQDAO0 0x2800 +#define UFS_INTEL_SQISAO0 0x2814 +#define UFS_INTEL_CQDAO0 0x281C +#define UFS_INTEL_CQISAO0 0x2824 +#define UFS_INTEL_MCQ_STRIDE 0x30 + +static int ufs_intel_op_runtime_config(struct ufs_hba *hba) +{ + struct ufshcd_mcq_opr_info_t *opr; + int i; + + hba->mcq_opr[OPR_SQD].offset = UFS_INTEL_SQDAO0; + hba->mcq_opr[OPR_SQIS].offset = UFS_INTEL_SQISAO0; + hba->mcq_opr[OPR_CQD].offset = UFS_INTEL_CQDAO0; + hba->mcq_opr[OPR_CQIS].offset = UFS_INTEL_CQISAO0; + + for (i = 0; i < OPR_MAX; i++) { + opr = &hba->mcq_opr[i]; + opr->stride = UFS_INTEL_MCQ_STRIDE; + opr->base = hba->mmio_base + opr->offset; + } + + return 0; +} + static int ufs_qemu_get_hba_mac(struct ufs_hba *hba) { return MAX_SUPP_MAC; @@ -527,6 +583,9 @@ static struct ufs_hba_variant_ops ufs_intel_mtl_hba_vops = { .exit = ufs_intel_common_exit, .hce_enable_notify = ufs_intel_hce_enable_notify, .link_startup_notify = ufs_intel_link_startup_notify, + .pwr_change_notify = ufs_intel_nvl_pwr_change_notify, + .mcq_config_resource = ufs_intel_mcq_config_resource, + .op_runtime_config = ufs_intel_op_runtime_config, .resume = ufs_intel_resume, .device_reset = ufs_intel_device_reset, }; diff --git a/drivers/ufs/host/ufshcd-pltfrm.c b/drivers/ufs/host/ufshcd-pltfrm.c index c2dafb583cf51..ac264786d2a6e 100644 --- a/drivers/ufs/host/ufshcd-pltfrm.c +++ b/drivers/ufs/host/ufshcd-pltfrm.c @@ -206,7 +206,11 @@ static void ufshcd_init_lanes_per_dir(struct ufs_hba *hba) dev_dbg(hba->dev, "%s: failed to read lanes-per-direction, ret=%d\n", __func__, ret); - hba->lanes_per_direction = UFSHCD_DEFAULT_LANES_PER_DIRECTION; + /* Old R-Car S4 DTBs lack "lanes-per-direction = <1>" */ + if (of_device_is_compatible(dev->of_node, "renesas,r8a779f0-ufs")) + hba->lanes_per_direction = 1; + else + hba->lanes_per_direction = UFSHCD_DEFAULT_LANES_PER_DIRECTION; } } diff --git a/drivers/uio/uio.c b/drivers/uio/uio.c index 1e4ade78ed849..e77d5e7d5f64c 100644 --- a/drivers/uio/uio.c +++ b/drivers/uio/uio.c @@ -1057,6 +1057,11 @@ int __uio_register_device(struct module *owner, err_request_irq: uio_dev_del_attributes(idev); err_uio_dev_add_attributes: + mutex_lock(&idev->info_lock); + idev->info = NULL; + mutex_unlock(&idev->info_lock); + wake_up_interruptible(&idev->wait); + kill_fasync(&idev->async_queue, SIGIO, POLL_HUP); device_del(&idev->dev); err_device_create: uio_free_minor(idev->minor); diff --git a/drivers/usb/atm/usbatm.c b/drivers/usb/atm/usbatm.c index 9600e1ec09930..7b0c791399eaa 100644 --- a/drivers/usb/atm/usbatm.c +++ b/drivers/usb/atm/usbatm.c @@ -917,8 +917,8 @@ static int usbatm_atm_init(struct usbatm_data *instance) instance->atm_dev = atm_dev; - atm_dev->ci_range.vpi_bits = ATM_CI_MAX; - atm_dev->ci_range.vci_bits = ATM_CI_MAX; + atm_dev->ci_range.vpi_bits = 8; + atm_dev->ci_range.vci_bits = 16; atm_dev->signal = ATM_PHY_SIG_UNKNOWN; /* temp init ATM device, set to 128kbit */ diff --git a/drivers/usb/atm/xusbatm.c b/drivers/usb/atm/xusbatm.c index 0befbf63d1cc8..5c1e1f5215555 100644 --- a/drivers/usb/atm/xusbatm.c +++ b/drivers/usb/atm/xusbatm.c @@ -79,7 +79,7 @@ static int xusbatm_bind(struct usbatm_data *usbatm, struct usb_interface *intf, const struct usb_device_id *id) { struct usb_device *usb_dev = interface_to_usbdev(intf); - int drv_ix = id - xusbatm_usb_ids; + int drv_ix = id->driver_info; int rx_alt = rx_altsetting[drv_ix]; int tx_alt = tx_altsetting[drv_ix]; struct usb_interface *rx_intf = xusbatm_find_intf(usb_dev, rx_alt, rx_endpoint[drv_ix]); @@ -168,7 +168,8 @@ static struct usb_driver xusbatm_usb_driver = { .name = xusbatm_driver_name, .probe = xusbatm_usb_probe, .disconnect = usbatm_usb_disconnect, - .id_table = xusbatm_usb_ids + .id_table = xusbatm_usb_ids, + .no_dynamic_id = 1, }; static int __init xusbatm_init(void) @@ -190,6 +191,7 @@ static int __init xusbatm_init(void) xusbatm_usb_ids[i].match_flags = USB_DEVICE_ID_MATCH_DEVICE; xusbatm_usb_ids[i].idVendor = vendor[i]; xusbatm_usb_ids[i].idProduct = product[i]; + xusbatm_usb_ids[i].driver_info = i; xusbatm_drivers[i].driver_name = xusbatm_driver_name; xusbatm_drivers[i].bind = xusbatm_bind; diff --git a/drivers/usb/c67x00/c67x00-sched.c b/drivers/usb/c67x00/c67x00-sched.c index a832f5696f2a9..ae6c94f41cb3c 100644 --- a/drivers/usb/c67x00/c67x00-sched.c +++ b/drivers/usb/c67x00/c67x00-sched.c @@ -761,13 +761,13 @@ static int c67x00_add_iso_urb(struct c67x00_hcd *c67x00, struct urb *urb) ret); urb->iso_frame_desc[urbp->cnt].actual_length = 0; urb->iso_frame_desc[urbp->cnt].status = ret; - if (urbp->cnt + 1 == urb->number_of_packets) - c67x00_giveback_urb(c67x00, urb, 0); } urbp->ep_data->next_frame = frame_add(urbp->ep_data->next_frame, urb->interval); urbp->cnt++; + if (ret && urbp->cnt == urb->number_of_packets) + c67x00_giveback_urb(c67x00, urb, 0); } return 0; } diff --git a/drivers/usb/cdns3/cdnsp-gadget.c b/drivers/usb/cdns3/cdnsp-gadget.c index 7a516e509198b..e84405352caab 100644 --- a/drivers/usb/cdns3/cdnsp-gadget.c +++ b/drivers/usb/cdns3/cdnsp-gadget.c @@ -1338,7 +1338,6 @@ static int cdnsp_run(struct cdnsp_device *pdev, cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); - ret = cdnsp_start(pdev); if (ret) { ret = -ENODEV; @@ -1837,6 +1836,82 @@ static void cdnsp_get_rev_cap(struct cdnsp_device *pdev) readl(&pdev->rev_cap->tx_buff_size)); } +static void cdnsp_set_event_deq(struct cdnsp_device *pdev) +{ + dma_addr_t deq; + u64 temp; + + deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg, + pdev->event_ring->dequeue); + + /* Update controller event ring dequeue pointer */ + temp = cdnsp_read_64(&pdev->ir_set->erst_dequeue); + temp &= ERST_PTR_MASK; + + /* + * Don't clear the EHB bit (which is RW1C) because + * there might be more events to service. + */ + temp &= ~ERST_EHB; + + cdnsp_write_64(((u64)deq & (u64)~ERST_PTR_MASK) | temp, + &pdev->ir_set->erst_dequeue); +} + +static void cdnsp_add_interrupter(struct cdnsp_device *pdev) +{ + u64 erst_base; + u32 erst_size; + + /* Set ERST count with the number of entries in the segment table. */ + erst_size = readl(&pdev->ir_set->erst_size); + erst_size &= ERST_SIZE_MASK; + erst_size |= ERST_NUM_SEGS; + writel(erst_size, &pdev->ir_set->erst_size); + + /* Set the segment table base address. */ + erst_base = cdnsp_read_64(&pdev->ir_set->erst_base); + erst_base &= ERST_PTR_MASK; + erst_base |= (pdev->erst.erst_dma_addr & (u64)~ERST_PTR_MASK); + cdnsp_write_64(erst_base, &pdev->ir_set->erst_base); + + /* Set the event ring dequeue address. */ + cdnsp_set_event_deq(pdev); +} + +/* Set up basic CDNSP registers */ +static void cdnsp_init(struct cdnsp_device *pdev) +{ + unsigned int val; + u64 val_64; + + val = readl(&pdev->op_regs->config_reg); + val |= ((val & ~MAX_DEVS) | CDNSP_DEV_MAX_SLOTS) | CONFIG_U3E; + writel(val, &pdev->op_regs->config_reg); + + /* Initialize the Command ring */ + cdnsp_ring_init(pdev, pdev->cmd_ring); + + /* Set the address in the Command Ring Control register */ + val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring); + val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) | + (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) | + pdev->cmd_ring->cycle_state; + cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring); + + /* Set Device Context Base Address Array pointer */ + cdnsp_write_64(pdev->dcbaa->dma, &pdev->op_regs->dcbaa_ptr); + + /* Set Doorbell array pointer */ + val = readl(&pdev->cap_regs->db_off); + val &= DBOFF_MASK; + pdev->dba = (void __iomem *)pdev->cap_regs + val; + + /* Initialize the Primary interrupter */ + cdnsp_ring_init(pdev, pdev->event_ring); + cdnsp_add_interrupter(pdev); +} + static int cdnsp_gen_setup(struct cdnsp_device *pdev) { int ret; @@ -1902,6 +1977,8 @@ static int cdnsp_gen_setup(struct cdnsp_device *pdev) if (ret) return ret; + cdnsp_init(pdev); + /* * Software workaround for U1: after transition * to U1 the controller starts gating clock, and in some cases, @@ -2031,9 +2108,6 @@ static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup) struct cdnsp_device *pdev = cdns->gadget_dev; unsigned long flags; - if (pdev->link_state == XDEV_U3) - return 0; - spin_lock_irqsave(&pdev->lock, flags); cdnsp_disconnect_gadget(pdev); cdnsp_stop(pdev); @@ -2047,12 +2121,38 @@ static int cdnsp_gadget_resume(struct cdns *cdns, bool lost_power) struct cdnsp_device *pdev = cdns->gadget_dev; enum usb_device_speed max_speed; unsigned long flags; + bool context_lost; + u32 val; int ret; if (!pdev->gadget_driver) return 0; spin_lock_irqsave(&pdev->lock, flags); + val = readl(&pdev->port3x_regs->mode_2); + context_lost = !!(val & CFG_3XPORT_U1_PIPE_CLK_GATE_EN) || lost_power; + + if (context_lost) { + cdnsp_halt(pdev); + cdnsp_set_apb_timeout_value(pdev); + + /* Reset the internal controller memory state and registers. */ + ret = cdnsp_reset(pdev); + if (ret) + goto unlock; + + val = readl(&pdev->port3x_regs->mode_2); + val &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN; + writel(val, &pdev->port3x_regs->mode_2); + + cdnsp_clear_cmd_ring(pdev); + + memset(pdev->event_ring->first_seg->trbs, 0, + sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT)); + + cdnsp_init(pdev); + } + max_speed = pdev->gadget_driver->max_speed; /* Limit speed if necessary. */ @@ -2060,9 +2160,10 @@ static int cdnsp_gadget_resume(struct cdns *cdns, bool lost_power) ret = cdnsp_run(pdev, max_speed); - if (pdev->link_state == XDEV_U3) + if (!context_lost && pdev->link_state == XDEV_U3) __cdnsp_gadget_wakeup(pdev); +unlock: spin_unlock_irqrestore(&pdev->lock, flags); return ret; diff --git a/drivers/usb/cdns3/cdnsp-gadget.h b/drivers/usb/cdns3/cdnsp-gadget.h index c44bca348a419..c3ae5040f9ccc 100644 --- a/drivers/usb/cdns3/cdnsp-gadget.h +++ b/drivers/usb/cdns3/cdnsp-gadget.h @@ -1510,6 +1510,7 @@ int cdnsp_endpoint_init(struct cdnsp_device *pdev, int cdnsp_ring_expansion(struct cdnsp_device *pdev, struct cdnsp_ring *ring, unsigned int num_trbs, gfp_t flags); +void cdnsp_ring_init(struct cdnsp_device *pdev, struct cdnsp_ring *ring); struct cdnsp_ring *cdnsp_dma_to_transfer_ring(struct cdnsp_ep *ep, u64 address); int cdnsp_alloc_stream_info(struct cdnsp_device *pdev, struct cdnsp_ep *pep, diff --git a/drivers/usb/cdns3/cdnsp-mem.c b/drivers/usb/cdns3/cdnsp-mem.c index 83f3384b735df..419309c8439e3 100644 --- a/drivers/usb/cdns3/cdnsp-mem.c +++ b/drivers/usb/cdns3/cdnsp-mem.c @@ -394,13 +394,6 @@ static struct cdnsp_ring *cdnsp_ring_alloc(struct cdnsp_device *pdev, if (ret) goto fail; - /* Only event ring does not use link TRB. */ - if (type != TYPE_EVENT) - ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |= - cpu_to_le32(LINK_TOGGLE); - - cdnsp_initialize_ring_info(ring); - trace_cdnsp_ring_alloc(ring); return ring; fail: kfree(ring); @@ -603,6 +596,7 @@ int cdnsp_alloc_stream_info(struct cdnsp_device *pdev, if (!cur_ring) goto cleanup_rings; + cdnsp_ring_init(pdev, cur_ring); cur_ring->stream_id = cur_stream; cur_ring->trb_address_map = &stream_info->trb_address_map; @@ -698,6 +692,8 @@ static int cdnsp_alloc_priv_device(struct cdnsp_device *pdev) if (!pdev->eps[0].ring) goto fail; + cdnsp_ring_init(pdev, pdev->eps[0].ring); + /* Point to output device context in dcbaa. */ pdev->dcbaa->dev_context_ptrs[1] = cpu_to_le64(pdev->out_ctx.dma); pdev->cmd.in_ctx = &pdev->in_ctx; @@ -991,6 +987,8 @@ int cdnsp_endpoint_init(struct cdnsp_device *pdev, if (!pep->ring) return -ENOMEM; + cdnsp_ring_init(pdev, pep->ring); + pep->skip = false; /* Fill the endpoint context */ @@ -1096,28 +1094,6 @@ void cdnsp_mem_cleanup(struct cdnsp_device *pdev) pdev->active_port = NULL; } -static void cdnsp_set_event_deq(struct cdnsp_device *pdev) -{ - dma_addr_t deq; - u64 temp; - - deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg, - pdev->event_ring->dequeue); - - /* Update controller event ring dequeue pointer */ - temp = cdnsp_read_64(&pdev->ir_set->erst_dequeue); - temp &= ERST_PTR_MASK; - - /* - * Don't clear the EHB bit (which is RW1C) because - * there might be more events to service. - */ - temp &= ~ERST_EHB; - - cdnsp_write_64(((u64)deq & (u64)~ERST_PTR_MASK) | temp, - &pdev->ir_set->erst_dequeue); -} - static void cdnsp_add_in_port(struct cdnsp_device *pdev, struct cdnsp_port *port, __le32 __iomem *addr) @@ -1226,6 +1202,36 @@ static int cdnsp_setup_port_arrays(struct cdnsp_device *pdev) return 0; } +static void cdnsp_initialize_ring_segments(struct cdnsp_device *pdev, struct cdnsp_ring *ring) +{ + struct cdnsp_segment *seg; + + /* Only event ring does not use link TRB. */ + if (ring->type == TYPE_EVENT) + return; + + seg = ring->first_seg; + + while (seg) { + struct cdnsp_segment *next = seg->next; + + cdnsp_link_segments(pdev, seg, next, ring->type); + if (next == ring->first_seg) + break; + + seg = next; + } + + ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |= cpu_to_le32(LINK_TOGGLE); +} + +void cdnsp_ring_init(struct cdnsp_device *pdev, struct cdnsp_ring *ring) +{ + cdnsp_initialize_ring_segments(pdev, ring); + cdnsp_initialize_ring_info(ring); + trace_cdnsp_ring_alloc(ring); +} + /* * Initialize memory for CDNSP (one-time init). * @@ -1237,10 +1243,8 @@ int cdnsp_mem_init(struct cdnsp_device *pdev) { struct device *dev = pdev->dev; int ret = -ENOMEM; - unsigned int val; dma_addr_t dma; u32 page_size; - u64 val_64; /* * Use 4K pages, since that's common and the minimum the @@ -1248,10 +1252,6 @@ int cdnsp_mem_init(struct cdnsp_device *pdev) */ page_size = 1 << 12; - val = readl(&pdev->op_regs->config_reg); - val |= ((val & ~MAX_DEVS) | CDNSP_DEV_MAX_SLOTS) | CONFIG_U3E; - writel(val, &pdev->op_regs->config_reg); - /* * Doorbell array must be physically contiguous * and 64-byte (cache line) aligned. @@ -1263,8 +1263,6 @@ int cdnsp_mem_init(struct cdnsp_device *pdev) pdev->dcbaa->dma = dma; - cdnsp_write_64(dma, &pdev->op_regs->dcbaa_ptr); - /* * Initialize the ring segment pool. The ring must be a contiguous * structure comprised of TRBs. The TRBs must be 16 byte aligned, @@ -1290,17 +1288,6 @@ int cdnsp_mem_init(struct cdnsp_device *pdev) if (!pdev->cmd_ring) goto destroy_device_pool; - /* Set the address in the Command Ring Control register */ - val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring); - val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) | - (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) | - pdev->cmd_ring->cycle_state; - cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring); - - val = readl(&pdev->cap_regs->db_off); - val &= DBOFF_MASK; - pdev->dba = (void __iomem *)pdev->cap_regs + val; - /* Set ir_set to interrupt register set 0 */ pdev->ir_set = &pdev->run_regs->ir_set[0]; @@ -1317,21 +1304,6 @@ int cdnsp_mem_init(struct cdnsp_device *pdev) if (ret) goto free_event_ring; - /* Set ERST count with the number of entries in the segment table. */ - val = readl(&pdev->ir_set->erst_size); - val &= ERST_SIZE_MASK; - val |= ERST_NUM_SEGS; - writel(val, &pdev->ir_set->erst_size); - - /* Set the segment table base address. */ - val_64 = cdnsp_read_64(&pdev->ir_set->erst_base); - val_64 &= ERST_PTR_MASK; - val_64 |= (pdev->erst.erst_dma_addr & (u64)~ERST_PTR_MASK); - cdnsp_write_64(val_64, &pdev->ir_set->erst_base); - - /* Set the event ring dequeue address. */ - cdnsp_set_event_deq(pdev); - ret = cdnsp_setup_port_arrays(pdev); if (ret) goto free_erst; diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c index 56d2ba824a0b2..282314eea7fc1 100644 --- a/drivers/usb/chipidea/ci_hdrc_imx.c +++ b/drivers/usb/chipidea/ci_hdrc_imx.c @@ -528,7 +528,7 @@ static int ci_hdrc_imx_probe(struct platform_device *pdev) if (data->wakeup_irq > 0) { irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s:wakeup", pdata.name); if (!irq_name) { - dev_err_probe(dev, -ENOMEM, "failed to create irq_name\n"); + ret = dev_err_probe(dev, -ENOMEM, "failed to create irq_name\n"); goto err_clk; } diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c index af9ae55dae14e..51cd9320a7366 100644 --- a/drivers/usb/class/usbtmc.c +++ b/drivers/usb/class/usbtmc.c @@ -71,7 +71,6 @@ struct usbtmc_dev_capabilities { * allocated for each USBTMC device in the driver's probe function. */ struct usbtmc_device_data { - const struct usb_device_id *id; struct usb_device *usb_dev; struct usb_interface *intf; struct list_head file_list; @@ -2394,7 +2393,6 @@ static int usbtmc_probe(struct usb_interface *intf, return -ENOMEM; data->intf = intf; - data->id = id; data->usb_dev = usb_get_dev(interface_to_usbdev(intf)); usb_set_intfdata(intf, data); kref_init(&data->kref); diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c index e191934623c73..486ced74405e3 100644 --- a/drivers/usb/core/devio.c +++ b/drivers/usb/core/devio.c @@ -1113,7 +1113,6 @@ static int usbdev_release(struct inode *inode, struct file *file) if (!ps->suspend_allowed) usb_autosuspend_device(dev); usb_unlock_device(dev); - usb_put_dev(dev); put_pid(ps->disc_pid); put_cred(ps->cred); @@ -1122,6 +1121,7 @@ static int usbdev_release(struct inode *inode, struct file *file) free_async(as); as = async_getcompleted(ps); } + usb_put_dev(dev); kfree(ps); return 0; diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index f63004417058e..7f33fe5ba03bd 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -228,14 +228,16 @@ static void usb_free_dynids(struct usb_driver *usb_drv) } static const struct usb_device_id *usb_match_dynamic_id(struct usb_interface *intf, - const struct usb_driver *drv) + const struct usb_driver *drv, + struct usb_device_id *id_copy) { struct usb_dynid *dynid; guard(mutex)(&usb_dynids_lock); list_for_each_entry(dynid, &drv->dynids.list, node) { if (usb_match_one_id(intf, &dynid->id)) { - return &dynid->id; + *id_copy = dynid->id; + return id_copy; } } return NULL; @@ -321,6 +323,7 @@ static int usb_probe_interface(struct device *dev) struct usb_interface *intf = to_usb_interface(dev); struct usb_device *udev = interface_to_usbdev(intf); const struct usb_device_id *id; + struct usb_device_id id_copy; int error = -ENODEV; int lpm_disable_error = -ENODEV; @@ -340,7 +343,7 @@ static int usb_probe_interface(struct device *dev) return error; } - id = usb_match_dynamic_id(intf, driver); + id = usb_match_dynamic_id(intf, driver, &id_copy); if (!id) id = usb_match_id(intf, driver->id_table); if (!id) @@ -892,6 +895,7 @@ static int usb_device_match(struct device *dev, const struct device_driver *drv) struct usb_interface *intf; const struct usb_driver *usb_drv; const struct usb_device_id *id; + struct usb_device_id id_copy; /* device drivers never match interfaces */ if (is_usb_device_driver(drv)) @@ -904,7 +908,7 @@ static int usb_device_match(struct device *dev, const struct device_driver *drv) if (id) return 1; - id = usb_match_dynamic_id(intf, usb_drv); + id = usb_match_dynamic_id(intf, usb_drv, &id_copy); if (id) return 1; } diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index d92bf887739d0..068c7bea31020 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -753,10 +753,12 @@ void usb_wakeup_notification(struct usb_device *hdev, { struct usb_hub *hub; struct usb_port *port_dev; + unsigned long flags; if (!hdev) return; + spin_lock_irqsave(&device_state_lock, flags); hub = usb_hub_to_struct_hub(hdev); if (hub) { port_dev = hub->ports[portnum - 1]; @@ -766,6 +768,7 @@ void usb_wakeup_notification(struct usb_device *hdev, set_bit(portnum, hub->wakeup_bits); kick_hub_wq(hub); } + spin_unlock_irqrestore(&device_state_lock, flags); } EXPORT_SYMBOL_GPL(usb_wakeup_notification); @@ -991,10 +994,12 @@ static int hub_hub_status(struct usb_hub *hub, mutex_lock(&hub->status_mutex); ret = get_hub_status(hub->hdev, &hub->status->hub); - if (ret < 0) { + if (ret < (int)sizeof(hub->status->hub)) { if (ret != -ENODEV) dev_err(hub->intfdev, "%s failed (err = %d)\n", __func__, ret); + if (ret >= 0) + ret = -EIO; } else { *status = le16_to_cpu(hub->status->hub.wHubStatus); *change = le16_to_cpu(hub->status->hub.wHubChange); diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index c8b02c27d27d4..102169b404196 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -4680,6 +4680,7 @@ static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on) { struct dwc2_hsotg *hsotg = to_hsotg(gadget); unsigned long flags; + int ret = 0; dev_dbg(hsotg->dev, "%s: is_on: %d op_state: %d\n", __func__, is_on, hsotg->op_state); @@ -4691,6 +4692,13 @@ static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on) } spin_lock_irqsave(&hsotg->lock, flags); + if (hsotg->in_ppd) { + ret = dwc2_exit_partial_power_down(hsotg, 0, true); + if (ret) { + dev_err(hsotg->dev, "exit partial_power_down failed\n"); + goto exit; + } + } if (is_on) { hsotg->enabled = 1; dwc2_hsotg_core_init_disconnected(hsotg, false); @@ -4704,9 +4712,10 @@ static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on) } hsotg->gadget.speed = USB_SPEED_UNKNOWN; +exit: spin_unlock_irqrestore(&hsotg->lock, flags); - return 0; + return ret; } static int dwc2_hsotg_vbus_session(struct usb_gadget *gadget, int is_active) diff --git a/drivers/usb/dwc3/dwc3-google.c b/drivers/usb/dwc3/dwc3-google.c index 60ee4cc99b286..a01ca23cb6a8a 100644 --- a/drivers/usb/dwc3/dwc3-google.c +++ b/drivers/usb/dwc3/dwc3-google.c @@ -442,6 +442,7 @@ static int dwc3_google_probe(struct platform_device *pdev) probe_data.dwc = &google->dwc; probe_data.res = res; probe_data.ignore_clocks_and_resets = true; + probe_data.properties = DWC3_DEFAULT_PROPERTIES; ret = dwc3_core_probe(&probe_data); if (ret) { ret = dev_err_probe(dev, ret, "failed to register DWC3 Core\n"); diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index bfe616194dfad..310b5ffb236aa 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -304,7 +304,7 @@ void dwc3_ep0_out_start(struct dwc3 *dwc) dwc3_ep->flags &= ~DWC3_EP_DELAY_STOP; if (dwc->connected) - dwc3_stop_active_transfer(dwc3_ep, true, true); + dwc3_stop_active_transfer(dwc3_ep, false, true); else dwc3_remove_requests(dwc, dwc3_ep, -ESHUTDOWN); } diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 1082e9c9afaa8..e46db7032089f 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1004,7 +1004,7 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) * controller to generate an ERDY to initiate the * stream. */ - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); /* * All stream eps will reinitiate stream on NoStream @@ -1032,7 +1032,7 @@ void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status) { struct dwc3_request *req; - dwc3_stop_active_transfer(dep, true, false); + dwc3_stop_active_transfer(dep, false, false); /* If endxfer is delayed, avoid unmapping requests */ if (dep->flags & DWC3_EP_DELAY_STOP) @@ -1720,7 +1720,7 @@ static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) if (ret == -EAGAIN) return ret; - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); list_for_each_entry_safe(req, tmp, &dep->started_list, list) dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED); @@ -1757,6 +1757,11 @@ static int __dwc3_gadget_get_frame(struct dwc3 *dwc) * the controller won't update the TRB progress on command * completion. It also won't clear the HWO bit in the TRB. * The command will also not complete immediately in that case. + * + * Older programming guide revisions recommended setting ForceRM to 1 + * when ending a transfer. Newer programming guide revisions now + * recommend keeping ForceRM cleared, and TRBs are properly updated + * on command completion. */ static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) { @@ -1882,7 +1887,7 @@ static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep) * to wait for the next XferNotReady to test the command again */ if (cmd_status == 0) { - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); return 0; } } @@ -2165,7 +2170,7 @@ static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, struct dwc3_request *t; /* wait until it is processed */ - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); /* * Remove any started request if the transfer is @@ -2242,7 +2247,7 @@ int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) return 0; } - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); list_for_each_entry_safe(req, tmp, &dep->started_list, list) dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED); @@ -3357,7 +3362,7 @@ static void dwc3_nostream_work(struct work_struct *work) dwc3_send_gadget_generic_command(dwc, cmd, dep->number); } else { dep->flags |= DWC3_EP_DELAY_START; - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); spin_unlock_irqrestore(&dwc->lock, flags); return; } @@ -3497,6 +3502,7 @@ static void dwc3_gadget_free_endpoints(struct dwc3 *dwc) } dwc3_debugfs_remove_endpoint_dir(dep); + cancel_delayed_work_sync(&dep->nostream_work); kfree(dep); } } @@ -3714,7 +3720,7 @@ static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep, if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && list_empty(&dep->started_list) && (list_empty(&dep->pending_list) || status == -EXDEV)) - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); else if (dwc3_gadget_ep_should_continue(dep)) if (__dwc3_gadget_kick_transfer(dep) == 0) no_started_trb = false; diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index 183a25f65ac89..dd6d6b11199e0 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -1177,7 +1177,7 @@ static ssize_t os_desc_qw_sign_show(struct config_item *item, char *page) struct gadget_info *gi = os_desc_item_to_gadget_info(item); int res; - res = utf16s_to_utf8s((wchar_t *) gi->qw_sign, OS_STRING_QW_SIGN_LEN, + res = utf16s_to_utf8s((wchar_t *) gi->qw_sign, OS_STRING_QW_SIGN_LEN / 2, UTF16_LITTLE_ENDIAN, page, PAGE_SIZE - 1); page[res++] = '\n'; @@ -1199,7 +1199,7 @@ static ssize_t os_desc_qw_sign_store(struct config_item *item, const char *page, mutex_lock(&gi->lock); res = utf8s_to_utf16s(page, l, UTF16_LITTLE_ENDIAN, (wchar_t *) gi->qw_sign, - OS_STRING_QW_SIGN_LEN); + OS_STRING_QW_SIGN_LEN / 2); if (res > 0) res = len; mutex_unlock(&gi->lock); diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 44218be1e6768..563390f05f660 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -548,6 +548,7 @@ static ssize_t ffs_ep0_read(struct file *file, char __user *buf, if (ffs_setup_state_clear_cancelled(ffs) == FFS_SETUP_CANCELLED) return -EIDRM; +retry: /* Acquire mutex */ ret = ffs_mutex_lock(&ffs->mutex, file->f_flags & O_NONBLOCK); if (ret < 0) @@ -582,10 +583,15 @@ static ssize_t ffs_ep0_read(struct file *file, char __user *buf, break; } - if (wait_event_interruptible_exclusive_locked_irq(ffs->ev.waitq, - ffs->ev.count)) { - ret = -EINTR; - break; + if (!ffs->ev.count) { + spin_unlock_irq(&ffs->ev.waitq.lock); + mutex_unlock(&ffs->mutex); + + if (wait_event_interruptible_exclusive(ffs->ev.waitq, + ffs->ev.count)) + return -EINTR; + + goto retry; } /* unlocks spinlock */ @@ -867,9 +873,15 @@ static void ffs_user_copy_worker(struct work_struct *work) bool kiocb_has_eventfd = io_data->kiocb->ki_flags & IOCB_EVENTFD; if (io_data->read && ret > 0) { - kthread_use_mm(io_data->mm); - ret = ffs_copy_to_iter(io_data->buf, ret, &io_data->data); - kthread_unuse_mm(io_data->mm); + if (mmget_not_zero(io_data->mm)) { + kthread_use_mm(io_data->mm); + ret = ffs_copy_to_iter(io_data->buf, ret, &io_data->data); + kthread_unuse_mm(io_data->mm); + mmput(io_data->mm); + } else { + ret = -EFAULT; + } + mmdrop(io_data->mm); } io_data->kiocb->ki_complete(io_data->kiocb, ret); @@ -1264,16 +1276,22 @@ static ssize_t ffs_epfile_write_iter(struct kiocb *kiocb, struct iov_iter *from) kiocb->private = p; - if (p->aio) + if (p->aio) { + mmgrab(p->mm); kiocb_set_cancel_fn(kiocb, ffs_aio_cancel); + } res = ffs_epfile_io(kiocb->ki_filp, p); if (res == -EIOCBQUEUED) return res; - if (p->aio) + if (p->aio) { + kiocb->ki_complete(kiocb, res); + mmdrop(p->mm); kfree(p); - else + return -EIOCBQUEUED; + } else { *from = p->data; + } return res; } @@ -1308,16 +1326,21 @@ static ssize_t ffs_epfile_read_iter(struct kiocb *kiocb, struct iov_iter *to) kiocb->private = p; - if (p->aio) + if (p->aio) { + mmgrab(p->mm); kiocb_set_cancel_fn(kiocb, ffs_aio_cancel); + } res = ffs_epfile_io(kiocb->ki_filp, p); if (res == -EIOCBQUEUED) return res; if (p->aio) { + kiocb->ki_complete(kiocb, res); + mmdrop(p->mm); kfree(p->to_free); kfree(p); + return -EIOCBQUEUED; } else { *to = p->data; } @@ -1682,13 +1705,13 @@ static int ffs_dmabuf_transfer(struct file *file, /* In the meantime, endpoint got disabled or changed. */ if (epfile->ep != ep) { ret = -ESHUTDOWN; - goto err_fence_put; + goto err_fence_free; } usb_req = usb_ep_alloc_request(ep->ep, GFP_ATOMIC); if (!usb_req) { ret = -ENOMEM; - goto err_fence_put; + goto err_fence_free; } /* @@ -1737,9 +1760,9 @@ static int ffs_dmabuf_transfer(struct file *file, return ret; -err_fence_put: +err_fence_free: spin_unlock_irq(&epfile->ffs->eps_lock); - dma_fence_put(&fence->base); + kfree(fence); err_resv_unlock: dma_resv_unlock(dmabuf->resv); err_attachment_put: diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c index b7b06cb79ff5e..8a6ae944d436c 100644 --- a/drivers/usb/gadget/function/f_mass_storage.c +++ b/drivers/usb/gadget/function/f_mass_storage.c @@ -2748,6 +2748,9 @@ int fsg_common_set_num_buffers(struct fsg_common *common, unsigned int n) struct fsg_buffhd *bh, *buffhds; int i; + if (n < 2) + return -EINVAL; + buffhds = kzalloc_objs(*buffhds, n); if (!buffhds) return -ENOMEM; @@ -2961,7 +2964,7 @@ EXPORT_SYMBOL_GPL(fsg_common_create_lun); int fsg_common_create_luns(struct fsg_common *common, struct fsg_config *cfg) { - char buf[8]; /* enough for 100000000 different numbers, decimal */ + char buf[14]; int i, rc; fsg_common_remove_luns(common); diff --git a/drivers/usb/gadget/function/f_midi.c b/drivers/usb/gadget/function/f_midi.c index fba8cf787d6c1..63fb6ee70a3d6 100644 --- a/drivers/usb/gadget/function/f_midi.c +++ b/drivers/usb/gadget/function/f_midi.c @@ -879,7 +879,6 @@ static int f_midi_bind(struct usb_configuration *c, struct usb_function *f) int status, n, jack = 1, i = 0, endpoint_descriptor_index = 0; midi->gadget = cdev->gadget; - INIT_WORK(&midi->work, f_midi_in_work); status = f_midi_register_card(midi); if (status < 0) goto fail_register; @@ -1377,6 +1376,7 @@ static struct usb_function *f_midi_alloc(struct usb_function_instance *fi) status = -ENOMEM; goto midi_free; } + INIT_WORK(&midi->work, f_midi_in_work); midi->out_ports = opts->out_ports; midi->index = opts->index; midi->buflen = opts->buflen; diff --git a/drivers/usb/gadget/function/f_midi2.c b/drivers/usb/gadget/function/f_midi2.c index 19fdac0243439..5b8b18281989c 100644 --- a/drivers/usb/gadget/function/f_midi2.c +++ b/drivers/usb/gadget/function/f_midi2.c @@ -1145,7 +1145,7 @@ static int f_midi2_alloc_ep_reqs(struct f_midi2_usb_ep *usb_ep) if (!usb_ep->reqs) return -EINVAL; - for (i = 0; i < midi2->info.num_reqs; i++) { + for (i = 0; i < usb_ep->num_reqs; i++) { if (usb_ep->reqs[i].req) continue; usb_ep->reqs[i].req = alloc_ep_req(usb_ep->usb_ep, @@ -1160,10 +1160,9 @@ static int f_midi2_alloc_ep_reqs(struct f_midi2_usb_ep *usb_ep) /* Free allocated requests */ static void f_midi2_free_ep_reqs(struct f_midi2_usb_ep *usb_ep) { - struct f_midi2 *midi2 = usb_ep->card; int i; - for (i = 0; i < midi2->info.num_reqs; i++) { + for (i = 0; i < usb_ep->num_reqs; i++) { if (!usb_ep->reqs[i].req) continue; free_ep_req(usb_ep->usb_ep, usb_ep->reqs[i].req); @@ -2178,13 +2177,13 @@ static ssize_t f_midi2_opts_bool_store(struct f_midi2_opts *opts, /* generic show/store for string */ static ssize_t f_midi2_opts_str_show(struct f_midi2_opts *opts, - const char *str, char *page) + const char **strp, char *page) { int result = 0; mutex_lock(&opts->lock); - if (str) - result = scnprintf(page, PAGE_SIZE, "%s\n", str); + if (*strp) + result = scnprintf(page, PAGE_SIZE, "%s\n", *strp); mutex_unlock(&opts->lock); return result; } @@ -2278,7 +2277,7 @@ static ssize_t f_midi2_block_opts_name_show(struct config_item *item, { struct f_midi2_block_opts *opts = to_f_midi2_block_opts(item); - return f_midi2_opts_str_show(opts->ep->opts, opts->info.name, page); + return f_midi2_opts_str_show(opts->ep->opts, &opts->info.name, page); } static ssize_t f_midi2_block_opts_name_store(struct config_item *item, @@ -2435,7 +2434,7 @@ static ssize_t f_midi2_ep_opts_##name##_show(struct config_item *item, \ char *page) \ { \ struct f_midi2_ep_opts *opts = to_f_midi2_ep_opts(item); \ - return f_midi2_opts_str_show(opts->opts, opts->info.name, page);\ + return f_midi2_opts_str_show(opts->opts, &opts->info.name, page);\ } \ \ static ssize_t f_midi2_ep_opts_##name##_store(struct config_item *item, \ @@ -2473,6 +2472,7 @@ static void f_midi2_ep_opts_release(struct config_item *item) { struct f_midi2_ep_opts *opts = to_f_midi2_ep_opts(item); + configfs_remove_default_groups(&opts->group); kfree(opts->info.ep_name); kfree(opts->info.product_id); kfree(opts); @@ -2589,7 +2589,7 @@ static ssize_t f_midi2_opts_iface_name_show(struct config_item *item, { struct f_midi2_opts *opts = to_f_midi2_opts(item); - return f_midi2_opts_str_show(opts, opts->info.iface_name, page); + return f_midi2_opts_str_show(opts, &opts->info.iface_name, page); } static ssize_t f_midi2_opts_iface_name_store(struct config_item *item, @@ -2639,6 +2639,7 @@ static void f_midi2_free_inst(struct usb_function_instance *f) opts = container_of(f, struct f_midi2_opts, func_inst); + configfs_remove_default_groups(&opts->func_inst.group); kfree(opts->info.iface_name); kfree(opts); } diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c index b3fa5a17fd2db..9e6d4f39900a2 100644 --- a/drivers/usb/gadget/function/f_tcm.c +++ b/drivers/usb/gadget/function/f_tcm.c @@ -1675,19 +1675,25 @@ static struct se_portal_group *usbg_make_tpg(struct se_wwn *wwn, opts = container_of(tpg_instances[i].func_inst, struct f_tcm_opts, func_inst); - mutex_lock(&opts->dep_lock); - if (!opts->ready) - goto unlock_dep; + if (!READ_ONCE(opts->ready)) + goto unlock_inst; if (opts->has_dep) { if (!try_module_get(opts->dependent)) - goto unlock_dep; + goto unlock_inst; } else { + /* + * configfs_depend_item_unlocked() may acquire the configfs + * root inode lock when the target belongs to a different + * subsystem. Calling it under dep_lock would create a + * circular dependency: + * dep_lock -> configfs inode lock -> su_mutex -> dep_lock + */ ret = configfs_depend_item_unlocked( wwn->wwn_group.cg_subsys, &opts->func_inst.group.cg_item); if (ret) - goto unlock_dep; + goto unlock_inst; } tpg = kzalloc_obj(struct usbg_tpg); @@ -1714,7 +1720,6 @@ static struct se_portal_group *usbg_make_tpg(struct se_wwn *wwn, tpg_instances[i].tpg = tpg; tpg->fi = tpg_instances[i].func_inst; - mutex_unlock(&opts->dep_lock); mutex_unlock(&tpg_instances_lock); return &tpg->se_tpg; @@ -1727,8 +1732,6 @@ static struct se_portal_group *usbg_make_tpg(struct se_wwn *wwn, module_put(opts->dependent); else configfs_undepend_item_unlocked(&opts->func_inst.group.cg_item); -unlock_dep: - mutex_unlock(&opts->dep_lock); unlock_inst: mutex_unlock(&tpg_instances_lock); @@ -2024,7 +2027,7 @@ static const struct target_core_fabric_ops usbg_ops = { .fabric_enable_tpg = usbg_enable_tpg, .fabric_drop_tpg = usbg_drop_tpg, .fabric_post_link = usbg_port_link, - .fabric_pre_unlink = usbg_port_unlink, + .fabric_post_unlink = usbg_port_unlink, .fabric_init_nodeacl = usbg_init_nodeacl, .tfc_wwn_attrs = usbg_wwn_attrs, @@ -2666,9 +2669,7 @@ static int tcm_set_name(struct usb_function_instance *f, const char *name) pr_debug("tcm: Activating %s\n", name); - mutex_lock(&opts->dep_lock); - opts->ready = true; - mutex_unlock(&opts->dep_lock); + WRITE_ONCE(opts->ready, true); return 0; } diff --git a/drivers/usb/gadget/function/f_uac1.c b/drivers/usb/gadget/function/f_uac1.c index 85c502e98f577..7a81cd176abd3 100644 --- a/drivers/usb/gadget/function/f_uac1.c +++ b/drivers/usb/gadget/function/f_uac1.c @@ -1594,7 +1594,8 @@ static ssize_t f_uac1_opts_##name##_store(struct config_item *item, \ const char *page, size_t len) \ { \ struct f_uac1_opts *opts = to_f_uac1_opts(item); \ - char *split_page = NULL; \ + char *buf = NULL; \ + char *split_page; \ int ret = -EINVAL; \ char *token; \ u32 num; \ @@ -1608,18 +1609,22 @@ static ssize_t f_uac1_opts_##name##_store(struct config_item *item, \ \ i = 0; \ memset(opts->name##s, 0x00, sizeof(opts->name##s)); \ - split_page = kstrdup(page, GFP_KERNEL); \ + buf = kstrdup(page, GFP_KERNEL); \ + split_page = buf; \ while ((token = strsep(&split_page, ",")) != NULL) { \ ret = kstrtou32(token, 0, &num); \ if (ret) \ goto end; \ - \ + if (i >= UAC_MAX_RATES) { \ + ret = -EINVAL; \ + goto end; \ + } \ opts->name##s[i++] = num; \ ret = len; \ }; \ \ end: \ - kfree(split_page); \ + kfree(buf); \ mutex_unlock(&opts->lock); \ return ret; \ } \ diff --git a/drivers/usb/gadget/function/f_uac1_legacy.c b/drivers/usb/gadget/function/f_uac1_legacy.c index 5d201a2e30e7f..3f52099a4fdd5 100644 --- a/drivers/usb/gadget/function/f_uac1_legacy.c +++ b/drivers/usb/gadget/function/f_uac1_legacy.c @@ -888,60 +888,10 @@ UAC1_INT_ATTRIBUTE(req_buf_size); UAC1_INT_ATTRIBUTE(req_count); UAC1_INT_ATTRIBUTE(audio_buf_size); -#define UAC1_STR_ATTRIBUTE(name) \ -static ssize_t f_uac1_opts_##name##_show(struct config_item *item, \ - char *page) \ -{ \ - struct f_uac1_legacy_opts *opts = to_f_uac1_opts(item); \ - int result; \ - \ - mutex_lock(&opts->lock); \ - result = sprintf(page, "%s\n", opts->name); \ - mutex_unlock(&opts->lock); \ - \ - return result; \ -} \ - \ -static ssize_t f_uac1_opts_##name##_store(struct config_item *item, \ - const char *page, size_t len) \ -{ \ - struct f_uac1_legacy_opts *opts = to_f_uac1_opts(item); \ - int ret = -EBUSY; \ - char *tmp; \ - \ - mutex_lock(&opts->lock); \ - if (opts->refcnt) \ - goto end; \ - \ - tmp = kstrndup(page, len, GFP_KERNEL); \ - if (tmp) { \ - ret = -ENOMEM; \ - goto end; \ - } \ - if (opts->name##_alloc) \ - kfree(opts->name); \ - opts->name##_alloc = true; \ - opts->name = tmp; \ - ret = len; \ - \ -end: \ - mutex_unlock(&opts->lock); \ - return ret; \ -} \ - \ -CONFIGFS_ATTR(f_uac1_opts_, name) - -UAC1_STR_ATTRIBUTE(fn_play); -UAC1_STR_ATTRIBUTE(fn_cap); -UAC1_STR_ATTRIBUTE(fn_cntl); - static struct configfs_attribute *f_uac1_attrs[] = { &f_uac1_opts_attr_req_buf_size, &f_uac1_opts_attr_req_count, &f_uac1_opts_attr_audio_buf_size, - &f_uac1_opts_attr_fn_play, - &f_uac1_opts_attr_fn_cap, - &f_uac1_opts_attr_fn_cntl, NULL, }; @@ -956,12 +906,6 @@ static void f_audio_free_inst(struct usb_function_instance *f) struct f_uac1_legacy_opts *opts; opts = container_of(f, struct f_uac1_legacy_opts, func_inst); - if (opts->fn_play_alloc) - kfree(opts->fn_play); - if (opts->fn_cap_alloc) - kfree(opts->fn_cap); - if (opts->fn_cntl_alloc) - kfree(opts->fn_cntl); kfree(opts); } diff --git a/drivers/usb/gadget/function/f_uac2.c b/drivers/usb/gadget/function/f_uac2.c index 897787d0803c1..d8cf710085a05 100644 --- a/drivers/usb/gadget/function/f_uac2.c +++ b/drivers/usb/gadget/function/f_uac2.c @@ -2012,7 +2012,8 @@ static ssize_t f_uac2_opts_##name##_store(struct config_item *item, \ const char *page, size_t len) \ { \ struct f_uac2_opts *opts = to_f_uac2_opts(item); \ - char *split_page = NULL; \ + char *buf = NULL; \ + char *split_page; \ int ret = -EINVAL; \ char *token; \ u32 num; \ @@ -2026,18 +2027,22 @@ static ssize_t f_uac2_opts_##name##_store(struct config_item *item, \ \ i = 0; \ memset(opts->name##s, 0x00, sizeof(opts->name##s)); \ - split_page = kstrdup(page, GFP_KERNEL); \ + buf = kstrdup(page, GFP_KERNEL); \ + split_page = buf; \ while ((token = strsep(&split_page, ",")) != NULL) { \ ret = kstrtou32(token, 0, &num); \ if (ret) \ goto end; \ - \ + if (i >= UAC_MAX_RATES) { \ + ret = -EINVAL; \ + goto end; \ + } \ opts->name##s[i++] = num; \ ret = len; \ }; \ \ end: \ - kfree(split_page); \ + kfree(buf); \ mutex_unlock(&opts->lock); \ return ret; \ } \ diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c index 73dc7e42875ff..d1bf3ea751973 100644 --- a/drivers/usb/gadget/function/f_uvc.c +++ b/drivers/usb/gadget/function/f_uvc.c @@ -889,9 +889,12 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f) v4l2_error: v4l2_device_unregister(&uvc->v4l2_dev); error: - if (uvc->control_req) + if (uvc->control_req) { usb_ep_free_request(cdev->gadget->ep0, uvc->control_req); + uvc->control_req = NULL; + } kfree(uvc->control_buf); + uvc->control_buf = NULL; usb_free_all_descriptors(f); return ret; @@ -1075,7 +1078,9 @@ static void uvc_function_unbind(struct usb_configuration *c, uvc->vdev_release_done = NULL; usb_ep_free_request(cdev->gadget->ep0, uvc->control_req); + uvc->control_req = NULL; kfree(uvc->control_buf); + uvc->control_buf = NULL; usb_free_all_descriptors(f); } diff --git a/drivers/usb/gadget/function/u_audio.c b/drivers/usb/gadget/function/u_audio.c index e53f2927b5398..391419c208d45 100644 --- a/drivers/usb/gadget/function/u_audio.c +++ b/drivers/usb/gadget/function/u_audio.c @@ -1177,6 +1177,20 @@ static struct snd_kcontrol_new u_audio_controls[] = { }, }; +static void u_audio_card_free(struct snd_card *card) +{ + struct snd_uac_chip *uac = card->private_data; + + if (!uac) + return; + + kfree(uac->p_prm.reqs); + kfree(uac->c_prm.reqs); + kfree(uac->p_prm.rbuf); + kfree(uac->c_prm.rbuf); + kfree(uac); +} + int g_audio_setup(struct g_audio *g_audio, const char *pcm_name, const char *card_name) { @@ -1256,6 +1270,8 @@ int g_audio_setup(struct g_audio *g_audio, const char *pcm_name, goto fail; uac->card = card; + card->private_data = uac; + card->private_free = u_audio_card_free; /* * Create first PCM device @@ -1424,6 +1440,8 @@ int g_audio_setup(struct g_audio *g_audio, const char *pcm_name, snd_fail: snd_card_free(card); + return err; + fail: kfree(uac->p_prm.reqs); kfree(uac->c_prm.reqs); @@ -1449,12 +1467,6 @@ void g_audio_cleanup(struct g_audio *g_audio) card = uac->card; if (card) snd_card_free_when_closed(card); - - kfree(uac->p_prm.reqs); - kfree(uac->c_prm.reqs); - kfree(uac->p_prm.rbuf); - kfree(uac->c_prm.rbuf); - kfree(uac); } EXPORT_SYMBOL_GPL(g_audio_cleanup); diff --git a/drivers/usb/gadget/function/u_uac1_legacy.h b/drivers/usb/gadget/function/u_uac1_legacy.h index b5df9bcbbeba7..b9ddae550ff3c 100644 --- a/drivers/usb/gadget/function/u_uac1_legacy.h +++ b/drivers/usb/gadget/function/u_uac1_legacy.h @@ -62,9 +62,6 @@ struct f_uac1_legacy_opts { char *fn_cap; char *fn_cntl; unsigned bound:1; - unsigned fn_play_alloc:1; - unsigned fn_cap_alloc:1; - unsigned fn_cntl_alloc:1; struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index 2f9700b3f1b64..9ba09118bb74a 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -821,7 +821,7 @@ int uvcg_video_init(struct uvc_video *video, struct uvc_device *uvc) /* Allocate a kthread for asynchronous hw submit handler. */ video->kworker = kthread_run_worker(0, "UVCG"); if (IS_ERR(video->kworker)) { - uvcg_err(&video->uvc->func, "failed to create UVCG kworker\n"); + uvcg_err(&uvc->func, "failed to create UVCG kworker\n"); return PTR_ERR(video->kworker); } diff --git a/drivers/usb/gadget/functions.c b/drivers/usb/gadget/functions.c index 203361a64212d..70e31c40e2673 100644 --- a/drivers/usb/gadget/functions.c +++ b/drivers/usb/gadget/functions.c @@ -70,7 +70,7 @@ void usb_put_function_instance(struct usb_function_instance *fi) { struct module *mod; - if (!fi) + if (!fi || !fi->fd) return; mod = fi->fd->mod; diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c index d87a8ab515107..12819e8c265d4 100644 --- a/drivers/usb/gadget/legacy/inode.c +++ b/drivers/usb/gadget/legacy/inode.c @@ -1251,10 +1251,11 @@ ep0_poll (struct file *fd, poll_table *wait) static long gadget_dev_ioctl (struct file *fd, unsigned code, unsigned long value) { struct dev_data *dev = fd->private_data; - struct usb_gadget *gadget = dev->gadget; + struct usb_gadget *gadget; long ret = -ENOTTY; spin_lock_irq(&dev->lock); + gadget = dev->gadget; if (dev->state == STATE_DEV_OPENED || dev->state == STATE_DEV_UNBOUND) { /* Not bound to a UDC */ diff --git a/drivers/usb/gadget/udc/aspeed_udc.c b/drivers/usb/gadget/udc/aspeed_udc.c index 75f9c831b21a6..54f81e6680094 100644 --- a/drivers/usb/gadget/udc/aspeed_udc.c +++ b/drivers/usb/gadget/udc/aspeed_udc.c @@ -1431,25 +1431,12 @@ static void ast_udc_init_hw(struct ast_udc_dev *udc) ast_udc_write(udc, 0, AST_UDC_EP0_CTRL); } -static void ast_udc_remove(struct platform_device *pdev) +static void ast_udc_cleanup(struct platform_device *pdev) { struct ast_udc_dev *udc = platform_get_drvdata(pdev); unsigned long flags; u32 ctrl; - usb_del_gadget_udc(&udc->gadget); - if (udc->driver) { - /* - * This is broken as only some cleanup is skipped, *udev is - * freed and the register mapping goes away. Any further usage - * probably crashes. Also the device is unbound, so the skipped - * cleanup is never catched up later. - */ - dev_alert(&pdev->dev, - "Driver is busy and still going away. Fasten your seat belts!\n"); - return; - } - spin_lock_irqsave(&udc->lock, flags); /* Disable upstream port connection */ @@ -1469,6 +1456,26 @@ static void ast_udc_remove(struct platform_device *pdev) udc->ep0_buf = NULL; } +static void ast_udc_remove(struct platform_device *pdev) +{ + struct ast_udc_dev *udc = platform_get_drvdata(pdev); + + usb_del_gadget_udc(&udc->gadget); + if (udc->driver) { + /* + * This is broken as only some cleanup is skipped, *udev is + * freed and the register mapping goes away. Any further usage + * probably crashes. Also the device is unbound, so the skipped + * cleanup is never catched up later. + */ + dev_alert(&pdev->dev, + "Driver is busy and still going away. Fasten your seat belts!\n"); + return; + } + + ast_udc_cleanup(pdev); +} + static int ast_udc_probe(struct platform_device *pdev) { enum usb_device_speed max_speed; @@ -1521,6 +1528,12 @@ static int ast_udc_probe(struct platform_device *pdev) AST_UDC_NUM_ENDPOINTS, &udc->ep0_buf_dma, GFP_KERNEL); + if (!udc->ep0_buf) { + clk_disable_unprepare(udc->clk); + rc = -ENOMEM; + goto err; + } + udc->gadget.speed = USB_SPEED_UNKNOWN; udc->gadget.max_speed = USB_SPEED_HIGH; udc->creq = udc->reg + AST_UDC_SETUP0; @@ -1550,20 +1563,20 @@ static int ast_udc_probe(struct platform_device *pdev) udc->irq = platform_get_irq(pdev, 0); if (udc->irq < 0) { rc = udc->irq; - goto err; + goto err_cleanup; } rc = devm_request_irq(&pdev->dev, udc->irq, ast_udc_isr, 0, KBUILD_MODNAME, udc); if (rc) { dev_err(&pdev->dev, "Failed to request interrupt\n"); - goto err; + goto err_cleanup; } rc = usb_add_gadget_udc(&pdev->dev, &udc->gadget); if (rc) { dev_err(&pdev->dev, "Failed to add gadget udc\n"); - goto err; + goto err_cleanup; } dev_info(&pdev->dev, "Initialized udc in USB%s mode\n", @@ -1571,9 +1584,10 @@ static int ast_udc_probe(struct platform_device *pdev) return 0; +err_cleanup: + ast_udc_cleanup(pdev); err: dev_err(&pdev->dev, "Failed to udc probe, rc:0x%x\n", rc); - ast_udc_remove(pdev); return rc; } diff --git a/drivers/usb/gadget/udc/at91_udc.c b/drivers/usb/gadget/udc/at91_udc.c index 5aa360ba4f039..8e5d7fb715492 100644 --- a/drivers/usb/gadget/udc/at91_udc.c +++ b/drivers/usb/gadget/udc/at91_udc.c @@ -1794,6 +1794,19 @@ static void at91udc_of_init(struct at91_udc *udc, struct device_node *np) udc->caps = match->data; } +/* + * The work handler re-arms this timer, so shut the timer down before + * draining the work; otherwise it restarts the polling cycle. + */ +static void at91_udc_shutdown_vbus_timer(struct at91_udc *udc) +{ + if (!(udc->board.vbus_pin && udc->board.vbus_polled)) + return; + + timer_shutdown_sync(&udc->vbus_timer); + cancel_work_sync(&udc->vbus_timer_work); +} + static int at91udc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1907,7 +1920,7 @@ static int at91udc_probe(struct platform_device *pdev) } retval = usb_add_gadget_udc(dev, &udc->gadget); if (retval) - goto err_unprepare_iclk; + goto err_shutdown_vbus; dev_set_drvdata(dev, udc); device_init_wakeup(dev, 1); create_debug_file(udc); @@ -1915,6 +1928,8 @@ static int at91udc_probe(struct platform_device *pdev) INFO("%s version %s\n", driver_name, DRIVER_VERSION); return 0; +err_shutdown_vbus: + at91_udc_shutdown_vbus_timer(udc); err_unprepare_iclk: clk_unprepare(udc->iclk); err_unprepare_fclk: @@ -1933,6 +1948,9 @@ static void at91udc_remove(struct platform_device *pdev) DBG("remove\n"); usb_del_gadget_udc(&udc->gadget); + + at91_udc_shutdown_vbus_timer(udc); + if (udc->driver) { dev_err(&pdev->dev, "Driver still in use but removing anyhow\n"); diff --git a/drivers/usb/gadget/udc/r8a66597-udc.c b/drivers/usb/gadget/udc/r8a66597-udc.c index e7a5d8553c0ea..d190e16d43fc0 100644 --- a/drivers/usb/gadget/udc/r8a66597-udc.c +++ b/drivers/usb/gadget/udc/r8a66597-udc.c @@ -1951,7 +1951,6 @@ static int r8a66597_probe(struct platform_device *pdev) return 0; err_add_udc: - r8a66597_free_request(&r8a66597->ep[0].ep, r8a66597->ep0_req); clean_up2: if (r8a66597->pdata->on_chip) clk_disable_unprepare(r8a66597->clk); diff --git a/drivers/usb/gadget/udc/snps_udc_plat.c b/drivers/usb/gadget/udc/snps_udc_plat.c index db842a6de643d..0ee3ed185d9c3 100644 --- a/drivers/usb/gadget/udc/snps_udc_plat.c +++ b/drivers/usb/gadget/udc/snps_udc_plat.c @@ -159,10 +159,9 @@ static int udc_plat_probe(struct platform_device *pdev) if (of_property_present(dev->of_node, "extcon")) { udc->edev = extcon_get_edev_by_phandle(dev, 0); if (IS_ERR(udc->edev)) { - if (PTR_ERR(udc->edev) == -EPROBE_DEFER) - return -EPROBE_DEFER; - dev_err(dev, "Invalid or missing extcon\n"); ret = PTR_ERR(udc->edev); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Invalid or missing extcon\n"); goto exit_phy; } diff --git a/drivers/usb/host/xhci-dbgtty.c b/drivers/usb/host/xhci-dbgtty.c index 2e7384c6b6ec2..7e44e62d6ea93 100644 --- a/drivers/usb/host/xhci-dbgtty.c +++ b/drivers/usb/host/xhci-dbgtty.c @@ -628,8 +628,8 @@ int dbc_tty_init(void) dbc_tty_driver = tty_alloc_driver(64, TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV); if (IS_ERR(dbc_tty_driver)) { - idr_destroy(&dbc_tty_minors); - return PTR_ERR(dbc_tty_driver); + ret = PTR_ERR(dbc_tty_driver); + goto fail; } dbc_tty_driver->driver_name = "dbc_serial"; @@ -649,10 +649,17 @@ int dbc_tty_init(void) ret = tty_register_driver(dbc_tty_driver); if (ret) { pr_err("Can't register dbc tty driver\n"); - tty_driver_kref_put(dbc_tty_driver); - idr_destroy(&dbc_tty_minors); + goto fail_put; } + return ret; + +fail_put: + tty_driver_kref_put(dbc_tty_driver); +fail: + idr_destroy(&dbc_tty_minors); + dbc_tty_driver = NULL; + return ret; } diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 4f98d8269625b..8908f5c27e526 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -842,21 +842,18 @@ static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, usb_hcd_giveback_urb(hcd, urb, status); } -static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, - struct xhci_ring *ring, struct xhci_td *td) +static void xhci_unmap_one_bounce_buffer(struct xhci_hcd *xhci, + struct xhci_ring *ring, struct xhci_td *td, + struct xhci_segment *seg) { struct device *dev = xhci_to_hcd(xhci)->self.sysdev; - struct xhci_segment *seg = td->bounce_seg; struct urb *urb = td->urb; size_t len; - if (!ring || !seg || !urb) - return; - if (usb_urb_dir_out(urb)) { dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, DMA_TO_DEVICE); - return; + goto done; } dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, @@ -872,10 +869,29 @@ static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, memcpy(urb->transfer_buffer + seg->bounce_offs, seg->bounce_buf, seg->bounce_len); } +done: seg->bounce_len = 0; seg->bounce_offs = 0; } +static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, + struct xhci_ring *ring, struct xhci_td *td) +{ + struct xhci_segment *seg; + int i = 0; + + if (!td->bounce_seg || !ring || !td->urb) + return; + + /* td->bounce_seg is the last one bounced, unmap them all */ + for (seg = td->start_seg; i++ < ring->num_segs; seg = seg->next) { + if (seg->bounce_len) + xhci_unmap_one_bounce_buffer(xhci, ring, td, seg); + if (seg == td->bounce_seg) + break; + } +} + static void xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td, struct xhci_ring *ep_ring, int status) { @@ -2025,7 +2041,7 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) vdev = xhci->devs[port->slot_id]; /* We might get interrupts after shared_hcd is removed */ - if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) { + if (port->rhub == &xhci->usb3_rhub && xhci_get_usb3_hcd(xhci) == NULL) { xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n"); bogus_port_status = true; goto cleanup; @@ -2600,6 +2616,17 @@ static bool xhci_spurious_success_tx_event(struct xhci_hcd *xhci, } } +static struct xhci_td *find_td_by_dma(struct xhci_ring *ep_ring, dma_addr_t dma) +{ + struct xhci_td *td; + + if (dma) + list_for_each_entry(td, &ep_ring->td_list, td_list) + if (trb_in_td(td, dma)) + return td; + return NULL; +} + /* * If this function returns an error condition, it means it got a Transfer * event with a corrupted Slot ID, Endpoint ID, or TRB DMA address. @@ -2794,8 +2821,11 @@ static int handle_tx_event(struct xhci_hcd *xhci, xhci_dequeue_td(xhci, td, ep_ring, td->status); } - /* If the TRB pointer is NULL, missed TDs will be skipped on the next event */ - if (trb_comp_code == COMP_MISSED_SERVICE_ERROR && !ep_trb_dma) + /* + * We don't know how many TDs were missed when ep_trb_dma is zero (as permitted by + * xHCI 1.0) or bogus. Bail out leaving ep->skip set, next event will sort it out. + */ + if (trb_comp_code == COMP_MISSED_SERVICE_ERROR && !find_td_by_dma(ep_ring, ep_trb_dma)) return 0; if (list_empty(&ep_ring->td_list)) { @@ -3674,7 +3704,7 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, &trb_buff_len, ring->enq_seg)) { send_addr = ring->enq_seg->bounce_dma; - /* assuming TD won't span 2 segs */ + /* TD bounced at least, and last on this seg */ td->bounce_seg = ring->enq_seg; } } diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 091c82ca8ee29..5c759ad28199f 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -5433,6 +5433,7 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) struct device *dev = hcd->self.sysdev; int retval; u32 hcs_params1; + u32 hc_capbase; /* Accept arbitrarily long scatter-gather lists */ hcd->self.sg_tablesize = ~0; @@ -5453,15 +5454,19 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) mutex_init(&xhci->mutex); xhci->main_hcd = hcd; xhci->cap_regs = hcd->regs; - xhci->op_regs = hcd->regs + - HC_LENGTH(readl(&xhci->cap_regs->hc_capbase)); + hc_capbase = readl(&xhci->cap_regs->hc_capbase); + if (hc_capbase == U32_MAX) { + xhci_warn(xhci, "Host controller not accessible, removed?\n"); + return -ENODEV; + } + xhci->op_regs = hcd->regs + HC_LENGTH(hc_capbase); xhci->run_regs = hcd->regs + (readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK); /* Cache read-only capability registers */ hcs_params1 = readl(&xhci->cap_regs->hcs_params1); xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2); xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3); - xhci->hci_version = HC_VERSION(readl(&xhci->cap_regs->hc_capbase)); + xhci->hci_version = HC_VERSION(hc_capbase); xhci->hcc_params = readl(&xhci->cap_regs->hcc_params); if (xhci->hci_version > 0x100) xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2); diff --git a/drivers/usb/image/mdc800.c b/drivers/usb/image/mdc800.c index ca287b770e8cb..f7caa1c5cbb79 100644 --- a/drivers/usb/image/mdc800.c +++ b/drivers/usb/image/mdc800.c @@ -1000,13 +1000,13 @@ static int __init usb_mdc800_init (void) mdc800->downloaded = 0; mdc800->written = 0; - mdc800->irq_urb_buffer=kmalloc (8, GFP_KERNEL); + mdc800->irq_urb_buffer=kzalloc (8, GFP_KERNEL); if (!mdc800->irq_urb_buffer) goto cleanup_on_fail; mdc800->write_urb_buffer=kmalloc (8, GFP_KERNEL); if (!mdc800->write_urb_buffer) goto cleanup_on_fail; - mdc800->download_urb_buffer=kmalloc (64, GFP_KERNEL); + mdc800->download_urb_buffer=kzalloc (64, GFP_KERNEL); if (!mdc800->download_urb_buffer) goto cleanup_on_fail; diff --git a/drivers/usb/misc/usb-ljca.c b/drivers/usb/misc/usb-ljca.c index 78e94dd89da55..fcb627b49eac6 100644 --- a/drivers/usb/misc/usb-ljca.c +++ b/drivers/usb/misc/usb-ljca.c @@ -595,6 +595,9 @@ static int ljca_enumerate_gpio(struct ljca_adapter *adap) if (gpio_num > LJCA_MAX_GPIO_NUM) return -EINVAL; + if (desc->bank_num > ARRAY_SIZE(valid_pin)) + return -EINVAL; + /* construct platform data */ gpio_info = kzalloc_obj(*gpio_info); if (!gpio_info) diff --git a/drivers/usb/misc/usbtest.c b/drivers/usb/misc/usbtest.c index 98071b25ac076..8759df49be287 100644 --- a/drivers/usb/misc/usbtest.c +++ b/drivers/usb/misc/usbtest.c @@ -3054,6 +3054,7 @@ static struct usb_driver usbtest_driver = { .disconnect = usbtest_disconnect, .suspend = usbtest_suspend, .resume = usbtest_resume, + .no_dynamic_id = 1, }; /*-------------------------------------------------------------------------*/ diff --git a/drivers/usb/mtu3/mtu3_core.c b/drivers/usb/mtu3/mtu3_core.c index 66dbfe1705d57..a40bf5bad2d5d 100644 --- a/drivers/usb/mtu3/mtu3_core.c +++ b/drivers/usb/mtu3/mtu3_core.c @@ -1037,9 +1037,14 @@ int ssusb_gadget_suspend(struct ssusb_mtk *ssusb, pm_message_t msg) if (!mtu->gadget_driver) return 0; - if (mtu->connected) + /* Prevent runtime suspend when active connection exists */ + if (mtu->connected && PMSG_IS_AUTO(msg)) return -EBUSY; + /* Perform soft disconnect for system suspend */ + if (mtu->softconnect && !PMSG_IS_AUTO(msg)) + mtu3_dev_on_off(mtu, 0); + mtu3_dev_suspend(mtu); synchronize_irq(mtu->irq); @@ -1055,5 +1060,9 @@ int ssusb_gadget_resume(struct ssusb_mtk *ssusb, pm_message_t msg) mtu3_dev_resume(mtu); + /* Restore soft connect for system resume */ + if (mtu->softconnect && !PMSG_IS_AUTO(msg)) + mtu3_dev_on_off(mtu, 1); + return 0; } diff --git a/drivers/usb/phy/phy-fsl-usb.c b/drivers/usb/phy/phy-fsl-usb.c index 35d79f11b03d3..1986f7e9d9be2 100644 --- a/drivers/usb/phy/phy-fsl-usb.c +++ b/drivers/usb/phy/phy-fsl-usb.c @@ -46,7 +46,7 @@ static const char driver_name[] = "fsl-usb2-otg"; -const pm_message_t otg_suspend_state = { +static const pm_message_t otg_suspend_state = { .event = 1, }; @@ -57,11 +57,11 @@ static struct fsl_otg *fsl_otg_dev; static int srp_wait_done; /* FSM timers */ -struct fsl_otg_timer *a_wait_vrise_tmr, *a_wait_bcon_tmr, *a_aidl_bdis_tmr, +static struct fsl_otg_timer *a_wait_vrise_tmr, *a_wait_bcon_tmr, *a_aidl_bdis_tmr, *b_ase0_brst_tmr, *b_se0_srp_tmr; /* Driver specific timers */ -struct fsl_otg_timer *b_data_pulse_tmr, *b_vbus_pulse_tmr, *b_srp_fail_tmr, +static struct fsl_otg_timer *b_data_pulse_tmr, *b_vbus_pulse_tmr, *b_srp_fail_tmr, *b_srp_wait_tmr, *a_wait_enum_tmr; static struct list_head active_timers; @@ -102,7 +102,7 @@ static void (*_fsl_writel)(u32 v, unsigned __iomem *p); #define fsl_writel(val, addr) writel(val, addr) #endif /* CONFIG_PPC32 */ -int write_ulpi(u8 addr, u8 data) +static int write_ulpi(u8 addr, u8 data) { u32 temp; @@ -115,7 +115,7 @@ int write_ulpi(u8 addr, u8 data) /* Operations that will be called from OTG Finite State Machine */ /* Charge vbus for vbus pulsing in SRP */ -void fsl_otg_chrg_vbus(struct otg_fsm *fsm, int on) +static void fsl_otg_chrg_vbus(struct otg_fsm *fsm, int on) { u32 tmp; @@ -133,7 +133,7 @@ void fsl_otg_chrg_vbus(struct otg_fsm *fsm, int on) } /* Discharge vbus through a resistor to ground */ -void fsl_otg_dischrg_vbus(int on) +static void fsl_otg_dischrg_vbus(int on) { u32 tmp; @@ -151,7 +151,7 @@ void fsl_otg_dischrg_vbus(int on) } /* A-device driver vbus, controlled through PP bit in PORTSC */ -void fsl_otg_drv_vbus(struct otg_fsm *fsm, int on) +static void fsl_otg_drv_vbus(struct otg_fsm *fsm, int on) { u32 tmp; @@ -169,7 +169,7 @@ void fsl_otg_drv_vbus(struct otg_fsm *fsm, int on) * Pull-up D+, signalling connect by periperal. Also used in * data-line pulsing in SRP */ -void fsl_otg_loc_conn(struct otg_fsm *fsm, int on) +static void fsl_otg_loc_conn(struct otg_fsm *fsm, int on) { u32 tmp; @@ -188,7 +188,7 @@ void fsl_otg_loc_conn(struct otg_fsm *fsm, int on) * port. In host mode, controller will automatically send SOF. * Suspend will block the data on the port. */ -void fsl_otg_loc_sof(struct otg_fsm *fsm, int on) +static void fsl_otg_loc_sof(struct otg_fsm *fsm, int on) { u32 tmp; @@ -203,7 +203,7 @@ void fsl_otg_loc_sof(struct otg_fsm *fsm, int on) } /* Start SRP pulsing by data-line pulsing, followed with v-bus pulsing. */ -void fsl_otg_start_pulse(struct otg_fsm *fsm) +static void fsl_otg_start_pulse(struct otg_fsm *fsm) { u32 tmp; @@ -219,7 +219,7 @@ void fsl_otg_start_pulse(struct otg_fsm *fsm) fsl_otg_add_timer(fsm, b_data_pulse_tmr); } -void b_data_pulse_end(unsigned long foo) +static void b_data_pulse_end(unsigned long foo) { #ifdef HA_DATA_PULSE #else @@ -230,7 +230,7 @@ void b_data_pulse_end(unsigned long foo) fsl_otg_pulse_vbus(); } -void fsl_otg_pulse_vbus(void) +static void fsl_otg_pulse_vbus(void) { srp_wait_done = 0; fsl_otg_chrg_vbus(&fsl_otg_dev->fsm, 1); @@ -238,7 +238,7 @@ void fsl_otg_pulse_vbus(void) fsl_otg_add_timer(&fsl_otg_dev->fsm, b_vbus_pulse_tmr); } -void b_vbus_pulse_end(unsigned long foo) +static void b_vbus_pulse_end(unsigned long foo) { fsl_otg_chrg_vbus(&fsl_otg_dev->fsm, 0); @@ -251,7 +251,7 @@ void b_vbus_pulse_end(unsigned long foo) fsl_otg_add_timer(&fsl_otg_dev->fsm, b_srp_wait_tmr); } -void b_srp_end(unsigned long foo) +static void b_srp_end(unsigned long foo) { fsl_otg_dischrg_vbus(0); srp_wait_done = 1; @@ -266,7 +266,7 @@ void b_srp_end(unsigned long foo) * a_host will start by SRP. It needs to set b_hnp_enable before * actually suspending to start HNP */ -void a_wait_enum(unsigned long foo) +static void a_wait_enum(unsigned long foo) { VDBG("a_wait_enum timeout\n"); if (!fsl_otg_dev->phy.otg->host->b_hnp_enable) @@ -276,13 +276,13 @@ void a_wait_enum(unsigned long foo) } /* The timeout callback function to set time out bit */ -void set_tmout(unsigned long indicator) +static void set_tmout(unsigned long indicator) { *(int *)indicator = 1; } /* Initialize timers */ -int fsl_otg_init_timers(struct otg_fsm *fsm) +static int fsl_otg_init_timers(struct otg_fsm *fsm) { /* FSM used timers */ a_wait_vrise_tmr = otg_timer_initializer(&set_tmout, TA_WAIT_VRISE, @@ -339,7 +339,7 @@ int fsl_otg_init_timers(struct otg_fsm *fsm) } /* Uninitialize timers */ -void fsl_otg_uninit_timers(void) +static void fsl_otg_uninit_timers(void) { /* FSM used timers */ kfree(a_wait_vrise_tmr); @@ -391,7 +391,7 @@ static struct fsl_otg_timer *fsl_otg_get_timer(enum otg_fsm_timer t) } /* Add timer to timer list */ -void fsl_otg_add_timer(struct otg_fsm *fsm, void *gtimer) +static void fsl_otg_add_timer(struct otg_fsm *fsm, void *gtimer) { struct fsl_otg_timer *timer = gtimer; struct fsl_otg_timer *tmp_timer; @@ -421,7 +421,7 @@ static void fsl_otg_fsm_add_timer(struct otg_fsm *fsm, enum otg_fsm_timer t) } /* Remove timer from the timer list; clear timeout status */ -void fsl_otg_del_timer(struct otg_fsm *fsm, void *gtimer) +static void fsl_otg_del_timer(struct otg_fsm *fsm, void *gtimer) { struct fsl_otg_timer *timer = gtimer; struct fsl_otg_timer *tmp_timer, *del_tmp; @@ -443,7 +443,7 @@ static void fsl_otg_fsm_del_timer(struct otg_fsm *fsm, enum otg_fsm_timer t) } /* Reset controller, not reset the bus */ -void otg_reset_controller(void) +static void otg_reset_controller(void) { u32 command; @@ -455,7 +455,7 @@ void otg_reset_controller(void) } /* Call suspend/resume routines in host driver */ -int fsl_otg_start_host(struct otg_fsm *fsm, int on) +static int fsl_otg_start_host(struct otg_fsm *fsm, int on) { struct usb_otg *otg = fsm->otg; struct device *dev; @@ -522,7 +522,7 @@ int fsl_otg_start_host(struct otg_fsm *fsm, int on) * Call suspend and resume function in udc driver * to stop and start udc driver. */ -int fsl_otg_start_gadget(struct otg_fsm *fsm, int on) +static int fsl_otg_start_gadget(struct otg_fsm *fsm, int on) { struct usb_otg *otg = fsm->otg; struct device *dev; @@ -704,7 +704,7 @@ static int fsl_otg_start_hnp(struct usb_otg *otg) * intact. It needs to have knowledge of some USB interrupts * such as port change. */ -irqreturn_t fsl_otg_isr(int irq, void *dev_id) +static irqreturn_t fsl_otg_isr(int irq, void *dev_id) { struct otg_fsm *fsm = &((struct fsl_otg *)dev_id)->fsm; struct usb_otg *otg = ((struct fsl_otg *)dev_id)->phy.otg; @@ -830,7 +830,7 @@ static int fsl_otg_conf(struct platform_device *pdev) } /* OTG Initialization */ -int usb_otg_start(struct platform_device *pdev) +static int usb_otg_start(struct platform_device *pdev) { struct fsl_otg *p_otg; struct usb_phy *otg_trans = usb_get_phy(USB_PHY_TYPE_USB2); @@ -1002,7 +1002,7 @@ static void fsl_otg_remove(struct platform_device *pdev) pdata->exit(pdev); } -struct platform_driver fsl_otg_driver = { +static struct platform_driver fsl_otg_driver = { .probe = fsl_otg_probe, .remove = fsl_otg_remove, .driver = { diff --git a/drivers/usb/phy/phy-fsl-usb.h b/drivers/usb/phy/phy-fsl-usb.h index 95bfe7f1b83a9..b754077875c39 100644 --- a/drivers/usb/phy/phy-fsl-usb.h +++ b/drivers/usb/phy/phy-fsl-usb.h @@ -373,6 +373,6 @@ struct fsl_otg_config { #define FSL_OTG_NAME "fsl-usb2-otg" -void fsl_otg_add_timer(struct otg_fsm *fsm, void *timer); -void fsl_otg_del_timer(struct otg_fsm *fsm, void *timer); -void fsl_otg_pulse_vbus(void); +static void fsl_otg_add_timer(struct otg_fsm *fsm, void *timer); +static void fsl_otg_del_timer(struct otg_fsm *fsm, void *timer); +static void fsl_otg_pulse_vbus(void); diff --git a/drivers/usb/renesas_usbhs/common.c b/drivers/usb/renesas_usbhs/common.c index 8c93bde4b8167..51d3035f82bed 100644 --- a/drivers/usb/renesas_usbhs/common.c +++ b/drivers/usb/renesas_usbhs/common.c @@ -813,9 +813,6 @@ static void usbhs_remove(struct platform_device *pdev) flush_delayed_work(&priv->notify_hotplug_work); - usbhs_platform_call(priv, hardware_exit, pdev); - reset_control_assert(priv->rsts); - /* * Explicitly free the IRQ to ensure the interrupt handler is * disabled and synchronized before freeing resources. @@ -832,6 +829,9 @@ static void usbhs_remove(struct platform_device *pdev) if (!usbhs_get_dparam(priv, runtime_pwctrl)) usbhsc_power_ctrl(priv, 0); + usbhs_platform_call(priv, hardware_exit, pdev); + reset_control_assert(priv->rsts); + usbhsc_clk_put(priv); pm_runtime_disable(&pdev->dev); } diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index 580f06f5ce5e3..e4ad14375d6a7 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -2689,12 +2689,26 @@ static void option_instat_callback(struct urb *urb) dev_dbg(dev, "%s: NULL req_pkt\n", __func__); return; } + + if (urb->actual_length < sizeof(*req_pkt)) { + dev_err(dev, "%s: short packet: %u bytes\n", __func__, + urb->actual_length); + return; + } + if ((req_pkt->bRequestType == 0xA1) && (req_pkt->bRequest == 0x20)) { + unsigned char signals; int old_dcd_state; - unsigned char signals = *((unsigned char *) - urb->transfer_buffer + - sizeof(struct usb_ctrlrequest)); + + if (urb->actual_length < sizeof(*req_pkt) + 1) { + dev_err(dev, "%s: short interrupt transfer: %u bytes\n", + __func__, urb->actual_length); + return; + } + + signals = *((unsigned char *)urb->transfer_buffer + + sizeof(*req_pkt)); dev_dbg(dev, "%s: signal x%x\n", __func__, signals); diff --git a/drivers/usb/serial/spcp8x5.c b/drivers/usb/serial/spcp8x5.c index c11d64bf08fba..3ef7ca2c0cf44 100644 --- a/drivers/usb/serial/spcp8x5.c +++ b/drivers/usb/serial/spcp8x5.c @@ -133,14 +133,14 @@ struct spcp8x5_private { static int spcp8x5_probe(struct usb_serial *serial, const struct usb_device_id *id) { - usb_set_serial_data(serial, (void *)id); + usb_set_serial_data(serial, (void *)id->driver_info); return 0; } static int spcp8x5_port_probe(struct usb_serial_port *port) { - const struct usb_device_id *id = usb_get_serial_data(port->serial); + unsigned int quirks = (unsigned int)(unsigned long)usb_get_serial_data(port->serial); struct spcp8x5_private *priv; priv = kzalloc_obj(*priv); @@ -148,7 +148,7 @@ static int spcp8x5_port_probe(struct usb_serial_port *port) return -ENOMEM; spin_lock_init(&priv->lock); - priv->quirks = id->driver_info; + priv->quirks = quirks; usb_set_serial_port_data(port, priv); @@ -235,18 +235,6 @@ static void spcp8x5_set_work_mode(struct usb_serial_port *port, u16 value, dev_err(&port->dev, "failed to set work mode: %d\n", ret); } -static int spcp8x5_carrier_raised(struct usb_serial_port *port) -{ - u8 msr; - int ret; - - ret = spcp8x5_get_msr(port, &msr); - if (ret || msr & MSR_STATUS_LINE_DCD) - return 1; - - return 0; -} - static void spcp8x5_dtr_rts(struct usb_serial_port *port, int on) { struct spcp8x5_private *priv = usb_get_serial_port_data(port); @@ -458,7 +446,6 @@ static struct usb_serial_driver spcp8x5_device = { .num_bulk_out = 1, .open = spcp8x5_open, .dtr_rts = spcp8x5_dtr_rts, - .carrier_raised = spcp8x5_carrier_raised, .set_termios = spcp8x5_set_termios, .init_termios = spcp8x5_init_termios, .tiocmget = spcp8x5_tiocmget, diff --git a/drivers/usb/storage/ene_ub6250.c b/drivers/usb/storage/ene_ub6250.c index ed49a3bc859ce..895f90c7a3fae 100644 --- a/drivers/usb/storage/ene_ub6250.c +++ b/drivers/usb/storage/ene_ub6250.c @@ -2357,7 +2357,9 @@ static int ene_ub6250_probe(struct usb_interface *intf, return result; /* probe card type */ + mutex_lock(&us->dev_mutex); result = ene_get_card_type(us, REG_CARD_STATUS, info->bbuf); + mutex_unlock(&us->dev_mutex); if (result != USB_STOR_XFER_GOOD) { usb_stor_disconnect(intf); return USB_STOR_TRANSPORT_ERROR; diff --git a/drivers/usb/storage/realtek_cr.c b/drivers/usb/storage/realtek_cr.c index af038b897c6be..c4b28744693b7 100644 --- a/drivers/usb/storage/realtek_cr.c +++ b/drivers/usb/storage/realtek_cr.c @@ -916,7 +916,6 @@ static int realtek_cr_autosuspend_setup(struct us_data *us) us->proto_handler = rts51x_invoke_transport; chip->timer_expires = 0; - timer_setup(&chip->rts51x_suspend_timer, rts51x_suspend_timer_fn, 0); fw5895_init(us); /* enable autosuspend function of the usb device */ @@ -934,10 +933,7 @@ static void realtek_cr_destructor(void *extra) return; #ifdef CONFIG_REALTEK_AUTOPM - if (ss_en) { - timer_delete(&chip->rts51x_suspend_timer); - chip->timer_expires = 0; - } + timer_shutdown_sync(&chip->rts51x_suspend_timer); #endif kfree(chip->status); } @@ -982,6 +978,9 @@ static int init_realtek_cr(struct us_data *us) us->extra = chip; us->extra_destructor = realtek_cr_destructor; +#ifdef CONFIG_REALTEK_AUTOPM + timer_setup(&chip->rts51x_suspend_timer, rts51x_suspend_timer_fn, 0); +#endif us->max_lun = chip->max_lun = rts51x_get_max_lun(us); chip->us = us; diff --git a/drivers/usb/typec/altmodes/thunderbolt.c b/drivers/usb/typec/altmodes/thunderbolt.c index 32250b94262a9..601d39ee17c47 100644 --- a/drivers/usb/typec/altmodes/thunderbolt.c +++ b/drivers/usb/typec/altmodes/thunderbolt.c @@ -303,6 +303,8 @@ static void tbt_altmode_remove(struct typec_altmode *alt) { struct tbt_altmode *tbt = typec_altmode_get_drvdata(alt); + disable_work_sync(&tbt->work); + for (int i = TYPEC_PLUG_SOP_PP; i >= 0; --i) { if (tbt->plug[i]) typec_altmode_put_plug(tbt->plug[i]); diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c index 3e39b800e6b5f..4eec90c82bae4 100644 --- a/drivers/usb/typec/hd3ss3220.c +++ b/drivers/usb/typec/hd3ss3220.c @@ -62,6 +62,7 @@ struct hd3ss3220 { int id_irq; struct regulator *vbus; + bool vbus_enabled; }; static int hd3ss3220_set_power_opmode(struct hd3ss3220 *hd3ss3220, int power_opmode) @@ -208,7 +209,7 @@ static void hd3ss3220_regulator_control(struct hd3ss3220 *hd3ss3220, bool on) { int ret; - if (regulator_is_enabled(hd3ss3220->vbus) == on) + if (hd3ss3220->vbus_enabled == on) return; if (on) @@ -216,9 +217,13 @@ static void hd3ss3220_regulator_control(struct hd3ss3220 *hd3ss3220, bool on) else ret = regulator_disable(hd3ss3220->vbus); - if (ret) + if (ret) { dev_err(hd3ss3220->dev, - "vbus regulator %s failed: %d\n", on ? "disable" : "enable", ret); + "vbus regulator %s failed: %d\n", on ? "enable" : "disable", ret); + return; + } + + hd3ss3220->vbus_enabled = on; } static void hd3ss3220_set_role(struct hd3ss3220 *hd3ss3220) diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 9b908c46bd7df..afa6fc1813978 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -57,6 +57,8 @@ static void *typec_switch_match(const struct fwnode_handle *fwnode, */ dev = class_find_device(&typec_mux_class, NULL, fwnode, switch_fwnode_match); + if (!dev) + return ERR_PTR(-EPROBE_DEFER); /* Skip duplicates */ for (i = 0; i < TYPEC_MUX_MAX_DEVS; i++) @@ -65,7 +67,7 @@ static void *typec_switch_match(const struct fwnode_handle *fwnode, return NULL; } - return dev ? to_typec_switch_dev(dev) : ERR_PTR(-EPROBE_DEFER); + return to_typec_switch_dev(dev); } /** @@ -275,7 +277,9 @@ static int mux_fwnode_match(struct device *dev, const void *fwnode) static void *typec_mux_match(const struct fwnode_handle *fwnode, const char *id, void *data) { + struct typec_mux_dev **mux_devs = data; struct device *dev; + int i; /* * Device graph (OF graph) does not give any means to identify the @@ -290,8 +294,18 @@ static void *typec_mux_match(const struct fwnode_handle *fwnode, dev = class_find_device(&typec_mux_class, NULL, fwnode, mux_fwnode_match); + if (!dev) + return ERR_PTR(-EPROBE_DEFER); - return dev ? to_typec_mux_dev(dev) : ERR_PTR(-EPROBE_DEFER); + /* Skip duplicates */ + for (i = 0; i < TYPEC_MUX_MAX_DEVS; i++) + if (to_typec_mux_dev(dev) == mux_devs[i]) { + put_device(dev); + return NULL; + } + + + return to_typec_mux_dev(dev); } /** @@ -316,7 +330,8 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) return ERR_PTR(-ENOMEM); count = fwnode_connection_find_matches(fwnode, "mode-switch", - NULL, typec_mux_match, + (void **)mux_devs, + typec_mux_match, (void **)mux_devs, ARRAY_SIZE(mux_devs)); if (count <= 0) { diff --git a/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c b/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c index e6b28648f4407..926fa017ebafc 100644 --- a/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c +++ b/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c @@ -543,6 +543,8 @@ static void qcom_pmic_typec_pdphy_stop(struct pmic_typec *tcpm) for (i = 0; i < pmic_typec_pdphy->nr_irqs; i++) disable_irq(pmic_typec_pdphy->irq_data[i].irq); + cancel_work_sync(&pmic_typec_pdphy->reset_work); + qcom_pmic_typec_pdphy_reset_on(pmic_typec_pdphy); regulator_disable(pmic_typec_pdphy->vdd_pdphy); diff --git a/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_port.c b/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_port.c index bf985efe1cd6b..d3523435f3e01 100644 --- a/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_port.c +++ b/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_port.c @@ -683,6 +683,9 @@ static int qcom_pmic_typec_port_start(struct pmic_typec *tcpm, enable_irq(pmic_typec_port->irq_data[i].irq); done: + if (ret) + disable_delayed_work_sync(&pmic_typec_port->cc_debounce_dwork); + return ret; } @@ -693,6 +696,8 @@ static void qcom_pmic_typec_port_stop(struct pmic_typec *tcpm) for (i = 0; i < pmic_typec_port->nr_irqs; i++) disable_irq(pmic_typec_port->irq_data[i].irq); + + disable_delayed_work_sync(&pmic_typec_port->cc_debounce_dwork); } int qcom_pmic_typec_port_probe(struct platform_device *pdev, diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index 7ac7000b2d139..6717ac914c6a6 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -38,6 +38,7 @@ struct tcpci { struct regmap *regmap; unsigned int alert_mask; + unsigned int rx_type_mask; bool controls_vbus; @@ -488,6 +489,8 @@ static int tcpci_set_pd_rx(struct tcpc_dev *tcpc, bool enable) if (tcpci->data->cable_comm_capable) reg |= TCPC_RX_DETECT_SOP1; } + + tcpci->rx_type_mask = reg; ret = regmap_write(tcpci->regmap, TCPC_RX_DETECT, reg); if (ret < 0) return ret; @@ -749,6 +752,7 @@ irqreturn_t tcpci_irq(struct tcpci *tcpci) if (status & TCPC_ALERT_RX_STATUS) { struct pd_message msg; unsigned int cnt, payload_cnt; + enum tcpm_transmit_type rx_type; u16 header; regmap_read(tcpci->regmap, TCPC_RX_BYTE_CNT, &cnt); @@ -773,10 +777,16 @@ irqreturn_t tcpci_irq(struct tcpci *tcpci) regmap_raw_read(tcpci->regmap, TCPC_RX_DATA, &msg.payload, payload_cnt); + ret = regmap_read(tcpci->regmap, TCPC_RX_BUF_FRAME_TYPE, &rx_type); + if (ret) + return ret; + /* Read complete, clear RX status alert bit */ tcpci_write16(tcpci, TCPC_ALERT, TCPC_ALERT_RX_STATUS); - tcpm_pd_receive(tcpci->port, &msg, TCPC_TX_SOP); + rx_type &= TCPC_RX_BUF_FRAME_TYPE_MASK; + if (tcpci->rx_type_mask & BIT(rx_type)) + tcpm_pd_receive(tcpci->port, &msg, rx_type); } if (tcpci->data->vbus_vsafe0v && (status & TCPC_ALERT_EXTENDED_STATUS)) { diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 89eec20a2064c..e49ab6d54a5de 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -6921,16 +6921,32 @@ static void tcpm_pd_event_handler(struct kthread_work *work) } } if (events & TCPM_SOURCING_VBUS) { - tcpm_log(port, "sourcing vbus"); /* * In fast role swap case TCPC autonomously sources vbus. Set vbus_source - * true as TCPM wouldn't have called tcpm_set_vbus. + * true conditionally as TCPM wouldn't have called tcpm_set_vbus. + * If TCPM calls tcpm_set_vbus to source vbus, vbus_source would already + * be true. * - * When vbus is sourced on the command on TCPM i.e. TCPM called - * tcpm_set_vbus to source vbus, vbus_source would already be true. + * When TCPM_FRS_EVENT and TCPM_SOURCING_VBUS arrive simultaneously, + * handling TCPM_FRS_EVENT above transitions the state to AMS_START + * with upcoming_state FR_SWAP_SEND. */ - port->vbus_source = true; - _tcpm_pd_vbus_on(port); + + if (tcpm_port_is_source(port) || + tcpm_port_is_debug_source(port) || + (port->state == AMS_START && port->upcoming_state == FR_SWAP_SEND) || + port->state == FR_SWAP_SEND || + port->state == FR_SWAP_SEND_TIMEOUT || + port->state == FR_SWAP_SNK_SRC_TRANSITION_TO_OFF || + port->state == FR_SWAP_SNK_SRC_NEW_SINK_READY || + port->state == FR_SWAP_SNK_SRC_SOURCE_VBUS_APPLIED) { + tcpm_log(port, "sourcing vbus"); + port->vbus_source = true; + _tcpm_pd_vbus_on(port); + } else { + tcpm_log(port, "Discarding sourcing vbus! Invalid state %s", + tcpm_states[port->state]); + } } if (events & TCPM_PORT_CLEAN) { tcpm_log(port, "port clean"); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index d5ee0af9058b9..159d1a2f9a1f5 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -114,7 +114,6 @@ struct tps6598x_intel_vid_status_reg { __le32 attention_vdo; __le16 enter_vdo; __le16 device_mode; - __le16 cable_mode; } __packed; /* Standard Task return codes */ @@ -700,9 +699,19 @@ static void cd321x_typec_update_mode(struct tps6598x *tps, struct cd321x_status cd321x->state.mode == TYPEC_TBT_MODE) return; - tbt_data.cable_mode = le16_to_cpu(st->intel_vid_status.cable_mode); - tbt_data.device_mode = le16_to_cpu(st->intel_vid_status.device_mode); - tbt_data.enter_vdo = le16_to_cpu(st->intel_vid_status.enter_vdo); + tbt_data.cable_mode = TBT_MODE | + TBT_SET_CABLE_SPEED(TPS_DATA_STATUS_TBT_CABLE_SPEED(st->data_status)) | + TBT_SET_CABLE_ROUNDED(TPS_DATA_STATUS_TBT_CABLE_GEN(st->data_status)); + if (st->data_status & TPS_DATA_STATUS_OPTICAL_CABLE) + tbt_data.cable_mode |= TBT_CABLE_OPTICAL; + if (st->data_status & TPS_DATA_STATUS_ACTIVE_LINK_TRAIN) + tbt_data.cable_mode |= TBT_CABLE_LINK_TRAINING; + if (st->data_status & TPS_DATA_STATUS_ACTIVE_CABLE) + tbt_data.cable_mode |= TBT_CABLE_ACTIVE_PASSIVE; + tbt_data.device_mode = TBT_MODE | + (u32)le16_to_cpu(st->intel_vid_status.device_mode) << 16; + tbt_data.enter_vdo = + (u32)le16_to_cpu(st->intel_vid_status.enter_vdo) << 16; cd321x->state.alt = cd321x->port_altmode_tbt; cd321x->state.mode = TYPEC_TBT_MODE; cd321x->state.data = &tbt_data; diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 03edbb77bbd6d..d172c84ada74e 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -206,10 +206,10 @@ #define TPS_DATA_STATUS_DP_PIN_ASSIGNMENT(x) \ TPS_FIELD_GET(TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK, (x)) #define TPS_DATA_STATUS_TBT_CABLE_SPEED_MASK GENMASK(27, 25) -#define TPS_DATA_STATUS_TBT_CABLE_SPEED \ +#define TPS_DATA_STATUS_TBT_CABLE_SPEED(x) \ TPS_FIELD_GET(TPS_DATA_STATUS_TBT_CABLE_SPEED_MASK, (x)) #define TPS_DATA_STATUS_TBT_CABLE_GEN_MASK GENMASK(29, 28) -#define TPS_DATA_STATUS_TBT_CABLE_GEN \ +#define TPS_DATA_STATUS_TBT_CABLE_GEN(x) \ TPS_FIELD_GET(TPS_DATA_STATUS_TBT_CABLE_GEN_MASK, (x)) /* Map data status to DP spec assignments */ diff --git a/drivers/usb/typec/ucsi/debugfs.c b/drivers/usb/typec/ucsi/debugfs.c index ff33a5e7c6b0d..a124105b6226d 100644 --- a/drivers/usb/typec/ucsi/debugfs.c +++ b/drivers/usb/typec/ucsi/debugfs.c @@ -162,6 +162,7 @@ void ucsi_debugfs_unregister(struct ucsi *ucsi) debugfs_remove_recursive(ucsi->debugfs->dentry); kfree(ucsi->debugfs); + ucsi->debugfs = NULL; } void ucsi_debugfs_init(void) diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c index 7067f2561b846..8d2032d0762ca 100644 --- a/drivers/usb/typec/ucsi/displayport.c +++ b/drivers/usb/typec/ucsi/displayport.c @@ -74,7 +74,7 @@ static int ucsi_displayport_enter(struct typec_altmode *alt, u32 *vdo) cur = 0xff; } - if (cur != 0xff) { + if (cur < UCSI_MAX_ALTMODES) { ret = dp->con->port_altmode[cur] == alt ? 0 : -EBUSY; goto err_unlock; } diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index 49f1c53721bbd..b3a7712a64507 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -91,7 +91,8 @@ int ucsi_sync_control_common(struct ucsi *ucsi, u64 command, u32 *cci, if (ret) goto out_clear_bit; - if (!wait_for_completion_timeout(&ucsi->complete, 5 * HZ)) + if (!wait_for_completion_timeout(&ucsi->complete, + msecs_to_jiffies(UCSI_TIMEOUT_MS))) ret = -ETIMEDOUT; out_clear_bit: @@ -2366,6 +2367,8 @@ void ucsi_unregister(struct ucsi *ucsi) cancel_delayed_work_sync(&ucsi->work); cancel_work_sync(&ucsi->resume_work); + ucsi_debugfs_unregister(ucsi); + /* Disable notifications */ ucsi->ops->async_control(ucsi, cmd); diff --git a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c index ca1b534cb183d..57b37fca150c2 100644 --- a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c +++ b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c @@ -105,6 +105,7 @@ struct gaokun_ucsi { struct notifier_block nb; u16 version; u8 num_ports; + bool registered; }; /* -------------------------------------------------------------------------- */ @@ -482,8 +483,13 @@ static void gaokun_ucsi_register_worker(struct work_struct *work) } ret = ucsi_register(ucsi); - if (ret) + if (ret) { dev_err_probe(ucsi->dev, ret, "ucsi register failed\n"); + gaokun_ec_unregister_notify(uec->ec, &uec->nb); + return; + } + + uec->registered = true; } static int gaokun_ucsi_probe(struct auxiliary_device *adev, @@ -528,8 +534,11 @@ static void gaokun_ucsi_remove(struct auxiliary_device *adev) int i; disable_delayed_work_sync(&uec->work); - gaokun_ec_unregister_notify(uec->ec, &uec->nb); - ucsi_unregister(uec->ucsi); + if (uec->registered) { + gaokun_ec_unregister_notify(uec->ec, &uec->nb); + ucsi_unregister(uec->ucsi); + } + for (i = 0; i < uec->num_ports; ++i) typec_mux_put(uec->ports[i].typec_mux); diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index ab6d6ab3b3d8d..2af1cec958848 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -724,7 +724,8 @@ static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, if (config->device_features & ~device_features) { IFCVF_ERR(pdev, "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n", config->device_features, device_features); - return -EINVAL; + ret = -EINVAL; + goto err; } device_features &= config->device_features; } diff --git a/drivers/vdpa/mlx5/core/mr.c b/drivers/vdpa/mlx5/core/mr.c index 77a479aeaa85a..b0c5ff23d022b 100644 --- a/drivers/vdpa/mlx5/core/mr.c +++ b/drivers/vdpa/mlx5/core/mr.c @@ -481,7 +481,7 @@ static int add_direct_chain(struct mlx5_vdpa_dev *mvdev, return 0; err_alloc: - list_for_each_entry_safe(dmr, n, &mr->head, list) { + list_for_each_entry_safe(dmr, n, &tmp, list) { list_del_init(&dmr->list); unmap_direct_mr(mvdev, dmr); kfree(dmr); diff --git a/drivers/vdpa/solidrun/snet_main.c b/drivers/vdpa/solidrun/snet_main.c index 28d55315df2a8..3e2cea1e45f3e 100644 --- a/drivers/vdpa/solidrun/snet_main.c +++ b/drivers/vdpa/solidrun/snet_main.c @@ -418,11 +418,15 @@ static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet) snet->vqs[i]->irq_name, snet->vqs[i]); if (ret) { SNET_ERR(pdev, "Failed to request IRQ\n"); - return ret; + goto err_free_irqs; } snet->vqs[i]->irq = irq; } return 0; + +err_free_irqs: + snet_free_irqs(snet); + return ret; } static void snet_set_status(struct vdpa_device *vdev, u8 status) diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index 4d116644851d9..c748fe4511635 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -233,8 +233,11 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr, kthread_init_work(&vdpasim->work, vdpasim_work_fn); vdpasim->worker = kthread_run_worker(0, "vDPA sim worker: %s", dev_attr->name); - if (IS_ERR(vdpasim->worker)) + if (IS_ERR(vdpasim->worker)) { + ret = PTR_ERR(vdpasim->worker); + vdpasim->worker = NULL; goto err_iommu; + } mutex_init(&vdpasim->mutex); spin_lock_init(&vdpasim->iommu_lock); @@ -746,18 +749,24 @@ static void vdpasim_free(struct vdpa_device *vdpa) struct vdpasim *vdpasim = vdpa_to_sim(vdpa); int i; - kthread_cancel_work_sync(&vdpasim->work); - kthread_destroy_worker(vdpasim->worker); + if (vdpasim->worker) { + kthread_cancel_work_sync(&vdpasim->work); + kthread_destroy_worker(vdpasim->worker); + } - for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { - vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov); - vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov); + if (vdpasim->vqs) { + for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { + vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov); + vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov); + } } vdpasim->dev_attr.free(vdpasim); - for (i = 0; i < vdpasim->dev_attr.nas; i++) - vhost_iotlb_reset(&vdpasim->iommu[i]); + if (vdpasim->iommu) { + for (i = 0; i < vdpasim->dev_attr.nas; i++) + vhost_iotlb_reset(&vdpasim->iommu[i]); + } kfree(vdpasim->iommu); kfree(vdpasim->iommu_pt); kfree(vdpasim->vqs); diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index f70f454dde8eb..76dd5b0828d7a 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -79,10 +79,11 @@ static void vdpasim_blk_buffer_unlock(struct vdpasim_blk *blk) static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector, u64 num_sectors, u64 max_sectors) { - if (start_sector > VDPASIM_BLK_CAPACITY) { + if (start_sector >= VDPASIM_BLK_CAPACITY) { dev_dbg(&vdpasim->vdpa.dev, "starting sector exceeds the capacity - start: 0x%llx capacity: 0x%x\n", start_sector, VDPASIM_BLK_CAPACITY); + return false; } if (num_sectors > max_sectors) { diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c index 29fd14ce5860b..a6514b5ccd865 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c @@ -225,10 +225,15 @@ static void vdpasim_net_work(struct vdpasim *vdpasim) break; } - ++tx_pkts; read = vringh_iov_pull_iotlb(&txq->vring, &txq->out_iov, net->buffer, PAGE_SIZE); + if (read <= 0) { + ++tx_errors; + vdpasim_net_complete(txq, 0); + continue; + } + ++tx_pkts; tx_bytes += read; if (!receive_filter(vdpasim, read)) { diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index 10dcf016bfb06..ed71e5835e60c 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -1751,11 +1751,11 @@ static long vduse_dev_compat_ioctl(struct file *file, unsigned int cmd, break; } default: - ret = -ENOIOCTLCMD; - break; + return vduse_dev_ioctl(file, cmd, + (unsigned long)compat_ptr(arg)); } - return vduse_dev_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); + return ret; } #else #define vduse_dev_compat_ioctl compat_ptr_ioctl @@ -2094,7 +2094,9 @@ static bool vduse_validate_config(struct vduse_dev_config *config, return false; } - if (config->vq_align > PAGE_SIZE) + if (config->vq_align < VRING_USED_ALIGN_SIZE || + !is_power_of_2(config->vq_align) || + config->vq_align > PAGE_SIZE) return false; if (config->config_size > PAGE_SIZE) diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index a10ed733f0e3a..9914f3ac69aef 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -1272,6 +1272,7 @@ static int vfio_msi_cap_len(struct vfio_pci_core_device *vdev, u8 pos) ret = init_pci_cap_msi_perm(vdev->msi_perm, len, flags); if (ret) { kfree(vdev->msi_perm); + vdev->msi_perm = NULL; return ret; } diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 3f11a9624b9c0..6a184588ff234 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -554,6 +554,9 @@ static void vfio_pci_core_map_bars(struct vfio_pci_core_device *vdev) vdev->barmap[bar] = IOMEM_ERR_PTR(-ENODEV); + if (pdev->non_mappable_bars) + continue; + if (!pci_resource_len(pdev, i)) continue; diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 6949b704166d5..4d9d7c2216ed5 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -1696,6 +1696,8 @@ static int vhost_net_set_features(struct vhost_net *n, const u64 *features) if (virtio_features_test_bit(features, VIRTIO_F_ACCESS_PLATFORM)) { if (vhost_init_device_iotlb(&n->dev)) goto out_unlock; + } else { + vhost_clear_device_iotlb(&n->dev); } for (i = 0; i < VHOST_NET_VQ_MAX; ++i) { @@ -1777,7 +1779,8 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl, return -EFAULT; /* Zero the trailing space provided by user-space, if any */ - if (clear_user(argp, size_mul(count - copied, sizeof(u64)))) + if (clear_user(argp + size_mul(copied, sizeof(u64)), + size_mul(count - copied, sizeof(u64)))) return -EFAULT; return 0; case VHOST_SET_FEATURES_ARRAY: diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index c3d913bd7cac7..a31786796d4ce 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -58,9 +58,12 @@ struct vhost_vdpa { struct cdev cdev; atomic_t opened; u32 nvqs; + u16 vq_num_max; int virtio_id; int minor; struct eventfd_ctx *config_ctx; + /* Serialises vhost_vdpa_config_cb() against config_ctx being replaced. */ + spinlock_t config_lock; int in_batch; struct vdpa_iova_range range; u32 batch_asid; @@ -194,10 +197,12 @@ static irqreturn_t vhost_vdpa_virtqueue_cb(void *private) static irqreturn_t vhost_vdpa_config_cb(void *private) { struct vhost_vdpa *v = private; - struct eventfd_ctx *config_ctx = v->config_ctx; + unsigned long flags; - if (config_ctx) - eventfd_signal(config_ctx); + spin_lock_irqsave(&v->config_lock, flags); + if (v->config_ctx) + eventfd_signal(v->config_ctx); + spin_unlock_irqrestore(&v->config_lock, flags); return IRQ_HANDLED; } @@ -236,7 +241,9 @@ static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid) static int _compat_vdpa_reset(struct vhost_vdpa *v) { struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; u32 flags = 0; + int ret; v->suspended = false; @@ -246,7 +253,14 @@ static int _compat_vdpa_reset(struct vhost_vdpa *v) VDPA_RESET_F_CLEAN_MAP : 0; } - return vdpa_reset(vdpa, flags); + v->vq_num_max = 0; + ret = vdpa_reset(vdpa, flags); + if (!ret) { + /* Some backends derive the max from mutable queue state. */ + v->vq_num_max = ops->get_vq_num_max(vdpa); + } + + return ret; } static int vhost_vdpa_reset(struct vhost_vdpa *v) @@ -518,15 +532,22 @@ static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp) static void vhost_vdpa_config_put(struct vhost_vdpa *v) { - if (v->config_ctx) { - eventfd_ctx_put(v->config_ctx); - v->config_ctx = NULL; - } + struct eventfd_ctx *ctx; + unsigned long flags; + + spin_lock_irqsave(&v->config_lock, flags); + ctx = v->config_ctx; + v->config_ctx = NULL; + spin_unlock_irqrestore(&v->config_lock, flags); + + if (ctx) + eventfd_ctx_put(ctx); } static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp) { struct vdpa_callback cb; + unsigned long flags; int fd; struct eventfd_ctx *ctx; @@ -536,18 +557,20 @@ static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp) return -EFAULT; ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + + spin_lock_irqsave(&v->config_lock, flags); swap(ctx, v->config_ctx); + spin_unlock_irqrestore(&v->config_lock, flags); - if (!IS_ERR_OR_NULL(ctx)) + /* + * The callback can no longer reach the old context, so this is the + * last reference to it. + */ + if (ctx) eventfd_ctx_put(ctx); - if (IS_ERR(v->config_ctx)) { - long ret = PTR_ERR(v->config_ctx); - - v->config_ctx = NULL; - return ret; - } - v->vdpa->config->set_config_cb(v->vdpa, &cb); return 0; @@ -648,9 +671,15 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, u32 idx; long r; - r = get_user(idx, (u32 __user *)argp); - if (r < 0) - return r; + if (cmd == VHOST_SET_VRING_NUM) { + if (copy_from_user(&s, argp, sizeof(s))) + return -EFAULT; + idx = s.index; + } else { + r = get_user(idx, (u32 __user *)argp); + if (r < 0) + return r; + } if (idx >= v->nvqs) return -ENOBUFS; @@ -659,6 +688,23 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, vq = &v->vqs[idx]; switch (cmd) { + case VHOST_SET_VRING_NUM: + mutex_lock(&vq->mutex); + if (vq->private_data) { + r = -EBUSY; + } else if (!s.num || s.num > 0xffff || + s.num > v->vq_num_max || + (s.num & (s.num - 1))) { + r = -EINVAL; + } else { + vq->num = s.num; + r = 0; + } + mutex_unlock(&vq->mutex); + if (r) + return r; + ops->set_vq_num(vdpa, idx, s.num); + return 0; case VHOST_VDPA_SET_VRING_ENABLE: if (copy_from_user(&s, argp, sizeof(s))) return -EFAULT; @@ -772,9 +818,6 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, ops->set_vq_cb(vdpa, idx, &cb); break; - case VHOST_SET_VRING_NUM: - ops->set_vq_num(vdpa, idx, vq->num); - break; } return r; @@ -1613,6 +1656,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa) } atomic_set(&v->opened, 0); + spin_lock_init(&v->config_lock); v->minor = minor; v->vdpa = vdpa; v->nvqs = vdpa->nvqs; diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 269efad90369e..f9511be1fa43f 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -344,6 +344,17 @@ static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq) vq->meta_iotlb[j] = NULL; } +/* Caller must hold the virtqueue mutex. */ +static void vhost_vq_invalidate_access(struct vhost_virtqueue *vq) +{ + vq->desc = NULL; + vq->avail = NULL; + vq->used = NULL; + vq->log_used = false; + vq->log_addr = -1ull; + __vhost_vq_meta_reset(vq); +} + static void vhost_vq_meta_reset(struct vhost_dev *d) { int i; @@ -1914,6 +1925,13 @@ int vq_meta_prefetch(struct vhost_virtqueue *vq) { unsigned int num = vq->num; + /* + * vhost_vq_invalidate_access() clears all three addresses together. + * A single zero address may be a valid GIOVA in IOTLB mode. + */ + if (!vq->desc && !vq->avail && !vq->used) + return 0; + if (!vq->iotlb) return 1; @@ -2283,6 +2301,40 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg } EXPORT_SYMBOL_GPL(vhost_vring_ioctl); +/* Caller must hold the device mutex. */ +void vhost_clear_device_iotlb(struct vhost_dev *d) +{ + struct vhost_iotlb *iotlb; + int i; + + iotlb = d->iotlb; + if (!iotlb) + return; + + vhost_dev_lock_vqs(d); + + /* + * vhost_dev_lock_vqs() takes all VQ mutexes in index order. Drop the + * device-wide view while they are held, then clear each per-VQ view + * and its cached ring access before releasing the locks. Workers + * cannot observe a mixed address-space state during this handoff. + */ + d->iotlb = NULL; + + for (i = 0; i < d->nvqs; ++i) { + struct vhost_virtqueue *vq = d->vqs[i]; + + vq->iotlb = NULL; + vhost_vq_invalidate_access(vq); + } + + vhost_dev_unlock_vqs(d); + vhost_clear_msg(d); + vhost_iotlb_free(iotlb); + wake_up_interruptible_poll(&d->wait, EPOLLIN | EPOLLRDNORM); +} +EXPORT_SYMBOL_GPL(vhost_clear_device_iotlb); + int vhost_init_device_iotlb(struct vhost_dev *d) { struct vhost_iotlb *niotlb, *oiotlb; @@ -2303,7 +2355,10 @@ int vhost_init_device_iotlb(struct vhost_dev *d) mutex_lock(&vq->mutex); vq->iotlb = niotlb; - __vhost_vq_meta_reset(vq); + if (oiotlb) + __vhost_vq_meta_reset(vq); + else + vhost_vq_invalidate_access(vq); mutex_unlock(&vq->mutex); } diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 0192ade6e7491..3c75e80893730 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -277,6 +277,7 @@ ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to, int noblock); ssize_t vhost_chr_write_iter(struct vhost_dev *dev, struct iov_iter *from); +void vhost_clear_device_iotlb(struct vhost_dev *d); int vhost_init_device_iotlb(struct vhost_dev *d); void vhost_iotlb_map_free(struct vhost_iotlb *iotlb, diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index 9aaab6bb8061c..abed1fbcf66cc 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -868,6 +868,8 @@ static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features) if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) { if (vhost_init_device_iotlb(&vsock->dev)) goto err; + } else { + vhost_clear_device_iotlb(&vsock->dev); } vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET); diff --git a/drivers/video/backlight/Kconfig b/drivers/video/backlight/Kconfig index 7aa1c4b21111f..39cb7dfb9c0fc 100644 --- a/drivers/video/backlight/Kconfig +++ b/drivers/video/backlight/Kconfig @@ -193,7 +193,7 @@ config BACKLIGHT_KTD253 config BACKLIGHT_KTD2801 tristate "Backlight Driver for Kinetic KTD2801" - depends on GPIOLIB || COMPILE_TEST + depends on GPIOLIB select LEDS_EXPRESSWIRE help Say Y to enable the backlight driver for the Kinetic KTD2801 1-wire diff --git a/drivers/video/backlight/aw99706.c b/drivers/video/backlight/aw99706.c index 18299faf06ad4..c4a66b59ecf13 100644 --- a/drivers/video/backlight/aw99706.c +++ b/drivers/video/backlight/aw99706.c @@ -60,7 +60,7 @@ #define AW99706_MTPLDOSEL_REG 0x1E #define AW99706_MTPRUN_REG 0x1F -#define RESV 0 +#define RESV U32_MAX /* Boost switching frequency table, in Hz */ static const u32 aw99706_sw_freq_tbl[] = { @@ -94,17 +94,19 @@ static int aw99706_dt_property_lookup(const struct aw99706_dt_prop *prop, int i; if (!prop->lookup_tbl) { + if (dt_val > (prop->mask >> __ffs(prop->mask))) + return -EINVAL; *val = dt_val; return 0; } for (i = 0; i < prop->tbl_size; i++) - if (prop->lookup_tbl[i] == dt_val) + if (prop->lookup_tbl[i] != RESV && prop->lookup_tbl[i] == dt_val) break; *val = i; - return i == prop->tbl_size ? -1 : 0; + return i == prop->tbl_size ? -EINVAL : 0; } #define MIN_ILED_MAX 5000 @@ -116,11 +118,14 @@ aw99706_dt_property_iled_max_convert(const struct aw99706_dt_prop *prop, u32 dt_val, u8 *val) { if (dt_val > MAX_ILED_MAX || dt_val < MIN_ILED_MAX) - return -1; + return -EINVAL; + + if ((dt_val - MIN_ILED_MAX) % STEP_ILED_MAX) + return -EINVAL; *val = (dt_val - MIN_ILED_MAX) / STEP_ILED_MAX; - return (dt_val - MIN_ILED_MAX) % STEP_ILED_MAX; + return 0; } static const struct aw99706_dt_prop aw99706_dt_props[] = { @@ -130,23 +135,23 @@ static const struct aw99706_dt_prop aw99706_dt_props[] = { AW99706_CFG0_REG, AW99706_DIM_MODE_MASK, 1, }, { - "awinic,sw-freq", aw99706_dt_property_lookup, + "awinic,sw-freq-hz", aw99706_dt_property_lookup, aw99706_sw_freq_tbl, ARRAY_SIZE(aw99706_sw_freq_tbl), AW99706_CFG1_REG, AW99706_SW_FREQ_MASK, 750000, }, { - "awinic,sw-ilmt", aw99706_dt_property_lookup, + "awinic,sw-ilmt-microamp", aw99706_dt_property_lookup, aw99706_sw_ilmt_tbl, ARRAY_SIZE(aw99706_sw_ilmt_tbl), AW99706_CFG1_REG, AW99706_SW_ILMT_MASK, 3000000, }, { - "awinic,iled-max", aw99706_dt_property_iled_max_convert, + "awinic,iled-max-microamp", aw99706_dt_property_iled_max_convert, NULL, 0, AW99706_CFG2_REG, AW99706_ILED_MAX_MASK, 20000, }, { - "awinic,uvlo-thres", aw99706_dt_property_lookup, + "awinic,uvlo-thres-microvolt", aw99706_dt_property_lookup, aw99706_ulvo_thres_tbl, ARRAY_SIZE(aw99706_ulvo_thres_tbl), AW99706_CFG2_REG, AW99706_UVLOSEL_MASK, 2200000, }, @@ -335,7 +340,7 @@ static int aw99706_bl_update_status(struct backlight_device *bl) { struct aw99706_device *aw = bl_get_data(bl); - return aw99706_update_brightness(aw, bl->props.brightness); + return aw99706_update_brightness(aw, backlight_get_brightness(bl)); } static const struct backlight_ops aw99706_bl_ops = { diff --git a/drivers/video/fbdev/clps711x-fb.c b/drivers/video/fbdev/clps711x-fb.c index 7a7db7100499e..6789773b22fbe 100644 --- a/drivers/video/fbdev/clps711x-fb.c +++ b/drivers/video/fbdev/clps711x-fb.c @@ -329,8 +329,6 @@ static int clps711x_fb_probe(struct platform_device *pdev) return 0; - unregister_framebuffer(info); - out_fb_dealloc_cmap: regmap_update_bits(cfb->syscon, SYSCON_OFFSET, SYSCON1_LCDEN, 0); fb_dealloc_cmap(&info->cmap); diff --git a/drivers/video/fbdev/kyro/fbdev.c b/drivers/video/fbdev/kyro/fbdev.c index d756b3603fa6a..c237389888229 100644 --- a/drivers/video/fbdev/kyro/fbdev.c +++ b/drivers/video/fbdev/kyro/fbdev.c @@ -369,6 +369,9 @@ static int kyro_dev_overlay_create(u32 ulWidth, static int kyro_dev_overlay_viewport_set(u32 x, u32 y, u32 ulWidth, u32 ulHeight) { + u32 right; + u32 bottom; + if (deviceInfo.ulOverlayOffset == 0) /* probably haven't called CreateOverlay yet */ return -EINVAL; @@ -378,11 +381,30 @@ static int kyro_dev_overlay_viewport_set(u32 x, u32 y, u32 ulWidth, u32 ulHeight (x < 2 && ulWidth + 2 == 0)) return -EINVAL; + /* + * SetOverlayViewPort() adjusts X coordinates by +2 (left) and +1 + * (right) before packing them into 16-bit register fields. + */ + if (x > U16_MAX - 2 || y > U16_MAX) + return -EINVAL; + + right = x + ulWidth; + bottom = y + ulHeight; + + if (right < x || bottom < y) + return -EINVAL; + + right--; + bottom--; + + if (right > U16_MAX - 1 || bottom > U16_MAX) + return -EINVAL; + /* Stop Ramdac Output */ DisableRamdacOutput(deviceInfo.pSTGReg); SetOverlayViewPort(deviceInfo.pSTGReg, - x, y, x + ulWidth - 1, y + ulHeight - 1); + x, y, right, bottom); EnableOverlayPlane(deviceInfo.pSTGReg); /* Start Ramdac Output */ diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c index 5e7963b4aa93c..8d1dcd4371139 100644 --- a/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c +++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c @@ -1150,14 +1150,14 @@ static int dsicm_probe(struct platform_device *pdev) dssdev->caps = OMAP_DSS_DISPLAY_CAP_MANUAL_UPDATE | OMAP_DSS_DISPLAY_CAP_TEAR_ELIM; + mutex_init(&ddata->lock); + r = omapdss_register_display(dssdev); if (r) { dev_err(dev, "Failed to register panel\n"); goto err_reg; } - mutex_init(&ddata->lock); - atomic_set(&ddata->do_update, 0); ddata->reset_gpio = devm_gpiod_get(&pdev->dev, "reset", GPIOD_OUT_LOW); diff --git a/drivers/video/fbdev/pvr2fb.c b/drivers/video/fbdev/pvr2fb.c index 9428716e2dc40..a6e7abca7a06f 100644 --- a/drivers/video/fbdev/pvr2fb.c +++ b/drivers/video/fbdev/pvr2fb.c @@ -639,7 +639,7 @@ static irqreturn_t __maybe_unused pvr2fb_interrupt(int irq, void *dev_id) } #ifdef CONFIG_PVR2_DMA -static ssize_t pvr2fb_write(struct fb_info *info, const char *buf, +static ssize_t pvr2fb_write(struct fb_info *info, const char __user *buf, size_t count, loff_t *ppos) { unsigned long dst, start, end, len; @@ -1077,7 +1077,7 @@ static struct pvr2_board { #ifdef CONFIG_PCI { pvr2fb_pci_init, pvr2fb_pci_exit, "PCI PVR2" }, #endif - { 0, }, + { }, }; static int __init pvr2fb_init(void) diff --git a/drivers/video/fbdev/ssd1307fb.c b/drivers/video/fbdev/ssd1307fb.c index 644b8d97b3811..c4fdecafd8560 100644 --- a/drivers/video/fbdev/ssd1307fb.c +++ b/drivers/video/fbdev/ssd1307fb.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -72,6 +73,13 @@ struct ssd1307fb_par { struct i2c_client *client; u32 height; struct fb_info *info; + /* Pending damage, with exclusive x2/y2, protected by damage_lock. */ + spinlock_t damage_lock; + bool damage_pending; + u32 damage_x1; + u32 damage_x2; + u32 damage_y1; + u32 damage_y2; u8 lookup_table[4]; u32 page_offset; u32 col_offset; @@ -302,19 +310,49 @@ static int ssd1307fb_blank(int blank_mode, struct fb_info *info) return ssd1307fb_write_cmd(par->client, SSD1307FB_DISPLAY_ON); } +static void ssd1307fb_schedule_damage(struct fb_info *info, u32 x, u32 y, + u32 width, u32 height) +{ + struct ssd1307fb_par *par = info->par; + unsigned long flags; + u32 x2, y2; + + if (!width || !height || x >= par->width || y >= par->height) + return; + + x2 = x + min(width, par->width - x); + y2 = y + min(height, par->height - y); + + spin_lock_irqsave(&par->damage_lock, flags); + if (par->damage_pending) { + par->damage_x1 = min(par->damage_x1, x); + par->damage_y1 = min(par->damage_y1, y); + par->damage_x2 = max(par->damage_x2, x2); + par->damage_y2 = max(par->damage_y2, y2); + } else { + par->damage_x1 = x; + par->damage_y1 = y; + par->damage_x2 = x2; + par->damage_y2 = y2; + par->damage_pending = true; + } + spin_unlock_irqrestore(&par->damage_lock, flags); + + /* Advance an already-pending mmap update as well. */ + mod_delayed_work(system_wq, &info->deferred_work, 0); +} + static void ssd1307fb_defio_damage_range(struct fb_info *info, off_t off, size_t len) { struct ssd1307fb_par *par = info->par; - ssd1307fb_update_display(par); + ssd1307fb_schedule_damage(info, 0, 0, par->width, par->height); } static void ssd1307fb_defio_damage_area(struct fb_info *info, u32 x, u32 y, u32 width, u32 height) { - struct ssd1307fb_par *par = info->par; - - ssd1307fb_update_rect(par, x, y, width, height); + ssd1307fb_schedule_damage(info, x, y, width, height); } FB_GEN_DEFAULT_DEFERRED_SYSMEM_OPS(ssd1307fb, @@ -329,7 +367,30 @@ static const struct fb_ops ssd1307fb_ops = { static void ssd1307fb_deferred_io(struct fb_info *info, struct list_head *pagereflist) { - ssd1307fb_update_display(info->par); + struct ssd1307fb_par *par = info->par; + unsigned long flags; + u32 x, y, width, height; + + spin_lock_irqsave(&par->damage_lock, flags); + if (!list_empty(pagereflist)) { + x = 0; + y = 0; + width = par->width; + height = par->height; + par->damage_pending = false; + } else if (par->damage_pending) { + x = par->damage_x1; + y = par->damage_y1; + width = par->damage_x2 - par->damage_x1; + height = par->damage_y2 - par->damage_y1; + par->damage_pending = false; + } else { + spin_unlock_irqrestore(&par->damage_lock, flags); + return; + } + spin_unlock_irqrestore(&par->damage_lock, flags); + + ssd1307fb_update_rect(par, x, y, width, height); } static int ssd1307fb_init(struct ssd1307fb_par *par) @@ -601,6 +662,7 @@ static int ssd1307fb_probe(struct i2c_client *client) par = info->par; par->info = info; par->client = client; + spin_lock_init(&par->damage_lock); par->device_info = device_get_match_data(dev); diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c index cc6a074f3165b..9a06cef756994 100644 --- a/drivers/video/fbdev/tdfxfb.c +++ b/drivers/video/fbdev/tdfxfb.c @@ -1385,7 +1385,7 @@ static int tdfxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) return err; - err = pci_enable_device(pdev); + err = pcim_enable_device(pdev); if (err) { printk(KERN_ERR "tdfxfb: Can't enable pdev: %d\n", err); return err; diff --git a/drivers/video/fbdev/uvesafb.c b/drivers/video/fbdev/uvesafb.c index 9d82326c744f2..ccc9dbc25813c 100644 --- a/drivers/video/fbdev/uvesafb.c +++ b/drivers/video/fbdev/uvesafb.c @@ -1907,6 +1907,8 @@ static int uvesafb_init(void) err = 0; } } + if (err) + cn_del_callback(&uvesafb_cn_id); return err; } diff --git a/drivers/video/fbdev/vfb.c b/drivers/video/fbdev/vfb.c index 5b7965f36c5ed..de137b2bdaedc 100644 --- a/drivers/video/fbdev/vfb.c +++ b/drivers/video/fbdev/vfb.c @@ -78,6 +78,13 @@ static int vfb_pan_display(struct fb_var_screeninfo *var, static int vfb_mmap(struct fb_info *info, struct vm_area_struct *vma); +static void vfb_destroy(struct fb_info *info) +{ + vfree(info->screen_buffer); + fb_dealloc_cmap(&info->cmap); + framebuffer_release(info); +} + static const struct fb_ops vfb_ops = { .owner = THIS_MODULE, __FB_DEFAULT_SYSMEM_OPS_RDWR, @@ -87,6 +94,7 @@ static const struct fb_ops vfb_ops = { .fb_pan_display = vfb_pan_display, __FB_DEFAULT_SYSMEM_OPS_DRAW, .fb_mmap = vfb_mmap, + .fb_destroy = vfb_destroy, }; /* @@ -485,9 +493,6 @@ static void vfb_remove(struct platform_device *dev) if (info) { unregister_framebuffer(info); - vfree(videomemory); - fb_dealloc_cmap(&info->cmap); - framebuffer_release(info); } } diff --git a/drivers/video/sticore.c b/drivers/video/sticore.c index 0d37e4b10447e..1d4477f204504 100644 --- a/drivers/video/sticore.c +++ b/drivers/video/sticore.c @@ -325,7 +325,7 @@ static void sti_rom_copy(unsigned long base, unsigned long count, void *dest) -static char default_sti_path[21] __read_mostly; +static char default_sti_path[32] __read_mostly; #ifndef MODULE static int __init sti_setup(char *str) @@ -1148,6 +1148,26 @@ static void sti_init_roms(void) pr_info("STI GSC/PCI core graphics driver " STI_DRIVERVERSION "\n"); + /* + * Find default console by hardware path which is either stored in + * console entry in stable storage or alternatively from console path + * in PAGE0 used by BCH and PDC. + */ + if (!default_sti_path[0]) { + struct pdc_module_path conspath; + struct device *dev = NULL; + + if (pdc_stable_read(0x60, &conspath, sizeof(conspath)) == PDC_OK) + dev = hwpath_to_device(&conspath.path); + if (!dev) + dev = hwpath_to_device(&PAGE0->mem_cons.dp.path); + if (dev && dev_is_pci(dev)) + print_pci_hwpath(to_pci_dev(dev), default_sti_path); + else if (dev && !dev_is_pci(dev)) + print_pa_hwpath(to_parisc_device(dev), default_sti_path); + pr_debug("default graphic card: %s\n", default_sti_path); + } + /* Register drivers for native & PCI cards */ register_parisc_driver(&pa_sti_driver); WARN_ON(pci_register_driver(&pci_sti_driver)); diff --git a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c index 32cd038cb79bd..dbbb2cc0e124a 100644 --- a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +++ b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c @@ -16,54 +16,38 @@ /** * struct arm_cca_token_info - a descriptor for the token buffer. - * @challenge: Pointer to the challenge data - * @challenge_size: Size of the challenge data * @granule: PA of the granule to which the token will be written * @offset: Offset within granule to start of buffer in bytes - * @result: result of rsi_attestation_token_continue operation */ struct arm_cca_token_info { - void *challenge; - unsigned long challenge_size; phys_addr_t granule; unsigned long offset; - unsigned long result; }; -static void arm_cca_attestation_init(void *param) -{ - struct arm_cca_token_info *info; - - info = (struct arm_cca_token_info *)param; - - info->result = rsi_attestation_token_init(info->challenge, - info->challenge_size); -} - /** * arm_cca_attestation_continue - Retrieve the attestation token data. * - * @param: pointer to the arm_cca_token_info + * @info: pointer to the arm_cca_token_info * * Attestation token generation is a long running operation and therefore * the token data may not be retrieved in a single call. Moreover, the * token retrieval operation must be requested on the same CPU on which the * attestation token generation was initialised. - * This helper function is therefore scheduled on the same CPU multiple + * This helper function must therefore be executed on the same CPU multiple * times until the entire token data is retrieved. */ -static void arm_cca_attestation_continue(void *param) +static unsigned long +arm_cca_attestation_continue(struct arm_cca_token_info *info) { + unsigned long ret; unsigned long len; unsigned long size; - struct arm_cca_token_info *info; - - info = (struct arm_cca_token_info *)param; size = RSI_GRANULE_SIZE - info->offset; - info->result = rsi_attestation_token_continue(info->granule, - info->offset, size, &len); + ret = rsi_attestation_token_continue(info->granule, info->offset, size, + &len); info->offset += len; + return ret; } /** @@ -74,8 +58,8 @@ static void arm_cca_attestation_continue(void *param) * * Initialise the attestation token generation using the challenge data * passed in the TSM descriptor. Allocate memory for the attestation token - * and schedule calls to retrieve the attestation token on the same CPU - * on which the attestation token generation was initialised. + * and retrieve the attestation token on the same CPU on which the + * attestation token generation was initialised. * * The challenge data must be at least 32 bytes and no more than 64 bytes. If * less than 64 bytes are provided it will be zero padded to 64 bytes. @@ -85,12 +69,11 @@ static void arm_cca_attestation_continue(void *param) * * %-EINVAL - A parameter was not valid. * * %-ENOMEM - Out of memory. * * %-EFAULT - Failed to get IPA for memory page(s). - * * A negative status code as returned by smp_call_function_single(). */ static int arm_cca_report_new(struct tsm_report *report, void *data) { - int ret; - int cpu; + int ret = 0; + unsigned long rsi_result; long max_size; unsigned long token_size = 0; struct arm_cca_token_info info; @@ -103,37 +86,33 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) /* * The attestation token 'init' and 'continue' calls must be - * performed on the same CPU. smp_call_function_single() is used - * instead of simply calling get_cpu() because of the need to - * allocate outblob based on the returned value from the 'init' - * call and that cannot be done in an atomic context. + * performed on the same CPU, so disable CPU migration around + * those operations. */ - cpu = smp_processor_id(); + migrate_disable(); - info.challenge = desc->inblob; - info.challenge_size = desc->inblob_len; - - ret = smp_call_function_single(cpu, arm_cca_attestation_init, - &info, true); - if (ret) - return ret; - max_size = info.result; - - if (max_size <= 0) - return -EINVAL; + max_size = rsi_attestation_token_init(desc->inblob, desc->inblob_len); + if (max_size <= 0) { + ret = -EINVAL; + goto exit_migrate_enable; + } /* Allocate outblob */ token = kvzalloc(max_size, GFP_KERNEL); - if (!token) - return -ENOMEM; + if (!token) { + ret = -ENOMEM; + goto exit_migrate_enable; + } /* * Since the outblob may not be physically contiguous, use a page * to bounce the buffer from RMM. */ buf = alloc_pages_exact(RSI_GRANULE_SIZE, GFP_KERNEL); - if (!buf) - return -ENOMEM; + if (!buf) { + ret = -ENOMEM; + goto exit_migrate_enable; + } /* Get the PA of the memory page(s) that were allocated */ info.granule = (unsigned long)virt_to_phys(buf); @@ -144,21 +123,15 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) info.offset = 0; do { /* - * Schedule a call to retrieve a sub-granule chunk - * of data per loop iteration. + * Retrieve a sub-granule chunk of data per loop + * iteration. */ - ret = smp_call_function_single(cpu, - arm_cca_attestation_continue, - (void *)&info, true); - if (ret != 0) { - token_size = 0; - goto exit_free_granule_page; - } - } while (info.result == RSI_INCOMPLETE && + rsi_result = arm_cca_attestation_continue(&info); + } while (rsi_result == RSI_INCOMPLETE && info.offset < RSI_GRANULE_SIZE); /* Break out in case of failure */ - if (info.result != RSI_SUCCESS && info.result != RSI_INCOMPLETE) { + if (rsi_result != RSI_SUCCESS && rsi_result != RSI_INCOMPLETE) { ret = -ENXIO; token_size = 0; goto exit_free_granule_page; @@ -173,12 +146,14 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) break; memcpy(&token[token_size], buf, info.offset); token_size += info.offset; - } while (info.result == RSI_INCOMPLETE); + } while (rsi_result == RSI_INCOMPLETE); report->outblob = no_free_ptr(token); exit_free_granule_page: report->outblob_len = token_size; free_pages_exact(buf, RSI_GRANULE_SIZE); +exit_migrate_enable: + migrate_enable(); return ret; } diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 299fa83be1d5f..b6c9e927bef57 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -401,6 +401,32 @@ static const struct cpumask *virtio_irq_get_affinity(struct device *_d, return dev->config->get_vq_affinity(dev, irq_vec); } +/** + * virtio_device_shutdown - break and reset a device on shutdown + * @dev: the device + * + * Drivers with their own .shutdown method should quiesce their activity and + * then call this to stop the device the way the generic shutdown path does. + */ +void virtio_device_shutdown(struct virtio_device *dev) +{ + /* + * Some devices get wedged if you kick them after they are + * reset. Mark all vqs as broken to make sure we don't. + */ + virtio_break_device(dev); + /* + * Guarantee that any callback will see vq->broken as true. + */ + virtio_synchronize_cbs(dev); + /* + * As IOMMUs are reset on shutdown, this will block device access to memory. + * Some devices get wedged if this happens, so reset to make sure it does not. + */ + dev->config->reset(dev); +} +EXPORT_SYMBOL_GPL(virtio_device_shutdown); + static void virtio_dev_shutdown(struct device *_d) { struct virtio_device *dev = dev_to_virtio(_d); @@ -419,20 +445,7 @@ static void virtio_dev_shutdown(struct device *_d) return; } - /* - * Some devices get wedged if you kick them after they are - * reset. Mark all vqs as broken to make sure we don't. - */ - virtio_break_device(dev); - /* - * Guarantee that any callback will see vq->broken as true. - */ - virtio_synchronize_cbs(dev); - /* - * As IOMMUs are reset on shutdown, this will block device access to memory. - * Some devices get wedged if this happens, so reset to make sure it does not. - */ - dev->config->reset(dev); + virtio_device_shutdown(dev); } static int virtio_dev_num_vf(struct device *dev) @@ -591,8 +604,8 @@ void unregister_virtio_device(struct virtio_device *dev) { int index = dev->index; /* save for after device release */ - device_unregister(&dev->dev); virtio_debug_device_exit(dev); + device_unregister(&dev->dev); ida_free(&virtio_index_ida, index); } EXPORT_SYMBOL_GPL(unregister_virtio_device); diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 581ac799d9749..70450e922d8bb 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -1095,32 +1096,51 @@ static void remove_common(struct virtio_balloon *vb) vb->vdev->config->del_vqs(vb->vdev); } -static void virtballoon_remove(struct virtio_device *vdev) +/* + * Stop all asynchronous balloon work. The device must still be alive so that + * in-flight requests can drain via the host before it is reset or freed. + */ +static void virtballoon_quiesce(struct virtio_balloon *vb) { - struct virtio_balloon *vb = vdev->priv; + struct virtio_device *vdev = vb->vdev; - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_REPORTING)) page_reporting_unregister(&vb->pr_dev_info); - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) unregister_oom_notifier(&vb->oom_nb); - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) virtio_balloon_unregister_shrinker(vb); + spin_lock_irq(&vb->stop_update_lock); vb->stop_update = true; spin_unlock_irq(&vb->stop_update_lock); cancel_work_sync(&vb->update_balloon_size_work); cancel_work_sync(&vb->update_balloon_stats_work); - if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) cancel_work_sync(&vb->report_free_page_work); +} + +static void virtballoon_remove(struct virtio_device *vdev) +{ + struct virtio_balloon *vb = vdev->priv; + + virtballoon_quiesce(vb); + + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) destroy_workqueue(vb->balloon_wq); - } remove_common(vb); mutex_destroy(&vb->balloon_lock); kfree(vb); } +static void virtballoon_shutdown(struct virtio_device *vdev) +{ + virtballoon_quiesce(vdev->priv); + virtio_device_shutdown(vdev); +} + #ifdef CONFIG_PM_SLEEP static int virtballoon_freeze(struct virtio_device *vdev) { @@ -1165,6 +1185,11 @@ static int virtballoon_validate(struct virtio_device *vdev) else if (!virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) __virtio_clear_bit(vdev, VIRTIO_BALLOON_F_REPORTING); + /* + * Disable indirect descriptors to avoid memory allocation in + * virtqueue_add during page reporting. + */ + __virtio_clear_bit(vdev, VIRTIO_RING_F_INDIRECT_DESC); __virtio_clear_bit(vdev, VIRTIO_F_ACCESS_PLATFORM); return 0; } @@ -1186,6 +1211,7 @@ static struct virtio_driver virtio_balloon_driver = { .validate = virtballoon_validate, .probe = virtballoon_probe, .remove = virtballoon_remove, + .shutdown = virtballoon_shutdown, .config_changed = virtballoon_changed, #ifdef CONFIG_PM_SLEEP .freeze = virtballoon_freeze, diff --git a/drivers/virtio/virtio_input.c b/drivers/virtio/virtio_input.c index deec24e8e6828..e3bd0b9616f94 100644 --- a/drivers/virtio/virtio_input.c +++ b/drivers/virtio/virtio_input.c @@ -49,9 +49,12 @@ static void virtinput_recv_events(struct virtqueue *vq) le16_to_cpu(event->code), le32_to_cpu(event->value)); spin_lock_irqsave(&vi->lock, flags); + if (!vi->ready) + continue; virtinput_queue_evtbuf(vi, event); } - virtqueue_kick(vq); + if (vi->ready) + virtqueue_kick(vq); } spin_unlock_irqrestore(&vi->lock, flags); } @@ -331,6 +334,7 @@ static int virtinput_probe(struct virtio_device *vdev) spin_lock_irqsave(&vi->lock, flags); vi->ready = false; spin_unlock_irqrestore(&vi->lock, flags); + virtio_reset_device(vdev); err_mt_init_slots: input_free_device(vi->idev); err_input_alloc: @@ -350,8 +354,9 @@ static void virtinput_remove(struct virtio_device *vdev) vi->ready = false; spin_unlock_irqrestore(&vi->lock, flags); - input_unregister_device(vi->idev); + /* Callbacks use vi->idev. */ virtio_reset_device(vdev); + input_unregister_device(vi->idev); while ((buf = virtqueue_detach_unused_buf(vi->sts)) != NULL) kfree(buf); vdev->config->del_vqs(vdev); diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 510b7c4efdff8..d5b148bde180a 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -87,6 +87,9 @@ struct virtio_mmio_device { void __iomem *base; unsigned long version; + + /* True if enable_irq_wake() succeeded for the shared IRQ. */ + bool wake_irq_enabled; }; /* Configuration interface */ @@ -329,11 +332,17 @@ static void vm_del_vqs(struct virtio_device *vdev) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); struct virtqueue *vq, *n; + int irq = platform_get_irq(vm_dev->pdev, 0); list_for_each_entry_safe(vq, n, &vdev->vqs, list) vm_del_vq(vq); - free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev); + if (vm_dev->wake_irq_enabled) { + disable_irq_wake(irq); + vm_dev->wake_irq_enabled = false; + } + + free_irq(irq, vm_dev); } static void vm_synchronize_cbs(struct virtio_device *vdev) @@ -460,8 +469,9 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, if (err) return err; - if (of_property_read_bool(vm_dev->pdev->dev.of_node, "wakeup-source")) - enable_irq_wake(irq); + if (of_property_read_bool(vm_dev->pdev->dev.of_node, "wakeup-source") && + !enable_irq_wake(irq)) + vm_dev->wake_irq_enabled = true; for (i = 0; i < nvqs; ++i) { struct virtqueue_info *vqi = &vqs_info[i]; diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 164f480b18a6f..b90c174450b20 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -120,7 +120,9 @@ static irqreturn_t vp_interrupt(int irq, void *opaque) if (isr & VIRTIO_PCI_ISR_CONFIG) vp_config_changed(irq, opaque); - return vp_vring_interrupt(irq, opaque); + vp_vring_interrupt(irq, opaque); + + return IRQ_HANDLED; } static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, @@ -499,7 +501,7 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, if (!avq_num) return 0; sprintf(avq->name, "avq.%u", avq->vq_index); - vq = vp_setup_vq(vdev, queue_idx++, vp_modern_avq_done, avq->name, + vq = vp_setup_vq(vdev, avq->vq_index, vp_modern_avq_done, avq->name, false, VIRTIO_MSI_NO_VECTOR, &vp_dev->admin_vq.info); if (IS_ERR(vq)) { diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index b438dc2ce1b80..a9e1ee9536b5b 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1670,7 +1670,7 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq, struct scatterlist *sg; unsigned int i, n, c, descs_used, err_idx, len; __le16 head_flags, flags; - u16 head, id, prev, curr, avail_used_flags; + u16 head, id, prev, curr, avail_used_flags, unpub_flags; int err; START_USE(vq); @@ -1798,15 +1798,30 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq, curr = vq->free_head; vq->packed.avail_used_flags = avail_used_flags; + unpub_flags = avail_used_flags ^ (1 << VRING_PACKED_DESC_F_AVAIL | + 1 << VRING_PACKED_DESC_F_USED); for (n = 0; n < total_sg; n++) { if (i == err_idx) break; + /* + * The mapping loop made every descriptor but the head + * available. Stamp the previous wrap counter's AVAIL and USED + * bits on those, so that a later and shorter chain at this head + * does not leave one of them available beyond its own last + * descriptor. Marking them used instead would hand + * is_used_desc_packed() a completion we never made. + */ + if (i != head) + desc[i].flags = cpu_to_le16(unpub_flags); vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]); curr = vq->packed.desc_extra[curr].next; i++; - if (i >= vq->packed.vring.num) + if (i >= vq->packed.vring.num) { i = 0; + unpub_flags ^= 1 << VRING_PACKED_DESC_F_AVAIL | + 1 << VRING_PACKED_DESC_F_USED; + } } END_USE(vq); @@ -1828,7 +1843,7 @@ static inline int virtqueue_add_packed_in_order(struct vring_virtqueue *vq, struct scatterlist *sg; unsigned int i, n, sg_count, err_idx, total_in_len = 0; __le16 head_flags, flags; - u16 head, avail_used_flags; + u16 head, avail_used_flags, unpub_flags; bool avail_wrap_counter; int err; @@ -1955,14 +1970,29 @@ static inline int virtqueue_add_packed_in_order(struct vring_virtqueue *vq, i = head; vq->packed.avail_used_flags = avail_used_flags; vq->packed.avail_wrap_counter = avail_wrap_counter; + unpub_flags = avail_used_flags ^ (1 << VRING_PACKED_DESC_F_AVAIL | + 1 << VRING_PACKED_DESC_F_USED); for (n = 0; n < total_sg; n++) { if (i == err_idx) break; + /* + * The mapping loop made every descriptor but the head + * available. Stamp the previous wrap counter's AVAIL and USED + * bits on those, so that a later and shorter chain at this head + * does not leave one of them available beyond its own last + * descriptor. Marking them used instead would hand + * is_used_desc_packed() a completion we never made. + */ + if (i != head) + desc[i].flags = cpu_to_le16(unpub_flags); vring_unmap_extra_packed(vq, &vq->packed.desc_extra[i]); i++; - if (i >= vq->packed.vring.num) + if (i >= vq->packed.vring.num) { i = 0; + unpub_flags ^= 1 << VRING_PACKED_DESC_F_AVAIL | + 1 << VRING_PACKED_DESC_F_USED; + } } END_USE(vq); diff --git a/drivers/virtio/virtio_rtc_driver.c b/drivers/virtio/virtio_rtc_driver.c index 4419735b0f0dc..74616ba5be119 100644 --- a/drivers/virtio/virtio_rtc_driver.c +++ b/drivers/virtio/virtio_rtc_driver.c @@ -574,8 +574,8 @@ static int viortc_msg_xfer(struct viortc_vq *vq, struct viortc_msg *msg, * read requests */ -/** timeout for clock readings, where timeouts are considered non-fatal */ -#define VIORTC_MSG_READ_TIMEOUT secs_to_jiffies(60) +/** timeout for runtime requests, where timeouts are considered non-fatal */ +#define VIORTC_MSG_TIMEOUT secs_to_jiffies(60) /** * viortc_read() - VIRTIO_RTC_REQ_READ wrapper @@ -600,7 +600,7 @@ int viortc_read(struct viortc_dev *viortc, u16 vio_clk_id, u64 *reading) VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - VIORTC_MSG_READ_TIMEOUT); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -642,7 +642,7 @@ int viortc_read_cross(struct viortc_dev *viortc, u16 vio_clk_id, u8 hw_counter, VIORTC_MSG_WRITE(hdl, hw_counter, &hw_counter); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - VIORTC_MSG_READ_TIMEOUT); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -809,7 +809,7 @@ int viortc_read_alarm(struct viortc_dev *viortc, u16 vio_clk_id, VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - 0); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -858,7 +858,7 @@ int viortc_set_alarm(struct viortc_dev *viortc, u16 vio_clk_id, u64 alarm_time, VIORTC_MSG_WRITE(hdl, flags, &flags); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - 0); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -900,7 +900,7 @@ int viortc_set_alarm_enabled(struct viortc_dev *viortc, u16 vio_clk_id, VIORTC_MSG_WRITE(hdl, flags, &flags); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - 0); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); diff --git a/drivers/w1/masters/ds2482.c b/drivers/w1/masters/ds2482.c index 0069e6f854d7f..7622c87828442 100644 --- a/drivers/w1/masters/ds2482.c +++ b/drivers/w1/masters/ds2482.c @@ -310,6 +310,10 @@ static u8 ds2482_w1_triplet(void *data, u8 dbit) mutex_unlock(&pdev->access_lock); + /* On bus error, decode to 3 (no device responded) to abort the search */ + if (status < 0) + status = 3 << 5; + /* Decode the status */ return (status >> 5); } diff --git a/drivers/w1/slaves/w1_ds28e17.c b/drivers/w1/slaves/w1_ds28e17.c index e53bc41bde3ca..b638963d4b595 100644 --- a/drivers/w1/slaves/w1_ds28e17.c +++ b/drivers/w1/slaves/w1_ds28e17.c @@ -389,6 +389,10 @@ static int w1_f19_i2c_master_transfer(struct i2c_adapter *adapter, * another simple read in that case. */ if (msgs[i+1].flags & I2C_M_RECV_LEN) { + if (msgs[i+1].buf[0] > I2C_SMBUS_BLOCK_MAX) { + i = -EPROTO; + goto error; + } result = w1_f19_i2c_read(sl, msgs[i+1].addr, &(msgs[i+1].buf[1]), msgs[i+1].buf[0]); if (result < 0) { @@ -415,6 +419,10 @@ static int w1_f19_i2c_master_transfer(struct i2c_adapter *adapter, * another simple read in that case. */ if (msgs[i].flags & I2C_M_RECV_LEN) { + if (msgs[i].buf[0] > I2C_SMBUS_BLOCK_MAX) { + i = -EPROTO; + goto error; + } result = w1_f19_i2c_read(sl, msgs[i].addr, &(msgs[i].buf[1]), diff --git a/drivers/watchdog/da9062_wdt.c b/drivers/watchdog/da9062_wdt.c index 426962547df16..4d558652e9e71 100644 --- a/drivers/watchdog/da9062_wdt.c +++ b/drivers/watchdog/da9062_wdt.c @@ -256,7 +256,7 @@ static int __maybe_unused da9062_wdt_suspend(struct device *dev) if (!wdt->use_sw_pm) return 0; - if (watchdog_active(wdd)) + if (watchdog_active(wdd) || watchdog_hw_running(wdd)) return da9062_wdt_stop(wdd); return 0; @@ -270,7 +270,7 @@ static int __maybe_unused da9062_wdt_resume(struct device *dev) if (!wdt->use_sw_pm) return 0; - if (watchdog_active(wdd)) + if (watchdog_active(wdd) || watchdog_hw_running(wdd)) return da9062_wdt_start(wdd); return 0; diff --git a/drivers/watchdog/da9063_wdt.c b/drivers/watchdog/da9063_wdt.c index 92e1b78ff4810..3703110e82fcc 100644 --- a/drivers/watchdog/da9063_wdt.c +++ b/drivers/watchdog/da9063_wdt.c @@ -271,7 +271,7 @@ static int da9063_wdt_suspend(struct device *dev) if (!da9063->use_sw_pm) return 0; - if (watchdog_active(wdd)) + if (watchdog_active(wdd) || watchdog_hw_running(wdd)) return da9063_wdt_stop(wdd); return 0; @@ -285,7 +285,7 @@ static int da9063_wdt_resume(struct device *dev) if (!da9063->use_sw_pm) return 0; - if (watchdog_active(wdd)) + if (watchdog_active(wdd) || watchdog_hw_running(wdd)) return da9063_wdt_start(wdd); return 0; diff --git a/drivers/watchdog/db8500_wdt.c b/drivers/watchdog/db8500_wdt.c index 97148ac0aa54a..70ccea13288d8 100644 --- a/drivers/watchdog/db8500_wdt.c +++ b/drivers/watchdog/db8500_wdt.c @@ -16,7 +16,7 @@ #include #include -#include +#include #define WATCHDOG_TIMEOUT 600 /* 10 minutes */ @@ -37,24 +37,24 @@ MODULE_PARM_DESC(nowayout, static int db8500_wdt_start(struct watchdog_device *wdd) { - return prcmu_enable_a9wdog(PRCMU_WDOG_ALL); + return db8500_prcmu_enable_a9wdog(PRCMU_WDOG_ALL); } static int db8500_wdt_stop(struct watchdog_device *wdd) { - return prcmu_disable_a9wdog(PRCMU_WDOG_ALL); + return db8500_prcmu_disable_a9wdog(PRCMU_WDOG_ALL); } static int db8500_wdt_keepalive(struct watchdog_device *wdd) { - return prcmu_kick_a9wdog(PRCMU_WDOG_ALL); + return db8500_prcmu_kick_a9wdog(PRCMU_WDOG_ALL); } static int db8500_wdt_set_timeout(struct watchdog_device *wdd, unsigned int timeout) { db8500_wdt_stop(wdd); - prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); + db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); db8500_wdt_start(wdd); return 0; @@ -91,10 +91,10 @@ static int db8500_wdt_probe(struct platform_device *pdev) watchdog_set_nowayout(&db8500_wdt, nowayout); /* disable auto off on sleep */ - prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false); + db8500_prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false); /* set HW initial value */ - prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); + db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); ret = devm_watchdog_register_device(dev, &db8500_wdt); if (ret) @@ -110,9 +110,9 @@ static int db8500_wdt_suspend(struct platform_device *pdev, { if (watchdog_active(&db8500_wdt)) { db8500_wdt_stop(&db8500_wdt); - prcmu_config_a9wdog(PRCMU_WDOG_CPU1, true); + db8500_prcmu_config_a9wdog(PRCMU_WDOG_CPU1, true); - prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); + db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); db8500_wdt_start(&db8500_wdt); } return 0; @@ -122,9 +122,9 @@ static int db8500_wdt_resume(struct platform_device *pdev) { if (watchdog_active(&db8500_wdt)) { db8500_wdt_stop(&db8500_wdt); - prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false); + db8500_prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false); - prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); + db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); db8500_wdt_start(&db8500_wdt); } return 0; diff --git a/drivers/watchdog/digicolor_wdt.c b/drivers/watchdog/digicolor_wdt.c index 073d37867f479..de1a3267a9723 100644 --- a/drivers/watchdog/digicolor_wdt.c +++ b/drivers/watchdog/digicolor_wdt.c @@ -25,6 +25,7 @@ struct dc_wdt { void __iomem *base; struct clk *clk; spinlock_t lock; + unsigned long rate; }; static unsigned timeout; @@ -61,7 +62,7 @@ static int dc_wdt_start(struct watchdog_device *wdog) { struct dc_wdt *wdt = watchdog_get_drvdata(wdog); - dc_wdt_set(wdt, wdog->timeout * clk_get_rate(wdt->clk)); + dc_wdt_set(wdt, wdog->timeout * wdt->rate); return 0; } @@ -79,7 +80,7 @@ static int dc_wdt_set_timeout(struct watchdog_device *wdog, unsigned int t) { struct dc_wdt *wdt = watchdog_get_drvdata(wdog); - dc_wdt_set(wdt, t * clk_get_rate(wdt->clk)); + dc_wdt_set(wdt, t * wdt->rate); wdog->timeout = t; return 0; @@ -90,7 +91,7 @@ static unsigned int dc_wdt_get_timeleft(struct watchdog_device *wdog) struct dc_wdt *wdt = watchdog_get_drvdata(wdog); uint32_t count = readl_relaxed(wdt->base + TIMER_A_COUNT); - return count / clk_get_rate(wdt->clk); + return count / wdt->rate; } static const struct watchdog_ops dc_wdt_ops = { @@ -130,7 +131,11 @@ static int dc_wdt_probe(struct platform_device *pdev) wdt->clk = devm_clk_get(dev, NULL); if (IS_ERR(wdt->clk)) return PTR_ERR(wdt->clk); - dc_wdt_wdd.max_timeout = U32_MAX / clk_get_rate(wdt->clk); + + wdt->rate = clk_get_rate(wdt->clk); + if (!wdt->rate) + return -EINVAL; + dc_wdt_wdd.max_timeout = U32_MAX / wdt->rate; dc_wdt_wdd.timeout = dc_wdt_wdd.max_timeout; dc_wdt_wdd.parent = dev; diff --git a/drivers/watchdog/msc313e_wdt.c b/drivers/watchdog/msc313e_wdt.c index d962589e2c55a..d62586f6e09d0 100644 --- a/drivers/watchdog/msc313e_wdt.c +++ b/drivers/watchdog/msc313e_wdt.c @@ -31,20 +31,39 @@ struct msc313e_wdt_priv { struct clk *clk; }; +static u32 msc313e_wdt_get_hw_timeout(struct msc313e_wdt_priv *priv) +{ + u16 low, high; + + low = readw(priv->base + REG_WDT_MAX_PRD_L); + high = readw(priv->base + REG_WDT_MAX_PRD_H); + + return ((u32)high << 16) | low; +} + +static void msc313e_wdt_set_hw_timeout(struct msc313e_wdt_priv *priv, + unsigned int timeout) +{ + u32 t = timeout * clk_get_rate(priv->clk); + + /* Clear before to prevent premature reset during non-atomic updates. */ + writew(1, priv->base + REG_WDT_CLR); + + writew(t & 0xffff, priv->base + REG_WDT_MAX_PRD_L); + writew((t >> 16) & 0xffff, priv->base + REG_WDT_MAX_PRD_H); + writew(1, priv->base + REG_WDT_CLR); +} + static int msc313e_wdt_start(struct watchdog_device *wdev) { struct msc313e_wdt_priv *priv = watchdog_get_drvdata(wdev); - u32 timeout; int err; err = clk_prepare_enable(priv->clk); if (err) return err; - timeout = wdev->timeout * clk_get_rate(priv->clk); - writew(timeout & 0xffff, priv->base + REG_WDT_MAX_PRD_L); - writew((timeout >> 16) & 0xffff, priv->base + REG_WDT_MAX_PRD_H); - writew(1, priv->base + REG_WDT_CLR); + msc313e_wdt_set_hw_timeout(priv, wdev->timeout); return 0; } @@ -60,6 +79,9 @@ static int msc313e_wdt_stop(struct watchdog_device *wdev) { struct msc313e_wdt_priv *priv = watchdog_get_drvdata(wdev); + /* Clear before to prevent premature reset during non-atomic updates. */ + writew(1, priv->base + REG_WDT_CLR); + writew(0, priv->base + REG_WDT_MAX_PRD_L); writew(0, priv->base + REG_WDT_MAX_PRD_H); writew(0, priv->base + REG_WDT_CLR); @@ -69,9 +91,13 @@ static int msc313e_wdt_stop(struct watchdog_device *wdev) static int msc313e_wdt_settimeout(struct watchdog_device *wdev, unsigned int new_time) { + struct msc313e_wdt_priv *priv = watchdog_get_drvdata(wdev); + wdev->timeout = new_time; - return msc313e_wdt_start(wdev); + if (watchdog_hw_running(wdev) || watchdog_active(wdev)) + msc313e_wdt_set_hw_timeout(priv, wdev->timeout); + return 0; } static const struct watchdog_info msc313e_wdt_ident = { @@ -97,6 +123,8 @@ static int msc313e_wdt_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct msc313e_wdt_priv *priv; + unsigned long rate; + int ret; priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); if (!priv) @@ -116,27 +144,51 @@ static int msc313e_wdt_probe(struct platform_device *pdev) priv->wdev.ops = &msc313e_wdt_ops, priv->wdev.parent = dev; priv->wdev.min_timeout = MSC313E_WDT_MIN_TIMEOUT; - priv->wdev.max_timeout = U32_MAX / clk_get_rate(priv->clk); + rate = clk_get_rate(priv->clk); + if (!rate) + return -EINVAL; + priv->wdev.max_timeout = U32_MAX / rate; priv->wdev.timeout = MSC313E_WDT_DEFAULT_TIMEOUT; - /* If the period is non-zero the WDT is running */ - if (readw(priv->base + REG_WDT_MAX_PRD_L) | (readw(priv->base + REG_WDT_MAX_PRD_H) << 16)) - set_bit(WDOG_HW_RUNNING, &priv->wdev.status); - watchdog_set_drvdata(&priv->wdev, priv); + platform_set_drvdata(pdev, priv); watchdog_init_timeout(&priv->wdev, timeout, dev); watchdog_stop_on_reboot(&priv->wdev); watchdog_stop_on_unregister(&priv->wdev); + watchdog_stop_ping_on_suspend(&priv->wdev); - return devm_watchdog_register_device(dev, &priv->wdev); + ret = clk_prepare_enable(priv->clk); + if (ret) + return ret; + + /* If the period is non-zero the WDT is running */ + if (msc313e_wdt_get_hw_timeout(priv)) { + msc313e_wdt_set_hw_timeout(priv, priv->wdev.timeout); + set_bit(WDOG_HW_RUNNING, &priv->wdev.status); + /* + * Keep the clock enabled. The watchdog core will skip the next + * start() and a future stop() will balance the CCF reference + * count. + */ + } else { + clk_disable_unprepare(priv->clk); + } + + ret = devm_watchdog_register_device(dev, &priv->wdev); + + /* If the WDT is running and anything goes wrong, disable the clock. */ + if (ret && test_bit(WDOG_HW_RUNNING, &priv->wdev.status)) + clk_disable_unprepare(priv->clk); + + return ret; } static int __maybe_unused msc313e_wdt_suspend(struct device *dev) { struct msc313e_wdt_priv *priv = dev_get_drvdata(dev); - if (watchdog_active(&priv->wdev)) + if (watchdog_active(&priv->wdev) || watchdog_hw_running(&priv->wdev)) msc313e_wdt_stop(&priv->wdev); return 0; @@ -145,11 +197,15 @@ static int __maybe_unused msc313e_wdt_suspend(struct device *dev) static int __maybe_unused msc313e_wdt_resume(struct device *dev) { struct msc313e_wdt_priv *priv = dev_get_drvdata(dev); + int ret = 0; - if (watchdog_active(&priv->wdev)) - msc313e_wdt_start(&priv->wdev); + if (watchdog_active(&priv->wdev) || watchdog_hw_running(&priv->wdev)) { + ret = msc313e_wdt_start(&priv->wdev); + if (ret) + dev_err(dev, "Failed to restart watchdog (err=%d)\n", ret); + } - return 0; + return ret; } static SIMPLE_DEV_PM_OPS(msc313e_wdt_pm_ops, msc313e_wdt_suspend, msc313e_wdt_resume); diff --git a/drivers/watchdog/pseries-wdt.c b/drivers/watchdog/pseries-wdt.c index 48d67f7c972a3..e97b943e1d3c5 100644 --- a/drivers/watchdog/pseries-wdt.c +++ b/drivers/watchdog/pseries-wdt.c @@ -12,61 +12,10 @@ #include #include #include +#include #define DRV_NAME "pseries-wdt" -/* - * H_WATCHDOG Input - * - * R4: "flags": - * - * Bits 48-55: "operation" - */ -#define PSERIES_WDTF_OP_START 0x100UL /* start timer */ -#define PSERIES_WDTF_OP_STOP 0x200UL /* stop timer */ -#define PSERIES_WDTF_OP_QUERY 0x300UL /* query timer capabilities */ - -/* - * Bits 56-63: "timeoutAction" (for "Start Watchdog" only) - */ -#define PSERIES_WDTF_ACTION_HARD_POWEROFF 0x1UL /* poweroff */ -#define PSERIES_WDTF_ACTION_HARD_RESTART 0x2UL /* restart */ -#define PSERIES_WDTF_ACTION_DUMP_RESTART 0x3UL /* dump + restart */ - -/* - * H_WATCHDOG Output - * - * R3: Return code - * - * H_SUCCESS The operation completed. - * - * H_BUSY The hypervisor is too busy; retry the operation. - * - * H_PARAMETER The given "flags" are somehow invalid. Either the - * "operation" or "timeoutAction" is invalid, or a - * reserved bit is set. - * - * H_P2 The given "watchdogNumber" is zero or exceeds the - * supported maximum value. - * - * H_P3 The given "timeoutInMs" is below the supported - * minimum value. - * - * H_NOOP The given "watchdogNumber" is already stopped. - * - * H_HARDWARE The operation failed for ineffable reasons. - * - * H_FUNCTION The H_WATCHDOG hypercall is not supported by this - * hypervisor. - * - * R4: - * - * - For the "Query Watchdog Capabilities" operation, a 64-bit - * structure: - */ -#define PSERIES_WDTQ_MIN_TIMEOUT(cap) (((cap) >> 48) & 0xffff) -#define PSERIES_WDTQ_MAX_NUMBER(cap) (((cap) >> 32) & 0xffff) - static const unsigned long pseries_wdt_action[] = { [0] = PSERIES_WDTF_ACTION_HARD_POWEROFF, [1] = PSERIES_WDTF_ACTION_HARD_RESTART, diff --git a/drivers/watchdog/rtd119x_wdt.c b/drivers/watchdog/rtd119x_wdt.c index 984905695dde5..0bfadb58917b9 100644 --- a/drivers/watchdog/rtd119x_wdt.c +++ b/drivers/watchdog/rtd119x_wdt.c @@ -98,6 +98,7 @@ static int rtd119x_wdt_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct rtd119x_watchdog_device *data; + unsigned long rate; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -111,10 +112,14 @@ static int rtd119x_wdt_probe(struct platform_device *pdev) if (IS_ERR(data->clk)) return PTR_ERR(data->clk); + rate = clk_get_rate(data->clk); + if (!rate) + return -EINVAL; + data->wdt_dev.info = &rtd119x_wdt_info; data->wdt_dev.ops = &rtd119x_wdt_ops; data->wdt_dev.timeout = 120; - data->wdt_dev.max_timeout = 0xffffffff / clk_get_rate(data->clk); + data->wdt_dev.max_timeout = 0xffffffff / rate; data->wdt_dev.min_timeout = 1; data->wdt_dev.parent = dev; diff --git a/drivers/watchdog/rzv2h_wdt.c b/drivers/watchdog/rzv2h_wdt.c index 3b6abb66a1da0..83540dd9a37b6 100644 --- a/drivers/watchdog/rzv2h_wdt.c +++ b/drivers/watchdog/rzv2h_wdt.c @@ -278,6 +278,7 @@ static int rzv2h_wdt_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct rzv2h_wdt_priv *priv; struct clk *count_clk; + unsigned long rate; int ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); @@ -314,8 +315,12 @@ static int rzv2h_wdt_probe(struct platform_device *pdev) return dev_err_probe(dev, -EINVAL, "Invalid count source\n"); } + rate = clk_get_rate(count_clk); + if (!rate) + return dev_err_probe(dev, -EINVAL, "Invalid clock rate\n"); + priv->wdev.max_hw_heartbeat_ms = (MILLI * priv->of_data->timeout_cycles * - priv->of_data->cks_div) / clk_get_rate(count_clk); + priv->of_data->cks_div) / rate; dev_dbg(dev, "max hw timeout of %dms\n", priv->wdev.max_hw_heartbeat_ms); ret = devm_pm_runtime_enable(dev); diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c index 7e99c3b1f3676..72ad59649ccc4 100644 --- a/drivers/watchdog/sp5100_tco.c +++ b/drivers/watchdog/sp5100_tco.c @@ -605,8 +605,10 @@ static int __init sp5100_tco_init(void) pr_info("SP5100/SB800 TCO WatchDog Timer Driver\n"); err = platform_driver_register(&sp5100_tco_driver); - if (err) + if (err) { + pci_dev_put(sp5100_tco_pci); return err; + } sp5100_tco_platform_device = platform_device_register_simple(TCO_DRIVER_NAME, -1, NULL, 0); @@ -619,6 +621,7 @@ static int __init sp5100_tco_init(void) unreg_platform_driver: platform_driver_unregister(&sp5100_tco_driver); + pci_dev_put(sp5100_tco_pci); return err; } @@ -626,6 +629,7 @@ static void __exit sp5100_tco_exit(void) { platform_device_unregister(sp5100_tco_platform_device); platform_driver_unregister(&sp5100_tco_driver); + pci_dev_put(sp5100_tco_pci); } module_init(sp5100_tco_init); diff --git a/drivers/watchdog/starfive-wdt.c b/drivers/watchdog/starfive-wdt.c index af55adc4a3c69..5a3254c83d3bb 100644 --- a/drivers/watchdog/starfive-wdt.c +++ b/drivers/watchdog/starfive-wdt.c @@ -371,7 +371,7 @@ static void starfive_wdt_stop(struct starfive_wdt *wdt) static int starfive_wdt_pm_start(struct watchdog_device *wdd) { struct starfive_wdt *wdt = watchdog_get_drvdata(wdd); - int ret = pm_runtime_get_sync(wdd->parent); + int ret = pm_runtime_resume_and_get(wdd->parent); if (ret < 0) return ret; diff --git a/drivers/watchdog/sunxi_wdt.c b/drivers/watchdog/sunxi_wdt.c index b6c761acc3de6..3db34524ed13b 100644 --- a/drivers/watchdog/sunxi_wdt.c +++ b/drivers/watchdog/sunxi_wdt.c @@ -128,6 +128,38 @@ static int sunxi_wdt_ping(struct watchdog_device *wdt_dev) return 0; } +static bool sunxi_wdt_is_running(struct watchdog_device *wdt_dev) +{ + struct sunxi_wdt_dev *sunxi_wdt = watchdog_get_drvdata(wdt_dev); + const struct sunxi_wdt_reg *regs = sunxi_wdt->wdt_regs; + + return readl(sunxi_wdt->wdt_base + regs->wdt_mode) & WDT_MODE_EN; +} + +static unsigned int sunxi_wdt_get_timeout(struct watchdog_device *wdt_dev) +{ + struct sunxi_wdt_dev *sunxi_wdt = watchdog_get_drvdata(wdt_dev); + const struct sunxi_wdt_reg *regs = sunxi_wdt->wdt_regs; + unsigned int timeout; + u32 interval; + + interval = readl(sunxi_wdt->wdt_base + regs->wdt_mode); + interval >>= regs->wdt_timeout_shift; + interval &= WDT_TIMEOUT_MASK; + /* Round the 0.5-second interval up to the minimum representable timeout. */ + if (!interval) + return WDT_MIN_TIMEOUT; + + for (timeout = WDT_MIN_TIMEOUT; + timeout < ARRAY_SIZE(wdt_timeout_map); timeout++) { + if (wdt_timeout_map[timeout] == interval) + return timeout; + } + + /* Reserved interval encoding. */ + return 0; +} + static int sunxi_wdt_set_timeout(struct watchdog_device *wdt_dev, unsigned int timeout) { @@ -259,6 +291,7 @@ static int sunxi_wdt_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct sunxi_wdt_dev *sunxi_wdt; + unsigned int running_timeout; int err; sunxi_wdt = devm_kzalloc(dev, sizeof(*sunxi_wdt), GFP_KERNEL); @@ -286,7 +319,17 @@ static int sunxi_wdt_probe(struct platform_device *pdev) watchdog_set_drvdata(&sunxi_wdt->wdt_dev, sunxi_wdt); - sunxi_wdt_stop(&sunxi_wdt->wdt_dev); + if (sunxi_wdt_is_running(&sunxi_wdt->wdt_dev)) { + running_timeout = sunxi_wdt_get_timeout(&sunxi_wdt->wdt_dev); + if (running_timeout) + sunxi_wdt->wdt_dev.timeout = running_timeout; + + err = sunxi_wdt_start(&sunxi_wdt->wdt_dev); + if (err) + return err; + + set_bit(WDOG_HW_RUNNING, &sunxi_wdt->wdt_dev.status); + } watchdog_stop_on_reboot(&sunxi_wdt->wdt_dev); err = devm_watchdog_register_device(dev, &sunxi_wdt->wdt_dev); diff --git a/drivers/watchdog/watchdog_hrtimer_pretimeout.c b/drivers/watchdog/watchdog_hrtimer_pretimeout.c index fbc7eecd8b203..49a05ea60c979 100644 --- a/drivers/watchdog/watchdog_hrtimer_pretimeout.c +++ b/drivers/watchdog/watchdog_hrtimer_pretimeout.c @@ -30,6 +30,7 @@ void watchdog_hrtimer_pretimeout_init(struct watchdog_device *wdd) void watchdog_hrtimer_pretimeout_start(struct watchdog_device *wdd) { if (!(wdd->info->options & WDIOF_PRETIMEOUT) && + wdd->pretimeout && !watchdog_pretimeout_invalid(wdd, wdd->pretimeout)) hrtimer_start(&wdd->wd_data->pretimeout_timer, ktime_set(wdd->timeout - wdd->pretimeout, 0), diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index e7f1d4ca6d753..e7f74ea7cd5eb 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -679,7 +679,7 @@ void xen_free_ballooned_pages(unsigned int nr_pages, struct page **pages) } EXPORT_SYMBOL(xen_free_ballooned_pages); -static int __init balloon_add_regions(void) +static int __init balloon_add_regions(bool append) { unsigned long start_pfn, pages; unsigned long pfn, extra_pfn_end; @@ -703,19 +703,26 @@ static int __init balloon_add_regions(void) balloon_append(pfn_to_page(pfn)); /* - * Extra regions are accounted for in the physmap, but need - * decreasing from current_pages and target_pages to balloon - * down the initial allocation, because they are already - * accounted for in total_pages. + * There are two different use-cases depending on how the + * initial memory target is fetched. For PVH dom0 and PV the + * target is usually set to reflect the domain assigned memory, + * and hence extra regions need adding. + * + * OTOH for HVM and PVH domU the target is set to the amount of + * RAM reported in the memory map, and hence extra regions need + * subtracting to reflect the real memory usage. */ pages = extra_pfn_end - start_pfn; - if (pages >= balloon_stats.current_pages || - pages >= balloon_stats.target_pages) { + if (append) { + balloon_stats.total_pages += pages; + } else if (pages >= balloon_stats.current_pages || + pages >= balloon_stats.target_pages) { WARN(1, "Extra pages underflow current target"); return -ERANGE; + } else { + balloon_stats.current_pages -= pages; + balloon_stats.target_pages -= pages; } - balloon_stats.current_pages -= pages; - balloon_stats.target_pages -= pages; } return 0; @@ -726,6 +733,7 @@ static int __init balloon_init(void) struct task_struct *task; long current_pages = 0; domid_t domid = DOMID_SELF; + bool append = true; int rc; if (!xen_domain()) @@ -745,6 +753,7 @@ static int __init balloon_init(void) } else { if (xen_unpopulated_pages >= get_num_physpages()) goto underflow; + append = false; current_pages = get_num_physpages() - xen_unpopulated_pages; } @@ -767,7 +776,7 @@ static int __init balloon_init(void) register_sysctl_init("xen/balloon", balloon_table); #endif - rc = balloon_add_regions(); + rc = balloon_add_regions(append); if (rc) return rc; diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c index fafb2b84fa5c4..9371f88ac5588 100644 --- a/drivers/xen/xenbus/xenbus_probe.c +++ b/drivers/xen/xenbus/xenbus_probe.c @@ -681,7 +681,8 @@ void xenbus_dev_changed(const char *node, struct xen_bus_type *bus) dev->otherend_id); if (state == XenbusStateInitialising && - (state != dev->state || backend != dev->otherend_id)) { + (state != dev->state || + (dev->otherend && backend != dev->otherend_id))) { /* * State has been reset, assume the old one vanished * and new one needs to be probed. diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c index d1cca4acb6f33..62274d377b3aa 100644 --- a/drivers/xen/xenbus/xenbus_xs.c +++ b/drivers/xen/xenbus/xenbus_xs.c @@ -915,19 +915,27 @@ int xs_init(void) int err; struct task_struct *task; - register_reboot_notifier(&xs_reboot_nb); + err = register_reboot_notifier(&xs_reboot_nb); + if (err) + return err; /* Initialize the shared memory rings to talk to xenstored */ err = xb_init_comms(); if (err) - return err; + goto err_unregister_reboot_notifier; task = kthread_run(xenwatch_thread, NULL, "xenwatch"); - if (IS_ERR(task)) - return PTR_ERR(task); + if (IS_ERR(task)) { + err = PTR_ERR(task); + goto err_unregister_reboot_notifier; + } /* shutdown watches for kexec boot */ xs_reset_watches(); return 0; + +err_unregister_reboot_notifier: + unregister_reboot_notifier(&xs_reboot_nb); + return err; } diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c index 1ac0b3dcc0778..13cf87a5f90cb 100644 --- a/fs/9p/vfs_addr.c +++ b/fs/9p/vfs_addr.c @@ -54,11 +54,37 @@ static void v9fs_begin_writeback(struct netfs_io_request *wreq) static void v9fs_issue_write(struct netfs_io_subrequest *subreq) { struct p9_fid *fid = subreq->rreq->netfs_priv; + struct inode *inode = subreq->rreq->inode; + struct netfs_inode *ictx = netfs_inode(inode); int err, len; len = p9_client_write(fid, subreq->start, &subreq->io_iter, &err); - if (len > 0) + if (len > 0) { + uoff_t end = subreq->start + len, i_size, remote, zp; + bool set = false; + + spin_lock(&inode->i_lock); + + /* We can read the sizes directly as we hold i_lock. */ + i_size = inode->i_size; + remote = ictx->_remote_i_size; + zp = ictx->_zero_point; + + if (end > i_size) { + i_size = end; + set = true; + } + if (end > remote) { + remote = end; + set = true; + } + + if (set) + netfs_write_sizes(inode, i_size, remote, zp); + spin_unlock(&inode->i_lock); + __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + } netfs_write_subrequest_terminated(subreq, len ?: err); } diff --git a/fs/afs/addr_list.c b/fs/afs/addr_list.c index 63bf096b721aa..73195d76b481a 100644 --- a/fs/afs/addr_list.c +++ b/fs/afs/addr_list.c @@ -394,8 +394,11 @@ void afs_set_peer_appdata(struct afs_server *server, struct rxrpc_peer *pn = new_alist->addrs[n].peer; struct rxrpc_peer *po = old_alist->addrs[o].peer; - if (pn == po) + if (pn == po) { + n++; + o++; continue; + } if (pn < po) { rxrpc_kernel_set_peer_data(pn, data); n++; diff --git a/fs/afs/dir_edit.c b/fs/afs/dir_edit.c index 3ead36a070487..c31303059444a 100644 --- a/fs/afs/dir_edit.c +++ b/fs/afs/dir_edit.c @@ -442,7 +442,7 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, /* Check and clear the entry. */ de = &block->dirents[slot]; if (de->u.valid != 1) - goto error_unmap; + goto error; trace_afs_edit_dir(vnode, why, afs_edit_dir_delete, b, slot, ntohl(de->u.vnode), ntohl(de->u.unique), @@ -458,7 +458,6 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, /* Clear the constituent entries. */ next = de->u.hash_next; memset(de, 0, sizeof(*de) * iter.nr_slots); - kunmap_local(block); /* Adjust the hash chain: if iter->prev_entry is 0, the hashtable head * index is previous; otherwise it's slot number of the previous entry. @@ -485,7 +484,6 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, pde = &pblock->dirents[ps]; prev_next = pde->u.hash_next; if (prev_next != htons(entry)) { - kunmap_local(pblock); pr_warn("%llx:%llx:%x: not prev in chain b=%x p=%x,%x e=%x %*s", vnode->fid.vid, vnode->fid.vnode, vnode->fid.unique, iter.bucket, iter.prev_entry, prev_next, entry, @@ -493,7 +491,6 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, goto error; } pde->u.hash_next = next; - kunmap_local(pblock); } netfs_single_mark_inode_dirty(&vnode->netfs.inode); @@ -503,18 +500,16 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, _debug("Remove %s from %u[%u]", name->name, b, slot); out_unmap: + afs_dir_end_iter(&iter); kunmap_local(meta); _leave(""); return; already_invalidated: - kunmap_local(block); trace_afs_edit_dir(vnode, why, afs_edit_dir_delete_inval, 0, 0, 0, 0, name->name); goto out_unmap; -error_unmap: - kunmap_local(block); error: trace_afs_edit_dir(vnode, why, afs_edit_dir_delete_error, 0, 0, 0, 0, name->name); diff --git a/fs/afs/dir_search.c b/fs/afs/dir_search.c index 104411c0692f5..11ebdfffcb1d3 100644 --- a/fs/afs/dir_search.c +++ b/fs/afs/dir_search.c @@ -75,10 +75,7 @@ union afs_xdr_dir_block *afs_dir_find_block(struct afs_dir_iter *iter, size_t bl _enter("%zx,%d", block, slot); - if (iter->block) { - kunmap_local(iter->block); - iter->block = NULL; - } + afs_dir_end_iter(iter); if (dvnode->directory_size < blend) goto fail; @@ -173,12 +170,8 @@ int afs_dir_search_bucket(struct afs_dir_iter *iter, const struct qstr *name, ret = -ENOENT; found: - if (iter->block) { - kunmap_local(iter->block); - iter->block = NULL; - } - bad: + afs_dir_end_iter(iter); if (ret == -ESTALE) afs_invalidate_dir(iter->dvnode, afs_dir_invalid_iter_stale); _leave(" = %d", ret); diff --git a/fs/afs/fs_probe.c b/fs/afs/fs_probe.c index a91ad1938d078..8c62334dbfe72 100644 --- a/fs/afs/fs_probe.c +++ b/fs/afs/fs_probe.c @@ -258,6 +258,7 @@ int afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server, lockdep_is_held(&server->fs_lock)); if (old) { estate->responsive_set = old->responsive_set; + old_alist = old->addresses; if (!new_alist) new_alist = old->addresses; } diff --git a/fs/afs/internal.h b/fs/afs/internal.h index 290873bac89b3..330654ed16ece 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -1133,6 +1133,14 @@ int afs_dir_search_bucket(struct afs_dir_iter *iter, const struct qstr *name, int afs_dir_search(struct afs_vnode *dvnode, const struct qstr *name, struct afs_fid *_fid, afs_dataversion_t *_dir_version); +static inline void afs_dir_end_iter(struct afs_dir_iter *iter) +{ + if (iter->block) { + kunmap_local(iter->block); + iter->block = NULL; + } +} + /* * dir_silly.c */ diff --git a/fs/afs/server.c b/fs/afs/server.c index 0fe162ea2a36d..189138bd6d712 100644 --- a/fs/afs/server.c +++ b/fs/afs/server.c @@ -242,7 +242,6 @@ struct afs_server *afs_lookup_server(struct afs_cell *cell, struct key *key, out: afs_put_addrlist(alist, afs_alist_trace_put_server_create); if (candidate) { - kfree(rcu_access_pointer(server->endpoint_state)); kfree(candidate); afs_dec_servers_outstanding(cell->net); } diff --git a/fs/autofs/inode.c b/fs/autofs/inode.c index c1e210cec4369..066c16f2ea569 100644 --- a/fs/autofs/inode.c +++ b/fs/autofs/inode.c @@ -51,6 +51,10 @@ void autofs_kill_sb(struct super_block *sb) if (sbi) { /* Free wait queues, close pipe */ autofs_catatonic_mode(sbi); + if (sbi->pipe) { + fput(sbi->pipe); + sbi->pipe = NULL; + } put_pid(sbi->oz_pgrp); } @@ -323,8 +327,10 @@ static int autofs_fill_super(struct super_block *s, struct fs_context *fc) return -ENOMEM; root_inode = autofs_get_inode(s, S_IFDIR | 0755); - if (!root_inode) + if (!root_inode) { + autofs_free_ino(ino); return -ENOMEM; + } root_inode->i_uid = ctx->uid; root_inode->i_gid = ctx->gid; diff --git a/fs/backing-file.c b/fs/backing-file.c index 080c99696cd0d..cc101143f9212 100644 --- a/fs/backing-file.c +++ b/fs/backing-file.c @@ -35,7 +35,7 @@ struct file *backing_file_open(const struct file *user_file, int flags, const struct path *real_path, const struct cred *cred) { - const struct path *user_path = &user_file->f_path; + const struct path *user_path = file_user_path(user_file); struct file *f; int error; diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c index f1863a891db64..d77604a8996bd 100644 --- a/fs/bpf_fs_kfuncs.c +++ b/fs/bpf_fs_kfuncs.c @@ -419,10 +419,6 @@ BTF_ID(func, bpf_lsm_inode_rmdir) BTF_ID(func, bpf_lsm_inode_setattr) BTF_ID(func, bpf_lsm_inode_setxattr) BTF_ID(func, bpf_lsm_inode_unlink) -#ifdef CONFIG_SECURITY_PATH -BTF_ID(func, bpf_lsm_path_unlink) -BTF_ID(func, bpf_lsm_path_rmdir) -#endif /* CONFIG_SECURITY_PATH */ BTF_SET_END(d_inode_locked_hooks) bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog) diff --git a/fs/btrfs/acl.c b/fs/btrfs/acl.c index e55b686fe1ab8..662cdd1cbdef6 100644 --- a/fs/btrfs/acl.c +++ b/fs/btrfs/acl.c @@ -15,6 +15,7 @@ #include "xattr.h" #include "acl.h" #include "misc.h" +#include "btrfs_inode.h" struct posix_acl *btrfs_get_acl(struct inode *inode, int type, bool rcu) { @@ -107,6 +108,9 @@ int btrfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, struct inode *inode = d_inode(dentry); umode_t old_mode = inode->i_mode; + if (btrfs_root_readonly(BTRFS_I(inode)->root)) + return -EROFS; + if (type == ACL_TYPE_ACCESS && acl) { ret = posix_acl_update_mode(idmap, inode, &inode->i_mode, &acl); diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 8def7abb728f7..c5bcd3c03d247 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -2047,6 +2047,11 @@ static int btrfs_reclaim_block_group(struct btrfs_block_group *bg, int *reclaime trace_btrfs_reclaim_block_group(bg); ret = btrfs_relocate_chunk(fs_info, bg->start, false); + if (btrfs_is_zoned(fs_info) && ret == -EAGAIN) { + btrfs_dec_block_group_ro(bg); + btrfs_debug(fs_info, "deferring reclaim of chunk %llu", bg->start); + return ret; + } if (ret) { btrfs_dec_block_group_ro(bg); btrfs_err(fs_info, "error relocating chunk %llu", @@ -2113,7 +2118,8 @@ void btrfs_reclaim_block_groups(struct btrfs_fs_info *fs_info, unsigned int limi spin_unlock(&fs_info->unused_bgs_lock); ret = btrfs_reclaim_block_group(bg, &reclaimed); - if (ret && !READ_ONCE(space_info->periodic_reclaim)) + if ((btrfs_is_zoned(fs_info) && ret == -EAGAIN) || + (ret && !READ_ONCE(space_info->periodic_reclaim))) btrfs_link_bg_list(bg, &retry_list); btrfs_put_block_group(bg); diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index f0c6758b7055d..0697b285e05fb 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -1130,20 +1130,15 @@ static_assert(PAGE_ALIGNED(CLUSTER_SIZE)); * * - Extent bits are locked */ -static int defrag_one_locked_target(struct btrfs_inode *inode, - struct defrag_target_range *target, - struct folio **folios, int nr_pages, - struct extent_state **cached_state) +static void defrag_one_locked_target(struct btrfs_inode *inode, + struct defrag_target_range *target, + struct folio **folios, int nr_pages, + struct extent_state **cached_state) { struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct extent_changeset *data_reserved = NULL; const u64 start = target->start; const u64 len = target->len; - int ret = 0; - ret = btrfs_delalloc_reserve_space(inode, &data_reserved, start, len); - if (ret < 0) - return ret; btrfs_clear_extent_bit(&inode->io_tree, start, start + len - 1, EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, cached_state); @@ -1164,10 +1159,6 @@ static int defrag_one_locked_target(struct btrfs_inode *inode, continue; btrfs_folio_clamp_set_dirty(fs_info, folio, start, len); } - btrfs_delalloc_release_extents(inode, len); - extent_changeset_free(data_reserved); - - return ret; } static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, @@ -1183,6 +1174,8 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, u64 cur = start; const unsigned int nr_pages = ((start + len - 1) >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; + struct extent_changeset *data_reserved = NULL; + u64 last_defrag_end = start; int ret = 0; ASSERT(nr_pages <= CLUSTER_SIZE / PAGE_SIZE); @@ -1192,6 +1185,22 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, if (!folios) return -ENOMEM; + /* + * Reserve delalloc space before locking the range and before locking + * and dirtying any folios - otherwise we could deadlock, for example + * after defrag of one range we dirty folios and keep them locked when + * we move to the next range, so reserving delalloc space right before + * each range could trigger flushing of delalloc and deadlock on the + * extent lock or trigger a transaction commit with flushoncommit, which + * can either deadlock on the lock of a folio made dirty in the previous + * range or the extent lock. + */ + ret = btrfs_delalloc_reserve_space(inode, &data_reserved, start, len); + if (ret < 0) { + kfree(folios); + return ret; + } + /* Prepare all pages */ for (int i = 0; cur < start + len && i < nr_pages; i++) { folios[i] = defrag_prepare_one_folio(inode, cur >> PAGE_SHIFT); @@ -1226,10 +1235,11 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, goto unlock_extent; list_for_each_entry(entry, &target_list, list) { - ret = defrag_one_locked_target(inode, entry, folios, nr_pages, - &cached_state); - if (ret < 0) - break; + defrag_one_locked_target(inode, entry, folios, nr_pages, &cached_state); + if (entry->start > last_defrag_end) + btrfs_delalloc_release_space(inode, data_reserved, last_defrag_end, + entry->start - last_defrag_end, true); + last_defrag_end = entry->start + entry->len; } list_for_each_entry_safe(entry, tmp, &target_list, list) { @@ -1246,6 +1256,12 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, folio_put(folios[i]); } kfree(folios); + btrfs_delalloc_release_extents(inode, len); + if (last_defrag_end < start + len) + btrfs_delalloc_release_space(inode, data_reserved, last_defrag_end, + start + len - last_defrag_end, true); + extent_changeset_free(data_reserved); + return ret; } diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index 318ddb7904292..7e78215b2049b 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -493,6 +493,7 @@ static int mark_block_group_to_copy(struct btrfs_fs_info *fs_info, path->reada = READA_FORWARD; path->search_commit_root = true; path->skip_locking = true; + path->need_commit_sem = true; key.objectid = src_dev->devid; key.type = BTRFS_DEV_EXTENT_KEY; diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 460326d34143c..19a1259b3b2f2 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -281,17 +281,24 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, em2 = btrfs_create_dio_extent(BTRFS_I(inode), dio_data, start, &file_extent, type); btrfs_dec_nocow_writers(bg); - if (type == BTRFS_ORDERED_PREALLOC) { - btrfs_free_extent_map(em); - *map = em2; - em = em2; - } - if (IS_ERR(em2)) { ret = PTR_ERR(em2); + btrfs_free_extent_map(em); + *map = NULL; goto out; } + /* + * True NOCOW writes don't need to create a new extent map, + * while PREALLOC writes must replace the existing one. + */ + if (em2) { + ASSERT(type == BTRFS_ORDERED_PREALLOC); + btrfs_free_extent_map(em); + *map = em2; + em = em2; + } + dio_data->nocow_done = true; } else { /* Our caller expects us to free the input extent map. */ diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 2f1666d9544e1..12b771bda43aa 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -271,14 +271,15 @@ int btree_csum_one_bio(struct btrfs_bio *bbio) return -EIO; /* - * If an extent_buffer is marked as EXTENT_BUFFER_ZONED_ZEROOUT, don't - * checksum it but zero-out its content. This is done to preserve - * ordering of I/O without unnecessarily writing out data. + * An extent_buffer marked EXTENT_BUFFER_ZONED_ZEROOUT is written out as + * zeros to preserve ordering of I/O without persisting the now + * unnecessary block. The bio is fed from the shared zero page (see + * write_one_eb()), so there is nothing to checksum here. Crucially, the + * buffer's own content is left intact: it may still be referenced, e.g. + * btrfs_free_tree_block() reads its header to add a delayed reference. */ - if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) { - memzero_extent_buffer(eb, 0, eb->len); + if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) return 0; - } if (WARN_ON_ONCE(found_start != eb->start)) return -EIO; @@ -2357,6 +2358,10 @@ static int validate_sys_chunk_array(const struct btrfs_fs_info *fs_info, key.type, cur); return -EUCLEAN; } + + if (unlikely(cur + sizeof(*chunk) > sys_array_size)) + goto short_read; + chunk = (struct btrfs_chunk *)(sb->sys_chunk_array + cur); num_stripes = btrfs_stack_chunk_num_stripes(chunk); if (unlikely(cur + btrfs_chunk_item_size(num_stripes) > sys_array_size)) @@ -3314,6 +3319,8 @@ static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info) */ rcu_read_lock(); xa_for_each(&fs_info->buffer_tree, index, eb) { + unsigned int refs; + /* Increase the ref so that the eb won't disappear. */ if (!refcount_inc_not_zero(&eb->refs)) continue; @@ -3323,17 +3330,27 @@ static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info) if (test_bit(EXTENT_BUFFER_READING, &eb->bflags)) wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE); + /* + * We hold the spinlock to make sure above + * EXTENT_BUFFER_READING flag is cleared with the held + * ref dropped. + * Or we can hit a race window and lead to false alerts. + */ + spin_lock(&eb->refs_lock); + refs = refcount_read(&eb->refs); + spin_unlock(&eb->refs_lock); + /* * The refs threshold is 2, one held by us at the beginning * of the loop, one for the ownership in the buffer tree. */ - if (unlikely(refcount_read(&eb->refs) > 2 || extent_buffer_under_io(eb))) { + if (unlikely(refs > 2 || extent_buffer_under_io(eb))) { WARN_ON_ONCE(IS_ENABLED(CONFIG_BTRFS_DEBUG)); btrfs_warn(fs_info, "unable to release extent buffer %llu owner %llu gen %llu refs %u flags 0x%lx", eb->start, btrfs_header_owner(eb), btrfs_header_generation(eb), - refcount_read(&eb->refs), eb->bflags); + refs, eb->bflags); } free_extent_buffer(eb); rcu_read_lock(); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index f032f0858f40f..ae6294fedd2d7 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -299,6 +300,25 @@ static noinline void unlock_delalloc_folio(const struct inode *inode, PAGE_UNLOCK); } +#ifdef CONFIG_BTRFS_DEBUG +/* + * Writeback must write-protect a folio when locking it for IO, before + * anything consumes its data (zeroing, inline copy, compression, + * checksumming). If this fails, then an mmap writer would be able to + * modify the data concurrently while we need it to be stable. + */ +void btrfs_check_folio_write_protected(struct folio *folio) +{ + if (folio_mkclean(folio)) { + const struct btrfs_inode *inode = BTRFS_I(folio->mapping->host); + + DEBUG_WARN("writable mmap PTEs, root %llu ino %llu pos %llu order %u", + btrfs_root_id(inode->root), btrfs_ino(inode), folio_pos(folio), + folio_order(folio)); + } +} +#endif + static noinline int lock_delalloc_folios(struct inode *inode, struct folio *locked_folio, u64 start, u64 end) @@ -332,6 +352,8 @@ static noinline int lock_delalloc_folios(struct inode *inode, folio_unlock(folio); goto out; } + /* Locked for writeback; revoke writable mmap PTEs before using the data. */ + folio_mkclean(folio); range_start = max_t(u64, folio_pos(folio), start); range_len = min_t(u64, folio_next_pos(folio), end + 1) - range_start; btrfs_folio_set_lock(fs_info, folio, range_start, range_len); @@ -1893,6 +1915,13 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, ASSERT(end <= folio_end, "start=%llu len=%u folio_start=%llu folio_size=%zu", start, len, folio_start, folio_size(folio)); + /* + * We are about to checksum and write out the data, so it must not be + * mmap writeable, or we could corrupt the data and end up with invalid + * checksums. + */ + btrfs_check_folio_write_protected(folio); + /* Truncate the submit bitmap to the current range. */ if (start > folio_start) bitmap_clear(bio_ctrl->submit_bitmap, 0, @@ -2350,14 +2379,17 @@ static struct extent_buffer *find_extent_buffer_nolock( static void end_bbio_meta_write(struct btrfs_bio *bbio) { struct extent_buffer *eb = bbio->private; - struct folio_iter fi; if (bbio->bio.bi_status != BLK_STS_OK) set_btree_ioerr(eb); - bio_for_each_folio_all(fi, &bbio->bio) { - btrfs_meta_folio_clear_writeback(fi.folio, eb); - } + /* + * Clear writeback on the buffer's own folios. The bio may carry the + * shared zero page instead (EXTENT_BUFFER_ZONED_ZEROOUT), so iterate + * the extent buffer folios rather than the bio folios. + */ + for (int i = 0; i < num_extent_folios(eb); i++) + btrfs_meta_folio_clear_writeback(eb->folios[i], eb); buffer_tree_clear_mark(eb, PAGECACHE_TAG_WRITEBACK); clear_and_wake_up_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); @@ -2398,7 +2430,8 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, struct btrfs_fs_info *fs_info = eb->fs_info; struct btrfs_bio *bbio; - prepare_eb_write(eb); + if (!test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) + prepare_eb_write(eb); bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES, REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc), @@ -2418,8 +2451,21 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, btrfs_meta_folio_set_writeback(folio, eb); if (!folio_test_dirty(folio)) wbc->nr_to_write -= folio_nr_pages(folio); - bio_add_folio_nofail(&bbio->bio, folio, range_len, - offset_in_folio(folio, range_start)); + if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) { + u32 off = 0; + + while (off < range_len) { + u32 add = min_t(u32, PAGE_SIZE, range_len - off); + + bio_add_folio_nofail(&bbio->bio, + page_folio(ZERO_PAGE(0)), + add, 0); + off += add; + } + } else { + bio_add_folio_nofail(&bbio->bio, folio, range_len, + offset_in_folio(folio, range_start)); + } wbc_account_cgroup_owner(wbc, folio, range_len); folio_unlock(folio); } @@ -2467,6 +2513,76 @@ void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start, } } +static int write_meta_extent_buffer(struct btrfs_eb_write_context *ctx, + struct writeback_control *wbc) +{ + struct extent_buffer *eb = ctx->eb; + int ret; + + ret = btrfs_check_meta_write_pointer(eb->fs_info, ctx); + if (ret) + return ret; + + if (!lock_extent_buffer_for_io(eb, wbc)) + return 0; + + /* Implies write in zoned mode. */ + if (ctx->zoned_bg) { + /* Mark the last eb in the block group. */ + btrfs_schedule_zone_finish_bg(ctx->zoned_bg, eb); + ctx->zoned_bg->meta_write_pointer += eb->len; + } + write_one_eb(eb, wbc); + return 0; +} + +/* + * On a zoned filesystem, write out the currently dirty metadata extent buffers + * of @bg. Used to flush the active metadata/system block group before the + * ascending-address walk in btree_writepages(), so that walk can pivot the + * active block group away (finishing it) instead of aborting the commit; see + * the caller for details. + */ +static void flush_active_meta_bg(struct address_space *mapping, + struct writeback_control *wbc, + struct btrfs_eb_write_context *ctx, + struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host); + unsigned long index = bg->start >> fs_info->nodesize_bits; + unsigned long end = (btrfs_block_group_end(bg) - 1) >> fs_info->nodesize_bits; + struct eb_batch batch; + unsigned int nr_ebs; + + ASSERT(btrfs_is_zoned(fs_info)); + lockdep_assert_held(&fs_info->zoned_meta_io_lock); + + eb_batch_init(&batch); + while (index <= end && + (nr_ebs = buffer_tree_get_ebs_tag(fs_info, &index, end, + PAGECACHE_TAG_DIRTY, &batch))) { + struct extent_buffer *eb; + + while ((eb = eb_batch_next(&batch)) != NULL) { + ctx->eb = eb; + + /* + * If the eb is behind the write pointer (-EBUSY, e.g. + * already being written by someone else) skip it and + * carry on. Only a hole at the write pointer (-EAGAIN) + * stops the flush. The main walk in btree_writepages() + * then deals with it. + */ + if (write_meta_extent_buffer(ctx, wbc) == -EAGAIN) { + eb_batch_release(&batch); + return; + } + } + eb_batch_release(&batch); + cond_resched(); + } +} + int btree_writepages(struct address_space *mapping, struct writeback_control *wbc) { struct btrfs_eb_write_context ctx = { .wbc = wbc }; @@ -2502,6 +2618,22 @@ int btree_writepages(struct address_space *mapping, struct writeback_control *wb else tag = PAGECACHE_TAG_DIRTY; btrfs_zoned_meta_io_lock(fs_info); + + /* + * On a zoned filesystem, flush the currently active metadata/system + * block group(s) first, under this same lock, so the ascending-address + * walk below can pivot the active block group instead of aborting the + * transaction commit with -EAGAIN. + */ + if (btrfs_is_zoned(fs_info) && wbc->sync_mode == WB_SYNC_ALL && + !wbc->for_sync) { + if (fs_info->active_meta_bg) + flush_active_meta_bg(mapping, wbc, &ctx, + fs_info->active_meta_bg); + if (fs_info->active_system_bg) + flush_active_meta_bg(mapping, wbc, &ctx, + fs_info->active_system_bg); + } retry: if (wbc->sync_mode == WB_SYNC_ALL) buffer_tree_tag_for_writeback(fs_info, index, end); @@ -2512,28 +2644,13 @@ int btree_writepages(struct address_space *mapping, struct writeback_control *wb while ((eb = eb_batch_next(&batch)) != NULL) { ctx.eb = eb; - ret = btrfs_check_meta_write_pointer(eb->fs_info, &ctx); - if (ret) { - if (ret == -EBUSY) - ret = 0; - - if (ret) { - done = true; - break; - } - continue; - } - - if (!lock_extent_buffer_for_io(eb, wbc)) - continue; - - /* Implies write in zoned mode. */ - if (ctx.zoned_bg) { - /* Mark the last eb in the block group. */ - btrfs_schedule_zone_finish_bg(ctx.zoned_bg, eb); - ctx.zoned_bg->meta_write_pointer += eb->len; + ret = write_meta_extent_buffer(&ctx, wbc); + if (ret == -EBUSY) { + ret = 0; + } else if (ret) { + done = true; + break; } - write_one_eb(eb, wbc); } nr_to_write_done = (wbc->nr_to_write <= 0); eb_batch_release(&batch); @@ -2703,6 +2820,8 @@ static int extent_write_cache_pages(struct address_space *mapping, continue; } + /* Locked for writeback; revoke writable mmap PTEs before using the data. */ + folio_mkclean(folio); ret = extent_writepage(folio, bio_ctrl); if (ret < 0) { done = true; @@ -3098,47 +3217,71 @@ static inline void btrfs_release_extent_buffer(struct extent_buffer *eb) kmem_cache_free(extent_buffer_cache, eb); } +/* + * Claim a slot to track an extent buffer in, evicting the coldest tracked buffer + * when the array is full. + * + * Slots fill in order until the array is full. After that a CLOCK (second + * chance) scan advances the hand, clearing one reference bit per step, until + * it lands on an unreferenced slot whose buffer is evicted. Clearing a bit per + * step bounds the scan to BTRFS_INHIBITED_EBS_SLOTS iterations. + */ +static int btrfs_inhibit_claim_slot(struct btrfs_trans_handle *trans) +{ + int slot; + + if (trans->nr_inhibited_ebs < BTRFS_INHIBITED_EBS_SLOTS) + return trans->nr_inhibited_ebs++; + + while (trans->inhibited_ebs_referenced & (1U << trans->inhibited_ebs_hand)) { + trans->inhibited_ebs_referenced &= ~(1U << trans->inhibited_ebs_hand); + trans->inhibited_ebs_hand = + (trans->inhibited_ebs_hand + 1) % BTRFS_INHIBITED_EBS_SLOTS; + } + slot = trans->inhibited_ebs_hand; + trans->inhibited_ebs_hand = (trans->inhibited_ebs_hand + 1) % BTRFS_INHIBITED_EBS_SLOTS; + + atomic_dec(&trans->inhibited_ebs[slot]->writeback_inhibitors); + free_extent_buffer(trans->inhibited_ebs[slot]); + + return slot; +} + /* * Inhibit writeback on buffer during transaction. * * @trans: transaction handle that will own the inhibitor * @eb: extent buffer to inhibit writeback on * - * Attempt to track this extent buffer in the transaction's inhibited set. If - * memory allocation fails, the buffer is simply not tracked. It may be written - * back and need re-COW, which is the original behavior. This is acceptable - * since inhibiting writeback is an optimization. + * Attempt to track this extent buffer in the transaction's inhibited set. When + * the set is full the coldest tracked buffer is evicted instead. An untracked + * buffer may be written back and need re-COW, which is the original behavior. + * This is acceptable since inhibiting writeback is an optimization. */ void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans, struct extent_buffer *eb) { - unsigned long index = eb->start >> trans->fs_info->nodesize_bits; - void *old; + int slot; lockdep_assert_held(&eb->lock); - /* Check if already inhibited by this handle. */ - old = xa_load(&trans->writeback_inhibited_ebs, index); - if (old == eb) - return; - - /* Take reference for the xarray entry. */ - refcount_inc(&eb->refs); - old = xa_store(&trans->writeback_inhibited_ebs, index, eb, GFP_NOFS); - if (xa_is_err(old)) { - /* Allocation failed, just skip inhibiting this buffer. */ - free_extent_buffer(eb); - return; + /* Already tracked: set its reference bit (second chance) and return. */ + for (int i = 0; i < trans->nr_inhibited_ebs; i++) { + if (trans->inhibited_ebs[i] == eb) { + trans->inhibited_ebs_referenced |= 1U << i; + return; + } } - /* Handle replacement of different eb at same index. */ - if (old && old != eb) { - struct extent_buffer *old_eb = old; - - atomic_dec(&old_eb->writeback_inhibitors); - free_extent_buffer(old_eb); - } + slot = btrfs_inhibit_claim_slot(trans); + /* + * Pin the eb while the array holds a raw pointer to it; the counter is + * what lock_extent_buffer_for_io() checks. + */ + refcount_inc(&eb->refs); atomic_inc(&eb->writeback_inhibitors); + trans->inhibited_ebs[slot] = eb; + trans->inhibited_ebs_referenced |= 1U << slot; } /* @@ -3146,14 +3289,13 @@ void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans, struct extent_ */ void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans) { - struct extent_buffer *eb; - unsigned long index; - - xa_for_each(&trans->writeback_inhibited_ebs, index, eb) { - atomic_dec(&eb->writeback_inhibitors); - free_extent_buffer(eb); + for (int i = 0; i < trans->nr_inhibited_ebs; i++) { + atomic_dec(&trans->inhibited_ebs[i]->writeback_inhibitors); + free_extent_buffer(trans->inhibited_ebs[i]); } - xa_destroy(&trans->writeback_inhibited_ebs); + trans->nr_inhibited_ebs = 0; + trans->inhibited_ebs_referenced = 0; + trans->inhibited_ebs_hand = 0; } static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info, @@ -3811,12 +3953,31 @@ static int release_extent_buffer(struct extent_buffer *eb) return 0; } -void free_extent_buffer(struct extent_buffer *eb) +static void clear_extent_buffer_reading(struct extent_buffer *eb) +{ + clear_and_wake_up_bit(EXTENT_BUFFER_READING, &eb->bflags); +} + +static void free_extent_buffer_clear_reading(struct extent_buffer *eb, + bool clear_reading) { int refs; + if (!eb) return; + /* + * We want to clear EXTENT_BUFFER_READING flag and decrease refs + * in the same critical section. + * This will make sure invalidate_and_check_btree_folios() won't + * see an eb with EXTENT_BUFFER_READING cleared but refs not yet + * decreased. + */ + if (clear_reading) { + spin_lock(&eb->refs_lock); + clear_extent_buffer_reading(eb); + } + refs = refcount_read(&eb->refs); while (1) { if (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags)) { @@ -3827,11 +3988,16 @@ void free_extent_buffer(struct extent_buffer *eb) } /* Optimization to avoid locking eb->refs_lock. */ - if (atomic_try_cmpxchg(&eb->refs.refs, &refs, refs - 1)) + if (atomic_try_cmpxchg(&eb->refs.refs, &refs, refs - 1)) { + if (clear_reading) + spin_unlock(&eb->refs_lock); return; + } } - spin_lock(&eb->refs_lock); + if (!clear_reading) + spin_lock(&eb->refs_lock); + if (refcount_read(&eb->refs) == 2 && test_bit(EXTENT_BUFFER_STALE, &eb->bflags) && !extent_buffer_under_io(eb) && @@ -3845,6 +4011,11 @@ void free_extent_buffer(struct extent_buffer *eb) release_extent_buffer(eb); } +void free_extent_buffer(struct extent_buffer *eb) +{ + return free_extent_buffer_clear_reading(eb, false); +} + void free_extent_buffer_stale(struct extent_buffer *eb) { if (!eb) @@ -3970,11 +4141,6 @@ void set_extent_buffer_uptodate(struct extent_buffer *eb) btrfs_meta_folio_set_uptodate(eb->folios[i], eb); } -static void clear_extent_buffer_reading(struct extent_buffer *eb) -{ - clear_and_wake_up_bit(EXTENT_BUFFER_READING, &eb->bflags); -} - static void end_bbio_meta_read(struct btrfs_bio *bbio) { struct extent_buffer *eb = bbio->private; @@ -3998,8 +4164,7 @@ static void end_bbio_meta_read(struct btrfs_bio *bbio) else clear_extent_buffer_uptodate(eb); - clear_extent_buffer_reading(eb); - free_extent_buffer(eb); + free_extent_buffer_clear_reading(eb, true); bio_put(&bbio->bio); } diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 9896e15ddc406..8699253376993 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -255,6 +255,11 @@ bool try_release_extent_mapping(struct folio *folio, gfp_t mask); int try_release_extent_buffer(struct folio *folio); int btrfs_read_folio(struct file *file, struct folio *folio); +#ifdef CONFIG_BTRFS_DEBUG +void btrfs_check_folio_write_protected(struct folio *folio); +#else +static inline void btrfs_check_folio_write_protected(struct folio *folio) { } +#endif void extent_write_locked_range(struct inode *inode, const struct folio *locked_folio, u64 start, u64 end, struct writeback_control *wbc, bool pages_dirty); diff --git a/fs/btrfs/fiemap.c b/fs/btrfs/fiemap.c index 6263e837093e4..929e978702c35 100644 --- a/fs/btrfs/fiemap.c +++ b/fs/btrfs/fiemap.c @@ -660,7 +660,7 @@ static int extent_fiemap(struct btrfs_inode *inode, range_end = round_up(start + len, sectorsize); prev_extent_end = range_start; - btrfs_lock_extent(&inode->io_tree, range_start, range_end, &cached_state); + btrfs_lock_extent(&inode->io_tree, range_start, range_end - 1, &cached_state); ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end); if (ret < 0) @@ -840,7 +840,7 @@ static int extent_fiemap(struct btrfs_inode *inode, } out_unlock: - btrfs_unlock_extent(&inode->io_tree, range_start, range_end, &cached_state); + btrfs_unlock_extent(&inode->io_tree, range_start, range_end - 1, &cached_state); if (ret == BTRFS_FIEMAP_FLUSH_CACHE) { btrfs_release_path(path); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index a2a2df2df7866..d892683927149 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -875,62 +875,56 @@ static noinline int prepare_one_folio(struct inode *inode, struct folio **folio_ /* * Locks the extent and properly waits for data=ordered extents to finish - * before allowing the folios to be modified if need. + * before allowing the folios to be modified. * * Return: - * 1 - the extent is locked - * 0 - the extent is not locked, and everything is OK + * 0 - the extent is locked * -EAGAIN - need to prepare the folios again */ static noinline int -lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct folio *folio, - loff_t pos, size_t write_bytes, - u64 *lockstart, u64 *lockend, bool nowait, - struct extent_state **cached_state) +lock_and_cleanup_extent(struct btrfs_inode *inode, struct folio *folio, + loff_t pos, size_t write_bytes, + u64 *lockstart, u64 *lockend, bool nowait, + struct extent_state **cached_state) { struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_ordered_extent *ordered; u64 start_pos; u64 last_pos; - int ret = 0; start_pos = round_down(pos, fs_info->sectorsize); last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1; - if (start_pos < inode->vfs_inode.i_size) { - struct btrfs_ordered_extent *ordered; - - if (nowait) { - if (!btrfs_try_lock_extent(&inode->io_tree, start_pos, - last_pos, cached_state)) { - folio_unlock(folio); - folio_put(folio); - return -EAGAIN; - } - } else { - btrfs_lock_extent(&inode->io_tree, start_pos, last_pos, - cached_state); - } - - ordered = btrfs_lookup_ordered_range(inode, start_pos, - last_pos - start_pos + 1); - if (ordered && - ordered->file_offset + ordered->num_bytes > start_pos && - ordered->file_offset <= last_pos) { - btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos, - cached_state); + if (nowait) { + if (!btrfs_try_lock_extent(&inode->io_tree, start_pos, + last_pos, cached_state)) { folio_unlock(folio); folio_put(folio); - btrfs_start_ordered_extent(ordered); - btrfs_put_ordered_extent(ordered); return -EAGAIN; } - if (ordered) - btrfs_put_ordered_extent(ordered); + } else { + btrfs_lock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); + } - *lockstart = start_pos; - *lockend = last_pos; - ret = 1; + ordered = btrfs_lookup_ordered_range(inode, start_pos, + last_pos - start_pos + 1); + if (ordered && + ordered->file_offset + ordered->num_bytes > start_pos && + ordered->file_offset <= last_pos) { + btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); + folio_unlock(folio); + folio_put(folio); + btrfs_start_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + return -EAGAIN; } + if (ordered) + btrfs_put_ordered_extent(ordered); + + *lockstart = start_pos; + *lockend = last_pos; /* * We should be called after prepare_one_folio() which should have locked @@ -938,7 +932,7 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct folio *folio, */ WARN_ON(!folio_test_locked(folio)); - return ret; + return 0; } /* @@ -1195,7 +1189,6 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, const u64 reserved_start = round_down(start, fs_info->sectorsize); u64 reserved_len; struct folio *folio = NULL; - int extents_locked; u64 lockstart; u64 lockend; bool only_release_metadata = false; @@ -1253,18 +1246,16 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, reserved_len = last_block - reserved_start; } - extents_locked = lock_and_cleanup_extent_if_need(inode, folio, start, - write_bytes, &lockstart, - &lockend, nowait, - &cached_state); - if (extents_locked < 0) { - if (!nowait && extents_locked == -EAGAIN) + ret = lock_and_cleanup_extent(inode, folio, start, write_bytes, + &lockstart, &lockend, nowait, &cached_state); + if (ret < 0) { + if (!nowait) goto again; btrfs_delalloc_release_extents(inode, reserved_len); release_space(inode, *data_reserved, reserved_start, reserved_len, only_release_metadata); - return extents_locked; + return ret; } copied = copy_folio_from_iter_atomic(folio, offset_in_folio(folio, start), @@ -1288,11 +1279,8 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, /* No copied bytes, unlock, release reserved space and exit. */ if (copied == 0) { - if (extents_locked) - btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, - &cached_state); - else - btrfs_free_extent_state(cached_state); + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, + &cached_state); btrfs_delalloc_release_extents(inode, reserved_len); release_space(inode, *data_reserved, reserved_start, reserved_len, only_release_metadata); @@ -1311,17 +1299,7 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, ret = btrfs_dirty_folio(inode, folio, start, copied, &cached_state, only_release_metadata); - /* - * If we have not locked the extent range, because the range's start - * offset is >= i_size, we might still have a non-NULL cached extent - * state, acquired while marking the extent range as delalloc through - * btrfs_dirty_page(). Therefore free any possible cached extent state - * to avoid a memory leak. - */ - if (extents_locked) - btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); - else - btrfs_free_extent_state(cached_state); + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); btrfs_delalloc_release_extents(inode, reserved_len); if (ret) { @@ -2535,8 +2513,10 @@ int btrfs_replace_file_extents(struct btrfs_inode *inode, inode_set_ctime_current(&inode->vfs_inode)); ret = btrfs_update_inode(trans, inode); - if (ret) + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); break; + } btrfs_end_transaction(trans); btrfs_btree_balance_dirty(fs_info); diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c index 1b3d82ae3de80..b7a4a6ade30f0 100644 --- a/fs/btrfs/free-space-tree.c +++ b/fs/btrfs/free-space-tree.c @@ -1353,7 +1353,7 @@ int btrfs_rebuild_free_space_tree(struct btrfs_fs_info *fs_info) if (unlikely(ret)) { btrfs_abort_transaction(trans, ret); btrfs_end_transaction(trans); - return ret; + goto out_clear; } node = rb_first_cached(&fs_info->block_group_cache_tree); @@ -1371,14 +1371,16 @@ int btrfs_rebuild_free_space_tree(struct btrfs_fs_info *fs_info) if (unlikely(ret)) { btrfs_abort_transaction(trans, ret); btrfs_end_transaction(trans); - return ret; + goto out_clear; } next: if (btrfs_should_end_transaction(trans)) { btrfs_end_transaction(trans); trans = btrfs_start_transaction(free_space_root, 1); - if (IS_ERR(trans)) - return PTR_ERR(trans); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_clear; + } } node = rb_next(node); } @@ -1390,6 +1392,10 @@ int btrfs_rebuild_free_space_tree(struct btrfs_fs_info *fs_info) ret = btrfs_commit_transaction(trans); clear_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags); return ret; + +out_clear: + clear_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags); + return ret; } static int __add_block_group_free_space(struct btrfs_trans_handle *trans, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 2534cd9284d54..7fe0a63d690a3 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -775,19 +775,28 @@ static inline void inode_should_defrag(struct btrfs_inode *inode, static int extent_range_clear_dirty_for_io(struct btrfs_inode *inode, u64 start, u64 end) { + pgoff_t index = start >> PAGE_SHIFT; const pgoff_t end_index = end >> PAGE_SHIFT; struct folio *folio; int ret = 0; - for (pgoff_t index = start >> PAGE_SHIFT; index <= end_index; index++) { + while (index <= end_index) { folio = filemap_get_folio(inode->vfs_inode.i_mapping, index); if (IS_ERR(folio)) { if (!ret) ret = PTR_ERR(folio); + index++; continue; } + /* + * We are about to compress the folio, so it must not be mmap + * writeable or we could corrupt the data as we attempt to + * compress it. + */ + btrfs_check_folio_write_protected(folio); btrfs_folio_clamp_clear_dirty(inode->root->fs_info, folio, start, end + 1 - start); + index = folio_next_index(folio); folio_put(folio); } return ret; @@ -877,11 +886,6 @@ static void compress_file_range(struct btrfs_work *work) inode_should_defrag(inode, start, end, end - start + 1, SZ_16K); - /* - * We need to call clear_page_dirty_for_io on each page in the range. - * Otherwise applications with the file mmap'd can wander in and change - * the page contents while we are compressing them. - */ ret = extent_range_clear_dirty_for_io(inode, start, end); /* @@ -2317,6 +2321,13 @@ static int run_delalloc_inline(struct btrfs_inode *inode, struct folio *locked_f int ret; ASSERT(folio_pos(locked_folio) == 0); + /* + * If an mmap writer could modify the folio while we copy it into an + * inline extent we might see only part of their modification then + * wrongly mark it clean again after copying, losing that write. So the + * folio must be write protected here. + */ + btrfs_check_folio_write_protected(locked_folio); if (btrfs_inode_can_compress(inode) && inode_need_compress(inode, 0, blocksize, true)) { @@ -2327,12 +2338,27 @@ static int run_delalloc_inline(struct btrfs_inode *inode, struct folio *locked_f } else if (inode->prop_compress) { compress_type = inode->prop_compress; } + /* + * We need to pass blocksize and not i_size, otherwise we can't + * create compressed inline extents for data smaller than sector + * size with lzo. + */ cb = btrfs_compress_bio(inode, 0, blocksize, compress_type, compress_level, 0); if (IS_ERR(cb)) { cb = NULL; /* Just fall back to non-compressed case. */ } else { compressed_size = cb->bbio.bio.bi_iter.bi_size; + /* + * If we did not save space, it's pointless and wasteful + * to have an inline compressed extent, so fallback to + * an uncompressed inline extent. + */ + if (compressed_size >= i_size) { + cleanup_compressed_bio(cb); + cb = NULL; + compressed_size = 0; + } } } if (!can_cow_file_range_inline(inode, 0, i_size, compressed_size)) { @@ -3424,6 +3450,9 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent) */ btrfs_remove_ordered_extent(ordered_extent); + /* Cleanup any remaining biocs attached to the OE. */ + btrfs_cleanup_ordered_bioc_list(ordered_extent); + /* once for us */ btrfs_put_ordered_extent(ordered_extent); /* once for the tree */ @@ -3862,7 +3891,8 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) if (ret) goto out; } - trans = btrfs_start_transaction(root, 1); + /* Only deletes the orphan. */ + trans = btrfs_start_transaction_fallback_global_rsv(root, 1); if (IS_ERR(trans)) { ret = PTR_ERR(trans); goto out; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index baa645e988126..a537036303a47 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -356,14 +356,21 @@ int btrfs_fileattr_set(struct mnt_idmap *idmap, inode_flags |= BTRFS_INODE_NODATACOW; } } else { - /* - * Revert back under same assumptions as above - */ - if (S_ISREG(inode->vfs_inode.i_mode)) { - if (inode->vfs_inode.i_size == 0) - inode_flags &= ~(BTRFS_INODE_NODATACOW | - BTRFS_INODE_NODATASUM); - } else { + /* We can only change NODATACOW for zero-sized regular file. */ + if (S_ISREG(inode->vfs_inode.i_mode) && (inode->vfs_inode.i_size == 0)) { + inode_flags &= ~BTRFS_INODE_NODATACOW; + /* + * There is currently no way to change NODATASUM flag + * through fileattr API. If we unconditionally keep the + * current NODATASUM flag, chattr +C then chattr -C will + * keep the NODATASUM flag, and no way to remove that + * flag. + * + * So respect the current mount option for NODATASUM flag. + */ + if (!btrfs_test_opt(fs_info, NODATASUM)) + inode_flags &= ~BTRFS_INODE_NODATASUM; + } else if (!S_ISREG(inode->vfs_inode.i_mode)) { inode_flags &= ~BTRFS_INODE_NODATACOW; } } diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 502fb4a55cb22..f64a4537be315 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -4339,12 +4339,13 @@ static int qgroup_free_reserved_data(struct btrfs_inode *inode, struct ulist_node *unode; struct ulist_iterator uiter; struct extent_changeset changeset; + const u32 sectorsize = root->fs_info->sectorsize; + const u64 aligned_start = round_down(start, sectorsize); + const u64 aligned_len = round_up(start + len, sectorsize) - aligned_start; u64 freed = 0; int ret; extent_changeset_init_bytes_only(&changeset); - len = round_up(start + len, root->fs_info->sectorsize); - start = round_down(start, root->fs_info->sectorsize); ULIST_ITER_INIT(&uiter); while ((unode = ulist_next(&reserved->range_changed, &uiter))) { @@ -4356,12 +4357,15 @@ static int qgroup_free_reserved_data(struct btrfs_inode *inode, extent_changeset_release(&changeset); - /* Only free range in range [start, start + len) */ - if (range_start >= start + len || - range_start + range_len <= start) + /* + * Only free the range within + * [aligned_start, aligned_start + aligned_len). + */ + if (range_start >= aligned_start + aligned_len || + range_start + range_len <= aligned_start) continue; - free_start = max(range_start, start); - free_len = min(start + len, range_start + range_len) - + free_start = max(range_start, aligned_start); + free_len = min(aligned_start + aligned_len, range_start + range_len) - free_start; /* * TODO: To also modify reserved->ranges_reserved to reflect diff --git a/fs/btrfs/raid-stripe-tree.c b/fs/btrfs/raid-stripe-tree.c index b210371ce91e3..d9e660447205f 100644 --- a/fs/btrfs/raid-stripe-tree.c +++ b/fs/btrfs/raid-stripe-tree.c @@ -310,8 +310,10 @@ static int update_raid_extent_item(struct btrfs_trans_handle *trans, ret = btrfs_search_slot(trans, trans->fs_info->stripe_root, key, path, 0, 1); - if (ret) - return (ret == 1 ? ret : -EINVAL); + if (ret > 0) + ret = -ENOENT; + if (ret < 0) + return ret; leaf = path->nodes[0]; slot = path->slots[0]; @@ -337,7 +339,6 @@ int btrfs_insert_one_raid_extent(struct btrfs_trans_handle *trans, stripe_extent = kzalloc(item_size, GFP_NOFS); if (unlikely(!stripe_extent)) { btrfs_abort_transaction(trans, -ENOMEM); - btrfs_end_transaction(trans); return -ENOMEM; } @@ -374,7 +375,7 @@ int btrfs_insert_raid_extent(struct btrfs_trans_handle *trans, struct btrfs_ordered_extent *ordered_extent) { struct btrfs_io_context *bioc; - int ret; + int ret = 0; if (!btrfs_fs_incompat(trans->fs_info, RAID_STRIPE_TREE)) return 0; @@ -382,17 +383,23 @@ int btrfs_insert_raid_extent(struct btrfs_trans_handle *trans, list_for_each_entry(bioc, &ordered_extent->bioc_list, rst_ordered_entry) { ret = btrfs_insert_one_raid_extent(trans, bioc); if (ret) - return ret; + break; } - while (!list_empty(&ordered_extent->bioc_list)) { - bioc = list_first_entry(&ordered_extent->bioc_list, + btrfs_cleanup_ordered_bioc_list(ordered_extent); + return ret; +} + +void btrfs_cleanup_ordered_bioc_list(struct btrfs_ordered_extent *ordered) +{ + while (!list_empty(&ordered->bioc_list)) { + struct btrfs_io_context *bioc; + + bioc = list_first_entry(&ordered->bioc_list, typeof(*bioc), rst_ordered_entry); list_del(&bioc->rst_ordered_entry); btrfs_put_bioc(bioc); } - - return 0; } int btrfs_get_raid_extent_offset(struct btrfs_fs_info *fs_info, diff --git a/fs/btrfs/raid-stripe-tree.h b/fs/btrfs/raid-stripe-tree.h index 69942ad431408..eb02cf48511bc 100644 --- a/fs/btrfs/raid-stripe-tree.h +++ b/fs/btrfs/raid-stripe-tree.h @@ -28,6 +28,7 @@ int btrfs_get_raid_extent_offset(struct btrfs_fs_info *fs_info, u32 stripe_index, struct btrfs_io_stripe *stripe); int btrfs_insert_raid_extent(struct btrfs_trans_handle *trans, struct btrfs_ordered_extent *ordered_extent); +void btrfs_cleanup_ordered_bioc_list(struct btrfs_ordered_extent *ordered); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS int btrfs_insert_one_raid_extent(struct btrfs_trans_handle *trans, diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index fc5c14b5adad7..e4b259717c251 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1537,6 +1537,33 @@ static void clear_reloc_root(struct btrfs_root *root) clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); } +/* Drop the reloc trees of a relocation that is being deferred and retried. */ +static void abort_reloc_roots(struct reloc_control *rc, struct list_head *list) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + struct btrfs_root *reloc_root, *tmp; + + list_for_each_entry_safe(reloc_root, tmp, list, root_list) { + struct btrfs_root *root; + + root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, false); + if (!IS_ERR(root)) { + if (root->reloc_root == reloc_root) { + clear_reloc_root(root); + btrfs_put_root(reloc_root); + } + btrfs_put_root(root); + } + + btrfs_set_root_refs(&reloc_root->root_item, 0); + memset(&reloc_root->root_item.drop_progress, 0, sizeof(struct btrfs_disk_key)); + btrfs_set_root_drop_level(&reloc_root->root_item, 0); + + list_del_init(&reloc_root->root_list); + list_add_tail(&reloc_root->reloc_dirty_list, &rc->dirty_subvol_roots); + } +} + static int clean_dirty_subvols(struct reloc_control *rc) { struct btrfs_root *root; @@ -1876,8 +1903,7 @@ int prepare_to_merge(struct reloc_control *rc, int err) return err; } -static noinline_for_stack -void merge_reloc_roots(struct reloc_control *rc) +static noinline_for_stack int merge_reloc_roots(struct reloc_control *rc) { struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct btrfs_root *root; @@ -1975,7 +2001,15 @@ void merge_reloc_roots(struct reloc_control *rc) goto again; } out: - if (ret) { + if (btrfs_is_zoned(fs_info) && ret == -EAGAIN) { + abort_reloc_roots(rc, &reloc_roots); + + /* New reloc root may be added. */ + mutex_lock(&fs_info->reloc_mutex); + list_splice_init(&rc->reloc_roots, &reloc_roots); + mutex_unlock(&fs_info->reloc_mutex); + abort_reloc_roots(rc, &reloc_roots); + } else if (ret) { btrfs_handle_fs_error(fs_info, ret, NULL); free_reloc_roots(&reloc_roots); @@ -2001,6 +2035,7 @@ void merge_reloc_roots(struct reloc_control *rc) * * The remaining nodes will be cleaned up by put_reloc_control(). */ + return ret; } static void free_block_list(struct rb_root *blocks) @@ -3730,7 +3765,9 @@ static noinline_for_stack int relocate_block_group(struct reloc_control *rc) */ err = prepare_to_merge(rc, err); - merge_reloc_roots(rc); + ret = merge_reloc_roots(rc); + if (ret && !err) + err = ret; rc->merge_reloc_tree = false; unset_reloc_control(rc); @@ -5555,6 +5592,24 @@ static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) return ret; } +static void release_recovered_fs_roots(struct list_head *roots, bool drop_reloc_refs) +{ + struct btrfs_root *root; + struct btrfs_root *next; + + list_for_each_entry_safe(root, next, roots, reloc_dirty_list) { + list_del_init(&root->reloc_dirty_list); + if (drop_reloc_refs) { + struct btrfs_root *reloc_root = root->reloc_root; + + ASSERT(reloc_root); + root->reloc_root = NULL; + btrfs_put_root(reloc_root); + } + btrfs_put_root(root); + } +} + /* * recover relocation interrupted by system crash. * @@ -5564,6 +5619,7 @@ static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) { LIST_HEAD(reloc_roots); + LIST_HEAD(recovered_roots); struct btrfs_key key; struct btrfs_root *fs_root; struct btrfs_root *reloc_root; @@ -5680,7 +5736,7 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) ret = PTR_ERR(fs_root); list_add_tail(&reloc_root->root_list, &reloc_roots); btrfs_end_transaction(trans); - goto out_unset; + goto out_drop_reloc_refs; } ret = __add_reloc_root(reloc_root, rc); @@ -5689,17 +5745,21 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) list_add_tail(&reloc_root->root_list, &reloc_roots); btrfs_put_root(fs_root); btrfs_end_transaction(trans); - goto out_unset; + goto out_drop_reloc_refs; } + ASSERT(list_empty(&fs_root->reloc_dirty_list)); fs_root->reloc_root = btrfs_grab_root(reloc_root); - btrfs_put_root(fs_root); + list_add_tail(&fs_root->reloc_dirty_list, &recovered_roots); } ret = btrfs_commit_transaction(trans); if (ret) - goto out_unset; + goto out_drop_reloc_refs; + release_recovered_fs_roots(&recovered_roots, false); - merge_reloc_roots(rc); + ret = merge_reloc_roots(rc); + if (ret) + goto out_unset; unset_reloc_control(rc); @@ -5713,6 +5773,8 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) ret2 = clean_dirty_subvols(rc); if (ret2 < 0 && !ret) ret = ret2; +out_drop_reloc_refs: + release_recovered_fs_roots(&recovered_roots, true); out_unset: unset_reloc_control(rc); reloc_chunk_end(fs_info); diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index d2f7ac5b6e961..3b3f04521369d 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -1041,6 +1041,10 @@ static void scrub_stripe_report_errors(struct scrub_ctx *sctx, skip: for_each_set_bit(sector_nr, &extent_bitmap, stripe->nr_sectors) { + const u64 sector_logical = stripe->logical + + ((u64)sector_nr << fs_info->sectorsize_bits); + const u64 sector_physical = physical + + ((u64)sector_nr << fs_info->sectorsize_bits); bool repaired = false; if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr)) { @@ -1069,12 +1073,12 @@ static void scrub_stripe_report_errors(struct scrub_ctx *sctx, if (dev) { btrfs_err_rl(fs_info, "scrub: fixed up error at logical %llu on dev %s physical %llu", - stripe->logical, btrfs_dev_name(dev), - physical); + sector_logical, btrfs_dev_name(dev), + sector_physical); } else { btrfs_err_rl(fs_info, "scrub: fixed up error at logical %llu on mirror %u", - stripe->logical, stripe->mirror_num); + sector_logical, stripe->mirror_num); } continue; } @@ -1083,30 +1087,30 @@ static void scrub_stripe_report_errors(struct scrub_ctx *sctx, if (dev) { btrfs_err_rl(fs_info, "scrub: unable to fixup (regular) error at logical %llu on dev %s physical %llu", - stripe->logical, btrfs_dev_name(dev), - physical); + sector_logical, btrfs_dev_name(dev), + sector_physical); } else { btrfs_err_rl(fs_info, "scrub: unable to fixup (regular) error at logical %llu on mirror %u", - stripe->logical, stripe->mirror_num); + sector_logical, stripe->mirror_num); } if (scrub_bitmap_test_bit_io_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("i/o error", dev, false, - stripe->logical, physical); + sector_logical, sector_physical); if (scrub_bitmap_test_bit_csum_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("checksum error", dev, false, - stripe->logical, physical); + sector_logical, sector_physical); if (scrub_bitmap_test_bit_meta_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("header error", dev, false, - stripe->logical, physical); + sector_logical, sector_physical); if (scrub_bitmap_test_bit_meta_gen_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("generation error", dev, false, - stripe->logical, physical); + sector_logical, sector_physical); } /* Update the device stats. */ diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 3ae480c7474b0..a888202397edb 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -2066,7 +2066,7 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen); if (ret <= 0) - return 0; + return ret; /* * If we have a parent root we need to verify that the parent dir was diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index f4e34898d5814..2608d03076e0e 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -1845,8 +1845,12 @@ static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf) f_fsid.val[0] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root) >> 32; f_fsid.val[1] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root); - /* Hash dev_t to avoid f_fsid collision with cloned filesystems. */ - if (fs_info->fs_devices->total_devices == 1) { + /* + * Hash dev_t to avoid f_fsid collisions with cloned filesystems. + * Only do this when a clone is present so the original filesystem + * (mounted first) maintains backward-compatible f_fsid behavior. + */ + if (fs_info->fs_devices->temp_fsid) { __kernel_fsid_t dev_fsid = u64_to_fsid(huge_encode_dev(fs_info->fs_devices->latest_dev->bdev->bd_dev)); diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 8f94197281008..b64c6d2855c9a 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -458,8 +458,19 @@ static int record_root_in_trans(struct btrfs_trans_handle *trans, * through btrfs_record_root_in_trans without having to take the * lock. smp_wmb() makes sure that all the writes above are * done before we pop in the zero below + * + * If @force is true, it means the call is from + * qgroup_account_snapshot(), which only requires radix tree + * tracking. + * We should not force reloc root creation here, as the root + * may have already been modified, and in that case + * root->commit_root has already been dropped. + * + * Using that commit root will cause the reloc root to refer + * to a deleted extent, causing extent tree corruption. */ - ret = btrfs_init_reloc_root(trans, root); + if (!force) + ret = btrfs_init_reloc_root(trans, root); smp_mb__before_atomic(); clear_bit(BTRFS_ROOT_IN_TRANS_SETUP, &root->state); } @@ -698,8 +709,6 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, goto alloc_fail; } - xa_init(&h->writeback_inhibited_ebs); - /* * If we are JOIN_NOLOCK we're already committing a transaction and * waiting on this guy, so we don't need to do the sb_start_intwrite @@ -2589,6 +2598,12 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) ret = btrfs_write_and_wait_transaction(trans); if (unlikely(ret)) { btrfs_err(fs_info, "error while writing out transaction: %d", ret); + /* + * Abort before releasing tree_log_mutex, so a log sync waiting + * on it sees the fs error and skips writing super_for_commit + * for this failed transaction. See btrfs_sync_log(). + */ + btrfs_abort_transaction(trans, ret); mutex_unlock(&fs_info->tree_log_mutex); goto scrub_continue; } diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 5e4b1106fd905..3a57f227b5ed2 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -7,12 +7,12 @@ #define BTRFS_TRANSACTION_H #include +#include #include #include #include #include #include -#include #include "btrfs_inode.h" #include "delayed-ref.h" @@ -23,6 +23,7 @@ struct btrfs_fs_info; struct btrfs_root_item; struct btrfs_root; struct btrfs_path; +struct extent_buffer; /* * Signal that a direct IO write is in progress, to avoid deadlock for sync @@ -136,6 +137,18 @@ enum { #define TRANS_EXTWRITERS (__TRANS_START | __TRANS_ATTACH) +/* + * Number of extent buffers a transaction handle tracks for writeback + * inhibition. The CLOCK reference bits pack into a u32 so this must not exceed + * 32, and keeping it a power of two lets the compiler reduce the CLOCK hand + * modulo to a mask. + */ +#define BTRFS_INHIBITED_EBS_SLOTS 8 + +static_assert(BTRFS_INHIBITED_EBS_SLOTS <= 32); +static_assert(BTRFS_INHIBITED_EBS_SLOTS != 0 && + (BTRFS_INHIBITED_EBS_SLOTS & (BTRFS_INHIBITED_EBS_SLOTS - 1)) == 0); + struct btrfs_trans_handle { u64 transid; u64 bytes_reserved; @@ -163,8 +176,14 @@ struct btrfs_trans_handle { struct btrfs_fs_info *fs_info; struct list_head new_bgs; struct btrfs_block_rsv delayed_rsv; - /* Extent buffers with writeback inhibited by this handle. */ - struct xarray writeback_inhibited_ebs; + + /* Extent buffers this handle has inhibited writeback on. */ + struct extent_buffer *inhibited_ebs[BTRFS_INHIBITED_EBS_SLOTS]; + /* CLOCK reference bit per slot. */ + u32 inhibited_ebs_referenced; + u32 nr_inhibited_ebs; + /* CLOCK hand. */ + u32 inhibited_ebs_hand; }; /* diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 0ce91396b517f..1326bbc1322dc 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -2100,7 +2100,7 @@ static int check_dev_extent_item(const struct extent_buffer *leaf, sectorsize))) { generic_err(leaf, slot, "invalid dev extent chunk offset, has %llu not aligned to %u", - btrfs_dev_extent_chunk_objectid(leaf, de), + btrfs_dev_extent_chunk_offset(leaf, de), sectorsize); return -EUCLEAN; } diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 875e4ddc68eae..c87f6a7e8d612 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -7307,6 +7307,22 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); if (ret < 0) goto out; + /* + * There can't be an inode ref key with offset 0 because inode numbers + * start at BTRFS_FIRST_FREE_OBJECTID. + */ + if (WARN_ON_ONCE(ret == 0)) { + btrfs_err(trans->fs_info, + "found inode ref key with offset 0 for root %llu inode %llu", + btrfs_root_id(root), ino); + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } + /* + * Set to 0 so that in case we don't do any work below, we won't return + * 1 and trigger an unnecessary transaction commit. + */ + ret = 0; while (true) { struct extent_buffer *leaf = path->nodes[0]; diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c index 983365a735416..1f05525386c93 100644 --- a/fs/btrfs/verity.c +++ b/fs/btrfs/verity.c @@ -93,6 +93,20 @@ static loff_t merkle_file_pos(const struct inode *inode) return rounded; } +/* + * Start a transaction for removing verity items or the verity orphan. + * + * Like unlink, this only deletes items and frees space in the end, so the + * reservation may come from the global reserve when the filesystem is full + * (-ENOSPC) and is not subject to the qgroup limit (-EDQUOT). Otherwise a + * failed enable could never be cleaned up in either situation. + */ +static struct btrfs_trans_handle *start_verity_cleanup_trans(struct btrfs_root *root, + unsigned int num_items) +{ + return btrfs_start_transaction_fallback_global_rsv(root, num_items); +} + /* * Drop all the items for this inode with this key_type. * @@ -120,7 +134,7 @@ static int drop_verity_items(struct btrfs_inode *inode, u8 key_type) while (1) { /* 1 for the item being dropped */ - trans = btrfs_start_transaction(root, 1); + trans = start_verity_cleanup_trans(root, 1); if (IS_ERR(trans)) return PTR_ERR(trans); @@ -466,7 +480,7 @@ static int rollback_verity(struct btrfs_inode *inode) * 1 for updating the inode flag * 1 for deleting the orphan */ - trans = btrfs_start_transaction(root, 2); + trans = start_verity_cleanup_trans(root, 2); if (IS_ERR(trans)) { ret = PTR_ERR(trans); trans = NULL; @@ -720,14 +734,18 @@ static struct page *btrfs_read_merkle_tree_page(struct inode *inode, goto out; folio_lock(folio); - /* If it's not uptodate after we have the lock, we got a read error. */ - if (!folio_test_uptodate(folio)) { + /* Folio was truncated from mapping. */ + if (!folio->mapping) { folio_unlock(folio); folio_put(folio); - return ERR_PTR(-EIO); + goto again; } - folio_unlock(folio); - goto out; + /* Another reader may have filled the folio while we waited. */ + if (folio_test_uptodate(folio)) { + folio_unlock(folio); + goto out; + } + goto read_folio; } folio = filemap_alloc_folio(mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS), @@ -744,6 +762,7 @@ static struct page *btrfs_read_merkle_tree_page(struct inode *inode, return ERR_PTR(ret); } +read_folio: /* * Merkle item keys are indexed from byte 0 in the merkle tree. * They have the form: @@ -753,6 +772,7 @@ static struct page *btrfs_read_merkle_tree_page(struct inode *inode, ret = read_key_bytes(BTRFS_I(inode), BTRFS_VERITY_MERKLE_ITEM_KEY, off, folio_address(folio), PAGE_SIZE, folio); if (ret < 0) { + folio_unlock(folio); folio_put(folio); return ERR_PTR(ret); } diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 6eab4cc73ce44..b90cbd1e3ebe3 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -3070,7 +3070,11 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path error_sysfs: btrfs_sysfs_remove_device(device); mutex_lock(&fs_info->fs_devices->device_list_mutex); + if (seeding_dev) + btrfs_assign_next_active_device(device, seed_devices->latest_dev); mutex_lock(&fs_info->chunk_mutex); + if (!list_empty(&device->post_commit_list)) + list_del_init(&device->post_commit_list); list_del_rcu(&device->dev_list); list_del(&device->dev_alloc_list); fs_info->fs_devices->num_devices--; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index a016cb471beb4..7f0dde6398d4d 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -2626,16 +2626,13 @@ static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_writ down_read(&dev_replace->rwsem); map = block_group->physical_map; for (i = 0; i < map->num_stripes; i++) { - ret = call_zone_finish(block_group, &map->stripes[i]); - if (ret) { - up_read(&dev_replace->rwsem); - return ret; - } + if (ret) + break; } up_read(&dev_replace->rwsem); - if (!fully_written) + if (!ret && !fully_written) btrfs_dec_block_group_ro(block_group); spin_lock(&fs_info->zone_active_bgs_lock); @@ -2648,7 +2645,7 @@ static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_writ clear_and_wake_up_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags); - return 0; + return ret; } int btrfs_zone_finish(struct btrfs_block_group *block_group) diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c index 86919293fd546..58d9ff76fe07b 100644 --- a/fs/btrfs/zstd.c +++ b/fs/btrfs/zstd.c @@ -307,8 +307,17 @@ struct list_head *zstd_get_workspace(struct btrfs_fs_info *fs_info, int level) DEFINE_WAIT(wait); prepare_to_wait(&zwsm->wait, &wait, TASK_UNINTERRUPTIBLE); - schedule(); + /* + * Re-check after being queued: zstd_put_workspace() only wakes + * a queue that already has a sleeper, so a workspace returned + * since the failed allocation woke nobody. + */ + ws = zstd_find_workspace(fs_info, level); + if (!ws) + schedule(); finish_wait(&zwsm->wait, &wait); + if (ws) + return ws; goto again; } diff --git a/fs/buffer.c b/fs/buffer.c index 9af5f061a1f85..955ab07b34eb8 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2177,6 +2177,7 @@ void block_commit_write(struct folio *folio, size_t from, size_t to) { size_t block_start, block_end; bool partial = false; + bool uptodate = folio_test_uptodate(folio); unsigned blocksize; struct buffer_head *bh, *head; @@ -2199,6 +2200,8 @@ void block_commit_write(struct folio *folio, size_t from, size_t to) clear_buffer_new(bh); block_start = block_end; + if (uptodate && block_start >= to) + break; bh = bh->b_this_page; } while (bh != head); diff --git a/fs/cachefiles/xattr.c b/fs/cachefiles/xattr.c index f8ae78b3f7b6d..c70bf67e52b01 100644 --- a/fs/cachefiles/xattr.c +++ b/fs/cachefiles/xattr.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "internal.h" #define CACHEFILES_COOKIE_TYPE_DATA 1 @@ -50,7 +51,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) _enter("%x,#%d", object->debug_id, len); - buf = kmalloc(sizeof(struct cachefiles_xattr) + len, GFP_KERNEL); + buf = kmalloc(sizeof(struct cachefiles_xattr) + max(len, sizeof(__be64)), GFP_KERNEL); if (!buf) return -ENOMEM; @@ -60,6 +61,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) buf->content = object->content_info; if (test_bit(FSCACHE_COOKIE_LOCAL_WRITE, &object->cookie->flags)) buf->content = CACHEFILES_CONTENT_DIRTY; + put_unaligned_be64(0, (__be64 *)buf->data); if (len > 0) memcpy(buf->data, fscache_get_aux(object->cookie), len); @@ -77,8 +79,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) trace_cachefiles_vfs_error(object, file_inode(file), ret, cachefiles_trace_setxattr_error); trace_cachefiles_coherency(object, file_inode(file)->i_ino, - be64_to_cpup((__be64 *)buf->data), - buf->content, + buf->data, buf->content, cachefiles_coherency_set_fail); if (ret != -ENOMEM) cachefiles_io_error_obj( @@ -86,8 +87,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) "Failed to set xattr with error %d", ret); } else { trace_cachefiles_coherency(object, file_inode(file)->i_ino, - be64_to_cpup((__be64 *)buf->data), - buf->content, + buf->data, buf->content, cachefiles_coherency_set_ok); } @@ -110,9 +110,10 @@ int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file int ret = -ESTALE; tlen = sizeof(struct cachefiles_xattr) + len; - buf = kmalloc(tlen, GFP_KERNEL); + buf = kmalloc(sizeof(struct cachefiles_xattr) + max(len, sizeof(__be64)), GFP_KERNEL); if (!buf) return -ENOMEM; + put_unaligned_be64(0, (__be64 *)buf->data); xlen = cachefiles_inject_read_error(); if (xlen == 0) @@ -148,8 +149,7 @@ int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file out: trace_cachefiles_coherency(object, file_inode(file)->i_ino, - be64_to_cpup((__be64 *)buf->data), - buf->content, why); + buf->data, buf->content, why); kfree(buf); return ret; } diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index ecf33b66610c8..e9f561bedf475 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1426,6 +1426,16 @@ void ceph_shift_unused_folios_left(struct folio_batch *fbatch) fbatch->nr = n; } +static void ceph_undo_wrbuffer_claim(struct inode *inode, struct folio *folio) +{ + struct ceph_snap_context *snapc = folio_detach_private(folio); + + if (!snapc) + return; + ceph_put_wrbuffer_cap_refs(ceph_inode(inode), 1, snapc); + ceph_put_snap_context(snapc); +} + static int ceph_submit_write(struct address_space *mapping, struct writeback_control *wbc, @@ -1489,6 +1499,7 @@ int ceph_submit_write(struct address_space *mapping, if (!page) continue; + ceph_undo_wrbuffer_claim(inode, page_folio(page)); redirty_page_for_writepage(wbc, page); unlock_page(page); } diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index d7283fb54cec4..bb5ef0eacd8f3 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -979,6 +979,27 @@ int __ceph_caps_revoking_other(struct ceph_inode_info *ci, return 0; } +/* + * Return true if any cap of this inode holds caps which the MDS has + * revoked, but which we have not released yet. + */ +static bool __ceph_is_any_revoking(const struct ceph_inode_info *ci) +{ + const struct rb_node *p; + + lockdep_assert_held(&ci->i_ceph_lock); + + for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) { + const struct ceph_cap *cap = + rb_entry(p, struct ceph_cap, ci_node); + + if (cap->implemented & ~cap->issued) + return true; + } + + return false; +} + int __ceph_caps_used(struct ceph_inode_info *ci) { int used = 0; @@ -1421,6 +1442,9 @@ static void __prep_cap(struct cap_msg_args *arg, struct ceph_cap *cap, cap->implemented &= cap->issued | used; cap->mds_wanted = want; + if ((ci->i_ceph_flags & CEPH_I_FLUSH_FORCE) != 0 && !__ceph_is_any_revoking(ci)) + clear_bit(CEPH_I_FLUSH_FORCE_BIT, &ci->i_ceph_flags); + arg->session = cap->session; arg->ino = ceph_vino(inode).ino; arg->cid = cap->cap_id; @@ -2038,6 +2062,14 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags) if (ci->i_ceph_flags & CEPH_I_FLUSH) flags |= CHECK_CAPS_FLUSH; + /* + * A revoke whose response was deferred (see handle_cap_grant()) must + * still be acknowledged. Replay the forced flush here so that even a + * check triggered by writeback/invalidation completion sends a cap + * message to the MDS. + */ + if (ci->i_ceph_flags & CEPH_I_FLUSH_FORCE) + flags |= CHECK_CAPS_FLUSH_FORCE; retry: /* Caps wanted by virtue of active open files. */ file_wanted = __ceph_caps_file_wanted(ci); @@ -2589,9 +2621,14 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc, } } - list_for_each_entry(cf, &ci->i_cap_flush_list, i_list) { - if (cf->tid < first_tid) + cf = list_first_entry(&ci->i_cap_flush_list, struct ceph_cap_flush, i_list); + while (&cf->i_list != &ci->i_cap_flush_list) { + struct ceph_cap_flush *next; + + if (cf->tid < first_tid) { + cf = list_next_entry(cf, i_list); continue; + } cap = ci->i_auth_cap; if (!(cap && cap->session == session)) { @@ -2601,6 +2638,7 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc, } first_tid = cf->tid + 1; + next = list_next_entry(cf, i_list); if (!cf->is_capsnap) { struct cap_msg_args arg; @@ -2641,6 +2679,7 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc, } spin_lock(&ci->i_ceph_lock); + cf = next; } } @@ -3757,13 +3796,30 @@ static void handle_cap_grant(struct inode *inode, BUG_ON(cap->issued & ~cap->implemented); /* don't let check_caps skip sending a response to MDS for revoke msgs */ - if (!revoke_wait && le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) { - cap->mds_wanted = 0; - flags |= CHECK_CAPS_FLUSH_FORCE; - if (cap == ci->i_auth_cap) - check_caps = 1; /* check auth cap only */ - else - check_caps = 2; /* check all caps */ + if (le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) { + if (revoke_wait) { + /* + * We can't ack the revoke yet: the response is deferred + * until the writeback or cache invalidation queued above + * completes. Set the CEPH_I_FLUSH_FORCE flag to remember + * that a forced cap message is owed so that deferred + * completion (ceph_put_wrbuffer_cap_refs() or the + * invalidate worker, both of which call ceph_check_caps()) + * actually sends one, even if by then the revoked caps look + * unused, the inode is retaining caps, or the MDS has + * re-granted them. Without this, the cap message is never + * sent and the MDS hangs ("isn't responding to + * mclientcaps(revoke)"). + */ + set_bit(CEPH_I_FLUSH_FORCE_BIT, &ci->i_ceph_flags); + } else { + cap->mds_wanted = 0; + flags |= CHECK_CAPS_FLUSH_FORCE; + if (cap == ci->i_auth_cap) + check_caps = 1; /* check auth cap only */ + else + check_caps = 2; /* check all caps */ + } } if (extra_info->inline_version > 0 && diff --git a/fs/ceph/crypto.c b/fs/ceph/crypto.c index 64d2407592772..1cc11f77413c1 100644 --- a/fs/ceph/crypto.c +++ b/fs/ceph/crypto.c @@ -298,6 +298,11 @@ int ceph_encode_encrypted_dname(struct inode *parent, char *buf, int elen) * Otherwise, base64 decode the string, and then ask fscrypt to format it * for userland presentation. * + * Though the fscrypt/crypto subsystems broadly expect all buffers to be in the + * linear-mapped region, this function slightly relaxes those requirements: + * fname->ctext, fname->name, and oname->name may be vmalloc(), but not + * tname->name. + * * Returns 0 on success or negative error code on error. */ int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, @@ -305,11 +310,15 @@ int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, { struct inode *dir = fname->dir; struct fscrypt_str _tname = FSTR_INIT(NULL, 0); + struct fscrypt_str _oname; struct fscrypt_str iname; char *name = fname->name; int name_len = fname->name_len; int ret; + if (WARN_ON_ONCE(tname && is_vmalloc_addr(tname->name))) + return -EIO; + /* Sanity check that the resulting name will fit in the buffer */ if (fname->name_len > NAME_MAX || fname->ctext_len > NAME_MAX) return -EIO; @@ -350,31 +359,50 @@ int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, goto out_inode; } + if ((!tname || !tname->name) && + (fname->ctext_len == 0 || + unlikely(is_vmalloc_addr(fname->ctext)) || + unlikely(is_vmalloc_addr(oname->name)))) { + ret = fscrypt_fname_alloc_buffer(NAME_MAX, &_tname); + if (ret) + goto out_inode; + tname = &_tname; + } + if (fname->ctext_len == 0) { int declen; - if (!tname) { - ret = fscrypt_fname_alloc_buffer(NAME_MAX, &_tname); - if (ret) - goto out_inode; - tname = &_tname; - } - - declen = base64_decode(name, name_len, - tname->name, false, BASE64_IMAP); + declen = base64_decode(name, name_len, tname->name, false, + BASE64_IMAP); if (declen <= 0) { ret = -EIO; goto out; } iname.name = tname->name; iname.len = declen; + } else if (unlikely(is_vmalloc_addr(fname->ctext))) { + memcpy(tname->name, fname->ctext, fname->ctext_len); + + iname.name = tname->name; + iname.len = fname->ctext_len; } else { iname.name = fname->ctext; iname.len = fname->ctext_len; } - ret = fscrypt_fname_disk_to_usr(dir, 0, 0, &iname, oname); - if (!ret && (dir != fname->dir)) { + _oname.name = unlikely(is_vmalloc_addr(oname->name)) ? + tname->name : oname->name; + _oname.len = oname->len; + + ret = fscrypt_fname_disk_to_usr(dir, 0, 0, &iname, &_oname); + if (ret) + goto out; + + if (unlikely(is_vmalloc_addr(oname->name))) + memcpy(oname->name, _oname.name, _oname.len); + oname->len = _oname.len; + + if (dir != fname->dir) { char tmp_buf[BASE64_CHARS(NAME_MAX)]; name_len = snprintf(tmp_buf, sizeof(tmp_buf), "_%.*s_%llu", diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index ef9e92e362d3e..12dd3dc539152 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -1174,7 +1174,7 @@ static struct dentry *ceph_mkdir(struct mnt_idmap *idmap, struct inode *dir, !req->r_reply_info.head->is_target && !req->r_reply_info.head->is_dentry) err = ceph_handle_notrace_create(dir, dentry); - ret = ERR_PTR(err); + ret = err ? ERR_PTR(err) : NULL; out_req: if (!IS_ERR(ret) && req->r_dentry != dentry) /* Some other dentry was spliced in */ @@ -1764,11 +1764,11 @@ static int __dir_lease_check(const struct dentry *dentry, if (ret > 0) { if (time_before(jiffies, di->time + lwc->dir_lease_ttl)) return STOP; + if (!lwc->expire_dir_lease) + return KEEP; /* Move dentry to tail of dir lease list if we don't want * to delete it. So dentries in the list are checked in a * round robin manner */ - if (!lwc->expire_dir_lease) - return TOUCH; if (dentry->d_lockref.count > 0 || (di->flags & CEPH_DENTRY_REFERENCED)) return TOUCH; @@ -1795,7 +1795,7 @@ int ceph_trim_dentries(struct ceph_mds_client *mdsc) lwc.dir_lease = false; lwc.nr_to_scan = CEPH_CAPS_PER_RELEASE * 2; freed = __dentry_leases_walk(mdsc, &lwc); - if (!lwc.nr_to_scan) /* more invalid leases */ + if (freed > 0 && !lwc.nr_to_scan) /* more invalid leases */ return -EAGAIN; if (lwc.nr_to_scan < CEPH_CAPS_PER_RELEASE) @@ -1805,6 +1805,10 @@ int ceph_trim_dentries(struct ceph_mds_client *mdsc) lwc.expire_dir_lease = freed < count; lwc.dir_lease_ttl = mdsc->fsc->mount_options->caps_wanted_delay_max * HZ; freed +=__dentry_leases_walk(mdsc, &lwc); + if (freed == 0 && count == 0) + /* no progress possible currently, retry futile */ + return 0; + if (!lwc.nr_to_scan) /* more to check */ return -EAGAIN; diff --git a/fs/ceph/export.c b/fs/ceph/export.c index b2f2af1046791..debb9634b9e3d 100644 --- a/fs/ceph/export.c +++ b/fs/ceph/export.c @@ -442,6 +442,16 @@ static struct dentry *ceph_fh_to_parent(struct super_block *sb, return dentry; } +static int ceph_export_copy_name(char *name, const char *src, u32 len) +{ + if (len > NAME_MAX) + return -ENAMETOOLONG; + + memcpy(name, src, len); + name[len] = '\0'; + return 0; +} + static int __get_snap_name(struct dentry *parent, char *name, struct dentry *child) { @@ -513,9 +523,8 @@ static int __get_snap_name(struct dentry *parent, char *name, BUG_ON(!rde->inode.in); if (ceph_snap(inode) == le64_to_cpu(rde->inode.in->snapid)) { - memcpy(name, rde->name, rde->name_len); - name[rde->name_len] = '\0'; - err = 0; + err = ceph_export_copy_name(name, rde->name, + rde->name_len); goto out; } } @@ -580,8 +589,8 @@ static int ceph_get_name(struct dentry *parent, char *name, rinfo = &req->r_reply_info; if (!IS_ENCRYPTED(dir)) { - memcpy(name, rinfo->dname, rinfo->dname_len); - name[rinfo->dname_len] = 0; + err = ceph_export_copy_name(name, rinfo->dname, + rinfo->dname_len); } else { struct fscrypt_str oname = FSTR_INIT(NULL, 0); struct ceph_fname fname = { .dir = dir, @@ -595,10 +604,9 @@ static int ceph_get_name(struct dentry *parent, char *name, goto out; err = ceph_fname_to_usr(&fname, NULL, &oname, NULL); - if (!err) { - memcpy(name, oname.name, oname.len); - name[oname.len] = 0; - } + if (!err) + err = ceph_export_copy_name(name, oname.name, + oname.len); ceph_fname_free_buffer(dir, &oname); } out: diff --git a/fs/ceph/file.c b/fs/ceph/file.c index a4a2a4b6a027d..a0b9c2b5a5832 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -2477,6 +2477,54 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from) if (err < 0) goto out; + /* + * For O_APPEND writes we may have waited for Fwx exclusive caps + * while the previous Fwx holder (another client) extended the + * file. i_size has been updated via the cap grant message from + * the MDS, but ki_pos is still the old EOF. Re-read i_size here + * (no extra MDS round-trip needed) and adjust ki_pos to the true + * EOF. Since we hold Fwx, no other client can change the file. + */ + if (iocb->ki_flags & IOCB_APPEND) { + loff_t cur_eof = i_size_read(inode); + + if (cur_eof != pos) { + doutc(cl, + "%p %llx.%llx O_APPEND: pos adjusted %lld -> %lld\n", + inode, ceph_vinop(inode), pos, cur_eof); + iocb->ki_pos = cur_eof; + pos = cur_eof; + if (pos >= limit) { + err = -EFBIG; + goto out_caps; + } + iov_iter_truncate(from, limit - pos); + count = iov_iter_count(from); + + /* + * ceph_get_caps() validated the old endoff + * against i_max_size; adjusting ki_pos forward + * may have shifted the write range beyond the + * granted max_size. Re-check and truncate if + * necessary. + */ + spin_lock(&ci->i_ceph_lock); + if (pos + count > (loff_t)ci->i_max_size) { + loff_t max_size = ci->i_max_size; + + spin_unlock(&ci->i_ceph_lock); + if (pos >= max_size) { + err = -EFBIG; + goto out_caps; + } + iov_iter_truncate(from, max_size - pos); + count = iov_iter_count(from); + } else { + spin_unlock(&ci->i_ceph_lock); + } + } + } + err = file_update_time(file); if (err) goto out_caps; diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 3c692ad02c857..0b6ff5b704f6d 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -541,9 +541,13 @@ static int parse_reply_info_readdir(void **p, void *end, * to do the base64_decode in-place. It's * safe because the decoded string should * always be shorter, which is 3/4 of origin - * string. + * string. If this message was allocated with + * vmalloc() (happens, but rarely), leave it + * NULL and let ceph_fname_to_usr() allocate + * suitable temporary working space instead. */ - tname.name = _name; + if (likely(!is_vmalloc_addr(_name))) + tname.name = _name; /* * Set oname to _name too, and this will be @@ -615,10 +619,36 @@ static int parse_reply_info_filelock(void **p, void *end, #define DELEGATED_INO_AVAILABLE xa_mk_value(1) +static int ceph_insert_deleg_ino(struct ceph_mds_session *s, u64 ino) +{ + struct ceph_client *cl = s->s_mdsc->fsc->client; + int err; + + /* + * Cap how many delegated inodes a single session may hold. This is + * the only place that grows the count, so atomic_add_unless() bounds + * it at exactly CEPH_MAX_DELEG_INOS; s_num_deleg_inos can never exceed + * that. + */ + if (!atomic_add_unless(&s->s_num_deleg_inos, 1, CEPH_MAX_DELEG_INOS)) { + pr_warn_ratelimited_client(cl, + "MDS session already holds %d delegated inodes\n", + CEPH_MAX_DELEG_INOS); + return -EOVERFLOW; + } + + err = xa_insert(&s->s_delegated_inos, ino, DELEGATED_INO_AVAILABLE, + GFP_KERNEL); + if (err) + atomic_dec(&s->s_num_deleg_inos); + return err; +} + static int ceph_parse_deleg_inos(void **p, void *end, struct ceph_mds_session *s) { struct ceph_client *cl = s->s_mdsc->fsc->client; + u64 msg_deleg_inos = 0; u32 sets; ceph_decode_32_safe(p, end, sets, bad); @@ -636,16 +666,34 @@ static int ceph_parse_deleg_inos(void **p, void *end, start, len); continue; } + + /* + * Bound the number of inodes one reply may delegate. + * ceph_insert_deleg_ino() separately caps the per-session + * population, so this only has to stop one reply from spinning + * the insert loop under an attacker-controlled len. + */ + if (len > (u64)CEPH_MAX_DELEG_INOS || + msg_deleg_inos > (u64)CEPH_MAX_DELEG_INOS - len) { + pr_warn_ratelimited_client(cl, + "MDS reply delegates too many inodes (have %llu, +%llu, max %d)\n", + msg_deleg_inos, len, CEPH_MAX_DELEG_INOS); + return -EIO; + } + msg_deleg_inos += len; + while (len--) { - int err = xa_insert(&s->s_delegated_inos, start++, - DELEGATED_INO_AVAILABLE, - GFP_KERNEL); + int err = ceph_insert_deleg_ino(s, start++); + if (!err) { doutc(cl, "added delegated inode 0x%llx\n", start - 1); } else if (err == -EBUSY) { pr_warn_client(cl, "MDS delegated inode 0x%llx more than once.\n", start - 1); + } else if (err == -EOVERFLOW) { + /* ceph_insert_deleg_ino() already warned. */ + return -EIO; } else { return err; } @@ -663,16 +711,17 @@ u64 ceph_get_deleg_ino(struct ceph_mds_session *s) xa_for_each(&s->s_delegated_inos, ino, val) { val = xa_erase(&s->s_delegated_inos, ino); - if (val == DELEGATED_INO_AVAILABLE) + if (val == DELEGATED_INO_AVAILABLE) { + atomic_dec(&s->s_num_deleg_inos); return ino; + } } return 0; } int ceph_restore_deleg_ino(struct ceph_mds_session *s, u64 ino) { - return xa_insert(&s->s_delegated_inos, ino, DELEGATED_INO_AVAILABLE, - GFP_KERNEL); + return ceph_insert_deleg_ino(s, ino); } #else /* BITS_PER_LONG == 64 */ /* @@ -1059,6 +1108,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, INIT_LIST_HEAD(&s->s_waiting); INIT_LIST_HEAD(&s->s_unsafe); xa_init(&s->s_delegated_inos); + atomic_set(&s->s_num_deleg_inos, 0); INIT_LIST_HEAD(&s->s_cap_releases); INIT_WORK(&s->s_cap_release_work, ceph_cap_release_work); @@ -4441,7 +4491,9 @@ static void handle_session(struct ceph_mds_session *session, pr_err_client(cl, "No memory for path\n"); goto fail; } - ceph_decode_copy(&p, cap_auths[i].match.path, _len); + ceph_decode_copy_safe(&p, end, + cap_auths[i].match.path, + _len, bad); /* Remove the tailing '/' */ while (_len && cap_auths[i].match.path[_len - 1] == '/') { @@ -4458,7 +4510,9 @@ static void handle_session(struct ceph_mds_session *session, pr_err_client(cl, "No memory for fs_name\n"); goto fail; } - ceph_decode_copy(&p, cap_auths[i].match.fs_name, _len); + ceph_decode_copy_safe(&p, end, + cap_auths[i].match.fs_name, + _len, bad); } ceph_decode_8_safe(&p, end, cap_auths[i].match.root_squash, bad); @@ -5106,6 +5160,7 @@ static int send_mds_reconnect(struct ceph_mds_client *mdsc, /* Serialized by s_mutex against concurrent ceph_get_deleg_ino(). */ xa_destroy(&session->s_delegated_inos); + atomic_set(&session->s_num_deleg_inos, 0); if (session->s_state == CEPH_MDS_SESSION_CLOSED || session->s_state == CEPH_MDS_SESSION_REJECTED) { pr_info_client(cl, "mds%d skipping reconnect, session %s\n", @@ -5834,9 +5889,11 @@ static void check_new_map(struct ceph_mds_client *mdsc, ceph_mdsmap_get_addr(newmap, i), sizeof(struct ceph_entity_addr))) { /* just close it */ + ceph_get_mds_session(s); mutex_unlock(&mdsc->mutex); mutex_lock(&s->s_mutex); mutex_lock(&mdsc->mutex); + ceph_put_mds_session(s); ceph_con_close(&s->s_con); mutex_unlock(&s->s_mutex); s->s_state = CEPH_MDS_SESSION_RESTARTING; @@ -5851,6 +5908,7 @@ static void check_new_map(struct ceph_mds_client *mdsc, newstate >= CEPH_MDS_STATE_RECONNECT) { int rc; + ceph_get_mds_session(s); mutex_unlock(&mdsc->mutex); clear_bit(i, targets); rc = send_mds_reconnect(mdsc, s); @@ -5859,6 +5917,7 @@ static void check_new_map(struct ceph_mds_client *mdsc, "mds%d reconnect failed: %d\n", i, rc); mutex_lock(&mdsc->mutex); + ceph_put_mds_session(s); } /* @@ -5871,9 +5930,11 @@ static void check_new_map(struct ceph_mds_client *mdsc, pr_info_client(cl, "mds%d recovery completed\n", s->s_mds); kick_requests(mdsc, i); + ceph_get_mds_session(s); mutex_unlock(&mdsc->mutex); mutex_lock(&s->s_mutex); mutex_lock(&mdsc->mutex); + ceph_put_mds_session(s); ceph_kick_flushing_caps(mdsc, s); mutex_unlock(&s->s_mutex); wake_up_session_caps(s, RECONNECT); @@ -6532,11 +6593,13 @@ int ceph_mds_check_access(struct ceph_mds_client *mdsc, char *tpath, int mask) doutc(cl, "tpath '%s', mask %d, caller_uid %d, caller_gid %d\n", tpath, mask, caller_uid, caller_gid); + mutex_lock(&mdsc->mutex); for (i = 0; i < mdsc->s_cap_auths_num; i++) { struct ceph_mds_cap_auth *s = &mdsc->s_cap_auths[i]; err = ceph_mds_auth_match(mdsc, s, cred, tpath); if (err < 0) { + mutex_unlock(&mdsc->mutex); put_cred(cred); return err; } else if (err > 0) { @@ -6558,6 +6621,7 @@ int ceph_mds_check_access(struct ceph_mds_client *mdsc, char *tpath, int mask) doutc(cl, "root_squash_perms %d, rw_perms_s %p\n", root_squash_perms, rw_perms_s); if (root_squash_perms && rw_perms_s == NULL) { + mutex_unlock(&mdsc->mutex); doutc(cl, "access allowed\n"); return 0; } @@ -6572,6 +6636,7 @@ int ceph_mds_check_access(struct ceph_mds_client *mdsc, char *tpath, int mask) !!(mask & MAY_READ), !!(mask & MAY_WRITE)); } doutc(cl, "access denied\n"); + mutex_unlock(&mdsc->mutex); return -EACCES; } diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index 0ece4c9e3529f..e7a262c9c2ab4 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -300,6 +300,7 @@ struct ceph_mds_session { struct list_head s_waiting; /* waiting requests */ struct list_head s_unsafe; /* unsafe requests */ struct xarray s_delegated_inos; + atomic_t s_num_deleg_inos; }; /* @@ -603,6 +604,7 @@ struct ceph_mds_client { struct rw_semaphore pool_perm_rwsem; struct rb_root pool_perm_tree; + /* protected by mutex */ u32 s_cap_auths_num; struct ceph_mds_cap_auth *s_cap_auths; diff --git a/fs/ceph/mdsmap.c b/fs/ceph/mdsmap.c index 450a4dc9662e5..53079ef34c3a1 100644 --- a/fs/ceph/mdsmap.c +++ b/fs/ceph/mdsmap.c @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -126,6 +127,7 @@ struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p, u8 mdsmap_v; u16 mdsmap_ev; u32 target; + size_t export_targets_len; m = kzalloc_obj(*m, GFP_NOFS); if (!m) @@ -224,8 +226,11 @@ struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p, *p += namelen; if (info_v >= 2) { ceph_decode_32_safe(p, end, num_export_targets, bad); + export_targets_len = size_mul(num_export_targets, + sizeof(u32)); + ceph_decode_need(p, end, export_targets_len, bad); pexport_targets = *p; - *p += num_export_targets * sizeof(u32); + *p += export_targets_len; } else { num_export_targets = 0; } @@ -264,6 +269,10 @@ struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p, goto nomem; for (j = 0; j < num_export_targets; j++) { target = ceph_decode_32(&pexport_targets); + if (target >= CEPH_MAX_MDS) { + err = -EIO; + goto corrupt; + } info->export_targets[j] = target; } } else { diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 1d6aab060780a..38f86294182cf 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -641,6 +641,15 @@ static inline int ceph_ino_compare(struct inode *inode, void *data) #define CEPH_MDS_INO_LOG_OFFSET (2 * CEPH_MAX_MDS) #define CEPH_INO_SYSTEM_BASE ((6*CEPH_MAX_MDS) + (CEPH_MAX_MDS * CEPH_NUM_STRAY)) +/* + * Upper bound on the number of delegated inodes a single MDS session may + * hold. The MDS normally hands out a small preallocation window (the + * userspace mds_client_prealloc_inos option defaults to 1000) and refills + * it as the client consumes entries. This leaves generous headroom while + * bounding the CPU and memory a malformed delegation interval can consume. + */ +#define CEPH_MAX_DELEG_INOS 8192 + static inline bool ceph_vino_is_reserved(const struct ceph_vino vino) { if (vino.ino >= CEPH_INO_SYSTEM_BASE || @@ -687,6 +696,10 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, #define CEPH_I_ASYNC_CREATE_BIT (12) /* async create in flight for this */ #define CEPH_I_SHUTDOWN_BIT (13) /* inode is no longer usable */ #define CEPH_I_ASYNC_CHECK_CAPS_BIT (14) /* check caps after async creating finishes */ +#define CEPH_I_FLUSH_FORCE_BIT (15) /* a revoke's response was deferred; + * force a cap message to the MDS once + * the deferred work completes + */ #define CEPH_I_DIR_ORDERED (1 << CEPH_I_DIR_ORDERED_BIT) #define CEPH_I_FLUSH (1 << CEPH_I_FLUSH_BIT) @@ -699,6 +712,7 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, #define CEPH_I_ODIRECT (1 << CEPH_I_ODIRECT_BIT) #define CEPH_I_ASYNC_CREATE (1 << CEPH_I_ASYNC_CREATE_BIT) #define CEPH_I_SHUTDOWN (1 << CEPH_I_SHUTDOWN_BIT) +#define CEPH_I_FLUSH_FORCE (1 << CEPH_I_FLUSH_FORCE_BIT) /* * Masks of ceph inode work. diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c index 860fc8e1867df..cc4ffbbcb719a 100644 --- a/fs/ceph/xattr.c +++ b/fs/ceph/xattr.c @@ -848,6 +848,7 @@ static int __build_xattrs(struct inode *inode) name = p; p += len; ceph_decode_32_safe(&p, end, len, bad); + ceph_decode_need(&p, end, len, bad); val = p; p += len; diff --git a/fs/configfs/dir.c b/fs/configfs/dir.c index 3c88f13f1ca2e..eda80c2a2d388 100644 --- a/fs/configfs/dir.c +++ b/fs/configfs/dir.c @@ -416,6 +416,15 @@ static void configfs_remove_dir(struct dentry *d) if (d_really_is_positive(d)) { if (unlikely(simple_rmdir(d_inode(parent), d))) pr_warn("remove_dir (%pd): attributes remain", d); + else + /* + * configfs_get_config_item() takes a hashed dentry as + * proof that ->s_element is still alive. Our caller + * is about to drop the last reference to the item and + * the VFS will not unhash until after we return, so + * unhash it here. + */ + d_drop(d); } pr_debug(" o %pd removing done (%d)\n", d, d_count(d)); diff --git a/fs/configfs/symlink.c b/fs/configfs/symlink.c index 31eb28b273091..3b31c714400f9 100644 --- a/fs/configfs/symlink.c +++ b/fs/configfs/symlink.c @@ -76,9 +76,9 @@ static int configfs_get_target_path(struct config_item *item, static int create_link(struct config_item *parent_item, struct config_item *item, + struct configfs_dirent *target_sd, struct dentry *dentry) { - struct configfs_dirent *target_sd = item->ci_dentry->d_fsdata; char *body; int ret; @@ -115,6 +115,7 @@ static int create_link(struct config_item *parent_item, static int get_target(const char *symname, struct config_item **target, + struct configfs_dirent **target_sd, struct super_block *sb) { struct path path __free(path_put) = {}; @@ -125,7 +126,20 @@ static int get_target(const char *symname, struct config_item **target, return ret; if (path.dentry->d_sb != sb) return -EPERM; - *target = configfs_get_config_item(path.dentry); + /* + * A hashed dentry guarantees that neither the item nor the dirent + * have been released yet, as removals unhash before dropping. + * Grab both references here. An item reference alone would not keep + * ->ci_dentry alive. + */ + spin_lock(&path.dentry->d_lock); + if (!d_unhashed(path.dentry)) { + struct configfs_dirent *sd = path.dentry->d_fsdata; + + *target = config_item_get(sd->s_element); + *target_sd = configfs_get(sd); + } + spin_unlock(&path.dentry->d_lock); if (!*target) return -ENOENT; return 0; @@ -139,6 +153,7 @@ int configfs_symlink(struct mnt_idmap *idmap, struct inode *dir, struct configfs_dirent *sd; struct config_item *parent_item; struct config_item *target_item = NULL; + struct configfs_dirent *target_sd = NULL; const struct config_item_type *type; sd = dentry->d_parent->d_fsdata; @@ -182,7 +197,7 @@ int configfs_symlink(struct mnt_idmap *idmap, struct inode *dir, * AV, a thoroughly annoyed bastard. */ inode_unlock(dir); - ret = get_target(symname, &target_item, dentry->d_sb); + ret = get_target(symname, &target_item, &target_sd, dentry->d_sb); inode_lock(dir); if (ret) goto out_put; @@ -196,13 +211,14 @@ int configfs_symlink(struct mnt_idmap *idmap, struct inode *dir, ret = type->ct_item_ops->allow_link(parent_item, target_item); if (!ret) { mutex_lock(&configfs_symlink_mutex); - ret = create_link(parent_item, target_item, dentry); + ret = create_link(parent_item, target_item, target_sd, dentry); mutex_unlock(&configfs_symlink_mutex); if (ret && type->ct_item_ops->drop_link) type->ct_item_ops->drop_link(parent_item, target_item); } + configfs_put(target_sd); config_item_put(target_item); out_put: diff --git a/fs/dax.c b/fs/dax.c index 6d175cd47a99b..cadbf76da2279 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -392,12 +392,12 @@ int dax_folio_reset_order(struct folio *folio) int order = folio_order(folio); /* - * DAX maintains the invariant that folio->share != 0 only when - * folio->mapping == NULL (enforced by dax_folio_make_shared()). - * Equivalently: folio->mapping != NULL implies folio->share == 0. - * Callers ensure share has been decremented to zero before - * calling here, so unconditionally clearing both fields is - * correct. + * Clear the mapping and the index/share union word. folio->share + * and folio->index occupy the same union in struct folio. For + * non-shared folios (mapping != NULL), the union holds folio->index + * (file page offset); for shared folios (mapping == NULL), it holds + * folio->share (reference count). Either way, we are releasing the + * folio and both fields should be zeroed. */ folio->mapping = NULL; folio->share = 0; @@ -480,11 +480,12 @@ static void dax_associate_entry(void *entry, struct address_space *mapping, unsigned long address, bool shared) { unsigned long size = dax_entry_size(entry), index; - struct folio *folio = dax_to_folio(entry); + struct folio *folio; if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) return; + folio = dax_to_folio(entry); index = linear_page_index(vma, address & ~(size - 1)); if (shared && (folio->mapping || dax_folio_is_shared(folio))) { if (folio->mapping) @@ -505,21 +506,23 @@ static void dax_associate_entry(void *entry, struct address_space *mapping, static void dax_disassociate_entry(void *entry, struct address_space *mapping, bool trunc) { - struct folio *folio = dax_to_folio(entry); + struct folio *folio; if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) return; + folio = dax_to_folio(entry); dax_folio_put(folio); } static struct page *dax_busy_page(void *entry) { - struct folio *folio = dax_to_folio(entry); + struct folio *folio; if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) return NULL; + folio = dax_to_folio(entry); if (folio_ref_count(folio) - folio_mapcount(folio)) return &folio->page; else diff --git a/fs/dcache.c b/fs/dcache.c index 3e9af9de70746..073c2ce2d4448 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1794,7 +1794,12 @@ static void do_one_tree(struct dentry *dentry) { shrink_dcache_tree(dentry, true); d_walk(dentry, dentry, umount_check); - d_drop(dentry); + spin_lock(&dentry->d_lock); + __d_drop(dentry); + /* A busy root survives the dput() below so don't leave it on ->s_roots. */ + if (unlikely(!hlist_unhashed(&dentry->d_sib))) + unlink_secondary_root(dentry); + spin_unlock(&dentry->d_lock); dput(dentry); } diff --git a/fs/debugfs/file.c b/fs/debugfs/file.c index edd6aafbfbaaf..08de6652a4f31 100644 --- a/fs/debugfs/file.c +++ b/fs/debugfs/file.c @@ -273,7 +273,8 @@ static int debugfs_locked_down(struct inode *inode, (!real_fops || (!real_fops->unlocked_ioctl && !real_fops->compat_ioctl && - !real_fops->mmap))) + !real_fops->mmap && + !real_fops->mmap_prepare))) return 0; if (security_locked_down(LOCKDOWN_DEBUGFS)) diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c index 74b02b55e3f6d..e67119b6029c7 100644 --- a/fs/ecryptfs/crypto.c +++ b/fs/ecryptfs/crypto.c @@ -1197,7 +1197,7 @@ static int ecryptfs_read_headers_virt(char *page_virt, } else set_default_header_data(crypt_stat); rc = ecryptfs_parse_packet_set(crypt_stat, (page_virt + offset), - ecryptfs_dentry); + PAGE_SIZE - offset, ecryptfs_dentry); out: return rc; } diff --git a/fs/ecryptfs/ecryptfs_kernel.h b/fs/ecryptfs/ecryptfs_kernel.h index f4f56a92bd563..7d2488a10b179 100644 --- a/fs/ecryptfs/ecryptfs_kernel.h +++ b/fs/ecryptfs/ecryptfs_kernel.h @@ -580,7 +580,8 @@ int ecryptfs_generate_key_packet_set(char *dest_base, size_t *len, size_t max); int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, - unsigned char *src, struct dentry *ecryptfs_dentry); + unsigned char *src, size_t src_size, + struct dentry *ecryptfs_dentry); int ecryptfs_truncate(struct dentry *dentry, loff_t new_length); ssize_t ecryptfs_getxattr_lower(struct dentry *lower_dentry, struct inode *lower_inode, diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c index ebebc9551f1ff..51651314b7a62 100644 --- a/fs/ecryptfs/keystore.c +++ b/fs/ecryptfs/keystore.c @@ -894,6 +894,12 @@ ecryptfs_parse_tag_70_packet(char **filename, size_t *filename_size, "rc = [%d]\n", __func__, rc); goto out; } + if (s->parsed_tag_70_packet_size < (ECRYPTFS_SIG_SIZE + 2)) { + ecryptfs_printk(KERN_WARNING, "Invalid packet size [%zd]\n", + s->parsed_tag_70_packet_size); + rc = -EINVAL; + goto out; + } s->block_aligned_filename_size = (s->parsed_tag_70_packet_size - ECRYPTFS_SIG_SIZE - 1); if ((1 + s->packet_size_len + s->parsed_tag_70_packet_size) @@ -1384,10 +1390,20 @@ parse_tag_3_packet(struct ecryptfs_crypt_stat *crypt_stat, } (*new_auth_tok)->session_key.encrypted_key_size = (body_size - (ECRYPTFS_SALT_SIZE + 5)); + /* + * Although encrypted_key_size is copied into the + * encrypted_key[ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES] buffer here, + * it later bounds operations on a smaller buffer: + * decrypt_passphrase_encrypted_session_key() sets decrypted_key_size = + * encrypted_key_size and decrypts into + * decrypted_key[ECRYPTFS_MAX_KEY_BYTES], then memcpy's into + * crypt_stat->key[ECRYPTFS_MAX_KEY_BYTES]. Limit to + * ECRYPTFS_MAX_KEY_BYTES to protect those smaller buffers. + */ if ((*new_auth_tok)->session_key.encrypted_key_size - > ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES) { + > ECRYPTFS_MAX_KEY_BYTES) { printk(KERN_WARNING "Tag 3 packet contains key larger " - "than ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES\n"); + "than ECRYPTFS_MAX_KEY_BYTES\n"); rc = -EINVAL; goto out_free; } @@ -1537,7 +1553,7 @@ parse_tag_11_packet(unsigned char *data, unsigned char *contents, } (*packet_size) += length_size; (*tag_11_contents_size) = (body_size - 14); - if (unlikely((*packet_size) + body_size + 1 > max_packet_size)) { + if (unlikely((*packet_size) + body_size > max_packet_size)) { printk(KERN_ERR "Packet size exceeds max\n"); rc = -EINVAL; goto out; @@ -1704,6 +1720,7 @@ decrypt_passphrase_encrypted_session_key(struct ecryptfs_auth_tok *auth_tok, * ecryptfs_parse_packet_set * @crypt_stat: The cryptographic context * @src: Virtual address of region of memory containing the packets + * @src_size: Size of the packet set buffer * @ecryptfs_dentry: The eCryptfs dentry associated with the packet set * * Get crypt_stat to have the file's session key if the requisite key @@ -1714,7 +1731,7 @@ decrypt_passphrase_encrypted_session_key(struct ecryptfs_auth_tok *auth_tok, * conditions. */ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, - unsigned char *src, + unsigned char *src, size_t src_size, struct dentry *ecryptfs_dentry) { size_t i = 0; @@ -1736,7 +1753,11 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, * added the our &auth_tok_list */ next_packet_is_auth_tok_packet = 1; while (next_packet_is_auth_tok_packet) { - size_t max_packet_size = ((PAGE_SIZE - 8) - i); + size_t max_packet_size; + + if (i >= src_size) + break; + max_packet_size = src_size - i; switch (src[i]) { case ECRYPTFS_TAG_3_PACKET_TYPE: @@ -1751,12 +1772,16 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, goto out_wipe_list; } i += packet_size; + if (i > src_size) { + rc = -EIO; + goto out_wipe_list; + } rc = parse_tag_11_packet((unsigned char *)&src[i], sig_tmp_space, ECRYPTFS_SIG_SIZE, &tag_11_contents_size, &tag_11_packet_size, - max_packet_size); + src_size - i); if (rc) { ecryptfs_printk(KERN_ERR, "No valid " "(ecryptfs-specific) literal " @@ -1768,6 +1793,10 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, goto out_wipe_list; } i += tag_11_packet_size; + if (i > src_size) { + rc = -EIO; + goto out_wipe_list; + } if (ECRYPTFS_SIG_SIZE != tag_11_contents_size) { ecryptfs_printk(KERN_ERR, "Expected " "signature of size [%d]; " @@ -1793,6 +1822,10 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, goto out_wipe_list; } i += packet_size; + if (i > src_size) { + rc = -EIO; + goto out_wipe_list; + } crypt_stat->flags |= ECRYPTFS_ENCRYPTED; break; case ECRYPTFS_TAG_11_PACKET_TYPE: diff --git a/fs/ecryptfs/messaging.c b/fs/ecryptfs/messaging.c index 03c60f0850ca6..73c7b8215e667 100644 --- a/fs/ecryptfs/messaging.c +++ b/fs/ecryptfs/messaging.c @@ -166,6 +166,7 @@ int ecryptfs_exorcise_daemon(struct ecryptfs_daemon *daemon) mutex_unlock(&daemon->mux); goto out; } + mutex_lock(&ecryptfs_msg_ctx_lists_mux); list_for_each_entry_safe(msg_ctx, msg_ctx_tmp, &daemon->msg_ctx_out_queue, daemon_out_list) { list_del(&msg_ctx->daemon_out_list); @@ -174,6 +175,7 @@ int ecryptfs_exorcise_daemon(struct ecryptfs_daemon *daemon) "the out queue of a dying daemon\n", __func__); ecryptfs_msg_ctx_alloc_to_free(msg_ctx); } + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); hlist_del(&daemon->euid_chain); mutex_unlock(&daemon->mux); kfree_sensitive(daemon); @@ -284,9 +286,16 @@ ecryptfs_send_message_locked(char *data, int data_len, u8 msg_type, mutex_unlock(&ecryptfs_msg_ctx_lists_mux); rc = ecryptfs_send_miscdev(data, data_len, *msg_ctx, msg_type, 0, daemon); - if (rc) + if (rc) { printk(KERN_ERR "%s: Error attempting to send message to " "userspace daemon; rc = [%d]\n", __func__, rc); + mutex_lock(&ecryptfs_msg_ctx_lists_mux); + mutex_lock(&(*msg_ctx)->mux); + ecryptfs_msg_ctx_alloc_to_free(*msg_ctx); + mutex_unlock(&(*msg_ctx)->mux); + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); + *msg_ctx = NULL; + } out: return rc; } diff --git a/fs/ecryptfs/miscdev.c b/fs/ecryptfs/miscdev.c index 5a7d081499227..68804399a5dfc 100644 --- a/fs/ecryptfs/miscdev.c +++ b/fs/ecryptfs/miscdev.c @@ -360,7 +360,7 @@ ecryptfs_miscdev_write(struct file *file, const char __user *buf, u32 seq; size_t packet_size, packet_size_length; char *data; - unsigned char packet_size_peek[ECRYPTFS_MAX_PKT_LEN_SIZE]; + unsigned char packet_size_peek[ECRYPTFS_MAX_PKT_LEN_SIZE] = { }; ssize_t rc; if (count == 0) { @@ -376,7 +376,8 @@ ecryptfs_miscdev_write(struct file *file, const char __user *buf, } if (copy_from_user(packet_size_peek, &buf[PKT_LEN_OFFSET], - sizeof(packet_size_peek))) { + min_t(size_t, count - PKT_LEN_OFFSET, + sizeof(packet_size_peek)))) { printk(KERN_WARNING "%s: Error while inspecting packet size\n", __func__); return -EFAULT; diff --git a/fs/ecryptfs/super.c b/fs/ecryptfs/super.c index 3bc21d677564d..686b2b4a9cb56 100644 --- a/fs/ecryptfs/super.c +++ b/fs/ecryptfs/super.c @@ -150,6 +150,13 @@ static int ecryptfs_show_options(struct seq_file *m, struct dentry *root) if (mount_crypt_stat->global_default_cipher_key_size) seq_printf(m, ",ecryptfs_key_bytes=%zd", mount_crypt_stat->global_default_cipher_key_size); + if (mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) { + seq_printf(m, ",ecryptfs_fn_cipher=%s", + mount_crypt_stat->global_default_fn_cipher_name); + if (mount_crypt_stat->global_default_fn_cipher_key_bytes) + seq_printf(m, ",ecryptfs_fn_key_bytes=%zd", + mount_crypt_stat->global_default_fn_cipher_key_bytes); + } if (mount_crypt_stat->flags & ECRYPTFS_PLAINTEXT_PASSTHROUGH_ENABLED) seq_printf(m, ",ecryptfs_passthrough"); if (mount_crypt_stat->flags & ECRYPTFS_XATTR_METADATA_ENABLED) diff --git a/fs/efivarfs/super.c b/fs/efivarfs/super.c index 733c19571f1cf..8d33f11db2a1b 100644 --- a/fs/efivarfs/super.c +++ b/fs/efivarfs/super.c @@ -89,12 +89,30 @@ static int efivarfs_statfs(struct dentry *dentry, struct kstatfs *buf) /* Some UEFI firmware does not implement QueryVariableInfo() */ storage_space = remaining_space = 0; if (efi_rt_services_supported(EFI_RT_SUPPORTED_QUERY_VARIABLE_INFO)) { - status = efivar_query_variable_info(attr, &storage_space, - &remaining_space, - &max_variable_size); - if (status != EFI_SUCCESS && status != EFI_UNSUPPORTED) - pr_warn_ratelimited("query_variable_info() failed: 0x%lx\n", - status); + static DEFINE_RATELIMIT_STATE(_rs, 2 * HZ, 5); + static u64 storage, remaining; + static DEFINE_SPINLOCK(lock); + + if (!__ratelimit(&_rs)) { + ratelimit_set_flags(&_rs, RATELIMIT_MSG_ON_RELEASE); + + spin_lock(&lock); + storage_space = storage; + remaining_space = remaining; + spin_unlock(&lock); + } else { + status = efivar_query_variable_info(attr, &storage_space, + &remaining_space, + &max_variable_size); + if (status != EFI_SUCCESS && status != EFI_UNSUPPORTED) + pr_warn("query_variable_info() failed: 0x%lx\n", + status); + + spin_lock(&lock); + storage = storage_space; + remaining = remaining_space; + spin_unlock(&lock); + } } /* diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig index 2dfc313588d28..37495e95a78d9 100644 --- a/fs/erofs/Kconfig +++ b/fs/erofs/Kconfig @@ -132,11 +132,13 @@ config EROFS_FS_ZIP_LZMA Say N if you want to disable LZMA compression support. config EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS - int "EROFS LZMA default maximum decompression streams" + int "EROFS LZMA default maximum decompression streams" if SMP depends on EROFS_FS_ZIP_LZMA range 1 NR_CPUS if SMP range 1 1 if !SMP - default 16 + default 16 if SMP && NR_CPUS >= 16 + default NR_CPUS if SMP + default 1 help By default EROFS allocates one LZMA decompression stream per CPU. Each stream can hold a dictionary of up to 8 MiB taken from the @@ -144,7 +146,7 @@ config EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS of memory. This caps the default; the lzma_streams module parameter still overrides it. - If unsure, keep the default of 16. + If unsure, keep the suggested default (16 or lower). config EROFS_FS_ZIP_DEFLATE bool "EROFS DEFLATE compressed data support" diff --git a/fs/erofs/data.c b/fs/erofs/data.c index 9aa48c8d67d12..b6cdffd0d8936 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -61,7 +61,7 @@ void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap) return NULL; if (!buf->base) buf->base = kmap_local_page(buf->page); - return buf->base + (offset & ~PAGE_MASK); + return buf->base + ((buf->off + offset) & ~PAGE_MASK); } int erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb, diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index 3c54e95964c9f..d387b27c4ee21 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -7,8 +7,6 @@ #include "compress.h" #include -#define LZ4_MAX_DISTANCE_PAGES (DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1) - static int z_erofs_load_lz4_config(struct super_block *sb, struct erofs_super_block *dsb, void *data, int size) { @@ -21,8 +19,6 @@ static int z_erofs_load_lz4_config(struct super_block *sb, erofs_err(sb, "invalid lz4 cfgs, size=%u", size); return -EINVAL; } - distance = le16_to_cpu(lz4->max_distance); - sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks); if (!sbi->lz4.max_pclusterblks) { sbi->lz4.max_pclusterblks = 1; /* reserved case */ @@ -39,45 +35,25 @@ static int z_erofs_load_lz4_config(struct super_block *sb, sbi->lz4.max_pclusterblks = 1; sbi->available_compr_algs = 1 << Z_EROFS_COMPRESSION_LZ4; } - - sbi->lz4.max_distance_pages = distance ? - DIV_ROUND_UP(distance, PAGE_SIZE) + 1 : - LZ4_MAX_DISTANCE_PAGES; return z_erofs_gbuf_growsize(sbi->lz4.max_pclusterblks); } /* - * Fill all gaps with bounce pages if it's a sparse page list. Also check if - * all physical pages are consecutive, which can be seen for moderate CR. + * Fill all gaps with bounce pages if it's a sparse page list (for example some + * folios are already uptodate and thus can be mapped into userspace). Also + * check if pages are physically consecutive, which can be seen for moderate CR. */ -static int z_erofs_lz4_prepare_dstpages(struct z_erofs_decompress_req *rq, - struct page **pagepool) +static int z_erofs_oneshot_prepare_dstpages(struct z_erofs_decompress_req *rq, + struct page **pagepool) { - struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL }; - unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES, - BITS_PER_LONG)] = { 0 }; - unsigned int lz4_max_distance_pages = - EROFS_SB(rq->sb)->lz4.max_distance_pages; void *kaddr = NULL; - unsigned int i, j, top; + unsigned int i; - top = 0; - for (i = j = 0; i < rq->outpages; ++i, ++j) { - struct page *const page = rq->out[i]; - struct page *victim; - - if (j >= lz4_max_distance_pages) - j = 0; - - /* 'valid' bounced can only be tested after a complete round */ - if (!rq->fillgaps && test_bit(j, bounced)) { - DBG_BUGON(i < lz4_max_distance_pages); - DBG_BUGON(top >= lz4_max_distance_pages); - availables[top++] = rq->out[i - lz4_max_distance_pages]; - } + for (i = 0; i < rq->outpages; ++i) { + struct page *page, *victim; + page = rq->out[i]; if (page) { - __clear_bit(j, bounced); if (!PageHighMem(page)) { if (!i) { kaddr = page_address(page); @@ -89,21 +65,14 @@ static int z_erofs_lz4_prepare_dstpages(struct z_erofs_decompress_req *rq, continue; } } - kaddr = NULL; - continue; - } - kaddr = NULL; - __set_bit(j, bounced); - - if (top) { - victim = availables[--top]; } else { victim = __erofs_allocpage(pagepool, rq->gfp, true); if (!victim) return -ENOMEM; set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); + rq->out[i] = victim; } - rq->out[i] = victim; + kaddr = NULL; } return kaddr ? 1 : 0; } @@ -266,7 +235,7 @@ static const char *z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, dst_maptype = 0; } else { /* general decoding path which can be used for all cases */ - ret = z_erofs_lz4_prepare_dstpages(rq, pagepool); + ret = z_erofs_oneshot_prepare_dstpages(rq, pagepool); if (ret < 0) return ERR_PTR(ret); if (ret > 0) { @@ -299,6 +268,7 @@ static const char *z_erofs_transform_plain(struct z_erofs_decompress_req *rq, return ERR_PTR(-EOPNOTSUPP); if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) { cur = bs - (rq->pageofs_out & (bs - 1)); + DBG_BUGON(rq->pageofs_in & (bs - 1)); pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK; cur = min(cur, rq->outputsize); if (cur && rq->out[0]) { diff --git a/fs/erofs/decompressor_lzma.c b/fs/erofs/decompressor_lzma.c index 6b0cdb446c6ad..9d15f94cbee17 100644 --- a/fs/erofs/decompressor_lzma.c +++ b/fs/erofs/decompressor_lzma.c @@ -5,6 +5,7 @@ struct z_erofs_lzma { struct z_erofs_lzma *next; struct xz_dec_microlzma *state; + unsigned int dict_size; u8 bounce[PAGE_SIZE]; }; @@ -128,11 +129,19 @@ static int z_erofs_load_lzma_config(struct super_block *sb, err = 0; /* 2. walk each isolated stream and grow max dict_size if needed */ for (strm = head; strm; strm = strm->next) { + struct xz_dec_microlzma *state; + + if (strm->dict_size >= dict_size) + continue; + state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); + if (!state) { + err = -ENOMEM; + break; + } if (strm->state) xz_dec_microlzma_end(strm->state); - strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); - if (!strm->state) - err = -ENOMEM; + strm->state = state; + strm->dict_size = dict_size; } /* 3. push back all to the global list and update max dict_size */ @@ -142,7 +151,8 @@ static int z_erofs_load_lzma_config(struct super_block *sb, spin_unlock(&z_erofs_lzma_lock); wake_up_all(&z_erofs_lzma_wq); - z_erofs_lzma_max_dictsize = dict_size; + if (!err) + z_erofs_lzma_max_dictsize = dict_size; mutex_unlock(&lzma_resize_mutex); return err; } diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 57bd21859c65d..be98c9222bf79 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -71,12 +71,8 @@ struct erofs_dev_context { bool flatdev; }; -/* all filesystem-wide lz4 configurations */ struct erofs_sb_lz4_info { - /* # of pages needed for EROFS lz4 rolling decompression */ - u16 max_distance_pages; - /* maximum possible blocks for pclusters in the filesystem */ - u16 max_pclusterblks; + u16 max_pclusterblks; /* maximum physical blocks for LZ4 pclusters */ }; struct erofs_xattr_prefix_item { @@ -269,7 +265,7 @@ struct erofs_inode { #ifdef CONFIG_EROFS_FS_ZIP struct { unsigned short z_advise; - unsigned char z_algorithmtype[2]; + unsigned char z_algofmt[2]; unsigned char z_lclusterbits; union { u64 z_tailextent_headlcn; diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c index 6734483a440fb..dfcec9376cd54 100644 --- a/fs/erofs/sysfs.c +++ b/fs/erofs/sysfs.c @@ -95,6 +95,7 @@ EROFS_ATTR_FEATURE(sb_chksum); EROFS_ATTR_FEATURE(ztailpacking); EROFS_ATTR_FEATURE(fragments); EROFS_ATTR_FEATURE(dedupe); +EROFS_ATTR_FEATURE(xattr_prefixes); EROFS_ATTR_FEATURE(48bit); EROFS_ATTR_FEATURE(metabox); @@ -108,6 +109,7 @@ static struct attribute *erofs_feat_attrs[] = { ATTR_LIST(ztailpacking), ATTR_LIST(fragments), ATTR_LIST(dedupe), + ATTR_LIST(xattr_prefixes), ATTR_LIST(48bit), ATTR_LIST(metabox), NULL, diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c index df7ea019526d7..57cfb75207824 100644 --- a/fs/erofs/xattr.c +++ b/fs/erofs/xattr.c @@ -620,8 +620,8 @@ int erofs_xattr_fill_inode_fingerprint(struct erofs_inode_fingerprint *fp, { struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); struct erofs_xattr_prefix_item *prefix; + int domainlen, valuelen, base_index; const char *infix; - int valuelen, base_index; if (!test_opt(&sbi->opt, INODE_SHARE)) return -EOPNOTSUPP; @@ -633,17 +633,18 @@ int erofs_xattr_fill_inode_fingerprint(struct erofs_inode_fingerprint *fp, valuelen = erofs_getxattr(inode, base_index, infix, NULL, 0); if (valuelen <= 0 || valuelen > (1 << sbi->blkszbits)) return -EFSCORRUPTED; - fp->size = valuelen + (domain_id ? strlen(domain_id) : 0); + domainlen = strlen(domain_id); + fp->size = domainlen + 1 + valuelen; fp->opaque = kmalloc(fp->size, GFP_KERNEL); if (!fp->opaque) return -ENOMEM; + memcpy(fp->opaque, domain_id, domainlen + 1); if (valuelen != erofs_getxattr(inode, base_index, infix, - fp->opaque, valuelen)) { + fp->opaque + domainlen + 1, valuelen)) { kfree(fp->opaque); fp->opaque = NULL; return -EFSCORRUPTED; } - memcpy(fp->opaque + valuelen, domain_id, fp->size - valuelen); return 0; } #endif diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 74520e9102596..f0759211b7234 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -128,7 +128,17 @@ struct z_erofs_pcluster_slab { #define _PCLP(n) { .maxpages = n } static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { - _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), + _PCLP(1), + _PCLP(4), +#if Z_EROFS_PCLUSTER_MAX_PAGES > 16 + _PCLP(16), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 64 + _PCLP(64), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 128 + _PCLP(128), +#endif _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES + 1) }; @@ -1254,7 +1264,7 @@ static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio) const struct z_erofs_decompressor *alg = z_erofs_decomp[pcl->algorithmformat]; bool try_free = true; - int i, j, jtop, err2, err = eio ? -EIO : 0; + int i, err2, err = eio ? -EIO : 0; struct page *page; bool overlapped; const char *reason; @@ -1343,7 +1353,6 @@ static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio) be->compressed_pages >= be->onstack_pages + Z_EROFS_ONSTACK_PAGES) kvfree(be->compressed_pages); - jtop = 0; z_erofs_fill_other_copies(be, err); for (i = 0; i < be->nr_pages; ++i) { page = be->decompressed_pages[i]; @@ -1351,22 +1360,11 @@ static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio) continue; DBG_BUGON(z_erofs_page_is_invalidated(page)); - if (!z_erofs_is_shortlived_page(page)) { + if (!z_erofs_is_shortlived_page(page)) erofs_onlinefolio_end(page_folio(page), err, true); - continue; - } - if (pcl->algorithmformat != Z_EROFS_COMPRESSION_LZ4) { + else erofs_pagepool_add(be->pagepool, page); - continue; - } - for (j = 0; j < jtop && be->decompressed_pages[j] != page; ++j) - ; - if (j >= jtop) /* this bounce page is newly detected */ - be->decompressed_pages[jtop++] = page; } - while (jtop) - erofs_pagepool_add(be->pagepool, - be->decompressed_pages[--jtop]); if (be->decompressed_pages != be->onstack_pages) kvfree(be->decompressed_pages); diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index 5811556a7b71f..b316a50efe383 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -483,14 +483,14 @@ static int z_erofs_map_blocks_fo(struct inode *inode, } if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { - if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) + if ((vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) && + !(map->m_flags & EROFS_MAP_META)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; - } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { - map->m_algorithmformat = vi->z_algorithmtype[1]; } else { - map->m_algorithmformat = vi->z_algorithmtype[0]; + map->m_algorithmformat = + vi->z_algofmt[m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2]; } if ((flags & EROFS_GET_BLOCKS_FIEMAP) || @@ -604,9 +604,14 @@ static int z_erofs_map_blocks_ext(struct inode *inode, if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) map->m_flags |= EROFS_MAP_PARTIAL_REF; map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; - if (fmt) - map->m_algorithmformat = fmt - 1; - else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) + if (fmt) { + map->m_algorithmformat = --fmt; + if (fmt >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", + fmt, map->m_la, vi->nid); + return -EOPNOTSUPP; + } + } else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else @@ -624,7 +629,7 @@ static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) struct super_block *const sb = inode->i_sb; struct z_erofs_map_header *h; erofs_off_t pos; - int err = 0; + int err = 0, nr; if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { /* @@ -667,12 +672,19 @@ static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) goto done; } - vi->z_algorithmtype[0] = h->h_algorithmtype & 15; - vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); else if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) vi->z_idata_size = le16_to_cpu(h->h_idata_size); + for (nr = 0; nr < 2; ++nr) { + vi->z_algofmt[nr] = (h->h_algorithmtype >> (4 * nr)) & 15; + if (vi->z_algofmt[nr] >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", + nr + 1, vi->z_algofmt[nr], vi->nid); + err = -EOPNOTSUPP; + goto out_unlock; + } + } if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | @@ -720,12 +732,8 @@ static int z_erofs_map_sanity_check(struct inode *inode, if (!(map->m_flags & EROFS_MAP_MAPPED)) return 0; - if (unlikely(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX)) { - erofs_err(inode->i_sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", - map->m_algorithmformat, map->m_la, EROFS_I(inode)->nid); - return -EOPNOTSUPP; - } + DBG_BUGON(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX); if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { if (!(sbi->available_compr_algs & BIT(map->m_algorithmformat))) { erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu", diff --git a/fs/erofs/zutil.c b/fs/erofs/zutil.c index a0ce9226bc029..6b9bfa2ef3b65 100644 --- a/fs/erofs/zutil.c +++ b/fs/erofs/zutil.c @@ -79,6 +79,8 @@ int z_erofs_gbuf_growsize(unsigned int nrpages) for (i = 0; i < z_erofs_gbuf_count; ++i) { gbuf = &z_erofs_gbufpool[i]; + if (gbuf->nrpages >= nrpages) + continue; tmp_pages = kzalloc_objs(*tmp_pages, nrpages); if (!tmp_pages) goto out; diff --git a/fs/exec.c b/fs/exec.c index c7b8f2d6366c4..834d622777d68 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -854,6 +855,7 @@ static int exec_mmap(struct linux_binprm *bprm) /* Notify parent that we're no longer interested in the old VM */ tsk = current; old_mm = current->mm; + /* Clean up futexes and release the mm */ exec_mm_release(tsk, old_mm); ret = down_write_killable(&tsk->signal->exec_update_lock); @@ -902,9 +904,10 @@ static int exec_mmap(struct linux_binprm *bprm) BUG_ON(active_mm != old_mm); /* Defer teardown to setup_new_exec(), outside the exec locks. */ bprm->old_mm = old_mm; - return 0; + } else { + mmdrop_lazy_tlb(active_mm); } - mmdrop_lazy_tlb(active_mm); + futex_exec_done(tsk); return 0; } @@ -1101,6 +1104,17 @@ void __set_task_comm(struct task_struct *tsk, const char *buf, bool exec) perf_event_comm(tsk, exec); } +static void posixtimer_exec(struct task_struct *me) +{ +#ifdef CONFIG_POSIX_TIMERS + spin_lock_irq(&me->sighand->siglock); + posix_cpu_timers_exit(me); + spin_unlock_irq(&me->sighand->siglock); + exit_itimers(me); + flush_itimer_signals(); +#endif +} + /* * Calling this is the point of no return. None of the failures will be * seen by userspace since either the process is already taking a fatal @@ -1134,6 +1148,16 @@ int begin_new_exec(struct linux_binprm * bprm) retval = de_thread(me); if (retval) goto out; + + /* + * This must be done here to ensure that POSIX CPU timers which were + * armed on the current task are dequeued from me::posix_cputimers. + * Otherwise in case of a TID switch the deletion of the related POSIX + * timer would not remove an enqueued timer because the TID lookup + * of the old TID fails. + */ + posixtimer_exec(me); + /* see the comment in check_unsafe_exec() */ current->fs->in_exec = 0; /* @@ -1146,6 +1170,20 @@ int begin_new_exec(struct linux_binprm * bprm) if (retval) goto out; + /* + * We have to apply CLOEXEC before we change whether the process is + * dumpable (in setup_new_exec) to avoid a race with a process in userspace + * trying to access the should-be-closed file descriptors of a process + * undergoing exec(2). + * + * This can block on filesystem ->flush() handlers, including waiting + * for FUSE daemons, so do it before exec_mmap takes the + * exec_update_lock. + * This must happen after the point of no return, and after unsharing + * the FD table. + */ + do_close_on_exec(me->files); + /* * Must be called _before_ exec_mmap() as bprm->mm is * not visible until then. Doing it here also ensures @@ -1174,14 +1212,6 @@ int begin_new_exec(struct linux_binprm * bprm) if (retval) goto out_unlock; -#ifdef CONFIG_POSIX_TIMERS - spin_lock_irq(&me->sighand->siglock); - posix_cpu_timers_exit(me); - spin_unlock_irq(&me->sighand->siglock); - exit_itimers(me); - flush_itimer_signals(); -#endif - /* * Make the signal table private. */ @@ -1196,14 +1226,6 @@ int begin_new_exec(struct linux_binprm * bprm) clear_syscall_work_syscall_user_dispatch(me); - /* - * We have to apply CLOEXEC before we change whether the process is - * dumpable (in setup_new_exec) to avoid a race with a process in userspace - * trying to access the should-be-closed file descriptors of a process - * undergoing exec(2). - */ - do_close_on_exec(me->files); - if (bprm->secureexec) { /* Make sure parent cannot signal privileged process. */ me->pdeath_signal = 0; diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h index 9be50949ce34f..1f020b041a3d6 100644 --- a/fs/exfat/exfat_fs.h +++ b/fs/exfat/exfat_fs.h @@ -294,7 +294,7 @@ struct exfat_inode_info { /* on-disk position of directory entry or 0 */ loff_t i_pos; loff_t valid_size; - /* page-aligned size that has been zeroed out for mmap */ + /* block-aligned size zeroed in the page cache (>= valid_size) */ loff_t zeroed_size; /* hash by i_location */ struct hlist_node i_hash_fat; diff --git a/fs/exfat/file.c b/fs/exfat/file.c index 5fc13378d35f7..5e9b47ecc614e 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "exfat_raw.h" #include "exfat_fs.h" @@ -654,6 +655,104 @@ int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) return blkdev_issue_flush(inode->i_sb->s_bdev); } +/* + * exfat_zero_new_range - zero [start, end) without overwriting uptodate blocks + * + * Uptodate blocks may contain data written through a shared mapping beyond + * valid_size. + */ +static int exfat_zero_new_range(struct inode *inode, loff_t start, loff_t end) +{ + struct address_space *mapping = inode->i_mapping; + unsigned int blocksize = i_blocksize(inode); + loff_t pos = start; + int err; + + while (pos < end) { + loff_t next = min_t(loff_t, + round_down(pos, PAGE_SIZE) + PAGE_SIZE, end); + struct folio *folio; + loff_t bpos; + + folio = filemap_get_folio(mapping, pos >> PAGE_SHIFT); + if (IS_ERR(folio)) { + err = iomap_zero_range(inode, pos, next - pos, NULL, + &exfat_iomap_ops, NULL, NULL); + if (err < 0) + return err; + pos = next; + continue; + } + + if (folio_test_uptodate(folio)) { + folio_lock(folio); + if (folio->mapping == mapping) + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + pos = next; + continue; + } + + /* + * Zero not-uptodate block runs. iomap_zero_range() requires an + * unlocked folio, so recheck ->mapping after each call. + */ + folio_lock(folio); + bpos = pos; + while (bpos < next) { + loff_t rstart, rend; + + if (folio->mapping != mapping) { + folio_unlock(folio); + err = iomap_zero_range(inode, bpos, next - bpos, + NULL, &exfat_iomap_ops, NULL, NULL); + if (err < 0) { + folio_put(folio); + return err; + } + folio_lock(folio); + break; + } + + if (iomap_is_partially_uptodate(folio, + offset_in_folio(folio, bpos), blocksize)) { + bpos += blocksize; + continue; + } + + rstart = bpos; + rend = min_t(loff_t, bpos + blocksize, next); + while (rend < next && + !iomap_is_partially_uptodate(folio, + offset_in_folio(folio, rend), blocksize)) + rend = min_t(loff_t, rend + blocksize, next); + + folio_unlock(folio); + err = iomap_zero_range(inode, rstart, rend - rstart, + NULL, &exfat_iomap_ops, NULL, NULL); + if (err < 0) { + folio_put(folio); + return err; + } + folio_lock(folio); + bpos = rend; + } + + /* + * Dirty only a fully uptodate folio. Dirtying a partial folio could + * write uninitialised cache contents over valid on-disk blocks. + */ + if (folio->mapping == mapping && folio_test_uptodate(folio)) + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + pos = next; + } + + return 0; +} + static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size) { struct exfat_inode_info *ei = EXFAT_I(inode); @@ -661,18 +760,41 @@ static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size) int ret = 0; if (old_valid_size < new_valid_size) { + /* Do not re-zero blocks already covered by zeroed_size. */ + loff_t gap_start = max(old_valid_size, ei->zeroed_size); + if (i_size_read(inode) < new_valid_size) { - i_size_write(inode, new_valid_size); - mark_inode_dirty(inode); + /* + * Allocate clusters before increasing i_size. The gap + * may already be zeroed, so the subsequent zeroing + * can be skipped. + */ + ret = exfat_cont_expand(inode, new_valid_size); + if (ret) + return ret; } - ret = iomap_zero_range(inode, old_valid_size, - new_valid_size - old_valid_size, NULL, - &exfat_write_iomap_ops, NULL, NULL); + /* + * Revoke writable PTEs while zeroing the gap. A racing mmap + * store re-faults through exfat_page_mkwrite() after valid_size + * is updated. + */ + filemap_invalidate_lock(inode->i_mapping); + if (gap_start < new_valid_size) + unmap_mapping_range(inode->i_mapping, gap_start, + new_valid_size - gap_start, 0); + ret = exfat_zero_new_range(inode, gap_start, new_valid_size); + filemap_invalidate_unlock(inode->i_mapping); if (ret) { truncate_setsize(inode, old_valid_size); exfat_truncate(inode); + return ret; } + + ei->valid_size = new_valid_size; + if (ei->zeroed_size < round_up(new_valid_size, i_blocksize(inode))) + ei->zeroed_size = round_up(new_valid_size, i_blocksize(inode)); + mark_inode_dirty(inode); } return ret; @@ -825,39 +947,39 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf) struct inode *inode = file_inode(vmf->vma->vm_file); struct exfat_inode_info *ei = EXFAT_I(inode); vm_fault_t ret; - loff_t new_valid_size, mmap_valid_size; + loff_t new_valid_size, mmap_valid_size, fault_page_start; if (!inode_trylock(inode)) return VM_FAULT_RETRY; mmap_valid_size = ((loff_t)vmf->pgoff + 1) << PAGE_SHIFT; + fault_page_start = ((loff_t)vmf->pgoff) << PAGE_SHIFT; new_valid_size = min(mmap_valid_size, i_size_read(inode)); if (ei->valid_size < new_valid_size) { - if (ei->zeroed_size < mmap_valid_size) { + if (ei->zeroed_size < fault_page_start) { int err; /* - * Only zero the range that hasn't been zeroed yet for - * this mmap write path. zeroed_size tracks the largest - * page-aligned offset that has already been zeroed. - * - * This prevents unnecessarily zeroing out the entire - * tail page on every page fault when userspace writes - * data byte-by-byte through mmap (after a small - * fallocate). It fixes data corruption in the tail page - * while preserving the existing valid_size semantics. + * Zero only the gap below the faulting page. The read + * fault populated its folio and iomap_page_mkwrite() + * will dirty it. */ - err = iomap_zero_range(inode, ei->zeroed_size, - mmap_valid_size - ei->zeroed_size, NULL, - &exfat_iomap_ops, NULL, NULL); + err = exfat_zero_new_range(inode, ei->zeroed_size, + fault_page_start); if (err < 0) { inode_unlock(inode); return vmf_fs_error(err); } - ei->zeroed_size = mmap_valid_size; } + /* + * Track zeroed_size by block, not page, because writeback stops + * at i_size recording blocks wholly beyond it could skip a + * later required zeroing. + */ + if (ei->zeroed_size < round_up(new_valid_size, i_blocksize(inode))) + ei->zeroed_size = round_up(new_valid_size, i_blocksize(inode)); ei->valid_size = new_valid_size; mark_inode_dirty(inode); } @@ -866,7 +988,7 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf) file_update_time(vmf->vma->vm_file); filemap_invalidate_lock_shared(inode->i_mapping); - ret = iomap_page_mkwrite(vmf, &exfat_write_iomap_ops, NULL); + ret = iomap_page_mkwrite(vmf, &exfat_iomap_ops, NULL); filemap_invalidate_unlock_shared(inode->i_mapping); sb_end_pagefault(inode->i_sb); inode_unlock(inode); @@ -876,7 +998,6 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf) static const struct vm_operations_struct exfat_file_vm_ops = { .fault = filemap_fault, - .map_pages = filemap_map_pages, .page_mkwrite = exfat_page_mkwrite, }; @@ -887,21 +1008,6 @@ static int exfat_file_mmap_prepare(struct vm_area_desc *desc) if (unlikely(exfat_forced_shutdown(file_inode(desc->file)->i_sb))) return -EIO; - if (vma_desc_test_all(desc, VMA_SHARED_BIT, VMA_MAYWRITE_BIT)) { - struct inode *inode = file_inode(file); - loff_t from, to; - int err; - - from = ((loff_t)desc->pgoff << PAGE_SHIFT); - to = min_t(loff_t, i_size_read(inode), - from + vma_desc_size(desc)); - if (EXFAT_I(inode)->valid_size < to) { - err = exfat_extend_valid_size(inode, to); - if (err) - return err; - } - } - file_accessed(file); desc->vm_ops = &exfat_file_vm_ops; return 0; diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c index 190fc6471f849..d4d3ed933a63d 100644 --- a/fs/exfat/iomap.c +++ b/fs/exfat/iomap.c @@ -175,11 +175,18 @@ static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, if (ei->valid_size < end) { ei->valid_size = end; - if (ei->zeroed_size < end) - ei->zeroed_size = end; dirtied = true; } + /* + * IOMAP_F_ZERO_TAIL zeroes the remainder of the last block. Track that + * block as zeroed so later valid_size extensions do not zero it again. + */ + if (iomap->flags & IOMAP_F_ZERO_TAIL) + end = round_up(end, i_blocksize(inode)); + if (ei->zeroed_size < end) + ei->zeroed_size = end; + if (dirtied || iomap->flags & IOMAP_F_SIZE_CHANGED) mark_inode_dirty(inode); diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 29808629cce56..269b1c9fba5f7 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -1258,12 +1258,9 @@ static int ext2_setsize(struct inode *inode, loff_t newsize) filemap_invalidate_unlock(inode->i_mapping); inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); - if (inode_needs_sync(inode)) { - mmb_sync(&EXT2_I(inode)->i_metadata_bhs); + mark_inode_dirty(inode); + if (inode_needs_sync(inode)) sync_inode_metadata(inode, 1); - } else { - mark_inode_dirty(inode); - } return 0; } diff --git a/fs/ext2/xattr.c b/fs/ext2/xattr.c index e55d16abf422f..be63f89402a38 100644 --- a/fs/ext2/xattr.c +++ b/fs/ext2/xattr.c @@ -777,6 +777,7 @@ ext2_xattr_set2(struct inode *inode, struct buffer_head *old_bh, /* Update the inode. */ EXT2_I(inode)->i_file_acl = new_bh ? new_bh->b_blocknr : 0; inode_set_ctime_current(inode); + mark_inode_dirty(inode); if (IS_SYNC(inode)) { error = sync_inode_metadata(inode, 1); /* In case sync failed due to ENOSPC the inode was actually @@ -789,8 +790,7 @@ ext2_xattr_set2(struct inode *inode, struct buffer_head *old_bh, } goto cleanup; } - } else - mark_inode_dirty(inode); + } error = 0; if (old_bh && old_bh != new_bh) { diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c index f41f320f4437b..3971986de0289 100644 --- a/fs/ext4/crypto.c +++ b/fs/ext4/crypto.c @@ -144,7 +144,13 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, if (inode->i_ino == EXT4_ROOT_INO) return -EPERM; - if (WARN_ON_ONCE(IS_DAX(inode) && i_size_read(inode))) + /* + * For new encrypted inodes, S_DAX is never set in the first place. + * + * For existing inodes, this is called only on empty directories. ext4 + * never sets S_DAX on directories. + */ + if (WARN_ON_ONCE(IS_DAX(inode))) return -EINVAL; if (ext4_test_inode_flag(inode, EXT4_INODE_DAX)) @@ -163,6 +169,14 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, */ if (handle) { + /* + * __ext4_new_inode() should have already set the encrypt flag + * on the inode and avoided enabling inline data. + */ + if (WARN_ON_ONCE(!IS_ENCRYPTED(inode))) + return -EINVAL; + if (WARN_ON_ONCE(ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))) + return -EINVAL; /* * Since the inode is new it is ok to pass the * XATTR_CREATE flag. This is necessary to match the @@ -170,21 +184,10 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, * function with the credits allocated for the new * inode. */ - res = ext4_xattr_set_handle(handle, inode, - EXT4_XATTR_INDEX_ENCRYPTION, - EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, - ctx, len, XATTR_CREATE); - if (!res) { - ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); - ext4_clear_inode_state(inode, - EXT4_STATE_MAY_INLINE_DATA); - /* - * Update inode->i_flags - S_ENCRYPTED will be enabled, - * S_DAX may be disabled - */ - ext4_set_inode_flags(inode, false); - } - return res; + return ext4_xattr_set_handle(handle, inode, + EXT4_XATTR_INDEX_ENCRYPTION, + EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, XATTR_CREATE); } res = dquot_initialize(inode); @@ -205,10 +208,7 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, ctx, len, 0); if (!res) { ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); - /* - * Update inode->i_flags - S_ENCRYPTED will be enabled, - * S_DAX may be disabled - */ + /* Update inode->i_flags to set S_ENCRYPTED. */ ext4_set_inode_flags(inode, false); res = ext4_mark_inode_dirty(handle, inode); if (res) diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c index 17edd678fa87b..8d7b81e6948e7 100644 --- a/fs/ext4/dir.c +++ b/fs/ext4/dir.c @@ -138,6 +138,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) struct buffer_head *bh = NULL; struct fscrypt_str fstr = FSTR_INIT(NULL, 0); struct dir_private_info *info = file->private_data; + bool has_csum = ext4_has_feature_metadata_csum(sb); err = fscrypt_prepare_readdir(inode); if (err) @@ -149,7 +150,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) return err; /* Can we just clear INDEX flag to ignore htree information? */ - if (!ext4_has_feature_metadata_csum(sb)) { + if (!has_csum) { /* * We don't set the inode dirty flag since it's not * critical that it gets flushed back to the disk. @@ -235,7 +236,10 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) * dirent right now. Scan from the start of the block * to make sure. */ if (!inode_eq_iversion(inode, info->cookie)) { - for (i = 0; i < sb->s_blocksize && i < offset; ) { + for (i = 0; + i <= sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode) && + i < offset;) { de = (struct ext4_dir_entry_2 *) (bh->b_data + i); /* It's too expensive to do a full @@ -257,6 +261,17 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) info->cookie = inode_query_iversion(inode); } + if (unlikely(offset < sb->s_blocksize && + offset > sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode))) { + EXT4_ERROR_FILE(file, bh->b_blocknr, + "bad entry in directory: %s - offset=%u, size=%lu", + "directory entry too close to block end", + offset, sb->s_blocksize); + ctx->pos = round_up(ctx->pos, sb->s_blocksize); + goto next_block; + } + while (ctx->pos < inode->i_size && offset < sb->s_blocksize) { de = (struct ext4_dir_entry_2 *) (bh->b_data + offset); @@ -312,6 +327,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) ctx->pos += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize); } +next_block: if ((ctx->pos < inode->i_size) && !dir_relax_shared(inode)) goto done; brelse(bh); diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index b37c136ea3ab3..ed4241797ae34 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -3138,13 +3138,15 @@ int do_journal_get_write_access(handle_t *handle, struct inode *inode, void ext4_set_inode_mapping_order(struct inode *inode); #define FALL_BACK_TO_NONDELALLOC 1 #define CONVERT_INLINE_DATA 2 +#define EXT4_WRITE_DATA_INLINE 4 typedef enum { EXT4_IGET_NORMAL = 0, EXT4_IGET_SPECIAL = 0x0001, /* OK to iget a system inode */ EXT4_IGET_HANDLE = 0x0002, /* Inode # is from a handle */ EXT4_IGET_BAD = 0x0004, /* Allow to iget a bad inode */ - EXT4_IGET_EA_INODE = 0x0008 /* Inode should contain an EA value */ + EXT4_IGET_EA_INODE = 0x0008, /* Inode should contain an EA value */ + EXT4_IGET_NOWAIT = 0x0010 /* Non-blocking lookup (skip if freeing) */ } ext4_iget_flags; extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, @@ -3184,7 +3186,7 @@ extern int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode); extern int ext4_chunk_trans_blocks(struct inode *, int nrblocks); extern int ext4_chunk_trans_extent(struct inode *inode, int nrblocks); extern int ext4_meta_trans_blocks(struct inode *inode, int lblocks, - int pextents); + int pextents, int alloc_extents); extern int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end); extern int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length, bool *did_zero); diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 91c97af64b317..44ab246a31762 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -4976,7 +4976,7 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, * it can tell if the extent in the cache is a split extent. * But for now let's assume pextents as 2 always. */ - credits = ext4_meta_trans_blocks(inode, max_blocks, 2); + credits = ext4_meta_trans_blocks(inode, max_blocks, 2, 0); } if (credits) { diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 8e2259799614e..fbb486d917b01 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -2196,8 +2196,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val) if (ret == 0) { /* Range is not mapped */ path = ext4_find_extent(inode, cur, path, 0); - if (IS_ERR(path)) + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; goto out; + } memset(&newex, 0, sizeof(newex)); newex.ee_block = cpu_to_le32(cur); ext4_ext_store_pblock( @@ -2209,8 +2212,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val) path = ext4_ext_insert_extent(NULL, inode, path, &newex, 0); up_write((&EXT4_I(inode)->i_data_sem)); - if (IS_ERR(path)) + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; goto out; + } goto next; } @@ -2257,10 +2263,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val) } ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); + ret = 0; out: ext4_free_ext_path(path); iput(inode); - return 0; + return ret; } /* Replay DEL_RANGE tag */ @@ -2320,9 +2327,10 @@ ext4_fc_replay_del_range(struct super_block *sb, u8 *val) ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); ext4_mark_inode_dirty(NULL, inode); + ret = 0; out: iput(inode); - return 0; + return ret; } static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb) diff --git a/fs/ext4/file.c b/fs/ext4/file.c index eb1a323962b10..130edf1ac2423 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -309,6 +309,13 @@ static ssize_t ext4_buffered_write_iter(struct kiocb *iocb, return -EOPNOTSUPP; inode_lock(inode); + + /* + * Prevent concurrent direct I/O and buffered I/O to the same file + * range. Wait for in-flight DIO to finish before dirtying pages. + */ + inode_dio_wait(inode); + ret = ext4_write_checks(iocb, from); if (ret <= 0) goto out; diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index a40cb27f8116c..a5831fc536db9 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -997,6 +997,8 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap, err = fscrypt_prepare_new_inode(dir, inode, &encrypt); if (err) goto out; + if (encrypt) + i_flags |= EXT4_ENCRYPT_FL; } err = dquot_initialize(inode); @@ -1306,6 +1308,8 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap, ei->i_extra_isize = sbi->s_want_extra_isize; ei->i_inline_off = 0; if (ext4_has_feature_inline_data(sb) && + /* Encrypted inodes cannot have inline data */ + !(ei->i_flags & EXT4_ENCRYPT_FL) && (!(ei->i_flags & (EXT4_DAX_FL|EXT4_EA_INODE_FL)) || S_ISDIR(mode))) ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); ret = inode; diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 8045e4ff270c7..7bb28735de911 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -812,7 +812,19 @@ int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, goto out; } ext4_write_lock_xattr(inode, &no_expand); - BUG_ON(!ext4_has_inline_data(inode)); + /* + * We could have raced with ext4_page_mkwrite() converting + * the inode and clearing the inline data flag, so we just + * release resources and retry the whole write. + */ + if (unlikely(!ext4_has_inline_data(inode))) { + ext4_write_unlock_xattr(inode, &no_expand); + brelse(iloc.bh); + folio_unlock(folio); + folio_put(folio); + ext4_journal_stop(handle); + return 0; + } /* * ei->i_inline_off may have changed since @@ -1454,6 +1466,8 @@ int ext4_read_inline_dir(struct file *file, /* for other entry, the real offset in * the buf has to be tuned accordingly. */ + if (i + ext4_dir_rec_len(1, NULL) > extra_size) + break; de = (struct ext4_dir_entry_2 *) (dir_buf + i - extra_offset); /* It's too expensive to do a full @@ -1488,10 +1502,17 @@ int ext4_read_inline_dir(struct file *file, continue; } + /* + * de lives at dir_buf + ctx->pos - extra_offset, within the + * kmalloc(inline_size) buffer. Make sure its header fits before + * ext4_check_dir_entry() dereferences de->rec_len. + */ + if (ctx->pos + ext4_dir_rec_len(1, NULL) > extra_size) + goto out; de = (struct ext4_dir_entry_2 *) (dir_buf + ctx->pos - extra_offset); if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf, - extra_size, ctx->pos)) + inline_size, ctx->pos)) goto out; if (le32_to_cpu(de->inode)) { if (!dir_emit(ctx, de->name, de->name_len, diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index ce99807c5f5b2..a33629b397b06 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1182,6 +1182,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, int nr_wait = 0; int i; bool should_journal_data = ext4_should_journal_data(inode); + bool folio_uptodate = folio_test_uptodate(folio); BUG_ON(!folio_test_locked(folio)); BUG_ON(to > folio_size(folio)); @@ -1193,13 +1194,13 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, head = create_empty_buffers(folio, blocksize, 0); block = EXT4_PG_TO_LBLK(inode, folio->index); - for (bh = head, block_start = 0; bh != head || !block_start; + for (bh = head, block_start = 0; + block_start < to || (!folio_uptodate && bh != head); block++, block_start = block_end, bh = bh->b_this_page) { block_end = block_start + blocksize; if (block_end <= from || block_start >= to) { - if (folio_test_uptodate(folio)) { + if (folio_uptodate) set_buffer_uptodate(bh); - } continue; } if (WARN_ON_ONCE(buffer_new(bh))) @@ -1220,7 +1221,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, if (should_journal_data) do_journal_get_write_access(handle, inode, bh); - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { /* * Unlike __block_write_begin() we leave * dirtying of new uptodate buffers to @@ -1237,7 +1238,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, continue; } } - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { set_buffer_uptodate(bh); continue; } @@ -1302,6 +1303,8 @@ static int ext4_write_begin(const struct kiocb *iocb, if (unlikely(ret)) return ret; + *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE); + trace_ext4_write_begin(inode, pos, len); /* * Reserve one block more for addition to orphan list in case @@ -1316,8 +1319,10 @@ static int ext4_write_begin(const struct kiocb *iocb, foliop); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } /* @@ -1450,8 +1455,7 @@ static int ext4_write_end(const struct kiocb *iocb, trace_ext4_write_end(inode, pos, len, copied); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -1560,8 +1564,7 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, BUG_ON(!ext4_handle_valid(handle)); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -2694,13 +2697,25 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) * page is already under writeback and we are not doing * a data integrity writeback, skip the page */ - if (!folio_test_dirty(folio) || - (folio_test_writeback(folio) && - (mpd->wbc->sync_mode == WB_SYNC_NONE)) || + if ((folio_test_writeback(folio) && + mpd->wbc->sync_mode == WB_SYNC_NONE) || unlikely(folio->mapping != mapping)) { folio_unlock(folio); continue; } + /* + * If the folio is clean, skip writing it back. + * Cycle the folio through the writeback state + * though, to clear stale xarray tags. + */ + if (!folio_test_dirty(folio)) { + if (!folio_test_writeback(folio)) { + __folio_start_writeback(folio, false); + folio_end_writeback(folio); + } + folio_unlock(folio); + continue; + } folio_wait_writeback(folio); BUG_ON(folio_test_writeback(folio)); @@ -2852,10 +2867,10 @@ static int ext4_do_writepages(struct mpage_da_data *mpd) if (ext4_should_dioread_nolock(inode)) { int bpf = ext4_journal_blocks_per_folio(inode); /* - * We may need to convert up to one extent per block in - * the folio and we may dirty the inode. + * We may need to convert up to one extent per block in the + * folio. */ - rsv_blocks = 1 + ext4_ext_index_trans_blocks(inode, bpf); + rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0); } if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) @@ -3161,8 +3176,10 @@ static int ext4_da_write_begin(const struct kiocb *iocb, foliop, fsdata, true); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } retry: @@ -3291,17 +3308,15 @@ static int ext4_da_write_end(const struct kiocb *iocb, struct folio *folio, void *fsdata) { struct inode *inode = mapping->host; - int write_mode = (int)(unsigned long)fsdata; + unsigned long write_mode = (unsigned long)fsdata; - if (write_mode == FALL_BACK_TO_NONDELALLOC) + if (write_mode & FALL_BACK_TO_NONDELALLOC) return ext4_write_end(iocb, mapping, pos, len, copied, folio, fsdata); trace_ext4_da_write_end(inode, pos, len, copied); - if (write_mode != CONVERT_INLINE_DATA && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) && - ext4_has_inline_data(inode)) + if (write_mode & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -3694,8 +3709,8 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, return ret; if (map->m_len < orig_mlen) { map->m_len = orig_mlen; - dio_credits = ext4_meta_trans_blocks(inode, orig_mlen, - map->m_len); + dio_credits = ext4_meta_trans_blocks(inode, map->m_len, + map->m_len, 0); } else { dio_credits = ext4_chunk_trans_blocks(inode, map->m_len); @@ -4219,6 +4234,14 @@ int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end) offset = from & (blocksize - 1); if (!offset || from >= end) return 0; + /* + * Inline data has no tail block to zero out. Note that a race with + * ext4_page_mkwrite() converting inline data to an extent without + * holding i_rwsem is safe, as that path zeroes the full block before + * copying in the inline data. + */ + if (ext4_has_inline_data(inode)) + return 0; /* If we are processing an encrypted inode during orphan list handling */ if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode)) return 0; @@ -5270,6 +5293,20 @@ void ext4_set_inode_mapping_order(struct inode *inode) mapping_set_folio_order_range(inode->i_mapping, min_order, max_order); } +static int ext4_iget_match(struct inode *inode, u64 ino, void *data) +{ + if (inode->i_ino != ino) + return 0; + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) { + spin_unlock(&inode->i_lock); + return -1; + } + __iget(inode); + spin_unlock(&inode->i_lock); + return 1; +} + struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, ext4_iget_flags flags, const char *function, unsigned int line) @@ -5298,9 +5335,24 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, return ERR_PTR(-EFSCORRUPTED); } - inode = iget_locked(sb, ino); - if (!inode) - return ERR_PTR(-ENOMEM); + if (flags & EXT4_IGET_NOWAIT) { + inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL); + if (!inode) + return ERR_PTR(-ENOENT); + + if (inode_state_read_once(inode) & I_NEW) + wait_on_new_inode(inode); + + if (unlikely(inode_unhashed(inode))) { + iput(inode); + return ERR_PTR(-ENOENT); + } + } else { + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + } + if (!(inode_state_read_once(inode) & I_NEW)) { ret = check_igot_inode(inode, flags, function, line); if (ret) { @@ -6307,17 +6359,17 @@ static int ext4_index_trans_blocks(struct inode *inode, int lblocks, } /* - * Account for index blocks, block groups bitmaps and block group - * descriptor blocks if modify datablocks and index blocks - * worse case, the indexs blocks spread over different block groups - * - * If datablocks are discontiguous, they are possible to spread over - * different block groups too. If they are contiguous, with flexbg, - * they could still across block group boundary. - * - * Also account for superblock, inode, quota and xattr blocks + * Calculate number of credits needed in a transaction to: + * * Allocate data blocks from @alloc_extents different groups - note that + * with flexbg a single physical extent can span multiple groups but + * single mballoc request only returns extent within one group. + * * Allocate metatadata (extent tree blocks, indirect blocks) to store + * pointers to @pextents data extents having @lblocks in total. + * * Modify extent tree / indirect block tree, inode, superblock, quota + * tracking, xattr blocks */ -int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) +int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents, + int alloc_extents) { ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); int gdpblocks; @@ -6334,7 +6386,7 @@ int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents) * Now let's see how many group bitmaps and group descriptors need * to account */ - groups = idxblocks + pextents; + groups = idxblocks + alloc_extents; gdpblocks = groups; if (groups > ngroups) groups = ngroups; @@ -6360,7 +6412,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) { int ret; - ret = ext4_meta_trans_blocks(inode, nrblocks, 1); + ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1); /* Account for data blocks for journalled mode */ if (ext4_should_journal_data(inode)) ret += nrblocks; @@ -6378,7 +6430,7 @@ int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) */ int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) { - return ext4_meta_trans_blocks(inode, nrblocks, 1); + return ext4_meta_trans_blocks(inode, nrblocks, 1, 1); } /* @@ -6510,6 +6562,16 @@ static int ext4_try_to_expand_extra_isize(struct inode *inode, if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) return -EOVERFLOW; + /* + * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize + * may move xattrs to external blocks and release ea_inodes via iput. + * When !SB_ACTIVE, iput triggers write_inode_now() which acquires + * s_writepages_rwsem, causing a deadlock with the caller's active + * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle). + */ + if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE))) + return -EBUSY; + /* * In nojournal mode, we can immediately attempt to expand * the inode. When journaled, we first need to obtain extra diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c index 477d43d7e2942..5d60ef10fe110 100644 --- a/fs/ext4/migrate.c +++ b/fs/ext4/migrate.c @@ -464,6 +464,7 @@ int ext4_ext_migrate(struct inode *inode) if (IS_ERR(tmp_inode)) { retval = PTR_ERR(tmp_inode); ext4_journal_stop(handle); + tmp_inode = NULL; goto out_unlock; } /* @@ -591,9 +592,9 @@ int ext4_ext_migrate(struct inode *inode) ext4_journal_stop(handle); out_tmp_inode: unlock_new_inode(tmp_inode); - iput(tmp_inode); out_unlock: ext4_writepages_up_write(inode->i_sb, alloc_ctx); + iput(tmp_inode); return retval; } diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index cc49ae04a6f64..3b9740c1c16d8 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -1467,6 +1467,8 @@ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size, /* this code is executed quadratically often */ /* do minimal checking `by hand' */ if (de->name + de->name_len <= dlimit && + (!ext4_hash_in_dirent(dir) || + (char *)de + ext4_dir_rec_len(de->name_len, dir) <= dlimit) && ext4_match(dir, fname, de)) { /* found a match - just to be sure, do * a full check */ diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c index 64ea476242338..e7fe1a99d4855 100644 --- a/fs/ext4/orphan.c +++ b/fs/ext4/orphan.c @@ -388,7 +388,7 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); - if (!es->s_last_orphan && !oi->of_blocks) { + if (!es->s_last_orphan && ext4_orphan_file_empty(sb)) { ext4_debug("no orphan inodes to clean up\n"); return; } @@ -572,6 +572,7 @@ int ext4_init_orphan_info(struct super_block *sb) int i, j; int ret; int free; + int loaded = 0; __le32 *bdata; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); struct ext4_orphan_block_tail *ot; @@ -613,6 +614,7 @@ int ext4_init_orphan_info(struct super_block *sb) ret = -EIO; goto out_free; } + loaded++; ot = ext4_orphan_block_tail(sb, oi->of_binfo[i].ob_bh); if (le32_to_cpu(ot->ob_magic) != EXT4_ORPHAN_BLOCK_MAGIC) { ext4_error(sb, "orphan file block %d: bad magic", i); @@ -635,8 +637,10 @@ int ext4_init_orphan_info(struct super_block *sb) iput(inode); return 0; out_free: - for (i--; i >= 0; i--) - brelse(oi->of_binfo[i].ob_bh); + while (loaded > 0) { + loaded--; + brelse(oi->of_binfo[loaded].ob_bh); + } kvfree(oi->of_binfo); out_put: iput(inode); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 245f67d10ded3..63de62a75a543 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -5369,7 +5369,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ext4_set_def_opts(sb, es); sbi->s_resuid = make_kuid(&init_user_ns, ext4_get_resuid(es)); - sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resuid(es)); + sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resgid(es)); sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ; sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME; sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME; diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 982a1f831e228..6fa41c48f3971 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -1550,7 +1550,7 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, while (ce) { ea_inode = ext4_iget(inode->i_sb, ce->e_value, - EXT4_IGET_EA_INODE); + EXT4_IGET_EA_INODE | EXT4_IGET_NOWAIT); if (IS_ERR(ea_inode)) goto next_entry; ext4_xattr_inode_set_class(ea_inode); @@ -2075,12 +2075,13 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, * stable so we can check the additional * reference fits. */ - ref = le32_to_cpu(BHDR(new_bh)->h_refcount) + 1; - if (ref > EXT4_XATTR_REFCOUNT_MAX) { + ref = le32_to_cpu(BHDR(new_bh)->h_refcount); + if (ref >= EXT4_XATTR_REFCOUNT_MAX) { /* * Undo everything and check mbcache * again. */ + clear_bit(MBE_REUSABLE_B, &ce->e_flags); unlock_buffer(new_bh); dquot_free_block(inode, EXT4_C2B(EXT4_SB(sb), @@ -2091,6 +2092,7 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, new_bh = NULL; goto inserted; } + ref++; BHDR(new_bh)->h_refcount = cpu_to_le32(ref); if (ref == EXT4_XATTR_REFCOUNT_MAX) clear_bit(MBE_REUSABLE_B, &ce->e_flags); @@ -2839,6 +2841,7 @@ int ext4_expand_extra_isize_ea(struct inode *inode, int new_extra_isize, s_min_extra_isize) { tried_min_extra_isize++; new_extra_isize = s_min_extra_isize; + error = 0; goto retry; } goto cleanup; diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index a765fda715363..631269c6f94d7 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1323,10 +1323,11 @@ struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index, if (folio_test_large(folio)) { pgoff_t folio_index = mapping_align_index(mapping, index); + unsigned long nr_pages = folio_nr_pages(folio); f2fs_folio_put(folio, true); invalidate_inode_pages2_range(mapping, folio_index, - folio_index + folio_nr_pages(folio) - 1); + folio_index + nr_pages - 1); f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); goto retry; } @@ -1532,8 +1533,11 @@ static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) old_blkaddr = dn->data_blkaddr; err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr, &sum, seg_type, NULL); - if (err) + if (err) { + if (old_blkaddr == NULL_ADDR) + dec_valid_block_count(sbi, dn->inode, count); return err; + } if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1); diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index af88db8fdb711..ff379aff44726 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -352,10 +352,6 @@ static void update_mem_info(struct f2fs_sb_info *sbi) get_cache: si->cache_mem = 0; - /* build gc */ - if (sbi->gc_thread) - si->cache_mem += sizeof(struct f2fs_gc_kthread); - /* build merge flush thread */ if (SM_I(sbi)->fcc_info) si->cache_mem += sizeof(struct flush_cmd_control); diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index a9563f7fcd884..fd0e2cd31a814 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -320,6 +320,7 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir, de = find_in_block(dir, dentry_folio, fname, &max_slots, use_hash); if (IS_ERR(de)) { + f2fs_folio_put(dentry_folio, false); *res_folio = ERR_CAST(de); de = NULL; break; @@ -460,7 +461,7 @@ void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, folio_mark_dirty(folio); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); f2fs_folio_put(folio, true); } @@ -615,7 +616,7 @@ void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode, clear_inode_flag(inode, FI_NEW_INODE); } inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); if (F2FS_I(dir)->i_current_depth != current_depth) f2fs_i_depth_write(dir, current_depth); @@ -927,7 +928,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio, f2fs_folio_put(folio, true); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); if (inode) f2fs_drop_nlink(dir, inode); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 8f3e632f315cc..b3c763a98cb8d 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1748,6 +1748,33 @@ struct decompress_io_ctx { #define MAX_COMPRESS_LOG_SIZE 8 #define MAX_COMPRESS_WINDOW_SIZE(log_size) ((PAGE_SIZE) << (log_size)) +struct f2fs_gc_kthread { + struct task_struct *f2fs_gc_task; + wait_queue_head_t gc_wait_queue_head; + + /* for gc sleep time */ + unsigned int urgent_sleep_time; + unsigned int min_sleep_time; + unsigned int max_sleep_time; + unsigned int no_gc_sleep_time; + + /* for changing gc mode */ + bool gc_wake; + + /* for GC_MERGE mount option */ + wait_queue_head_t fggc_wq; /* + * caller of f2fs_balance_fs() + * will wait on this wait queue. + */ + + /* for gc control for zoned devices */ + unsigned int no_zoned_gc_percent; + unsigned int boost_zoned_gc_percent; + unsigned int valid_thresh_ratio; + unsigned int boost_gc_multiple; + unsigned int boost_gc_greedy; +}; + struct f2fs_sb_info { struct super_block *sb; /* pointer to VFS super block */ struct proc_dir_entry *s_proc; /* proc entry */ @@ -1883,7 +1910,7 @@ struct f2fs_sb_info { * semaphore for GC, avoid * race between GC and GC or CP */ - struct f2fs_gc_kthread *gc_thread; /* GC thread */ + struct f2fs_gc_kthread gc_thread; /* GC thread */ struct atgc_management am; /* atgc management */ unsigned int cur_victim_sec; /* current victim section num */ unsigned int gc_mode; /* current GC state */ @@ -4096,10 +4123,26 @@ static inline struct inode *fio_inode(struct f2fs_io_info *fio) #define MIN_FRAGMENT_SIZE 1 #define MAX_FRAGMENT_SIZE 512 -static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi) +static inline bool f2fs_need_rand_blk(struct f2fs_sb_info *sbi, + enum log_type type) +{ + if (type == CURSEG_COLD_DATA_PINNED) + return false; + return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK; +} + +static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi, + enum log_type type) +{ + if (type == CURSEG_COLD_DATA_PINNED) + return false; + return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG; +} + +static inline bool f2fs_need_rand_seg_blk(struct f2fs_sb_info *sbi, + enum log_type type) { - return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG || - F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK; + return f2fs_need_rand_blk(sbi, type) || f2fs_need_rand_seg(sbi, type); } /* @@ -4228,7 +4271,7 @@ int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control); void f2fs_build_gc_manager(struct f2fs_sb_info *sbi); int f2fs_gc_range(struct f2fs_sb_info *sbi, unsigned int start_seg, unsigned int end_seg, - bool dry_run, unsigned int dry_run_sections); + bool dry_run, unsigned int dry_run_sections, bool lock); int f2fs_resize_fs(struct file *filp, __u64 block_count); int __init f2fs_create_garbage_collection_cache(void); void f2fs_destroy_garbage_collection_cache(void); diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 4b52c56d71f07..f5c169dced2dd 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -36,23 +36,69 @@ #include #include -static void f2fs_zero_post_eof_page(struct inode *inode, - loff_t new_size, bool lock) +static int fill_zero(struct inode *inode, pgoff_t index, + loff_t start, loff_t len); + +static int do_zero_post_eof_page(struct inode *inode, loff_t new_size) +{ + loff_t old_size = i_size_read(inode); + unsigned int offset, len; + pgoff_t index; + int err; + + offset = old_size & (PAGE_SIZE - 1); + + if (!offset) + return 0; + + len = min_t(loff_t, PAGE_SIZE - offset, new_size - old_size); + index = old_size >> PAGE_SHIFT; + + if (f2fs_has_inline_data(inode)) { + /* data post eof should be always zero */ + if (new_size <= MAX_INLINE_DATA(inode)) + return 0; + err = f2fs_convert_inline_inode(inode); + if (err) + return err; + } + + err = fill_zero(inode, index, offset, len); + if (err) + return err; + return filemap_write_and_wait_range(inode->i_mapping, + old_size, old_size + len - 1); +} + +static int f2fs_zero_post_eof_page(struct inode *inode, + loff_t new_size, bool lock, bool writeback) { loff_t old_size = i_size_read(inode); + bool strict = + F2FS_OPTION(F2FS_I_SB(inode)).fsync_mode == FSYNC_MODE_STRICT; if (old_size >= new_size) - return; + return 0; - if (mapping_empty(inode->i_mapping)) - return; + if (!strict && mapping_empty(inode->i_mapping)) + return 0; if (lock) filemap_invalidate_lock(inode->i_mapping); /* zero or drop pages only in range of [old_size, new_size] */ - truncate_inode_pages_range(inode->i_mapping, old_size, new_size); + truncate_inode_pages_range(inode->i_mapping, old_size, new_size - 1); if (lock) filemap_invalidate_unlock(inode->i_mapping); + + if (!writeback || !strict) + return 0; + /* + * In fsync_mode=strict, when we expand an unaligned EOF size, we + * should zero post EOF data and writeback the data immediately, + * so that it can avoid exposing stale data after metadata flush + * and POR. + */ + return do_zero_post_eof_page(inode, new_size); } static vm_fault_t f2fs_filemap_fault(struct vm_fault *vmf) @@ -132,7 +178,10 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) f2fs_bug_on(sbi, f2fs_has_inline_data(inode)); - f2fs_zero_post_eof_page(inode, (folio->index + 1) << PAGE_SHIFT, true); + err = f2fs_zero_post_eof_page(inode, + (folio->index + 1) << PAGE_SHIFT, true, false); + if (err) + goto out_pagefault; file_update_time(vmf->vma->vm_file); filemap_invalidate_lock_shared(inode->i_mapping); @@ -189,7 +238,7 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) out_sem: filemap_invalidate_unlock_shared(inode->i_mapping); - +out_pagefault: sb_end_pagefault(inode->i_sb); out: ret = vmf_fs_error(err); @@ -1107,17 +1156,23 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, !IS_ALIGNED(attr->ia_size, F2FS_BLK_TO_BYTES(fi->i_cluster_size))) return -EINVAL; - /* - * To prevent scattered pin block generation, we don't allow - * smaller/equal size unaligned truncation for pinned file. - * We only support overwrite IO to pinned file, so don't - * care about larger size truncation. - */ - if (f2fs_is_pinned_file(inode) && - attr->ia_size <= i_size_read(inode) && - !IS_ALIGNED(attr->ia_size, - F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi)))) - return -EINVAL; + + if (f2fs_is_pinned_file(inode)) { + /* + * It may break section-aligned fallocate recovery + * mechanism, so do not allow larger size truncation. + */ + if (attr->ia_size > i_size_read(inode)) + return -EINVAL; + /* + * To prevent scattered pin block generation, we don't + * allow smaller/equal size unaligned truncation for + * pinned file. + */ + else if (!IS_ALIGNED(attr->ia_size, + F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi)))) + return -EINVAL; + } } if (is_quota_modification(idmap, inode, attr)) { @@ -1169,8 +1224,12 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, f2fs_down_write(&fi->i_gc_rwsem[WRITE]); filemap_invalidate_lock(inode->i_mapping); - if (attr->ia_size > old_size) - f2fs_zero_post_eof_page(inode, attr->ia_size, false); + if (attr->ia_size > old_size) { + err = f2fs_zero_post_eof_page(inode, + attr->ia_size, false, true); + if (err) + goto err_out; + } truncate_setsize(inode, attr->ia_size); if (attr->ia_size <= old_size) @@ -1179,6 +1238,7 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, * do not trim all blocks after i_size if target size is * larger than i_size. */ +err_out: filemap_invalidate_unlock(inode->i_mapping); f2fs_up_write(&fi->i_gc_rwsem[WRITE]); if (err) @@ -1290,7 +1350,9 @@ static int f2fs_punch_hole(struct inode *inode, loff_t offset, loff_t len) if (ret) return ret; - f2fs_zero_post_eof_page(inode, offset + len, true); + ret = f2fs_zero_post_eof_page(inode, offset + len, true, false); + if (ret) + return ret; pg_start = ((unsigned long long) offset) >> PAGE_SHIFT; pg_end = ((unsigned long long) offset + len) >> PAGE_SHIFT; @@ -1577,7 +1639,9 @@ static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); filemap_invalidate_lock(inode->i_mapping); - f2fs_zero_post_eof_page(inode, offset + len, false); + ret = f2fs_zero_post_eof_page(inode, offset + len, false, false); + if (ret) + goto out_unlock; f2fs_lock_op(sbi, &lc); f2fs_drop_extent_tree(inode); @@ -1585,6 +1649,7 @@ static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) ret = __exchange_data_block(inode, inode, end, start, nrpages - end, true); f2fs_unlock_op(sbi, &lc); +out_unlock: filemap_invalidate_unlock(inode->i_mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); return ret; @@ -1617,11 +1682,14 @@ static int f2fs_collapse_range(struct inode *inode, loff_t offset, loff_t len) /* write out all moved pages, if possible */ filemap_invalidate_lock(inode->i_mapping); - filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + ret = filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + if (ret) + goto out_unlock; truncate_pagecache(inode, offset); new_size = i_size_read(inode) - len; ret = f2fs_truncate_blocks(inode, new_size, true); +out_unlock: filemap_invalidate_unlock(inode->i_mapping); if (!ret) f2fs_i_size_write(inode, new_size); @@ -1703,7 +1771,9 @@ static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, if (ret) return ret; - f2fs_zero_post_eof_page(inode, offset + len, true); + ret = f2fs_zero_post_eof_page(inode, offset + len, true, false); + if (ret) + return ret; pg_start = ((unsigned long long) offset) >> PAGE_SHIFT; pg_end = ((unsigned long long) offset + len) >> PAGE_SHIFT; @@ -1838,7 +1908,9 @@ static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); filemap_invalidate_lock(mapping); - f2fs_zero_post_eof_page(inode, offset + len, false); + ret = f2fs_zero_post_eof_page(inode, offset + len, false, false); + if (ret) + goto out_unlock; truncate_pagecache(inode, offset); while (!ret && idx > pg_start) { @@ -1856,6 +1928,7 @@ static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) idx + delta, nr, false); f2fs_unlock_op(sbi, &lc); } +out_unlock: filemap_invalidate_unlock(mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); if (ret) @@ -1898,7 +1971,9 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, if (err) return err; - f2fs_zero_post_eof_page(inode, offset + len, true); + err = f2fs_zero_post_eof_page(inode, offset + len, true, true); + if (err) + return err; f2fs_balance_fs(sbi, true); @@ -1919,8 +1994,9 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, block_t sec_len; if (map.m_lblk % sec_blks) { - map.m_lblk = rounddown(map.m_lblk, sec_blks); - map.m_len = pg_end - map.m_lblk; + pg_start = rounddown(map.m_lblk, sec_blks); + map.m_lblk = pg_start; + map.m_len = pg_end - pg_start; if (off_end) map.m_len++; } @@ -3083,6 +3159,9 @@ static int f2fs_ioc_defragment(struct file *filp, unsigned long arg) if (f2fs_readonly(sbi->sb)) return -EROFS; + if (IS_DEVICE_ALIASING(inode)) + return -EOPNOTSUPP; + if (copy_from_user(&range, (struct f2fs_defragment __user *)arg, sizeof(range))) return -EFAULT; @@ -3121,8 +3200,9 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, struct inode *dst = file_inode(file_out); struct f2fs_sb_info *sbi = F2FS_I_SB(src); struct f2fs_lock_context lc; - size_t olen = len, dst_max_i_size = 0; - size_t dst_osize; + size_t olen = len; + loff_t dst_max_i_size = 0; + loff_t dst_osize, dst_end; int ret; if (file_in->f_path.mnt != file_out->f_path.mnt || @@ -3135,7 +3215,8 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, if (!S_ISREG(src->i_mode) || !S_ISREG(dst->i_mode)) return -EINVAL; - if (IS_ENCRYPTED(src) || IS_ENCRYPTED(dst)) + if (IS_ENCRYPTED(src) || IS_ENCRYPTED(dst) || + IS_DEVICE_ALIASING(src) || IS_DEVICE_ALIASING(dst)) return -EOPNOTSUPP; if (pos_out < 0 || pos_in < 0) @@ -3144,8 +3225,6 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, if (src == dst) { if (pos_in == pos_out) return 0; - if (pos_out > pos_in && pos_out < pos_in + len) - return -EINVAL; } inode_lock(src); @@ -3171,6 +3250,8 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, goto out_unlock; if (len == 0) olen = len = src->i_size - pos_in; + if (src == dst && pos_out > pos_in && pos_out < pos_in + len) + goto out_unlock; if (pos_in + len == src->i_size) len = ALIGN(src->i_size, F2FS_BLKSIZE) - pos_in; if (len == 0) { @@ -3179,8 +3260,15 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, } dst_osize = dst->i_size; - if (pos_out + olen > dst->i_size) - dst_max_i_size = pos_out + olen; + if (olen > LLONG_MAX - pos_out) + goto out_unlock; + dst_end = pos_out + olen; + if (dst_end > dst->i_size) { + ret = inode_newsize_ok(dst, dst_end); + if (ret) + goto out_unlock; + dst_max_i_size = dst_end; + } /* verify the end result is block aligned */ if (!IS_ALIGNED(pos_in, F2FS_BLKSIZE) || @@ -4473,7 +4561,7 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len) page_idx = folio_next_index(folio); } while (page_len < len); - do { + while (redirty_idx < page_idx) { folio = filemap_lock_folio(mapping, redirty_idx); /* It will never fail, when folio has pinned above */ @@ -4486,7 +4574,7 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len) redirty_idx = folio_next_index(folio); folio_unlock(folio); folio_put_refs(folio, 2); - } while (redirty_idx < page_idx); + } return ret; } @@ -5002,8 +5090,10 @@ static ssize_t f2fs_write_checks(struct kiocb *iocb, struct iov_iter *from) if (err) return err; - f2fs_zero_post_eof_page(inode, - iocb->ki_pos + iov_iter_count(from), true); + err = f2fs_zero_post_eof_page(inode, + iocb->ki_pos + iov_iter_count(from), true, true); + if (err) + return err; return count; } @@ -5289,9 +5379,8 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) bool dio; bool may_need_sync = true; int preallocated; - const loff_t pos = iocb->ki_pos; - const ssize_t count = iov_iter_count(from); ssize_t ret; + loff_t bufio_start_pos; if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) { ret = -EIO; @@ -5312,15 +5401,17 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) inode_lock(inode); } + ret = f2fs_write_checks(iocb, from); + if (ret <= 0) + goto out_unlock; + if (f2fs_is_pinned_file(inode) && - !f2fs_overwrite_io(inode, pos, count)) { + !f2fs_overwrite_io(inode, iocb->ki_pos, iov_iter_count(from))) { ret = -EIO; goto out_unlock; } - ret = f2fs_write_checks(iocb, from); - if (ret <= 0) - goto out_unlock; + bufio_start_pos = iocb->ki_pos; /* Determine whether we will do a direct write or a buffered write. */ dio = f2fs_should_use_dio(inode, iocb, from); @@ -5375,8 +5466,8 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) */ if (ret > 0 && !dio && (iocb->ki_flags & IOCB_DIRECT)) f2fs_flush_buffered_write(iocb->ki_filp->f_mapping, - orig_pos, - orig_pos + ret - 1); + bufio_start_pos, + bufio_start_pos + ret - 1); return ret; } diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index ffaa7ba76a1bd..57ef3876d7afd 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -31,9 +31,9 @@ static unsigned int count_bits(const unsigned long *addr, static int gc_thread_func(void *data) { struct f2fs_sb_info *sbi = data; - struct f2fs_gc_kthread *gc_th = sbi->gc_thread; - wait_queue_head_t *wq = &sbi->gc_thread->gc_wait_queue_head; - wait_queue_head_t *fggc_wq = &sbi->gc_thread->fggc_wq; + struct f2fs_gc_kthread *gc_th = &sbi->gc_thread; + wait_queue_head_t *wq = &sbi->gc_thread.gc_wait_queue_head; + wait_queue_head_t *fggc_wq = &sbi->gc_thread.fggc_wq; unsigned int wait_ms; struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, @@ -193,13 +193,9 @@ static int gc_thread_func(void *data) int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) { - struct f2fs_gc_kthread *gc_th; + struct f2fs_gc_kthread *gc_th = &sbi->gc_thread; dev_t dev = sbi->sb->s_bdev->bd_dev; - gc_th = f2fs_kmalloc(sbi, sizeof(struct f2fs_gc_kthread), GFP_KERNEL); - if (!gc_th) - return -ENOMEM; - gc_th->urgent_sleep_time = DEF_GC_THREAD_URGENT_SLEEP_TIME; gc_th->valid_thresh_ratio = DEF_GC_THREAD_VALID_THRESH_RATIO; gc_th->boost_gc_multiple = BOOST_GC_MULTIPLE; @@ -221,16 +217,14 @@ int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) gc_th->gc_wake = false; - sbi->gc_thread = gc_th; - init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head); - init_waitqueue_head(&sbi->gc_thread->fggc_wq); - sbi->gc_thread->f2fs_gc_task = kthread_run(gc_thread_func, sbi, + init_waitqueue_head(&gc_th->gc_wait_queue_head); + init_waitqueue_head(&gc_th->fggc_wq); + gc_th->f2fs_gc_task = kthread_run(gc_thread_func, sbi, "f2fs_gc-%u:%u", MAJOR(dev), MINOR(dev)); if (IS_ERR(gc_th->f2fs_gc_task)) { int err = PTR_ERR(gc_th->f2fs_gc_task); - kfree(gc_th); - sbi->gc_thread = NULL; + gc_th->f2fs_gc_task = NULL; return err; } @@ -241,14 +235,14 @@ int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi) { - struct f2fs_gc_kthread *gc_th = sbi->gc_thread; + struct f2fs_gc_kthread *gc_th = &sbi->gc_thread; - if (!gc_th) + if (!gc_th->f2fs_gc_task) return; + kthread_stop(gc_th->f2fs_gc_task); + gc_th->f2fs_gc_task = NULL; wake_up_all(&gc_th->fggc_wq); - kfree(gc_th); - sbi->gc_thread = NULL; } static int select_gc_type(struct f2fs_sb_info *sbi, int gc_type) @@ -316,7 +310,7 @@ static void select_policy(struct f2fs_sb_info *sbi, int gc_type, p->max_search = sbi->max_victim_search; /* let's select beginning hot/small space first. */ - if (f2fs_need_rand_seg(sbi)) { + if (f2fs_need_rand_seg_blk(sbi, type)) { p->offset = get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); SIT_I(sbi)->last_victim[p->gc_mode] = p->offset; @@ -796,7 +790,7 @@ int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result, if (one_time) { p.one_time_gc = one_time; if (has_enough_free_secs(sbi, 0, NR_PERSISTENT_LOG)) - valid_thresh_ratio = sbi->gc_thread->valid_thresh_ratio; + valid_thresh_ratio = sbi->gc_thread.valid_thresh_ratio; } retry: @@ -1807,9 +1801,9 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, if (f2fs_sb_has_blkzoned(sbi) && !has_enough_free_blocks(sbi, - sbi->gc_thread->boost_zoned_gc_percent)) + sbi->gc_thread.boost_zoned_gc_percent)) window_granularity *= - sbi->gc_thread->boost_gc_multiple; + sbi->gc_thread.boost_gc_multiple; end_segno = start_segno + window_granularity; } @@ -2156,8 +2150,9 @@ void f2fs_build_gc_manager(struct f2fs_sb_info *sbi) int f2fs_gc_range(struct f2fs_sb_info *sbi, unsigned int start_seg, unsigned int end_seg, - bool dry_run, unsigned int dry_run_sections) + bool dry_run, unsigned int dry_run_sections, bool lock) { + struct f2fs_lock_context lc; unsigned int segno; unsigned int gc_secs = dry_run_sections; @@ -2170,28 +2165,46 @@ int f2fs_gc_range(struct f2fs_sb_info *sbi, .ilist = LIST_HEAD_INIT(gc_list.ilist), .iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS), }; + int err = 0; + + if (lock) + f2fs_down_write_trace(&sbi->gc_lock, &lc); /* * avoid migrating empty section, as it can be allocated by * log in parallel. */ if (!get_valid_blocks(sbi, segno, true)) - continue; + goto next; if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, segno))) - continue; + goto next; do_garbage_collect(sbi, segno, &gc_list, FG_GC, true, false); put_gc_inode(&gc_list); - if (!dry_run && get_valid_blocks(sbi, segno, true)) - return -EAGAIN; + /* reset all pinned status during fggc */ + f2fs_unpin_all_sections(sbi, true); + + if (!dry_run && get_valid_blocks(sbi, segno, true)) { + err = -EAGAIN; + goto next; + } if (dry_run && dry_run_sections && - !get_valid_blocks(sbi, segno, true) && --gc_secs == 0) - break; + !get_valid_blocks(sbi, segno, true)) { + --gc_secs; + goto next; + } if (fatal_signal_pending(current)) - return -ERESTARTSYS; + err = -ERESTARTSYS; +next: + if (lock) + f2fs_up_write_trace(&sbi->gc_lock, &lc); + if (err) + return err; + if (dry_run && dry_run_sections && !gc_secs) + return 0; } return 0; @@ -2200,8 +2213,9 @@ int f2fs_gc_range(struct f2fs_sb_info *sbi, static int free_segment_range(struct f2fs_sb_info *sbi, unsigned int secs, bool dry_run) { - unsigned int next_inuse, start, end; + unsigned int secno, next_inuse, start, end, end_secno; struct cp_control cpc = { CP_RESIZE, 0, 0, 0 }; + unsigned int freed_secs = 0; int gc_mode, gc_type; int err = 0; int type; @@ -2210,6 +2224,7 @@ static int free_segment_range(struct f2fs_sb_info *sbi, MAIN_SECS(sbi) -= secs; start = MAIN_SECS(sbi) * SEGS_PER_SEC(sbi); end = MAIN_SEGS(sbi) - 1; + end_secno = GET_SEC_FROM_SEG(sbi, end); mutex_lock(&DIRTY_I(sbi)->seglist_lock); for (gc_mode = 0; gc_mode < MAX_GC_POLICY; gc_mode++) @@ -2221,15 +2236,23 @@ static int free_segment_range(struct f2fs_sb_info *sbi, sbi->next_victim_seg[gc_type] = NULL_SEGNO; mutex_unlock(&DIRTY_I(sbi)->seglist_lock); + spin_lock(&FREE_I(sbi)->segmap_lock); + for (secno = MAIN_SECS(sbi); secno <= end_secno; secno++) { + if (!test_bit(secno, FREE_I(sbi)->free_secmap)) + freed_secs++; + } + FREE_I(sbi)->free_sections -= freed_secs; + spin_unlock(&FREE_I(sbi)->segmap_lock); + /* Move out cursegs from the target range */ - for (type = CURSEG_HOT_DATA; type < NR_CURSEG_PERSIST_TYPE; type++) { + for (type = CURSEG_HOT_DATA; type < NR_CURSEG_TYPE; type++) { err = f2fs_allocate_segment_for_resize(sbi, type, start, end); if (err) goto out; } /* do GC to move out valid blocks in the range */ - err = f2fs_gc_range(sbi, start, end, dry_run, 0); + err = f2fs_gc_range(sbi, start, end, dry_run, 0, false); if (err || dry_run) goto out; @@ -2245,6 +2268,9 @@ static int free_segment_range(struct f2fs_sb_info *sbi, f2fs_bug_on(sbi, 1); } out: + spin_lock(&FREE_I(sbi)->segmap_lock); + FREE_I(sbi)->free_sections += freed_secs; + spin_unlock(&FREE_I(sbi)->segmap_lock); MAIN_SECS(sbi) += secs; return err; } diff --git a/fs/f2fs/gc.h b/fs/f2fs/gc.h index 6c4d4567571eb..b015742fb455b 100644 --- a/fs/f2fs/gc.h +++ b/fs/f2fs/gc.h @@ -45,32 +45,7 @@ #define NR_GC_CHECKPOINT_SECS (3) /* data/node/dentry sections */ -struct f2fs_gc_kthread { - struct task_struct *f2fs_gc_task; - wait_queue_head_t gc_wait_queue_head; - - /* for gc sleep time */ - unsigned int urgent_sleep_time; - unsigned int min_sleep_time; - unsigned int max_sleep_time; - unsigned int no_gc_sleep_time; - - /* for changing gc mode */ - bool gc_wake; - - /* for GC_MERGE mount option */ - wait_queue_head_t fggc_wq; /* - * caller of f2fs_balance_fs() - * will wait on this wait queue. - */ - /* for gc control for zoned devices */ - unsigned int no_zoned_gc_percent; - unsigned int boost_zoned_gc_percent; - unsigned int valid_thresh_ratio; - unsigned int boost_gc_multiple; - unsigned int boost_gc_greedy; -}; struct gc_inode_list { struct list_head ilist; @@ -197,6 +172,6 @@ static inline bool need_to_boost_gc(struct f2fs_sb_info *sbi) { if (f2fs_sb_has_blkzoned(sbi)) return !has_enough_free_blocks(sbi, - sbi->gc_thread->boost_zoned_gc_percent); + sbi->gc_thread.boost_zoned_gc_percent); return has_enough_invalid_blocks(sbi); } diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index e2f7bedf15525..aec06fb4fd76b 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -732,7 +732,7 @@ void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, f2fs_folio_put(folio, true); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); if (inode) f2fs_drop_nlink(dir, inode); diff --git a/fs/f2fs/iostat.c b/fs/f2fs/iostat.c index ae265e3e9b2c9..12d4e18a6a50e 100644 --- a/fs/f2fs/iostat.c +++ b/fs/f2fs/iostat.c @@ -332,6 +332,12 @@ void f2fs_destroy_iostat_processing(void) int f2fs_init_iostat(struct f2fs_sb_info *sbi) { + /* + * The f2fs_iostat tracepoint emits a fixed number of read folio order + * buckets; make sure every order fits so none is silently dropped. + */ + BUILD_BUG_ON(NR_PAGE_ORDERS > F2FS_IOSTAT_RD_FOLIO_ORDERS); + /* init iostat info */ spin_lock_init(&sbi->iostat_lock); spin_lock_init(&sbi->iostat_lat_lock); diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index cac03b8e91a15..a8db3928c8b55 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -707,15 +707,16 @@ static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir, * performance regression. */ if (!err) { - filemap_write_and_wait_range(inode->i_mapping, 0, - disk_link.len - 1); + err = filemap_write_and_wait_range(inode->i_mapping, 0, + disk_link.len - 1); - if (IS_DIRSYNC(dir)) + if (!err && IS_DIRSYNC(dir)) f2fs_sync_fs(sbi->sb, 1); - } else { - f2fs_unlink(dir, dentry); } + if (err) + f2fs_unlink(dir, dentry); + f2fs_balance_fs(sbi, true); goto out_free_encrypted_link; @@ -1076,7 +1077,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, f2fs_up_write(&F2FS_I(old_inode)->i_sem); inode_set_ctime_current(old_inode); - f2fs_mark_inode_dirty_sync(old_inode, false); + f2fs_mark_inode_dirty_sync(old_inode, true); f2fs_delete_entry(old_entry, old_folio, old_dir, NULL); old_folio = NULL; @@ -1246,7 +1247,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_i_links_write(old_dir, old_nlink > 0); f2fs_up_write(&F2FS_I(old_dir)->i_sem); } - f2fs_mark_inode_dirty_sync(old_dir, false); + f2fs_mark_inode_dirty_sync(old_dir, true); /* update directory entry info of new dir inode */ f2fs_set_link(new_dir, new_entry, new_folio, old_inode); @@ -1265,7 +1266,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_i_links_write(new_dir, new_nlink > 0); f2fs_up_write(&F2FS_I(new_dir)->i_sem); } - f2fs_mark_inode_dirty_sync(new_dir, false); + f2fs_mark_inode_dirty_sync(new_dir, true); if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT) { f2fs_add_ino_entry(sbi, old_dir->i_ino, TRANS_DIR_INO); diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 38917e4a73194..1671535fbb10a 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -660,6 +660,7 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, __builtin_return_address(0), ni->ino, ni->nid, ni->blk_addr, ni->version, ni->flag); f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); + return -EFSCORRUPTED; } /* cache nat entry */ @@ -1617,7 +1618,7 @@ static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, if (!err) return folio; out_err: - folio_clear_uptodate(folio); + clear_node_folio_dirty(folio); out_put_err: /* ENOENT comes from read_node_folio which is not an error. */ if (err != -ENOENT) @@ -1789,7 +1790,7 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync, /* get old block addr of this node page */ nid = nid_of_node(folio); - if (f2fs_sanity_check_node_footer(sbi, folio, nid, + if (f2fs_sanity_check_node_footer(sbi, folio, folio->index, NODE_TYPE_REGULAR, false)) { fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_NID); @@ -2009,6 +2010,11 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, f2fs_debug(sbi, "Retry to write fsync mark: ino=%u, idx=%lx", ino, last_folio->index); folio_lock(last_folio); + if (unlikely(!is_node_folio(last_folio))) { + f2fs_folio_put(last_folio, true); + ret = -EAGAIN; + goto out; + } f2fs_folio_wait_writeback(last_folio, NODE, true, true); folio_mark_dirty(last_folio); folio_unlock(last_folio); diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index 89af8407b6673..3fecfdbd59586 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -158,6 +158,22 @@ static int init_recovered_filename(const struct inode *dir, return 0; } +static const char *recover_printable_name(struct inode *inode, + struct f2fs_inode *raw, + int *name_len) +{ + static const char encrypted_name[] = ""; + + if (file_enc_name(inode)) { + *name_len = sizeof(encrypted_name) - 1; + return encrypted_name; + } + + *name_len = min_t(unsigned int, le32_to_cpu(raw->i_namelen), + F2FS_NAME_LEN); + return raw->i_name; +} + static int recover_dentry(struct inode *inode, struct folio *ifolio, struct list_head *dir_list) { @@ -170,7 +186,8 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio, struct inode *dir, *einode; struct fsync_inode_entry *entry; int err = 0; - char *name; + const char *name; + int name_len; entry = get_fsync_inode(dir_list, pino); if (!entry) { @@ -229,12 +246,9 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio, out_put: f2fs_folio_put(folio, false); out: - if (file_enc_name(inode)) - name = ""; - else - name = raw_inode->i_name; - f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %s, dir = %llu, err = %d", - __func__, ino_of_node(ifolio), name, + name = recover_printable_name(inode, raw_inode, &name_len); + f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %.*s, dir = %llu, err = %d", + __func__, ino_of_node(ifolio), name_len, name, IS_ERR(dir) ? 0 : dir->i_ino, err); return err; } @@ -282,7 +296,8 @@ static int recover_inode(struct inode *inode, struct folio *folio) { struct f2fs_inode *raw = F2FS_INODE(folio); struct f2fs_inode_info *fi = F2FS_I(inode); - char *name; + const char *name; + int name_len; int err; inode->i_mode = le16_to_cpu(raw->i_mode); @@ -331,13 +346,11 @@ static int recover_inode(struct inode *inode, struct folio *folio) f2fs_mark_inode_dirty_sync(inode, true); - if (file_enc_name(inode)) - name = ""; - else - name = F2FS_INODE(folio)->i_name; + name = recover_printable_name(inode, raw, &name_len); - f2fs_notice(F2FS_I_SB(inode), "recover_inode: ino = %x, name = %s, inline = %x", - ino_of_node(folio), name, raw->i_inline); + f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %.*s, inline = %x", + __func__, ino_of_node(folio), name_len, name, + raw->i_inline); return 0; } diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index d71ddb3ee9184..011f8172e9234 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -452,15 +452,14 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) f2fs_submit_merged_write(sbi, DATA); f2fs_submit_all_merged_ipu_writes(sbi); - if (test_opt(sbi, GC_MERGE) && sbi->gc_thread && - sbi->gc_thread->f2fs_gc_task) { + if (test_opt(sbi, GC_MERGE) && sbi->gc_thread.f2fs_gc_task) { DEFINE_WAIT(wait); - prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait, + prepare_to_wait(&sbi->gc_thread.fggc_wq, &wait, TASK_UNINTERRUPTIBLE); - wake_up(&sbi->gc_thread->gc_wait_queue_head); + wake_up(&sbi->gc_thread.gc_wait_queue_head); io_schedule(); - finish_wait(&sbi->gc_thread->fggc_wq, &wait); + finish_wait(&sbi->gc_thread.fggc_wq, &wait); } else { struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, @@ -2937,7 +2936,7 @@ static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) sanity_check_seg_type(sbi, seg_type); if (__is_large_section(sbi)) { - if (f2fs_need_rand_seg(sbi)) { + if (f2fs_need_rand_seg_blk(sbi, type)) { unsigned int hint = GET_SEC_FROM_SEG(sbi, curseg->segno); if (GET_SEC_FROM_SEG(sbi, curseg->segno + 1) != hint) @@ -2946,7 +2945,7 @@ static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) GET_SEG_FROM_SEC(sbi, hint + 1) - 1); } return curseg->segno; - } else if (f2fs_need_rand_seg(sbi)) { + } else if (f2fs_need_rand_seg_blk(sbi, type)) { return get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); } @@ -3002,7 +3001,7 @@ static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) curseg->next_segno = segno; reset_curseg(sbi, type, 1); curseg->alloc_type = LFS; - if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + if (f2fs_need_rand_blk(sbi, type)) curseg->fragment_remained_chunk = get_random_u32_inclusive(1, sbi->max_fragment_chunk); return 0; @@ -3358,14 +3357,15 @@ int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi) f2fs_unlock_op(sbi, &lc); if (f2fs_sb_has_blkzoned(sbi) && err == -EAGAIN && gc_required) { - f2fs_down_write_trace(&sbi->gc_lock, &lc); err = f2fs_gc_range(sbi, 0, sbi->first_seq_zone_segno - 1, - true, ZONED_PIN_SEC_REQUIRED_COUNT); - f2fs_up_write_trace(&sbi->gc_lock, &lc); - - gc_required = false; - if (!err) + true, ZONED_PIN_SEC_REQUIRED_COUNT, true); + if (err) + return err; + err = f2fs_sync_fs(sbi->sb, 1); + if (!err) { + gc_required = false; goto retry; + } } return err; @@ -3820,7 +3820,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); } else { curseg->next_blkoff++; - if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + if (f2fs_need_rand_blk(sbi, type)) f2fs_randomize_chunk(sbi, curseg); } if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno)) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 9760e4efeffec..7f6c4e9d3735a 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2456,9 +2456,9 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) seq_puts(seq, "adaptive"); else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS) seq_puts(seq, "lfs"); - else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG) + else if (f2fs_need_rand_seg(sbi, NO_CHECK_TYPE)) seq_puts(seq, "fragment:segment"); - else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + else if (f2fs_need_rand_blk(sbi, NO_CHECK_TYPE)) seq_puts(seq, "fragment:block"); seq_printf(seq, ",active_logs=%u", F2FS_OPTION(sbi).active_logs); if (test_opt(sbi, RESERVE_ROOT) || test_opt(sbi, RESERVE_NODE)) @@ -2943,11 +2943,11 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) if ((flags & SB_RDONLY) || (F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_OFF && !test_opt(sbi, GC_MERGE))) { - if (sbi->gc_thread) { + if (sbi->gc_thread.f2fs_gc_task) { f2fs_stop_gc_thread(sbi); need_restart_gc = true; } - } else if (!sbi->gc_thread) { + } else if (!sbi->gc_thread.f2fs_gc_task) { err = f2fs_start_gc_thread(sbi); if (err) goto restore_opts; diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 665687244c939..aa1621419932a 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -75,7 +75,7 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, static unsigned char *__struct_ptr(struct f2fs_sb_info *sbi, int struct_type) { if (struct_type == GC_THREAD) - return (unsigned char *)sbi->gc_thread; + return (unsigned char *)&sbi->gc_thread; else if (struct_type == SM_INFO) return (unsigned char *)SM_I(sbi); else if (struct_type == DCC_INFO) @@ -557,7 +557,7 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return -EINVAL; cprc->ckpt_thread_ioprio = IOPRIO_PRIO_VALUE(class, level); - if (test_opt(sbi, MERGE_CHECKPOINT)) { + if (cprc->f2fs_issue_ckpt) { ret = set_task_ioprio(cprc->f2fs_issue_ckpt, cprc->ckpt_thread_ioprio); if (ret) @@ -664,20 +664,20 @@ static ssize_t __sbi_store(struct f2fs_attr *a, sbi->gc_mode = GC_NORMAL; } else if (t == 1) { sbi->gc_mode = GC_URGENT_HIGH; - if (sbi->gc_thread) { - sbi->gc_thread->gc_wake = true; + if (sbi->gc_thread.f2fs_gc_task) { + sbi->gc_thread.gc_wake = true; wake_up_interruptible_all( - &sbi->gc_thread->gc_wait_queue_head); + &sbi->gc_thread.gc_wait_queue_head); wake_up_discard_thread(sbi, true); } } else if (t == 2) { sbi->gc_mode = GC_URGENT_LOW; } else if (t == 3) { sbi->gc_mode = GC_URGENT_MID; - if (sbi->gc_thread) { - sbi->gc_thread->gc_wake = true; + if (sbi->gc_thread.f2fs_gc_task) { + sbi->gc_thread.gc_wake = true; wake_up_interruptible_all( - &sbi->gc_thread->gc_wait_queue_head); + &sbi->gc_thread.gc_wait_queue_head); } } else { return -EINVAL; @@ -934,14 +934,14 @@ static ssize_t __sbi_store(struct f2fs_attr *a, if (!strcmp(a->attr.name, "gc_boost_gc_multiple")) { if (t < 1 || t > SEGS_PER_SEC(sbi)) return -EINVAL; - sbi->gc_thread->boost_gc_multiple = (unsigned int)t; + sbi->gc_thread.boost_gc_multiple = (unsigned int)t; return count; } if (!strcmp(a->attr.name, "gc_boost_gc_greedy")) { if (t > GC_GREEDY) return -EINVAL; - sbi->gc_thread->boost_gc_greedy = (unsigned int)t; + sbi->gc_thread.boost_gc_greedy = (unsigned int)t; return count; } @@ -989,8 +989,8 @@ static ssize_t __sbi_store(struct f2fs_attr *a, if (sbi->cprc_info.f2fs_issue_ckpt) set_user_nice(sbi->cprc_info.f2fs_issue_ckpt, PRIO_TO_NICE(sbi->critical_task_priority)); - if (sbi->gc_thread && sbi->gc_thread->f2fs_gc_task) - set_user_nice(sbi->gc_thread->f2fs_gc_task, + if (sbi->gc_thread.f2fs_gc_task) + set_user_nice(sbi->gc_thread.f2fs_gc_task, PRIO_TO_NICE(sbi->critical_task_priority)); return count; } @@ -1007,13 +1007,15 @@ static ssize_t f2fs_sbi_store(struct f2fs_attr *a, ssize_t ret; bool gc_entry = (!strcmp(a->attr.name, "gc_urgent") || a->struct_type == GC_THREAD); + bool thread_entry = !strcmp(a->attr.name, "ckpt_thread_ioprio") || + !strcmp(a->attr.name, "critical_task_priority"); - if (gc_entry) { + if (gc_entry || thread_entry) { if (!down_read_trylock(&sbi->sb->s_umount)) return -EAGAIN; } ret = __sbi_store(a, sbi, buf, count); - if (gc_entry) + if (gc_entry || thread_entry) up_read(&sbi->sb->s_umount); return ret; diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index ed33e5110f2ab..6728d1488cadb 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -119,7 +119,7 @@ static int f2fs_xattr_advise_set(const struct xattr_handler *handler, unsigned char old_advise = F2FS_I(inode)->i_advise; unsigned char new_advise; - if (!inode_owner_or_capable(&nop_mnt_idmap, inode)) + if (!inode_owner_or_capable(idmap, inode)) return -EPERM; if (value == NULL) return -EINVAL; diff --git a/fs/fat/misc.c b/fs/fat/misc.c index 3027ef53af214..2a0fea26a99a8 100644 --- a/fs/fat/misc.c +++ b/fs/fat/misc.c @@ -133,7 +133,11 @@ int fat_chain_add(struct inode *inode, int new_dclus, int nr_cluster) ret = fat_ent_read(inode, &fatent, last); if (ret >= 0) { int wait = inode_needs_sync(inode); + int old = ret; + ret = fat_ent_write(inode, &fatent, new_dclus, wait); + if (ret < 0) + fat_ent_write(inode, &fatent, old, wait); fatent_brelse(&fatent); } if (ret < 0) diff --git a/fs/fat/nfs.c b/fs/fat/nfs.c index 509eea96a457d..6e1b371711edd 100644 --- a/fs/fat/nfs.c +++ b/fs/fat/nfs.c @@ -250,8 +250,10 @@ struct inode *fat_rebuild_parent(struct super_block *sb, int parent_logstart) MSDOS_I(dummy_grand_parent)->i_pos = -1; } - if (!fat_scan_logstart(dummy_grand_parent, clus_to_match, &sinfo)) + if (!fat_scan_logstart(dummy_grand_parent, clus_to_match, &sinfo)) { parent = fat_build_inode(sb, sinfo.de, sinfo.i_pos); + brelse(sinfo.bh); + } brelse(parent_bh); iput(dummy_grand_parent); diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index fdb8766d275a1..c0363c3bdb0e9 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -726,19 +726,34 @@ static bool isw_prepare_wbs_switch(struct bdi_writeback *new_wb, struct inode_switch_wbs_context *isw, struct list_head *list, int *nr) { - struct inode *inode; + struct inode *inode, *tmp; + LIST_HEAD(scanned); + bool full = false; + + /* + * Walk from the oldest end and move scanned inodes to the newest + * end, so the next scan resumes at unscanned inodes instead of + * re-walking an ever-growing run of prepared and skipped ones. + * For b_dirty_time this keeps the oldest unscanned inode at the + * end move_expired_inodes() picks from; b_attached is unordered. + */ + list_for_each_entry_safe_reverse(inode, tmp, list, i_io_list) { + list_move(&inode->i_io_list, &scanned); - list_for_each_entry(inode, list, i_io_list) { if (!inode_prepare_wbs_switch(inode, new_wb)) continue; isw->inodes[*nr] = inode; (*nr)++; - if (*nr >= WB_MAX_INODES_PER_ISW - 1) - return true; + if (*nr >= WB_MAX_INODES_PER_ISW - 1) { + full = true; + break; + } } - return false; + list_splice(&scanned, list); + + return full; } /** diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 3c15b5ba16d71..4079cf8e5974a 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -530,7 +530,8 @@ static int cuse_channel_open(struct inode *inode, struct file *file) INIT_LIST_HEAD(&cc->list); - cc->fc.chan->initialized = 1; + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&cc->fc.chan->initialized, 1); rc = cuse_send_init(cc); if (rc) { fuse_dev_put(fud); @@ -653,6 +654,11 @@ static void __exit cuse_exit(void) { misc_deregister(&cuse_miscdev); class_destroy(cuse_class); + /* + * Wait for pending call_rcu() callbacks that call back into + * this module via fc->release (cuse_fc_release). + */ + rcu_barrier(); } module_init(cuse_init); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 5763a7cd3b37f..a82cf763f50db 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -75,17 +75,23 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa fch->minor = param->minor; fch->max_write = param->max_write; fch->max_pages = param->max_pages; + + if (param->io_uring_enabled) + fuse_uring_conn_init(fch); } - /* Make sure stores before this are seen on another CPU */ - smp_wmb(); - fch->initialized = 1; + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&fch->initialized, 1); wake_up_all(&fch->blocked_waitq); } static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) { - return !fch->initialized || (for_background && fch->blocked) || + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized)) + return true; + + return (for_background && fch->blocked) || (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); } @@ -120,9 +126,6 @@ static struct fuse_req *fuse_get_req(struct fuse_chan *fch, bool for_background) goto out; } - /* Matches smp_wmb() in fuse_chan_set_initialized() */ - smp_rmb(); - err = -ENOTCONN; if (!fch->connected) goto out; @@ -240,7 +243,8 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq, void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) { spin_lock(&fiq->lock); - if (list_empty(&req->intr_entry)) { + /* Repeat FR_SENT test after obtaining the lock to prevent race with fuse_resend() */ + if (list_empty(&req->intr_entry) && test_bit(FR_SENT, &req->flags)) { list_add_tail(&req->intr_entry, &fiq->interrupts); /* * Pairs with smp_mb() implied by test_and_set_bit() @@ -411,11 +415,6 @@ void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc) fch->conn = fc; } -void fuse_chan_io_uring_enable(struct fuse_chan *fch) -{ - fch->io_uring = 1; -} - void fuse_pqueue_init(struct fuse_pqueue *fpq) { spin_lock_init(&fpq->lock); @@ -725,7 +724,7 @@ static void request_wait_answer(struct fuse_req *req) if (req->args->abort_on_kill) { fuse_chan_abort(fch, false); - return; + goto wait_for_finish; } if (test_bit(FR_URING, &req->flags)) @@ -736,6 +735,7 @@ static void request_wait_answer(struct fuse_req *req) return; } +wait_for_finish: /* * Either request is already in userspace, or it was forced. * Wait it out. @@ -1760,7 +1760,7 @@ static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos, void fuse_chan_resend(struct fuse_chan *fch) { struct fuse_dev *fud; - struct fuse_req *req, *next; + struct fuse_req *req; struct fuse_iqueue *fiq = &fch->iq; LIST_HEAD(to_queue); unsigned int i; @@ -1775,24 +1775,20 @@ void fuse_chan_resend(struct fuse_chan *fch) struct fuse_pqueue *fpq = &fud->pq; spin_lock(&fpq->lock); - for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) - list_splice_tail_init(&fpq->processing[i], &to_queue); + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) { + struct list_head *this_queue = &fpq->processing[i]; + + list_for_each_entry(req, this_queue, list) + clear_bit(FR_SENT, &req->flags); + list_splice_tail_init(this_queue, &to_queue); + } spin_unlock(&fpq->lock); } spin_unlock(&fch->lock); - list_for_each_entry_safe(req, next, &to_queue, list) { - set_bit(FR_PENDING, &req->flags); - clear_bit(FR_SENT, &req->flags); - /* mark the request as resend request */ - req->in.h.unique |= FUSE_UNIQUE_RESEND; - } - spin_lock(&fiq->lock); if (!fiq->connected) { spin_unlock(&fiq->lock); - list_for_each_entry(req, &to_queue, list) - clear_bit(FR_PENDING, &req->flags); fuse_dev_end_requests(&to_queue); return; } @@ -1801,6 +1797,10 @@ void fuse_chan_resend(struct fuse_chan *fch) * intr_entry on fiq->interrupts after the request is re-queued. */ list_for_each_entry(req, &to_queue, list) { + set_bit(FR_PENDING, &req->flags); + /* mark the request as resend request */ + req->in.h.unique |= FUSE_UNIQUE_RESEND; + if (test_bit(FR_INTERRUPTED, &req->flags)) list_del_init(&req->intr_entry); } @@ -1888,7 +1888,8 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, * initialized and connected state */ err = -EINVAL; - if (!fch->initialized || !fch->connected) + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized) || !fch->connected) goto copy_finish; /* Don't try to move folios (yet) */ diff --git a/fs/fuse/dev.h b/fs/fuse/dev.h index aed69fd14c41d..8d25378c09183 100644 --- a/fs/fuse/dev.h +++ b/fs/fuse/dev.h @@ -22,6 +22,7 @@ struct fuse_chan_param { unsigned int minor; unsigned int max_write; unsigned int max_pages; + bool io_uring_enabled; }; struct fuse_chan *fuse_chan_new(void); @@ -34,7 +35,6 @@ void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val); unsigned int fuse_chan_num_waiting(struct fuse_chan *fch); void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc); void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param); -void fuse_chan_io_uring_enable(struct fuse_chan *fch); ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args); int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags); int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 77c8cec43d9c6..8448fdbd90c9a 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -204,7 +204,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) return; for (qid = 0; qid < ring->nr_queues; qid++) { - struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); struct fuse_ring_ent *ent, *next; if (!queue) @@ -223,7 +223,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) kfree(queue->fpq.processing); kfree(queue); - ring->queues[qid] = NULL; + WRITE_ONCE(ring->queues[qid], NULL); } kfree(ring->queues); @@ -238,7 +238,6 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) { struct fuse_ring *ring; size_t nr_queues = num_possible_cpus(); - struct fuse_ring *res = NULL; size_t max_payload_size; ring = kzalloc_obj(*ring, GFP_KERNEL_ACCOUNT); @@ -258,12 +257,6 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) spin_unlock(&fch->lock); goto out_err; } - if (fch->ring) { - /* race, another thread created the ring in the meantime */ - spin_unlock(&fch->lock); - res = fch->ring; - goto out_err; - } init_waitqueue_head(&ring->stop_waitq); @@ -278,7 +271,13 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) out_err: kfree(ring->queues); kfree(ring); - return res; + return NULL; +} + +void fuse_uring_conn_init(struct fuse_chan *fch) +{ + if (fuse_uring_create(fch)) + fch->io_uring = 1; } static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, @@ -321,9 +320,11 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, } /* - * write_once and lock as the caller mostly doesn't take the lock at all + * fch->lock serializes concurrent creators for this qid. + * smp_store_release() are for the lockless readers who must see a + * fully initialized queue after &ring->queues[qid] is set */ - WRITE_ONCE(ring->queues[qid], queue); + smp_store_release(&ring->queues[qid], queue); spin_unlock(&fch->lock); return queue; @@ -434,7 +435,7 @@ static void fuse_uring_log_ent_state(struct fuse_ring *ring) struct fuse_ring_ent *ent; for (qid = 0; qid < ring->nr_queues; qid++) { - struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); if (!queue) continue; @@ -744,6 +745,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; + struct fuse_in_header in_header; int err; err = -EIO; @@ -765,8 +767,9 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, } /* copy fuse_in_header */ - return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->in.h, - sizeof(req->in.h)); + in_header = req->in.h; + return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &in_header, + sizeof(in_header)); } static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, @@ -871,11 +874,13 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, unsigned int issue_flags) { struct fuse_ring *ring = ent->queue->ring; + struct fuse_out_header out_header; ssize_t err = -EFAULT; - if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h, - sizeof(req->out.h))) + if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &out_header, + sizeof(out_header))) goto out; + req->out.h = out_header; err = fuse_uring_out_header_has_err(&req->out.h, req); if (err) { @@ -967,7 +972,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (qid >= ring->nr_queues) return -EINVAL; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) return err; fpq = &queue->fpq; @@ -1035,7 +1040,7 @@ static bool is_ring_ready(struct fuse_ring *ring, int current_qid) if (current_qid == qid) continue; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) { ready = false; break; @@ -1176,26 +1181,21 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, struct fuse_ring *ring = smp_load_acquire(&fch->ring); struct fuse_ring_queue *queue; struct fuse_ring_ent *ent; - int err; unsigned int qid = READ_ONCE(cmd_req->qid); - err = -ENOMEM; - if (!ring) { - ring = fuse_uring_create(fch); - if (!ring) - return err; - } + if (!ring) + return -EINVAL; if (qid >= ring->nr_queues) { pr_info_ratelimited("fuse: Invalid ring qid %u\n", qid); return -EINVAL; } - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) { queue = fuse_uring_create_queue(ring, qid); if (!queue) - return err; + return -ENOMEM; } /* @@ -1237,23 +1237,30 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) } fch = fud->chan; - /* Once a connection has io-uring enabled on it, it can't be disabled */ - if (!enable_uring && !fch->io_uring) { - pr_info_ratelimited("fuse-io-uring is disabled\n"); - return -EOPNOTSUPP; - } + /* + * The ring is sized from values negotiated by FUSE_INIT + * + * Pairs with smp_store_release() in fuse_chan_set_initialized() + */ + if (!smp_load_acquire(&fch->initialized)) + return -EAGAIN; if (fch->abort_with_err) return -ECONNABORTED; if (!fch->connected) return -ENOTCONN; - /* - * fuse_uring_register() needs the ring to be initialized, - * we need to know the max payload size - */ - if (!fch->initialized) - return -EAGAIN; + /* Once a connection has io-uring enabled on it, it can't be disabled */ + if (!enable_uring && !fch->io_uring) { + pr_info_ratelimited("fuse-io-uring is disabled by module parameter\n"); + return -EOPNOTSUPP; + } + + if (!fch->io_uring) { + pr_info_ratelimited( + "fuse-io-uring not enabled on this connection\n"); + return -EOPNOTSUPP; + } switch (cmd_op) { case FUSE_IO_URING_CMD_REGISTER: @@ -1330,7 +1337,7 @@ static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) ring->nr_queues)) qid = 0; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); WARN_ONCE(!queue, "Missing queue for qid %d\n", qid); return queue; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 55f8d04e4b0b2..d721a4fc0215c 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -135,6 +135,7 @@ struct fuse_ring { bool ready; }; +void fuse_uring_conn_init(struct fuse_chan *fch); void fuse_uring_stop_queues(struct fuse_ring *ring); void fuse_uring_abort_end_requests(struct fuse_ring *ring); int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); @@ -174,6 +175,10 @@ static inline bool fuse_uring_ready(struct fuse_chan *fch) #else /* CONFIG_FUSE_IO_URING */ +static inline void fuse_uring_conn_init(struct fuse_chan *fch) +{ +} + static inline void fuse_uring_abort(struct fuse_chan *fch) { } diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 0e2a1039fa43e..4466db3e38274 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -96,6 +96,7 @@ static void fuse_advise_use_readdirplus(struct inode *dir) struct fuse_dentry { u64 time; + u64 epoch; union { struct rcu_head rcu; struct rb_node node; @@ -236,6 +237,13 @@ void fuse_dentry_tree_cleanup(void) WARN_ON_ONCE(!RB_EMPTY_ROOT(&dentry_hash[i].tree)); } +void fuse_dentry_set_epoch(struct dentry *dentry, u64 epoch) +{ + struct fuse_dentry *fd = dentry->d_fsdata; + + fd->epoch = epoch; +} + static inline void __fuse_dentry_settime(struct dentry *dentry, u64 time) { ((struct fuse_dentry *) dentry->d_fsdata)->time = time; @@ -387,10 +395,11 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, struct fuse_mount *fm; struct fuse_conn *fc; struct fuse_inode *fi; + struct fuse_dentry *fd = entry->d_fsdata; int ret; fc = get_fuse_conn_super(dir->i_sb); - if (entry->d_time < atomic_read(&fc->epoch)) + if (fd->epoch < atomic_read(&fc->epoch)) goto invalid; inode = d_inode_rcu(entry); @@ -480,10 +489,10 @@ static int fuse_dentry_init(struct dentry *dentry) RB_CLEAR_NODE(&fd->node); dentry->d_fsdata = fd; /* - * Initialising d_time (epoch) to '0' ensures the dentry is invalid + * Initialising epoch to '0' ensures the dentry is invalid * if compared to fc->epoch, which is initialized to '1'. */ - dentry->d_time = 0; + fuse_dentry_set_epoch(dentry, 0); return 0; } @@ -641,7 +650,7 @@ static struct dentry *fuse_lookup(struct inode *dir, struct dentry *entry, goto out_err; entry = newent ? newent : entry; - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); if (outarg_valid) fuse_change_entry_timeout(entry, &outarg); else @@ -898,7 +907,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, } kfree(forget); d_instantiate(entry, inode); - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); fuse_change_entry_timeout(entry, &outentry); fuse_dir_changed(dir); err = generic_file_open(inode, file); @@ -1028,10 +1037,10 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun return d; if (d) { - d->d_time = epoch; + fuse_dentry_set_epoch(d, epoch); fuse_change_entry_timeout(d, &outarg); } else { - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); fuse_change_entry_timeout(entry, &outarg); } fuse_dir_changed(dir); @@ -2161,10 +2170,8 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, filemap_invalidate_lock(mapping); fault_blocked = true; err = fuse_dax_break_layouts(inode, 0, -1); - if (err) { - filemap_invalidate_unlock(mapping); - return err; - } + if (err) + goto unlock; } if (attr->ia_valid & ATTR_OPEN) { @@ -2191,7 +2198,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, ATTR_TIMES_SET)) { err = write_inode_now(inode, true); if (err) - return err; + goto unlock; fuse_set_nowrite(inode); fuse_release_nowrite(inode); @@ -2299,6 +2306,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); +unlock: if (fault_blocked) filemap_invalidate_unlock(mapping); return err; diff --git a/fs/fuse/file.c b/fs/fuse/file.c index ceada75310b88..5820def81d9f5 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -272,7 +272,7 @@ static int fuse_open(struct inode *inode, struct file *file) filemap_invalidate_lock(inode->i_mapping); err = fuse_dax_break_layouts(inode, 0, -1); if (err) - goto out_inode_unlock; + goto out_unlock; } if (is_wb_truncate || dax_truncate) @@ -296,9 +296,9 @@ static int fuse_open(struct inode *inode, struct file *file) else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) invalidate_inode_pages2(inode->i_mapping); } +out_unlock: if (dax_truncate) filemap_invalidate_unlock(inode->i_mapping); -out_inode_unlock: if (is_wb_truncate || dax_truncate) inode_unlock(inode); @@ -1801,13 +1801,14 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = file_inode(iocb->ki_filp); struct address_space *mapping = inode->i_mapping; - loff_t pos = iocb->ki_pos; ssize_t res; bool exclusive; fuse_dio_lock(iocb, from, &exclusive); res = generic_write_checks(iocb, from); if (res > 0) { + loff_t pos = iocb->ki_pos; + task_io_account_write(res); if (!is_sync_kiocb(iocb)) { res = fuse_direct_IO(iocb, from); @@ -1822,7 +1823,7 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) /* * As in generic_file_direct_write(), invalidate after * write, to invalidate read-ahead cache that may have - * with the write. + * competed with the write. */ invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 85f738c531225..c8d4c5f3af7e8 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1054,6 +1054,8 @@ u64 fuse_time_to_jiffies(u64 sec, u32 nsec); void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o); +void fuse_dentry_set_epoch(struct dentry *dentry, u64 epoch); + /* * Initialize fuse_conn */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index d975073c60296..e89e2b0a57a6a 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -791,6 +791,9 @@ static int fuse_opt_fd(struct fs_context *fsc, struct file *file) { struct fuse_fs_context *ctx = fsc->fs_private; + if (ctx->fud) + return invalfc(fsc, "Multiple fd specified"); + if (file->f_op != &fuse_dev_operations) return invalfc(fsc, "fd is not a fuse device"); /* @@ -1272,6 +1275,7 @@ static void process_init_reply(struct fuse_args *args, int error) struct fuse_mount *fm = ia->fm; struct fuse_conn *fc = fm->fc; struct fuse_init_out *arg = &ia->out; + bool io_uring_enabled = false; bool ok = true; if (error || arg->major != FUSE_KERNEL_VERSION) @@ -1402,7 +1406,7 @@ static void process_init_reply(struct fuse_args *args, int error) ok = false; } if (flags & FUSE_OVER_IO_URING && fuse_uring_enabled()) - fuse_chan_io_uring_enable(fc->chan); + io_uring_enabled = true; if (flags & FUSE_REQUEST_TIMEOUT) timeout = arg->request_timeout; @@ -1432,6 +1436,7 @@ static void process_init_reply(struct fuse_args *args, int error) .minor = fc->minor, .max_write = fc->max_write, .max_pages = fc->max_pages, + .io_uring_enabled = io_uring_enabled, }; fuse_chan_set_initialized(fc->chan, &cp); } @@ -1474,10 +1479,6 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) flags |= FUSE_PASSTHROUGH; - /* - * This is just an information flag for fuse server. No need to check - * the reply - server is either sending IORING_OP_URING_CMD or not. - */ if (fuse_uring_enabled()) flags |= FUSE_OVER_IO_URING; @@ -1639,6 +1640,8 @@ static int fuse_fill_super_submount(struct super_block *sb, fuse_fill_attr_from_inode(&root_attr, parent_fi); root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0, fuse_get_evict_ctr(fm->fc)); + if (!root) + return -ENOMEM; /* * This inode is just a duplicate, so it is not looked up and * its nlookup should not be incremented. fuse_iget() does diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 0e13214917473..5ca87151d70d9 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -260,7 +260,7 @@ static int fuse_direntplus_link(struct file *file, } if (fc->readdirplus_auto) set_bit(FUSE_I_INIT_RDPLUS, &get_fuse_inode(inode)->state); - dentry->d_time = epoch; + fuse_dentry_set_epoch(dentry, epoch); fuse_change_entry_timeout(dentry, o); dput(dentry); diff --git a/fs/hfsplus/catalog.c b/fs/hfsplus/catalog.c index 776ce36cf076b..fe11c4b6dd997 100644 --- a/fs/hfsplus/catalog.c +++ b/fs/hfsplus/catalog.c @@ -204,7 +204,7 @@ int hfsplus_find_cat(struct super_block *sb, u32 cnid, return err; type = be16_to_cpu(tmp.type); - if (type != HFSPLUS_FOLDER_THREAD && type != HFSPLUS_FILE_THREAD) { + if (!is_hfs_thread_record_type(type)) { pr_err("found bad thread record in catalog\n"); return -EIO; } @@ -350,23 +350,22 @@ int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str) goto out; if (!str) { - int len; + hfsplus_cat_entry entry = {0}; hfsplus_cat_build_key_with_cnid(sb, fd.search_key, cnid); - err = hfs_brec_find(&fd, hfs_find_rec_by_key); + err = hfsplus_brec_read_cat(&fd, &entry); if (err) goto out; - off = fd.entryoffset + - offsetof(struct hfsplus_cat_thread, nodeName); - fd.search_key->cat.parent = cpu_to_be32(dir->i_ino); - hfs_bnode_read(fd.bnode, - &fd.search_key->cat.name.length, off, 2); - len = be16_to_cpu(fd.search_key->cat.name.length) * 2; - hfs_bnode_read(fd.bnode, - &fd.search_key->cat.name.unicode, - off + 2, len); - fd.search_key->key_len = cpu_to_be16(6 + len); + type = be16_to_cpu(entry.type); + if (!is_hfs_thread_record_type(type)) { + pr_err("found bad thread record in catalog\n"); + err = -EIO; + goto out; + } + + hfsplus_cat_build_key_uni(fd.search_key, dir->i_ino, + &entry.thread.nodeName); } else { err = hfsplus_cat_build_key(sb, fd.search_key, dir->i_ino, str); if (unlikely(err)) diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h index ec04b82ad9278..7c8667d5a49c1 100644 --- a/fs/hfsplus/hfsplus_fs.h +++ b/fs/hfsplus/hfsplus_fs.h @@ -521,6 +521,12 @@ static inline u32 hfsplus_cat_thread_size(const struct hfsplus_cat_thread *threa be16_to_cpu(thread->nodeName.length) * sizeof(hfsplus_unichr); } +static inline +bool is_hfs_thread_record_type(u16 type) +{ + return type == HFSPLUS_FOLDER_THREAD || type == HFSPLUS_FILE_THREAD; +} + int hfsplus_brec_read_cat(struct hfs_find_data *fd, hfsplus_cat_entry *entry); /* diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 216e1a0dd0b23..6a1d7e778cb0e 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1419,7 +1419,8 @@ hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc) goto out_free; return 0; out_free: - kfree(sbinfo->spool); + if (sbinfo->spool) + hugepage_put_subpool(sbinfo->spool); kfree(sbinfo); return -ENOMEM; } diff --git a/fs/inode.c b/fs/inode.c index 31c5b9ee3a81d..c6b08e8c5f627 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -883,7 +883,6 @@ void evict_inodes(struct super_block *sb) struct inode *inode; LIST_HEAD(dispose); -again: spin_lock(&sb->s_inode_list_lock); list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { if (icount_read_once(inode)) @@ -902,19 +901,19 @@ void evict_inodes(struct super_block *sb) inode_state_set(inode, I_FREEING); inode_lru_list_del(inode); spin_unlock(&inode->i_lock); - list_add(&inode->i_lru, &dispose); /* - * We can have a ton of inodes to evict at unmount time given - * enough memory, check to see if we need to go to sleep for a - * bit so we don't livelock. + * Keep this inode out of dispose so it stays on s_inodes while + * the list lock is dropped. I_FREEING prevents new references + * and leaves eviction to us, so we can resume the walk from it. */ if (need_resched()) { spin_unlock(&sb->s_inode_list_lock); cond_resched(); dispose_list(&dispose); - goto again; + spin_lock(&sb->s_inode_list_lock); } + list_add(&inode->i_lru, &dispose); } spin_unlock(&sb->s_inode_list_lock); @@ -2833,8 +2832,8 @@ struct timespec64 inode_set_ctime_to_ts(struct inode *inode, struct timespec64 t { trace_inode_set_ctime_to_ts(inode, &ts); set_normalized_timespec64(&ts, ts.tv_sec, ts.tv_nsec); - inode->i_ctime_sec = ts.tv_sec; - inode->i_ctime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_ctime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_ctime_nsec, ts.tv_nsec); return ts; } EXPORT_SYMBOL(inode_set_ctime_to_ts); @@ -2908,7 +2907,7 @@ struct timespec64 inode_set_ctime_current(struct inode *inode) */ cns = smp_load_acquire(&inode->i_ctime_nsec); if (cns & I_CTIME_QUERIED) { - struct timespec64 ctime = { .tv_sec = inode->i_ctime_sec, + struct timespec64 ctime = { .tv_sec = inode_get_ctime_sec(inode), .tv_nsec = cns & ~I_CTIME_QUERIED }; if (timespec64_compare(&now, &ctime) <= 0) { @@ -2920,7 +2919,7 @@ struct timespec64 inode_set_ctime_current(struct inode *inode) mgtime_counter_inc(mg_ctime_updates); /* No need to cmpxchg if it's exactly the same */ - if (cns == now.tv_nsec && inode->i_ctime_sec == now.tv_sec) { + if (cns == now.tv_nsec && inode_get_ctime_sec(inode) == now.tv_sec) { trace_ctime_xchg_skip(inode, &now); goto out; } @@ -2929,7 +2928,7 @@ struct timespec64 inode_set_ctime_current(struct inode *inode) /* Try to swap the nsec value into place. */ if (try_cmpxchg(&inode->i_ctime_nsec, &cur, now.tv_nsec)) { /* If swap occurred, then we're (mostly) done */ - inode->i_ctime_sec = now.tv_sec; + WRITE_ONCE(inode->i_ctime_sec, now.tv_sec); trace_ctime_ns_xchg(inode, cns, now.tv_nsec, cur); mgtime_counter_inc(mg_ctime_swaps); } else { @@ -2944,7 +2943,7 @@ struct timespec64 inode_set_ctime_current(struct inode *inode) goto retry; } /* Otherwise, keep the existing ctime */ - now.tv_sec = inode->i_ctime_sec; + now.tv_sec = inode_get_ctime_sec(inode); now.tv_nsec = cur & ~I_CTIME_QUERIED; } out: @@ -2977,7 +2976,7 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 u /* pairs with try_cmpxchg below */ cur = smp_load_acquire(&inode->i_ctime_nsec); cur_ts.tv_nsec = cur & ~I_CTIME_QUERIED; - cur_ts.tv_sec = inode->i_ctime_sec; + cur_ts.tv_sec = inode_get_ctime_sec(inode); /* If the update is older than the existing value, skip it. */ if (timespec64_compare(&update, &cur_ts) <= 0) @@ -3003,7 +3002,7 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 u retry: old = cur; if (try_cmpxchg(&inode->i_ctime_nsec, &cur, update.tv_nsec)) { - inode->i_ctime_sec = update.tv_sec; + WRITE_ONCE(inode->i_ctime_sec, update.tv_sec); mgtime_counter_inc(mg_ctime_swaps); return update; } @@ -3019,7 +3018,7 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 u goto retry; /* Otherwise, it was a new timestamp. */ - cur_ts.tv_sec = inode->i_ctime_sec; + cur_ts.tv_sec = inode_get_ctime_sec(inode); cur_ts.tv_nsec = cur & ~I_CTIME_QUERIED; return cur_ts; } diff --git a/fs/iomap/bio.c b/fs/iomap/bio.c index dc8ac7e370a5c..48100c614431c 100644 --- a/fs/iomap/bio.c +++ b/fs/iomap/bio.c @@ -179,10 +179,11 @@ int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter, if (srcmap->flags & IOMAP_F_INTEGRITY) fs_bio_integrity_alloc(&bio); error = submit_bio_wait(&bio); - if (srcmap->flags & IOMAP_F_INTEGRITY) { + if (bio_integrity(&bio)) { if (!error) error = fs_bio_integrity_verify(&bio, sector, len); fs_bio_integrity_free(&bio); } + bio_uninit(&bio); return error; } diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index e2cd5f92babed..01daed391856c 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -358,6 +358,7 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter, iomap_max_bio_size(&iter->iomap), alignment); else ret = bio_iov_iter_get_pages(bio, dio->submit.iter, + bdev_dma_alignment(bio->bi_bdev), alignment - 1); if (unlikely(ret)) goto out_put_bio; diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c index e4a29829591a7..c445a38b6285d 100644 --- a/fs/iomap/iter.c +++ b/fs/iomap/iter.c @@ -6,12 +6,19 @@ #include #include "trace.h" -static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter) +/* + * Release the iter folio batch. Note that the iomap flag is meant to control + * the I/O path for the mapping and may not be set in error situations. + */ +static inline void iomap_iter_clean_fbatch(const struct iomap_iter *iter, + struct iomap *iomap) { - if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) { + if (!iter->fbatch) + return; + iomap->flags &= ~IOMAP_F_FOLIO_BATCH; + if (folio_batch_count(iter->fbatch)) { folio_batch_release(iter->fbatch); folio_batch_reinit(iter->fbatch); - iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH; } } @@ -40,51 +47,27 @@ static inline void iomap_iter_done(struct iomap_iter *iter) } /** - * iomap_iter - iterate over a ranges in a file - * @iter: iteration structue - * @ops: iomap ops provided by the file system + * iomap_iter_continue - decide whether iteration should continue + * @iter: iteration structure + * @iomap: the mapping that was just processed + * @srcmap: the source mapping that was just processed * - * Iterate over filesystem-provided space mappings for the provided file range. + * Helper normally called via iomap_iter_next(). Called after the previous + * mapping has been finished to determine whether there is more of the file + * range left to process. * - * This function handles cleanup of resources acquired for iteration when the - * filesystem indicates there are no more space mappings, which means that this - * function must be called in a loop that continues as long it returns a - * positive value. If 0 or a negative value is returned, the caller must not - * return to the loop body. Within a loop body, there are two ways to break out - * of the loop body: leave @iter.status unchanged, or set it to a negative - * errno. + * Returns 1 if there is more work to do, in which case @iomap and @srcmap are + * cleared so the caller can produce the next mapping; zero if the range is + * fully consumed; or a negative errno on error. */ -int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) +int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, + struct iomap *srcmap, int ret) { - bool stale = iter->iomap.flags & IOMAP_F_STALE; - ssize_t advanced; - u64 olen; - int ret; + const bool stale = iomap->flags & IOMAP_F_STALE; + const ssize_t advanced = iter->pos - iter->iter_start_pos; - trace_iomap_iter(iter, ops, _RET_IP_); - - if (!iter->iomap.length) - goto begin; - - /* - * Calculate how far the iter was advanced and the original length bytes - * for ->iomap_end(). - */ - advanced = iter->pos - iter->iter_start_pos; - olen = iter->len + advanced; - - if (ops->iomap_end) { - ret = ops->iomap_end(iter->inode, iter->iter_start_pos, - iomap_length_trim(iter, iter->iter_start_pos, - olen), - advanced, iter->flags, &iter->iomap); - if (ret < 0 && !advanced) - return ret; - } - - /* detect old return semantics where this would advance */ - if (WARN_ON_ONCE(iter->status > 0)) - iter->status = -EIO; + if (ret < 0 && !advanced) + return ret; /* * Use iter->len to determine whether to continue onto the next mapping. @@ -92,25 +75,60 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) * advanced at all (i.e. no work was done for some reason) unless the * mapping has been marked stale and needs to be reprocessed. */ - if (iter->status < 0) + if (WARN_ON_ONCE(iter->status > 0)) + /* detect old return semantics where this would advance */ + ret = -EIO; + else if (iter->status < 0) ret = iter->status; else if (iter->len == 0 || (!advanced && !stale)) ret = 0; else ret = 1; - iomap_iter_clean_fbatch(iter); - iter->status = 0; + + iomap_iter_clean_fbatch(iter, iomap); + if (ret <= 0) return ret; - memset(&iter->iomap, 0, sizeof(iter->iomap)); - memset(&iter->srcmap, 0, sizeof(iter->srcmap)); + memset(iomap, 0, sizeof(*iomap)); + memset(srcmap, 0, sizeof(*srcmap)); -begin: - ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags, - &iter->iomap, &iter->srcmap); - if (ret < 0) - return ret; - iomap_iter_done(iter); - return 1; + return ret; +} +EXPORT_SYMBOL_GPL(iomap_iter_continue); + +/** + * iomap_iter - iterate over ranges in a file + * @iter: iteration structure + * @ops: iomap ops provided by the filesystem + * + * Iterate over filesystem-provided space mappings for the provided file range. + * + * This function handles cleanup of resources acquired for iteration when the + * filesystem indicates there are no more space mappings, which means that this + * function must be called in a loop that continues as long it returns a + * positive value. If 0 or a negative value is returned, the caller must not + * return to the loop body. Within a loop body, there are two ways to break out + * of the loop body: leave @iter.status unchanged, or set it to a negative + * errno. + */ +int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) +{ + int ret; + + trace_iomap_iter(iter, ops, _RET_IP_); + + if (ops->iomap_next) + ret = ops->iomap_next(iter, &iter->iomap, &iter->srcmap); + else + ret = iomap_iter_next(iter, &iter->iomap, &iter->srcmap, + ops->iomap_begin, ops->iomap_end); + + iter->status = 0; + if (ret > 0) + iomap_iter_done(iter); + else if (ret < 0) + iomap_iter_clean_fbatch(iter, &iter->iomap); + + return ret; } diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c index 397568b9c7e7d..f9869d62b8509 100644 --- a/fs/isofs/compress.c +++ b/fs/isofs/compress.c @@ -65,12 +65,14 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, /* Empty block? */ if (block_size == 0) { for ( i = 0 ; i < pcount ; i++ ) { + unsigned int off = i ? 0 : poffset; + if (!pages[i]) continue; - memzero_page(pages[i], 0, PAGE_SIZE); + memzero_page(pages[i], off, PAGE_SIZE - off); SetPageUptodate(pages[i]); } - return ((loff_t)pcount) << PAGE_SHIFT; + return (((loff_t)pcount) << PAGE_SHIFT) - poffset; } /* Because zlib is not thread-safe, do all the I/O at the top. */ @@ -291,6 +293,7 @@ static int zisofs_fill_pages(struct inode *inode, int full_page, int pcount, memzero_page(*pages, poffset, PAGE_SIZE - poffset); SetPageUptodate(*pages); } + brelse(bh); return 0; } diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 1508e2f544621..5132737120108 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -358,15 +358,16 @@ int jbd2_cleanup_journal_tail(journal_t *journal) /* * journal_shrink_one_cp_list * - * Find all the written-back checkpoint buffers in the given list - * and try to release them. If the whole transaction is released, set - * the 'released' parameter. Return the number of released checkpointed - * buffers. + * Find written-back checkpoint buffers in the given list and try to release + * them. If 'nr_to_scan' is set, scan at most that many buffers. If the whole + * transaction is released, set the 'released' parameter. Return the number of + * released checkpointed buffers. * * Called with j_list_lock held. */ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, enum jbd2_shrink_type type, + unsigned long *nr_to_scan, bool *released) { struct journal_head *last_jh; @@ -375,13 +376,15 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, int ret; *released = false; - if (!jh) + if (!jh || (nr_to_scan && !*nr_to_scan)) return 0; last_jh = jh->b_cpprev; do { jh = next_jh; next_jh = jh->b_cpnext; + if (nr_to_scan) + (*nr_to_scan)--; if (type == JBD2_SHRINK_DESTROY) { ret = __jbd2_journal_remove_checkpoint(jh); @@ -389,7 +392,7 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, ret = jbd2_journal_try_remove_checkpoint(jh); if (ret < 0) { if (type == JBD2_SHRINK_BUSY_SKIP) - continue; + goto next; break; } } @@ -400,9 +403,10 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, break; } +next: if (need_resched()) break; - } while (jh != last_jh); + } while (jh != last_jh && (!nr_to_scan || *nr_to_scan)); return nr_freed; } @@ -424,7 +428,6 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, tid_t first_tid = 0, last_tid = 0, next_tid = 0; tid_t tid = 0; unsigned long nr_freed = 0; - unsigned long freed; bool first_set = false; again: @@ -457,10 +460,9 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, next_transaction = transaction->t_cpnext; tid = transaction->t_tid; - freed = journal_shrink_one_cp_list(transaction->t_checkpoint_list, - JBD2_SHRINK_BUSY_SKIP, &released); - nr_freed += freed; - (*nr_to_scan) -= min(*nr_to_scan, freed); + nr_freed += journal_shrink_one_cp_list(transaction->t_checkpoint_list, + JBD2_SHRINK_BUSY_SKIP, + nr_to_scan, &released); if (*nr_to_scan == 0) break; if (need_resched() || spin_needbreak(&journal->j_list_lock)) @@ -516,7 +518,7 @@ void __jbd2_journal_clean_checkpoint_list(journal_t *journal, transaction = next_transaction; next_transaction = transaction->t_cpnext; journal_shrink_one_cp_list(transaction->t_checkpoint_list, - type, &released); + type, NULL, &released); /* * This function only frees up some memory if possible so we * dont have an obligation to finish processing. Bail out if diff --git a/fs/kernfs/mount.c b/fs/kernfs/mount.c index f183a96778b9a..a57399021c8b0 100644 --- a/fs/kernfs/mount.c +++ b/fs/kernfs/mount.c @@ -434,8 +434,8 @@ void kernfs_kill_sb(struct super_block *sb) up_write(&root->kernfs_supers_rwsem); /* - * Remove the superblock from fs_supers/s_instances - * so we can't find it, before freeing kernfs_super_info. + * Mark the superblock dead so sget_fc() can't find it, + * before freeing kernfs_super_info. */ kill_anon_super(sb); kfree(info); diff --git a/fs/lockd/clntproc.c b/fs/lockd/clntproc.c index f06faf577cea7..f8018bfe9c646 100644 --- a/fs/lockd/clntproc.c +++ b/fs/lockd/clntproc.c @@ -487,9 +487,12 @@ static const struct file_lock_operations nlmclnt_lock_ops = { static void nlmclnt_locks_init_private(struct file_lock *fl, struct nlm_host *host) { fl->fl_u.nfs_fl.state = 0; + fl->fl_ops = NULL; fl->fl_u.nfs_fl.owner = nlmclnt_find_lockowner(host, fl->c.flc_owner); INIT_LIST_HEAD(&fl->fl_u.nfs_fl.list); + if (!fl->fl_u.nfs_fl.owner) + return; fl->fl_ops = &nlmclnt_lock_ops; } diff --git a/fs/lockd/lockd.h b/fs/lockd/lockd.h index e418a50c41803..14cc952fe81a2 100644 --- a/fs/lockd/lockd.h +++ b/fs/lockd/lockd.h @@ -314,7 +314,7 @@ void nsm_release(struct nsm_handle *nsm); * This is used in garbage collection and resource reclaim * A return value != 0 means destroy the lock/block/share */ -typedef int (*nlm_host_match_fn_t)(void *cur, struct nlm_host *ref); +typedef int (*nlm_host_match_fn_t)(void *owner, struct nlm_host *ref); /* * Server-side lock handling diff --git a/fs/lockd/nlm3xdr_gen.c b/fs/lockd/nlm3xdr_gen.c index 9ed5a41b5daf3..df14692ce37f2 100644 --- a/fs/lockd/nlm3xdr_gen.c +++ b/fs/lockd/nlm3xdr_gen.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Generated by xdrgen. Manual edits will be lost. // XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x -// XDR specification modification time: Thu Apr 23 10:56:34 2026 +// XDR specification modification time: Mon Jun 29 20:42:29 2026 #include diff --git a/fs/lockd/nlm3xdr_gen.h b/fs/lockd/nlm3xdr_gen.h index c99038e99805f..3824ffe2aae46 100644 --- a/fs/lockd/nlm3xdr_gen.h +++ b/fs/lockd/nlm3xdr_gen.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x */ -/* XDR specification modification time: Thu Apr 23 10:56:34 2026 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM3_DECL_H #define _LINUX_XDRGEN_NLM3_DECL_H diff --git a/fs/lockd/nlm4xdr_gen.c b/fs/lockd/nlm4xdr_gen.c index 1c8c221db456c..5a60aff067147 100644 --- a/fs/lockd/nlm4xdr_gen.c +++ b/fs/lockd/nlm4xdr_gen.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Generated by xdrgen. Manual edits will be lost. // XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x -// XDR specification modification time: Thu Dec 25 13:10:19 2025 +// XDR specification modification time: Mon Jun 29 20:42:29 2026 #include @@ -20,6 +20,16 @@ xdrgen_decode_fsh4_mode(struct xdr_stream *xdr, fsh4_mode *ptr) if (xdr_stream_decode_u32(xdr, &val) < 0) return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsm_DN: + case fsm_DR: + case fsm_DW: + case fsm_DRW: + break; + default: + return false; + } *ptr = val; return true; } @@ -31,6 +41,16 @@ xdrgen_decode_fsh4_access(struct xdr_stream *xdr, fsh4_access *ptr) if (xdr_stream_decode_u32(xdr, &val) < 0) return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsa_NONE: + case fsa_R: + case fsa_W: + case fsa_RW: + break; + default: + return false; + } *ptr = val; return true; } @@ -62,7 +82,30 @@ xdrgen_decode_int32(struct xdr_stream *xdr, int32 *ptr) static bool __maybe_unused xdrgen_decode_nlm4_stats(struct xdr_stream *xdr, nlm4_stats *ptr) { - return xdr_stream_decode_be32(xdr, ptr) == 0; + __be32 raw; + u32 val; + + if (xdr_stream_decode_be32(xdr, &raw) < 0) + return false; + val = be32_to_cpu(raw); + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NLM4_GRANTED: + case NLM4_DENIED: + case NLM4_DENIED_NOLOCKS: + case NLM4_BLOCKED: + case NLM4_DENIED_GRACE_PERIOD: + case NLM4_DEADLCK: + case NLM4_ROFS: + case NLM4_STALE_FH: + case NLM4_FBIG: + case NLM4_FAILED: + break; + default: + return false; + } + *ptr = raw; + return true; } static bool __maybe_unused diff --git a/fs/lockd/nlm4xdr_gen.h b/fs/lockd/nlm4xdr_gen.h index b6008b296a3e3..ce9dda0a052a3 100644 --- a/fs/lockd/nlm4xdr_gen.h +++ b/fs/lockd/nlm4xdr_gen.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */ -/* XDR specification modification time: Thu Dec 25 13:10:19 2025 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM4_DECL_H #define _LINUX_XDRGEN_NLM4_DECL_H diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c index 490551369ef2a..ee90e743064af 100644 --- a/fs/lockd/svc.c +++ b/fs/lockd/svc.c @@ -47,7 +47,7 @@ static struct svc_program nlmsvc_program; -const struct nlmsvc_binding *nlmsvc_ops; +const struct nlmsvc_binding __rcu *nlmsvc_ops; EXPORT_SYMBOL_GPL(nlmsvc_ops); static DEFINE_MUTEX(nlmsvc_mutex); @@ -142,7 +142,7 @@ lockd(void *vrqstp) nlmsvc_retry_blocked(rqstp); svc_recv(rqstp, 0); } - if (nlmsvc_ops) + if (rcu_access_pointer(nlmsvc_ops)) nlmsvc_invalidate_all(); nlm_shutdown_hosts(); cancel_delayed_work_sync(&ln->grace_period_end); diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c index 78e675470c4b2..b73004a7987e8 100644 --- a/fs/lockd/svc4proc.c +++ b/fs/lockd/svc4proc.c @@ -128,7 +128,7 @@ nlm4svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored) { struct nlm_host *host; - if (!nlmsvc_ops) + if (!rcu_access_pointer(nlmsvc_ops)) return NULL; host = nlmsvc_lookup_host(rqstp, caller.data, caller.len); if (!host) @@ -872,7 +872,8 @@ static __be32 nlm4svc_proc_granted_msg(struct svc_rqst *rqstp) struct nlm4_testargs_wrapper *argp = rqstp->rq_argp; struct nlm_host *host; - host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + host = nlmsvc_lookup_host(rqstp, argp->xdrgen.alock.caller_name.data, + argp->xdrgen.alock.caller_name.len); if (!host) return rpc_system_err; @@ -894,7 +895,7 @@ static __be32 nlm4svc_proc_granted_res(struct svc_rqst *rqstp) { struct nlm4_res_wrapper *argp = rqstp->rq_argp; - if (!nlmsvc_ops) + if (!rcu_access_pointer(nlmsvc_ops)) return rpc_success; if (nlm4_netobj_to_cookie(&argp->cookie, &argp->xdrgen.cookie)) diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c index 386a881b520fd..d410b8c698937 100644 --- a/fs/lockd/svcproc.c +++ b/fs/lockd/svcproc.c @@ -133,7 +133,7 @@ nlm3svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored) { struct nlm_host *host; - if (!nlmsvc_ops) + if (!rcu_access_pointer(nlmsvc_ops)) return NULL; host = nlmsvc_lookup_host(rqstp, caller.data, caller.len); if (!host) @@ -924,7 +924,7 @@ static __be32 nlmsvc_proc_granted_res(struct svc_rqst *rqstp) { struct nlm_res_wrapper *argp = rqstp->rq_argp; - if (!nlmsvc_ops) + if (!rcu_access_pointer(nlmsvc_ops)) return rpc_success; if (nlm_netobj_to_cookie(&argp->cookie, &argp->xdrgen.cookie)) diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c index a0d1a6fbf61ee..ffab76278bb26 100644 --- a/fs/lockd/svcsubs.c +++ b/fs/lockd/svcsubs.c @@ -90,22 +90,35 @@ int lock_to_openmode(struct file_lock *lock) static __be32 nlm_do_fopen(struct svc_rqst *rqstp, struct nlm_file *file, int mode) { + const struct nlmsvc_binding *ops; __be32 nlmerr = nlm__int__failed; __be32 deferred = 0; int error; int m; + rcu_read_lock(); + ops = rcu_dereference(nlmsvc_ops); + if (!ops || !try_module_get(ops->owner)) { + rcu_read_unlock(); + return nlm__int__failed; + } + rcu_read_unlock(); + for (m = O_RDONLY; m <= O_WRONLY; m++) { struct file **fp = &file->f_file[m]; if (mode != O_RDWR && mode != m) continue; - if (*fp) + if (*fp) { + module_put(ops->owner); return nlm_granted; + } - error = nlmsvc_ops->fopen(rqstp, &file->f_handle, fp, m); - if (!error) + error = ops->fopen(rqstp, &file->f_handle, fp, m); + if (!error) { + module_put(ops->owner); return nlm_granted; + } dprintk("lockd: open failed (errno %d)\n", error); switch (error) { @@ -122,6 +135,7 @@ static __be32 nlm_do_fopen(struct svc_rqst *rqstp, } } + module_put(ops->owner); return deferred ? deferred : nlmerr; } @@ -185,6 +199,33 @@ nlm_lookup_file(struct svc_rqst *rqstp, struct nlm_file **result, goto out_unlock; } +/* + * Release the struct file references held by a nlm_file. + */ +static void nlm_release_files(struct nlm_file *file) +{ + const struct nlmsvc_binding *ops; + bool have_ops; + + rcu_read_lock(); + ops = rcu_dereference(nlmsvc_ops); + have_ops = ops && try_module_get(ops->owner); + rcu_read_unlock(); + + if (have_ops) { + if (file->f_file[O_RDONLY]) + ops->fclose(file->f_file[O_RDONLY]); + if (file->f_file[O_WRONLY]) + ops->fclose(file->f_file[O_WRONLY]); + module_put(ops->owner); + } else { + if (file->f_file[O_RDONLY]) + fput(file->f_file[O_RDONLY]); + if (file->f_file[O_WRONLY]) + fput(file->f_file[O_WRONLY]); + } +} + /* * Delete a file after having released all locks, blocks and shares */ @@ -194,10 +235,7 @@ nlm_delete_file(struct nlm_file *file) nlm_debug_print_file("closing file", file); if (!hlist_unhashed(&file->f_list)) { hlist_del(&file->f_list); - if (file->f_file[O_RDONLY]) - nlmsvc_ops->fclose(file->f_file[O_RDONLY]); - if (file->f_file[O_WRONLY]) - nlmsvc_ops->fclose(file->f_file[O_WRONLY]); + nlm_release_files(file); kfree(file); } else { printk(KERN_WARNING "lockd: attempt to release unknown file!\n"); @@ -312,12 +350,10 @@ nlm_file_inuse(struct nlm_file *file) return 0; } -static void nlm_close_files(struct nlm_file *file) +static void nlm_file_release(struct nlm_file *file) { - if (file->f_file[O_RDONLY]) - nlmsvc_ops->fclose(file->f_file[O_RDONLY]); - if (file->f_file[O_WRONLY]) - nlmsvc_ops->fclose(file->f_file[O_WRONLY]); + if (!nlm_file_inuse(file)) + nlm_delete_file(file); } /* @@ -327,32 +363,41 @@ static int nlm_traverse_files(void *data, nlm_host_match_fn_t match, int (*is_failover_file)(void *data, struct nlm_file *file)) { - struct hlist_node *next; - struct nlm_file *file; + struct nlm_file *file, *next; int i, ret = 0; mutex_lock(&nlm_file_mutex); for (i = 0; i < FILE_NRHASH; i++) { - hlist_for_each_entry_safe(file, next, &nlm_files[i], f_list) { - if (is_failover_file && !is_failover_file(data, file)) - continue; + file = hlist_entry_safe(nlm_files[i].first, + struct nlm_file, f_list); + if (file) file->f_count++; - mutex_unlock(&nlm_file_mutex); - - /* Traverse locks, blocks and shares of this file - * and update file->f_locks count */ - if (nlm_inspect_file(data, file, match)) - ret = 1; + while (file) { + /* + * Pin the next neighbour before we drop the mutex + * for nlm_inspect_file(); a concurrent + * nlm_release_file() under the same mutex would + * otherwise be free to unlink and kfree it during + * the unlock window, leaving us to dereference a + * freed slab when we walked to next afterwards. + */ + next = hlist_entry_safe(file->f_list.next, + struct nlm_file, f_list); + if (next) + next->f_count++; + + if (!is_failover_file || is_failover_file(data, file)) { + mutex_unlock(&nlm_file_mutex); + + if (nlm_inspect_file(data, file, match)) + ret = 1; + + mutex_lock(&nlm_file_mutex); + } - mutex_lock(&nlm_file_mutex); file->f_count--; - /* No more references to this file. Let go of it. */ - if (list_empty(&file->f_blocks) && !file->f_locks - && !file->f_shares && !file->f_count) { - hlist_del(&file->f_list); - nlm_close_files(file); - kfree(file); - } + nlm_file_release(file); + file = next; } } mutex_unlock(&nlm_file_mutex); @@ -512,7 +557,7 @@ EXPORT_SYMBOL_GPL(nlmsvc_unlock_all_by_sb); static int nlmsvc_match_ip(void *datap, struct nlm_host *host) { - return rpc_cmp_addr(nlm_srcaddr(host), datap); + return rpc_cmp_addr(nlm_srcaddr(datap), (struct sockaddr *)host); } /** diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index 7fdfa4f27e349..105194de6e13e 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -54,6 +54,42 @@ static void netfs_rreq_expand(struct netfs_io_request *rreq, } } +/* + * Drop the folio refs acquired from the readahead API. + */ +static void netfs_bulk_drop_ra_refs(struct netfs_io_request *rreq) +{ + struct folio_batch fbatch; + struct folio *folio; + pgoff_t nr_pages = DIV_ROUND_UP(rreq->len, PAGE_SIZE); + pgoff_t first = rreq->start / PAGE_SIZE; + XA_STATE(xas, &rreq->mapping->i_pages, first); + + folio_batch_init(&fbatch); + + rcu_read_lock(); + + xas_for_each(&xas, folio, first + nr_pages - 1) { + if (xas_retry(&xas, folio)) + continue; + + if (!folio_batch_add(&fbatch, folio)) + folio_batch_release(&fbatch); + } + + rcu_read_unlock(); + folio_batch_release(&fbatch); + trace_netfs_rreq(rreq, netfs_rreq_trace_ra_put_ref); + clear_bit_unlock(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags); + wake_up(&rreq->waitq); +} + +static void netfs_maybe_bulk_drop_ra_refs(struct netfs_io_request *rreq) +{ + if (test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags)) + netfs_bulk_drop_ra_refs(rreq); +} + /* * Begin an operation, and fetch the stored zero point value from the cookie if * available. @@ -74,12 +110,8 @@ static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_in * * Returns the limited size if successful and -ENOMEM if insufficient memory * available. - * - * [!] NOTE: This must be run in the same thread as ->issue_read() was called - * in as we access the readahead_control struct. */ -static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq, - struct readahead_control *ractl) +static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) { struct netfs_io_request *rreq = subreq->rreq; size_t rsize = subreq->len; @@ -87,30 +119,6 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq, if (subreq->source == NETFS_DOWNLOAD_FROM_SERVER) rsize = umin(rsize, rreq->io_streams[0].sreq_max_len); - if (ractl) { - /* If we don't have sufficient folios in the rolling buffer, - * extract a folioq's worth from the readahead region at a time - * into the buffer. Note that this acquires a ref on each page - * that we will need to release later - but we don't want to do - * that until after we've started the I/O. - */ - struct folio_batch put_batch; - - folio_batch_init(&put_batch); - while (rreq->submitted < subreq->start + rsize) { - ssize_t added; - - added = rolling_buffer_load_from_ra(&rreq->buffer, ractl, - &put_batch); - if (added < 0) { - folio_batch_release(&put_batch); - return added; - } - rreq->submitted += added; - } - folio_batch_release(&put_batch); - } - subreq->len = rsize; if (unlikely(rreq->io_streams[0].sreq_max_segs)) { size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize, @@ -203,17 +211,68 @@ static void netfs_issue_read(struct netfs_io_request *rreq, } } +/* + * Mark folios that we want to copy to the cache. For filesystems that use + * netfslib fully, we set folio->private to NETFS_FOLIO_COPY_TO_CACHE; + * otherwise we set the deprecated PG_private_2. + */ +static void netfs_mark_copy_to_cache(struct netfs_io_request *rreq, + struct folio_queue **fq, + unsigned int *offset, + int *slot, + size_t len, + bool copy) +{ + while (len > 0) { + struct folio *folio; + size_t fsize, overlap; + + if (!*fq) + break; + if (*slot >= folioq_count(*fq)) { + *fq = (*fq)->next; + *slot = 0; + *offset = 0; + continue; + } + + /* Determine how much the subreq overlaps the folio, if at all. */ + fsize = folioq_folio_size(*fq, *slot); + overlap = min(len, fsize - *offset); + + if (overlap > 0 && copy) { + folio = folioq_folio(*fq, *slot); + if (unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags))) { + if (!folio_test_private_2(folio)) + folio_start_private_2(folio); + } else { + if (!folio_get_private(folio)) + folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE); + } + trace_netfs_folio(folio, netfs_folio_trace_mark_copy); + } + + len -= overlap; + *offset += overlap; + if (*offset >= fsize) { + *slot += 1; + *offset = 0; + } + } +} + /* * Perform a read to the pagecache from a series of sources of different types, * slicing up the region to be read according to available cache blocks and * network rsize. */ -static void netfs_read_to_pagecache(struct netfs_io_request *rreq, - struct readahead_control *ractl) +static void netfs_read_to_pagecache(struct netfs_io_request *rreq) { + struct folio_queue *fq = rreq->buffer.tail; unsigned long long start = rreq->start; + unsigned int offset = 0; ssize_t size = rreq->len; - int ret = 0; + int ret = 0, slot = 0; do { struct netfs_io_subrequest *subreq; @@ -288,7 +347,7 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq, break; issue: - slice = netfs_prepare_read_iterator(subreq, ractl); + slice = netfs_prepare_read_iterator(subreq); if (slice < 0) { ret = slice; netfs_cancel_read(subreq, ret); @@ -301,7 +360,15 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq, set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); } + if (fq) { + /* See if the cache indicated this should be cached. */ + bool copy = test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); + + netfs_mark_copy_to_cache(rreq, &fq, &slot, &offset, slice, copy); + } + netfs_issue_read(rreq, subreq); + netfs_maybe_bulk_drop_ra_refs(rreq); if (test_bit(NETFS_RREQ_PAUSE, &rreq->flags)) netfs_wait_for_paused_read(rreq); @@ -339,7 +406,8 @@ void netfs_readahead(struct readahead_control *ractl) { struct netfs_io_request *rreq; struct netfs_inode *ictx = netfs_inode(ractl->mapping->host); - unsigned long long start = readahead_pos(ractl); + ssize_t added; + uoff_t start = readahead_pos(ractl); size_t size = readahead_length(ractl); int ret; @@ -360,11 +428,24 @@ void netfs_readahead(struct readahead_control *ractl) netfs_rreq_expand(rreq, ractl); - rreq->submitted = rreq->start; - if (rolling_buffer_init(&rreq->buffer, rreq->debug_id, ITER_DEST, rreq->gfp) < 0) + /* Load the folios to be read into a bvecq chain. Note that this + * acquires a ref on each folio that we will need to release later - + * but we don't want to do that until after we've started the I/O. + */ + added = rolling_buffer_bulk_load_from_ra(&rreq->buffer, ractl, + rreq->debug_id, rreq->gfp); + if (added < 0) { + ret = added; goto cleanup_free; - netfs_read_to_pagecache(rreq, ractl); + } + __set_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags); + + rreq->submitted = rreq->start + added; + rreq->cleaned_to = rreq->start; + netfs_read_set_unlock_at(rreq); + netfs_read_to_pagecache(rreq); + netfs_maybe_bulk_drop_ra_refs(rreq); return netfs_put_request(rreq, netfs_rreq_trace_put_return); cleanup_free: @@ -387,6 +468,7 @@ static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct fo if (added < 0) return added; rreq->submitted = rreq->start + added; + rreq->progress_at = added; return 0; } @@ -400,15 +482,14 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) struct netfs_group *group = netfs_folio_group(folio); struct netfs_folio *finfo = netfs_folio_info(folio); struct netfs_inode *ctx = netfs_inode(mapping->host); - struct folio *sink = NULL; - struct bio_vec *bvec; + struct bio_vec *bvec = NULL; unsigned int from = finfo->dirty_offset; unsigned int to = from + finfo->dirty_len; - unsigned int off = 0, i = 0; + unsigned int off = 0; size_t flen = folio_size(folio); size_t nr_bvec = flen / PAGE_SIZE + 2; size_t part; - int ret; + int ret, i = 0, sink_from = -1, sink_to = -1; _enter("%lx", folio->index); @@ -433,31 +514,30 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) if (!bvec) goto discard; - sink = folio_alloc(GFP_KERNEL, 0); - if (!sink) { - kfree(bvec); - goto discard; - } - trace_netfs_folio(folio, netfs_folio_trace_read_gaps); - rreq->direct_bv = bvec; - rreq->direct_bv_count = nr_bvec; if (from > 0) { bvec_set_folio(&bvec[i++], folio, from, 0); off = from; } + sink_from = i; while (off < to) { + struct folio *sink = folio_alloc(GFP_KERNEL, 0); + + if (!sink) + goto discard; part = min_t(size_t, to - off, PAGE_SIZE); - bvec_set_folio(&bvec[i++], sink, part, 0); + bvec_set_folio(&bvec[i], sink, part, 0); off += part; + sink_to = i; + i++; } if (to < flen) bvec_set_folio(&bvec[i++], folio, flen - to, to); iov_iter_bvec(&rreq->buffer.iter, ITER_DEST, bvec, i, rreq->len); rreq->submitted = rreq->start + flen; - netfs_read_to_pagecache(rreq, NULL); + netfs_read_to_pagecache(rreq); ret = netfs_wait_for_read(rreq); if (ret >= 0) { @@ -471,8 +551,10 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) folio_mark_uptodate(folio); } - if (sink) - folio_put(sink); + if (sink_to >= 0) + for (; sink_from <= sink_to; sink_from++) + folio_put(bvec_folio(&bvec[sink_from])); + kfree(bvec); folio_unlock(folio); netfs_put_request(rreq, netfs_rreq_trace_put_return); return ret < 0 ? ret : 0; @@ -481,6 +563,10 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) netfs_put_failed_request(rreq); alloc_error: folio_unlock(folio); + if (sink_to >= 0) + for (; sink_from <= sink_to; sink_from++) + folio_put(bvec_folio(&bvec[sink_from])); + kfree(bvec); return ret; } @@ -532,7 +618,7 @@ int netfs_read_folio(struct file *file, struct folio *folio) if (ret < 0) goto discard; - netfs_read_to_pagecache(rreq, NULL); + netfs_read_to_pagecache(rreq); ret = netfs_wait_for_read(rreq); netfs_put_request(rreq, netfs_rreq_trace_put_return); return ret < 0 ? ret : 0; @@ -689,7 +775,7 @@ int netfs_write_begin(struct netfs_inode *ctx, if (ret < 0) goto error_put; - netfs_read_to_pagecache(rreq, NULL); + netfs_read_to_pagecache(rreq); ret = netfs_wait_for_read(rreq); netfs_put_request(rreq, netfs_rreq_trace_put_return); if (ret < 0) @@ -754,7 +840,7 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio, if (ret < 0) goto error_put; - netfs_read_to_pagecache(rreq, NULL); + netfs_read_to_pagecache(rreq); ret = netfs_wait_for_read(rreq); netfs_put_request(rreq, netfs_rreq_trace_put_return); return ret < 0 ? ret : 0; diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c index c16fbad286a17..2361277416c73 100644 --- a/fs/netfs/direct_write.c +++ b/fs/netfs/direct_write.c @@ -21,7 +21,7 @@ static void netfs_unbuffered_write_done(struct netfs_io_request *wreq) /* Okay, declare that all I/O is complete. */ trace_netfs_rreq(wreq, netfs_rreq_trace_write_done); - if (!wreq->error) + if (wreq->transferred) netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred); if (wreq->origin == NETFS_DIO_WRITE && @@ -51,7 +51,7 @@ static void netfs_unbuffered_write_done(struct netfs_io_request *wreq) wreq->iocb->ki_pos += written; if (wreq->iocb->ki_complete) { trace_netfs_rreq(wreq, netfs_rreq_trace_ki_complete); - wreq->iocb->ki_complete(wreq->iocb, wreq->error ?: written); + wreq->iocb->ki_complete(wreq->iocb, written ?: wreq->error); } wreq->iocb = VFS_PTR_POISON; } @@ -95,7 +95,7 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) { struct netfs_io_subrequest *subreq = NULL; struct netfs_io_stream *stream = &wreq->io_streams[0]; - int ret; + int ret = 0; _enter("%llx", wreq->len); @@ -110,6 +110,11 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) if (!subreq) { netfs_prepare_write(wreq, stream, wreq->start + wreq->transferred); subreq = stream->construct; + if (!subreq) { + wreq->error = -ENOMEM; + ret = -ENOMEM; + break; + } stream->construct = NULL; } @@ -121,8 +126,14 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) } iov_iter_truncate(&subreq->io_iter, wreq->len - wreq->transferred); - if (!iov_iter_count(&subreq->io_iter)) + if (!iov_iter_count(&subreq->io_iter)) { + pr_warn("netfs: Unexpected zero-length iterator R=%08x\n", + wreq->debug_id); + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); + netfs_write_subrequest_terminated(subreq, -EIO); + wreq->error = -EIO; break; + } subreq->len = netfs_limit_iter(&subreq->io_iter, 0, stream->sreq_max_len, @@ -139,13 +150,11 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { retry = true; } else if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) { - ret = subreq->error; - wreq->error = ret; + wreq->error = subreq->error; netfs_see_subrequest(subreq, netfs_sreq_trace_see_failed); subreq = NULL; break; } - ret = 0; if (!retry) { netfs_unbuffered_write_collect(wreq, stream, subreq); @@ -288,11 +297,11 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter * ret = -EIOCBQUEUED; } else { ret = netfs_unbuffered_write(wreq); - if (ret < 0) { - _debug("begin = %zd", ret); - } else { + if (wreq->transferred) { iocb->ki_pos += wreq->transferred; - ret = wreq->transferred ?: wreq->error; + ret = wreq->transferred; + } else if (wreq->error) { + ret = wreq->error; } netfs_put_request(wreq, netfs_rreq_trace_put_complete); diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index 420ee7b26580f..c79c8e69d60ca 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -79,6 +79,7 @@ ssize_t netfs_wait_for_read(struct netfs_io_request *rreq); ssize_t netfs_wait_for_write(struct netfs_io_request *rreq); void netfs_wait_for_paused_read(struct netfs_io_request *rreq); void netfs_wait_for_paused_write(struct netfs_io_request *rreq); +void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq); /* * objects.c @@ -109,6 +110,8 @@ static inline void netfs_see_subrequest(struct netfs_io_subrequest *subreq, /* * read_collect.c */ +void netfs_cancel_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio); +void netfs_read_set_unlock_at(struct netfs_io_request *rreq); bool netfs_read_collection(struct netfs_io_request *rreq); void netfs_read_collection_worker(struct work_struct *work); void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error); diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c index 5d554512ed23a..f5c1c463f4ff7 100644 --- a/fs/netfs/misc.c +++ b/fs/netfs/misc.c @@ -563,3 +563,22 @@ void netfs_wait_for_paused_write(struct netfs_io_request *rreq) { return netfs_wait_for_pause(rreq, netfs_write_collection); } + +/* + * Wait for the readahead-acquired refs to be put. + */ +void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq) +{ + DEFINE_WAIT(myself); + + for (;;) { + trace_netfs_rreq(rreq, netfs_rreq_trace_wait_put_ra_refs); + prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE); + if (!test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags)) + break; + schedule(); + } + + trace_netfs_rreq(rreq, netfs_rreq_trace_waited_put_ra_refs); + finish_wait(&rreq->waitq, &myself); +} diff --git a/fs/netfs/objects.c b/fs/netfs/objects.c index 01461a74642d6..ad549daa9c794 100644 --- a/fs/netfs/objects.c +++ b/fs/netfs/objects.c @@ -34,31 +34,39 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, rreq = mempool_alloc(mempool, gfp); } else { - rreq = mempool->alloc(gfp, mempool->pool_data); + rreq = mempool_alloc_noreserve(mempool, gfp); if (!rreq) return ERR_PTR(-ENOMEM); } memset(rreq, 0, kmem_cache_size(cache)); INIT_WORK(&rreq->cleanup_work, netfs_free_request); - rreq->gfp = gfp; - rreq->start = start; - rreq->len = len; - rreq->origin = origin; - rreq->netfs_ops = ctx->ops; - rreq->mapping = mapping; - rreq->inode = inode; - rreq->i_size = i_size_read(inode); - rreq->debug_id = atomic_inc_return(&debug_ids); - rreq->wsize = INT_MAX; + rreq->gfp = gfp; + rreq->start = start; + rreq->collected_to = start; + rreq->cleaned_to = start; + rreq->len = len; + rreq->progress_at = 0; + rreq->origin = origin; + rreq->netfs_ops = ctx->ops; + rreq->mapping = mapping; + rreq->inode = inode; + rreq->i_size = i_size_read(inode); + rreq->debug_id = atomic_inc_return(&debug_ids); + rreq->wsize = INT_MAX; rreq->io_streams[0].sreq_max_len = ULONG_MAX; rreq->io_streams[0].sreq_max_segs = 0; spin_lock_init(&rreq->lock); - INIT_LIST_HEAD(&rreq->io_streams[0].subrequests); - INIT_LIST_HEAD(&rreq->io_streams[1].subrequests); init_waitqueue_head(&rreq->waitq); refcount_set(&rreq->ref, 2); + for (int s = 0; s < NR_IO_STREAMS; s++) { + struct netfs_io_stream *stream = &rreq->io_streams[s]; + + INIT_LIST_HEAD(&stream->subrequests); + stream->collected_to = rreq->start; + } + if (origin == NETFS_READAHEAD || origin == NETFS_READPAGE || origin == NETFS_READ_GAPS || @@ -206,7 +214,7 @@ struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq struct kmem_cache *cache = mempool->pool_data; if (rreq->gfp == GFP_KERNEL) - subreq = mempool->alloc(rreq->gfp, mempool->pool_data); + subreq = mempool_alloc_noreserve(mempool, rreq->gfp); else subreq = mempool_alloc(mempool, rreq->gfp); if (!subreq) diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c index 23660a5901246..a94197ef01811 100644 --- a/fs/netfs/read_collect.c +++ b/fs/netfs/read_collect.c @@ -19,7 +19,6 @@ #define MADE_PROGRESS 0x04 /* Made progress cleaning up a stream or the folio set */ #define BUFFERED 0x08 /* The pagecache needs cleaning up */ #define NEED_RETRY 0x10 /* A front op requests retrying */ -#define COPY_TO_CACHE 0x40 /* Need to copy subrequest to cache */ #define ABANDON_SREQ 0x80 /* Need to abandon untransferred part of subrequest */ /* @@ -34,6 +33,30 @@ static void netfs_clear_unread(struct netfs_io_subrequest *subreq) __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); } +/* + * Cancel the copy-to-cache mark on a folio. + */ +void netfs_cancel_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio) +{ + if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { + if (folio_get_private(folio) == NETFS_FOLIO_COPY_TO_CACHE) { + folio_detach_private(folio); + trace_netfs_folio(folio, netfs_folio_trace_cancel_copy); + } else if (netfs_folio_group(folio) == NETFS_FOLIO_COPY_TO_CACHE) { + struct netfs_folio *finfo = netfs_folio_info(folio); + + finfo->netfs_group = NULL; + trace_netfs_folio(folio, netfs_folio_trace_cancel_copy); + } + } else { + // TODO: Use of PG_private_2 is deprecated. + if (folio_test_private_2(folio)) { + folio_end_private_2(folio); + trace_netfs_folio(folio, netfs_folio_trace_cancel_copy); + } + } +} + /* * Flush, mark and unlock a folio that's now completely read. If we want to * cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it @@ -48,37 +71,37 @@ static void netfs_unlock_read_folio(struct netfs_io_request *rreq, if (unlikely(folio_pos(folio) < rreq->abandon_to)) { trace_netfs_folio(folio, netfs_folio_trace_abandon); + netfs_cancel_copy_to_cache(rreq, folio); goto just_unlock; } flush_dcache_folio(folio); folio_mark_uptodate(folio); - if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { - finfo = netfs_folio_info(folio); - if (finfo) { - trace_netfs_folio(folio, netfs_folio_trace_filled_gaps); - if (finfo->netfs_group) - folio_change_private(folio, finfo->netfs_group); - else - folio_detach_private(folio); - kfree(finfo); - } + if (unlikely(test_bit(NETFS_RREQ_CANCEL_CACHING, &rreq->flags))) + netfs_cancel_copy_to_cache(rreq, folio); - if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags)) { - if (!WARN_ON_ONCE(folio_get_private(folio) != NULL)) { - trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); - folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE); - folio_mark_dirty(folio); - } + if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { + if (netfs_folio_group(folio) == NETFS_FOLIO_COPY_TO_CACHE) { + trace_netfs_folio(folio, netfs_folio_trace_sched_copy); + folio_mark_dirty(folio); } else { + finfo = netfs_folio_info(folio); + if (finfo) { + trace_netfs_folio(folio, netfs_folio_trace_filled_gaps); + if (finfo->netfs_group) + folio_change_private(folio, finfo->netfs_group); + else + folio_detach_private(folio); + kfree(finfo); + } trace_netfs_folio(folio, netfs_folio_trace_read_done); } folioq_clear(folioq, slot); } else { // TODO: Use of PG_private_2 is deprecated. - if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags)) + if (folio_test_private_2(folio)) netfs_pgpriv2_copy_to_cache(rreq, folio); } @@ -94,6 +117,35 @@ static void netfs_unlock_read_folio(struct netfs_io_request *rreq, folioq_clear(folioq, slot); } +/* + * Determine how much to gather before unlocking more folios. + */ +void netfs_read_set_unlock_at(struct netfs_io_request *rreq) +{ + struct folio_queue *folioq = rreq->buffer.tail; + unsigned int slot = rreq->buffer.first_tail_slot; + size_t cleaned_to = rreq->cleaned_to - rreq->start; + size_t progress_at = cleaned_to; + size_t minimum = 256 * 1024; + + while (progress_at < rreq->len) { + if (slot >= folioq_count(folioq)) { + folioq = folioq->next; + if (!folioq) + break; + slot = 0; + } + + progress_at += folioq_folio_size(folioq, slot); + if (progress_at - cleaned_to >= minimum) + break; + slot++; + } + + WRITE_ONCE(rreq->progress_at, progress_at); + trace_netfs_read_progress_at(rreq); +} + /* * Unlock any folios we've finished with. */ @@ -112,30 +164,31 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq, if (slot >= folioq_nr_slots(folioq)) { folioq = rolling_buffer_delete_spent(&rreq->buffer); if (!folioq) { - rreq->front_folio_order = 0; + WRITE_ONCE(rreq->progress_at, rreq->len); return; } slot = 0; } + /* We have to wait for readahead refs to have been released before we + * can unlock any folios as the ref-dropper walks i_pages and the only + * thing preventing these folios from being removed is the folio lock. + */ + if (test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags)) + netfs_wait_for_put_ra_refs(rreq); + for (;;) { struct folio *folio; unsigned long long fpos, fend; - unsigned int order; size_t fsize; - if (*notes & COPY_TO_CACHE) - set_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags); - folio = folioq_folio(folioq, slot); if (WARN_ONCE(!folio_test_locked(folio), "R=%08x: folio %lx is not locked\n", rreq->debug_id, folio->index)) trace_netfs_folio(folio, netfs_folio_trace_not_locked); - order = folioq_folio_order(folioq, slot); - rreq->front_folio_order = order; - fsize = PAGE_SIZE << order; + fsize = folioq_folio_size(folioq, slot); fpos = folio_pos(folio); fend = fpos + fsize; @@ -149,8 +202,6 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq, WRITE_ONCE(rreq->cleaned_to, fpos + fsize); *notes |= MADE_PROGRESS; - clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags); - /* Clean up the head folioq. If we clear an entire folioq, then * we can get rid of it provided it's not also the tail folioq * being filled by the issuer. @@ -172,6 +223,8 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq, rreq->buffer.tail = folioq; done: rreq->buffer.first_tail_slot = slot; + + netfs_read_set_unlock_at(rreq); } /* @@ -232,7 +285,7 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq) * subreqs. */ if (notes & BUFFERED) { - size_t fsize = PAGE_SIZE << rreq->front_folio_order; + uoff_t unlock_at = rreq->start + rreq->progress_at; /* Clear the tail of a short read. */ if (!(notes & HIT_PENDING) && @@ -248,16 +301,13 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq) stream->collected_to = front->start + transferred; rreq->collected_to = stream->collected_to; - if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &front->flags)) - notes |= COPY_TO_CACHE; - if (test_bit(NETFS_SREQ_FAILED, &front->flags)) { rreq->abandon_to = front->start + front->len; front->transferred = front->len; transferred = front->len; trace_netfs_rreq(rreq, netfs_rreq_trace_set_abandon); } - if (front->start + transferred >= rreq->cleaned_to + fsize || + if (front->start + transferred >= unlock_at || test_bit(NETFS_SREQ_HIT_EOF, &front->flags)) netfs_read_unlock_folios(rreq, ¬es); } else { @@ -385,6 +435,11 @@ static void netfs_rreq_assess_single(struct netfs_io_request *rreq) netfs_single_mark_inode_dirty(rreq->inode); } + /* To do DIO, the cache has to round the size up, so we need to undo + * the rounding. + */ + rreq->transferred = min(rreq->transferred, rreq->i_size); + if (rreq->iocb) { rreq->iocb->ki_pos += rreq->transferred; if (rreq->iocb->ki_complete) { @@ -477,20 +532,22 @@ void netfs_read_collection_worker(struct work_struct *work) void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq) { struct netfs_io_request *rreq = subreq->rreq; - struct netfs_io_stream *stream = &rreq->io_streams[0]; - size_t fsize = PAGE_SIZE << rreq->front_folio_order; - - trace_netfs_sreq(subreq, netfs_sreq_trace_progress); + struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr]; + size_t progress_at = READ_ONCE(rreq->progress_at); + uoff_t update_at = rreq->start + progress_at; + uoff_t transferred_to = subreq->start + subreq->transferred; /* If we are at the head of the queue, wake up the collector, * getting a ref to it if we were the ones to do so. */ - if (subreq->start + subreq->transferred > rreq->cleaned_to + fsize && + if (progress_at < rreq->len && + transferred_to >= update_at && (rreq->origin == NETFS_READAHEAD || rreq->origin == NETFS_READPAGE || rreq->origin == NETFS_READ_FOR_WRITE) && list_is_first(&subreq->rreq_link, &stream->subrequests) ) { + trace_netfs_sreq(subreq, netfs_sreq_trace_progress); __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); netfs_wake_collector(rreq); } diff --git a/fs/netfs/read_pgpriv2.c b/fs/netfs/read_pgpriv2.c index c31190993b762..a4b7bb88cbdb6 100644 --- a/fs/netfs/read_pgpriv2.c +++ b/fs/netfs/read_pgpriv2.c @@ -54,8 +54,8 @@ static void netfs_pgpriv2_copy_folio(struct netfs_io_request *creq, struct folio /* Attach the folio to the rolling buffer. */ if (rolling_buffer_append(&creq->buffer, folio, 0, creq->gfp) < 0) { + set_bit(NETFS_RREQ_CANCEL_CACHING, &creq->flags); folio_end_private_2(folio); - clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &creq->flags); return; } @@ -122,13 +122,14 @@ static struct netfs_io_request *netfs_pgpriv2_begin_copy_to_cache( netfs_put_failed_request(creq); cancel: rreq->copy_to_cache = ERR_PTR(-ENOBUFS); - clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags); + set_bit(NETFS_RREQ_CANCEL_CACHING, &rreq->flags); return ERR_PTR(-ENOBUFS); } /* * [DEPRECATED] Mark page as requiring copy-to-cache using PG_private_2 and add - * it to the copy write request. + * it to the copy write request. PG_private_2 should already be set on the + * folio. */ void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio) { @@ -136,11 +137,13 @@ void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *fo if (!creq) creq = netfs_pgpriv2_begin_copy_to_cache(rreq, folio); - if (IS_ERR(creq)) + if (IS_ERR(creq)) { + set_bit(NETFS_RREQ_CANCEL_CACHING, &rreq->flags); + netfs_cancel_copy_to_cache(rreq, folio); return; + } - trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); - folio_start_private_2(folio); + trace_netfs_folio(folio, netfs_folio_trace_pgpriv2_copy); netfs_pgpriv2_copy_folio(creq, folio); } diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c index 2b42758e01ec9..4f6a36c6e214f 100644 --- a/fs/netfs/read_retry.c +++ b/fs/netfs/read_retry.c @@ -292,11 +292,22 @@ void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) { struct folio_queue *p; + /* We have to wait for readahead refs to have been released before we + * can unlock any folios as the ref-dropper walks i_pages and the only + * thing preventing these folios from being removed is the folio lock. + */ + if (test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags)) + netfs_wait_for_put_ra_refs(rreq); + for (p = rreq->buffer.tail; p; p = p->next) { for (int slot = 0; slot < folioq_count(p); slot++) { struct folio *folio = folioq_folio(p, slot); - if (folio && !folioq_is_marked2(p, slot)) { + if (!folio) + continue; + netfs_cancel_copy_to_cache(rreq, folio); + + if (!folioq_is_marked2(p, slot)) { if (folio == rreq->no_unlock_folio && test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) { diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c index 8833550d2eb60..de67ac41548d1 100644 --- a/fs/netfs/read_single.c +++ b/fs/netfs/read_single.c @@ -170,6 +170,8 @@ ssize_t netfs_read_single(struct inode *inode, struct file *file, struct iov_ite if (IS_ERR(rreq)) return PTR_ERR(rreq); + rreq->progress_at = rreq->len; + ret = netfs_single_begin_cache_read(rreq, ictx); if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) goto cleanup_free; diff --git a/fs/netfs/rolling_buffer.c b/fs/netfs/rolling_buffer.c index 8c0026836f9c1..d30d5ef6d86ec 100644 --- a/fs/netfs/rolling_buffer.c +++ b/fs/netfs/rolling_buffer.c @@ -29,7 +29,7 @@ struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp, struct folio_queue *fq; if (gfp == GFP_KERNEL) - fq = netfs_folioq_pool.alloc(gfp, netfs_folioq_pool.pool_data); + fq = mempool_alloc_noreserve(&netfs_folioq_pool, gfp); else fq = mempool_alloc(&netfs_folioq_pool, gfp); if (fq) { @@ -115,42 +115,65 @@ int rolling_buffer_make_space(struct rolling_buffer *roll, gfp_t gfp) } /* - * Decant the list of folios to read into a rolling buffer. + * Decant the entire list of folios to read into a rolling buffer. */ -ssize_t rolling_buffer_load_from_ra(struct rolling_buffer *roll, - struct readahead_control *ractl, - struct folio_batch *put_batch) +ssize_t rolling_buffer_bulk_load_from_ra(struct rolling_buffer *roll, + struct readahead_control *ractl, + unsigned int rreq_id, gfp_t gfp) { struct folio_queue *fq; - struct page **vec; - int nr, ix, to; - ssize_t size = 0; + ssize_t loaded = 0; - if (rolling_buffer_make_space(roll, GFP_KERNEL) < 0) - return -ENOMEM; + while (ractl->_nr_pages - ractl->_batch_count > 0) { + unsigned int nr; - fq = roll->head; - vec = (struct page **)fq->vec.folios; - nr = __readahead_batch(ractl, vec + folio_batch_count(&fq->vec), - folio_batch_space(&fq->vec)); - ix = fq->vec.nr; - to = ix + nr; - fq->vec.nr = to; - for (; ix < to; ix++) { - struct folio *folio = folioq_folio(fq, ix); - unsigned int order = folio_order(folio); - - fq->orders[ix] = order; - size += PAGE_SIZE << order; - trace_netfs_folio(folio, netfs_folio_trace_read); - if (!folio_batch_add(put_batch, folio)) - folio_batch_release(put_batch); + /* Allocate a folioq to put some folios into and attach it to + * the rolling buffer. + */ + fq = netfs_folioq_alloc(rreq_id, gfp, + netfs_trace_folioq_make_space); + if (!fq) + goto nomem_unlock; + fq->prev = roll->head; + if (!roll->tail) + roll->tail = fq; + else + roll->head->next = fq; + roll->head = fq; + + /* Get a batch of folios and note their orders. */ + nr = __readahead_batch(ractl, (struct page **)fq->vec.folios, + folioq_nr_slots(fq)); + if (WARN_ON_ONCE(!nr)) + break; + fq->vec.nr = nr; + + for (int slot = 0; slot < nr; slot++) { + struct folio *folio = folioq_folio(fq, slot); + unsigned int order; + + order = folio_order(folio); + fq->orders[slot] = order; + loaded += PAGE_SIZE << order; + trace_netfs_folio(folio, netfs_folio_trace_read); + } } - WRITE_ONCE(roll->iter.count, roll->iter.count + size); - /* Store the counter after setting the slot. */ - smp_store_release(&roll->next_head_slot, to); - return size; + WRITE_ONCE(roll->iter.count, loaded); + iov_iter_folio_queue(&roll->iter, ITER_DEST, roll->tail, 0, 0, loaded); + return loaded; + +nomem_unlock: + for (fq = roll->tail; fq; fq = fq->next) { + for (int slot = 0; slot < folioq_count(fq); slot++) { + folio_unlock(fq->vec.folios[slot]); + folioq_mark(fq, slot); + } + } + rolling_buffer_clear(roll); + roll->head = NULL; + roll->tail = NULL; + return -ENOMEM; } /* diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index 2d9cfcd43658f..851f6f93ad45a 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -170,6 +170,8 @@ void netfs_prepare_write(struct netfs_io_request *wreq, rolling_buffer_make_space(&wreq->buffer, wreq->gfp); subreq = netfs_alloc_subrequest(wreq); + if (!subreq) + return; subreq->source = stream->source; subreq->start = start; subreq->stream_nr = stream->stream_nr; diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c index bb35f88501cea..db4bb0a6283e5 100644 --- a/fs/nfs/blocklayout/dev.c +++ b/fs/nfs/blocklayout/dev.c @@ -85,15 +85,17 @@ bl_free_device(struct pnfs_block_dev *dev) { bl_unregister_dev(dev); - if (dev->nr_children) { + if (dev->children) { int i; for (i = 0; i < dev->nr_children; i++) bl_free_device(&dev->children[i]); kfree(dev->children); - } else { - if (dev->bdev_file) - fput(dev->bdev_file); + dev->children = NULL; + dev->nr_children = 0; + } else if (dev->bdev_file) { + fput(dev->bdev_file); + dev->bdev_file = NULL; } } @@ -437,6 +439,7 @@ bl_parse_scsi(struct nfs_server *server, struct pnfs_block_dev *d, out_blkdev_put: fput(d->bdev_file); + d->bdev_file = NULL; return error; } @@ -472,8 +475,11 @@ bl_parse_concat(struct nfs_server *server, struct pnfs_block_dev *d, for (i = 0; i < v->concat.volumes_count; i++) { ret = bl_parse_deviceid(server, &d->children[i], volumes, v->concat.volumes[i], gfp_mask); - if (ret) + if (ret) { + bl_free_device(&d->children[i]); + bl_free_device(d); return ret; + } d->nr_children++; d->children[i].start += len; @@ -501,8 +507,11 @@ bl_parse_stripe(struct nfs_server *server, struct pnfs_block_dev *d, for (i = 0; i < v->stripe.volumes_count; i++) { ret = bl_parse_deviceid(server, &d->children[i], volumes, v->stripe.volumes[i], gfp_mask); - if (ret) + if (ret) { + bl_free_device(&d->children[i]); + bl_free_device(d); return ret; + } d->nr_children++; len += d->children[i].len; diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c index ff4e9fd38e83f..bc282b744f34f 100644 --- a/fs/nfs/callback.c +++ b/fs/nfs/callback.c @@ -231,8 +231,9 @@ int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt) cb_info->users++; err_net: if (!cb_info->users) { + xprt_svc_shutdown_bc(xprt); svc_set_num_threads(cb_info->serv, 0, 0); - svc_destroy(&cb_info->serv); + xprt_svc_destroy_nullify_bc(xprt, &cb_info->serv); } err_create: mutex_unlock(&nfs_callback_mutex); @@ -254,6 +255,7 @@ void nfs_callback_down(int minorversion, struct net *net, struct rpc_xprt *xprt) mutex_lock(&nfs_callback_mutex); serv = cb_info->serv; + xprt_svc_shutdown_bc(xprt); nfs_callback_down_net(minorversion, serv, net); cb_info->users--; if (cb_info->users == 0) { diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c index 4382baddc9eed..a6406707ede6c 100644 --- a/fs/nfs/callback_xdr.c +++ b/fs/nfs/callback_xdr.c @@ -417,7 +417,7 @@ static __be32 decode_cb_sequence_args(struct svc_rqst *rqstp, args->csa_nrclists = ntohl(*p++); args->csa_rclists = NULL; if (args->csa_nrclists) { - args->csa_rclists = kmalloc_objs(*args->csa_rclists, + args->csa_rclists = kzalloc_objs(*args->csa_rclists, args->csa_nrclists); if (unlikely(args->csa_rclists == NULL)) return htonl(NFS4ERR_RESOURCE); diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 4dcb91ab30397..60386330aeeca 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -215,9 +215,21 @@ static void nfs_cb_idr_remove_locked(struct nfs_client *clp) { struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); - if (clp->cl_cb_ident) + if (clp->cl_cb_ident) { idr_remove(&nn->cb_ident_idr, clp->cl_cb_ident); + clp->cl_cb_ident = 0; + } +} + +void nfs_cb_idr_remove(struct nfs_client *clp) +{ + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + + spin_lock(&nn->nfs_client_lock); + nfs_cb_idr_remove_locked(clp); + spin_unlock(&nn->nfs_client_lock); } +EXPORT_SYMBOL_GPL(nfs_cb_idr_remove); static void pnfs_init_server(struct nfs_server *server) { diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 9546d2195c25f..ab3e441464a66 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -447,11 +447,14 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred, struct nfs_inode *nfsi = NFS_I(inode); struct nfs_delegation *delegation, *old_delegation; struct nfs_delegation *freeme = NULL; + bool orphaned = false; int status = 0; delegation = kmalloc_obj(*delegation, GFP_KERNEL_ACCOUNT); - if (delegation == NULL) + if (delegation == NULL) { + nfs4_proc_delegreturn(inode, cred, stateid, NULL, 0); return -ENOMEM; + } nfs4_stateid_copy(&delegation->stateid, stateid); refcount_set(&delegation->refcount, 1); delegation->type = type; @@ -500,11 +503,15 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred, goto out; } if (test_and_set_bit(NFS_DELEGATION_RETURNING, - &old_delegation->flags)) + &old_delegation->flags)) { + orphaned = true; goto out; + } } - if (!nfs_detach_delegations_locked(nfsi, old_delegation, clp)) + if (!nfs_detach_delegations_locked(nfsi, old_delegation, clp)) { + orphaned = true; goto out; + } freeme = old_delegation; add_new: /* @@ -539,8 +546,11 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred, nfs_update_delegated_mtime(inode); out: spin_unlock(&clp->cl_lock); - if (delegation != NULL) + if (delegation != NULL) { + if (orphaned) + nfs_do_return_delegation(inode, delegation, 0); __nfs_free_delegation(delegation); + } if (freeme != NULL) { nfs_do_return_delegation(inode, freeme, 0); nfs_mark_delegation_revoked(server, freeme); @@ -594,7 +604,7 @@ static int nfs_end_delegation_return(struct inode *inode, spin_lock(&server->delegations_lock); if (list_empty(&delegation->entry)) refcount_inc(&delegation->refcount); - list_move_tail(&delegation->entry, &server->delegations_return); + list_move_tail(&delegation->entry, &server->delegations_delayed); spin_unlock(&server->delegations_lock); set_bit(NFS4CLNT_DELEGRETURN_DELAYED, &server->nfs_client->cl_state); abort: diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index c7caffb31935c..e97ce5240c18e 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2319,6 +2319,13 @@ int nfs_atomic_open_v23(struct inode *dir, struct dentry *dentry, if (open_flags & O_CREAT) { error = nfs_do_create(dir, dentry, mode, open_flags); if (!error) { + /* With UNCHECKED mode, a server may return NFS3_OK for + * a pre-existing non-regular file (e.g. a symlink). + * Let the VFS handle it; calling finish_open() would + * hit no_open() and return -ENXIO. + */ + if (!d_is_reg(dentry)) + return finish_no_open(file, NULL); file->f_mode |= FMODE_CREATED; return finish_open(file, dentry, NULL); } else if (error != -EEXIST || open_flags & O_EXCL) diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c index 7226989ee4d53..9ddcaf86fcef0 100644 --- a/fs/nfs/filelayout/filelayoutdev.c +++ b/fs/nfs/filelayout/filelayoutdev.c @@ -170,7 +170,8 @@ nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, goto out_err_free_deviceid; } - dsaddr->ds_list[i] = nfs4_pnfs_ds_add(net, &dsaddrs, gfp_flags); + dsaddr->ds_list[i] = nfs4_pnfs_ds_add(net, &dsaddrs, 4, + gfp_flags); if (!dsaddr->ds_list[i]) goto out_err_drain_dsaddrs; trace_fl_getdevinfo(server, &pdev->dev_id, dsaddr->ds_list[i]->ds_remotestr); diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index c4aa995026f67..ef26fcab9c103 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -1322,7 +1322,8 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task, struct pnfs_layout_hdr *lo = lseg->pls_layout; struct inode *inode = lo->plh_inode; struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); - struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table; + struct nfs4_slot_table *tbl = nfs4_has_session(clp) ? + &clp->cl_session->fc_slot_table : clp->cl_slot_tbl; switch (op_status) { case NFS4_OK: diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c index 1109462a96991..c716fd99e8ef9 100644 --- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c +++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c @@ -153,7 +153,8 @@ nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, new_ds->ds_versions = ds_versions; new_ds->ds_versions_cnt = version_count; - new_ds->ds = nfs4_pnfs_ds_add(net, &dsaddrs, gfp_flags); + new_ds->ds = nfs4_pnfs_ds_add(net, &dsaddrs, ds_versions[0].version, + gfp_flags); if (!new_ds->ds) goto out_err_drain_dsaddrs; diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 5bcd4027d2033..e538736bb165a 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -1531,9 +1531,7 @@ int nfs_clear_invalid_mapping(struct address_space *mapping) ret = nfs_invalidate_mapping(inode, mapping); trace_nfs_invalidate_mapping_exit(inode, ret); - clear_bit_unlock(NFS_INO_INVALIDATING, bitlock); - smp_mb__after_atomic(); - wake_up_bit(bitlock, NFS_INO_INVALIDATING); + clear_and_wake_up_bit(NFS_INO_INVALIDATING, bitlock); out: return ret; } diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index e4533f5836321..864fa092bcead 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -225,6 +225,7 @@ void nfs_server_copy_userdata(struct nfs_server *, struct nfs_server *); extern void nfs_put_client(struct nfs_client *); extern void nfs_free_client(struct nfs_client *); +void nfs_cb_idr_remove(struct nfs_client *clp); extern struct nfs_client *nfs4_find_client_ident(struct net *, int); extern struct nfs_client * nfs4_find_client_sessionid(struct net *, const struct sockaddr *, diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index 71c271a1700a9..5bc6dc9c1135a 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -261,6 +261,7 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init) return clp; error: + nfs_cb_idr_remove(clp); nfs_free_client(clp); return ERR_PTR(err); } @@ -915,20 +916,22 @@ static int nfs4_server_common_setup(struct nfs_server *server, return error; /* data servers support only a subset of NFSv4.1 */ - if (is_ds_only_client(server->nfs_client)) - return -EPROTONOSUPPORT; + if (is_ds_only_client(server->nfs_client)) { + error = -EPROTONOSUPPORT; + goto out_free_delegation_hash; + } /* We must ensure the session is initialised first */ error = nfs4_init_session(server->nfs_client); if (error < 0) - return error; + goto out_free_delegation_hash; nfs_server_set_init_caps(server); /* Probe the root fh to retrieve its FSID and filehandle */ error = nfs4_get_rootfh(server, mntfh, auth_probe); if (error < 0) - return error; + goto out_free_delegation_hash; dprintk("Server FSID: %llx:%llx\n", (unsigned long long) server->fsid.major, @@ -937,7 +940,7 @@ static int nfs4_server_common_setup(struct nfs_server *server, error = nfs_probe_server(server, mntfh); if (error < 0) - return error; + goto out_free_delegation_hash; nfs4_session_limit_rwsize(server); nfs4_session_limit_xasize(server); @@ -949,6 +952,11 @@ static int nfs4_server_common_setup(struct nfs_server *server, server->mount_time = jiffies; server->destroy = nfs4_destroy_server; return 0; + +out_free_delegation_hash: + kfree(server->delegation_hash_table); + server->delegation_hash_table = NULL; + return error; } /* diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 5709c6fea85bb..2edd16e4981a2 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -7762,6 +7762,7 @@ static int nfs4_add_lease(struct file *file, int arg, struct file_lease **lease, { struct inode *inode = file_inode(file); fmode_t type = arg == F_RDLCK ? FMODE_READ : FMODE_WRITE; + fl_owner_t owner = (*lease)->c.flc_owner; int ret; /* No delegation, no lease */ @@ -7771,7 +7772,8 @@ static int nfs4_add_lease(struct file *file, int arg, struct file_lease **lease, if (ret || nfs4_have_delegation(inode, type, 0)) return ret; /* We raced with a delegation return */ - nfs4_delete_lease(file, priv); + dprintk("%s: raced with a delegation return\n", __func__); + nfs4_delete_lease(file, &owner); return -EAGAIN; } diff --git a/fs/nfs/nfs4session.c b/fs/nfs/nfs4session.c index 5c128957a0a41..993f0db7cf5e4 100644 --- a/fs/nfs/nfs4session.c +++ b/fs/nfs/nfs4session.c @@ -632,16 +632,22 @@ int nfs4_init_ds_session(struct nfs_client *clp, unsigned long lease_time) int ret; spin_lock(&clp->cl_lock); - if (test_and_clear_bit(NFS4_SESSION_INITING, &session->session_state)) { - /* - * Do not set NFS_CS_CHECK_LEASE_TIME instead set the - * DS lease to be equal to the MDS lease. - */ + /* + * Do not set NFS_CS_CHECK_LEASE_TIME instead set the + * DS lease to be equal to the MDS lease. + * + * A v4.0 DS has no session, so seed the lease every time. + */ + if (!session || + test_and_clear_bit(NFS4_SESSION_INITING, &session->session_state)) { clp->cl_lease_time = lease_time; clp->cl_last_renewal = jiffies; } spin_unlock(&clp->cl_lock); + if (!session) + return 0; + ret = nfs41_check_session_ready(clp); if (ret) return ret; diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 7715e2bd5871b..fe5af86f9c486 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1389,7 +1389,7 @@ pnfs_layout_need_return(struct pnfs_layout_hdr *lo) return false; return pnfs_mark_layout_stateid_return(lo, &lo->plh_return_segs, lo->plh_return_iomode, - lo->plh_return_seq) != EBUSY; + lo->plh_return_seq) != -EBUSY; } static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) @@ -2100,15 +2100,6 @@ static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); } -static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) -{ - unsigned long *bitlock = &lo->plh_flags; - - clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); - smp_mb__after_atomic(); - wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); -} - static void _add_to_server_list(struct pnfs_layout_hdr *lo, struct nfs_server *server) { @@ -2284,7 +2275,8 @@ pnfs_update_layout(struct inode *ino, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_INVALID_OPEN); nfs4_schedule_stateid_recovery(server, ctx->state); - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, + &lo->plh_flags); pnfs_put_layout_hdr(lo); goto lookup_again; } @@ -2353,7 +2345,8 @@ pnfs_update_layout(struct inode *ino, if (!exception.retry) goto out_put_layout_hdr; if (first) - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, + &lo->plh_flags); trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); pnfs_put_layout_hdr(lo); @@ -2365,7 +2358,7 @@ pnfs_update_layout(struct inode *ino, out_put_layout_hdr: if (first) - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_EXIT); pnfs_put_layout_hdr(lo); @@ -2457,7 +2450,7 @@ static void _lgopen_prepare_attached(struct nfs4_opendata *data, lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, nfs_io_gfp_mask()); if (!lgp) { - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); nfs_layoutget_end(lo); pnfs_put_layout_hdr(lo); return; @@ -2561,7 +2554,8 @@ void nfs4_lgopen_release(struct nfs4_layoutget *lgp) { if (lgp != NULL) { if (lgp->lo) { - pnfs_clear_first_layoutget(lgp->lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, + &lgp->lo->plh_flags); nfs_layoutget_end(lgp->lo); } pnfs_layoutget_free(lgp); @@ -2643,6 +2637,7 @@ pnfs_layout_process(struct nfs4_layoutget *lgp) spin_unlock(&ino->i_lock); lseg->pls_layout = lo; NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); + pnfs_free_lseg_list(&free_me); return ERR_PTR(-EAGAIN); } @@ -3273,15 +3268,6 @@ pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) } EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); -static void pnfs_clear_layoutcommitting(struct inode *inode) -{ - unsigned long *bitlock = &NFS_I(inode)->flags; - - clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); - smp_mb__after_atomic(); - wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); -} - /* * There can be multiple RW segments. */ @@ -3306,7 +3292,7 @@ static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *lis pnfs_put_lseg(lseg); } - pnfs_clear_layoutcommitting(inode); + clear_and_wake_up_bit(NFS_INO_LAYOUTCOMMITTING, &NFS_I(inode)->flags); } void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) @@ -3446,7 +3432,7 @@ pnfs_layoutcommit_inode(struct inode *inode, bool sync) spin_unlock(&inode->i_lock); kfree(data); clear_layoutcommitting: - pnfs_clear_layoutcommitting(inode); + clear_and_wake_up_bit(NFS_INO_LAYOUTCOMMITTING, &NFS_I(inode)->flags); goto out; } EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index eb39859c216c9..f06a03ac4fa37 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -63,6 +63,7 @@ struct nfs4_pnfs_ds { const struct net *ds_net; struct nfs_client *ds_clp; refcount_t ds_count; + u32 ds_version; /* cache key, with ds_addrs */ unsigned long ds_state; #define NFS4DS_CONNECTING 0 /* ds is establishing connection */ }; @@ -417,7 +418,7 @@ void pnfs_generic_write_commit_done(struct rpc_task *task, void *data); void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds); struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, - gfp_t gfp_flags); + u32 version, gfp_t gfp_flags); void nfs4_pnfs_v3_ds_connect_unload(void); int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, struct nfs4_deviceid_node *devid, unsigned int timeo, diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 648c95b78eeab..49c621cde43a8 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -603,15 +603,17 @@ _same_data_server_addrs_locked(const struct list_head *dsaddrs1, } /* - * Lookup DS by addresses. nfs4_ds_cache_lock is held + * Lookup DS by addresses and NFS version. nfs4_ds_cache_lock is held */ static struct nfs4_pnfs_ds * -_data_server_lookup_locked(const struct nfs_net *nn, const struct list_head *dsaddrs) +_data_server_lookup_locked(const struct nfs_net *nn, + const struct list_head *dsaddrs, u32 version) { struct nfs4_pnfs_ds *ds; list_for_each_entry(ds, &nn->nfs4_data_server_cache, ds_node) - if (_same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs)) + if (ds->ds_version == version && + _same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs)) return ds; return NULL; } @@ -719,7 +721,8 @@ nfs4_pnfs_remotestr(struct list_head *dsaddrs, gfp_t gfp_flags) * uncached and return cached struct nfs4_pnfs_ds. */ struct nfs4_pnfs_ds * -nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_flags) +nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, u32 version, + gfp_t gfp_flags) { struct nfs_net *nn = net_generic(net, nfs_net_id); struct nfs4_pnfs_ds *tmp_ds, *ds = NULL; @@ -738,7 +741,7 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_fla remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags); spin_lock(&nn->nfs4_data_server_lock); - tmp_ds = _data_server_lookup_locked(nn, dsaddrs); + tmp_ds = _data_server_lookup_locked(nn, dsaddrs, version); if (tmp_ds == NULL) { INIT_LIST_HEAD(&ds->ds_addrs); list_splice_init(dsaddrs, &ds->ds_addrs); @@ -747,6 +750,7 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_fla INIT_LIST_HEAD(&ds->ds_node); ds->ds_net = net; ds->ds_clp = NULL; + ds->ds_version = version; list_add(&ds->ds_node, &nn->nfs4_data_server_cache); dprintk("%s add new data server %s\n", __func__, ds->ds_remotestr); diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c index dd715cdb6c043..85aa03a7b020c 100644 --- a/fs/nfs_common/nfslocalio.c +++ b/fs/nfs_common/nfslocalio.c @@ -292,8 +292,22 @@ struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *uuid, localio = nfs_to->nfsd_open_local_fh(net, uuid->dom, rpc_clnt, cred, nfs_fh, pnf, fmode); if (!IS_ERR(localio) && nfs_uuid_add_file(uuid, nfl) < 0) { - /* Delete the cached file when racing with nfs_uuid_put() */ + /* + * Delete the cached file when racing with nfs_uuid_put(). + * Since nfl->nfs_uuid was never published via + * rcu_assign_pointer(), nfs_close_local_fh() will early-return + * and cannot clean up after us. Drop the slot's file ref and + * its paired net ref, then drop the caller-owned nfsd_file ref + * (+1) and the entry-time nfsd_net ref carried via nf->nf_net, + * and return -ENXIO so the caller never dereferences the + * now-cleared localio. + */ + struct nfsd_file __rcu *tmp = + (struct nfsd_file __force __rcu *)localio; + nfs_to_nfsd_file_put_local(pnf); + nfs_to_nfsd_file_put_local(&tmp); + localio = ERR_PTR(-ENXIO); } nfs_to_nfsd_net_put(net); diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index a47c90f40422b..1f678583f6128 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -1006,7 +1006,8 @@ static int nfsd_nl_parse_one_export(struct cache_detail *cd, goto out_uuid; err = 0; - nfsd4_setup_layout_type(&exp); + if (exp.ex_flags & NFSEXP_PNFS) + nfsd4_setup_layout_type(&exp); } expp = svc_export_lookup(&exp); diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 24511c3208db9..1ea2bfd518255 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -55,6 +55,17 @@ /* We only care about NFSD_MAY_READ/WRITE for this cache */ #define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO) +/* If the shrinker runs between calls to list_lru_walk_node() in + * nfsd_file_gc(), the "remaining" count will be wrong. This could + * result in premature freeing of some files. This may not matter much + * but is easy to fix with this spinlock which temporarily disables + * the shrinker. + * + * It also serializes callers of nfsd_file_dispose_list_delayed() + * against per-net shutdown. + */ +static DEFINE_SPINLOCK(nfsd_gc_lock); + static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits); static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions); static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations); @@ -62,11 +73,6 @@ static DEFINE_PER_CPU(unsigned long, nfsd_file_releases); static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age); static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions); -struct nfsd_fcache_disposal { - spinlock_t lock; - struct list_head freeme; -}; - static struct kmem_cache *nfsd_file_slab; static struct kmem_cache *nfsd_file_mark_slab; static struct list_lru nfsd_file_lru; @@ -327,8 +333,11 @@ static void nfsd_file_lru_add(struct nfsd_file *nf) refcount_inc(&nf->nf_ref); if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru)) trace_nfsd_file_lru_add(nf); - else - WARN_ON(1); + else { + refcount_dec(&nf->nf_ref); + WARN_ON_ONCE(1); + return; + } nfsd_file_schedule_laundrette(); } @@ -419,25 +428,31 @@ nfsd_file_dispose_list(struct list_head *dispose) } /** - * nfsd_file_dispose_list_delayed - move list of dead files to net's freeme list + * nfsd_file_dispose_list_delayed - queue dead files for nfsd thread disposal * @dispose: list of nfsd_files to be disposed * - * Transfers each file to the "freeme" list for its nfsd_net, to eventually - * be disposed of by the per-net garbage collector. + * Transfers each file to the dispose list in its nfsd_net and wakes an nfsd + * thread to do the actual close. This keeps the cost of fput() in the nfsd + * threads rather than in the shrinker or GC worker. + * + * All callers must hold nfsd_gc_lock, so that nfsd_file_cache_shutdown_net() + * can synchronize against them before draining the per-net dispose list. + * This guarantees nf_net is still live when we call net_generic(). */ static void nfsd_file_dispose_list_delayed(struct list_head *dispose) { - while(!list_empty(dispose)) { + lockdep_assert_held(&nfsd_gc_lock); + + while (!list_empty(dispose)) { struct nfsd_file *nf = list_first_entry(dispose, struct nfsd_file, nf_gc); struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id); - struct nfsd_fcache_disposal *l = nn->fcache_disposal; struct svc_serv *serv; - spin_lock(&l->lock); - list_move_tail(&nf->nf_gc, &l->freeme); - spin_unlock(&l->lock); + spin_lock(&nn->fcache_dispose_lock); + list_move_tail(&nf->nf_gc, &nn->fcache_dispose_list); + spin_unlock(&nn->fcache_dispose_lock); /* * The filecache laundrette is shut down after the @@ -461,21 +476,28 @@ nfsd_file_dispose_list_delayed(struct list_head *dispose) */ void nfsd_file_net_dispose(struct nfsd_net *nn) { - struct nfsd_fcache_disposal *l = nn->fcache_disposal; - - if (!list_empty(&l->freeme)) { + if (!list_empty(&nn->fcache_dispose_list)) { LIST_HEAD(dispose); int i; - spin_lock(&l->lock); - for (i = 0; i < 8 && !list_empty(&l->freeme); i++) - list_move(l->freeme.next, &dispose); - spin_unlock(&l->lock); - if (!list_empty(&l->freeme)) - /* Wake up another thread to share the work + spin_lock(&nn->fcache_dispose_lock); + for (i = 0; i < 8 && !list_empty(&nn->fcache_dispose_list); i++) + list_move(nn->fcache_dispose_list.next, &dispose); + spin_unlock(&nn->fcache_dispose_lock); + if (!list_empty(&nn->fcache_dispose_list)) { + /* + * Wake up another thread to share the work * *before* doing any actual disposing. + * + * The filecache laundrette is shut down after + * the nn->nfsd_serv pointer is cleared, but + * before the svc_serv is freed. */ - svc_wake_up(nn->nfsd_serv); + struct svc_serv *serv = nn->nfsd_serv; + + if (serv) + svc_wake_up(serv); + } nfsd_file_dispose_list(&dispose); } } @@ -552,13 +574,6 @@ nfsd_file_gc_cb(struct list_head *item, struct list_lru_one *lru, return nfsd_file_lru_cb(item, lru, arg); } -/* If the shrinker runs between calls to list_lru_walk_node() in - * nfsd_file_gc(), the "remaining" count will be wrong. This could - * result in premature freeing of some files. This may not matter much - * but is easy to fix with this spinlock which temporarily disables - * the shrinker. - */ -static DEFINE_SPINLOCK(nfsd_gc_lock); static void nfsd_file_gc(void) { @@ -581,9 +596,9 @@ nfsd_file_gc(void) remaining = 0; } } + nfsd_file_dispose_list_delayed(&dispose); spin_unlock(&nfsd_gc_lock); trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru)); - nfsd_file_dispose_list_delayed(&dispose); } static void @@ -611,9 +626,9 @@ nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc) ret = list_lru_shrink_walk(&nfsd_file_lru, sc, nfsd_file_lru_cb, &dispose); + nfsd_file_dispose_list_delayed(&dispose); spin_unlock(&nfsd_gc_lock); trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru)); - nfsd_file_dispose_list_delayed(&dispose); return ret; } @@ -686,11 +701,11 @@ nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose) } /** - * nfsd_file_close_inode - attempt a delayed close of a nfsd_file + * nfsd_file_close_inode - attempt a deferred close of a nfsd_file * @inode: inode of the file to attempt to remove * * Close out any open nfsd_files that can be reaped for @inode. The - * actual freeing is deferred to the dispose_list_delayed infrastructure. + * actual freeing is deferred to the nfsd service threads. * * This is used by the fsnotify callbacks and setlease notifier. */ @@ -699,8 +714,10 @@ nfsd_file_close_inode(struct inode *inode) { LIST_HEAD(dispose); + spin_lock(&nfsd_gc_lock); nfsd_file_queue_for_close(inode, &dispose); nfsd_file_dispose_list_delayed(&dispose); + spin_unlock(&nfsd_gc_lock); } /** @@ -940,42 +957,14 @@ __nfsd_file_cache_purge(struct net *net) nfsd_file_dispose_list(&dispose); } -static struct nfsd_fcache_disposal * -nfsd_alloc_fcache_disposal(void) -{ - struct nfsd_fcache_disposal *l; - - l = kmalloc_obj(*l); - if (!l) - return NULL; - spin_lock_init(&l->lock); - INIT_LIST_HEAD(&l->freeme); - return l; -} - -static void -nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l) -{ - nfsd_file_dispose_list(&l->freeme); - kfree(l); -} - -static void -nfsd_free_fcache_disposal_net(struct net *net) -{ - struct nfsd_net *nn = net_generic(net, nfsd_net_id); - struct nfsd_fcache_disposal *l = nn->fcache_disposal; - - nfsd_free_fcache_disposal(l); -} - int nfsd_file_cache_start_net(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - nn->fcache_disposal = nfsd_alloc_fcache_disposal(); - return nn->fcache_disposal ? 0 : -ENOMEM; + spin_lock_init(&nn->fcache_dispose_lock); + INIT_LIST_HEAD(&nn->fcache_dispose_list); + return 0; } /** @@ -994,8 +983,18 @@ nfsd_file_cache_purge(struct net *net) void nfsd_file_cache_shutdown_net(struct net *net) { + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + nfsd_file_cache_purge(net); - nfsd_free_fcache_disposal_net(net); + /* + * Ensure any in-progress shrinker, GC, or fsnotify/lease callback + * (all of which hold nfsd_gc_lock while calling + * nfsd_file_dispose_list_delayed()) has fully completed before + * draining the per-net dispose list. + */ + spin_lock(&nfsd_gc_lock); + spin_unlock(&nfsd_gc_lock); + nfsd_file_dispose_list(&nn->fcache_dispose_list); } void @@ -1227,7 +1226,7 @@ nfsd_file_do_acquire(struct svc_rqst *rqstp, struct net *net, nf->nf_mark = nfsd_file_mark_find_or_create(inode); if (type != S_IFREG || nf->nf_mark) { - if (file) { + if (file && (file->f_mode & FMODE_OPENED)) { get_file(file); nf->nf_file = file; status = nfs_ok; @@ -1374,12 +1373,12 @@ nfsd_file_acquire_local(struct net *net, struct svc_cred *cred, * @rqstp: the RPC transaction being executed * @fhp: the NFS filehandle of the file just created * @may_flags: NFSD_MAY_ settings for the file - * @file: cached, already-open file (may be NULL) + * @file: cached, already-open file (may be NULL or not yet opened) * @pnf: OUT: new or found "struct nfsd_file" object * * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist, - * and @file is non-NULL, use it to instantiate a new nfsd_file instead of - * opening a new one. + * and @file has FMODE_OPENED set, use it to instantiate a new nfsd_file + * instead of opening a new one. * * Return values: * %nfs_ok - @pnf points to an nfsd_file with its reference diff --git a/fs/nfsd/flexfilelayoutxdr.c b/fs/nfsd/flexfilelayoutxdr.c index f9f7e38cba13f..374e52d3064a6 100644 --- a/fs/nfsd/flexfilelayoutxdr.c +++ b/fs/nfsd/flexfilelayoutxdr.c @@ -30,19 +30,24 @@ nfsd4_ff_encode_layoutget(struct xdr_stream *xdr, struct ff_idmap uid; struct ff_idmap gid; - fh_len = 4 + fl->fh.size; + fh_len = 4 + xdr_align_size(fl->fh.size); uid.len = sprintf(uid.buf, "%u", from_kuid(&init_user_ns, fl->uid)); gid.len = sprintf(gid.buf, "%u", from_kgid(&init_user_ns, fl->gid)); - /* 8 + len for recording the length, name, and padding */ - ds_len = 20 + sizeof(stateid_opaque_t) + 4 + fh_len + - 8 + uid.len + 8 + gid.len; + /* data server entry: deviceid + efficiency + stateid + fh list + + * user + group + flags + stats_collect_hint + */ + ds_len = 16 + 4 + 4 + sizeof(stateid_opaque_t) + 4 + fh_len + + 4 + xdr_align_size(uid.len) + + 4 + xdr_align_size(gid.len) + + 4 + 4; + /* mirror: ds_count + ds */ mirror_len = 4 + ds_len; - /* The layout segment */ - len = 20 + mirror_len; + /* stripe_unit + mirror_count + mirror */ + len = 12 + mirror_len; p = xdr_reserve_space(xdr, sizeof(__be32) + len); if (!p) @@ -94,7 +99,8 @@ nfsd4_ff_encode_getdeviceinfo(struct xdr_stream *xdr, } /* len + padding for two strings */ - addr_len = 16 + da->netaddr.netid_len + da->netaddr.addr_len; + addr_len = 8 + xdr_align_size(da->netaddr.netid_len) + + xdr_align_size(da->netaddr.addr_len); ver_len = 20; len = 4 + ver_len + 4 + addr_len; diff --git a/fs/nfsd/localio.c b/fs/nfsd/localio.c index be710d809a3ba..c3eb0557b3e1b 100644 --- a/fs/nfsd/localio.c +++ b/fs/nfsd/localio.c @@ -97,11 +97,15 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom, } nfsd_file_get(localio); again: + rcu_read_lock(); new = unrcu_pointer(cmpxchg(pnf, NULL, RCU_INITIALIZER(localio))); if (new) { /* Some other thread installed an nfsd_file */ - if (nfsd_file_get(new) == NULL) + if (nfsd_file_get(new) == NULL) { + rcu_read_unlock(); goto again; + } + rcu_read_unlock(); /* * Drop the ref we were going to install (both file and * net) and the one we were going to return (only file). @@ -110,6 +114,8 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom, nfsd_net_put(net); nfsd_file_put(localio); localio = new; + } else { + rcu_read_unlock(); } } else nfsd_net_put(net); diff --git a/fs/nfsd/lockd.c b/fs/nfsd/lockd.c index 6fe1325815e0e..72a5b499839d8 100644 --- a/fs/nfsd/lockd.c +++ b/fs/nfsd/lockd.c @@ -92,6 +92,7 @@ nlm_fclose(struct file *filp) } static const struct nlmsvc_binding nfsd_nlm_ops = { + .owner = THIS_MODULE, .fopen = nlm_fopen, /* open file for locking */ .fclose = nlm_fclose, /* close file */ }; @@ -100,11 +101,12 @@ void nfsd_lockd_init(void) { dprintk("nfsd: initializing lockd\n"); - nlmsvc_ops = &nfsd_nlm_ops; + rcu_assign_pointer(nlmsvc_ops, &nfsd_nlm_ops); } void nfsd_lockd_shutdown(void) { - nlmsvc_ops = NULL; + RCU_INIT_POINTER(nlmsvc_ops, NULL); + synchronize_rcu(); } diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 27da1a3edacbf..a7bd7b67fa4f9 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -28,6 +28,16 @@ struct cld_net; struct nfsd_net_cb; struct nfsd4_client_tracking_ops; +enum nfsd_net_flag { + NFSD_NET_GRACE_ENDED, + NFSD_NET_GRACE_END_FORCED, + NFSD_NET_IN_GRACE, + NFSD_NET_SOMEBODY_RECLAIMED, + NFSD_NET_TRACK_RECLAIM_COMPLETES, + NFSD_NET_UP, + NFSD_NET_LOCKD_UP, +}; + enum { /* cache misses due only to checksum comparison failures */ NFSD_STATS_PAYLOAD_MISSES, @@ -66,9 +76,9 @@ struct nfsd_net { struct cache_detail *nametoid_cache; struct lock_manager nfsd4_manager; - bool grace_ended; - bool grace_end_forced; + unsigned long flags; time64_t boot_time; + time64_t boot_time_bt; /* same instant in CLOCK_BOOTTIME */ struct dentry *nfsd_client_dir; @@ -84,6 +94,7 @@ struct nfsd_net { */ struct list_head *reclaim_str_hashtbl; int reclaim_str_hashtbl_size; + struct rw_semaphore reclaim_str_hashtbl_lock; struct list_head *conf_id_hashtbl; struct rb_root conf_name_tree; struct list_head *unconf_id_hashtbl; @@ -96,12 +107,16 @@ struct nfsd_net { * close_lru holds (open) stateowner queue ordered by nfs4_stateowner.so_time * for last close replay. * - * All of the above fields are protected by the client_mutex. + * reclaim_str_hashtbl[], reclaim_str_hashtbl_size are protected by + * reclaim_str_hashtbl_lock. + * + * All of the remaining fields are protected by the client_lock. */ struct list_head client_lru; struct list_head close_lru; - /* protects del_recall_lru and delegation hash/unhash */ + /* protects del_recall_lru and delegation hash/unhash; + * nests outside client_lock */ spinlock_t deleg_lock ____cacheline_aligned; struct list_head del_recall_lru; @@ -110,26 +125,21 @@ struct nfsd_net { struct delayed_work laundromat_work; - /* client_lock protects the client lru list and session hash table */ + /* client_lock protects the client lru list and session hash + * table; nests inside deleg_lock */ spinlock_t client_lock; /* protects blocked_locks_lru */ spinlock_t blocked_locks_lock; struct file *rec_file; - bool in_grace; const struct nfsd4_client_tracking_ops *client_tracking_ops; time64_t nfsd4_lease; time64_t nfsd4_grace; - bool somebody_reclaimed; - bool track_reclaim_completes; atomic_t nr_reclaim_complete; - bool nfsd_net_up; - bool lockd_up; - seqlock_t writeverf_lock; unsigned char writeverf[8]; @@ -209,7 +219,8 @@ struct nfsd_net { /* utsname taken from the process that starts the server */ char nfsd_name[UNX_MAXNODENAME+1]; - struct nfsd_fcache_disposal *fcache_disposal; + spinlock_t fcache_dispose_lock; + struct list_head fcache_dispose_list; siphash_key_t siphash_key; diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 76305b86c1a95..2998640f259de 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -115,14 +115,19 @@ static __be32 nfsacld_proc_setacl(struct svc_rqst *rqstp) inode_lock(inode); - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_ACCESS, - argp->acl_access); - if (error) - goto out_drop_lock; - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_DEFAULT, - argp->acl_default); - if (error) - goto out_drop_lock; + error = 0; + if (argp->mask & NFS_ACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_ACCESS, argp->acl_access); + if (error) + goto out_drop_lock; + } + if (argp->mask & NFS_DFACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_DEFAULT, argp->acl_default); + if (error) + goto out_drop_lock; + } inode_unlock(inode); @@ -248,22 +253,21 @@ nfsaclsvc_encode_getaclres(struct svc_rqst *rqstp, struct xdr_stream *xdr) if (!svcxdr_encode_stat(xdr, resp->status)) return false; - - if (dentry == NULL || d_really_is_negative(dentry)) - return true; - inode = d_inode(dentry); - - if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) - return false; - if (xdr_stream_encode_u32(xdr, resp->mask) < 0) - return false; - - if (!nfs_stream_encode_acl(xdr, inode, resp->acl_access, - resp->mask & NFS_ACL, 0)) - return false; - if (!nfs_stream_encode_acl(xdr, inode, resp->acl_default, - resp->mask & NFS_DFACL, NFS_ACL_DEFAULT)) - return false; + switch (resp->status) { + case nfs_ok: + inode = d_inode(dentry); + if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) + return false; + if (xdr_stream_encode_u32(xdr, resp->mask) < 0) + return false; + if (!nfs_stream_encode_acl(xdr, inode, resp->acl_access, + resp->mask & NFS_ACL, 0)) + return false; + if (!nfs_stream_encode_acl(xdr, inode, resp->acl_default, + resp->mask & NFS_DFACL, NFS_ACL_DEFAULT)) + return false; + break; + } return true; } diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index e87731380be85..a87f9d7f32be2 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -105,12 +105,17 @@ static __be32 nfsd3_proc_setacl(struct svc_rqst *rqstp) inode_lock(inode); - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_ACCESS, - argp->acl_access); - if (error) - goto out_drop_lock; - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_DEFAULT, - argp->acl_default); + error = 0; + if (argp->mask & NFS_ACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_ACCESS, argp->acl_access); + if (error) + goto out_drop_lock; + } + if (argp->mask & NFS_DFACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_DEFAULT, argp->acl_default); + } out_drop_lock: inode_unlock(inode); diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index aeda7a802bdf7..617a70d132926 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -29,6 +29,25 @@ static int nfs3_ftypes[] = { S_IFIFO, /* NF3FIFO */ }; +/* + * Reject a client-supplied atime or mtime whose nanoseconds field is out + * of range. Such a value is well-formed on the wire but is not a valid + * timespec64, and storing it verbatim can corrupt on-disk timestamps. + * tv_nsec is a long, so it is cast to unsigned long (the same width) to + * catch both an over-large value and one that became negative when an + * out-of-range u32 wire nseconds was assigned to a 32-bit long. + */ +static bool nfsd3_time_in_range(const struct iattr *iap) +{ + if ((iap->ia_valid & ATTR_ATIME_SET) && + (unsigned long)iap->ia_atime.tv_nsec >= NSEC_PER_SEC) + return false; + if ((iap->ia_valid & ATTR_MTIME_SET) && + (unsigned long)iap->ia_mtime.tv_nsec >= NSEC_PER_SEC) + return false; + return true; +} + static __be32 nfsd3_map_status(__be32 status) { switch (status) { @@ -101,9 +120,14 @@ nfsd3_proc_setattr(struct svc_rqst *rqstp) SVCFH_fmt(&argp->fh)); fh_copy(&resp->fh, &argp->fh); + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } if (argp->check_guard) guardtime = &argp->guardtime; resp->status = nfsd_setattr(rqstp, &resp->fh, &attrs, guardtime); +out: resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -265,6 +289,8 @@ nfsd3_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, trace_nfsd_vfs_create(rqstp, fhp, S_IFREG, argp->name, argp->len); + if (!nfsd3_time_in_range(iap)) + return nfserr_inval; if (isdotent(argp->name, argp->len)) return nfserr_exist; if (!(iap->ia_valid & ATTR_MODE)) @@ -400,8 +426,13 @@ nfsd3_proc_mkdir(struct svc_rqst *rqstp) argp->attrs.ia_valid &= ~ATTR_SIZE; fh_copy(&resp->dirfh, &argp->fh); fh_init(&resp->fh, NFS3_FHSIZE); + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len, &attrs, S_IFDIR, 0, &resp->fh); +out: resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -415,6 +446,10 @@ nfsd3_proc_symlink(struct svc_rqst *rqstp) .na_iattr = &argp->attrs, }; + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } if (argp->tlen == 0) { resp->status = nfserr_inval; goto out; @@ -471,6 +506,11 @@ nfsd3_proc_mknod(struct svc_rqst *rqstp) goto out; } + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } + type = nfs3_ftypes[argp->ftype]; resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len, &attrs, type, rdev, &resp->fh); diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 50827405468d2..7c868afc329e5 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -108,6 +108,8 @@ static int decode_cb_fattr4(struct xdr_stream *xdr, uint32_t *bitmap, if (!xdrgen_decode_fattr4_time_deleg_access(xdr, &access)) return -EIO; + if (access.nseconds >= NSEC_PER_SEC) + return -EIO; fattr->ncf_cb_atime.tv_sec = access.seconds; fattr->ncf_cb_atime.tv_nsec = access.nseconds; @@ -117,6 +119,8 @@ static int decode_cb_fattr4(struct xdr_stream *xdr, uint32_t *bitmap, if (!xdrgen_decode_fattr4_time_deleg_modify(xdr, &modify)) return -EIO; + if (modify.nseconds >= NSEC_PER_SEC) + return -EIO; fattr->ncf_cb_mtime.tv_sec = modify.seconds; fattr->ncf_cb_mtime.tv_nsec = modify.nseconds; @@ -456,13 +460,20 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr, const struct nfsd4_callback *cb, struct nfs4_cb_compound_hdr *hdr) { - struct nfsd4_session *session = cb->cb_clp->cl_cb_session; + struct nfsd4_session *session; struct nfsd4_referring_call_list *rcl; __be32 *p; if (hdr->minorversion == 0) return; + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + return; + } + encode_nfs_cb_opnum4(xdr, OP_CB_SEQUENCE); encode_sessionid4(xdr, session); @@ -478,6 +489,7 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr, encode_referring_call_list4(xdr, rcl); hdr->nops++; + rcu_read_unlock(); } static void update_cb_slot_table(struct nfsd4_session *ses, u32 target) @@ -529,21 +541,32 @@ static void update_cb_slot_table(struct nfsd4_session *ses, u32 target) static int decode_cb_sequence4resok(struct xdr_stream *xdr, struct nfsd4_callback *cb) { - struct nfsd4_session *session = cb->cb_clp->cl_cb_session; + struct nfsd4_session *session; int status = -ESERVERFAULT; __be32 *p; u32 seqid, slotid, target; + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + cb->cb_seq_status = -NFS4ERR_BADSESSION; + return -NFS4ERR_BADSESSION; + } + /* * If the server returns different values for sessionID, slotID or * sequence number, the server is looney tunes. */ p = xdr_inline_decode(xdr, NFS4_MAX_SESSIONID_LEN + 4 + 4 + 4 + 4); - if (unlikely(p == NULL)) + if (unlikely(p == NULL)) { + rcu_read_unlock(); goto out_overflow; + } if (memcmp(p, session->se_sessionid.data, NFS4_MAX_SESSIONID_LEN)) { dprintk("NFS: %s Invalid session id\n", __func__); + rcu_read_unlock(); goto out; } p += XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN); @@ -551,12 +574,14 @@ static int decode_cb_sequence4resok(struct xdr_stream *xdr, seqid = be32_to_cpup(p++); if (seqid != session->se_cb_seq_nr[cb->cb_held_slot]) { dprintk("NFS: %s Invalid sequence number\n", __func__); + rcu_read_unlock(); goto out; } slotid = be32_to_cpup(p++); if (slotid != cb->cb_held_slot) { dprintk("NFS: %s Invalid slotid\n", __func__); + rcu_read_unlock(); goto out; } @@ -564,6 +589,7 @@ static int decode_cb_sequence4resok(struct xdr_stream *xdr, target = be32_to_cpup(p++); update_cb_slot_table(session, target); + rcu_read_unlock(); status = 0; out: cb->cb_seq_status = status; @@ -1150,9 +1176,8 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c } else { if (!conn->cb_xprt || !ses) return -EINVAL; - clp->cl_cb_session = ses; args.bc_xprt = conn->cb_xprt; - args.prognumber = clp->cl_cb_session->se_cb_prog; + args.prognumber = ses->se_cb_prog; args.protocol = conn->cb_xprt->xpt_class->xcl_ident | XPRT_TRANSPORT_BC; args.authflavor = ses->se_cb_sec.flavor; @@ -1170,8 +1195,10 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c return -ENOMEM; } - if (clp->cl_minorversion != 0) + if (clp->cl_minorversion != 0) { clp->cl_cb_conn.cb_xprt = conn->cb_xprt; + rcu_assign_pointer(clp->cl_cb_session, ses); + } clp->cl_cb_client = client; clp->cl_cb_cred = cred; rcu_read_lock(); @@ -1278,18 +1305,33 @@ static int grab_slot(struct nfsd4_session *ses) static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task) { struct nfs4_client *clp = cb->cb_clp; - struct nfsd4_session *ses = clp->cl_cb_session; + struct nfsd4_session *ses; if (cb->cb_held_slot >= 0) return true; + + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (!ses) { + rcu_read_unlock(); + rpc_sleep_on(&clp->cl_cb_waitq, task, NULL); + return false; + } cb->cb_held_slot = grab_slot(ses); if (cb->cb_held_slot < 0) { + rcu_read_unlock(); rpc_sleep_on(&clp->cl_cb_waitq, task, NULL); /* Race breaker */ - cb->cb_held_slot = grab_slot(ses); + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (ses) + cb->cb_held_slot = grab_slot(ses); + rcu_read_unlock(); if (cb->cb_held_slot < 0) return false; rpc_wake_up_queued_task(&clp->cl_cb_waitq, task); + } else { + rcu_read_unlock(); } return true; } @@ -1297,12 +1339,17 @@ static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task) static void nfsd41_cb_release_slot(struct nfsd4_callback *cb) { struct nfs4_client *clp = cb->cb_clp; - struct nfsd4_session *ses = clp->cl_cb_session; + struct nfsd4_session *ses; if (cb->cb_held_slot >= 0) { - spin_lock(&ses->se_lock); - ses->se_cb_slot_avail |= BIT(cb->cb_held_slot); - spin_unlock(&ses->se_lock); + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (ses) { + spin_lock(&ses->se_lock); + ses->se_cb_slot_avail |= BIT(cb->cb_held_slot); + spin_unlock(&ses->se_lock); + } + rcu_read_unlock(); cb->cb_held_slot = -1; rpc_wake_up_next(&clp->cl_cb_waitq); } @@ -1434,22 +1481,35 @@ static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata) trace_nfsd_cb_rpc_prepare(clp); cb->cb_seq_status = 1; cb->cb_status = 0; - if (minorversion && !nfsd41_cb_get_slot(cb, task)) - return; + if (minorversion) { + if (!rcu_access_pointer(clp->cl_cb_session)) { + rpc_exit(task, -EIO); + return; + } + if (!nfsd41_cb_get_slot(cb, task)) + return; + } rpc_call_start(task); } /* Returns true if CB_COMPOUND processing should continue */ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback *cb) { - struct nfsd4_session *session = cb->cb_clp->cl_cb_session; + struct nfsd4_session *session; bool ret = false; if (cb->cb_held_slot < 0) goto requeue; + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + goto requeue; + } + /* This is the operation status code for CB_SEQUENCE */ - trace_nfsd_cb_seq_status(task, cb); + trace_nfsd_cb_seq_status(task, cb, session); switch (cb->cb_seq_status) { case 0: /* @@ -1481,12 +1541,16 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback fallthrough; case -NFS4ERR_BADSESSION: nfsd4_mark_cb_fault(cb->cb_clp); + rcu_read_unlock(); goto requeue; case -NFS4ERR_DELAY: cb->cb_seq_status = 1; - if (RPC_SIGNALLED(task) || !rpc_restart_call(task)) + if (RPC_SIGNALLED(task) || !rpc_restart_call(task)) { + rcu_read_unlock(); goto requeue; + } rpc_delay(task, 2 * HZ); + rcu_read_unlock(); return false; case -NFS4ERR_SEQ_MISORDERED: case -NFS4ERR_BADSLOT: @@ -1498,11 +1562,13 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback */ nfsd4_mark_cb_fault(cb->cb_clp); cb->cb_held_slot = -1; + rcu_read_unlock(); goto retry_nowait; default: nfsd4_mark_cb_fault(cb->cb_clp); } - trace_nfsd_cb_free_slot(task, cb); + trace_nfsd_cb_free_slot(task, cb, session); + rcu_read_unlock(); nfsd41_cb_release_slot(cb); return ret; retry_nowait: @@ -1624,7 +1690,15 @@ static struct nfsd4_conn * __nfsd4_find_backchannel(struct nfs4_client *clp) * Note there isn't a lot of locking in this code; instead we depend on * the fact that it is run from clp->cl_callback_wq, which won't run two * work items at once. So, for example, clp->cl_callback_wq handles all - * access of cl_cb_client and all calls to rpc_create or rpc_shutdown_client. + * access of cl_cb_client, and all calls to rpc_create or + * rpc_shutdown_client. + * + * cl_cb_session is written only from cl_callback_wq (via + * rcu_assign_pointer) and read from rpciod under rcu_read_lock (via + * rcu_dereference) by encode_cb_sequence4args(), decode_cb_sequence4resok(), + * nfsd4_cb_sequence_done(), and the cb-slot helpers. Sessions are freed + * with kfree_rcu() so that rpciod readers in an RCU read-side critical + * section never dereference a freed session. */ static void nfsd4_process_cb_update(struct nfsd4_callback *cb) { @@ -1676,6 +1750,7 @@ static void nfsd4_process_cb_update(struct nfsd4_callback *cb) nfsd4_mark_cb_down(clp); if (c) svc_xprt_put(c->cn_xprt); + rcu_assign_pointer(clp->cl_cb_session, ses); return; } } diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c index f34320e4c2f4a..279ff1e9dffbe 100644 --- a/fs/nfsd/nfs4layouts.c +++ b/fs/nfsd/nfs4layouts.c @@ -247,13 +247,20 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate, nfsd4_init_cb(&ls->ls_recall, clp, &nfsd4_cb_layout_ops, NFSPROC4_CLNT_CB_LAYOUT); - if (parent->sc_type == SC_TYPE_DELEG) + if (parent->sc_type == SC_TYPE_DELEG) { + spin_lock(&fp->fi_lock); ls->ls_file = nfsd_file_get(fp->fi_deleg_file); - else + spin_unlock(&fp->fi_lock); + } else { ls->ls_file = find_any_file(fp); - BUG_ON(!ls->ls_file); + } + if (!ls->ls_file) { + nfs4_put_stid(stp); + return NULL; + } ls->ls_fenced = false; + ls->ls_fence_inflight = false; ls->ls_fence_delay = 0; INIT_DELAYED_WORK(&ls->ls_fence_work, nfsd4_layout_fence_worker); @@ -791,15 +798,6 @@ nfsd4_layout_fence_worker(struct work_struct *work) struct nfs4_client *clp; struct nfsd_net *nn; - /* - * The workqueue clears WORK_STRUCT_PENDING before invoking - * this callback. Re-arm immediately so that - * delayed_work_pending() returns true while the fence - * operation is in progress, preventing - * lm_breaker_timedout() from taking a duplicate reference. - */ - mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, 0); - spin_lock(&ls->ls_lock); if (list_empty(&ls->ls_layouts)) { spin_unlock(&ls->ls_lock); @@ -809,6 +807,9 @@ nfsd4_layout_fence_worker(struct work_struct *work) nfsd4_close_layout(ls); ls->ls_fenced = true; + spin_lock(&ls->ls_lock); + ls->ls_fence_inflight = false; + spin_unlock(&ls->ls_lock); nfs4_put_stid(&ls->ls_stid); return; } @@ -894,18 +895,26 @@ nfsd4_layout_lm_breaker_timedout(struct file_lease *fl) if ((!nfsd4_layout_ops[ls->ls_layout_type]->fence_client) || ls->ls_fenced) return true; - if (delayed_work_pending(&ls->ls_fence_work)) - return false; /* * Make sure layout has not been returned yet before - * taking a reference count on the layout stateid. + * taking a reference count on the layout stateid. The + * ls_fence_inflight flag is set together with the sc_count + * increment under ls_lock so that a fence worker invocation + * already in progress (which has cleared WORK_STRUCT_PENDING + * but not yet reached dispose:) cannot be coalesced with a + * fresh schedule that takes an extra unmatched reference. */ spin_lock(&ls->ls_lock); + if (ls->ls_fence_inflight) { + spin_unlock(&ls->ls_lock); + return false; + } if (list_empty(&ls->ls_layouts) || !refcount_inc_not_zero(&ls->ls_stid.sc_count)) { spin_unlock(&ls->ls_lock); return true; } + ls->ls_fence_inflight = true; spin_unlock(&ls->ls_lock); mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, 0); diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 8561540ab2db9..939749709e416 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -306,10 +306,6 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, goto out; } - status = fh_compose(resfhp, fhp->fh_export, child, fhp); - if (status != nfs_ok) - goto out; - v_mtime = 0; v_atime = 0; if (nfsd4_create_is_exclusive(open->op_createmode)) { @@ -335,6 +331,10 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, if (status != nfs_ok) goto out; + status = fh_compose(resfhp, fhp->fh_export, child, fhp); + if (status != nfs_ok) + goto out; + switch (open->op_createmode) { case NFS4_CREATE_UNCHECKED: if (!d_is_reg(child)) @@ -385,6 +385,10 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, open->op_created = true; fh_fill_post_attrs(fhp); + status = fh_compose(resfhp, fhp->fh_export, child, fhp); + if (status != nfs_ok) + goto out; + /* A newly created file already has a file size of zero. */ if ((iap->ia_valid & ATTR_SIZE) && (iap->ia_size == 0)) iap->ia_valid &= ~ATTR_SIZE; @@ -663,7 +667,7 @@ nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, pr_warn("nfsd4_process_open2 failed to open newly-created file: status=%u\n", be32_to_cpu(status)); if (reclaim && !status) - nn->somebody_reclaimed = true; + set_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags); out: if (open->op_filp) { fput(open->op_filp); @@ -677,8 +681,6 @@ nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, nfsd4_cleanup_open_state(cstate, open); nfsd4_bump_seqid(cstate, status); out_err: - posix_acl_release(open->op_dpacl); - posix_acl_release(open->op_pacl); return status; } @@ -700,6 +702,13 @@ static __be32 nfsd4_open_omfg(struct svc_rqst *rqstp, struct nfsd4_compound_stat return nfsd4_open(rqstp, cstate, &op->u); } +static void +nfsd4_open_release(union nfsd4_op_u *u) +{ + posix_acl_release(u->open.op_dpacl); + posix_acl_release(u->open.op_pacl); +} + /* * filehandle-manipulating ops. */ @@ -839,6 +848,20 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (status) goto out_aftermask; + /* Sanitize cr_type to avoid returning ATTRNOTSUPP. */ + switch (create->cr_type) { + case NF4LNK: + case NF4BLK: + case NF4CHR: + case NF4SOCK: + case NF4FIFO: + case NF4DIR: + break; + default: + status = nfserr_badtype; + goto out_aftermask; + } + if (create->cr_acl) { if (attrs.na_dpacl || attrs.na_pacl) { status = nfserr_inval; @@ -846,6 +869,8 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, } status = nfsd4_acl_to_attr(create->cr_type, create->cr_acl, &attrs); + if (status != nfs_ok) + goto out_aftermask; } current->fs->umask = create->cr_umask; switch (create->cr_type) { @@ -1253,7 +1278,7 @@ nfsd4_setattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (deleg_attrs) { status = nfserr_bad_stateid; - if (st->sc_type & SC_TYPE_DELEG) { + if (st && (st->sc_type & SC_TYPE_DELEG)) { struct nfs4_delegation *dp = delegstateid(st); /* Only for *_ATTRS_DELEG flavors */ @@ -1494,6 +1519,9 @@ static void nfs4_put_copy(struct nfsd4_copy *copy) { if (!refcount_dec_and_test(©->refcount)) return; + /* Drop the task_struct pinned in nfsd4_copy(); NULL on sync copies. */ + if (copy->copy_task) + put_task_struct(copy->copy_task); kfree(copy->cp_src); kfree(copy); } @@ -1503,20 +1531,18 @@ static void release_copy_files(struct nfsd4_copy *copy); static void nfsd4_stop_copy(struct nfsd4_copy *copy) { trace_nfsd_copy_async_cancel(copy); - if (!test_and_set_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags)) { - kthread_stop(copy->copy_task); - if (!test_bit(NFSD4_COPY_F_CB_ERROR, ©->cp_flags)) - copy->nfserr = nfs_ok; - set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); - } - /* - * The copy was removed from async_copies before this function - * was called, so the reaper cannot clean it up. Release files - * here regardless of who won the STOPPED race. If the thread - * set STOPPED, it has finished using the files. If STOPPED - * was set here, kthread_stop() waited for the thread to exit. + * Join the kthread before releasing its resources. The task_struct is + * pinned in nfsd4_copy(), so kthread_stop() is safe even after the + * one-shot kthread has exited. The caller already unlinked the copy, + * so this runs once per copy. */ + set_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags); + kthread_stop(copy->copy_task); + if (!test_bit(NFSD4_COPY_F_CB_ERROR, ©->cp_flags)) + copy->nfserr = nfs_ok; + set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); + release_copy_files(copy); nfs4_put_copy(copy); } @@ -1530,7 +1556,13 @@ static struct nfsd4_copy *nfsd4_unhash_copy(struct nfs4_client *clp) copy = list_first_entry(&clp->async_copies, struct nfsd4_copy, copies); refcount_inc(©->refcount); - copy->cp_clp = NULL; + /* + * Unlinking hides the copy from the reaper, so drop its + * s2s_cp_stateids entry here while cp_clp is still valid. + */ + nfs4_free_copy_state(copy); + /* Pairs with smp_load_acquire() in nfsd4_send_cb_offload(). */ + smp_store_release(©->cp_clp, NULL); if (!list_empty(©->copies)) list_del_init(©->copies); } @@ -1542,8 +1574,11 @@ void nfsd4_shutdown_copy(struct nfs4_client *clp) { struct nfsd4_copy *copy; - while ((copy = nfsd4_unhash_copy(clp)) != NULL) + while ((copy = nfsd4_unhash_copy(clp)) != NULL) { nfsd4_stop_copy(copy); + /* Reaper can't reach the unhashed copy; drop its membership ref. */ + nfs4_put_copy(copy); + } } static bool nfsd4_copy_on_sb(const struct nfsd4_copy *copy, @@ -1562,6 +1597,10 @@ static bool nfsd4_copy_on_sb(const struct nfsd4_copy *copy, * nfsd4_cancel_copy_by_sb - cancel async copy operations on @sb * @net: net namespace containing the copy operations * @sb: targeted superblock + * + * Context: Caller must hold nfsd_mutex with NFSD_NET_UP set. Outside + * that window nn->conf_id_hashtbl is unallocated or freed, + * so the walk would dereference a NULL or dangling pointer. */ void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb) { @@ -1571,6 +1610,7 @@ void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb) unsigned int idhashval; LIST_HEAD(to_cancel); + lockdep_assert_held(&nfsd_mutex); spin_lock(&nn->client_lock); for (idhashval = 0; idhashval < CLIENT_HASH_SIZE; idhashval++) { struct list_head *head = &nn->conf_id_hashtbl[idhashval]; @@ -1606,7 +1646,11 @@ void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb) struct nfs4_client *clp = copy->cp_clp; list_del_init(©->copies); + /* Reaper can't reach it; drop the s2s entry while cp_clp is valid. */ + nfs4_free_copy_state(copy); nfsd4_stop_copy(copy); + /* Drop the membership ref the reaper would have dropped. */ + nfs4_put_copy(copy); nfsd4_put_client(clp); } } @@ -1901,6 +1945,8 @@ static void nfsd4_cb_offload_release(struct nfsd4_callback *cb) set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags); nfsd4_put_client(cb->cb_clp); + /* Drop the copy reference taken in nfsd4_send_cb_offload(). */ + nfs4_put_copy(copy); } static int nfsd4_cb_offload_done(struct nfsd4_callback *cb, @@ -1950,6 +1996,7 @@ static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy, /* See RFC 7862 p.67: */ if (bytes_total == 0) bytes_total = ULLONG_MAX; + since = READ_ONCE(dst->f_wb_err); do { /* Only async copies can be stopped here */ if (kthread_should_stop()) @@ -1965,13 +2012,14 @@ static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy, } while (bytes_total > 0 && nfsd4_copy_is_async(copy)); /* for a non-zero asynchronous copy do a commit of data */ if (nfsd4_copy_is_async(copy) && copy->cp_res.wr_bytes_written > 0) { - since = READ_ONCE(dst->f_wb_err); end = copy->cp_dst_pos + copy->cp_res.wr_bytes_written - 1; status = vfs_fsync_range(dst, copy->cp_dst_pos, end, 0); if (!status) status = filemap_check_wb_err(dst->f_mapping, since); if (!status) set_bit(NFSD4_COPY_F_COMMITTED, ©->cp_flags); + else if (status != -EAGAIN && status != -ESTALE) + nfsd_reset_write_verifier(copy->cp_nn); } return bytes_copied; } @@ -2030,28 +2078,29 @@ static void release_copy_files(struct nfsd4_copy *copy) } } +/* + * Called from the reaper and from nfsd4_copy()'s error path; in both + * cases the copy is already unreachable from clp->async_copies. + */ static void cleanup_async_copy(struct nfsd4_copy *copy) { nfs4_free_copy_state(copy); release_copy_files(copy); - if (copy->cp_clp) { - spin_lock(©->cp_clp->async_lock); - if (!list_empty(©->copies)) - list_del_init(©->copies); - spin_unlock(©->cp_clp->async_lock); - } nfs4_put_copy(copy); } static void nfsd4_send_cb_offload(struct nfsd4_copy *copy) { struct nfsd4_cb_offload *cbo = ©->cp_cb_offload; - struct nfs4_client *clp = copy->cp_clp; + struct nfs4_client *clp; /* - * cp_clp is NULL when called via nfsd4_shutdown_copy() during - * client destruction. Skip the callback; the client is gone. + * Pairs with smp_store_release(&cp_clp) in find_async_copy() and + * nfsd4_unhash_copy(); the set_bit/clear_bit writers are unordered. + * cp_clp is NULL once the copy was canceled; skip the callback, the + * canceling path owns the notification. */ + clp = smp_load_acquire(©->cp_clp); if (!clp) { set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags); return; @@ -2063,10 +2112,12 @@ static void nfsd4_send_cb_offload(struct nfsd4_copy *copy) cbo->co_retries = 5; /* - * Hold a reference on the client while the callback is in flight. - * Released in nfsd4_cb_offload_release(). + * Hold the client and the copy across the in-flight callback; co_cb is + * embedded in the copy, so it must outlive the callback. Both are + * dropped in nfsd4_cb_offload_release(). */ kref_get(&clp->cl_nfsdfs.cl_ref); + refcount_inc(©->refcount); nfsd4_init_cb(&cbo->co_cb, clp, &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD); @@ -2118,16 +2169,20 @@ static int nfsd4_do_async_copy(void *data) do_callback: if (!test_bit(NFSD4_COPY_F_CB_ERROR, ©->cp_flags)) copy->nfserr = nfserr; - /* The kthread exits forthwith. Ensure that a subsequent - * OFFLOAD_CANCEL won't try to kill it again. */ - set_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags); - + /* + * Don't set NFSD4_COPY_F_STOPPED here: it tells a teardown caller it + * may skip kthread_stop(), which would then release nf_dst and the + * client while still in use. Only nfsd4_stop_copy() sets it, after + * joining. + */ set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); trace_nfsd_copy_async_done(copy); atomic_dec(©->cp_nn->pending_async_copies); if (copy->cp_res.wr_bytes_written > 0 && copy->attr_update) nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0); nfsd4_send_cb_offload(copy); + /* Drop the kthread's reference (taken in nfsd4_copy()); copy may be freed after this. */ + nfs4_put_copy(copy); return 0; } @@ -2153,21 +2208,21 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, } status = nfsd4_setup_inter_ssc(rqstp, cstate, copy); if (status) { - trace_nfsd_copy_done(copy, status); - return nfserr_offload_denied; + status = nfserr_offload_denied; + goto out; } } else { trace_nfsd_copy_intra(copy); status = nfsd4_setup_intra_ssc(rqstp, cstate, copy); - if (status) { - trace_nfsd_copy_done(copy, status); - return status; - } + if (status) + goto out; } memcpy(©->fh, &cstate->current_fh.fh_handle, sizeof(struct knfsd_fh)); if (nfsd4_copy_is_async(copy)) { + struct task_struct *task; + async_copy = kzalloc_obj(struct nfsd4_copy); if (!async_copy) goto out_err; @@ -2182,11 +2237,12 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, async_copy->cp_src = kmalloc_obj(*async_copy->cp_src); if (!async_copy->cp_src) goto out_dec_async_copy_err; - if (!nfs4_init_copy_state(nn, copy)) + dup_copy_fields(copy, async_copy); + + if (!nfs4_init_copy_state(nn, async_copy)) goto out_dec_async_copy_err; - memcpy(&result->cb_stateid, ©->cp_stateid.cs_stid, + memcpy(&result->cb_stateid, &async_copy->cp_stateid.cs_stid, sizeof(result->cb_stateid)); - dup_copy_fields(copy, async_copy); if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) & FMODE_NOCMTIME) != 0) async_copy->attr_update = true; @@ -2195,15 +2251,27 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, NFS4_MAX_SESSIONID_LEN); async_copy->cp_cb_offload.co_referring_slotid = cstate->slot->sl_index; async_copy->cp_cb_offload.co_referring_seqno = cstate->slot->sl_seqid; - async_copy->copy_task = kthread_create(nfsd4_do_async_copy, - async_copy, "%s", "copy thread"); - if (IS_ERR(async_copy->copy_task)) + task = kthread_create(nfsd4_do_async_copy, async_copy, + "%s", "copy thread"); + if (IS_ERR(task)) goto out_dec_async_copy_err; + /* + * Pin the task_struct so kthread_stop() is safe after this + * one-shot kthread exits. Released by nfs4_put_copy(). + */ + get_task_struct(task); + async_copy->copy_task = task; + /* + * Take the kthread's ref and wake it before publishing, so the + * publisher touches async_copy no further and teardown can + * drain it. + */ + refcount_inc(&async_copy->refcount); + wake_up_process(async_copy->copy_task); spin_lock(&async_copy->cp_clp->async_lock); list_add(&async_copy->copies, &async_copy->cp_clp->async_copies); spin_unlock(&async_copy->cp_clp->async_lock); - wake_up_process(async_copy->copy_task); status = nfs_ok; } else { status = nfsd4_do_copy(copy, copy->nf_src->nf_file, @@ -2257,8 +2325,18 @@ find_async_copy(struct nfs4_client *clp, stateid_t *stateid) spin_lock(&clp->async_lock); copy = find_async_copy_locked(clp, stateid); - if (copy) + if (copy) { refcount_inc(©->refcount); + nfs4_free_copy_state(copy); + /* + * Mirror nfsd4_unhash_copy(): unlink and clear cp_clp under + * async_lock so the reaper can't reach it. Caller drops the + * membership ref after nfsd4_stop_copy(). + */ + smp_store_release(©->cp_clp, NULL); + if (!list_empty(©->copies)) + list_del_init(©->copies); + } spin_unlock(&clp->async_lock); return copy; } @@ -2277,8 +2355,11 @@ nfsd4_offload_cancel(struct svc_rqst *rqstp, struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); return manage_cpntf_state(nn, &os->stateid, clp, NULL); - } else + } else { nfsd4_stop_copy(copy); + /* find_async_copy() unlinked it from the reaper; drop the membership ref. */ + nfs4_put_copy(copy); + } return nfs_ok; } @@ -2292,7 +2373,6 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); struct nfs4_stid *stid = NULL; struct nfs4_cpntf_state *cps; - struct nfs4_client *clp = cstate->clp; status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh, &cn->cpn_src_stateid, RD_STATE, NULL, @@ -2306,12 +2386,14 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, cn->cpn_lease_time.tv_nsec = 0; status = nfserrno(-ENOMEM); + /* + * The returned cps is published and fully initialized, and carries an + * extra reference for us; drop it once we are done with it. + */ cps = nfs4_alloc_init_cpntf_state(nn, stid); if (!cps) goto out; memcpy(&cn->cpn_cnr_stateid, &cps->cp_stateid.cs_stid, sizeof(stateid_t)); - memcpy(&cps->cp_p_stateid, &stid->sc_stateid, sizeof(stateid_t)); - memcpy(&cps->cp_p_clid, &clp->cl_clientid, sizeof(clientid_t)); /* For now, only return one server address in cpn_src, the * address used by the client to connect to this server. @@ -2320,10 +2402,11 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, status = nfsd4_set_netaddr((struct sockaddr *)&rqstp->rq_daddr, &cn->cpn_src->u.nl4_addr); WARN_ON_ONCE(status); - if (status) { - nfs4_put_cpntf_state(nn, cps); - goto out; - } + /* + * Drop our extra reference. The membership reference keeps the entry + * alive for a later inter-server READ, or until the laundromat reaps it. + */ + nfs4_put_cpntf_state(nn, cps); out: nfs4_put_stid(stid); return status; @@ -3119,9 +3202,22 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) op->status = nfsd4_open_omfg(rqstp, cstate, op); goto encode_op; } - if (!current_fh->fh_dentry && - !HAS_FH_FLAG(current_fh, NFSD4_FH_FOREIGN)) { - if (!(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { + if (!current_fh->fh_dentry) { + if (HAS_FH_FLAG(current_fh, NFSD4_FH_FOREIGN)) { + /* + * FOREIGN fh from inter-SSC PUTFH: only + * SAVEFH may proceed with a NULL fh_dentry. + * Per RFC 7862 S15.2.3, validation of a + * foreign fh is deferred to the operation + * that consumes it, and NFS4ERR_STALE is + * returned at that point. + */ + if (op->opnum != OP_SAVEFH && + !(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { + op->status = nfserr_stale; + goto encode_op; + } + } else if (!(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { op->status = nfserr_nofilehandle; goto encode_op; } @@ -3185,6 +3281,9 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) status = op->status; } + if (op->opdesc && op->opdesc->op_release) + op->opdesc->op_release(&op->u); + trace_nfsd_compound_status(args->client_opcnt, resp->opcnt, status, nfsd4_op_name(op->opnum)); @@ -3507,7 +3606,7 @@ static u32 nfsd4_get_dir_delegation_rsize(const struct svc_rqst *rqstp, op_encode_stateid_maxsz + 2 /* gddr_notification */ + 2 /* gddr_child_attributes */ + - 2 /* gddr_dir_attributes */); + 2 /* gddr_dir_attributes */) * sizeof(__be32); } #ifdef CONFIG_NFSD_PNFS @@ -3684,6 +3783,7 @@ static const struct nfsd4_operation nfsd4_ops[] = { }, [OP_OPEN] = { .op_func = nfsd4_open, + .op_release = nfsd4_open_release, .op_flags = OP_HANDLES_WRONGSEC | OP_MODIFIES_SOMETHING, .op_name = "OP_OPEN", .op_rsize_bop = nfsd4_open_rsize, diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index 6ea25a52d2f4e..d513971fb119d 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -167,7 +167,7 @@ nfsd4_create_clid_dir(struct nfs4_client *clp) end_creating(dentry); out: if (status == 0) { - if (nn->in_grace) + if (test_bit(NFSD_NET_IN_GRACE, &nn->flags)) __nfsd4_create_reclaim_record_grace(clp, dname, nn); vfs_fsync(nn->rec_file, 0); } else { @@ -285,10 +285,12 @@ __nfsd4_remove_reclaim_record_grace(const char *dname, int len, return; } name.len = len; + down_write(&nn->reclaim_str_hashtbl_lock); crp = nfsd4_find_reclaim_client(name, nn); - kfree(name.data); if (crp) nfs4_remove_reclaim_record(crp, nn); + up_write(&nn->reclaim_str_hashtbl_lock); + kfree(name.data); } static void @@ -317,7 +319,7 @@ nfsd4_remove_clid_dir(struct nfs4_client *clp) nfs4_reset_creds(original_cred); if (status == 0) { vfs_fsync(nn->rec_file, 0); - if (nn->in_grace) + if (test_bit(NFSD_NET_IN_GRACE, &nn->flags)) __nfsd4_remove_reclaim_record_grace(dname, HEXDIR_LEN, nn); } @@ -373,7 +375,7 @@ nfsd4_recdir_purge_old(struct nfsd_net *nn) { int status; - nn->in_grace = false; + clear_bit(NFSD_NET_IN_GRACE, &nn->flags); if (!nn->rec_file) return; status = mnt_want_write_file(nn->rec_file); @@ -455,7 +457,7 @@ nfsd4_init_recdir(struct net *net) nfs4_reset_creds(original_cred); if (!status) - nn->in_grace = true; + set_bit(NFSD_NET_IN_GRACE, &nn->flags); return status; } @@ -484,6 +486,7 @@ nfs4_legacy_state_init(struct net *net) for (i = 0; i < CLIENT_HASH_SIZE; i++) INIT_LIST_HEAD(&nn->reclaim_str_hashtbl[i]); nn->reclaim_str_hashtbl_size = 0; + init_rwsem(&nn->reclaim_str_hashtbl_lock); return 0; } @@ -598,13 +601,16 @@ nfsd4_check_legacy_client(struct nfs4_client *clp) goto out_enoent; } name.len = HEXDIR_LEN; + down_read(&nn->reclaim_str_hashtbl_lock); crp = nfsd4_find_reclaim_client(name, nn); - kfree(name.data); if (crp) { set_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); crp->cr_clp = clp; - return 0; } + up_read(&nn->reclaim_str_hashtbl_lock); + kfree(name.data); + if (crp) + return 0; out_enoent: return -ENOENT; @@ -1176,6 +1182,7 @@ nfsd4_cld_check(struct nfs4_client *clp) return 0; /* look for it in the reclaim hashtable otherwise */ + down_read(&nn->reclaim_str_hashtbl_lock); crp = nfsd4_find_reclaim_client(clp->cl_name, nn); if (crp) goto found; @@ -1191,6 +1198,7 @@ nfsd4_cld_check(struct nfs4_client *clp) if (!name.data) { dprintk("%s: failed to allocate memory for name.data!\n", __func__); + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; } name.len = HEXDIR_LEN; @@ -1201,9 +1209,11 @@ nfsd4_cld_check(struct nfs4_client *clp) } #endif + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; found: crp->cr_clp = clp; + up_read(&nn->reclaim_str_hashtbl_lock); return 0; } @@ -1215,6 +1225,7 @@ nfsd4_cld_check_v2(struct nfs4_client *clp) struct cld_net *cn = nn->cld_net; #endif struct nfs4_client_reclaim *crp; + unsigned int princhashlen; char *principal = NULL; /* did we already find that this client is stable? */ @@ -1222,6 +1233,7 @@ nfsd4_cld_check_v2(struct nfs4_client *clp) return 0; /* look for it in the reclaim hashtable otherwise */ + down_read(&nn->reclaim_str_hashtbl_lock); crp = nfsd4_find_reclaim_client(clp->cl_name, nn); if (crp) goto found; @@ -1237,6 +1249,7 @@ nfsd4_cld_check_v2(struct nfs4_client *clp) if (!name.data) { dprintk("%s: failed to allocate memory for name.data\n", __func__); + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; } name.len = HEXDIR_LEN; @@ -1247,23 +1260,31 @@ nfsd4_cld_check_v2(struct nfs4_client *clp) } #endif + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; found: - if (crp->cr_princhash.len) { + princhashlen = crp->cr_princhash.len; + if (princhashlen) { u8 digest[SHA256_DIGEST_SIZE]; + u8 *pdata; if (clp->cl_cred.cr_raw_principal) principal = clp->cl_cred.cr_raw_principal; else if (clp->cl_cred.cr_principal) principal = clp->cl_cred.cr_principal; - if (principal == NULL) + if (principal == NULL) { + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; + } sha256(principal, strlen(principal), digest); - if (memcmp(crp->cr_princhash.data, digest, - crp->cr_princhash.len)) + pdata = crp->cr_princhash.data; + if (memcmp(pdata, digest, princhashlen)) { + up_read(&nn->reclaim_str_hashtbl_lock); return -ENOENT; + } } crp->cr_clp = clp; + up_read(&nn->reclaim_str_hashtbl_lock); return 0; } @@ -1362,7 +1383,8 @@ nfs4_cld_state_init(struct net *net) for (i = 0; i < CLIENT_HASH_SIZE; i++) INIT_LIST_HEAD(&nn->reclaim_str_hashtbl[i]); nn->reclaim_str_hashtbl_size = 0; - nn->track_reclaim_completes = true; + init_rwsem(&nn->reclaim_str_hashtbl_lock); + set_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags); atomic_set(&nn->nr_reclaim_complete, 0); return 0; @@ -1373,7 +1395,7 @@ nfs4_cld_state_shutdown(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - nn->track_reclaim_completes = false; + clear_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags); kfree(nn->reclaim_str_hashtbl); } diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index a42f34842d776..258621750eb7f 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -202,18 +202,28 @@ renew_client_locked(struct nfs4_client *clp) clp->cl_state = NFSD4_ACTIVE; } +/* + * Finish a cl_rpc_users unpin with the client_lock held. A + * revocation walk clears @renew so the client whose state it is + * revoking is not revived; every other caller renews the lease of + * a still-active client. + */ +static void __put_client_locked(struct nfs4_client *clp, bool renew) +{ + if (is_client_expired(clp)) + wake_up_all(&expiry_wq); + else if (renew) + renew_client_locked(clp); +} + static void put_client_renew_locked(struct nfs4_client *clp) { struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); lockdep_assert_held(&nn->client_lock); - if (!atomic_dec_and_test(&clp->cl_rpc_users)) - return; - if (!is_client_expired(clp)) - renew_client_locked(clp); - else - wake_up_all(&expiry_wq); + if (atomic_dec_and_test(&clp->cl_rpc_users)) + __put_client_locked(clp, true); } static void put_client_renew(struct nfs4_client *clp) @@ -222,10 +232,27 @@ static void put_client_renew(struct nfs4_client *clp) if (!atomic_dec_and_lock(&clp->cl_rpc_users, &nn->client_lock)) return; - if (!is_client_expired(clp)) - renew_client_locked(clp); - else - wake_up_all(&expiry_wq); + __put_client_locked(clp, true); + spin_unlock(&nn->client_lock); +} + +static void put_client_no_renew_locked(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + if (atomic_dec_and_test(&clp->cl_rpc_users)) + __put_client_locked(clp, false); +} + +static void put_client_no_renew(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (!atomic_dec_and_lock(&clp->cl_rpc_users, &nn->client_lock)) + return; + __put_client_locked(clp, false); spin_unlock(&nn->client_lock); } @@ -326,6 +353,16 @@ free_blocked_lock(struct nfsd4_blocked_lock *nbl) kref_put(&nbl->nbl_kref, free_nbl); } +/* A blocked lock's flc_owner is its nfs4_lockowner. */ +static struct nfs4_client * +nbl_client(struct nfsd4_blocked_lock *nbl) +{ + struct nfs4_lockowner *lo; + + lo = (struct nfs4_lockowner *)nbl->nbl_lock.c.flc_owner; + return lo->lo_owner.so_client; +} + static void remove_blocked_locks(struct nfs4_lockowner *lo) { @@ -939,7 +976,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla * Create a unique stateid_t to represent each COPY. */ static int nfs4_init_cp_state(struct nfsd_net *nn, copy_stateid_t *stid, - unsigned char cs_type) + unsigned char cs_type, struct nfs4_stid *p_stid) { int new_id; @@ -949,19 +986,34 @@ static int nfs4_init_cp_state(struct nfsd_net *nn, copy_stateid_t *stid, idr_preload(GFP_KERNEL); spin_lock(&nn->s2s_cp_lock); new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, stid, 0, 0, GFP_NOWAIT); - stid->cs_stid.si_opaque.so_id = new_id; - stid->cs_stid.si_generation = 1; + if (new_id >= 0) { + stid->cs_stid.si_opaque.so_id = new_id; + stid->cs_stid.si_generation = 1; + /* + * Set cs_type and link onto sc_cp_list under the same lock + * that installed the IDR entry, so a concurrent + * manage_cpntf_state() sees either no entry or a fully + * linked cp_list. + */ + stid->cs_type = cs_type; + if (p_stid) { + struct nfs4_cpntf_state *cps = + container_of(stid, struct nfs4_cpntf_state, + cp_stateid); + + list_add(&cps->cp_list, &p_stid->sc_cp_list); + } + } spin_unlock(&nn->s2s_cp_lock); idr_preload_end(); if (new_id < 0) return 0; - stid->cs_type = cs_type; return 1; } int nfs4_init_copy_state(struct nfsd_net *nn, struct nfsd4_copy *copy) { - return nfs4_init_cp_state(nn, ©->cp_stateid, NFS4_COPY_STID); + return nfs4_init_cp_state(nn, ©->cp_stateid, NFS4_COPY_STID, NULL); } struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn, @@ -972,13 +1024,21 @@ struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn, cps = kzalloc_obj(struct nfs4_cpntf_state); if (!cps) return NULL; + /* So a stale list_del_init() before linking is a no-op. */ + INIT_LIST_HEAD(&cps->cp_list); cps->cpntf_time = ktime_get_boottime_seconds(); - refcount_set(&cps->cp_stateid.cs_count, 1); - if (!nfs4_init_cp_state(nn, &cps->cp_stateid, NFS4_COPYNOTIFY_STID)) + /* + * Fully initialize the entry before nfs4_init_cp_state() publishes it, + * since a concurrent OFFLOAD_CANCEL could then free it. Take an extra + * reference for the caller (dropped with nfs4_put_cpntf_state()). + */ + memcpy(&cps->cp_p_stateid, &p_stid->sc_stateid, sizeof(stateid_t)); + memcpy(&cps->cp_p_clid, &p_stid->sc_client->cl_clientid, + sizeof(clientid_t)); + refcount_set(&cps->cp_stateid.cs_count, 2); + if (!nfs4_init_cp_state(nn, &cps->cp_stateid, NFS4_COPYNOTIFY_STID, + p_stid)) goto out_free; - spin_lock(&nn->s2s_cp_lock); - list_add(&cps->cp_list, &p_stid->sc_cp_list); - spin_unlock(&nn->s2s_cp_lock); return cps; out_free: kfree(cps); @@ -998,18 +1058,66 @@ void nfs4_free_copy_state(struct nfsd4_copy *copy) spin_unlock(&nn->s2s_cp_lock); } +/* + * Drop the parent's reference on an already-unlinked cpntf entry. If a + * concurrent holder still owns a reference, its nfs4_put_cpntf_state() does + * the final free. + * + * nn->s2s_cp_lock must be held. + */ +static void put_cpntf_state_unlinked_locked(struct nfs4_cpntf_state *cps) +{ + WARN_ON_ONCE(cps->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID); + WARN_ON_ONCE(!list_empty(&cps->cp_list)); + + if (refcount_dec_and_test(&cps->cp_stateid.cs_count)) + kfree(cps); +} + +/* + * Unhash from the IDR and sc_cp_list. Gated on list_empty() to avoid + * evicting a recycled so_id. + */ +static void nfsd4_unhash_cpntf_state(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) +{ + lockdep_assert_held(&nn->s2s_cp_lock); + + if (!list_empty(&cps->cp_list)) { + list_del_init(&cps->cp_list); + idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.cs_stid.si_opaque.so_id); + } +} + +/* + * Revoke a copy-notify stateid: unlink it from the IDR and sc_cp_list first + * so no new finder can discover it, then drop the membership reference. Every + * revoke path (cancel, laundromat, drain) must use this rather than + * _free_cpntf_state_locked(), which unlinks only at refcount zero and so could + * let a second revoke free the entry under a concurrent reader. + * + * nn->s2s_cp_lock must be held. + */ +static void revoke_cpntf_state_locked(struct nfsd_net *nn, + struct nfs4_cpntf_state *cps) +{ + nfsd4_unhash_cpntf_state(nn, cps); + put_cpntf_state_unlinked_locked(cps); +} + static void nfs4_free_cpntf_statelist(struct net *net, struct nfs4_stid *stid) { - struct nfs4_cpntf_state *cps; + struct nfs4_cpntf_state *cps, *tmp; struct nfsd_net *nn; nn = net_generic(net, nfsd_net_id); spin_lock(&nn->s2s_cp_lock); - while (!list_empty(&stid->sc_cp_list)) { - cps = list_first_entry(&stid->sc_cp_list, - struct nfs4_cpntf_state, cp_list); - _free_cpntf_state_locked(nn, cps); - } + /* + * Revoke unlinks each entry before dropping the parent's reference, so + * the drain terminates in one pass per entry regardless of cs_count; a + * concurrent holder does the final kfree via nfs4_put_cpntf_state(). + */ + list_for_each_entry_safe(cps, tmp, &stid->sc_cp_list, cp_list) + revoke_cpntf_state_locked(nn, cps); spin_unlock(&nn->s2s_cp_lock); } @@ -1168,9 +1276,9 @@ alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp, void nfs4_put_stid(struct nfs4_stid *s) { - struct svc_export *exp = s->sc_export; struct nfs4_file *fp = s->sc_file; struct nfs4_client *clp = s->sc_client; + struct svc_export *exp; might_lock(&clp->cl_lock); @@ -1181,6 +1289,8 @@ nfs4_put_stid(struct nfs4_stid *s) idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id); if (s->sc_status & SC_STATUS_ADMIN_REVOKED) atomic_dec(&s->sc_client->cl_admin_revoked); + /* Read under cl_lock to serialize with drop_stid_export(). */ + exp = s->sc_export; nfs4_free_cpntf_statelist(clp->net, s); spin_unlock(&clp->cl_lock); s->sc_free(s); @@ -1637,6 +1747,7 @@ static void free_ol_stateid_reaplist(struct list_head *reaplist) { struct nfs4_ol_stateid *stp; + struct svc_export *exp; struct nfs4_file *fp; might_sleep(); @@ -1646,7 +1757,10 @@ free_ol_stateid_reaplist(struct list_head *reaplist) st_locks); list_del(&stp->st_locks); fp = stp->st_stid.sc_file; + exp = stp->st_stid.sc_export; stp->st_stid.sc_free(&stp->st_stid); + if (exp) + exp_put(exp); if (fp) put_nfs4_file(fp); } @@ -1873,14 +1987,20 @@ static void revoke_one_stid(struct nfsd_net *nn, struct nfs4_client *clp, * being released. Thus nfsd will no longer prevent the filesystem from being * unmounted. * - * The clients which own the states will subsequently being notified that the + * The clients which own the states will subsequently be notified that the * states have been "admin-revoked". + * + * Context: Caller must hold nfsd_mutex with NFSD_NET_UP set. Outside + * that window nn->conf_id_hashtbl is unallocated or freed, + * so the walk would dereference a NULL or dangling pointer. */ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) { unsigned int idhashval; unsigned int sc_types; + lockdep_assert_held(&nfsd_mutex); + sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT; spin_lock(&nn->client_lock); @@ -1889,9 +2009,19 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) struct nfs4_client *clp; retry: list_for_each_entry(clp, head, cl_idhash) { - struct nfs4_stid *stid = find_one_sb_stid(clp, sb, - sc_types); + struct nfs4_stid *stid; + + /* + * force_expire_client() ignores cl_rpc_users once + * its wait_event() has passed, so pinning cannot + * keep an already-expiring client alive; the + * expiry path revokes its states instead. + */ + if (is_client_expired(clp)) + continue; + stid = find_one_sb_stid(clp, sb, sc_types); if (stid) { + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); revoke_one_stid(nn, clp, stid); nfs4_put_stid(stid); @@ -1904,6 +2034,7 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) */ nn->nfs40_last_revoke = ktime_get_boottime_seconds(); + put_client_no_renew_locked(clp); goto retry; } } @@ -1946,12 +2077,18 @@ static struct nfs4_stid *find_one_export_stid(struct nfs4_client *clp, * * Userspace (exportfs -u) sends this after removing the last client * for a path, enabling the underlying filesystem to be unmounted. + * + * Context: Caller must hold nfsd_mutex with NFSD_NET_UP set. Outside + * that window nn->conf_id_hashtbl is unallocated or freed, + * so the walk would dereference a NULL or dangling pointer. */ void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path) { unsigned int idhashval; unsigned int sc_types; + lockdep_assert_held(&nfsd_mutex); + sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT; spin_lock(&nn->client_lock); @@ -1960,10 +2097,14 @@ void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path) struct nfs4_client *clp; retry: list_for_each_entry(clp, head, cl_idhash) { - struct nfs4_stid *stid = find_one_export_stid( - clp, path, - sc_types); + struct nfs4_stid *stid; + + /* Skip or pin clp as in nfsd4_revoke_states(). */ + if (is_client_expired(clp)) + continue; + stid = find_one_export_stid(clp, path, sc_types); if (stid) { + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); revoke_one_stid(nn, clp, stid); nfs4_put_stid(stid); @@ -1971,6 +2112,7 @@ void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path) if (clp->cl_minorversion == 0) nn->nfs40_last_revoke = ktime_get_boottime_seconds(); + put_client_no_renew_locked(clp); goto retry; } } @@ -2295,7 +2437,7 @@ static void __free_session(struct nfsd4_session *ses) { free_session_slots(ses, 0); xa_destroy(&ses->se_slots); - kfree(ses); + kfree_rcu(ses, rcu_head); } static void free_session(struct nfsd4_session *ses) @@ -2641,14 +2783,24 @@ __destroy_client(struct nfs4_client *clp) release_openowner(oo); } for (i = 0; i < OWNER_HASH_SIZE; i++) { - struct nfs4_stateowner *so, *tmp; + struct nfs4_stateowner *so; - list_for_each_entry_safe(so, tmp, &clp->cl_ownerstr_hashtbl[i], - so_strhash) { + spin_lock(&clp->cl_lock); + while (!list_empty(&clp->cl_ownerstr_hashtbl[i])) { + so = list_first_entry(&clp->cl_ownerstr_hashtbl[i], + struct nfs4_stateowner, so_strhash); /* Should be no openowners at this point */ WARN_ON_ONCE(so->so_is_open_owner); + nfs4_get_stateowner(so); + unhash_lockowner_locked(lockowner(so)); + spin_unlock(&clp->cl_lock); + remove_blocked_locks(lockowner(so)); + nfs4_put_stateowner(so); + + spin_lock(&clp->cl_lock); } + spin_unlock(&clp->cl_lock); } nfsd4_return_all_client_layouts(clp); nfsd4_shutdown_copy(clp); @@ -2675,16 +2827,23 @@ static void inc_reclaim_complete(struct nfs4_client *clp) { struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); - if (!nn->track_reclaim_completes) + if (!test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags)) return; - if (!nfsd4_find_reclaim_client(clp->cl_name, nn)) + + down_read(&nn->reclaim_str_hashtbl_lock); + if (!nfsd4_find_reclaim_client(clp->cl_name, nn)) { + up_read(&nn->reclaim_str_hashtbl_lock); return; + } if (atomic_inc_return(&nn->nr_reclaim_complete) == nn->reclaim_str_hashtbl_size) { + up_read(&nn->reclaim_str_hashtbl_lock); printk(KERN_INFO "NFSD: all clients done reclaiming, ending NFSv4 grace period (net %x)\n", clp->net->ns.inum); nfsd4_end_grace(nn); + return; } + up_read(&nn->reclaim_str_hashtbl_lock); } static void expire_client(struct nfs4_client *clp) @@ -3259,6 +3418,11 @@ static void force_expire_client(struct nfs4_client *clp) trace_nfsd_clid_admin_expired(&clp->cl_clientid); + /* + * cl_time is cleared under client_lock before the wait so a + * revocation walk pinning cl_rpc_users under it either skips + * this client or is seen by this wait_event(). + */ spin_lock(&nn->client_lock); clp->cl_time = 0; spin_unlock(&nn->client_lock); @@ -3414,7 +3578,7 @@ static struct nfs4_client *create_client(struct xdr_netobj name, clp->cl_time = ktime_get_boottime_seconds(); copy_verf(clp, verf); memcpy(&clp->cl_addr, sa, sizeof(struct sockaddr_storage)); - clp->cl_cb_session = NULL; + RCU_INIT_POINTER(clp->cl_cb_session, NULL); clp->net = net; clp->cl_nfsd_dentry = nfsd_client_mkdir( nn, &clp->cl_nfsdfs, @@ -4496,6 +4660,19 @@ static void nfsd4_construct_sequence_response(struct nfsd4_session *session, seq->status_flags |= SEQ4_STATUS_ADMIN_STATE_REVOKED; } +static bool nfsd4_slots_inuse(struct nfsd4_session *ses, int from) +{ + int i; + + for (i = from; i < ses->se_fchannel.maxreqs; i++) { + struct nfsd4_slot *slot = xa_load(&ses->se_slots, i); + + if (slot->sl_flags & NFSD4_SLOT_INUSE) + return true; + } + return false; +} + __be32 nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) @@ -4575,7 +4752,9 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (session->se_target_maxslots < session->se_fchannel.maxreqs && slot->sl_generation == session->se_slot_gen && - seq->maxslots <= session->se_target_maxslots) + seq->maxslots <= session->se_target_maxslots && + seq->slotid < session->se_target_maxslots && + !nfsd4_slots_inuse(session, session->se_target_maxslots)) /* Client acknowledged our reduce maxreqs */ free_session_slots(session, session->se_target_maxslots); @@ -5018,8 +5197,6 @@ nfsd4_init_leases_net(struct nfsd_net *nn) nn->nfsd4_lease = 90; /* default lease time */ nn->nfsd4_grace = 90; - nn->somebody_reclaimed = false; - nn->track_reclaim_completes = false; nn->clverifier_counter = get_random_u32(); nn->clientid_base = get_random_u32(); nn->clientid_counter = nn->clientid_base + 1; @@ -5172,6 +5349,7 @@ static void nfsd4_drop_revoked_stid(struct nfs4_stid *s) case SC_TYPE_DELEG: dp = delegstateid(s); list_del_init(&dp->dl_recall_lru); + s->sc_status |= SC_STATUS_FREED; spin_unlock(&cl->cl_lock); nfs4_put_stid(s); break; @@ -5615,8 +5793,10 @@ static void nfsd_break_one_deleg(struct nfs4_delegation *dp) refcount_inc(&dp->dl_stid.sc_count); queued = nfsd4_run_cb(&dp->dl_recall); WARN_ON_ONCE(!queued); - if (!queued) + if (!queued) { refcount_dec(&dp->dl_stid.sc_count); + clear_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags); + } } /* Called from break_lease() with flc_lock held. */ @@ -6728,12 +6908,21 @@ nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, static void nfsd4_end_grace(struct nfsd_net *nn) { - /* do nothing if grace period already ended */ - if (nn->grace_ended) + /* + * nfsd4_end_grace() can be entered concurrently from the + * laundromat workqueue and from an nfsd compound thread + * handling RECLAIM_COMPLETE. Without serialization, both + * callers can observe NFSD_NET_GRACE_ENDED clear and proceed + * into nfsd4_record_grace_done(). For tracking ops whose + * grace_done drains reclaim_str_hashtbl, that results in + * list corruption and a double free of every + * nfs4_client_reclaim entry. Use an atomic test-and-set so + * exactly one caller proceeds. + */ + if (test_and_set_bit(NFSD_NET_GRACE_ENDED, &nn->flags)) return; trace_nfsd_grace_complete(nn); - nn->grace_ended = true; /* * If the server goes down again right now, an NFSv4 * client will still be allowed to reclaim after it comes back up, @@ -6774,10 +6963,10 @@ bool nfsd4_force_end_grace(struct nfsd_net *nn) { if (!nn->client_tracking_ops) return false; - if (READ_ONCE(nn->grace_ended)) + if (test_bit(NFSD_NET_GRACE_ENDED, &nn->flags)) return false; /* laundromat_work must be initialised now, though it might be disabled */ - WRITE_ONCE(nn->grace_end_forced, true); + set_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags); /* mod_delayed_work() doesn't queue work after * nfs4_state_shutdown_net() has called disable_delayed_work_sync() */ @@ -6791,18 +6980,22 @@ bool nfsd4_force_end_grace(struct nfsd_net *nn) */ static bool clients_still_reclaiming(struct nfsd_net *nn) { - time64_t double_grace_period_end = nn->boot_time + + time64_t double_grace_period_end = nn->boot_time_bt + 2 * nn->nfsd4_lease; - if (READ_ONCE(nn->grace_end_forced)) - return false; - if (nn->track_reclaim_completes && - atomic_read(&nn->nr_reclaim_complete) == - nn->reclaim_str_hashtbl_size) + if (test_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags)) return false; - if (!nn->somebody_reclaimed) + if (test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags)) { + int size; + + down_read(&nn->reclaim_str_hashtbl_lock); + size = nn->reclaim_str_hashtbl_size; + up_read(&nn->reclaim_str_hashtbl_lock); + if (atomic_read(&nn->nr_reclaim_complete) == size) + return false; + } + if (!test_and_clear_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags)) return false; - nn->somebody_reclaimed = false; /* * If we've given them *two* lease times to reclaim, and they're * still not done, give up: @@ -6859,30 +7052,36 @@ static void nfsd4_ssc_shutdown_umount(struct nfsd_net *nn) static void nfsd4_ssc_expire_umount(struct nfsd_net *nn) { bool do_wakeup = false; - struct nfsd4_ssc_umount_item *ni = NULL; - struct nfsd4_ssc_umount_item *tmp; + struct nfsd4_ssc_umount_item *ni; +restart: spin_lock(&nn->nfsd_ssc_lock); - list_for_each_entry_safe(ni, tmp, &nn->nfsd_ssc_mount_list, nsui_list) { - if (time_after(jiffies, ni->nsui_expire)) { - if (refcount_read(&ni->nsui_refcnt) > 1) - continue; + list_for_each_entry(ni, &nn->nfsd_ssc_mount_list, nsui_list) { + if (!time_after(jiffies, ni->nsui_expire)) + break; + if (refcount_read(&ni->nsui_refcnt) > 1) + continue; - /* mark being unmount */ - ni->nsui_busy = true; - spin_unlock(&nn->nfsd_ssc_lock); - mntput(ni->nsui_vfsmount); - spin_lock(&nn->nfsd_ssc_lock); + /* Prevent concurrent setup during unmount */ + ni->nsui_busy = true; + spin_unlock(&nn->nfsd_ssc_lock); + mntput(ni->nsui_vfsmount); + spin_lock(&nn->nfsd_ssc_lock); - /* waiters need to start from begin of list */ - list_del(&ni->nsui_list); - kfree(ni); + /* Force concurrent scanners to restart */ + list_del(&ni->nsui_list); + kfree(ni); - /* wakeup ssc_connect waiters */ - do_wakeup = true; - continue; - } - break; + /* wakeup ssc_connect waiters */ + do_wakeup = true; + /* + * Concurrent nfsd4_ssc_cancel_dul() can free any item + * on the list under nfsd_ssc_lock while mntput() runs + * above. Restart from the head; the list is short and + * the expire worker is periodic, so this is cheap. + */ + spin_unlock(&nn->nfsd_ssc_lock); + goto restart; } if (do_wakeup) wake_up_all(&nn->nfsd_ssc_waitq); @@ -7032,16 +7231,20 @@ static void nfs40_clean_admin_revoked(struct nfsd_net *nn, if (atomic_read(&clp->cl_admin_revoked) == 0) continue; + if (is_client_expired(clp)) + continue; spin_lock(&clp->cl_lock); idr_for_each_entry_ul(&clp->cl_stateids, stid, tmp, id) if (stid->sc_status & SC_STATUS_ADMIN_REVOKED) { refcount_inc(&stid->sc_count); + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); /* this function drops ->cl_lock */ nfsd4_drop_revoked_stid(stid); nfs4_put_stid(stid); spin_lock(&nn->client_lock); + put_client_no_renew_locked(clp); goto retry; } spin_unlock(&clp->cl_lock); @@ -7062,6 +7265,7 @@ nfs4_laundromat(struct nfsd_net *nn) .new_timeo = nn->nfsd4_lease }; struct nfs4_cpntf_state *cps; + struct nfs4_client *clp; copy_stateid_t *cps_t; int i; @@ -7076,7 +7280,7 @@ nfs4_laundromat(struct nfsd_net *nn) cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid); if (cps->cp_stateid.cs_type == NFS4_COPYNOTIFY_STID && state_expired(<, cps->cpntf_time)) - _free_cpntf_state_locked(nn, cps); + revoke_cpntf_state_locked(nn, cps); } spin_unlock(&nn->s2s_cp_lock); nfsd4_async_copy_reaper(nn); @@ -7090,6 +7294,18 @@ nfs4_laundromat(struct nfsd_net *nn) dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); if (!state_expired(<, dp->dl_time)) break; + clp = dp->dl_stid.sc_client; + spin_lock(&nn->client_lock); + if (is_client_expired(clp)) { + spin_unlock(&nn->client_lock); + continue; + } + /* + * Pin without reviving: get_client_locked() would + * flip a courtesy client back to NFSD4_ACTIVE. + */ + atomic_inc(&clp->cl_rpc_users); + spin_unlock(&nn->client_lock); refcount_inc(&dp->dl_stid.sc_count); unhash_delegation_locked(dp, SC_STATUS_REVOKED); list_add(&dp->dl_recall_lru, &reaplist); @@ -7098,8 +7314,10 @@ nfs4_laundromat(struct nfsd_net *nn) while (!list_empty(&reaplist)) { dp = list_first_entry(&reaplist, struct nfs4_delegation, dl_recall_lru); + clp = dp->dl_stid.sc_client; list_del_init(&dp->dl_recall_lru); revoke_delegation(dp); + put_client_no_renew(clp); } spin_lock(&nn->client_lock); @@ -7109,11 +7327,16 @@ nfs4_laundromat(struct nfsd_net *nn) if (!state_expired(<, oo->oo_time)) break; list_del_init(&oo->oo_close_lru); + clp = oo->oo_owner.so_client; + if (is_client_expired(clp)) + continue; stp = oo->oo_last_closed_stid; oo->oo_last_closed_stid = NULL; + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); nfs4_put_stid(&stp->st_stid); spin_lock(&nn->client_lock); + put_client_no_renew_locked(clp); } spin_unlock(&nn->client_lock); @@ -7129,22 +7352,29 @@ nfs4_laundromat(struct nfsd_net *nn) * indefinitely once the lock does become free. */ BUG_ON(!list_empty(&reaplist)); + spin_lock(&nn->client_lock); spin_lock(&nn->blocked_locks_lock); - while (!list_empty(&nn->blocked_locks_lru)) { - nbl = list_first_entry(&nn->blocked_locks_lru, - struct nfsd4_blocked_lock, nbl_lru); + list_for_each_safe(pos, next, &nn->blocked_locks_lru) { + nbl = list_entry(pos, struct nfsd4_blocked_lock, nbl_lru); if (!state_expired(<, nbl->nbl_time)) break; + clp = nbl_client(nbl); + if (is_client_expired(clp)) + continue; + atomic_inc(&clp->cl_rpc_users); list_move(&nbl->nbl_lru, &reaplist); list_del_init(&nbl->nbl_list); } spin_unlock(&nn->blocked_locks_lock); + spin_unlock(&nn->client_lock); while (!list_empty(&reaplist)) { nbl = list_first_entry(&reaplist, struct nfsd4_blocked_lock, nbl_lru); + clp = nbl_client(nbl); list_del_init(&nbl->nbl_lru); free_blocked_lock(nbl); + put_client_no_renew(clp); } #ifdef CONFIG_NFSD_V4_2_INTER_SSC /* service the server-to-server copy delayed unmount list */ @@ -7195,12 +7425,12 @@ deleg_reaper(struct nfsd_net *nn) continue; if (atomic_read(&clp->cl_delegs_in_recall)) continue; - if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags)) - continue; if (ktime_get_boottime_seconds() - clp->cl_ra_time < 5) continue; if (clp->cl_cb_state != NFSD4_CB_UP) continue; + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags)) + continue; /* release in nfsd4_cb_recall_any_release */ kref_get(&clp->cl_nfsdfs.cl_ref); @@ -7463,16 +7693,14 @@ nfs4_check_file(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfs4_stid *s, out: return status; } -static void -_free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) + +static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) { WARN_ON_ONCE(cps->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID); - if (!refcount_dec_and_test(&cps->cp_stateid.cs_count)) - return; - list_del(&cps->cp_list); - idr_remove(&nn->s2s_cp_stateids, - cps->cp_stateid.cs_stid.si_opaque.so_id); - kfree(cps); + if (refcount_dec_and_test(&cps->cp_stateid.cs_count)) { + nfsd4_unhash_cpntf_state(nn, cps); + kfree(cps); + } } /* * A READ from an inter server to server COPY will have a @@ -7497,10 +7725,20 @@ __be32 manage_cpntf_state(struct nfsd_net *nn, stateid_t *st, state = NULL; goto unlock; } - if (!clp) + if (!clp) { refcount_inc(&state->cp_stateid.cs_count); - else - _free_cpntf_state_locked(nn, state); + } else if (memcmp(&clp->cl_clientid, &state->cp_p_clid, + sizeof(clientid_t))) { + /* + * OFFLOAD_CANCEL: only the creating client may cancel. + * so_id is guessable, so without this check any client + * could free another's cpntf state. + */ + state = NULL; + goto unlock; + } else { + revoke_cpntf_state_locked(nn, state); + } } unlock: spin_unlock(&nn->s2s_cp_lock); @@ -7787,7 +8025,7 @@ nfs4_preprocess_seqid_op(struct nfsd4_compound_state *cstate, u32 seqid, return status; stp = openlockstateid(s); if (nfsd4_cstate_assign_replay(cstate, stp->st_stateowner) == -EAGAIN) { - nfs4_put_stateowner(stp->st_stateowner); + nfs4_put_stid(&stp->st_stid); goto retry; } @@ -8036,6 +8274,10 @@ nfsd4_delegreturn(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (status) goto put_stateid; + status = nfs4_check_fh(&cstate->current_fh, &dp->dl_stid); + if (status) + goto put_stateid; + trace_nfsd_deleg_return(stateid); destroy_delegation(dp); smp_mb__after_atomic(); @@ -8506,6 +8748,9 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, status = nfserr_no_grace; if (!locks_in_grace(net) && lock->lk_reclaim) goto out; + if (lock->lk_reclaim && + test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &cstate->clp->cl_flags)) + goto out; if (lock->lk_reclaim) flags |= FL_RECLAIM; @@ -8542,10 +8787,11 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, goto out; } - if (lock->lk_type & (NFS4_READW_LT | NFS4_WRITEW_LT) && - nfsd4_has_session(cstate) && - locks_can_async_lock(nf->nf_file->f_op)) - flags |= FL_SLEEP; + if ((lock->lk_type == NFS4_READW_LT || + lock->lk_type == NFS4_WRITEW_LT) && + nfsd4_has_session(cstate) && + locks_can_async_lock(nf->nf_file->f_op)) + flags |= FL_SLEEP; nbl = find_or_allocate_block(lock_sop, &fp->fi_fhandle, nn); if (!nbl) { @@ -8587,7 +8833,7 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, nfs4_inc_and_copy_stateid(&lock->lk_resp_stateid, &lock_stp->st_stid); status = 0; if (lock->lk_reclaim) - nn->somebody_reclaimed = true; + set_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags); break; case FILE_LOCK_DEFERRED: kref_put(&nbl->nbl_kref, free_nbl); @@ -8963,9 +9209,13 @@ bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn) { struct nfs4_client_reclaim *crp; + bool found; + down_read(&nn->reclaim_str_hashtbl_lock); crp = nfsd4_find_reclaim_client(name, nn); - return (crp && crp->cr_clp); + found = (crp && crp->cr_clp); + up_read(&nn->reclaim_str_hashtbl_lock); + return found; } /* @@ -8978,10 +9228,39 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, unsigned int strhashval; struct nfs4_client_reclaim *crp; + down_write(&nn->reclaim_str_hashtbl_lock); + + /* + * A reclaim record for this client name may already exist (for + * example, populated at boot from the recovery directory before + * an in-grace RECLAIM_COMPLETE or an nfsdcld downcall delivers + * the same name). Dedup here so reclaim_str_hashtbl_size stays + * equal to the number of distinct client names; inc_reclaim_complete + * relies on that equality to end the grace period via the fast path. + */ + crp = nfsd4_find_reclaim_client(name, nn); + if (crp) { + if (princhash.len && crp->cr_princhash.len == 0) { + void *pdata = kmemdup(princhash.data, princhash.len, + GFP_KERNEL); + if (pdata) { + crp->cr_princhash.data = pdata; + crp->cr_princhash.len = princhash.len; + } else { + dprintk("%s: failed to allocate memory for princhash.data!\n", + __func__); + crp = NULL; + } + } + up_write(&nn->reclaim_str_hashtbl_lock); + return crp; + } + name.data = kmemdup(name.data, name.len, GFP_KERNEL); if (!name.data) { dprintk("%s: failed to allocate memory for name.data!\n", __func__); + up_write(&nn->reclaim_str_hashtbl_lock); return NULL; } if (princhash.len) { @@ -8990,6 +9269,7 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, dprintk("%s: failed to allocate memory for princhash.data!\n", __func__); kfree(name.data); + up_write(&nn->reclaim_str_hashtbl_lock); return NULL; } } else @@ -9009,6 +9289,7 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, kfree(name.data); kfree(princhash.data); } + up_write(&nn->reclaim_str_hashtbl_lock); return crp; } @@ -9028,6 +9309,7 @@ nfs4_release_reclaim(struct nfsd_net *nn) struct nfs4_client_reclaim *crp = NULL; int i; + down_write(&nn->reclaim_str_hashtbl_lock); for (i = 0; i < CLIENT_HASH_SIZE; i++) { while (!list_empty(&nn->reclaim_str_hashtbl[i])) { crp = list_entry(nn->reclaim_str_hashtbl[i].next, @@ -9036,6 +9318,7 @@ nfs4_release_reclaim(struct nfsd_net *nn) } } WARN_ON_ONCE(nn->reclaim_str_hashtbl_size); + up_write(&nn->reclaim_str_hashtbl_lock); } /* @@ -9113,8 +9396,9 @@ static int nfs4_state_create_net(struct net *net) nn->conf_name_tree = RB_ROOT; nn->unconf_name_tree = RB_ROOT; nn->boot_time = ktime_get_real_seconds(); - nn->grace_ended = false; - nn->grace_end_forced = false; + nn->boot_time_bt = ktime_get_boottime_seconds(); + clear_bit(NFSD_NET_GRACE_ENDED, &nn->flags); + clear_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags); nn->nfsd4_manager.block_opens = true; INIT_LIST_HEAD(&nn->nfsd4_manager.list); INIT_LIST_HEAD(&nn->client_lru); @@ -9200,7 +9484,8 @@ nfs4_state_start_net(struct net *net) nfsd4_client_tracking_init(net); /* safe for laundromat to run now */ enable_delayed_work(&nn->laundromat_work); - if (nn->track_reclaim_completes && nn->reclaim_str_hashtbl_size == 0) + if (test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags) && + nn->reclaim_str_hashtbl_size == 0) goto skip_grace; printk(KERN_INFO "NFSD: starting %lld-second grace period (net %x)\n", nn->nfsd4_grace, net->ns.inum); diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index e17488a911f76..6a0a97d39bf6a 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -449,9 +449,18 @@ nfsd4_decode_posixacl(struct nfsd4_compoundargs *argp, struct posix_acl **acl) if (xdr_stream_decode_u32(argp->xdr, &count) < 0) return nfserr_bad_xdr; + /* + * The NFSv4 POSIX ACL draft doesn't define a max number of ACE's, but + * the NFSACL spec does. For NFSv4, cap the number of entries to the v3 + * limit, as we want to ensure that ACLs set via NFSv4 POSIX ACL + * extensions are retrievable via NFSACL. + */ + if (count > NFS_ACL_MAX_ENTRIES) + return nfserr_inval; + *acl = posix_acl_alloc(count, GFP_KERNEL); if (*acl == NULL) - return nfserr_resource; + return nfserr_jukebox; (*acl)->a_count = count; for (ace = (*acl)->a_entries; ace < (*acl)->a_entries + count; ace++) { @@ -628,6 +637,8 @@ nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen, if (!xdrgen_decode_fattr4_time_deleg_access(argp->xdr, &access)) return nfserr_bad_xdr; + if (access.nseconds >= NSEC_PER_SEC) + return nfserr_inval; iattr->ia_atime.tv_sec = access.seconds; iattr->ia_atime.tv_nsec = access.nseconds; iattr->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET | ATTR_DELEG; @@ -637,6 +648,8 @@ nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen, if (!xdrgen_decode_fattr4_time_deleg_modify(argp->xdr, &modify)) return nfserr_bad_xdr; + if (modify.nseconds >= NSEC_PER_SEC) + return nfserr_inval; iattr->ia_mtime.tv_sec = modify.seconds; iattr->ia_mtime.tv_nsec = modify.nseconds; iattr->ia_ctime.tv_sec = modify.seconds; @@ -955,6 +968,10 @@ nfsd4_decode_create(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) case NF4LNK: if (xdr_stream_decode_u32(argp->xdr, &create->cr_datalen) < 0) return nfserr_bad_xdr; + if (create->cr_datalen == 0) + return nfserr_inval; + if (create->cr_datalen > NFS4_MAXPATHLEN) + return nfserr_nametoolong; p = xdr_inline_decode(argp->xdr, create->cr_datalen); if (!p) return nfserr_bad_xdr; @@ -2106,6 +2123,7 @@ static __be32 nfsd4_decode_nl4_server(struct nfsd4_compoundargs *argp, { struct nfs42_netaddr *naddr; __be32 *p; + u32 str_len; if (xdr_stream_decode_u32(argp->xdr, &ns->nl4_type) < 0) return nfserr_bad_xdr; @@ -2135,6 +2153,18 @@ static __be32 nfsd4_decode_nl4_server(struct nfsd4_compoundargs *argp, return nfserr_bad_xdr; memcpy(naddr->addr, p, naddr->addr_len); break; + case NL4_NAME: + case NL4_URL: + /* + * Well-formed XDR, but only NL4_NETADDR is supported. Consume + * the utf8str_cis to keep the stream aligned, then return + * NFS4ERR_NOTSUPP rather than the misleading NFS4ERR_BADXDR. + */ + if (xdr_stream_decode_u32(argp->xdr, &str_len) < 0) + return nfserr_bad_xdr; + if (!xdr_inline_decode(argp->xdr, str_len)) + return nfserr_bad_xdr; + return nfserr_notsupp; default: return nfserr_bad_xdr; } @@ -6390,9 +6420,6 @@ nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op) write_bytes_to_xdr_buf(xdr->buf, op_status_offset, &op->status, XDR_UNIT); release: - if (opdesc && opdesc->op_release) - opdesc->op_release(&op->u); - /* * Account for pages consumed while encoding this operation. * The xdr_stream primitives don't manage rq_next_page. @@ -6424,9 +6451,12 @@ void nfsd4_release_compoundargs(struct svc_rqst *rqstp) { struct nfsd4_compoundargs *args = rqstp->rq_argp; + args->opcnt = 0; if (args->ops != args->iops) { - vfree(args->ops); + void *old_ops = args->ops; + args->ops = args->iops; + kvfree_rcu_mightsleep(old_ops); } while (args->to_free) { struct svcxdr_tmpbuf *tb = args->to_free; diff --git a/fs/nfsd/nfs4xdr_gen.c b/fs/nfsd/nfs4xdr_gen.c index 824497051b876..d92dbae5db0fc 100644 --- a/fs/nfsd/nfs4xdr_gen.c +++ b/fs/nfsd/nfs4xdr_gen.c @@ -1,16 +1,16 @@ // SPDX-License-Identifier: GPL-2.0 // Generated by xdrgen. Manual edits will be lost. // XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x -// XDR specification modification time: Thu Jan 8 23:12:07 2026 +// XDR specification modification time: Tue Jun 30 11:57:21 2026 #include #include "nfs4xdr_gen.h" static bool __maybe_unused -xdrgen_decode_int64_t(struct xdr_stream *xdr, int64_t *ptr) +xdrgen_decode_int32_t(struct xdr_stream *xdr, int32_t *ptr) { - return xdrgen_decode_hyper(xdr, ptr); + return xdrgen_decode_int(xdr, ptr); } static bool __maybe_unused @@ -19,6 +19,154 @@ xdrgen_decode_uint32_t(struct xdr_stream *xdr, uint32_t *ptr) return xdrgen_decode_unsigned_int(xdr, ptr); } +static bool __maybe_unused +xdrgen_decode_int64_t(struct xdr_stream *xdr, int64_t *ptr) +{ + return xdrgen_decode_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_uint64_t(struct xdr_stream *xdr, uint64_t *ptr) +{ + return xdrgen_decode_unsigned_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_nfsstat4(struct xdr_stream *xdr, nfsstat4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NFS4_OK: + case NFS4ERR_PERM: + case NFS4ERR_NOENT: + case NFS4ERR_IO: + case NFS4ERR_NXIO: + case NFS4ERR_ACCESS: + case NFS4ERR_EXIST: + case NFS4ERR_XDEV: + case NFS4ERR_NOTDIR: + case NFS4ERR_ISDIR: + case NFS4ERR_INVAL: + case NFS4ERR_FBIG: + case NFS4ERR_NOSPC: + case NFS4ERR_ROFS: + case NFS4ERR_MLINK: + case NFS4ERR_NAMETOOLONG: + case NFS4ERR_NOTEMPTY: + case NFS4ERR_DQUOT: + case NFS4ERR_STALE: + case NFS4ERR_BADHANDLE: + case NFS4ERR_BAD_COOKIE: + case NFS4ERR_NOTSUPP: + case NFS4ERR_TOOSMALL: + case NFS4ERR_SERVERFAULT: + case NFS4ERR_BADTYPE: + case NFS4ERR_DELAY: + case NFS4ERR_SAME: + case NFS4ERR_DENIED: + case NFS4ERR_EXPIRED: + case NFS4ERR_LOCKED: + case NFS4ERR_GRACE: + case NFS4ERR_FHEXPIRED: + case NFS4ERR_SHARE_DENIED: + case NFS4ERR_WRONGSEC: + case NFS4ERR_CLID_INUSE: + case NFS4ERR_RESOURCE: + case NFS4ERR_MOVED: + case NFS4ERR_NOFILEHANDLE: + case NFS4ERR_MINOR_VERS_MISMATCH: + case NFS4ERR_STALE_CLIENTID: + case NFS4ERR_STALE_STATEID: + case NFS4ERR_OLD_STATEID: + case NFS4ERR_BAD_STATEID: + case NFS4ERR_BAD_SEQID: + case NFS4ERR_NOT_SAME: + case NFS4ERR_LOCK_RANGE: + case NFS4ERR_SYMLINK: + case NFS4ERR_RESTOREFH: + case NFS4ERR_LEASE_MOVED: + case NFS4ERR_ATTRNOTSUPP: + case NFS4ERR_NO_GRACE: + case NFS4ERR_RECLAIM_BAD: + case NFS4ERR_RECLAIM_CONFLICT: + case NFS4ERR_BADXDR: + case NFS4ERR_LOCKS_HELD: + case NFS4ERR_OPENMODE: + case NFS4ERR_BADOWNER: + case NFS4ERR_BADCHAR: + case NFS4ERR_BADNAME: + case NFS4ERR_BAD_RANGE: + case NFS4ERR_LOCK_NOTSUPP: + case NFS4ERR_OP_ILLEGAL: + case NFS4ERR_DEADLOCK: + case NFS4ERR_FILE_OPEN: + case NFS4ERR_ADMIN_REVOKED: + case NFS4ERR_CB_PATH_DOWN: + case NFS4ERR_BADIOMODE: + case NFS4ERR_BADLAYOUT: + case NFS4ERR_BAD_SESSION_DIGEST: + case NFS4ERR_BADSESSION: + case NFS4ERR_BADSLOT: + case NFS4ERR_COMPLETE_ALREADY: + case NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case NFS4ERR_DELEG_ALREADY_WANTED: + case NFS4ERR_BACK_CHAN_BUSY: + case NFS4ERR_LAYOUTTRYLATER: + case NFS4ERR_LAYOUTUNAVAILABLE: + case NFS4ERR_NOMATCHING_LAYOUT: + case NFS4ERR_RECALLCONFLICT: + case NFS4ERR_UNKNOWN_LAYOUTTYPE: + case NFS4ERR_SEQ_MISORDERED: + case NFS4ERR_SEQUENCE_POS: + case NFS4ERR_REQ_TOO_BIG: + case NFS4ERR_REP_TOO_BIG: + case NFS4ERR_REP_TOO_BIG_TO_CACHE: + case NFS4ERR_RETRY_UNCACHED_REP: + case NFS4ERR_UNSAFE_COMPOUND: + case NFS4ERR_TOO_MANY_OPS: + case NFS4ERR_OP_NOT_IN_SESSION: + case NFS4ERR_HASH_ALG_UNSUPP: + case NFS4ERR_CLIENTID_BUSY: + case NFS4ERR_PNFS_IO_HOLE: + case NFS4ERR_SEQ_FALSE_RETRY: + case NFS4ERR_BAD_HIGH_SLOT: + case NFS4ERR_DEADSESSION: + case NFS4ERR_ENCR_ALG_UNSUPP: + case NFS4ERR_PNFS_NO_LAYOUT: + case NFS4ERR_NOT_ONLY_OP: + case NFS4ERR_WRONG_CRED: + case NFS4ERR_WRONG_TYPE: + case NFS4ERR_DIRDELEG_UNAVAIL: + case NFS4ERR_REJECT_DELEG: + case NFS4ERR_RETURNCONFLICT: + case NFS4ERR_DELEG_REVOKED: + case NFS4ERR_PARTNER_NOTSUPP: + case NFS4ERR_PARTNER_NO_AUTH: + case NFS4ERR_UNION_NOTSUPP: + case NFS4ERR_OFFLOAD_DENIED: + case NFS4ERR_WRONG_LFS: + case NFS4ERR_BADLABEL: + case NFS4ERR_OFFLOAD_NO_REQS: + case NFS4ERR_NOXATTR: + case NFS4ERR_XATTR2BIG: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_attrlist4(struct xdr_stream *xdr, attrlist4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, 0); +} + static bool __maybe_unused xdrgen_decode_bitmap4(struct xdr_stream *xdr, bitmap4 *ptr) { @@ -30,6 +178,24 @@ xdrgen_decode_bitmap4(struct xdr_stream *xdr, bitmap4 *ptr) return true; } +static bool __maybe_unused +xdrgen_decode_verifier4(struct xdr_stream *xdr, verifier4 *ptr) +{ + return xdr_stream_decode_opaque_fixed(xdr, ptr, NFS4_VERIFIER_SIZE) == 0; +} + +static bool __maybe_unused +xdrgen_decode_nfs_cookie4(struct xdr_stream *xdr, nfs_cookie4 *ptr) +{ + return xdrgen_decode_uint64_t(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_nfs_fh4(struct xdr_stream *xdr, nfs_fh4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, NFS4_FHSIZE); +} + static bool __maybe_unused xdrgen_decode_utf8string(struct xdr_stream *xdr, utf8string *ptr) { @@ -54,6 +220,29 @@ xdrgen_decode_utf8str_mixed(struct xdr_stream *xdr, utf8str_mixed *ptr) return xdrgen_decode_utf8string(xdr, ptr); } +static bool __maybe_unused +xdrgen_decode_component4(struct xdr_stream *xdr, component4 *ptr) +{ + return xdrgen_decode_utf8str_cs(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_linktext4(struct xdr_stream *xdr, linktext4 *ptr) +{ + return xdrgen_decode_utf8str_cs(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_pathname4(struct xdr_stream *xdr, pathname4 *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->count) < 0) + return false; + for (u32 i = 0; i < ptr->count; i++) + if (!xdrgen_decode_component4(xdr, &ptr->element[i])) + return false; + return true; +} + static bool __maybe_unused xdrgen_decode_nfstime4(struct xdr_stream *xdr, struct nfstime4 *ptr) { @@ -64,6 +253,26 @@ xdrgen_decode_nfstime4(struct xdr_stream *xdr, struct nfstime4 *ptr) return true; } +static bool __maybe_unused +xdrgen_decode_fattr4(struct xdr_stream *xdr, struct fattr4 *ptr) +{ + if (!xdrgen_decode_bitmap4(xdr, &ptr->attrmask)) + return false; + if (!xdrgen_decode_attrlist4(xdr, &ptr->attr_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_stateid4(struct xdr_stream *xdr, struct stateid4 *ptr) +{ + if (!xdrgen_decode_uint32_t(xdr, &ptr->seqid)) + return false; + if (xdr_stream_decode_opaque_fixed(xdr, ptr->other, 12) < 0) + return false; + return true; +} + static bool __maybe_unused xdrgen_decode_fattr4_offline(struct xdr_stream *xdr, fattr4_offline *ptr) { @@ -366,9 +575,160 @@ xdrgen_decode_fattr4_posix_access_acl(struct xdr_stream *xdr, fattr4_posix_acces */ static bool __maybe_unused -xdrgen_encode_int64_t(struct xdr_stream *xdr, const int64_t value) +xdrgen_decode_notify_type4(struct xdr_stream *xdr, notify_type4 *ptr) { - return xdrgen_encode_hyper(xdr, value); + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NOTIFY4_CHANGE_CHILD_ATTRS: + case NOTIFY4_CHANGE_DIR_ATTRS: + case NOTIFY4_REMOVE_ENTRY: + case NOTIFY4_ADD_ENTRY: + case NOTIFY4_RENAME_ENTRY: + case NOTIFY4_CHANGE_COOKIE_VERIFIER: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_entry4(struct xdr_stream *xdr, struct notify_entry4 *ptr) +{ + if (!xdrgen_decode_component4(xdr, &ptr->ne_file)) + return false; + if (!xdrgen_decode_fattr4(xdr, &ptr->ne_attrs)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_prev_entry4(struct xdr_stream *xdr, struct prev_entry4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->pe_prev_entry)) + return false; + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->pe_prev_entry_cookie)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_remove4(struct xdr_stream *xdr, struct notify_remove4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->nrm_old_entry)) + return false; + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->nrm_old_entry_cookie)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_add4(struct xdr_stream *xdr, struct notify_add4 *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->nad_old_entry.count) < 0) + return false; + if (ptr->nad_old_entry.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_old_entry.count; i++) + if (!xdrgen_decode_notify_remove4(xdr, &ptr->nad_old_entry.element[i])) + return false; + if (!xdrgen_decode_notify_entry4(xdr, &ptr->nad_new_entry)) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->nad_new_entry_cookie.count) < 0) + return false; + if (ptr->nad_new_entry_cookie.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_new_entry_cookie.count; i++) + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->nad_new_entry_cookie.element[i])) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->nad_prev_entry.count) < 0) + return false; + if (ptr->nad_prev_entry.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_prev_entry.count; i++) + if (!xdrgen_decode_prev_entry4(xdr, &ptr->nad_prev_entry.element[i])) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->nad_last_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_attr4(struct xdr_stream *xdr, struct notify_attr4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->na_changed_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_rename4(struct xdr_stream *xdr, struct notify_rename4 *ptr) +{ + if (!xdrgen_decode_notify_remove4(xdr, &ptr->nrn_old_entry)) + return false; + if (!xdrgen_decode_notify_add4(xdr, &ptr->nrn_new_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_verifier4(struct xdr_stream *xdr, struct notify_verifier4 *ptr) +{ + if (!xdrgen_decode_verifier4(xdr, &ptr->nv_old_cookieverf)) + return false; + if (!xdrgen_decode_verifier4(xdr, &ptr->nv_new_cookieverf)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notifylist4(struct xdr_stream *xdr, notifylist4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, 0); +} + +static bool __maybe_unused +xdrgen_decode_notify4(struct xdr_stream *xdr, struct notify4 *ptr) +{ + if (!xdrgen_decode_bitmap4(xdr, &ptr->notify_mask)) + return false; + if (!xdrgen_decode_notifylist4(xdr, &ptr->notify_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_CB_NOTIFY4args(struct xdr_stream *xdr, struct CB_NOTIFY4args *ptr) +{ + if (!xdrgen_decode_stateid4(xdr, &ptr->cna_stateid)) + return false; + if (!xdrgen_decode_nfs_fh4(xdr, &ptr->cna_fh)) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->cna_changes.count) < 0) + return false; + for (u32 i = 0; i < ptr->cna_changes.count; i++) + if (!xdrgen_decode_notify4(xdr, &ptr->cna_changes.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_CB_NOTIFY4res(struct xdr_stream *xdr, struct CB_NOTIFY4res *ptr) +{ + if (!xdrgen_decode_nfsstat4(xdr, &ptr->cnr_status)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_int32_t(struct xdr_stream *xdr, const int32_t value) +{ + return xdrgen_encode_int(xdr, value); } static bool __maybe_unused @@ -377,6 +737,30 @@ xdrgen_encode_uint32_t(struct xdr_stream *xdr, const uint32_t value) return xdrgen_encode_unsigned_int(xdr, value); } +static bool __maybe_unused +xdrgen_encode_int64_t(struct xdr_stream *xdr, const int64_t value) +{ + return xdrgen_encode_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_uint64_t(struct xdr_stream *xdr, const uint64_t value) +{ + return xdrgen_encode_unsigned_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_nfsstat4(struct xdr_stream *xdr, nfsstat4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_attrlist4(struct xdr_stream *xdr, const attrlist4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + static bool __maybe_unused xdrgen_encode_bitmap4(struct xdr_stream *xdr, const bitmap4 value) { @@ -388,6 +772,24 @@ xdrgen_encode_bitmap4(struct xdr_stream *xdr, const bitmap4 value) return true; } +static bool __maybe_unused +xdrgen_encode_verifier4(struct xdr_stream *xdr, const verifier4 value) +{ + return xdr_stream_encode_opaque_fixed(xdr, value, NFS4_VERIFIER_SIZE) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_nfs_cookie4(struct xdr_stream *xdr, const nfs_cookie4 value) +{ + return xdrgen_encode_uint64_t(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_nfs_fh4(struct xdr_stream *xdr, const nfs_fh4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + static bool __maybe_unused xdrgen_encode_utf8string(struct xdr_stream *xdr, const utf8string value) { @@ -412,6 +814,29 @@ xdrgen_encode_utf8str_mixed(struct xdr_stream *xdr, const utf8str_mixed value) return xdrgen_encode_utf8string(xdr, value); } +static bool __maybe_unused +xdrgen_encode_component4(struct xdr_stream *xdr, const component4 value) +{ + return xdrgen_encode_utf8str_cs(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_linktext4(struct xdr_stream *xdr, const linktext4 value) +{ + return xdrgen_encode_utf8str_cs(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_pathname4(struct xdr_stream *xdr, const pathname4 value) +{ + if (xdr_stream_encode_u32(xdr, value.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value.count; i++) + if (!xdrgen_encode_component4(xdr, value.element[i])) + return false; + return true; +} + static bool __maybe_unused xdrgen_encode_nfstime4(struct xdr_stream *xdr, const struct nfstime4 *value) { @@ -422,6 +847,26 @@ xdrgen_encode_nfstime4(struct xdr_stream *xdr, const struct nfstime4 *value) return true; } +static bool __maybe_unused +xdrgen_encode_fattr4(struct xdr_stream *xdr, const struct fattr4 *value) +{ + if (!xdrgen_encode_bitmap4(xdr, value->attrmask)) + return false; + if (!xdrgen_encode_attrlist4(xdr, value->attr_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_stateid4(struct xdr_stream *xdr, const struct stateid4 *value) +{ + if (!xdrgen_encode_uint32_t(xdr, value->seqid)) + return false; + if (xdr_stream_encode_opaque_fixed(xdr, value->other, 12) < 0) + return false; + return true; +} + static bool __maybe_unused xdrgen_encode_fattr4_offline(struct xdr_stream *xdr, const fattr4_offline value) { @@ -567,3 +1012,137 @@ xdrgen_encode_fattr4_posix_access_acl(struct xdr_stream *xdr, const fattr4_posix return false; return true; } + +static bool __maybe_unused +xdrgen_encode_notify_type4(struct xdr_stream *xdr, notify_type4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_notify_entry4(struct xdr_stream *xdr, const struct notify_entry4 *value) +{ + if (!xdrgen_encode_component4(xdr, value->ne_file)) + return false; + if (!xdrgen_encode_fattr4(xdr, &value->ne_attrs)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_prev_entry4(struct xdr_stream *xdr, const struct prev_entry4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->pe_prev_entry)) + return false; + if (!xdrgen_encode_nfs_cookie4(xdr, value->pe_prev_entry_cookie)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_remove4(struct xdr_stream *xdr, const struct notify_remove4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->nrm_old_entry)) + return false; + if (!xdrgen_encode_nfs_cookie4(xdr, value->nrm_old_entry_cookie)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_add4(struct xdr_stream *xdr, const struct notify_add4 *value) +{ + if (value->nad_old_entry.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_old_entry.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_old_entry.count; i++) + if (!xdrgen_encode_notify_remove4(xdr, &value->nad_old_entry.element[i])) + return false; + if (!xdrgen_encode_notify_entry4(xdr, &value->nad_new_entry)) + return false; + if (value->nad_new_entry_cookie.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_new_entry_cookie.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_new_entry_cookie.count; i++) + if (!xdrgen_encode_nfs_cookie4(xdr, value->nad_new_entry_cookie.element[i])) + return false; + if (value->nad_prev_entry.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_prev_entry.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_prev_entry.count; i++) + if (!xdrgen_encode_prev_entry4(xdr, &value->nad_prev_entry.element[i])) + return false; + if (!xdrgen_encode_bool(xdr, value->nad_last_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_attr4(struct xdr_stream *xdr, const struct notify_attr4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->na_changed_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_rename4(struct xdr_stream *xdr, const struct notify_rename4 *value) +{ + if (!xdrgen_encode_notify_remove4(xdr, &value->nrn_old_entry)) + return false; + if (!xdrgen_encode_notify_add4(xdr, &value->nrn_new_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_verifier4(struct xdr_stream *xdr, const struct notify_verifier4 *value) +{ + if (!xdrgen_encode_verifier4(xdr, value->nv_old_cookieverf)) + return false; + if (!xdrgen_encode_verifier4(xdr, value->nv_new_cookieverf)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notifylist4(struct xdr_stream *xdr, const notifylist4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_notify4(struct xdr_stream *xdr, const struct notify4 *value) +{ + if (!xdrgen_encode_bitmap4(xdr, value->notify_mask)) + return false; + if (!xdrgen_encode_notifylist4(xdr, value->notify_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_CB_NOTIFY4args(struct xdr_stream *xdr, const struct CB_NOTIFY4args *value) +{ + if (!xdrgen_encode_stateid4(xdr, &value->cna_stateid)) + return false; + if (!xdrgen_encode_nfs_fh4(xdr, value->cna_fh)) + return false; + if (xdr_stream_encode_u32(xdr, value->cna_changes.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->cna_changes.count; i++) + if (!xdrgen_encode_notify4(xdr, &value->cna_changes.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_CB_NOTIFY4res(struct xdr_stream *xdr, const struct CB_NOTIFY4res *value) +{ + if (!xdrgen_encode_nfsstat4(xdr, value->cnr_status)) + return false; + return true; +} diff --git a/fs/nfsd/nfs4xdr_gen.h b/fs/nfsd/nfs4xdr_gen.h index 1c487f1a11ab2..d7725bd5648ae 100644 --- a/fs/nfsd/nfs4xdr_gen.h +++ b/fs/nfsd/nfs4xdr_gen.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x */ -/* XDR specification modification time: Thu Jan 8 23:12:07 2026 */ +/* XDR specification modification time: Tue Jun 30 11:57:21 2026 */ #ifndef _LINUX_XDRGEN_NFS4_1_DECL_H #define _LINUX_XDRGEN_NFS4_1_DECL_H @@ -21,10 +21,13 @@ bool xdrgen_encode_fattr4_time_deleg_access(struct xdr_stream *xdr, const fattr4 bool xdrgen_decode_fattr4_time_deleg_modify(struct xdr_stream *xdr, fattr4_time_deleg_modify *ptr); bool xdrgen_encode_fattr4_time_deleg_modify(struct xdr_stream *xdr, const fattr4_time_deleg_modify *value); + bool xdrgen_decode_aclmodel4(struct xdr_stream *xdr, aclmodel4 *ptr); bool xdrgen_encode_aclmodel4(struct xdr_stream *xdr, aclmodel4 value); + bool xdrgen_decode_aclscope4(struct xdr_stream *xdr, aclscope4 *ptr); bool xdrgen_encode_aclscope4(struct xdr_stream *xdr, aclscope4 value); + bool xdrgen_decode_posixacetag4(struct xdr_stream *xdr, posixacetag4 *ptr); bool xdrgen_encode_posixacetag4(struct xdr_stream *xdr, posixacetag4 value); diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c index 154468ceccdc1..a9fa9dd410b2f 100644 --- a/fs/nfsd/nfscache.c +++ b/fs/nfsd/nfscache.c @@ -200,14 +200,14 @@ int nfsd_reply_cache_init(struct nfsd_net *nn) nn->nfsd_reply_cache_shrinker->seeks = 1; nn->nfsd_reply_cache_shrinker->private_data = nn; - shrinker_register(nn->nfsd_reply_cache_shrinker); - for (i = 0; i < hashsize; i++) { INIT_LIST_HEAD(&nn->drc_hashtbl[i].lru_head); spin_lock_init(&nn->drc_hashtbl[i].cache_lock); } nn->drc_hashsize = hashsize; + shrinker_register(nn->nfsd_reply_cache_shrinker); + return 0; out_shrinker: kvfree(nn->drc_hashtbl); @@ -275,7 +275,7 @@ nfsd_prune_bucket_locked(struct nfsd_net *nn, struct nfsd_drc_bucket *b, nfsd_cacherep_unlink_locked(nn, b, rp); list_add(&rp->c_lru, dispose); - if (max && ++freed > max) + if (max && ++freed >= max) break; } } diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index fa92e31d19d6f..bc16fc7ca24f3 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -296,14 +296,15 @@ static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size) * 2. Is that directory a mount point, or * 3. Is that directory the root of an exported file system? */ - nfsd4_cancel_copy_by_sb(netns(file), path.dentry->d_sb); error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb); mutex_lock(&nfsd_mutex); nn = net_generic(netns(file), nfsd_net_id); - if (nn->nfsd_serv) + if (test_bit(NFSD_NET_UP, &nn->flags)) { + nfsd4_cancel_copy_by_sb(netns(file), path.dentry->d_sb); nfsd4_revoke_states(nn, path.dentry->d_sb); - else + } else { error = -EINVAL; + } mutex_unlock(&nfsd_mutex); path_put(&path); @@ -420,6 +421,7 @@ static ssize_t write_threads(struct file *file, char *buf, size_t size) char *mesg = buf; int rv; struct net *net = netns(file); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); if (size > 0) { int newthreads; @@ -430,7 +432,10 @@ static ssize_t write_threads(struct file *file, char *buf, size_t size) return -EINVAL; trace_nfsd_ctl_threads(net, newthreads); mutex_lock(&nfsd_mutex); - rv = nfsd_svc(1, &newthreads, net, file->f_cred, NULL); + if (newthreads > 0 || nn->nfsd_serv != NULL) + rv = nfsd_svc(1, &newthreads, net, file->f_cred, NULL); + else + rv = 0; mutex_unlock(&nfsd_mutex); if (rv < 0) return rv; @@ -1111,7 +1116,7 @@ static ssize_t write_v4_end_grace(struct file *file, char *buf, size_t size) } return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%c\n", - nn->grace_ended ? 'Y' : 'N'); + test_bit(NFSD_NET_GRACE_ENDED, &nn->flags) ? 'Y' : 'N'); } #endif @@ -1414,8 +1419,8 @@ static int create_proc_exports_entry(void) unsigned int nfsd_net_id; struct nfsd_genl_rqstp { - struct sockaddr rq_daddr; - struct sockaddr rq_saddr; + struct sockaddr_storage rq_daddr; + struct sockaddr_storage rq_saddr; unsigned long rq_flags; ktime_t rq_stime; __be32 rq_xid; @@ -1448,9 +1453,9 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, nla_put_s64(skb, NFSD_A_RPC_STATUS_SERVICE_TIME, ktime_to_us(genl_rqstp->rq_stime), NFSD_A_RPC_STATUS_PAD)) - return -ENOBUFS; + goto out_cancel; - switch (genl_rqstp->rq_saddr.sa_family) { + switch (genl_rqstp->rq_saddr.ss_family) { case AF_INET: { const struct sockaddr_in *s_in, *d_in; @@ -1464,7 +1469,7 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, s_in->sin_port) || nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT, d_in->sin_port)) - return -ENOBUFS; + goto out_cancel; break; } case AF_INET6: { @@ -1480,7 +1485,7 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, s_in->sin6_port) || nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT, d_in->sin6_port)) - return -ENOBUFS; + goto out_cancel; break; } } @@ -1488,10 +1493,14 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, for (i = 0; i < genl_rqstp->rq_opcnt; i++) if (nla_put_u32(skb, NFSD_A_RPC_STATUS_COMPOUND_OPS, genl_rqstp->rq_opnum[i])) - return -ENOBUFS; + goto out_cancel; genlmsg_end(skb, hdr); return 0; + +out_cancel: + genlmsg_cancel(skb, hdr); + return -ENOBUFS; } /** @@ -1519,18 +1528,28 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, for (i = 0; i < nn->nfsd_serv->sv_nrpools; i++) { struct svc_rqst *rqstp; + long thread_skip = 0; if (i < cb->args[0]) /* already consumed */ continue; + /* + * The saved thread index only applies to the pool the dump + * was resumed in. Subsequent pools must start from thread 0, + * otherwise their first cb->args[1] threads are silently + * skipped. + */ + if (i == cb->args[0]) + thread_skip = cb->args[1]; + rqstp_index = 0; list_for_each_entry_rcu(rqstp, &nn->nfsd_serv->sv_pools[i].sp_all_threads, rq_all) { - struct nfsd_genl_rqstp genl_rqstp; + struct nfsd_genl_rqstp genl_rqstp = {}; unsigned int status_counter; - if (rqstp_index++ < cb->args[1]) /* already consumed */ + if (rqstp_index++ < thread_skip) /* already consumed */ continue; /* * Acquire rq_status_counter before parsing the rqst @@ -1551,9 +1570,9 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, genl_rqstp.rq_stime = rqstp->rq_stime; genl_rqstp.rq_opcnt = 0; memcpy(&genl_rqstp.rq_daddr, svc_daddr(rqstp), - sizeof(struct sockaddr)); + sizeof(struct sockaddr_storage)); memcpy(&genl_rqstp.rq_saddr, svc_addr(rqstp), - sizeof(struct sockaddr)); + sizeof(struct sockaddr_storage)); #ifdef CONFIG_NFSD_V4 if (rqstp->rq_vers == NFS4_VERSION && @@ -1572,17 +1591,26 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, #endif /* CONFIG_NFSD_V4 */ /* - * Acquire rq_status_counter before reporting the rqst - * fields to the user. + * Read-side load-load fence: order the field reads + * above before the counter re-read below, mirroring + * the smp_rmb() in the standard seqcount retry. The + * begin-side smp_load_acquire() above pairs with the + * smp_store_release() in nfsd_dispatch(). */ - if (smp_load_acquire(&rqstp->rq_status_counter) != - status_counter) + smp_rmb(); + if (READ_ONCE(rqstp->rq_status_counter) != status_counter) continue; ret = nfsd_genl_rpc_status_compose_msg(skb, cb, &genl_rqstp); - if (ret) + if (ret) { + if (skb->len) { + cb->args[0] = i; + cb->args[1] = rqstp_index - 1; + ret = skb->len; + } goto out; + } } } @@ -1943,6 +1971,60 @@ int nfsd_nl_version_get_doit(struct sk_buff *skb, struct genl_info *info) return err; } +/** + * nfsd_nl_validate_listeners - sanity-check the listener list from userland + * @info: netlink metadata and command arguments + * + * Walk every NFSD_A_SERVER_SOCK_ADDR attribute and confirm that each entry + * is well-formed: it parses against the policy, carries both an address and + * a transport name, and the address is long enough for its family. Doing + * this up front lets the callers below assume every entry is valid and + * guarantees we make no changes when the request is malformed. + * + * Return: 0 if every entry is valid, or a negative errno otherwise. + */ +static int nfsd_nl_validate_listeners(struct genl_info *info) +{ + const struct nlattr *attr; + int rem; + + nlmsg_for_each_attr_type(attr, NFSD_A_SERVER_SOCK_ADDR, info->nlhdr, + GENL_HDRLEN, rem) { + struct nlattr *tb[NFSD_A_SOCK_MAX + 1]; + struct sockaddr *sa; + int err; + + err = nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, + nfsd_sock_nl_policy, info->extack); + if (err < 0) + return err; + + if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) + return -EINVAL; + + sa = nla_data(tb[NFSD_A_SOCK_ADDR]); + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(sa->sa_family)) + return -EINVAL; + + switch (sa->sa_family) { + case AF_INET: + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < + sizeof(struct sockaddr_in)) + return -EINVAL; + break; + case AF_INET6: + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < + sizeof(struct sockaddr_in6)) + return -EINVAL; + break; + default: + return -EAFNOSUPPORT; + } + } + + return 0; +} + /** * nfsd_nl_listener_set_doit - set the nfs running sockets * @skb: reply buffer @@ -1961,6 +2043,15 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) bool delete = false; int err, rem; + /* + * Validate the entire listener list before making any changes, so a + * malformed request fails cleanly without creating a serv or touching + * the existing listeners. + */ + err = nfsd_nl_validate_listeners(info); + if (err) + return err; + mutex_lock(&nfsd_mutex); err = nfsd_create_serv(net); @@ -1987,16 +2078,11 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) const char *xcl_name; struct sockaddr *sa; + /* validated up front in nfsd_nl_validate_listeners() */ if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, nfsd_sock_nl_policy, info->extack) < 0) continue; - if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) - continue; - - if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa)) - continue; - xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]); sa = nla_data(tb[NFSD_A_SOCK_ADDR]); @@ -2048,16 +2134,11 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) struct sockaddr *sa; int ret; + /* validated up front in nfsd_nl_validate_listeners() */ if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, nfsd_sock_nl_policy, info->extack) < 0) continue; - if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) - continue; - - if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa)) - continue; - xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]); sa = nla_data(tb[NFSD_A_SOCK_ADDR]); @@ -2343,7 +2424,7 @@ int nfsd_nl_unlock_filesystem_doit(struct sk_buff *skb, error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb); mutex_lock(&nfsd_mutex); - if (nn->nfsd_serv) { + if (test_bit(NFSD_NET_UP, &nn->flags)) { nfsd4_cancel_copy_by_sb(net, path.dentry->d_sb); nfsd4_revoke_states(nn, path.dentry->d_sb); } else { @@ -2390,7 +2471,7 @@ int nfsd_nl_unlock_export_doit(struct sk_buff *skb, struct genl_info *info) return error; mutex_lock(&nfsd_mutex); - if (nn->nfsd_serv) { + if (test_bit(NFSD_NET_UP, &nn->flags)) { nfsd_file_close_export(net, &path); nfsd4_revoke_export_states(nn, &path); } else @@ -2512,11 +2593,12 @@ static int __init init_nfsd(void) { int retval; - nfsd_debugfs_init(); - retval = nfsd4_init_slabs(); if (retval) return retval; + + nfsd_debugfs_init(); + retval = nfsd4_init_pnfs(); if (retval) goto out_free_slabs; @@ -2561,8 +2643,8 @@ static int __init init_nfsd(void) out_free_pnfs: nfsd4_exit_pnfs(); out_free_slabs: - nfsd4_free_slabs(); nfsd_debugfs_exit(); + nfsd4_free_slabs(); return retval; } @@ -2577,9 +2659,9 @@ static void __exit exit_nfsd(void) unregister_pernet_subsys(&nfsd_net_ops); nfsd_drc_slab_free(); nfsd_lockd_shutdown(); - nfsd4_free_slabs(); nfsd4_exit_pnfs(); nfsd_debugfs_exit(); + nfsd4_free_slabs(); } MODULE_AUTHOR("Olaf Kirch "); diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 11bce03b90316..d33d49b61e6af 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -332,13 +332,14 @@ void nfsd_lockd_shutdown(void); #define nfserr_noxattr cpu_to_be32(NFS4ERR_NOXATTR) /* - * Error codes for internal use. We use enum to choose numbers that are - * not already assigned, then covert to be32 resulting in a number that - * cannot conflict with any existing be32 nfserr value. + * Error codes for internal use. These are based at an impossible + * nfsstat4 value so that, once converted to be32, they cannot conflict + * with any value defined by the protocol (compare the nlm__int__* codes + * in fs/lockd/lockd.h). */ enum { /* end-of-file indicator in readdir */ - NFSERR_EOF = NFS4ERR_FIRST_FREE, + NFSERR_EOF = 30000, #define nfserr_eof cpu_to_be32(NFSERR_EOF) /* replay detected */ diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 429ca5c6ec087..ab53de1c280d5 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -70,10 +70,8 @@ nfsd_mode_check(struct dentry *dentry, umode_t requested) if (requested == 0) /* the caller doesn't care */ return nfs_ok; if (mode == requested) { - if (mode == S_IFDIR && !d_can_lookup(dentry)) { - WARN_ON_ONCE(1); + if (mode == S_IFDIR && !d_can_lookup(dentry)) return nfserr_notdir; - } return nfs_ok; } if (mode == S_IFLNK) { @@ -344,15 +342,19 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct net *net, if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOWCC) fhp->fh_no_wcc = true; fhp->fh_64bit_cookies = true; - if (exp->ex_flags & NFSEXP_V4ROOT) + if (exp->ex_flags & NFSEXP_V4ROOT) { + dput(dentry); goto out; + } break; case NFS_FHSIZE: fhp->fh_no_wcc = true; if (EX_WGATHER(exp)) fhp->fh_use_wgather = true; - if (exp->ex_flags & NFSEXP_V4ROOT) + if (exp->ex_flags & NFSEXP_V4ROOT) { + dput(dentry); goto out; + } } fhp->fh_dentry = dentry; diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 8873033d1e82f..a73d5c259cd93 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -82,6 +82,7 @@ nfsd_proc_setattr(struct svc_rqst *rqstp) .na_iattr = iap, }; struct svc_fh *fhp; + int hosterr; dprintk("nfsd: SETATTR %s, valid=%x, size=%ld\n", SVCFH_fmt(&argp->fh), @@ -117,6 +118,12 @@ nfsd_proc_setattr(struct svc_rqst *rqstp) if (resp->status != nfs_ok) goto out; + hosterr = fh_want_write(fhp); + if (hosterr) { + resp->status = nfserrno(hosterr); + goto out; + } + if (delta < 0) delta = -delta; if (delta < MAX_TOUCH_TIME_ERROR && diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 4f1ab3222a4d4..5e0a0c1cbb910 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -351,7 +351,7 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred) struct nfsd_net *nn = net_generic(net, nfsd_net_id); int ret; - if (nn->nfsd_net_up) + if (test_bit(NFSD_NET_UP, &nn->flags)) return 0; ret = nfsd_startup_generic(); @@ -364,11 +364,11 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred) goto out_socks; } - if (nfsd_needs_lockd(nn) && !nn->lockd_up) { + if (nfsd_needs_lockd(nn) && !test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) { ret = lockd_up(net, cred); if (ret) goto out_socks; - nn->lockd_up = true; + set_bit(NFSD_NET_LOCKD_UP, &nn->flags); } ret = nfsd_file_cache_start_net(net); @@ -386,7 +386,7 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred) if (ret) goto out_reply_cache; - nn->nfsd_net_up = true; + set_bit(NFSD_NET_UP, &nn->flags); return 0; out_reply_cache: @@ -394,9 +394,9 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred) out_filecache: nfsd_file_cache_shutdown_net(net); out_lockd: - if (nn->lockd_up) { + if (test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) { lockd_down(net); - nn->lockd_up = false; + clear_bit(NFSD_NET_LOCKD_UP, &nn->flags); } out_socks: nfsd_shutdown_generic(); @@ -407,7 +407,7 @@ static void nfsd_shutdown_net(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - if (nn->nfsd_net_up) { + if (test_bit(NFSD_NET_UP, &nn->flags)) { percpu_ref_kill_and_confirm(&nn->nfsd_net_ref, nfsd_net_done); wait_for_completion(&nn->nfsd_net_confirm_done); @@ -415,18 +415,18 @@ static void nfsd_shutdown_net(struct net *net) nfs4_state_shutdown_net(net); nfsd_reply_cache_shutdown(nn); nfsd_file_cache_shutdown_net(net); - if (nn->lockd_up) { + if (test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) { lockd_down(net); - nn->lockd_up = false; + clear_bit(NFSD_NET_LOCKD_UP, &nn->flags); } wait_for_completion(&nn->nfsd_net_free_done); } percpu_ref_exit(&nn->nfsd_net_ref); - if (nn->nfsd_net_up) + if (test_bit(NFSD_NET_UP, &nn->flags)) nfsd_shutdown_generic(); - nn->nfsd_net_up = false; + clear_bit(NFSD_NET_UP, &nn->flags); } static DEFINE_SPINLOCK(nfsd_notifier_lock); @@ -815,7 +815,7 @@ nfsd_acl_init_request(struct svc_rqst *rqstp, ret->mismatch.lovers = NFSD_ACL_NRVERS; for (i = NFSD_ACL_MINVERS; i < NFSD_ACL_NRVERS; i++) { - if (nfsd_support_acl_version(rqstp->rq_vers) && + if (nfsd_support_acl_version(i) && nfsd_vers(nn, i, NFSD_TEST)) { ret->mismatch.lovers = i; break; @@ -825,7 +825,7 @@ nfsd_acl_init_request(struct svc_rqst *rqstp, return rpc_prog_unavail; ret->mismatch.hivers = NFSD_ACL_MINVERS; for (i = NFSD_ACL_NRVERS - 1; i >= NFSD_ACL_MINVERS; i--) { - if (nfsd_support_acl_version(rqstp->rq_vers) && + if (nfsd_support_acl_version(i) && nfsd_vers(nn, i, NFSD_TEST)) { ret->mismatch.hivers = i; break; @@ -960,6 +960,20 @@ nfsd(void *vrqstp) return 0; } +/* + * Set rq_status_counter back to an even value, indicating that the rqstp + * fields are no longer meaningful to a lockless reader. This pairs with the + * odd-valued store made once the request has been decoded, and must run on + * every return path that follows it so that the seq-lock like protocol used + * by nfsd_nl_rpc_status_get_dumpit() is not left permanently odd. The store + * also advances the counter so a concurrent reader detects the transition. + */ +static void nfsd_status_counter_set_idle(struct svc_rqst *rqstp) +{ + smp_store_release(&rqstp->rq_status_counter, + (rqstp->rq_status_counter | 1) + 1); +} + /** * nfsd_dispatch - Process an NFS or NFSACL or LOCALIO Request * @rqstp: incoming request @@ -1022,14 +1036,9 @@ int nfsd_dispatch(struct svc_rqst *rqstp) if (!proc->pc_encode(rqstp, &rqstp->rq_res_stream)) goto out_encode_err; - /* - * Release rq_status_counter setting it to an even value after the rpc - * request has been properly processed. - */ - smp_store_release(&rqstp->rq_status_counter, rqstp->rq_status_counter + 1); - nfsd_cache_update(rqstp, rp, ntli->ntli_cachetype, nfs_reply); out_cached_reply: + nfsd_status_counter_set_idle(rqstp); return 1; out_decode_err: @@ -1040,12 +1049,14 @@ int nfsd_dispatch(struct svc_rqst *rqstp) out_update_drop: nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL); out_dropit: + nfsd_status_counter_set_idle(rqstp); return 0; out_encode_err: trace_nfsd_cant_encode_err(rqstp); nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL); *statp = rpc_system_err; + nfsd_status_counter_set_idle(rqstp); return 1; } diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index ae71e06213175..86c9dd4b334bd 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -9,6 +9,16 @@ #include "xdr.h" #include "auth.h" +/* + * Sun convention: a sattr time-useconds field of one full second (an + * otherwise out-of-range value) means "set this time to the current + * server time." It's needed to make permissions checks for the "touch" + * program across NFSv2 mounts work correctly. See description of + * sattr in section 6.1 of "NFS Illustrated" by Brent Callaghan, + * Addison-Wesley, ISBN 0-201-32750-5 + */ +#define NFS2_SATTR_SET_TO_SERVER_TIME (1000000) + /* * Mapping of S_IF* types to NFS file types */ @@ -172,27 +182,29 @@ svcxdr_decode_sattr(struct svc_rqst *rqstp, struct xdr_stream *xdr, tmp1 = be32_to_cpup(p++); tmp2 = be32_to_cpup(p++); if (tmp1 != (u32)-1 && tmp2 != (u32)-1) { + /* + * Range test here to prevent the multiplication from + * wrapping to a valid (but incorrect) value on 32-bit + * platforms. + */ + if (tmp2 > NFS2_SATTR_SET_TO_SERVER_TIME) + return false; iap->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET; iap->ia_atime.tv_sec = tmp1; iap->ia_atime.tv_nsec = tmp2 * NSEC_PER_USEC; + if (tmp2 == NFS2_SATTR_SET_TO_SERVER_TIME) + iap->ia_valid &= ~ATTR_ATIME_SET; } tmp1 = be32_to_cpup(p++); tmp2 = be32_to_cpup(p++); if (tmp1 != (u32)-1 && tmp2 != (u32)-1) { + if (tmp2 > NFS2_SATTR_SET_TO_SERVER_TIME) + return false; iap->ia_valid |= ATTR_MTIME | ATTR_MTIME_SET; iap->ia_mtime.tv_sec = tmp1; iap->ia_mtime.tv_nsec = tmp2 * NSEC_PER_USEC; - /* - * Passing the invalid value useconds=1000000 for mtime - * is a Sun convention for "set both mtime and atime to - * current server time". It's needed to make permissions - * checks for the "touch" program across v2 mounts to - * Solaris and Irix boxes work correctly. See description of - * sattr in section 6.1 of "NFS Illustrated" by - * Brent Callaghan, Addison-Wesley, ISBN 0-201-32750-5 - */ - if (tmp2 == 1000000) + if (tmp2 == NFS2_SATTR_SET_TO_SERVER_TIME) iap->ia_valid &= ~(ATTR_ATIME_SET|ATTR_MTIME_SET); } diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index dec83e92650d1..f44ea672670f5 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -384,6 +384,7 @@ struct nfsd4_session { u16 se_slot_gen; bool se_dead; u32 se_target_maxslots; + struct rcu_head rcu_head; }; /* formatted contents of nfs4_sessionid */ @@ -496,7 +497,7 @@ struct nfs4_client { #define NFSD4_CB_FAULT 3 int cl_cb_state; struct nfsd4_callback cl_cb_null; - struct nfsd4_session *cl_cb_session; + struct nfsd4_session __rcu *cl_cb_session; /* for all client information that callback code might need: */ spinlock_t cl_lock; @@ -754,6 +755,7 @@ struct nfs4_layout_stateid { struct delayed_work ls_fence_work; unsigned int ls_fence_delay; bool ls_fenced; + bool ls_fence_inflight; }; static inline struct nfs4_layout_stateid *layoutstateid(struct nfs4_stid *s) diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 1c5a1e50f9465..171e8fdbafb60 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -271,7 +271,7 @@ TRACE_EVENT_CONDITION(nfsd_fh_verify, TP_CONDITION(rqstp != NULL), TP_STRUCT__entry( __field(unsigned int, netns_ino) - __sockaddr(server, rqstp->rq_xprt->xpt_remotelen) + __sockaddr(server, rqstp->rq_xprt->xpt_locallen) __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) __field(u32, xid) __field(u32, fh_hash) @@ -310,7 +310,7 @@ TRACE_EVENT_CONDITION(nfsd_fh_verify_err, TP_CONDITION(rqstp != NULL && error), TP_STRUCT__entry( __field(unsigned int, netns_ino) - __sockaddr(server, rqstp->rq_xprt->xpt_remotelen) + __sockaddr(server, rqstp->rq_xprt->xpt_locallen) __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) __field(u32, xid) __field(u32, fh_hash) @@ -1677,6 +1677,7 @@ TRACE_EVENT(nfsd_cb_setup_err, { OP_CB_RECALL, "CB_RECALL" }, \ { OP_CB_LAYOUTRECALL, "CB_LAYOUTRECALL" }, \ { OP_CB_RECALL_ANY, "CB_RECALL_ANY" }, \ + { OP_CB_NOTIFY, "CB_NOTIFY" }, \ { OP_CB_NOTIFY_LOCK, "CB_NOTIFY_LOCK" }, \ { OP_CB_OFFLOAD, "CB_OFFLOAD" }) @@ -1727,9 +1728,10 @@ DEFINE_NFSD_CB_LIFETIME_EVENT(bc_shutdown); TRACE_EVENT(nfsd_cb_seq_status, TP_PROTO( const struct rpc_task *task, - const struct nfsd4_callback *cb + const struct nfsd4_callback *cb, + const struct nfsd4_session *session ), - TP_ARGS(task, cb), + TP_ARGS(task, cb, session), TP_STRUCT__entry( __field(unsigned int, task_id) __field(unsigned int, client_id) @@ -1741,8 +1743,6 @@ TRACE_EVENT(nfsd_cb_seq_status, __field(int, seq_status) ), TP_fast_assign( - const struct nfs4_client *clp = cb->cb_clp; - const struct nfsd4_session *session = clp->cl_cb_session; const struct nfsd4_sessionid *sid = (struct nfsd4_sessionid *)&session->se_sessionid; @@ -1768,9 +1768,10 @@ TRACE_EVENT(nfsd_cb_seq_status, TRACE_EVENT(nfsd_cb_free_slot, TP_PROTO( const struct rpc_task *task, - const struct nfsd4_callback *cb + const struct nfsd4_callback *cb, + const struct nfsd4_session *session ), - TP_ARGS(task, cb), + TP_ARGS(task, cb, session), TP_STRUCT__entry( __field(unsigned int, task_id) __field(unsigned int, client_id) @@ -1781,8 +1782,6 @@ TRACE_EVENT(nfsd_cb_free_slot, __field(u32, slot_seqno) ), TP_fast_assign( - const struct nfs4_client *clp = cb->cb_clp; - const struct nfsd4_session *session = clp->cl_cb_session; const struct nfsd4_sessionid *sid = (struct nfsd4_sessionid *)&session->se_sessionid; diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 1e89c7ff9493b..f73012bc742f8 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -137,8 +137,10 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp, follow_flags = LOOKUP_AUTOMOUNT; err = follow_down(&path, follow_flags); - if (err < 0) + if (err < 0) { + path_put(&path); goto out; + } if (path.mnt == exp->ex_path.mnt && path.dentry == dentry && nfsd_mountpoint(dentry, exp) == 2) { /* This is only a mountpoint in some other namespace */ @@ -419,21 +421,22 @@ nfsd_sanitize_attrs(struct inode *inode, struct iattr *iap) } static __be32 -nfsd_get_write_access(struct svc_rqst *rqstp, struct svc_fh *fhp, - struct iattr *iap) +nfsd_may_truncate(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct iattr *iap) { struct inode *inode = d_inode(fhp->fh_dentry); - if (iap->ia_size < inode->i_size) { - __be32 err; + if (iap->ia_size >= i_size_read(inode)) + return nfs_ok; - err = nfsd_permission(&rqstp->rq_cred, - fhp->fh_export, fhp->fh_dentry, - NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE); - if (err) - return err; - } - return nfserrno(get_write_access(inode)); + return nfsd_permission(&rqstp->rq_cred, fhp->fh_export, fhp->fh_dentry, + NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE); +} + +static __be32 +nfsd_get_write_access(struct svc_fh *fhp) +{ + return nfserrno(get_write_access(d_inode(fhp->fh_dentry))); } static int __nfsd_setattr(struct dentry *dentry, struct iattr *iap) @@ -560,12 +563,17 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, * setattr call. */ if (size_change) { - err = nfsd_get_write_access(rqstp, fhp, iap); + err = nfsd_get_write_access(fhp); if (err) return err; } inode_lock(inode); + if (size_change) { + err = nfsd_may_truncate(rqstp, fhp, iap); + if (err) + goto out_unlock; + } err = fh_fill_pre_attrs(fhp); if (err) goto out_unlock; @@ -1374,6 +1382,7 @@ nfsd_direct_write(struct svc_rqst *rqstp, struct svc_fh *fhp, struct file *file = nf->nf_file; unsigned int nsegs, i; ssize_t host_err; + size_t expected; nsegs = nfsd_write_dio_iters_init(nf, rqstp->rq_bvec, nvecs, kiocb, *cnt, segments); @@ -1395,11 +1404,13 @@ nfsd_direct_write(struct svc_rqst *rqstp, struct svc_fh *fhp, kiocb->ki_flags |= IOCB_DONTCACHE; } + expected = iov_iter_count(&segments[i].iter); + host_err = vfs_iocb_iter_write(file, kiocb, &segments[i].iter); if (host_err < 0) return host_err; *cnt += host_err; - if (host_err < segments[i].iter.count) + if (host_err < (ssize_t)expected) break; /* partial write */ } diff --git a/fs/nilfs2/bmap.c b/fs/nilfs2/bmap.c index 5f0f1f283af02..83f6ea30cc8b5 100644 --- a/fs/nilfs2/bmap.c +++ b/fs/nilfs2/bmap.c @@ -175,7 +175,7 @@ static int nilfs_bmap_do_delete(struct nilfs_bmap *bmap, __u64 key) return ret; } - return bmap->b_ops->bop_delete(bmap, key); + return bmap->b_ops->bop_delete(bmap, key, false); } /** diff --git a/fs/nilfs2/bmap.h b/fs/nilfs2/bmap.h index 4656df3927228..a72f3c308a5d2 100644 --- a/fs/nilfs2/bmap.h +++ b/fs/nilfs2/bmap.h @@ -63,7 +63,7 @@ struct nilfs_bmap_operations { int (*bop_lookup_contig)(const struct nilfs_bmap *, __u64, __u64 *, unsigned int); int (*bop_insert)(struct nilfs_bmap *, __u64, __u64); - int (*bop_delete)(struct nilfs_bmap *, __u64); + int (*bop_delete)(struct nilfs_bmap *bmap, __u64 key, bool deform); void (*bop_clear)(struct nilfs_bmap *); int (*bop_propagate)(struct nilfs_bmap *, struct buffer_head *); diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c index 64d5f7c5ab440..64bac66af25b4 100644 --- a/fs/nilfs2/btree.c +++ b/fs/nilfs2/btree.c @@ -1425,6 +1425,28 @@ static void nilfs_btree_shrink(struct nilfs_bmap *btree, path[level].bp_bh = NULL; } +/** + * nilfs_btree_discard - discard the last node for the mapping transformation + * @btree: bmap struct of btree + * @path: array of nilfs_btree_path struct + * @level: level of the B-tree node being operated on + * @keyp: argument for passing a key (unused) + * @ptrp: argument for passing a pointer (unused) + */ +static void nilfs_btree_discard(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, int level, + __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *root = nilfs_btree_get_root(btree); + + nilfs_btree_node_delete(root, 0, NULL, NULL, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + nilfs_btree_node_set_level(root, level); + + nilfs_btnode_delete(path[level].bp_bh); + path[level].bp_bh = NULL; +} + static void nilfs_btree_nop(struct nilfs_bmap *btree, struct nilfs_btree_path *path, int level, __u64 *keyp, __u64 *ptrp) @@ -1435,7 +1457,7 @@ static int nilfs_btree_prepare_delete(struct nilfs_bmap *btree, struct nilfs_btree_path *path, int *levelp, struct nilfs_bmap_stats *stats, - struct inode *dat) + struct inode *dat, bool deform) { struct buffer_head *bh; struct nilfs_btree_node *node, *parent, *sib; @@ -1522,15 +1544,17 @@ static int nilfs_btree_prepare_delete(struct nilfs_bmap *btree, if (nilfs_btree_node_get_nchildren(node) - 1 <= NILFS_BTREE_ROOT_NCHILDREN_MAX) { path[level].bp_op = nilfs_btree_shrink; - stats->bs_nblocks += 2; - level++; - path[level].bp_op = nilfs_btree_nop; - goto shrink_root_child; + } else if (deform) { + path[level].bp_op = nilfs_btree_discard; } else { path[level].bp_op = nilfs_btree_do_delete; stats->bs_nblocks++; goto out; } + stats->bs_nblocks += 2; + level++; + path[level].bp_op = nilfs_btree_nop; + goto shrink_root_child; } } @@ -1581,7 +1605,7 @@ static void nilfs_btree_commit_delete(struct nilfs_bmap *btree, nilfs_bmap_set_dirty(btree); } -static int nilfs_btree_delete(struct nilfs_bmap *btree, __u64 key) +static int nilfs_btree_delete(struct nilfs_bmap *btree, __u64 key, bool deform) { struct nilfs_btree_path *path; @@ -1601,7 +1625,8 @@ static int nilfs_btree_delete(struct nilfs_bmap *btree, __u64 key) dat = NILFS_BMAP_USE_VBN(btree) ? nilfs_bmap_get_dat(btree) : NULL; - ret = nilfs_btree_prepare_delete(btree, path, &level, &stats, dat); + ret = nilfs_btree_prepare_delete(btree, path, &level, &stats, dat, + deform); if (ret < 0) goto out; nilfs_btree_commit_delete(btree, path, level, dat); diff --git a/fs/nilfs2/direct.c b/fs/nilfs2/direct.c index 8bd0b1374e25f..b8643d3aa2f88 100644 --- a/fs/nilfs2/direct.c +++ b/fs/nilfs2/direct.c @@ -144,7 +144,7 @@ static int nilfs_direct_insert(struct nilfs_bmap *bmap, __u64 key, __u64 ptr) return ret; } -static int nilfs_direct_delete(struct nilfs_bmap *bmap, __u64 key) +static int nilfs_direct_delete(struct nilfs_bmap *bmap, __u64 key, bool deform) { union nilfs_bmap_ptr_req req; struct inode *dat; @@ -234,7 +234,7 @@ int nilfs_direct_delete_and_convert(struct nilfs_bmap *bmap, /* no need to allocate any resource for conversion */ /* delete */ - ret = bmap->b_ops->bop_delete(bmap, key); + ret = bmap->b_ops->bop_delete(bmap, key, true); if (ret < 0) return ret; diff --git a/fs/nilfs2/ioctl.c b/fs/nilfs2/ioctl.c index b73f2c5d10f09..0957316e58b84 100644 --- a/fs/nilfs2/ioctl.c +++ b/fs/nilfs2/ioctl.c @@ -527,6 +527,7 @@ static int nilfs_ioctl_get_bdescs(struct inode *inode, struct file *filp, * Return: 0 on success, or one of the following negative error codes on * failure: * * %-EEXIST - Block conflict detected. + * * %-EINVAL - Invalid virtual block descriptor. * * %-EIO - I/O error. * * %-ENOENT - Requested block doesn't exist. * * %-ENOMEM - Insufficient memory available. @@ -536,15 +537,30 @@ static int nilfs_ioctl_move_inode_block(struct inode *inode, struct list_head *buffers) { struct buffer_head *bh; + __u64 limit_blkidx = (__u64)inode->i_sb->s_maxbytes >> inode->i_blkbits; int ret; - if (vdesc->vd_flags == 0) + /* + * vblocknr 0 is reserved as an invalid pointer. Also, limit_blkidx + * ensures that the page index converted from vd_vblocknr never + * overflows the page cache limit and respects the architecture's bmap + * key width. + */ + if (unlikely(vdesc->vd_vblocknr == 0 || + vdesc->vd_vblocknr >= limit_blkidx)) + return -EINVAL; + + if (vdesc->vd_flags == 0) { + if (unlikely(vdesc->vd_offset >= limit_blkidx)) + return -EINVAL; + ret = nilfs_gccache_submit_read_data( inode, vdesc->vd_offset, vdesc->vd_blocknr, vdesc->vd_vblocknr, &bh); - else + } else { ret = nilfs_gccache_submit_read_node( inode, vdesc->vd_blocknr, vdesc->vd_vblocknr, &bh); + } if (unlikely(ret < 0)) { if (ret == -ENOENT) diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index a9d8aa65416fd..1d00bce21c377 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c @@ -243,6 +243,7 @@ static void nilfs_copy_folio(struct folio *dst, struct folio *src, int nilfs_copy_dirty_pages(struct address_space *dmap, struct address_space *smap) { + struct inode *smap_inode = smap->host; struct folio_batch fbatch; unsigned int i; pgoff_t index = 0; @@ -258,8 +259,19 @@ int nilfs_copy_dirty_pages(struct address_space *dmap, struct folio *folio = fbatch.folios[i], *dfolio; folio_lock(folio); - if (unlikely(!folio_test_dirty(folio))) - NILFS_FOLIO_BUG(folio, "inconsistent dirty state"); + if (unlikely(!folio_test_dirty(folio))) { + if (WARN_ONCE(!sb_rdonly(smap_inode->i_sb), + "inconsistent dirty state\n")) + goto unlock_folio; + + /* + * If the filesystem has been forced to read-only + * due to metadata corruption. + */ + folio_unlock(folio); + err = -EROFS; + break; + } dfolio = filemap_grab_folio(dmap, folio->index); if (IS_ERR(dfolio)) { @@ -277,6 +289,7 @@ int nilfs_copy_dirty_pages(struct address_space *dmap, folio_unlock(dfolio); folio_put(dfolio); +unlock_folio: folio_unlock(folio); } folio_batch_release(&fbatch); diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index 9332f5ac60833..2189267894d2c 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -2561,6 +2561,10 @@ int nilfs_clean_segments(struct super_block *sb, struct nilfs_argv *argv, break; nilfs_warn(sb, "error %d cleaning segments", err); + + if (unlikely(err == -EROFS)) + goto out_unlock; + set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(sci->sc_interval); } diff --git a/fs/nilfs2/the_nilfs.c b/fs/nilfs2/the_nilfs.c index 7b23e373a106e..f3805e7aabeb5 100644 --- a/fs/nilfs2/the_nilfs.c +++ b/fs/nilfs2/the_nilfs.c @@ -461,6 +461,12 @@ static int nilfs_store_disk_layout(struct the_nilfs *nilfs, nilfs->ns_inode_size); return -EINVAL; } + if (NILFS_SR_BYTES(nilfs->ns_inode_size) > nilfs->ns_blocksize) { + nilfs_err(nilfs->ns_sb, + "too large inode size for super root: %d bytes", + nilfs->ns_inode_size); + return -EINVAL; + } nilfs->ns_first_ino = le32_to_cpu(sbp->s_first_ino); if (nilfs->ns_first_ino < NILFS_USER_INO) { diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index a3555bebad63e..a208a7ec1692a 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -599,9 +599,9 @@ static struct fanotify_event *fanotify_alloc_perm_event(const void *data, pevent->hdr.pad = 0; pevent->hdr.len = 0; pevent->state = FAN_EVENT_INIT; + pevent->watchdog_cnt = 0; pevent->path = *path; - /* NULL ppos means no range info */ - pevent->ppos = range ? &range->pos : NULL; + pevent->pos = range ? range->pos : FANOTIFY_NO_RANGE; pevent->count = range ? range->count : 0; path_get(path); diff --git a/fs/notify/fanotify/fanotify.h b/fs/notify/fanotify/fanotify.h index a0619e7694d57..3710543dbf827 100644 --- a/fs/notify/fanotify/fanotify.h +++ b/fs/notify/fanotify/fanotify.h @@ -428,6 +428,8 @@ FANOTIFY_ME(struct fanotify_event *event) return container_of(event, struct fanotify_mnt_event, fae); } +#define FANOTIFY_NO_RANGE ((loff_t)-1) + /* * Structure for permission fanotify events. It gets allocated and freed in * fanotify_handle_event() since we wait there for user response. When the @@ -438,7 +440,7 @@ FANOTIFY_ME(struct fanotify_event *event) struct fanotify_perm_event { struct fanotify_event fae; struct path path; - const loff_t *ppos; /* optional file range info */ + loff_t pos; /* FANOTIFY_NO_RANGE if unavailable */ size_t count; u32 response; /* userspace answer to the event */ unsigned short state; /* state of the event */ @@ -468,7 +470,7 @@ static inline bool fanotify_event_has_access_range(struct fanotify_event *event) if (!(event->mask & FANOTIFY_PRE_CONTENT_EVENTS)) return false; - return FANOTIFY_PERM(event)->ppos; + return FANOTIFY_PERM(event)->pos != FANOTIFY_NO_RANGE; } static inline struct fanotify_event *FANOTIFY_E(struct fsnotify_event *fse) diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index b604e3da58ad2..a32c6634d5927 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -112,7 +112,12 @@ static DECLARE_DELAYED_WORK(perm_group_work, perm_group_watchdog); static void perm_group_watchdog_schedule(void) { - schedule_delayed_work(&perm_group_work, secs_to_jiffies(perm_group_timeout)); + int timeout = READ_ONCE(perm_group_timeout); + + if (!timeout) + return; + + schedule_delayed_work(&perm_group_work, secs_to_jiffies(timeout)); } static void perm_group_watchdog(struct work_struct *work) @@ -675,12 +680,9 @@ static size_t copy_range_info_to_user(struct fanotify_event *event, if (WARN_ON_ONCE(info_len > count)) return -EFAULT; - if (WARN_ON_ONCE(!pevent->ppos)) - return -EINVAL; - info.hdr.info_type = FAN_EVENT_INFO_TYPE_RANGE; info.hdr.len = info_len; - info.offset = *(pevent->ppos); + info.offset = pevent->pos; info.count = pevent->count; if (copy_to_user(buf, &info, info_len)) @@ -1145,11 +1147,13 @@ static long fanotify_ioctl(struct file *file, unsigned int cmd, unsigned long ar { struct fsnotify_group *group; struct fsnotify_event *fsn_event; + unsigned int info_mode; void __user *p; int ret = -ENOTTY; size_t send_len = 0; group = file->private_data; + info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES); p = (void __user *) arg; @@ -1157,7 +1161,8 @@ static long fanotify_ioctl(struct file *file, unsigned int cmd, unsigned long ar case FIONREAD: spin_lock(&group->notification_lock); list_for_each_entry(fsn_event, &group->notification_list, list) - send_len += FAN_EVENT_METADATA_LEN; + send_len += fanotify_event_len(info_mode, + FANOTIFY_E(fsn_event)); spin_unlock(&group->notification_lock); ret = put_user(send_len, (int __user *) p); break; @@ -1316,16 +1321,18 @@ static bool fanotify_mark_update_flags(struct fsnotify_mark *fsn_mark, static bool fanotify_mark_add_to_mask(struct fsnotify_mark *fsn_mark, __u32 mask, unsigned int fan_flags) { + __u32 old_mask; bool recalc; spin_lock(&fsn_mark->lock); - if (!(fan_flags & FANOTIFY_MARK_IGNORE_BITS)) + if (!(fan_flags & FANOTIFY_MARK_IGNORE_BITS)) { + old_mask = fsn_mark->mask; fsn_mark->mask |= mask; - else + recalc = old_mask != fsn_mark->mask; + } else { fsn_mark->ignore_mask |= mask; - - recalc = fsnotify_calc_mask(fsn_mark) & - ~fsnotify_conn_mask(fsn_mark->connector); + recalc = true; + } recalc |= fanotify_mark_update_flags(fsn_mark, fan_flags); spin_unlock(&fsn_mark->lock); diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index ed37491c16189..5f19c24ec187f 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -539,7 +539,6 @@ static int inotify_update_existing_watch(struct fsnotify_group *group, { struct fsnotify_mark *fsn_mark; struct inotify_inode_mark *i_mark; - __u32 old_mask, new_mask; int replace = !(arg & IN_MASK_ADD); int create = (arg & IN_MASK_CREATE); int ret; @@ -555,27 +554,15 @@ static int inotify_update_existing_watch(struct fsnotify_group *group, i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); spin_lock(&fsn_mark->lock); - old_mask = fsn_mark->mask; if (replace) { fsn_mark->mask = 0; fsn_mark->flags &= ~INOTIFY_MARK_FLAGS; } fsn_mark->mask |= inotify_arg_to_mask(inode, arg); fsn_mark->flags |= inotify_arg_to_flags(arg); - new_mask = fsn_mark->mask; spin_unlock(&fsn_mark->lock); - if (old_mask != new_mask) { - /* more bits in old than in new? */ - int dropped = (old_mask & ~new_mask); - /* more bits in this fsn_mark than the inode's mask? */ - int do_inode = (new_mask & ~READ_ONCE(inode->i_fsnotify_mask)); - - /* update the inode with this new fsn_mark */ - if (dropped || do_inode) - fsnotify_recalc_mask(fsn_mark->connector); - - } + fsnotify_recalc_mask(fsn_mark->connector); /* return the wd */ ret = i_mark->wd; diff --git a/fs/ntfs/attrib.c b/fs/ntfs/attrib.c index 58f32aac5f614..b10ef76c98f33 100644 --- a/fs/ntfs/attrib.c +++ b/fs/ntfs/attrib.c @@ -693,6 +693,8 @@ static bool ntfs_non_resident_attr_value_is_valid(const struct attr_record *a) u32 attr_len; u32 min_len; u16 mp_offset; + u16 name_offset; + u32 name_end; attr_len = le32_to_cpu(a->length); min_len = offsetof(struct attr_record, data.non_resident.initialized_size) + @@ -701,7 +703,27 @@ static bool ntfs_non_resident_attr_value_is_valid(const struct attr_record *a) return false; mp_offset = le16_to_cpu(a->data.non_resident.mapping_pairs_offset); - return mp_offset >= min_len && mp_offset <= attr_len; + if (mp_offset < min_len || mp_offset > attr_len) + return false; + + if (a->name_length) { + name_offset = le16_to_cpu(a->name_offset); + + if (name_offset < min_len || name_offset >= attr_len) + return false; + + name_end = name_offset + a->name_length * sizeof(__le16); + if (name_end > attr_len || name_end > mp_offset) + return false; + } + + /* Ensure there's room for the compressed_size field if needed. */ + if (!(a->flags & (ATTR_IS_SPARSE | ATTR_COMPRESSION_MASK)) && + attr_len - mp_offset < + sizeof(a->data.non_resident.compressed_size)) + return false; + + return true; } static bool ntfs_attr_value_is_valid(struct ntfs_volume *vol, @@ -1710,8 +1732,8 @@ static struct attr_def *ntfs_attr_find_in_attrdef(const struct ntfs_volume *vol, struct attr_def *ad; WARN_ON(!type); - for (ad = vol->attrdef; (u8 *)ad - (u8 *)vol->attrdef < - vol->attrdef_size && ad->type; ++ad) { + for (ad = vol->attrdef; (u8 *)ad - (u8 *)vol->attrdef <= + vol->attrdef_size - (s32)sizeof(*ad) && ad->type; ++ad) { /* We have not found it yet, carry on searching. */ if (likely(le32_to_cpu(ad->type) < le32_to_cpu(type))) continue; @@ -2473,7 +2495,7 @@ int ntfs_resident_attr_record_add(struct ntfs_inode *ni, __le32 type, return offset; put_err_out: ntfs_attr_put_search_ctx(ctx); - return -EIO; + return err; } /* @@ -2612,7 +2634,7 @@ static int ntfs_non_resident_attr_record_add(struct ntfs_inode *ni, __le32 type, return offset; put_err_out: ntfs_attr_put_search_ctx(ctx); - return -1; + return err; } /* @@ -2890,7 +2912,7 @@ int ntfs_attr_add(struct ntfs_inode *ni, __le32 type, attr_ni = NULL; /* Allocate new extent. */ - err = ntfs_mft_record_alloc(ni->vol, 0, &attr_ni, ni, NULL); + err = ntfs_mft_record_alloc(ni->vol, 0, &attr_ni, ni, NULL, -1); if (err) { ntfs_error(sb, "Failed to allocate extent record"); goto err_out; @@ -3523,7 +3545,7 @@ int ntfs_attr_record_move_away(struct ntfs_attr_search_ctx *ctx, int extra) * new extent and move attribute to it. */ ni = NULL; - err = ntfs_mft_record_alloc(base_ni->vol, 0, &ni, base_ni, NULL); + err = ntfs_mft_record_alloc(base_ni->vol, 0, &ni, base_ni, NULL, -1); if (err) { ntfs_error(sb, "Couldn't allocate MFT record, err : %d", err); return err; @@ -3531,8 +3553,13 @@ int ntfs_attr_record_move_away(struct ntfs_attr_search_ctx *ctx, int extra) unmap_mft_record(ni); err = ntfs_attr_record_move_to(ctx, ni); - if (err) + if (err) { ntfs_error(sb, "Couldn't move attribute to MFT record"); + if (ntfs_mft_record_free(base_ni->vol, ni)) + ntfs_error(sb, "Couldn't free empty MFT record"); + else + ntfs_inode_close(ni); + } return err; } @@ -3542,7 +3569,8 @@ int ntfs_attr_record_move_away(struct ntfs_attr_search_ctx *ctx, int extra) * update allocated and compressed size. */ static int ntfs_attr_update_meta(struct attr_record *a, struct ntfs_inode *ni, - struct mft_record *m, struct ntfs_attr_search_ctx *ctx) + struct mft_record *m, struct ntfs_attr_search_ctx *ctx, + struct ntfs_inode *locked_ni, bool defer_attrlist) { int sparse, err = 0; struct ntfs_inode *base_ni; @@ -3578,6 +3606,8 @@ static int ntfs_attr_update_meta(struct attr_record *a, struct ntfs_inode *ni, le16_to_cpu(a->data.non_resident.mapping_pairs_offset) == 8) && !(le32_to_cpu(m->bytes_allocated) - le32_to_cpu(m->bytes_in_use))) { + if (defer_attrlist) + return -ENOSPC; if (!NInoAttrList(base_ni)) { err = ntfs_inode_add_attrlist(base_ni); if (err) @@ -3591,7 +3621,7 @@ static int ntfs_attr_update_meta(struct attr_record *a, struct ntfs_inode *ni, goto out; } - err = ntfs_attrlist_update(base_ni); + err = ntfs_attrlist_update_locked(base_ni, locked_ni); if (err) goto out; err = -EAGAIN; @@ -3671,6 +3701,8 @@ static int ntfs_attr_update_meta(struct attr_record *a, struct ntfs_inode *ni, * ntfs_attr_update_mapping_pairs - update mapping pairs for ntfs attribute * @ni: non-resident ntfs inode for which we need update * @from_vcn: update runlist starting this VCN + * @locked_ni: inode whose runlist write lock is already held + * @defer_attrlist: return -ENOSPC instead of updating an attribute list * * Build mapping pairs from @na->rl and write them to the disk. Also, this * function updates sparse bit, allocated and compressed size (allocates/frees @@ -3680,7 +3712,10 @@ static int ntfs_attr_update_meta(struct attr_record *a, struct ntfs_inode *ni, * call to this function. Vice-versa @na->compressed_size will be calculated and * set to correct value during this function. */ -int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn) +static int __ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, + s64 from_vcn, + struct ntfs_inode *locked_ni, + bool defer_attrlist) { struct ntfs_attr_search_ctx *ctx; struct ntfs_inode *base_ni; @@ -3772,7 +3807,8 @@ int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn) continue; } - err = ntfs_attr_update_meta(a, ni, m, ctx); + err = ntfs_attr_update_meta(a, ni, m, ctx, locked_ni, + defer_attrlist); if (err < 0) { if (err == -EAGAIN) { ntfs_attr_put_search_ctx(ctx); @@ -3812,18 +3848,28 @@ int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn) */ if (ni->type == AT_ATTRIBUTE_LIST) { ntfs_attr_put_search_ctx(ctx); - if (ntfs_inode_free_space(base_ni, mp_size - - cur_max_mp_size)) { - ntfs_debug("Attribute list is too big. Defragment the volume\n"); - return -ENOSPC; + ctx = NULL; + if (locked_ni == ni || defer_attrlist) { + err = -ENOSPC; + goto put_err_out; } - if (ntfs_attrlist_update(base_ni)) - return -EIO; + err = ntfs_inode_free_space(base_ni, mp_size - + cur_max_mp_size); + if (err) + return err; + err = ntfs_attrlist_update_locked( + base_ni, locked_ni); + if (err) + return err; goto retry; } /* Add attribute list if it isn't present, and retry. */ if (!NInoAttrList(base_ni)) { + if (defer_attrlist) { + err = -ENOSPC; + goto put_err_out; + } ntfs_attr_put_search_ctx(ctx); if (ntfs_inode_add_attrlist(base_ni)) { ntfs_error(sb, "Can not add attrlist"); @@ -3851,13 +3897,21 @@ int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn) } } + if (defer_attrlist && + (ctx->ntfs_ino->nr_extents == -1 || + NInoAttrList(ctx->ntfs_ino)) && + ctx->attr->type != AT_ATTRIBUTE_LIST) { + err = -ENOSPC; + goto put_err_out; + } + /* Update lowest vcn. */ a->data.non_resident.lowest_vcn = cpu_to_le64(stop_vcn); mark_mft_record_dirty(ctx->ntfs_ino); if ((ctx->ntfs_ino->nr_extents == -1 || NInoAttrList(ctx->ntfs_ino)) && ctx->attr->type != AT_ATTRIBUTE_LIST) { ctx->al_entry->lowest_vcn = cpu_to_le64(stop_vcn); - err = ntfs_attrlist_update(base_ni); + err = ntfs_attrlist_update_locked(base_ni, locked_ni); if (err) goto put_err_out; } @@ -3944,7 +3998,10 @@ int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn) unsigned int de_cnt = 0; /* Allocate new mft record. */ - err = ntfs_mft_record_alloc(ni->vol, 0, &ext_ni, base_ni, NULL); + err = ntfs_mft_record_alloc(ni->vol, 0, &ext_ni, base_ni, NULL, + base_ni->mft_no == FILE_MFT && + ni->type == AT_DATA && + ni->name == AT_UNNAMED ? stop_vcn : -1); if (err) { ntfs_error(sb, "Failed to allocate extent record"); goto put_err_out; @@ -4029,6 +4086,19 @@ int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn) return err; } +int ntfs_attr_update_mapping_pairs_locked(struct ntfs_inode *ni, + s64 from_vcn, + struct ntfs_inode *locked_ni) +{ + return __ntfs_attr_update_mapping_pairs(ni, from_vcn, locked_ni, + false); +} + +int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn) +{ + return ntfs_attr_update_mapping_pairs_locked(ni, from_vcn, NULL); +} + /* * ntfs_attr_make_resident - convert a non-resident to a resident attribute * @ni: open ntfs attribute to make resident @@ -4162,7 +4232,9 @@ static int ntfs_attr_make_resident(struct ntfs_inode *ni, struct ntfs_attr_searc * * Reduce the size of a non-resident, open ntfs attribute @na to @newsize bytes. */ -static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsize) +static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, + const s64 newsize, + struct ntfs_inode *locked_ni) { struct ntfs_volume *vol; struct ntfs_attr_search_ctx *ctx; @@ -4170,6 +4242,7 @@ static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsiz s64 nr_freed_clusters; int err; struct ntfs_inode *base_ni; + bool runlist_locked = locked_ni == ni; ntfs_debug("Inode 0x%llx attr 0x%x new size %lld\n", (unsigned long long)ni->mft_no, ni->type, (long long)newsize); @@ -4215,18 +4288,24 @@ static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsiz * clusters if there is a change. */ if (ntfs_bytes_to_cluster(vol, ni->allocated_size) != first_free_vcn) { - struct ntfs_attr_search_ctx *ctx; + /* + * ntfs_cluster_free() and ntfs_rl_truncate_nolock() + * both require this lock. + */ + if (!runlist_locked) + down_write(&ni->runlist.lock); err = ntfs_attr_map_whole_runlist(ni); if (err) { ntfs_debug("Eeek! ntfs_attr_map_whole_runlist failed.\n"); - return err; + goto unlock_runlist; } ctx = ntfs_attr_get_search_ctx(ni, NULL); if (!ctx) { ntfs_error(vol->sb, "%s: Failed to get search context", __func__); - return -ENOMEM; + err = -ENOMEM; + goto unlock_runlist; } /* Deallocate all clusters starting with the first free one. */ @@ -4234,7 +4313,8 @@ static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsiz if (nr_freed_clusters < 0) { ntfs_debug("Eeek! Freeing of clusters failed. Aborting...\n"); ntfs_attr_put_search_ctx(ctx); - return (int)nr_freed_clusters; + err = (int)nr_freed_clusters; + goto unlock_runlist; } ntfs_attr_put_search_ctx(ctx); @@ -4247,7 +4327,8 @@ static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsiz kvfree(ni->runlist.rl); ni->runlist.rl = NULL; ntfs_error(vol->sb, "Eeek! Run list truncation failed.\n"); - return -EIO; + err = -EIO; + goto unlock_runlist; } /* Prepare to mapping pairs update. */ @@ -4263,11 +4344,13 @@ static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsiz VFS_I(base_ni)->i_blocks = ni->allocated_size >> 9; /* Write mapping pairs for new runlist. */ - err = ntfs_attr_update_mapping_pairs(ni, 0 /*first_free_vcn*/); + err = ntfs_attr_update_mapping_pairs_locked(ni, 0, ni); if (err) { ntfs_debug("Eeek! Mapping pairs update failed. Leaving inconstant metadata. Run chkdsk.\n"); - return err; + goto unlock_runlist; } + if (!runlist_locked) + up_write(&ni->runlist.lock); } /* Get the first attribute record. */ @@ -4309,7 +4392,11 @@ static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsiz /* If the attribute now has zero size, make it resident. */ if (!newsize && !NInoEncrypted(ni) && !NInoCompressed(ni)) { + if (!runlist_locked) + down_write(&ni->runlist.lock); err = ntfs_attr_make_resident(ni, ctx); + if (!runlist_locked) + up_write(&ni->runlist.lock); if (err) { /* If couldn't make resident, just continue. */ if (err != -EPERM) @@ -4326,6 +4413,11 @@ static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsiz put_err_out: ntfs_attr_put_search_ctx(ctx); return err; + +unlock_runlist: + if (!runlist_locked) + up_write(&ni->runlist.lock); + return err; } /* @@ -4334,13 +4426,14 @@ static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsiz * @prealloc_size: preallocation size (in bytes) to which to expand the attribute * @newsize: new size (in bytes) to which to expand the attribute * @holes: how to create a hole if expanding - * @need_lock: whether mrec lock is needed or not + * @locked_ni: inode whose runlist lock is already held * * Expand the size of a non-resident, open ntfs attribute @na to @newsize bytes, * by allocating new clusters. */ static int ntfs_non_resident_attr_expand(struct ntfs_inode *ni, const s64 newsize, - const s64 prealloc_size, unsigned int holes, bool need_lock) + const s64 prealloc_size, unsigned int holes, + struct ntfs_inode *locked_ni) { s64 lcn_seek_from; s64 first_free_vcn; @@ -4487,13 +4580,39 @@ static int ntfs_non_resident_attr_expand(struct ntfs_inode *ni, const s64 newsiz ntfs_bytes_to_cluster(vol, ni->allocated_size), first_free_vcn - ntfs_bytes_to_cluster(vol, ni->allocated_size), - lcn_seek_from, DATA_ZONE, false, false, false); + lcn_seek_from, DATA_ZONE, false, + ni->type == AT_ATTRIBUTE_LIST, false); if (IS_ERR(rl)) { ntfs_debug("Cluster allocation failed (%lld)", (long long)first_free_vcn - ntfs_bytes_to_cluster(vol, ni->allocated_size)); return PTR_ERR(rl); } + /* + * A contiguous ATTRIBUTE_LIST allocation keeps its mapping + * pairs small enough to fit in the base MFT record. The + * allocator can return a short run when contiguity was + * requested, so discard it and retry normally if necessary. + */ + if (ni->type == AT_ATTRIBUTE_LIST && + (rl->vcn != ntfs_bytes_to_cluster(vol, + ni->allocated_size) || + rl->length != first_free_vcn - + ntfs_bytes_to_cluster(vol, ni->allocated_size) || + rl[1].length)) { + ntfs_cluster_free_from_rl(vol, rl); + kvfree(rl); + rl = ntfs_cluster_alloc(vol, + ntfs_bytes_to_cluster(vol, + ni->allocated_size), + first_free_vcn - + ntfs_bytes_to_cluster(vol, + ni->allocated_size), + lcn_seek_from, DATA_ZONE, false, + false, false); + if (IS_ERR(rl)) + return PTR_ERR(rl); + } } if (!NInoCompressed(ni)) { @@ -4512,7 +4631,8 @@ static int ntfs_non_resident_attr_expand(struct ntfs_inode *ni, const s64 newsiz /* Prepare to mapping pairs update. */ ni->allocated_size = ntfs_cluster_to_bytes(vol, first_free_vcn); - err = ntfs_attr_update_mapping_pairs(ni, 0); + err = ntfs_attr_update_mapping_pairs_locked( + ni, 0, locked_ni); if (err) { ntfs_debug("Mapping pairs update failed"); goto rollback; @@ -4556,11 +4676,11 @@ static int ntfs_non_resident_attr_expand(struct ntfs_inode *ni, const s64 newsiz ntfs_debug("Leaking clusters"); /* Now, truncate the runlist itself. */ - if (need_lock) + if (ni != locked_ni) down_write(&ni->runlist.lock); err2 = ntfs_rl_truncate_nolock(vol, &ni->runlist, ntfs_bytes_to_cluster(vol, org_alloc_size)); - if (need_lock) + if (ni != locked_ni) up_write(&ni->runlist.lock); if (err2) { /* @@ -4574,11 +4694,11 @@ static int ntfs_non_resident_attr_expand(struct ntfs_inode *ni, const s64 newsiz /* Prepare to mapping pairs update. */ ni->allocated_size = org_alloc_size; /* Restore mapping pairs. */ - if (need_lock) + if (ni != locked_ni) down_read(&ni->runlist.lock); - if (ntfs_attr_update_mapping_pairs(ni, 0)) + if (__ntfs_attr_update_mapping_pairs(ni, 0, locked_ni, true)) ntfs_error(sb, "Failed to restore old mapping pairs"); - if (need_lock) + if (ni != locked_ni) up_read(&ni->runlist.lock); if (NInoSparse(ni) || NInoCompressed(ni)) { @@ -4683,7 +4803,8 @@ static int ntfs_resident_attr_resize(struct ntfs_inode *attr_ni, const s64 newsi mark_mft_record_dirty(ctx->ntfs_ino); ntfs_attr_put_search_ctx(ctx); /* Resize non-resident attribute */ - return ntfs_non_resident_attr_expand(attr_ni, newsize, prealloc_size, holes, true); + return ntfs_non_resident_attr_expand( + attr_ni, newsize, prealloc_size, holes, NULL); } else if (err != -ENOSPC && err != -EPERM) { ntfs_error(sb, "Failed to make attribute non-resident"); goto put_err_out; @@ -4804,7 +4925,7 @@ static int ntfs_resident_attr_resize(struct ntfs_inode *attr_ni, const s64 newsi } /* Allocate new mft record. */ - err = ntfs_mft_record_alloc(base_ni->vol, 0, &ext_ni, base_ni, NULL); + err = ntfs_mft_record_alloc(base_ni->vol, 0, &ext_ni, base_ni, NULL, -1); if (err) { ntfs_error(sb, "Couldn't allocate MFT record"); goto put_err_out; @@ -4858,13 +4979,14 @@ int __ntfs_attr_truncate_vfs(struct ntfs_inode *ni, const s64 newsize, if (NInoNonResident(ni)) { if (newsize > i_size) { down_write(&ni->runlist.lock); - err = ntfs_non_resident_attr_expand(ni, newsize, 0, - NVolDisableSparse(ni->vol) ? - HOLES_NO : HOLES_OK, - false); + err = ntfs_non_resident_attr_expand( + ni, newsize, 0, + NVolDisableSparse(ni->vol) ? + HOLES_NO : HOLES_OK, ni); up_write(&ni->runlist.lock); } else - err = ntfs_non_resident_attr_shrink(ni, newsize); + err = ntfs_non_resident_attr_shrink( + ni, newsize, NULL); } else err = ntfs_resident_attr_resize(ni, newsize, 0, NVolDisableSparse(ni->vol) ? @@ -4873,7 +4995,9 @@ int __ntfs_attr_truncate_vfs(struct ntfs_inode *ni, const s64 newsize, return err; } -int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, const s64 prealloc_size) +int ntfs_attr_expand_locked(struct ntfs_inode *ni, const s64 newsize, + const s64 prealloc_size, + struct ntfs_inode *locked_ni) { int err = 0; @@ -4886,7 +5010,8 @@ int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, const s64 preallo ntfs_debug("Entering for inode 0x%llx, attr 0x%x, size %lld\n", (unsigned long long)ni->mft_no, ni->type, newsize); - if (ni->data_size == newsize) { + if (ni->data_size == newsize && + (!prealloc_size || prealloc_size <= ni->allocated_size)) { ntfs_debug("Size is already ok\n"); return 0; } @@ -4901,10 +5026,11 @@ int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, const s64 preallo } if (NInoNonResident(ni)) { - if (newsize > ni->data_size) - err = ntfs_non_resident_attr_expand(ni, newsize, prealloc_size, - NVolDisableSparse(ni->vol) ? - HOLES_NO : HOLES_OK, true); + if (newsize > ni->data_size || prealloc_size > ni->allocated_size) + err = ntfs_non_resident_attr_expand( + ni, newsize, prealloc_size, + NVolDisableSparse(ni->vol) ? + HOLES_NO : HOLES_OK, locked_ni); } else err = ntfs_resident_attr_resize(ni, newsize, prealloc_size, NVolDisableSparse(ni->vol) ? @@ -4915,6 +5041,12 @@ int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, const s64 preallo return err; } +int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, + const s64 prealloc_size) +{ + return ntfs_attr_expand_locked(ni, newsize, prealloc_size, NULL); +} + /* * ntfs_attr_truncate_i - resize an ntfs attribute * @ni: open ntfs inode to resize @@ -4927,7 +5059,9 @@ int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, const s64 preallo * newly allocated space is marked as not initialised and no real allocation * on disk is performed. */ -int ntfs_attr_truncate_i(struct ntfs_inode *ni, const s64 newsize, unsigned int holes) +int ntfs_attr_truncate_i_locked(struct ntfs_inode *ni, const s64 newsize, + unsigned int holes, + struct ntfs_inode *locked_ni) { int err; @@ -4961,15 +5095,23 @@ int ntfs_attr_truncate_i(struct ntfs_inode *ni, const s64 newsize, unsigned int if (NInoNonResident(ni)) { if (newsize > ni->data_size) - err = ntfs_non_resident_attr_expand(ni, newsize, 0, holes, true); + err = ntfs_non_resident_attr_expand( + ni, newsize, 0, holes, locked_ni); else - err = ntfs_non_resident_attr_shrink(ni, newsize); + err = ntfs_non_resident_attr_shrink( + ni, newsize, locked_ni); } else err = ntfs_resident_attr_resize(ni, newsize, 0, holes); ntfs_debug("Return status %d\n", err); return err; } +int ntfs_attr_truncate_i(struct ntfs_inode *ni, const s64 newsize, + unsigned int holes) +{ + return ntfs_attr_truncate_i_locked(ni, newsize, holes, NULL); +} + /* * Resize an attribute, creating a hole if relevant */ @@ -4987,10 +5129,11 @@ int ntfs_attr_map_cluster(struct ntfs_inode *ni, s64 vcn_start, s64 *lcn_start, struct ntfs_volume *vol = ni->vol; struct ntfs_attr_search_ctx *ctx; struct runlist_element *rl, *rlc; + struct runlist_element *old_rl = NULL; s64 vcn = vcn_start, lcn, clu_count; s64 lcn_seek_from = -1; int err = 0; - size_t new_rl_count; + size_t new_rl_count, old_rl_count; err = ntfs_attr_map_whole_runlist(ni); if (err) @@ -5083,6 +5226,19 @@ int ntfs_attr_map_cluster(struct ntfs_inode *ni, s64 vcn_start, s64 *lcn_start, WARN_ON(rlc->vcn != vcn); lcn = rlc->lcn; clu_count = rlc->length; + old_rl_count = ni->runlist.count; + old_rl = kmemdup(ni->runlist.rl, + old_rl_count * sizeof(*old_rl), GFP_NOFS); + if (!old_rl) { + err = -ENOMEM; + if (ntfs_cluster_free_from_rl(vol, rlc)) { + ntfs_error(vol->sb, + "Failed to free cluster allocation after runlist backup failure."); + NVolSetErrors(vol); + } + kvfree(rlc); + goto out; + } rl = ntfs_runlists_merge(&ni->runlist, rlc, 0, &new_rl_count); if (IS_ERR(rl)) { @@ -5106,15 +5262,32 @@ int ntfs_attr_map_cluster(struct ntfs_inode *ni, s64 vcn_start, s64 *lcn_start, if (update_mp) { ntfs_attr_reinit_search_ctx(ctx); - err = ntfs_attr_update_mapping_pairs(ni, 0); + err = ntfs_attr_update_mapping_pairs_locked(ni, 0, ni); if (err) { int err2; err2 = ntfs_cluster_free(ni, vcn, clu_count, ctx); - if (err2 < 0) + if (err2 < 0 || err2 != clu_count) { ntfs_error(vol->sb, - "Failed to free cluster allocation. Leaving inconstant metadata.\n"); - goto out; + "Failed to free cluster allocation. Leaving inconsistent metadata.\n"); + NVolSetErrors(vol); + goto out; + } + + /* + * Restore the runlist before repairing the on-disk + * mapping pairs. + */ + kvfree(ni->runlist.rl); + ni->runlist.rl = old_rl; + ni->runlist.count = old_rl_count; + old_rl = NULL; + if (ntfs_attr_update_mapping_pairs_locked( + ni, 0, ni)) { + ntfs_error(vol->sb, + "Failed to restore mapping pairs after allocation rollback.\n"); + NVolSetErrors(vol); + } } } else { VFS_I(ni)->i_blocks += clu_count << (vol->cluster_size_bits - 9); @@ -5126,6 +5299,7 @@ int ntfs_attr_map_cluster(struct ntfs_inode *ni, s64 vcn_start, s64 *lcn_start, *lcn_count = clu_count; *balloc = true; out: + kvfree(old_rl); ntfs_attr_put_search_ctx(ctx); return err; } @@ -5363,7 +5537,7 @@ int ntfs_non_resident_attr_insert_range(struct ntfs_inode *ni, s64 start_vcn, s6 ni->data_size += ntfs_cluster_to_bytes(vol, len); if (ntfs_cluster_to_bytes(vol, start_vcn) < ni->initialized_size) ni->initialized_size += ntfs_cluster_to_bytes(vol, len); - ret = ntfs_attr_update_mapping_pairs(ni, 0); + ret = ntfs_attr_update_mapping_pairs_locked(ni, 0, ni); up_write(&ni->runlist.lock); if (ret) return ret; @@ -5448,7 +5622,7 @@ int ntfs_non_resident_attr_collapse_range(struct ntfs_inode *ni, s64 start_vcn, } if (ni->allocated_size > 0) { - ret = ntfs_attr_update_mapping_pairs(ni, 0); + ret = ntfs_attr_update_mapping_pairs_locked(ni, 0, ni); if (ret) { up_write(&ni->runlist.lock); goto out_rl; @@ -5526,7 +5700,7 @@ int ntfs_non_resident_attr_punch_hole(struct ntfs_inode *ni, s64 start_vcn, s64 ni->runlist.rl = rl; ni->runlist.count = new_rl_count; - ret = ntfs_attr_update_mapping_pairs(ni, 0); + ret = ntfs_attr_update_mapping_pairs_locked(ni, 0, ni); up_write(&ni->runlist.lock); if (ret) { kvfree(punch_rl); @@ -5666,7 +5840,7 @@ int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bo lcn << vol->cluster_size_bits, alloc_cnt << vol->cluster_size_bits); - if (err > 0) + if (err) goto out; } @@ -5702,7 +5876,7 @@ int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bo if (NInoRunlistDirty(ni)) { mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); down_write(&ni->runlist.lock); - err = ntfs_attr_update_mapping_pairs(ni, 0); + err = ntfs_attr_update_mapping_pairs_locked(ni, 0, ni); if (err) ntfs_error(ni->vol->sb, "Updating mapping pairs failed"); else diff --git a/fs/ntfs/attrib.h b/fs/ntfs/attrib.h index e2224fbfaabe9..6b4fa9f576401 100644 --- a/fs/ntfs/attrib.h +++ b/fs/ntfs/attrib.h @@ -112,7 +112,13 @@ int ntfs_non_resident_attr_punch_hole(struct ntfs_inode *ni, s64 start_vcn, s64 int __ntfs_attr_truncate_vfs(struct ntfs_inode *ni, const s64 newsize, const s64 i_size); int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, const s64 prealloc_size); +int ntfs_attr_expand_locked(struct ntfs_inode *ni, const s64 newsize, + const s64 prealloc_size, + struct ntfs_inode *locked_ni); int ntfs_attr_truncate_i(struct ntfs_inode *ni, const s64 newsize, unsigned int holes); +int ntfs_attr_truncate_i_locked(struct ntfs_inode *ni, const s64 newsize, + unsigned int holes, + struct ntfs_inode *locked_ni); int ntfs_attr_truncate(struct ntfs_inode *ni, const s64 newsize); int ntfs_attr_rm(struct ntfs_inode *ni); int ntfs_attr_exist(struct ntfs_inode *ni, const __le32 type, __le16 *name, @@ -133,6 +139,9 @@ int ntfs_resident_attr_record_add(struct ntfs_inode *ni, __le32 type, __le16 *name, u8 name_len, u8 *val, u32 size, __le16 flags); int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn); +int ntfs_attr_update_mapping_pairs_locked(struct ntfs_inode *ni, + s64 from_vcn, + struct ntfs_inode *locked_ni); struct runlist_element *ntfs_attr_vcn_to_rl(struct ntfs_inode *ni, s64 vcn, s64 *lcn); /* diff --git a/fs/ntfs/attrlist.c b/fs/ntfs/attrlist.c index be3086d343381..bb191953dcb1b 100644 --- a/fs/ntfs/attrlist.c +++ b/fs/ntfs/attrlist.c @@ -12,6 +12,9 @@ #include "mft.h" #include "attrib.h" #include "attrlist.h" +#include "lcnalloc.h" + +#define NTFS_MAX_ATTR_LIST_SIZE (256 * 1024) /* * ntfs_attrlist_need - check whether inode need attribute list @@ -51,11 +54,155 @@ int ntfs_attrlist_need(struct ntfs_inode *ni) return 0; } -int ntfs_attrlist_update(struct ntfs_inode *base_ni) +/* + * Repack the $MFT/$ATTRIBUTE_LIST data into one run. + * + * The mapping pairs for an $ATTRIBUTE_LIST must remain in the base MFT + * record. Once that record has no room left, extending a fragmented list + * can require one more mapping-pairs byte than the record can hold. There + * is no attribute that can legally be moved out in that state: $STANDARD_ + * INFORMATION, $ATTRIBUTE_LIST, and the first $MFT/$DATA extent all have to + * stay in the base record. Move the list data to one contiguous run. The + * caller supplies the minimum allocation size so a recovery can use the + * smallest useful run while normal updates can still request the maximum + * legal list size as a reserve. + */ +static int ntfs_attrlist_repack(struct inode *attr_vi, + struct ntfs_inode *attr_ni, s64 min_alloc_size, + struct ntfs_inode *locked_ni) +{ + struct ntfs_volume *vol = attr_ni->vol; + struct runlist_element *old_rl, *new_rl; + u8 *data = NULL; + s64 data_size, alloc_size, nr_clusters, written; + s64 old_alloc_size; + size_t old_rl_count, new_rl_count; + unsigned long flags; + int err, restore_err; + if (attr_ni->mft_no != FILE_MFT || !NInoNonResident(attr_ni) || + min_alloc_size < 0) + return -EINVAL; + /* The buffered I/O below can reacquire the attribute runlist lock. */ + if (attr_ni == locked_ni) + return -ENOSPC; + + err = ntfs_attr_map_whole_runlist(attr_ni); + if (err) + return err; + + data_size = attr_ni->data_size; + if (data_size < 0) + return -EIO; + + if (data_size) { + data = kvmalloc(data_size, GFP_NOFS); + if (!data) + return -ENOMEM; + + written = ntfs_inode_attr_pread(attr_vi, 0, data_size, data); + if (written != data_size) { + err = written < 0 ? (int)written : -EIO; + goto out_free_data; + } + } + + old_alloc_size = attr_ni->allocated_size; + alloc_size = max_t(s64, old_alloc_size, min_alloc_size); + nr_clusters = ntfs_bytes_to_cluster(vol, + alloc_size + vol->cluster_size - 1); + if (nr_clusters <= 0) { + err = -EFBIG; + goto out_free_data; + } + + /* A single run keeps the mapping pairs at the minimum size. */ + new_rl = ntfs_cluster_alloc(vol, 0, nr_clusters, -1, DATA_ZONE, + true, true, false); + if (IS_ERR(new_rl)) { + err = PTR_ERR(new_rl); + goto out_free_data; + } + + new_rl_count = 0; + if (new_rl->vcn == 0 && new_rl->length == nr_clusters && + !new_rl[1].length) + new_rl_count = 2; + + if (new_rl_count != 2) { + ntfs_cluster_free_from_rl(vol, new_rl); + kvfree(new_rl); + err = -ENOSPC; + goto out_free_data; + } + old_rl = attr_ni->runlist.rl; + old_rl_count = attr_ni->runlist.count; + down_write(&attr_ni->runlist.lock); + attr_ni->runlist.rl = new_rl; + attr_ni->runlist.count = new_rl_count; + up_write(&attr_ni->runlist.lock); + + write_lock_irqsave(&attr_ni->size_lock, flags); + attr_ni->allocated_size = ntfs_cluster_to_bytes(vol, nr_clusters); + write_unlock_irqrestore(&attr_ni->size_lock, flags); + + /* Populate the replacement extent before publishing its mapping pairs. */ + if (data_size) { + written = ntfs_inode_attr_pwrite(attr_vi, 0, data_size, data, true); + if (written != data_size) { + err = written < 0 ? (int)written : -EIO; + goto restore_old_runlist; + } + } + + err = ntfs_attr_update_mapping_pairs_locked(attr_ni, 0, locked_ni); + if (err) + goto restore_old_runlist; + + /* The new mapping is now authoritative; release the old data runs. */ + if (ntfs_cluster_free_from_rl(vol, old_rl)) { + ntfs_error(vol->sb, + "Failed to free old ATTRIBUTE_LIST extent: inode %#llx", + (long long)attr_ni->mft_no); + NVolSetErrors(vol); + } + kvfree(old_rl); + kvfree(data); + return 0; + +restore_old_runlist: + down_write(&attr_ni->runlist.lock); + attr_ni->runlist.rl = old_rl; + attr_ni->runlist.count = old_rl_count; + up_write(&attr_ni->runlist.lock); + + write_lock_irqsave(&attr_ni->size_lock, flags); + attr_ni->allocated_size = old_alloc_size; + write_unlock_irqrestore(&attr_ni->size_lock, flags); + + restore_err = ntfs_attr_update_mapping_pairs_locked( + attr_ni, 0, locked_ni); + if (restore_err) { + ntfs_error(vol->sb, "Failed to restore ATTRIBUTE_LIST mapping pairs (%d)", + restore_err); + NVolSetErrors(vol); + } + + ntfs_cluster_free_from_rl(vol, new_rl); + kvfree(new_rl); + err = err ? err : restore_err; + +out_free_data: + kvfree(data); + return err; +} + +int ntfs_attrlist_update_locked(struct ntfs_inode *base_ni, + struct ntfs_inode *locked_ni) { struct inode *attr_vi; struct ntfs_inode *attr_ni; - int err; + s64 written; + int err, retry_err; /* * generic_shutdown_super() clears SB_ACTIVE before evicting cached @@ -72,23 +219,66 @@ int ntfs_attrlist_update(struct ntfs_inode *base_ni) return err; } attr_ni = NTFS_I(attr_vi); + /* Truncation and page-cache writes can reacquire this runlist lock. */ + if (attr_ni == locked_ni) { + iput(attr_vi); + return -ENOSPC; + } - err = ntfs_attr_truncate_i(attr_ni, base_ni->attr_list_size, HOLES_NO); - if (err == -ENOSPC && attr_ni->mft_no == FILE_MFT) { - err = ntfs_attr_truncate(attr_ni, 0); - if (err || ntfs_attr_truncate_i(attr_ni, base_ni->attr_list_size, HOLES_NO) != 0) { + err = ntfs_attr_truncate_i_locked( + attr_ni, base_ni->attr_list_size, HOLES_NO, locked_ni); + if (err == -ENOSPC && attr_ni->mft_no == FILE_MFT && + NInoNonResident(attr_ni)) { + retry_err = ntfs_attrlist_repack(attr_vi, attr_ni, + base_ni->attr_list_size, locked_ni); + if (retry_err) { + ntfs_error(base_ni->vol->sb, "Failed to repack attribute list"); iput(attr_vi); + return retry_err; + } + + retry_err = ntfs_attr_truncate_i_locked( + attr_ni, base_ni->attr_list_size, + HOLES_NO, locked_ni); + if (retry_err) { ntfs_error(base_ni->vol->sb, - "Failed to truncate attribute list of inode %#llx", - (long long)base_ni->mft_no); - return -EIO; + "Failed to resize attribute list after repack"); + iput(attr_vi); + return retry_err; } } else if (err) { iput(attr_vi); ntfs_error(base_ni->vol->sb, "Failed to truncate attribute list of inode %#llx", (long long)base_ni->mft_no); - return -EIO; + return err; + } + + /* + * Reserve the maximum legal list size while the MFT metadata area is + * still easy to allocate contiguously. This prevents a later list entry + * from needing another mapping-pairs byte in the full base MFT record. + * Failure to obtain the optional reserve must not reject the current + * metadata update; the repack retry above remains available if needed. + */ + if (base_ni->mft_no == FILE_MFT && NInoNonResident(attr_ni) && + attr_ni->allocated_size < NTFS_MAX_ATTR_LIST_SIZE) { + retry_err = ntfs_attr_expand_locked( + attr_ni, base_ni->attr_list_size, + NTFS_MAX_ATTR_LIST_SIZE, locked_ni); + if (retry_err == -ENOSPC) { + retry_err = ntfs_attrlist_repack( + attr_vi, attr_ni, + NTFS_MAX_ATTR_LIST_SIZE, locked_ni); + if (retry_err == -ENOSPC) + retry_err = 0; + } + if (retry_err) { + ntfs_error(base_ni->vol->sb, + "Failed to reserve attribute list space"); + iput(attr_vi); + return retry_err; + } } i_size_write(attr_vi, base_ni->attr_list_size); @@ -96,14 +286,15 @@ int ntfs_attrlist_update(struct ntfs_inode *base_ni) if (NInoNonResident(attr_ni) && !NInoAttrListNonResident(base_ni)) NInoSetAttrListNonResident(base_ni); - if (ntfs_inode_attr_pwrite(attr_vi, 0, base_ni->attr_list_size, - base_ni->attr_list, false) != - base_ni->attr_list_size) { + written = ntfs_inode_attr_pwrite(attr_vi, 0, base_ni->attr_list_size, + base_ni->attr_list, false); + if (written != base_ni->attr_list_size) { + err = written < 0 ? (int)written : -EIO; iput(attr_vi); ntfs_error(base_ni->vol->sb, "Failed to write attribute list of inode %#llx", (long long)base_ni->mft_no); - return -EIO; + return err; } NInoSetAttrListDirty(base_ni); @@ -111,6 +302,11 @@ int ntfs_attrlist_update(struct ntfs_inode *base_ni) return 0; } +int ntfs_attrlist_update(struct ntfs_inode *base_ni) +{ + return ntfs_attrlist_update_locked(base_ni, NULL); +} + /* * ntfs_attrlist_entry_add - add an attribute list attribute entry * @ni: opened ntfs inode, which contains that attribute diff --git a/fs/ntfs/attrlist.h b/fs/ntfs/attrlist.h index 1892a3934d3ab..10cc2cc8e2081 100644 --- a/fs/ntfs/attrlist.h +++ b/fs/ntfs/attrlist.h @@ -16,5 +16,7 @@ int ntfs_attrlist_need(struct ntfs_inode *ni); int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr); int ntfs_attrlist_entry_rm(struct ntfs_attr_search_ctx *ctx); int ntfs_attrlist_update(struct ntfs_inode *base_ni); +int ntfs_attrlist_update_locked(struct ntfs_inode *base_ni, + struct ntfs_inode *locked_ni); #endif /* defined _NTFS_ATTRLIST_H */ diff --git a/fs/ntfs/compress.c b/fs/ntfs/compress.c index 76bd806b41edd..89a151901b9de 100644 --- a/fs/ntfs/compress.c +++ b/fs/ntfs/compress.c @@ -96,26 +96,6 @@ void free_compression_buffers(void) mutex_unlock(&ntfs_cb_lock); } -/* - * zero_partial_compressed_page - zero out of bounds compressed page region - * @page: page to zero - * @initialized_size: initialized size of the attribute - */ -static void zero_partial_compressed_page(struct page *page, - const s64 initialized_size) -{ - u8 *kp = page_address(page); - unsigned int kp_ofs; - - ntfs_debug("Zeroing page region outside initialized size."); - if (((s64)page->__folio_index << PAGE_SHIFT) >= initialized_size) { - clear_page(kp); - return; - } - kp_ofs = initialized_size & ~PAGE_MASK; - memset(kp + kp_ofs, 0, PAGE_SIZE - kp_ofs); -} - /* * handle_bounds_compressed_page - test for&handle out of bounds compressed page * @page: page to check and handle @@ -125,9 +105,21 @@ static void zero_partial_compressed_page(struct page *page, static inline void handle_bounds_compressed_page(struct page *page, const loff_t i_size, const s64 initialized_size) { - if ((page->__folio_index >= (initialized_size >> PAGE_SHIFT)) && - (initialized_size < i_size)) - zero_partial_compressed_page(page, initialized_size); + loff_t pos = page_offset(page); + + if ((pos + PAGE_SIZE > initialized_size) && + (initialized_size < i_size)) { + size_t offset; + + ntfs_debug("Zeroing page region outside initialized size."); + if (pos >= initialized_size) + offset = 0; + else + offset = offset_in_page(initialized_size); + zero_user_segment(page, offset, PAGE_SIZE); + } else { + flush_dcache_page(page); + } } /* @@ -185,6 +177,7 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], /* Variables for uncompressed data / destination. */ struct page *dp; /* Current destination page being worked on. */ + u8 *dp_kaddr; /* Local kmap for the current destination page. */ u8 *dp_addr; /* Current pointer into dp. */ u8 *dp_sb_start; /* Start of current sub-block in dp. */ u8 *dp_sb_end; /* End of current sb in dp (dp_sb_start + NTFS_SB_SIZE). */ @@ -199,6 +192,7 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], /* Default error code. */ int err = -EOVERFLOW; + dp_kaddr = NULL; ntfs_debug("Entering, cb_size = 0x%x.", cb_size); do_next_sb: ntfs_debug("Beginning sub-block at offset = 0x%zx in the cb.", @@ -231,8 +225,6 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], */ handle_bounds_compressed_page(dp, i_size, initialized_size); - flush_dcache_page(dp); - kunmap_local(page_address(dp)); SetPageUptodate(dp); unlock_page(dp); if (di == xpage) @@ -278,7 +270,8 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], } /* We have a valid destination page. Setup the destination pointers. */ - dp_addr = (u8 *)page_address(dp) + do_sb_start; + dp_kaddr = kmap_local_page(dp); + dp_addr = dp_kaddr + do_sb_start; /* Now, we are ready to process the current sub-block (sb). */ if (!(le16_to_cpup((__le16 *)cb) & NTFS_SB_IS_COMPRESSED)) { @@ -299,6 +292,8 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], /* Advance destination position to next sub-block. */ *dest_ofs += NTFS_SB_SIZE; *dest_ofs &= ~PAGE_MASK; + kunmap_local(dp_kaddr); + dp_kaddr = NULL; if (!(*dest_ofs)) { finalize_page: /* @@ -333,6 +328,8 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], } /* We have finished the current sub-block. */ *dest_ofs &= ~PAGE_MASK; + kunmap_local(dp_kaddr); + dp_kaddr = NULL; if (!(*dest_ofs)) goto finalize_page; goto do_next_sb; @@ -352,7 +349,7 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], u8 *dp_back_addr; /* Check if we are done / still in range. */ - if (cb >= cb_sb_end || dp_addr > dp_sb_end) + if (cb >= cb_sb_end || dp_addr >= dp_sb_end) break; /* Determine token type and parse appropriately.*/ @@ -438,6 +435,8 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], goto do_next_tag; return_overflow: + if (dp_kaddr) + kunmap_local(dp_kaddr); ntfs_error(NULL, "Failed. Returning -EOVERFLOW."); goto return_error; } @@ -465,14 +464,14 @@ int ntfs_read_compressed_block(struct folio *folio) struct page *page = &folio->page; loff_t i_size; s64 initialized_size; - struct address_space *mapping = page->mapping; + struct address_space *mapping = folio->mapping; struct ntfs_inode *ni = NTFS_I(mapping->host); struct ntfs_volume *vol = ni->vol; struct super_block *sb = vol->sb; struct runlist_element *rl; unsigned long flags; u8 *cb, *cb_pos, *cb_end; - unsigned long offset, index = page->__folio_index; + unsigned long offset, index = folio->index; u32 cb_size = ni->itype.compressed.block_size; u64 cb_size_mask = cb_size - 1UL; s64 vcn; @@ -566,7 +565,6 @@ int ntfs_read_compressed_block(struct folio *folio) * least wasting our time. */ if (!PageDirty(page) && (!PageUptodate(page))) { - kmap_local_page(page); continue; } unlock_page(page); @@ -652,8 +650,7 @@ int ntfs_read_compressed_block(struct folio *folio) } lock_page(lpage); - memcpy(cb_pos, page_address(lpage) + page_ofs, - vol->cluster_size); + memcpy_from_page(cb_pos, lpage, page_ofs, vol->cluster_size); unlock_page(lpage); put_page(lpage); cb_pos += vol->cluster_size; @@ -692,14 +689,7 @@ int ntfs_read_compressed_block(struct folio *folio) for (; cur_page < cb_max_page; cur_page++) { page = pages[cur_page]; if (page) { - if (likely(!cur_ofs)) - clear_page(page_address(page)); - else - memset(page_address(page) + cur_ofs, 0, - PAGE_SIZE - - cur_ofs); - flush_dcache_page(page); - kunmap_local(page_address(page)); + memzero_page(page, cur_ofs, PAGE_SIZE - cur_ofs); SetPageUptodate(page); unlock_page(page); if (cur_page == xpage) @@ -717,8 +707,7 @@ int ntfs_read_compressed_block(struct folio *folio) if (cb_max_ofs && cb_pos < cb_end) { page = pages[cur_page]; if (page) - memset(page_address(page) + cur_ofs, 0, - cb_max_ofs - cur_ofs); + memzero_page(page, cur_ofs, cb_max_ofs - cur_ofs); /* * No need to update cb_pos at this stage: * cb_pos += cb_max_ofs - cur_ofs; @@ -739,7 +728,7 @@ int ntfs_read_compressed_block(struct folio *folio) for (; cur_page < cb_max_page; cur_page++) { page = pages[cur_page]; if (page) - memcpy(page_address(page) + cur_ofs, cb_pos, + memcpy_to_page(page, cur_ofs, cb_pos, PAGE_SIZE - cur_ofs); cb_pos += PAGE_SIZE - cur_ofs; cur_ofs = 0; @@ -750,7 +739,7 @@ int ntfs_read_compressed_block(struct folio *folio) if (cb_max_ofs && cb_pos < cb_end) { page = pages[cur_page]; if (page) - memcpy(page_address(page) + cur_ofs, cb_pos, + memcpy_to_page(page, cur_ofs, cb_pos, cb_max_ofs - cur_ofs); cb_pos += cb_max_ofs - cur_ofs; cur_ofs = cb_max_ofs; @@ -767,8 +756,6 @@ int ntfs_read_compressed_block(struct folio *folio) */ handle_bounds_compressed_page(page, i_size, initialized_size); - flush_dcache_page(page); - kunmap_local(page_address(page)); SetPageUptodate(page); unlock_page(page); if (cur2_page == xpage) @@ -804,7 +791,6 @@ int ntfs_read_compressed_block(struct folio *folio) page = pages[prev_cur_page]; if (page) { flush_dcache_page(page); - kunmap_local(page_address(page)); unlock_page(page); if (prev_cur_page != xpage) put_page(page); @@ -822,14 +808,15 @@ int ntfs_read_compressed_block(struct folio *folio) for (cur_page = 0; cur_page < max_page; cur_page++) { page = pages[cur_page]; if (page) { + folio = page_folio(page); + ntfs_error(vol->sb, "Still have pages left! Terminating them with extreme prejudice. Inode 0x%llx, page index 0x%lx.", - ni->mft_no, page->__folio_index); - flush_dcache_page(page); - kunmap_local(page_address(page)); - unlock_page(page); + ni->mft_no, folio->index); + flush_dcache_folio(folio); + folio_unlock(folio); if (cur_page != xpage) - put_page(page); + folio_put(folio); pages[cur_page] = NULL; } } @@ -864,7 +851,6 @@ int ntfs_read_compressed_block(struct folio *folio) page = pages[i]; if (page) { flush_dcache_page(page); - kunmap_local(page_address(page)); unlock_page(page); if (i != xpage) put_page(page); @@ -1316,7 +1302,6 @@ static int ntfs_write_cb(struct ntfs_inode *ni, loff_t pos, struct page **pages, } pages_disk[i] = pg; lock_page(pg); - kmap_local_page(pg); } outbuf = vmap(pages_disk, pages_count, VM_MAP, PAGE_KERNEL); @@ -1405,7 +1390,7 @@ static int ntfs_write_cb(struct ntfs_inode *ni, loff_t pos, struct page **pages, ni->runlist.count = new_rl_count; ni->runlist.rl = rl; - err = ntfs_attr_update_mapping_pairs(ni, 0); + err = ntfs_attr_update_mapping_pairs_locked(ni, 0, ni); up_write(&ni->runlist.lock); if (err) { err = -EIO; @@ -1451,7 +1436,6 @@ static int ntfs_write_cb(struct ntfs_inode *ni, loff_t pos, struct page **pages, for (i = 0; i < pages_count; i++) { pg = pages_disk[i]; if (pg) { - kunmap_local(page_address(pg)); unlock_page(pg); put_page(pg); } diff --git a/fs/ntfs/ea.c b/fs/ntfs/ea.c index 0cd192752b7cd..75a251ec4ebcb 100644 --- a/fs/ntfs/ea.c +++ b/fs/ntfs/ea.c @@ -196,6 +196,9 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len, struct ea_attr *p_ea; u32 ea_info_qsize = 0; char *ea_buf = NULL; + char *new_ea_buf; + char *old_ea_buf = NULL; + struct ea_information old_ea_info; size_t new_ea_size = ALIGN(struct_size(p_ea, ea_name, 1 + name_len + val_size), 4); s64 ea_off, ea_info_size, all_ea_size, ea_size; @@ -249,6 +252,22 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len, err = -EEXIST; goto out; } + if ((flags & XATTR_REPLACE) && !val_size) { + old_ea_info = *p_ea_info; + old_ea_buf = kvmemdup(ea_buf, all_ea_size, GFP_NOFS); + if (!old_ea_buf) { + err = -ENOMEM; + goto out; + } + } + + /* Check the final $EA size before removing the old entry. */ + if (val_size && + ntfs_attr_size_bounds_check(ni->vol, AT_EA, + ea_info_qsize - ea_size + new_ea_size)) { + err = -EFBIG; + goto out; + } p_ea = (struct ea_attr *)(ea_buf + ea_off); @@ -267,17 +286,39 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len, ea_info_qsize -= ea_size; p_ea_info->ea_query_length = cpu_to_le32(ea_info_qsize); - err = ntfs_write_ea(ni, AT_EA_INFORMATION, (char *)p_ea_info, 0, - sizeof(struct ea_information), false); - if (err) - goto out; + if ((flags & XATTR_REPLACE) && !val_size && !ea_info_qsize) { + err = ntfs_attr_remove(ni, AT_EA, AT_UNNAMED, 0); + if (err) + goto out; - err = ntfs_write_ea(ni, AT_EA, ea_buf, 0, ea_info_qsize, true); - if (err) + err = ntfs_attr_remove(ni, AT_EA_INFORMATION, AT_UNNAMED, 0); + if (err) { + /* Restore the original $EA if $EA_INFORMATION removal failed. */ + ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, old_ea_buf, + all_ea_size); + ea_info_qsize = le32_to_cpu(old_ea_info.ea_query_length); + } goto out; + } if ((flags & XATTR_REPLACE) && !val_size) { - /* Remove xattr. */ + err = ntfs_write_ea(ni, AT_EA, ea_buf, 0, ea_info_qsize, + true); + if (err) { + ntfs_write_ea(ni, AT_EA, old_ea_buf, 0, + all_ea_size, false); + goto out; + } + + err = ntfs_write_ea(ni, AT_EA_INFORMATION, (char *)p_ea_info, + 0, sizeof(struct ea_information), false); + if (err) { + ntfs_write_ea(ni, AT_EA, old_ea_buf, 0, + all_ea_size, false); + ntfs_write_ea(ni, AT_EA_INFORMATION, + (char *)&old_ea_info, 0, + sizeof(old_ea_info), false); + } goto out; } } else { @@ -285,22 +326,30 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len, err = -ENODATA; goto out; } - } - kvfree(ea_buf); + if (ntfs_attr_size_bounds_check(ni->vol, AT_EA, + ea_info_qsize + new_ea_size)) { + err = -EFBIG; + goto out; + } + } alloc_new_ea: - ea_buf = kzalloc(new_ea_size, GFP_NOFS); - if (!ea_buf) { + new_ea_buf = kvzalloc(ea_info_qsize + new_ea_size, GFP_NOFS); + if (!new_ea_buf) { err = -ENOMEM; goto out; } + if (ea_info_qsize) + memcpy(new_ea_buf, ea_buf, ea_info_qsize); + kvfree(ea_buf); + ea_buf = new_ea_buf; + p_ea = (struct ea_attr *)(ea_buf + ea_info_qsize); /* * EA and REPARSE_POINT compatibility not checked any more, * required by Windows 10, but having both may lead to * problems with earlier versions. */ - p_ea = (struct ea_attr *)ea_buf; memcpy(p_ea->ea_name, name, name_len); p_ea->ea_name_length = name_len; p_ea->ea_name[name_len] = 0; @@ -312,8 +361,7 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len, p_ea_info->ea_length = cpu_to_le16(ea_packed); p_ea_info->ea_query_length = cpu_to_le32(ea_info_qsize + new_ea_size); - if (ea_packed > 0xffff || - ntfs_attr_size_bounds_check(ni->vol, AT_EA, new_ea_size)) { + if (ea_packed > 0xffff) { err = -EFBIG; goto out; } @@ -322,13 +370,13 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len, * no EA or EA_INFORMATION : add them */ if (!ntfs_attr_exist(ni, AT_EA, AT_UNNAMED, 0)) { - err = ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, (char *)p_ea, - new_ea_size); + err = ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, ea_buf, + ea_info_qsize + new_ea_size); if (err) goto out; } else { - err = ntfs_write_ea(ni, AT_EA, (char *)p_ea, ea_info_qsize, - new_ea_size, false); + err = ntfs_write_ea(ni, AT_EA, ea_buf, 0, + ea_info_qsize + new_ea_size, true); if (err) goto out; } @@ -342,12 +390,15 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len, *packed_ea_size = p_ea_info->ea_length; mark_mft_record_dirty(ni); out: - if (ea_info_qsize > 0) - NInoSetHasEA(ni); - else - NInoClearHasEA(ni); + if (!err) { + if (ea_info_qsize > 0) + NInoSetHasEA(ni); + else + NInoClearHasEA(ni); + } kvfree(ea_buf); + kvfree(old_ea_buf); kvfree(p_ea_info); return err; @@ -357,37 +408,35 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len, * Check for the presence of an EA "$LXDEV" (used by WSL) * and return its value as a device address */ -int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags) +int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags, + bool *has_lxmod) { int err; __le32 v; + *has_lxmod = false; + if (!(flags & NTFS_VOL_UID)) { /* Load uid to lxuid EA */ err = ntfs_get_ea(inode, "$LXUID", sizeof("$LXUID") - 1, &v, sizeof(v)); - if (err < 0) - return err; - if (err != sizeof(v)) - return -EIO; - i_uid_write(inode, le32_to_cpu(v)); + if (err == sizeof(v)) + i_uid_write(inode, le32_to_cpu(v)); } if (!(flags & NTFS_VOL_GID)) { /* Load gid to lxgid EA */ err = ntfs_get_ea(inode, "$LXGID", sizeof("$LXGID") - 1, &v, sizeof(v)); - if (err < 0) - return err; - if (err != sizeof(v)) - return -EIO; - i_gid_write(inode, le32_to_cpu(v)); + if (err == sizeof(v)) + i_gid_write(inode, le32_to_cpu(v)); } /* Load mode to lxmod EA */ err = ntfs_get_ea(inode, "$LXMOD", sizeof("$LXMOD") - 1, &v, sizeof(v)); if (err == sizeof(v)) { inode->i_mode = le32_to_cpu(v); + *has_lxmod = true; } else { /* Everyone gets all permissions. */ inode->i_mode |= 0777; @@ -544,7 +593,7 @@ static int ntfs_getxattr(const struct xattr_handler *handler, if (!buffer) { err = sizeof(u8); } else if (size < sizeof(u8)) { - err = -ENODATA; + err = -ERANGE; } else { err = sizeof(u8); *(u8 *)buffer = (u8)(le32_to_cpu(ni->flags) & 0x3F); @@ -557,7 +606,7 @@ static int ntfs_getxattr(const struct xattr_handler *handler, if (!buffer) { err = sizeof(u32); } else if (size < sizeof(u32)) { - err = -ENODATA; + err = -ERANGE; } else { err = sizeof(u32); *(u32 *)buffer = le32_to_cpu(ni->flags); @@ -704,6 +753,12 @@ static int ntfs_new_attr_flags(struct ntfs_inode *ni, __le32 fattr) return err; } +static bool ntfs_is_reserved_lxattr(const char *name) +{ + return !strcmp(name, "$LXUID") || !strcmp(name, "$LXGID") || + !strcmp(name, "$LXMOD") || !strcmp(name, "$LXDEV"); +} + static int ntfs_setxattr(const struct xattr_handler *handler, struct mnt_idmap *idmap, struct dentry *unused, struct inode *inode, const char *name, const void *value, @@ -716,6 +771,9 @@ static int ntfs_setxattr(const struct xattr_handler *handler, if (NVolShutdown(ni->vol)) return -EIO; + if (ntfs_is_reserved_lxattr(name) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + if (!strcmp(name, SYSTEM_DOS_ATTRIB)) { if (sizeof(u8) != size) { err = -EINVAL; @@ -768,8 +826,10 @@ static int ntfs_setxattr(const struct xattr_handler *handler, mutex_unlock(&ni->mrec_lock); out: - inode_set_ctime_current(inode); - mark_inode_dirty(inode); + if (!err) { + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + } return err; } diff --git a/fs/ntfs/ea.h b/fs/ntfs/ea.h index 1f63bd55e057b..acb39c2a6fbc8 100644 --- a/fs/ntfs/ea.h +++ b/fs/ntfs/ea.h @@ -10,7 +10,8 @@ extern const struct xattr_handler *const ntfs_xattr_handlers[]; int ntfs_ea_set_wsl_not_symlink(struct ntfs_inode *ni, mode_t mode, dev_t dev); -int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags); +int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags, + bool *has_lxmod); int ntfs_ea_set_wsl_inode(struct inode *inode, dev_t rdev, __le16 *ea_size, unsigned int flags); ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size); diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index 6a7b638e523d9..8f682dbe51eaf 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -111,7 +111,8 @@ static int ntfs_trim_prealloc(struct inode *vi) ntfs_error(vol->sb, "Preallocated block rollback failed"); } else { ni->allocated_size = ntfs_cluster_to_bytes(vol, vcn_tr); - err = ntfs_attr_update_mapping_pairs(ni, 0); + err = ntfs_attr_update_mapping_pairs_locked( + ni, 0, ni); if (err) ntfs_error(vol->sb, "Failed to rollback mapping pairs for prealloc"); @@ -346,14 +347,12 @@ int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (ia_valid & ATTR_MODE) flags |= NTFS_EA_MODE; - if (S_ISDIR(vi->i_mode)) - vi->i_mode &= ~vol->dmask; - else - vi->i_mode &= ~vol->fmask; - mutex_lock(&ni->mrec_lock); - ntfs_ea_set_wsl_inode(vi, 0, NULL, flags); + err = ntfs_ea_set_wsl_inode(vi, 0, NULL, flags); mutex_unlock(&ni->mrec_lock); + if (err) + goto out; + } mark_inode_dirty(vi); diff --git a/fs/ntfs/index.c b/fs/ntfs/index.c index faa7ee920a3ad..8c8eb13859ab6 100644 --- a/fs/ntfs/index.c +++ b/fs/ntfs/index.c @@ -616,6 +616,31 @@ static struct index_root *ntfs_ir_lookup2(struct ntfs_inode *ni, __le16 *name, u return ir; } +static int ntfs_ir_move_to_base(struct ntfs_index_context *icx) +{ + struct ntfs_attr_search_ctx *ctx = NULL; + struct index_root *ir; + bool moved = false; + int ret = 0; + + ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx); + if (!ir) + return -ENOENT; + + if (ctx->ntfs_ino->mft_no != icx->idx_ni->mft_no) { + ret = ntfs_attr_record_move_to(ctx, icx->idx_ni); + if (!ret) { + moved = true; + ret = ntfs_attrlist_update(icx->idx_ni); + } + } + + ntfs_attr_put_search_ctx(ctx); + if (!ret && moved) + ret = ntfs_inode_free_empty_extents(icx->idx_ni); + return ret; +} + /* * Find a key in the index block. */ @@ -989,6 +1014,7 @@ static s64 ntfs_ibm_pos_to_vcn(struct ntfs_index_context *icx, s64 pos) static int ntfs_ibm_add(struct ntfs_index_context *icx) { u8 bmp[8]; + int ret; ntfs_debug("Entering\n"); @@ -998,10 +1024,11 @@ static int ntfs_ibm_add(struct ntfs_index_context *icx) * AT_BITMAP must be at least 8 bytes. */ memset(bmp, 0, sizeof(bmp)); - if (ntfs_attr_add(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len, - bmp, sizeof(bmp))) { + ret = ntfs_attr_add(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len, + bmp, sizeof(bmp)); + if (ret) { ntfs_error(icx->idx_ni->vol->sb, "Failed to add AT_BITMAP"); - return -EINVAL; + return ret; } return 0; @@ -1074,6 +1101,7 @@ static s64 ntfs_ibm_get_free(struct ntfs_index_context *icx) { u8 *bm; int bit; + int ret; s64 vcn, byte, size; ntfs_debug("Entering\n"); @@ -1081,7 +1109,7 @@ static s64 ntfs_ibm_get_free(struct ntfs_index_context *icx) bm = ntfs_attr_readall(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len, &size); if (!bm) - return (s64)-1; + return -EIO; for (byte = 0; byte < size; byte++) { if (bm[byte] == 255) @@ -1099,10 +1127,12 @@ static s64 ntfs_ibm_get_free(struct ntfs_index_context *icx) out: ntfs_debug("allocated vcn: %lld\n", vcn); - if (ntfs_ibm_set(icx, vcn)) - vcn = (s64)-1; + ret = ntfs_ibm_set(icx, vcn); kvfree(bm); + if (ret) + return ret; + return vcn; } @@ -1112,6 +1142,7 @@ static struct index_block *ntfs_ir_to_ib(struct index_root *ir, s64 ib_vcn) struct index_entry *ie_last; char *ies_start, *ies_end; int i; + u32 ib_cap; ntfs_debug("Entering\n"); @@ -1127,6 +1158,16 @@ static struct index_block *ntfs_ir_to_ib(struct index_root *ir, s64 ib_vcn) * as well, which can never have any data. */ i = (char *)ie_last - ies_start + le16_to_cpu(ie_last->length); + + /* Entries must fit in the allocated index block */ + ib_cap = le32_to_cpu(ib->index.allocated_size) - + le32_to_cpu(ib->index.entries_offset); + if ((u32)i > ib_cap) { + ntfs_error(NULL, "Entries (%d B) exceed IB capacity", i); + kvfree(ib); + return NULL; + } + memcpy(ntfs_ie_get_first(&ib->index), ies_start, i); ib->index.flags = ir->index.flags; @@ -1264,7 +1305,7 @@ static int ntfs_ir_reparent(struct ntfs_index_context *icx) new_ib_vcn = ntfs_ibm_get_free(icx); if (new_ib_vcn < 0) { - ret = -EINVAL; + ret = (int)new_ib_vcn; goto out; } @@ -1335,19 +1376,42 @@ static int ntfs_ir_reparent(struct ntfs_index_context *icx) * When there is no space to build a non-resident * index, we may have to move the root to an extent */ - if ((ret == -ENOSPC) && (ctx->al_entry || !ntfs_inode_add_attrlist(icx->idx_ni))) { - ntfs_attr_put_search_ctx(ctx); - ctx = NULL; - ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx); - if (ir && !ntfs_attr_record_move_away(ctx, ix_root_size - - le32_to_cpu(ctx->attr->data.resident.value_length))) { - if (ntfs_attrlist_update(ctx->base_ntfs_ino ? - ctx->base_ntfs_ino : ctx->ntfs_ino)) + if (ret == -ENOSPC) { + if (!ctx->al_entry) { + ret = ntfs_inode_add_attrlist(icx->idx_ni); + if (ret) goto clear_bmp; + ntfs_attr_put_search_ctx(ctx); ctx = NULL; goto retry; } + + if (ctx->ntfs_ino->mft_no != icx->idx_ni->mft_no) + goto clear_bmp; + + ret = ntfs_attr_record_move_away(ctx, ix_root_size - + le32_to_cpu(ctx->attr->data.resident.value_length)); + if (ret) + goto clear_bmp; + + ret = ntfs_attrlist_update(icx->idx_ni); + if (ret) { + int rollback_ret; + + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + rollback_ret = ntfs_ir_move_to_base(icx); + if (rollback_ret) + ntfs_error(icx->idx_ni->vol->sb, + "Failed to roll back INDEX_ROOT relocation: %d", + rollback_ret); + goto clear_bmp; + } + + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + goto retry; } clear_bmp: ntfs_ibm_clear(icx, new_ib_vcn); @@ -1579,7 +1643,7 @@ static int ntfs_ib_split(struct ntfs_index_context *icx, struct index_block *ib) median = ntfs_ie_get_median(&ib->index); new_vcn = ntfs_ibm_get_free(icx); if (new_vcn < 0) { - ret = -EINVAL; + ret = (int)new_vcn; goto out; } diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index 7381a18cfadd9..cfb753be4f6b1 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -682,6 +682,7 @@ static int ntfs_read_locked_inode(struct inode *vi) unsigned int name_len = 4, flags = 0; int extend_sys = 0; dev_t dev = 0; + bool has_lxmod = false; bool vol_err = true; ntfs_debug("Entering for i_ino 0x%llx.", ni->mft_no); @@ -862,7 +863,7 @@ static int ntfs_read_locked_inode(struct inode *vi) err = ntfs_attr_lookup(AT_EA_INFORMATION, NULL, 0, 0, 0, NULL, 0, ctx); if (!err) { NInoSetHasEA(ni); - ntfs_ea_get_wsl_inode(vi, &dev, flags); + ntfs_ea_get_wsl_inode(vi, &dev, flags, &has_lxmod); } if (ni->flags & FILE_ATTR_REPARSE_POINT) { @@ -886,16 +887,18 @@ static int ntfs_read_locked_inode(struct inode *vi) if (S_ISDIR(vi->i_mode)) { /* - * Apply the directory permissions mask set in the mount - * options. + * Apply the directory permissions mask set in the mount options + * when no per-file WSL mode is present. */ - vi->i_mode &= ~vol->dmask; + if (!has_lxmod) + vi->i_mode &= ~vol->dmask; /* Things break without this kludge! */ if (vi->i_nlink > 1) set_nlink(vi, 1); } else { - /* Apply the file permissions mask set in the mount options. */ - vi->i_mode &= ~vol->fmask; + /* Apply the file permissions mask when no WSL mode is present. */ + if (!has_lxmod) + vi->i_mode &= ~vol->fmask; } /* @@ -1229,7 +1232,8 @@ static int ntfs_read_locked_inode(struct inode *vi) if (m) unmap_mft_record(ni); err_out: - if (err != -EOPNOTSUPP && err != -ENOMEM && vol_err == true) { + if (err != -EOPNOTSUPP && err != -ENOMEM && + err != -EINTR && err != -ERESTARTSYS && vol_err == true) { ntfs_error(vol->sb, "Failed with error code %i. Marking corrupt inode 0x%llx as bad. Run chkdsk.", err, ni->mft_no); @@ -1455,12 +1459,13 @@ static int ntfs_read_locked_attr_inode(struct inode *base_vi, struct inode *vi) ntfs_attr_put_search_ctx(ctx); unmap_mft_record(base_ni); err_out: - if (err != -ENOENT) + if (err != -ENOENT && err != -EINTR && err != -ERESTARTSYS) ntfs_error(vol->sb, "Failed with error code %i while reading attribute inode (mft_no 0x%llx, type 0x%x, name_len %i). Marking corrupt inode and base inode 0x%llx as bad. Run chkdsk.", err, ni->mft_no, ni->type, ni->name_len, base_ni->mft_no); - if (err != -ENOENT && err != -ENOMEM) + if (err != -ENOENT && err != -ENOMEM && + err != -EINTR && err != -ERESTARTSYS) NVolSetErrors(vol); return err; } @@ -1664,8 +1669,9 @@ static int ntfs_read_locked_index_inode(struct inode *base_vi, struct inode *vi) /* Get the index bitmap attribute inode. */ bvi = ntfs_attr_iget(base_vi, AT_BITMAP, ni->name, ni->name_len); if (IS_ERR(bvi)) { - ntfs_error(vi->i_sb, "Failed to get bitmap attribute."); err = PTR_ERR(bvi); + if (err != -EINTR && err != -ERESTARTSYS) + ntfs_error(vi->i_sb, "Failed to get bitmap attribute."); goto unm_err_out; } bni = NTFS_I(bvi); @@ -1709,10 +1715,12 @@ static int ntfs_read_locked_index_inode(struct inode *base_vi, struct inode *vi) if (m) unmap_mft_record(base_ni); err_out: - ntfs_error(vi->i_sb, - "Failed with error code %i while reading index inode (mft_no 0x%llx, name_len %i.", - err, ni->mft_no, ni->name_len); - if (err != -EOPNOTSUPP && err != -ENOMEM) + if (err != -EINTR && err != -ERESTARTSYS) + ntfs_error(vi->i_sb, + "Failed with error code %i while reading index inode (mft_no 0x%llx, name_len %i.", + err, ni->mft_no, ni->name_len); + if (err != -EOPNOTSUPP && err != -ENOMEM && + err != -EINTR && err != -ERESTARTSYS) NVolSetErrors(vol); return err; } @@ -2769,7 +2777,7 @@ int __ntfs_write_inode(struct inode *vi, int sync) if (NInoNonResident(ni) && NInoRunlistDirty(ni)) { down_write(&ni->runlist.lock); - err = ntfs_attr_update_mapping_pairs(ni, 0); + err = ntfs_attr_update_mapping_pairs_locked(ni, 0, ni); if (!err) NInoClearRunlistDirty(ni); up_write(&ni->runlist.lock); @@ -3046,6 +3054,7 @@ int ntfs_inode_add_attrlist(struct ntfs_inode *ni) struct attr_list_entry *ale = NULL; struct mft_record *ni_mrec; u32 attr_al_len; + bool free_empty_extents = true; if (!ni) return -EINVAL; @@ -3145,6 +3154,7 @@ int ntfs_inode_add_attrlist(struct ntfs_inode *ni) ntfs_error(ni->vol->sb, "Couldn't add $ATTRIBUTE_LIST to MFT"); goto rollback; } + free_empty_extents = false; err = ntfs_attrlist_update(ni); if (err < 0) @@ -3164,6 +3174,8 @@ int ntfs_inode_add_attrlist(struct ntfs_inode *ni) CASE_SENSITIVE, 0, NULL, 0, ctx)) { if (ntfs_attr_record_rm(ctx)) ntfs_error(ni->vol->sb, "Rollback failed to remove attrlist"); + else + free_empty_extents = true; } else { ntfs_error(ni->vol->sb, "Rollback failed to find attrlist"); } @@ -3202,6 +3214,11 @@ int ntfs_inode_add_attrlist(struct ntfs_inode *ni) ni->attr_list_size = 0; NInoClearAttrList(ni); NInoClearAttrListDirty(ni); + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + if (free_empty_extents && ntfs_inode_free_empty_extents(ni)) + ntfs_error(ni->vol->sb, "Rollback failed to free empty extent"); + goto err_out; put_err_out: ntfs_attr_put_search_ctx(ctx); err_out: @@ -3288,6 +3305,55 @@ int ntfs_inode_close(struct ntfs_inode *ni) return err; } +/* + * ntfs_inode_free_empty_extents - free empty extent MFT records + * @ni: base inode whose empty extent records should be freed + * + * The caller must ensure that no on-disk attribute list references an empty + * extent record and must hold @ni->mrec_lock to serialize the extent array. + */ +int ntfs_inode_free_empty_extents(struct ntfs_inode *ni) +{ + int err = 0, i = 0; + + if (!ni || ni->nr_extents < 0) + return -EINVAL; + + mutex_lock(&ni->extent_lock); + while (i < ni->nr_extents) { + struct ntfs_inode *ext_ni = ni->ext.extent_ntfs_inos[i]; + struct mft_record *m; + int ret; + + m = map_mft_record(ext_ni); + if (IS_ERR(m)) { + if (!err) + err = PTR_ERR(m); + i++; + continue; + } + if (le32_to_cpu(m->bytes_in_use) - + le16_to_cpu(m->attrs_offset) != 8) { + unmap_mft_record(ext_ni); + i++; + continue; + } + unmap_mft_record(ext_ni); + + ret = ntfs_mft_record_free(ni->vol, ext_ni); + if (ret) { + if (!err) + err = ret; + i++; + continue; + } + ntfs_inode_close(ext_ni); + /* ntfs_inode_close() removed this entry from the extent array. */ + } + mutex_unlock(&ni->extent_lock); + return err; +} + void ntfs_destroy_ext_inode(struct ntfs_inode *ni) { ntfs_debug("Entering."); @@ -3387,6 +3453,9 @@ int ntfs_inode_free_space(struct ntfs_inode *ni, int size) * Chkdsk complain if $STANDARD_INFORMATION is not in the base MFT * record. * + * $INDEX_ROOT must remain resident, but its attribute record may be moved + * to an extent MFT record when the base record needs room for the list. + * * Also we can't move $ATTRIBUTE_LIST from base MFT_RECORD, so position * search context on first attribute after $STANDARD_INFORMATION and * $ATTRIBUTE_LIST. @@ -3428,9 +3497,6 @@ int ntfs_inode_free_space(struct ntfs_inode *ni, int size) ctx->attr->type == AT_DATA) goto retry; - if (ctx->attr->type == AT_INDEX_ROOT) - goto retry; - record_size = le32_to_cpu(ctx->attr->length); /* Move away attribute. */ @@ -3652,7 +3718,7 @@ static s64 __ntfs_inode_non_resident_attr_pwrite(struct inode *vi, FGP_CREAT | FGP_LOCK, mapping_gfp_mask(mapping)); if (IS_ERR(folio)) { - ret = -ENOMEM; + ret = PTR_ERR(folio); break; } } else { @@ -3684,6 +3750,7 @@ static s64 __ntfs_inode_non_resident_attr_pwrite(struct inode *vi, u64 rl_length = 0; s64 vcn; struct runlist_element *rl; + int bio_err; lcn_count = max_t(s64, 1, ntfs_bytes_to_cluster(vol, attr_len)); vcn = ntfs_pidx_to_cluster(vol, folio->index); @@ -3727,8 +3794,15 @@ static s64 __ntfs_inode_non_resident_attr_pwrite(struct inode *vi, goto err_unlock_folio; } - submit_bio_wait(bio); + bio_err = submit_bio_wait(bio); bio_put(bio); + if (bio_err) { + ntfs_error(vi->i_sb, + "Synchronous attribute write failed (%d)", + bio_err); + ret = bio_err; + goto err_unlock_folio; + } vcn += rl_length; offset += length; } while (lcn_count != 0); diff --git a/fs/ntfs/inode.h b/fs/ntfs/inode.h index 9aacd5787ffea..bb48f55b50ddc 100644 --- a/fs/ntfs/inode.h +++ b/fs/ntfs/inode.h @@ -338,6 +338,7 @@ int ntfs_get_block_mft_record(struct ntfs_inode *mft_ni, struct ntfs_inode *ni); int __ntfs_write_inode(struct inode *vi, int sync); int ntfs_inode_attach_all_extents(struct ntfs_inode *ni); int ntfs_inode_add_attrlist(struct ntfs_inode *ni); +int ntfs_inode_free_empty_extents(struct ntfs_inode *ni); void ntfs_destroy_ext_inode(struct ntfs_inode *ni); int ntfs_inode_free_space(struct ntfs_inode *ni, int size); s64 ntfs_inode_attr_pread(struct inode *vi, s64 pos, s64 count, u8 *buf); diff --git a/fs/ntfs/iomap.c b/fs/ntfs/iomap.c index 52eecf5cb256a..e6c2515014c35 100644 --- a/fs/ntfs/iomap.c +++ b/fs/ntfs/iomap.c @@ -81,7 +81,7 @@ const struct iomap_write_ops ntfs_iomap_folio_ops = { }; static int ntfs_read_iomap_begin_resident(struct inode *inode, loff_t offset, loff_t length, - unsigned int flags, struct iomap *iomap) + unsigned int flags, struct iomap *iomap, bool keep_mrec_lock) { struct ntfs_inode *base_ni, *ni = NTFS_I(inode); struct ntfs_attr_search_ctx *ctx; @@ -95,6 +95,8 @@ static int ntfs_read_iomap_begin_resident(struct inode *inode, loff_t offset, lo else base_ni = ni; + mutex_lock(&base_ni->mrec_lock); + ctx = ntfs_attr_get_search_ctx(base_ni, NULL); if (!ctx) { err = -ENOMEM; @@ -138,6 +140,13 @@ static int ntfs_read_iomap_begin_resident(struct inode *inode, loff_t offset, lo if (ctx) ntfs_attr_put_search_ctx(ctx); + if (!err && keep_mrec_lock && iomap->type == IOMAP_INLINE) { + iomap->private = base_ni; + return 0; + } + + mutex_unlock(&base_ni->mrec_lock); + return err; } @@ -261,24 +270,37 @@ static int ntfs_read_iomap_begin_non_resident(struct inode *inode, loff_t offset static int __ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap, - bool need_unwritten) + bool need_unwritten, bool keep_mrec_lock) { if (NInoNonResident(NTFS_I(inode))) return ntfs_read_iomap_begin_non_resident(inode, offset, length, flags, iomap, need_unwritten); return ntfs_read_iomap_begin_resident(inode, offset, length, - flags, iomap); + flags, iomap, keep_mrec_lock); } static int ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) { return __ntfs_read_iomap_begin(inode, offset, length, flags, iomap, - srcmap, true); + srcmap, true, true); } +static int ntfs_read_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, struct iomap *iomap) +{ + struct ntfs_inode *base_ni = iomap->private; + + if (base_ni) + mutex_unlock(&base_ni->mrec_lock); + return written; +} + +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_read_iomap_next, + ntfs_read_iomap_begin, ntfs_read_iomap_end); + const struct iomap_ops ntfs_read_iomap_ops = { - .iomap_begin = ntfs_read_iomap_begin, + .iomap_next = ntfs_read_iomap_next, }; /* @@ -318,7 +340,7 @@ static int ntfs_seek_iomap_begin(struct inode *inode, loff_t offset, loff_t leng unsigned int flags, struct iomap *iomap, struct iomap *srcmap) { return __ntfs_read_iomap_begin(inode, offset, length, flags, iomap, - srcmap, false); + srcmap, false, false); } static int ntfs_zero_read_iomap_end(struct inode *inode, loff_t pos, loff_t length, @@ -329,13 +351,17 @@ static int ntfs_zero_read_iomap_end(struct inode *inode, loff_t pos, loff_t leng return written; } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_zero_read_iomap_next, + ntfs_seek_iomap_begin, ntfs_zero_read_iomap_end); + static const struct iomap_ops ntfs_zero_read_iomap_ops = { - .iomap_begin = ntfs_seek_iomap_begin, - .iomap_end = ntfs_zero_read_iomap_end, + .iomap_next = ntfs_zero_read_iomap_next, }; +static DEFINE_IOMAP_ITER_NEXT(ntfs_seek_iomap_next, ntfs_seek_iomap_begin); + const struct iomap_ops ntfs_seek_iomap_ops = { - .iomap_begin = ntfs_seek_iomap_begin, + .iomap_next = ntfs_seek_iomap_next, }; int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length) @@ -764,9 +790,11 @@ static int ntfs_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, return written; } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_write_iomap_next, + ntfs_write_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_write_iomap_ops = { - .iomap_begin = ntfs_write_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_write_iomap_next, }; static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, @@ -777,9 +805,11 @@ static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, NTFS_IOMAP_FLAGS_MKWRITE); } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_page_mkwrite_iomap_next, + ntfs_page_mkwrite_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_page_mkwrite_iomap_ops = { - .iomap_begin = ntfs_page_mkwrite_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_page_mkwrite_iomap_next, }; static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, @@ -790,9 +820,11 @@ static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, NTFS_IOMAP_FLAGS_DIO); } +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_dio_iomap_next, + ntfs_dio_iomap_begin, ntfs_write_iomap_end); + const struct iomap_ops ntfs_dio_iomap_ops = { - .iomap_begin = ntfs_dio_iomap_begin, - .iomap_end = ntfs_write_iomap_end, + .iomap_next = ntfs_dio_iomap_next, }; static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc, diff --git a/fs/ntfs/lcnalloc.c b/fs/ntfs/lcnalloc.c index 835a041023a2a..0d6cd08ee2e76 100644 --- a/fs/ntfs/lcnalloc.c +++ b/fs/ntfs/lcnalloc.c @@ -53,10 +53,10 @@ int ntfs_cluster_free_from_rl_nolock(struct ntfs_volume *vol, if (rl->lcn < 0) continue; err = ntfs_bitmap_clear_run(lcnbmp_vi, rl->lcn, rl->length); - if (unlikely(err && (!ret || ret == -ENOMEM) && ret != err)) - ret = err; - else + if (likely(!err)) nr_freed += rl->length; + else if (!ret || ret == -ENOMEM) + ret = err; } ntfs_inc_free_clusters(vol, nr_freed); ntfs_debug("Done."); @@ -298,7 +298,12 @@ struct runlist_element *ntfs_cluster_alloc(struct ntfs_volume *vol, const s64 st clusters = count; rlpos = rlsize = 0; mapping = lcnbmp_vi->i_mapping; - i_size = i_size_read(lcnbmp_vi); + /* + * lcn_empty_bits_per_page is sized from nr_clusters, but $Bitmap can + * cover more clusters than that; bound the scan by the array. + */ + i_size = min_t(s64, i_size_read(lcnbmp_vi), + ((s64)vol->nr_clusters + 7) >> 3); while (1) { ntfs_debug("Start of outer while loop: done_zones 0x%x, search_zone %i, pass %i, zone_start 0x%llx, zone_end 0x%llx, bmp_initial_pos 0x%llx, bmp_pos 0x%llx, rlpos %i, rlsize %i.", done_zones, search_zone, pass, @@ -1040,8 +1045,9 @@ s64 __ntfs_cluster_free(struct ntfs_inode *ni, const s64 start_vcn, s64 count, "Failed to rollback (error %i). Leaving inconsistent metadata! Unmount and run chkdsk.", (int)delta); NVolSetErrors(vol); + } else { + ntfs_dec_free_clusters(vol, delta); } - ntfs_dec_free_clusters(vol, delta); up_write(&vol->lcnbmp_lock); memalloc_nofs_restore(memalloc_flags); ntfs_error(vol->sb, "Aborting (error %i).", err); diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c index fd20d7abd6f5e..8dcbc93a8ccac 100644 --- a/fs/ntfs/mft.c +++ b/fs/ntfs/mft.c @@ -213,7 +213,8 @@ struct mft_record *map_mft_record(struct ntfs_inode *ni) return m; atomic_dec(&ni->count); - ntfs_error(ni->vol->sb, "Failed with error code %lu.", -PTR_ERR(m)); + if (PTR_ERR(m) != -EINTR && PTR_ERR(m) != -ERESTARTSYS) + ntfs_error(ni->vol->sb, "Failed with error code %lu.", -PTR_ERR(m)); return m; } @@ -580,7 +581,7 @@ int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int syn err = pre_write_mst_fixup((struct ntfs_record *)fixup_m, vol->mft_record_size); if (err) { ntfs_error(vol->sb, "Failed to apply mst fixups!"); - goto err_out; + goto unmap_err_out; } folio_size = vol->mft_record_size / ni->mft_lcn_count; @@ -645,6 +646,8 @@ int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int syn return 0; put_bio_out: bio_put(bio); +unmap_err_out: + kunmap_local(kaddr); err_out: /* * The caller should mark the base inode as bad so no more I/O @@ -839,12 +842,126 @@ static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no, static const char *es = " Leaving inconsistent metadata. Unmount and run chkdsk."; -#define RESERVED_MFT_RECORDS 64 +#define FIRST_NORMAL_MFT_RECORD 24 +#define MFT_RECORD_RESERVE 4 + +/* + * Records 12-15 are marked in use by Windows but normally have no name + * and no links. Keep them as the last bootstrap option when a volume + * mounted without an in-memory tail reserve needs its first $MFT metadata + * extent. + */ +static bool mft_reserved_is_free(struct ntfs_volume *vol, + struct ntfs_inode *mft_ni, s64 mft_no) +{ + struct attr_record *a; + struct mft_record *m; + struct folio *folio; + void *mapped; + pgoff_t index = NTFS_MFT_NR_TO_PIDX(vol, mft_no); + unsigned int ofs = NTFS_MFT_NR_TO_POFS(vol, mft_no); + u32 attrs_offset, bytes_in_use; + bool available = false, have_std = false; + int i; + + for (i = 0; i < mft_ni->nr_extents; i++) { + if (mft_ni->ext.extent_ntfs_inos[i] && + mft_ni->ext.extent_ntfs_inos[i]->mft_no == mft_no) + return false; + } + m = kmalloc(vol->mft_record_size, GFP_NOFS); + if (!m) + return false; + + folio = read_mapping_folio(vol->mft_ino->i_mapping, index, NULL); + if (IS_ERR(folio)) + goto free_m; + + folio_lock(folio); + mapped = kmap_local_folio(folio, 0); + memcpy(m, (u8 *)mapped + ofs, vol->mft_record_size); + kunmap_local(mapped); + folio_unlock(folio); + folio_put(folio); + if (post_read_mst_fixup((struct ntfs_record *)m, vol->mft_record_size)) + goto free_m; + + if (!ntfs_is_mft_record(m->magic) || + !(m->flags & MFT_RECORD_IN_USE) || m->base_mft_record || + m->link_count) + goto out; + + attrs_offset = le16_to_cpu(m->attrs_offset); + bytes_in_use = le32_to_cpu(m->bytes_in_use); + if (attrs_offset > bytes_in_use || bytes_in_use > vol->mft_record_size || + bytes_in_use - attrs_offset < sizeof(a->type)) + goto out; + + for (a = (struct attr_record *)((u8 *)m + attrs_offset); + (u8 *)a + sizeof(a->type) <= (u8 *)m + bytes_in_use;) { + u32 len; + + if (a->type == AT_END) { + if ((u8 *)a + sizeof(a->type) + sizeof(a->length) > + (u8 *)m + bytes_in_use) + break; + /* Also accept a record emptied by an earlier bootstrap. */ + available = have_std || + (u8 *)a == (u8 *)m + attrs_offset; + break; + } + if (a->type == AT_FILE_NAME) + break; + len = le32_to_cpu(a->length); + if (len < offsetof(struct attr_record, data) || + (u8 *)a + len > (u8 *)m + bytes_in_use) + break; + if (a->type == AT_STANDARD_INFORMATION) { + u32 value_len, value_ofs; + + if (have_std || a->non_resident || + len < offsetof(struct attr_record, + data.resident.reserved) + 1) + break; + value_len = le32_to_cpu(a->data.resident.value_length); + value_ofs = le16_to_cpu(a->data.resident.value_offset); + if (value_ofs > len || value_len > len - value_ofs) + break; + have_std = true; + } + a = (struct attr_record *)((u8 *)a + len); + } +out: + kfree(m); + return available; +free_m: + kfree(m); + return false; +} + +static s64 mft_reserve_end(const u8 *buf, s64 buf_start, s64 buf_end, + s64 start, s64 pass_end, s64 initialized_mft_records) +{ + s64 end = start + 1; + s64 limit = min_t(s64, start + MFT_RECORD_RESERVE, pass_end); + + if (limit > initialized_mft_records) + limit = initialized_mft_records; + if (limit > buf_end) + limit = buf_end; + while (end < limit && + !(buf[(end - buf_start) >> 3] & + (1 << ((end - buf_start) & 7)))) + end++; + return end; +} /* - * ntfs_mft_bitmap_find_and_alloc_free_rec_nolock - see name + * mft_bitmap_alloc_free_rec - find and allocate a free MFT record * @vol: volume on which to search for a free mft record * @base_ni: open base inode if allocating an extent mft record or NULL + * @max_mft_no: first record which must not be allocated, or -1 + * @new_reserve_end: if not NULL, end of a free run starting after the result * * Search for a free mft record in the mft bitmap attribute on the ntfs volume * @vol. @@ -860,10 +977,12 @@ static const char *es = " Leaving inconsistent metadata. Unmount and run chkds * * Locking: Caller must hold vol->mftbmp_lock for writing. */ -static s64 ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(struct ntfs_volume *vol, - struct ntfs_inode *base_ni) +static s64 mft_bitmap_alloc_free_rec(struct ntfs_volume *vol, + struct ntfs_inode *base_ni, + s64 max_mft_no, s64 *new_reserve_end) { s64 pass_end, ll, data_pos, pass_start, ofs, bit; + s64 initialized_mft_records; unsigned long flags; struct address_space *mftbmp_mapping; u8 *buf = NULL, *byte; @@ -880,30 +999,36 @@ static s64 ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(struct ntfs_volume *vo read_lock_irqsave(&NTFS_I(vol->mft_ino)->size_lock, flags); pass_end = NTFS_I(vol->mft_ino)->allocated_size >> vol->mft_record_size_bits; + initialized_mft_records = NTFS_I(vol->mft_ino)->initialized_size >> + vol->mft_record_size_bits; read_unlock_irqrestore(&NTFS_I(vol->mft_ino)->size_lock, flags); read_lock_irqsave(&NTFS_I(vol->mftbmp_ino)->size_lock, flags); ll = NTFS_I(vol->mftbmp_ino)->initialized_size << 3; read_unlock_irqrestore(&NTFS_I(vol->mftbmp_ino)->size_lock, flags); if (pass_end > ll) pass_end = ll; - pass = 1; - if (!base_ni) - data_pos = vol->mft_data_pos; - else - data_pos = base_ni->mft_no + 1; - if (data_pos < RESERVED_MFT_RECORDS) - data_pos = RESERVED_MFT_RECORDS; - if (data_pos >= pass_end) { - data_pos = RESERVED_MFT_RECORDS; + if (max_mft_no >= 0 && pass_end > max_mft_no) + pass_end = max_mft_no; + if (base_ni && base_ni->mft_no == FILE_MFT) { + data_pos = FILE_first_user; pass = 2; - /* This happens on a freshly formatted volume. */ if (data_pos >= pass_end) return -ENOSPC; - } - - if (base_ni && base_ni->mft_no == FILE_MFT) { - data_pos = 0; - pass = 2; + } else { + pass = 1; + if (!base_ni) + data_pos = vol->mft_data_pos; + else + data_pos = base_ni->mft_no + 1; + if (data_pos < FIRST_NORMAL_MFT_RECORD) + data_pos = FIRST_NORMAL_MFT_RECORD; + if (data_pos >= pass_end) { + data_pos = FIRST_NORMAL_MFT_RECORD; + pass = 2; + /* This happens on a freshly formatted volume. */ + if (data_pos >= pass_end) + return -ENOSPC; + } } pass_start = data_pos; @@ -938,38 +1063,28 @@ static s64 ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(struct ntfs_volume *vo size, data_pos, bit); for (; bit < size && data_pos + bit < pass_end; bit &= ~7ull, bit += 8) { - /* - * If we're extending $MFT and running out of the first - * mft record (base record) then give up searching since - * no guarantee that the found record will be accessible. - */ - if (base_ni && base_ni->mft_no == FILE_MFT && bit > 400) { - folio_unlock(folio); - kunmap_local(buf); - folio_put(folio); - return -ENOSPC; - } - byte = buf + (bit >> 3); if (*byte == 0xff) continue; - b = ffz((unsigned long)*byte); - if (b < 8 && b >= (bit & 7)) { + b = bit & 7; + for (; b < 8; b++) { + if (*byte & (1 << b)) + continue; ll = data_pos + (bit & ~7ull) + b; + if (ll >= pass_end) + break; + /* Keep the dynamic tail reserve for $MFT metadata. */ + if ((!base_ni || base_ni->mft_no != FILE_MFT) && + ll >= vol->mft_record_reserve_pos && + ll < vol->mft_record_reserve_end) + continue; if (unlikely(ll >= (1ll << 32))) { folio_unlock(folio); kunmap_local(buf); folio_put(folio); return -ENOSPC; } - *byte |= 1 << b; - folio_mark_dirty(folio); - folio_unlock(folio); - kunmap_local(buf); - folio_put(folio); - ntfs_debug("Done. (Found and allocated mft record 0x%llx.)", - ll); - return ll; + goto found; } } ntfs_debug("After inner for loop: size 0x%x, data_pos 0x%llx, bit 0x%llx", @@ -992,7 +1107,8 @@ static s64 ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(struct ntfs_volume *vo * part of the zone which we omitted earlier. */ pass_end = pass_start; - data_pos = pass_start = RESERVED_MFT_RECORDS; + data_pos = FIRST_NORMAL_MFT_RECORD; + pass_start = FIRST_NORMAL_MFT_RECORD; ntfs_debug("pass %i, pass_start 0x%llx, pass_end 0x%llx.", pass, pass_start, pass_end); if (data_pos >= pass_end) @@ -1002,9 +1118,22 @@ static s64 ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(struct ntfs_volume *vo /* No free mft records in currently initialized mft bitmap. */ ntfs_debug("Done. (No free mft records left in currently initialized mft bitmap.)"); return -ENOSPC; +found: + if (new_reserve_end) + *new_reserve_end = mft_reserve_end(buf, data_pos, + data_pos + size, ll, pass_end, + initialized_mft_records); + *byte |= 1 << b; + folio_mark_dirty(folio); + folio_unlock(folio); + kunmap_local(buf); + folio_put(folio); + ntfs_debug("Done. (Found and allocated mft record 0x%llx.)", ll); + return ll; } -static int ntfs_mft_attr_extend(struct ntfs_inode *ni) +static int ntfs_mft_attr_extend(struct ntfs_inode *ni, + struct ntfs_inode *locked_ni) { int ret = 0; struct ntfs_inode *base_ni; @@ -1025,7 +1154,7 @@ static int ntfs_mft_attr_extend(struct ntfs_inode *ni) } } - ret = ntfs_attr_update_mapping_pairs(ni, 0); + ret = ntfs_attr_update_mapping_pairs_locked(ni, 0, locked_ni); if (ret) pr_err("MP update failed\n"); @@ -1213,7 +1342,7 @@ static int ntfs_mft_bitmap_extend_allocation_nolock(struct ntfs_volume *vol) ret = ntfs_attr_record_resize(ctx->mrec, a, mp_size + le16_to_cpu(a->data.non_resident.mapping_pairs_offset)); if (unlikely(ret)) { - ret = ntfs_mft_attr_extend(mftbmp_ni); + ret = ntfs_mft_attr_extend(mftbmp_ni, mftbmp_ni); if (!ret) goto extended_ok; if (ret != -EAGAIN) @@ -1324,7 +1453,9 @@ static int ntfs_mft_bitmap_extend_allocation_nolock(struct ntfs_volume *vol) NVolSetErrors(vol); } mark_mft_record_dirty(ctx->ntfs_ino); - } else if (status.mp_extended && ntfs_attr_update_mapping_pairs(mftbmp_ni, 0)) { + } else if (status.mp_extended && + ntfs_attr_update_mapping_pairs_locked(mftbmp_ni, 0, + mftbmp_ni)) { ntfs_error(vol->sb, "Failed to restore mapping pairs.%s", es); NVolSetErrors(vol); } @@ -1412,7 +1543,6 @@ static int ntfs_mft_bitmap_extend_initialized_nolock(struct ntfs_volume *vol) ret = ntfs_attr_set(mftbmp_ni, old_initialized_size, 8, 0); if (likely(!ret)) { ntfs_debug("Done. (Wrote eight initialized bytes to mft bitmap."); - ntfs_inc_free_mft_records(vol, 8 * 8); return 0; } ntfs_error(vol->sb, "Failed to write to mft bitmap."); @@ -1469,8 +1599,9 @@ static int ntfs_mft_bitmap_extend_initialized_nolock(struct ntfs_volume *vol) * @vol: volume on which to extend the mft data attribute * * Extend the mft data attribute on the ntfs volume @vol by 16 mft records - * worth of clusters or if not enough space for this by one mft record worth - * of clusters. + * worth of clusters or if not enough space for this by two mft records worth + * of clusters. Keeping at least two new records breaks the recursion between + * extending $MFT and allocating a record for a new $MFT attribute extent. * * Note: Only changes allocated_size, i.e. does not touch initialized_size or * data_size. @@ -1524,10 +1655,8 @@ static int ntfs_mft_data_extend_allocation_nolock(struct ntfs_volume *vol) } lcn = rl->lcn + rl->length; ntfs_debug("Last lcn of mft data attribute is 0x%llx.", lcn); - /* Minimum allocation is one mft record worth of clusters. */ - min_nr = NTFS_B_TO_CLU(vol, vol->mft_record_size); - if (!min_nr) - min_nr = 1; + /* Keep room for the allocating record and at least one MFT reserve. */ + min_nr = DIV_ROUND_UP_ULL((u64)vol->mft_record_size * 2, vol->cluster_size); /* Want to allocate 16 mft records worth of clusters. */ nr = vol->mft_record_size << 4 >> vol->cluster_size_bits; if (!nr) @@ -1651,7 +1780,7 @@ static int ntfs_mft_data_extend_allocation_nolock(struct ntfs_volume *vol) ret = ntfs_attr_record_resize(ctx->mrec, a, mp_size + le16_to_cpu(a->data.non_resident.mapping_pairs_offset)); if (unlikely(ret)) { - ret = ntfs_mft_attr_extend(mft_ni); + ret = ntfs_mft_attr_extend(mft_ni, NULL); if (!ret) goto extended_ok; if (ret != -EAGAIN) @@ -1926,6 +2055,7 @@ static int ntfs_mft_record_format(const struct ntfs_volume *vol, const s64 mft_n * @ni: [OUT] on success, set to the allocated ntfs inode * @base_ni: [IN] open base inode if allocating an extent mft record or NULL * @ni_mrec: [OUT] on successful return this is the mapped mft record + * @mft_data_vcn: [IN] lowest VCN of a new $MFT/$DATA extent, or -1 * * Allocate an mft record in $MFT/$DATA of an open ntfs volume @vol. * @@ -1953,30 +2083,23 @@ static int ntfs_mft_record_format(const struct ntfs_volume *vol, const s64 mft_n * optimize this we start scanning at the place specified by @base_ni or if * @base_ni is NULL we start where we last stopped and we perform wrap around * when we reach the end. Note, we do not try to allocate mft records below - * number 64 because numbers 0 to 15 are the defined system files anyway and 16 - * to 64 are special in that they are used for storing extension mft records - * for the $DATA attribute of $MFT. This is required to avoid the possibility - * of creating a runlist with a circular dependency which once written to disk - * can never be read in again. Windows will only use records 16 to 24 for - * normal files if the volume is completely out of space. We never use them - * which means that when the volume is really out of space we cannot create any - * more files while Windows can still create up to 8 small files. We can start - * doing this at some later time, it does not matter much for now. + * number 24 because numbers 0 to 15 are the defined system files and records + * 16 to 23 are kept for metadata compatibility. Records reserved dynamically + * at the initialized MFT tail are skipped by normal allocation and consumed by + * $MFT metadata extent allocation. * * When scanning the mft bitmap, we only search up to the last allocated mft - * record. If there are no free records left in the range 64 to number of + * record. If there are no free records left in the range 24 to number of * allocated mft records, then we extend the $MFT/$DATA attribute in order to * create free mft records. We extend the allocated size of $MFT/$DATA by 16 * records at a time or one cluster, if cluster size is above 16kiB. If there - * is not sufficient space to do this, we try to extend by a single mft record - * or one cluster, if cluster size is above the mft record size. + * is not sufficient space to do this, we try to extend by two mft records or + * one cluster, if a cluster already contains at least two mft records. * - * No matter how many mft records we allocate, we initialize only the first - * allocated mft record, incrementing mft data size and initialized size - * accordingly, open an struct ntfs_inode for it and return it to the caller, unless - * there are less than 64 mft records, in which case we allocate and initialize - * mft records until we reach record 64 which we consider as the first free mft - * record for use by normal files. + * When extending the initialized MFT tail, we also initialize up to four + * additional records and reserve them in memory for future $MFT metadata + * extents. If there are less than 24 mft records, records are initialized + * until record 24, which is the first record used for normal files. * * If during any stage we overflow the initialized data in the mft bitmap, we * extend the initialized size (and data size) by 8 bytes, allocating another @@ -2012,9 +2135,13 @@ static int ntfs_mft_record_format(const struct ntfs_volume *vol, const s64 mft_n */ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, struct ntfs_inode **ni, struct ntfs_inode *base_ni, - struct mft_record **ni_mrec) + struct mft_record **ni_mrec, const s64 mft_data_vcn) { s64 ll, bit, old_data_initialized, old_data_size; + s64 nr_new_mft_records = 0; + s64 max_mft_no = -1, reserve_start = -1, reserve_end = -1; + s64 candidate_reserve_end = -1; + s64 *reserve_endp; unsigned long flags; struct folio *folio; struct ntfs_inode *mft_ni, *mftbmp_ni; @@ -2025,7 +2152,9 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, unsigned int ofs; int err; __le16 seq_no, usn; - bool record_formatted = false; + bool record_formatted = false, from_reserve = false, tail_alloc = false; + bool reserve_created = false; + bool forced_reserved_record = false; unsigned int memalloc_flags; if (base_ni && *ni) @@ -2034,6 +2163,21 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, /* @mode and @base_ni are mutually exclusive. */ if (mode && base_ni) return -EINVAL; + if (mft_data_vcn >= 0 && + (!base_ni || base_ni->mft_no != FILE_MFT)) + return -EINVAL; + if (mft_data_vcn >= 0) { + u64 vbo; + + if ((u64)mft_data_vcn > (U64_MAX >> vol->cluster_size_bits)) + return -EOVERFLOW; + vbo = (u64)mft_data_vcn << vol->cluster_size_bits; + /* + * The whole extent record must be reachable without this + * extent, including when an MFT record spans multiple clusters. + */ + max_mft_no = vbo >> vol->mft_record_size_bits; + } if (base_ni) ntfs_debug("Entering (allocating an extent mft record for base mft record 0x%llx).", @@ -2048,10 +2192,39 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, mutex_lock(&mft_ni->mrec_lock); mftbmp_ni = NTFS_I(vol->mftbmp_ino); search_free_rec: + from_reserve = false; + reserve_created = false; + candidate_reserve_end = -1; if (!base_ni || base_ni->mft_no != FILE_MFT) down_write(&vol->mftbmp_lock); - bit = ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(vol, base_ni); + if (base_ni && base_ni->mft_no == FILE_MFT && + vol->mft_record_reserve_pos < vol->mft_record_reserve_end && + (max_mft_no < 0 || vol->mft_record_reserve_pos < max_mft_no)) { + bit = vol->mft_record_reserve_pos; + err = ntfs_bitmap_set_bit(vol->mftbmp_ino, bit); + if (unlikely(err)) { + ntfs_error(vol->sb, + "Failed to allocate reserved MFT record 0x%llx.", + bit); + goto err_out; + } + vol->mft_record_reserve_pos++; + from_reserve = true; + ntfs_debug("Allocated MFT metadata record 0x%llx from tail reserve.", + bit); + goto have_alloc_rec; + } + reserve_endp = vol->mft_record_reserve_pos >= + vol->mft_record_reserve_end ? &candidate_reserve_end : NULL; + bit = mft_bitmap_alloc_free_rec(vol, base_ni, max_mft_no, reserve_endp); if (bit >= 0) { + if (candidate_reserve_end > bit + 1) { + vol->mft_record_reserve_pos = bit + 1; + vol->mft_record_reserve_end = candidate_reserve_end; + reserve_created = true; + ntfs_debug("Reserved free MFT records [0x%llx, 0x%llx) for metadata.", + bit + 1, candidate_reserve_end); + } ntfs_debug("Found and allocated free record (#1), bit 0x%llx.", (long long)bit); goto have_alloc_rec; @@ -2066,6 +2239,24 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, } if (base_ni && base_ni->mft_no == FILE_MFT) { + static const u8 bootstrap_records[] = { + FILE_reserved15, FILE_reserved12, FILE_reserved13, + FILE_reserved14, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(bootstrap_records); i++) { + if (max_mft_no >= 0 && bootstrap_records[i] >= max_mft_no) + continue; + if (!mft_reserved_is_free(vol, mft_ni, + bootstrap_records[i])) + continue; + bit = bootstrap_records[i]; + forced_reserved_record = true; + ntfs_debug("Using reserved MFT record %lld to bootstrap metadata extension.", + bit); + goto have_alloc_rec; + } memalloc_nofs_restore(memalloc_flags); return bit; } @@ -2085,10 +2276,10 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, old_data_initialized = mftbmp_ni->initialized_size; read_unlock_irqrestore(&mftbmp_ni->size_lock, flags); if (old_data_initialized << 3 > ll && - old_data_initialized > RESERVED_MFT_RECORDS / 8) { + old_data_initialized << 3 > FIRST_NORMAL_MFT_RECORD) { bit = ll; - if (bit < RESERVED_MFT_RECORDS) - bit = RESERVED_MFT_RECORDS; + if (bit < FIRST_NORMAL_MFT_RECORD) + bit = FIRST_NORMAL_MFT_RECORD; if (unlikely(bit >= (1ll << 32))) goto max_err_out; ntfs_debug("Found free record (#2), bit 0x%llx.", @@ -2174,6 +2365,11 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, read_lock_irqsave(&mft_ni->size_lock, flags); old_data_initialized = mft_ni->initialized_size; read_unlock_irqrestore(&mft_ni->size_lock, flags); + tail_alloc = (!base_ni || base_ni->mft_no != FILE_MFT) && + bit >= (old_data_initialized >> vol->mft_record_size_bits) && + vol->mft_record_reserve_pos >= vol->mft_record_reserve_end; + if (tail_alloc) + ll = (bit + 2) << vol->mft_record_size_bits; if (ll <= old_data_initialized) { ntfs_debug("Allocated mft record already initialized."); goto mft_rec_already_initialized; @@ -2206,6 +2402,29 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, mft_ni->initialized_size); } read_unlock_irqrestore(&mft_ni->size_lock, flags); + if (tail_alloc) { + s64 bitmap_records; + + read_lock_irqsave(&mft_ni->size_lock, flags); + reserve_end = mft_ni->allocated_size >> + vol->mft_record_size_bits; + read_unlock_irqrestore(&mft_ni->size_lock, flags); + read_lock_irqsave(&mftbmp_ni->size_lock, flags); + bitmap_records = mftbmp_ni->initialized_size << 3; + read_unlock_irqrestore(&mftbmp_ni->size_lock, flags); + if (reserve_end > bitmap_records) + reserve_end = bitmap_records; + if (reserve_end > bit + 1 + MFT_RECORD_RESERVE) + reserve_end = bit + 1 + MFT_RECORD_RESERVE; + reserve_start = bit + 1; + if (reserve_end > reserve_start) { + ll = reserve_end << vol->mft_record_size_bits; + } else { + reserve_start = -1; + reserve_end = -1; + ll = (bit + 1) << vol->mft_record_size_bits; + } + } } else if (ll > mft_ni->allocated_size) { err = -ENOSPC; goto undo_mftbmp_alloc_nolock; @@ -2274,14 +2493,25 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, mark_mft_record_dirty(ctx->ntfs_ino); ntfs_attr_put_search_ctx(ctx); unmap_mft_record(mft_ni); + if (reserve_start >= 0 && reserve_end > reserve_start) { + vol->mft_record_reserve_pos = reserve_start; + vol->mft_record_reserve_end = reserve_end; + ntfs_debug("Reserved MFT records [0x%llx, 0x%llx) for metadata.", + reserve_start, reserve_end); + } read_lock_irqsave(&mft_ni->size_lock, flags); ntfs_debug("Status of mft data after mft record initialization: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.", mft_ni->allocated_size, i_size_read(vol->mft_ino), mft_ni->initialized_size); WARN_ON(i_size_read(vol->mft_ino) > mft_ni->allocated_size); WARN_ON(mft_ni->initialized_size > i_size_read(vol->mft_ino)); + nr_new_mft_records = (i_size_read(vol->mft_ino) - old_data_size) >> + vol->mft_record_size_bits; read_unlock_irqrestore(&mft_ni->size_lock, flags); mft_rec_already_initialized: + /* Account for newly visible MFT records before dropping the lock. */ + if (nr_new_mft_records > 0) + ntfs_inc_free_mft_records(vol, nr_new_mft_records); /* * We can finally drop the mft bitmap lock as the mft data attribute * has been fully updated. The only disparity left is that the @@ -2313,8 +2543,8 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, /* If we just formatted the mft record no need to do it again. */ if (!record_formatted) { /* Sanity check that the mft record is really not in use. */ - if (ntfs_is_file_record(m->magic) && - (m->flags & MFT_RECORD_IN_USE)) { + if (!forced_reserved_record && ntfs_is_file_record(m->magic) && + (m->flags & MFT_RECORD_IN_USE)) { ntfs_warning(vol->sb, "Mft record 0x%llx was marked free in mft bitmap but is marked used itself. Unmount and run chkdsk.", bit); @@ -2333,7 +2563,17 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, * wrong with the previous mft record. */ seq_no = m->sequence_number; - usn = *(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)); + /* + * The mft record still holds unvalidated, MST-protected on-disk + * bytes, so m->usa_ofs is untrusted here. Only preserve the old + * update sequence number if that offset is in bounds; otherwise + * leave usn zero so it is not restored below. + */ + if (!(le16_to_cpu(m->usa_ofs) & 1) && + le16_to_cpu(m->usa_ofs) + sizeof(usn) <= vol->mft_record_size) + usn = *(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)); + else + usn = 0; err = ntfs_mft_record_layout(vol, bit, m); if (unlikely(err)) { ntfs_error(vol->sb, "Failed to layout allocated mft record 0x%llx.", @@ -2377,9 +2617,13 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, ntfs_error(vol->sb, "Failed to map allocated extent mft record 0x%llx.", bit); err = PTR_ERR(m_tmp); - /* Set the mft record itself not in use. */ - m->flags &= cpu_to_le16( - ~le16_to_cpu(MFT_RECORD_IN_USE)); + if (forced_reserved_record) { + m->base_mft_record = 0; + m->flags |= MFT_RECORD_IN_USE; + } else { + /* Set the mft record itself not in use. */ + m->flags &= cpu_to_le16(~le16_to_cpu(MFT_RECORD_IN_USE)); + } /* Make sure the mft record is written out to disk. */ ntfs_mft_mark_dirty(folio); folio_unlock(folio); @@ -2452,7 +2696,8 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, (*ni)->mft_no = bit; if (ni_mrec) *ni_mrec = (*ni)->mrec; - ntfs_dec_free_mft_records(vol, 1); + if (!forced_reserved_record) + ntfs_dec_free_mft_records(vol, 1); return 0; undo_data_init: write_lock_irqsave(&mft_ni->size_lock, flags); @@ -2464,10 +2709,13 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, if (!base_ni || base_ni->mft_no != FILE_MFT) down_write(&vol->mftbmp_lock); undo_mftbmp_alloc_nolock: - if (ntfs_bitmap_clear_bit(vol->mftbmp_ino, bit)) { + if (!forced_reserved_record && ntfs_bitmap_clear_bit(vol->mftbmp_ino, bit)) { ntfs_error(vol->sb, "Failed to clear bit in mft bitmap.%s", es); NVolSetErrors(vol); } + if ((from_reserve || reserve_created) && + vol->mft_record_reserve_pos == bit + 1) + vol->mft_record_reserve_pos = bit; if (!base_ni || base_ni->mft_no != FILE_MFT) up_write(&vol->mftbmp_lock); err_out: @@ -2503,9 +2751,11 @@ int ntfs_mft_record_free(struct ntfs_volume *vol, struct ntfs_inode *ni) int err; u16 seq_no; __le16 old_seq_no; + __le64 old_base_mft_record; struct mft_record *ni_mrec; unsigned int memalloc_flags; struct ntfs_inode *base_ni; + bool keep_reserved; if (!vol || !ni) return -EINVAL; @@ -2518,9 +2768,23 @@ int ntfs_mft_record_free(struct ntfs_volume *vol, struct ntfs_inode *ni) /* Cache the mft reference for later. */ mft_no = ni->mft_no; - - /* Mark the mft record as not in use. */ - ni_mrec->flags &= ~MFT_RECORD_IN_USE; + if (likely(ni->nr_extents >= 0)) + base_ni = ni; + else + base_ni = ni->ext.base_ntfs_ino; + keep_reserved = mft_no >= FILE_reserved12 && + mft_no <= FILE_reserved15 && + base_ni->mft_no == FILE_MFT; + + old_base_mft_record = ni_mrec->base_mft_record; + if (keep_reserved) { + /* Restore the special, unnamed form used by reserved records. */ + ni_mrec->base_mft_record = 0; + ni_mrec->flags |= MFT_RECORD_IN_USE; + } else { + /* Mark the mft record as not in use. */ + ni_mrec->flags &= ~MFT_RECORD_IN_USE; + } /* Increment the sequence number, skipping zero, if it is not zero. */ old_seq_no = ni_mrec->sequence_number; @@ -2549,24 +2813,28 @@ int ntfs_mft_record_free(struct ntfs_volume *vol, struct ntfs_inode *ni) if (err) goto sync_rollback; - if (likely(ni->nr_extents >= 0)) - base_ni = ni; - else - base_ni = ni->ext.base_ntfs_ino; + if (keep_reserved) { + unmap_mft_record(ni); + return 0; + } /* Clear the bit in the $MFT/$BITMAP corresponding to this record. */ memalloc_flags = memalloc_nofs_save(); if (base_ni->mft_no != FILE_MFT) down_write(&vol->mftbmp_lock); err = ntfs_bitmap_clear_bit(vol->mftbmp_ino, mft_no); + if (!err) + ntfs_inc_free_mft_records(vol, 1); + if (!err && base_ni->mft_no == FILE_MFT && + mft_no + 1 == vol->mft_record_reserve_pos && + mft_no < vol->mft_record_reserve_end) + vol->mft_record_reserve_pos = mft_no; if (base_ni->mft_no != FILE_MFT) up_write(&vol->mftbmp_lock); memalloc_nofs_restore(memalloc_flags); if (err) goto bitmap_rollback; - unmap_mft_record(ni); - ntfs_inc_free_mft_records(vol, 1); return 0; /* Rollback what we did... */ @@ -2584,6 +2852,7 @@ int ntfs_mft_record_free(struct ntfs_volume *vol, struct ntfs_inode *ni) "Eeek! Rollback failed in %s. Leaving inconsistent metadata!\n", __func__); ni_mrec->flags |= MFT_RECORD_IN_USE; ni_mrec->sequence_number = old_seq_no; + ni_mrec->base_mft_record = old_base_mft_record; NInoSetDirty(ni); write_mft_record(ni, ni_mrec, 0); unmap_mft_record(ni); diff --git a/fs/ntfs/mft.h b/fs/ntfs/mft.h index 75a51a98d0f6e..ed5c1d595c0d3 100644 --- a/fs/ntfs/mft.h +++ b/fs/ntfs/mft.h @@ -78,7 +78,7 @@ static inline int write_mft_record(struct ntfs_inode *ni, struct mft_record *m, int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, struct ntfs_inode **ni, struct ntfs_inode *base_ni, - struct mft_record **ni_mrec); + struct mft_record **ni_mrec, const s64 mft_data_vcn); int ntfs_mft_record_free(struct ntfs_volume *vol, struct ntfs_inode *ni); int ntfs_mft_records_write(const struct ntfs_volume *vol, const u64 mref, const s64 count, struct mft_record *b); diff --git a/fs/ntfs/namei.c b/fs/ntfs/namei.c index cd403b1d99eed..dfd05e604f9c1 100644 --- a/fs/ntfs/namei.c +++ b/fs/ntfs/namei.c @@ -61,12 +61,12 @@ static int ntfs_check_bad_windows_name(struct ntfs_volume *vol, const __le16 *wc, unsigned int wc_len) { - if (ntfs_check_bad_char(wc, wc_len)) - return -EINVAL; - if (!NVolCheckWindowsNames(vol)) return 0; + if (ntfs_check_bad_char(wc, wc_len)) + return -EINVAL; + /* Check for trailing space or dot. */ if (wc_len > 0 && (wc[wc_len - 1] == cpu_to_le16(' ') || @@ -424,8 +424,6 @@ static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *d * directories, also setup the index values to the defaults. */ if (S_ISDIR(mode)) { - mode &= ~vol->dmask; - NInoSetMstProtected(ni); ni->itype.index.block_size = 4096; ni->itype.index.block_size_bits = ntfs_ffs(4096) - 1; @@ -439,8 +437,6 @@ static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *d ni->itype.index.vcn_size_bits = vol->sector_size_bits; } - } else { - mode &= ~vol->fmask; } if (IS_RDONLY(vi)) @@ -484,7 +480,7 @@ static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *d mark_inode_dirty(dir); err = ntfs_mft_record_alloc(dir_ni->vol, mode, &ni, NULL, - &ni_mrec); + &ni_mrec, -1); if (err) { iput(vi); return ERR_PTR(err); diff --git a/fs/ntfs/reparse.c b/fs/ntfs/reparse.c index fa523dc3691ea..f8b88a110cb67 100644 --- a/fs/ntfs/reparse.c +++ b/fs/ntfs/reparse.c @@ -332,7 +332,7 @@ unsigned int ntfs_reparse_tag_dt_types(struct ntfs_volume *vol, unsigned long mr vi = ntfs_iget(vol->sb, mref); if (IS_ERR(vi)) - return PTR_ERR(vi); + return DT_UNKNOWN; reparse_attr = (struct reparse_point *)ntfs_attr_readall(NTFS_I(vi), AT_REPARSE_POINT, NULL, 0, &attr_size); @@ -620,8 +620,9 @@ static int update_reparse_data(struct ntfs_inode *ni, struct ntfs_index_context goto put_rp_inode; } - if (set_reparse_index(ni, xr, ((const struct reparse_point *)value)->reparse_tag) && - oldsize > 0) { + err = set_reparse_index(ni, xr, + ((const struct reparse_point *)value)->reparse_tag); + if (err && oldsize > 0) { /* * If cannot index, try to remove the reparse * data and log the error. There will be an diff --git a/fs/ntfs/runlist.c b/fs/ntfs/runlist.c index 8e0fd400e7f78..00373e450ea7a 100644 --- a/fs/ntfs/runlist.c +++ b/fs/ntfs/runlist.c @@ -772,6 +772,25 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume * return ERR_PTR(-EIO); } + /* + * An empty mapping-pairs array is valid only for a zero-length + * attribute. + */ + if (!*buf && + (vcn || + le64_to_cpu(attr->data.non_resident.highest_vcn) != + (u64)(vcn - 1) || + le64_to_cpu(attr->data.non_resident.allocated_size) || + le64_to_cpu(attr->data.non_resident.data_size) || + le64_to_cpu(attr->data.non_resident.initialized_size))) { + ntfs_error(vol->sb, "Invalid empty mapping pairs array."); + return ERR_PTR(-EIO); + } + if (!vcn && !*buf && old_runlist && old_runlist->rl) { + *new_rl_count = old_runlist->count; + return old_runlist->rl; + } + /* Current position in runlist array. */ rlpos = 0; /* Allocate first page and set current runlist size to one page. */ @@ -884,12 +903,41 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume * ntfs_error(vol->sb, "lcn == -1"); } #endif + /* Check lcn is within the volume. */ + if (unlikely(lcn >= (s64)vol->nr_clusters)) { + ntfs_error(vol->sb, + "LCN >= nr_clusters in mapping pairs array."); + goto err_out; + } + /* Check lcn is not below -1. */ if (unlikely(lcn < -1)) { ntfs_error(vol->sb, "Invalid s64 < -1 in mapping pairs array."); goto err_out; } + if (lcn >= 0) { + s64 run_end; + + /* + * Ensure that the run stays within the volume. + * A valid starting LCN is not sufficient because + * the run length comes from disk. + */ + if (unlikely(check_add_overflow(lcn, + rl[rlpos].length, + &run_end))) { + ntfs_error(vol->sb, + "Run length overflow in mapping pairs array."); + goto err_out; + } + if (unlikely(run_end > (s64)vol->nr_clusters)) { + ntfs_error(vol->sb, + "Run extends beyond volume boundary."); + goto err_out; + } + } + /* chkdsk accepts zero-sized runs only for holes */ if ((lcn != -1) && !rl[rlpos].length) { ntfs_error(vol->sb, diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c index 8abe7bee4c0d9..b5371d269f747 100644 --- a/fs/ntfs/super.c +++ b/fs/ntfs/super.c @@ -557,8 +557,8 @@ static bool is_boot_sector_ntfs(const struct super_block *sb, * Check sectors per cluster value is valid and the cluster size * is not above the maximum (2MB). */ - if (b->bpb.sectors_per_cluster > 0x80 && - b->bpb.sectors_per_cluster < 0xf4) + if (b->bpb.sectors_per_cluster < 0xf4 && + !is_power_of_2(b->bpb.sectors_per_cluster)) goto not_ntfs; /* Check reserved/unused fields are really zero. */ @@ -645,7 +645,7 @@ static bool parse_ntfs_boot_sector(struct ntfs_volume *vol, { unsigned int sectors_per_cluster, sectors_per_cluster_bits, nr_hidden_sects; int clusters_per_mft_record, clusters_per_index_record; - s64 ll; + u64 ll; vol->sector_size = le16_to_cpu(b->bpb.bytes_per_sector); vol->sector_size_bits = ffs(vol->sector_size) - 1; @@ -755,23 +755,23 @@ static bool parse_ntfs_boot_sector(struct ntfs_volume *vol, * the same as it is much faster on 32-bit CPUs. */ ll = le64_to_cpu(b->number_of_sectors) >> sectors_per_cluster_bits; - if ((u64)ll >= 1ULL << 32) { + if (ll >= 1ULL << 32) { ntfs_error(vol->sb, "Cannot handle 64-bit clusters."); return false; } vol->nr_clusters = ll; ntfs_debug("vol->nr_clusters = 0x%llx", vol->nr_clusters); ll = le64_to_cpu(b->mft_lcn); - if (ll >= vol->nr_clusters) { - ntfs_error(vol->sb, "MFT LCN (%lli, 0x%llx) is beyond end of volume. Weird.", + if (ll >= (u64)vol->nr_clusters) { + ntfs_error(vol->sb, "MFT LCN (%llu, 0x%llx) is beyond end of volume. Weird.", ll, ll); return false; } vol->mft_lcn = ll; ntfs_debug("vol->mft_lcn = 0x%llx", vol->mft_lcn); ll = le64_to_cpu(b->mftmirr_lcn); - if (ll >= vol->nr_clusters) { - ntfs_error(vol->sb, "MFTMirr LCN (%lli, 0x%llx) is beyond end of volume. Weird.", + if (ll >= (u64)vol->nr_clusters) { + ntfs_error(vol->sb, "MFTMirr LCN (%llu, 0x%llx) is beyond end of volume. Weird.", ll, ll); return false; } @@ -1241,9 +1241,9 @@ static bool load_and_init_attrdef(struct ntfs_volume *vol) goto failed; } NInoSetSparseDisabled(NTFS_I(ino)); - /* The size of FILE_AttrDef must be above 0 and fit inside 31 bits. */ + /* FILE_AttrDef must hold at least one entry and fit inside 31 bits. */ i_size = i_size_read(ino); - if (i_size <= 0 || i_size > 0x7fffffff) + if (i_size < (s64)sizeof(struct attr_def) || i_size > 0x7fffffff) goto iput_failed; vol->attrdef = kvzalloc(i_size, GFP_NOFS); if (!vol->attrdef) @@ -1862,7 +1862,8 @@ static int ntfs_sync_fs(struct super_block *sb, int wait) return 0; /* If there are some dirty buffers in the bdev inode */ - if (ntfs_clear_volume_flags(vol, VOLUME_IS_DIRTY)) { + if (!NVolErrors(vol) && + ntfs_clear_volume_flags(vol, VOLUME_IS_DIRTY)) { ntfs_warning(sb, "Failed to clear dirty bit in volume information flags. Run chkdsk."); err = -EIO; } @@ -2063,8 +2064,7 @@ static unsigned long __get_nr_free_mft_records(struct ntfs_volume *vol, /* If errors occurred we may well have gone below zero, fix this. */ if (nr_free < 0) nr_free = 0; - else - atomic64_set(&vol->free_mft_records, nr_free); + atomic64_set(&vol->free_mft_records, nr_free); ntfs_debug("Exiting."); return nr_free; @@ -2130,7 +2130,14 @@ static int ntfs_statfs(struct dentry *dentry, struct kstatfs *sfs) read_unlock_irqrestore(&mft_ni->size_lock, flags); /* Free inodes in fs (based on current total count). */ - sfs->f_ffree = atomic64_read(&vol->free_mft_records); + size = atomic64_read(&vol->free_mft_records); + if (unlikely(size < 0 || size > (s64)sfs->f_files)) + ntfs_warning(vol->sb, "Invalid free MFT record count %lld.", size); + if (size < 0) + size = 0; + else if (size > (s64)sfs->f_files) + size = sfs->f_files; + sfs->f_ffree = size; /* * File system id. This is extremely *nix flavour dependent and even diff --git a/fs/ntfs/volume.h b/fs/ntfs/volume.h index 65fd3908af261..bc85a95922458 100644 --- a/fs/ntfs/volume.h +++ b/fs/ntfs/volume.h @@ -55,6 +55,10 @@ * @attrdef_size: Size of the attribute definition table in bytes. * @attrdef: Table of attribute definitions. Obtained from FILE_AttrDef. * @mft_data_pos: Mft record number at which to allocate the next mft record. + * @mft_record_reserve_pos: First record in the in-memory MFT metadata reserve + * (protected by mftbmp_lock). + * @mft_record_reserve_end: First record beyond the MFT metadata reserve + * (protected by mftbmp_lock). * @mft_zone_start: First cluster of the mft zone. * @mft_zone_end: First cluster beyond the mft zone. * @mft_zone_pos: Current position in the mft zone. @@ -119,6 +123,8 @@ struct ntfs_volume { s32 attrdef_size; struct attr_def *attrdef; s64 mft_data_pos; + s64 mft_record_reserve_pos; + s64 mft_record_reserve_end; s64 mft_zone_start; s64 mft_zone_end; s64 mft_zone_pos; @@ -252,17 +258,11 @@ static inline void ntfs_dec_free_clusters(struct ntfs_volume *vol, s64 nr) static inline void ntfs_inc_free_mft_records(struct ntfs_volume *vol, s64 nr) { - if (!NVolFreeClusterKnown(vol)) - return; - atomic64_add(nr, &vol->free_mft_records); } static inline void ntfs_dec_free_mft_records(struct ntfs_volume *vol, s64 nr) { - if (!NVolFreeClusterKnown(vol)) - return; - atomic64_sub(nr, &vol->free_mft_records); } diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c index 2b49bc0775582..2c18cc1b4fb74 100644 --- a/fs/ntfs3/frecord.c +++ b/fs/ntfs3/frecord.c @@ -768,10 +768,23 @@ int ni_create_attr_list(struct ntfs_inode *ni) rs = sbi->record_size; /* - * Skip estimating exact memory requirement. - * Looks like one record_size is always enough. + * Compute the exact size of the attribute list. Each attribute in the + * record yields one ATTR_LIST_ENTRY of le_size(name_len) bytes. The + * minimum on-disk attribute is SIZEOF_RESIDENT (0x18) bytes, but an + * unnamed one expands to le_size(0) (0x20) here, so a record crafted + * with many such attributes needs more than a single record_size; the + * previous fixed kzalloc(record_size) could therefore be overflowed by + * an attacker-controlled record. */ - le = kzalloc(al_aligned(rs), GFP_NOFS); + lsize = 0; + attr = NULL; + while ((attr = mi_enum_attr(ni, &ni->mi, attr))) + lsize += le_size(attr->name_len); + + if (!lsize) + return -EINVAL; + + le = kzalloc(al_aligned(lsize), GFP_NOFS); if (!le) return -ENOMEM; @@ -781,7 +794,6 @@ int ni_create_attr_list(struct ntfs_inode *ni) attr = NULL; nb = 0; free_b = 0; - attr = NULL; for (; (attr = mi_enum_attr(ni, &ni->mi, attr)); le = Add2Ptr(le, sz)) { sz = le_size(attr->name_len); @@ -2443,6 +2455,15 @@ int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, err = unc_size; else if (!unc_size || unc_size > frame_size) err = -EINVAL; + else if (unc_size < frame_size) { + /* + * Partial decompress: zero the [unc_size, frame_size) + * tail. decompress_lznt() leaves it untouched, so + * without this the freshly vmapped pages would expose + * uninitialized kernel memory to userspace. + */ + memset(frame_mem + unc_size, 0, frame_size - unc_size); + } } if (!err && valid_size < frame_vbo + frame_size) { size_t ok = valid_size - frame_vbo; diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index f038c799e7ac4..b05e48b28625a 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -647,6 +647,14 @@ static inline void *enum_rstbl(struct RESTART_TABLE *t, void *c) return NULL; } +/* + * dp_range_ok - true if [j, j + count) fits in a page_lcns[cap] array. + */ +static inline bool dp_range_ok(size_t j, u32 count, u32 cap) +{ + return j < cap && count <= cap - j; +} + /* * find_dp - Search for a @vcn in Dirty Page Table. */ @@ -789,6 +797,20 @@ static bool check_rstbl(const struct RESTART_TABLE *rt, size_t bytes) return true; } +static bool check_dp_table(const struct RESTART_TABLE *dptbl) +{ + u32 rsize = le16_to_cpu(dptbl->size); + struct DIR_PAGE_ENTRY *dp = NULL; + + while ((dp = enum_rstbl((struct RESTART_TABLE *)dptbl, dp))) { + if (struct_size(dp, page_lcns, le32_to_cpu(dp->lcns_follow)) > + rsize) + return false; + } + + return true; +} + /* * free_rsttbl_idx - Free a previously allocated index a Restart Table. */ @@ -853,6 +875,9 @@ static inline struct RESTART_TABLE *extend_rsttbl(struct RESTART_TABLE *tbl, u32 used = le16_to_cpu(tbl->used); struct RESTART_TABLE *rt; + if (used + add > U16_MAX) + return NULL; + rt = init_rsttbl(esize, used + add); if (!rt) return NULL; @@ -2276,7 +2301,15 @@ static int read_log_rec_buf(struct ntfs_log *log, */ for (;;) { bool usa_error; - u32 tail = log->page_size - off; + u32 tail; + + /* off comes from the on-disk restart area; bound it. */ + if (off > log->page_size) { + err = -EINVAL; + goto out; + } + + tail = log->page_size - off; if (tail >= data_len) tail = data_len; @@ -4295,6 +4328,11 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) goto out; } + if (!check_dp_table(rt)) { + err = -EINVAL; + goto out; + } + dptbl = kmemdup(rt, t32, GFP_NOFS); if (!dptbl) { err = -ENOMEM; @@ -5087,6 +5125,13 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) /* Shorten length by any Lcns which were deleted. */ saved_len = dlen; + if (!dp_range_ok(le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn), + le16_to_cpu(lrh->lcns_follow), + le32_to_cpu(dp->lcns_follow))) { + err = -EINVAL; + goto out; + } + for (i = le16_to_cpu(lrh->lcns_follow); i; i--) { size_t j; u32 alen, voff; diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c index bc7469d0a34d4..7c4db816c43d3 100644 --- a/fs/ntfs3/fsntfs.c +++ b/fs/ntfs3/fsntfs.c @@ -2302,8 +2302,8 @@ int ntfs_reparse_init(struct ntfs_sb_info *sbi) goto out; } - root_r = resident_data(attr); - if (root_r->type != ATTR_ZERO || + root_r = resident_data_ex(attr, sizeof(struct INDEX_ROOT)); + if (!root_r || root_r->type != ATTR_ZERO || root_r->rule != NTFS_COLLATION_TYPE_UINTS) { err = -EINVAL; goto out; @@ -2340,8 +2340,8 @@ int ntfs_objid_init(struct ntfs_sb_info *sbi) goto out; } - root = resident_data(attr); - if (root->type != ATTR_ZERO || + root = resident_data_ex(attr, sizeof(struct INDEX_ROOT)); + if (!root || root->type != ATTR_ZERO || root->rule != NTFS_COLLATION_TYPE_UINTS) { err = -EINVAL; goto out; diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 0c9bd669117de..ea4b9193b07d9 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -1648,10 +1648,10 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, goto out6; /* - * Call 'd_instantiate' after inode->i_op is set + * Call 'd_instantiate_new' after inode->i_op is set * but before finish_open. */ - d_instantiate(dentry, inode); + d_instantiate_new(dentry, inode); /* Set original time. inode times (i_ctime) may be changed in ntfs_init_acl. */ inode_set_atime_to_ts(inode, ni->i_crtime); @@ -1699,9 +1699,6 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, if (!fnd) ni_unlock(dir_ni); - if (!err) - unlock_new_inode(inode); - return err; } diff --git a/fs/ntfs3/record.c b/fs/ntfs3/record.c index 32bdb034c2a30..4f12ce15b03b4 100644 --- a/fs/ntfs3/record.c +++ b/fs/ntfs3/record.c @@ -202,7 +202,7 @@ struct ATTRIB *mi_enum_attr(struct ntfs_inode *ni, struct mft_inode *mi, u32 used = le32_to_cpu(rec->used); u32 t32, off, asize, prev_type; u16 t16; - u64 data_size, alloc_size, tot_size; + u64 svcn, evcn, data_size, alloc_size, tot_size; if (!attr) { u32 total = le32_to_cpu(rec->total); @@ -310,10 +310,38 @@ struct ATTRIB *mi_enum_attr(struct ntfs_inode *ni, struct mft_inode *mi, if (t32 && le16_to_cpu(attr->name_off) + t32 > t16) goto out; - /* Check start/end vcn. */ - if (le64_to_cpu(attr->nres.svcn) > le64_to_cpu(attr->nres.evcn) + 1) + /* + * Check start/end vcn. svcn == 0 with evcn == -1 (U64_MAX) is the + * sentinel for an empty non-resident attribute (no allocated + * clusters) and must be accepted: "svcn > evcn + 1" tolerates it, + * since "(u64)-1 + 1" is 0 and "0 > 0" is false. + * + * For a non-empty attribute evcn is a cluster index and must lie + * within the volume (sbi->used.bitmap.nbits, set up in + * ntfs_init_from_boot() before any caller of mi_enum_attr() runs). + * Bounding evcn also prevents a malformed value close to U64_MAX + * from slipping through the near-wrap "evcn + 1" upper bound. + */ + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + if (svcn > evcn + 1) goto out; + if (is_attr_ext(attr)) { + /* sparsed/compressed attribute. */ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + /* No limits. */ +#else + /* Check evcn fits into 32 bits. */ + if (evcn != U64_MAX && evcn >= (1ull << 32)) + goto out; +#endif + } else { + /* Check out of volume for normal attribute. */ + if (evcn != U64_MAX && evcn >= mi->sbi->used.bitmap.nbits) + goto out; + } + data_size = le64_to_cpu(attr->nres.data_size); if (le64_to_cpu(attr->nres.valid_size) > data_size) goto out; diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index 3305fe406cb22..4205a212154b8 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -65,6 +65,7 @@ #include #include #include +#include #include #include #include @@ -957,7 +958,7 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, struct ntfs_sb_info *sbi = sb->s_fs_info; int err; u32 mb, gb, boot_sector_size, sct_per_clst, record_size; - u64 sectors, clusters, mlcn, mlcn2, dev_size0; + u64 sectors, clusters, mlcn, mlcn2, mft_pos, mft2_pos, dev_size0; struct NTFS_BOOT *boot; struct buffer_head *bh; struct MFT_REC *rec; @@ -1026,7 +1027,15 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, mlcn2 = le64_to_cpu(boot->mft2_clst); sectors = le64_to_cpu(boot->sectors_per_volume); - if (mlcn * sct_per_clst >= sectors || mlcn2 * sct_per_clst >= sectors) { + /* + * Convert mlcn/mlcn2 to sector positions before comparing with + * 'sectors'. All three are u64 values that come from the boot + * sector, so use check_mul_overflow() to keep a wraparound from + * silently bypassing the comparison. + */ + if (check_mul_overflow(mlcn, (u64)sct_per_clst, &mft_pos) || + check_mul_overflow(mlcn2, (u64)sct_per_clst, &mft2_pos) || + mft_pos >= sectors || mft2_pos >= sectors) { ntfs_err( sb, "%s: start of MFT 0x%llx (0x%llx) is out of volume 0x%llx.", diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index 04814dd293752..6697362e655bf 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -146,26 +146,29 @@ static int ntfs_read_ea(struct ntfs_inode *ni, struct EA_FULL **ea, for (off = 0; off < size; off += ea_size) { const struct EA_FULL *ef = Add2Ptr(ea_p, off); u32 bytes = size - off; + size_t need; /* Check if we can use field ea->size. */ if (bytes < sizeof(ef->size)) goto out1; + /* Check if we can use fields ef->name_len and ef->elength. */ + if (bytes < offsetof(struct EA_FULL, name)) + goto out1; + + /* Size needed to hold this record's name and value. */ + need = struct_size(ef, name, + 1 + ef->name_len + le16_to_cpu(ef->elength)); + if (ef->size) { ea_size = le32_to_cpu(ef->size); - if (ea_size > bytes) + /* ef->size must fit the list and cover the record. */ + if (ea_size > bytes || ea_size < need) goto out1; continue; } - /* Check if we can use fields ef->name_len and ef->elength. */ - if (bytes < offsetof(struct EA_FULL, name)) - goto out1; - - ea_size = ALIGN(struct_size(ef, name, - 1 + ef->name_len + - le16_to_cpu(ef->elength)), - 4); + ea_size = ALIGN(need, 4); if (ea_size > bytes) goto out1; } diff --git a/fs/ocfs2/acl.c b/fs/ocfs2/acl.c index af1e2cedb217f..090ec60fb576a 100644 --- a/fs/ocfs2/acl.c +++ b/fs/ocfs2/acl.c @@ -110,8 +110,7 @@ static void *ocfs2_acl_to_xattr(const struct posix_acl *acl, size_t *size) return ocfs2_acl; } -static struct posix_acl *ocfs2_get_acl_nolock(struct inode *inode, - int type, +static struct posix_acl *ocfs2_get_acl_nolock(struct inode *inode, int type, struct buffer_head *di_bh) { int name_index; @@ -349,63 +348,105 @@ int ocfs2_acl_chmod(struct inode *inode, struct buffer_head *bh) * Initialize the ACLs of a new inode. If parent directory has default ACL, * then clone to new inode. Called from ocfs2_mknod. */ -int ocfs2_init_acl(handle_t *handle, - struct inode *inode, - struct inode *dir, - struct buffer_head *di_bh, - struct buffer_head *dir_bh, - struct ocfs2_alloc_context *meta_ac, - struct ocfs2_alloc_context *data_ac) +void ocfs2_acl_init_release(struct ocfs2_acl_state *state) +{ + posix_acl_release(state->default_acl); + posix_acl_release(state->acl); + state->default_acl = NULL; + state->acl = NULL; +} + +int ocfs2_acl_init_prepare(struct inode *inode, struct inode *dir, + struct buffer_head *dir_bh, + struct ocfs2_acl_state *state) { struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - struct posix_acl *acl = NULL; - int ret = 0, ret2; - umode_t mode; - - if (!S_ISLNK(inode->i_mode)) { - if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { - down_read(&OCFS2_I(dir)->ip_xattr_sem); - acl = ocfs2_get_acl_nolock(dir, ACL_TYPE_DEFAULT, - dir_bh); - up_read(&OCFS2_I(dir)->ip_xattr_sem); - if (IS_ERR(acl)) - return PTR_ERR(acl); + int ret = 0; + + state->default_acl = NULL; + state->acl = NULL; + state->mode = inode->i_mode; + + if (S_ISLNK(inode->i_mode)) + return 0; + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + down_read(&OCFS2_I(dir)->ip_xattr_sem); + state->default_acl = + ocfs2_get_acl_nolock(dir, ACL_TYPE_DEFAULT, dir_bh); + up_read(&OCFS2_I(dir)->ip_xattr_sem); + if (IS_ERR(state->default_acl)) { + ret = PTR_ERR(state->default_acl); + state->default_acl = NULL; + return ret; } - if (!acl) { - mode = inode->i_mode & ~current_umask(); - ret = ocfs2_acl_set_mode(inode, di_bh, handle, mode); - if (ret) { - mlog_errno(ret); + if (state->default_acl) { + state->acl = posix_acl_dup(state->default_acl); + if (!state->acl) { + ret = -ENOMEM; goto cleanup; } + ret = __posix_acl_create(&state->acl, GFP_NOFS, + &state->mode); + if (ret < 0) + goto cleanup; + if (ret == 0) { + posix_acl_release(state->acl); + state->acl = NULL; + } + if (!S_ISDIR(inode->i_mode)) { + posix_acl_release(state->default_acl); + state->default_acl = NULL; + } + } else { + state->mode &= ~current_umask(); } + } else { + state->mode &= ~current_umask(); } - if ((osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) && acl) { - if (S_ISDIR(inode->i_mode)) { + + return 0; +cleanup: + ocfs2_acl_init_release(state); + return ret; +} + +int ocfs2_init_acl(handle_t *handle, struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_acl_state *state) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + int ret = 0; + + if (S_ISLNK(inode->i_mode)) + return 0; + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (S_ISDIR(inode->i_mode) && state->default_acl) { ret = ocfs2_set_acl(handle, inode, di_bh, - ACL_TYPE_DEFAULT, acl, - meta_ac, data_ac); + ACL_TYPE_DEFAULT, + state->default_acl, meta_ac, + data_ac); if (ret) - goto cleanup; + return ret; } - mode = inode->i_mode; - ret = __posix_acl_create(&acl, GFP_NOFS, &mode); - if (ret < 0) - return ret; + } - ret2 = ocfs2_acl_set_mode(inode, di_bh, handle, mode); - if (ret2) { - mlog_errno(ret2); - ret = ret2; - goto cleanup; - } - if (ret > 0) { - ret = ocfs2_set_acl(handle, inode, - di_bh, ACL_TYPE_ACCESS, - acl, meta_ac, data_ac); + ret = ocfs2_acl_set_mode(inode, di_bh, handle, state->mode); + if (ret) { + mlog_errno(ret); + return ret; + } + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (state->acl) { + ret = ocfs2_set_acl(handle, inode, di_bh, + ACL_TYPE_ACCESS, state->acl, + meta_ac, data_ac); } } -cleanup: - posix_acl_release(acl); + return ret; } diff --git a/fs/ocfs2/acl.h b/fs/ocfs2/acl.h index 667c6f03fa606..a91f9ce278d6d 100644 --- a/fs/ocfs2/acl.h +++ b/fs/ocfs2/acl.h @@ -20,9 +20,20 @@ struct posix_acl *ocfs2_iop_get_acl(struct inode *inode, int type, bool rcu); int ocfs2_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, struct posix_acl *acl, int type); extern int ocfs2_acl_chmod(struct inode *, struct buffer_head *); -extern int ocfs2_init_acl(handle_t *, struct inode *, struct inode *, - struct buffer_head *, struct buffer_head *, - struct ocfs2_alloc_context *, - struct ocfs2_alloc_context *); +struct ocfs2_acl_state { + struct posix_acl *default_acl; + struct posix_acl *acl; + umode_t mode; +}; + +int ocfs2_acl_init_prepare(struct inode *inode, struct inode *dir, + struct buffer_head *dir_bh, + struct ocfs2_acl_state *state); +void ocfs2_acl_init_release(struct ocfs2_acl_state *state); +int ocfs2_init_acl(handle_t *handle, struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_acl_state *state); #endif /* OCFS2_ACL_H */ diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c index d12784aaaa4b1..8b5d8c2d7d367 100644 --- a/fs/ocfs2/cluster/heartbeat.c +++ b/fs/ocfs2/cluster/heartbeat.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,14 @@ static DECLARE_RWSEM(o2hb_callback_sem); * whenever any of the threads sees activity from the node in its region. */ static DEFINE_SPINLOCK(o2hb_live_lock); +/* + * Serializes region pin/unpin dependency management (o2hb_dependent_users + * and the o2nm_depend_item()/o2nm_undepend_item() calls). o2hb_region_pin() + * has to drop o2hb_live_lock across the sleeping o2nm_depend_item(), so the + * spinlock alone can no longer keep pin and unpin mutually exclusive; this + * mutex, taken outside o2hb_live_lock, does. + */ +static DEFINE_MUTEX(o2hb_dependency_mutex); static struct list_head o2hb_live_slots[O2NM_MAX_NODES]; static unsigned long o2hb_live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; static LIST_HEAD(o2hb_node_events); @@ -138,7 +147,7 @@ static unsigned int o2hb_dependent_users; * In global heartbeat mode, we pin/unpin all o2hb regions. This solution * works for both file system and userdlm domains. */ -static int o2hb_region_pin(const char *region_uuid); +static int o2hb_region_pin(const char *region_uuid, bool from_callback); static void o2hb_region_unpin(const char *region_uuid); /* Only sets a new threshold if there are no active regions. @@ -203,6 +212,7 @@ struct o2hb_region { /* protected by the hr_callback_sem */ struct task_struct *hr_task; + u8 hr_node_num; unsigned int hr_blocks; unsigned long long hr_start_block; @@ -257,6 +267,9 @@ struct o2hb_region { /* Message key for negotiate timeout message. */ unsigned int hr_key; struct list_head hr_handler_list; + /* Serializes timeout arming against failed-start and teardown. */ + struct mutex hr_arming_mutex; + bool hr_stopping; /* last hb status, 0 for success, other value for error. */ int hr_last_hb_status; @@ -321,9 +334,14 @@ static void o2hb_write_timeout(struct work_struct *work) static void o2hb_arm_timeout(struct o2hb_region *reg) { + mutex_lock(®->hr_arming_mutex); + + if (reg->hr_stopping) + goto out_unlock; + /* Arm writeout only after thread reaches steady state */ if (atomic_read(®->hr_steady_iterations) != 0) - return; + goto out_unlock; mlog(ML_HEARTBEAT, "Queue write timeout for %u ms\n", O2HB_MAX_WRITE_TIMEOUT_MS); @@ -342,6 +360,18 @@ static void o2hb_arm_timeout(struct o2hb_region *reg) schedule_delayed_work(®->hr_nego_timeout_work, msecs_to_jiffies(O2HB_NEGO_TIMEOUT_MS)); bitmap_zero(reg->hr_nego_node_bitmap, O2NM_MAX_NODES); + +out_unlock: + mutex_unlock(®->hr_arming_mutex); +} + +static void o2hb_queue_nego_timeout(struct o2hb_region *reg, + unsigned long delay) +{ + mutex_lock(®->hr_arming_mutex); + if (!reg->hr_stopping) + schedule_delayed_work(®->hr_nego_timeout_work, delay); + mutex_unlock(®->hr_arming_mutex); } static void o2hb_disarm_timeout(struct o2hb_region *reg) @@ -350,12 +380,25 @@ static void o2hb_disarm_timeout(struct o2hb_region *reg) cancel_delayed_work_sync(®->hr_nego_timeout_work); } -static int o2hb_send_nego_msg(int key, int type, u8 target) +static void o2hb_set_region_stopping(struct o2hb_region *reg, bool stopping) +{ + mutex_lock(®->hr_arming_mutex); + reg->hr_stopping = stopping; + mutex_unlock(®->hr_arming_mutex); +} + +static void o2hb_quiesce_timeout(struct o2hb_region *reg) +{ + o2hb_set_region_stopping(reg, true); + o2hb_disarm_timeout(reg); +} + +static int o2hb_send_nego_msg(int key, int type, u8 target, u8 node_num) { struct o2hb_nego_msg msg; int status, ret; - msg.node_num = o2nm_this_node(); + msg.node_num = node_num; again: ret = o2net_send_message(type, key, &msg, sizeof(msg), target, &status); @@ -373,8 +416,10 @@ static void o2hb_nego_timeout(struct work_struct *work) unsigned long live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; int master_node, i, ret; struct o2hb_region *reg; + u8 node_num; reg = container_of(work, struct o2hb_region, hr_nego_timeout_work.work); + node_num = reg->hr_node_num; /* don't negotiate timeout if last hb failed since it is very * possible io failed. Should let write timeout fence self. */ @@ -385,10 +430,10 @@ static void o2hb_nego_timeout(struct work_struct *work) /* lowest node as master node to make negotiate decision. */ master_node = find_first_bit(live_node_bitmap, O2NM_MAX_NODES); - if (master_node == o2nm_this_node()) { + if (master_node == node_num) { if (!test_bit(master_node, reg->hr_nego_node_bitmap)) { printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg).\n", - o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000, + node_num, O2HB_NEGO_TIMEOUT_MS / 1000, config_item_name(®->hr_item), reg_bdev(reg)); set_bit(master_node, reg->hr_nego_node_bitmap); } @@ -397,8 +442,7 @@ static void o2hb_nego_timeout(struct work_struct *work) /* check negotiate bitmap every second to do timeout * approve decision. */ - schedule_delayed_work(®->hr_nego_timeout_work, - msecs_to_jiffies(1000)); + o2hb_queue_nego_timeout(reg, msecs_to_jiffies(1000)); return; } @@ -417,7 +461,7 @@ static void o2hb_nego_timeout(struct work_struct *work) mlog(ML_HEARTBEAT, "send NEGO_APPROVE msg to node %d\n", i); ret = o2hb_send_nego_msg(reg->hr_key, - O2HB_NEGO_APPROVE_MSG, i); + O2HB_NEGO_APPROVE_MSG, i, node_num); if (ret) mlog(ML_ERROR, "send NEGO_APPROVE msg to node %d fail %d\n", i, ret); @@ -425,10 +469,10 @@ static void o2hb_nego_timeout(struct work_struct *work) } else { /* negotiate timeout with master node. */ printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg), negotiate timeout with node %d.\n", - o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000, config_item_name(®->hr_item), + node_num, O2HB_NEGO_TIMEOUT_MS / 1000, config_item_name(®->hr_item), reg_bdev(reg), master_node); ret = o2hb_send_nego_msg(reg->hr_key, O2HB_NEGO_TIMEOUT_MSG, - master_node); + master_node, node_num); if (ret) mlog(ML_ERROR, "send NEGO_TIMEOUT msg to node %d fail %d\n", master_node, ret); @@ -601,7 +645,9 @@ static int o2hb_issue_node_write(struct o2hb_region *reg, o2hb_bio_wait_init(write_wc); - slot = o2nm_this_node(); + slot = reg->hr_node_num; + if (slot >= O2NM_MAX_NODES) + return -EINVAL; bio = o2hb_setup_one_bio(reg, write_wc, &slot, slot+1, REQ_OP_WRITE | REQ_SYNC); @@ -670,8 +716,12 @@ static int o2hb_check_own_slot(struct o2hb_region *reg) struct o2hb_disk_slot *slot; struct o2hb_disk_heartbeat_block *hb_block; char *errstr; + u8 node_num = reg->hr_node_num; + + if (node_num >= O2NM_MAX_NODES) + return 0; - slot = ®->hr_slots[o2nm_this_node()]; + slot = ®->hr_slots[node_num]; /* Don't check on our 1st timestamp */ if (!slot->ds_last_time) return 0; @@ -712,7 +762,10 @@ static inline void o2hb_prepare_block(struct o2hb_region *reg, struct o2hb_disk_slot *slot; struct o2hb_disk_heartbeat_block *hb_block; - node_num = o2nm_this_node(); + node_num = reg->hr_node_num; + if (node_num >= O2NM_MAX_NODES) + return; + slot = ®->hr_slots[node_num]; hb_block = (struct o2hb_disk_heartbeat_block *)slot->ds_raw_block; @@ -1206,7 +1259,7 @@ static int o2hb_thread(void *data) set_user_nice(current, MIN_NICE); /* Pin node */ - ret = o2nm_depend_this_node(); + ret = o2nm_depend_node(reg->hr_node_num); if (ret) { mlog(ML_ERROR, "Node has been deleted, ret = %d\n", ret); reg->hr_node_deleted = 1; @@ -1215,7 +1268,8 @@ static int o2hb_thread(void *data) } while (!kthread_should_stop() && - !reg->hr_unclean_stop && !reg->hr_aborted_start) { + !reg->hr_unclean_stop && !reg->hr_aborted_start && + o2nm_this_node() == reg->hr_node_num) { /* We track the time spent inside * o2hb_do_disk_heartbeat so that we avoid more than * hr_timeout_ms between disk writes. On busy systems @@ -1264,7 +1318,7 @@ static int o2hb_thread(void *data) } /* Unpin node */ - o2nm_undepend_this_node(); + o2nm_undepend_node(reg->hr_node_num); mlog(ML_HEARTBEAT|ML_KTHREAD, "o2hb thread exiting\n"); @@ -1456,13 +1510,38 @@ void o2hb_init(void) o2hb_debug_init(); } -/* if we're already in a callback then we're already serialized by the sem */ -static void o2hb_fill_node_map_from_callback(unsigned long *map, - unsigned int bits) +static void __o2hb_fill_node_map(unsigned long *map, unsigned int bits) { bitmap_copy(map, o2hb_live_node_bitmap, bits); } +void o2hb_callback_read_lock(void) +{ + down_read(&o2hb_callback_sem); +} + +void o2hb_callback_read_unlock(void) +{ + up_read(&o2hb_callback_sem); +} + +void o2hb_synchronize_callbacks(void) +{ + down_write(&o2hb_callback_sem); + up_write(&o2hb_callback_sem); +} + +/* + * Callers must already hold o2hb_callback_sem for read or write so the copy + * stays serialized with callback delivery. + */ +void o2hb_fill_node_map_locked(unsigned long *map, unsigned int bits) +{ + spin_lock(&o2hb_live_lock); + __o2hb_fill_node_map(map, bits); + spin_unlock(&o2hb_live_lock); +} + /* * get a map of all nodes that are heartbeating in any regions */ @@ -1470,11 +1549,9 @@ void o2hb_fill_node_map(unsigned long *map, unsigned int bits) { /* callers want to serialize this map and callbacks so that they * can trust that they don't miss nodes coming to the party */ - down_read(&o2hb_callback_sem); - spin_lock(&o2hb_live_lock); - o2hb_fill_node_map_from_callback(map, bits); - spin_unlock(&o2hb_live_lock); - up_read(&o2hb_callback_sem); + o2hb_callback_read_lock(); + o2hb_fill_node_map_locked(map, bits); + o2hb_callback_read_unlock(); } EXPORT_SYMBOL_GPL(o2hb_fill_node_map); @@ -1522,6 +1599,8 @@ static void o2hb_region_release(struct config_item *item) mlog(ML_HEARTBEAT, "hb region release (%pg)\n", reg_bdev(reg)); + o2hb_quiesce_timeout(reg); + o2net_unregister_and_flush_handler_list(®->hr_handler_list); o2hb_unmap_slot_data(reg); if (reg->hr_bdev_file) @@ -1537,7 +1616,6 @@ static void o2hb_region_release(struct config_item *item) list_del(®->hr_all_item); spin_unlock(&o2hb_live_lock); - o2net_unregister_handler_list(®->hr_handler_list); kfree(reg); } @@ -1791,7 +1869,8 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, /* We can't heartbeat without having had our node number * configured yet. */ - if (o2nm_this_node() == O2NM_MAX_NODES) + reg->hr_node_num = o2nm_this_node(); + if (reg->hr_node_num == O2NM_MAX_NODES) return -EINVAL; ret = kstrtol(p, 0, &fd); @@ -1851,9 +1930,6 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, goto out; } - INIT_DELAYED_WORK(®->hr_write_timeout_work, o2hb_write_timeout); - INIT_DELAYED_WORK(®->hr_nego_timeout_work, o2hb_nego_timeout); - /* * A node is considered live after it has beat LIVE_THRESHOLD * times. We're not steady until we've given them a chance @@ -1873,6 +1949,7 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, atomic_set(®->hr_steady_iterations, live_threshold); /* unsteady_iterations is triple the steady_iterations */ atomic_set(®->hr_unsteady_iterations, (live_threshold * 3)); + o2hb_set_region_stopping(reg, false); hb_task = kthread_run(o2hb_thread, reg, "o2hb-%s", reg->hr_item.ci_name); @@ -1922,6 +1999,8 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, out: if (ret < 0) { + o2hb_quiesce_timeout(reg); + spin_lock(&o2hb_live_lock); hb_task = reg->hr_task; reg->hr_task = NULL; @@ -2036,6 +2115,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g ret = -ENAMETOOLONG; goto free; } + reg->hr_node_num = O2NM_MAX_NODES; spin_lock(&o2hb_live_lock); reg->hr_region_num = 0; @@ -2060,6 +2140,10 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g */ reg->hr_key = crc32_le(reg->hr_region_num + O2NM_MAX_REGIONS, name, strlen(name)); + mutex_init(®->hr_arming_mutex); + reg->hr_stopping = true; + INIT_DELAYED_WORK(®->hr_write_timeout_work, o2hb_write_timeout); + INIT_DELAYED_WORK(®->hr_nego_timeout_work, o2hb_nego_timeout); INIT_LIST_HEAD(®->hr_handler_list); ret = o2net_register_handler(O2HB_NEGO_TIMEOUT_MSG, reg->hr_key, sizeof(struct o2hb_nego_msg), @@ -2080,7 +2164,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g return ®->hr_item; unregister_handler: - o2net_unregister_handler_list(®->hr_handler_list); + o2net_unregister_and_flush_handler_list(®->hr_handler_list); remove_item: spin_lock(&o2hb_live_lock); list_del(®->hr_all_item); @@ -2099,6 +2183,8 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group, struct o2hb_region *reg = to_o2hb_region(item); int quorum_region = 0; + o2hb_quiesce_timeout(reg); + /* stop the thread when the user removes the region dir */ spin_lock(&o2hb_live_lock); hb_task = reg->hr_task; @@ -2142,6 +2228,7 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group, * If global heartbeat active and there are dependent users, * pin all regions if quorum region count <= CUT_OFF */ + mutex_lock(&o2hb_dependency_mutex); spin_lock(&o2hb_live_lock); if (!o2hb_dependent_users) @@ -2149,10 +2236,11 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group, if (bitmap_weight(o2hb_quorum_region_bitmap, O2NM_MAX_REGIONS) <= O2HB_PIN_CUT_OFF) - o2hb_region_pin(NULL); + o2hb_region_pin(NULL, true); unlock: spin_unlock(&o2hb_live_lock); + mutex_unlock(&o2hb_dependency_mutex); } static ssize_t o2hb_heartbeat_group_dead_threshold_show(struct config_item *item, @@ -2290,48 +2378,113 @@ EXPORT_SYMBOL_GPL(o2hb_setup_callback); * In local, we only pin the matching region. In global we pin all the active * regions. */ -static int o2hb_region_pin(const char *region_uuid) +static int o2hb_region_pin(const char *region_uuid, bool from_callback) { - int ret = 0, found = 0; - struct o2hb_region *reg; + int ret = 0, found; + struct o2hb_region *reg, *pinned; char *uuid; assert_spin_locked(&o2hb_live_lock); - list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) { - if (reg->hr_item_dropped) - continue; + do { + found = 0; + pinned = NULL; - uuid = config_item_name(®->hr_item); + list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) { + if (reg->hr_item_dropped) + continue; - /* local heartbeat */ - if (region_uuid) { - if (strcmp(region_uuid, uuid)) + uuid = config_item_name(®->hr_item); + + /* local heartbeat */ + if (region_uuid) { + if (strcmp(region_uuid, uuid)) + continue; + found = 1; + } + + if (reg->hr_item_pinned || reg->hr_item_dropped) { + if (found) + break; continue; - found = 1; + } + + /* + * Found a region that needs pinning. Take a reference + * so it stays alive while we drop the lock below. + */ + pinned = reg; + config_item_get(®->hr_item); + break; } - if (reg->hr_item_pinned || reg->hr_item_dropped) - goto skip_pin; + if (!pinned) + break; + + uuid = config_item_name(&pinned->hr_item); + + /* + * o2nm_depend_item() -> configfs_depend_item() can sleep (it + * takes the configfs root inode rwsem), so it must not run + * under o2hb_live_lock. Drop the lock across it; @pinned is + * kept alive by the reference taken above. The region list may + * change while unlocked, so we rescan from the top afterwards. + */ + spin_unlock(&o2hb_live_lock); /* Ignore ENOENT only for local hb (userdlm domain) */ - ret = o2nm_depend_item(®->hr_item); + if (from_callback) + ret = o2nm_depend_item_unlocked(&pinned->hr_item); + else + ret = o2nm_depend_item(&pinned->hr_item); + + spin_lock(&o2hb_live_lock); if (!ret) { - mlog(ML_CLUSTER, "Pin region %s\n", uuid); - reg->hr_item_pinned = 1; - } else { - if (ret == -ENOENT && found) - ret = 0; - else { - mlog(ML_ERROR, "Pin region %s fails with %d\n", - uuid, ret); + /* + * o2hb_live_lock was dropped across o2nm_depend_item(). + * o2hb_set_quorum_device() runs in the heartbeat thread + * without o2hb_dependency_mutex, so for global heartbeat + * it may have crossed O2HB_PIN_CUT_OFF and unpinned the + * regions while we slept. If that happened this pin is + * no longer wanted; undo it and stop rather than + * resurrecting it on the rescan below. + */ + if (!region_uuid && + bitmap_weight(o2hb_quorum_region_bitmap, + O2NM_MAX_REGIONS) > O2HB_PIN_CUT_OFF) { + o2nm_undepend_item(&pinned->hr_item); + spin_unlock(&o2hb_live_lock); + config_item_put(&pinned->hr_item); + spin_lock(&o2hb_live_lock); break; } + mlog(ML_CLUSTER, "Pin region %s\n", uuid); + pinned->hr_item_pinned = 1; + } else if (ret == -ENOENT && (found || !region_uuid)) { + /* + * For local hb (found): ignore ENOENT from userdlm + * domains as before. For global hb (!region_uuid): + * the region may have been detached from configfs + * while the lock was dropped — skip it and continue + * pinning the remaining regions. + */ + ret = 0; + } else { + mlog(ML_ERROR, "Pin region %s fails with %d\n", + uuid, ret); } -skip_pin: - if (found) - break; - } + + /* + * config_item_put() may drop the last reference and run + * o2hb_region_release(), which also grabs o2hb_live_lock and + * can sleep, so it must happen with the lock released. + */ + spin_unlock(&o2hb_live_lock); + config_item_put(&pinned->hr_item); + spin_lock(&o2hb_live_lock); + + /* local hb pins a single matching region */ + } while (!ret && !region_uuid); return ret; } @@ -2376,12 +2529,13 @@ static int o2hb_region_inc_user(const char *region_uuid) { int ret = 0; + mutex_lock(&o2hb_dependency_mutex); spin_lock(&o2hb_live_lock); /* local heartbeat */ if (!o2hb_global_heartbeat_active()) { - ret = o2hb_region_pin(region_uuid); - goto unlock; + ret = o2hb_region_pin(region_uuid, false); + goto unlock; } /* @@ -2393,16 +2547,23 @@ static int o2hb_region_inc_user(const char *region_uuid) goto unlock; if (bitmap_weight(o2hb_quorum_region_bitmap, - O2NM_MAX_REGIONS) <= O2HB_PIN_CUT_OFF) - ret = o2hb_region_pin(NULL); + O2NM_MAX_REGIONS) <= O2HB_PIN_CUT_OFF) { + ret = o2hb_region_pin(NULL, false); + if (ret) { + o2hb_region_unpin(NULL); + o2hb_dependent_users--; + } + } unlock: spin_unlock(&o2hb_live_lock); + mutex_unlock(&o2hb_dependency_mutex); return ret; } static void o2hb_region_dec_user(const char *region_uuid) { + mutex_lock(&o2hb_dependency_mutex); spin_lock(&o2hb_live_lock); /* local heartbeat */ @@ -2421,6 +2582,7 @@ static void o2hb_region_dec_user(const char *region_uuid) unlock: spin_unlock(&o2hb_live_lock); + mutex_unlock(&o2hb_dependency_mutex); } int o2hb_register_callback(const char *region_uuid, @@ -2495,7 +2657,7 @@ int o2hb_check_node_heartbeating_no_sem(u8 node_num) unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; spin_lock(&o2hb_live_lock); - o2hb_fill_node_map_from_callback(testing_map, O2NM_MAX_NODES); + __o2hb_fill_node_map(testing_map, O2NM_MAX_NODES); spin_unlock(&o2hb_live_lock); if (!test_bit(node_num, testing_map)) { mlog(ML_HEARTBEAT, @@ -2512,7 +2674,7 @@ int o2hb_check_node_heartbeating_from_callback(u8 node_num) { unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; - o2hb_fill_node_map_from_callback(testing_map, O2NM_MAX_NODES); + o2hb_fill_node_map_locked(testing_map, O2NM_MAX_NODES); if (!test_bit(node_num, testing_map)) { mlog(ML_HEARTBEAT, "node (%u) does not have heartbeating enabled.\n", diff --git a/fs/ocfs2/cluster/heartbeat.h b/fs/ocfs2/cluster/heartbeat.h index 8ef8c1b9eeb76..2ca2b657583c0 100644 --- a/fs/ocfs2/cluster/heartbeat.h +++ b/fs/ocfs2/cluster/heartbeat.h @@ -58,6 +58,11 @@ int o2hb_register_callback(const char *region_uuid, struct o2hb_callback_func *hc); void o2hb_unregister_callback(const char *region_uuid, struct o2hb_callback_func *hc); +void o2hb_callback_read_lock(void); +void o2hb_callback_read_unlock(void); +void o2hb_synchronize_callbacks(void); +void o2hb_fill_node_map_locked(unsigned long *map, + unsigned int bits); void o2hb_fill_node_map(unsigned long *map, unsigned int bits); void o2hb_exit(void); diff --git a/fs/ocfs2/cluster/nodemanager.c b/fs/ocfs2/cluster/nodemanager.c index 4025631545508..e08850a5d7364 100644 --- a/fs/ocfs2/cluster/nodemanager.c +++ b/fs/ocfs2/cluster/nodemanager.c @@ -326,6 +326,7 @@ static ssize_t o2nm_node_local_store(struct config_item *item, const char *page, struct o2nm_node *node = to_o2nm_node(item); struct o2nm_cluster *cluster; unsigned long tmp; + bool starting = false; char *p = (char *)page; ssize_t ret; @@ -362,11 +363,13 @@ static ssize_t o2nm_node_local_store(struct config_item *item, const char *page, ret = o2net_start_listening(node); if (ret) goto out; + starting = true; } if (!tmp && cluster->cl_has_local && cluster->cl_local_node == node->nd_num) { o2net_stop_listening(node); + cluster->cl_has_local = 0; cluster->cl_local_node = O2NM_INVALID_NODE_NUM; } @@ -374,6 +377,8 @@ static ssize_t o2nm_node_local_store(struct config_item *item, const char *page, if (node->nd_local) { cluster->cl_has_local = tmp; cluster->cl_local_node = node->nd_num; + if (starting) + o2net_complete_start_listening(node); } ret = count; @@ -777,17 +782,23 @@ int o2nm_depend_item(struct config_item *item) return configfs_depend_item(&o2nm_cluster_group.cs_subsys, item); } +int o2nm_depend_item_unlocked(struct config_item *item) +{ + return configfs_depend_item_unlocked(&o2nm_cluster_group.cs_subsys, + item); +} + void o2nm_undepend_item(struct config_item *item) { configfs_undepend_item(item); } -int o2nm_depend_this_node(void) +int o2nm_depend_node(u8 node_num) { int ret = 0; struct o2nm_node *local_node; - local_node = o2nm_get_node_by_num(o2nm_this_node()); + local_node = o2nm_get_node_by_num(node_num); if (!local_node) { ret = -EINVAL; goto out; @@ -800,17 +811,27 @@ int o2nm_depend_this_node(void) return ret; } -void o2nm_undepend_this_node(void) +void o2nm_undepend_node(u8 node_num) { struct o2nm_node *local_node; - local_node = o2nm_get_node_by_num(o2nm_this_node()); + local_node = o2nm_get_node_by_num(node_num); BUG_ON(!local_node); o2nm_undepend_item(&local_node->nd_item); o2nm_node_put(local_node); } +int o2nm_depend_this_node(void) +{ + return o2nm_depend_node(o2nm_this_node()); +} + +void o2nm_undepend_this_node(void) +{ + o2nm_undepend_node(o2nm_this_node()); +} + static void __exit exit_o2nm(void) { diff --git a/fs/ocfs2/cluster/nodemanager.h b/fs/ocfs2/cluster/nodemanager.h index 3490e77a952d5..ca3483fb54504 100644 --- a/fs/ocfs2/cluster/nodemanager.h +++ b/fs/ocfs2/cluster/nodemanager.h @@ -64,7 +64,10 @@ void o2nm_node_get(struct o2nm_node *node); void o2nm_node_put(struct o2nm_node *node); int o2nm_depend_item(struct config_item *item); +int o2nm_depend_item_unlocked(struct config_item *item); void o2nm_undepend_item(struct config_item *item); +int o2nm_depend_node(u8 node_num); +void o2nm_undepend_node(u8 node_num); int o2nm_depend_this_node(void); void o2nm_undepend_this_node(void); diff --git a/fs/ocfs2/cluster/tcp.c b/fs/ocfs2/cluster/tcp.c index 09a1f3b77bb8b..474fe1414cee8 100644 --- a/fs/ocfs2/cluster/tcp.c +++ b/fs/ocfs2/cluster/tcp.c @@ -38,6 +38,8 @@ */ #include +#include +#include #include #include #include @@ -102,9 +104,16 @@ static struct socket *o2net_listen_sock; * quorum work is queued as sock containers are shutdown.. stop_listening * tears down all the node's sock containers, preventing future shutdowns * and queued quorum work, before canceling delayed quorum work and - * destroying the work queue. + * destroying the work queue. Handler teardown can also race local listener + * shutdown, so keep a waitable destroying pointer until the old ordered + * queue has finished draining. */ static struct workqueue_struct *o2net_wq; +static struct workqueue_struct *o2net_wq_destroying; +static DEFINE_MUTEX(o2net_wq_mutex); +static DECLARE_COMPLETION(o2net_wq_destroyed); +/* Heartbeat callbacks stay registered across local-node off/on. */ +static bool o2net_listening; static struct work_struct o2net_listen_work; static struct o2hb_callback_func o2net_hb_up, o2net_hb_down; @@ -884,6 +893,27 @@ void o2net_unregister_handler_list(struct list_head *list) } EXPORT_SYMBOL_GPL(o2net_unregister_handler_list); +static void o2net_flush_wq(void) +{ + mutex_lock(&o2net_wq_mutex); + if (o2net_wq_destroying) { + mutex_unlock(&o2net_wq_mutex); + wait_for_completion(&o2net_wq_destroyed); + return; + } + + if (o2net_wq) + flush_workqueue(o2net_wq); + mutex_unlock(&o2net_wq_mutex); +} + +void o2net_unregister_and_flush_handler_list(struct list_head *list) +{ + o2net_unregister_handler_list(list); + o2net_flush_wq(); +} +EXPORT_SYMBOL_GPL(o2net_unregister_and_flush_handler_list); + static struct o2net_msg_handler *o2net_handler_get(u32 msg_type, u32 key) { struct o2net_msg_handler *nmh; @@ -1692,6 +1722,19 @@ static void o2net_still_up(struct work_struct *work) /* ------------------------------------------------------------ */ +static void o2net_hb_node_up(struct o2net_node *nn) +{ + /* ensure an immediate connect attempt */ + nn->nn_last_connect_attempt = jiffies - + (msecs_to_jiffies(o2net_reconnect_delay()) + 1); + + spin_lock(&nn->nn_lock); + atomic_set(&nn->nn_timeout, 0); + if (nn->nn_persistent_error) + o2net_set_nn_state(nn, NULL, 0, 0); + spin_unlock(&nn->nn_lock); +} + void o2net_disconnect_node(struct o2nm_node *node) { struct o2net_node *nn = o2net_nn_from_num(node->nd_num); @@ -1702,52 +1745,48 @@ void o2net_disconnect_node(struct o2nm_node *node) o2net_set_nn_state(nn, NULL, 0, -ENOTCONN); spin_unlock(&nn->nn_lock); - if (o2net_wq) { - cancel_delayed_work(&nn->nn_connect_expired); - cancel_delayed_work(&nn->nn_connect_work); - cancel_delayed_work(&nn->nn_still_up); - flush_workqueue(o2net_wq); - } + cancel_delayed_work(&nn->nn_connect_expired); + cancel_delayed_work(&nn->nn_connect_work); + cancel_delayed_work(&nn->nn_still_up); + o2net_flush_wq(); } static void o2net_hb_node_down_cb(struct o2nm_node *node, int node_num, void *data) { + u8 this_node; + o2quo_hb_down(node_num); if (!node) - return; + goto out; + + this_node = o2nm_this_node(); + if (!READ_ONCE(o2net_listening) || this_node == O2NM_MAX_NODES) + goto out; - if (node_num != o2nm_this_node()) + if (node_num != this_node) o2net_disconnect_node(node); +out: BUG_ON(atomic_read(&o2net_connected_peers) < 0); } static void o2net_hb_node_up_cb(struct o2nm_node *node, int node_num, void *data) { - struct o2net_node *nn = o2net_nn_from_num(node_num); + u8 this_node; o2quo_hb_up(node_num); BUG_ON(!node); - /* ensure an immediate connect attempt */ - nn->nn_last_connect_attempt = jiffies - - (msecs_to_jiffies(o2net_reconnect_delay()) + 1); + this_node = o2nm_this_node(); + if (!READ_ONCE(o2net_listening) || this_node == O2NM_MAX_NODES) + return; - if (node_num != o2nm_this_node()) { - /* believe it or not, accept and node heartbeating testing - * can succeed for this node before we got here.. so - * only use set_nn_state to clear the persistent error - * if that hasn't already happened */ - spin_lock(&nn->nn_lock); - atomic_set(&nn->nn_timeout, 0); - if (nn->nn_persistent_error) - o2net_set_nn_state(nn, NULL, 0, 0); - spin_unlock(&nn->nn_lock); - } + if (node_num != this_node) + o2net_hb_node_up(o2net_nn_from_num(node_num)); } void o2net_unregister_hb_callbacks(void) @@ -1756,6 +1795,37 @@ void o2net_unregister_hb_callbacks(void) o2hb_unregister_callback(NULL, &o2net_hb_down); } +/* + * Delay heartbeat-driven network work until the local node is fully published + * through o2nm_this_node(), then replay the nodes that are already live while + * callback delivery stays blocked. + */ +void o2net_complete_start_listening(struct o2nm_node *node) +{ + unsigned long live_nodes[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long node_num; + u8 local_node; + + local_node = o2nm_this_node(); + if (WARN_ON_ONCE(local_node == O2NM_MAX_NODES)) + return; + if (WARN_ON_ONCE(local_node != node->nd_num)) + return; + if (WARN_ON_ONCE(!o2net_wq)) + return; + + o2hb_callback_read_lock(); + WRITE_ONCE(o2net_listening, true); + o2hb_fill_node_map_locked(live_nodes, O2NM_MAX_NODES); + for_each_set_bit(node_num, live_nodes, O2NM_MAX_NODES) { + if (node_num == local_node) + continue; + + o2net_hb_node_up(o2net_nn_from_num(node_num)); + } + o2hb_callback_read_unlock(); +} + int o2net_register_hb_callbacks(void) { int ret; @@ -2023,6 +2093,36 @@ static int o2net_open_listening_sock(__be32 addr, __be16 port) return ret; } +static void o2net_destroy_wq(void) +{ + struct workqueue_struct *wq; + + mutex_lock(&o2net_wq_mutex); + if (o2net_wq_destroying) { + mutex_unlock(&o2net_wq_mutex); + wait_for_completion(&o2net_wq_destroyed); + return; + } + + wq = o2net_wq; + if (!wq) { + mutex_unlock(&o2net_wq_mutex); + return; + } + + reinit_completion(&o2net_wq_destroyed); + o2net_wq_destroying = wq; + mutex_unlock(&o2net_wq_mutex); + + destroy_workqueue(wq); + + mutex_lock(&o2net_wq_mutex); + o2net_wq = NULL; + o2net_wq_destroying = NULL; + complete_all(&o2net_wq_destroyed); + mutex_unlock(&o2net_wq_mutex); +} + /* * called from node manager when we should bring up our network listening * socket. node manager handles all the serialization to only call this @@ -2033,22 +2133,44 @@ static int o2net_open_listening_sock(__be32 addr, __be16 port) int o2net_start_listening(struct o2nm_node *node) { int ret = 0; + struct workqueue_struct *wq; + + if (WARN_ON_ONCE(READ_ONCE(o2net_listening))) + return -EBUSY; + + mutex_lock(&o2net_wq_mutex); + if (o2net_wq_destroying) { + mutex_unlock(&o2net_wq_mutex); + return -EBUSY; + } + if (WARN_ON_ONCE(o2net_wq)) { + mutex_unlock(&o2net_wq_mutex); + return -EBUSY; + } + mutex_unlock(&o2net_wq_mutex); - BUG_ON(o2net_wq != NULL); BUG_ON(o2net_listen_sock != NULL); mlog(ML_KTHREAD, "starting o2net thread...\n"); - o2net_wq = alloc_ordered_workqueue("o2net", WQ_MEM_RECLAIM); - if (o2net_wq == NULL) { + wq = alloc_ordered_workqueue("o2net", WQ_MEM_RECLAIM); + if (!wq) { mlog(ML_ERROR, "unable to launch o2net thread\n"); return -ENOMEM; /* ? */ } + mutex_lock(&o2net_wq_mutex); + if (unlikely(o2net_wq_destroying || o2net_wq)) { + mutex_unlock(&o2net_wq_mutex); + destroy_workqueue(wq); + return -EBUSY; + } + o2net_wq = wq; + mutex_unlock(&o2net_wq_mutex); + ret = o2net_open_listening_sock(node->nd_ipv4_address, node->nd_ipv4_port); if (ret) { - destroy_workqueue(o2net_wq); - o2net_wq = NULL; + o2net_destroy_wq(); } else o2quo_conn_up(node->nd_num); @@ -2065,6 +2187,9 @@ void o2net_stop_listening(struct o2nm_node *node) BUG_ON(o2net_wq == NULL); BUG_ON(o2net_listen_sock == NULL); + WRITE_ONCE(o2net_listening, false); + o2hb_synchronize_callbacks(); + /* stop the listening socket from generating work */ write_lock_bh(&sock->sk->sk_callback_lock); sock->sk->sk_data_ready = sock->sk->sk_user_data; @@ -2081,8 +2206,7 @@ void o2net_stop_listening(struct o2nm_node *node) /* finish all work and tear down the work queue */ mlog(ML_KTHREAD, "waiting for o2net thread to exit....\n"); - destroy_workqueue(o2net_wq); - o2net_wq = NULL; + o2net_destroy_wq(); sock_release(o2net_listen_sock); o2net_listen_sock = NULL; diff --git a/fs/ocfs2/cluster/tcp.h b/fs/ocfs2/cluster/tcp.h index a75b551d31c7b..a11bcee289479 100644 --- a/fs/ocfs2/cluster/tcp.h +++ b/fs/ocfs2/cluster/tcp.h @@ -89,6 +89,7 @@ int o2net_register_handler(u32 msg_type, u32 key, u32 max_len, o2net_post_msg_handler_func *post_func, struct list_head *unreg_list); void o2net_unregister_handler_list(struct list_head *list); +void o2net_unregister_and_flush_handler_list(struct list_head *list); void o2net_fill_node_map(unsigned long *map, unsigned bytes); @@ -96,6 +97,7 @@ struct o2nm_node; int o2net_register_hb_callbacks(void); void o2net_unregister_hb_callbacks(void); int o2net_start_listening(struct o2nm_node *node); +void o2net_complete_start_listening(struct o2nm_node *node); void o2net_stop_listening(struct o2nm_node *node); void o2net_disconnect_node(struct o2nm_node *node); int o2net_num_connected_peers(void); diff --git a/fs/ocfs2/dir.c b/fs/ocfs2/dir.c index d7fc3cccf2f4b..5ee894966b12a 100644 --- a/fs/ocfs2/dir.c +++ b/fs/ocfs2/dir.c @@ -625,6 +625,28 @@ static int ocfs2_validate_dx_root(struct super_block *sb, le16_to_cpu(el->l_count)); goto bail; } + } else { + struct ocfs2_dx_entry_list *dl_list = &dx_root->dr_entries; + + if (le16_to_cpu(dl_list->de_count) != + ocfs2_dx_entries_per_root(sb)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has invalid de_count %u (expected %u)\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + le16_to_cpu(dl_list->de_count), + ocfs2_dx_entries_per_root(sb)); + goto bail; + } + + if (le16_to_cpu(dl_list->de_num_used) > + le16_to_cpu(dl_list->de_count)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has invalid de_num_used %u (de_count %u)\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + le16_to_cpu(dl_list->de_num_used), + le16_to_cpu(dl_list->de_count)); + goto bail; + } } bail: @@ -664,10 +686,25 @@ static int ocfs2_validate_dx_leaf(struct super_block *sb, return ret; } - if (!OCFS2_IS_VALID_DX_LEAF(dx_leaf)) { - ret = ocfs2_error(sb, "Dir Index Leaf has bad signature %.*s\n", - 7, dx_leaf->dl_signature); - } + if (!OCFS2_IS_VALID_DX_LEAF(dx_leaf)) + return ocfs2_error(sb, "Dir Index Leaf has bad signature %.*s\n", + 7, dx_leaf->dl_signature); + + if (le16_to_cpu(dx_leaf->dl_list.de_count) != + ocfs2_dx_entries_per_leaf(sb)) + return ocfs2_error(sb, + "Dir Index Leaf # %llu has invalid de_count %u (expected %u)\n", + (unsigned long long)le64_to_cpu(dx_leaf->dl_blkno), + le16_to_cpu(dx_leaf->dl_list.de_count), + ocfs2_dx_entries_per_leaf(sb)); + + if (le16_to_cpu(dx_leaf->dl_list.de_num_used) > + le16_to_cpu(dx_leaf->dl_list.de_count)) + return ocfs2_error(sb, + "Dir Index Leaf # %llu has invalid de_num_used %u (de_count %u)\n", + (unsigned long long)le64_to_cpu(dx_leaf->dl_blkno), + le16_to_cpu(dx_leaf->dl_list.de_num_used), + le16_to_cpu(dx_leaf->dl_list.de_count)); return ret; } @@ -1917,7 +1954,7 @@ static int ocfs2_dir_foreach_blk_el(struct inode *inode, i += le16_to_cpu(de->rec_len); } offset = i; - ctx->pos = (ctx->pos & ~(sb->s_blocksize - 1)) + ctx->pos = (ctx->pos & ~((loff_t)sb->s_blocksize - 1)) | offset; *f_version = inode_query_iversion(inode); } diff --git a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c index aee3b4c56dcc6..612969867ff92 100644 --- a/fs/ocfs2/dlm/dlmmaster.c +++ b/fs/ocfs2/dlm/dlmmaster.c @@ -3099,6 +3099,12 @@ int dlm_migrate_request_handler(struct o2net_msg *msg, u32 len, void *data, name = migrate->name; namelen = migrate->namelen; + if (namelen > DLM_LOCKID_NAME_MAX) { + mlog(ML_ERROR, "%s: invalid name length %u in migrate request\n", + dlm->name, namelen); + ret = -EINVAL; + goto leave; + } hash = dlm_lockid_hash(name, namelen); /* preallocate.. if this fails, abort */ diff --git a/fs/ocfs2/dlm/dlmrecovery.c b/fs/ocfs2/dlm/dlmrecovery.c index 9b97bf73df228..9d4a2695b9594 100644 --- a/fs/ocfs2/dlm/dlmrecovery.c +++ b/fs/ocfs2/dlm/dlmrecovery.c @@ -1357,6 +1357,15 @@ int dlm_mig_lockres_handler(struct o2net_msg *msg, u32 len, void *data, if (!dlm_grab(dlm)) return -EINVAL; + if (mres->lockname_len > DLM_LOCKID_NAME_MAX || + mres->num_locks > DLM_MAX_MIGRATABLE_LOCKS || + be16_to_cpu(msg->data_len) < struct_size(mres, ml, mres->num_locks)) { + mlog(ML_ERROR, "%s: invalid lockres migration message from %u\n", + dlm->name, mres->master); + dlm_put(dlm); + return -EINVAL; + } + if (!dlm_joined(dlm)) { mlog(ML_ERROR, "Domain %s not joined! " "lockres %.*s, master %u\n", diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index 662dbc845b8bd..180107a11046c 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -1528,6 +1528,22 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + if ((le32_to_cpu(di->i_flags) & OCFS2_ORPHANED_FL) && + le16_to_cpu(di->i_orphaned_slot) >= OCFS2_SB(sb)->max_slots) { + rc = ocfs2_error(sb, "Invalid dinode %llu: orphaned slot %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_orphaned_slot)); + goto bail; + } + + if ((le32_to_cpu(di->i_flags) & OCFS2_DIO_ORPHANED_FL) && + le16_to_cpu(di->i_dio_orphaned_slot) >= OCFS2_SB(sb)->max_slots) { + rc = ocfs2_error(sb, "Invalid dinode %llu: DIO orphaned slot %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_dio_orphaned_slot)); + goto bail; + } + /* * Reject dinodes whose i_mode does not name one of the seven * canonical POSIX file types. ocfs2_populate_inode() copies @@ -1608,6 +1624,10 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + rc = ocfs2_validate_inode_xattr(sb, bh->b_blocknr, di); + if (rc) + goto bail; + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) { struct ocfs2_inline_data *data = &di->id2.i_data; @@ -1948,8 +1968,6 @@ int ocfs2_read_inode_block_full(struct inode *inode, struct buffer_head **bh, rc = ocfs2_read_blocks(INODE_CACHE(inode), OCFS2_I(inode)->ip_blkno, 1, &tmp, flags, ocfs2_validate_inode_block); - if (rc < 0) - make_bad_inode(inode); /* If ocfs2_read_blocks() got us a new bh, pass it up. */ if (!rc && !*bh) *bh = tmp; diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c index 1277666c77cd2..4e9f8dd63dcb2 100644 --- a/fs/ocfs2/namei.c +++ b/fs/ocfs2/namei.c @@ -256,6 +256,7 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, sigset_t oldset; int did_block_signals = 0; struct ocfs2_dentry_lock *dl = NULL; + struct ocfs2_acl_state acl_state = { 0 }; trace_ocfs2_mknod(dir, dentry, dentry->d_name.len, dentry->d_name.name, (unsigned long long)OCFS2_I(dir)->ip_blkno, @@ -330,14 +331,13 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, } } - /* calculate meta data/clusters for setting security and acl xattr */ - status = ocfs2_calc_xattr_init(dir, parent_fe_bh, mode, - &si, &want_clusters, - &xattr_credits, &want_meta); - if (status < 0) { - mlog_errno(status); + status = ocfs2_acl_init_prepare(inode, dir, parent_fe_bh, &acl_state); + if (status < 0) goto leave; - } + + /* calculate meta data/clusters for setting security and acl xattr */ + ocfs2_calc_xattr_init(dir, mode, &si, &want_clusters, &xattr_credits, + &want_meta, &acl_state); /* Reserve a cluster if creating an extent based directory. */ if (S_ISDIR(mode) && !ocfs2_supports_inline_data(osb)) { @@ -411,8 +411,8 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, inc_nlink(dir); } - status = ocfs2_init_acl(handle, inode, dir, new_fe_bh, parent_fe_bh, - meta_ac, data_ac); + status = ocfs2_init_acl(handle, inode, new_fe_bh, meta_ac, data_ac, + &acl_state); if (status < 0) { mlog_errno(status); @@ -477,6 +477,8 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, brelse(parent_fe_bh); kfree(si.value); + ocfs2_acl_init_release(&acl_state); + ocfs2_free_dir_lookup_result(&lookup); if (inode_ac) diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index 7323bde70caaf..d9f22b4a26546 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -116,6 +116,33 @@ static int ocfs2_validate_refcount_block(struct super_block *sb, le32_to_cpu(rb->rf_fs_generation)); goto out; } + + /* + * rf_records (rl_count/rl_used/rl_recs[]) is only meaningful when + * this block is not an interior tree block (OCFS2_REFCOUNT_TREE_FL); + * in that case the same union bytes hold an extent list (rf_list) + * instead, which is validated by ocfs2_validate_extent_block(). + */ + if (!(le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL)) { + if (le16_to_cpu(rb->rf_records.rl_count) != + ocfs2_refcount_recs_per_rb(sb)) { + rc = ocfs2_error(sb, + "Refcount block #%llu has an invalid rl_count of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(rb->rf_records.rl_count)); + goto out; + } + + if (le16_to_cpu(rb->rf_records.rl_used) > + le16_to_cpu(rb->rf_records.rl_count)) { + rc = ocfs2_error(sb, + "Refcount block #%llu has an invalid rl_used of %u (rl_count %u)\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(rb->rf_records.rl_used), + le16_to_cpu(rb->rf_records.rl_count)); + goto out; + } + } out: return rc; } @@ -3360,10 +3387,9 @@ static int ocfs2_replace_cow(struct ocfs2_cow_context *context) cow_start += num_clusters; } - if (ocfs2_dealloc_has_cluster(&context->dealloc)) { + if (ocfs2_dealloc_has_cluster(&context->dealloc)) ocfs2_schedule_truncate_log_flush(osb, 1); - ocfs2_run_deallocs(osb, &context->dealloc); - } + ocfs2_run_deallocs(osb, &context->dealloc); return ret; } @@ -3846,10 +3872,9 @@ static int ocfs2_attach_refcount_tree(struct inode *inode, ocfs2_unlock_refcount_tree(osb, ref_tree, 1); brelse(ref_root_bh); - if (!ret && ocfs2_dealloc_has_cluster(&dealloc)) { + if (!ret && ocfs2_dealloc_has_cluster(&dealloc)) ocfs2_schedule_truncate_log_flush(osb, 1); - ocfs2_run_deallocs(osb, &dealloc); - } + ocfs2_run_deallocs(osb, &dealloc); out: /* * Empty the extent map so that we may get the right extent @@ -4135,10 +4160,9 @@ static int ocfs2_create_reflink_node(struct inode *s_inode, ocfs2_unlock_refcount_tree(osb, ref_tree, 1); brelse(ref_root_bh); out: - if (ocfs2_dealloc_has_cluster(&dealloc)) { + if (ocfs2_dealloc_has_cluster(&dealloc)) ocfs2_schedule_truncate_log_flush(osb, 1); - ocfs2_run_deallocs(osb, &dealloc); - } + ocfs2_run_deallocs(osb, &dealloc); return ret; } @@ -4691,10 +4715,9 @@ loff_t ocfs2_reflink_remap_blocks(struct inode *s_inode, } out: - if (ocfs2_dealloc_has_cluster(&dealloc)) { + if (ocfs2_dealloc_has_cluster(&dealloc)) ocfs2_schedule_truncate_log_flush(osb, 1); - ocfs2_run_deallocs(osb, &dealloc); - } + ocfs2_run_deallocs(osb, &dealloc); return ret; } diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c index a4a2b87a45fe3..20c3aec6b9873 100644 --- a/fs/ocfs2/suballoc.c +++ b/fs/ocfs2/suballoc.c @@ -2759,7 +2759,7 @@ static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle, fe->i_clusters = cpu_to_le32(tmp_used - le16_to_cpu(cl->cl_cpg)); spin_lock(&OCFS2_I(alloc_inode)->ip_lock); - OCFS2_I(alloc_inode)->ip_clusters -= le32_to_cpu(fe->i_clusters); + OCFS2_I(alloc_inode)->ip_clusters = le32_to_cpu(fe->i_clusters); fe->i_size = cpu_to_le64(ocfs2_clusters_to_bytes(alloc_inode->i_sb, le32_to_cpu(fe->i_clusters))); spin_unlock(&OCFS2_I(alloc_inode)->ip_lock); diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index fcddd3c13acdd..bfafe059bedff 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -390,6 +390,12 @@ static int ocfs2_init_xattr_bucket(struct ocfs2_xattr_bucket *bucket, return rc; } +static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno, + struct ocfs2_xattr_header *xh, + size_t region_size); +static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket, + u64 blkno); + /* Read the xattr bucket at xb_blkno */ static int ocfs2_read_xattr_bucket(struct ocfs2_xattr_bucket *bucket, u64 xb_blkno) @@ -408,6 +414,8 @@ static int ocfs2_read_xattr_bucket(struct ocfs2_xattr_bucket *bucket, spin_unlock(&OCFS2_SB(bucket->bu_inode->i_sb)->osb_xattr_lock); if (rc) mlog_errno(rc); + else + rc = ocfs2_validate_xattr_bucket(bucket, xb_blkno); } if (rc) @@ -509,6 +517,22 @@ static int ocfs2_validate_xattr_block(struct super_block *sb, le32_to_cpu(xb->xb_fs_generation)); } + if (!(le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED)) { + size_t region_offset = + offsetof(struct ocfs2_xattr_block, xb_attrs.xb_header); + + if (bh->b_size < region_offset) + return ocfs2_error(sb, + "Invalid xattr block %llu: block size %zu is too small\n", + (unsigned long long)bh->b_blocknr, + bh->b_size); + + return ocfs2_validate_xattr_entries_flat(sb, bh->b_blocknr, + &xb->xb_attrs.xb_header, + bh->b_size - + region_offset); + } + return 0; } @@ -611,15 +635,11 @@ int ocfs2_calc_security_init(struct inode *dir, return ret; } -int ocfs2_calc_xattr_init(struct inode *dir, - struct buffer_head *dir_bh, - umode_t mode, - struct ocfs2_security_xattr_info *si, - int *want_clusters, - int *xattr_credits, - int *want_meta) +void ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, + struct ocfs2_security_xattr_info *si, + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state) { - int ret = 0; struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); int s_size = 0, a_size = 0, acl_len = 0, new_clusters; @@ -628,24 +648,20 @@ int ocfs2_calc_xattr_init(struct inode *dir, si->value_len); if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { - down_read(&OCFS2_I(dir)->ip_xattr_sem); - acl_len = ocfs2_xattr_get_nolock(dir, dir_bh, - OCFS2_XATTR_INDEX_POSIX_ACL_DEFAULT, - "", NULL, 0); - up_read(&OCFS2_I(dir)->ip_xattr_sem); - if (acl_len > 0) { - a_size = ocfs2_xattr_entry_real_size(0, acl_len); - if (S_ISDIR(mode)) - a_size <<= 1; - } else if (acl_len != 0 && acl_len != -ENODATA) { - ret = acl_len; - mlog_errno(ret); - return ret; + if (acl_state->default_acl && S_ISDIR(mode)) { + acl_len = acl_state->default_acl->a_count * + sizeof(struct ocfs2_acl_entry); + a_size += ocfs2_xattr_entry_real_size(0, acl_len); + } + if (acl_state->acl) { + acl_len = acl_state->acl->a_count * + sizeof(struct ocfs2_acl_entry); + a_size += ocfs2_xattr_entry_real_size(0, acl_len); } } if (!(s_size + a_size)) - return ret; + return; /* * The max space of security xattr taken inline is @@ -683,17 +699,34 @@ int ocfs2_calc_xattr_init(struct inode *dir, new_clusters); *want_clusters += new_clusters; } - if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL && - acl_len > OCFS2_XATTR_INLINE_SIZE) { - /* for directory, it has DEFAULT and ACCESS two types of acls */ - new_clusters = (S_ISDIR(mode) ? 2 : 1) * - ocfs2_clusters_for_bytes(dir->i_sb, acl_len); - *xattr_credits += ocfs2_clusters_to_blocks(dir->i_sb, - new_clusters); - *want_clusters += new_clusters; + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (acl_state->default_acl && S_ISDIR(mode)) { + acl_len = acl_state->default_acl->a_count * + sizeof(struct ocfs2_acl_entry); + if (acl_len > OCFS2_XATTR_INLINE_SIZE) { + new_clusters = + ocfs2_clusters_for_bytes(dir->i_sb, + acl_len); + *xattr_credits += + ocfs2_clusters_to_blocks(dir->i_sb, + new_clusters); + *want_clusters += new_clusters; + } + } + if (acl_state->acl) { + acl_len = acl_state->acl->a_count * + sizeof(struct ocfs2_acl_entry); + if (acl_len > OCFS2_XATTR_INLINE_SIZE) { + new_clusters = + ocfs2_clusters_for_bytes(dir->i_sb, + acl_len); + *xattr_credits += + ocfs2_clusters_to_blocks(dir->i_sb, + new_clusters); + *want_clusters += new_clusters; + } + } } - - return ret; } static int ocfs2_xattr_extend_allocation(struct inode *inode, @@ -740,12 +773,10 @@ static int ocfs2_xattr_extend_allocation(struct inode *inode, prev_clusters; if (why != RESTART_NONE && clusters_to_add) { - /* - * We can only fail in case the alloc file doesn't give - * up enough clusters. - */ - BUG_ON(why == RESTART_META); - + if (why == RESTART_META) { + status = -ENOSPC; + break; + } credits = ocfs2_calc_extend_credits(inode->i_sb, &vb->vb_xv->xr_list); status = ocfs2_extend_trans(handle, credits); @@ -950,41 +981,206 @@ static int ocfs2_xattr_list_entries(struct inode *inode, return result; } -static int ocfs2_xattr_ibody_lookup_header(struct inode *inode, - struct ocfs2_dinode *di, - struct ocfs2_xattr_header **header) +static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno, + struct ocfs2_xattr_header *xh, + size_t region_size) +{ + u16 xattr_count = le16_to_cpu(xh->xh_count); + size_t entries_limit = region_size; + size_t nv_limit = region_size; + size_t max_entries; + int i; + + if (region_size < sizeof(*xh)) + return ocfs2_error(sb, + "Invalid xattr in block %llu: region size %zu is too small\n", + (unsigned long long)blkno, region_size); + + max_entries = (entries_limit - sizeof(*xh)) / + sizeof(struct ocfs2_xattr_entry); + + if (xattr_count > max_entries) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry count %u exceeds maximum %zu\n", + (unsigned long long)blkno, + xattr_count, max_entries); + + for (i = 0; i < xattr_count; i++) { + struct ocfs2_xattr_entry *xe = &xh->xh_entries[i]; + size_t name_offset = le16_to_cpu(xe->xe_name_offset); + size_t value_offset; + + if (name_offset > nv_limit || + xe->xe_name_len > nv_limit - name_offset) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d name is out of bounds\n", + (unsigned long long)blkno, i); + + value_offset = name_offset + OCFS2_XATTR_SIZE(xe->xe_name_len); + if (value_offset > nv_limit) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d value starts out of bounds\n", + (unsigned long long)blkno, i); + + if (ocfs2_xattr_is_local(xe)) { + if (le64_to_cpu(xe->xe_value_size) > + nv_limit - value_offset) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d value is out of bounds\n", + (unsigned long long)blkno, + i); + } else if (sizeof(struct ocfs2_xattr_value_root) > + nv_limit - value_offset) { + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d value root is out of bounds\n", + (unsigned long long)blkno, i); + } + } + + return 0; +} + +static int ocfs2_xattr_ibody_lookup_header_raw(struct super_block *sb, + u64 blkno, + struct ocfs2_dinode *di, + struct ocfs2_xattr_header **header, + u16 *inline_size_ret) { + struct ocfs2_xattr_header *xh; u16 xattr_count; size_t max_entries; u16 inline_size = le16_to_cpu(di->i_xattr_inline_size); - if (inline_size > inode->i_sb->s_blocksize || + if (inline_size > sb->s_blocksize || inline_size < sizeof(struct ocfs2_xattr_header)) { - ocfs2_error(inode->i_sb, - "Invalid xattr inline size %u in inode %llu\n", - inline_size, - (unsigned long long)OCFS2_I(inode)->ip_blkno); + ocfs2_error(sb, + "Invalid inode %llu: xattr inline size %u\n", + (unsigned long long)blkno, inline_size); return -EFSCORRUPTED; } - *header = (struct ocfs2_xattr_header *) - ((void *)di + inode->i_sb->s_blocksize - inline_size); + xh = (struct ocfs2_xattr_header *) + ((void *)di + sb->s_blocksize - inline_size); - xattr_count = le16_to_cpu((*header)->xh_count); + xattr_count = le16_to_cpu(xh->xh_count); max_entries = (inline_size - sizeof(struct ocfs2_xattr_header)) / sizeof(struct ocfs2_xattr_entry); if (xattr_count > max_entries) { - ocfs2_error(inode->i_sb, + ocfs2_error(sb, "xattr entry count %u exceeds maximum %zu in inode %llu\n", xattr_count, max_entries, - (unsigned long long)OCFS2_I(inode)->ip_blkno); + (unsigned long long)blkno); return -EFSCORRUPTED; } + *header = xh; + if (inline_size_ret) + *inline_size_ret = inline_size; + return 0; } +int ocfs2_validate_inode_xattr(struct super_block *sb, u64 blkno, + struct ocfs2_dinode *di) +{ + struct ocfs2_xattr_header *xh; + u16 inline_size; + int ret; + + if (!(le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_XATTR_FL)) + return 0; + + ret = ocfs2_xattr_ibody_lookup_header_raw(sb, blkno, di, &xh, + &inline_size); + if (ret) + return ret; + + return ocfs2_validate_xattr_entries_flat(sb, blkno, xh, inline_size); +} + +static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket, + u64 blkno) +{ + struct super_block *sb = bucket->bu_inode->i_sb; + struct ocfs2_xattr_header *xh = bucket_xh(bucket); + u16 xattr_count = le16_to_cpu(xh->xh_count); + size_t region_size = (size_t)sb->s_blocksize * bucket->bu_blocks; + size_t entries_limit = sb->s_blocksize; + size_t nv_limit = sb->s_blocksize; + size_t max_entries; + int i; + + if (region_size < sizeof(*xh)) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: region size %zu is too small\n", + (unsigned long long)blkno, region_size); + + if (entries_limit < sizeof(*xh)) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entries limit %zu is too small\n", + (unsigned long long)blkno, + entries_limit); + + max_entries = (entries_limit - sizeof(*xh)) / + sizeof(struct ocfs2_xattr_entry); + + if (xattr_count > max_entries) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry count %u exceeds maximum %zu\n", + (unsigned long long)blkno, + xattr_count, max_entries); + + for (i = 0; i < xattr_count; i++) { + struct ocfs2_xattr_entry *xe = &xh->xh_entries[i]; + size_t name_offset = le16_to_cpu(xe->xe_name_offset); + size_t block_off = name_offset >> sb->s_blocksize_bits; + size_t block_offset = name_offset % nv_limit; + size_t value_offset; + + if (name_offset >= region_size || block_off >= bucket->bu_blocks) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d name is out of bounds\n", + (unsigned long long)blkno, i); + + if (xe->xe_name_len > nv_limit - block_offset) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d name crosses block boundary\n", + (unsigned long long)blkno, i); + + value_offset = block_offset + OCFS2_XATTR_SIZE(xe->xe_name_len); + if (value_offset > nv_limit) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d value starts out of bounds\n", + (unsigned long long)blkno, i); + + if (ocfs2_xattr_is_local(xe)) { + if (le64_to_cpu(xe->xe_value_size) > + nv_limit - value_offset) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d value is out of bounds\n", + (unsigned long long)blkno, + i); + } else if (sizeof(struct ocfs2_xattr_value_root) > + nv_limit - value_offset) { + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d value root is out of bounds\n", + (unsigned long long)blkno, i); + } + } + + return 0; +} + +static int ocfs2_xattr_ibody_lookup_header(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_header **header) +{ + return ocfs2_xattr_ibody_lookup_header_raw(inode->i_sb, + OCFS2_I(inode)->ip_blkno, + di, header, NULL); +} + int ocfs2_has_inline_xattr_value_outside(struct inode *inode, struct ocfs2_dinode *di) { @@ -3254,6 +3450,14 @@ static int ocfs2_calc_xattr_set_need(struct inode *inode, } else credits += OCFS2_SUBALLOC_ALLOC + 1; + /* + * Reserve metadata for the new xattr's value extent tree. + * The not_found path above adds credits for this tree but + * omits meta_add, leaving meta_ac NULL for large values. + */ + if (xi->xi_value_len > OCFS2_XATTR_INLINE_SIZE) + meta_add += ocfs2_extend_meta_needed(&def_xv.xv.xr_list); + /* * This cluster will be used either for new bucket or for * new xattr block. @@ -3483,9 +3687,10 @@ static int __ocfs2_xattr_set_handle(struct inode *inode, } /* - * This function only called duing creating inode - * for init security/acl xattrs of the new inode. - * All transanction credits have been reserved in mknod. + * This helper is only for setting initial ACL or security xattrs on an inode + * that is still unpublished, unhashed, and unattached to a dentry. + * Ordinary xattr updates must use ocfs2_xattr_set(). + * All transaction credits have been reserved in mknod or symlink callers. */ int ocfs2_xattr_set_handle(handle_t *handle, struct inode *inode, @@ -3542,8 +3747,6 @@ int ocfs2_xattr_set_handle(handle_t *handle, xis.inode_bh = xbs.inode_bh = di_bh; di = (struct ocfs2_dinode *)di_bh->b_data; - down_write(&OCFS2_I(inode)->ip_xattr_sem); - ret = ocfs2_xattr_ibody_find(inode, name_index, name, &xis); if (ret) goto cleanup; @@ -3556,7 +3759,6 @@ int ocfs2_xattr_set_handle(handle_t *handle, ret = __ocfs2_xattr_set_handle(inode, di, &xi, &xis, &xbs, &ctxt); cleanup: - up_write(&OCFS2_I(inode)->ip_xattr_sem); brelse(xbs.xattr_bh); ocfs2_xattr_bucket_free(xbs.bucket); @@ -7235,10 +7437,9 @@ int ocfs2_reflink_xattrs(struct inode *old_inode, ref_tree, 1); brelse(ref_root_bh); - if (ocfs2_dealloc_has_cluster(&dealloc)) { + if (ocfs2_dealloc_has_cluster(&dealloc)) ocfs2_schedule_truncate_log_flush(OCFS2_SB(old_inode->i_sb), 1); - ocfs2_run_deallocs(OCFS2_SB(old_inode->i_sb), &dealloc); - } + ocfs2_run_deallocs(OCFS2_SB(old_inode->i_sb), &dealloc); out: return ret; @@ -7257,6 +7458,7 @@ int ocfs2_init_security_and_acl(struct inode *dir, { int ret = 0; struct buffer_head *dir_bh = NULL; + struct ocfs2_acl_state acl_state = { 0 }; ret = ocfs2_init_security_get(inode, dir, qstr, NULL); if (ret) { @@ -7269,10 +7471,17 @@ int ocfs2_init_security_and_acl(struct inode *dir, mlog_errno(ret); goto leave; } - ret = ocfs2_init_acl(NULL, inode, dir, NULL, dir_bh, NULL, NULL); + + ret = ocfs2_acl_init_prepare(inode, dir, dir_bh, &acl_state); + if (ret) + goto unlock; + + ret = ocfs2_init_acl(NULL, inode, NULL, NULL, NULL, &acl_state); if (ret) mlog_errno(ret); +unlock: + ocfs2_acl_init_release(&acl_state); ocfs2_inode_unlock(dir, 0); brelse(dir_bh); leave: diff --git a/fs/ocfs2/xattr.h b/fs/ocfs2/xattr.h index 65e9aa7439199..887cc1a18b1af 100644 --- a/fs/ocfs2/xattr.h +++ b/fs/ocfs2/xattr.h @@ -43,6 +43,8 @@ int ocfs2_xattr_set_handle(handle_t *, struct inode *, struct buffer_head *, struct ocfs2_alloc_context *); int ocfs2_has_inline_xattr_value_outside(struct inode *inode, struct ocfs2_dinode *di); +int ocfs2_validate_inode_xattr(struct super_block *sb, u64 blkno, + struct ocfs2_dinode *di); int ocfs2_xattr_remove(struct inode *, struct buffer_head *); int ocfs2_init_security_get(struct inode *, struct inode *, const struct qstr *, @@ -55,9 +57,12 @@ int ocfs2_init_security_set(handle_t *, struct inode *, int ocfs2_calc_security_init(struct inode *, struct ocfs2_security_xattr_info *, int *, int *, struct ocfs2_alloc_context **); -int ocfs2_calc_xattr_init(struct inode *, struct buffer_head *, - umode_t, struct ocfs2_security_xattr_info *, - int *, int *, int *); + +struct ocfs2_acl_state; +void ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, + struct ocfs2_security_xattr_info *si, + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state); /* * xattrs can live inside an inode, as part of an external xattr block, diff --git a/fs/orangefs/devorangefs-req.c b/fs/orangefs/devorangefs-req.c index 33ee8cb32f831..e5c60da7e6773 100644 --- a/fs/orangefs/devorangefs-req.c +++ b/fs/orangefs/devorangefs-req.c @@ -474,6 +474,7 @@ static ssize_t orangefs_devreq_write_iter(struct kiocb *iocb, op->downcall.trailer_size, iter)) { gossip_err("%s: failed to copy trailer.\n", __func__); vfree(op->downcall.trailer_buf); + op->downcall.trailer_buf = NULL; goto Efault; } diff --git a/fs/orangefs/orangefs-debugfs.c b/fs/orangefs/orangefs-debugfs.c index 69bd73a2b5564..9f94919a6bc62 100644 --- a/fs/orangefs/orangefs-debugfs.c +++ b/fs/orangefs/orangefs-debugfs.c @@ -569,6 +569,7 @@ static int orangefs_prepare_cdm_array(char *debug_array_string) cds_delimiter = strchr(cds_head, '\n'); *cds_delimiter = '\0'; + cds_head = skip_spaces(cds_head); keyword_len = strcspn(cds_head, " "); cdm_array[i].keyword = kzalloc(keyword_len + 1, GFP_KERNEL); diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index a033743dbf515..a8d47b1e644b8 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -188,7 +188,6 @@ struct dentry *ovl_create_real(struct ovl_fs *ofs, struct dentry *parent, if (!err && ofs->casefold != ovl_dentry_casefolded(newdentry)) { pr_warn_ratelimited("wrong inherited casefold (%pd2)\n", newdentry); - end_creating(newdentry); err = -EINVAL; } break; diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index b75df37f70ac3..3fea90fe37c75 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -254,8 +254,10 @@ static inline struct dentry *ovl_do_mkdir(struct ovl_fs *ofs, { struct dentry *ret; + /* vfs_mkdir() drops @dentry on failure and may replace it on success */ + pr_debug("mkdir(%pd2, 0%o)\n", dentry, mode); ret = vfs_mkdir(ovl_upper_mnt_idmap(ofs), dir, dentry, mode, NULL); - pr_debug("mkdir(%pd2, 0%o) = %i\n", dentry, mode, PTR_ERR_OR_ZERO(ret)); + pr_debug("...mkdir = %i\n", PTR_ERR_OR_ZERO(ret)); return ret; } diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 60b808b85fc43..43eb0c053dacd 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1543,7 +1543,7 @@ int ovl_fill_super(struct super_block *sb, struct fs_context *fc) struct ovl_fs *ofs = sb->s_fs_info; int err; - err = -EIO; + err = -EINVAL; /* The fscontext fd may have been passed to another user namespace. */ if (fc->user_ns != current_user_ns()) goto out_err; diff --git a/fs/pidfs.c b/fs/pidfs.c index b57ecc96e9672..a6a643f15d08a 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -532,6 +532,7 @@ static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) struct task_struct *task __free(put_task) = NULL; struct nsproxy *nsp __free(put_nsproxy) = NULL; struct ns_common *ns_common = NULL; + int error; if (!pidfs_ioctl_valid(cmd)) return -ENOIOCTLCMD; @@ -555,20 +556,33 @@ static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) if (arg) return -EINVAL; + /* + * We're trying to open a file descriptor to the namespace so perform a + * filesystem cred ptrace check. Hold @task's exec_update_lock for the + * duration of the ptrace check and the namespace lookup so that the + * credentials used for the access decision match those of @task at the + * time its namespace is read, preventing a concurrent execve() from + * swapping the task's credentials in between the check and the use. We + * mirror nsfs behavior. + */ + error = down_read_killable(&task->signal->exec_update_lock); + if (error) + return error; + + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { + error = -EACCES; + goto out_unlock; + } + scoped_guard(task_lock, task) { nsp = task->nsproxy; if (nsp) get_nsproxy(nsp); } - if (!nsp) - return -ESRCH; /* just pretend it didn't exist */ - - /* - * We're trying to open a file descriptor to the namespace so perform a - * filesystem cred ptrace check. Also, we mirror nsfs behavior. - */ - if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) - return -EACCES; + if (!nsp) { + error = -ESRCH; /* just pretend it didn't exist */ + goto out_unlock; + } switch (cmd) { /* Namespaces that hang of nsproxy. */ @@ -650,11 +664,16 @@ static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) #endif break; default: - return -ENOIOCTLCMD; + error = -ENOIOCTLCMD; } - if (!ns_common) - return -EOPNOTSUPP; + if (!error && !ns_common) + error = -EOPNOTSUPP; + +out_unlock: + up_read(&task->signal->exec_update_lock); + if (error) + return error; /* open_namespace() unconditionally consumes the reference */ return open_namespace(ns_common); diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index a932a1fea8182..723ba366324a8 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -628,14 +628,22 @@ void mon_event_count(void *info) rr->err = 0; } -static struct rdt_ctrl_domain *get_ctrl_domain_from_cpu(int cpu, - struct rdt_resource *r) +/* + * Find the software controller's ctrl domain that contains @cpu on resource @r. + * + * Only called from the mbm_over worker via update_mba_bw() where the returned + * domain is kept alive by cancel_delayed_work_sync() in + * resctrl_offline_ctrl_domain(). This drains this worker and then waits on + * rdtgroup_mutex held here before the architecture can free the ctrl domain. + * + * Context: Call from RCU read-side critical section. + */ +static struct rdt_ctrl_domain *get_sc_ctrl_domain_from_cpu(int cpu, + struct rdt_resource *r) { struct rdt_ctrl_domain *d; - lockdep_assert_cpus_held(); - - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) { /* Find the domain that contains this CPU */ if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) return d; @@ -696,7 +704,8 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_l3_mon_domain *dom_m if (WARN_ON_ONCE(!pmbm_data)) return; - dom_mba = get_ctrl_domain_from_cpu(smp_processor_id(), r_mba); + guard(rcu)(); + dom_mba = get_sc_ctrl_domain_from_cpu(smp_processor_id(), r_mba); if (!dom_mba) { pr_warn_once("Failure to get domain for MBA update\n"); return; @@ -799,11 +808,25 @@ void cqm_handle_limbo(struct work_struct *work) unsigned long delay = msecs_to_jiffies(CQM_LIMBOCHECK_INTERVAL); struct rdt_l3_mon_domain *d; - cpus_read_lock(); + /* + * Safe to run without CPU hotplug lock. Work is guaranteed to be + * canceled before the domain structure is removed. + */ mutex_lock(&rdtgroup_mutex); + /* + * Ensure the worker is dedicated to a CPU as intended and not + * relocated by workqueue subsystem as part of CPU going offline. + */ + if (!is_percpu_thread()) + goto out_unlock; + d = container_of(work, struct rdt_l3_mon_domain, cqm_limbo.work); + /* Domain is going offline */ + if (cpumask_empty(&d->hdr.cpu_mask)) + goto out_unlock; + __check_limbo(d, false); if (has_busy_rmid(d)) { @@ -813,8 +836,8 @@ void cqm_handle_limbo(struct work_struct *work) delay); } +out_unlock: mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); } /** @@ -846,7 +869,10 @@ void mbm_handle_overflow(struct work_struct *work) struct list_head *head; struct rdt_resource *r; - cpus_read_lock(); + /* + * Safe to run without CPU hotplug lock. Work is guaranteed to be + * canceled before the domain structure is removed. + */ mutex_lock(&rdtgroup_mutex); /* @@ -856,9 +882,24 @@ void mbm_handle_overflow(struct work_struct *work) if (!resctrl_mounted || !resctrl_arch_mon_capable()) goto out_unlock; + /* + * Ensure the worker is dedicated to a CPU and not relocated by + * workqueue subsystem as part of CPU going offline since reading + * events depend on smp_processor_id(). After passing this check + * smp_processor_id() is valid for entire duration of this worker + * since it runs with rdtgroup_mutex held and the offline handler needs + * rdtgroup_mutex to offline the CPU being run on here. + */ + if (!is_percpu_thread()) + goto out_unlock; + r = resctrl_arch_get_resource(RDT_RESOURCE_L3); d = container_of(work, struct rdt_l3_mon_domain, mbm_over.work); + /* Domain is going offline */ + if (cpumask_empty(&d->hdr.cpu_mask)) + goto out_unlock; + list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { mbm_update(r, d, prgrp); @@ -880,7 +921,6 @@ void mbm_handle_overflow(struct work_struct *work) out_unlock: mutex_unlock(&rdtgroup_mutex); - cpus_read_unlock(); } /** diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index cc9966ff6cdfb..11c88d593f1ef 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -2608,15 +2608,24 @@ static void rdtgroup_kn_get(struct rdtgroup *rdtgrp, struct kernfs_node *kn) static void rdtgroup_kn_put(struct rdtgroup *rdtgrp, struct kernfs_node *kn) { - if (atomic_dec_and_test(&rdtgrp->waitcount) && - (rdtgrp->flags & RDT_DELETED)) { + bool needs_free; + + if (!atomic_dec_and_mutex_lock(&rdtgrp->waitcount, &rdtgroup_mutex)) { + kernfs_unbreak_active_protection(kn); + return; + } + + needs_free = rdtgrp->flags & RDT_DELETED; + + mutex_unlock(&rdtgroup_mutex); + + kernfs_unbreak_active_protection(kn); + + if (needs_free) { if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) rdtgroup_pseudo_lock_remove(rdtgrp); - kernfs_unbreak_active_protection(kn); rdtgroup_remove(rdtgrp); - } else { - kernfs_unbreak_active_protection(kn); } } @@ -4333,6 +4342,29 @@ static void domain_destroy_l3_mon_state(struct rdt_l3_mon_domain *d) void resctrl_offline_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d) { + /* + * mbm_handle_overflow() may dereference this ctrl domain via + * update_mba_bw()->get_sc_ctrl_domain_from_cpu(). The architecture has + * unlinked the domain from the RCU list and waited a grace period, so + * no new worker iteration can find it; drain any worker that already + * holds a pointer to it before the architecture frees the domain. + * + * Software controller is enabled/disabled on mount/unmount with + * cpus_read_lock() held. Running here with cpus_write_lock() so + * there are no concurrent changes to software controller status. + */ + if (r->rid == RDT_RESOURCE_MBA && is_mba_sc(r)) { + struct rdt_resource *l3 = resctrl_arch_get_resource(RDT_RESOURCE_L3); + struct rdt_l3_mon_domain *mon_d; + + list_for_each_entry_rcu(mon_d, &l3->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (mon_d->hdr.id == d->hdr.id) { + cancel_delayed_work_sync(&mon_d->mbm_over); + break; + } + } + } + mutex_lock(&rdtgroup_mutex); if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA) @@ -4345,6 +4377,24 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h { struct rdt_l3_mon_domain *d; + /* + * Called by architecture under CPU hotplug lock as it prepares to remove + * the domain which is guaranteed to be accessible here. + * The domain has been unlinked from the RCU list and a grace period + * has elapsed, so no new worker can be scheduled. Drain any worker that + * is in flight or pending before letting architecture proceed to free + * the domain that has the workers' struct delayed_work embedded. + * Do so before taking rdtgroup_mutex since the workers also acquire it. + */ + if (r->rid == RDT_RESOURCE_L3 && + domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) { + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); + if (resctrl_is_mbm_enabled()) + cancel_delayed_work_sync(&d->mbm_over); + if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID)) + cancel_delayed_work_sync(&d->cqm_limbo); + } + mutex_lock(&rdtgroup_mutex); /* @@ -4361,8 +4411,6 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h goto out_unlock; d = container_of(hdr, struct rdt_l3_mon_domain, hdr); - if (resctrl_is_mbm_enabled()) - cancel_delayed_work(&d->mbm_over); if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID) && has_busy_rmid(d)) { /* * When a package is going down, forcefully @@ -4373,7 +4421,6 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h * package never comes back. */ __check_limbo(d, true); - cancel_delayed_work(&d->cqm_limbo); } domain_destroy_l3_mon_state(d); @@ -4554,12 +4601,16 @@ void resctrl_offline_cpu(unsigned int cpu) d = get_mon_domain_from_cpu(cpu, l3); if (d) { if (resctrl_is_mbm_enabled() && cpu == d->mbm_work_cpu) { - cancel_delayed_work(&d->mbm_over); + mutex_unlock(&rdtgroup_mutex); + cancel_delayed_work_sync(&d->mbm_over); + mutex_lock(&rdtgroup_mutex); mbm_setup_overflow_handler(d, 0, cpu); } if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID) && cpu == d->cqm_work_cpu && has_busy_rmid(d)) { - cancel_delayed_work(&d->cqm_limbo); + mutex_unlock(&rdtgroup_mutex); + cancel_delayed_work_sync(&d->cqm_limbo); + mutex_lock(&rdtgroup_mutex); cqm_setup_limbo_handler(d, 0, cpu); } } diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c index 88d5e9a32f28b..647fa26da4d24 100644 --- a/fs/smb/client/cached_dir.c +++ b/fs/smb/client/cached_dir.c @@ -8,6 +8,7 @@ #include #include "cifsglob.h" #include "cifsproto.h" +#include "../common/smb2status.h" #include "cifs_debug.h" #include "smb2proto.h" #include "cached_dir.h" @@ -323,25 +324,37 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, rc = compound_send_recv(xid, ses, server, flags, 2, rqst, resp_buftype, rsp_iov); - if (rc) { - if (rc == -EREMCHG) { - tcon->need_reconnect = true; - pr_warn_once("server share %s deleted\n", - tcon->tree_name); - } - goto oshr_free; + if (rc == -EREMCHG) { + tcon->need_reconnect = true; + pr_warn_once("server share %s deleted\n", + tcon->tree_name); } - cfid->is_open = true; - spin_lock(&cfids->cfid_list_lock); + if (!rsp_iov[0].iov_base || rsp_iov[0].iov_len < sizeof(*o_rsp)) { + if (!rc) + rc = -EIO; + goto oshr_free; + } o_rsp = (struct smb2_create_rsp *)rsp_iov[0].iov_base; + if (o_rsp->hdr.Status != STATUS_SUCCESS) { + if (!rc) + rc = -EIO; + goto oshr_free; + } + oparms.fid->persistent_fid = o_rsp->PersistentFileId; oparms.fid->volatile_fid = o_rsp->VolatileFileId; #ifdef CONFIG_CIFS_DEBUG2 oparms.fid->mid = le64_to_cpu(o_rsp->hdr.MessageId); #endif /* CIFS_DEBUG2 */ + cfid->is_open = true; + atomic_inc(&tcon->num_remote_opens); + if (rc) + goto oshr_free; + + spin_lock(&cfids->cfid_list_lock); if (o_rsp->OplockLevel != SMB2_OPLOCK_LEVEL_LEASE) { spin_unlock(&cfids->cfid_list_lock); @@ -408,7 +421,6 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, close_cached_dir(cfid); } else { *ret_cfid = cfid; - atomic_inc(&tcon->num_remote_opens); } kfree(utf16_path); diff --git a/fs/smb/client/cifs_debug.c b/fs/smb/client/cifs_debug.c index 4ed4f55a0bb72..3761d3ad60882 100644 --- a/fs/smb/client/cifs_debug.c +++ b/fs/smb/client/cifs_debug.c @@ -754,7 +754,7 @@ static ssize_t cifs_stats_proc_write(struct file *file, atomic_set(&server->smb2slowcmd[i], 0); server->time_per_cmd[i] = 0; server->slowest_cmd[i] = 0; - server->fastest_cmd[0] = 0; + server->fastest_cmd[i] = 0; } #endif /* CONFIG_CIFS_STATS2 */ list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index 12005f46307de..c5e47a835f99d 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -100,8 +100,23 @@ cifs_idmap_key_destroy(struct key *key) kfree(key->payload.data[0]); } +static int +cifs_idmap_key_vet_description(const char *description) +{ + /* + * cifs.idmap descriptions are authority-bearing inputs to the + * cifs.idmap upcall helper. Only allow the kernel to create this + * type of key using the private root_cred installed in + * init_cifs_idmap; reject userspace request_key(2)/add_key(2). + */ + if (current_cred() != root_cred) + return -EPERM; + return 0; +} + static struct key_type cifs_idmap_key_type = { .name = "cifs.idmap", + .vet_description = cifs_idmap_key_vet_description, .instantiate = cifs_idmap_key_instantiate, .destroy = cifs_idmap_key_destroy, .describe = user_describe, @@ -1081,13 +1096,13 @@ unsigned int setup_special_user_owner_ACE(struct smb_ace *pntace) static void populate_new_aces(char *nacl_base, struct smb_sid *pownersid, struct smb_sid *pgrpsid, - __u64 *pnmode, u16 *pnum_aces, u16 *pnsize, + __u64 *pnmode, u16 *pnum_aces, u32 *pnsize, bool modefromsid, bool posix) { __u64 nmode; u16 num_aces = 0; - u16 nsize = 0; + u32 nsize = 0; __u64 user_mode; __u64 group_mode; __u64 other_mode; @@ -1186,17 +1201,17 @@ static void populate_new_aces(char *nacl_base, *pnsize = nsize; } -static __u16 replace_sids_and_copy_aces(struct smb_acl *pdacl, struct smb_acl *pndacl, - struct smb_sid *pownersid, struct smb_sid *pgrpsid, - struct smb_sid *pnownersid, struct smb_sid *pngrpsid, - int *aclflag) +static int replace_sids_and_copy_aces(struct smb_acl *pdacl, struct smb_acl *pndacl, + struct smb_sid *pownersid, struct smb_sid *pgrpsid, + struct smb_sid *pnownersid, struct smb_sid *pngrpsid, + int *aclflag, u16 *pnsize) { int i; u16 size = 0; struct smb_ace *pntace = NULL; char *acl_base = NULL; u16 src_num_aces = 0; - u16 nsize = 0; + u32 nsize = 0; struct smb_ace *pnntace = NULL; char *nacl_base = NULL; u16 ace_size = 0; @@ -1225,9 +1240,12 @@ static __u16 replace_sids_and_copy_aces(struct smb_acl *pdacl, struct smb_acl *p size += le16_to_cpu(pntace->size); nsize += ace_size; + if (nsize > U16_MAX) + return -EOVERFLOW; } - return nsize; + *pnsize = nsize; + return 0; } static int set_chmod_dacl(struct smb_acl *pdacl, struct smb_acl *pndacl, @@ -1239,7 +1257,7 @@ static int set_chmod_dacl(struct smb_acl *pdacl, struct smb_acl *pndacl, struct smb_ace *pntace = NULL; char *acl_base = NULL; u16 src_num_aces = 0; - u16 nsize = 0; + u32 nsize = 0; struct smb_ace *pnntace = NULL; char *nacl_base = NULL; u16 num_aces = 0; @@ -1290,6 +1308,8 @@ static int set_chmod_dacl(struct smb_acl *pdacl, struct smb_acl *pndacl, nsize += cifs_copy_ace(pnntace, pntace, NULL); num_aces++; + if (nsize > U16_MAX) + return -EOVERFLOW; next_ace: size += le16_to_cpu(pntace->size); @@ -1306,6 +1326,10 @@ static int set_chmod_dacl(struct smb_acl *pdacl, struct smb_acl *pndacl, } finalize_dacl: + /* The DACL size field is 16-bit on the wire, see MS-DTYP 2.4.5 */ + if (nsize > U16_MAX) + return -EOVERFLOW; + pndacl->num_aces = cpu_to_le16(num_aces); pndacl->size = cpu_to_le16(nsize); @@ -1331,6 +1355,7 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, { int rc = 0; struct smb_sid *owner_sid_ptr, *group_sid_ptr; + unsigned int sbflags = cifs_sb_flags(cifs_sb); struct smb_acl *dacl_ptr; /* no need for SACL ptr */ char *end_of_acl; __u32 dacloffset, osidoffset, gsidoffset; @@ -1349,17 +1374,21 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, cifs_dbg(NOISY, "revision %d type 0x%x ooffset 0x%x goffset 0x%x sacloffset 0x%x dacloffset 0x%x\n", pntsd->revision, pntsd->type, osidoffset, gsidoffset, le32_to_cpu(pntsd->sacloffset), dacloffset); -/* cifs_dump_mem("owner_sid: ", owner_sid_ptr, 64); */ + fattr->cf_uid = cifs_sb->ctx->linux_uid; + fattr->cf_gid = cifs_sb->ctx->linux_gid; + rc = sid_from_sd(pntsd, acl_len, osidoffset, &owner_sid_ptr); if (rc) { cifs_dbg(FYI, "%s: Error %d parsing Owner SID\n", __func__, rc); return rc; } - rc = sid_to_id(cifs_sb, owner_sid_ptr, fattr, SIDOWNER); - if (rc) { - cifs_dbg(FYI, "%s: Error %d mapping Owner SID to uid\n", - __func__, rc); - return rc; + if (!(sbflags & CIFS_MOUNT_OVERR_UID)) { + rc = sid_to_id(cifs_sb, owner_sid_ptr, fattr, SIDOWNER); + if (rc) { + cifs_dbg(FYI, "%s: Error %d mapping Owner SID to uid\n", + __func__, rc); + return rc; + } } rc = sid_from_sd(pntsd, acl_len, gsidoffset, &group_sid_ptr); @@ -1368,11 +1397,13 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, __func__, rc); return rc; } - rc = sid_to_id(cifs_sb, group_sid_ptr, fattr, SIDGROUP); - if (rc) { - cifs_dbg(FYI, "%s: Error %d mapping Group SID to gid\n", - __func__, rc); - return rc; + if (!(sbflags & CIFS_MOUNT_OVERR_GID)) { + rc = sid_to_id(cifs_sb, group_sid_ptr, fattr, SIDGROUP); + if (rc) { + cifs_dbg(FYI, "%s: Error %d mapping Group SID to gid\n", + __func__, rc); + return rc; + } } if (dacloffset) { @@ -1451,6 +1482,8 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, rc = set_chmod_dacl(dacl_ptr, ndacl_ptr, owner_sid_ptr, group_sid_ptr, pnmode, mode_from_sid, posix); + if (rc) + return rc; sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size); /* copy the non-dacl portion of secdesc */ @@ -1526,10 +1559,12 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, if (dacloffset) { /* Replace ACEs for old owner with new one */ - size = replace_sids_and_copy_aces(dacl_ptr, ndacl_ptr, - owner_sid_ptr, group_sid_ptr, - nowner_sid_ptr, ngroup_sid_ptr, - aclflag); + rc = replace_sids_and_copy_aces(dacl_ptr, ndacl_ptr, + owner_sid_ptr, group_sid_ptr, + nowner_sid_ptr, ngroup_sid_ptr, + aclflag, &size); + if (rc) + goto chown_chgrp_exit; ndacl_ptr->size = cpu_to_le16(size); } @@ -1815,11 +1850,13 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, cifs_put_tlink(tlink); return rc; } - if (mode_from_sid) - nsecdesclen += - le16_to_cpu(dacl_ptr->num_aces) * sizeof(struct smb_ace); - else /* cifsacl */ - nsecdesclen += le16_to_cpu(dacl_ptr->size); + /* + * Worst case: every ACE is rewritten with a new SID of + * SID_MAX_SUB_AUTHORITIES sub-auths -> sizeof(smb_ace) each, + * plus the smb_acl header replace_sids_and_copy_aces() emits. + */ + nsecdesclen += sizeof(struct smb_acl) + + le16_to_cpu(dacl_ptr->num_aces) * sizeof(struct smb_ace); } } diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index a1dacc7d8f74b..cb3a397b109ce 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -1413,8 +1413,19 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, */ lock_two_nondirectories(target_inode, src_inode); - if (len == 0) - len = src_inode->i_size - off; + if (len == 0) { + loff_t src_size = i_size_read(src_inode); + + if (off > src_size) { + rc = -EINVAL; + goto unlock; + } + len = src_size - off; + if (!len) { + rc = 0; + goto unlock; + } + } cifs_dbg(FYI, "clone range\n"); diff --git a/fs/smb/client/cifsfs.h b/fs/smb/client/cifsfs.h index 854e672a4e37a..651670c19c2b2 100644 --- a/fs/smb/client/cifsfs.h +++ b/fs/smb/client/cifsfs.h @@ -147,6 +147,7 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, struct file *src_file, long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg); void cifs_setsize(struct inode *inode, loff_t offset); +void cifs_resize_file_locked(struct inode *inode, loff_t offset); struct fs_context; struct smb3_fs_context; diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c index 1f77512252e77..6e3b6fbd9c55d 100644 --- a/fs/smb/client/cifssmb.c +++ b/fs/smb/client/cifssmb.c @@ -615,6 +615,11 @@ CIFSTCon(const unsigned int xid, struct cifs_ses *ses, tcon->tid = smb_buffer_response->Tid; bcc_ptr = pByteArea(smb_buffer_response); bytes_left = get_bcc(smb_buffer_response); + if (bytes_left < 2) { + rc = smb_EIO2(smb_eio_trace_tcon_bcc_too_small, + bytes_left, 2); + goto out; + } length = strnlen(bcc_ptr, bytes_left - 2); if (smb_buffer->Flags2 & SMBFLG2_UNICODE) is_unicode = true; @@ -670,6 +675,7 @@ CIFSTCon(const unsigned int xid, struct cifs_ses *ses, reset_cifs_unix_caps(xid, tcon, NULL, NULL); } } +out: cifs_buf_release(smb_buffer); return rc; } @@ -1713,8 +1719,17 @@ CIFSSMBRead(const unsigned int xid, struct cifs_io_parms *io_parms, pSMBr = (READ_RSP *)rsp_iov.iov_base; if (rc) { cifs_dbg(VFS, "Send error in read = %d\n", rc); + } else if (rsp_iov.iov_len < tcon->ses->server->vals->read_rsp_size) { + /* check that the received response can hold a whole READ_RSP */ + cifs_dbg(FYI, "%s: server returned short header. got=%zu expected=%zu\n", + __func__, rsp_iov.iov_len, + tcon->ses->server->vals->read_rsp_size); + rc = smb_EIO2(smb_eio_trace_read_rsp_short, + rsp_iov.iov_len, tcon->ses->server->vals->read_rsp_size); + *nbytes = 0; } else { - int data_length = le16_to_cpu(pSMBr->DataLengthHigh); + unsigned int data_length = le16_to_cpu(pSMBr->DataLengthHigh); + __u16 data_offset = le16_to_cpu(pSMBr->DataOffset); data_length = data_length << 16; data_length += le16_to_cpu(pSMBr->DataLength); *nbytes = data_length; @@ -1722,14 +1737,21 @@ CIFSSMBRead(const unsigned int xid, struct cifs_io_parms *io_parms, /*check that DataLength would not go beyond end of SMB */ if ((data_length > CIFSMaxBufSize) || (data_length > count)) { - cifs_dbg(FYI, "bad length %d for count %d\n", - data_length, count); + cifs_dbg(FYI, "%s: bad length %u for count %u\n", + __func__, data_length, count); rc = smb_EIO2(smb_eio_trace_read_overlarge, data_length, count); *nbytes = 0; + } else if (data_offset < sizeof(*pSMBr) || + (size_t)data_offset + data_length > rsp_iov.iov_len) { + /* check that the data lies within the received response */ + cifs_dbg(FYI, "%s: bad data offset %u length %u for response of %zu\n", + __func__, data_offset, data_length, rsp_iov.iov_len); + rc = smb_EIO2(smb_eio_trace_read_bad_offset, + data_offset, data_length); + *nbytes = 0; } else { - pReadData = (char *) (&pSMBr->hdr.Protocol) + - le16_to_cpu(pSMBr->DataOffset); + pReadData = (char *) (&pSMBr->hdr.Protocol) + data_offset; /* if (rc = copy_to_user(buf, pReadData, data_length)) { cifs_dbg(VFS, "Faulting on read rc = %d\n",rc); rc = -EFAULT; @@ -3058,7 +3080,7 @@ int cifs_query_reparse_point(const unsigned int xid, end = 2 + get_bcc(&io_rsp->hdr) + (__u8 *)&io_rsp->ByteCount; start = (__u8 *)&io_rsp->hdr.Protocol + data_offset; - if (start >= end) { + if (start >= end || (size_t)(end - start) < sizeof(*buf)) { rc = smb_EIO2(smb_eio_trace_qreparse_data_area, (unsigned long)start - (unsigned long)io_rsp, (unsigned long)end - (unsigned long)io_rsp); diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index ba749ec25a59f..b7ad267ac28ba 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -174,6 +174,8 @@ cifs_signal_cifsd_for_reconnect(struct TCP_Server_Info *server, nserver = ses->chans[i].server; if (!nserver) continue; + if (!list_empty(&nserver->rlist)) + continue; nserver->srv_count++; list_add(&nserver->rlist, &reco); } @@ -182,11 +184,15 @@ cifs_signal_cifsd_for_reconnect(struct TCP_Server_Info *server, } } + spin_lock(&cifs_tcp_ses_lock); list_for_each_entry_safe(server, nserver, &reco, rlist) { list_del_init(&server->rlist); set_need_reco(server); + spin_unlock(&cifs_tcp_ses_lock); cifs_put_tcp_session(server, 0); + spin_lock(&cifs_tcp_ses_lock); } + spin_unlock(&cifs_tcp_ses_lock); } /* @@ -1067,6 +1073,7 @@ clean_demultiplex_info(struct TCP_Server_Info *server) spin_unlock(&server->srv_lock); cancel_delayed_work_sync(&server->echo); + cancel_delayed_work_sync(&server->reconnect); spin_lock(&server->srv_lock); server->tcpStatus = CifsExiting; @@ -1823,6 +1830,7 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx, spin_lock_init(&tcp_ses->mid_counter_lock); INIT_LIST_HEAD(&tcp_ses->tcp_ses_list); INIT_LIST_HEAD(&tcp_ses->smb_ses_list); + INIT_LIST_HEAD(&tcp_ses->rlist); INIT_DELAYED_WORK(&tcp_ses->echo, cifs_echo_request); INIT_DELAYED_WORK(&tcp_ses->reconnect, smb2_reconnect_server); mutex_init(&tcp_ses->reconnect_mutex); @@ -1926,6 +1934,7 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx, kfree(tcp_ses->leaf_fullpath); if (tcp_ses->ssocket) sock_release(tcp_ses->ssocket); + smbd_destroy(tcp_ses); kfree(tcp_ses); } return ERR_PTR(rc); diff --git a/fs/smb/client/dfs_cache.c b/fs/smb/client/dfs_cache.c index 8cd93cd2f00f9..86dba25b7a5a8 100644 --- a/fs/smb/client/dfs_cache.c +++ b/fs/smb/client/dfs_cache.c @@ -122,6 +122,8 @@ static inline void free_tgts(struct cache_entry *ce) kfree(t->name); kfree(t); } + + WRITE_ONCE(ce->tgthint, NULL); } static inline void flush_cache_ent(struct cache_entry *ce) @@ -869,13 +871,22 @@ int dfs_cache_find(const unsigned int xid, struct cifs_ses *ses, const struct nl goto out_free_path; } - if (ref) - rc = setup_referral(path, ce, ref, get_tgt_name(ce)); - else + if (ref) { + char *target = get_tgt_name(ce); + + if (IS_ERR(target)) { + rc = PTR_ERR(target); + goto out_unlock; + } + rc = setup_referral(path, ce, ref, target); + } else { rc = 0; + } + if (!rc && tgt_list) rc = get_targets(ce, tgt_list); +out_unlock: up_read(&htable_rw_lock); out_free_path: @@ -915,10 +926,17 @@ int dfs_cache_noreq_find(const char *path, struct dfs_info3_param *ref, goto out_unlock; } - if (ref) - rc = setup_referral(path, ce, ref, get_tgt_name(ce)); - else + if (ref) { + char *target = get_tgt_name(ce); + + if (IS_ERR(target)) { + rc = PTR_ERR(target); + goto out_unlock; + } + rc = setup_referral(path, ce, ref, target); + } else { rc = 0; + } if (!rc && tgt_list) rc = get_targets(ce, tgt_list); @@ -959,7 +977,8 @@ void dfs_cache_noreq_update_tgthint(const char *path, const struct dfs_cache_tgt t = READ_ONCE(ce->tgthint); - if (unlikely(!strcasecmp(it->it_name, t->name))) + /* Check 't' in case ce->tgthint was cleared by free_tgts() */ + if (t && unlikely(!strcasecmp(it->it_name, t->name))) goto out_unlock; list_for_each_entry(t, &ce->tlist, list) { diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c index 88a4a1787ff04..19e6b1a6b9b03 100644 --- a/fs/smb/client/dir.c +++ b/fs/smb/client/dir.c @@ -203,7 +203,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, struct tcon_link *tlink, unsigned int oflags, umode_t mode, __u32 *oplock, struct cifs_fid *fid, struct cifs_open_info_data *buf, - struct inode **inode) + struct inode **inode, bool *opened) { int rc = -ENOENT; int create_options = CREATE_NOT_DIR; @@ -220,6 +220,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, __le32 lease_flags = 0; *inode = NULL; + *opened = false; *oplock = 0; if (tcon->ses->server->oplocks) *oplock = REQ_OPLOCK; @@ -236,6 +237,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, oflags, oplock, &fid->netfid, xid); switch (rc) { case 0: + *opened = true; if (newinode == NULL) { /* query inode info */ goto cifs_create_get_file_info; @@ -257,11 +259,9 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, /* * The server may allow us to open things like * FIFOs, but the client isn't set up to deal - * with that. If it's not a regular file, just - * close it and proceed as if it were a normal - * lookup. + * with that. Keep the handle until the caller + * can finish the lookup. */ - CIFSSMBClose(xid, tcon, fid->netfid); goto cifs_create_get_file_info; } /* success, no need to query */ @@ -388,6 +388,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, } return rc; } + *opened = true; if (rdwr_for_fscache == 2) cifs_invalidate_cache(dir, FSCACHE_INVAL_DIO_WRITE); @@ -479,7 +480,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, return rc; out_err: - if (server->ops->close) + if (*opened && server->ops->close) server->ops->close(xid, tcon, fid); if (newinode) iput(newinode); @@ -491,7 +492,7 @@ static int cifs_do_create(struct inode *dir, struct dentry *direntry, unsigned int oflags, umode_t mode, __u32 *oplock, struct cifs_fid *fid, struct cifs_open_info_data *buf, - struct inode **inode) + struct inode **inode, bool *opened) { void *page = alloc_dentry_path(); const char *full_path; @@ -500,10 +501,11 @@ static int cifs_do_create(struct inode *dir, struct dentry *direntry, full_path = build_path_from_dentry(direntry, page); if (IS_ERR(full_path)) { rc = PTR_ERR(full_path); + *opened = false; } else { rc = __cifs_do_create(dir, direntry, full_path, xid, tlink, oflags, mode, oplock, - fid, buf, inode); + fid, buf, inode, opened); } free_dentry_path(page); return rc; @@ -533,6 +535,8 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry, struct inode *inode; unsigned int xid; __u32 oplock; + bool is_regular; + bool opened; int rc; if (unlikely(cifs_forced_shutdown(cifs_sb))) @@ -585,12 +589,26 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry, cifs_add_pending_open(&fid, tlink, &open); rc = cifs_do_create(dir, direntry, xid, tlink, oflags, mode, - &oplock, &fid, &buf, &inode); + &oplock, &fid, &buf, &inode, &opened); if (rc) { cifs_del_pending_open(&open); goto out; } + is_regular = S_ISREG(inode->i_mode); + if (!is_regular || !opened) { + if (opened && server->ops->close) + server->ops->close(xid, tcon, &fid); + cifs_del_pending_open(&open); + if (S_ISLNK(inode->i_mode) && + (oflags & (O_NOFOLLOW | __O_REGULAR)) == + (O_NOFOLLOW | __O_REGULAR) && !(oflags & O_EXCL)) { + iput(inode); + rc = -ELOOP; + goto out; + } + } + if (d_in_lookup(direntry)) { alias = d_splice_alias(inode, direntry); if (!IS_ERR_OR_NULL(alias)) @@ -599,9 +617,15 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry, d_instantiate(direntry, inode); } - if ((oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL)) + if (is_regular && opened && + (oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL)) file->f_mode |= FMODE_CREATED; + if (!is_regular || !opened) { + rc = finish_no_open(file, NULL); + goto out; + } + rc = finish_open(file, direntry, generic_file_open); if (rc) { if (server->ops->close) @@ -664,6 +688,7 @@ int cifs_create(struct mnt_idmap *idmap, struct inode *dir, struct inode *inode; struct cifs_fid fid; __u32 oplock; + bool opened; struct cifs_open_info_data buf = {}; cifs_dbg(FYI, "cifs_create parent inode = 0x%p name is: %pd and dentry = 0x%p\n", @@ -686,10 +711,10 @@ int cifs_create(struct mnt_idmap *idmap, struct inode *dir, server->ops->new_lease_key(&fid); rc = cifs_do_create(dir, direntry, xid, tlink, oflags, - mode, &oplock, &fid, &buf, &inode); + mode, &oplock, &fid, &buf, &inode, &opened); if (!rc) { d_instantiate(direntry, inode); - if (server->ops->close) + if (opened && server->ops->close) server->ops->close(xid, tcon, &fid); } @@ -1082,6 +1107,7 @@ int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, struct inode *inode; unsigned int xid; __u32 oplock; + bool opened; int namelen; int rc; @@ -1120,7 +1146,7 @@ int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, namelen = scnprintf(name, namesize, CIFS_TMPNAME_PREFIX "%x", atomic_inc_return(&cifs_tmpcounter)); rc = __cifs_do_create(dir, dentry, path, xid, tlink, oflags, - mode, &oplock, &fid, NULL, &inode); + mode, &oplock, &fid, NULL, &inode, &opened); if (!rc) { rc = d_mark_tmpfile_name(file, &QSTR_LEN(name, namelen)); if (rc) { @@ -1138,6 +1164,8 @@ int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, } while (unlikely(rc == -EEXIST) && ++retries < max_retries); if (rc) { + if (rc == -ENOENT) + rc = -EOPNOTSUPP; cifs_del_pending_open(&open); goto out; } diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c index ac89c1ba56b1f..7294ccd4782f3 100644 --- a/fs/smb/client/file.c +++ b/fs/smb/client/file.c @@ -1016,6 +1016,7 @@ static int cifs_do_truncate(const unsigned int xid, struct dentry *dentry) if (!rc) { netfs_resize_file(&cinode->netfs, 0, true); cifs_setsize(inode, 0); + cifs_invalidate_cache(inode, 0); } } if (cfile) @@ -1490,11 +1491,18 @@ int cifs_close(struct inode *inode, struct file *file) trace_smb3_close_cached(tcon->tid, tcon->ses->Suid, cfile->fid.persistent_fid, cifs_sb->ctx->closetimeo); - queue_delayed_work(deferredclose_wq, - &cfile->deferred, cifs_sb->ctx->closetimeo); - cfile->deferred_close_scheduled = true; - spin_unlock(&cinode->deferred_lock); - return 0; + /* + * Each queued execution owns one reference. + * If nothing was queued, the reference of + * the closing file is dropped below. + */ + if (queue_delayed_work(deferredclose_wq, + &cfile->deferred, + cifs_sb->ctx->closetimeo)) { + cfile->deferred_close_scheduled = true; + spin_unlock(&cinode->deferred_lock); + return 0; + } } spin_unlock(&cinode->deferred_lock); _cifsFileInfo_put(cfile, true, false); @@ -3322,9 +3330,12 @@ void cifs_oplock_break(struct work_struct *work) wait_on_bit(&cinode->flags, CIFS_INODE_PENDING_WRITERS, TASK_UNINTERRUPTIBLE); - tlink = cifs_sb_tlink(cifs_sb); - if (IS_ERR(tlink)) + tlink = cifs_get_tlink(cfile->tlink); + if (IS_ERR_OR_NULL(tlink)) { + /* drop the reference taken when the break was queued */ + _cifsFileInfo_put(cfile, false /* do not wait for ourself */, false); goto out; + } tcon = tlink_tcon(tlink); server = tcon->ses->server; diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index 0afff761aab9c..db797a5621566 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -851,6 +851,7 @@ static void smb311_posix_info_to_fattr(struct cifs_fattr *fattr, struct smb311_posix_qinfo *info = &data->posix_fi; struct cifs_sb_info *cifs_sb = CIFS_SB(sb); struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + unsigned int sbflags = cifs_sb_flags(cifs_sb); memset(fattr, 0, sizeof(*fattr)); @@ -895,8 +896,12 @@ static void smb311_posix_info_to_fattr(struct cifs_fattr *fattr, fattr->cf_symlink_target = data->symlink_target; data->symlink_target = NULL; } - sid_to_id(cifs_sb, &data->posix_owner, fattr, SIDOWNER); - sid_to_id(cifs_sb, &data->posix_group, fattr, SIDGROUP); + fattr->cf_uid = cifs_sb->ctx->linux_uid; + fattr->cf_gid = cifs_sb->ctx->linux_gid; + if (!(sbflags & CIFS_MOUNT_OVERR_UID)) + sid_to_id(cifs_sb, &data->posix_owner, fattr, SIDOWNER); + if (!(sbflags & CIFS_MOUNT_OVERR_GID)) + sid_to_id(cifs_sb, &data->posix_group, fattr, SIDGROUP); cifs_dbg(FYI, "POSIX query info: mode 0x%x uniqueid 0x%llx nlink %d\n", fattr->cf_mode, fattr->cf_uniqueid, fattr->cf_nlink); @@ -2983,14 +2988,14 @@ int cifs_getattr(struct mnt_idmap *idmap, const struct path *path, stat->attributes |= STATX_ATTR_ENCRYPTED; /* - * If on a multiuser mount without unix extensions or cifsacl being - * enabled, and the admin hasn't overridden them, set the ownership - * to the fsuid/fsgid of the current process. + * If on a multiuser mount without unix extensions, posix extensions + * or cifsacl being enabled, and the admin hasn't overridden them, + * set the ownership to the fsuid/fsgid of the current process. */ sbflags = cifs_sb_flags(cifs_sb); if ((sbflags & CIFS_MOUNT_MULTIUSER) && !(sbflags & CIFS_MOUNT_CIFS_ACL) && - !tcon->unix_ext) { + !tcon->unix_ext && !tcon->posix_extensions) { if (!(sbflags & CIFS_MOUNT_OVERR_UID)) stat->uid = current_fsuid(); if (!(sbflags & CIFS_MOUNT_OVERR_GID)) @@ -3057,9 +3062,27 @@ void cifs_setsize(struct inode *inode, loff_t offset) inode->i_blocks = blocks; spin_unlock(&inode->i_lock); inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + if (offset > old_size) + pagecache_isize_extended(inode, old_size, offset); truncate_pagecache(inode, offset); netfs_wait_for_outstanding_io(inode); - fscache_resize_cookie(cifs_inode_cookie(inode), offset); +} + +void cifs_resize_file_locked(struct inode *inode, loff_t offset) +{ + struct fscache_cookie *cookie = cifs_inode_cookie(inode); + + lockdep_assert_held_write(&inode->i_rwsem); + + netfs_resize_file(netfs_inode(inode), offset, true); + cifs_setsize(inode, offset); + + if (!cookie) + return; + + fscache_use_cookie(cookie, true); + fscache_resize_cookie(cookie, offset); + cifs_fscache_unuse_inode_cookie(inode, true); } int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, @@ -3100,6 +3123,7 @@ int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, size, false); cifs_dbg(FYI, "%s: set_file_size: rc = %d\n", __func__, rc); cifsFileInfo_put(open_file); + tcon = NULL; } } @@ -3125,10 +3149,8 @@ int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, cifs_put_tlink(tlink); set_size_out: - if (rc == 0) { - netfs_resize_file(&cifsInode->netfs, size, true); - cifs_setsize(inode, size); - } + if (rc == 0) + cifs_resize_file_locked(inode, size); return rc; } @@ -3214,9 +3236,13 @@ cifs_setattr_unix(struct dentry *direntry, struct iattr *attrs) attrs->ia_valid &= ~(ATTR_CTIME | ATTR_MTIME); } - /* skip mode change if it's just for clearing setuid/setgid */ - if (attrs->ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) - attrs->ia_valid &= ~ATTR_MODE; + /* + * This function is only called when Unix extensions are in effect, + * so the mode is always sent to and stored on the server. Do not + * skip the mode change when clearing setuid/setgid bits: dropping + * ATTR_MODE here would leave those bits set on the server after a + * write, which is a security issue. + */ args = kmalloc_obj(*args); if (args == NULL) { @@ -3289,13 +3315,6 @@ cifs_setattr_unix(struct dentry *direntry, struct iattr *attrs) if (rc) goto out; - if ((attrs->ia_valid & ATTR_SIZE) && - attrs->ia_size != i_size_read(inode)) { - truncate_setsize(inode, attrs->ia_size); - netfs_resize_file(&cifsInode->netfs, attrs->ia_size, true); - fscache_resize_cookie(cifs_inode_cookie(inode), attrs->ia_size); - } - setattr_copy(&nop_mnt_idmap, inode, attrs); mark_inode_dirty(inode); @@ -3425,8 +3444,23 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) attrs->ia_valid &= ~(ATTR_UID | ATTR_GID); } - /* skip mode change if it's just for clearing setuid/setgid */ - if (attrs->ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) + /* + * Skip the mode change if it is only being done to clear the + * setuid/setgid bits *and* the mode is emulated via the DOS + * read-only attribute (the default, non-ACL case), which cannot + * represent the setuid/setgid bits anyway. + * + * When the mode is instead stored on the server - i.e. with the + * cifsacl or modefromsid mount options (via an ACL) or with the + * SMB3.1.1 POSIX extensions - the cleared mode must be pushed to + * the server. Dropping ATTR_MODE here would leave the setuid/ + * setgid bit set on the server after a write, which is a security + * issue (the bits are not stripped as they are on local + * filesystems). + */ + if ((attrs->ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) && + !((sbflags & (CIFS_MOUNT_CIFS_ACL | CIFS_MOUNT_MODE_FROM_SID)) || + cifs_sb_master_tcon(cifs_sb)->posix_extensions)) attrs->ia_valid &= ~ATTR_MODE; if (attrs->ia_valid & ATTR_MODE) { @@ -3501,13 +3535,6 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) if (rc) goto cifs_setattr_exit; - if ((attrs->ia_valid & ATTR_SIZE) && - attrs->ia_size != i_size_read(inode)) { - truncate_setsize(inode, attrs->ia_size); - netfs_resize_file(&cifsInode->netfs, attrs->ia_size, true); - fscache_resize_cookie(cifs_inode_cookie(inode), attrs->ia_size); - } - setattr_copy(&nop_mnt_idmap, inode, attrs); mark_inode_dirty(inode); diff --git a/fs/smb/client/misc.c b/fs/smb/client/misc.c index 46e1382e8e04b..05168284f2051 100644 --- a/fs/smb/client/misc.c +++ b/fs/smb/client/misc.c @@ -378,10 +378,11 @@ void cifs_queue_oplock_break(struct cifsFileInfo *cfile) * open_file_lock to enforce the validity of it for the oplock * break handler. The matching put is done at the end of the * handler. + * + * Only take a reference if the work is actually queued. */ - cifsFileInfo_get(cfile); - - queue_work(cifsoplockd_wq, &cfile->oplock_break); + if (queue_work(cifsoplockd_wq, &cfile->oplock_break)) + cifsFileInfo_get(cfile); } void cifs_done_oplock_break(struct cifsInodeInfo *cinode) @@ -787,7 +788,11 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, node->ref_flag = le16_to_cpu(ref->ReferralEntryFlags); /* copy DfsPath */ - if (le16_to_cpu(ref->DfsPathOffset) > data_end - (char *)ref) { + if (le16_to_cpu(ref->DfsPathOffset) < sizeof(*ref) || + le16_to_cpu(ref->DfsPathOffset) > data_end - (char *)ref) { + cifs_dbg(VFS, "%s: DfsPathOffset %u out of range [%zu, %td]\n", + __func__, le16_to_cpu(ref->DfsPathOffset), + sizeof(*ref), data_end - (char *)ref); rc = -EINVAL; goto parse_DFS_referrals_exit; } @@ -801,7 +806,11 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, } /* copy link target UNC */ - if (le16_to_cpu(ref->NetworkAddressOffset) > data_end - (char *)ref) { + if (le16_to_cpu(ref->NetworkAddressOffset) < sizeof(*ref) || + le16_to_cpu(ref->NetworkAddressOffset) > data_end - (char *)ref) { + cifs_dbg(VFS, "%s: NetworkAddressOffset %u out of range [%zu, %td]\n", + __func__, le16_to_cpu(ref->NetworkAddressOffset), + sizeof(*ref), data_end - (char *)ref); rc = -EINVAL; goto parse_DFS_referrals_exit; } @@ -891,8 +900,14 @@ static void tcon_super_cb(struct super_block *sb, void *arg) t1->ses->dfs_root_ses == t2->ses->dfs_root_ses) && t1->ses->server == t2->ses->server && t2->origin_fullpath && - dfs_src_pathname_equal(t2->origin_fullpath, t1->origin_fullpath)) + dfs_src_pathname_equal(t2->origin_fullpath, t1->origin_fullpath)) { + /* + * Take the active reference while iterate_supers_type() still + * holds s_umount shared. + */ + cifs_sb_active(sb); sd->sb = sb; + } spin_unlock(&t2->tc_lock); } @@ -909,15 +924,8 @@ static struct super_block *__cifs_get_super(void (*f)(struct super_block *, void for (; *fs_type; fs_type++) { iterate_supers_type(*fs_type, f, &sd); - if (sd.sb) { - /* - * Grab an active reference in order to prevent automounts (DFS links) - * of expiring and then freeing up our cifs superblock pointer while - * we're doing failover. - */ - cifs_sb_active(sd.sb); + if (sd.sb) return sd.sb; - } } pr_warn_once("%s: could not find dfs superblock\n", __func__); return ERR_PTR(-EINVAL); diff --git a/fs/smb/client/readdir.c b/fs/smb/client/readdir.c index 32a75afca8f52..9530e5b015642 100644 --- a/fs/smb/client/readdir.c +++ b/fs/smb/client/readdir.c @@ -242,9 +242,11 @@ static void cifs_posix_to_fattr(struct cifs_fattr *fattr, struct smb2_posix_info *info, struct cifs_sb_info *cifs_sb) { + unsigned int sbflags = cifs_sb_flags(cifs_sb); struct smb2_posix_info_parsed parsed; + int rc; - posix_info_parse(info, NULL, &parsed); + rc = posix_info_parse(info, NULL, &parsed); memset(fattr, 0, sizeof(*fattr)); fattr->cf_uniqueid = le64_to_cpu(info->Inode); @@ -281,8 +283,17 @@ cifs_posix_to_fattr(struct cifs_fattr *fattr, struct smb2_posix_info *info, le32_to_cpu(info->ReparseTag), le32_to_cpu(info->Mode)); - sid_to_id(cifs_sb, &parsed.owner, fattr, SIDOWNER); - sid_to_id(cifs_sb, &parsed.group, fattr, SIDGROUP); + fattr->cf_uid = cifs_sb->ctx->linux_uid; + fattr->cf_gid = cifs_sb->ctx->linux_gid; + if (rc < 0) { + cifs_dbg(VFS, "%s: failed to parse SIDs: %d\n", + __func__, rc); + } else { + if (!(sbflags & CIFS_MOUNT_OVERR_UID)) + sid_to_id(cifs_sb, &parsed.owner, fattr, SIDOWNER); + if (!(sbflags & CIFS_MOUNT_OVERR_GID)) + sid_to_id(cifs_sb, &parsed.group, fattr, SIDGROUP); + } } static void __dir_info_to_fattr(struct cifs_fattr *fattr, const void *info) diff --git a/fs/smb/client/reparse.c b/fs/smb/client/reparse.c index 5cc5b0410d489..3a27773186ae4 100644 --- a/fs/smb/client/reparse.c +++ b/fs/smb/client/reparse.c @@ -3,6 +3,7 @@ * Copyright (c) 2024 Paulo Alcantara */ +#include #include #include #include @@ -159,15 +160,24 @@ static int create_native_symlink(const unsigned int xid, struct inode *inode, convert_delimiter(sym, sep); /* - * For absolute NT symlinks it is required to pass also leading - * backslash and to not mangle NT object prefix "\\??\\" and not to - * mangle colon in drive letter. But cifs_convert_path_to_utf16() - * removes leading backslash and replaces '?' and ':'. So temporary - * mask these characters in NT object prefix by '_' and then change - * them back. + * Absolute NT symlinks must retain the leading backslash, "\\??\\" + * prefix and drive-letter colon. cifs_convert_path_to_utf16() strips + * the leading backslash and maps '?' and ':', so temporarily mask + * these characters with '_' and restore them after conversion. + * + * When symlinkroot is unset, sym comes directly from the caller. + * Validate the complete "\\??\\X:" prefix before using fixed offsets + * or subtracting the NT prefix length below. Require an ASCII drive + * letter so the prefix occupies six characters in UTF-16 too. */ - if (!(sbflags & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') + if (!(sbflags & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') { + if (!strstarts(sym, "\\??\\") || !isascii(sym[4]) || + !isalpha(sym[4]) || sym[5] != ':') { + rc = -EINVAL; + goto out; + } sym[0] = sym[1] = sym[2] = sym[5] = '_'; + } /* * On a POSIX paths mount the symlink target is stored verbatim, so @@ -971,7 +981,8 @@ int smb2_parse_native_symlink(char **target, const char *buf, unsigned int len, linux_target[i*3 + 1] = '.'; linux_target[i*3 + 2] = sep; } - memcpy(linux_target + levels*3, smb_target+1, smb_target_len); /* +1 to skip leading sep */ + /* +1 to skip leading sep */ + memcpy(linux_target + levels*3, smb_target+1, smb_target_len-1); } else { /* * This is either an absolute symlink in POSIX-style format @@ -1137,25 +1148,31 @@ static bool wsl_to_fattr(struct cifs_open_info_data *data, struct cifs_sb_info *cifs_sb, u32 tag, struct cifs_fattr *fattr) { + unsigned int sbflags = cifs_sb_flags(cifs_sb); + kuid_t uid = cifs_sb->ctx->linux_uid; + kgid_t gid = cifs_sb->ctx->linux_gid; struct smb2_file_full_ea_info *ea; bool have_xattr_dev = false; + dev_t rdev = 0; + umode_t mode; u32 next = 0; + mode = fattr->cf_mode & ~S_IFMT; switch (tag) { case IO_REPARSE_TAG_LX_SYMLINK: - fattr->cf_mode |= S_IFLNK; + mode |= S_IFLNK; break; case IO_REPARSE_TAG_LX_FIFO: - fattr->cf_mode |= S_IFIFO; + mode |= S_IFIFO; break; case IO_REPARSE_TAG_AF_UNIX: - fattr->cf_mode |= S_IFSOCK; + mode |= S_IFSOCK; break; case IO_REPARSE_TAG_LX_CHR: - fattr->cf_mode |= S_IFCHR; + mode |= S_IFCHR; break; case IO_REPARSE_TAG_LX_BLK: - fattr->cf_mode |= S_IFBLK; + mode |= S_IFBLK; break; } @@ -1177,26 +1194,31 @@ static bool wsl_to_fattr(struct cifs_open_info_data *data, nlen = ea->ea_name_length; v = (void *)((u8 *)ea->ea_data + ea->ea_name_length + 1); - if (!strncmp(name, SMB2_WSL_XATTR_UID, nlen)) - fattr->cf_uid = wsl_make_kuid(cifs_sb, v); - else if (!strncmp(name, SMB2_WSL_XATTR_GID, nlen)) - fattr->cf_gid = wsl_make_kgid(cifs_sb, v); - else if (!strncmp(name, SMB2_WSL_XATTR_MODE, nlen)) { + if (!strncmp(name, SMB2_WSL_XATTR_UID, nlen)) { + if (!(sbflags & CIFS_MOUNT_OVERR_UID)) + uid = wsl_make_kuid(cifs_sb, v); + } else if (!strncmp(name, SMB2_WSL_XATTR_GID, nlen)) { + if (!(sbflags & CIFS_MOUNT_OVERR_GID)) + gid = wsl_make_kgid(cifs_sb, v); + } else if (!strncmp(name, SMB2_WSL_XATTR_MODE, nlen)) { /* File type in reparse point tag and in xattr mode must match. */ - if (S_DT(fattr->cf_mode) != S_DT(le32_to_cpu(*(__le32 *)v))) + if (S_DT(mode) != S_DT(get_unaligned_le32(v))) return false; - fattr->cf_mode = (umode_t)le32_to_cpu(*(__le32 *)v); + mode = get_unaligned_le32(v); } else if (!strncmp(name, SMB2_WSL_XATTR_DEV, nlen)) { - fattr->cf_rdev = reparse_mkdev(v); + rdev = reparse_mkdev(v); have_xattr_dev = true; } } while (next); out: - /* Major and minor numbers for char and block devices are mandatory. */ if (!have_xattr_dev && (tag == IO_REPARSE_TAG_LX_CHR || tag == IO_REPARSE_TAG_LX_BLK)) return false; + fattr->cf_uid = uid; + fattr->cf_gid = gid; + fattr->cf_mode = mode; + fattr->cf_rdev = rdev; return true; } @@ -1205,6 +1227,7 @@ static bool posix_reparse_to_fattr(struct cifs_sb_info *cifs_sb, struct cifs_open_info_data *data) { struct reparse_nfs_data_buffer *buf = (struct reparse_nfs_data_buffer *)data->reparse.buf; + umode_t ftype; if (buf == NULL) return true; @@ -1220,7 +1243,7 @@ static bool posix_reparse_to_fattr(struct cifs_sb_info *cifs_sb, WARN_ON_ONCE(1); return false; } - fattr->cf_mode |= S_IFCHR; + ftype = S_IFCHR; fattr->cf_rdev = reparse_mkdev(buf->DataBuffer); break; case NFS_SPECFILE_BLK: @@ -1228,22 +1251,23 @@ static bool posix_reparse_to_fattr(struct cifs_sb_info *cifs_sb, WARN_ON_ONCE(1); return false; } - fattr->cf_mode |= S_IFBLK; + ftype = S_IFBLK; fattr->cf_rdev = reparse_mkdev(buf->DataBuffer); break; case NFS_SPECFILE_FIFO: - fattr->cf_mode |= S_IFIFO; + ftype = S_IFIFO; break; case NFS_SPECFILE_SOCK: - fattr->cf_mode |= S_IFSOCK; + ftype = S_IFSOCK; break; case NFS_SPECFILE_LNK: - fattr->cf_mode |= S_IFLNK; + ftype = S_IFLNK; break; default: WARN_ON_ONCE(1); return false; } + fattr->cf_mode = (fattr->cf_mode & ~S_IFMT) | ftype; return true; } @@ -1271,6 +1295,7 @@ bool cifs_reparse_point_to_fattr(struct cifs_sb_info *cifs_sb, break; case 0: /* SMB1 symlink */ case IO_REPARSE_TAG_SYMLINK: + fattr->cf_mode &= ~S_IFMT; fattr->cf_mode |= S_IFLNK; break; default: diff --git a/fs/smb/client/reparse.h b/fs/smb/client/reparse.h index 0164dc47bdfd5..0506d25eea5c3 100644 --- a/fs/smb/client/reparse.h +++ b/fs/smb/client/reparse.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "fs_context.h" #include "cifsglob.h" #include "../common/smbfsctl.h" @@ -23,7 +24,7 @@ static inline dev_t reparse_mkdev(void *ptr) { - u64 v = le64_to_cpu(*(__le64 *)ptr); + u64 v = get_unaligned_le64(ptr); return MKDEV(v & 0xffffffff, v >> 32); } @@ -31,7 +32,7 @@ static inline dev_t reparse_mkdev(void *ptr) static inline kuid_t wsl_make_kuid(struct cifs_sb_info *cifs_sb, void *ptr) { - u32 uid = le32_to_cpu(*(__le32 *)ptr); + u32 uid = get_unaligned_le32(ptr); if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_UID) return cifs_sb->ctx->linux_uid; @@ -41,7 +42,7 @@ static inline kuid_t wsl_make_kuid(struct cifs_sb_info *cifs_sb, static inline kgid_t wsl_make_kgid(struct cifs_sb_info *cifs_sb, void *ptr) { - u32 gid = le32_to_cpu(*(__le32 *)ptr); + u32 gid = get_unaligned_le32(ptr); if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_GID) return cifs_sb->ctx->linux_gid; diff --git a/fs/smb/client/sess.c b/fs/smb/client/sess.c index 7cf7dd104f7c3..e095f41b58828 100644 --- a/fs/smb/client/sess.c +++ b/fs/smb/client/sess.c @@ -149,9 +149,9 @@ int cifs_try_adding_channels(struct cifs_ses *ses) int old_chan_count, new_chan_count; int left; int rc = 0; - int tries = 0; + int tries = 0, attempts; size_t iface_weight = 0, iface_min_speed = 0; - struct cifs_server_iface *iface = NULL, *niface = NULL; + struct cifs_server_iface *iface = NULL, *candidate = NULL; struct cifs_server_iface *last_iface = NULL; spin_lock(&ses->chan_lock); @@ -197,67 +197,89 @@ int cifs_try_adding_channels(struct cifs_ses *ses) break; } - if (!iface) - iface = list_first_entry(&ses->iface_list, struct cifs_server_iface, - iface_head); last_iface = list_last_entry(&ses->iface_list, struct cifs_server_iface, iface_head); iface_min_speed = last_iface->speed; + spin_unlock(&ses->iface_lock); - list_for_each_entry_safe_from(iface, niface, &ses->iface_list, - iface_head) { - /* do not mix rdma and non-rdma interfaces */ - if (iface->rdma_capable != ses->server->rdma) - continue; - - /* skip ifaces that are unusable */ - if (!iface->is_active || - (is_ses_using_iface(ses, iface) && - !iface->rss_capable)) - continue; + attempts = 0; + while (left > 0) { + spin_lock(&ses->iface_lock); - /* check if we already allocated enough channels */ - iface_weight = iface->speed / iface_min_speed; + /* + * iface_lock must be dropped while opening a channel, + * and a concurrent interface refresh may remove and + * free entries during that window, so no list entry + * may be kept across it without a reference. Scan + * the list from the beginning each time and only pass + * a referenced candidate to cifs_ses_add_channel(); + * weight_fulfilled tracks the progress so that no + * iface is selected beyond its weight. + */ + candidate = NULL; + list_for_each_entry(iface, &ses->iface_list, iface_head) { + /* do not mix rdma and non-rdma interfaces */ + if (iface->rdma_capable != ses->server->rdma) + continue; + + /* skip ifaces that are unusable */ + if (!iface->is_active || + (is_ses_using_iface(ses, iface) && + !iface->rss_capable)) + continue; + + /* check if we already allocated enough channels */ + iface_weight = iface->speed / iface_min_speed; + + if (iface->weight_fulfilled >= iface_weight) + continue; + + /* take ref before unlock */ + kref_get(&iface->refcount); + candidate = iface; + break; + } - if (iface->weight_fulfilled >= iface_weight) - continue; + if (!candidate) { + /* no usable iface. reset weight_fulfilled and start over */ + list_for_each_entry(iface, &ses->iface_list, iface_head) + iface->weight_fulfilled = 0; + spin_unlock(&ses->iface_lock); + break; + } - /* take ref before unlock */ - kref_get(&iface->refcount); + attempts++; + if (attempts > 3 * ses->chan_max) { + kref_put(&candidate->refcount, release_iface); + spin_unlock(&ses->iface_lock); + break; + } spin_unlock(&ses->iface_lock); - rc = cifs_ses_add_channel(ses, iface); + rc = cifs_ses_add_channel(ses, candidate); spin_lock(&ses->iface_lock); if (rc) { cifs_dbg(VFS, "failed to open extra channel on iface:%pIS rc=%d\n", - &iface->sockaddr, + &candidate->sockaddr, rc); /* failure to add chan should increase weight */ - iface->weight_fulfilled++; - kref_put(&iface->refcount, release_iface); + candidate->weight_fulfilled++; + kref_put(&candidate->refcount, release_iface); + spin_unlock(&ses->iface_lock); continue; } - iface->num_channels++; - iface->weight_fulfilled++; + candidate->num_channels++; + candidate->weight_fulfilled++; cifs_info("successfully opened new channel on iface:%pIS\n", - &iface->sockaddr); - break; - } - - /* reached end of list. reset weight_fulfilled and start over */ - if (list_entry_is_head(iface, &ses->iface_list, iface_head)) { - list_for_each_entry(iface, &ses->iface_list, iface_head) - iface->weight_fulfilled = 0; + &candidate->sockaddr); spin_unlock(&ses->iface_lock); - iface = NULL; - continue; - } - spin_unlock(&ses->iface_lock); - left--; - new_chan_count++; + left--; + new_chan_count++; + break; + } } return new_chan_count - old_chan_count; diff --git a/fs/smb/client/smb1misc.c b/fs/smb/client/smb1misc.c index ba56023010d84..cdfbbff24b72a 100644 --- a/fs/smb/client/smb1misc.c +++ b/fs/smb/client/smb1misc.c @@ -80,7 +80,8 @@ is_valid_oplock_break(char *buffer, struct TCP_Server_Info *srv) (struct smb_com_transaction_change_notify_rsp *)buf; struct file_notify_information *pnotify; __u32 data_offset = 0; - size_t len = srv->total_read - srv->pdu_size; + /* total_read excludes the RFC1002 preamble */ + size_t len = srv->total_read; if (get_bcc(buf) > sizeof(struct file_notify_information)) { data_offset = le32_to_cpu(pSMBr->DataOffset); diff --git a/fs/smb/client/smb1ops.c b/fs/smb/client/smb1ops.c index dc5a8c1da6231..954c1a8ef724a 100644 --- a/fs/smb/client/smb1ops.c +++ b/fs/smb/client/smb1ops.c @@ -717,7 +717,7 @@ static int cifs_query_path_info(const unsigned int xid, ea->ea_value_length = cpu_to_le16(SMB2_WSL_XATTR_DEV_SIZE); memcpy(&ea->ea_data[0], SMB2_WSL_XATTR_DEV, SMB2_WSL_XATTR_NAME_LEN + 1); data->wsl.eas_len += ALIGN(sizeof(*ea) + SMB2_WSL_XATTR_NAME_LEN + 1 + - SMB2_WSL_XATTR_MODE_SIZE, 4); + SMB2_WSL_XATTR_DEV_SIZE, 4); rc = 0; } else if (rc >= 0) { /* It is an error if EA $LXDEV has wrong size. */ diff --git a/fs/smb/client/smb1transport.c b/fs/smb/client/smb1transport.c index 966f2cf83a517..42e95cc1bd892 100644 --- a/fs/smb/client/smb1transport.c +++ b/fs/smb/client/smb1transport.c @@ -375,12 +375,31 @@ coalesce_t2(char *second_buf, struct smb_hdr *target_hdr, unsigned int *pdu_len) data_area_of_tgt = (char *)&pSMBt->hdr.Protocol + get_unaligned_le16(&pSMBt->t2_rsp.DataOffset); - /* validate target area */ data_area_of_src = (char *)&pSMBs->hdr.Protocol + get_unaligned_le16(&pSMBs->t2_rsp.DataOffset); data_area_of_tgt += total_in_tgt; + /* + * DataOffset fields are server-supplied and not validated against + * buffer bounds; check both data pointers before mutating the + * target header. + */ + if (data_area_of_tgt < (char *)target_hdr + + sizeof(struct smb_t2_rsp) + sizeof(__le16) || + data_area_of_tgt + total_in_src > + (char *)target_hdr + CIFSMaxBufSize + MAX_CIFS_HDR_SIZE) { + cifs_dbg(VFS, "%s: target data area out of bounds\n", __func__); + return -EPROTO; + } + if (data_area_of_src < second_buf + + sizeof(struct smb_t2_rsp) + sizeof(__le16) || + data_area_of_src + total_in_src > + second_buf + smbCalcSize((struct smb_hdr *)second_buf)) { + cifs_dbg(VFS, "%s: secondary data area out of bounds\n", __func__); + return -EPROTO; + } + total_in_tgt += total_in_src; /* is the result too big for the field? */ if (total_in_tgt > USHRT_MAX) { @@ -430,10 +449,18 @@ bool cifs_check_trans2(struct mid_q_entry *mid, struct TCP_Server_Info *server, char *buf, int malformed) { - if (malformed) - return false; - if (check2ndT2(buf) <= 0) + if (malformed || check2ndT2(buf) <= 0) { + /* mid->multiRsp blocks the server buf detach in handle_mid(); + * returning false here would leak resp_buf and leave a dangling + * server->smallbuf/bigbuf after the user thread frees resp_buf. + */ + if (mid->multiRsp) { + mid->multiEnd = true; + dequeue_mid(server, mid, true); + return true; + } return false; + } mid->multiRsp = true; if (mid->resp_buf) { /* merge response - fix up 1st*/ diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c index f35b6488d810d..fb2fccbe86677 100644 --- a/fs/smb/client/smb2file.c +++ b/fs/smb/client/smb2file.c @@ -61,7 +61,10 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) cifs_dbg(FYI, "%s: skipping unhandled error context: 0x%x\n", __func__, le32_to_cpu(p->ErrorId)); - len = ALIGN(le32_to_cpu(p->ErrorDataLength), 8); + len = le32_to_cpu(p->ErrorDataLength); + if (len > end - ((u8 *)p + sizeof(*p))) + return ERR_PTR(-EINVAL); + len = ALIGN(len, 8); if (len > end - ((u8 *)p + sizeof(*p))) return ERR_PTR(-EINVAL); diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index 213bc298cdf22..970503a7674e6 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -40,9 +40,11 @@ static struct reparse_data_buffer *reparse_buf_ptr(struct kvec *iov) buf = (struct reparse_data_buffer *)((u8 *)io + off); len = sizeof(*buf); - rdlen = le16_to_cpu(buf->ReparseDataLength); + if (count < len) + return ERR_PTR(smb_EIO2(smb_eio_trace_reparse_rdlen, count, 0)); - if (count < len || count < rdlen + len) + rdlen = le16_to_cpu(buf->ReparseDataLength); + if (count < rdlen + len) return ERR_PTR(smb_EIO2(smb_eio_trace_reparse_rdlen, count, rdlen)); return buf; } @@ -74,6 +76,17 @@ static int parse_posix_sids(struct cifs_open_info_data *data, sidsbuf = (u8 *)qi + le16_to_cpu(qi->OutputBufferOffset) + qi_len; sidsbuf_end = sidsbuf + out_len - qi_len; + if (sidsbuf_end < sidsbuf) { + cifs_dbg(VFS, "%s: server-supplied out_len %u caused pointer wraparound\n", + __func__, out_len); + return -EINVAL; + } + if (sidsbuf_end > (u8 *)rsp_iov->iov_base + rsp_iov->iov_len) { + cifs_dbg(VFS, "%s: server-supplied out_len %u overruns iov by %td bytes\n", + __func__, out_len, + sidsbuf_end - ((u8 *)rsp_iov->iov_base + rsp_iov->iov_len)); + return -EINVAL; + } owner_len = posix_info_sid_size(sidsbuf, sidsbuf_end); if (owner_len == -1) @@ -574,6 +587,7 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, idata->fi.Attributes = create_rsp->FileAttributes; idata->fi.AllocationSize = create_rsp->AllocationSize; idata->fi.EndOfFile = create_rsp->EndofFile; + idata->contains_posix_file_info = false; if (le32_to_cpu(idata->fi.NumberOfLinks) == 0) idata->fi.NumberOfLinks = cpu_to_le32(1); /* dummy value */ idata->fi.DeletePending = 0; /* successful open = not delete pending */ @@ -582,8 +596,10 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, /* smb2_parse_contexts() fills idata->fi.IndexNumber */ rc = smb2_parse_contexts(server, &rsp_iov[0], &oparms->fid->epoch, oparms->fid->lease_key, &oplock, &idata->fi, NULL); - if (rc) + if (rc) { cifs_dbg(VFS, "rc: %d parsing context of compound op\n", rc); + tmp_rc = rc; + } } for (i = 0; i < num_cmds; i++) { @@ -596,7 +612,6 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, switch (cmds[i]) { case SMB2_OP_QUERY_INFO: idata = in_iov[i].iov_base; - idata->contains_posix_file_info = false; if (rc == 0 && cfile && cfile->symlink_target) { idata->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); if (!idata->symlink_target) @@ -609,6 +624,8 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, le16_to_cpu(qi_rsp->OutputBufferOffset), le32_to_cpu(qi_rsp->OutputBufferLength), &rsp_iov[i + 1], sizeof(idata->fi), (char *)&idata->fi); + if (!rc) + idata->contains_posix_file_info = false; } SMB2_query_info_free(&rqst[num_rqst++]); if (rc) @@ -620,7 +637,6 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, break; case SMB2_OP_POSIX_QUERY_INFO: idata = in_iov[i].iov_base; - idata->contains_posix_file_info = true; if (rc == 0 && cfile && cfile->symlink_target) { idata->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); if (!idata->symlink_target) @@ -634,6 +650,8 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, le32_to_cpu(qi_rsp->OutputBufferLength), &rsp_iov[i + 1], sizeof(idata->posix_fi) /* add SIDs */, (char *)&idata->posix_fi); + if (!rc) + idata->contains_posix_file_info = true; } if (rc == 0) rc = parse_posix_sids(idata, &rsp_iov[i + 1]); @@ -705,7 +723,6 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, idata = in_iov[i].iov_base; idata->reparse.io.iov = *iov; idata->reparse.io.buftype = resp_buftype[i + 1]; - idata->contains_posix_file_info = false; /* BB VERIFY */ rbuf = reparse_buf_ptr(iov); if (IS_ERR(rbuf)) { rc = PTR_ERR(rbuf); @@ -727,7 +744,6 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, case SMB2_OP_QUERY_WSL_EA: if (!rc) { idata = in_iov[i].iov_base; - idata->contains_posix_file_info = false; qi_rsp = rsp_iov[i + 1].iov_base; data[0] = (u8 *)qi_rsp + le16_to_cpu(qi_rsp->OutputBufferOffset); size[0] = le32_to_cpu(qi_rsp->OutputBufferLength); @@ -1104,7 +1120,7 @@ smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name, struct kvec close_iov; int resp_buftype[2]; struct cifs_fid fid; - int flags = 0; + int flags = CIFS_CP_CREATE_CLOSE_OP; __u8 oplock; int rc; diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index 9068175e57cd0..596388acb31e7 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -821,7 +821,8 @@ smb2_cancelled_close_fid(struct work_struct *work) */ static int __smb2_handle_cancelled_cmd(struct cifs_tcon *tcon, __u16 cmd, __u64 mid, - __u64 persistent_fid, __u64 volatile_fid) + __u64 persistent_fid, __u64 volatile_fid, + bool account_remote_open) { struct close_cancelled_open *cancelled; @@ -835,6 +836,8 @@ __smb2_handle_cancelled_cmd(struct cifs_tcon *tcon, __u16 cmd, __u64 mid, cancelled->cmd = cmd; cancelled->mid = mid; INIT_WORK(&cancelled->work, smb2_cancelled_close_fid); + if (account_remote_open) + atomic_inc(&tcon->num_remote_opens); WARN_ON(queue_work(cifsiod_wq, &cancelled->work) == false); return 0; @@ -871,7 +874,7 @@ smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, spin_unlock(&tcon->tc_lock); rc = __smb2_handle_cancelled_cmd(tcon, SMB2_CLOSE_HE, 0, - persistent_fid, volatile_fid); + persistent_fid, volatile_fid, false); if (rc) cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cancelled_close); @@ -899,7 +902,7 @@ smb2_handle_cancelled_mid(struct mid_q_entry *mid, struct TCP_Server_Info *serve le16_to_cpu(hdr->Command), le64_to_cpu(hdr->MessageId), rsp->PersistentFileId, - rsp->VolatileFileId); + rsp->VolatileFileId, true); if (rc) cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cancelled_mid); diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index 192649fec25d5..46507f8bc68be 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -785,9 +785,9 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, break; } /* Validate that Next doesn't point beyond the buffer */ - if (next > bytes_left) { - cifs_dbg(VFS, "%s: invalid Next pointer %zu > %zd\n", - __func__, next, bytes_left); + if (next < sizeof(*p) || next > bytes_left) { + cifs_dbg(VFS, "%s: invalid Next pointer %zu out of range [%zu, %zd]\n", + __func__, next, sizeof(*p), bytes_left); rc = -EINVAL; goto out; } @@ -1053,8 +1053,9 @@ move_smb2_ea_to_cifs(char *dst, size_t dst_size, char *name, *value; size_t buf_size = dst_size; size_t name_len, value_len, user_name_len; + u32 next_off; - while (src_size > 0) { + while (src_size >= sizeof(*src)) { name_len = (size_t)src->ea_name_length; value_len = (size_t)le16_to_cpu(src->ea_value_length); @@ -1110,14 +1111,22 @@ move_smb2_ea_to_cifs(char *dst, size_t dst_size, if (!src->next_entry_offset) break; - if (src_size < le32_to_cpu(src->next_entry_offset)) { - /* stop before overrun buffer */ - rc = -ERANGE; - break; + next_off = le32_to_cpu(src->next_entry_offset); + if (next_off < sizeof(*src) || src_size < next_off) { + cifs_dbg(FYI, "EA next_entry_offset %u out of range [%zu, %zu]\n", + next_off, sizeof(*src), src_size); + rc = smb_EIO2(smb_eio_trace_ea_next_offset, + next_off, src_size); + goto out; + } + src_size -= next_off; + src = (void *)((char *)src + next_off); + if (src_size > 0 && src_size < sizeof(*src)) { + cifs_dbg(FYI, "EA next_entry_offset %u left truncated entry (%zu bytes)\n", + next_off, src_size); + rc = smb_EIO2(smb_eio_trace_ea_next_offset, next_off, src_size); + goto out; } - src_size -= le32_to_cpu(src->next_entry_offset); - src = (void *)((char *)src + - le32_to_cpu(src->next_entry_offset)); } /* didn't find the named attribute */ @@ -1839,31 +1848,31 @@ smb2_ioctl_query_info(const unsigned int xid, * * @tcon: destination file tcon * @bytes_left: how many bytes are left to copy + * @chunk_size: maximum size of a single chunk * * Return: maximum number of chunks with which Chunks[] can be filled. */ static inline u32 -calc_chunk_count(struct cifs_tcon *tcon, u64 bytes_left) +calc_chunk_count(struct cifs_tcon *tcon, u64 bytes_left, u32 chunk_size) { u32 max_chunks = READ_ONCE(tcon->max_chunks); u32 max_bytes_copy = READ_ONCE(tcon->max_bytes_copy); - u32 max_bytes_chunk = READ_ONCE(tcon->max_bytes_chunk); u64 need; u32 allowed; - if (!max_bytes_chunk || !max_bytes_copy || !max_chunks) + if (!chunk_size || !max_bytes_copy || !max_chunks) return 0; /* chunks needed for the remaining bytes */ - need = DIV_ROUND_UP_ULL(bytes_left, max_bytes_chunk); + need = DIV_ROUND_UP_ULL(bytes_left, chunk_size); /* chunks allowed per cc request */ - allowed = DIV_ROUND_UP(max_bytes_copy, max_bytes_chunk); + allowed = DIV_ROUND_UP(max_bytes_copy, chunk_size); return (u32)umin(need, umin(max_chunks, allowed)); } /** - * smb2_copychunk_range - server-side copy of data range + * __smb2_copychunk_range - server-side copy of data range * * @xid: transaction id * @src_file: source file @@ -1875,15 +1884,15 @@ calc_chunk_count(struct cifs_tcon *tcon, u64 bytes_left) * Obtains a resume key for @src_file and issues FSCTL_SRV_COPYCHUNK_WRITE * IOCTLs, splitting the request into chunks limited by tcon->max_*. * - * Return: @len on success; negative errno on failure. + * Return: 0 on success; negative errno on failure. */ -static ssize_t -smb2_copychunk_range(const unsigned int xid, - struct cifsFileInfo *src_file, - struct cifsFileInfo *dst_file, - u64 src_off, - u64 len, - u64 dst_off) +static int +__smb2_copychunk_range(const unsigned int xid, + struct cifsFileInfo *src_file, + struct cifsFileInfo *dst_file, + u64 src_off, + u64 len, + u64 dst_off) { int rc = 0; unsigned int ret_data_len = 0; @@ -1891,12 +1900,14 @@ smb2_copychunk_range(const unsigned int xid, struct copychunk_ioctl_rsp *cc_rsp = NULL; struct cifs_tcon *tcon; struct srv_copychunk *chunk; - u32 chunks, chunk_count, chunk_bytes; + u32 chunks, chunk_count, chunk_bytes, chunk_size; u32 copy_bytes, copy_bytes_left; u32 chunks_written, bytes_written; u64 total_bytes_left = len; u64 src_off_prev, dst_off_prev; + u64 max_chunk = 0; u32 retries = 0; + bool reverse = false; tcon = tlink_tcon(dst_file->tlink); @@ -1904,8 +1915,50 @@ smb2_copychunk_range(const unsigned int xid, dst_file->fid.volatile_fid, tcon->tid, tcon->ses->Suid, src_off, dst_off, len); + /* + * Same-file left shifts are safe in forward order. For a right shift, + * let L be the copy length, delta the distance between the source and + * destination, and C the normal chunk size: + * + * delta >= L: copy forwards using C + * delta < L: + * delta >= C: copy backwards using C + * delta < C: copy backwards with chunks limited to delta + * + * Copying backwards prevents one chunk from overwriting data needed by + * a later chunk. Limiting the chunk size to delta prevents an individual + * chunk from overlapping itself. + * This limit can be removed once all supported servers handle overlapping + * descriptors safely. + * + * A small right shift over a large range may therefore require many + * chunks. + */ + if (src_file == dst_file && dst_off > src_off) { + u64 delta = dst_off - src_off; + + if (delta < len) { + reverse = true; + max_chunk = delta; + } + } + + /* + * A backward copy walks the offsets down from the end of the range. + * Do this once, outside the retry loop, so a retry does not move the + * offsets again. + */ + if (reverse) { + src_off += len; + dst_off += len; + } + retry: - chunk_count = calc_chunk_count(tcon, total_bytes_left); + chunk_size = READ_ONCE(tcon->max_bytes_chunk); + if (max_chunk && max_chunk < chunk_size) + chunk_size = (u32)max_chunk; + + chunk_count = calc_chunk_count(tcon, total_bytes_left, chunk_size); if (!chunk_count) { rc = -EOPNOTSUPP; goto out; @@ -1946,16 +1999,21 @@ smb2_copychunk_range(const unsigned int xid, while (copy_bytes_left > 0 && chunks < chunk_count) { chunk = &cc_req->Chunks[chunks++]; + chunk_bytes = umin(copy_bytes_left, chunk_size); + if (reverse) { + src_off -= chunk_bytes; + dst_off -= chunk_bytes; + } + chunk->SourceOffset = cpu_to_le64(src_off); chunk->TargetOffset = cpu_to_le64(dst_off); - - chunk_bytes = umin(copy_bytes_left, tcon->max_bytes_chunk); - chunk->Length = cpu_to_le32(chunk_bytes); /* Buffer is zeroed, no need to set chunk->Reserved = 0 */ - src_off += chunk_bytes; - dst_off += chunk_bytes; + if (!reverse) { + src_off += chunk_bytes; + dst_off += chunk_bytes; + } copy_bytes_left -= chunk_bytes; copy_bytes += chunk_bytes; @@ -2003,6 +2061,18 @@ smb2_copychunk_range(const unsigned int xid, goto out; } + /* + * A successful COPYCHUNK should copy every descriptor (MS-SMB2 + * 3.3.5.15.6). Reject a short backward copy because the rewind + * below only supports forward copying. + */ + if (unlikely(reverse && bytes_written < copy_bytes)) { + cifs_tcon_dbg(VFS, "Copychunk short write %u/%u (reverse)\n", + bytes_written, copy_bytes); + rc = -EIO; + goto out; + } + /* Partial write: rewind */ if (bytes_written < copy_bytes) { u32 delta = copy_bytes - bytes_written; @@ -2064,10 +2134,27 @@ smb2_copychunk_range(const unsigned int xid, trace_smb3_copychunk_done(xid, src_file->fid.volatile_fid, dst_file->fid.volatile_fid, tcon->tid, tcon->ses->Suid, src_off, dst_off, len); - return len; + return 0; } } +static ssize_t +smb2_copychunk_range(const unsigned int xid, + struct cifsFileInfo *src_file, + struct cifsFileInfo *dst_file, + u64 src_off, + u64 len, + u64 dst_off) +{ + int rc; + + rc = __smb2_copychunk_range(xid, src_file, dst_file, src_off, len, + dst_off); + if (rc) + return rc; + return len; +} + static int smb2_flush_file(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_fid *fid) @@ -2222,8 +2309,7 @@ smb2_duplicate_extents(const unsigned int xid, rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false); if (rc) goto duplicate_extents_out; - netfs_resize_file(netfs_inode(inode), dest_off + len, true); - cifs_setsize(inode, dest_off + len); + cifs_resize_file_locked(inode, dest_off + len); } rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid, trgtfile->fid.volatile_fid, @@ -2374,8 +2460,14 @@ smb3_enum_snapshots(const unsigned int xid, struct cifs_tcon *tcon, * and retry the ioctl again with larger array size sufficient * to hold all of the snapshot GMT tokens on the second try. */ - if (snapshot_in.snapshot_array_size < GMT_TOKEN_SIZE) + if (snapshot_in.snapshot_array_size < GMT_TOKEN_SIZE) { + if (ret_data_len < sizeof(struct smb_snapshot_array)) { + rc = -EIO; + kfree(retbuf); + return rc; + } ret_data_len = sizeof(struct smb_snapshot_array); + } /* * We return struct SRV_SNAPSHOT_ARRAY, followed by @@ -3442,6 +3534,13 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, trace_smb3_zero_enter(xid, cfile->fid.persistent_fid, tcon->tid, ses->Suid, offset, len); + new_size = offset + len; + if (!keep_size && i_size_read(inode) < new_size) { + rc = inode_newsize_ok(inode, new_size); + if (rc) + goto out; + } + filemap_invalidate_lock(inode->i_mapping); netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); @@ -3465,6 +3564,9 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, if (keep_size == false && !CIFS_CACHE_READ(cifsi)) goto zero_range_exit; + fscache_invalidate(cifs_inode_cookie(inode), NULL, + i_size_read(inode), 0); + rc = smb3_zero_data(file, tcon, offset, len, xid); if (rc < 0) goto zero_range_exit; @@ -3472,7 +3574,6 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, /* * do we also need to change the size of the file? */ - new_size = offset + len; if (keep_size == false && (unsigned long long)i_size_read(inode) < new_size) { rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, cfile->pid, new_size); @@ -3489,6 +3590,7 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, zero_range_exit: filemap_invalidate_unlock(inode->i_mapping); + out: free_xid(xid); if (rc) trace_smb3_zero_err(xid, cfile->fid.persistent_fid, tcon->tid, @@ -3534,6 +3636,8 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon, */ truncate_pagecache_range(inode, offset, offset + len - 1); netfs_wait_for_outstanding_io(inode); + fscache_invalidate(cifs_inode_cookie(inode), NULL, + i_size_read(inode), 0); cifs_dbg(FYI, "Offset %lld len %lld\n", offset, len); @@ -3776,8 +3880,7 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, } new_eof = off + len; - netfs_resize_file(&cifsi->netfs, new_eof, true); - cifs_setsize(inode, new_eof); + cifs_resize_file_locked(inode, new_eof); qrc = SMB2_query_info(xid, tcon, cfile->fid.persistent_fid, @@ -3825,8 +3928,7 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, if (rc) goto out; - netfs_resize_file(&cifsi->netfs, new_eof, true); - cifs_setsize(inode, new_eof); + cifs_resize_file_locked(inode, new_eof); qrc = SMB2_query_info(xid, tcon, cfile->fid.persistent_fid, @@ -3941,18 +4043,26 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon, } filemap_invalidate_lock(inode->i_mapping); - rc = filemap_write_and_wait_range(inode->i_mapping, off, old_eof - 1); + rc = filemap_write_and_wait_range(inode->i_mapping, + round_down(off, PAGE_SIZE), + old_eof - 1); if (rc < 0) goto out_2; - truncate_pagecache_range(inode, off, old_eof); + netfs_wait_for_outstanding_io(inode); + /* + * Invalidate cached folios from the page containing off to EOF before + * moving data on the server, so subsequent reads do not see stale data. + */ + truncate_pagecache_range(inode, round_down(off, PAGE_SIZE), -1); + fscache_invalidate(cifs_inode_cookie(inode), NULL, old_eof, 0); + spin_lock(&inode->i_lock); netfs_write_zero_point(inode, old_eof); spin_unlock(&inode->i_lock); - netfs_wait_for_outstanding_io(inode); - rc = smb2_copychunk_range(xid, cfile, cfile, off + len, - old_eof - off - len, off); + rc = __smb2_copychunk_range(xid, cfile, cfile, off + len, + old_eof - off - len, off); if (rc < 0) goto out_2; @@ -3985,7 +4095,7 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon, struct cifsFileInfo *cfile = file->private_data; struct inode *inode = file_inode(file); struct cifsInodeInfo *cifsi = CIFS_I(inode); - __u64 count, old_eof, new_eof; + loff_t old_eof, new_eof; xid = get_xid(); @@ -3995,15 +4105,32 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon, goto out; } - count = old_eof - off; - new_eof = old_eof + len; + if (check_add_overflow(old_eof, len, &new_eof)) { + rc = -EFBIG; + goto out; + } + rc = inode_newsize_ok(inode, new_eof); + if (rc) + goto out; + + /* SET_ZERO_DATA creates a hole only in a sparse file. */ + rc = smb2_set_sparse(xid, tcon, cfile, inode, true); + if (rc) + goto out; filemap_invalidate_lock(inode->i_mapping); - rc = filemap_write_and_wait_range(inode->i_mapping, off, new_eof - 1); + rc = filemap_write_and_wait_range(inode->i_mapping, + round_down(off, PAGE_SIZE), + old_eof - 1); if (rc < 0) goto out_2; - truncate_pagecache_range(inode, off, old_eof); netfs_wait_for_outstanding_io(inode); + /* + * Invalidate cached folios from the page containing off to EOF before + * moving data on the server, so subsequent reads do not see stale data. + */ + truncate_pagecache_range(inode, round_down(off, PAGE_SIZE), -1); + fscache_invalidate(cifs_inode_cookie(inode), NULL, old_eof, 0); rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, cfile->pid, new_eof); @@ -4016,7 +4143,12 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon, spin_unlock(&inode->i_lock); fscache_resize_cookie(cifs_inode_cookie(inode), i_size_read(inode)); - rc = smb2_copychunk_range(xid, cfile, cfile, off, count, off + len); + /* + * Move [off, old_eof) right by len. The helper copies backwards if the + * source and destination ranges overlap. + */ + rc = __smb2_copychunk_range(xid, cfile, cfile, off, old_eof - off, + off + len); if (rc < 0) goto out_2; spin_lock(&inode->i_lock); @@ -4437,25 +4569,37 @@ smb3_create_lease_buf(u8 *lease_key, u8 oplock, u8 *parent_lease_key, __le32 fla static __u8 smb2_parse_lease_buf(void *buf, __u16 *epoch, char *lease_key) { - struct create_lease *lc = (struct create_lease *)buf; + struct create_context *cc = buf; + struct lease_context lc; *epoch = 0; /* not used */ - if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) + if (le32_to_cpu(cc->DataLength) != sizeof(lc)) + return 0; + + memcpy(&lc, (u8 *)cc + le16_to_cpu(cc->DataOffset), sizeof(lc)); + if (lc.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) return SMB2_OPLOCK_LEVEL_NOCHANGE; - return le32_to_cpu(lc->lcontext.LeaseState); + return le32_to_cpu(lc.LeaseState); } static __u8 smb3_parse_lease_buf(void *buf, __u16 *epoch, char *lease_key) { - struct create_lease_v2 *lc = (struct create_lease_v2 *)buf; + struct create_context *cc = buf; + struct lease_context_v2 lc; - *epoch = le16_to_cpu(lc->lcontext.Epoch); - if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) + if (le32_to_cpu(cc->DataLength) != sizeof(lc)) { + *epoch = 0; + return 0; + } + + memcpy(&lc, (u8 *)cc + le16_to_cpu(cc->DataOffset), sizeof(lc)); + *epoch = le16_to_cpu(lc.Epoch); + if (lc.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) return SMB2_OPLOCK_LEVEL_NOCHANGE; if (lease_key) - memcpy(lease_key, &lc->lcontext.LeaseKey, SMB2_LEASE_KEY_SIZE); - return le32_to_cpu(lc->lcontext.LeaseState); + memcpy(lease_key, lc.LeaseKey, SMB2_LEASE_KEY_SIZE); + return le32_to_cpu(lc.LeaseState); } static unsigned int @@ -5245,11 +5389,13 @@ receive_encrypted_standard(struct TCP_Server_Info *server, length = decrypt_raw_data(server, buf, buf_size, NULL, false); if (length) return length; + pdu_length = buf_size; next_is_large = server->large_buf; one_more: shdr = (struct smb2_hdr *)buf; next_cmd = le32_to_cpu(shdr->NextCommand); + server->total_read = next_cmd ? next_cmd : pdu_length; if (*num_mids >= MAX_COMPOUND) { cifs_server_dbg(VFS, "too many PDUs in compound\n"); @@ -5257,8 +5403,15 @@ receive_encrypted_standard(struct TCP_Server_Info *server, } if (next_cmd) { - if (WARN_ON_ONCE(next_cmd > pdu_length)) + if (next_cmd < MID_HEADER_SIZE(server) || + next_cmd > pdu_length || + pdu_length - next_cmd < MID_HEADER_SIZE(server)) { + unsigned int max_next = pdu_length > (unsigned int)MID_HEADER_SIZE(server) ? + pdu_length - (unsigned int)MID_HEADER_SIZE(server) : 0; + cifs_server_dbg(VFS, "invalid NextCommand offset %u out of range [%zu, %u]\n", + next_cmd, MID_HEADER_SIZE(server), max_next); return -1; + } if (next_is_large) next_buffer = (char *)cifs_buf_get(); else @@ -5294,6 +5447,7 @@ receive_encrypted_standard(struct TCP_Server_Info *server, server->bigbuf = buf = next_buffer; else server->smallbuf = buf = next_buffer; + next_buffer = NULL; goto one_more; } else if (ret != 0) { /* diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index 4ce165e40657f..3d7ead36d1a0e 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -189,18 +189,19 @@ cifs_chan_skip_or_disable(struct cifs_ses *ses, spin_unlock(&ses->chan_lock); /* - * the above reference of server by channel - * needs to be dropped without holding chan_lock - * as cifs_put_tcp_session takes a higher lock - * i.e. cifs_tcp_ses_lock + * signal the channel and its primary server to + * reconnect before dropping the above reference of + * server by channel, which is done without holding + * chan_lock as cifs_put_tcp_session takes a higher + * lock i.e. cifs_tcp_ses_lock */ - cifs_put_tcp_session(server, from_reconnect); - cifs_signal_cifsd_for_reconnect(server, false); /* mark primary server as needing reconnect */ pserver = server->primary_server; cifs_signal_cifsd_for_reconnect(pserver, false); + + cifs_put_tcp_session(server, from_reconnect); skip_terminate: return -EHOSTDOWN; } @@ -2378,23 +2379,32 @@ create_reconnect_durable_buf(struct cifs_fid *fid) static void parse_query_id_ctxt(struct create_context *cc, struct smb2_file_all_info *buf) { - struct create_disk_id_rsp *pdisk_id = (struct create_disk_id_rsp *)cc; + u16 doff = le16_to_cpu(cc->DataOffset); + u32 dlen = le32_to_cpu(cc->DataLength); + u8 *beg; - cifs_dbg(FYI, "parse query id context 0x%llx 0x%llx\n", - pdisk_id->DiskFileId, pdisk_id->VolumeId); - buf->IndexNumber = pdisk_id->DiskFileId; + if (dlen < sizeof(__le64)) + return; + + beg = (u8 *)cc + doff; + memcpy(&buf->IndexNumber, beg, sizeof(__le64)); + cifs_dbg(FYI, "parse query id context 0x%llx\n", + le64_to_cpu(buf->IndexNumber)); } static void parse_posix_ctxt(struct create_context *cc, struct smb2_file_all_info *info, struct create_posix_rsp *posix) { - int sid_len; u8 *beg = (u8 *)cc + le16_to_cpu(cc->DataOffset); - u8 *end = beg + le32_to_cpu(cc->DataLength); + u32 dlen = le32_to_cpu(cc->DataLength); + u8 *end = beg + dlen; + int sid_len; u8 *sid; memset(posix, 0, sizeof(*posix)); + if (dlen < 3 * sizeof(__le32)) + return; posix->nlink = get_unaligned_le32(beg); posix->reparse_tag = get_unaligned_le32(beg + 4); @@ -2430,6 +2440,7 @@ int smb2_parse_contexts(struct TCP_Server_Info *server, struct smb2_create_rsp *rsp = rsp_iov->iov_base; struct create_context *cc; size_t rem, off, len; + size_t cc_len; size_t doff, dlen; size_t noff, nlen; char *name; @@ -2452,29 +2463,41 @@ int smb2_parse_contexts(struct TCP_Server_Info *server, buf->IndexNumber = 0; while (rem >= sizeof(*cc)) { + off = le32_to_cpu(cc->Next); + if (off) { + if ((off & 0x7) || off >= rem || off < sizeof(*cc)) + return -EINVAL; + cc_len = off; + } else { + cc_len = rem; + } + doff = le16_to_cpu(cc->DataOffset); dlen = le32_to_cpu(cc->DataLength); - if (check_add_overflow(doff, dlen, &len) || len > rem) + if (doff < sizeof(*cc) || + check_add_overflow(doff, dlen, &len) || len > cc_len) return -EINVAL; noff = le16_to_cpu(cc->NameOffset); nlen = le16_to_cpu(cc->NameLength); - if (noff + nlen > doff) + if (noff < sizeof(*cc) || + check_add_overflow(noff, nlen, &len) || len > cc_len || + (dlen && len > doff)) return -EINVAL; name = (char *)cc + noff; switch (nlen) { case 4: - if (!strncmp(name, SMB2_CREATE_REQUEST_LEASE, 4)) { + if (dlen && !strncmp(name, SMB2_CREATE_REQUEST_LEASE, 4)) { *oplock = server->ops->parse_lease_buf(cc, epoch, lease_key); - } else if (buf && + } else if (dlen && buf && !strncmp(name, SMB2_CREATE_QUERY_ON_DISK_ID, 4)) { parse_query_id_ctxt(cc, buf); } break; case 16: - if (posix && !memcmp(name, smb3_create_tag_posix, 16)) + if (dlen && posix && !memcmp(name, smb3_create_tag_posix, 16)) parse_posix_ctxt(cc, buf, posix); break; default: @@ -2486,13 +2509,18 @@ int smb2_parse_contexts(struct TCP_Server_Info *server, } off = le32_to_cpu(cc->Next); - if (!off) + if (!off) { + rem = 0; break; + } if (check_sub_overflow(rem, off, &rem)) return -EINVAL; cc = (struct create_context *)((u8 *)cc + off); } + if (rem) + return -EINVAL; + if (rsp->OplockLevel != SMB2_OPLOCK_LEVEL_LEASE) *oplock = rsp->OplockLevel; @@ -3372,6 +3400,7 @@ SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path, #endif /* CIFS_DEBUG2 */ if (file_info) { + buf->contains_posix_file_info = false; file_info->CreationTime = rsp->CreationTime; file_info->LastAccessTime = rsp->LastAccessTime; file_info->LastWriteTime = rsp->LastWriteTime; @@ -3387,6 +3416,9 @@ SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path, rc = smb2_parse_contexts(server, &rsp_iov, &oparms->fid->epoch, oparms->fid->lease_key, oplock, file_info, posix); + if (rc) + SMB2_close(xid, tcon, oparms->fid->persistent_fid, + oparms->fid->volatile_fid); trace_smb3_open_done(xid, rsp->PersistentFileId, tcon->tid, ses->Suid, oparms->create_options, oparms->desired_access, @@ -4564,8 +4596,10 @@ smb2_new_read_req(void **buf, unsigned int *total_len, if (rc) return rc; - if (server == NULL) - return -ECONNABORTED; + if (!server) { + rc = -ECONNABORTED; + goto free_req; + } shdr = &req->hdr; shdr->Id.SyncId.ProcessId = cpu_to_le32(io_parms->pid); @@ -4596,8 +4630,10 @@ smb2_new_read_req(void **buf, unsigned int *total_len, rdata->mr = smbd_register_mr(server->smbd_conn, &rdata->subreq.io_iter, true, need_invalidate); - if (!rdata->mr) - return -EAGAIN; + if (!rdata->mr) { + rc = -EAGAIN; + goto free_req; + } req->Channel = SMB2_CHANNEL_RDMA_V1_INVALIDATE; if (need_invalidate) @@ -4638,6 +4674,10 @@ smb2_new_read_req(void **buf, unsigned int *total_len, *buf = req; return rc; + +free_req: + cifs_small_buf_release(req); + return rc; } static void @@ -4885,6 +4925,7 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) smb2_should_replay(tcon, &rdata->retries, &rdata->cur_sleep)) { + rdata->replay = true; trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_retry_needed); __set_bit(NETFS_SREQ_NEED_RETRY, &rdata->subreq.flags); } diff --git a/fs/smb/client/trace.h b/fs/smb/client/trace.h index 5b21ad3c15fb5..bb8d0197cb54f 100644 --- a/fs/smb/client/trace.h +++ b/fs/smb/client/trace.h @@ -27,6 +27,7 @@ EM(smb_eio_trace_copychunk_overcopy_c, "copychunk_overcopy_c") \ EM(smb_eio_trace_create_rsp_too_small, "create_rsp_too_small") \ EM(smb_eio_trace_dfsref_no_rsp, "dfsref_no_rsp") \ + EM(smb_eio_trace_ea_next_offset, "ea_next_offset") \ EM(smb_eio_trace_ea_overrun, "ea_overrun") \ EM(smb_eio_trace_extract_will_pin, "extract_will_pin") \ EM(smb_eio_trace_forced_shutdown, "forced_shutdown") \ @@ -79,6 +80,7 @@ EM(smb_eio_trace_qreparse_setup_count, "qreparse_setup_count") \ EM(smb_eio_trace_qreparse_sizes_wrong, "qreparse_sizes_wrong") \ EM(smb_eio_trace_qsym_bcc_too_small, "qsym_bcc_too_small") \ + EM(smb_eio_trace_read_bad_offset, "read_bad_offset") \ EM(smb_eio_trace_read_mid_state_unknown, "read_mid_state_unknown") \ EM(smb_eio_trace_read_overlarge, "read_overlarge") \ EM(smb_eio_trace_read_rsp_malformed, "read_rsp_malformed") \ @@ -133,6 +135,7 @@ EM(smb_eio_trace_sym_slash, "sym_slash") \ EM(smb_eio_trace_sym_target_len, "sym_target_len") \ EM(smb_eio_trace_symlink_file_size, "symlink_file_size") \ + EM(smb_eio_trace_tcon_bcc_too_small, "tcon_bcc_too_small") \ EM(smb_eio_trace_tdis_in_reconnect, "tdis_in_reconnect") \ EM(smb_eio_trace_tx_chained_async, "tx_chained_async") \ EM(smb_eio_trace_tx_compress_failed, "tx_compress_failed") \ @@ -213,7 +216,7 @@ #define EM(a, b) a, #define E_(a, b) a -enum smb_eio_trace { smb_eio_traces } __mode(byte); +enum smb_eio_trace { smb_eio_traces }; enum smb3_rw_credits_trace { smb3_rw_credits_traces } __mode(byte); enum smb3_tcon_ref_trace { smb3_tcon_ref_traces } __mode(byte); diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c index fdf4e50c27ceb..6e21b5f8754a1 100644 --- a/fs/smb/client/transport.c +++ b/fs/smb/client/transport.c @@ -101,12 +101,11 @@ void __release_mid(struct TCP_Server_Info *server, struct mid_q_entry *midEntry) trace_smb3_slow_rsp(smb_cmd, midEntry->mid, midEntry->pid, midEntry->when_sent, midEntry->when_received); if (cifsFYI & CIFS_TIMER) { - pr_debug("slow rsp: cmd %d mid %llu", - midEntry->command, midEntry->mid); - cifs_info("A: 0x%lx S: 0x%lx R: 0x%lx\n", - now - midEntry->when_alloc, - now - midEntry->when_sent, - now - midEntry->when_received); + pr_debug("slow rsp: cmd %d mid %llu A: 0x%lx S: 0x%lx R: 0x%lx\n", + midEntry->command, midEntry->mid, + now - midEntry->when_alloc, + now - midEntry->when_sent, + now - midEntry->when_received); } } #endif @@ -806,6 +805,18 @@ cifs_cancelled_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) release_mid(server, mid); } +static void +cifs_mark_compound_mids_cancelled(struct mid_q_entry **mid, int count) +{ + int i; + + for (i = 0; i < count; i++) { + spin_lock(&mid[i]->mid_lock); + mid[i]->wait_cancelled = true; + spin_unlock(&mid[i]->mid_lock); + } +} + /* * cifs_pick_channel - pick an eligible channel for network operations * @@ -866,6 +877,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, int *resp_buf_type, struct kvec *resp_iov) { int i, j, optype, rc = 0; + int num_processed = 0; struct mid_q_entry *mid[MAX_COMPOUND]; bool cancelled_mid[MAX_COMPOUND] = {false}; struct cifs_credits credits[MAX_COMPOUND] = { @@ -966,6 +978,10 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, if (rc < 0) { revert_current_mid(server, num_rqst); server->sequence_number -= 2; + for (i = 0; i < num_rqst; i++) { + delete_mid(server, mid[i]); + cancelled_mid[i] = true; + } } cifs_server_unlock(server); @@ -1012,6 +1028,14 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, break; } if (rc != 0) { + /* + * A completed CREATE earlier in the compound chain may have + * opened a remote handle even though a later wait was + * interrupted. Mark it cancelled so __release_mid() invokes + * the existing unmatched-open cleanup. + */ + cifs_mark_compound_mids_cancelled(mid, i); + for (; i < num_rqst; i++) { cifs_server_dbg(FYI, "Cancelling wait for mid %llu cmd: %d\n", mid[i]->mid, le16_to_cpu(mid[i]->command)); @@ -1034,6 +1058,14 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, rc = cifs_sync_mid_result(mid[i], server); if (rc != 0) { + /* + * A previous CREATE may have completed before this + * response failed. Mark it cancelled so its remote + * handle is closed when the mid is released. + */ + cifs_mark_compound_mids_cancelled(mid, i); + /* Keep their response buffers for cancelled-mid cleanup. */ + num_processed = 0; /* mark this mid as cancelled to not free it below */ cancelled_mid[i] = true; goto out; @@ -1043,13 +1075,24 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, mid[i]->mid_state != MID_RESPONSE_READY) { rc = smb_EIO1(smb_eio_trace_rx_mid_unready, mid[i]->mid_state); cifs_dbg(FYI, "Bad MID state?\n"); + cifs_mark_compound_mids_cancelled(mid, i); + num_processed = 0; goto out; } rc = server->ops->check_receive(mid[i], server, flags & CIFS_LOG_ERROR); + num_processed = i + 1; + } - if (resp_iov) { +out: + /* + * Delay moving response buffers out of their mids until response + * synchronization completes. This lets cancelled-mid cleanup inspect + * an earlier CREATE response if a later MID fails. + */ + if (resp_iov) { + for (i = 0; i < num_processed; i++) { buf = (char *)mid[i]->resp_buf; resp_iov[i].iov_base = buf; resp_iov[i].iov_len = mid[i]->resp_buf_size; @@ -1068,21 +1111,22 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, /* * Compounding is never used during session establish. */ - spin_lock(&ses->ses_lock); - if ((ses->ses_status == SES_NEW) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) { - struct kvec iov = { - .iov_base = resp_iov[0].iov_base, - .iov_len = resp_iov[0].iov_len - }; - spin_unlock(&ses->ses_lock); - cifs_server_lock(server); - smb311_update_preauth_hash(ses, server, &iov, 1); - cifs_server_unlock(server); + if (num_processed == num_rqst) { spin_lock(&ses->ses_lock); + if ((ses->ses_status == SES_NEW) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) { + struct kvec iov = { + .iov_base = resp_iov[0].iov_base, + .iov_len = resp_iov[0].iov_len + }; + spin_unlock(&ses->ses_lock); + cifs_server_lock(server); + smb311_update_preauth_hash(ses, server, &iov, 1); + cifs_server_unlock(server); + spin_lock(&ses->ses_lock); + } + spin_unlock(&ses->ses_lock); } - spin_unlock(&ses->ses_lock); -out: /* * This will dequeue all mids. After this it is important that the * demultiplex_thread will not process any of these mids any further. diff --git a/fs/smb/common/smb2pdu.h b/fs/smb/common/smb2pdu.h index e7ff52b8aba5f..37b660b2047b0 100644 --- a/fs/smb/common/smb2pdu.h +++ b/fs/smb/common/smb2pdu.h @@ -847,8 +847,8 @@ struct smb2_lock_req { __le16 StructureSize; /* Must be 48 */ __le16 LockCount; /* - * The least significant four bits are the index, the other 28 bits are - * the lock sequence number (0 to 64). See MS-SMB2 2.2.26 + * The least significant four bits are the lock sequence number. The + * other 28 bits are the index (0 to 64). See MS-SMB2 2.2.26. */ __le32 LockSequenceNumber; __u64 PersistentFileId; diff --git a/fs/smb/common/smbglob.h b/fs/smb/common/smbglob.h index 4e33d91cdc9db..d9c7e6e7af295 100644 --- a/fs/smb/common/smbglob.h +++ b/fs/smb/common/smbglob.h @@ -39,6 +39,7 @@ struct smb_version_values { size_t create_mxac_size; size_t create_disk_id_size; size_t create_posix_size; + size_t create_aapl_size; }; static inline unsigned int get_rfc1002_len(void *buf) diff --git a/fs/smb/server/Kconfig b/fs/smb/server/Kconfig index 08d8b7a965a64..221ec9717a834 100644 --- a/fs/smb/server/Kconfig +++ b/fs/smb/server/Kconfig @@ -9,7 +9,6 @@ config SMB_SERVER select CRYPTO select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_ARC4 - select CRYPTO_LIB_DES select CRYPTO_LIB_MD5 select CRYPTO_LIB_SHA256 select CRYPTO_LIB_SHA512 diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index 4e7b6f0e6b8cd..8d22d28a12bf9 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -24,7 +24,6 @@ #include #include -#include #include "server.h" #include "smb_common.h" @@ -439,6 +438,7 @@ int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, resp_ext = ksmbd_ipc_login_request_ext(resp->login_response.account); user = ksmbd_alloc_user(&resp->login_response, resp_ext); + kvfree(resp_ext); if (!user) { ksmbd_debug(AUTH, "login failure\n"); retval = -ENOMEM; @@ -462,6 +462,7 @@ int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, memcpy(out_blob, resp->payload + resp->session_key_len, resp->spnego_blob_len); *out_len = resp->spnego_blob_len; + sess->kerberos_expiry = resp->session_expiry; retval = 0; out: kvfree(resp); @@ -716,8 +717,21 @@ static int ksmbd_get_encryption_key(struct ksmbd_work *work, __u64 ses_id, if (enc) sess = work->sess; - else - sess = ksmbd_session_lookup_all(work->conn, ses_id); + else { + /* + * A previous-session replacement leaves the old encryption key in + * place. Use it to authenticate an encrypted request, then let + * session validation reject the expired session. This preserves the + * encrypted STATUS_USER_SESSION_DELETED response without reviving + * the session. + */ + sess = ksmbd_session_lookup_all_states(work->conn, ses_id); + if (sess && sess->state != SMB2_SESSION_VALID && + (sess->state != SMB2_SESSION_EXPIRED || !sess->enc)) { + ksmbd_user_session_put(sess); + sess = NULL; + } + } if (!sess) return -EINVAL; diff --git a/fs/smb/server/compress.c b/fs/smb/server/compress.c index 01d1771ff6636..5162fb84c755a 100644 --- a/fs/smb/server/compress.c +++ b/fs/smb/server/compress.c @@ -14,24 +14,14 @@ #define SMB_COMPRESS_MIN_LEN PAGE_SIZE -/** - * ksmbd_decompress_request() - replace a compressed request with its SMB2 PDU - * @conn: connection which owns the current RFC1002 request buffer - * - * Derive the uncompressed size from the transform variant, enforce ksmbd's - * normal message limits, and ask the common decoder to validate every payload. - * On success, replace conn->request_buf with a regular RFC1002-framed SMB2 - * message so the rest of the request path needs no compression awareness. - * - * Return: 0 on success, otherwise a negative errno. - */ -int ksmbd_decompress_request(struct ksmbd_conn *conn) +static int __ksmbd_decompress_request(struct ksmbd_conn *conn, + void *request_buf, void **out_buf) { struct smb2_compression_hdr *hdr; - unsigned int pdu_size = get_rfc1002_len(conn->request_buf); + unsigned int pdu_size = get_rfc1002_len(request_buf); u32 orig_size, offset, out_size; u32 max_allowed_pdu_size; - char *buf, *out; + char *out; int rc; if (pdu_size < sizeof(struct smb2_compression_hdr)) @@ -41,7 +31,7 @@ int ksmbd_decompress_request(struct ksmbd_conn *conn) conn->compress_algorithm == SMB3_COMPRESS_NONE) return -EINVAL; - hdr = smb_get_msg(conn->request_buf); + hdr = smb_get_msg(request_buf); if (hdr->ProtocolId != SMB2_COMPRESSION_TRANSFORM_ID) return -EINVAL; @@ -74,19 +64,69 @@ int ksmbd_decompress_request(struct ksmbd_conn *conn) if (!out) return -ENOMEM; - buf = (char *)hdr; *(__be32 *)out = cpu_to_be32(out_size); rc = smb_compression_decompress(conn->compress_algorithm, conn->compress_chained, conn->compress_pattern, - buf, pdu_size, out + 4, out_size); + (char *)hdr, pdu_size, out + 4, out_size); if (rc) { kvfree(out); return rc; } + *out_buf = out; + return 0; +} + +/** + * ksmbd_decompress_request() - replace a compressed request with its SMB2 PDU + * @conn: connection which owns the current RFC1002 request buffer + * + * Derive the uncompressed size from the transform variant, enforce ksmbd's + * normal message limits, and ask the common decoder to validate every payload. + * On success, replace conn->request_buf with a regular RFC1002-framed SMB2 + * message so the rest of the request path needs no compression awareness. + * + * Return: 0 on success, otherwise a negative errno. + */ +int ksmbd_decompress_request(struct ksmbd_conn *conn) +{ + void *out_buf; + int rc; + + rc = __ksmbd_decompress_request(conn, conn->request_buf, &out_buf); + if (rc) + return rc; + kvfree(conn->request_buf); - conn->request_buf = out; + conn->request_buf = out_buf; + return 0; +} + +/** + * ksmbd_decompress_work_request() - decompress an encrypted work request + * @work: work item whose request buffer contains a compression transform + * + * SMB3 encrypts a compressed message by applying compression first and + * encryption second. The receive loop can therefore only decode the + * compression transform before work allocation for an unencrypted request; + * an encrypted request must be decompressed after its encryption layer has + * been removed. + * + * Return: 0 on success, otherwise a negative errno. + */ +int ksmbd_decompress_work_request(struct ksmbd_work *work) +{ + void *out_buf; + int rc; + + rc = __ksmbd_decompress_request(work->conn, work->request_buf, + &out_buf); + if (rc) + return rc; + + kvfree(work->request_buf); + work->request_buf = out_buf; return 0; } diff --git a/fs/smb/server/compress.h b/fs/smb/server/compress.h index 663c6f44f09b0..13df2eb221e8c 100644 --- a/fs/smb/server/compress.h +++ b/fs/smb/server/compress.h @@ -11,6 +11,7 @@ #include "../common/compress/compress.h" int ksmbd_decompress_request(struct ksmbd_conn *conn); +int ksmbd_decompress_work_request(struct ksmbd_work *work); int ksmbd_compress_response(struct ksmbd_work *work); #endif /* __KSMBD_COMPRESS_H__ */ diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c index ef6f202f4024c..4f042a8a2fdbd 100644 --- a/fs/smb/server/connection.c +++ b/fs/smb/server/connection.c @@ -11,7 +11,9 @@ #include "server.h" #include "smb_common.h" #include "mgmt/ksmbd_ida.h" +#include "mgmt/user_session.h" #include "connection.h" +#include "vfs_cache.h" #include "compress.h" #include "transport_tcp.h" #include "transport_rdma.h" @@ -20,6 +22,8 @@ static DEFINE_MUTEX(init_lock); static struct ksmbd_conn_ops default_conn_ops; +static struct delayed_work session_expiration_work; +static bool stopping_session_expiration_work; DEFINE_HASHTABLE(conn_list, CONN_HASH_BITS); DECLARE_RWSEM(conn_list_lock); @@ -33,9 +37,9 @@ static int proc_show_clients(struct seq_file *m, void *v) struct timespec64 now, t; int i; - seq_printf(m, "#%-20s %-10s %-10s %-10s %-10s %-10s\n", - "", "", "", "", - "", ""); + seq_printf(m, "#%-40s %-10s %-10s %-12s %-10s %s\n", + "", "", "", "", + "", ""); down_read(&conn_list_lock); hash_for_each(conn_list, i, conn, hlist) { @@ -44,11 +48,11 @@ static int proc_show_clients(struct seq_file *m, void *v) t = timespec64_sub(now, t); #if IS_ENABLED(CONFIG_IPV6) if (!conn->inet_addr) - seq_printf(m, "%-20pI6c", &conn->inet6_addr); + seq_printf(m, " %-40pI6c", &conn->inet6_addr); else #endif - seq_printf(m, "%-20pI4", &conn->inet_addr); - seq_printf(m, " 0x%-10x %-10u %-12d %-10d %ptT\n", + seq_printf(m, " %-40pI4", &conn->inet_addr); + seq_printf(m, " 0x%-8x %-10u %-12d %-10d %ptT\n", conn->dialect, conn->total_credits, atomic_read(&conn->stats.open_files_count), @@ -82,18 +86,28 @@ static void delete_proc_clients(void) {} static struct workqueue_struct *ksmbd_conn_wq; +static void ksmbd_session_expiration_worker(struct work_struct *work); + int ksmbd_conn_wq_init(void) { ksmbd_conn_wq = alloc_workqueue("ksmbd-conn-release", WQ_UNBOUND | WQ_MEM_RECLAIM, 0); if (!ksmbd_conn_wq) return -ENOMEM; + + WRITE_ONCE(stopping_session_expiration_work, false); + INIT_DELAYED_WORK(&session_expiration_work, + ksmbd_session_expiration_worker); + queue_delayed_work(ksmbd_conn_wq, &session_expiration_work, + KSMBD_SESSION_EXPIRATION_INTERVAL); return 0; } void ksmbd_conn_wq_destroy(void) { if (ksmbd_conn_wq) { + WRITE_ONCE(stopping_session_expiration_work, true); + cancel_delayed_work_sync(&session_expiration_work); destroy_workqueue(ksmbd_conn_wq); ksmbd_conn_wq = NULL; } @@ -185,6 +199,7 @@ void ksmbd_conn_free(struct ksmbd_conn *conn) kvfree(conn->request_buf); kfree(conn->preauth_info); kfree(conn->mechToken); + ksmbd_preauth_session_destroy(conn); ksmbd_conn_put(conn); } @@ -202,6 +217,7 @@ struct ksmbd_conn *ksmbd_conn_alloc(void) return NULL; conn->need_neg = true; + conn->creation_time = jiffies; ksmbd_conn_set_new(conn); conn->local_nls = load_nls("utf8"); if (!conn->local_nls) @@ -223,6 +239,7 @@ struct ksmbd_conn *ksmbd_conn_alloc(void) init_waitqueue_head(&conn->r_count_q); INIT_LIST_HEAD(&conn->requests); INIT_LIST_HEAD(&conn->async_requests); + INIT_LIST_HEAD(&conn->preauth_sess_table); spin_lock_init(&conn->request_lock); spin_lock_init(&conn->credits_lock); ida_init(&conn->async_ida); @@ -291,6 +308,26 @@ void ksmbd_conn_try_dequeue_request(struct ksmbd_work *work) wake_up_all(&conn->req_running_q); } +static void ksmbd_conn_cancel_async_requests(struct ksmbd_conn *conn) +{ + struct ksmbd_work *work, *tmp; + + ksmbd_debug(CONN, "Cancel pending async requests on releasing connection\n"); + spin_lock(&conn->request_lock); + list_for_each_entry_safe(work, tmp, &conn->async_requests, + async_request_entry) { + if (cmpxchg(&work->state, KSMBD_WORK_ACTIVE, + KSMBD_WORK_CANCELLED) != KSMBD_WORK_ACTIVE) + continue; + + ksmbd_debug(CONN, "Cancel async request id %d\n", + work->async_id); + if (work->cancel_fn) + work->cancel_fn(work->cancel_argv); + } + spin_unlock(&conn->request_lock); +} + void ksmbd_conn_lock(struct ksmbd_conn *conn) { mutex_lock(&conn->srv_mutex); @@ -301,25 +338,64 @@ void ksmbd_conn_unlock(struct ksmbd_conn *conn) mutex_unlock(&conn->srv_mutex); } -void ksmbd_all_conn_set_status(u64 sess_id, u32 status) +static bool ksmbd_session_is_bound_to_conn(struct ksmbd_session *sess, + struct ksmbd_conn *conn) +{ + bool found; + + rcu_read_lock(); + found = xa_load(&conn->sessions, sess->id) == sess; + rcu_read_unlock(); + if (found) + return true; + + down_read(&sess->chann_lock); + found = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + return found; +} + +void ksmbd_all_conn_set_status(struct ksmbd_session *sess, u32 status) { struct ksmbd_conn *conn; int bkt; down_read(&conn_list_lock); hash_for_each(conn_list, bkt, conn, hlist) { - if (conn->binding || xa_load(&conn->sessions, sess_id)) - WRITE_ONCE(conn->status, status); + if (ksmbd_session_is_bound_to_conn(sess, conn)) { + spin_lock(&conn->request_lock); + if (!ksmbd_conn_exiting(conn) && + !ksmbd_conn_releasing(conn)) + WRITE_ONCE(conn->status, status); + spin_unlock(&conn->request_lock); + } } up_read(&conn_list_lock); } +void ksmbd_conn_abort(struct ksmbd_conn *conn) +{ + bool shutdown = false; + + spin_lock(&conn->request_lock); + if (!ksmbd_conn_exiting(conn) && !ksmbd_conn_releasing(conn)) { + ksmbd_conn_set_exiting(conn); + shutdown = true; + } + spin_unlock(&conn->request_lock); + wake_up_all(&conn->req_running_q); + + if (shutdown && conn->transport->ops->shutdown) + conn->transport->ops->shutdown(conn->transport); +} + void ksmbd_conn_wait_idle(struct ksmbd_conn *conn) { wait_event(conn->req_running_q, atomic_read(&conn->req_running) < 2); } -int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id) +int ksmbd_conn_wait_idle_sess(struct ksmbd_conn *curr_conn, + struct ksmbd_session *sess) { struct ksmbd_conn *conn; int rc, retry_count = 0, max_timeout = 120; @@ -329,9 +405,12 @@ int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id) if (retry_count >= max_timeout) return -EIO; + /* A blocked byte-range lock cannot drain until teardown wakes it. */ + ksmbd_wake_session_blocked_works(sess); + down_read(&conn_list_lock); hash_for_each(conn_list, bkt, conn, hlist) { - if (conn->binding || xa_load(&conn->sessions, sess_id)) { + if (ksmbd_session_is_bound_to_conn(sess, conn)) { rcount = (conn == curr_conn) ? 2 : 1; if (atomic_read(&conn->req_running) >= rcount) { rc = wait_event_timeout(conn->req_running_q, @@ -421,6 +500,20 @@ bool ksmbd_conn_alive(struct ksmbd_conn *conn) if (kthread_should_stop()) return false; + /* + * Stale connections that have not completed NEGOTIATE and SESSION_SETUP + * must be disconnected. Do not race a request that is currently + * completing authentication. + */ + if (!atomic_read(&conn->req_running) && + time_after(jiffies, conn->creation_time + + KSMBD_UNAUTHENTICATED_CONN_TIMEOUT) && + (READ_ONCE(conn->need_neg) || + !ksmbd_conn_has_valid_or_expired_session(conn))) { + ksmbd_debug(CONN, "Connection setup timed out\n"); + return false; + } + if (atomic_read(&conn->stats.open_files_count) > 0) return true; @@ -438,6 +531,51 @@ bool ksmbd_conn_alive(struct ksmbd_conn *conn) return true; } +static void ksmbd_session_expiration_worker(struct work_struct *work) +{ + struct ksmbd_conn *conn, *target; + int bkt; + + if (!ksmbd_server_running()) + goto reschedule; + + ksmbd_expire_sessions(); + + /* + * An old connection without a Valid or Expired session must be + * disconnected. Process one connection at a time without holding + * conn_list_lock across transport shutdown. + */ +again: + target = NULL; + down_read(&conn_list_lock); + hash_for_each(conn_list, bkt, conn, hlist) { + if (ksmbd_conn_exiting(conn) || ksmbd_conn_releasing(conn) || + atomic_read(&conn->req_running) || + time_before_eq(jiffies, conn->creation_time + + KSMBD_UNAUTHENTICATED_CONN_TIMEOUT) || + (!READ_ONCE(conn->need_neg) && + ksmbd_conn_has_valid_or_expired_session(conn))) + continue; + + target = ksmbd_conn_get(conn); + break; + } + up_read(&conn_list_lock); + + if (target) { + ksmbd_debug(CONN, "Connection setup timed out\n"); + ksmbd_conn_abort(target); + ksmbd_conn_put(target); + goto again; + } + +reschedule: + if (!READ_ONCE(stopping_session_expiration_work)) + queue_delayed_work(ksmbd_conn_wq, &session_expiration_work, + KSMBD_SESSION_EXPIRATION_INTERVAL); +} + /* "+2" for BCC field (ByteCount, 2 bytes) */ #define SMB1_MIN_SUPPORTED_PDU_SIZE (sizeof(struct smb_hdr) + 2) #define SMB2_MIN_SUPPORTED_PDU_SIZE (sizeof(struct smb2_pdu)) @@ -566,6 +704,7 @@ int ksmbd_conn_handler_loop(void *p) } ksmbd_conn_set_releasing(conn); + ksmbd_conn_cancel_async_requests(conn); /* Wait till all reference dropped to the Server object*/ ksmbd_debug(CONN, "Wait for all pending requests(%d)\n", atomic_read(&conn->r_count)); wait_event(conn->r_count_q, atomic_read(&conn->r_count) == 0); @@ -623,7 +762,8 @@ int ksmbd_conn_transport_init(void) } out: mutex_unlock(&init_lock); - create_proc_clients(); + if (create_proc_clients()) + pr_warn("Unable to create clients procfs entry\n"); return ret; } @@ -657,8 +797,10 @@ static void stop_sessions(void) * handler exited its receive loop for an unrelated * reason). */ - if (READ_ONCE(conn->status) != KSMBD_SESS_RELEASING) + spin_lock(&conn->request_lock); + if (!ksmbd_conn_releasing(conn)) ksmbd_conn_set_exiting(conn); + spin_unlock(&conn->request_lock); target = conn; break; } diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h index 0e4ebfac55588..909f73c6e3c14 100644 --- a/fs/smb/server/connection.h +++ b/fs/smb/server/connection.h @@ -22,6 +22,7 @@ #include "ksmbd_work.h" struct smbdirect_buffer_descriptor_v1; +struct ksmbd_session; #define KSMBD_SOCKET_BACKLOG 16 @@ -66,6 +67,7 @@ struct ksmbd_conn { struct rw_semaphore session_lock; /* smb session 1 per user */ struct xarray sessions; + unsigned long creation_time; unsigned long last_active; /* How many request are running currently */ atomic_t req_running; @@ -124,6 +126,7 @@ struct ksmbd_conn { bool binding; atomic_t refcnt; bool is_aapl; + bool aapl_readdir_attr; /* READDIR_ATTR negotiated */ struct work_struct release_work; }; @@ -158,6 +161,8 @@ struct ksmbd_transport { #define KSMBD_TCP_RECV_TIMEOUT (7 * HZ) #define KSMBD_TCP_SEND_TIMEOUT (5 * HZ) +#define KSMBD_SESSION_EXPIRATION_INTERVAL (5 * HZ) +#define KSMBD_UNAUTHENTICATED_CONN_TIMEOUT (45 * HZ) #define KSMBD_TCP_PEER_SOCKADDR(c) ((struct sockaddr *)&((c)->peer_addr)) #define CONN_HASH_BITS 12 @@ -166,11 +171,13 @@ extern struct rw_semaphore conn_list_lock; bool ksmbd_conn_alive(struct ksmbd_conn *conn); void ksmbd_conn_wait_idle(struct ksmbd_conn *conn); -int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id); +int ksmbd_conn_wait_idle_sess(struct ksmbd_conn *curr_conn, + struct ksmbd_session *sess); struct ksmbd_conn *ksmbd_conn_alloc(void); void ksmbd_conn_free(struct ksmbd_conn *conn); struct ksmbd_conn *ksmbd_conn_get(struct ksmbd_conn *conn); void ksmbd_conn_put(struct ksmbd_conn *conn); +void ksmbd_conn_abort(struct ksmbd_conn *conn); int ksmbd_conn_wq_init(void); void ksmbd_conn_wq_destroy(void); bool ksmbd_conn_lookup_dialect(struct ksmbd_conn *c); @@ -279,5 +286,5 @@ static inline void ksmbd_conn_set_releasing(struct ksmbd_conn *conn) WRITE_ONCE(conn->status, KSMBD_SESS_RELEASING); } -void ksmbd_all_conn_set_status(u64 sess_id, u32 status); +void ksmbd_all_conn_set_status(struct ksmbd_session *sess, u32 status); #endif /* __CONNECTION_H__ */ diff --git a/fs/smb/server/ksmbd_netlink.h b/fs/smb/server/ksmbd_netlink.h index 8ccd57fd904bc..af1e760453d9c 100644 --- a/fs/smb/server/ksmbd_netlink.h +++ b/fs/smb/server/ksmbd_netlink.h @@ -113,7 +113,8 @@ struct ksmbd_startup_request { __u32 max_connections; /* Number of maximum simultaneous connections */ __s8 bind_interfaces_only; __u32 max_ip_connections; /* Number of maximum connection per ip address */ - __s8 reserved[499]; /* Reserved room */ + __s8 aapl_model[32]; /* AAPL model string for Finder icon, e.g. "Xserve" */ + __s8 reserved[467]; /* Reserved room */ __u32 ifc_list_sz; /* interfaces list size */ __s8 ____payload[]; } __packed; @@ -285,6 +286,7 @@ struct ksmbd_spnego_authen_response { * stored in SecurityBuffer of SMB2 SESSION * SETUP response */ + __u64 session_expiry; /* Kerberos ticket expiry time */ __u8 payload[]; /* session key + AP_REP */ }; @@ -377,6 +379,8 @@ enum KSMBD_TREE_CONN_STATUS { #define KSMBD_SHARE_FLAG_UPDATE BIT(14) #define KSMBD_SHARE_FLAG_CROSSMNT BIT(15) #define KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY BIT(16) +#define KSMBD_SHARE_FLAG_HIDE_UNREADABLE BIT(17) +#define KSMBD_SHARE_FLAG_TIME_MACHINE BIT(18) /* * Tree connect request flags. diff --git a/fs/smb/server/ksmbd_work.c b/fs/smb/server/ksmbd_work.c index e2c2f45264bea..f35335307670f 100644 --- a/fs/smb/server/ksmbd_work.c +++ b/fs/smb/server/ksmbd_work.c @@ -11,6 +11,7 @@ #include "server.h" #include "connection.h" #include "ksmbd_work.h" +#include "vfs_cache.h" #include "mgmt/ksmbd_ida.h" static struct kmem_cache *work_cache; @@ -56,6 +57,7 @@ struct ksmbd_work *ksmbd_alloc_work_struct(void) INIT_LIST_HEAD(&work->request_entry); INIT_LIST_HEAD(&work->async_request_entry); INIT_LIST_HEAD(&work->fp_entry); + INIT_LIST_HEAD(&work->notify_entry); INIT_LIST_HEAD(&work->aux_read_list); work->iov_alloc_cnt = ARRAY_SIZE(work->iov_inline); work->iov = work->iov_inline; @@ -85,6 +87,9 @@ void ksmbd_free_work_struct(struct ksmbd_work *work) if (work->async_id) ksmbd_release_id(&work->conn->async_ida, work->async_id); + if (work->owns_conn_ref) + ksmbd_conn_put(work->conn); + ksmbd_fd_put(work, work->request_open); kmem_cache_free(work_cache, work); } diff --git a/fs/smb/server/ksmbd_work.h b/fs/smb/server/ksmbd_work.h index 88104f0cf3636..0844aa929f55d 100644 --- a/fs/smb/server/ksmbd_work.h +++ b/fs/smb/server/ksmbd_work.h @@ -12,6 +12,7 @@ struct ksmbd_conn; struct ksmbd_session; struct ksmbd_tree_connect; +struct ksmbd_file; #define KSMBD_WORK_INLINE_IOVS 4 @@ -81,7 +82,7 @@ struct ksmbd_work { /* Contiguous SMB2 compression transform owned by this work item. */ void *compress_buf; - unsigned char state; + unsigned int state; /* No response for cancelled request */ bool send_no_response:1; /* Request is encrypted */ @@ -90,7 +91,11 @@ struct ksmbd_work { bool compress_response:1; /* Is this SYNC or ASYNC ksmbd_work */ bool asynchronous:1; + /* Work owns a reference to @conn. */ + bool owns_conn_ref:1; bool need_invalidate_rkey:1; + bool request_open_chseq_tracked:1; + bool session_setup_reauth:1; unsigned int remote_key; /* cancel works */ @@ -98,12 +103,21 @@ struct ksmbd_work { void **cancel_argv; void (*cancel_fn)(void **argv); + /* + * Refcounted open associated with the SMB2 command currently being + * processed. + */ + struct ksmbd_file *request_open; + __le16 request_open_chseq; + struct work_struct work; /* List head at conn->requests */ struct list_head request_entry; /* List head at conn->async_requests */ struct list_head async_request_entry; struct list_head fp_entry; + /* List head at ksmbd_file->notify_pendings */ + struct list_head notify_entry; }; /** diff --git a/fs/smb/server/mgmt/share_config.c b/fs/smb/server/mgmt/share_config.c index 6f97f8d39657c..de524df6dd9c6 100644 --- a/fs/smb/server/mgmt/share_config.c +++ b/fs/smb/server/mgmt/share_config.c @@ -28,6 +28,62 @@ struct ksmbd_veto_pattern { struct list_head list; }; +#ifdef CONFIG_PROC_FS +static const struct ksmbd_const_name ksmbd_share_flag_names[] = { + {KSMBD_SHARE_FLAG_AVAILABLE, "available"}, + {KSMBD_SHARE_FLAG_BROWSEABLE, "browseable"}, + {KSMBD_SHARE_FLAG_WRITEABLE, "writeable"}, + {KSMBD_SHARE_FLAG_READONLY, "read-only"}, + {KSMBD_SHARE_FLAG_GUEST_OK, "guest-ok"}, + {KSMBD_SHARE_FLAG_GUEST_ONLY, "guest-only"}, + {KSMBD_SHARE_FLAG_STORE_DOS_ATTRS, "store-dos-attrs"}, + {KSMBD_SHARE_FLAG_OPLOCKS, "oplocks"}, + {KSMBD_SHARE_FLAG_PIPE, "pipe"}, + {KSMBD_SHARE_FLAG_HIDE_DOT_FILES, "hide-dot-files"}, + {KSMBD_SHARE_FLAG_INHERIT_OWNER, "inherit-owner"}, + {KSMBD_SHARE_FLAG_STREAMS, "streams"}, + {KSMBD_SHARE_FLAG_FOLLOW_SYMLINKS, "follow-symlinks"}, + {KSMBD_SHARE_FLAG_ACL_XATTR, "acl-xattr"}, + {KSMBD_SHARE_FLAG_UPDATE, "update"}, + {KSMBD_SHARE_FLAG_CROSSMNT, "crossmnt"}, + {KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY, "continuous-availability"}, +}; + +static int proc_show_shares(struct seq_file *m, void *v) +{ + struct ksmbd_share_config *share; + int i; + + down_read(&shares_table_lock); + hash_for_each(shares_table, i, share, hlist) { + seq_printf(m, "name:\t%s\n", share->name); + seq_printf(m, "type:\t%s\n", + test_share_config_flag(share, KSMBD_SHARE_FLAG_PIPE) ? + "pipe" : "disk"); + seq_printf(m, "tree_connects:\t%d\n", + atomic_read(&share->tree_connections)); + seq_printf(m, "file_mask:\t0%07o\n", share->create_mask); + seq_printf(m, "directory_mask:\t0%07o\n", share->directory_mask); + seq_puts(m, "flags:\t"); + ksmbd_proc_show_flag_names(m, ksmbd_share_flag_names, + ARRAY_SIZE(ksmbd_share_flag_names), + share->flags); + seq_puts(m, "\n\n"); + } + up_read(&shares_table_lock); + return 0; +} + +int create_proc_shares(void) +{ + if (!ksmbd_proc_create("shares", proc_show_shares, NULL)) + return -ENOMEM; + return 0; +} +#else +int create_proc_shares(void) { return 0; } +#endif + static unsigned int share_name_hash(const char *name) { return jhash(name, strlen(name), 0); @@ -88,9 +144,9 @@ static struct ksmbd_share_config *__share_lookup(const char *name) static int parse_veto_list(struct ksmbd_share_config *share, char *veto_list, - int veto_list_sz) + size_t veto_list_sz) { - int sz = 0; + size_t sz; if (!veto_list_sz) return 0; @@ -98,7 +154,7 @@ static int parse_veto_list(struct ksmbd_share_config *share, while (veto_list_sz > 0) { struct ksmbd_veto_pattern *p; - sz = strlen(veto_list); + sz = strnlen(veto_list, veto_list_sz); if (!sz) break; @@ -106,7 +162,7 @@ static int parse_veto_list(struct ksmbd_share_config *share, if (!p) return -ENOMEM; - p->pattern = kstrdup(veto_list, KSMBD_DEFAULT_GFP); + p->pattern = kstrndup(veto_list, sz, KSMBD_DEFAULT_GFP); if (!p->pattern) { kfree(p); return -ENOMEM; @@ -114,6 +170,9 @@ static int parse_veto_list(struct ksmbd_share_config *share, list_add(&p->list, &share->veto_list); + if (sz == veto_list_sz) + break; + veto_list += sz + 1; veto_list_sz -= (sz + 1); } @@ -156,21 +215,38 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work, share->flags = resp->flags; atomic_set(&share->refcount, 1); + ksmbd_share_tree_conn_init(share); INIT_LIST_HEAD(&share->veto_list); share->name = kstrdup(name, KSMBD_DEFAULT_GFP); + if (!share->name) { + kill_share(share); + share = NULL; + goto out; + } if (!test_share_config_flag(share, KSMBD_SHARE_FLAG_PIPE)) { - int path_len = PATH_MAX; - - if (resp->payload_sz) - path_len = resp->payload_sz - resp->veto_list_sz; + size_t path_len; - share->path = kstrndup(ksmbd_share_config_path(resp), path_len, - KSMBD_DEFAULT_GFP); - if (!share->path) { - ret = -ENOMEM; + if (resp->payload_sz <= resp->veto_list_sz) { + ret = -EINVAL; } else { - ret = 0; + path_len = resp->payload_sz - resp->veto_list_sz; + if (resp->veto_list_sz) + path_len--; + + if (!path_len) { + ret = -EINVAL; + } else { + share->path = kstrndup( + ksmbd_share_config_path(resp), + path_len, KSMBD_DEFAULT_GFP); + if (!share->path) + ret = -ENOMEM; + else + ret = 0; + } + } + if (share->path) { share->path_sz = strlen(share->path); while (share->path_sz > 1 && share->path[share->path_sz - 1] == '/') @@ -203,7 +279,7 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work, share->path = NULL; } } - if (ret || !share->name) { + if (ret) { kill_share(share); share = NULL; goto out; diff --git a/fs/smb/server/mgmt/share_config.h b/fs/smb/server/mgmt/share_config.h index d4ac2dd4de204..d157545fe7d1d 100644 --- a/fs/smb/server/mgmt/share_config.h +++ b/fs/smb/server/mgmt/share_config.h @@ -24,6 +24,9 @@ struct ksmbd_share_config { struct path vfs_path; atomic_t refcount; +#ifdef CONFIG_PROC_FS + atomic_t tree_connections; +#endif struct hlist_node hlist; unsigned short create_mask; unsigned short directory_mask; @@ -60,6 +63,27 @@ static inline int test_share_config_flag(struct ksmbd_share_config *share, return share->flags & flag; } +#ifdef CONFIG_PROC_FS +static inline void ksmbd_share_tree_conn_init(struct ksmbd_share_config *share) +{ + atomic_set(&share->tree_connections, 0); +} + +static inline void ksmbd_share_tree_conn_inc(struct ksmbd_share_config *share) +{ + atomic_inc(&share->tree_connections); +} + +static inline void ksmbd_share_tree_conn_dec(struct ksmbd_share_config *share) +{ + atomic_dec(&share->tree_connections); +} +#else +static inline void ksmbd_share_tree_conn_init(struct ksmbd_share_config *share) {} +static inline void ksmbd_share_tree_conn_inc(struct ksmbd_share_config *share) {} +static inline void ksmbd_share_tree_conn_dec(struct ksmbd_share_config *share) {} +#endif + void ksmbd_share_config_del(struct ksmbd_share_config *share); void __ksmbd_share_config_put(struct ksmbd_share_config *share); @@ -74,4 +98,5 @@ struct ksmbd_share_config *ksmbd_share_config_get(struct ksmbd_work *work, const char *name); bool ksmbd_share_veto_filename(struct ksmbd_share_config *share, const char *filename); +int create_proc_shares(void); #endif /* __SHARE_CONFIG_MANAGEMENT_H__ */ diff --git a/fs/smb/server/mgmt/tree_connect.c b/fs/smb/server/mgmt/tree_connect.c index 58e5b8592da46..dd1db3554caed 100644 --- a/fs/smb/server/mgmt/tree_connect.c +++ b/fs/smb/server/mgmt/tree_connect.c @@ -82,12 +82,15 @@ ksmbd_tree_conn_connect(struct ksmbd_work *work, const char *share_name) down_write(&sess->tree_conns_lock); ret = xa_err(xa_store(&sess->tree_conns, tree_conn->id, tree_conn, KSMBD_DEFAULT_GFP)); + if (!ret) + atomic_inc(&tree_conn->refcount); up_write(&sess->tree_conns_lock); if (ret) { status.ret = -ENOMEM; goto out_error; } ksmbd_counter_inc(KSMBD_COUNTER_TREE_CONNS); + ksmbd_share_tree_conn_inc(sc); kvfree(resp); return status; @@ -116,6 +119,7 @@ static int __ksmbd_tree_conn_disconnect(struct ksmbd_session *sess, ret = ksmbd_ipc_tree_disconnect_request(sess->id, tree_conn->id); ksmbd_release_tree_conn_id(sess, tree_conn->id); ksmbd_counter_dec(KSMBD_COUNTER_TREE_CONNS); + ksmbd_share_tree_conn_dec(tree_conn->share_conf); if (atomic_dec_and_test(&tree_conn->refcount)) { ksmbd_share_config_put(tree_conn->share_conf); kfree(tree_conn); @@ -127,6 +131,12 @@ int ksmbd_tree_conn_disconnect(struct ksmbd_session *sess, struct ksmbd_tree_connect *tree_conn) { down_write(&sess->tree_conns_lock); + if (tree_conn->t_state == TREE_DISCONNECTED || + xa_load(&sess->tree_conns, tree_conn->id) != tree_conn) { + up_write(&sess->tree_conns_lock); + return -ENOENT; + } + tree_conn->t_state = TREE_DISCONNECTED; xa_erase(&sess->tree_conns, tree_conn->id); up_write(&sess->tree_conns_lock); diff --git a/fs/smb/server/mgmt/user_config.c b/fs/smb/server/mgmt/user_config.c index cf45841d9d1b9..0be08cf1896c4 100644 --- a/fs/smb/server/mgmt/user_config.c +++ b/fs/smb/server/mgmt/user_config.c @@ -26,6 +26,7 @@ struct ksmbd_user *ksmbd_login_user(const char *account) resp_ext = ksmbd_ipc_login_request_ext(account); user = ksmbd_alloc_user(resp, resp_ext); + kvfree(resp_ext); out: kvfree(resp); return user; @@ -36,6 +37,17 @@ struct ksmbd_user *ksmbd_alloc_user(struct ksmbd_login_response *resp, { struct ksmbd_user *user; + /* + * resp->hash_sz is a __u16 taken from the mountd IPC login response but + * resp->hash[] is only KSMBD_REQ_MAX_HASH_SZ bytes. A malformed or + * malicious response can set hash_sz far beyond that (up to 65535), + * making the memcpy() below read past the response object + * (slab-out-of-bounds in ksmbd_alloc_user()). Reject any oversized + * hash rather than trust the length. + */ + if (resp->hash_sz > sizeof(resp->hash)) + return NULL; + user = kmalloc_obj(struct ksmbd_user, KSMBD_DEFAULT_GFP); if (!user) return NULL; diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index f99c86284ba3d..c3c0557dcc19c 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -22,6 +22,7 @@ static DEFINE_IDA(session_ida); #define SESSION_HASH_BITS 12 +#define KSMBD_MAX_PENDING_SESSIONS 1 static DEFINE_HASHTABLE(sessions_table, SESSION_HASH_BITS); static DECLARE_RWSEM(sessions_table_lock); @@ -90,9 +91,15 @@ static int show_proc_session(struct seq_file *m, void *v) sess = (struct ksmbd_session *)m->private; ksmbd_user_session_get(sess); + seq_printf(m, "%-20s\t%s\n", "user", session_user_name(sess)); + seq_printf(m, "%-20s\t%llu\n", "id", sess->id); + seq_printf(m, "%-20s\t%s\n", "state", session_state_string(sess)); + i = 0; down_read(&sess->chann_lock); xa_for_each(&sess->ksmbd_chann_list, id, chan) { + const char *name; + #if IS_ENABLED(CONFIG_IPV6) if (chan->conn->inet_addr) seq_printf(m, "%-20s\t%pI4\n", "client", @@ -104,29 +111,37 @@ static int show_proc_session(struct seq_file *m, void *v) seq_printf(m, "%-20s\t%pI4\n", "client", &chan->conn->inet_addr); #endif - seq_printf(m, "%-20s\t%s\n", "user", session_user_name(sess)); - seq_printf(m, "%-20s\t%llu\n", "id", sess->id); - seq_printf(m, "%-20s\t%s\n", "state", - session_state_string(sess)); - seq_printf(m, "%-20s\t", "capabilities"); ksmbd_proc_show_flag_names(m, ksmbd_sess_cap_const_names, ARRAY_SIZE(ksmbd_sess_cap_const_names), chan->conn->vals->req_capabilities); + seq_putc(m, '\n'); if (sess->sign) { - seq_printf(m, "%-20s\t", "signing"); - ksmbd_proc_show_const_name(m, "%s\t", - ksmbd_signing_const_names, - ARRAY_SIZE(ksmbd_signing_const_names), - le16_to_cpu(chan->conn->signing_algorithm)); - } else if (sess->enc) { - seq_printf(m, "%-20s\t", "encryption"); - ksmbd_proc_show_const_name(m, "%s\t", - ksmbd_cipher_const_names, - ARRAY_SIZE(ksmbd_cipher_const_names), - le16_to_cpu(chan->conn->cipher_type)); + unsigned int algorithm = + le16_to_cpu(chan->conn->signing_algorithm); + + name = ksmbd_proc_const_name(ksmbd_signing_const_names, + ARRAY_SIZE(ksmbd_signing_const_names), + algorithm); + if (name) + seq_printf(m, "%-20s\t%s\n", "signing", name); + else + seq_printf(m, "%-20s\t0x%04x\n", "signing", + algorithm); + } + if (sess->enc) { + unsigned int cipher = le16_to_cpu(chan->conn->cipher_type); + + name = ksmbd_proc_const_name(ksmbd_cipher_const_names, + ARRAY_SIZE(ksmbd_cipher_const_names), + cipher); + if (name) + seq_printf(m, "%-20s\t%s\n", "encryption", name); + else + seq_printf(m, "%-20s\t0x%04x\n", "encryption", + cipher); } i++; } @@ -152,35 +167,6 @@ static int show_proc_session(struct seq_file *m, void *v) return 0; } -void ksmbd_proc_show_flag_names(struct seq_file *m, - const struct ksmbd_const_name *table, - int count, - unsigned int flags) -{ - int i; - - for (i = 0; i < count; i++) { - if (table[i].const_value & flags) - seq_printf(m, "0x%08x\t", table[i].const_value); - } - seq_putc(m, '\n'); -} - -void ksmbd_proc_show_const_name(struct seq_file *m, - const char *format, - const struct ksmbd_const_name *table, - int count, - unsigned int const_value) -{ - int i; - - for (i = 0; i < count; i++) { - if (table[i].const_value & const_value) - seq_printf(m, format, table[i].name); - } - seq_putc(m, '\n'); -} - static int create_proc_session(struct ksmbd_session *sess) { char name[30]; @@ -188,6 +174,8 @@ static int create_proc_session(struct ksmbd_session *sess) snprintf(name, sizeof(name), "sessions/%llu", sess->id); sess->proc_entry = ksmbd_proc_create(name, show_proc_session, sess); + if (!sess->proc_entry) + return -ENOMEM; return 0; } @@ -395,6 +383,12 @@ void ksmbd_session_destroy(struct ksmbd_session *sess) kfree(sess); } +static void ksmbd_session_remove_from_table(struct ksmbd_session *sess) +{ + hash_del(&sess->hlist); + ksmbd_counter_dec(KSMBD_COUNTER_SESSIONS); +} + struct ksmbd_session *__session_lookup(unsigned long long id) { struct ksmbd_session *sess; @@ -408,35 +402,153 @@ struct ksmbd_session *__session_lookup(unsigned long long id) return NULL; } -static void ksmbd_expire_session(struct ksmbd_conn *conn) +static bool ksmbd_too_many_session_setups(struct ksmbd_conn *conn) { unsigned long id; struct ksmbd_session *sess; + unsigned int pending = 0; down_write(&sessions_table_lock); down_write(&conn->session_lock); xa_for_each(&conn->sessions, id, sess) { + if (READ_ONCE(sess->state) != SMB2_SESSION_IN_PROGRESS) + continue; + if (atomic_read(&sess->refcnt) <= 1 && - (sess->state != SMB2_SESSION_VALID || - time_after(jiffies, - sess->last_active + SMB2_SESSION_TIMEOUT))) { + time_after(jiffies, sess->last_active + + KSMBD_UNAUTHENTICATED_CONN_TIMEOUT)) { xa_erase(&conn->sessions, sess->id); - hash_del(&sess->hlist); + ksmbd_session_remove_from_table(sess); ksmbd_session_destroy(sess); continue; } + pending++; } up_write(&conn->session_lock); up_write(&sessions_table_lock); + return pending >= KSMBD_MAX_PENDING_SESSIONS; } int ksmbd_session_register(struct ksmbd_conn *conn, struct ksmbd_session *sess) { + int ret; + sess->dialect = conn->dialect; memcpy(sess->ClientGUID, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); - ksmbd_expire_session(conn); - return xa_err(xa_store(&conn->sessions, sess->id, sess, KSMBD_DEFAULT_GFP)); + /* Bound abandoned SessionId-zero authentication exchanges. */ + if (ksmbd_too_many_session_setups(conn)) + ret = -ENOSPC; + else + ret = xa_err(xa_store(&conn->sessions, sess->id, sess, + KSMBD_DEFAULT_GFP)); + if (ret) { + down_write(&sessions_table_lock); + ksmbd_session_remove_from_table(sess); + up_write(&sessions_table_lock); + ksmbd_user_session_put(sess); + } + + return ret; +} + +void ksmbd_session_unregister(struct ksmbd_conn *conn, + struct ksmbd_session *sess) +{ + struct ksmbd_conn *session_conns[KSMBD_MAX_CHANNELS]; + struct channel *chann; + unsigned long index; + unsigned int nr_conns = 0, i; + bool removed = false; + + down_write(&sessions_table_lock); + if (!hlist_unhashed(&sess->hlist)) { + /* Keep each channel connection stable under sessions_table_lock. */ + down_read(&sess->chann_lock); + xa_for_each(&sess->ksmbd_chann_list, index, chann) { + if (nr_conns == ARRAY_SIZE(session_conns)) + break; + session_conns[nr_conns++] = chann->conn; + } + up_read(&sess->chann_lock); + + ksmbd_session_remove_from_table(sess); + removed = true; + } + + down_write(&conn->session_lock); + if (xa_load(&conn->sessions, sess->id) == sess) + xa_erase(&conn->sessions, sess->id); + up_write(&conn->session_lock); + for (i = 0; i < nr_conns; i++) { + if (session_conns[i] == conn) + continue; + down_write(&session_conns[i]->session_lock); + if (xa_load(&session_conns[i]->sessions, sess->id) == sess) + xa_erase(&session_conns[i]->sessions, sess->id); + up_write(&session_conns[i]->session_lock); + } + up_write(&sessions_table_lock); + + if (removed) + ksmbd_user_session_put(sess); +} + +bool ksmbd_conn_has_valid_or_expired_session(struct ksmbd_conn *conn) +{ + struct ksmbd_session *sess; + unsigned long id; + int state, bkt; + bool found = false; + + down_read(&conn->session_lock); + xa_for_each(&conn->sessions, id, sess) { + state = READ_ONCE(sess->state); + if (state == SMB2_SESSION_VALID || + state == SMB2_SESSION_EXPIRED) { + found = true; + break; + } + } + up_read(&conn->session_lock); + if (found) + return true; + + /* A session bound through SMB3 multichannel is not in conn->sessions. */ + down_read(&sessions_table_lock); + hash_for_each(sessions_table, bkt, sess, hlist) { + state = READ_ONCE(sess->state); + if (state != SMB2_SESSION_VALID && + state != SMB2_SESSION_EXPIRED) + continue; + + down_read(&sess->chann_lock); + found = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + if (found) + break; + } + up_read(&sessions_table_lock); + return found; +} + +void ksmbd_expire_sessions(void) +{ + struct ksmbd_session *sess; + u64 now = ktime_get_real_seconds(); + int bkt; + + down_read(&sessions_table_lock); + hash_for_each(sessions_table, bkt, sess, hlist) { + if (READ_ONCE(sess->state) != SMB2_SESSION_VALID || + !sess->kerberos_expiry || now < sess->kerberos_expiry) + continue; + + if (cmpxchg(&sess->state, SMB2_SESSION_VALID, + SMB2_SESSION_EXPIRED) == SMB2_SESSION_VALID) + ksmbd_counter_inc(KSMBD_COUNTER_SESSION_TIMEOUTS); + } + up_read(&sessions_table_lock); } static int ksmbd_chann_del(struct ksmbd_conn *conn, struct ksmbd_session *sess) @@ -453,7 +565,7 @@ static int ksmbd_chann_del(struct ksmbd_conn *conn, struct ksmbd_session *sess) return 0; } -void ksmbd_sessions_deregister(struct ksmbd_conn *conn) +void ksmbd_conn_sessions_cleanup(struct ksmbd_conn *conn) { struct ksmbd_session *sess; unsigned long id; @@ -464,7 +576,7 @@ void ksmbd_sessions_deregister(struct ksmbd_conn *conn) hash_for_each_safe(sessions_table, bkt, tmp, sess, hlist) { if (!ksmbd_chann_del(conn, sess) && xa_empty(&sess->ksmbd_chann_list)) { - hash_del(&sess->hlist); + ksmbd_session_remove_from_table(sess); down_write(&conn->session_lock); xa_erase(&conn->sessions, sess->id); up_write(&conn->session_lock); @@ -475,18 +587,10 @@ void ksmbd_sessions_deregister(struct ksmbd_conn *conn) down_write(&conn->session_lock); xa_for_each(&conn->sessions, id, sess) { - unsigned long chann_id; - struct channel *chann; - - xa_for_each(&sess->ksmbd_chann_list, chann_id, chann) { - if (chann->conn != conn) - ksmbd_conn_set_exiting(chann->conn); - } - ksmbd_chann_del(conn, sess); if (xa_empty(&sess->ksmbd_chann_list)) { xa_erase(&conn->sessions, sess->id); - hash_del(&sess->hlist); + ksmbd_session_remove_from_table(sess); if (atomic_dec_and_test(&sess->refcnt)) ksmbd_session_destroy(sess); } @@ -539,19 +643,35 @@ struct ksmbd_session *ksmbd_session_lookup_slowpath(unsigned long long id) return sess; } -struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, - unsigned long long id) +struct ksmbd_session *ksmbd_session_lookup_all_states(struct ksmbd_conn *conn, + unsigned long long id) { struct ksmbd_session *sess; + bool channel_found; sess = ksmbd_session_lookup(conn, id); - if (!sess && conn->binding) { + if (!sess) { sess = ksmbd_session_lookup_slowpath(id); - if (sess && !xa_load(&sess->ksmbd_chann_list, (long)conn)) { + if (!sess) + return NULL; + + down_read(&sess->chann_lock); + channel_found = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + if (!channel_found) { ksmbd_user_session_put(sess); sess = NULL; } } + return sess; +} + +struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, + unsigned long long id) +{ + struct ksmbd_session *sess; + + sess = ksmbd_session_lookup_all_states(conn, id); if (sess && sess->state != SMB2_SESSION_VALID) { ksmbd_user_session_put(sess); sess = NULL; @@ -592,6 +712,17 @@ struct preauth_session *ksmbd_preauth_session_alloc(struct ksmbd_conn *conn, return sess; } +void ksmbd_preauth_session_destroy(struct ksmbd_conn *conn) +{ + struct preauth_session *sess, *tmp; + + list_for_each_entry_safe(sess, tmp, &conn->preauth_sess_table, + preauth_entry) { + list_del(&sess->preauth_entry); + kfree(sess); + } +} + void destroy_previous_session(struct ksmbd_conn *conn, struct ksmbd_user *user, u64 id) { @@ -612,16 +743,28 @@ void destroy_previous_session(struct ksmbd_conn *conn, memcmp(user->passkey, prev_user->passkey, user->passkey_sz)) goto out; - ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_RECONNECT); - err = ksmbd_conn_wait_idle_sess_id(conn, id); + down_write(&prev_sess->chann_lock); + if (prev_sess->tearing_down) { + up_write(&prev_sess->chann_lock); + goto out; + } + prev_sess->tearing_down = true; + up_write(&prev_sess->chann_lock); + + ksmbd_all_conn_set_status(prev_sess, KSMBD_SESS_NEED_RECONNECT); + err = ksmbd_conn_wait_idle_sess(conn, prev_sess); if (err) { - ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_SETUP); + down_write(&prev_sess->chann_lock); + prev_sess->tearing_down = false; + up_write(&prev_sess->chann_lock); + ksmbd_all_conn_set_status(prev_sess, KSMBD_SESS_GOOD); goto out; } ksmbd_destroy_file_table(prev_sess); + prev_sess->kerberos_expiry = 0; prev_sess->state = SMB2_SESSION_EXPIRED; - ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_SETUP); + ksmbd_all_conn_set_status(prev_sess, KSMBD_SESS_NEED_SETUP); ksmbd_launch_ksmbd_durable_scavenger(); out: up_write(&conn->session_lock); @@ -691,10 +834,11 @@ static struct ksmbd_session *__session_create(int protocol) down_write(&sessions_table_lock); hash_add(sessions_table, &sess->hlist, sess->id); + ksmbd_counter_inc(KSMBD_COUNTER_SESSIONS); up_write(&sessions_table_lock); - create_proc_session(sess); - ksmbd_counter_inc(KSMBD_COUNTER_SESSIONS); + if (create_proc_session(sess)) + pr_warn_ratelimited("Unable to create session %llu procfs entry\n", sess->id); return sess; error: diff --git a/fs/smb/server/mgmt/user_session.h b/fs/smb/server/mgmt/user_session.h index 4637a8c8436d0..217258551d6d0 100644 --- a/fs/smb/server/mgmt/user_session.h +++ b/fs/smb/server/mgmt/user_session.h @@ -42,11 +42,13 @@ struct ksmbd_session { bool sign; bool enc; + bool tearing_down; int state; __u8 *Preauth_HashValue; char sess_key[CIFS_KEY_SIZE]; + u64 kerberos_expiry; struct hlist_node hlist; struct rw_semaphore chann_lock; @@ -70,6 +72,8 @@ struct ksmbd_session { struct rw_semaphore rpc_lock; }; +#define KSMBD_MAX_CHANNELS 32 + static inline int test_session_flag(struct ksmbd_session *sess, int bit) { return sess->flags & bit; @@ -96,14 +100,21 @@ bool is_ksmbd_session_in_connection(struct ksmbd_conn *conn, unsigned long long id); int ksmbd_session_register(struct ksmbd_conn *conn, struct ksmbd_session *sess); -void ksmbd_sessions_deregister(struct ksmbd_conn *conn); +void ksmbd_session_unregister(struct ksmbd_conn *conn, + struct ksmbd_session *sess); +void ksmbd_conn_sessions_cleanup(struct ksmbd_conn *conn); +bool ksmbd_conn_has_valid_or_expired_session(struct ksmbd_conn *conn); +void ksmbd_expire_sessions(void); struct ksmbd_session *__session_lookup(unsigned long long id); struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, unsigned long long id); +struct ksmbd_session *ksmbd_session_lookup_all_states(struct ksmbd_conn *conn, + unsigned long long id); void destroy_previous_session(struct ksmbd_conn *conn, struct ksmbd_user *user, u64 id); struct preauth_session *ksmbd_preauth_session_alloc(struct ksmbd_conn *conn, u64 sess_id); +void ksmbd_preauth_session_destroy(struct ksmbd_conn *conn); struct preauth_session *ksmbd_preauth_session_lookup(struct ksmbd_conn *conn, unsigned long long id); diff --git a/fs/smb/server/misc.h b/fs/smb/server/misc.h index 3909104e18ad4..1faaddd0f5f75 100644 --- a/fs/smb/server/misc.h +++ b/fs/smb/server/misc.h @@ -43,7 +43,7 @@ struct ksmbd_const_name { const char *name; }; -void ksmbd_proc_init(void); +int ksmbd_proc_init(void); void ksmbd_proc_cleanup(void); void ksmbd_proc_reset(void); struct proc_dir_entry *ksmbd_proc_create(const char *name, @@ -53,13 +53,10 @@ void ksmbd_proc_show_flag_names(struct seq_file *m, const struct ksmbd_const_name *table, int count, unsigned int flags); -void ksmbd_proc_show_const_name(struct seq_file *m, - const char *format, - const struct ksmbd_const_name *table, - int count, - unsigned int const_value); +const char *ksmbd_proc_const_name(const struct ksmbd_const_name *table, + int count, unsigned int const_value); #else -static inline void ksmbd_proc_init(void) {} +static inline int ksmbd_proc_init(void) { return 0; } static inline void ksmbd_proc_cleanup(void) {} static inline void ksmbd_proc_reset(void) {} #endif diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c index 79787099afdc6..04dc572ed1bbf 100644 --- a/fs/smb/server/oplock.c +++ b/fs/smb/server/oplock.c @@ -16,6 +16,7 @@ #include "mgmt/user_session.h" #include "mgmt/share_config.h" #include "mgmt/tree_connect.h" +#include "server.h" static LIST_HEAD(lease_table_list); static DEFINE_RWLOCK(lease_list_lock); @@ -89,6 +90,7 @@ static struct oplock_info *alloc_opinfo(struct ksmbd_work *work, opinfo->conn = ksmbd_conn_get(work->conn); opinfo->level = SMB2_OPLOCK_LEVEL_NONE; opinfo->op_state = OPLOCK_STATE_NONE; + spin_lock_init(&opinfo->state_lock); opinfo->pending_break = 0; opinfo->fid = id; opinfo->Tid = Tid; @@ -545,14 +547,23 @@ void close_id_del_oplock(struct ksmbd_file *fp) opinfo_del(opinfo); rcu_assign_pointer(fp->f_opinfo, NULL); - if (opinfo->op_state == OPLOCK_ACK_WAIT) { - opinfo->op_state = OPLOCK_CLOSING; - wake_up_interruptible_all(&opinfo->oplock_q); - if (opinfo->is_lease) { - atomic_set(&opinfo->breaking_cnt, 0); - wake_up_interruptible_all(&opinfo->oplock_brk); - } - } + spin_lock(&opinfo->state_lock); + if (opinfo->op_state == OPLOCK_ACK_WAIT && opinfo->is_lease) + atomic_set(&opinfo->breaking_cnt, 0); + /* + * An opinfo that has been removed from the inode list is terminal. Keep + * this transition and releasing pending_break under state_lock. a breaker + * takes the same lock before it acquires pending_break or sets ACK_WAIT. + */ + opinfo->op_state = OPLOCK_CLOSING; + clear_bit_unlock(0, &opinfo->pending_break); + spin_unlock(&opinfo->state_lock); + wake_up_interruptible_all(&opinfo->oplock_q); + if (opinfo->is_lease) + wake_up_interruptible_all(&opinfo->oplock_brk); + /* memory barrier is needed for wake_up_bit() */ + smp_mb__after_atomic(); + wake_up_bit(&opinfo->pending_break, 0); opinfo_count_dec(fp); atomic_dec(&opinfo->refcount); @@ -734,12 +745,18 @@ static bool wait_for_break_ack(struct oplock_info *opinfo) /* is this a timeout ? */ if (!rc) { + spin_lock(&opinfo->state_lock); + if (opinfo->op_state == OPLOCK_CLOSING) { + spin_unlock(&opinfo->state_lock); + return false; + } if (opinfo->is_lease) { opinfo->o_lease->state = SMB2_LEASE_NONE_LE; lease_update_oplock_levels(opinfo->o_lease); } opinfo->level = SMB2_OPLOCK_LEVEL_NONE; opinfo->op_state = OPLOCK_STATE_NONE; + spin_unlock(&opinfo->state_lock); return true; } @@ -754,9 +771,35 @@ static void wake_up_oplock_break(struct oplock_info *opinfo) wake_up_bit(&opinfo->pending_break, 0); } +static bool oplock_break_set_ack_wait(struct oplock_info *opinfo) +{ + bool ret = false; + + spin_lock(&opinfo->state_lock); + if (opinfo->op_state != OPLOCK_CLOSING) { + opinfo->op_state = OPLOCK_ACK_WAIT; + ret = true; + } + spin_unlock(&opinfo->state_lock); + + return ret; +} + static int oplock_break_pending(struct oplock_info *opinfo, int req_op_level) { - while (test_and_set_bit(0, &opinfo->pending_break)) { + for (;;) { + bool closing; + + spin_lock(&opinfo->state_lock); + closing = opinfo->op_state == OPLOCK_CLOSING; + if (!closing && !test_and_set_bit(0, &opinfo->pending_break)) { + spin_unlock(&opinfo->state_lock); + break; + } + spin_unlock(&opinfo->state_lock); + if (closing) + return -ENOENT; + if (opinfo->is_lease) opinfo->o_lease->reuse_epoch = true; @@ -765,9 +808,12 @@ static int oplock_break_pending(struct oplock_info *opinfo, int req_op_level) /* Not immediately break to none. */ opinfo->open_trunc = 0; - if (opinfo->op_state == OPLOCK_CLOSING) + spin_lock(&opinfo->state_lock); + closing = opinfo->op_state == OPLOCK_CLOSING; + spin_unlock(&opinfo->state_lock); + if (closing) return -ENOENT; - else if (opinfo->level <= req_op_level) { + if (opinfo->level <= req_op_level) { if (opinfo->is_lease == false) return 1; @@ -878,31 +924,69 @@ static void __smb2_oplock_break_noti(struct work_struct *wk) ksmbd_conn_put(conn); } +/* + * Select and pin the connection used for an oplock break before doing any + * allocations which may sleep. The caller of oplock_break() holds a live + * reference on ci (a file being opened, a file being operated on, or an + * explicit ksmbd_inode_lookup_lock() reference in the parent lease break + * paths), so the inode cannot be freed during the call and its lock is + * reachable without dereferencing opinfo->o_fp, which is not pinned by + * the oplock reference and may be freed by a concurrent close. + * + * opinfo->conn is cleared under ci->m_lock by session_fd_check() when the + * durable handle owning the oplock is disconnected, reassigned by + * ksmbd_reopen_durable_fd() under the same lock, and the last + * ksmbd_conn_put() of the old connection frees it. Holding the read lock + * excludes both writers, so the connection cannot be freed while it is + * selected. + */ +static struct ksmbd_conn *smb2_oplock_break_conn_get(struct oplock_info *opinfo, + struct ksmbd_inode *ci) +{ + struct ksmbd_conn *conn; + + down_read(&ci->m_lock); + conn = READ_ONCE(opinfo->conn); + if (conn && !ksmbd_conn_releasing(conn)) + conn = ksmbd_conn_get(conn); + else + conn = NULL; + up_read(&ci->m_lock); + + return conn; +} + /** * smb2_oplock_break_noti() - send smb2 exclusive/batch to level2 oplock * break command from server to client * @opinfo: oplock info object + * @ci: inode owning the break target's oplock list, pinned by + * the caller * * Return: 0 on success, otherwise error */ -static int smb2_oplock_break_noti(struct oplock_info *opinfo) +static int smb2_oplock_break_noti(struct oplock_info *opinfo, + struct ksmbd_inode *ci) { struct ksmbd_conn *conn; struct oplock_break_info *br_info; int ret = 0; struct ksmbd_work *work; - conn = READ_ONCE(opinfo->conn); + conn = smb2_oplock_break_conn_get(opinfo, ci); if (!conn) return ksmbd_invalidate_durable_fd(opinfo->fid); work = ksmbd_alloc_work_struct(); - if (!work) + if (!work) { + ksmbd_conn_put(conn); return -ENOMEM; + } br_info = kmalloc_obj(struct oplock_break_info, KSMBD_DEFAULT_GFP); if (!br_info) { ksmbd_free_work_struct(work); + ksmbd_conn_put(conn); return -ENOMEM; } @@ -911,7 +995,8 @@ static int smb2_oplock_break_noti(struct oplock_info *opinfo) br_info->open_trunc = opinfo->open_trunc; work->request_buf = (char *)br_info; - work->conn = ksmbd_conn_get(conn); + /* Transfer the reference acquired by smb2_oplock_break_conn_get(). */ + work->conn = conn; work->sess = opinfo->sess; ksmbd_conn_r_count_inc(conn); @@ -1074,9 +1159,9 @@ static void wait_lease_breaking(struct oplock_info *opinfo) } } -static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, - struct ksmbd_work *in_work, bool share_break) -{ +static int oplock_break(struct oplock_info *brk_opinfo, struct ksmbd_inode *ci, + int req_op_level, struct ksmbd_work *in_work, + bool share_break, bool sync_lease_break){ int err = 0; bool sent_interim = false; @@ -1145,7 +1230,11 @@ static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, if (lease->state & (SMB2_LEASE_WRITE_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE)) { - brk_opinfo->op_state = OPLOCK_ACK_WAIT; + if (!oplock_break_set_ack_wait(brk_opinfo)) { + atomic_dec_if_positive(&brk_opinfo->breaking_cnt); + wake_up_oplock_break(brk_opinfo); + return -ENOENT; + } } else atomic_dec(&brk_opinfo->breaking_cnt); @@ -1192,11 +1281,27 @@ static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, return err < 0 ? err : 0; if (brk_opinfo->level == SMB2_OPLOCK_LEVEL_BATCH || - brk_opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE) - brk_opinfo->op_state = OPLOCK_ACK_WAIT; + brk_opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE) { + if (!oplock_break_set_ack_wait(brk_opinfo)) { + wake_up_oplock_break(brk_opinfo); + return -ENOENT; + } + } + + /* + * Keep a conflicting CREATE asynchronous while waiting for an + * oplock-break acknowledgement. Besides avoiding a blocked client + * request, this lets a replay arrive while the original CREATE is + * still pending and be rejected with FILE_NOT_AVAILABLE. + */ + if (in_work) { + setup_async_work(in_work, NULL, NULL); + smb2_send_interim_resp(in_work, STATUS_PENDING); + release_async_work(in_work); + } } - err = smb2_oplock_break_noti(brk_opinfo); + err = smb2_oplock_break_noti(brk_opinfo, ci); ksmbd_debug(OPLOCK, "oplock granted = %d\n", brk_opinfo->level); if (brk_opinfo->op_state == OPLOCK_CLOSING) @@ -1224,13 +1329,14 @@ static int oplock_break_add(struct list_head *head, struct oplock_info *opinfo) return 0; } -static void oplock_break_drain_none(struct list_head *head) +static void oplock_break_drain_none(struct list_head *head, + struct ksmbd_inode *ci) { struct oplock_break_entry *ent, *tmp; list_for_each_entry_safe(ent, tmp, head, list) { - oplock_break(ent->opinfo, SMB2_OPLOCK_LEVEL_NONE, NULL, false); - list_del(&ent->list); + oplock_break(ent->opinfo, ci, SMB2_OPLOCK_LEVEL_NONE, NULL, + false, false); list_del(&ent->list); opinfo_put(ent->opinfo); kfree(ent); } @@ -1377,7 +1483,7 @@ void smb_send_parent_lease_break_noti(struct ksmbd_file *fp, } up_read(&p_ci->m_lock); - oplock_break_drain_none(&brk_list); + oplock_break_drain_none(&brk_list, p_ci); ksmbd_inode_put(p_ci); } @@ -1421,7 +1527,7 @@ void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) } up_read(&p_ci->m_lock); - oplock_break_drain_none(&brk_list); + oplock_break_drain_none(&brk_list, p_ci); ksmbd_inode_put(p_ci); } @@ -1435,12 +1541,13 @@ void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) * @tid: Tree id of connection * @lctx: lease context information on file open * @share_ret: share mode + * @replay: whether this is a replayed CREATE request * * Return: 0 on success, otherwise error */ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, struct ksmbd_file *fp, __u16 tid, - struct lease_ctx_info *lctx, int share_ret) + struct lease_ctx_info *lctx, int share_ret, bool replay) { int err = 0; int break_level = SMB2_OPLOCK_LEVEL_II; @@ -1524,6 +1631,21 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, prev_op_has_lease = prev_opinfo->is_lease; if (prev_op_has_lease) prev_op_state = prev_opinfo->o_lease->state; + /* + * A replay received while this open is waiting for an oplock or lease + * break must not observe an intermediate level and proceed as a new + * open. This check has to precede break_needed. an oplock may already + * have been downgraded from Batch to II while its acknowledgement is + * still pending. + */ + if (replay && + (test_bit(0, &prev_opinfo->pending_break) || + prev_opinfo->op_state == OPLOCK_ACK_WAIT)) { + err = -EINPROGRESS; + opinfo_put(prev_opinfo); + goto err_out; + } + if (share_ret < 0 && prev_opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE) { err = share_ret; @@ -1541,9 +1663,8 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, prev_durable_detached = prev_op_snapshot.durable_detached; prev_fid = prev_op_snapshot.fid; - err = oplock_break(prev_opinfo, break_level, work, - share_ret < 0 && prev_opinfo->is_lease); - if (prev_durable_detached || (prev_durable_open && err == -ENOENT)) + err = oplock_break(prev_opinfo, ci, break_level, work, + share_ret < 0 && prev_opinfo->is_lease, false); if (prev_durable_detached || (prev_durable_open && err == -ENOENT)) ksmbd_invalidate_durable_fd(prev_fid); opinfo_put(prev_opinfo); if (err == -EAGAIN) { @@ -1555,7 +1676,14 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, goto set_lev; } if (err == -ENOENT) { - if (req_op_level != SMB2_OPLOCK_LEVEL_NONE) + /* + * A pending durable CREATE can lose the previous oplock when + * its holder closes the file. In that case grant the original + * request its full caching state. Other opens still need the + * normal shared-open downgrade below. + */ + if (!prev_durable_open && + req_op_level != SMB2_OPLOCK_LEVEL_NONE) req_op_level = SMB2_OPLOCK_LEVEL_II; goto set_lev; } @@ -1640,8 +1768,8 @@ static bool smb_break_all_write_oplock(struct ksmbd_work *work, } brk_opinfo->open_trunc = is_trunc; - oplock_break(brk_opinfo, SMB2_OPLOCK_LEVEL_II, work, false); - sent_break = true; + oplock_break(brk_opinfo, fp->f_ci, SMB2_OPLOCK_LEVEL_II, work, false, + false); sent_break = true; opinfo_put(brk_opinfo); return sent_break; @@ -1725,13 +1853,16 @@ static void __smb_break_all_levII_oplock(struct ksmbd_work *work, if (!brk_op->is_lease && !send_oplock_break) { brk_op->level = SMB2_OPLOCK_LEVEL_NONE; - brk_op->op_state = OPLOCK_STATE_NONE; + spin_lock(&brk_op->state_lock); + if (brk_op->op_state != OPLOCK_CLOSING) + brk_op->op_state = OPLOCK_STATE_NONE; + spin_unlock(&brk_op->state_lock); } else { - oplock_break(brk_op, + oplock_break(brk_op, ci, brk_op->is_lease && !is_trunc ? SMB2_OPLOCK_LEVEL_II : SMB2_OPLOCK_LEVEL_NONE, send_interim && !sent_interim ? work : NULL, - false); + false, false); } sent_interim = true; list_del(&ent->list); @@ -2138,6 +2269,82 @@ void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp) SIDUNIX_GROUP, (struct smb_sid *)&buf->SidBuffer[28]); } +/** + * create_aapl_rsp_buf() - build AAPL kAAPL_SERVER_QUERY response + * @cc: buffer to write the create context into (AAPL_RSP_MAX_SIZE bytes) + * @vol_caps: volume capability flags (SMB2_CRTCTX_AAPL_* volume bits) + * @req_bitmap: the client's request bitmap, echoed back in reply_bitmap + * + * Response format follows the layout observed from macOS's own smbd, and + * matches the client-side parsing in AAPL's published public client kernel + * source (public client behavior reference, kAAPL_SERVER_QUERY + * case): reply_bitmap, then server_caps/vol_caps/model-info fields present + * only when their reply_bitmap bit is set: + * reply_bitmap = req_bitmap masked to the fields we support + * server_caps = AAPL_SERVER_CAPS_KSMBD when requested + * vol_caps = caller-supplied + * model string = server_conf.aapl_model (default "Xserve") in UTF-16LE, + * when SMB2_CRTCTX_AAPL_MODEL_INFO requested + * + * Sending reply_bitmap with MODEL_INFO set but no model string causes + * smbfs.kext to enter a broken disconnect path requiring a macOS reboot. + */ +void create_aapl_rsp_buf(char *cc, __u64 vol_caps, __u64 req_bitmap) +{ + struct create_aapl_rsp *buf; + u64 reply_bitmap; + u32 data_len; + + buf = (struct create_aapl_rsp *)cc; + memset(buf, 0, AAPL_RSP_MAX_SIZE); + + reply_bitmap = req_bitmap & (SMB2_CRTCTX_AAPL_SERVER_CAPS | + SMB2_CRTCTX_AAPL_VOLUME_CAPS | + SMB2_CRTCTX_AAPL_MODEL_INFO); + + /* base data: cmd(4)+reserved(4)+reply_bitmap(8)+server_caps(8)+vol_caps(8) */ + data_len = 32; + if (reply_bitmap & SMB2_CRTCTX_AAPL_MODEL_INFO) + data_len += 4 + 4 + AAPL_MODEL_UTF16_BYTES; /* pad2+model_bytes+string */ + + buf->ccontext.DataOffset = cpu_to_le16(offsetof(struct create_aapl_rsp, cmd)); + buf->ccontext.DataLength = cpu_to_le32(data_len); + buf->ccontext.NameOffset = cpu_to_le16(offsetof(struct create_aapl_rsp, Name)); + buf->ccontext.NameLength = cpu_to_le16(SMB2_CREATE_AAPL_LEN); + buf->Name[0] = 'A'; + buf->Name[1] = 'A'; + buf->Name[2] = 'P'; + buf->Name[3] = 'L'; + + buf->cmd = cpu_to_le32(SMB2_CRTCTX_AAPL_SERVER_QUERY); + buf->reply_bitmap = cpu_to_le64(reply_bitmap); + buf->server_caps = (reply_bitmap & SMB2_CRTCTX_AAPL_SERVER_CAPS) ? + cpu_to_le64(AAPL_SERVER_CAPS_KSMBD) : 0; + buf->vol_caps = (reply_bitmap & SMB2_CRTCTX_AAPL_VOLUME_CAPS) ? + cpu_to_le64(vol_caps) : 0; + + if (reply_bitmap & SMB2_CRTCTX_AAPL_MODEL_INFO) { + __le32 *p = (__le32 *)((u8 *)buf + sizeof(*buf)); + __le16 *model_str = (__le16 *)(p + 2); + const char *src = server_conf.aapl_model[0] ? + server_conf.aapl_model : "Xserve"; + int i, model_bytes = 0; + + /* Convert ASCII model string to UTF-16LE in-place */ + for (i = 0; src[i] && i < AAPL_MODEL_MAX_CHARS; i++) { + model_str[i] = cpu_to_le16((unsigned char)src[i]); + model_bytes += 2; + } + + p[0] = 0; /* pad2 */ + p[1] = cpu_to_le32(model_bytes); + + /* Update DataLength to reflect actual model string size */ + buf->ccontext.DataLength = + cpu_to_le32(data_len - AAPL_MODEL_UTF16_BYTES + model_bytes); + } +} + /* * Find lease object(opinfo) for given lease key/fid from lease * break/file close path. diff --git a/fs/smb/server/oplock.h b/fs/smb/server/oplock.h index 3f581d22bb676..54785df84525e 100644 --- a/fs/smb/server/oplock.h +++ b/fs/smb/server/oplock.h @@ -66,6 +66,7 @@ struct oplock_info { struct ksmbd_file *o_fp; int level; int op_state; + spinlock_t state_lock; unsigned long pending_break; u64 fid; atomic_t breaking_cnt; @@ -96,7 +97,7 @@ struct oplock_break_info { int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, struct ksmbd_file *fp, __u16 tid, - struct lease_ctx_info *lctx, int share_ret); + struct lease_ctx_info *lctx, int share_ret, bool replay); void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, int is_trunc); void smb_break_all_levII_oplock_no_interim(struct ksmbd_work *work, @@ -125,6 +126,7 @@ void create_durable_v2_rsp_buf(char *cc, struct ksmbd_file *fp); void create_mxac_rsp_buf(char *cc, int maximal_access); void create_disk_id_rsp_buf(char *cc, __u64 file_id, __u64 vol_id); void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp); +void create_aapl_rsp_buf(char *cc, __u64 vol_caps, __u64 req_bitmap); struct create_context *smb2_find_context_vals(void *open_req, const char *tag, int tag_len); struct oplock_info *lookup_lease_in_table(struct ksmbd_conn *conn, char *lease_key); diff --git a/fs/smb/server/proc.c b/fs/smb/server/proc.c index 101a2cc45a447..19f0f2cfbf543 100644 --- a/fs/smb/server/proc.c +++ b/fs/smb/server/proc.c @@ -11,10 +11,12 @@ #include #include "misc.h" +#include "connection.h" #include "server.h" #include "stats.h" #include "smb_common.h" #include "smb2pdu.h" +#include "vfs_cache.h" static struct proc_dir_entry *ksmbd_proc_fs; struct ksmbd_counters ksmbd_counters; @@ -27,6 +29,42 @@ struct proc_dir_entry *ksmbd_proc_create(const char *name, show, v); } +void ksmbd_proc_show_flag_names(struct seq_file *m, + const struct ksmbd_const_name *table, + int count, unsigned int flags) +{ + unsigned int remaining = flags; + bool separator = false; + int i; + + for (i = 0; i < count; i++) { + unsigned int flag = table[i].const_value; + + if (!flag || (remaining & flag) != flag) + continue; + seq_printf(m, "%s%s", separator ? "," : "", table[i].name); + separator = true; + remaining &= ~flag; + } + + if (remaining) + seq_printf(m, "%s0x%08x", separator ? "," : "", remaining); + else if (!separator) + seq_puts(m, "none"); +} + +const char *ksmbd_proc_const_name(const struct ksmbd_const_name *table, + int count, unsigned int const_value) +{ + int i; + + for (i = 0; i < count; i++) { + if (table[i].const_value == const_value) + return table[i].name; + } + return NULL; +} + struct ksmbd_const_smb2_process_req { unsigned int const_value; const char *name; @@ -54,32 +92,129 @@ static const struct ksmbd_const_smb2_process_req smb2_process_req[KSMBD_COUNTER_ {le16_to_cpu(SMB2_OPLOCK_BREAK), "SMB2_OPLOCK_BREAK"}, }; +static const char *ksmbd_server_state_string(void) +{ + switch (READ_ONCE(server_conf.state)) { + case SERVER_STATE_STARTING_UP: + return "starting"; + case SERVER_STATE_RUNNING: + return "running"; + case SERVER_STATE_RESETTING: + return "resetting"; + case SERVER_STATE_SHUTTING_DOWN: + return "shutdown"; + default: + return "unknown"; + } +} + +static const char *ksmbd_signing_mode_string(void) +{ + switch (server_conf.signing) { + case KSMBD_CONFIG_OPT_DISABLED: + return "disabled"; + case KSMBD_CONFIG_OPT_MANDATORY: + return "mandatory"; + case KSMBD_CONFIG_OPT_AUTO: + return "auto"; + default: + return "unknown"; + } +} + +static void proc_show_runtime_totals(struct seq_file *m) +{ + struct ksmbd_conn *conn; + unsigned int clients = 0; + unsigned int open_files = 0; + int i; + + down_read(&conn_list_lock); + hash_for_each(conn_list, i, conn, hlist) { + clients++; + open_files += atomic_read(&conn->stats.open_files_count); + } + up_read(&conn_list_lock); + + seq_printf(m, "clients:\t%u\n", clients); + seq_printf(m, "open_files:\t%u\n", open_files); +} + static int proc_show_ksmbd_stats(struct seq_file *m, void *v) { int i; seq_puts(m, "Server\n"); - seq_printf(m, "name: %s\n", ksmbd_server_string()); - seq_printf(m, "netbios: %s\n", ksmbd_netbios_name()); - seq_printf(m, "work group: %s\n", ksmbd_work_group()); - seq_printf(m, "min protocol: %s\n", ksmbd_get_protocol_string(server_conf.min_protocol)); - seq_printf(m, "max protocol: %s\n", ksmbd_get_protocol_string(server_conf.max_protocol)); - seq_printf(m, "flags: 0x%08x\n", server_conf.flags); - seq_printf(m, "share_fake_fscaps: 0x%08x\n", + seq_printf(m, "state:\t%s\n", ksmbd_server_state_string()); + seq_printf(m, "name:\t%s\n", ksmbd_server_string()); + seq_printf(m, "netbios:\t%s\n", ksmbd_netbios_name()); + seq_printf(m, "work_group:\t%s\n", ksmbd_work_group()); + seq_printf(m, "min_protocol:\t%s\n", ksmbd_get_protocol_string(server_conf.min_protocol)); + seq_printf(m, "max_protocol:\t%s\n", ksmbd_get_protocol_string(server_conf.max_protocol)); + seq_printf(m, "flags:\t0x%08x\n", server_conf.flags); + seq_printf(m, "tcp_port:\t%u\n", server_conf.tcp_port); + seq_printf(m, "signing:\t%s\n", ksmbd_signing_mode_string()); + seq_printf(m, "signing_enforced:\t%s\n", + server_conf.enforced_signing ? "yes" : "no"); + seq_printf(m, "bind_interfaces_only:\t%s\n", + server_conf.bind_interfaces_only ? "yes" : "no"); + seq_printf(m, "max_connections:\t%u\n", server_conf.max_connections); + seq_printf(m, "max_connections_per_ip:\t%u\n", + server_conf.max_ip_connections); + seq_printf(m, "max_inflight_requests:\t%u\n", + server_conf.max_inflight_req); + seq_printf(m, "deadtime_seconds:\t%lu\n", server_conf.deadtime / HZ); + seq_printf(m, "ipc_timeout_seconds:\t%u\n", server_conf.ipc_timeout / HZ); + if (server_conf.ipc_last_active) + seq_printf(m, "ipc_last_active_seconds:\t%lu\n", + jiffies_to_msecs(jiffies - server_conf.ipc_last_active) / + MSEC_PER_SEC); + else + seq_puts(m, "ipc_last_active_seconds:\tnever\n"); + seq_printf(m, "durable_scavenger:\t%s\n", + ksmbd_durable_scavenger_active() ? "running" : "stopped"); + seq_printf(m, "share_fake_fscaps:\t0x%08x\n", server_conf.share_fake_fscaps); - seq_printf(m, "sessions: %lld\n", + proc_show_runtime_totals(m); + seq_printf(m, "sessions:\t%lld\n", ksmbd_counter_sum(KSMBD_COUNTER_SESSIONS)); - seq_printf(m, "tree connects: %lld\n", + seq_printf(m, "session_timeouts:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_SESSION_TIMEOUTS)); + seq_printf(m, "tree_connects:\t%lld\n", ksmbd_counter_sum(KSMBD_COUNTER_TREE_CONNS)); - seq_printf(m, "read bytes: %lld\n", + seq_printf(m, "requests:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_REQUESTS)); + seq_printf(m, "read_bytes:\t%lld\n", ksmbd_counter_sum(KSMBD_COUNTER_READ_BYTES)); - seq_printf(m, "written bytes: %lld\n", + seq_printf(m, "written_bytes:\t%lld\n", ksmbd_counter_sum(KSMBD_COUNTER_WRITE_BYTES)); seq_puts(m, "\nSMB2\n"); for (i = 0; i < KSMBD_COUNTER_MAX_REQS; i++) - seq_printf(m, "%-20s:\t%lld\n", smb2_process_req[i].name, + seq_printf(m, "%s:\t%lld\n", smb2_process_req[i].name, ksmbd_counter_sum(KSMBD_COUNTER_FIRST_REQ + i)); + + seq_puts(m, "\nSMB2 status\n"); + seq_printf(m, "success:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_STATUS_SUCCESS)); + seq_printf(m, "informational:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_STATUS_INFORMATIONAL)); + seq_printf(m, "warning:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_STATUS_WARNING)); + seq_printf(m, "error:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_STATUS_ERROR)); + seq_printf(m, "access_denied:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_ACCESS_DENIED)); + seq_printf(m, "not_found:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_NOT_FOUND)); + seq_printf(m, "invalid_parameter:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_INVALID_PARAMETER)); + seq_printf(m, "sharing_violation:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_SHARING_VIOLATION)); + seq_printf(m, "not_supported:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_NOT_SUPPORTED)); + seq_printf(m, "other:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_OTHER)); return 0; } @@ -106,14 +241,14 @@ void ksmbd_proc_reset(void) percpu_counter_set(&ksmbd_counters.counters[i], 0); } -void ksmbd_proc_init(void) +int ksmbd_proc_init(void) { int i; - int retval; + int retval = -ENOMEM; ksmbd_proc_fs = proc_mkdir("fs/ksmbd", NULL); if (!ksmbd_proc_fs) - return; + return retval; if (!proc_mkdir_mode("sessions", 0400, ksmbd_proc_fs)) goto err_out; @@ -124,11 +259,14 @@ void ksmbd_proc_init(void) goto err_out; } - if (!ksmbd_proc_create("server", proc_show_ksmbd_stats, NULL)) + if (!ksmbd_proc_create("server", proc_show_ksmbd_stats, NULL)) { + retval = -ENOMEM; goto err_out; + } ksmbd_proc_reset(); - return; + return 0; err_out: ksmbd_proc_cleanup(); + return retval; } diff --git a/fs/smb/server/server.c b/fs/smb/server/server.c index 960c4c897c11f..e113b23d96f94 100644 --- a/fs/smb/server/server.c +++ b/fs/smb/server/server.c @@ -15,6 +15,7 @@ #include "server.h" #include "smb_common.h" +#include "smb2pdu.h" #include "../common/smb2status.h" #include "connection.h" #include "transport_ipc.h" @@ -155,8 +156,11 @@ static int __process_request(struct ksmbd_work *work, struct ksmbd_conn *conn, } ret = cmds->proc(work); - if (conn->ops->inc_reqs) - conn->ops->inc_reqs(command); + if (conn->ops->inc_reqs) { + struct smb2_hdr *rsp = ksmbd_resp_buf_curr(work); + + conn->ops->inc_reqs(command, rsp->Status); + } if (ret < 0) ksmbd_debug(CONN, "Failed to process %u [%d]\n", command, ret); @@ -182,9 +186,33 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, if (conn->ops->is_transform_hdr && conn->ops->is_transform_hdr(work->request_buf)) { rc = conn->ops->decrypt_req(work); - if (rc < 0) + if (rc < 0) { + ksmbd_conn_abort(conn); return; + } work->encrypted = true; + + /* + * SMB3 applies compression before encryption. The receive loop + * handles a plain compression transform before allocating work, but + * an encrypted request exposes that transform only after decryption. + */ + if (((struct smb2_hdr *)smb_get_msg(work->request_buf))->ProtocolId == + SMB2_COMPRESSION_TRANSFORM_ID) { + rc = ksmbd_decompress_work_request(work); + if (rc < 0) { + ksmbd_conn_abort(conn); + return; + } + } + + /* The decrypted payload must now be a complete SMB2 request. */ + if (((struct smb2_hdr *)smb_get_msg(work->request_buf))->ProtocolId != + SMB2_PROTO_NUMBER || + get_rfc1002_len(work->request_buf) < sizeof(struct smb2_pdu)) { + ksmbd_conn_abort(conn); + return; + } } if (conn->ops->allocate_rsp_buf(work)) @@ -204,6 +232,9 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, if (rc == -EINVAL) conn->ops->set_rsp_status(work, STATUS_INVALID_PARAMETER); + else if (rc == -EKEYEXPIRED) + conn->ops->set_rsp_status(work, + STATUS_NETWORK_SESSION_EXPIRED); else conn->ops->set_rsp_status(work, STATUS_USER_SESSION_DELETED); @@ -211,7 +242,11 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, struct smb2_hdr *rsp_hdr; rsp_hdr = ksmbd_resp_buf_curr(work); - rsp_hdr->Flags |= SMB2_FLAGS_SIGNED; + if (rc == -EKEYEXPIRED && work->sess && + conn->ops->set_sign_rsp) + conn->ops->set_sign_rsp(work); + else + rsp_hdr->Flags |= SMB2_FLAGS_SIGNED; } goto send; } else if (rc > 0) { @@ -229,8 +264,10 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, } rc = __process_request(work, conn, &command); - if (rc == SERVER_HANDLER_ABORT) + if (rc == SERVER_HANDLER_ABORT) { + smb2_complete_request_open(work); break; + } /* * Call smb2_set_rsp_credits() function to set number of credits @@ -243,10 +280,13 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, if (rc < 0) { conn->ops->set_rsp_status(work, STATUS_INVALID_PARAMETER); + smb2_complete_request_open(work); goto send; } } + smb2_complete_request_open(work); + is_chained = is_chained_smb2_message(work); if (work->sess && @@ -262,6 +302,7 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, } while (is_chained == true); send: + smb2_complete_request_open(work); /* * Release any credit charge still outstanding for this request. On * the normal path smb2_set_rsp_credits() already returned it, but the @@ -362,7 +403,7 @@ static int ksmbd_server_process_request(struct ksmbd_conn *conn) static int ksmbd_server_terminate_conn(struct ksmbd_conn *conn) { - ksmbd_sessions_deregister(conn); + ksmbd_conn_sessions_cleanup(conn); destroy_lease_table(conn); return 0; } @@ -603,18 +644,25 @@ static int __init ksmbd_server_init(void) return ret; } - ksmbd_proc_init(); - create_proc_sessions(); + ret = ksmbd_proc_init(); + if (ret) + goto err_unregister; + + if (create_proc_sessions()) + pr_warn("Unable to create sessions procfs entry\n"); + + if (create_proc_shares()) + pr_warn("Unable to create shares procfs entry\n"); ksmbd_server_tcp_callbacks_init(); ret = server_conf_init(); if (ret) - goto err_unregister; + goto err_proc_cleanup; ret = ksmbd_work_pool_init(); if (ret) - goto err_unregister; + goto err_proc_cleanup; ret = ksmbd_init_file_cache(); if (ret) @@ -660,6 +708,8 @@ static int __init ksmbd_server_init(void) ksmbd_exit_file_cache(); err_destroy_work_pools: ksmbd_work_pool_destroy(); +err_proc_cleanup: + ksmbd_proc_cleanup(); err_unregister: class_unregister(&ksmbd_control_class); diff --git a/fs/smb/server/server.h b/fs/smb/server/server.h index b8a7317be86b4..4d4d268b59d5a 100644 --- a/fs/smb/server/server.h +++ b/fs/smb/server/server.h @@ -48,6 +48,8 @@ struct ksmbd_server_config { char *conf[SERVER_CONF_WORK_GROUP + 1]; struct task_struct *dh_task; bool bind_interfaces_only; + /* AAPL model string for Finder icon, e.g. "Xserve" */ + char aapl_model[32]; }; extern struct ksmbd_server_config server_conf; diff --git a/fs/smb/server/smb2ops.c b/fs/smb/server/smb2ops.c index c9a32ee096b58..8c1f5b5c98287 100644 --- a/fs/smb/server/smb2ops.c +++ b/fs/smb/server/smb2ops.c @@ -37,6 +37,7 @@ static struct smb_version_values smb21_server_values = { .create_mxac_size = sizeof(struct create_mxac_rsp), .create_disk_id_size = sizeof(struct create_disk_id_rsp), .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, }; static struct smb_version_values smb30_server_values = { @@ -64,6 +65,7 @@ static struct smb_version_values smb30_server_values = { .create_mxac_size = sizeof(struct create_mxac_rsp), .create_disk_id_size = sizeof(struct create_disk_id_rsp), .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, }; static struct smb_version_values smb302_server_values = { @@ -91,6 +93,7 @@ static struct smb_version_values smb302_server_values = { .create_mxac_size = sizeof(struct create_mxac_rsp), .create_disk_id_size = sizeof(struct create_disk_id_rsp), .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, }; static struct smb_version_values smb311_server_values = { @@ -118,6 +121,7 @@ static struct smb_version_values smb311_server_values = { .create_mxac_size = sizeof(struct create_mxac_rsp), .create_disk_id_size = sizeof(struct create_disk_id_rsp), .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, }; static struct smb_version_ops smb2_0_server_ops = { @@ -266,8 +270,10 @@ void init_smb3_02_server(struct ksmbd_conn *conn) if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_MULTI_CHANNEL; - if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE) - conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_PERSISTENT_HANDLES; + /* + * Durable handles are in-memory only. Do not advertise persistent + * handles until CA recovery and fencing are implemented. + */ } /** @@ -290,10 +296,7 @@ int init_smb3_11_server(struct ksmbd_conn *conn) if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_MULTI_CHANNEL; - if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE) - conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_PERSISTENT_HANDLES; - - INIT_LIST_HEAD(&conn->preauth_sess_table); + /* See init_smb3_02_server(): persistent handles require CA recovery. */ return 0; } diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index 76f63f9adc725..07de36aac0bff 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "glob.h" #include "../common/smbfsctl.h" @@ -55,6 +56,10 @@ static void __wbuf(struct ksmbd_work *work, void **req, void **rsp) } } +static struct ksmbd_work *smb2_notify_cancel_claim(void **argv); +static void smb2_notify_cancel_fn(void **argv); +static void smb2_complete_notify_cancel(struct ksmbd_work *in_work); + #define WORK_BUFFERS(w, rq, rs) __wbuf((w), (void **)&(rq), (void **)&(rs)) #define SMB2_CREATE_FILE_ATTRIBUTE_MASK \ @@ -64,28 +69,8 @@ static void __wbuf(struct ksmbd_work *work, void **req, void **rsp) /* Windows reports automatic write-time updates at roughly 15 ms resolution. */ #define KSMBD_WRITE_TIME_RESOLUTION (15ULL * 10000) -/** - * check_session_id() - check for valid session id in smb header - * @conn: connection instance - * @id: session id from smb header - * - * Return: 1 if valid session id, otherwise 0 - */ -static inline bool check_session_id(struct ksmbd_conn *conn, u64 id) -{ - struct ksmbd_session *sess; - - if (id == 0 || id == -1) - return false; - - sess = ksmbd_session_lookup_all(conn, id); - if (sess) { - ksmbd_user_session_put(sess); - return true; - } - pr_err("Invalid user session id: %llu\n", id); - return false; -} +/* MAXFILESIZE in [MS-FSA] 2.1.5.3 Server Requests a Write. */ +#define SMB2_MAX_FILE_SIZE 0xfffffff0000ULL struct channel *lookup_chann_list(struct ksmbd_session *sess, struct ksmbd_conn *conn) { @@ -98,8 +83,6 @@ struct channel *lookup_chann_list(struct ksmbd_session *sess, struct ksmbd_conn return chann; } -#define KSMBD_MAX_CHANNELS 32 - static int register_session_channel(struct ksmbd_session *sess, struct ksmbd_conn *conn, const char *sess_key) @@ -110,6 +93,11 @@ static int register_session_channel(struct ksmbd_session *sess, int rc = 0; down_write(&sess->chann_lock); + if (sess->tearing_down) { + rc = -ESHUTDOWN; + goto out; + } + if (xa_load(&sess->ksmbd_chann_list, (long)conn)) goto out; @@ -551,6 +539,8 @@ static void init_chained_smb2_rsp(struct ksmbd_work *work) */ rsp_hdr->Flags = (SMB2_FLAGS_SERVER_TO_REDIR | SMB2_FLAGS_RELATED_OPERATIONS); + if (rcv_hdr->Flags & SMB2_FLAGS_REPLAY_OPERATION) + rsp_hdr->Flags |= SMB2_FLAGS_REPLAY_OPERATION; rsp_hdr->NextCommand = 0; rsp_hdr->MessageId = rcv_hdr->MessageId; rsp_hdr->Id.SyncId.ProcessId = rcv_hdr->Id.SyncId.ProcessId; @@ -643,6 +633,8 @@ int init_smb2_rsp_hdr(struct ksmbd_work *work) * Message is response. We don't grant oplock yet. */ rsp_hdr->Flags = (SMB2_FLAGS_SERVER_TO_REDIR); + if (rcv_hdr->Flags & SMB2_FLAGS_REPLAY_OPERATION) + rsp_hdr->Flags |= SMB2_FLAGS_REPLAY_OPERATION; rsp_hdr->NextCommand = 0; rsp_hdr->MessageId = rcv_hdr->MessageId; rsp_hdr->Id.SyncId.ProcessId = rcv_hdr->Id.SyncId.ProcessId; @@ -653,6 +645,192 @@ int init_smb2_rsp_hdr(struct ksmbd_work *work) return 0; } +static __le16 smb3_hdr_channel_sequence(struct smb2_hdr *hdr) +{ + return ((struct smb3_hdr_req *)hdr)->ChannelSequence; +} + +static bool smb3_hdr_replay(struct smb2_hdr *hdr) +{ + return hdr->Flags & SMB2_FLAGS_REPLAY_OPERATION; +} + +static int smb3_verify_channel_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_hdr *hdr, + bool allow_stale) +{ + __le16 chseq_le; + u16 chseq, old_chseq; + int ret = 0; + + if (work->conn->dialect < SMB30_PROT_ID) + return 0; + + chseq_le = smb3_hdr_channel_sequence(hdr); + chseq = le16_to_cpu(chseq_le); + + spin_lock(&fp->f_lock); + old_chseq = le16_to_cpu(fp->channel_sequence); + if (smb3_hdr_replay(hdr)) { + if (chseq == old_chseq && fp->outstanding_pre_requests == 0) { + fp->outstanding_requests++; + } else if ((u16)(chseq - old_chseq) <= 0x7fff && + fp->outstanding_pre_requests == 0) { + fp->outstanding_pre_requests += fp->outstanding_requests; + fp->outstanding_requests = 1; + fp->channel_sequence = chseq_le; + } else if (allow_stale) { + fp->outstanding_pre_requests++; + } else { + ret = -EAGAIN; + } + } else { + if (chseq == old_chseq) { + fp->outstanding_requests++; + } else if ((u16)(chseq - old_chseq) <= 0x7fff) { + fp->outstanding_pre_requests += fp->outstanding_requests; + fp->outstanding_requests = 1; + fp->channel_sequence = chseq_le; + } else if (allow_stale) { + fp->outstanding_pre_requests++; + } else { + ret = -EAGAIN; + } + } + spin_unlock(&fp->f_lock); + + return ret; +} + +static void smb3_complete_channel_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + __le16 chseq_le) +{ + u16 chseq; + + if (work->conn->dialect < SMB30_PROT_ID) + return; + + chseq = le16_to_cpu(chseq_le); + + spin_lock(&fp->f_lock); + if (chseq == le16_to_cpu(fp->channel_sequence)) { + if (fp->outstanding_requests) + fp->outstanding_requests--; + } else { + if (fp->outstanding_pre_requests) + fp->outstanding_pre_requests--; + } + spin_unlock(&fp->f_lock); +} + +static int smb2_set_request_open(struct ksmbd_work *work, struct ksmbd_file *fp, + struct smb2_hdr *hdr, bool verify_chseq, + bool allow_stale_chseq) +{ + struct ksmbd_file *open; + int ret; + + smb2_complete_request_open(work); + + open = ksmbd_file_get(fp); + if (!open) + return -ESTALE; + + if (verify_chseq) { + ret = smb3_verify_channel_sequence(work, fp, hdr, + allow_stale_chseq); + if (ret) { + ksmbd_fd_put(work, open); + return ret; + } + work->request_open_chseq_tracked = true; + } + + work->request_open = open; + work->request_open_chseq = smb3_hdr_channel_sequence(hdr); + return 0; +} + +void smb2_complete_request_open(struct ksmbd_work *work) +{ + struct ksmbd_file *open = work->request_open; + + if (!open) + return; + + if (work->request_open_chseq_tracked) + smb3_complete_channel_sequence(work, open, + work->request_open_chseq); + + work->request_open = NULL; + work->request_open_chseq_tracked = false; + ksmbd_fd_put(work, open); +} + +static bool smb2_lock_sequence_applicable(struct ksmbd_work *work, + struct ksmbd_file *fp) +{ + return fp->is_resilient || fp->is_durable || fp->is_persistent || + (work->conn->dialect >= SMB30_PROT_ID && + (work->conn->vals->req_capabilities & + SMB2_GLOBAL_CAP_MULTI_CHANNEL)); +} + +static bool smb2_verify_lock_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_lock_req *req) +{ + u32 val, index; + u8 sequence; + bool replay = false; + + if (work->conn->dialect == SMB20_PROT_ID || + !smb2_lock_sequence_applicable(work, fp)) + return false; + + val = le32_to_cpu(req->LockSequenceNumber); + sequence = val & 0xf; + index = val >> 4; + if (!index || index > KSMBD_LOCK_SEQ_ARRAY_SIZE) + return false; + + spin_lock(&fp->f_lock); + if (fp->lock_seq[index - 1].valid) { + if (fp->lock_seq[index - 1].sequence == sequence) + replay = true; + else + fp->lock_seq[index - 1].valid = false; + } + spin_unlock(&fp->f_lock); + + return replay; +} + +static void smb2_update_lock_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_lock_req *req) +{ + u32 val, index; + u8 sequence; + + if (work->conn->dialect == SMB20_PROT_ID || + !smb2_lock_sequence_applicable(work, fp)) + return; + + val = le32_to_cpu(req->LockSequenceNumber); + sequence = val & 0xf; + index = val >> 4; + if (!index || index > KSMBD_LOCK_SEQ_ARRAY_SIZE) + return; + + spin_lock(&fp->f_lock); + fp->lock_seq[index - 1].valid = true; + fp->lock_seq[index - 1].sequence = sequence; + spin_unlock(&fp->f_lock); +} + /** * smb2_allocate_rsp_buf() - allocate smb2 response buffer * @work: smb work containing smb request buffer @@ -697,6 +875,53 @@ int smb2_allocate_rsp_buf(struct ksmbd_work *work) return 0; } +static bool smb2_session_expired_cmd_allowed(struct ksmbd_work *work, + unsigned int cmd) +{ + struct smb2_lock_req *req; + unsigned int len, lock_count, i; + + if (cmd == SMB2_CANCEL_HE || cmd == SMB2_CLOSE_HE || + cmd == SMB2_LOGOFF_HE) + return true; + if (cmd != SMB2_LOCK_HE) + return false; + + req = ksmbd_req_buf_next(work); + if (req->hdr.NextCommand) + len = le32_to_cpu(req->hdr.NextCommand); + else { + len = get_rfc1002_len(work->request_buf); + if (len < work->next_smb2_rcv_hdr_off) + return false; + len -= work->next_smb2_rcv_hdr_off; + } + + lock_count = le16_to_cpu(req->LockCount); + if (!lock_count || len < offsetof(struct smb2_lock_req, locks) || + lock_count > (len - offsetof(struct smb2_lock_req, locks)) / + sizeof(struct smb2_lock_element)) + return false; + + for (i = 0; i < lock_count; i++) { + if (le32_to_cpu(req->locks[i].Flags) != SMB2_LOCKFLAG_UNLOCK) + return false; + } + return true; +} + +static bool smb2_session_kerberos_expired(struct ksmbd_session *sess) +{ + if (!sess->kerberos_expiry || + ktime_get_real_seconds() < sess->kerberos_expiry) + return false; + + if (cmpxchg(&sess->state, SMB2_SESSION_VALID, + SMB2_SESSION_EXPIRED) == SMB2_SESSION_VALID) + ksmbd_counter_inc(KSMBD_COUNTER_SESSION_TIMEOUTS); + return true; +} + /** * smb2_check_user_session() - check for valid session for a user * @work: smb work containing smb request buffer @@ -711,19 +936,36 @@ int smb2_check_user_session(struct ksmbd_work *work) unsigned long long sess_id; /* - * SMB2_ECHO, SMB2_NEGOTIATE, SMB2_SESSION_SETUP command do not - * require a session id, so no need to validate user session's for - * these commands. + * SMB2_NEGOTIATE and SMB2_SESSION_SETUP do not require a session id. + * SMB2_ECHO may omit it, but an echo carrying a session id still needs + * the session attached to work so that its signature can be checked and + * the response can be signed, including after Kerberos expiry. */ - if (cmd == SMB2_ECHO_HE || cmd == SMB2_NEGOTIATE_HE || - cmd == SMB2_SESSION_SETUP_HE) + if (cmd == SMB2_NEGOTIATE_HE || cmd == SMB2_SESSION_SETUP_HE) + return 0; + + sess_id = le64_to_cpu(req_hdr->SessionId); + if (cmd == SMB2_ECHO_HE) { + /* + * ECHO remains valid without a live session, including after + * LOGOFF. Attach an existing session only to authenticate a signed + * ECHO and sign its response; a stale SessionId is not an error. + */ + if (!work->next_smb2_rcv_hdr_off && sess_id) + work->sess = ksmbd_session_lookup_all_states(conn, sess_id); + if (work->sess) { + if (!smb2_session_kerberos_expired(work->sess) && + work->sess->state != SMB2_SESSION_VALID) { + ksmbd_user_session_put(work->sess); + work->sess = NULL; + } + } return 0; + } if (!ksmbd_conn_good(conn)) return -EIO; - sess_id = le64_to_cpu(req_hdr->SessionId); - /* * If request is not the first in Compound request, * Just validate session id in header with work->sess->id. @@ -738,18 +980,41 @@ int smb2_check_user_session(struct ksmbd_work *work) sess_id, work->sess->id); return -EINVAL; } + smb2_session_kerberos_expired(work->sess); if (work->sess->state != SMB2_SESSION_VALID) { pr_err("compound request on a non-valid session (state %d)\n", work->sess->state); - return -EINVAL; + if (smb2_session_kerberos_expired(work->sess) && + smb2_session_expired_cmd_allowed(work, cmd)) + return 1; + return smb2_session_kerberos_expired(work->sess) ? + -EKEYEXPIRED : -EINVAL; } return 1; } /* Check for validity of user session */ - work->sess = ksmbd_session_lookup_all(conn, sess_id); - if (work->sess) + work->sess = ksmbd_session_lookup_all_states(conn, sess_id); + if (work->sess) { + if (smb2_session_kerberos_expired(work->sess)) { + return smb2_session_expired_cmd_allowed(work, cmd) ? + 1 : -EKEYEXPIRED; + } + if (work->sess->state != SMB2_SESSION_VALID) { + /* + * Keep the reference for an encrypted request so the caller can + * return STATUS_USER_SESSION_DELETED encrypted with the old key. + */ + if (work->encrypted && + work->sess->state == SMB2_SESSION_EXPIRED && + work->sess->enc) + return -ENOENT; + ksmbd_user_session_put(work->sess); + work->sess = NULL; + return -ENOENT; + } return 1; + } ksmbd_debug(SMB, "Invalid user session, Uid %llu\n", sess_id); return -ENOENT; } @@ -1393,7 +1658,7 @@ int smb2_handle_negotiate(struct ksmbd_work *work) KSMBD_DEFAULT_GFP); if (!conn->preauth_info) { rc = -ENOMEM; - rsp->hdr.Status = STATUS_INVALID_PARAMETER; + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; goto err_out; } @@ -1490,7 +1755,7 @@ int smb2_handle_negotiate(struct ksmbd_work *work) ksmbd_conn_set_need_setup(conn); err_out: - if (rc) + if (rc && rsp->hdr.Status == STATUS_SUCCESS) rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; if (!rc) @@ -1820,7 +2085,9 @@ static int krb5_authenticate(struct ksmbd_work *work, struct ksmbd_session *sess = work->sess; char *in_blob, *out_blob; char channel_key[CIFS_KEY_SIZE] = {}; - char *auth_key = conn->binding ? channel_key : sess->sess_key; + char reauth_key[CIFS_KEY_SIZE] = {}; + char *auth_key = conn->binding ? channel_key : + (work->session_setup_reauth ? reauth_key : sess->sess_key); u64 prev_sess_id; bool binding = conn->binding; int in_len, out_len; @@ -1839,7 +2106,7 @@ static int krb5_authenticate(struct ksmbd_work *work, if (retval) { ksmbd_debug(SMB, "krb5 authentication failed\n"); if (retval != -EKEYREJECTED) - retval = -EINVAL; + retval = -EPERM; goto out; } @@ -1855,12 +2122,21 @@ static int krb5_authenticate(struct ksmbd_work *work, * that it is reauthentication. And the user/password * has been verified, so return it here. */ - if (sess->state == SMB2_SESSION_VALID) { + if (sess->state == SMB2_SESSION_VALID && !work->session_setup_reauth) { if (conn->binding) goto binding_session; return 0; } + /* + * Reauthentication verifies the new Kerberos credentials but keeps + * the established SMB session keys. + */ + if (work->session_setup_reauth) { + retval = 0; + goto out; + } + if ((rsp->SessionFlags != SMB2_SESSION_FLAG_IS_GUEST_LE && (conn->sign || server_conf.enforced_signing)) || (req->SecurityMode & SMB2_NEGOTIATE_SIGNING_REQUIRED)) @@ -1898,6 +2174,7 @@ static int krb5_authenticate(struct ksmbd_work *work, } retval = 0; out: + memzero_explicit(reauth_key, sizeof(reauth_key)); if (binding) memzero_explicit(channel_key, sizeof(channel_key)); return retval; @@ -1916,7 +2193,7 @@ int smb2_sess_setup(struct ksmbd_work *work) struct ksmbd_conn *conn = work->conn; struct smb2_sess_setup_req *req; struct smb2_sess_setup_rsp *rsp; - struct ksmbd_session *sess; + struct ksmbd_session *sess = NULL; struct negotiate_message *negblob; unsigned int negblob_len, negblob_off; int rc = 0; @@ -2041,8 +2318,13 @@ int smb2_sess_setup(struct ksmbd_work *work) } if (sess->state == SMB2_SESSION_EXPIRED) { - rc = -EFAULT; - goto out_err; + if (sess->kerberos_expiry && + ktime_get_real_seconds() >= sess->kerberos_expiry) { + work->session_setup_reauth = true; + } else { + rc = -EFAULT; + goto out_err; + } } if (ksmbd_conn_need_reconnect(conn)) { @@ -2052,6 +2334,9 @@ int smb2_sess_setup(struct ksmbd_work *work) goto out_err; } + if (work->session_setup_reauth) + WRITE_ONCE(sess->state, SMB2_SESSION_IN_PROGRESS); + conn->binding = false; } work->sess = sess; @@ -2086,10 +2371,8 @@ int smb2_sess_setup(struct ksmbd_work *work) if (conn->preferred_auth_mech & (KSMBD_AUTH_KRB5 | KSMBD_AUTH_MSKRB5)) { rc = krb5_authenticate(work, req, rsp); - if (rc) { - rc = -EINVAL; + if (rc) goto out_err; - } if (!ksmbd_conn_need_reconnect(conn)) { ksmbd_conn_set_good(conn); @@ -2165,6 +2448,14 @@ int smb2_sess_setup(struct ksmbd_work *work) } if (rc < 0) { + bool setup_in_progress = sess && + READ_ONCE(sess->state) == SMB2_SESSION_IN_PROGRESS && + !(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING); + + /* Authentication errors must not leave the new session published. */ + if (setup_in_progress) + ksmbd_session_unregister(conn, sess); + if (sess && conn->dialect == SMB311_PROT_ID && (req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { struct preauth_session *preauth_sess; @@ -2198,8 +2489,10 @@ int smb2_sess_setup(struct ksmbd_work *work) * For binding requests, session belongs to another * connection. Do not expire it. */ - if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { + if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING) && + !setup_in_progress) { sess->last_active = jiffies; + sess->kerberos_expiry = 0; sess->state = SMB2_SESSION_EXPIRED; } /* @@ -2251,6 +2544,7 @@ int smb2_tree_connect(struct ksmbd_work *work) struct ksmbd_session *sess = work->sess; char *treename = NULL, *name = NULL; struct ksmbd_tree_conn_status status; + struct ksmbd_tree_connect *tree_conn = NULL; struct ksmbd_share_config *share = NULL; int rc = -EINVAL; @@ -2277,9 +2571,10 @@ int smb2_tree_connect(struct ksmbd_work *work) name, treename); status = ksmbd_tree_conn_connect(work, name); - if (status.ret == KSMBD_TREE_CONN_STATUS_OK) + if (status.ret == KSMBD_TREE_CONN_STATUS_OK) { + tree_conn = status.tree_conn; rsp->hdr.Id.SyncId.TreeId = cpu_to_le32(status.tree_conn->id); - else + } else goto out_err1; share = status.tree_conn->share_conf; @@ -2311,16 +2606,22 @@ int smb2_tree_connect(struct ksmbd_work *work) status.tree_conn->posix_extensions = true; down_write(&sess->tree_conns_lock); - status.tree_conn->t_state = TREE_CONNECTED; + if (status.tree_conn->t_state == TREE_DISCONNECTED) { + status.ret = KSMBD_TREE_CONN_STATUS_ERROR; + share = NULL; + } else { + status.tree_conn->t_state = TREE_CONNECTED; + } up_write(&sess->tree_conns_lock); + if (status.ret != KSMBD_TREE_CONN_STATUS_OK) + goto out_err1; rsp->StructureSize = cpu_to_le16(16); out_err1: - if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE && share && - test_share_config_flag(share, - KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY)) - rsp->Capabilities = SMB2_SHARE_CAP_CONTINUOUS_AVAILABILITY; - else - rsp->Capabilities = 0; + /* + * A configured CA share is not continuously available until persistent + * open recovery, ownership fencing, and failover are implemented. + */ + rsp->Capabilities = 0; rsp->Reserved = 0; /* default manual caching */ rsp->ShareFlags = SMB2_SHAREFLAG_MANUAL_CACHING; @@ -2328,10 +2629,19 @@ int smb2_tree_connect(struct ksmbd_work *work) if (conn->dialect == SMB311_PROT_ID && conn->compress_algorithm != SMB3_COMPRESS_NONE) rsp->ShareFlags |= cpu_to_le32(SMB2_SHAREFLAG_COMPRESS_DATA); + if (share && test_share_config_flag(share, + KSMBD_SHARE_FLAG_HIDE_UNREADABLE)) + rsp->ShareFlags |= + cpu_to_le32(SMB2_SHAREFLAG_ACCESS_BASED_DIRECTORY_ENUM); rc = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_tree_connect_rsp)); - if (rc) + if (rc) { + if (status.ret == KSMBD_TREE_CONN_STATUS_OK) { + ksmbd_tree_conn_disconnect(sess, status.tree_conn); + status.tree_conn = NULL; + } status.ret = KSMBD_TREE_CONN_STATUS_NOMEM; + } if (!IS_ERR(treename)) kfree(treename); @@ -2367,6 +2677,9 @@ int smb2_tree_connect(struct ksmbd_work *work) if (status.ret != KSMBD_TREE_CONN_STATUS_OK) smb2_set_err_rsp(work); + if (tree_conn) + ksmbd_tree_connect_put(tree_conn); + return rc; } @@ -2470,17 +2783,6 @@ int smb2_tree_disconnect(struct ksmbd_work *work) ksmbd_close_tree_conn_fds(work); - down_write(&sess->tree_conns_lock); - if (tcon->t_state == TREE_DISCONNECTED) { - up_write(&sess->tree_conns_lock); - rsp->hdr.Status = STATUS_NETWORK_NAME_DELETED; - err = -ENOENT; - goto err_out; - } - - tcon->t_state = TREE_DISCONNECTED; - up_write(&sess->tree_conns_lock); - err = ksmbd_tree_conn_disconnect(sess, tcon); if (err) { rsp->hdr.Status = STATUS_NETWORK_NAME_DELETED; @@ -2515,7 +2817,6 @@ int smb2_session_logoff(struct ksmbd_work *work) struct ksmbd_session *sess = work->sess; struct smb2_logoff_req *req; struct smb2_logoff_rsp *rsp; - u64 sess_id; int err; WORK_BUFFERS(work, req, rsp); @@ -2529,25 +2830,52 @@ int smb2_session_logoff(struct ksmbd_work *work) smb2_set_err_rsp(work); return -ENOENT; } - sess_id = le64_to_cpu(req->hdr.SessionId); - ksmbd_all_conn_set_status(sess_id, KSMBD_SESS_NEED_RECONNECT); + + down_write(&sess->chann_lock); + if (sess->tearing_down) { + up_write(&sess->chann_lock); + ksmbd_conn_unlock(conn); + rsp->hdr.Status = STATUS_USER_SESSION_DELETED; + smb2_set_err_rsp(work); + return -ENOENT; + } + sess->tearing_down = true; + up_write(&sess->chann_lock); + + ksmbd_all_conn_set_status(sess, KSMBD_SESS_NEED_RECONNECT); ksmbd_conn_unlock(conn); + err = ksmbd_conn_wait_idle_sess(conn, sess); + if (err) { + down_write(&sess->chann_lock); + sess->tearing_down = false; + up_write(&sess->chann_lock); + ksmbd_all_conn_set_status(sess, KSMBD_SESS_GOOD); + rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; + smb2_set_err_rsp(work); + return err; + } + ksmbd_close_session_fds(work); - ksmbd_conn_wait_idle(conn); if (ksmbd_tree_conn_session_logoff(sess)) { ksmbd_debug(SMB, "Invalid tid %d\n", req->hdr.Id.SyncId.TreeId); rsp->hdr.Status = STATUS_NETWORK_NAME_DELETED; smb2_set_err_rsp(work); - return -ENOENT; + err = -ENOENT; + } else { + err = 0; } down_write(&conn->session_lock); + sess->kerberos_expiry = 0; sess->state = SMB2_SESSION_EXPIRED; up_write(&conn->session_lock); - ksmbd_all_conn_set_status(sess_id, KSMBD_SESS_NEED_SETUP); + ksmbd_all_conn_set_status(sess, KSMBD_SESS_NEED_SETUP); + + if (err) + return err; rsp->StructureSize = cpu_to_le16(4); err = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_logoff_rsp)); @@ -2974,12 +3302,86 @@ struct durable_info { unsigned short int type; bool persistent; bool reconnected; + bool replay; + bool replay_consumed; bool app_instance_id; unsigned int timeout; char *CreateGuid; char AppInstanceId[SMB2_CREATE_GUID_SIZE]; }; +static int smb2_check_durable_replay(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct lease_ctx_info *lc, + bool persistent) +{ + struct oplock_info *opinfo; + int ret = 0; + + if (!fp->is_durable && !fp->is_persistent) + return -EACCES; + + if (ksmbd_vfs_compare_durable_owner(fp, work->sess->user) == false) + return -EACCES; + + if (fp->is_persistent && !persistent) + return -EINVAL; + + opinfo = opinfo_get(fp); + if (!opinfo) + return 0; + + if (opinfo->sess && opinfo->sess->id != work->sess->id) { + ret = -ENOEXEC; + goto out; + } + + if (opinfo->is_lease) { + if (!lc || + memcmp(opinfo->o_lease->lease_key, lc->lease_key, + SMB2_LEASE_KEY_SIZE)) { + ret = -EACCES; + goto out; + } + } else { + if (lc) { + ret = -EACCES; + goto out; + } + + if (fp->is_durable && opinfo->level != SMB2_OPLOCK_LEVEL_BATCH) + ret = -EACCES; + } +out: + opinfo_put(opinfo); + return ret; +} + +static bool smb2_durable_replay_consumed(struct ksmbd_file *fp) +{ + bool consumed; + + spin_lock(&fp->f_lock); + consumed = fp->durable_replay_consumed; + spin_unlock(&fp->f_lock); + + return consumed; +} + +static void smb2_mark_durable_replay_consumed(struct ksmbd_file *fp) +{ + spin_lock(&fp->f_lock); + fp->durable_replay_consumed = true; + spin_unlock(&fp->f_lock); +} + +static bool smb2_durable_replay_differs(struct ksmbd_file *fp, + struct smb2_create_req *req) +{ + return fp->cdoption != req->CreateDisposition || + fp->create_file_attributes != req->FileAttributes; +} + static int parse_durable_handle_context(struct ksmbd_work *work, struct smb2_create_req *req, struct lease_ctx_info *lc, @@ -3107,6 +3509,10 @@ static int parse_durable_handle_context(struct ksmbd_work *work, durable_v2_blob = (struct create_durable_req_v2 *)context; ksmbd_debug(SMB, "Request for durable v2 open\n"); + dh_info->CreateGuid = durable_v2_blob->dcontext.CreateGuid; + dh_info->persistent = + le32_to_cpu(durable_v2_blob->dcontext.Flags) & + SMB2_DHANDLE_FLAG_PERSISTENT; dh_info->fp = ksmbd_lookup_fd_cguid(durable_v2_blob->dcontext.CreateGuid); if (dh_info->fp) { if (!memcmp(conn->ClientGUID, dh_info->fp->client_guid, @@ -3117,12 +3523,73 @@ static int parse_durable_handle_context(struct ksmbd_work *work, goto out; } - if (dh_info->fp->conn) { + if (dh_info->fp->f_state == FP_NEW) { + /* Original CREATE is still pending. */ ksmbd_put_durable_fd(dh_info->fp); - err = -EBADF; + err = -EAGAIN; goto out; } - dh_info->reconnected = true; + + if (!dh_info->fp->is_durable && + !dh_info->fp->is_persistent) { + /* + * A DurableHandleReqV2 CREATE can complete + * without granting durability (for example, if + * it requested no oplock). Its CreateGuid still + * identifies a completed CREATE for replay. + */ + if (dh_info->fp->conn && + ksmbd_vfs_compare_durable_owner( + dh_info->fp, work->sess->user)) { + if (smb2_durable_replay_consumed( + dh_info->fp)) { + ksmbd_put_durable_fd(dh_info->fp); + dh_info->fp = NULL; + dh_info->type = dh_idx; + dh_info->replay_consumed = true; + break; + } + if (smb2_durable_replay_differs( + dh_info->fp, req)) + smb2_mark_durable_replay_consumed( + dh_info->fp); + dh_info->replay = true; + dh_info->type = dh_idx; + goto out; + } + ksmbd_put_durable_fd(dh_info->fp); + err = -EACCES; + goto out; + } + + if (dh_info->fp->conn && + smb2_durable_replay_consumed(dh_info->fp)) { + ksmbd_put_durable_fd(dh_info->fp); + dh_info->fp = NULL; + dh_info->type = dh_idx; + dh_info->replay_consumed = true; + break; + } + + err = smb2_check_durable_replay(work, + dh_info->fp, + lc, + dh_info->persistent); + if (err) { + ksmbd_put_durable_fd(dh_info->fp); + goto out; + } + + if (dh_info->fp->conn) { + if (smb2_durable_replay_differs(dh_info->fp, + req)) + smb2_mark_durable_replay_consumed( + dh_info->fp); + dh_info->replay = true; + } else { + dh_info->reconnected = true; + } + dh_info->type = dh_idx; goto out; } ksmbd_put_durable_fd(dh_info->fp); @@ -3131,10 +3598,6 @@ static int parse_durable_handle_context(struct ksmbd_work *work, if ((lc && (lc->req_state & SMB2_LEASE_HANDLE_CACHING_LE)) || req_op_level == SMB2_OPLOCK_LEVEL_BATCH) { - dh_info->CreateGuid = - durable_v2_blob->dcontext.CreateGuid; - dh_info->persistent = - le32_to_cpu(durable_v2_blob->dcontext.Flags); dh_info->timeout = le32_to_cpu(durable_v2_blob->dcontext.Timeout); dh_info->type = dh_idx; @@ -3216,6 +3679,9 @@ int smb2_open(struct ksmbd_work *work) int rc = 0; int contxt_cnt = 0, query_disk_id = 0; bool maximal_access_ctxt = false, posix_ctxt = false; + bool aapl_ctxt = false; + bool durable_rsp = true; + __u64 aapl_req_bitmap = 0, aapl_client_caps = 0; int s_type = 0; int next_off = 0; char *name = NULL; @@ -3333,6 +3799,21 @@ int smb2_open(struct ksmbd_work *work) if (rc) goto err_out2; + if (dh_info.replay == true) { + fp = dh_info.fp; + if (ksmbd_override_fsids(work)) { + rc = -ENOMEM; + goto err_out2; + } + + file_info = FILE_OPENED; + rc = ksmbd_vfs_getattr(&fp->filp->f_path, &stat); + if (rc) + goto err_out2; + + goto reconnected_fp; + } + if (dh_info.reconnected == true) { rc = smb2_check_durable_oplock(conn, share, dh_info.fp, lc, sess->user, name); @@ -3601,7 +4082,8 @@ int smb2_open(struct ksmbd_work *work) if (file_present && !(req->CreateOptions & FILE_DELETE_ON_CLOSE_LE)) { rc = smb_check_perm_dacl(conn, &path, &daccess, - sess->user->uid); + req->DesiredAccess, + sess->user->uid, false); if (rc) goto err_out; } @@ -3731,7 +4213,23 @@ int smb2_open(struct ksmbd_work *work) goto err_out; } + /* + * Publish the client and create GUID before an oplock/lease break can + * make this CREATE pending. A replay of that in-flight CREATE must find + * this FP_NEW handle and fail with STATUS_FILE_NOT_AVAILABLE instead of + * waiting on the same break again. + */ + memcpy(fp->client_guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); + if (dh_info.CreateGuid) { + memcpy(fp->create_guid, dh_info.CreateGuid, SMB2_CREATE_GUID_SIZE); + fp->durable_replay_consumed = dh_info.replay_consumed; + rc = ksmbd_vfs_set_durable_owner(fp, sess->user); + if (rc) + goto err_out; + } + fp->cdoption = req->CreateDisposition; + fp->create_file_attributes = req->FileAttributes; fp->daccess = daccess; fp->saccess = req->ShareAccess; fp->coption = req->CreateOptions; @@ -3899,7 +4397,8 @@ int smb2_open(struct ksmbd_work *work) rc = smb_grant_oplock(work, req_op_level, fp->persistent_id, fp, le32_to_cpu(req->hdr.Id.SyncId.TreeId), - lc, share_ret); + lc, share_ret, + smb3_hdr_replay(&req->hdr)); if (rc < 0) goto err_out1; } @@ -3954,7 +4453,32 @@ int smb2_open(struct ksmbd_work *work) query_disk_id = 1; } - if (conn->is_aapl == false) { + if (test_share_config_flag(share, KSMBD_SHARE_FLAG_TIME_MACHINE)) { + context = smb2_find_context_vals(req, SMB2_CREATE_AAPL, 4); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out1; + } else if (context) { + struct aapl_server_query_req *aapl_req; + + if (le32_to_cpu(context->DataLength) < + sizeof(struct aapl_server_query_req)) { + rc = -EINVAL; + goto err_out1; + } + + aapl_req = (struct aapl_server_query_req *) + ((char *)context + + le16_to_cpu(context->DataOffset)); + if (le32_to_cpu(aapl_req->cmd) == + SMB2_CRTCTX_AAPL_SERVER_QUERY) { + conn->is_aapl = true; + aapl_ctxt = true; + aapl_req_bitmap = le64_to_cpu(aapl_req->req_bitmap); + aapl_client_caps = le64_to_cpu(aapl_req->client_caps); + } + } + } else if (conn->is_aapl == false) { context = smb2_find_context_vals(req, SMB2_CREATE_AAPL, 4); if (IS_ERR(context)) { rc = PTR_ERR(context); @@ -3987,7 +4511,7 @@ int smb2_open(struct ksmbd_work *work) if (created) smb2_new_xattrs(tcon, &path, fp); - memcpy(fp->client_guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); + fp->create_action = cpu_to_le32(file_info); if (dh_info.type == DURABLE_REQ_V2 || dh_info.type == DURABLE_REQ) { if (dh_info.type == DURABLE_REQ_V2 && dh_info.persistent && @@ -3996,10 +4520,7 @@ int smb2_open(struct ksmbd_work *work) fp->is_persistent = true; else fp->is_durable = true; - if (dh_info.type == DURABLE_REQ_V2) { - memcpy(fp->create_guid, dh_info.CreateGuid, - SMB2_CREATE_GUID_SIZE); if (dh_info.app_instance_id) memcpy(fp->app_instance_id, dh_info.AppInstanceId, @@ -4009,14 +4530,33 @@ int smb2_open(struct ksmbd_work *work) min_t(unsigned int, dh_info.timeout, DURABLE_HANDLE_MAX_TIMEOUT); else - fp->durable_timeout = 60; + fp->durable_timeout = 60000; } } -reconnected_fp: + /* + * conn->is_aapl detection above (this function's create-context + * parsing) is skipped on the reconnect path below, since a + * reconnect always arrives on a fresh connection -- if the client + * cares, it sends its own AAPL context on this same CREATE, which + * this function's normal (non-reconnect) parsing already handles. + */ + reconnected_fp: + if (dh_info.replay) + file_info = le32_to_cpu(fp->create_action); rsp->StructureSize = cpu_to_le16(89); opinfo = opinfo_get(fp); rsp->OplockLevel = opinfo != NULL ? opinfo->level : 0; + /* + * A durable CREATE replay does not modify the existing open. When + * replayed without an oplock, however, its response reflects that + * request and cannot include a new durable-handle response context. + */ + if (dh_info.replay && !lc && + req_op_level == SMB2_OPLOCK_LEVEL_NONE) { + rsp->OplockLevel = SMB2_OPLOCK_LEVEL_NONE; + durable_rsp = false; + } rsp->Flags = 0; rsp->CreateAction = cpu_to_le32(file_info); rsp->CreationTime = cpu_to_le64(fp->create_time); @@ -4110,7 +4650,8 @@ int smb2_open(struct ksmbd_work *work) next_off = conn->vals->create_disk_id_size; } - if (dh_info.type == DURABLE_REQ || dh_info.type == DURABLE_REQ_V2) { + if (durable_rsp && + (dh_info.type == DURABLE_REQ || dh_info.type == DURABLE_REQ_V2)) { struct create_context *durable_ccontext; durable_ccontext = (struct create_context *)(rsp->Buffer + @@ -4138,6 +4679,10 @@ int smb2_open(struct ksmbd_work *work) } if (posix_ctxt) { + struct create_context *posix_ccontext; + + posix_ccontext = (struct create_context *)(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength)); contxt_cnt++; create_posix_rsp_buf(rsp->Buffer + le32_to_cpu(rsp->CreateContextsLength), @@ -4147,6 +4692,29 @@ int smb2_open(struct ksmbd_work *work) iov_len += conn->vals->create_posix_size; if (next_ptr) *next_ptr = cpu_to_le32(next_off); + next_ptr = &posix_ccontext->Next; + next_off = conn->vals->create_posix_size; + } + + /* + * AAPL create context response: see smb2pdu.h for the capability + * rationale. Scoped to TIME_MACHINE shares only. + */ + if (aapl_ctxt) { + if (aapl_client_caps & SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR) + conn->aapl_readdir_attr = true; + + contxt_cnt++; + create_aapl_rsp_buf(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength), + SMB2_CRTCTX_AAPL_FULL_SYNC, + aapl_req_bitmap); + le32_add_cpu(&rsp->CreateContextsLength, + conn->vals->create_aapl_size); + iov_len += conn->vals->create_aapl_size; + if (next_ptr) + *next_ptr = cpu_to_le32(next_off); + /* AAPL is last; next_ptr need not be updated */ } if (contxt_cnt > 0) { @@ -4163,8 +4731,11 @@ int smb2_open(struct ksmbd_work *work) err_out2: if (!rc) { - rc = ksmbd_update_fstate(&work->sess->file_table, fp, - FP_INITED); + if (!dh_info.replay) + rc = ksmbd_update_fstate(&work->sess->file_table, fp, + FP_INITED); + if (!rc) + rc = smb2_set_request_open(work, fp, &req->hdr, false, false); if (!rc) rc = ksmbd_iov_pin_rsp(work, (void *)rsp, iov_len); } @@ -4173,8 +4744,12 @@ int smb2_open(struct ksmbd_work *work) rsp->hdr.Status = STATUS_INVALID_PARAMETER; else if (rc == -EOPNOTSUPP) rsp->hdr.Status = STATUS_NOT_SUPPORTED; - else if (rc == -EACCES || rc == -ESTALE || rc == -EXDEV) - rsp->hdr.Status = STATUS_ACCESS_DENIED; + else if (rc == -EACCES || rc == -ESTALE || rc == -EXDEV) { + if (req->DesiredAccess & FILE_ACCESS_SYSTEM_SECURITY_LE) + rsp->hdr.Status = STATUS_PRIVILEGE_NOT_HELD; + else + rsp->hdr.Status = STATUS_ACCESS_DENIED; + } else if (rc == -ENOENT) rsp->hdr.Status = STATUS_OBJECT_NAME_INVALID; else if (rc == -EPERM) @@ -4191,15 +4766,22 @@ int smb2_open(struct ksmbd_work *work) rsp->hdr.Status = STATUS_OBJECT_NAME_COLLISION; else if (rc == -EMFILE) rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + else if (rc == -EINPROGRESS) + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + else if (rc == -EAGAIN) + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; if (!rsp->hdr.Status) rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; - if (fp) + if (fp && !dh_info.replay) ksmbd_fd_put(work, fp); smb2_set_err_rsp(work); ksmbd_debug(SMB, "Error response: %x\n", rsp->hdr.Status); } + if (dh_info.replay) + ksmbd_put_durable_fd(dh_info.fp); + if (dh_info.reconnected) { /* * If reconnect succeeded, fp was republished in the @@ -4572,6 +5154,7 @@ static int process_query_dir_entries(struct smb2_query_dir_private *priv) for (i = 0; i < priv->d_info->num_entry; i++) { struct dentry *dent; + struct path path; if (dentry_name(priv->d_info, priv->info_level)) return -EINVAL; @@ -4594,6 +5177,23 @@ static int process_query_dir_entries(struct smb2_query_dir_private *priv) continue; } + if (test_share_config_flag(priv->work->tcon->share_conf, + KSMBD_SHARE_FLAG_HIDE_UNREADABLE)) { + __le32 daccess = FILE_READ_DATA_LE | FILE_READ_EA_LE | + FILE_READ_ATTRIBUTES_LE; + + path.mnt = priv->dir_fp->filp->f_path.mnt; + path.dentry = dent; + rc = smb_check_perm_dacl(priv->work->conn, &path, + &daccess, daccess, + priv->work->sess->user->uid, + true); + if (rc) { + dput(dent); + continue; + } + } + ksmbd_kstat.kstat = &kstat; if (priv->info_level != FILE_NAMES_INFORMATION) { rc = ksmbd_vfs_fill_dentry_attrs(priv->work, @@ -5022,26 +5622,31 @@ int smb2_query_dir(struct ksmbd_work *work) /** * buffer_check_err() - helper function to check buffer errors * @reqOutputBufferLength: max buffer length expected in command response + * @fixed_len: minimum fixed response length * @rsp: query info response buffer contains output buffer length - * @rsp_org: base response buffer pointer in case of chained response * * Return: 0 on success, otherwise error */ static int buffer_check_err(int reqOutputBufferLength, - struct smb2_query_info_rsp *rsp, - void *rsp_org) + unsigned int fixed_len, + struct smb2_query_info_rsp *rsp) { - if (reqOutputBufferLength < le32_to_cpu(rsp->OutputBufferLength)) { + unsigned int output_len = le32_to_cpu(rsp->OutputBufferLength); + + if (reqOutputBufferLength < fixed_len) { pr_err("Invalid Buffer Size Requested\n"); rsp->hdr.Status = STATUS_INFO_LENGTH_MISMATCH; - *(__be32 *)rsp_org = cpu_to_be32(sizeof(struct smb2_hdr)); return -EINVAL; } + + if (reqOutputBufferLength < output_len) { + rsp->hdr.Status = STATUS_BUFFER_OVERFLOW; + rsp->OutputBufferLength = cpu_to_le32(reqOutputBufferLength); + } return 0; } -static void get_standard_info_pipe(struct smb2_query_info_rsp *rsp, - void *rsp_org) +static void get_standard_info_pipe(struct smb2_query_info_rsp *rsp) { struct smb2_file_standard_info *sinfo; @@ -5056,8 +5661,7 @@ static void get_standard_info_pipe(struct smb2_query_info_rsp *rsp, cpu_to_le32(sizeof(struct smb2_file_standard_info)); } -static void get_internal_info_pipe(struct smb2_query_info_rsp *rsp, u64 num, - void *rsp_org) +static void get_internal_info_pipe(struct smb2_query_info_rsp *rsp, u64 num) { struct smb2_file_internal_info *file_info; @@ -5071,8 +5675,7 @@ static void get_internal_info_pipe(struct smb2_query_info_rsp *rsp, u64 num, static int smb2_get_info_file_pipe(struct ksmbd_session *sess, struct smb2_query_info_req *req, - struct smb2_query_info_rsp *rsp, - void *rsp_org) + struct smb2_query_info_rsp *rsp) { u64 id; int rc; @@ -5097,14 +5700,16 @@ static int smb2_get_info_file_pipe(struct ksmbd_session *sess, switch (req->FileInfoClass) { case FILE_STANDARD_INFORMATION: - get_standard_info_pipe(rsp, rsp_org); + get_standard_info_pipe(rsp); rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), - rsp, rsp_org); + le32_to_cpu(rsp->OutputBufferLength), + rsp); break; case FILE_INTERNAL_INFORMATION: - get_internal_info_pipe(rsp, id, rsp_org); + get_internal_info_pipe(rsp, id); rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), - rsp, rsp_org); + le32_to_cpu(rsp->OutputBufferLength), + rsp); break; default: ksmbd_debug(SMB, "smb2_info_file_pipe for %u not supported\n", @@ -5386,7 +5991,6 @@ static int get_file_all_info(struct ksmbd_work *work, char *filename; u64 time; int ret, buf_free_len, filename_len; - struct smb2_query_info_req *req = ksmbd_req_buf_next(work); if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { ksmbd_debug(SMB, "no right to read the attributes : 0x%x\n", @@ -5399,10 +6003,9 @@ static int get_file_all_info(struct ksmbd_work *work, return PTR_ERR(filename); filename_len = strlen(filename); - buf_free_len = smb2_calc_max_out_buf_len(work, + buf_free_len = smb2_resp_buf_len(work, offsetof(struct smb2_query_info_rsp, Buffer) + - offsetof(struct smb2_file_all_info, FileName), - le32_to_cpu(req->OutputBufferLength)); + offsetof(struct smb2_file_all_info, FileName)); if (buf_free_len < (filename_len + 1) * 2) { kfree(filename); return -EINVAL; @@ -5493,7 +6096,6 @@ static int get_file_stream_info(struct ksmbd_work *work, ssize_t xattr_list_len; int nbytes = 0, streamlen, stream_name_len, next, idx = 0; int buf_free_len; - struct smb2_query_info_req *req = ksmbd_req_buf_next(work); int ret; ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, @@ -5503,10 +6105,8 @@ static int get_file_stream_info(struct ksmbd_work *work, file_info = (struct smb2_file_stream_info *)rsp->Buffer; - buf_free_len = - smb2_calc_max_out_buf_len(work, - offsetof(struct smb2_query_info_rsp, Buffer), - le32_to_cpu(req->OutputBufferLength)); + buf_free_len = smb2_resp_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer)); if (buf_free_len < 0) goto out; @@ -5819,13 +6419,13 @@ static int smb2_get_info_file(struct ksmbd_work *work, struct ksmbd_file *fp; int fileinfoclass = 0; int rc = 0; + unsigned int fixed_len; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; if (test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_PIPE)) { /* smb2 info file called for pipe */ - rc = smb2_get_info_file_pipe(work->sess, req, rsp, - work->response_buf); + rc = smb2_get_info_file_pipe(work->sess, req, rsp); goto iov_pin_out; } @@ -5922,9 +6522,26 @@ static int smb2_get_info_file(struct ksmbd_work *work, fileinfoclass); rc = -EOPNOTSUPP; } - if (!rc) + if (!rc) { + fixed_len = le32_to_cpu(rsp->OutputBufferLength); + switch (fileinfoclass) { + case FILE_ALL_INFORMATION: + fixed_len = FILE_ALL_INFORMATION_SIZE; + break; + case FILE_ALTERNATE_NAME_INFORMATION: + fixed_len = FILE_ALTERNATE_NAME_INFORMATION_SIZE; + break; + case FILE_NORMALIZED_NAME_INFORMATION: + fixed_len = FILE_NORMALIZED_NAME_INFORMATION_SIZE; + break; + case FILE_STREAM_INFORMATION: + fixed_len = FILE_STREAM_INFORMATION_SIZE; + break; + } rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), - rsp, work->response_buf); + fixed_len, + rsp); + } ksmbd_fd_put(work, fp); iov_pin_out: @@ -5945,6 +6562,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, struct kstatfs stfs; struct path path; int rc = 0, len; + unsigned int fixed_len = 0; if (!share->path) return -EIO; @@ -5979,6 +6597,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info->DeviceCharacteristics |= cpu_to_le32(FILE_READ_ONLY_DEVICE); rsp->OutputBufferLength = cpu_to_le32(8); + fixed_len = 8; break; } case FS_ATTRIBUTE_INFORMATION: @@ -6031,6 +6650,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info->FileSystemNameLen = cpu_to_le32(len); sz = sizeof(FILE_SYSTEM_ATTRIBUTE_INFO) + len; rsp->OutputBufferLength = cpu_to_le32(sz); + fixed_len = 16; break; } case FS_VOLUME_INFORMATION: @@ -6058,6 +6678,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info->SupportsObjects = 0; sz = sizeof(struct filesystem_vol_info) + len; rsp->OutputBufferLength = cpu_to_le32(sz); + fixed_len = 24; break; } case FS_SIZE_INFORMATION: @@ -6070,6 +6691,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info->SectorsPerAllocationUnit = cpu_to_le32(1); info->BytesPerSector = cpu_to_le32(stfs.f_bsize); rsp->OutputBufferLength = cpu_to_le32(24); + fixed_len = 24; break; } case FS_FULL_SIZE_INFORMATION: @@ -6085,6 +6707,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info->SectorsPerAllocationUnit = cpu_to_le32(1); info->BytesPerSector = cpu_to_le32(stfs.f_bsize); rsp->OutputBufferLength = cpu_to_le32(32); + fixed_len = 32; break; } case FS_OBJECT_ID_INFORMATION: @@ -6105,6 +6728,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info->extended_info.rel_date = 0; memcpy(info->extended_info.version_string, "1.1.0", strlen("1.1.0")); rsp->OutputBufferLength = cpu_to_le32(64); + fixed_len = 64; break; } case FS_SECTOR_SIZE_INFORMATION: @@ -6126,6 +6750,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info->ByteOffsetForSectorAlignment = 0; info->ByteOffsetForPartitionAlignment = 0; rsp->OutputBufferLength = cpu_to_le32(28); + fixed_len = 28; break; } case FS_CONTROL_INFORMATION: @@ -6146,6 +6771,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info->DefaultQuotaLimit = cpu_to_le64(SMB2_NO_FID); info->Padding = 0; rsp->OutputBufferLength = cpu_to_le32(48); + fixed_len = 48; break; } case FS_POSIX_INFORMATION: @@ -6166,6 +6792,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, info->TotalFileNodes = cpu_to_le64(stfs.f_files); info->FreeFileNodes = cpu_to_le64(stfs.f_ffree); rsp->OutputBufferLength = cpu_to_le32(56); + fixed_len = 56; } break; } @@ -6174,7 +6801,8 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, return -EOPNOTSUPP; } rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), - rsp, work->response_buf); + fixed_len, + rsp); path_put(&path); if (!rc) @@ -6275,7 +6903,8 @@ static int smb2_get_info_sec(struct ksmbd_work *work, rsp->OutputBufferLength = cpu_to_le32(secdesclen); rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), - rsp, work->response_buf); + le32_to_cpu(rsp->OutputBufferLength), + rsp); if (rc) goto err_out; @@ -6405,7 +7034,6 @@ int smb2_close(struct ksmbd_work *work) u64 sess_id; struct smb2_close_req *req; struct smb2_close_rsp *rsp; - struct ksmbd_conn *conn = work->conn; struct ksmbd_file *fp; u64 time; int err = 0; @@ -6428,7 +7056,7 @@ int smb2_close(struct ksmbd_work *work) sess_id = work->compound_sid; work->compound_sid = 0; - if (check_session_id(conn, sess_id)) { + if (work->sess && work->sess->id == sess_id) { work->compound_sid = sess_id; } else { rsp->hdr.Status = STATUS_USER_SESSION_DELETED; @@ -7101,6 +7729,7 @@ int smb2_set_info(struct ksmbd_work *work) struct smb2_set_info_rsp *rsp; struct ksmbd_file *fp = NULL; int rc = 0; + bool chseq_err = false; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; ksmbd_debug(SMB, "Received smb2 set info request\n"); @@ -7140,6 +7769,13 @@ int smb2_set_info(struct ksmbd_work *work) goto err_out; } + rc = smb2_set_request_open(work, fp, &req->hdr, true, false); + if (rc) { + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + chseq_err = true; + goto err_out; + } + saved_cred = override_creds(fp->filp->f_cred); switch (req->InfoType) { case SMB2_O_INFO_FILE: @@ -7184,7 +7820,7 @@ int smb2_set_info(struct ksmbd_work *work) rsp->hdr.Status = STATUS_OBJECT_NAME_INVALID; else if (rc == -EBUSY || rc == -ENOTEMPTY) rsp->hdr.Status = STATUS_DIRECTORY_NOT_EMPTY; - else if (rc == -EAGAIN) + else if (rc == -EAGAIN && !chseq_err) rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; else if (rc == -EBADF || rc == -ESTALE) rsp->hdr.Status = STATUS_INVALID_HANDLE; @@ -7226,13 +7862,18 @@ static noinline int smb2_read_pipe(struct ksmbd_work *work) } aux_payload_buf = - kvmalloc(rpc_resp->payload_sz, KSMBD_DEFAULT_GFP); + kvmalloc(ALIGN(rpc_resp->payload_sz, 8), + KSMBD_DEFAULT_GFP); if (!aux_payload_buf) { err = -ENOMEM; goto out; } memcpy(aux_payload_buf, rpc_resp->payload, rpc_resp->payload_sz); + if (rpc_resp->payload_sz & 7) + memset(aux_payload_buf + rpc_resp->payload_sz, 0, + ALIGN(rpc_resp->payload_sz, 8) - + rpc_resp->payload_sz); nbytes = rpc_resp->payload_sz; err = ksmbd_iov_pin_rsp_read(work, (void *)rsp, @@ -7393,6 +8034,10 @@ int smb2_read(struct ksmbd_work *work) goto out; } + err = smb2_set_request_open(work, fp, &req->hdr, true, true); + if (err) + goto out; + if (!(fp->daccess & (FILE_READ_DATA_LE | FILE_READ_ATTRIBUTES_LE))) { pr_err("Not permitted to read : 0x%x\n", fp->daccess); err = -EACCES; @@ -7640,6 +8285,7 @@ int smb2_write(struct ksmbd_work *work) char *data_buf; bool writethrough = false, is_rdma_channel = false; bool async_interim = false; + bool chseq_err = false; int err = 0; unsigned int max_write_size = work->conn->vals->max_write_size; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; @@ -7670,8 +8316,10 @@ int smb2_write(struct ksmbd_work *work) } offset = le64_to_cpu(req->Offset); - if (offset < 0) - return -EINVAL; + if (offset < 0) { + err = -EINVAL; + goto out; + } length = le32_to_cpu(req->Length); if (req->Channel == SMB2_CHANNEL_RDMA_V1 || @@ -7685,6 +8333,19 @@ int smb2_write(struct ksmbd_work *work) length = le32_to_cpu(req->RemainingBytes); } + if (length) { + u64 end = (u64)offset + length; + + if (end > SMB2_MAX_FILE_SIZE) { + err = -EINVAL; + goto out; + } + if (end == SMB2_MAX_FILE_SIZE) { + err = -EFBIG; + goto out; + } + } + if (is_rdma_channel == true) { unsigned int ch_offset = le16_to_cpu(req->WriteChannelInfoOffset); @@ -7714,6 +8375,13 @@ int smb2_write(struct ksmbd_work *work) goto out; } + err = smb2_set_request_open(work, fp, &req->hdr, true, false); + if (err) { + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + chseq_err = true; + goto out; + } + if (!(fp->daccess & (FILE_WRITE_DATA_LE | FILE_READ_ATTRIBUTES_LE))) { pr_err("Not permitted to write : 0x%x\n", fp->daccess); err = -EACCES; @@ -7785,7 +8453,7 @@ int smb2_write(struct ksmbd_work *work) if (async_interim) release_async_work(work); - if (err == -EAGAIN) + if (err == -EAGAIN && !chseq_err) rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; else if (err == -ENOSPC || err == -EFBIG) rsp->hdr.Status = STATUS_DISK_FULL; @@ -7797,7 +8465,7 @@ int smb2_write(struct ksmbd_work *work) rsp->hdr.Status = STATUS_SHARING_VIOLATION; else if (err == -EINVAL) rsp->hdr.Status = STATUS_INVALID_PARAMETER; - else + else if (rsp->hdr.Status == 0) rsp->hdr.Status = STATUS_INVALID_HANDLE; smb2_set_err_rsp(work); @@ -7864,6 +8532,7 @@ int smb2_cancel(struct ksmbd_work *work) struct smb2_hdr *hdr = smb_get_msg(work->request_buf); struct smb2_hdr *chdr; struct ksmbd_work *iter; + struct ksmbd_work *cancelled_notify = NULL; struct list_head *command_list; if (work->next_smb2_rcv_hdr_off) @@ -7893,19 +8562,31 @@ int smb2_cancel(struct ksmbd_work *work) * still on conn->async_requests with a live cancel_fn * pointing at the freed file_lock. */ - if (iter->state != KSMBD_WORK_ACTIVE) + if (cmpxchg(&iter->state, KSMBD_WORK_ACTIVE, + KSMBD_WORK_CANCELLED) != KSMBD_WORK_ACTIVE) break; ksmbd_debug(SMB, "smb2 with AsyncId %llu cancelled command = 0x%x\n", le64_to_cpu(hdr->Id.AsyncId), le16_to_cpu(chdr->Command)); - iter->state = KSMBD_WORK_CANCELLED; - if (iter->cancel_fn) + if (iter->cancel_fn == smb2_notify_cancel_fn) + cancelled_notify = + smb2_notify_cancel_claim(iter->cancel_argv); + else if (iter->cancel_fn) iter->cancel_fn(iter->cancel_argv); break; } spin_unlock(&conn->request_lock); + + /* + * Complete a cancelled notify before this CANCEL handler returns. + * Deferring it to the system workqueue lets a following request and + * its response overtake STATUS_CANCELLED, leaving clients waiting + * for the original notify even though the cancellation was accepted. + */ + if (cancelled_notify) + smb2_complete_notify_cancel(cancelled_notify); } else { command_list = &conn->requests; @@ -7917,11 +8598,16 @@ int smb2_cancel(struct ksmbd_work *work) iter == work) continue; + if (cmpxchg(&iter->state, KSMBD_WORK_ACTIVE, + KSMBD_WORK_CANCELLED) != KSMBD_WORK_ACTIVE) + break; + ksmbd_debug(SMB, "smb2 with mid %llu cancelled command = 0x%x\n", le64_to_cpu(hdr->MessageId), le16_to_cpu(chdr->Command)); - iter->state = KSMBD_WORK_CANCELLED; + if (iter->cancel_fn) + iter->cancel_fn(iter->cancel_argv); break; } spin_unlock(&conn->request_lock); @@ -8026,6 +8712,19 @@ static void smb2_remove_blocked_lock(void **argv) locks_wake_up(flock); } +static void smb2_free_lock(struct file_lock *flock) +{ + ksmbd_vfs_posix_lock_unblock(flock); + locks_free_lock(flock); +} + +static void smb2_free_blocked_lock(struct file_lock *flock) +{ + ksmbd_vfs_posix_lock_unblock(flock); + locks_wake_up(flock); + locks_free_lock(flock); +} + static inline bool lock_defer_pending(struct file_lock *fl) { /* check pending lock waiters */ @@ -8058,6 +8757,7 @@ int smb2_lock(struct ksmbd_work *work) LIST_HEAD(rollback_list); int prior_lock = 0, bkt; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + bool lock_replayed; WORK_BUFFERS(work, req, rsp); @@ -8086,6 +8786,14 @@ int smb2_lock(struct ksmbd_work *work) goto out2; } + err = smb2_set_request_open(work, fp, &req->hdr, false, false); + if (err) + goto out2; + + lock_replayed = smb2_verify_lock_sequence(work, fp, req); + if (lock_replayed) + goto lock_success; + filp = fp->filp; lock_count = le16_to_cpu(req->LockCount); lock_ele = req->locks; @@ -8202,14 +8910,14 @@ int smb2_lock(struct ksmbd_work *work) cmp_lock->end == smb_lock->end && !lock_defer_pending(cmp_lock->fl)) { nolock = 0; - list_del(&cmp_lock->flist); - list_del(&cmp_lock->clist); + list_del_init(&cmp_lock->flist); + list_del_init(&cmp_lock->clist); cmp_lock->conn = NULL; spin_unlock(&conn->llist_lock); up_read(&conn_list_lock); ksmbd_conn_put(conn); - locks_free_lock(cmp_lock->fl); + smb2_free_lock(cmp_lock->fl); kfree(cmp_lock); goto out_check_cl; } @@ -8291,11 +8999,12 @@ int smb2_lock(struct ksmbd_work *work) ksmbd_debug(SMB, "would have to wait for getting lock\n"); - list_add(&smb_lock->llist, &rollback_list); argv = kmalloc(sizeof(void *), KSMBD_DEFAULT_GFP); if (!argv) { err = -ENOMEM; + smb2_free_blocked_lock(flock); + kfree(smb_lock); goto out; } argv[0] = flock; @@ -8306,8 +9015,11 @@ int smb2_lock(struct ksmbd_work *work) if (rc) { kfree(argv); err = -ENOMEM; + smb2_free_blocked_lock(flock); + kfree(smb_lock); goto out; } + list_add(&smb_lock->llist, &rollback_list); spin_lock(&fp->f_lock); list_add(&work->fp_entry, &fp->blocked_works); spin_unlock(&fp->f_lock); @@ -8340,16 +9052,10 @@ int smb2_lock(struct ksmbd_work *work) rsp->hdr.Status = STATUS_RANGE_NOT_LOCKED; kfree(smb_lock); - goto out2; + /* rollback_list may still hold earlier grants */ + goto out; } else if (!rc) { list_add(&smb_lock->llist, &rollback_list); - smb_lock->conn = ksmbd_conn_get(work->conn); - spin_lock(&work->conn->llist_lock); - list_add_tail(&smb_lock->clist, - &work->conn->lock_list); - list_add_tail(&smb_lock->flist, - &fp->lock_list); - spin_unlock(&work->conn->llist_lock); ksmbd_debug(SMB, "successful in taking lock\n"); } else { locks_free_lock(flock); @@ -8363,6 +9069,7 @@ int smb2_lock(struct ksmbd_work *work) if (atomic_read(&fp->f_ci->op_count) > 1) smb_break_all_oplock(work, fp); +lock_success: rsp->StructureSize = cpu_to_le16(4); ksmbd_debug(SMB, "successful in taking lock\n"); rsp->hdr.Status = STATUS_SUCCESS; @@ -8371,6 +9078,23 @@ int smb2_lock(struct ksmbd_work *work) if (err) goto out; + /* publish only once the whole batch has committed */ + if (!list_empty(&rollback_list)) { + spin_lock(&work->conn->llist_lock); + list_for_each_entry_safe(smb_lock, tmp, &rollback_list, llist) { + list_del_init(&smb_lock->llist); + smb_lock->conn = ksmbd_conn_get(work->conn); + list_add_tail(&smb_lock->clist, + &work->conn->lock_list); + list_add_tail(&smb_lock->flist, + &fp->lock_list); + } + spin_unlock(&work->conn->llist_lock); + } + + if (!lock_replayed) + smb2_update_lock_sequence(work, fp, req); + ksmbd_fd_put(work, fp); return 0; @@ -8398,16 +9122,7 @@ int smb2_lock(struct ksmbd_work *work) } list_del(&smb_lock->llist); - conn = smb_lock->conn; - spin_lock(&conn->llist_lock); - if (!list_empty(&smb_lock->flist)) - list_del(&smb_lock->flist); - list_del(&smb_lock->clist); - smb_lock->conn = NULL; - spin_unlock(&conn->llist_lock); - ksmbd_conn_put(conn); - - locks_free_lock(smb_lock->fl); + smb2_free_lock(smb_lock->fl); if (rlock) locks_free_lock(rlock); kfree(smb_lock); @@ -8875,10 +9590,12 @@ int smb2_ioctl(struct ksmbd_work *work) struct smb2_ioctl_req *req; struct smb2_ioctl_rsp *rsp; unsigned int cnt_code, nbytes = 0, out_buf_len, in_buf_len; - u64 id = KSMBD_NO_FID; + u64 id = KSMBD_NO_FID, pid = KSMBD_NO_FID; struct ksmbd_conn *conn = work->conn; int ret = 0; char *buffer; + bool no_fileid_ioctl = false; + bool chseq_err = false; ksmbd_debug(SMB, "Received smb2 ioctl request\n"); @@ -8891,14 +9608,17 @@ int smb2_ioctl(struct ksmbd_work *work) ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); id = work->compound_fid; + pid = work->compound_pfid; } } else { req = smb_get_msg(work->request_buf); rsp = smb_get_msg(work->response_buf); } - if (!has_file_id(id)) + if (!has_file_id(id)) { id = req->VolatileFileId; + pid = req->PersistentFileId; + } if (req->Flags != cpu_to_le32(SMB2_0_IOCTL_IS_FSCTL)) { ret = -EOPNOTSUPP; @@ -8908,6 +9628,40 @@ int smb2_ioctl(struct ksmbd_work *work) buffer = (char *)req + le32_to_cpu(req->InputOffset); cnt_code = le32_to_cpu(req->CtlCode); + switch (cnt_code) { + case FSCTL_DFS_GET_REFERRALS: + case FSCTL_DFS_GET_REFERRALS_EX: + case FSCTL_QUERY_NETWORK_INTERFACE_INFO: + case FSCTL_VALIDATE_NEGOTIATE_INFO: + case FSCTL_PIPE_WAIT: + no_fileid_ioctl = true; + break; + default: + break; + } + + if (!no_fileid_ioctl && has_file_id(id)) { + struct ksmbd_file *fp; + + fp = ksmbd_lookup_fd_slow(work, id, pid); + if (!fp) { + if (cnt_code == FSCTL_DUPLICATE_EXTENTS_TO_FILE) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + goto out2; + } + ret = -ENOENT; + goto out; + } + + ret = smb2_set_request_open(work, fp, &req->hdr, true, false); + ksmbd_fd_put(work, fp); + if (ret) { + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + chseq_err = true; + goto out; + } + } + ret = smb2_calc_max_out_buf_len(work, offsetof(struct smb2_ioctl_rsp, Buffer), le32_to_cpu(req->MaxOutputResponse)); @@ -9298,7 +10052,7 @@ int smb2_ioctl(struct ksmbd_work *work) rsp->hdr.Status = STATUS_NOT_SUPPORTED; else if (ret == -ENOSPC) rsp->hdr.Status = STATUS_BUFFER_TOO_SMALL; - else if (ret < 0 || rsp->hdr.Status == 0) + else if (!chseq_err && (ret < 0 || rsp->hdr.Status == 0)) rsp->hdr.Status = STATUS_INVALID_PARAMETER; out2: @@ -9338,6 +10092,14 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) return; } + ret = smb2_set_request_open(work, fp, &req->hdr, false, false); + if (ret) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + smb2_set_err_rsp(work); + ksmbd_fd_put(work, fp); + return; + } + opinfo = opinfo_get(fp); if (!opinfo) { pr_err("unexpected null oplock_info\n"); @@ -9350,6 +10112,22 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) if (opinfo->op_state != OPLOCK_ACK_WAIT) { ksmbd_debug(SMB, "unexpected oplock state 0x%x\n", opinfo->op_state); + if (smb3_hdr_replay(&req->hdr) && + opinfo->op_state == OPLOCK_STATE_NONE) { + rsp->StructureSize = cpu_to_le16(24); + rsp->OplockLevel = opinfo->level; + rsp->Reserved = 0; + rsp->Reserved2 = 0; + rsp->VolatileFid = volatile_id; + rsp->PersistentFid = persistent_id; + ret = ksmbd_iov_pin_rsp(work, rsp, + sizeof(struct smb2_oplock_break)); + if (ret) + ksmbd_debug(SMB, + "failed to pin replayed oplock break response: %d\n", + ret); + goto out_no_state_change; + } if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) status = STATUS_INVALID_OPLOCK_PROTOCOL; else @@ -9416,8 +10194,12 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) smb2_set_err_rsp(work); out: - opinfo->op_state = OPLOCK_STATE_NONE; + spin_lock(&opinfo->state_lock); + if (opinfo->op_state != OPLOCK_CLOSING) + opinfo->op_state = OPLOCK_STATE_NONE; + spin_unlock(&opinfo->state_lock); wake_up_interruptible_all(&opinfo->oplock_q); +out_no_state_change: opinfo_put(opinfo); ksmbd_fd_put(work, fp); } @@ -9470,11 +10252,15 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) if (opinfo->op_state == OPLOCK_STATE_NONE) { pr_err("unexpected lease break state 0x%x\n", opinfo->op_state); + if (smb3_hdr_replay(&req->hdr)) + goto replay_rsp; rsp->hdr.Status = STATUS_UNSUCCESSFUL; goto err_out; } if (!atomic_read(&opinfo->breaking_cnt)) { + if (smb3_hdr_replay(&req->hdr)) + goto replay_rsp; rsp->hdr.Status = STATUS_UNSUCCESSFUL; goto err_out; } @@ -9502,13 +10288,29 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) if (ret) goto err_out; - opinfo->op_state = OPLOCK_STATE_NONE; + spin_lock(&opinfo->state_lock); + if (opinfo->op_state != OPLOCK_CLOSING) + opinfo->op_state = OPLOCK_STATE_NONE; + spin_unlock(&opinfo->state_lock); wake_up_interruptible_all(&opinfo->oplock_q); - atomic_dec(&opinfo->breaking_cnt); + atomic_dec_if_positive(&opinfo->breaking_cnt); wake_up_interruptible_all(&opinfo->oplock_brk); opinfo_put(opinfo); return; +replay_rsp: + rsp->StructureSize = cpu_to_le16(36); + rsp->Reserved = 0; + rsp->Flags = 0; + memcpy(rsp->LeaseKey, req->LeaseKey, 16); + rsp->LeaseState = lease->state; + rsp->LeaseDuration = 0; + ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_lease_ack)); + if (ret) + goto err_out; + opinfo_put(opinfo); + return; + err_out: smb2_set_err_rsp(work); opinfo_put(opinfo); @@ -9548,6 +10350,127 @@ int smb2_oplock_break(struct ksmbd_work *work) return 0; } +/* + * Cancel handler for a deferred CHANGE_NOTIFY. Races against + * __ksmbd_close_fd()'s notify_pendings drain (vfs_cache.c), which can run + * concurrently on a different connection closing the same handle -- only + * one of the two may claim and free in_work, so both sides check + * list_empty() under fp->f_lock before touching it (list_del_init() + * leaves a node empty, so whichever side removes it first is the owner; + * the loser must not touch in_work again, since the winner may already be + * freeing it). + * + * smb2_cancel() holds conn->request_lock (a spinlock) for the entire + * time it walks conn->async_requests and calls this function -- so this + * runs with preemption disabled and must not sleep or re-acquire that + * same lock. release_async_work() does both (it takes conn->request_lock + * itself, and frees things that can involve sleeping paths), so calling + * it from here would self-deadlock the very thread processing the + * client's CANCEL command. ksmbd_conn_write() can also sleep (it takes + * conn's write mutex). So: do only the non-sleeping, no-relock cleanup + * inline here. smb2_cancel() sends and frees the claimed notify after it + * drops request_lock, preserving response order for a client CANCEL. The + * connection teardown caller has no such post-unlock path, so its wrapper + * defers the send and free to a workqueue. + */ +struct notify_cancel_ctx { + struct work_struct work; + struct ksmbd_work *in_work; +}; + +static void smb2_send_notify_cancelled(struct ksmbd_work *work) +{ + struct smb2_hdr *hdr = smb_get_msg(work->response_buf); + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess; + + sess = ksmbd_session_lookup(conn, le64_to_cpu(hdr->SessionId)); + if (sess) { + work->sess = sess; + if (work->encrypted && sess->enc && conn->ops->encrypt_resp) { + conn->ops->encrypt_resp(work); + } else if (conn->ops->is_sign_req && conn->ops->set_sign_rsp && + conn->ops->is_sign_req(work, + conn->ops->get_cmd_val(work))) { + conn->ops->set_sign_rsp(work); + } + } + + ksmbd_conn_write(work); + if (sess) { + ksmbd_user_session_put(sess); + work->sess = NULL; + } +} + +static void smb2_notify_cancel_deferred(struct work_struct *w) +{ + struct notify_cancel_ctx *ctx = + container_of(w, struct notify_cancel_ctx, work); + + smb2_complete_notify_cancel(ctx->in_work); + kfree(ctx); +} + +static struct ksmbd_work *smb2_notify_cancel_claim(void **argv) +{ + struct ksmbd_work *in_work = (struct ksmbd_work *)argv[0]; + struct ksmbd_file *fp = (struct ksmbd_file *)argv[1]; + bool claimed; + + spin_lock(&fp->f_lock); + claimed = !list_empty(&in_work->notify_entry); + if (claimed) + list_del_init(&in_work->notify_entry); + spin_unlock(&fp->f_lock); + + if (!claimed) + return NULL; + + /* conn->request_lock is held by smb2_cancel() or connection teardown. */ + in_work->cancel_fn = NULL; + kfree(in_work->cancel_argv); + in_work->cancel_argv = NULL; + return in_work; +} + +static void smb2_complete_notify_cancel(struct ksmbd_work *in_work) +{ + struct smb2_hdr *in_hdr = smb_get_msg(in_work->response_buf); + + in_hdr->Status = STATUS_CANCELLED; + smb2_send_notify_cancelled(in_work); + release_async_work(in_work); + ksmbd_free_work_struct(in_work); +} + +static void smb2_notify_cancel_fn(void **argv) +{ + struct ksmbd_work *in_work = smb2_notify_cancel_claim(argv); + struct ksmbd_conn *conn; + struct notify_cancel_ctx *ctx; + + if (!in_work) + return; + conn = in_work->conn; + + ctx = kmalloc(sizeof(*ctx), GFP_ATOMIC); + if (!ctx) { + /* Can't defer the response -- free without sending one. */ + list_del_init(&in_work->async_request_entry); + in_work->asynchronous = false; + if (in_work->async_id) { + ksmbd_release_id(&conn->async_ida, in_work->async_id); + in_work->async_id = 0; + } + ksmbd_free_work_struct(in_work); + return; + } + ctx->in_work = in_work; + INIT_WORK(&ctx->work, smb2_notify_cancel_deferred); + schedule_work(&ctx->work); +} + /** * smb2_notify() - handler for smb2 notify request * @work: smb work containing notify command buffer @@ -9558,6 +10481,9 @@ int smb2_notify(struct ksmbd_work *work) { struct smb2_change_notify_req *req; struct smb2_change_notify_rsp *rsp; + struct ksmbd_work *in_work; + struct smb2_hdr *in_hdr; + struct ksmbd_file *fp; ksmbd_debug(SMB, "Received smb2 notify\n"); @@ -9572,9 +10498,149 @@ int smb2_notify(struct ksmbd_work *work) return -EIO; } - smb2_set_err_rsp(work); - rsp->hdr.Status = STATUS_NOT_IMPLEMENTED; - return -EOPNOTSUPP; + /* + * macOS backupd sends CHANGE_NOTIFY with FileId=FFFF...FFFF (share-root + * sentinel) to watch for changes on the share root without holding an + * open handle. Respond STATUS_PENDING + STATUS_NOTIFY_CLEANUP immediately; + * without this, backupd aborts Time Machine setup on STATUS_FILE_CLOSED. + */ + if (req->VolatileFileId == SMB2_NO_FID && + req->PersistentFileId == SMB2_NO_FID) { + in_work = ksmbd_alloc_work_struct(); + if (!in_work || allocate_interim_rsp_buf(in_work)) { + if (in_work) + ksmbd_free_work_struct(in_work); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + if (setup_async_work(work, NULL, NULL)) { + ksmbd_free_work_struct(in_work); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + smb2_send_interim_resp(work, STATUS_PENDING); + in_work->conn = work->conn; + in_hdr = smb_get_msg(in_work->response_buf); + memcpy(in_hdr, ksmbd_resp_buf_next(work), + __SMB2_HEADER_STRUCTURE_SIZE); + in_hdr->Flags |= SMB2_FLAGS_ASYNC_COMMAND; + in_hdr->Id.AsyncId = cpu_to_le64(work->async_id); + smb2_set_err_rsp(in_work); + in_hdr->Status = STATUS_NOTIFY_CLEANUP; + in_work->async_id = work->async_id; + work->async_id = 0; + release_async_work(work); + ksmbd_conn_write(in_work); + ksmbd_free_work_struct(in_work); + work->send_no_response = 1; + return 0; + } + + /* + * KSMBD does not implement a real change-notification backend. + * Genuine SMB2 servers (and macOS smbfs) never complete a + * CHANGE_NOTIFY spontaneously: it is satisfied only by a real + * directory change, or with STATUS_NOTIFY_CLEANUP when the watched + * handle is closed. Completing it early (e.g. on a timer) makes + * Finder treat the cleanup as "directory changed" and re-enumerate + * the directory forever, leaving items unopenable. Returning + * STATUS_NOT_IMPLEMENTED here (like stock ksmbd) makes macOS smbfs + * hard-freeze on unmount, so this must stay deferred. + */ + fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); + if (!fp) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + smb2_set_err_rsp(work); + return 0; + } + + in_work = ksmbd_alloc_work_struct(); + if (!in_work || allocate_interim_rsp_buf(in_work)) { + if (in_work) + ksmbd_free_work_struct(in_work); + ksmbd_fd_put(work, fp); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + /* + * in_work is synthetic (not from the normal request-receiving + * pipeline), so it has no request_buf of its own. It gets registered + * into conn->async_requests below, and smb2_cancel() unconditionally + * computes smb_get_msg(iter->request_buf) for every entry in that + * list while searching for a match -- give it its own small buffer + * (not an alias of response_buf: ksmbd_free_work_struct() kvfree()s + * both separately, so aliasing them would double-free) so that stays + * a harmless read instead of a near-NULL dereference. + */ + in_work->request_buf = kzalloc(MAX_CIFS_SMALL_BUFFER_SIZE, KSMBD_DEFAULT_GFP); + if (!in_work->request_buf) { + ksmbd_free_work_struct(in_work); + ksmbd_fd_put(work, fp); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + memcpy(smb_get_msg(in_work->request_buf), req, + __SMB2_HEADER_STRUCTURE_SIZE); + + if (setup_async_work(work, NULL, NULL)) { + ksmbd_free_work_struct(in_work); + ksmbd_fd_put(work, fp); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + + smb2_send_interim_resp(work, STATUS_PENDING); + + /* Keep the async IDA alive until the deferred work is released. */ + in_work->conn = ksmbd_conn_get(work->conn); + in_work->owns_conn_ref = true; + in_work->encrypted = work->encrypted; + in_hdr = smb_get_msg(in_work->response_buf); + memcpy(in_hdr, ksmbd_resp_buf_next(work), __SMB2_HEADER_STRUCTURE_SIZE); + in_hdr->Flags |= SMB2_FLAGS_ASYNC_COMMAND; + in_hdr->Id.AsyncId = cpu_to_le64(work->async_id); + smb2_set_err_rsp(in_work); + in_hdr->Status = STATUS_NOTIFY_CLEANUP; + + /* + * Transfer ownership of the async id to in_work; it stays reserved + * until in_work is freed after the deferred response is sent on + * close, so it can't be reused for an unrelated async response. + */ + in_work->async_id = work->async_id; + work->async_id = 0; + release_async_work(work); + + /* + * work itself is about to be recycled by the normal request-processing + * pipeline, so it can't stay the target of a future CANCEL -- register + * in_work instead, reusing the same async_id, so a client-sent CANCEL + * for this notify actually finds something to cancel instead of + * silently doing nothing until the handle eventually closes. + */ + in_work->asynchronous = true; + in_work->cancel_argv = kmalloc_array(2, sizeof(void *), KSMBD_DEFAULT_GFP); + if (in_work->cancel_argv) { + in_work->cancel_argv[0] = in_work; + in_work->cancel_argv[1] = fp; + in_work->cancel_fn = smb2_notify_cancel_fn; + } + spin_lock(&work->conn->request_lock); + list_add_tail(&in_work->async_request_entry, &work->conn->async_requests); + spin_unlock(&work->conn->request_lock); + + spin_lock(&fp->f_lock); + list_add_tail(&in_work->notify_entry, &fp->notify_pendings); + spin_unlock(&fp->f_lock); + + ksmbd_fd_put(work, fp); + work->send_no_response = 1; + return 0; } /** @@ -9867,33 +10933,33 @@ bool smb3_is_transform_hdr(void *buf) int smb3_decrypt_req(struct ksmbd_work *work) { - struct ksmbd_session *sess; char *buf = work->request_buf; unsigned int pdu_length = get_rfc1002_len(buf); struct kvec iov[2]; - int buf_data_size = pdu_length - sizeof(struct smb2_transform_hdr); + unsigned int buf_data_size; struct smb2_transform_hdr *tr_hdr = smb_get_msg(buf); + unsigned int original_msg_size; int rc = 0; - if (pdu_length < sizeof(struct smb2_transform_hdr) || - buf_data_size < sizeof(struct smb2_hdr)) { + if (pdu_length < sizeof(struct smb2_transform_hdr)) { pr_err("Transform message is too small (%u)\n", pdu_length); return -ECONNABORTED; } - if (buf_data_size < le32_to_cpu(tr_hdr->OriginalMessageSize)) { - pr_err("Transform message is broken\n"); + buf_data_size = pdu_length - sizeof(struct smb2_transform_hdr); + original_msg_size = le32_to_cpu(tr_hdr->OriginalMessageSize); + if (buf_data_size < sizeof(struct smb2_compression_hdr) || + original_msg_size < sizeof(struct smb2_compression_hdr)) { + pr_err("Transform message is too small (%u)\n", + pdu_length); return -ECONNABORTED; } - sess = ksmbd_session_lookup_all(work->conn, le64_to_cpu(tr_hdr->SessionId)); - if (!sess) { - pr_err("invalid session id(%llx) in transform header\n", - le64_to_cpu(tr_hdr->SessionId)); + if (buf_data_size < original_msg_size) { + pr_err("Transform message is broken\n"); return -ECONNABORTED; } - ksmbd_user_session_put(sess); iov[0].iov_base = buf; iov[0].iov_len = sizeof(struct smb2_transform_hdr) + 4; @@ -9903,8 +10969,9 @@ int smb3_decrypt_req(struct ksmbd_work *work) if (rc) return rc; - memmove(buf + 4, iov[1].iov_base, buf_data_size); - *(__be32 *)buf = cpu_to_be32(buf_data_size); + /* Drop the AEAD authentication tag from the inner RFC1002 frame. */ + memmove(buf + 4, iov[1].iov_base, original_msg_size); + *(__be32 *)buf = cpu_to_be32(original_msg_size); return rc; } diff --git a/fs/smb/server/smb2pdu.h b/fs/smb/server/smb2pdu.h index aa06c8c905f1f..bf30fa8544b7f 100644 --- a/fs/smb/server/smb2pdu.h +++ b/fs/smb/server/smb2pdu.h @@ -61,11 +61,68 @@ struct preauth_integrity_info { #define SMB2_SESSION_IN_PROGRESS BIT(0) #define SMB2_SESSION_VALID BIT(1) -#define SMB2_SESSION_TIMEOUT (10 * HZ) - /* Apple Defined Contexts */ #define SMB2_CREATE_AAPL "AAPL" +/* + * AAPL SMB2 extension -- kAAPL_SERVER_QUERY create context. + * + * Command code and bitmap values are the existing + * SMB2_CRTCTX_AAPL_* constants in fs/smb/common/smb2pdu.h. + * + * Omitting the model string when reply_bitmap includes + * SMB2_CRTCTX_AAPL_MODEL_INFO causes smbfs.kext to enter a broken + * disconnect path requiring a reboot. + * + * Layout: ccontext(16) + Name[4] + Pad[4] + cmd(4) + reserved(4) + + * reply_bitmap(8) + server_caps(8) + vol_caps(8) + * When MODEL_INFO requested, appended: pad2(4) + model_bytes(4) + UTF-16LE + */ +#define SMB2_CREATE_AAPL_LEN 4 + +/* + * Server capability flags (server_caps field) -- SMB2_CRTCTX_AAPL_UNIX_BASED: + * prevents macOS Windows-compat mode (question-mark icons). + * SMB2_CRTCTX_AAPL_SUPPORTS_OSX_COPYFILE: enables server-side file copy via + * FSCTL_SRV_COPYCHUNK. SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR: inline + * FinderInfo per FIND entry, set when client also advertises the bit; + * format: EaSize=max_access, ShortName[0..7]=rfork_size, + * ShortName[8..23]=FinderInfo(16B), Reserved2=unix_mode. + */ +#define AAPL_SERVER_CAPS_KSMBD (SMB2_CRTCTX_AAPL_UNIX_BASED | \ + SMB2_CRTCTX_AAPL_SUPPORTS_OSX_COPYFILE | \ + SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR) + +/* Model string: up to 31 ASCII chars */ +#define AAPL_MODEL_MAX_CHARS 31 +#define AAPL_MODEL_UTF16_BYTES (AAPL_MODEL_MAX_CHARS * 2) + +/* + * Max AAPL response: header(24) + base data(32) + pad2(4) + model_bytes(4) + * + model(62), 8-byte aligned: ALIGN(126, 8) = 128 bytes. + */ +#define AAPL_RSP_MAX_SIZE 128 + +/* AAPL server query request (client->server) */ +struct aapl_server_query_req { + __le32 cmd; + __le32 reserved; + __le64 req_bitmap; + __le64 client_caps; +} __packed; + +struct create_aapl_rsp { + struct create_context_hdr ccontext; + __u8 Name[4]; + __u8 Pad[4]; + __le32 cmd; + __le32 reserved; + __le64 reply_bitmap; + __le64 server_caps; + __le64 vol_caps; + /* when MODEL_INFO requested: __le32 pad2; __le32 model_bytes; __le16 model[] */ +} __packed; + #define DURABLE_HANDLE_MAX_TIMEOUT 300000 struct create_alloc_size_req { @@ -140,6 +197,7 @@ struct file_sparse { #define FILE_ALLOCATION_INFORMATION_SIZE 19 #define FILE_END_OF_FILE_INFORMATION_SIZE 20 #define FILE_ALTERNATE_NAME_INFORMATION_SIZE 8 +#define FILE_NORMALIZED_NAME_INFORMATION_SIZE 8 #define FILE_STREAM_INFORMATION_SIZE 32 #define FILE_PIPE_INFORMATION_SIZE 23 #define FILE_PIPE_LOCAL_INFORMATION_SIZE 24 @@ -347,6 +405,7 @@ bool smb3_encryption_negotiated(struct ksmbd_conn *conn); /* smb2 misc functions */ int ksmbd_smb2_check_message(struct ksmbd_work *work); +void smb2_complete_request_open(struct ksmbd_work *work); /* smb2 command handlers */ int smb2_handle_negotiate(struct ksmbd_work *work); diff --git a/fs/smb/server/smb_common.h b/fs/smb/server/smb_common.h index b090b56743c4c..7b9c5cfcb63b0 100644 --- a/fs/smb/server/smb_common.h +++ b/fs/smb/server/smb_common.h @@ -135,7 +135,7 @@ struct file_id_both_directory_info { struct smb_version_ops { u16 (*get_cmd_val)(struct ksmbd_work *swork); - void (*inc_reqs)(unsigned int cmd); + void (*inc_reqs)(unsigned int cmd, __le32 status); int (*init_rsp_hdr)(struct ksmbd_work *swork); void (*set_rsp_status)(struct ksmbd_work *swork, __le32 err); int (*allocate_rsp_buf)(struct ksmbd_work *work); diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index c13f07a09ab8b..ae1527aa62697 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -380,10 +380,10 @@ void free_acl_state(struct posix_acl_state *state) kfree(state->groups); } -static void parse_dacl(struct mnt_idmap *idmap, - struct smb_acl *pdacl, char *end_of_acl, - struct smb_sid *pownersid, struct smb_sid *pgrpsid, - struct smb_fattr *fattr) +static int parse_dacl(struct mnt_idmap *idmap, + struct smb_acl *pdacl, char *end_of_acl, + struct smb_sid *pownersid, struct smb_sid *pgrpsid, + struct smb_fattr *fattr) { int i, ret; u16 num_aces = 0; @@ -397,13 +397,13 @@ static void parse_dacl(struct mnt_idmap *idmap, bool owner_found = false, group_found = false, others_found = false; if (!pdacl) - return; + return 0; /* validate that we do not go past end of acl */ if (end_of_acl < (char *)pdacl + sizeof(struct smb_acl) || end_of_acl < (char *)pdacl + le16_to_cpu(pdacl->size)) { pr_err("ACL too small to parse DACL\n"); - return; + return -EINVAL; } ksmbd_debug(SMB, "DACL revision %d size %d num aces %d\n", @@ -415,31 +415,31 @@ static void parse_dacl(struct mnt_idmap *idmap, num_aces = le16_to_cpu(pdacl->num_aces); if (num_aces <= 0) - return; + return 0; dacl_size = le16_to_cpu(pdacl->size); if (dacl_size < sizeof(struct smb_acl)) - return; + return -EINVAL; if (num_aces > (dacl_size - sizeof(struct smb_acl)) / (offsetof(struct smb_ace, sid) + offsetof(struct smb_sid, sub_auth) + sizeof(__le16))) - return; + return -EINVAL; ret = init_acl_state(&acl_state, num_aces); if (ret) - return; + return ret; ret = init_acl_state(&default_acl_state, num_aces); if (ret) { free_acl_state(&acl_state); - return; + return ret; } ppace = kmalloc_objs(struct smb_ace *, num_aces, KSMBD_DEFAULT_GFP); if (!ppace) { free_acl_state(&default_acl_state); free_acl_state(&acl_state); - return; + return -ENOMEM; } /* @@ -448,8 +448,10 @@ static void parse_dacl(struct mnt_idmap *idmap, * user/group/other have no permissions */ for (i = 0; i < num_aces; ++i) { - if (end_of_acl - acl_base < acl_size) - break; + if (end_of_acl - acl_base < acl_size) { + ret = -EINVAL; + goto out; + } ppace[i] = (struct smb_ace *)(acl_base + acl_size); acl_base = (char *)ppace[i]; @@ -462,8 +464,10 @@ static void parse_dacl(struct mnt_idmap *idmap, (end_of_acl - acl_base < acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth) || (le16_to_cpu(ppace[i]->size) < - acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth)) - break; + acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth)) { + ret = -EINVAL; + goto out; + } acl_size = le16_to_cpu(ppace[i]->size); ppace[i]->access_req = @@ -521,8 +525,8 @@ static void parse_dacl(struct mnt_idmap *idmap, temp_fattr.cf_uid = INVALID_UID; ret = sid_to_id(idmap, &ppace[i]->sid, SIDOWNER, &temp_fattr); if (ret || uid_eq(temp_fattr.cf_uid, INVALID_UID)) { - pr_err("%s: Error %d mapping Owner SID to uid\n", - __func__, ret); + pr_err_ratelimited("%s: Error %d mapping Owner SID to uid\n", + __func__, ret); continue; } @@ -538,7 +542,6 @@ static void parse_dacl(struct mnt_idmap *idmap, ((acl_mode & 0700) >> 6) | 0004; } } - kfree(ppace); if (owner_found) { /* The owner must be set to at least read-only. */ @@ -581,10 +584,12 @@ static void parse_dacl(struct mnt_idmap *idmap, fattr->cf_acls = posix_acl_alloc(acl_state.users->n + acl_state.groups->n + 4, KSMBD_DEFAULT_GFP); - if (fattr->cf_acls) { - cf_pace = fattr->cf_acls->a_entries; - posix_state_to_acl(&acl_state, cf_pace); + if (!fattr->cf_acls) { + ret = -ENOMEM; + goto out; } + cf_pace = fattr->cf_acls->a_entries; + posix_state_to_acl(&acl_state, cf_pace); } } @@ -595,14 +600,20 @@ static void parse_dacl(struct mnt_idmap *idmap, fattr->cf_dacls = posix_acl_alloc(default_acl_state.users->n + default_acl_state.groups->n + 4, KSMBD_DEFAULT_GFP); - if (fattr->cf_dacls) { - cf_pdace = fattr->cf_dacls->a_entries; - posix_state_to_acl(&default_acl_state, cf_pdace); + if (!fattr->cf_dacls) { + ret = -ENOMEM; + goto out; } + cf_pdace = fattr->cf_dacls->a_entries; + posix_state_to_acl(&default_acl_state, cf_pdace); } } + ret = 0; +out: + kfree(ppace); free_acl_state(&acl_state); free_acl_state(&default_acl_state); + return ret; } static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, @@ -963,8 +974,10 @@ int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, if (dacloffset < sizeof(struct smb_ntsd)) return -EINVAL; - parse_dacl(idmap, dacl_ptr, end_of_acl, - owner_sid_ptr, group_sid_ptr, fattr); + rc = parse_dacl(idmap, dacl_ptr, end_of_acl, + owner_sid_ptr, group_sid_ptr, fattr); + if (rc) + return rc; } return 0; @@ -1432,7 +1445,8 @@ bool smb_inherit_flags(int flags, bool is_dir) } int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, - __le32 *pdaccess, int uid) + __le32 *pdaccess, __le32 raw_daccess, int uid, + bool strict) { struct mnt_idmap *idmap = mnt_idmap(path->mnt); struct smb_ntsd *pntsd = NULL; @@ -1442,10 +1456,11 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, unsigned int dacl_offset; size_t dacl_struct_end; struct smb_sid sid; - int granted = le32_to_cpu(*pdaccess & ~FILE_MAXIMAL_ACCESS_LE); + int requested = le32_to_cpu(*pdaccess & ~FILE_MAXIMAL_ACCESS_LE); + int granted = requested; struct smb_ace *ace; int i, found = 0; - unsigned int access_bits = 0; + unsigned int access_bits = 0, denied = 0; struct smb_ace *others_ace = NULL; struct posix_acl_entry *pa_entry; unsigned int sid_type = SIDOWNER; @@ -1479,12 +1494,15 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, goto err_out; } + if (!uid) + sid_type = SIDUNIX_USER; + id_to_sid(uid, sid_type, &sid); + if (*pdaccess & FILE_MAXIMAL_ACCESS_LE) { - granted = READ_CONTROL | WRITE_DAC | FILE_READ_ATTRIBUTES | + access_bits = READ_CONTROL | WRITE_DAC | FILE_READ_ATTRIBUTES | DELETE; - ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); - aces_size = acl_size - sizeof(struct smb_acl); + aces_size = pdacl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { if (aces_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE) @@ -1495,17 +1513,43 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, CIFS_SID_BASE_SIZE) break; aces_size -= ace_size; - granted |= le32_to_cpu(ace->access_req); + + if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE + + sizeof(__le32) * ace->sid.num_subauth) + break; + + if (ace->flags & INHERIT_ONLY_ACE || + (compare_sids(&sid, &ace->sid) && + compare_sids(&sid_unix_NFS_mode, &ace->sid) && + compare_sids(&sid_everyone, &ace->sid) && + compare_sids(&sid_authusers, &ace->sid))) + goto next_ace; + + switch (ace->type) { + case ACCESS_ALLOWED_ACE_TYPE: + access_bits |= le32_to_cpu(ace->access_req); + break; + case ACCESS_DENIED_ACE_TYPE: + case ACCESS_DENIED_CALLBACK_ACE_TYPE: + denied |= ~access_bits & + le32_to_cpu(ace->access_req); + break; + } +next_ace: ace = (struct smb_ace *)((char *)ace + le16_to_cpu(ace->size)); } + access_bits &= ~denied; + if ((raw_daccess & FILE_GENERIC_EXECUTE_LE) && + S_ISREG(d_inode(path->dentry)->i_mode) && + (access_bits & GENERIC_READ_FLAGS) == GENERIC_READ_FLAGS) + access_bits |= FILE_EXECUTE; + granted = requested | access_bits; } - if (!uid) - sid_type = SIDUNIX_USER; - id_to_sid(uid, sid_type, &sid); - ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); - aces_size = acl_size - sizeof(struct smb_acl); + aces_size = pdacl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { if (aces_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE) @@ -1527,25 +1571,16 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, found = 1; break; } - if (!compare_sids(&sid_everyone, &ace->sid)) + if (!compare_sids(&sid_everyone, &ace->sid) || + !compare_sids(&sid_authusers, &ace->sid)) others_ace = ace; ace = (struct smb_ace *)((char *)ace + le16_to_cpu(ace->size)); } - if (*pdaccess & FILE_MAXIMAL_ACCESS_LE && found) { - granted = READ_CONTROL | WRITE_DAC | FILE_READ_ATTRIBUTES | - DELETE; - - granted |= le32_to_cpu(ace->access_req); - - if (!pdacl->num_aces) - granted = GENERIC_ALL_FLAGS; - } - if (IS_ENABLED(CONFIG_FS_POSIX_ACL)) { posix_acls = get_inode_acl(d_inode(path->dentry), ACL_TYPE_ACCESS); - if (!IS_ERR_OR_NULL(posix_acls) && !found) { + if (!IS_ERR_OR_NULL(posix_acls) && !found && !others_ace) { unsigned int id = -1; pa_entry = posix_acls->a_entries; @@ -1583,19 +1618,27 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, } } - switch (ace->type) { - case ACCESS_ALLOWED_ACE_TYPE: - access_bits = le32_to_cpu(ace->access_req); - break; - case ACCESS_DENIED_ACE_TYPE: - case ACCESS_DENIED_CALLBACK_ACE_TYPE: - access_bits = le32_to_cpu(~ace->access_req); - break; + if (!(*pdaccess & FILE_MAXIMAL_ACCESS_LE)) { + switch (ace->type) { + case ACCESS_ALLOWED_ACE_TYPE: + access_bits = le32_to_cpu(ace->access_req); + break; + case ACCESS_DENIED_ACE_TYPE: + case ACCESS_DENIED_CALLBACK_ACE_TYPE: + access_bits = le32_to_cpu(~ace->access_req); + break; + } } check_access_bits: - if (granted & - ~(access_bits | FILE_READ_ATTRIBUTES | READ_CONTROL | WRITE_DAC | DELETE)) { + if (strict) { + access_bits &= granted; + } else { + access_bits |= FILE_READ_ATTRIBUTES | READ_CONTROL | + WRITE_DAC | DELETE; + } + + if (granted & ~access_bits) { ksmbd_debug(SMB, "Access denied with winACL, granted : %x, access_req : %x\n", granted, le32_to_cpu(ace->access_req)); rc = -EACCES; diff --git a/fs/smb/server/smbacl.h b/fs/smb/server/smbacl.h index ab21ba2cd4df3..01810c16cc046 100644 --- a/fs/smb/server/smbacl.h +++ b/fs/smb/server/smbacl.h @@ -95,7 +95,8 @@ bool smb_inherit_flags(int flags, bool is_dir); int smb_inherit_dacl(struct ksmbd_conn *conn, const struct path *path, unsigned int uid, unsigned int gid); int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, - __le32 *pdaccess, int uid); + __le32 *pdaccess, __le32 raw_daccess, int uid, + bool strict); int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon, const struct path *path, struct smb_ntsd *pntsd, int ntsd_len, bool type_check, bool get_write); diff --git a/fs/smb/server/stats.h b/fs/smb/server/stats.h index b60c30c690770..8b32b8b4e8be4 100644 --- a/fs/smb/server/stats.h +++ b/fs/smb/server/stats.h @@ -9,12 +9,25 @@ #ifndef __KSMBD_STATS_H__ #define __KSMBD_STATS_H__ +#include "../common/smb2status.h" + #define KSMBD_COUNTER_MAX_REQS 19 enum { KSMBD_COUNTER_SESSIONS = 0, + KSMBD_COUNTER_SESSION_TIMEOUTS, KSMBD_COUNTER_TREE_CONNS, KSMBD_COUNTER_REQUESTS, + KSMBD_COUNTER_STATUS_SUCCESS, + KSMBD_COUNTER_STATUS_INFORMATIONAL, + KSMBD_COUNTER_STATUS_WARNING, + KSMBD_COUNTER_STATUS_ERROR, + KSMBD_COUNTER_ERROR_ACCESS_DENIED, + KSMBD_COUNTER_ERROR_NOT_FOUND, + KSMBD_COUNTER_ERROR_INVALID_PARAMETER, + KSMBD_COUNTER_ERROR_SHARING_VIOLATION, + KSMBD_COUNTER_ERROR_NOT_SUPPORTED, + KSMBD_COUNTER_ERROR_OTHER, KSMBD_COUNTER_READ_BYTES, KSMBD_COUNTER_WRITE_BYTES, KSMBD_COUNTER_FIRST_REQ, @@ -50,10 +63,49 @@ static inline void ksmbd_counter_sub(int type, s64 value) percpu_counter_sub(&ksmbd_counters.counters[type], value); } -static inline void ksmbd_counter_inc_reqs(unsigned int cmd) +static inline void ksmbd_counter_inc_reqs(unsigned int cmd, __le32 status) { - if (cmd < KSMBD_COUNTER_MAX_REQS) + unsigned int severity = le32_to_cpu(status) >> 30; + int type; + + switch (severity) { + case 0: + type = KSMBD_COUNTER_STATUS_SUCCESS; + break; + case 1: + type = KSMBD_COUNTER_STATUS_INFORMATIONAL; + break; + case 2: + type = KSMBD_COUNTER_STATUS_WARNING; + break; + default: + type = KSMBD_COUNTER_STATUS_ERROR; + break; + } + percpu_counter_inc(&ksmbd_counters.counters[type]); + + if (severity == 3) { + if (status == STATUS_ACCESS_DENIED) + type = KSMBD_COUNTER_ERROR_ACCESS_DENIED; + else if (status == STATUS_OBJECT_NAME_NOT_FOUND || + status == STATUS_NO_SUCH_FILE) + type = KSMBD_COUNTER_ERROR_NOT_FOUND; + else if (status == STATUS_INVALID_PARAMETER) + type = KSMBD_COUNTER_ERROR_INVALID_PARAMETER; + else if (status == STATUS_SHARING_VIOLATION) + type = KSMBD_COUNTER_ERROR_SHARING_VIOLATION; + else if (status == STATUS_NOT_SUPPORTED || + status == STATUS_NOT_IMPLEMENTED) + type = KSMBD_COUNTER_ERROR_NOT_SUPPORTED; + else + type = KSMBD_COUNTER_ERROR_OTHER; + percpu_counter_inc(&ksmbd_counters.counters[type]); + } + + if (cmd < KSMBD_COUNTER_MAX_REQS) { + percpu_counter_inc(&ksmbd_counters.counters[KSMBD_COUNTER_REQUESTS]); percpu_counter_inc(&ksmbd_counters.counters[KSMBD_COUNTER_FIRST_REQ + cmd]); + } } static inline s64 ksmbd_counter_sum(int type) @@ -66,7 +118,7 @@ static inline void ksmbd_counter_inc(int type) {} static inline void ksmbd_counter_dec(int type) {} static inline void ksmbd_counter_add(int type, s64 value) {} static inline void ksmbd_counter_sub(int type, s64 value) {} -static inline void ksmbd_counter_inc_reqs(unsigned int cmd) {} +static inline void ksmbd_counter_inc_reqs(unsigned int cmd, __le32 status) {} static inline s64 ksmbd_counter_sum(int type) { return 0; } #endif diff --git a/fs/smb/server/transport_ipc.c b/fs/smb/server/transport_ipc.c index 0c581b9624d3e..5ad33fbfcbeff 100644 --- a/fs/smb/server/transport_ipc.c +++ b/fs/smb/server/transport_ipc.c @@ -322,6 +322,15 @@ static int ipc_server_config_on_startup(struct ksmbd_startup_request *req) goto out; } server_conf.share_fake_fscaps = req->share_fake_fscaps; + + /* AAPL model string for Finder icon */ + if (req->aapl_model[0]) + strscpy(server_conf.aapl_model, req->aapl_model, + sizeof(server_conf.aapl_model)); + else + strscpy(server_conf.aapl_model, "Xserve", + sizeof(server_conf.aapl_model)); + ksmbd_init_domain(req->sub_auth); if (req->smb2_max_read) @@ -497,6 +506,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_rpc_command *resp = entry->response; + if (entry->msg_sz < sizeof(struct ksmbd_rpc_command)) + return -EINVAL; + if (check_add_overflow(sizeof(struct ksmbd_rpc_command), resp->payload_sz, &msg_sz)) return -EINVAL; @@ -506,6 +518,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_spnego_authen_response *resp = entry->response; + if (entry->msg_sz < sizeof(struct ksmbd_spnego_authen_response)) + return -EINVAL; + msg_sz = sizeof(struct ksmbd_spnego_authen_response) + resp->session_key_len + resp->spnego_blob_len; break; @@ -514,20 +529,33 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_share_config_response *resp = entry->response; - if (resp->payload_sz) { - if (resp->payload_sz < resp->veto_list_sz) - return -EINVAL; + if (entry->msg_sz < sizeof(struct ksmbd_share_config_response)) + return -EINVAL; - if (check_add_overflow(sizeof(struct ksmbd_share_config_response), - resp->payload_sz, &msg_sz)) - return -EINVAL; - } + if (strnlen(resp->share_name, sizeof(resp->share_name)) == + sizeof(resp->share_name)) + return -EINVAL; + + if (resp->veto_list_sz > resp->payload_sz) + return -EINVAL; + + if (resp->flags != KSMBD_SHARE_FLAG_INVALID && + !(resp->flags & KSMBD_SHARE_FLAG_PIPE) && + resp->payload_sz <= resp->veto_list_sz) + return -EINVAL; + + if (check_add_overflow(sizeof(struct ksmbd_share_config_response), + resp->payload_sz, &msg_sz)) + return -EINVAL; break; } case KSMBD_EVENT_LOGIN_REQUEST_EXT: { struct ksmbd_login_response_ext *resp = entry->response; + if (entry->msg_sz < sizeof(struct ksmbd_login_response_ext)) + return -EINVAL; + if (resp->ngroups) { if (resp->ngroups < 0 || resp->ngroups > NGROUPS_MAX) { diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index 13b711ea575d8..668fc333c5be3 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -39,6 +39,7 @@ struct tcp_transport { static const struct ksmbd_transport_ops ksmbd_tcp_transport_ops; static void tcp_stop_kthread(struct task_struct *kthread); +static void ksmbd_tcp_stop_listener(struct interface *iface); static struct interface *alloc_iface(char *ifname); static void ksmbd_tcp_disconnect(struct ksmbd_transport *t); @@ -315,13 +316,20 @@ static int ksmbd_tcp_run_kthread(struct interface *iface) int rc; struct task_struct *kthread; - kthread = kthread_run(ksmbd_kthread_fn, (void *)iface, "ksmbd-%s", - iface->name); + kthread = kthread_create(ksmbd_kthread_fn, (void *)iface, "ksmbd-%s", + iface->name); if (IS_ERR(kthread)) { rc = PTR_ERR(kthread); return rc; } + + /* + * The listener can exit after its socket is shutdown, so keep the + * task_struct alive until the caller has stopped it. + */ + get_task_struct(kthread); iface->ksmbd_kthread = kthread; + wake_up_process(kthread); return 0; } @@ -434,6 +442,11 @@ static void ksmbd_tcp_disconnect(struct ksmbd_transport *t) atomic_dec(&active_num_conn); } +static void ksmbd_tcp_shutdown(struct ksmbd_transport *t) +{ + kernel_sock_shutdown(TCP_TRANS(t)->sock, SHUT_RDWR); +} + static void tcp_destroy_socket(struct socket *ksmbd_socket) { int ret; @@ -580,12 +593,7 @@ static int ksmbd_netdev_event(struct notifier_block *nb, unsigned long event, if (iface && iface->state == IFACE_STATE_CONFIGURED) { ksmbd_debug(CONN, "netdev-down event: netdev(%s) is going down\n", iface->name); - kernel_sock_shutdown(iface->ksmbd_socket, SHUT_RDWR); - tcp_stop_kthread(iface->ksmbd_kthread); - iface->ksmbd_kthread = NULL; - sock_release(iface->ksmbd_socket); - iface->ksmbd_socket = NULL; - + ksmbd_tcp_stop_listener(iface); iface->state = IFACE_STATE_DOWN; break; } @@ -613,11 +621,25 @@ static void tcp_stop_kthread(struct task_struct *kthread) if (!kthread) return; - ret = kthread_stop(kthread); + ret = kthread_stop_put(kthread); if (ret) pr_err("failed to stop forker thread\n"); } +static void ksmbd_tcp_stop_listener(struct interface *iface) +{ + if (iface->ksmbd_socket) + kernel_sock_shutdown(iface->ksmbd_socket, SHUT_RDWR); + + tcp_stop_kthread(iface->ksmbd_kthread); + iface->ksmbd_kthread = NULL; + + if (iface->ksmbd_socket) { + sock_release(iface->ksmbd_socket); + iface->ksmbd_socket = NULL; + } +} + void ksmbd_tcp_destroy(void) { struct interface *iface, *tmp; @@ -625,6 +647,7 @@ void ksmbd_tcp_destroy(void) unregister_netdevice_notifier(&ksmbd_netdev_notifier); list_for_each_entry_safe(iface, tmp, &iface_list, entry) { + ksmbd_tcp_stop_listener(iface); list_del(&iface->entry); kfree(iface->name); kfree(iface); @@ -680,5 +703,6 @@ static const struct ksmbd_transport_ops ksmbd_tcp_transport_ops = { .read = ksmbd_tcp_read, .writev = ksmbd_tcp_writev, .disconnect = ksmbd_tcp_disconnect, + .shutdown = ksmbd_tcp_shutdown, .free_transport = ksmbd_tcp_free_transport, }; diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c index d324585c0566b..233d127f27015 100644 --- a/fs/smb/server/vfs.c +++ b/fs/smb/server/vfs.c @@ -297,9 +297,6 @@ static int check_lock_range(struct file *filp, loff_t start, loff_t end, struct file_lock_context *ctx = locks_inode_context(file_inode(filp)); int error = 0; - if (start == end) - return 0; - if (!ctx || list_empty_careful(&ctx->flc_posix)) return 0; diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c index a141025581afe..5eae49ed1c799 100644 --- a/fs/smb/server/vfs_cache.c +++ b/fs/smb/server/vfs_cache.c @@ -66,7 +66,7 @@ static const struct ksmbd_const_name ksmbd_lease_const_names[] = { static const struct ksmbd_const_name ksmbd_oplock_const_names[] = { {SMB2_OPLOCK_LEVEL_NONE, "OPLOCK_NONE"}, {SMB2_OPLOCK_LEVEL_II, "OPLOCK_II"}, - {SMB2_OPLOCK_LEVEL_EXCLUSIVE, "OPLOCK_EXECL"}, + {SMB2_OPLOCK_LEVEL_EXCLUSIVE, "OPLOCK_EXCLUSIVE"}, {SMB2_OPLOCK_LEVEL_BATCH, "OPLOCK_BATCH"}, }; @@ -76,14 +76,14 @@ static int proc_show_files(struct seq_file *m, void *v) unsigned int id; struct oplock_info *opinfo; - seq_printf(m, "#%-10s %-10s %-10s %-10s %-15s %-10s %-10s %s\n", + seq_printf(m, "#%-10s %-18s %-18s %-10s %-16s %-10s %-10s %s\n", "", "", "", "", "", "", "", ""); read_lock(&global_ft.lock); idr_for_each_entry(global_ft.idr, fp, id) { - seq_printf(m, "%#-10x %#-10llx %#-10llx %#-10x", + seq_printf(m, " %#-10x %#-18llx %#-18llx %#-10x", fp->tcon ? fp->tcon->id : 0, fp->persistent_id, fp->volatile_id, @@ -93,6 +93,7 @@ static int proc_show_files(struct seq_file *m, void *v) opinfo = rcu_dereference(fp->f_opinfo); if (opinfo) { const struct ksmbd_const_name *const_names; + const char *name; int count; unsigned int level; @@ -106,11 +107,14 @@ static int proc_show_files(struct seq_file *m, void *v) level = opinfo->level; } rcu_read_unlock(); - ksmbd_proc_show_const_name(m, " %-15s", - const_names, count, level); + name = ksmbd_proc_const_name(const_names, count, level); + if (name) + seq_printf(m, " %-16s", name); + else + seq_printf(m, " 0x%-14x", level); } else { rcu_read_unlock(); - seq_printf(m, " %-15s", " "); + seq_printf(m, " %-16s", " "); } seq_printf(m, " %#010x %#010x %s\n", @@ -124,7 +128,8 @@ static int proc_show_files(struct seq_file *m, void *v) static int create_proc_files(void) { - ksmbd_proc_create("files", proc_show_files, NULL); + if (!ksmbd_proc_create("files", proc_show_files, NULL)) + return -ENOMEM; return 0; } #else @@ -135,6 +140,16 @@ static bool durable_scavenger_running; static DEFINE_MUTEX(durable_scavenger_lock); static wait_queue_head_t dh_wq; +bool ksmbd_durable_scavenger_active(void) +{ + bool active; + + mutex_lock(&durable_scavenger_lock); + active = durable_scavenger_running; + mutex_unlock(&durable_scavenger_lock); + return active; +} + void ksmbd_set_fd_limit(unsigned long limit) { limit = min(limit, get_max_files()); @@ -497,6 +512,7 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) { struct file *filp; struct ksmbd_lock *smb_lock, *tmp_lock; + struct ksmbd_work *cn_work; fd_limit_close(); ksmbd_remove_durable_fd(fp); @@ -510,8 +526,9 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) if (!IS_ERR_OR_NULL(filp)) fput(filp); - /* because the reference count of fp is 0, it is guaranteed that - * there are not accesses to fp->lock_list. + /* + * The zero fp reference count serializes access to fp->lock_list, but + * the VFS may still have blocked requests chained below these locks. */ list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { struct ksmbd_conn *conn = smb_lock->conn; @@ -524,11 +541,58 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) ksmbd_conn_put(conn); } - list_del(&smb_lock->flist); + list_del_init(&smb_lock->flist); + ksmbd_vfs_posix_lock_unblock(smb_lock->fl); locks_free_lock(smb_lock->fl); kfree(smb_lock); } + /* + * Complete any CHANGE_NOTIFY left pending on this handle now that + * it is closed. KSMBD never completes CHANGE_NOTIFY spontaneously + * (no real change-notification backend), only on close -- matching + * genuine SMB2/macOS smbfs semantics and avoiding the Finder + * "directory changed, re-enumerate everything" loop. + * + * smb2_notify() on another connection can be adding to + * notify_pendings under fp->f_lock at the same time this handle is + * closed, and a client-sent CANCEL can concurrently be racing to + * claim the same entry via smb2_notify_cancel_fn() (smb2pdu.c). + * Pop one entry at a time under the lock via list_del_init() rather + * than a bulk list_splice_init(): list_del_init() leaves the node + * self-linked ("empty"), which is what the cancel path checks under + * the same lock to tell whether it lost the race -- a bulk splice + * would instead relink every entry into a shared local list, so an + * entry claimed here would still read as "not empty" to a racing + * cancel_fn, and both sides could end up freeing the same work. + * ksmbd_conn_write() can sleep (it takes conn's write mutex), so it + * must not be called while fp->f_lock is held -- release the lock + * before processing each popped entry, then reacquire it for the + * next. + */ + for (;;) { + spin_lock(&fp->f_lock); + if (list_empty(&fp->notify_pendings)) { + spin_unlock(&fp->f_lock); + break; + } + cn_work = list_first_entry(&fp->notify_pendings, + struct ksmbd_work, notify_entry); + list_del_init(&cn_work->notify_entry); + spin_unlock(&fp->f_lock); + + ksmbd_conn_write(cn_work); + /* + * release_async_work() removes cn_work from + * conn->async_requests, frees cancel_argv, and releases+zeroes + * async_id -- all needed before ksmbd_free_work_struct(), which + * only releases async_id itself if still nonzero (i.e. if this + * hadn't already been done). + */ + release_async_work(cn_work); + ksmbd_free_work_struct(cn_work); + } + /* * Drop fp's strong reference on conn (taken in ksmbd_open_fd() / * ksmbd_reopen_durable_fd()). Durable fps that reached the @@ -629,6 +693,11 @@ static struct ksmbd_file *ksmbd_fp_get(struct ksmbd_file *fp) return fp; } +struct ksmbd_file *ksmbd_file_get(struct ksmbd_file *fp) +{ + return ksmbd_fp_get(fp); +} + static struct ksmbd_file *__ksmbd_lookup_fd(struct ksmbd_file_table *ft, u64 id) { @@ -672,12 +741,25 @@ static void set_close_state_blocked_works(struct ksmbd_file *fp) spin_lock(&fp->f_lock); list_for_each_entry(cancel_work, &fp->blocked_works, fp_entry) { - cancel_work->state = KSMBD_WORK_CLOSED; - cancel_work->cancel_fn(cancel_work->cancel_argv); + if (xchg(&cancel_work->state, KSMBD_WORK_CLOSED) == + KSMBD_WORK_ACTIVE) + cancel_work->cancel_fn(cancel_work->cancel_argv); } spin_unlock(&fp->f_lock); } +void ksmbd_wake_session_blocked_works(struct ksmbd_session *sess) +{ + struct ksmbd_file_table *ft = &sess->file_table; + struct ksmbd_file *fp; + unsigned int id; + + read_lock(&ft->lock); + idr_for_each_entry(ft->idr, fp, id) + set_close_state_blocked_works(fp); + read_unlock(&ft->lock); +} + int ksmbd_close_fd(struct ksmbd_work *work, u64 id) { struct ksmbd_file *fp; @@ -1050,6 +1132,7 @@ struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp) INIT_LIST_HEAD(&fp->blocked_works); INIT_LIST_HEAD(&fp->node); INIT_LIST_HEAD(&fp->lock_list); + INIT_LIST_HEAD(&fp->notify_pendings); spin_lock_init(&fp->f_lock); mutex_init(&fp->readdir_lock); atomic_set(&fp->refcount, 1); @@ -1446,9 +1529,12 @@ void ksmbd_launch_ksmbd_durable_scavenger(void) server_conf.dh_task = kthread_run(ksmbd_durable_scavenger, (void *)NULL, "ksmbd-durable-scavenger"); - if (IS_ERR(server_conf.dh_task)) + if (IS_ERR(server_conf.dh_task)) { pr_err("cannot start conn thread, err : %ld\n", PTR_ERR(server_conf.dh_task)); + server_conf.dh_task = NULL; + durable_scavenger_running = false; + } mutex_unlock(&durable_scavenger_lock); } @@ -1471,7 +1557,7 @@ void ksmbd_stop_durable_scavenger(void) } /* - * ksmbd_vfs_copy_durable_owner - Copy owner info for durable reconnect + * ksmbd_vfs_set_durable_owner - Store owner info for durable replay/reconnect * @fp: ksmbd file pointer to store owner info * @user: user pointer to copy from * @@ -1480,8 +1566,8 @@ void ksmbd_stop_durable_scavenger(void) * * Return: 0 on success, or negative error code on failure */ -static int ksmbd_vfs_copy_durable_owner(struct ksmbd_file *fp, - struct ksmbd_user *user) +int ksmbd_vfs_set_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user) { char *name; @@ -1552,7 +1638,7 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, if (WARN_ON_ONCE(!fp->conn)) return false; - if (ksmbd_vfs_copy_durable_owner(fp, user)) + if (ksmbd_vfs_set_durable_owner(fp, user)) return false; /* @@ -1620,7 +1706,8 @@ void ksmbd_close_session_fds(struct ksmbd_work *work) int ksmbd_init_global_file_table(void) { - create_proc_files(); + if (create_proc_files()) + pr_warn("Unable to create files procfs entry\n"); return ksmbd_init_file_table(&global_ft); } @@ -1722,7 +1809,7 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) down_write(&ci->m_lock); list_for_each_entry_rcu(op, &ci->m_op_list, op_entry, lockdep_is_held(&ci->m_lock)) { - if (op->conn) + if (op->conn || op->o_fp != fp) continue; op->conn = ksmbd_conn_get(fp->conn); op->sess = work->sess; diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h index b9e27307a26c1..1c6fc4349b883 100644 --- a/fs/smb/server/vfs_cache.h +++ b/fs/smb/server/vfs_cache.h @@ -82,6 +82,13 @@ struct durable_owner { char *name; }; +#define KSMBD_LOCK_SEQ_ARRAY_SIZE 64 + +struct ksmbd_lock_sequence { + bool valid; + u8 sequence; +}; + struct ksmbd_file { struct file *filp; u64 persistent_id; @@ -101,6 +108,7 @@ struct ksmbd_file { __le32 saccess; __le32 coption; __le32 cdoption; + __le32 create_file_attributes; __u64 create_time; __u64 change_time; __u64 allocation_size; @@ -121,6 +129,8 @@ struct ksmbd_file { unsigned int durable_timeout; unsigned int durable_scavenger_timeout; + /* CREATE action returned when this durable handle was established. */ + __le32 create_action; /* if ls is happening on directory, below is valid*/ struct ksmbd_readdir_data readdir_data; @@ -132,9 +142,20 @@ struct ksmbd_file { bool is_persistent; bool is_resilient; bool durable_reconnect_disabled; + bool durable_replay_consumed; bool is_posix_ctxt; struct durable_owner owner; + __le16 channel_sequence; + unsigned int outstanding_requests; + unsigned int outstanding_pre_requests; + struct ksmbd_lock_sequence lock_seq[KSMBD_LOCK_SEQ_ARRAY_SIZE]; + + /* + * Pending CHANGE_NOTIFY completions for this handle, sent with + * STATUS_NOTIFY_CLEANUP when the handle is closed. + */ + struct list_head notify_pendings; }; static inline void set_ctx_actor(struct dir_context *ctx, @@ -167,6 +188,9 @@ struct ksmbd_file *ksmbd_lookup_fd_fast(struct ksmbd_work *work, u64 id); struct ksmbd_file *ksmbd_lookup_foreign_fd(struct ksmbd_work *work, u64 id); struct ksmbd_file *ksmbd_lookup_fd_slow(struct ksmbd_work *work, u64 id, u64 pid); +int ksmbd_vfs_set_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user); +struct ksmbd_file *ksmbd_file_get(struct ksmbd_file *fp); void ksmbd_fd_put(struct ksmbd_work *work, struct ksmbd_file *fp); struct ksmbd_inode *ksmbd_inode_lookup_lock(struct dentry *d); void ksmbd_inode_put(struct ksmbd_inode *ci); @@ -185,8 +209,10 @@ unsigned int ksmbd_open_durable_fd(struct ksmbd_file *fp); struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp); void ksmbd_launch_ksmbd_durable_scavenger(void); void ksmbd_stop_durable_scavenger(void); +bool ksmbd_durable_scavenger_active(void); void ksmbd_close_tree_conn_fds(struct ksmbd_work *work); void ksmbd_close_session_fds(struct ksmbd_work *work); +void ksmbd_wake_session_blocked_works(struct ksmbd_session *sess); int ksmbd_close_inode_fds(struct ksmbd_work *work, struct inode *inode); int ksmbd_init_global_file_table(void); void ksmbd_free_global_file_table(void); diff --git a/fs/smb/smbdirect/accept.c b/fs/smb/smbdirect/accept.c index 5297400058385..039c9bfbd8acd 100644 --- a/fs/smb/smbdirect/accept.c +++ b/fs/smb/smbdirect/accept.c @@ -145,11 +145,21 @@ int smbdirect_accept_connect_request(struct smbdirect_socket *sc, rdma_accept_failed: /* - * smbdirect_connection_destroy_qp() calls ib_drain_qp(), - * so that smbdirect_accept_negotiate_recv_done() will - * call smbdirect_connection_put_recv_io() + * The recv_io posted above is now owned by the QP (recv_io was set to + * NULL after a successful post). smbdirect_connection_destroy_qp() + * calls ib_drain_qp(), whose completion + * (smbdirect_accept_negotiate_recv_done) returns the recv_io to the + * free list via smbdirect_connection_put_recv_io(). It therefore MUST + * run BEFORE smbdirect_connection_destroy_mem_pools(): otherwise the + * posted recv_io is still outstanding when kmem_cache_destroy() runs + * ("Slab cache still has objects") and is later freed into an + * already-destroyed mempool (mempool_free_bulk NULL-ptr-deref). */ + smbdirect_connection_destroy_qp(sc); + smbdirect_connection_destroy_mem_pools(sc); + return ret; post_recv_io_failed: + /* post failed: recv_io was not accepted by the QP, still in hand */ if (recv_io) smbdirect_connection_put_recv_io(recv_io); get_recv_io_failed: diff --git a/fs/smb/smbdirect/connection.c b/fs/smb/smbdirect/connection.c index 8adf580975344..fe99537207176 100644 --- a/fs/smb/smbdirect/connection.c +++ b/fs/smb/smbdirect/connection.c @@ -403,12 +403,21 @@ void smbdirect_connection_destroy_qp(struct smbdirect_socket *sc) sc->ib.qp = NULL; rdma_destroy_qp(sc->rdma.cm_id); } + /* + * These CQs were created with ib_alloc_cq_any(), which arms an internal + * completion handler (ib_cq_poll_work for IB_POLL_WORKQUEUE). They MUST be + * torn down with ib_free_cq(), which cancel_work_sync()es that poll work + * before freeing the CQ. ib_destroy_cq() skips that step, so a completion + * posted late by the (software) provider — e.g. rxe posting an RNR error + * from rxe_receiver after rdma_destroy_qp() — re-queues ib_cq_poll_work on + * an already-freed CQ (KASAN slab-use-after-free in ib_cq_poll_work). + */ if (sc->ib.recv_cq) { - ib_destroy_cq(sc->ib.recv_cq); + ib_free_cq(sc->ib.recv_cq); sc->ib.recv_cq = NULL; } if (sc->ib.send_cq) { - ib_destroy_cq(sc->ib.send_cq); + ib_free_cq(sc->ib.send_cq); sc->ib.send_cq = NULL; } if (sc->ib.pd) { diff --git a/fs/smb/smbdirect/socket.c b/fs/smb/smbdirect/socket.c index 39cca7219c4df..bb02df6158b97 100644 --- a/fs/smb/smbdirect/socket.c +++ b/fs/smb/smbdirect/socket.c @@ -305,12 +305,26 @@ void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc, * disconnect all pending and ready sockets * * First we move ready sockets to pending again. + * + * Only a socket that was a listener (listen.backlog != -1) owns a + * populated listen.ready/pending list. Guarding on that also keeps + * lockdep quiet: without it, the listener holds sc->listen.lock while + * the loop recurses into each child psc, which takes psc->listen.lock. + * Those are always different instances of the same lock class -- a + * child never listens, so the nesting is strictly listener -> child + * and cannot really deadlock -- but lockdep only sees the class and + * reports "possible recursive locking". A child has empty listen + * lists and nothing to do here, so skipping it loses nothing, and a + * pending child stays on its listener's list for the free path + * (smbdirect_socket_destroy) to reap. */ - spin_lock_irqsave(&sc->listen.lock, flags); - list_splice_init(&sc->listen.ready, &sc->listen.pending); - list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) - smbdirect_socket_schedule_cleanup(psc, sc->first_error); - spin_unlock_irqrestore(&sc->listen.lock, flags); + if (sc->listen.backlog != -1) { /* was a listener */ + spin_lock_irqsave(&sc->listen.lock, flags); + list_splice_init(&sc->listen.ready, &sc->listen.pending); + list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) + smbdirect_socket_schedule_cleanup(psc, sc->first_error); + spin_unlock_irqrestore(&sc->listen.lock, flags); + } switch (sc->status) { case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: @@ -405,12 +419,20 @@ static void smbdirect_socket_cleanup_work(struct work_struct *work) * disconnect all pending and ready sockets * * First we move ready sockets to pending again. + * + * Guarded on listen.backlog != -1 for the same reason as in + * __smbdirect_socket_schedule_cleanup(): only a listener owns a + * populated listen list, and skipping the block for a child avoids + * nesting psc->listen.lock under a listener's listen.lock (different + * instances of one class -- harmless, but lockdep cannot tell). */ - spin_lock_irqsave(&sc->listen.lock, flags); - list_splice_init(&sc->listen.ready, &sc->listen.pending); - list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) - smbdirect_socket_schedule_cleanup(psc, sc->first_error); - spin_unlock_irqrestore(&sc->listen.lock, flags); + if (sc->listen.backlog != -1) { /* was a listener */ + spin_lock_irqsave(&sc->listen.lock, flags); + list_splice_init(&sc->listen.ready, &sc->listen.pending); + list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) + smbdirect_socket_schedule_cleanup(psc, sc->first_error); + spin_unlock_irqrestore(&sc->listen.lock, flags); + } switch (sc->status) { case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: @@ -473,6 +495,7 @@ static void smbdirect_socket_destroy(struct smbdirect_socket *sc) struct smbdirect_recv_io *recv_io; struct smbdirect_recv_io *recv_tmp; LIST_HEAD(all_list); + LIST_HEAD(pending_list); unsigned long flags; smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, @@ -530,24 +553,29 @@ static void smbdirect_socket_destroy(struct smbdirect_socket *sc) * disconnect all pending and ready sockets * * We move ready sockets to pending again. + * + * Capture them here -- rdma_lock_handler(sc->rdma.cm_id) is held above, + * so a concurrent CM CONNECT_REQUEST cannot add more; sc->listen.lock + * below only protects the list splice itself -- but DEFER releasing + * them until the listener's cm_id is destroyed: + * + * - smbdirect_socket_release() -> smbdirect_socket_destroy() takes the + * child's own rdma_lock_handler() lock (&id_priv->handler_mutex). + * The listener's and the child's cm_id are always different + * instances, so the nesting cannot really deadlock, but lockdep only + * sees one lock class and reports "possible recursive locking". + * + * - rdma_destroy_id() of a child before the listener's own + * rdma_destroy_id() below lets _cma_cancel_listens() walk the freed + * child id_priv (KASAN slab-use-after-free in __mutex_lock()). + * + * The children are independent sockets whose teardown does not need + * the listener's handler lock. */ spin_lock_irqsave(&sc->listen.lock, flags); - list_splice_tail_init(&sc->listen.ready, &all_list); - list_splice_tail_init(&sc->listen.pending, &all_list); + list_splice_tail_init(&sc->listen.ready, &pending_list); + list_splice_tail_init(&sc->listen.pending, &pending_list); spin_unlock_irqrestore(&sc->listen.lock, flags); - psockets = list_count_nodes(&all_list); - if (sc->listen.backlog != -1) /* was a listener */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "release %zu pending sockets\n", psockets); - list_for_each_entry_safe(psc, tsc, &all_list, accept.list) { - list_del_init(&psc->accept.list); - psc->accept.listener = NULL; - smbdirect_socket_release(psc); - } - if (sc->listen.backlog != -1) /* was a listener */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "released %zu pending sockets\n", psockets); - INIT_LIST_HEAD(&all_list); /* It's not possible for upper layer to get to reassembly */ if (sc->listen.backlog == -1) /* was not a listener */ @@ -577,6 +605,26 @@ static void smbdirect_socket_destroy(struct smbdirect_socket *sc) sc->rdma.cm_id = NULL; } + /* + * The listener's rdma_lock_handler() lock is dropped and its cm_id is + * destroyed, so it is safe to release the child sockets captured + * above: each release recurses into smbdirect_socket_destroy() and + * takes that child's own handler_mutex without nesting it under the + * listener's, and _cma_cancel_listens() can no longer reach them. + */ + psockets = list_count_nodes(&pending_list); + if (sc->listen.backlog != -1) /* was a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "release %zu pending sockets\n", psockets); + list_for_each_entry_safe(psc, tsc, &pending_list, accept.list) { + list_del_init(&psc->accept.list); + psc->accept.listener = NULL; + smbdirect_socket_release(psc); + } + if (sc->listen.backlog != -1) /* was a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "released %zu pending sockets\n", psockets); + if (sc->listen.backlog == -1) /* was not a listener */ smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "destroying mem pools\n"); diff --git a/fs/squashfs/cache.c b/fs/squashfs/cache.c index 67abd4dff2222..2807b80d46b72 100644 --- a/fs/squashfs/cache.c +++ b/fs/squashfs/cache.c @@ -299,7 +299,7 @@ int squashfs_copy_data(void *buffer, struct squashfs_cache_entry *entry, { int remaining = length; - if (length == 0) + if (length == 0 || offset < 0) return 0; else if (buffer == NULL) return min(length, entry->length - offset); diff --git a/fs/squashfs/xz_wrapper.c b/fs/squashfs/xz_wrapper.c index 0a4ff3ec9c8cd..6610af2414493 100644 --- a/fs/squashfs/xz_wrapper.c +++ b/fs/squashfs/xz_wrapper.c @@ -57,10 +57,10 @@ static void *squashfs_xz_comp_opts(struct squashfs_sb_info *msblk, opts->dict_size = le32_to_cpu(comp_opts->dictionary_size); - /* the dictionary size should be 2^n or 2^n+2^(n+1) */ + /* the dictionary size should be positive and 2^n or 2^n+2^(n+1) */ n = ffs(opts->dict_size) - 1; - if (opts->dict_size != (1 << n) && opts->dict_size != (1 << n) + - (1 << (n + 1))) { + if (opts->dict_size <= 0 || (opts->dict_size != (1 << n) && + opts->dict_size != (1 << n) + (1 << (n + 1)))) { err = -EIO; goto out; } diff --git a/fs/stat.c b/fs/stat.c index 89909746bed19..c461c30542340 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -53,7 +53,7 @@ void fill_mg_cmtime(struct kstat *stat, u32 request_mask, struct inode *inode) } stat->mtime = inode_get_mtime(inode); - stat->ctime.tv_sec = inode->i_ctime_sec; + stat->ctime.tv_sec = inode_get_ctime_sec(inode); stat->ctime.tv_nsec = (u32)atomic_read(pcn); if (!(stat->ctime.tv_nsec & I_CTIME_QUERIED)) stat->ctime.tv_nsec = ((u32)atomic_fetch_or(I_CTIME_QUERIED, pcn)); diff --git a/fs/super.c b/fs/super.c index ffdcc6a2e0deb..addf3ad54a583 100644 --- a/fs/super.c +++ b/fs/super.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include /* for the emergency remount stuff */ @@ -102,7 +103,7 @@ static bool super_flags(const struct super_block *sb, unsigned int flags) * creation will succeed and SB_BORN is set by vfs_get_tree() or we're * woken and we'll see SB_DYING. * - * The caller must have acquired a temporary reference on @sb->s_count. + * The caller must have acquired a temporary reference on @sb->s_passive. * * Return: The function returns true if SB_BORN was set and with * s_umount held. The function returns false if SB_DYING was @@ -367,7 +368,7 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags, spin_lock_init(&s->s_inode_wblist_lock); fserror_mount(s); - s->s_count = 1; + refcount_set(&s->s_passive, 1); atomic_set(&s->s_active, 1); mutex_init(&s->s_vfs_rename_mutex); lockdep_set_class(&s->s_vfs_rename_mutex, &type->s_vfs_rename_key); @@ -403,33 +404,28 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags, /* Superblock refcounting */ /* - * Drop a superblock's refcount. The caller must hold sb_lock. + * Drop a superblock's passive reference. Must be called WITHOUT sb_lock held; + * put_super() acquires sb_lock itself when the final reference is dropped. */ -static void __put_super(struct super_block *s) +void put_super(struct super_block *s) { - if (!--s->s_count) { + if (refcount_dec_and_test(&s->s_passive)) { + struct file_system_type *type = s->s_type; + + spin_lock(&sb_lock); list_del_init(&s->s_list); + hlist_del_init(&s->s_instances); + spin_unlock(&sb_lock); + WARN_ON(s->s_dentry_lru.node); WARN_ON(s->s_inode_lru.node); WARN_ON(s->s_mounts); call_rcu(&s->rcu, destroy_super_rcu); + /* The unlink above may touch type->fs_supers, so drop it last. */ + put_filesystem(type); } } -/** - * put_super - drop a temporary reference to superblock - * @sb: superblock in question - * - * Drops a temporary reference, frees superblock if there's no - * references left. - */ -void put_super(struct super_block *sb) -{ - spin_lock(&sb_lock); - __put_super(sb); - spin_unlock(&sb_lock); -} - static void kill_super_notify(struct super_block *sb) { lockdep_assert_not_held(&sb->s_umount); @@ -438,24 +434,17 @@ static void kill_super_notify(struct super_block *sb) if (sb->s_flags & SB_DEAD) return; - /* - * Remove it from @fs_supers so it isn't found by new - * sget_fc() walkers anymore. Any concurrent mounter still - * managing to grab a temporary reference is guaranteed to - * already see SB_DYING and will wait until we notify them about - * SB_DEAD. - */ - spin_lock(&sb_lock); - hlist_del_init(&sb->s_instances); - spin_unlock(&sb_lock); - /* * Let concurrent mounts know that this thing is really dead. - * We don't need @sb->s_umount here as every concurrent caller - * will see SB_DYING and either discard the superblock or wait - * for SB_DEAD. + * sget_fc() skips SB_DEAD superblocks and calls test() under + * sb_lock, so set it under sb_lock: once we return no test() + * runs on this superblock anymore and none will start. Everyone + * else already saw SB_DYING and either discarded the superblock + * or waits for SB_DEAD. */ + spin_lock(&sb_lock); super_wake(sb, SB_DEAD); + spin_unlock(&sb_lock); } /** @@ -478,15 +467,10 @@ void deactivate_locked_super(struct super_block *s) kill_super_notify(s); - /* - * Since list_lru_destroy() may sleep, we cannot call it from - * put_super(), where we hold the sb_lock. Therefore we destroy - * the lru lists right now. - */ + /* list_lru_destroy() may sleep; put_super() callers may not. */ list_lru_destroy(&s->s_dentry_lru); list_lru_destroy(&s->s_inode_lru); - put_filesystem(fs); put_super(s); } else { super_unlock_excl(s); @@ -529,7 +513,7 @@ static bool grab_super(struct super_block *sb) { bool locked; - sb->s_count++; + refcount_inc(&sb->s_passive); spin_unlock(&sb_lock); locked = super_lock_excl(sb); if (locked) { @@ -556,7 +540,7 @@ static bool grab_super(struct super_block *sb) * lock held in read mode in case of success. On successful return, * the caller must drop the s_umount lock when done. * - * Note that unlike get_super() et.al. this one does *not* bump ->s_count. + * Note that unlike get_super() et.al. this one does *not* bump ->s_passive. * The reason why it's safe is that we are OK with doing trylock instead * of down_read(). There's a couple of places that are OK with that, but * it's very much not a general-purpose interface. @@ -673,12 +657,12 @@ void generic_shutdown_super(struct super_block *sb) } /* * Broadcast to everyone that grabbed a temporary reference to this - * superblock before we removed it from @fs_supers that the superblock - * is dying. Every walker of @fs_supers outside of sget_fc() will now - * discard this superblock and treat it as dead. + * superblock that it is dying. Every walker of @fs_supers outside + * of sget_fc() will now discard this superblock and treat it as + * dead. * - * We leave the superblock on @fs_supers so it can be found by - * sget_fc() until we passed sb->kill_sb(). + * sget_fc() keeps finding the superblock until SB_DEAD is set, so + * a concurrent mounter waits until we passed sb->kill_sb(). */ super_wake(sb, SB_DYING); super_unlock_excl(sb); @@ -757,6 +741,9 @@ struct super_block *sget_fc(struct fs_context *fc, spin_lock(&sb_lock); if (test) { hlist_for_each_entry(old, &fc->fs_type->fs_supers, s_instances) { + /* Only unlinked at the last passive reference. */ + if (super_flags(old, SB_DEAD)) + continue; if (test(old, fc)) goto share_extant_sb; } @@ -851,14 +838,17 @@ static void __iterate_supers(void (*f)(struct super_block *, void *), void *arg, struct super_block *sb, *p = NULL; bool excl = flags & SUPER_ITER_EXCL; - guard(spinlock)(&sb_lock); + spin_lock(&sb_lock); for (sb = first_super(flags); !list_entry_is_head(sb, &super_blocks, s_list); sb = next_super(sb, flags)) { if (super_flags(sb, SB_DYING)) continue; - sb->s_count++; + + if (!refcount_inc_not_zero(&sb->s_passive)) + continue; + spin_unlock(&sb_lock); if (flags & SUPER_ITER_UNLOCKED) { @@ -868,13 +858,14 @@ static void __iterate_supers(void (*f)(struct super_block *, void *), void *arg, super_unlock(sb, excl); } - spin_lock(&sb_lock); if (p) - __put_super(p); + put_super(p); p = sb; + spin_lock(&sb_lock); } + spin_unlock(&sb_lock); if (p) - __put_super(p); + put_super(p); } void iterate_supers(void (*f)(struct super_block *, void *), void *arg) @@ -903,7 +894,9 @@ void iterate_supers_type(struct file_system_type *type, if (super_flags(sb, SB_DYING)) continue; - sb->s_count++; + if (!refcount_inc_not_zero(&sb->s_passive)) + continue; + spin_unlock(&sb_lock); locked = super_lock_shared(sb); @@ -912,14 +905,14 @@ void iterate_supers_type(struct file_system_type *type, super_unlock_shared(sb); } - spin_lock(&sb_lock); if (p) - __put_super(p); + put_super(p); p = sb; + spin_lock(&sb_lock); } - if (p) - __put_super(p); spin_unlock(&sb_lock); + if (p) + put_super(p); } EXPORT_SYMBOL(iterate_supers_type); @@ -935,15 +928,17 @@ struct super_block *user_get_super(dev_t dev, bool excl) if (sb->s_dev != dev) continue; - sb->s_count++; + if (!refcount_inc_not_zero(&sb->s_passive)) + continue; + spin_unlock(&sb_lock); locked = super_lock(sb, excl); if (locked) return sb; + put_super(sb); spin_lock(&sb_lock); - __put_super(sb); break; } spin_unlock(&sb_lock); @@ -1374,9 +1369,7 @@ static struct super_block *bdev_super_lock(struct block_device *bdev, bool excl) lockdep_assert_not_held(&bdev->bd_disk->open_mutex); /* Make sure sb doesn't go away from under us */ - spin_lock(&sb_lock); - sb->s_count++; - spin_unlock(&sb_lock); + refcount_inc(&sb->s_passive); mutex_unlock(&bdev->bd_holder_lock); @@ -2018,7 +2011,7 @@ int freeze_super(struct super_block *sb, enum freeze_holder who, const void *fre int ret; if (!super_lock_excl(sb)) { - WARN_ON_ONCE("Dying superblock while freezing!"); + WARN_ONCE(1, "Dying superblock while freezing!"); return -EINVAL; } atomic_inc(&sb->s_active); @@ -2122,11 +2115,14 @@ static int thaw_super_locked(struct super_block *sb, enum freeze_holder who, goto out_unlock; /* - * All freezers share a single active reference. - * So just unlock in case there are any left. + * All freezers share a single active reference. If other freezers + * remain, drop our hold and report success; the superblock stays + * frozen until the last holder thaws it. */ - if (freeze_dec(sb, who)) + if (freeze_dec(sb, who)) { + error = 0; goto out_unlock; + } if (sb_rdonly(sb)) { sb->s_writers.frozen = SB_UNFROZEN; @@ -2182,7 +2178,7 @@ int thaw_super(struct super_block *sb, enum freeze_holder who, const void *freeze_owner) { if (!super_lock_excl(sb)) { - WARN_ON_ONCE("Dying superblock while thawing!"); + WARN_ONCE(1, "Dying superblock while thawing!"); return -EINVAL; } return thaw_super_locked(sb, who, freeze_owner); diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index a524584353279..9ce620a310169 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -438,6 +438,8 @@ static inline struct eventfs_inode *init_ei(struct eventfs_inode *ei, const char if (!ei->name) return NULL; kref_init(&ei->kref); + INIT_LIST_HEAD(&ei->children); + INIT_LIST_HEAD(&ei->list); return ei; } @@ -725,8 +727,6 @@ struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode ei->entries = entries; ei->nr_entries = size; ei->data = data; - INIT_LIST_HEAD(&ei->children); - INIT_LIST_HEAD(&ei->list); scoped_guard(mutex, &eventfs_mutex) { if (!parent->is_freed) @@ -798,9 +798,6 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry ei->attr.uid = uid; ei->attr.gid = gid; - INIT_LIST_HEAD(&ei->children); - INIT_LIST_HEAD(&ei->list); - ti = get_tracefs(inode); ti->flags |= TRACEFS_EVENT_INODE; ti->private = ei; diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c index a4a0158f712d3..1f770795ee70f 100644 --- a/fs/ubifs/auth.c +++ b/fs/ubifs/auth.c @@ -217,7 +217,7 @@ int ubifs_sb_verify_signature(struct ubifs_info *c, signode = snod->node; - if (le32_to_cpu(signode->len) > snod->len + sizeof(struct ubifs_sig_node)) { + if (le32_to_cpu(signode->len) > snod->len - sizeof(struct ubifs_sig_node)) { ubifs_err(c, "invalid signature len %d", le32_to_cpu(signode->len)); err = -EINVAL; goto out_destroy; diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 67bcf83758c8f..e1fdf9f9c9f50 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -336,65 +336,6 @@ const struct address_space_operations udf_aops = { .migrate_folio = buffer_migrate_folio, }; -/* - * Expand file stored in ICB to a normal one-block-file - * - * This function requires i_mutex held - */ -int udf_expand_file_adinicb(struct inode *inode) -{ - struct folio *folio; - struct udf_inode_info *iinfo = UDF_I(inode); - int err; - - WARN_ON_ONCE(!inode_is_locked(inode)); - if (!iinfo->i_lenAlloc) { - down_write(&iinfo->i_data_sem); - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) - iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; - else - iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; - up_write(&iinfo->i_data_sem); - mark_inode_dirty(inode); - return 0; - } - - folio = __filemap_get_folio(inode->i_mapping, 0, - FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); - if (IS_ERR(folio)) - return PTR_ERR(folio); - - if (!folio_test_uptodate(folio)) - udf_adinicb_read_folio(folio); - down_write(&iinfo->i_data_sem); - memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, - iinfo->i_lenAlloc); - iinfo->i_lenAlloc = 0; - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) - iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; - else - iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; - folio_mark_dirty(folio); - folio_unlock(folio); - up_write(&iinfo->i_data_sem); - err = filemap_fdatawrite(inode->i_mapping); - if (err) { - /* Restore everything back so that we don't lose data... */ - folio_lock(folio); - down_write(&iinfo->i_data_sem); - memcpy_from_folio(iinfo->i_data + iinfo->i_lenEAttr, - folio, 0, inode->i_size); - folio_unlock(folio); - iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; - iinfo->i_lenAlloc = inode->i_size; - up_write(&iinfo->i_data_sem); - } - folio_put(folio); - mark_inode_dirty(inode); - - return err; -} - #define UDF_MAP_CREATE 0x01 /* Mapping can allocate new blocks */ #define UDF_MAP_NOPREALLOC 0x02 /* Do not preallocate blocks */ @@ -455,6 +396,76 @@ static int udf_map_block(struct inode *inode, struct udf_map_rq *map) return ret; } +/* + * Expand file stored in ICB to a normal one-block-file + * + * This function requires i_mutex held + */ +int udf_expand_file_adinicb(struct inode *inode) +{ + struct folio *folio; + struct udf_inode_info *iinfo = UDF_I(inode); + struct udf_map_rq map = { + .lblk = 0, + .iflags = UDF_MAP_CREATE, + }; + int err; + + WARN_ON_ONCE(!inode_is_locked(inode)); + if (!iinfo->i_lenAlloc) { + down_write(&iinfo->i_data_sem); + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + up_write(&iinfo->i_data_sem); + mark_inode_dirty(inode); + return 0; + } + + folio = __filemap_get_folio(inode->i_mapping, 0, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (!folio_test_uptodate(folio)) + udf_adinicb_read_folio(folio); + down_write(&iinfo->i_data_sem); + memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, + iinfo->i_lenAlloc); + iinfo->i_lenAlloc = 0; + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + up_write(&iinfo->i_data_sem); + + /* Allocate the block underlying the data */ + err = udf_map_block(inode, &map); + if (err < 0) + goto restore; + + folio_mark_dirty(folio); + folio_unlock(folio); + err = filemap_fdatawrite(inode->i_mapping); + if (err) { + /* Restore everything back so that we don't lose data... */ + folio_lock(folio); +restore: + down_write(&iinfo->i_data_sem); + memcpy_from_folio(iinfo->i_data + iinfo->i_lenEAttr, + folio, 0, inode->i_size); + iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; + iinfo->i_lenAlloc = inode->i_size; + up_write(&iinfo->i_data_sem); + folio_unlock(folio); + } + folio_put(folio); + mark_inode_dirty(inode); + + return err; +} + static int __udf_get_block(struct inode *inode, sector_t block, struct buffer_head *bh_result, int flags) { @@ -531,7 +542,7 @@ static int udf_do_extend_file(struct inode *inode, sb->s_blocksize - 1) & ~(sb->s_blocksize - 1)); iinfo->i_lenExtents = (iinfo->i_lenExtents + sb->s_blocksize - 1) & - ~(sb->s_blocksize - 1); + ~((u64)sb->s_blocksize - 1); } add = 0; diff --git a/fs/udf/partition.c b/fs/udf/partition.c index 2b85c9501bed8..ad8dcedca263d 100644 --- a/fs/udf/partition.c +++ b/fs/udf/partition.c @@ -55,7 +55,7 @@ uint32_t udf_get_pblock_virt15(struct super_block *sb, uint32_t block, map = &sbi->s_partmaps[partition]; vdata = &map->s_type_specific.s_virtual; - if (block > vdata->s_num_entries) { + if (block >= vdata->s_num_entries) { udf_debug("Trying to access block beyond end of VAT (%u max %u)\n", block, vdata->s_num_entries); return 0xFFFFFFFF; diff --git a/fs/udf/super.c b/fs/udf/super.c index 7b85f5a2b79fc..9686078bba649 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -2054,6 +2054,17 @@ static int udf_load_vrs(struct super_block *sb, struct udf_options *uopt, return 0; } +static void udf_mark_buffer_dirty(struct buffer_head *bh) +{ + /* + * We set buffer uptodate unconditionally here to avoid spurious + * warnings from mark_buffer_dirty() when previous EIO has marked + * the buffer as !uptodate + */ + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); +} + static void udf_finalize_lvid(struct logicalVolIntegrityDesc *lvid) { struct timespec64 ts; @@ -2089,7 +2100,7 @@ static void udf_open_lvid(struct super_block *sb) UDF_SET_FLAG(sb, UDF_FLAG_INCONSISTENT); udf_finalize_lvid(lvid); - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; mutex_unlock(&sbi->s_alloc_mutex); /* Make opening of filesystem visible on the media immediately */ @@ -2122,14 +2133,8 @@ static void udf_close_lvid(struct super_block *sb) if (!UDF_QUERY_FLAG(sb, UDF_FLAG_INCONSISTENT)) lvid->integrityType = cpu_to_le32(LVID_INTEGRITY_TYPE_CLOSE); - /* - * We set buffer uptodate unconditionally here to avoid spurious - * warnings from mark_buffer_dirty() when previous EIO has marked - * the buffer as !uptodate - */ - set_buffer_uptodate(bh); udf_finalize_lvid(lvid); - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; mutex_unlock(&sbi->s_alloc_mutex); /* Make closing of filesystem visible on the media immediately */ @@ -2411,7 +2416,7 @@ static int udf_sync_fs(struct super_block *sb, int wait) * Blockdevice will be synced later so we don't have to submit * the buffer for IO */ - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; } mutex_unlock(&sbi->s_alloc_mutex); diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index fe03745d09b18..a05d1888a2bab 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -36,6 +36,8 @@ static int udf_pc_to_char(struct super_block *sb, unsigned char *from, /* Reserve one byte for terminating \0 */ tolen--; while (elen < fromlen) { + if (fromlen - elen < sizeof(struct pathComponent)) + return -EIO; pc = (struct pathComponent *)(from + elen); elen += sizeof(struct pathComponent); switch (pc->componentType) { diff --git a/fs/ufs/cylinder.c b/fs/ufs/cylinder.c index a2813270c303e..b930ee1cf8530 100644 --- a/fs/ufs/cylinder.c +++ b/fs/ufs/cylinder.c @@ -68,6 +68,16 @@ static bool ufs_read_cylinder(struct super_block *sb, ucpi->c_clustersumoff = fs32_to_cpu(sb, ucg->cg_u.cg_44.cg_clustersumoff); ucpi->c_clusteroff = fs32_to_cpu(sb, ucg->cg_u.cg_44.cg_clusteroff); ucpi->c_nclusterblks = fs32_to_cpu(sb, ucg->cg_u.cg_44.cg_nclusterblks); + + /* these on-disk values become array and bitmap indices */ + if (ucpi->c_cgx != cgno || + ucpi->c_rotor >= uspi->s_fpg || + ucpi->c_frotor >= uspi->s_fpg || + ucpi->c_irotor >= uspi->s_ipg) { + ufs_error(sb, __func__, + "inconsistent metadata in cylinder group %u\n", cgno); + goto failed; + } UFSD("EXIT\n"); return true; diff --git a/fs/ufs/dir.c b/fs/ufs/dir.c index e62fe56676710..ce43cf20b07c9 100644 --- a/fs/ufs/dir.c +++ b/fs/ufs/dir.c @@ -590,7 +590,7 @@ int ufs_empty_dir(struct inode * inode) kaddr = ufs_get_folio(inode, i, &folio); if (IS_ERR(kaddr)) - continue; + return 0; de = (struct ufs_dir_entry *)kaddr; kaddr += ufs_last_byte(inode, i) - UFS_DIR_REC_LEN(1); diff --git a/fs/ufs/super.c b/fs/ufs/super.c index c4831a8b9b3fb..7deecb3954e23 100644 --- a/fs/ufs/super.c +++ b/fs/ufs/super.c @@ -1199,6 +1199,15 @@ static int ufs_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_maxbytes = ufs_max_bytes(sb); sb->s_max_links = UFS_LINK_MAX; + ufs_setup_cstotal(sb); + /* + * Read cylinder group structures + */ + if (!sb_rdonly(sb)) + if (!ufs_read_cylinder_structures(sb)) + goto failed; + + /* create the root dentry last, once UFS_SB(sb) is fully set up */ inode = ufs_iget(sb, UFS_ROOTINO); if (IS_ERR(inode)) { ret = PTR_ERR(inode); @@ -1210,14 +1219,6 @@ static int ufs_fill_super(struct super_block *sb, struct fs_context *fc) goto failed; } - ufs_setup_cstotal(sb); - /* - * Read cylinder group structures - */ - if (!sb_rdonly(sb)) - if (!ufs_read_cylinder_structures(sb)) - goto failed; - UFSD("EXIT\n"); return 0; diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 86c5c09a5db4f..b6288395f8533 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -325,6 +325,13 @@ xfs_attr3_leaf_verify_entry( */ if (ent->flags & XFS_ATTR_LOCAL) { lentry = xfs_attr3_leaf_name_local(leaf, idx); + + /* Validate lentry pointer is within bounds before field access */ + if ((char *)lentry >= buf_end) + return __this_address; + if ((char *)lentry + offsetof(struct xfs_attr_leaf_name_local, nameval) > buf_end) + return __this_address; + namesize = xfs_attr_leaf_entsize_local(lentry->namelen, be16_to_cpu(lentry->valuelen)); name_end = (char *)lentry + namesize; @@ -332,6 +339,13 @@ xfs_attr3_leaf_verify_entry( return __this_address; } else { rentry = xfs_attr3_leaf_name_remote(leaf, idx); + + /* Validate rentry pointer is within bounds before field access */ + if ((char *)rentry >= buf_end) + return __this_address; + if ((char *)rentry + offsetof(struct xfs_attr_leaf_name_remote, name) > buf_end) + return __this_address; + namesize = xfs_attr_leaf_entsize_remote(rentry->namelen); name_end = (char *)rentry + namesize; if (rentry->namelen == 0) diff --git a/fs/xfs/libxfs/xfs_btree_mem.c b/fs/xfs/libxfs/xfs_btree_mem.c index 37136a70e56d6..1d83a4251ceeb 100644 --- a/fs/xfs/libxfs/xfs_btree_mem.c +++ b/fs/xfs/libxfs/xfs_btree_mem.c @@ -117,6 +117,7 @@ xfbtree_init( struct xfs_buftarg *btp, const struct xfs_btree_ops *ops) { + unsigned long long owner = xfbt->owner; unsigned int blocklen = xfbtree_rec_bytes(mp, ops); unsigned int keyptr_len; int error; @@ -133,6 +134,7 @@ xfbtree_init( memset(xfbt, 0, sizeof(*xfbt)); xfbt->target = btp; + xfbt->owner = owner; /* Set up min/maxrecs for this btree. */ keyptr_len = ops->key_len + sizeof(__be64); diff --git a/fs/xfs/libxfs/xfs_btree_staging.c b/fs/xfs/libxfs/xfs_btree_staging.c index c3c7ea54895a6..c5b7613e5216a 100644 --- a/fs/xfs/libxfs/xfs_btree_staging.c +++ b/fs/xfs/libxfs/xfs_btree_staging.c @@ -337,8 +337,10 @@ xfs_btree_bload_prep_block( xfs_btree_set_sibling(cur, *blockp, &new_ptr, XFS_BB_RIGHTSIB); ret = xfs_btree_bload_drop_buf(bbl, buffers_list, bpp); - if (ret) + if (ret) { + xfs_buf_relse(new_bp); return ret; + } /* Initialize the new btree block. */ xfs_btree_init_block_cur(cur, new_bp, level, nr_this_block); diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index 9debb95d86fa0..98f02d68e938f 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -130,7 +130,7 @@ xfs_da_state_reset( state->mp = state->args->dp->i_mount; } -static inline int xfs_dabuf_nfsb(struct xfs_mount *mp, int whichfork) +inline int xfs_dabuf_nfsb(struct xfs_mount *mp, int whichfork) { if (whichfork == XFS_DATA_FORK) return mp->m_dir_geo->fsbcount; diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index afcf2d3c7a21c..a718b1ceb0aab 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -244,4 +244,6 @@ xfs_failaddr_t xfs_da3_node_header_check(struct xfs_buf *bp, xfs_ino_t owner); extern struct kmem_cache *xfs_da_state_cache; +int xfs_dabuf_nfsb(struct xfs_mount *mp, int whichfork); + #endif /* __XFS_DA_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 89501e8bd2f84..843c333044415 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -583,7 +583,7 @@ xfs_defer_finish_one( const struct xfs_defer_op_type *ops = dfp->dfp_ops; struct xfs_btree_cur *state = NULL; struct list_head *li, *n; - int error; + int error = 0; trace_xfs_defer_pending_finish(tp->t_mountp, dfp); diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 3efed37cb98a8..49eda8d0994de 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -959,6 +959,16 @@ xmi_can_exchange_reflink_flags( { struct xfs_mount *mp = req->ip1->i_mount; + /* + * The INO1_WRITTEN optimization can skip exchanging hole and + * unwritten mappings, which means we cannot guarantee that all + * shared extents actually moved to the other file. Clearing the + * reflink flag of an inode that still holds shared extents breaks + * the CoW write path, so refuse to exchange the flags in that case. + */ + if (req->flags & XFS_EXCHMAPS_INO1_WRITTEN) + return false; + /* * The INO1_WRITTEN optimization can skip exchanging hole and * unwritten mappings, which means we cannot guarantee that all diff --git a/fs/xfs/libxfs/xfs_metafile.c b/fs/xfs/libxfs/xfs_metafile.c index 71f004e9dc645..1f54d39003c27 100644 --- a/fs/xfs/libxfs/xfs_metafile.c +++ b/fs/xfs/libxfs/xfs_metafile.c @@ -297,14 +297,14 @@ xfs_metafile_resv_init( goto out_unlock; /* - * Space taken by the per-AG metadata btrees are accounted on-disk as - * used space. We therefore only hide the space that is reserved but - * not used by the trees. + * Space taken by metadata btrees are accounted on-disk as used space. + * We therefore only hide the space that is reserved but not used by + * the trees. */ if (used > target) target = used; else if (target > dblocks_avail) - target = dblocks_avail; + target = max(dblocks_avail, used); hidden_space = target - used; error = xfs_dec_fdblocks(mp, hidden_space, true); diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index 8d111c9b6527e..a2f2f5fa640e4 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -193,7 +193,7 @@ xfs_parent_addname( const struct xfs_name *parent_name, struct xfs_inode *child) { - int error; + int error, local; error = xfs_parent_iread_extents(tp, child); if (error) @@ -203,6 +203,10 @@ xfs_parent_addname( xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, I_INO(child), parent_name); + /* Growing the attr fork needs a real reservation in args->total. */ + ppargs->args.total = xfs_attr_calc_size(&ppargs->args, &local); + ASSERT(local); + return xfs_attr_setname(&ppargs->args, 0); } @@ -239,7 +243,7 @@ xfs_parent_replacename( const struct xfs_name *new_name, struct xfs_inode *child) { - int error; + int error, local; error = xfs_parent_iread_extents(tp, child); if (error) @@ -249,6 +253,10 @@ xfs_parent_replacename( xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, I_INO(child), old_name); + /* Growing the attr fork needs a real reservation in args->total. */ + ppargs->args.total = xfs_attr_calc_size(&ppargs->args, &local); + ASSERT(local); + xfs_inode_to_parent_rec(&ppargs->new_rec, new_dp); ppargs->args.new_name = new_name->name; diff --git a/fs/xfs/libxfs/xfs_rtrefcount_btree.c b/fs/xfs/libxfs/xfs_rtrefcount_btree.c index 22acc1411aacd..dcc89b8e149b0 100644 --- a/fs/xfs/libxfs/xfs_rtrefcount_btree.c +++ b/fs/xfs/libxfs/xfs_rtrefcount_btree.c @@ -489,8 +489,11 @@ xfs_rtrefcountbt_maxlevels_ondisk(void) minrecs[0] = xfs_rtrefcountbt_block_maxrecs(blocklen, true) / 2; minrecs[1] = xfs_rtrefcountbt_block_maxrecs(blocklen, false) / 2; - /* We need at most one record for every block in an rt group. */ - return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_RGBLOCKS); + /* + * We need at most one record for every block in an rt group, and + * one extra level for the inode root. + */ + return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_RGBLOCKS) + 1; } int __init @@ -614,7 +617,7 @@ xfs_rtrefcountbt_from_disk( fpp = xfs_rtrefcount_droot_ptr_addr(dblock, 1, maxrecs); tpp = xfs_rtrefcount_broot_ptr_addr(mp, rblock, 1, rblocklen); numrecs = be16_to_cpu(dblock->bb_numrecs); - memcpy(tkp, fkp, 2 * sizeof(*fkp) * numrecs); + memcpy(tkp, fkp, sizeof(*fkp) * numrecs); memcpy(tpp, fpp, sizeof(*fpp) * numrecs); } else { frp = xfs_rtrefcount_droot_rec_addr(dblock, 1); @@ -700,7 +703,7 @@ xfs_rtrefcountbt_to_disk( fpp = xfs_rtrefcount_broot_ptr_addr(mp, rblock, 1, rblocklen); tpp = xfs_rtrefcount_droot_ptr_addr(dblock, 1, maxrecs); numrecs = be16_to_cpu(rblock->bb_numrecs); - memcpy(tkp, fkp, 2 * sizeof(*fkp) * numrecs); + memcpy(tkp, fkp, sizeof(*fkp) * numrecs); memcpy(tpp, fpp, sizeof(*fpp) * numrecs); } else { frp = xfs_rtrefcount_rec_addr(rblock, 1); diff --git a/fs/xfs/libxfs/xfs_rtrmap_btree.c b/fs/xfs/libxfs/xfs_rtrmap_btree.c index c264bc5651c0a..a15e460a1ec7f 100644 --- a/fs/xfs/libxfs/xfs_rtrmap_btree.c +++ b/fs/xfs/libxfs/xfs_rtrmap_btree.c @@ -618,7 +618,7 @@ xfs_rtrmapbt_mem_cursor( struct xfs_btree_cur *cur; cur = xfs_btree_alloc_cursor(mp, tp, &xfs_rtrmapbt_mem_ops, - mp->m_rtrmap_maxlevels, xfs_rtrmapbt_cur_cache); + xfs_rtrmapbt_maxlevels_ondisk(), xfs_rtrmapbt_cur_cache); cur->bc_mem.xfbtree = xfbt; cur->bc_nlevels = xfbt->nlevels; cur->bc_group = xfs_group_hold(rtg_group(rtg)); @@ -716,10 +716,12 @@ xfs_rtrmapbt_maxlevels_ondisk(void) * happens, which means that we must compute the max height based on * what the btree will look like if it consumes almost all the blocks * in the data device due to maximal sharing factor. + * + * Add one extra level for the inode root. */ max_dblocks = -1U; /* max ag count */ max_dblocks *= XFS_MAX_CRC_AG_BLOCKS; - return xfs_btree_space_to_height(minrecs, max_dblocks); + return xfs_btree_space_to_height(minrecs, max_dblocks) + 1; } int __init diff --git a/fs/xfs/scrub/agheader.c b/fs/xfs/scrub/agheader.c index 1fa66aa68e169..fa5d32ec020ae 100644 --- a/fs/xfs/scrub/agheader.c +++ b/fs/xfs/scrub/agheader.c @@ -418,6 +418,13 @@ xchk_superblock( xchk_block_set_corrupt(sc, bp); } + if (xfs_has_zoned(mp)) { + if (sb->sb_rtstart != cpu_to_be64(mp->m_sb.sb_rtstart)) + xchk_block_set_corrupt(sc, bp); + if (sb->sb_rtreserved != cpu_to_be64(mp->m_sb.sb_rtreserved)) + xchk_block_set_corrupt(sc, bp); + } + /* Everything else must be zero. */ sblen = xchk_superblock_ondisk_size(mp); if (memchr_inv((char *)sb + sblen, 0, BBTOB(bp->b_length) - sblen)) diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index 2104512f1ee19..a66b611588c47 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -668,14 +668,16 @@ xrep_agfl_init_header( struct xfs_scrub *sc, struct xfs_buf *agfl_bp, struct xagb_bitmap *agfl_extents, - xfs_agblock_t flcount) + xfs_agblock_t flcount, + struct xfs_agfl *old_agfl) { struct xrep_agfl_fill af = { .sc = sc, .flcount = flcount, }; struct xfs_mount *mp = sc->mp; - struct xfs_agfl *agfl; + struct xfs_agfl *agfl = XFS_BUF_TO_AGFL(agfl_bp); + const size_t agfl_sz = BBTOB(agfl_bp->b_length); int error; ASSERT(flcount <= xfs_agfl_size(mp)); @@ -684,8 +686,8 @@ xrep_agfl_init_header( * Start rewriting the header by setting the bno[] array to * NULLAGBLOCK, then setting AGFL header fields. */ - agfl = XFS_BUF_TO_AGFL(agfl_bp); - memset(agfl, 0xFF, BBTOB(agfl_bp->b_length)); + memcpy(old_agfl, agfl, agfl_sz); + memset(agfl, 0xFF, agfl_sz); agfl->agfl_magicnum = cpu_to_be32(XFS_AGFL_MAGIC); agfl->agfl_seqno = cpu_to_be32(pag_agno(sc->sa.pag)); uuid_copy(&agfl->agfl_uuid, &mp->m_sb.sb_meta_uuid); @@ -697,16 +699,23 @@ xrep_agfl_init_header( */ xagb_bitmap_init(&af.used_extents); af.agfl_bno = xfs_buf_to_agfl_bno(agfl_bp); - xagb_bitmap_walk(agfl_extents, xrep_agfl_fill, &af); + error = xagb_bitmap_walk(agfl_extents, xrep_agfl_fill, &af); + if (error && error != -ECANCELED) + goto err_undo; error = xagb_bitmap_disunion(agfl_extents, &af.used_extents); if (error) - return error; + goto err_undo; /* Write new AGFL to disk. */ xfs_trans_buf_set_type(sc->tp, agfl_bp, XFS_BLFT_AGFL_BUF); - xfs_trans_log_buf(sc->tp, agfl_bp, 0, BBTOB(agfl_bp->b_length) - 1); + xfs_trans_log_buf(sc->tp, agfl_bp, 0, agfl_sz - 1); xagb_bitmap_destroy(&af.used_extents); return 0; + +err_undo: + xagb_bitmap_destroy(&af.used_extents); + memcpy(agfl, old_agfl, agfl_sz); + return error; } /* Repair the AGFL. */ @@ -718,6 +727,7 @@ xrep_agfl( struct xfs_mount *mp = sc->mp; struct xfs_buf *agf_bp; struct xfs_buf *agfl_bp; + struct xfs_agfl *old_agfl; xfs_agblock_t flcount; int error; @@ -725,6 +735,10 @@ xrep_agfl( if (!xfs_has_rmapbt(mp)) return -EOPNOTSUPP; + old_agfl = kzalloc(BBTOB(XFS_FSS_TO_BB(mp, 1)), XCHK_GFP_FLAGS); + if (!old_agfl) + return -ENOMEM; + xagb_bitmap_init(&agfl_extents); /* @@ -734,7 +748,7 @@ xrep_agfl( */ error = xfs_alloc_read_agf(sc->sa.pag, sc->tp, 0, &agf_bp); if (error) - return error; + goto err_old_agfl; /* * Make sure we have the AGFL buffer, as scrub might have decided it @@ -745,7 +759,7 @@ xrep_agfl( XFS_AGFL_DADDR(mp)), XFS_FSS_TO_BB(mp, 1), 0, &agfl_bp, NULL); if (error) - return error; + goto err_old_agfl; agfl_bp->b_ops = &xfs_agfl_buf_ops; /* Gather all the extents we're going to put on the new AGFL. */ @@ -762,10 +776,11 @@ xrep_agfl( * we adjust the AGF flcount (which can fail) so avoid updating any * buffers until we know that part works. */ - xrep_agfl_update_agf(sc, agf_bp, flcount); - error = xrep_agfl_init_header(sc, agfl_bp, &agfl_extents, flcount); + error = xrep_agfl_init_header(sc, agfl_bp, &agfl_extents, flcount, + old_agfl); if (error) goto err; + xrep_agfl_update_agf(sc, agf_bp, flcount); /* * Ok, the AGFL should be ready to go now. Roll the transaction to @@ -785,6 +800,8 @@ xrep_agfl( err: xagb_bitmap_destroy(&agfl_extents); +err_old_agfl: + kfree(old_agfl); return error; } diff --git a/fs/xfs/scrub/alloc_repair.c b/fs/xfs/scrub/alloc_repair.c index dce6ab0429dcf..95e318e4f3a6c 100644 --- a/fs/xfs/scrub/alloc_repair.c +++ b/fs/xfs/scrub/alloc_repair.c @@ -571,7 +571,7 @@ xrep_abt_dispose_one( * allocation, and blocks that didn't get used can be freed via the usual * (deferred) means. */ -STATIC void +STATIC int xrep_abt_dispose_reservations( struct xrep_abt *ra, int error) @@ -582,9 +582,13 @@ xrep_abt_dispose_reservations( goto junkit; list_for_each_entry_safe(resv, n, &ra->new_bnobt.resv_list, list) { - error = xrep_abt_dispose_one(ra, resv); - if (error) + int error2 = xrep_abt_dispose_one(ra, resv); + + if (error2) { + if (!error) + error = error2; goto junkit; + } } junkit: @@ -596,6 +600,7 @@ xrep_abt_dispose_reservations( xrep_newbt_cancel(&ra->new_bnobt); xrep_newbt_cancel(&ra->new_cntbt); + return error; } /* Retrieve free space data for bulk load. */ @@ -801,7 +806,9 @@ xrep_abt_build_new_trees( goto err_newbt; /* Dispose of any unused blocks and the accounting information. */ - xrep_abt_dispose_reservations(ra, error); + error = xrep_abt_dispose_reservations(ra, error); + if (error) + return error; return xrep_roll_ag_trans(sc); @@ -812,8 +819,7 @@ xrep_abt_build_new_trees( xfs_btree_del_cursor(cnt_cur, error); xfs_btree_del_cursor(bno_cur, error); err_newbt: - xrep_abt_dispose_reservations(ra, error); - return error; + return xrep_abt_dispose_reservations(ra, error); } /* diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 6e6af142f1fb8..28f92e9ba72b2 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -1294,7 +1294,7 @@ xrep_xattr_swap_prep( .geo = sc->mp->m_attr_geo, .whichfork = XFS_ATTR_FORK, .trans = sc->tp, - .total = 1, + .total = xfs_dabuf_nfsb(sc->mp, XFS_ATTR_FORK), .owner = I_INO(sc->ip), }; diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c index 401c278725d21..3b0f1dbd9147a 100644 --- a/fs/xfs/scrub/bmap.c +++ b/fs/xfs/scrub/bmap.c @@ -274,7 +274,7 @@ xchk_bmap_xref_rmap_cow( unsigned long long rmap_end; uint64_t owner = XFS_RMAP_OWN_COW; - if (!info->sc->sa.rmap_cur || xchk_skip_xref(info->sc->sm)) + if (xchk_skip_xref(info->sc->sm)) return; /* Find the rmap record for this irec. */ diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 2a037aae904d0..19d974c7e2b7a 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -492,7 +492,7 @@ xchk_directory_data_bestfree( goto out; xchk_buffer_recheck(sc, bp); - if (xfs_has_crc(sc->mp)) { + if (!is_block && xfs_has_crc(sc->mp)) { struct xfs_dir3_data_hdr *hdr3 = bp->b_addr; if (hdr3->pad) diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 1c088cfba10ea..2cfcf1c35679e 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -484,18 +484,24 @@ xrep_dir_recover_data( while (offset < end) { struct xfs_dir2_data_unused *dup = bp->b_addr + offset; struct xfs_dir2_data_entry *dep = bp->b_addr + offset; + unsigned int advance; if (xchk_should_terminate(rd->sc, &error)) return error; /* Skip unused entries. */ if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + if (!dup->length) + break; offset += be16_to_cpu(dup->length); continue; } /* Don't walk off the end of the block. */ - offset += xfs_dir2_data_entsize(rd->sc->mp, dep->namelen); + advance = xfs_dir2_data_entsize(rd->sc->mp, dep->namelen); + if (!advance) + break; + offset += advance; if (offset > end) break; @@ -721,7 +727,7 @@ xrep_dir_replay_removename( const struct xfs_name *name, xfs_extlen_t total) { - struct xfs_inode *dp = rd->args.dp; + struct xfs_inode *dp = rd->sc->tempip; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); @@ -1375,9 +1381,24 @@ xrep_dir_live_update( if (p->delta > 0) error = xrep_dir_stash_createname(rd, p->name, I_INO(p->ip)); - else - error = xrep_dir_stash_removename(rd, p->name, + else { + /* + * xfs_dentry_to_name in unlink or rename-exchange can + * pass us names with ftype FT_UNKNOWN, but we really + * must know the ftype of the child that is being + * removed so that we can do nlink updates correctly + * without holding inode references. + */ + struct xfs_name name = { + .name = p->name->name, + .len = p->name->len, + .type = xfs_mode_to_ftype( + VFS_IC(p->ip)->i_mode), + }; + + error = xrep_dir_stash_removename(rd, &name, I_INO(p->ip)); + } mutex_unlock(&rd->pscan.lock); if (error) goto out_abort; @@ -1467,7 +1488,7 @@ xrep_dir_swap_prep( .geo = sc->mp->m_dir_geo, .whichfork = XFS_DATA_FORK, .trans = sc->tp, - .total = 1, + .total = xfs_dabuf_nfsb(sc->mp, XFS_DATA_FORK), .owner = I_INO(sc->ip), }; diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index b2cf6e5439d91..9b0ab23166124 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -259,6 +259,7 @@ xchk_dirtree_create_path( dl->nr_paths++; return 0; out_path: + xino_bitmap_destroy(&path->seen_inodes); kfree(path); return error; } @@ -368,12 +369,38 @@ xchk_dirpath_step_up( struct xfs_inode *dp; xfs_ino_t parent_ino = be64_to_cpu(dl->pptr_rec.p_ino); unsigned int lock_mode; - int error; + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; /* Grab and lock the parent directory. */ error = xchk_iget(sc, parent_ino, &dp); - if (error) + switch (error) { + case -EINVAL: + case -ENOENT: + mutex_lock(&dl->lock); + + if (dl->stale) { + /* live update detected a change in this path */ + error = -ESTALE; + } else { + /* inode doesn't exist, path invalid */ + error = -EFSCORRUPTED; + + trace_xchk_dirpath_badino(dl->sc, path->path_nr, + path->nr_steps, &dl->xname, + &dl->pptr_rec); + } + + mutex_unlock(&dl->lock); + return error; + case 0: + /* keep going */ + break; + default: return error; + } lock_mode = xfs_ilock_attr_map_shared(dp); mutex_lock(&dl->lock); diff --git a/fs/xfs/scrub/dirtree_repair.c b/fs/xfs/scrub/dirtree_repair.c index bbf6acf6fd400..1d1eafcf6eb59 100644 --- a/fs/xfs/scrub/dirtree_repair.c +++ b/fs/xfs/scrub/dirtree_repair.c @@ -479,6 +479,7 @@ xrep_dirtree_unlink( } if (xfs_has_parent(sc->mp)) { + memset(&dl->ppargs, 0, sizeof(dl->ppargs)); error = xfs_parent_removename(sc->tp, &dl->ppargs, dp, &dl->xname, sc->ip); if (error) @@ -618,6 +619,7 @@ xrep_dirtree_create_adoption_path( return 0; out_path: + xino_bitmap_destroy(&path->seen_inodes); kfree(path); return error; } diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c index 04b6b96b0a30e..eab3ac2704bef 100644 --- a/fs/xfs/scrub/findparent.c +++ b/fs/xfs/scrub/findparent.c @@ -139,32 +139,52 @@ xrep_findparent_dirent( return 0; } -/* - * If this is a directory, walk the dirents looking for any that point to the - * scrub target inode. - */ -STATIC int -xrep_findparent_walk_directory( - struct xrep_findparent_info *fpi) +static inline bool +xrep_findparent_want_scan_file( + const struct xrep_findparent_info *fpi) { - struct xfs_scrub *sc = fpi->sc; - struct xfs_inode *dp = fpi->dp; - unsigned int lock_mode; - int error = 0; + const struct xfs_scrub *sc = fpi->sc; + const struct xfs_inode *dp = fpi->dp; + + /* Only directories can be parents */ + if (!S_ISDIR(VFS_IC(dp)->i_mode)) + return false; /* * The inode being scanned cannot be its own parent, nor can any * temporary directory we created to stage this repair. */ if (dp == sc->ip || dp == sc->tempip) - return 0; + return false; /* * Similarly, temporary files created to stage a repair cannot be the * parent of this inode. */ if (xrep_is_tempfile(dp)) + return false; + + return true; +} + +/* + * If this is a directory, walk the dirents looking for any that point to the + * scrub target inode. + */ +STATIC int +xrep_findparent_walk_file( + struct xrep_findparent_info *fpi) +{ + struct xfs_scrub *sc = fpi->sc; + struct xfs_inode *dp = fpi->dp; + unsigned int lock_mode; + int error = 0; + + if (!xrep_findparent_want_scan_file(fpi)) { + if (fpi->parent_scan) + xchk_iscan_mark_visited(&fpi->parent_scan->iscan, dp); return 0; + } /* * Scan the directory to see if there it contains an entry pointing to @@ -201,6 +221,8 @@ xrep_findparent_walk_directory( goto out_unlock; out_unlock: + if (fpi->parent_scan) + xchk_iscan_mark_visited(&fpi->parent_scan->iscan, dp); xfs_iunlock(dp, lock_mode); return error; } @@ -308,11 +330,7 @@ xrep_findparent_scan( ASSERT(S_ISDIR(VFS_IC(sc->ip)->i_mode)); while ((ret = xchk_iscan_iter(&pscan->iscan, &fpi.dp)) == 1) { - if (S_ISDIR(VFS_I(fpi.dp)->i_mode)) - ret = xrep_findparent_walk_directory(&fpi); - else - ret = 0; - xchk_iscan_mark_visited(&pscan->iscan, fpi.dp); + ret = xrep_findparent_walk_file(&fpi); xchk_irele(sc, fpi.dp); if (ret) break; @@ -401,7 +419,7 @@ xrep_findparent_confirm( goto out_rele; } - error = xrep_findparent_walk_directory(&fpi); + error = xrep_findparent_walk_file(&fpi); if (error) goto out_rele; diff --git a/fs/xfs/scrub/health.c b/fs/xfs/scrub/health.c index 2171bcf0f6c1f..487ecc5f9f3c2 100644 --- a/fs/xfs/scrub/health.c +++ b/fs/xfs/scrub/health.c @@ -202,9 +202,9 @@ xchk_update_health( * there's no sick flag defined for it, so we branch here ahead of the * mask check. */ - if (sc->sm->sm_type == XFS_SCRUB_TYPE_HEALTHY && - !(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) { - xchk_mark_all_healthy(sc->mp); + if (sc->sm->sm_type == XFS_SCRUB_TYPE_HEALTHY) { + if (!(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + xchk_mark_all_healthy(sc->mp); return; } diff --git a/fs/xfs/scrub/ialloc.c b/fs/xfs/scrub/ialloc.c index 19c0b1b2a787b..9270ad075fe0a 100644 --- a/fs/xfs/scrub/ialloc.c +++ b/fs/xfs/scrub/ialloc.c @@ -85,6 +85,8 @@ xchk_inobt_xref_finobt( goto no_record; error = xfs_inobt_get_rec(cur, &frec, &has_record); + if (error) + return error; if (!has_record) return -EFSCORRUPTED; @@ -188,6 +190,8 @@ xchk_finobt_xref_inobt( goto no_record; error = xfs_inobt_get_rec(cur, &irec, &has_record); + if (error) + return error; if (!has_record) return -EFSCORRUPTED; diff --git a/fs/xfs/scrub/inode.c b/fs/xfs/scrub/inode.c index 65b13e3119168..46e9bf4a43179 100644 --- a/fs/xfs/scrub/inode.c +++ b/fs/xfs/scrub/inode.c @@ -607,7 +607,7 @@ xchk_dinode( } /* di_forkoff */ - if (XFS_DFORK_BOFF(dip) >= mp->m_sb.sb_inodesize) + if (dip->di_forkoff >= (XFS_LITINO(mp) >> 3)) xchk_ino_set_corrupt(sc, ino); if (naextents != 0 && dip->di_forkoff == 0) xchk_ino_set_corrupt(sc, ino); diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index 8bc508336aa5f..b87c221462338 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -1702,7 +1702,7 @@ xrep_inode_blockcounts( &acount); if (error) return error; - if (count >= sc->mp->m_sb.sb_dblocks) + if (acount >= sc->mp->m_sb.sb_dblocks) return -EFSCORRUPTED; error = xrep_ino_ensure_extent_count(sc, XFS_ATTR_FORK, nextents); diff --git a/fs/xfs/scrub/metapath.c b/fs/xfs/scrub/metapath.c index ff1ff762b3003..c1fa0e1796531 100644 --- a/fs/xfs/scrub/metapath.c +++ b/fs/xfs/scrub/metapath.c @@ -397,7 +397,7 @@ xrep_metapath_unlink( /* Figure out if we're removing a parent pointer too. */ if (xfs_has_parent(mp)) { - xfs_inode_to_parent_rec(&rec, ip); + xfs_inode_to_parent_rec(&rec, mpath->dp); error = xfs_parent_lookup(sc->tp, ip, &mpath->xname, &rec, &mpath->pptr_args); switch (error) { @@ -556,6 +556,8 @@ xrep_metapath_try_unlink( error = xchk_metapath_ilock_parent_and_child(mpath, ip); if (error) { xchk_trans_cancel(sc); + if (ip) + xchk_irele(sc, ip); return error; } xfs_trans_ijoin(sc->tp, mpath->dp, 0); diff --git a/fs/xfs/scrub/newbt.c b/fs/xfs/scrub/newbt.c index c82f4631fd9c2..584076b2a6eed 100644 --- a/fs/xfs/scrub/newbt.c +++ b/fs/xfs/scrub/newbt.c @@ -193,9 +193,11 @@ xrep_newbt_add_blocks( struct xrep_newbt_resv *resv; int error; - resv = kmalloc_obj(struct xrep_newbt_resv, XCHK_GFP_FLAGS); - if (!resv) - return -ENOMEM; + /* + * We have no way to clean up the allocated space *and* return an + * ENOMEM if we fail to allocate this control structure. + */ + resv = kmalloc_obj(struct xrep_newbt_resv, GFP_KERNEL | __GFP_NOFAIL); INIT_LIST_HEAD(&resv->list); resv->agbno = XFS_FSB_TO_AGBNO(mp, args->fsbno); diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 3aca66869b800..21e31eeaa042f 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -192,12 +192,16 @@ xrep_orphanage_create( /* Make sure the orphanage is owned by root. */ error = xrep_chown_orphanage(sc, XFS_I(orphanage_inode)); if (error) - goto out_dput_orphanage; + goto out_rele_orphanage; /* Stash the reference for later and bail out. */ sc->orphanage = XFS_I(orphanage_inode); sc->orphanage_ilock_flags = 0; + orphanage_inode = NULL; +out_rele_orphanage: + if (orphanage_inode) + xchk_irele(sc, XFS_I(orphanage_inode)); out_dput_orphanage: end_creating(orphanage_dentry); out_dput_root: diff --git a/fs/xfs/scrub/quota_repair.c b/fs/xfs/scrub/quota_repair.c index 487bd4f68ebb2..59302e8afc7ef 100644 --- a/fs/xfs/scrub/quota_repair.c +++ b/fs/xfs/scrub/quota_repair.c @@ -325,7 +325,7 @@ xrep_quota_block( * If there's nothing that would impede a dqiterate, we're * done. */ - if ((ddq->d_type & XFS_DQTYPE_REC_MASK) != dqtype || + if ((ddq->d_type & XFS_DQTYPE_REC_MASK) == dqtype && id == be32_to_cpu(ddq->d_id)) { xfs_trans_brelse(sc->tp, bp); return 0; @@ -363,11 +363,18 @@ xrep_quota_block( ddq->d_rtbcount, &ddq->d_rtbtimer, defq->rtb.time); + /* + * This transaction operates on raw disk buffers, so we don't + * have a dquot log item to assign the LSN for us. Instead, + * set it to zero so that log recovery will always replay any + * logged dquot item atop this buffer. + */ + dqblk->dd_lsn = 0; + /* We only support v5 filesystems so always set these. */ uuid_copy(&dqblk->dd_uuid, &sc->mp->m_sb.sb_meta_uuid); xfs_update_cksum((char *)dqblk, sizeof(struct xfs_dqblk), XFS_DQUOT_CRC_OFF); - dqblk->dd_lsn = 0; } switch (dqtype) { case XFS_DQTYPE_USER: @@ -455,8 +462,7 @@ xrep_quota_data_fork( if (truncate) { /* Erase everything after the block containing the max dquot */ - error = xfs_bunmapi_range(&sc->tp, sc->ip, 0, - max_dqid_off * sc->mp->m_sb.sb_blocksize, + error = xfs_bunmapi_range(&sc->tp, sc->ip, 0, max_dqid_off + 1, XFS_MAX_FILEOFF); if (error) goto out; diff --git a/fs/xfs/scrub/quotacheck.c b/fs/xfs/scrub/quotacheck.c index c199d128538e5..c32030a054406 100644 --- a/fs/xfs/scrub/quotacheck.c +++ b/fs/xfs/scrub/quotacheck.c @@ -263,8 +263,10 @@ xqcheck_mod_live_ino_dqtrx( dqa->tx_id = p->tx_id; error = rhashtable_insert_fast(&xqc->shadow_dquot_acct, &dqa->hash, xqcheck_dqacct_hash_params); - if (error) + if (error) { + kfree(dqa); goto out_abort; + } } /* Find the shadow dqtrx (or an empty slot) here. */ diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c index fcd14c1703ea0..f698b9be3dd1d 100644 --- a/fs/xfs/scrub/reap.c +++ b/fs/xfs/scrub/reap.c @@ -601,7 +601,7 @@ xreap_configure_agextent_limits( /* Maximum overhead of invalidating one buffer. */ const unsigned int per_binval = - xfs_buf_inval_log_space(1, XFS_B_TO_FSBT(mp, max_binval)); + xfs_buf_inval_log_space(1, XFS_FSB_TO_B(mp, max_binval)); /* * For each transaction in a reap chain, we can delete some number of @@ -680,7 +680,7 @@ xreap_configure_agcow_limits( /* Overhead of invalidating one buffer */ const unsigned int per_binval = - xfs_buf_inval_log_space(1, XFS_B_TO_FSBT(mp, max_binval)); + xfs_buf_inval_log_space(1, XFS_FSB_TO_B(mp, max_binval)); /* * For each transaction in a reap chain, we can delete some number of @@ -1399,7 +1399,7 @@ xreap_bmapi_binval( * far we've gotten. */ if (!xreap_inc_binval(rs)) { - imap->br_blockcount = agbno_next - bno; + imap->br_blockcount = bno - agbno; goto out; } } diff --git a/fs/xfs/scrub/refcount.c b/fs/xfs/scrub/refcount.c index 4e1bf23e5b897..f8c51d8fbb3da 100644 --- a/fs/xfs/scrub/refcount.c +++ b/fs/xfs/scrub/refcount.c @@ -410,7 +410,7 @@ xchk_refcount_mergeable( const struct xfs_refcount_irec *r1 = &rrc->prev_rec; /* Ignore if prev_rec is not yet initialized. */ - if (r1->rc_blockcount > 0) + if (r1->rc_blockcount == 0) return false; if (r1->rc_domain != r2->rc_domain) @@ -581,8 +581,12 @@ xchk_xref_is_cow_staging( if (rc.rc_domain != XFS_REFC_DOMAIN_COW) xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); + /* Can't start after bno */ + if (rc.rc_startblock > agbno) + xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); + /* Must be at least as long as what was passed in */ - if (rc.rc_blockcount < len) + if (rc.rc_startblock + rc.rc_blockcount < agbno + len) xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); } diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index 11697a8b2a1d1..c2a437416227a 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -399,6 +399,7 @@ xrep_calc_rtgroup_resblks( struct xfs_mount *mp = sc->mp; struct xfs_scrub_metadata *sm = sc->sm; uint64_t usedlen; + xfs_extlen_t refcbt_sz = 0; xfs_extlen_t rmapbt_sz = 0; if (!(sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR)) @@ -411,13 +412,27 @@ xrep_calc_rtgroup_resblks( usedlen = xfs_rtbxlen_to_blen(mp, xfs_rtgroup_extents(mp, sm->sm_agno)); ASSERT(usedlen <= XFS_MAX_RGBLOCKS); + if (xfs_has_reflink(mp)) + refcbt_sz = xfs_rtrefcountbt_calc_size(mp, usedlen); + if (xfs_has_rmapbt(mp)) rmapbt_sz = xfs_rtrmapbt_calc_size(mp, usedlen); + /* + * Guess how many blocks we need to rebuild the rmapbt. For + * non-reflink filesystems we can't have more records than used blocks. + * However, with reflink it's possible to have more than one rmap + * record per rtgroup block. We don't know how many rmaps there could + * be in the rtgroup, so we start off with what we hope is an generous + * over-estimation. + */ + if (refcbt_sz > 0 && rmapbt_sz > 0) + rmapbt_sz *= 2; + trace_xrep_calc_rtgroup_resblks_btsize(mp, sm->sm_agno, usedlen, - rmapbt_sz); + rmapbt_sz, refcbt_sz); - return rmapbt_sz; + return max(rmapbt_sz, refcbt_sz); } #endif /* CONFIG_XFS_RT */ diff --git a/fs/xfs/scrub/rgsuper.c b/fs/xfs/scrub/rgsuper.c index 2bd2c0351b35f..6e2abe5dc27cd 100644 --- a/fs/xfs/scrub/rgsuper.c +++ b/fs/xfs/scrub/rgsuper.c @@ -36,8 +36,10 @@ xchk_rgsuperblock_xref( if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) return; - xchk_xref_is_used_rt_space(sc, xfs_rgbno_to_rtb(sc->sr.rtg, 0), 1); - xchk_xref_is_only_rt_owned_by(sc, 0, 1, &XFS_RMAP_OINFO_FS); + xchk_xref_is_used_rt_space(sc, xfs_rgbno_to_rtb(sc->sr.rtg, 0), + sc->mp->m_sb.sb_rextsize); + xchk_xref_is_only_rt_owned_by(sc, 0, sc->mp->m_sb.sb_rextsize, + &XFS_RMAP_OINFO_FS); } int diff --git a/fs/xfs/scrub/rtrefcount.c b/fs/xfs/scrub/rtrefcount.c index 4e7c540c8d230..3d916d71a135d 100644 --- a/fs/xfs/scrub/rtrefcount.c +++ b/fs/xfs/scrub/rtrefcount.c @@ -20,6 +20,7 @@ #include "xfs_metafile.h" #include "xfs_rtrefcount_btree.h" #include "xfs_rtalloc.h" +#include "xfs_ag.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/btree.h" @@ -375,7 +376,7 @@ xchk_rtrefcount_mergeable( const struct xfs_refcount_irec *r1 = &rrc->prev_rec; /* Ignore if prev_rec is not yet initialized. */ - if (r1->rc_blockcount > 0) + if (r1->rc_blockcount == 0) return false; if (r1->rc_startblock + r1->rc_blockcount != r2->rc_startblock) @@ -428,7 +429,7 @@ static inline void xchk_rtrefcountbt_xref_gaps( struct xfs_scrub *sc, struct xchk_rtrefcbt_records *rrc, - xfs_rtblock_t bno) + xfs_rgblock_t bno) { struct xfs_rmap_irec low; struct xfs_rmap_irec high; @@ -504,30 +505,75 @@ xchk_rtrefcountbt_rec( return 0; } +/* Count the number of blocks used by the rtrefcount btree file in this AG. */ +static int +xchk_rtrefcount_count_agblocks( + struct xfs_scrub *sc, + xfs_agnumber_t agno, + const struct xfs_owner_info *btree_oinfo, + xfs_filblks_t *blocks) +{ + xfs_filblks_t agblocks = 0; + int error; + + error = xchk_ag_init_existing(sc, agno, &sc->sa); + if (error) + goto out_free; + + /* + * If we don't have an rmap cursor, we can't complete the cross + * referencing, so return EFSCORRUPTED to end the loop and trigger the + * XFAIL flag. + */ + if (!sc->sa.rmap_cur) { + error = -EFSCORRUPTED; + goto out_free; + } + + error = xchk_count_rmap_ownedby_ag(sc, sc->sa.rmap_cur, btree_oinfo, + &agblocks); + if (error) + goto out_free; + + *blocks += agblocks; +out_free: + xchk_ag_free(sc, &sc->sa); + return error; +} + /* Make sure we have as many refc blocks as the rmap says. */ STATIC void -xchk_refcount_xref_rmap( +xchk_rtrefcount_xref_rmap( struct xfs_scrub *sc, const struct xfs_owner_info *btree_oinfo, xfs_extlen_t cow_blocks) { xfs_filblks_t refcbt_blocks = 0; - xfs_filblks_t blocks; - int error; + xfs_filblks_t blocks = 1; /* one for the iroot */ + xfs_agnumber_t agno; + int error = 0; - if (!sc->sr.rmap_cur || !sc->sa.rmap_cur || xchk_skip_xref(sc->sm)) + if (!xfs_has_rmapbt(sc->mp) || xchk_skip_xref(sc->sm)) return; /* Check that we saw as many refcbt blocks as the rmap knows about. */ error = xfs_btree_count_blocks(sc->sr.refc_cur, &refcbt_blocks); if (!xchk_btree_process_error(sc, sc->sr.refc_cur, 0, &error)) return; - error = xchk_count_rmap_ownedby_ag(sc, sc->sa.rmap_cur, btree_oinfo, - &blocks); - if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + + for (agno = 0; agno < sc->mp->m_sb.sb_agcount; agno++) { + error = xchk_rtrefcount_count_agblocks(sc, agno, btree_oinfo, + &blocks); + if (error) + break; + } + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) return; if (blocks != refcbt_blocks) - xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); + xchk_fblock_xref_set_corrupt(sc, XFS_DATA_FORK, 0); + + if (!sc->sr.rmap_cur || xchk_skip_xref(sc->sm)) + return; /* Check that we saw as many cow blocks as the rmap knows about. */ error = xchk_count_rmap_ownedby_ag(sc, sc->sr.rmap_cur, @@ -538,7 +584,7 @@ xchk_refcount_xref_rmap( xchk_btree_xref_set_corrupt(sc, sc->sr.rmap_cur, 0); } -/* Scrub the refcount btree for some AG. */ +/* Scrub the refcount btree for some rtgroup. */ int xchk_rtrefcountbt( struct xfs_scrub *sc) @@ -564,11 +610,11 @@ xchk_rtrefcountbt( /* * Check that all blocks between the last refcount > 1 record and the - * end of the rt volume have at most one reverse mapping. + * end of the rtgroup have at most one reverse mapping. */ - xchk_rtrefcountbt_xref_gaps(sc, &rrc, sc->mp->m_sb.sb_rblocks); - - xchk_refcount_xref_rmap(sc, &btree_oinfo, rrc.cow_blocks); + xchk_rtrefcountbt_xref_gaps(sc, &rrc, + xfs_rtx_to_rgbno(sc->sr.rtg, sc->mp->m_sb.sb_rgextents)); + xchk_rtrefcount_xref_rmap(sc, &btree_oinfo, rrc.cow_blocks); return 0; } @@ -609,8 +655,12 @@ xchk_xref_is_rt_cow_staging( if (rc.rc_domain != XFS_REFC_DOMAIN_COW) xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); + /* Can't start after bno */ + if (rc.rc_startblock > bno) + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); + /* Must be at least as long as what was passed in */ - if (rc.rc_blockcount < len) + if (rc.rc_startblock + rc.rc_blockcount < bno + len) xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); } diff --git a/fs/xfs/scrub/rtsummary_repair.c b/fs/xfs/scrub/rtsummary_repair.c index f065c3e51ce2e..ed763290aec1e 100644 --- a/fs/xfs/scrub/rtsummary_repair.c +++ b/fs/xfs/scrub/rtsummary_repair.c @@ -164,9 +164,10 @@ xrep_rtsummary( /* * Now exchange the contents. Nothing in repair uses the temporary * buffer, so we can reuse it for the tempfile exchrange information. + * Use XFS_MAX_FILEOFF here so that we correct the rtsummary file size. */ error = xrep_tempexch_trans_reserve(sc, XFS_DATA_FORK, 0, - rts->rsumblocks, &rts->tempexch); + XFS_MAX_FILEOFF, &rts->tempexch); if (error) return error; diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 8742445c86f45..12c228b7f4770 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -765,8 +765,7 @@ xfs_scrub_metadata( out_teardown: error = xchk_teardown(sc, error); out_sc: - if (error != -ENOENT) - xchk_stats_merge(mp, sm, &run); + xchk_stats_merge(mp, sm, error, &run); kfree(sc); out: trace_xchk_done(XFS_I(file_inode(file)), sm, error); diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index 6d7d3523b71f2..7ade1d005efef 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -40,7 +40,7 @@ static inline int xchk_maybe_relax(struct xchk_relax *widget) return 0; widget->resched_nr = 0; - if (unlikely(widget->next_resched <= jiffies)) { + if (unlikely(time_after_eq(jiffies, widget->next_resched))) { cond_resched(); widget->next_resched = XCHK_RELAX_NEXT; } diff --git a/fs/xfs/scrub/stats.c b/fs/xfs/scrub/stats.c index ef3f6abdb7061..da0c05ffe5cd0 100644 --- a/fs/xfs/scrub/stats.c +++ b/fs/xfs/scrub/stats.c @@ -84,6 +84,7 @@ static const char *name_map[XFS_SCRUB_TYPE_NR] = { [XFS_SCRUB_TYPE_RGSUPER] = "rgsuper", [XFS_SCRUB_TYPE_RTRMAPBT] = "rtrmapbt", [XFS_SCRUB_TYPE_RTREFCBT] = "rtrefcountbt", + [XFS_SCRUB_TYPE_HEALTHY] = "healthy", }; /* Format the scrub stats into a text buffer, similar to pcp style. */ @@ -99,25 +100,31 @@ xchk_stats_format( int ret = 0; for (i = 0; i < XFS_SCRUB_TYPE_NR; i++, css++) { + struct xchk_scrub_stats fss; + if (!name_map[i]) continue; + spin_lock(&css->css_lock); + memcpy(&fss, css, offsetof(struct xchk_scrub_stats, css_lock)); + spin_unlock(&css->css_lock); + ret = scnprintf(buf, remaining, "%s %u %u %u %u %u %u %u %u %u %llu %u %u %llu\n", name_map[i], - (unsigned int)css->invocations, - (unsigned int)css->clean, - (unsigned int)css->corrupt, - (unsigned int)css->preen, - (unsigned int)css->xfail, - (unsigned int)css->xcorrupt, - (unsigned int)css->incomplete, - (unsigned int)css->warning, - (unsigned int)css->retries, - (unsigned long long)css->checktime_us, - (unsigned int)css->repair_invocations, - (unsigned int)css->repair_success, - (unsigned long long)css->repairtime_us); + (unsigned int)fss.invocations, + (unsigned int)fss.clean, + (unsigned int)fss.corrupt, + (unsigned int)fss.preen, + (unsigned int)fss.xfail, + (unsigned int)fss.xcorrupt, + (unsigned int)fss.incomplete, + (unsigned int)fss.warning, + (unsigned int)fss.retries, + (unsigned long long)fss.checktime_us, + (unsigned int)fss.repair_invocations, + (unsigned int)fss.repair_success, + (unsigned long long)fss.repairtime_us); if (ret <= 0) break; @@ -188,31 +195,37 @@ STATIC void xchk_stats_merge_one( struct xchk_stats *cs, const struct xfs_scrub_metadata *sm, + int error, const struct xchk_stats_run *run) { struct xchk_scrub_stats *css; + unsigned int sm_flags = sm->sm_flags; if (sm->sm_type >= XFS_SCRUB_TYPE_NR) { ASSERT(sm->sm_type < XFS_SCRUB_TYPE_NR); return; } + /* caller applies this same transformation after we return */ + if (error == -EFSCORRUPTED || error == -EFSBADCRC) + sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + css = &cs->cs_stats[sm->sm_type]; spin_lock(&css->css_lock); css->invocations++; - if (!(sm->sm_flags & XFS_SCRUB_OFLAG_UNCLEAN)) + if (!(sm_flags & XFS_SCRUB_OFLAG_UNCLEAN)) css->clean++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + if (sm_flags & XFS_SCRUB_OFLAG_CORRUPT) css->corrupt++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_PREEN) + if (sm_flags & XFS_SCRUB_OFLAG_PREEN) css->preen++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_XFAIL) + if (sm_flags & XFS_SCRUB_OFLAG_XFAIL) css->xfail++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_XCORRUPT) + if (sm_flags & XFS_SCRUB_OFLAG_XCORRUPT) css->xcorrupt++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_INCOMPLETE) + if (sm_flags & XFS_SCRUB_OFLAG_INCOMPLETE) css->incomplete++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_WARNING) + if (sm_flags & XFS_SCRUB_OFLAG_WARNING) css->warning++; css->retries += run->retries; css->checktime_us += howmany_64(run->scrub_ns, NSEC_PER_USEC); @@ -230,10 +243,14 @@ void xchk_stats_merge( struct xfs_mount *mp, const struct xfs_scrub_metadata *sm, + int error, const struct xchk_stats_run *run) { - xchk_stats_merge_one(&global_stats, sm, run); - xchk_stats_merge_one(mp->m_scrub_stats, sm, run); + if (error == -ENOENT) + return; + + xchk_stats_merge_one(&global_stats, sm, error, run); + xchk_stats_merge_one(mp->m_scrub_stats, sm, error, run); } /* debugfs boilerplate */ diff --git a/fs/xfs/scrub/stats.h b/fs/xfs/scrub/stats.h index b358ad8d8b90a..221052b95dd07 100644 --- a/fs/xfs/scrub/stats.h +++ b/fs/xfs/scrub/stats.h @@ -27,7 +27,7 @@ void xchk_stats_register(struct xchk_stats *cs, struct dentry *parent); void xchk_stats_unregister(struct xchk_stats *cs); void xchk_stats_merge(struct xfs_mount *mp, const struct xfs_scrub_metadata *sm, - const struct xchk_stats_run *run); + int error, const struct xchk_stats_run *run); static inline u64 xchk_stats_now(void) { return ktime_get_ns(); } static inline u64 xchk_stats_elapsed_ns(u64 since) @@ -53,7 +53,7 @@ static inline u64 xchk_stats_elapsed_ns(u64 since) # define xchk_stats_unregister(cs) ((void)0) # define xchk_stats_now() (0) # define xchk_stats_elapsed_ns(x) (0 * (x)) -# define xchk_stats_merge(mp, sm, run) ((void)0) +# define xchk_stats_merge(mp, sm, error, run) ((void)0) #endif /* CONFIG_XFS_ONLINE_SCRUB_STATS */ #endif /* __XFS_SCRUB_STATS_H__ */ diff --git a/fs/xfs/scrub/symlink_repair.c b/fs/xfs/scrub/symlink_repair.c index 91c86ea0e0f15..1819613642333 100644 --- a/fs/xfs/scrub/symlink_repair.c +++ b/fs/xfs/scrub/symlink_repair.c @@ -291,7 +291,7 @@ xrep_symlink_swap_prep( if (error) return error; - xfs_trans_log_inode(sc->tp, sc->ip, 0); + xfs_trans_log_inode(sc->tp, sc->tempip, logflags); error = xfs_defer_finish(&sc->tp); if (error) diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index 98820003b9298..59a9213a3c7dd 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -649,6 +649,19 @@ xrep_tempexch_prep_request( return 0; } +static inline unsigned int +xrep_tempexch_estimate_sf_resblks( + struct xfs_scrub *sc, + int whichfork) +{ + /* repairing a symlink target */ + if (S_ISLNK(VFS_I(sc->ip)->i_mode) && whichfork == XFS_DATA_FORK) + return 1; + + /* everything else is a directory or an xattr structure */ + return xfs_dabuf_nfsb(sc->mp, whichfork); +} + /* * Fill out the mapping exchange resource estimation structures in preparation * for exchanging the contents of a metadata file that we've rebuilt in the @@ -663,6 +676,8 @@ xrep_tempexch_estimate( struct xfs_ifork *ifp; struct xfs_ifork *tifp; int whichfork = xfs_exchmaps_reqfork(req); + unsigned int sf_resblks = + xrep_tempexch_estimate_sf_resblks(sc, whichfork); int state = 0; /* @@ -693,9 +708,9 @@ xrep_tempexch_estimate( * plus the block we converted. */ req->ip1_bcount = sc->tempip->i_nblocks; - req->ip2_bcount = 1; + req->ip2_bcount = sf_resblks; req->nr_exchanges = 1 + tifp->if_nextents; - req->resblks = 1; + req->resblks = sf_resblks; break; case 2: /* @@ -707,10 +722,10 @@ xrep_tempexch_estimate( * is (worst case) the extent count of the file being repaired * plus the block we converted. */ - req->ip1_bcount = 1; + req->ip1_bcount = sf_resblks; req->ip2_bcount = sc->ip->i_nblocks; req->nr_exchanges = 1 + ifp->if_nextents; - req->resblks = 1; + req->resblks = sf_resblks; break; case 3: /* @@ -722,10 +737,10 @@ xrep_tempexch_estimate( * fileoff 0. Presumably, the caller could not exchange the * two inode fork areas directly. */ - req->ip1_bcount = 1; - req->ip2_bcount = 1; + req->ip1_bcount = sf_resblks; + req->ip2_bcount = sf_resblks; req->nr_exchanges = 1; - req->resblks = 2; + req->resblks = 2 * sf_resblks; break; } diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 14aa0ec1f09e4..cb85f75ce1018 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -1640,7 +1640,7 @@ DECLARE_EVENT_CLASS(xchk_pptr_class, __entry->dev = ip->i_mount->m_super->s_dev; __entry->ino = I_INO(ip); __entry->namelen = name->len; - memcpy(__get_str(name), name, name->len); + memcpy(__get_str(name), name->name, name->len); __entry->far_ino = far_ino; ), TP_printk("dev %d:%d ino 0x%llx name '%.*s' far_ino 0x%llx", @@ -1706,6 +1706,39 @@ DEFINE_EVENT(xchk_dirtree_class, name, \ DEFINE_XCHK_DIRTREE_EVENT(xchk_dirtree_create_path); DEFINE_XCHK_DIRTREE_EVENT(xchk_dirpath_walk_upwards); +TRACE_EVENT(xchk_dirpath_badino, + TP_PROTO(struct xfs_scrub *sc, unsigned int path_nr, + unsigned int step_nr, const struct xfs_name *name, + const struct xfs_parent_rec *pptr), + TP_ARGS(sc, path_nr, step_nr, name, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, path_nr) + __field(unsigned int, step_nr) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->path_nr = path_nr; + __entry->step_nr = step_nr; + __entry->parent_ino = be64_to_cpu(pptr->p_ino); + __entry->parent_gen = be32_to_cpu(pptr->p_gen); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d path %u step %u parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->path_nr, + __entry->step_nr, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +); + DECLARE_EVENT_CLASS(xchk_dirpath_class, TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, unsigned int path_nr, unsigned int step_nr, @@ -2343,25 +2376,29 @@ TRACE_EVENT(xrep_calc_ag_resblks_btsize, #ifdef CONFIG_XFS_RT TRACE_EVENT(xrep_calc_rtgroup_resblks_btsize, TP_PROTO(struct xfs_mount *mp, xfs_rgnumber_t rgno, - xfs_rgblock_t usedlen, xfs_rgblock_t rmapbt_sz), - TP_ARGS(mp, rgno, usedlen, rmapbt_sz), + xfs_rgblock_t usedlen, xfs_rgblock_t rmapbt_sz, + xfs_rgblock_t refcbt_sz), + TP_ARGS(mp, rgno, usedlen, rmapbt_sz, refcbt_sz), TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_rgnumber_t, rgno) __field(xfs_rgblock_t, usedlen) __field(xfs_rgblock_t, rmapbt_sz) + __field(xfs_rgblock_t, refcbt_sz) ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; __entry->rgno = rgno; __entry->usedlen = usedlen; __entry->rmapbt_sz = rmapbt_sz; + __entry->refcbt_sz = refcbt_sz; ), - TP_printk("dev %d:%d rgno 0x%x usedlen %u rmapbt %u", + TP_printk("dev %d:%d rgno 0x%x usedlen %u rmapbt %u refcountbt %u", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->rgno, __entry->usedlen, - __entry->rmapbt_sz) + __entry->rmapbt_sz, + __entry->refcbt_sz) ); #endif /* CONFIG_XFS_RT */ diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 48d7dfd3e15f0..44e344fc51e91 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -108,7 +108,7 @@ xfs_buf_free( ASSERT(list_empty(&bp->b_lru)); if (!xfs_buftarg_is_mem(bp->b_target) && size >= PAGE_SIZE) - mm_account_reclaimed_pages(howmany(size, PAGE_SHIFT)); + mm_account_reclaimed_pages(howmany(size, PAGE_SIZE)); if (is_vmalloc_addr(bp->b_addr)) vfree(bp->b_addr); diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index b4f6c594808ce..e696ee36c2e8d 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -139,10 +139,14 @@ xfs_qm_adjust_dqlimits( dq->q_ino.softlimit = defq->ino.soft; if (!dq->q_ino.hardlimit) dq->q_ino.hardlimit = defq->ino.hard; - if (!dq->q_rtb.softlimit) + if (!dq->q_rtb.softlimit) { dq->q_rtb.softlimit = defq->rtb.soft; - if (!dq->q_rtb.hardlimit) + prealloc = 1; + } + if (!dq->q_rtb.hardlimit) { dq->q_rtb.hardlimit = defq->rtb.hard; + prealloc = 1; + } if (prealloc) xfs_dquot_set_prealloc_limits(dq); diff --git a/fs/xfs/xfs_exchmaps_item.c b/fs/xfs/xfs_exchmaps_item.c index c3745d33e54ea..dd5d92ca1010f 100644 --- a/fs/xfs/xfs_exchmaps_item.c +++ b/fs/xfs/xfs_exchmaps_item.c @@ -344,7 +344,17 @@ xfs_xmi_validate( if (!xfs_verify_fileext(mp, xlf->xmi_startoff1, xlf->xmi_blockcount)) return false; - return xfs_verify_fileext(mp, xlf->xmi_startoff2, xlf->xmi_blockcount); + if (!xfs_verify_fileext(mp, xlf->xmi_startoff2, xlf->xmi_blockcount)) + return false; + + if (xlf->xmi_flags & XFS_EXCHMAPS_SET_SIZES) { + if ((int64_t)xlf->xmi_isize1 < 0) + return false; + if ((int64_t)xlf->xmi_isize2 < 0) + return false; + } + + return true; } /* @@ -403,6 +413,13 @@ xfs_xmi_item_recover_intent( *ipp1 = ip1; *ipp2 = ip2; xmi = xfs_exchmaps_init_intent(req); + + /* Restore intended file sizes from recovered logged item */ + if (req->flags & XFS_EXCHMAPS_SET_SIZES) { + xmi->xmi_isize1 = xlf->xmi_isize1; + xmi->xmi_isize2 = xlf->xmi_isize2; + } + xfs_defer_add_item(dfp, &xmi->xmi_list); return xmi; diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c index 94965a6c2187f..fafb4e3f065c7 100644 --- a/fs/xfs/xfs_exchrange.c +++ b/fs/xfs/xfs_exchrange.c @@ -504,6 +504,9 @@ xfs_exchange_range_finish( { int error; + if (fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN) + return 0; + error = file_remove_privs(fxr->file1); if (error) return error; @@ -630,6 +633,9 @@ xfs_exchrange_prep( if (error) return error; + if (fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN) + return 0; + trace_xfs_exchrange_flush(fxr, ip1, ip2); /* Flush the relevant ranges of both files. */ @@ -706,9 +712,11 @@ xfs_exchrange_contents( * other file write would do. This may involve turning on support for * logged xattrs if either file has security capabilities. */ - error = xfs_exchange_range_finish(fxr); - if (error) - goto out_unlock; + if (!(fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN)) { + error = xfs_exchange_range_finish(fxr); + if (error) + goto out_unlock; + } out_unlock: xfs_iunlock2_io_mmap(ip1, ip2); @@ -783,9 +791,12 @@ xfs_exchange_range( if (ret) return ret; - fsnotify_modify(fxr->file1); - if (fxr->file2 != fxr->file1) - fsnotify_modify(fxr->file2); + if (!(fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN)) { + fsnotify_modify(fxr->file1); + if (fxr->file2 != fxr->file1) + fsnotify_modify(fxr->file2); + } + return 0; } @@ -896,7 +907,7 @@ xfs_ioc_commit_range( if (copy_from_user(&args, argp, sizeof(args))) return -EFAULT; - if (args.flags & ~XFS_EXCHANGE_RANGE_ALL_FLAGS) + if (args.pad || (args.flags & ~XFS_EXCHANGE_RANGE_ALL_FLAGS)) return -EINVAL; if (kern_f->magic != XCR_FRESH_MAGIC) return -EBUSY; diff --git a/fs/xfs/xfs_healthmon.c b/fs/xfs/xfs_healthmon.c index 4521ffdab9f1a..451575dceae04 100644 --- a/fs/xfs/xfs_healthmon.c +++ b/fs/xfs/xfs_healthmon.c @@ -245,7 +245,9 @@ xfs_healthmon_merge_events( case XFS_HEALTHMON_DIOWRITE: case XFS_HEALTHMON_DATALOST: /* logically adjacent file ranges can merge */ - if (existing->fino != new->fino || existing->fgen != new->fgen) + if (existing->fino != new->fino || + existing->fgen != new->fgen || + existing->error != new->error) return false; if (existing->fpos + existing->flen == new->fpos) { @@ -272,12 +274,13 @@ __xfs_healthmon_insert( { struct timespec64 now; + lockdep_assert_held(&hm->lock); + ktime_get_coarse_real_ts64(&now); event->time_ns = (now.tv_sec * NSEC_PER_SEC) + now.tv_nsec; event->next = hm->first_event; - if (!hm->first_event) - hm->first_event = event; + hm->first_event = event; if (!hm->last_event) hm->last_event = event; xfs_healthmon_bump_events(hm); @@ -294,6 +297,8 @@ __xfs_healthmon_push( { struct timespec64 now; + lockdep_assert_held(&hm->lock); + ktime_get_coarse_real_ts64(&now); event->time_ns = (now.tv_sec * NSEC_PER_SEC) + now.tv_nsec; @@ -330,8 +335,10 @@ xfs_healthmon_clear_lost_prev( if (hm->events < XFS_HEALTHMON_MAX_EVENTS) event = kmemdup(&lost_event, sizeof(struct xfs_healthmon_event), GFP_NOFS); - if (!event) + if (!event) { + xfs_healthmon_bump_lost(hm); return -ENOMEM; + } __xfs_healthmon_push(hm, event); cleared: @@ -415,8 +422,10 @@ xfs_healthmon_unmount( * There's nothing actionable for userspace after an unmount. Once * we've inserted the unmount event, hm no longer owns that event. */ + mutex_lock(&hm->lock); __xfs_healthmon_insert(hm, hm->unmount_event); hm->unmount_event = NULL; + mutex_unlock(&hm->lock); xfs_healthmon_detach(hm); xfs_healthmon_put(hm); @@ -738,6 +747,13 @@ static const unsigned int type_map[] = { [XFS_HEALTHMON_DATALOST] = XFS_HEALTH_MONITOR_TYPE_DATALOST, }; +static inline bool +xfs_healthmon_check_outbuffer_space(const struct xfs_healthmon *hm) +{ + return hm->bufhead + sizeof(struct xfs_health_monitor_event) <= + hm->bufsize; +} + /* Render event as a V0 structure */ STATIC int xfs_healthmon_format_v0( @@ -804,10 +820,10 @@ xfs_healthmon_format_v0( break; } - ASSERT(hm->bufhead + sizeof(hme) <= hm->bufsize); + ASSERT(xfs_healthmon_check_outbuffer_space(hm)); /* copy formatted object to the outbuf */ - if (hm->bufhead + sizeof(hme) <= hm->bufsize) { + if (xfs_healthmon_check_outbuffer_space(hm)) { memcpy(hm->buffer + hm->bufhead, &hme, sizeof(hme)); hm->bufhead += sizeof(hme); } @@ -890,7 +906,11 @@ xfs_healthmon_format_pop( { struct xfs_healthmon_event *event; - if (hm->bufhead + sizeof(*event) > hm->bufsize) + /* + * Don't bother if there's not enough space to format even one event in + * the outbuffer. + */ + if (!xfs_healthmon_check_outbuffer_space(hm)) return NULL; mutex_lock(&hm->lock); diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 9d8dd30bd927a..d462e17d47935 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1656,7 +1656,7 @@ xfs_blockgc_free_dquots( do_work = true; } - if (XFS_IS_UQUOTA_ENFORCED(mp) && gdqp && xfs_dquot_lowsp(gdqp)) { + if (XFS_IS_GQUOTA_ENFORCED(mp) && gdqp && xfs_dquot_lowsp(gdqp)) { icw.icw_gid = make_kgid(mp->m_super->s_user_ns, gdqp->q_id); icw.icw_flags |= XFS_ICWALK_FLAG_GID; do_work = true; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index fdb011e6ef600..8e4acebf72d67 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2736,12 +2736,13 @@ xlog_recover_iunlink_bucket( { struct xfs_mount *mp = pag_mount(pag); struct xfs_inode *prev_ip = NULL; - struct xfs_inode *ip; xfs_agino_t prev_agino, agino; int error = 0; agino = be32_to_cpu(agi->agi_unlinked[bucket]); while (agino != NULLAGINO) { + struct xfs_inode *ip; + error = xfs_iget(mp, NULL, xfs_agino_to_ino(pag, agino), 0, 0, &ip); if (error) @@ -2750,11 +2751,11 @@ xlog_recover_iunlink_bucket( ASSERT(VFS_I(ip)->i_nlink == 0); ASSERT(VFS_I(ip)->i_mode != 0); xfs_iflags_clear(ip, XFS_IRECOVERY); - agino = ip->i_next_unlinked; if (prev_ip) { ip->i_prev_unlinked = prev_agino; xfs_irele(prev_ip); + prev_ip = NULL; /* * Ensure the inode is removed from the unlinked list @@ -2766,18 +2767,20 @@ xlog_recover_iunlink_bucket( * complete. */ error = xfs_inodegc_flush(mp); - if (error) - break; + if (error) { + xfs_irele(ip); + return error; + } } prev_agino = agino; + agino = ip->i_next_unlinked; prev_ip = ip; } if (prev_ip) { int error2; - ip->i_prev_unlinked = prev_agino; xfs_irele(prev_ip); error2 = xfs_inodegc_flush(mp); diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 896b24f87ac9d..f9c28d3864db3 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -1432,16 +1432,22 @@ xfs_qm_flush_one( error = xfs_dquot_use_attached_buf(dqp, &bp); if (error) - goto out_unlock; + goto out_dqflock; if (!bp) { error = -EFSCORRUPTED; - goto out_unlock; + goto out_dqflock; } error = xfs_qm_dqflush(dqp, bp); if (!error) xfs_buf_delwri_queue(bp, buffer_list); xfs_buf_relse(bp); + mutex_unlock(&dqp->q_qlock); + xfs_qm_dqrele(dqp); + return error; + +out_dqflock: + xfs_dqfunlock(dqp); out_unlock: mutex_unlock(&dqp->q_qlock); xfs_qm_dqrele(dqp); diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 8bccf89a77668..682c6e1b45e3e 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -508,6 +508,7 @@ xfs_refcount_recover_work( struct xfs_cui_log_item *cuip = CUI_ITEM(lip); struct xfs_trans *tp; struct xfs_mount *mp = lip->li_log->l_mp; + unsigned int dblocks; bool isrt = xfs_cui_item_isrt(lip); int i; int error = 0; @@ -543,8 +544,11 @@ xfs_refcount_recover_work( * full btree split on either end of the refcount range. */ resv = xlog_recover_resv(&M_RES(mp)->tr_itruncate); - error = xfs_trans_alloc(mp, &resv, mp->m_refc_maxlevels * 2, 0, - XFS_TRANS_RESERVE, &tp); + if (isrt) + dblocks = mp->m_rtrefc_maxlevels * 2; + else + dblocks = mp->m_refc_maxlevels * 2; + error = xfs_trans_alloc(mp, &resv, dblocks, 0, XFS_TRANS_RESERVE, &tp); if (error) return error; diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 2a3a73a8566d1..000cff1ce324f 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -573,6 +573,7 @@ xfs_rmap_recover_work( struct xfs_rui_log_item *ruip = RUI_ITEM(lip); struct xfs_trans *tp; struct xfs_mount *mp = lip->li_log->l_mp; + unsigned int dblocks; bool isrt = xfs_rui_item_isrt(lip); int i; int error = 0; @@ -596,8 +597,11 @@ xfs_rmap_recover_work( } resv = xlog_recover_resv(&M_RES(mp)->tr_itruncate); - error = xfs_trans_alloc(mp, &resv, mp->m_rmap_maxlevels, 0, - XFS_TRANS_RESERVE, &tp); + if (isrt) + dblocks = mp->m_rtrmap_maxlevels; + else + dblocks = mp->m_rmap_maxlevels; + error = xfs_trans_alloc(mp, &resv, dblocks, 0, XFS_TRANS_RESERVE, &tp); if (error) return error; diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 7bfbd9f6f0dff..1b36cf12d4e3e 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -1029,6 +1029,15 @@ xfs_trans_roll( * duplicate transaction that gets returned. */ error = __xfs_trans_commit(tp, true); + + tp = *tpp; + /* + * __xfs_trans_commit cleared the NOFS flag by calling into + * xfs_trans_free. Set it again here before doing memory + * allocations. + */ + xfs_trans_set_context(tp); + if (error) return error; @@ -1040,13 +1049,6 @@ xfs_trans_roll( * either nothing be locked across this call, or that anything that is * locked be logged in the prior and the next transactions. */ - tp = *tpp; - /* - * __xfs_trans_commit cleared the NOFS flag by calling into - * xfs_trans_free. Set it again here before doing memory - * allocations. - */ - xfs_trans_set_context(tp); error = xfs_log_regrant(tp->t_mountp, tp->t_ticket); if (error) return error; diff --git a/fs/xfs/xfs_verify_media.c b/fs/xfs/xfs_verify_media.c index 5ead3976d5115..b75c81f8fcc03 100644 --- a/fs/xfs/xfs_verify_media.c +++ b/fs/xfs/xfs_verify_media.c @@ -268,6 +268,8 @@ xfs_verify_media( struct xfs_buftarg *btp = NULL; struct bio *bio; struct folio *folio; + xfs_daddr_t dev_start = 0; + xfs_daddr_t dev_end = 0; xfs_daddr_t daddr; uint64_t bbcount; int error = 0; @@ -277,24 +279,33 @@ xfs_verify_media( switch (me->me_dev) { case XFS_DEV_DATA: btp = mp->m_ddev_targp; + dev_end = XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks); break; case XFS_DEV_LOG: - if (mp->m_logdev_targp != mp->m_ddev_targp) + if (mp->m_logdev_targp != mp->m_ddev_targp) { btp = mp->m_logdev_targp; + dev_end = XFS_FSB_TO_BB(mp, mp->m_sb.sb_logblocks); + } break; case XFS_DEV_RT: btp = mp->m_rtdev_targp; + dev_start = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart); + dev_end = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart + + mp->m_sb.sb_rblocks); break; } if (!btp) return -ENODEV; /* - * If the caller told us to verify beyond the end of the disk, tell the - * user exactly where that was. + * If the caller told us to verify before the start or beyond the end + * of the disk volume, tell the user exactly where the volume starts + * and ends. */ - if (me->me_end_daddr > btp->bt_nr_sectors) - me->me_end_daddr = btp->bt_nr_sectors; + if (me->me_end_daddr > dev_end) + me->me_end_daddr = dev_end; + if (me->me_start_daddr < dev_start) + me->me_start_daddr = dev_start; /* start and end have to be aligned to the lba size */ if (!IS_ALIGNED(BBTOB(me->me_start_daddr | me->me_end_daddr), @@ -323,8 +334,7 @@ xfs_verify_media( * verifying. */ daddr = me->me_start_daddr; - bbcount = min_t(sector_t, me->me_end_daddr, btp->bt_nr_sectors) - - me->me_start_daddr; + bbcount = me->me_end_daddr - me->me_start_daddr; folio = xfs_verify_alloc_folio(xfs_verify_iosize(me, btp, bbcount)); if (!folio) diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index 714d111d80530..61782dfdfe5a5 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -686,6 +686,7 @@ void acpi_bus_unregister_driver(struct acpi_driver *driver); int acpi_bus_scan(acpi_handle handle); void acpi_bus_trim(struct acpi_device *start); acpi_status acpi_bus_get_ejd(acpi_handle handle, acpi_handle * ejd); +struct device *acpi_bus_get_primary_device(struct acpi_device *adev); int acpi_match_device_ids(struct acpi_device *device, const struct acpi_device_id *ids); void acpi_set_modalias(struct acpi_device *adev, const char *default_id, @@ -1000,6 +1001,11 @@ int acpi_scan_add_dep(acpi_handle handle, struct acpi_handle_list *dep_devices); u32 arch_acpi_add_auto_dep(acpi_handle handle); #else /* CONFIG_ACPI */ +static inline struct device *acpi_bus_get_primary_device(struct acpi_device *adev) +{ + return NULL; +} + static inline bool acpi_of_match_device(const struct acpi_device *adev, const struct of_device_id *of_match_table, const struct of_device_id **of_id) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 5659f4b5a1252..ee9c5d354a856 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -839,12 +839,20 @@ .stab.index 0 : { *(.stab.index) } \ .stab.indexstr 0 : { *(.stab.indexstr) } +#ifdef CONFIG_KLP_BUILD +#define KLP_SYMID \ + .klp.symid 0 : { *(.klp.symid) } +#else +#define KLP_SYMID +#endif + /* Required sections not related to debugging. */ #define ELF_DETAILS \ .comment 0 : { *(.comment) } \ .symtab 0 : { *(.symtab) } \ .strtab 0 : { *(.strtab) } \ - .shstrtab 0 : { *(.shstrtab) } + .shstrtab 0 : { *(.shstrtab) } \ + KLP_SYMID #define MODINFO \ .modinfo : { *(.modinfo) . = ALIGN(8); } diff --git a/include/clocksource/samsung_pwm.h b/include/clocksource/samsung_pwm.h index 9b435caa95fef..36f6f246e5594 100644 --- a/include/clocksource/samsung_pwm.h +++ b/include/clocksource/samsung_pwm.h @@ -15,7 +15,7 @@ * spinlock is not shared between both drivers. */ #ifdef CONFIG_CLKSRC_SAMSUNG_PWM -extern spinlock_t samsung_pwm_lock; +extern raw_spinlock_t samsung_pwm_lock; #endif struct samsung_pwm_variant { diff --git a/include/cxl/mailbox.h b/include/cxl/mailbox.h index c4e99e2e3a9d4..d008b9db07aa6 100644 --- a/include/cxl/mailbox.h +++ b/include/cxl/mailbox.h @@ -50,6 +50,7 @@ struct cxl_mbox_cmd { * @payload_size: Size of space for payload * (CXL 3.1 8.2.8.4.3 Mailbox Capabilities Register) * @mbox_mutex: mutex protects device mailbox and firmware + * @feat_mutex: serializes multi-part Get/Set Feature transfers * @mbox_wait: rcuwait for mailbox * @mbox_send: @dev specific transport for transmitting mailbox commands * @feat_cap: Features capability @@ -60,6 +61,7 @@ struct cxl_mailbox { DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); size_t payload_size; struct mutex mbox_mutex; /* lock to protect mailbox context */ + struct mutex feat_mutex; struct rcuwait mbox_wait; int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd); enum cxl_features_capability feat_cap; diff --git a/include/drm/bridge/dw_dp.h b/include/drm/bridge/dw_dp.h index 25363541e69d5..22105c3e8e4d6 100644 --- a/include/drm/bridge/dw_dp.h +++ b/include/drm/bridge/dw_dp.h @@ -24,4 +24,5 @@ struct dw_dp_plat_data { struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder, const struct dw_dp_plat_data *plat_data); +void dw_dp_unbind(struct dw_dp *dp); #endif /* __DW_DP__ */ diff --git a/include/drm/drm_pagemap.h b/include/drm/drm_pagemap.h index 95eb4b66b0577..ebbd3b0ddf367 100644 --- a/include/drm/drm_pagemap.h +++ b/include/drm/drm_pagemap.h @@ -2,6 +2,7 @@ #ifndef _DRM_PAGEMAP_H_ #define _DRM_PAGEMAP_H_ +#include #include #include #include @@ -339,6 +340,9 @@ struct drm_pagemap_migrate_details { #if IS_ENABLED(CONFIG_ZONE_DEVICE) +#define DRM_PAGEMAP_ZDD_FLAG_MIGRATED BIT(0) +#define DRM_PAGEMAP_ZDD_FLAG_MASK DRM_PAGEMAP_ZDD_FLAG_MIGRATED + int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, struct mm_struct *mm, unsigned long start, unsigned long end, @@ -373,7 +377,9 @@ static inline struct drm_pagemap_zdd *drm_pagemap_page_zone_device_data(struct p { struct folio *folio = page_folio(page); - return folio_zone_device_data(folio); + return (struct drm_pagemap_zdd *) + ((unsigned long)folio_zone_device_data(folio) & + ~DRM_PAGEMAP_ZDD_FLAG_MASK); } #else diff --git a/include/keys/request_key_auth-type.h b/include/keys/request_key_auth-type.h index 01e42ee5f4099..464636278c4f8 100644 --- a/include/keys/request_key_auth-type.h +++ b/include/keys/request_key_auth-type.h @@ -22,7 +22,7 @@ struct request_key_auth { const struct cred *cred; void *callout_info; size_t callout_len; - pid_t pid; + struct pid *pid; char op[8]; } __randomize_layout; diff --git a/include/kunit/run-in-irq-context.h b/include/kunit/run-in-irq-context.h index bfe60d6cf28d8..3802b6fb218ed 100644 --- a/include/kunit/run-in-irq-context.h +++ b/include/kunit/run-in-irq-context.h @@ -38,11 +38,13 @@ static enum hrtimer_restart kunit_irq_test_timer_func(struct hrtimer *timer) softirq_calls = atomic_read(&state->softirq_func_calls); /* - * If the timer is firing too often for the softirq or task to ever have - * a chance to run, increase the timer interval. This is needed on very - * slow systems. + * If the hrtimer is running much faster than the bh_work or the task, + * then it is firing too fast and might be starving those contexts as + * well as the actual system timer tick. Increase the interval. */ - if (hardirq_calls >= 20 && (softirq_calls == 0 || task_calls == 0)) + if (hardirq_calls >= 20 && + (hardirq_calls / 2 > softirq_calls || + hardirq_calls / 2 > task_calls)) state->interval = ktime_add_ns(state->interval, 250); if (!state->func(state->test_specific_state)) diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h index c3dda32fd8030..0ea2a8bf7efb7 100644 --- a/include/linux/bio-integrity.h +++ b/include/linux/bio-integrity.h @@ -17,6 +17,9 @@ enum bip_flags { BIP_MEMPOOL = 1 << 15, /* buffer backed by mempool */ }; +/* flags that require generate/verify action. */ +#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG) + struct bio_integrity_payload { struct bvec_iter bip_iter; diff --git a/include/linux/bio.h b/include/linux/bio.h index 8f33f717b14f5..ce34ea49ef358 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -477,7 +477,7 @@ int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data, size_t len, enum req_op op); int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, - unsigned len_align_mask); + unsigned mem_align_mask, unsigned len_align_mask); void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter); void __bio_release_pages(struct bio *bio, bool mark_dirty); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7719f65284456..c2c327dc0a72f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -570,7 +570,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (IS_ERR_OR_NULL(rec)) { if (long_memcpy) - bpf_long_memcpy(dst, src, round_up(size, 8)); + bpf_long_memcpy(dst, src, size); else memcpy(dst, src, size); return; @@ -593,7 +593,7 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) { - bpf_obj_memcpy(map->record, dst, src, map->value_size, true); + bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true); } static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) @@ -894,6 +894,7 @@ enum bpf_arg_type { ARG_PTR_TO_CTX, /* pointer to context */ ARG_ANYTHING, /* any (initialized) argument is ok */ + ARG_SCALAR, /* scalar argument */ ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ @@ -1118,21 +1119,6 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn) return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } -/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an - * atomic load or store, and false if it is a read-modify-write instruction. - */ -static inline bool -bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) -{ - switch (atomic_insn->imm) { - case BPF_LOAD_ACQ: - case BPF_STORE_REL: - return true; - default: - return false; - } -} - struct bpf_prog_ops { int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); @@ -1524,6 +1510,7 @@ int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, struct bpf_tracing_multi_link *link); int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link); +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags); /* * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn @@ -1647,6 +1634,7 @@ static inline int bpf_trampoline_multi_detach(struct bpf_prog *prog, { return -ENOTSUPP; } +static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif struct bpf_func_info_aux { @@ -1865,8 +1853,9 @@ struct bpf_prog_aux { struct bpf_prog { u16 pages; /* Number of allocated pages */ - u16 jited:1, /* Is our filter JIT'ed? */ + u32 jited:1, /* Is our filter JIT'ed? */ jit_requested:1,/* archs need to JIT the prog */ + jit_required:1, /* program strictly requires JIT compiler */ gpl_compatible:1, /* Is filter GPL compatible? */ cb_access:1, /* Is control block accessed? */ dst_needed:1, /* Do we need dst entry? */ @@ -3146,7 +3135,7 @@ int btf_struct_access(struct bpf_verifier_log *log, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict); + bool strict, bool walk_flex_arrays); int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf *btf, @@ -3170,7 +3159,6 @@ const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); @@ -3509,11 +3497,6 @@ static inline void bpf_task_storage_free(struct task_struct *task) { } -static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return false; -} - static inline const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) @@ -4169,7 +4152,7 @@ bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image) } #endif -static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type) +static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) { switch (map_type) { case BPF_MAP_TYPE_PERCPU_ARRAY: @@ -4196,13 +4179,13 @@ static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 all return -EINVAL; if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) { - if (!bpf_map_supports_cpu_flags(map->map_type)) + if (!bpf_map_is_percpu_map(map->map_type)) return -EINVAL; if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS)) return -EINVAL; cpu = flags >> 32; - if ((flags & BPF_F_CPU) && cpu >= num_possible_cpus()) + if ((flags & BPF_F_CPU) && (cpu >= nr_cpu_ids || !cpu_possible(cpu))) return -ERANGE; } diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 39a851e690ec4..7ad03ffb9300f 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -385,6 +385,8 @@ struct bpf_func_state { * | number of simulations is tracked in frame N */ u32 callback_depth; + /* Instructions processed in this frame and callees on the current path. */ + u32 insns_subtotal; /* The following fields should be last. See copy_func_state() */ /* The state of the stack. Each element of the array describes BPF_REG_SIZE @@ -704,6 +706,8 @@ struct bpf_insn_aux_data { */ u32 calls_callback:1; u32 indirect_target:1; /* if it is an indirect jump target */ + /* true if some jump or call instruction targets this instruction */ + u32 jump_target:1; /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -803,7 +807,8 @@ struct bpf_subprog_info { u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */ u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; - u32 insn_processed; + u32 insns_total; + u32 insns_self; /* offsets in range [stack_depth .. fastcall_stack_off) * are used for bpf_fastcall spills and fills. */ @@ -1115,6 +1120,16 @@ static inline void mark_jmp_point(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].jmp_point = true; } +static inline void mark_jump_target(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].jump_target = true; +} + +static inline bool bpf_is_jump_target(struct bpf_verifier_env *env, int insn_idx) +{ + return env->insn_aux_data[insn_idx].jump_target; +} + static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env) { struct bpf_verifier_state *cur = env->cur_state; @@ -1307,7 +1322,9 @@ static inline bool bpf_type_has_unsafe_modifiers(u32 type) static inline bool type_is_ptr_alloc_obj(u32 type) { - return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC; + return base_type(type) == PTR_TO_BTF_ID && + type_flag(type) & MEM_ALLOC && + !(type_flag(type) & PTR_UNTRUSTED); } static inline bool type_is_non_owning_ref(u32 type) diff --git a/include/linux/compaction.h b/include/linux/compaction.h index f29ef0653546e..66a2f70e9e019 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -58,6 +58,7 @@ enum compact_result { }; struct alloc_context; /* in mm/internal.h */ +struct capture_control; /* in mm/internal.h */ /* * Number of free order-0 pages that should be available above given watermark @@ -92,7 +93,7 @@ extern int fragmentation_index(struct zone *zone, unsigned int order); extern enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, unsigned int alloc_flags, const struct alloc_context *ac, enum compact_priority prio, - struct page **page); + struct capture_control *capc); extern void reset_isolation_suitable(pg_data_t *pgdat); extern bool compaction_suitable(struct zone *zone, int order, unsigned long watermark, int highest_zoneidx); diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 65d76a38974ba..bf3999daa080a 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -273,6 +273,7 @@ static inline void dl_rebuild_rd_accounting(void) static inline void rebuild_sched_domains(void) { + guard(cpus_read_lock)(); partition_sched_domains(1, NULL, NULL); } diff --git a/include/linux/designware_i2c.h b/include/linux/designware_i2c.h new file mode 100644 index 0000000000000..53f37f18a7229 --- /dev/null +++ b/include/linux/designware_i2c.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Synopsys DesignWare I2C register definitions + * + * Copyright (C) 2026, Intel Corporation + */ + +#ifndef __LINUX_DESIGNWARE_I2C_H +#define __LINUX_DESIGNWARE_I2C_H + +#include + +/* + * Registers offset + */ +#define DW_IC_CON 0x00 +#define DW_IC_TAR 0x04 +#define DW_IC_SAR 0x08 +#define DW_IC_DATA_CMD 0x10 +#define DW_IC_SS_SCL_HCNT 0x14 +#define DW_IC_SS_SCL_LCNT 0x18 +#define DW_IC_FS_SCL_HCNT 0x1c +#define DW_IC_FS_SCL_LCNT 0x20 +#define DW_IC_HS_SCL_HCNT 0x24 +#define DW_IC_HS_SCL_LCNT 0x28 +#define DW_IC_INTR_STAT 0x2c +#define DW_IC_INTR_MASK 0x30 +#define DW_IC_RAW_INTR_STAT 0x34 +#define DW_IC_RX_TL 0x38 +#define DW_IC_TX_TL 0x3c +#define DW_IC_CLR_INTR 0x40 +#define DW_IC_CLR_RX_UNDER 0x44 +#define DW_IC_CLR_RX_OVER 0x48 +#define DW_IC_CLR_TX_OVER 0x4c +#define DW_IC_CLR_RD_REQ 0x50 +#define DW_IC_CLR_TX_ABRT 0x54 +#define DW_IC_CLR_RX_DONE 0x58 +#define DW_IC_CLR_ACTIVITY 0x5c +#define DW_IC_CLR_STOP_DET 0x60 +#define DW_IC_CLR_START_DET 0x64 +#define DW_IC_CLR_GEN_CALL 0x68 +#define DW_IC_ENABLE 0x6c +#define DW_IC_STATUS 0x70 +#define DW_IC_TXFLR 0x74 +#define DW_IC_RXFLR 0x78 +#define DW_IC_SDA_HOLD 0x7c +#define DW_IC_TX_ABRT_SOURCE 0x80 +#define DW_IC_ENABLE_STATUS 0x9c +#define DW_IC_CLR_RESTART_DET 0xa8 +#define DW_IC_SMBUS_INTR_STAT 0xc8 +#define DW_IC_SMBUS_INTR_MASK 0xcc +#define DW_IC_CLR_SMBUS_INTR 0xd4 +#define DW_IC_COMP_PARAM_1 0xf4 +#define DW_IC_COMP_VERSION 0xf8 +#define DW_IC_COMP_TYPE 0xfc + +/* DW_IC_CON bits */ +#define DW_IC_CON_MASTER BIT(0) +#define DW_IC_CON_SPEED_STD (1 << 1) +#define DW_IC_CON_SPEED_FAST (2 << 1) +#define DW_IC_CON_SPEED_HIGH (3 << 1) +#define DW_IC_CON_SPEED_MASK GENMASK(2, 1) +#define DW_IC_CON_10BITADDR_SLAVE BIT(3) +#define DW_IC_CON_10BITADDR_MASTER BIT(4) +#define DW_IC_CON_RESTART_EN BIT(5) +#define DW_IC_CON_SLAVE_DISABLE BIT(6) +#define DW_IC_CON_STOP_DET_IFADDRESSED BIT(7) +#define DW_IC_CON_TX_EMPTY_CTRL BIT(8) +#define DW_IC_CON_RX_FIFO_FULL_HLD_CTRL BIT(9) +#define DW_IC_CON_BUS_CLEAR_CTRL BIT(11) + +/* DW_IC_DATA_CMD bits */ +#define DW_IC_DATA_CMD_DAT GENMASK(7, 0) +#define DW_IC_DATA_CMD_FIRST_DATA_BYTE BIT(11) + +/* DW_IC_INTR_* bits */ +#define DW_IC_INTR_RX_UNDER BIT(0) +#define DW_IC_INTR_RX_OVER BIT(1) +#define DW_IC_INTR_RX_FULL BIT(2) +#define DW_IC_INTR_TX_OVER BIT(3) +#define DW_IC_INTR_TX_EMPTY BIT(4) +#define DW_IC_INTR_RD_REQ BIT(5) +#define DW_IC_INTR_TX_ABRT BIT(6) +#define DW_IC_INTR_RX_DONE BIT(7) +#define DW_IC_INTR_ACTIVITY BIT(8) +#define DW_IC_INTR_STOP_DET BIT(9) +#define DW_IC_INTR_START_DET BIT(10) +#define DW_IC_INTR_GEN_CALL BIT(11) +#define DW_IC_INTR_RESTART_DET BIT(12) +#define DW_IC_INTR_MST_ON_HOLD BIT(13) + +/* DW_IC_ENABLE bits */ +#define DW_IC_ENABLE_ENABLE BIT(0) +#define DW_IC_ENABLE_ABORT BIT(1) + +/* DW_IC_STATUS bits */ +#define DW_IC_STATUS_ACTIVITY BIT(0) +#define DW_IC_STATUS_TFE BIT(2) +#define DW_IC_STATUS_RFNE BIT(3) +#define DW_IC_STATUS_MASTER_ACTIVITY BIT(5) +#define DW_IC_STATUS_SLAVE_ACTIVITY BIT(6) +#define DW_IC_STATUS_MASTER_HOLD_TX_FIFO_EMPTY BIT(7) + +/* DW_IC_SMBUS_INTR_* bits */ +#define DW_IC_SMBUS_INTR_ALERT BIT(10) + +#endif /* __LINUX_DESIGNWARE_I2C_H */ diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h index 7b2968612b7e6..6746838940649 100644 --- a/include/linux/dm-io.h +++ b/include/linux/dm-io.h @@ -27,7 +27,7 @@ struct page_list { struct page *page; }; -typedef void (*io_notify_fn)(unsigned int long error, void *context); +typedef void (*io_notify_fn)(unsigned long int error, unsigned long int unsup, void *context); enum dm_io_mem_type { DM_IO_PAGE_LIST,/* Page list */ @@ -80,8 +80,8 @@ void dm_io_client_destroy(struct dm_io_client *client); * error occurred doing io to the corresponding region. */ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, - struct dm_io_region *region, unsigned int long *sync_error_bits, - unsigned short ioprio); + struct dm_io_region *region, unsigned long int *sync_error_bits, + unsigned long int *sync_unsup_bits, unsigned short ioprio); #endif /* __KERNEL__ */ #endif /* _LINUX_DM_IO_H */ diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index d1203da56fc5f..d15b2b31d3c91 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -567,6 +567,7 @@ void dma_buf_unpin(struct dma_buf_attachment *attach); struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info); int dma_buf_fd(struct dma_buf *dmabuf, int flags); +void dma_buf_fd_install(struct dma_buf *dmabuf, int fd); struct dma_buf *dma_buf_get(int fd); void dma_buf_put(struct dma_buf *dmabuf); diff --git a/include/linux/dma-fence-array.h b/include/linux/dma-fence-array.h index 1b1d87579c382..0c49d7ccefb64 100644 --- a/include/linux/dma-fence-array.h +++ b/include/linux/dma-fence-array.h @@ -28,7 +28,6 @@ struct dma_fence_array_cb { /** * struct dma_fence_array - fence to represent an array of fences * @base: fence base class - * @lock: spinlock for fence handling * @num_fences: number of fences in the array * @num_pending: fences in the array still pending * @fences: array of the fences diff --git a/include/linux/dma-fence-chain.h b/include/linux/dma-fence-chain.h index df3beadf15157..705c4394ac0d0 100644 --- a/include/linux/dma-fence-chain.h +++ b/include/linux/dma-fence-chain.h @@ -20,7 +20,6 @@ * @prev: previous fence of the chain * @prev_seqno: original previous seqno before garbage collection * @fence: encapsulated fence - * @lock: spinlock for fence handling */ struct dma_fence_chain { struct dma_fence base; @@ -81,9 +80,8 @@ dma_fence_chain_contained(struct dma_fence *fence) } /** - * dma_fence_chain_alloc - * - * Returns a new struct dma_fence_chain object or NULL on failure. + * dma_fence_chain_alloc - Returns a new &struct dma_fence_chain object or + * %NULL on failure. * * This specialized allocator has to be a macro for its allocations to be * accounted separately (to have a separate alloc_tag). The typecast is @@ -93,7 +91,8 @@ dma_fence_chain_contained(struct dma_fence *fence) kmalloc_obj(struct dma_fence_chain) /** - * dma_fence_chain_free + * dma_fence_chain_free - Frees an allocated but not used + * &struct dma_fence_chain object. * @chain: chain node to free * * Frees up an allocated but not used struct dma_fence_chain object. This diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h index b52ab692b22e8..c4a76bc4006b9 100644 --- a/include/linux/dma-fence.h +++ b/include/linux/dma-fence.h @@ -141,6 +141,9 @@ struct dma_fence_ops { * compute the name at runtime, without having it to store permanently * for each fence, or build a cache of some sort. * + * The returned string is RCU protected and can be freed after the fence + * signaled and a RCU grace period passed. + * * This callback is mandatory. */ const char * (*get_driver_name)(struct dma_fence *fence); @@ -153,6 +156,9 @@ struct dma_fence_ops { * having it to store permanently for each fence, or build a cache of * some sort. * + * The returned string is RCU protected and can be freed after the fence + * signaled and a RCU grace period passed. + * * This callback is mandatory. */ const char * (*get_timeline_name)(struct dma_fence *fence); diff --git a/include/linux/efi.h b/include/linux/efi.h index ccbc35479684a..24221a8424121 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1212,8 +1212,8 @@ efi_call_acpi_prm_handler(efi_status_t (__efiapi *handler_addr)(u64, void *), /* * efi_runtime_service() function identifiers. - * "NONE" is used by efi_recover_from_page_fault() to check if the page - * fault happened while executing an efi runtime service. + * "NONE" is used by efi_crash_gracefully_on_page_fault() to check if the + * page fault happened while executing an efi runtime service. */ enum efi_rts_ids { EFI_NONE, diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index 416a3352261fe..f7740c5f6967c 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -96,6 +96,9 @@ static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned l ret = arch_ptrace_report_syscall_entry(regs); if (ret || (work & SYSCALL_WORK_SYSCALL_EMU)) return -1L; + + /* ptrace might have changed work flags */ + work = READ_ONCE(current_thread_info()->syscall_work); } /* Do seccomp after ptrace, to catch any tracer changes. */ diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 12683b5d125e4..97a1adbd9eaee 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -944,6 +944,7 @@ struct kernel_ethtool_ts_info { #define ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM BIT(6) #define ETHTOOL_OP_NEEDS_RTNL_RSS BIT(7) #define ETHTOOL_OP_NEEDS_RTNL_GLINK BIT(8) +#define ETHTOOL_OP_NEEDS_RTNL_TEST BIT(9) /** * struct ethtool_ops - optional netdev operations @@ -981,6 +982,7 @@ struct kernel_ethtool_ts_info { * - netdev_update_features() * - netif_set_real_num_tx_queues() * - ethtool_op_get_link() (syncs link watch under rtnl_lock) + * - netif_open() / netif_close() (used by @self_test) * * @get_drvinfo: Report driver/device information. Modern drivers no * longer have to implement this callback. Most fields are diff --git a/include/linux/filter.h b/include/linux/filter.h index 14acb2455746f..e0e83bf33fd1b 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -383,6 +383,61 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn) /* Legacy alias */ #define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF) +/* + * Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an + * atomic load or store, and false if it is a read-modify-write instruction. + */ +static inline bool +bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) +{ + switch (atomic_insn->imm) { + case BPF_LOAD_ACQ: + case BPF_STORE_REL: + return true; + default: + return false; + } +} + +/* + * A load-acquire is the only BPF_STX class instruction that reads into + * dst_reg from src_reg + off16, i.e. it has the operand roles of a BPF_LDX. + * Unlike bpf_atomic_is_load_store(), @insn is not assumed to be a BPF_ATOMIC + * instruction here, so that callers which walk all instruction classes can + * use this directly. + */ +static inline bool bpf_atomic_is_load_acq(const struct bpf_insn *insn) +{ + return BPF_CLASS(insn->code) == BPF_STX && + (BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && + insn->imm == BPF_LOAD_ACQ; +} + +/* + * Given an instruction @insn, return the number of the BPF register that a + * BPF_ATOMIC reads the value at its memory operand into, or -1 if there is + * no such register. That is the register a BPF_PROBE_ATOMIC has to clear when + * the access faults. Like bpf_atomic_is_load_acq(), @insn is not assumed to + * be a BPF_ATOMIC here. + */ +static inline int bpf_atomic_load_reg(const struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_STX || + (BPF_MODE(insn->code) != BPF_ATOMIC && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)) + return -1; + + switch (insn->imm) { + case BPF_LOAD_ACQ: + return insn->dst_reg; + case BPF_CMPXCHG: + return BPF_REG_0; + default: + return (insn->imm & BPF_FETCH) ? insn->src_reg : -1; + } +} + /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ #define BPF_ST_MEM(SIZE, DST, OFF, IMM) \ diff --git a/include/linux/font.h b/include/linux/font.h index 6845f02d739a0..877efe165d2ab 100644 --- a/include/linux/font.h +++ b/include/linux/font.h @@ -49,6 +49,8 @@ static inline unsigned int font_glyph_pitch(unsigned int width) * scanlines, which is usually the glyph's height in scanlines. Fonts * coming from user space can sometimes have a different vertical pitch * with empty scanlines between two adjacent glyphs. + * + * Returns: the number of bytes per glyph */ static inline unsigned int font_glyph_size(unsigned int width, unsigned int vpitch) { @@ -60,7 +62,7 @@ static inline unsigned int font_glyph_size(unsigned int width, unsigned int vpit */ /** - * font_data_t - Raw font data + * typedef font_data_t - Raw font data * * Values of type font_data_t store a pointer to raw font data. The format * is monochrome. Each bit sets a pixel of a stored glyph. Font data does diff --git a/include/linux/fs.h b/include/linux/fs.h index 50ce731a2b78f..09de7bf6f1d28 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1598,12 +1598,12 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, static inline time64_t inode_get_atime_sec(const struct inode *inode) { - return inode->i_atime_sec; + return READ_ONCE(inode->i_atime_sec); } static inline long inode_get_atime_nsec(const struct inode *inode) { - return inode->i_atime_nsec; + return READ_ONCE(inode->i_atime_nsec); } static inline struct timespec64 inode_get_atime(const struct inode *inode) @@ -1617,8 +1617,8 @@ static inline struct timespec64 inode_get_atime(const struct inode *inode) static inline struct timespec64 inode_set_atime_to_ts(struct inode *inode, struct timespec64 ts) { - inode->i_atime_sec = ts.tv_sec; - inode->i_atime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_atime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_atime_nsec, ts.tv_nsec); return ts; } @@ -1633,12 +1633,12 @@ static inline struct timespec64 inode_set_atime(struct inode *inode, static inline time64_t inode_get_mtime_sec(const struct inode *inode) { - return inode->i_mtime_sec; + return READ_ONCE(inode->i_mtime_sec); } static inline long inode_get_mtime_nsec(const struct inode *inode) { - return inode->i_mtime_nsec; + return READ_ONCE(inode->i_mtime_nsec); } static inline struct timespec64 inode_get_mtime(const struct inode *inode) @@ -1651,8 +1651,8 @@ static inline struct timespec64 inode_get_mtime(const struct inode *inode) static inline struct timespec64 inode_set_mtime_to_ts(struct inode *inode, struct timespec64 ts) { - inode->i_mtime_sec = ts.tv_sec; - inode->i_mtime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_mtime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_mtime_nsec, ts.tv_nsec); return ts; } @@ -1677,12 +1677,12 @@ static inline struct timespec64 inode_set_mtime(struct inode *inode, static inline time64_t inode_get_ctime_sec(const struct inode *inode) { - return inode->i_ctime_sec; + return READ_ONCE(inode->i_ctime_sec); } static inline long inode_get_ctime_nsec(const struct inode *inode) { - return inode->i_ctime_nsec & ~I_CTIME_QUERIED; + return READ_ONCE(inode->i_ctime_nsec) & ~I_CTIME_QUERIED; } static inline struct timespec64 inode_get_ctime(const struct inode *inode) diff --git a/include/linux/fs/super_types.h b/include/linux/fs/super_types.h index ef7941e9dc79a..68747182abf9a 100644 --- a/include/linux/fs/super_types.h +++ b/include/linux/fs/super_types.h @@ -145,7 +145,7 @@ struct super_block { unsigned long s_magic; struct dentry *s_root; struct rw_semaphore s_umount; - int s_count; + refcount_t s_passive; atomic_t s_active; #ifdef CONFIG_SECURITY void *s_security; diff --git a/include/linux/fsl/ntmp.h b/include/linux/fsl/ntmp.h index d3b6c476b91a2..764ef28926085 100644 --- a/include/linux/fsl/ntmp.h +++ b/include/linux/fsl/ntmp.h @@ -75,8 +75,10 @@ struct ntmp_user { /* NTMP table bitmaps for resource management */ u32 ett_bitmap_size; u32 ect_bitmap_size; + u16 maft_num_entries; unsigned long *ett_gid_bitmap; /* only valid for switch */ unsigned long *ect_gid_bitmap; /* only valid for switch */ + unsigned long *maft_eid_bitmap; /* only valid for ENETC */ }; struct maft_entry_data { diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 02bc5027523a8..bd76a16a63afb 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -866,8 +866,9 @@ unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec); unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec); extern ftrace_func_t ftrace_trace_function; +struct trace_array; -int ftrace_regex_open(struct ftrace_ops *ops, int flag, +int ftrace_regex_open(struct trace_array *tr, struct ftrace_ops *ops, int flag, struct inode *inode, struct file *file); ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos); @@ -1077,7 +1078,7 @@ static inline unsigned long ftrace_location(unsigned long ip) * have them defined when ftrace is not enabled, but these * functions may still be called. Use a macro instead of inline. */ -#define ftrace_regex_open(ops, flag, inod, file) ({ -ENODEV; }) +#define ftrace_regex_open(tr, ops, flag, inode, file) ({ -ENODEV; }) #define ftrace_set_early_filter(ops, buf, enable) do { } while (0) #define ftrace_set_filter_ip(ops, ip, remove, reset) ({ -ENODEV; }) #define ftrace_set_filter_ips(ops, ips, cnt, remove, reset) ({ -ENODEV; }) diff --git a/include/linux/futex.h b/include/linux/futex.h index 51f4ccdc90927..51d5faa1266f2 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -73,6 +73,7 @@ static inline void futex_init_task(struct task_struct *tsk) void futex_exit_recursive(struct task_struct *tsk); void futex_exit_release(struct task_struct *tsk); void futex_exec_release(struct task_struct *tsk); +void futex_exec_done(struct task_struct *tsk); long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3); @@ -91,6 +92,7 @@ static inline void futex_init_task(struct task_struct *tsk) { } static inline void futex_exit_recursive(struct task_struct *tsk) { } static inline void futex_exit_release(struct task_struct *tsk) { } static inline void futex_exec_release(struct task_struct *tsk) { } +static inline void futex_exec_done(struct task_struct *tsk) { } static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3) { diff --git a/include/linux/hid.h b/include/linux/hid.h index b240baa95ab5b..4e2422ec03db7 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -642,6 +642,7 @@ enum hid_battery_status { * @max: maximum battery value from HID descriptor * @report_type: HID report type (input/feature) * @report_id: HID report ID for this battery + * @report_offset: bit offset of the capacity field within its report * @charge_status: current charging status * @status: battery reporting status * @capacity: current battery capacity (0-100) @@ -657,6 +658,7 @@ struct hid_battery { __s32 max; __s32 report_type; __s32 report_id; + __s32 report_offset; __s32 charge_status; enum hid_battery_status status; __s32 capacity; diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 2abaf99321e90..f883c03bd9653 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -171,6 +171,7 @@ extern int movable_gigantic_pages __read_mostly; extern int sysctl_hugetlb_shm_group __read_mostly; extern struct list_head huge_boot_pages[MAX_NUMNODES]; +void hugetlb_bootmem_struct_page_init(void); void hugetlb_bootmem_alloc(void); extern nodemask_t hugetlb_bootmem_nodes; void hugetlb_bootmem_set_nodes(void); @@ -1261,6 +1262,8 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm) { } +pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep); + static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { @@ -1293,6 +1296,10 @@ static inline bool hugetlbfs_pagecache_present( static inline void hugetlb_bootmem_alloc(void) { } + +static inline void hugetlb_bootmem_struct_page_init(void) +{ +} #endif /* CONFIG_HUGETLB_PAGE */ static inline spinlock_t *huge_pte_lock(struct hstate *h, diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h index 4d2a687933240..2edf40b112ae0 100644 --- a/include/linux/i3c/master.h +++ b/include/linux/i3c/master.h @@ -228,6 +228,8 @@ struct i3c_dev_desc { * every time the I3C device is rediscovered with a different dynamic * address assigned * @bus: I3C bus this device is attached to + * @node: unregistered device list node, only for use by + * i3c_master_register_new_i3c_devs(), it is not protected by a lock * * I3C device object exposed to I3C device drivers. The takes care of linking * this object to the relevant &struct_i3c_dev_desc one. @@ -238,6 +240,7 @@ struct i3c_device { struct device dev; struct i3c_dev_desc *desc; struct i3c_bus *bus; + struct list_head node; }; /* diff --git a/include/linux/ieee80211-mesh.h b/include/linux/ieee80211-mesh.h index 7eb15834531cf..9e548b9173df1 100644 --- a/include/linux/ieee80211-mesh.h +++ b/include/linux/ieee80211-mesh.h @@ -361,8 +361,7 @@ ieee80211_mesh_hwmp_perr_get_rcode(const u8 *ie, u8 dst_idx) /* IEEE Std 802.11-2016 9.4.2.113 PREQ element */ static inline bool ieee80211_mesh_preq_size_ok(const u8 *pos, u8 elen) { - struct ieee80211_mesh_hwmp_preq_bottom *preq_elem_bottom = - ieee80211_mesh_hwmp_preq_get_bottom(pos); + struct ieee80211_mesh_hwmp_preq_bottom *preq_elem_bottom; u8 target_count; int needed; @@ -378,6 +377,7 @@ static inline bool ieee80211_mesh_preq_size_ok(const u8 *pos, u8 elen) if (elen < needed) return false; + preq_elem_bottom = ieee80211_mesh_hwmp_preq_get_bottom(pos); target_count = preq_elem_bottom->target_count; /* IEEE Std 802.11-2016 Table 14-10 to 14-16 */ if (target_count < 1) diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 20cc16ea4e5ab..4846032bf4ffc 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -365,6 +365,9 @@ static inline int __vlan_insert_inner_tag(struct sk_buff *skb, const u8 meta_len = mac_len > ETH_TLEN ? skb_metadata_len(skb) : 0; struct vlan_ethhdr *veth; + if (unlikely(!pskb_may_pull(skb, mac_len))) + return -EINVAL; + if (skb_cow_head(skb, meta_len + VLAN_HLEN) < 0) return -ENOMEM; diff --git a/include/linux/igmp.h b/include/linux/igmp.h index 3a2d35a9f3078..a0cf0398519fd 100644 --- a/include/linux/igmp.h +++ b/include/linux/igmp.h @@ -57,20 +57,21 @@ struct ip_mc_socklist { }; struct ip_sf_list { - struct ip_sf_list *sf_next; + struct ip_sf_list __rcu *sf_next; unsigned long sf_count[2]; /* include/exclude counts */ __be32 sf_inaddr; unsigned char sf_gsresp; /* include in g & s response? */ unsigned char sf_oldin; /* change state */ unsigned char sf_crcount; /* retrans. left to send */ + struct rcu_head rcu; }; struct ip_mc_list { struct in_device *interface; __be32 multiaddr; unsigned int sfmode; - struct ip_sf_list *sources; - struct ip_sf_list *tomb; + struct ip_sf_list __rcu *sources; + struct ip_sf_list __rcu *tomb; unsigned long sfcount[2]; union { struct ip_mc_list *next; diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 87151a5b62c1b..dac5b2cb8aaaa 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -20,6 +20,14 @@ enum { * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set. */ IOU_F_TWQ_LAZY_WAKE = 1, + + /* + * Set when task_work is queued from a waitqueue wakeup handler, where + * an arbitrary provider waitqueue lock is held. Signaling the CQ ring + * eventfd inline from there can recurse back into that lock through + * epoll, so the eventfd signal must be deferred. + */ + IOU_F_TWQ_IN_WAKE = 2, }; enum io_uring_cmd_flags { diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 56b43d594e6e2..12eb908380a2d 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -212,24 +212,36 @@ struct iomap_write_ops { #define IOMAP_ATOMIC (1 << 9) /* torn-write protection */ #define IOMAP_DONTCACHE (1 << 10) -struct iomap_ops { - /* - * Return the existing mapping at pos, or reserve space starting at - * pos for up to length, as long as we can do it as a single mapping. - * The actual length is returned in iomap->length. - */ - int (*iomap_begin)(struct inode *inode, loff_t pos, loff_t length, - unsigned flags, struct iomap *iomap, - struct iomap *srcmap); +/* + * Return the existing mapping at pos, or reserve space starting at pos for up + * to length, as long as we can do it as a single mapping. + * The actual length is returned in iomap->length. + */ +typedef int (*iomap_iter_begin_fn)(struct inode *inode, loff_t pos, + loff_t length, unsigned flags, struct iomap *iomap, + struct iomap *srcmap); - /* - * Commit and/or unreserve space previous allocated using iomap_begin. - * Written indicates the length of the successful write operation which - * needs to be commited, while the rest needs to be unreserved. - * Written might be zero if no data was written. - */ - int (*iomap_end)(struct inode *inode, loff_t pos, loff_t length, - ssize_t written, unsigned flags, struct iomap *iomap); +/* + * Commit and/or unreserve space previously allocated by iomap_iter_begin_fn. + * Written indicates the length of the successful write operation which needs + * to be committed, while the rest needs to be unreserved. + * Written might be zero if no data was written. + */ +typedef int (*iomap_iter_end_fn)(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned flags, struct iomap *iomap); + +/* + * Produce the next mapping (finishing the previous one if needed). + * Return 1 to continue iterating, 0 if the range is fully consumed, or a + * negative error on failure. + */ +typedef int (*iomap_iter_next_fn)(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap); + +struct iomap_ops { + iomap_iter_begin_fn iomap_begin; + iomap_iter_end_fn iomap_end; + iomap_iter_next_fn iomap_next; }; /** @@ -317,6 +329,71 @@ static inline const struct iomap *iomap_iter_srcmap(const struct iomap_iter *i) return &i->iomap; } +int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, + struct iomap *srcmap, int ret); + +/** + * iomap_iter_next - finish the previous mapping and produce the next one + * @iter: iteration structure + * @iomap: mapping to finish and then repopulate + * @srcmap: source mapping to finish and then repopulate + * @begin: callback that produces a mapping for the current position + * @end: optional callback that finishes the previous mapping, or NULL + * + * Inline helper that implements the common body of an ->iomap_next() + * callback: it finishes the previous mapping via @end (if present), decides + * via iomap_iter_continue() whether to keep going, and obtains the next + * mapping via @begin. + * + * This helper is marked __always_inline so that when a caller passes + * compile-time-constant @begin and @end callbacks, the compiler can call them + * directly, avoiding the indirect-call overhead. + * + * Returns 1 to continue iterating, 0 once the range is fully consumed, or a + * negative errno on error. + */ +static __always_inline int iomap_iter_next(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap, + iomap_iter_begin_fn begin, iomap_iter_end_fn end) +{ + int ret = 0; + + if (iomap->length) { + if (end) { + /* + * Calculate how far the iter was advanced and the + * original length bytes for end(). + */ + ssize_t advanced = iter->pos - iter->iter_start_pos; + loff_t len; + + len = iomap_length_trim(iter, iter->iter_start_pos, + iter->len + advanced); + + ret = end(iter->inode, iter->iter_start_pos, len, + advanced, iter->flags, iomap); + } + ret = iomap_iter_continue(iter, iomap, srcmap, ret); + if (ret <= 0) + return ret; + } + + ret = begin(iter->inode, iter->pos, iter->len, iter->flags, iomap, + srcmap); + + return ret < 0 ? ret : 1; +} + +#define DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, end_fn) \ +int name(const struct iomap_iter *iter, struct iomap *iomap, \ + struct iomap *srcmap) \ +{ \ + return iomap_iter_next(iter, iomap, srcmap, begin_fn, end_fn); \ +} + +#define DEFINE_IOMAP_ITER_NEXT(name, begin_fn) \ + DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, NULL) + /* * Return the file offset for the first unchanged block after a short write. * diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index f78787e654f4c..2c2fb39f049c3 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -6,6 +6,7 @@ #define __LINUX_IRQCHIP_ARM_GIC_V5_H #include +#include #include #include @@ -324,6 +325,7 @@ struct gicv5_irs_chip_data { struct list_head entry; struct fwnode_handle *fwnode; void __iomem *irs_base; + struct resource res; u32 flags; u32 spi_min; u32 spi_range; diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index 8c4f3bb244295..e6de7ae55bda0 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -181,6 +181,7 @@ struct kprobe_blacklist_entry { struct list_head list; unsigned long start_addr; unsigned long end_addr; + struct rcu_head rcu; }; #ifdef CONFIG_KPROBES diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ab8cfaec82d31..c00fc1740ce58 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2608,6 +2608,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 28f086c4a1873..d929d83abf3be 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -126,6 +126,15 @@ struct nd_mapping_desc { struct bio; struct resource; struct nd_region; + +/* + * Provider flush callback return values: + * 0: flush completed synchronously + * <0: flush failed + * >0: flush completion was queued and @bio will be completed later + */ +#define NVDIMM_FLUSH_ASYNC 1 + struct nd_region_desc { struct resource *res; struct nd_mapping_desc *mapping; diff --git a/include/linux/list.h b/include/linux/list.h index 09d979976b3b8..59f8aa0905d3b 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -150,10 +150,13 @@ static inline bool __list_del_entry_valid(struct list_head *entry) * * This is only for internal list manipulation where we know * the prev/next entries already! + * + * Must be inlined to ensure it can be safely called + * with initdata arguments. */ -static inline void __list_add(struct list_head *new, - struct list_head *prev, - struct list_head *next) +static __always_inline void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next) { if (!__list_add_valid(new, prev, next)) return; @@ -171,8 +174,12 @@ static inline void __list_add(struct list_head *new, * * Insert a new entry after the specified head. * This is good for implementing stacks. + * + * Must be inlined to ensure it can be safely called + * with initdata arguments. */ -static inline void list_add(struct list_head *new, struct list_head *head) +static __always_inline void list_add(struct list_head *new, + struct list_head *head) { __list_add(new, head, head->next); } diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h index 88722e5caf020..63ea5417de849 100644 --- a/include/linux/liveupdate.h +++ b/include/linux/liveupdate.h @@ -173,7 +173,9 @@ struct liveupdate_flb_ops { * @lock: A mutex that protects all fields within this structure, providing * the synchronization service for the FLB's ops. * @finished: True once the FLB's finish() callback has run. - * @retrieved: True once the FLB's retrieve() callback has run. + * @retrieve_status: Status code indicating whether retrieve() has been + * attempted. 0 means not attempted, 1 means successful, + * and negative value means it failed with that error code. */ struct luo_flb_private_state { refcount_t count; @@ -181,7 +183,7 @@ struct luo_flb_private_state { void *obj; struct mutex lock; bool finished; - bool retrieved; + int retrieve_status; }; /* @@ -243,6 +245,7 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp); void liveupdate_flb_put_incoming(struct liveupdate_flb *flb); int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp); +void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb); #else /* CONFIG_LIVEUPDATE */ @@ -292,5 +295,9 @@ static inline int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, return -EOPNOTSUPP; } +static inline void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb) +{ +} + #endif /* CONFIG_LIVEUPDATE */ #endif /* _LINUX_LIVEUPDATE_H */ diff --git a/include/linux/lockd/bind.h b/include/linux/lockd/bind.h index b614e0deea726..db8207d4059f3 100644 --- a/include/linux/lockd/bind.h +++ b/include/linux/lockd/bind.h @@ -16,17 +16,23 @@ struct svc_rqst; struct rpc_task; struct rpc_clnt; struct super_block; +struct module; -/* - * This is the set of functions for lockd->nfsd communication +/** + * struct nlmsvc_binding - lockd -> nfsd callback table + * @owner: module that provides this binding. + * @fopen: open a file by NFS file handle on behalf of an NLM request. + * @fclose: close a file that was previously opened via @fopen. + * Implementations MUST be semantically equivalent to fput(). */ struct nlmsvc_binding { + struct module *owner; int (*fopen)(struct svc_rqst *rqstp, struct nfs_fh *f, struct file **filp, int flags); void (*fclose)(struct file *filp); }; -extern const struct nlmsvc_binding *nlmsvc_ops; +extern const struct nlmsvc_binding __rcu *nlmsvc_ops; /* * Similar to nfs_client_initdata, but without the NFS-specific diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 4a5631906affb..0d712e7be6ddc 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -572,7 +572,7 @@ void maple_tree_init(void); void mas_destroy(struct ma_state *mas); void *mas_prev(struct ma_state *mas, unsigned long min); -void *mas_prev_range(struct ma_state *mas, unsigned long max); +void *mas_prev_range(struct ma_state *mas, unsigned long min); void *mas_next(struct ma_state *mas, unsigned long max); void *mas_next_range(struct ma_state *mas, unsigned long max); diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 8170bb8066a22..32621fa690ac7 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -270,10 +270,15 @@ struct mem_cgroup { #endif int kmemcg_id; - struct memcg_vmstats_percpu __percpu *vmstats_percpu; - #ifdef CONFIG_CGROUP_WRITEBACK struct list_head cgwb_list; +#endif + + /* Keep the hot per-CPU stats pointer away from memory event counters. */ + struct memcg_vmstats_percpu __percpu *vmstats_percpu + ____cacheline_aligned_in_smp; + +#ifdef CONFIG_CGROUP_WRITEBACK struct wb_domain cgwb_domain; struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; #endif diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 7c9d66729c609..06c58cb057791 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -289,7 +289,7 @@ static inline void __remove_memory(u64 start, u64 size) {} /* Default online_type (MMOP_*) when new memory blocks are added. */ extern enum mmop mhp_get_default_online_type(void); extern void mhp_set_default_online_type(enum mmop online_type); -extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat); +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat); extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, diff --git a/include/linux/mempool.h b/include/linux/mempool.h index a0fa6d43e0dc4..6da502aef2f71 100644 --- a/include/linux/mempool.h +++ b/include/linux/mempool.h @@ -70,6 +70,13 @@ int mempool_alloc_bulk_noprof(struct mempool *pool, void **elem, #define mempool_alloc_bulk(...) \ alloc_hooks(mempool_alloc_bulk_noprof(__VA_ARGS__)) +/* + * Allocate a new element without dipping into the pool's reserves or + * waiting. Returns NULL on failure. + */ +#define mempool_alloc_noreserve(_pool, _gfp) \ + alloc_hooks((_pool)->alloc(_gfp, (_pool)->pool_data)) + void *mempool_alloc_preallocated(struct mempool *pool) __malloc; void mempool_free(void *element, struct mempool *pool); unsigned int mempool_free_bulk(struct mempool *pool, void **elem, diff --git a/include/linux/mfd/db8500-prcmu.h b/include/linux/mfd/db8500-prcmu.h index a62de3d155edc..c939c9a1170a0 100644 --- a/include/linux/mfd/db8500-prcmu.h +++ b/include/linux/mfd/db8500-prcmu.h @@ -12,6 +12,9 @@ #include #include +#include + +#include /* For clock identifiers */ /* * Registers @@ -24,6 +27,38 @@ #define DB8500_PRCM_DSI_SW_RESET_DSI1_SW_RESETN BIT(1) #define DB8500_PRCM_DSI_SW_RESET_DSI2_SW_RESETN BIT(2) +/* Offset for the firmware version within the TCPM */ +#define DB8500_PRCMU_FW_VERSION_OFFSET 0xA4 + +#define DB8500_PRCMU_LEGACY_OFFSET 0xDD4 + +/* + * CLKOUT sources + */ +#define PRCMU_CLKSRC_CLK38M 0x00 +#define PRCMU_CLKSRC_ACLK 0x01 +#define PRCMU_CLKSRC_SYSCLK 0x02 +#define PRCMU_CLKSRC_LCDCLK 0x03 +#define PRCMU_CLKSRC_SDMMCCLK 0x04 +#define PRCMU_CLKSRC_TVCLK 0x05 +#define PRCMU_CLKSRC_TIMCLK 0x06 +#define PRCMU_CLKSRC_CLK009 0x07 +/* These are only valid for CLKOUT1: */ +#define PRCMU_CLKSRC_SIAMMDSPCLK 0x40 +#define PRCMU_CLKSRC_I2CCLK 0x41 +#define PRCMU_CLKSRC_MSP02CLK 0x42 +#define PRCMU_CLKSRC_ARMPLL_OBSCLK 0x43 +#define PRCMU_CLKSRC_HSIRXCLK 0x44 +#define PRCMU_CLKSRC_HSITXCLK 0x45 +#define PRCMU_CLKSRC_ARMCLKFIX 0x46 +#define PRCMU_CLKSRC_HDMICLK 0x47 + +/* + * Definitions for controlling ESRAM0 in deep sleep. + */ +#define ESRAM0_DEEP_SLEEP_STATE_OFF 1 +#define ESRAM0_DEEP_SLEEP_STATE_RET 2 + /* This portion previously known as */ /** @@ -451,10 +486,173 @@ enum prcmu_power_status { PRCMU_ARMPENDINGIT_ER = 0x93, }; +/* PRCMU Wakeup defines */ +enum prcmu_wakeup_index { + PRCMU_WAKEUP_INDEX_RTC, + PRCMU_WAKEUP_INDEX_RTT0, + PRCMU_WAKEUP_INDEX_RTT1, + PRCMU_WAKEUP_INDEX_HSI0, + PRCMU_WAKEUP_INDEX_HSI1, + PRCMU_WAKEUP_INDEX_USB, + PRCMU_WAKEUP_INDEX_ABB, + PRCMU_WAKEUP_INDEX_ABB_FIFO, + PRCMU_WAKEUP_INDEX_ARM, + PRCMU_WAKEUP_INDEX_CD_IRQ, + NUM_PRCMU_WAKEUP_INDICES +}; + +#define PRCMU_WAKEUP(_name) (BIT(PRCMU_WAKEUP_INDEX_##_name)) + +/** + * enum prcmu_wdog_id - PRCMU watchdog IDs + * @PRCMU_WDOG_ALL: use all timers + * @PRCMU_WDOG_CPU1: use first CPU timer only + * @PRCMU_WDOG_CPU2: use second CPU timer conly + */ +enum prcmu_wdog_id { + PRCMU_WDOG_ALL = 0x00, + PRCMU_WDOG_CPU1 = 0x01, + PRCMU_WDOG_CPU2 = 0x02, +}; + +/** + * enum ape_opp - APE OPP states definition + * @APE_OPP_INIT: + * @APE_NO_CHANGE: The APE operating point is unchanged + * @APE_100_OPP: The new APE operating point is ape100opp + * @APE_50_OPP: 50% + * @APE_50_PARTLY_25_OPP: 50%, except some clocks at 25%. + */ +enum ape_opp { + APE_OPP_INIT = 0x00, + APE_NO_CHANGE = 0x01, + APE_100_OPP = 0x02, + APE_50_OPP = 0x03, + APE_50_PARTLY_25_OPP = 0xFF, +}; + +/** + * enum arm_opp - ARM OPP states definition + * @ARM_OPP_INIT: + * @ARM_NO_CHANGE: The ARM operating point is unchanged + * @ARM_100_OPP: The new ARM operating point is arm100opp + * @ARM_50_OPP: The new ARM operating point is arm50opp + * @ARM_MAX_OPP: Operating point is "max" (more than 100) + * @ARM_MAX_FREQ100OPP: Set max opp if available, else 100 + * @ARM_EXTCLK: The new ARM operating point is armExtClk + */ +enum arm_opp { + ARM_OPP_INIT = 0x00, + ARM_NO_CHANGE = 0x01, + ARM_100_OPP = 0x02, + ARM_50_OPP = 0x03, + ARM_MAX_OPP = 0x04, + ARM_MAX_FREQ100OPP = 0x05, + ARM_EXTCLK = 0x07 +}; + +/** + * enum ddr_opp - DDR OPP states definition + * @DDR_100_OPP: The new DDR operating point is ddr100opp + * @DDR_50_OPP: The new DDR operating point is ddr50opp + * @DDR_25_OPP: The new DDR operating point is ddr25opp + */ +enum ddr_opp { + DDR_100_OPP = 0x00, + DDR_50_OPP = 0x01, + DDR_25_OPP = 0x02, +}; + +/** + * enum ddr_pwrst - DDR power states definition + * @DDR_PWR_STATE_UNCHANGED: SDRAM and DDR controller state is unchanged + * @DDR_PWR_STATE_ON: + * @DDR_PWR_STATE_OFFLOWLAT: + * @DDR_PWR_STATE_OFFHIGHLAT: + */ +enum ddr_pwrst { + DDR_PWR_STATE_UNCHANGED = 0x00, + DDR_PWR_STATE_ON = 0x01, + DDR_PWR_STATE_OFFLOWLAT = 0x02, + DDR_PWR_STATE_OFFHIGHLAT = 0x03 +}; + /* * Definitions for autonomous power management configuration. */ +/* EPOD (power domain) IDs */ + +/* + * DB8500 EPODs + * - EPOD_ID_SVAMMDSP: power domain for SVA MMDSP + * - EPOD_ID_SVAPIPE: power domain for SVA pipe + * - EPOD_ID_SIAMMDSP: power domain for SIA MMDSP + * - EPOD_ID_SIAPIPE: power domain for SIA pipe + * - EPOD_ID_SGA: power domain for SGA + * - EPOD_ID_B2R2_MCDE: power domain for B2R2 and MCDE + * - EPOD_ID_ESRAM12: power domain for ESRAM 1 and 2 + * - EPOD_ID_ESRAM34: power domain for ESRAM 3 and 4 + * - NUM_EPOD_ID: number of power domains + * + * TODO: These should be prefixed. + */ +#define EPOD_ID_SVAMMDSP 0 +#define EPOD_ID_SVAPIPE 1 +#define EPOD_ID_SIAMMDSP 2 +#define EPOD_ID_SIAPIPE 3 +#define EPOD_ID_SGA 4 +#define EPOD_ID_B2R2_MCDE 5 +#define EPOD_ID_ESRAM12 6 +#define EPOD_ID_ESRAM34 7 +#define NUM_EPOD_ID 8 + +/* + * state definition for EPOD (power domain) + * - EPOD_STATE_NO_CHANGE: The EPOD should remain unchanged + * - EPOD_STATE_OFF: The EPOD is switched off + * - EPOD_STATE_RAMRET: The EPOD is switched off with its internal RAM in + * retention + * - EPOD_STATE_ON_CLK_OFF: The EPOD is switched on, clock is still off + * - EPOD_STATE_ON: Same as above, but with clock enabled + */ +#define EPOD_STATE_NO_CHANGE 0x00 +#define EPOD_STATE_OFF 0x01 +#define EPOD_STATE_RAMRET 0x02 +#define EPOD_STATE_ON_CLK_OFF 0x03 +#define EPOD_STATE_ON 0x04 + +#define PRCMU_FW_PROJECT_U8500 2 +#define PRCMU_FW_PROJECT_U8400 3 +#define PRCMU_FW_PROJECT_U9500 4 /* Customer specific */ +#define PRCMU_FW_PROJECT_U8500_MBB 5 +#define PRCMU_FW_PROJECT_U8500_C1 6 +#define PRCMU_FW_PROJECT_U8500_C2 7 +#define PRCMU_FW_PROJECT_U8500_C3 8 +#define PRCMU_FW_PROJECT_U8500_C4 9 +#define PRCMU_FW_PROJECT_U9500_MBL 10 +#define PRCMU_FW_PROJECT_U8500_SSG1 11 /* Samsung specific */ +#define PRCMU_FW_PROJECT_U8500_MBL2 12 /* Customer specific */ +#define PRCMU_FW_PROJECT_U8520 13 +#define PRCMU_FW_PROJECT_U8420 14 +#define PRCMU_FW_PROJECT_U8500_SSG2 15 /* Samsung specific */ +#define PRCMU_FW_PROJECT_U8420_SYSCLK 17 +#define PRCMU_FW_PROJECT_A9420 20 +/* [32..63] 9540 and derivatives */ +#define PRCMU_FW_PROJECT_U9540 32 +/* [64..95] 8540 and derivatives */ +#define PRCMU_FW_PROJECT_L8540 64 +/* [96..126] 8580 and derivatives */ +#define PRCMU_FW_PROJECT_L8580 96 + +#define PRCMU_FW_PROJECT_NAME_LEN 20 + +/* PRCMU QoS APE OPP class */ +#define PRCMU_QOS_APE_OPP 1 +#define PRCMU_QOS_DDR_OPP 2 +#define PRCMU_QOS_ARM_OPP 3 +#define PRCMU_QOS_DEFAULT_VALUE -1 + #define PRCMU_AUTO_PM_OFF 0 #define PRCMU_AUTO_PM_ON 1 @@ -469,6 +667,14 @@ enum prcmu_auto_pm_policy { PRCMU_AUTO_PM_POLICY_DSP_CLK_OFF_HWP_CLK_OFF, }; +struct prcmu_fw_version { + u32 project; /* Notice, project shifted with 8 on ux540 */ + u8 api_version; + u8 func_version; + u8 errata; + char project_name[PRCMU_FW_PROJECT_NAME_LEN]; +}; + /** * struct prcmu_auto_pm_config - Autonomous power management configuration. * @sia_auto_pm_enable: SIA autonomous pm enable. (PRCMU_AUTO_PM_{OFF,ON}) @@ -501,6 +707,9 @@ void prcmu_configure_auto_pm(struct prcmu_auto_pm_config *sleep, bool prcmu_is_auto_pm_enabled(void); int prcmu_config_clkout(u8 clkout, u8 source, u8 div); +unsigned long prcmu_clock_rate(u8 clock); +long prcmu_round_clock_rate(u8 clock, unsigned long rate); +int prcmu_set_clock_rate(u8 clock, unsigned long rate); int prcmu_set_clock_divider(u8 clock, u8 divider); int db8500_prcmu_config_hotdog(u8 threshold); int db8500_prcmu_config_hotmon(u8 low, u8 high); @@ -508,6 +717,8 @@ int db8500_prcmu_start_temp_sense(u16 cycles32k); int db8500_prcmu_stop_temp_sense(void); int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size); int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size); +int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, + u8 *mask, u8 size); int prcmu_ac_wake_req(void); void prcmu_ac_sleep_req(void); @@ -610,6 +821,21 @@ static inline int prcmu_config_clkout(u8 clkout, u8 source, u8 div) return 0; } +static inline unsigned long prcmu_clock_rate(u8 clock) +{ + return 0; +} + +static inline long prcmu_round_clock_rate(u8 clock, unsigned long rate) +{ + return 0; +} + +static inline int prcmu_set_clock_rate(u8 clock, unsigned long rate) +{ + return 0; +} + static inline int prcmu_set_clock_divider(u8 clock, u8 divider) { return 0; @@ -637,12 +863,18 @@ static inline int db8500_prcmu_stop_temp_sense(void) static inline int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size) { - return -ENOSYS; + return -EINVAL; } static inline int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size) { - return -ENOSYS; + return -EINVAL; +} + +static inline int prcmu_abb_write_masked(u8 slave, u8 reg, + u8 *value, u8 *mask, u8 size) +{ + return -EINVAL; } static inline int prcmu_ac_wake_req(void) @@ -745,4 +977,20 @@ static inline void db8500_prcmu_write_masked(unsigned int reg, u32 mask, #endif /* !CONFIG_MFD_DB8500_PRCMU */ +static inline int prcmu_qos_add_requirement(int prcmu_qos_class, + char *name, s32 value) +{ + return 0; +} + +static inline int prcmu_qos_update_requirement(int prcmu_qos_class, + char *name, s32 new_value) +{ + return 0; +} + +static inline void prcmu_qos_remove_requirement(int prcmu_qos_class, char *name) +{ +} + #endif /* __MFD_DB8500_PRCMU_H */ diff --git a/include/linux/mfd/dbx500-prcmu.h b/include/linux/mfd/dbx500-prcmu.h deleted file mode 100644 index 828362b7860c6..0000000000000 --- a/include/linux/mfd/dbx500-prcmu.h +++ /dev/null @@ -1,575 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST Ericsson SA 2011 - * - * STE Ux500 PRCMU API - */ -#ifndef __MACH_PRCMU_H -#define __MACH_PRCMU_H - -#include -#include -#include - -#include /* For clock identifiers */ - -/* Offset for the firmware version within the TCPM */ -#define DB8500_PRCMU_FW_VERSION_OFFSET 0xA4 -#define DBX540_PRCMU_FW_VERSION_OFFSET 0xA8 - -/* PRCMU Wakeup defines */ -enum prcmu_wakeup_index { - PRCMU_WAKEUP_INDEX_RTC, - PRCMU_WAKEUP_INDEX_RTT0, - PRCMU_WAKEUP_INDEX_RTT1, - PRCMU_WAKEUP_INDEX_HSI0, - PRCMU_WAKEUP_INDEX_HSI1, - PRCMU_WAKEUP_INDEX_USB, - PRCMU_WAKEUP_INDEX_ABB, - PRCMU_WAKEUP_INDEX_ABB_FIFO, - PRCMU_WAKEUP_INDEX_ARM, - PRCMU_WAKEUP_INDEX_CD_IRQ, - NUM_PRCMU_WAKEUP_INDICES -}; -#define PRCMU_WAKEUP(_name) (BIT(PRCMU_WAKEUP_INDEX_##_name)) - -/* EPOD (power domain) IDs */ - -/* - * DB8500 EPODs - * - EPOD_ID_SVAMMDSP: power domain for SVA MMDSP - * - EPOD_ID_SVAPIPE: power domain for SVA pipe - * - EPOD_ID_SIAMMDSP: power domain for SIA MMDSP - * - EPOD_ID_SIAPIPE: power domain for SIA pipe - * - EPOD_ID_SGA: power domain for SGA - * - EPOD_ID_B2R2_MCDE: power domain for B2R2 and MCDE - * - EPOD_ID_ESRAM12: power domain for ESRAM 1 and 2 - * - EPOD_ID_ESRAM34: power domain for ESRAM 3 and 4 - * - NUM_EPOD_ID: number of power domains - * - * TODO: These should be prefixed. - */ -#define EPOD_ID_SVAMMDSP 0 -#define EPOD_ID_SVAPIPE 1 -#define EPOD_ID_SIAMMDSP 2 -#define EPOD_ID_SIAPIPE 3 -#define EPOD_ID_SGA 4 -#define EPOD_ID_B2R2_MCDE 5 -#define EPOD_ID_ESRAM12 6 -#define EPOD_ID_ESRAM34 7 -#define NUM_EPOD_ID 8 - -/* - * state definition for EPOD (power domain) - * - EPOD_STATE_NO_CHANGE: The EPOD should remain unchanged - * - EPOD_STATE_OFF: The EPOD is switched off - * - EPOD_STATE_RAMRET: The EPOD is switched off with its internal RAM in - * retention - * - EPOD_STATE_ON_CLK_OFF: The EPOD is switched on, clock is still off - * - EPOD_STATE_ON: Same as above, but with clock enabled - */ -#define EPOD_STATE_NO_CHANGE 0x00 -#define EPOD_STATE_OFF 0x01 -#define EPOD_STATE_RAMRET 0x02 -#define EPOD_STATE_ON_CLK_OFF 0x03 -#define EPOD_STATE_ON 0x04 - -/* - * CLKOUT sources - */ -#define PRCMU_CLKSRC_CLK38M 0x00 -#define PRCMU_CLKSRC_ACLK 0x01 -#define PRCMU_CLKSRC_SYSCLK 0x02 -#define PRCMU_CLKSRC_LCDCLK 0x03 -#define PRCMU_CLKSRC_SDMMCCLK 0x04 -#define PRCMU_CLKSRC_TVCLK 0x05 -#define PRCMU_CLKSRC_TIMCLK 0x06 -#define PRCMU_CLKSRC_CLK009 0x07 -/* These are only valid for CLKOUT1: */ -#define PRCMU_CLKSRC_SIAMMDSPCLK 0x40 -#define PRCMU_CLKSRC_I2CCLK 0x41 -#define PRCMU_CLKSRC_MSP02CLK 0x42 -#define PRCMU_CLKSRC_ARMPLL_OBSCLK 0x43 -#define PRCMU_CLKSRC_HSIRXCLK 0x44 -#define PRCMU_CLKSRC_HSITXCLK 0x45 -#define PRCMU_CLKSRC_ARMCLKFIX 0x46 -#define PRCMU_CLKSRC_HDMICLK 0x47 - -/** - * enum prcmu_wdog_id - PRCMU watchdog IDs - * @PRCMU_WDOG_ALL: use all timers - * @PRCMU_WDOG_CPU1: use first CPU timer only - * @PRCMU_WDOG_CPU2: use second CPU timer conly - */ -enum prcmu_wdog_id { - PRCMU_WDOG_ALL = 0x00, - PRCMU_WDOG_CPU1 = 0x01, - PRCMU_WDOG_CPU2 = 0x02, -}; - -/** - * enum ape_opp - APE OPP states definition - * @APE_OPP_INIT: - * @APE_NO_CHANGE: The APE operating point is unchanged - * @APE_100_OPP: The new APE operating point is ape100opp - * @APE_50_OPP: 50% - * @APE_50_PARTLY_25_OPP: 50%, except some clocks at 25%. - */ -enum ape_opp { - APE_OPP_INIT = 0x00, - APE_NO_CHANGE = 0x01, - APE_100_OPP = 0x02, - APE_50_OPP = 0x03, - APE_50_PARTLY_25_OPP = 0xFF, -}; - -/** - * enum arm_opp - ARM OPP states definition - * @ARM_OPP_INIT: - * @ARM_NO_CHANGE: The ARM operating point is unchanged - * @ARM_100_OPP: The new ARM operating point is arm100opp - * @ARM_50_OPP: The new ARM operating point is arm50opp - * @ARM_MAX_OPP: Operating point is "max" (more than 100) - * @ARM_MAX_FREQ100OPP: Set max opp if available, else 100 - * @ARM_EXTCLK: The new ARM operating point is armExtClk - */ -enum arm_opp { - ARM_OPP_INIT = 0x00, - ARM_NO_CHANGE = 0x01, - ARM_100_OPP = 0x02, - ARM_50_OPP = 0x03, - ARM_MAX_OPP = 0x04, - ARM_MAX_FREQ100OPP = 0x05, - ARM_EXTCLK = 0x07 -}; - -/** - * enum ddr_opp - DDR OPP states definition - * @DDR_100_OPP: The new DDR operating point is ddr100opp - * @DDR_50_OPP: The new DDR operating point is ddr50opp - * @DDR_25_OPP: The new DDR operating point is ddr25opp - */ -enum ddr_opp { - DDR_100_OPP = 0x00, - DDR_50_OPP = 0x01, - DDR_25_OPP = 0x02, -}; - -/* - * Definitions for controlling ESRAM0 in deep sleep. - */ -#define ESRAM0_DEEP_SLEEP_STATE_OFF 1 -#define ESRAM0_DEEP_SLEEP_STATE_RET 2 - -/** - * enum ddr_pwrst - DDR power states definition - * @DDR_PWR_STATE_UNCHANGED: SDRAM and DDR controller state is unchanged - * @DDR_PWR_STATE_ON: - * @DDR_PWR_STATE_OFFLOWLAT: - * @DDR_PWR_STATE_OFFHIGHLAT: - */ -enum ddr_pwrst { - DDR_PWR_STATE_UNCHANGED = 0x00, - DDR_PWR_STATE_ON = 0x01, - DDR_PWR_STATE_OFFLOWLAT = 0x02, - DDR_PWR_STATE_OFFHIGHLAT = 0x03 -}; - -#define DB8500_PRCMU_LEGACY_OFFSET 0xDD4 - -#define PRCMU_FW_PROJECT_U8500 2 -#define PRCMU_FW_PROJECT_U8400 3 -#define PRCMU_FW_PROJECT_U9500 4 /* Customer specific */ -#define PRCMU_FW_PROJECT_U8500_MBB 5 -#define PRCMU_FW_PROJECT_U8500_C1 6 -#define PRCMU_FW_PROJECT_U8500_C2 7 -#define PRCMU_FW_PROJECT_U8500_C3 8 -#define PRCMU_FW_PROJECT_U8500_C4 9 -#define PRCMU_FW_PROJECT_U9500_MBL 10 -#define PRCMU_FW_PROJECT_U8500_SSG1 11 /* Samsung specific */ -#define PRCMU_FW_PROJECT_U8500_MBL2 12 /* Customer specific */ -#define PRCMU_FW_PROJECT_U8520 13 -#define PRCMU_FW_PROJECT_U8420 14 -#define PRCMU_FW_PROJECT_U8500_SSG2 15 /* Samsung specific */ -#define PRCMU_FW_PROJECT_U8420_SYSCLK 17 -#define PRCMU_FW_PROJECT_A9420 20 -/* [32..63] 9540 and derivatives */ -#define PRCMU_FW_PROJECT_U9540 32 -/* [64..95] 8540 and derivatives */ -#define PRCMU_FW_PROJECT_L8540 64 -/* [96..126] 8580 and derivatives */ -#define PRCMU_FW_PROJECT_L8580 96 - -#define PRCMU_FW_PROJECT_NAME_LEN 20 -struct prcmu_fw_version { - u32 project; /* Notice, project shifted with 8 on ux540 */ - u8 api_version; - u8 func_version; - u8 errata; - char project_name[PRCMU_FW_PROJECT_NAME_LEN]; -}; - -#include - -#if defined(CONFIG_UX500_SOC_DB8500) - -static inline void __init prcmu_early_init(void) -{ - db8500_prcmu_early_init(); -} - -static inline int prcmu_set_power_state(u8 state, bool keep_ulp_clk, - bool keep_ap_pll) -{ - return db8500_prcmu_set_power_state(state, keep_ulp_clk, - keep_ap_pll); -} - -static inline u8 prcmu_get_power_state_result(void) -{ - return db8500_prcmu_get_power_state_result(); -} - -static inline int prcmu_set_epod(u16 epod_id, u8 epod_state) -{ - return db8500_prcmu_set_epod(epod_id, epod_state); -} - -static inline void prcmu_enable_wakeups(u32 wakeups) -{ - db8500_prcmu_enable_wakeups(wakeups); -} - -static inline void prcmu_disable_wakeups(void) -{ - prcmu_enable_wakeups(0); -} - -static inline void prcmu_config_abb_event_readout(u32 abb_events) -{ - db8500_prcmu_config_abb_event_readout(abb_events); -} - -static inline void prcmu_get_abb_event_buffer(void __iomem **buf) -{ - db8500_prcmu_get_abb_event_buffer(buf); -} - -int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size); -int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size); -int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, u8 *mask, u8 size); - -int prcmu_config_clkout(u8 clkout, u8 source, u8 div); - -static inline int prcmu_request_clock(u8 clock, bool enable) -{ - return db8500_prcmu_request_clock(clock, enable); -} - -unsigned long prcmu_clock_rate(u8 clock); -long prcmu_round_clock_rate(u8 clock, unsigned long rate); -int prcmu_set_clock_rate(u8 clock, unsigned long rate); - -static inline int prcmu_get_ddr_opp(void) -{ - return db8500_prcmu_get_ddr_opp(); -} - -static inline int prcmu_set_arm_opp(u8 opp) -{ - return db8500_prcmu_set_arm_opp(opp); -} - -static inline int prcmu_get_arm_opp(void) -{ - return db8500_prcmu_get_arm_opp(); -} - -static inline int prcmu_set_ape_opp(u8 opp) -{ - return db8500_prcmu_set_ape_opp(opp); -} - -static inline int prcmu_get_ape_opp(void) -{ - return db8500_prcmu_get_ape_opp(); -} - -static inline int prcmu_request_ape_opp_100_voltage(bool enable) -{ - return db8500_prcmu_request_ape_opp_100_voltage(enable); -} - -static inline void prcmu_system_reset(u16 reset_code) -{ - db8500_prcmu_system_reset(reset_code); -} - -static inline u16 prcmu_get_reset_code(void) -{ - return db8500_prcmu_get_reset_code(); -} - -int prcmu_ac_wake_req(void); -void prcmu_ac_sleep_req(void); -static inline void prcmu_modem_reset(void) -{ - db8500_prcmu_modem_reset(); -} - -static inline bool prcmu_is_ac_wake_requested(void) -{ - return db8500_prcmu_is_ac_wake_requested(); -} - -static inline int prcmu_config_esram0_deep_sleep(u8 state) -{ - return db8500_prcmu_config_esram0_deep_sleep(state); -} - -static inline int prcmu_config_hotdog(u8 threshold) -{ - return db8500_prcmu_config_hotdog(threshold); -} - -static inline int prcmu_config_hotmon(u8 low, u8 high) -{ - return db8500_prcmu_config_hotmon(low, high); -} - -static inline int prcmu_start_temp_sense(u16 cycles32k) -{ - return db8500_prcmu_start_temp_sense(cycles32k); -} - -static inline int prcmu_stop_temp_sense(void) -{ - return db8500_prcmu_stop_temp_sense(); -} - -static inline u32 prcmu_read(unsigned int reg) -{ - return db8500_prcmu_read(reg); -} - -static inline void prcmu_write(unsigned int reg, u32 value) -{ - db8500_prcmu_write(reg, value); -} - -static inline void prcmu_write_masked(unsigned int reg, u32 mask, u32 value) -{ - db8500_prcmu_write_masked(reg, mask, value); -} - -static inline int prcmu_enable_a9wdog(u8 id) -{ - return db8500_prcmu_enable_a9wdog(id); -} - -static inline int prcmu_disable_a9wdog(u8 id) -{ - return db8500_prcmu_disable_a9wdog(id); -} - -static inline int prcmu_kick_a9wdog(u8 id) -{ - return db8500_prcmu_kick_a9wdog(id); -} - -static inline int prcmu_load_a9wdog(u8 id, u32 timeout) -{ - return db8500_prcmu_load_a9wdog(id, timeout); -} - -static inline int prcmu_config_a9wdog(u8 num, bool sleep_auto_off) -{ - return db8500_prcmu_config_a9wdog(num, sleep_auto_off); -} -#else - -static inline void prcmu_early_init(void) {} - -static inline int prcmu_set_power_state(u8 state, bool keep_ulp_clk, - bool keep_ap_pll) -{ - return 0; -} - -static inline int prcmu_set_epod(u16 epod_id, u8 epod_state) -{ - return 0; -} - -static inline void prcmu_enable_wakeups(u32 wakeups) {} - -static inline void prcmu_disable_wakeups(void) {} - -static inline int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size) -{ - return -ENOSYS; -} - -static inline int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size) -{ - return -ENOSYS; -} - -static inline int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, u8 *mask, - u8 size) -{ - return -ENOSYS; -} - -static inline int prcmu_config_clkout(u8 clkout, u8 source, u8 div) -{ - return 0; -} - -static inline int prcmu_request_clock(u8 clock, bool enable) -{ - return 0; -} - -static inline long prcmu_round_clock_rate(u8 clock, unsigned long rate) -{ - return 0; -} - -static inline int prcmu_set_clock_rate(u8 clock, unsigned long rate) -{ - return 0; -} - -static inline unsigned long prcmu_clock_rate(u8 clock) -{ - return 0; -} - -static inline int prcmu_set_ape_opp(u8 opp) -{ - return 0; -} - -static inline int prcmu_get_ape_opp(void) -{ - return APE_100_OPP; -} - -static inline int prcmu_request_ape_opp_100_voltage(bool enable) -{ - return 0; -} - -static inline int prcmu_set_arm_opp(u8 opp) -{ - return 0; -} - -static inline int prcmu_get_arm_opp(void) -{ - return ARM_100_OPP; -} - -static inline int prcmu_get_ddr_opp(void) -{ - return DDR_100_OPP; -} - -static inline void prcmu_system_reset(u16 reset_code) {} - -static inline u16 prcmu_get_reset_code(void) -{ - return 0; -} - -static inline int prcmu_ac_wake_req(void) -{ - return 0; -} - -static inline void prcmu_ac_sleep_req(void) {} - -static inline void prcmu_modem_reset(void) {} - -static inline bool prcmu_is_ac_wake_requested(void) -{ - return false; -} - -static inline int prcmu_config_esram0_deep_sleep(u8 state) -{ - return 0; -} - -static inline void prcmu_config_abb_event_readout(u32 abb_events) {} - -static inline void prcmu_get_abb_event_buffer(void __iomem **buf) -{ - *buf = NULL; -} - -static inline int prcmu_config_hotdog(u8 threshold) -{ - return 0; -} - -static inline int prcmu_config_hotmon(u8 low, u8 high) -{ - return 0; -} - -static inline int prcmu_start_temp_sense(u16 cycles32k) -{ - return 0; -} - -static inline int prcmu_stop_temp_sense(void) -{ - return 0; -} - -static inline u32 prcmu_read(unsigned int reg) -{ - return 0; -} - -static inline void prcmu_write(unsigned int reg, u32 value) {} - -static inline void prcmu_write_masked(unsigned int reg, u32 mask, u32 value) {} - -#endif - -static inline void prcmu_set(unsigned int reg, u32 bits) -{ - prcmu_write_masked(reg, bits, bits); -} - -static inline void prcmu_clear(unsigned int reg, u32 bits) -{ - prcmu_write_masked(reg, bits, 0); -} - -/* PRCMU QoS APE OPP class */ -#define PRCMU_QOS_APE_OPP 1 -#define PRCMU_QOS_DDR_OPP 2 -#define PRCMU_QOS_ARM_OPP 3 -#define PRCMU_QOS_DEFAULT_VALUE -1 - -static inline int prcmu_qos_add_requirement(int prcmu_qos_class, - char *name, s32 value) -{ - return 0; -} - -static inline int prcmu_qos_update_requirement(int prcmu_qos_class, - char *name, s32 new_value) -{ - return 0; -} - -static inline void prcmu_qos_remove_requirement(int prcmu_qos_class, char *name) -{ -} - -#endif /* __MACH_PRCMU_H */ diff --git a/include/linux/mlx5/vport.h b/include/linux/mlx5/vport.h index ee34d3ed335f3..d5113ecb59588 100644 --- a/include/linux/mlx5/vport.h +++ b/include/linux/mlx5/vport.h @@ -61,7 +61,8 @@ u8 mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport); int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport, u8 other_vport, u8 state); int mlx5_query_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 op_mod, - u16 vport, u8 other_vport, u32 *max_tx_speed); + u16 vport, u8 other_vport, + u32 *max_tx_speed, u8 *state); int mlx5_modify_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 opmod, u16 vport, u8 other_vport, u16 max_tx_speed); int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev, diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index b18c2b2e7d2c6..2b109cdaaad86 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1211,7 +1211,7 @@ struct mm_struct { struct mm_mm_cid mm_cid; /* sched_cache related statistics */ - struct sched_cache_stat sc_stat; + struct sched_cache_group *sched_cache_grp; #ifdef CONFIG_MMU atomic_long_t pgtables_bytes; /* size of all page tables */ #endif @@ -1609,8 +1609,9 @@ static inline unsigned int mm_cid_size(void) #endif /* CONFIG_SCHED_MM_CID */ #ifdef CONFIG_SCHED_CACHE -void mm_init_sched(struct mm_struct *mm, - struct sched_cache_time __percpu *pcpu_sched); +int mm_init_sched(struct mm_struct *mm, + struct sched_cache_time __percpu *pcpu_sched); +void mm_destroy_sched(struct mm_struct *mm); static inline int mm_alloc_sched_noprof(struct mm_struct *mm) { @@ -1620,17 +1621,11 @@ static inline int mm_alloc_sched_noprof(struct mm_struct *mm) if (!pcpu_sched) return -ENOMEM; - mm_init_sched(mm, pcpu_sched); - return 0; + return mm_init_sched(mm, pcpu_sched); } #define mm_alloc_sched(...) alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__)) -static inline void mm_destroy_sched(struct mm_struct *mm) -{ - free_percpu(mm->sc_stat.pcpu_sched); - mm->sc_stat.pcpu_sched = NULL; -} #else /* !CONFIG_SCHED_CACHE */ static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; } diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 6b5c2390cc301..e1d348d430c1a 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -622,6 +622,8 @@ static inline void mmap_read_unlock(struct mm_struct *mm) DEFINE_GUARD(mmap_read_lock, struct mm_struct *, mmap_read_lock(_T), mmap_read_unlock(_T)) DEFINE_GUARD_COND(mmap_read_lock, _try, mmap_read_trylock(_T)) +DEFINE_GUARD(mmap_write_lock, struct mm_struct *, + mmap_write_lock(_T), mmap_write_unlock(_T)) static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) { diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 8840b126979ff..14bcdcd44d1d5 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -832,8 +832,13 @@ struct xps_dev_maps { #define TC_BITMASK 15 /* HW offloaded queuing disciplines txq count and offset maps */ struct netdev_tc_txq { - u16 count; - u16 offset; + union { + struct { + u16 count; + u16 offset; + }; + u32 combined; + }; }; #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE) @@ -894,6 +899,7 @@ struct net_device_path { u8 h_dest[ETH_ALEN]; } encap; struct { + struct dst_entry *dst; union { struct in_addr src_v4; struct in6_addr src_v6; @@ -3422,8 +3428,9 @@ void dev_remove_offload(struct packet_offload *po); int dev_get_iflink(const struct net_device *dev); int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb); -int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr, +int dev_fill_forward_path(struct net_device_path_ctx *ctx, struct net_device_path_stack *stack); +void dev_fill_forward_path_release(struct net_device_path_stack *stack); struct net_device *dev_get_by_name(struct net *net, const char *name); struct net_device *dev_get_by_name_rcu(struct net *net, const char *name); struct net_device *__dev_get_by_name(struct net *net, const char *name); diff --git a/include/linux/netfs.h b/include/linux/netfs.h index d0b62d53eea99..67c2d9ef7c623 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -247,6 +247,7 @@ struct netfs_io_request { unsigned long long submitted; /* Amount submitted for I/O so far */ unsigned long long len; /* Length of the request */ size_t transferred; /* Amount to be indicated as transferred */ + size_t progress_at; /* Report read progress when hit this much read */ long error; /* 0 or error that occurred */ unsigned long long i_size; /* Size of the file */ unsigned long long start; /* Start position */ @@ -263,7 +264,6 @@ struct netfs_io_request { atomic_t subreq_counter; /* Next subreq->debug_index */ unsigned int nr_group_rel; /* Number of refs to release on ->group */ spinlock_t lock; /* Lock for queuing subreqs */ - unsigned char front_folio_order; /* Order (size) of front folio */ enum netfs_io_origin origin; /* Origin of the request */ bool direct_bv_unpin; /* T if direct_bv[] must be unpinned */ refcount_t ref; @@ -276,9 +276,10 @@ struct netfs_io_request { #define NETFS_RREQ_SHORT_TRANSFER 5 /* Set if we have a short transfer */ #define NETFS_RREQ_OFFLOAD_COLLECTION 8 /* Offload collection to workqueue */ #define NETFS_RREQ_NO_UNLOCK_FOLIO 9 /* Don't unlock no_unlock_folio on completion */ -#define NETFS_RREQ_FOLIO_COPY_TO_CACHE 10 /* Copy current folio to cache from read */ +#define NETFS_RREQ_CANCEL_CACHING 10 /* Set to cancel caching */ #define NETFS_RREQ_UPLOAD_TO_SERVER 11 /* Need to write to the server */ #define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */ +#define NETFS_RREQ_NEED_PUT_RA_REFS 17 /* Need to put the folio refs RA gave us */ #define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark * write to cache on read */ const struct netfs_request_ops *netfs_ops; diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h index d87be1f25273a..44e5e9fa12e1e 100644 --- a/include/linux/nfs4.h +++ b/include/linux/nfs4.h @@ -171,133 +171,6 @@ Needs to be updated if more operations are defined in future.*/ #define LAST_NFS42_OP OP_REMOVEXATTR #define LAST_NFS4_OP LAST_NFS42_OP -enum nfsstat4 { - NFS4_OK = 0, - NFS4ERR_PERM = 1, - NFS4ERR_NOENT = 2, - NFS4ERR_IO = 5, - NFS4ERR_NXIO = 6, - NFS4ERR_ACCESS = 13, - NFS4ERR_EXIST = 17, - NFS4ERR_XDEV = 18, - /* Unused/reserved 19 */ - NFS4ERR_NOTDIR = 20, - NFS4ERR_ISDIR = 21, - NFS4ERR_INVAL = 22, - NFS4ERR_FBIG = 27, - NFS4ERR_NOSPC = 28, - NFS4ERR_ROFS = 30, - NFS4ERR_MLINK = 31, - NFS4ERR_NAMETOOLONG = 63, - NFS4ERR_NOTEMPTY = 66, - NFS4ERR_DQUOT = 69, - NFS4ERR_STALE = 70, - NFS4ERR_BADHANDLE = 10001, - NFS4ERR_BAD_COOKIE = 10003, - NFS4ERR_NOTSUPP = 10004, - NFS4ERR_TOOSMALL = 10005, - NFS4ERR_SERVERFAULT = 10006, - NFS4ERR_BADTYPE = 10007, - NFS4ERR_DELAY = 10008, - NFS4ERR_SAME = 10009, - NFS4ERR_DENIED = 10010, - NFS4ERR_EXPIRED = 10011, - NFS4ERR_LOCKED = 10012, - NFS4ERR_GRACE = 10013, - NFS4ERR_FHEXPIRED = 10014, - NFS4ERR_SHARE_DENIED = 10015, - NFS4ERR_WRONGSEC = 10016, - NFS4ERR_CLID_INUSE = 10017, - NFS4ERR_RESOURCE = 10018, - NFS4ERR_MOVED = 10019, - NFS4ERR_NOFILEHANDLE = 10020, - NFS4ERR_MINOR_VERS_MISMATCH = 10021, - NFS4ERR_STALE_CLIENTID = 10022, - NFS4ERR_STALE_STATEID = 10023, - NFS4ERR_OLD_STATEID = 10024, - NFS4ERR_BAD_STATEID = 10025, - NFS4ERR_BAD_SEQID = 10026, - NFS4ERR_NOT_SAME = 10027, - NFS4ERR_LOCK_RANGE = 10028, - NFS4ERR_SYMLINK = 10029, - NFS4ERR_RESTOREFH = 10030, - NFS4ERR_LEASE_MOVED = 10031, - NFS4ERR_ATTRNOTSUPP = 10032, - NFS4ERR_NO_GRACE = 10033, - NFS4ERR_RECLAIM_BAD = 10034, - NFS4ERR_RECLAIM_CONFLICT = 10035, - NFS4ERR_BADXDR = 10036, - NFS4ERR_LOCKS_HELD = 10037, - NFS4ERR_OPENMODE = 10038, - NFS4ERR_BADOWNER = 10039, - NFS4ERR_BADCHAR = 10040, - NFS4ERR_BADNAME = 10041, - NFS4ERR_BAD_RANGE = 10042, - NFS4ERR_LOCK_NOTSUPP = 10043, - NFS4ERR_OP_ILLEGAL = 10044, - NFS4ERR_DEADLOCK = 10045, - NFS4ERR_FILE_OPEN = 10046, - NFS4ERR_ADMIN_REVOKED = 10047, - NFS4ERR_CB_PATH_DOWN = 10048, - - /* nfs41 */ - NFS4ERR_BADIOMODE = 10049, - NFS4ERR_BADLAYOUT = 10050, - NFS4ERR_BAD_SESSION_DIGEST = 10051, - NFS4ERR_BADSESSION = 10052, - NFS4ERR_BADSLOT = 10053, - NFS4ERR_COMPLETE_ALREADY = 10054, - NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, - NFS4ERR_DELEG_ALREADY_WANTED = 10056, - NFS4ERR_BACK_CHAN_BUSY = 10057, /* backchan reqs outstanding */ - NFS4ERR_LAYOUTTRYLATER = 10058, - NFS4ERR_LAYOUTUNAVAILABLE = 10059, - NFS4ERR_NOMATCHING_LAYOUT = 10060, - NFS4ERR_RECALLCONFLICT = 10061, - NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, - NFS4ERR_SEQ_MISORDERED = 10063, /* unexpected seq.id in req */ - NFS4ERR_SEQUENCE_POS = 10064, /* [CB_]SEQ. op not 1st op */ - NFS4ERR_REQ_TOO_BIG = 10065, /* request too big */ - NFS4ERR_REP_TOO_BIG = 10066, /* reply too big */ - NFS4ERR_REP_TOO_BIG_TO_CACHE = 10067, /* rep. not all cached */ - NFS4ERR_RETRY_UNCACHED_REP = 10068, /* retry & rep. uncached */ - NFS4ERR_UNSAFE_COMPOUND = 10069, /* retry/recovery too hard */ - NFS4ERR_TOO_MANY_OPS = 10070, /* too many ops in [CB_]COMP */ - NFS4ERR_OP_NOT_IN_SESSION = 10071, /* op needs [CB_]SEQ. op */ - NFS4ERR_HASH_ALG_UNSUPP = 10072, /* hash alg. not supp. */ - /* Error 10073 is unused. */ - NFS4ERR_CLIENTID_BUSY = 10074, /* clientid has state */ - NFS4ERR_PNFS_IO_HOLE = 10075, /* IO to _SPARSE file hole */ - NFS4ERR_SEQ_FALSE_RETRY = 10076, /* retry not original */ - NFS4ERR_BAD_HIGH_SLOT = 10077, /* sequence arg bad */ - NFS4ERR_DEADSESSION = 10078, /* persistent session dead */ - NFS4ERR_ENCR_ALG_UNSUPP = 10079, /* SSV alg mismatch */ - NFS4ERR_PNFS_NO_LAYOUT = 10080, /* direct I/O with no layout */ - NFS4ERR_NOT_ONLY_OP = 10081, /* bad compound */ - NFS4ERR_WRONG_CRED = 10082, /* permissions:state change */ - NFS4ERR_WRONG_TYPE = 10083, /* current operation mismatch */ - NFS4ERR_DIRDELEG_UNAVAIL = 10084, /* no directory delegation */ - NFS4ERR_REJECT_DELEG = 10085, /* on callback */ - NFS4ERR_RETURNCONFLICT = 10086, /* outstanding layoutreturn */ - NFS4ERR_DELEG_REVOKED = 10087, /* deleg./layout revoked */ - - /* nfs42 */ - NFS4ERR_PARTNER_NOTSUPP = 10088, - NFS4ERR_PARTNER_NO_AUTH = 10089, - NFS4ERR_UNION_NOTSUPP = 10090, - NFS4ERR_OFFLOAD_DENIED = 10091, - NFS4ERR_WRONG_LFS = 10092, - NFS4ERR_BADLABEL = 10093, - NFS4ERR_OFFLOAD_NO_REQS = 10094, - - /* xattr (RFC8276) */ - NFS4ERR_NOXATTR = 10095, - NFS4ERR_XATTR2BIG = 10096, - - /* can be used for internal errors */ - NFS4ERR_FIRST_FREE -}; - /* error codes for internal client use */ #define NFS4ERR_RESET_TO_MDS 12001 #define NFS4ERR_RESET_TO_PNFS 12002 diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 11c5b31cfc7da..e8144048cd0ec 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -354,7 +354,7 @@ struct nfs4_layoutreturn { struct nfs4_xdr_opaque_data ld_private; }; -#define PNFS_LAYOUTSTATS_MAXSIZE 256 +#define PNFS_LAYOUTSTATS_MAXSIZE 384 struct nfs42_layoutstat_args; struct nfs42_layoutstat_devinfo; diff --git a/include/linux/oa_tc6.h b/include/linux/oa_tc6.h index 15f58e3c56c74..71cffcb041b1c 100644 --- a/include/linux/oa_tc6.h +++ b/include/linux/oa_tc6.h @@ -10,6 +10,56 @@ #include #include +/* OPEN Alliance TC6 registers */ +/* Standard Capabilities Register */ +#define OA_TC6_REG_STDCAP 0x0002 +#define OA_TC6_STDCAP_DIRECT_PHY_REG_ACCESS BIT(8) + +/* Reset Control and Status Register */ +#define OA_TC6_REG_RESET 0x0003 +#define OA_TC6_RESET_SWRESET BIT(0) /* Software Reset */ + +/* Configuration Register #0 */ +#define OA_TC6_REG_CONFIG0 0x0004 +#define OA_TC6_CONFIG0_SYNC BIT(15) +#define OA_TC6_CONFIG0_ZARFE_ENABLE BIT(12) +#define OA_TC6_CONFIG0_PROTE BIT(5) + +/* Status Register #0 */ +#define OA_TC6_REG_STATUS0 0x0008 +#define OA_TC6_STATUS0_RESETC BIT(6) /* Reset Complete */ +#define OA_TC6_STATUS0_HEADER_ERROR BIT(5) +#define OA_TC6_STATUS0_LOSS_OF_FRAME_ERROR BIT(4) +#define OA_TC6_STATUS0_RX_BUFFER_OVERFLOW_ERROR BIT(3) +#define OA_TC6_STATUS0_TX_PROTOCOL_ERROR BIT(0) + +/* Buffer Status Register */ +#define OA_TC6_REG_BUFFER_STATUS 0x000B +#define OA_TC6_BUFFER_STATUS_TX_CREDITS_AVAILABLE GENMASK(15, 8) +#define OA_TC6_BUFFER_STATUS_RX_CHUNKS_AVAILABLE GENMASK(7, 0) + +/* Interrupt Mask Register #0 */ +#define OA_TC6_REG_INT_MASK0 0x000C +#define OA_TC6_INT_MASK0_HEADER_ERR_MASK BIT(5) +#define OA_TC6_INT_MASK0_LOSS_OF_FRAME_ERR_MASK BIT(4) +#define OA_TC6_INT_MASK0_RX_BUFFER_OVERFLOW_ERR_MASK BIT(3) +#define OA_TC6_INT_MASK0_TX_PROTOCOL_ERR_MASK BIT(0) +#define OA_TC6_INT_MASK0_ALL_INTERRUPTS (GENMASK(5, 0) | \ + GENMASK(12, 7)) + +/* PHY Clause 22 registers base address and mask */ +#define OA_TC6_PHY_STD_REG_ADDR_BASE 0xFF00 +#define OA_TC6_PHY_STD_REG_ADDR_MASK 0x1F + +/* PHY – Clause 45 registers memory map selector (MMS) as per table 6 in the + * OPEN Alliance specification. + */ +#define OA_TC6_PHY_C45_PCS_MMS2 2 /* MMD 3 */ +#define OA_TC6_PHY_C45_PMA_PMD_MMS3 3 /* MMD 1 */ +#define OA_TC6_PHY_C45_VS_PLCA_MMS4 4 /* MMD 31 */ +#define OA_TC6_PHY_C45_AUTO_NEG_MMS5 5 /* MMD 7 */ +#define OA_TC6_PHY_C45_POWER_UNIT_MMS6 6 /* MMD 13 */ + struct oa_tc6; struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev); diff --git a/include/linux/pci-ecam.h b/include/linux/pci-ecam.h index d930651473b4d..044f67ced6ff8 100644 --- a/include/linux/pci-ecam.h +++ b/include/linux/pci-ecam.h @@ -81,6 +81,9 @@ void __iomem *pci_ecam_map_bus(struct pci_bus *bus, unsigned int devfn, /* default ECAM ops */ extern const struct pci_ecam_ops pci_generic_ecam_ops; +/* default CAM ops */ +extern const struct pci_ecam_ops pci_generic_cam_ops; + #if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) extern const struct pci_ecam_ops pci_32b_ops; /* 32-bit accesses only */ extern const struct pci_ecam_ops pci_32b_read_ops; /* 32-bit read only */ diff --git a/include/linux/pci.h b/include/linux/pci.h index 64b308b6e61c1..66a4fda05793d 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -79,14 +79,18 @@ * and, if ARI Forwarding is enabled, functions may appear to be on multiple * devices. */ -#define PCI_SLOT_ALL_DEVICES 0xfe +#define PCI_SLOT_ALL_DEVICES 0xfeff + +/* Used to identify a slot as a placeholder */ +#define PCI_SLOT_PLACEHOLDER 0xffff /* pci_slot represents a physical slot */ struct pci_slot { struct pci_bus *bus; /* Bus this slot is on */ struct list_head list; /* Node in list of slots */ struct hotplug_slot *hotplug; /* Hotplug info (move here) */ - unsigned char number; /* Device nr, or PCI_SLOT_ALL_DEVICES */ + u16 number; /* Device nr, or PCI_SLOT_ALL_DEVICES */ + unsigned int per_func_slot:1; /* Allow per function slot */ struct kobject kobj; }; @@ -979,6 +983,7 @@ struct module; * function returns zero when the driver chooses to * take "ownership" of the device or an error code * (negative number) otherwise. + * The pci_device_id parameter is only valid during probe. * The probe function always gets called from process * context, so it can sleep. * @remove: The remove() function gets called whenever a device diff --git a/include/linux/perf/riscv_pmu.h b/include/linux/perf/riscv_pmu.h index f82a28040594b..ecaa40370830a 100644 --- a/include/linux/perf/riscv_pmu.h +++ b/include/linux/perf/riscv_pmu.h @@ -55,7 +55,7 @@ struct riscv_pmu { irqreturn_t (*handle_irq)(int irq_num, void *dev); - unsigned long cmask; + DECLARE_BITMAP(cmask, RISCV_MAX_COUNTERS); u64 (*ctr_read)(struct perf_event *event); int (*ctr_get_idx)(struct perf_event *event); int (*ctr_get_width)(int idx); diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 48d851fbd8ea5..310681cccb50a 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1467,23 +1467,6 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data, return size; } -/* - * Clear all bitfields in the perf_branch_entry. - * The to and from fields are not cleared because they are - * systematically modified by caller. - */ -static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *br) -{ - br->mispred = 0; - br->predicted = 0; - br->in_tx = 0; - br->abort = 0; - br->cycles = 0; - br->type = 0; - br->spec = PERF_BR_SPEC_NA; - br->reserved = 0; -} - extern void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data, diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h index 6dbc5a1bf6a8c..d2fa1c41bfd2b 100644 --- a/include/linux/rbtree_augmented.h +++ b/include/linux/rbtree_augmented.h @@ -87,18 +87,18 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, } /* - * Template for declaring augmented rbtree callbacks (generic case) + * Template for declaring augmented rbtree callbacks (generic multi fields) * * RBSTATIC: 'static' or empty * RBNAME: name of the rb_augment_callbacks structure * RBSTRUCT: struct type of the tree nodes * RBFIELD: name of struct rb_node field within RBSTRUCT - * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree - * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data + * RBCOPY: name of function that copies the RBAUGMENTED datas + * RBCOMPUTE: name of function that recomputes the RBAUGMENTED datas */ -#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ +#define RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBCOPY, RBCOMPUTE) \ static inline void \ RBNAME ## _propagate(struct rb_node *rb, struct rb_node *stop) \ { \ @@ -114,14 +114,14 @@ RBNAME ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - new->RBAUGMENTED = old->RBAUGMENTED; \ + RBCOPY(new, old); \ } \ static void \ RBNAME ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - new->RBAUGMENTED = old->RBAUGMENTED; \ + RBCOPY(new, old); \ RBCOMPUTE(old, false); \ } \ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ @@ -130,6 +130,27 @@ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ .rotate = RBNAME ## _rotate \ }; +/* + * Template for declaring augmented rbtree callbacks (generic single field) + * + * RBSTATIC: 'static' or empty + * RBNAME: name of the rb_augment_callbacks structure + * RBSTRUCT: struct type of the tree nodes + * RBFIELD: name of struct rb_node field within RBSTRUCT + * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree + * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data + */ + +#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ +static inline void \ +RBNAME ## _copy_single(RBSTRUCT *new, RBSTRUCT *old) \ +{ \ + new->RBAUGMENTED = old->RBAUGMENTED; \ +} \ +RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBNAME ## _copy_single, RBCOMPUTE) + /* * Template for declaring augmented rbtree callbacks, * computing RBAUGMENTED scalar as max(RBCOMPUTE(node)) for all subtree nodes. diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index 7c1546d480082..de98462d58889 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -145,7 +146,6 @@ struct rproc_ops { * a message. * @RPROC_RUNNING: device is up and running * @RPROC_CRASHED: device has crashed; need to start recovery - * @RPROC_DELETED: device is deleted * @RPROC_ATTACHED: device has been booted by another entity and the core * has attached to it * @RPROC_DETACHED: device has been booted by another entity and waiting @@ -163,10 +163,9 @@ enum rproc_state { RPROC_SUSPENDED = 1, RPROC_RUNNING = 2, RPROC_CRASHED = 3, - RPROC_DELETED = 4, - RPROC_ATTACHED = 5, - RPROC_DETACHED = 6, - RPROC_LAST = 7, + RPROC_ATTACHED = 4, + RPROC_DETACHED = 5, + RPROC_LAST = 6, }; /** @@ -259,7 +258,10 @@ enum rproc_features { * @subdevs: list of subdevices, to following the running state * @notifyids: idr for dynamically assigning rproc-wide unique notify ids * @index: index of this rproc device + * @attach_work: workqueue for attaching rproc * @crash_handler: workqueue for handling a crash + * @crash_handler_lock: serializes crash handler queueing and deletion + * @deleting: remoteproc deletion has begun * @crash_cnt: crash counter * @recovery_disabled: flag that state if recovery was disabled * @max_notifyid: largest allocated notify id. @@ -301,7 +303,10 @@ struct rproc { struct list_head subdevs; struct idr notifyids; int index; + struct work_struct attach_work; struct work_struct crash_handler; + spinlock_t crash_handler_lock; + bool deleting; unsigned int crash_cnt; bool recovery_disabled; int max_notifyid; diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index 0670742b2d601..eac3e9080c3c0 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -3,8 +3,9 @@ #define _LINUX_RING_BUFFER_H #include -#include #include +#include +#include #include @@ -218,14 +219,15 @@ bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer); size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu); struct buffer_data_read_page; -struct buffer_data_read_page * -ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu); +int ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu, + struct buffer_data_read_page **rpage); void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, struct buffer_data_read_page *page); int ring_buffer_read_page(struct trace_buffer *buffer, struct buffer_data_read_page *data_page, size_t len, int cpu, int full); void *ring_buffer_read_page_data(struct buffer_data_read_page *page); +unsigned int ring_buffer_read_page_size(struct buffer_data_read_page *rpage); struct trace_seq; @@ -278,11 +280,25 @@ static inline struct ring_buffer_desc *__first_ring_buffer_desc(struct trace_buf return (struct ring_buffer_desc *)(&desc->__data[0]); } +/* + * Returns the number of pages for a ring_buffer_desc. The caller must ensure it + * does not overflow ring_buffer_desc::nr_page_va. + */ +static inline unsigned long __calc_nr_pages_ring_buffer_desc(size_t size) +{ + /* Takes into account the reader page */ + return max(DIV_ROUND_UP(size, PAGE_SIZE - BUF_PAGE_HDR_SIZE), 2UL) + 1; +} + static inline size_t trace_buffer_desc_size(size_t buffer_size, unsigned int nr_cpus) { - unsigned int nr_pages = max(DIV_ROUND_UP(buffer_size, PAGE_SIZE), 2UL) + 1; + unsigned long nr_pages = __calc_nr_pages_ring_buffer_desc(buffer_size); struct ring_buffer_desc *rbdesc; + /* Capped by ring_buffer_desc::nr_page_va */ + if (nr_pages > UINT_MAX) + return SIZE_MAX; + return size_add(offsetof(struct trace_buffer_desc, __data), size_mul(nr_cpus, struct_size(rbdesc, page_va, nr_pages))); } diff --git a/include/linux/rolling_buffer.h b/include/linux/rolling_buffer.h index 9e5dad29669cf..a97f7cfaacaad 100644 --- a/include/linux/rolling_buffer.h +++ b/include/linux/rolling_buffer.h @@ -45,9 +45,9 @@ struct rolling_buffer_snapshot { int rolling_buffer_init(struct rolling_buffer *roll, unsigned int rreq_id, unsigned int direction, gfp_t gfp); int rolling_buffer_make_space(struct rolling_buffer *roll, gfp_t gfp); -ssize_t rolling_buffer_load_from_ra(struct rolling_buffer *roll, - struct readahead_control *ractl, - struct folio_batch *put_batch); +ssize_t rolling_buffer_bulk_load_from_ra(struct rolling_buffer *roll, + struct readahead_control *ractl, + unsigned int rreq_id, gfp_t gfp); ssize_t rolling_buffer_append(struct rolling_buffer *roll, struct folio *folio, unsigned int flags, gfp_t gfp); struct folio_queue *rolling_buffer_delete_spent(struct rolling_buffer *roll); diff --git a/include/linux/rsc_table.h b/include/linux/rsc_table.h index c6d6d553d8f11..4cef11a2e3a2c 100644 --- a/include/linux/rsc_table.h +++ b/include/linux/rsc_table.h @@ -337,17 +337,22 @@ static inline int rsc_table_for_each_entry(struct resource_table *table, int i, ret; for (i = 0; i < table->num; i++) { - int offset = table->offset[i]; - struct fw_rsc_hdr *hdr = (void *)table + offset; - int avail = table_sz - offset - sizeof(*hdr); - int rsc_offset = offset + sizeof(*hdr); - void *rsc = (void *)hdr + sizeof(*hdr); + u32 offset = table->offset[i]; + struct fw_rsc_hdr *hdr; + int avail, rsc_offset; + void *rsc; - if (avail < 0) { + if (offset < sizeof(*table) || offset >= table_sz || + table_sz - offset < sizeof(*hdr)) { dev_err(dev, "rsc table is truncated\n"); return -EINVAL; } + hdr = (void *)table + offset; + avail = table_sz - offset - sizeof(*hdr); + rsc_offset = offset + sizeof(*hdr); + rsc = (void *)hdr + sizeof(*hdr); + ret = cb(hdr->type, rsc, rsc_offset, avail, data); if (ret) return ret; diff --git a/include/linux/rtsx_usb.h b/include/linux/rtsx_usb.h index 276b509c03e36..0fc5a74700a8b 100644 --- a/include/linux/rtsx_usb.h +++ b/include/linux/rtsx_usb.h @@ -61,6 +61,9 @@ struct rtsx_ucr { struct timer_list sg_timer; struct mutex dev_mutex; + + u16 card_status_cache; + bool card_status_valid; }; /* buffer size */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 373bcc0598d10..1713fb6b33edc 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1543,6 +1543,14 @@ struct task_struct { /* Collect coverage from softirq context: */ unsigned int kcov_softirq; + + /* Temporary storage for preempting remote coverage collection: */ + unsigned int kcov_saved_mode; + unsigned int kcov_saved_size; + void *kcov_saved_area; + struct kcov *kcov_saved_kcov; + int kcov_saved_sequence; + #endif #ifdef CONFIG_MEMCG_V1 @@ -1767,7 +1775,7 @@ static inline bool is_lazy_mmu_mode_active(void) } #endif -extern struct pid *cad_pid; +extern struct pid __rcu *cad_pid; /* * Per process flags @@ -2390,7 +2398,7 @@ struct sched_cache_time { unsigned long epoch; }; -struct sched_cache_stat { +struct sched_cache_group { struct sched_cache_time __percpu *pcpu_sched; raw_spinlock_t lock; unsigned long epoch; @@ -2398,11 +2406,13 @@ struct sched_cache_stat { unsigned long next_scan; unsigned long footprint; int cpu; + refcount_t refcnt; + struct rcu_head rcu; } ____cacheline_aligned_in_smp; #else -struct sched_cache_stat { }; +struct sched_cache_group { }; #endif diff --git a/include/linux/sched/rt.h b/include/linux/sched/rt.h index 4e3338103654c..922935cc33833 100644 --- a/include/linux/sched/rt.h +++ b/include/linux/sched/rt.h @@ -52,8 +52,10 @@ static inline bool rt_or_dl_task_policy(struct task_struct *tsk) #ifdef CONFIG_RT_MUTEXES extern void rt_mutex_pre_schedule(void); +extern void rt_mutex_futex_pre_schedule(void); extern void rt_mutex_schedule(void); extern void rt_mutex_post_schedule(void); +extern void rt_mutex_futex_post_schedule(void); /* * Must hold either p->pi_lock or task_rq(p)->lock. diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 584ae88b435e9..d45a5476b97de 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -562,10 +562,7 @@ static inline sigset_t *sigmask_to_save(void) return res; } -static inline int kill_cad_pid(int sig, int priv) -{ - return kill_pid(cad_pid, sig, priv); -} +int kill_cad_pid(int sig, int priv); /* These can be the second arg to send_sig_info/send_group_sig_info. */ #define SEND_SIG_NOINFO ((struct kernel_siginfo *) 0) diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index b5d9d7c2b8add..f96812d71c515 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -281,9 +281,9 @@ static inline int task_node(const struct task_struct *p) } #ifdef CONFIG_SCHED_CACHE -extern void sched_update_llc_bytes(unsigned int cpu); +extern void sched_update_llc_bytes(const struct cpumask *cpus); #else -static inline void sched_update_llc_bytes(unsigned int cpu) { } +static inline void sched_update_llc_bytes(const struct cpumask *cpus) { } #endif #endif /* _LINUX_SCHED_TOPOLOGY_H */ diff --git a/include/linux/sched/user.h b/include/linux/sched/user.h index 4cc52698e214e..8d7e5521f7cdd 100644 --- a/include/linux/sched/user.h +++ b/include/linux/sched/user.h @@ -25,7 +25,8 @@ struct user_struct { #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) || \ defined(CONFIG_NET) || defined(CONFIG_IO_URING) || \ - defined(CONFIG_VFIO_PCI_ZDEV_KVM) || IS_ENABLED(CONFIG_IOMMUFD) + defined(CONFIG_VFIO_PCI_ZDEV_KVM) || IS_ENABLED(CONFIG_IOMMUFD) || \ + defined(CONFIG_SECRETMEM) atomic_long_t locked_vm; #endif #ifdef CONFIG_WATCH_QUEUE diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 22eda1d54a0e8..ce125cd244439 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1834,22 +1834,6 @@ static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, } } -static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val) -{ - skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL); - skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; -} - -static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb) -{ - return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL; -} - -static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb) -{ - return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL); -} - static inline void net_zcopy_put(struct ubuf_info *uarg) { if (uarg) @@ -1872,8 +1856,7 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success) struct ubuf_info *uarg = skb_zcopy(skb); if (uarg) { - if (!skb_zcopy_is_nouarg(skb)) - uarg->ops->complete(skb, uarg, zerocopy_success); + uarg->ops->complete(skb, uarg, zerocopy_success); skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY; } @@ -3082,6 +3065,11 @@ static inline bool skb_transport_header_was_set(const struct sk_buff *skb) return skb->transport_header != (typeof(skb->transport_header))~0U; } +static inline void skb_unset_transport_header(struct sk_buff *skb) +{ + skb->transport_header = (typeof(skb->transport_header))~0U; +} + static inline unsigned char *skb_transport_header(const struct sk_buff *skb) { DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb)); @@ -3126,6 +3114,30 @@ static inline void skb_set_transport_header(struct sk_buff *skb, skb->transport_header += offset; } +/** + * skb_set_transport_header_careful - conditionally set transport header + * @skb: buffer to alter + * @offset: offset to add to skb->data + * + * Hardened version of skb_set_transport_header(). + * + * Returns: true if the operation was a success. + */ +static inline bool __must_check +skb_set_transport_header_careful(struct sk_buff *skb, const int offset) +{ + long thoff = skb->data - skb->head + offset; + + if (unlikely(thoff != (typeof(skb->transport_header))thoff)) + return false; + + if (unlikely(thoff == (typeof(skb->transport_header))~0U)) + return false; + + skb->transport_header = thoff; + return true; +} + static inline unsigned char *skb_network_header(const struct sk_buff *skb) { return skb->head + skb->network_header; @@ -4343,7 +4355,10 @@ skb_header_pointer_careful(const struct sk_buff *skb, int offset, static inline void * __must_check skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len) { - if (likely(skb_headlen(skb) - offset >= len)) + unsigned int uoffset = (unsigned int)offset; + + if (likely(uoffset <= skb_headlen(skb) && + (unsigned int)len <= skb_headlen(skb) - uoffset)) return skb->data + offset; return NULL; } diff --git a/include/linux/soc/qcom/geni-se.h b/include/linux/soc/qcom/geni-se.h index c5e6ab85df099..0991ef7ec0928 100644 --- a/include/linux/soc/qcom/geni-se.h +++ b/include/linux/soc/qcom/geni-se.h @@ -124,6 +124,7 @@ struct geni_se { #define SE_DMA_RX_FSM_RST 0xd58 #define SE_HW_PARAM_0 0xe24 #define SE_HW_PARAM_1 0xe28 +#define SE_HW_PARAM_2 0xe2c /* GENI_FORCE_DEFAULT_REG fields */ #define FORCE_DEFAULT BIT(0) @@ -291,6 +292,9 @@ struct geni_se { #define RX_FIFO_DEPTH_MSK GENMASK(21, 16) #define RX_FIFO_DEPTH_SHFT 16 +/* SE_HW_PARAM_2 fields */ +#define PROG_RAM_DEPTH_MSK GENMASK(10, 0) + #define HW_VER_MAJOR_MASK GENMASK(31, 28) #define HW_VER_MAJOR_SHFT 28 #define HW_VER_MINOR_MASK GENMASK(27, 16) diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h index 83d2c2a7116c0..f35c29440f4de 100644 --- a/include/linux/soc/qcom/ubwc.h +++ b/include/linux/soc/qcom/ubwc.h @@ -8,6 +8,7 @@ #define __QCOM_UBWC_H__ #include +#include #include #include diff --git a/include/linux/sunrpc/bc_xprt.h b/include/linux/sunrpc/bc_xprt.h index 98939cb664cfa..59d0cc889beba 100644 --- a/include/linux/sunrpc/bc_xprt.h +++ b/include/linux/sunrpc/bc_xprt.h @@ -32,6 +32,7 @@ int xprt_setup_bc(struct rpc_xprt *xprt, unsigned int min_reqs); void xprt_destroy_bc(struct rpc_xprt *xprt, unsigned int max_reqs); void xprt_free_bc_rqst(struct rpc_rqst *req); unsigned int xprt_bc_max_slots(struct rpc_xprt *xprt); +void xprt_svc_shutdown_bc(struct rpc_xprt *xprt); void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv); /* @@ -71,6 +72,10 @@ static inline void xprt_free_bc_request(struct rpc_rqst *req) { } +static inline void xprt_svc_shutdown_bc(struct rpc_xprt *xprt) +{ +} + static inline void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv) { svc_destroy(serv); diff --git a/include/linux/sunrpc/svc_rdma_pcl.h b/include/linux/sunrpc/svc_rdma_pcl.h index 7516ad0fae808..655681cf8fedc 100644 --- a/include/linux/sunrpc/svc_rdma_pcl.h +++ b/include/linux/sunrpc/svc_rdma_pcl.h @@ -97,7 +97,7 @@ pcl_next_chunk(const struct svc_rdma_pcl *pcl, struct svc_rdma_chunk *chunk) */ #define pcl_for_each_segment(pos, chunk) \ for (pos = &(chunk)->ch_segments[0]; \ - pos <= &(chunk)->ch_segments[(chunk)->ch_segcount - 1]; \ + pos < &(chunk)->ch_segments[(chunk)->ch_segcount]; \ pos++) /** diff --git a/include/linux/sunrpc/xdrgen/_defs.h b/include/linux/sunrpc/xdrgen/_defs.h index 20c7270aa64d2..8f3776ef32295 100644 --- a/include/linux/sunrpc/xdrgen/_defs.h +++ b/include/linux/sunrpc/xdrgen/_defs.h @@ -25,6 +25,8 @@ typedef struct { #define XDR_void (0) #define XDR_bool (1) +#define XDR_short (1) +#define XDR_unsigned_short (1) #define XDR_int (1) #define XDR_unsigned_int (1) #define XDR_long (1) diff --git a/include/linux/sunrpc/xdrgen/nfs4_1.h b/include/linux/sunrpc/xdrgen/nfs4_1.h index 4ac54bdbd335b..ed6df7690b5fb 100644 --- a/include/linux/sunrpc/xdrgen/nfs4_1.h +++ b/include/linux/sunrpc/xdrgen/nfs4_1.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x */ -/* XDR specification modification time: Thu Jan 8 23:12:07 2026 */ +/* XDR specification modification time: Tue Jun 30 11:57:21 2026 */ #ifndef _LINUX_XDRGEN_NFS4_1_DEF_H #define _LINUX_XDRGEN_NFS4_1_DEF_H @@ -9,15 +9,149 @@ #include #include -typedef s64 int64_t; +typedef s32 int32_t; typedef u32 uint32_t; +typedef s64 int64_t; + +typedef u64 uint64_t; + +enum { NFS4_VERIFIER_SIZE = 8 }; + +enum { NFS4_FHSIZE = 128 }; + +enum nfsstat4 { + NFS4_OK = 0, + NFS4ERR_PERM = 1, + NFS4ERR_NOENT = 2, + NFS4ERR_IO = 5, + NFS4ERR_NXIO = 6, + NFS4ERR_ACCESS = 13, + NFS4ERR_EXIST = 17, + NFS4ERR_XDEV = 18, + NFS4ERR_NOTDIR = 20, + NFS4ERR_ISDIR = 21, + NFS4ERR_INVAL = 22, + NFS4ERR_FBIG = 27, + NFS4ERR_NOSPC = 28, + NFS4ERR_ROFS = 30, + NFS4ERR_MLINK = 31, + NFS4ERR_NAMETOOLONG = 63, + NFS4ERR_NOTEMPTY = 66, + NFS4ERR_DQUOT = 69, + NFS4ERR_STALE = 70, + NFS4ERR_BADHANDLE = 10001, + NFS4ERR_BAD_COOKIE = 10003, + NFS4ERR_NOTSUPP = 10004, + NFS4ERR_TOOSMALL = 10005, + NFS4ERR_SERVERFAULT = 10006, + NFS4ERR_BADTYPE = 10007, + NFS4ERR_DELAY = 10008, + NFS4ERR_SAME = 10009, + NFS4ERR_DENIED = 10010, + NFS4ERR_EXPIRED = 10011, + NFS4ERR_LOCKED = 10012, + NFS4ERR_GRACE = 10013, + NFS4ERR_FHEXPIRED = 10014, + NFS4ERR_SHARE_DENIED = 10015, + NFS4ERR_WRONGSEC = 10016, + NFS4ERR_CLID_INUSE = 10017, + NFS4ERR_RESOURCE = 10018, + NFS4ERR_MOVED = 10019, + NFS4ERR_NOFILEHANDLE = 10020, + NFS4ERR_MINOR_VERS_MISMATCH = 10021, + NFS4ERR_STALE_CLIENTID = 10022, + NFS4ERR_STALE_STATEID = 10023, + NFS4ERR_OLD_STATEID = 10024, + NFS4ERR_BAD_STATEID = 10025, + NFS4ERR_BAD_SEQID = 10026, + NFS4ERR_NOT_SAME = 10027, + NFS4ERR_LOCK_RANGE = 10028, + NFS4ERR_SYMLINK = 10029, + NFS4ERR_RESTOREFH = 10030, + NFS4ERR_LEASE_MOVED = 10031, + NFS4ERR_ATTRNOTSUPP = 10032, + NFS4ERR_NO_GRACE = 10033, + NFS4ERR_RECLAIM_BAD = 10034, + NFS4ERR_RECLAIM_CONFLICT = 10035, + NFS4ERR_BADXDR = 10036, + NFS4ERR_LOCKS_HELD = 10037, + NFS4ERR_OPENMODE = 10038, + NFS4ERR_BADOWNER = 10039, + NFS4ERR_BADCHAR = 10040, + NFS4ERR_BADNAME = 10041, + NFS4ERR_BAD_RANGE = 10042, + NFS4ERR_LOCK_NOTSUPP = 10043, + NFS4ERR_OP_ILLEGAL = 10044, + NFS4ERR_DEADLOCK = 10045, + NFS4ERR_FILE_OPEN = 10046, + NFS4ERR_ADMIN_REVOKED = 10047, + NFS4ERR_CB_PATH_DOWN = 10048, + NFS4ERR_BADIOMODE = 10049, + NFS4ERR_BADLAYOUT = 10050, + NFS4ERR_BAD_SESSION_DIGEST = 10051, + NFS4ERR_BADSESSION = 10052, + NFS4ERR_BADSLOT = 10053, + NFS4ERR_COMPLETE_ALREADY = 10054, + NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, + NFS4ERR_DELEG_ALREADY_WANTED = 10056, + NFS4ERR_BACK_CHAN_BUSY = 10057, + NFS4ERR_LAYOUTTRYLATER = 10058, + NFS4ERR_LAYOUTUNAVAILABLE = 10059, + NFS4ERR_NOMATCHING_LAYOUT = 10060, + NFS4ERR_RECALLCONFLICT = 10061, + NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, + NFS4ERR_SEQ_MISORDERED = 10063, + NFS4ERR_SEQUENCE_POS = 10064, + NFS4ERR_REQ_TOO_BIG = 10065, + NFS4ERR_REP_TOO_BIG = 10066, + NFS4ERR_REP_TOO_BIG_TO_CACHE = 10067, + NFS4ERR_RETRY_UNCACHED_REP = 10068, + NFS4ERR_UNSAFE_COMPOUND = 10069, + NFS4ERR_TOO_MANY_OPS = 10070, + NFS4ERR_OP_NOT_IN_SESSION = 10071, + NFS4ERR_HASH_ALG_UNSUPP = 10072, + NFS4ERR_CLIENTID_BUSY = 10074, + NFS4ERR_PNFS_IO_HOLE = 10075, + NFS4ERR_SEQ_FALSE_RETRY = 10076, + NFS4ERR_BAD_HIGH_SLOT = 10077, + NFS4ERR_DEADSESSION = 10078, + NFS4ERR_ENCR_ALG_UNSUPP = 10079, + NFS4ERR_PNFS_NO_LAYOUT = 10080, + NFS4ERR_NOT_ONLY_OP = 10081, + NFS4ERR_WRONG_CRED = 10082, + NFS4ERR_WRONG_TYPE = 10083, + NFS4ERR_DIRDELEG_UNAVAIL = 10084, + NFS4ERR_REJECT_DELEG = 10085, + NFS4ERR_RETURNCONFLICT = 10086, + NFS4ERR_DELEG_REVOKED = 10087, + NFS4ERR_PARTNER_NOTSUPP = 10088, + NFS4ERR_PARTNER_NO_AUTH = 10089, + NFS4ERR_UNION_NOTSUPP = 10090, + NFS4ERR_OFFLOAD_DENIED = 10091, + NFS4ERR_WRONG_LFS = 10092, + NFS4ERR_BADLABEL = 10093, + NFS4ERR_OFFLOAD_NO_REQS = 10094, + NFS4ERR_NOXATTR = 10095, + NFS4ERR_XATTR2BIG = 10096, +}; + +typedef enum nfsstat4 nfsstat4; + +typedef opaque attrlist4; + typedef struct { u32 count; uint32_t *element; } bitmap4; +typedef u8 verifier4[NFS4_VERIFIER_SIZE]; + +typedef uint64_t nfs_cookie4; + +typedef opaque nfs_fh4; + typedef opaque utf8string; typedef utf8string utf8str_cis; @@ -26,11 +160,30 @@ typedef utf8string utf8str_cs; typedef utf8string utf8str_mixed; +typedef utf8str_cs component4; + +typedef utf8str_cs linktext4; + +typedef struct { + u32 count; + component4 *element; +} pathname4; + struct nfstime4 { int64_t seconds; uint32_t nseconds; }; +struct fattr4 { + bitmap4 attrmask; + attrlist4 attr_vals; +}; + +struct stateid4 { + uint32_t seqid; + u8 other[12]; +}; + typedef bool fattr4_offline; enum { FATTR4_OFFLINE = 83 }; @@ -216,11 +369,98 @@ enum { FATTR4_POSIX_DEFAULT_ACL = 91 }; enum { FATTR4_POSIX_ACCESS_ACL = 92 }; -#define NFS4_int64_t_sz \ - (XDR_hyper) +enum notify_type4 { + NOTIFY4_CHANGE_CHILD_ATTRS = 0, + NOTIFY4_CHANGE_DIR_ATTRS = 1, + NOTIFY4_REMOVE_ENTRY = 2, + NOTIFY4_ADD_ENTRY = 3, + NOTIFY4_RENAME_ENTRY = 4, + NOTIFY4_CHANGE_COOKIE_VERIFIER = 5, +}; + +typedef enum notify_type4 notify_type4; + +struct notify_entry4 { + component4 ne_file; + struct fattr4 ne_attrs; +}; + +struct prev_entry4 { + struct notify_entry4 pe_prev_entry; + nfs_cookie4 pe_prev_entry_cookie; +}; + +struct notify_remove4 { + struct notify_entry4 nrm_old_entry; + nfs_cookie4 nrm_old_entry_cookie; +}; + +struct notify_add4 { + struct { + u32 count; + struct notify_remove4 *element; + } nad_old_entry; + struct notify_entry4 nad_new_entry; + struct { + u32 count; + nfs_cookie4 *element; + } nad_new_entry_cookie; + struct { + u32 count; + struct prev_entry4 *element; + } nad_prev_entry; + bool nad_last_entry; +}; + +struct notify_attr4 { + struct notify_entry4 na_changed_entry; +}; + +struct notify_rename4 { + struct notify_remove4 nrn_old_entry; + struct notify_add4 nrn_new_entry; +}; + +struct notify_verifier4 { + verifier4 nv_old_cookieverf; + verifier4 nv_new_cookieverf; +}; + +typedef opaque notifylist4; + +struct notify4 { + bitmap4 notify_mask; + notifylist4 notify_vals; +}; + +struct CB_NOTIFY4args { + struct stateid4 cna_stateid; + nfs_fh4 cna_fh; + struct { + u32 count; + struct notify4 *element; + } cna_changes; +}; + +struct CB_NOTIFY4res { + nfsstat4 cnr_status; +}; + +#define NFS4_int32_t_sz \ + (XDR_int) #define NFS4_uint32_t_sz \ (XDR_unsigned_int) +#define NFS4_int64_t_sz \ + (XDR_hyper) +#define NFS4_uint64_t_sz \ + (XDR_unsigned_hyper) +#define NFS4_nfsstat4_sz (XDR_int) +#define NFS4_attrlist4_sz (XDR_unsigned_int) #define NFS4_bitmap4_sz (XDR_unsigned_int) +#define NFS4_verifier4_sz (XDR_QUADLEN(NFS4_VERIFIER_SIZE)) +#define NFS4_nfs_cookie4_sz \ + (NFS4_uint64_t_sz) +#define NFS4_nfs_fh4_sz (XDR_unsigned_int + XDR_QUADLEN(NFS4_FHSIZE)) #define NFS4_utf8string_sz (XDR_unsigned_int) #define NFS4_utf8str_cis_sz \ (NFS4_utf8string_sz) @@ -228,8 +468,17 @@ enum { FATTR4_POSIX_ACCESS_ACL = 92 }; (NFS4_utf8string_sz) #define NFS4_utf8str_mixed_sz \ (NFS4_utf8string_sz) +#define NFS4_component4_sz \ + (NFS4_utf8str_cs_sz) +#define NFS4_linktext4_sz \ + (NFS4_utf8str_cs_sz) +#define NFS4_pathname4_sz (XDR_unsigned_int) #define NFS4_nfstime4_sz \ (NFS4_int64_t_sz + NFS4_uint32_t_sz) +#define NFS4_fattr4_sz \ + (NFS4_bitmap4_sz + NFS4_attrlist4_sz) +#define NFS4_stateid4_sz \ + (NFS4_uint32_t_sz + XDR_QUADLEN(12)) #define NFS4_fattr4_offline_sz \ (XDR_bool) #define NFS4_open_arguments4_sz \ @@ -259,5 +508,27 @@ enum { FATTR4_POSIX_ACCESS_ACL = 92 }; (NFS4_aclscope4_sz) #define NFS4_fattr4_posix_default_acl_sz (XDR_unsigned_int) #define NFS4_fattr4_posix_access_acl_sz (XDR_unsigned_int) +#define NFS4_notify_type4_sz (XDR_int) +#define NFS4_notify_entry4_sz \ + (NFS4_component4_sz + NFS4_fattr4_sz) +#define NFS4_prev_entry4_sz \ + (NFS4_notify_entry4_sz + NFS4_nfs_cookie4_sz) +#define NFS4_notify_remove4_sz \ + (NFS4_notify_entry4_sz + NFS4_nfs_cookie4_sz) +#define NFS4_notify_add4_sz \ + (XDR_unsigned_int + (1 * (NFS4_notify_remove4_sz)) + NFS4_notify_entry4_sz + XDR_unsigned_int + (1 * (NFS4_nfs_cookie4_sz)) + XDR_unsigned_int + (1 * (NFS4_prev_entry4_sz)) + XDR_bool) +#define NFS4_notify_attr4_sz \ + (NFS4_notify_entry4_sz) +#define NFS4_notify_rename4_sz \ + (NFS4_notify_remove4_sz + NFS4_notify_add4_sz) +#define NFS4_notify_verifier4_sz \ + (NFS4_verifier4_sz + NFS4_verifier4_sz) +#define NFS4_notifylist4_sz (XDR_unsigned_int) +#define NFS4_notify4_sz \ + (NFS4_bitmap4_sz + NFS4_notifylist4_sz) +#define NFS4_CB_NOTIFY4args_sz \ + (NFS4_stateid4_sz + NFS4_nfs_fh4_sz + XDR_unsigned_int) +#define NFS4_CB_NOTIFY4res_sz \ + (NFS4_nfsstat4_sz) #endif /* _LINUX_XDRGEN_NFS4_1_DEF_H */ diff --git a/include/linux/sunrpc/xdrgen/nlm3.h b/include/linux/sunrpc/xdrgen/nlm3.h index 897e7d91807c6..0fc627031d8a9 100644 --- a/include/linux/sunrpc/xdrgen/nlm3.h +++ b/include/linux/sunrpc/xdrgen/nlm3.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x */ -/* XDR specification modification time: Thu Apr 23 10:56:34 2026 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM3_DEF_H #define _LINUX_XDRGEN_NLM3_DEF_H diff --git a/include/linux/sunrpc/xdrgen/nlm4.h b/include/linux/sunrpc/xdrgen/nlm4.h index e95e8f105624d..77860a3d1c1e9 100644 --- a/include/linux/sunrpc/xdrgen/nlm4.h +++ b/include/linux/sunrpc/xdrgen/nlm4.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */ -/* XDR specification modification time: Thu Dec 25 13:10:19 2025 */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ #ifndef _LINUX_XDRGEN_NLM4_DEF_H #define _LINUX_XDRGEN_NLM4_DEF_H diff --git a/include/linux/uio.h b/include/linux/uio.h index a9bc5b3067e32..fe2e985d74d24 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -389,9 +389,17 @@ ssize_t iov_iter_extract_pages(struct iov_iter *i, struct page ***pages, size_t maxsize, unsigned int maxpages, iov_iter_extraction_t extraction_flags, size_t *offset0); +/* + * Block-layer consumers (e.g. bio_iov_iter_get_pages()) require that the + * segments of an ITER_BVEC iterator are already aligned to the target device's + * DMA alignment, and forward them as-is. In-kernel users that build their own + * bvecs must not create sub-aligned segments; iov_iter_extract_bvecs() enforces + * the same for the segments it extracts via @mem_align_mask. + */ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, size_t max_size, unsigned short *nr_vecs, - unsigned short max_vecs, iov_iter_extraction_t extraction_flags); + unsigned short max_vecs, unsigned mem_align_mask, + iov_iter_extraction_t extraction_flags); /** * iov_iter_extract_will_pin - Indicate how pages from the iterator will be retained diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec8..18be159bbc341 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -186,9 +186,6 @@ struct xol_area; struct uprobes_state { struct xol_area *xol_area; -#ifdef CONFIG_X86_64 - struct hlist_head head_tramps; -#endif }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, @@ -238,8 +235,6 @@ extern void uprobe_handle_trampoline(struct pt_regs *regs); extern void *arch_uretprobe_trampoline(unsigned long *psize); extern unsigned long uprobe_get_trampoline_vaddr(void); extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len); -extern void arch_uprobe_clear_state(struct mm_struct *mm); -extern void arch_uprobe_init_state(struct mm_struct *mm); extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr); extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr); extern unsigned long arch_uprobe_get_xol_area(void); diff --git a/include/linux/usb.h b/include/linux/usb.h index 1da4ad1610bca..49ab8dbb885f6 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -1185,7 +1185,8 @@ extern ssize_t usb_show_dynids(struct usb_dynids *dynids, char *buf); * interface. It may also use usb_set_interface() to specify the * appropriate altsetting. If unwilling to manage the interface, * return -ENODEV, if genuine IO errors occurred, an appropriate - * negative errno value. + * negative errno value. The usb_device_id parameter is only valid during + * probe. * @disconnect: Called when the interface is no longer accessible, usually * because its device has been (or is being) disconnected or the * driver module is being unloaded. diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h index 337a5485af7c7..ee360dedeaa65 100644 --- a/include/linux/usb/pd.h +++ b/include/linux/usb/pd.h @@ -493,7 +493,7 @@ static inline unsigned int pdo_epr_avs_apdo_min_voltage_mv(u32 pdo) static inline unsigned int pdo_epr_avs_apdo_max_voltage_mv(u32 pdo) { - return FIELD_GET(PDO_EPR_AVS_APDO_MIN_VOLT, pdo) * 100; + return FIELD_GET(PDO_EPR_AVS_APDO_MAX_VOLT, pdo) * 100; } static inline unsigned int pdo_epr_avs_apdo_pdp_w(u32 pdo) diff --git a/include/linux/usb/tcpci.h b/include/linux/usb/tcpci.h index f7f5cfbdef120..9b46a6bc762cc 100644 --- a/include/linux/usb/tcpci.h +++ b/include/linux/usb/tcpci.h @@ -144,6 +144,7 @@ #define TCPC_RX_BUF_FRAME_TYPE 0x31 #define TCPC_RX_BUF_FRAME_TYPE_SOP 0 #define TCPC_RX_BUF_FRAME_TYPE_SOP1 1 +#define TCPC_RX_BUF_FRAME_TYPE_MASK GENMASK(2, 0) #define TCPC_RX_HDR 0x32 #define TCPC_RX_DATA 0x34 /* through 0x4f */ diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 93e573c565635..f923e42cfd011 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -213,6 +213,7 @@ int virtio_device_freeze(struct virtio_device *dev); int virtio_device_restore(struct virtio_device *dev); #endif void virtio_reset_device(struct virtio_device *dev); +void virtio_device_shutdown(struct virtio_device *dev); int virtio_device_reset_prepare(struct virtio_device *dev); int virtio_device_reset_done(struct virtio_device *dev); diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h index f36d21b5bc19e..c381b916c1b54 100644 --- a/include/linux/virtio_net.h +++ b/include/linux/virtio_net.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -179,6 +180,9 @@ static inline int __virtio_net_hdr_to_skb(struct sk_buff *skb, if (skb->ip_summed == CHECKSUM_PARTIAL && skb->csum_offset != offsetof(struct tcphdr, check)) return -EINVAL; + + BUILD_BUG_ON(TCP_MIN_GSO_SIZE * GSO_MAX_SEGS < GSO_MAX_SIZE); + gso_size = max(gso_size, TCP_MIN_GSO_SIZE); break; } diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index d87dc7f77f4e8..a39c20c77efcd 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -62,7 +62,7 @@ struct vm_struct { #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC unsigned int page_order; #endif - unsigned int nr_pages; + unsigned long nr_pages; phys_addr_t phys_addr; const void *caller; unsigned long requested_size; diff --git a/include/linux/wait.h b/include/linux/wait.h index dce055e6add39..7e215330199cf 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -1228,6 +1228,7 @@ long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_en void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); +int woken_wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); #define DEFINE_WAIT_FUNC(name, function) \ diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index ace7379d627d2..553d7b23e3adf 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -32,6 +32,7 @@ int out_of_line_wait_on_bit_timeout(unsigned long *word, int, wait_bit_action_f int out_of_line_wait_on_bit_lock(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode); struct wait_queue_head *bit_waitqueue(unsigned long *word, int bit); extern void __init wait_bit_init(void); +extern struct wait_bit_key *__var_wake_key(struct wait_queue_entry *wq_entry, void *arg); int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); diff --git a/include/media/rc-map.h b/include/media/rc-map.h index d90e4611b0664..d95ed3e96de28 100644 --- a/include/media/rc-map.h +++ b/include/media/rc-map.h @@ -262,7 +262,6 @@ struct rc_map *rc_map_get(const char *name); #define RC_MAP_GENIUS_TVGO_A11MCE "rc-genius-tvgo-a11mce" #define RC_MAP_GOTVIEW7135 "rc-gotview7135" #define RC_MAP_HAUPPAUGE "rc-hauppauge" -#define RC_MAP_HAUPPAUGE_NEW "rc-hauppauge" #define RC_MAP_HISI_POPLAR "rc-hisi-poplar" #define RC_MAP_HISI_TV_DEMO "rc-hisi-tv-demo" #define RC_MAP_IMON_MCE "rc-imon-mce" @@ -309,7 +308,6 @@ struct rc_map *rc_map_get(const char *name); #define RC_MAP_PROTEUS_2309 "rc-proteus-2309" #define RC_MAP_PURPLETV "rc-purpletv" #define RC_MAP_PV951 "rc-pv951" -#define RC_MAP_RC5_TV "rc-rc5-tv" #define RC_MAP_RC6_MCE "rc-rc6-mce" #define RC_MAP_REAL_AUDIO_220_32_KEYS "rc-real-audio-220-32-keys" #define RC_MAP_REDDO "rc-reddo" diff --git a/include/media/v4l2-hevc.h b/include/media/v4l2-hevc.h new file mode 100644 index 0000000000000..973c96be16be4 --- /dev/null +++ b/include/media/v4l2-hevc.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper functions for HEVC stateless codecs. + */ + +#ifndef _MEDIA_V4L2_HEVC_H +#define _MEDIA_V4L2_HEVC_H + +#include +#include + +/** + * v4l2_hevc_pps_num_tile_columns - number of HEVC tile columns, bounded + * @pps: the V4L2 HEVC PPS control + * + * Return the number of tile columns (num_tile_columns_minus1 + 1) clamped to + * the capacity of column_width_minus1[]. The control validation already + * rejects out-of-range counts; this keeps the consuming drivers bounded too. + */ +static inline unsigned int +v4l2_hevc_pps_num_tile_columns(const struct v4l2_ctrl_hevc_pps *pps) +{ + return min_t(unsigned int, pps->num_tile_columns_minus1 + 1, + ARRAY_SIZE(pps->column_width_minus1)); +} + +/** + * v4l2_hevc_pps_num_tile_rows - number of HEVC tile rows, bounded + * @pps: the V4L2 HEVC PPS control + * + * Return the number of tile rows (num_tile_rows_minus1 + 1) clamped to the + * capacity of row_height_minus1[]. + */ +static inline unsigned int +v4l2_hevc_pps_num_tile_rows(const struct v4l2_ctrl_hevc_pps *pps) +{ + return min_t(unsigned int, pps->num_tile_rows_minus1 + 1, + ARRAY_SIZE(pps->row_height_minus1)); +} + +#endif /* _MEDIA_V4L2_HEVC_H */ diff --git a/include/net/bluetooth/coredump.h b/include/net/bluetooth/coredump.h index 72f51b587a046..00c12c7ac042f 100644 --- a/include/net/bluetooth/coredump.h +++ b/include/net/bluetooth/coredump.h @@ -61,6 +61,7 @@ struct hci_devcoredump { #ifdef CONFIG_DEV_COREDUMP void hci_devcd_reset(struct hci_dev *hdev); +void hci_devcd_shutdown(struct hci_dev *hdev); void hci_devcd_rx(struct work_struct *work); void hci_devcd_timeout(struct work_struct *work); @@ -75,6 +76,7 @@ int hci_devcd_abort(struct hci_dev *hdev); #else static inline void hci_devcd_reset(struct hci_dev *hdev) {} +static inline void hci_devcd_shutdown(struct hci_dev *hdev) {} static inline void hci_devcd_rx(struct work_struct *work) {} static inline void hci_devcd_timeout(struct work_struct *work) {} diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 3df59849dcbea..8115435b5d2af 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -919,9 +919,9 @@ static inline void hci_discovery_filter_clear(struct hci_dev *hdev) hdev->discovery.result_filtering = false; hdev->discovery.report_invalid_rssi = true; hdev->discovery.rssi = HCI_RSSI_INVALID; - hdev->discovery.uuid_count = 0; spin_lock(&hdev->discovery.lock); + hdev->discovery.uuid_count = 0; kfree(hdev->discovery.uuids); hdev->discovery.uuids = NULL; spin_unlock(&hdev->discovery.lock); diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index ef6ce1c20a4f0..3d9a32094347c 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -699,7 +699,12 @@ struct l2cap_rx_busy { struct l2cap_pinfo { struct bt_sock bt; + + /* With owning sk_socket chan may be read without lock, other access + * should hold lock_sock. + */ struct l2cap_chan *chan; + struct list_head rx_busy; }; diff --git a/include/net/cfg802154.h b/include/net/cfg802154.h index 76d2cd2e2b309..2e960441ea49e 100644 --- a/include/net/cfg802154.h +++ b/include/net/cfg802154.h @@ -376,6 +376,7 @@ struct cfg802154_mac_pkt { struct list_head node; struct sk_buff *skb; struct ieee802154_sub_if_data *sdata; + netdevice_tracker dev_tracker; u8 page; u8 channel; }; diff --git a/include/net/codel.h b/include/net/codel.h index aa80f744826cd..183d43c2bd434 100644 --- a/include/net/codel.h +++ b/include/net/codel.h @@ -140,6 +140,11 @@ struct codel_vars { /* needed shift to get a Q0.32 number from rec_inv_sqrt */ #define REC_INV_SQRT_SHIFT (32 - REC_INV_SQRT_BITS) +/* Cap on drops per codel_dequeue() call: the loop's work depends on the + * idle gap and backlog, both outside our control; resync when exceeded. + */ +#define CODEL_MAX_DROPS_PER_DEQUEUE 256 + /** * struct codel_stats - contains codel shared variables and stats * @maxpacket: largest packet we've seen so far diff --git a/include/net/codel_impl.h b/include/net/codel_impl.h index 2c1f0ec309e9f..8f26132d45b7f 100644 --- a/include/net/codel_impl.h +++ b/include/net/codel_impl.h @@ -93,12 +93,17 @@ static void codel_Newton_step(struct codel_vars *vars) * CoDel control_law is t + interval/sqrt(count) * We maintain in rec_inv_sqrt the reciprocal value of sqrt(count) to avoid * both sqrt() and divide operation. + * + * Clamp the increment to at least 1 tick: a very small interval (or a + * large count) can truncate it to zero, stalling the dropping loop. */ static codel_time_t codel_control_law(codel_time_t t, codel_time_t interval, u32 rec_inv_sqrt) { - return t + reciprocal_scale(interval, rec_inv_sqrt << REC_INV_SQRT_SHIFT); + return t + max_t(u32, 1, + reciprocal_scale(interval, + rec_inv_sqrt << REC_INV_SQRT_SHIFT)); } static bool codel_should_drop(const struct sk_buff *skb, @@ -154,6 +159,7 @@ static struct sk_buff *codel_dequeue(void *ctx, codel_skb_dequeue_t dequeue_func) { struct sk_buff *skb = dequeue_func(vars, ctx); + unsigned int drops = 0; codel_time_t now; bool drop; @@ -180,6 +186,14 @@ static struct sk_buff *codel_dequeue(void *ctx, */ while (vars->dropping && codel_time_after_eq(now, vars->drop_next)) { + if (++drops > CODEL_MAX_DROPS_PER_DEQUEUE) { + /* fell far behind the schedule */ + WRITE_ONCE(vars->drop_next, + codel_control_law(now, + params->interval, + vars->rec_inv_sqrt)); + break; + } /* dont care of possible wrap * since there is no more divide. */ diff --git a/include/net/dst.h b/include/net/dst.h index 307073eae7f83..dbedfe72e1fd1 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -455,7 +455,8 @@ static inline unsigned int dst_dev_overhead(struct dst_entry *dst, struct sk_buff *skb) { if (likely(dst)) - return LL_RESERVED_SPACE(dst->dev); + return max_t(unsigned int, skb->mac_len, + LL_RESERVED_SPACE(dst->dev)); return skb->mac_len; } diff --git a/include/net/gue.h b/include/net/gue.h index caefd6da86939..d377155fd0b31 100644 --- a/include/net/gue.h +++ b/include/net/gue.h @@ -84,8 +84,9 @@ static inline size_t guehdr_priv_flags_len(__be32 flags) } /* Validate standard and private flags. Returns non-zero (meaning invalid) - * if there is an unknown standard or private flags, or the options length for - * the flags exceeds the options length specific in hlen of the GUE header. + * if there is an unknown standard or private flags, if the options length for + * the flags exceeds the options length specified in hlen of the GUE header, or + * if a private option contains invalid data. */ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen) { @@ -103,8 +104,8 @@ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen) /* Private flags are last four bytes accounted in * guehdr_flags_len */ - __be32 pflags = *(__be32 *)((void *)&guehdr[1] + - len - GUE_LEN_PRIV); + void *data = (void *)&guehdr[1] + len; + __be32 pflags = *(__be32 *)(data - GUE_LEN_PRIV); if (pflags & ~GUE_PFLAGS_ALL) return 1; @@ -112,6 +113,16 @@ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen) len += guehdr_priv_flags_len(pflags); if (len > optlen) return 1; + + if (pflags & GUE_PFLAG_REMCSUM) { + __be16 *pd = data; + + /* The field offset pd[1] must not be less + * than the start pd[0]. + */ + if (ntohs(pd[1]) < ntohs(pd[0])) + return 1; + } } return 0; diff --git a/include/net/if_inet6.h b/include/net/if_inet6.h index 238ad3349456a..795fb41b45f5d 100644 --- a/include/net/if_inet6.h +++ b/include/net/if_inet6.h @@ -88,8 +88,6 @@ struct ip6_sf_socklist { struct in6_addr sl_addr[] __counted_by(sl_max); }; -#define IP6_SFBLOCK 10 /* allocate this many at once */ - struct ipv6_mc_socklist { struct in6_addr addr; int ifindex; diff --git a/include/net/inetpeer.h b/include/net/inetpeer.h index f475757daafba..414e9adf4c511 100644 --- a/include/net/inetpeer.h +++ b/include/net/inetpeer.h @@ -35,6 +35,7 @@ struct inetpeer_addr { struct inet_peer { struct rb_node rb_node; + u64 hash; struct inetpeer_addr daddr; u32 metrics[RTAX_MAX]; @@ -125,6 +126,9 @@ static inline int inetpeer_addr_cmp(const struct inetpeer_addr *a, { int i, n; + if (a->family != b->family) + return a->family < b->family ? -1 : 1; + if (a->family == AF_INET) n = sizeof(a->a4) / sizeof(u32); else diff --git a/include/net/ip.h b/include/net/ip.h index 7f2fe1a8401b1..6f602df72ee62 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -506,6 +506,31 @@ static inline unsigned int ip_dst_mtu_maybe_forward(const struct dst_entry *dst, return res; } +/* Configured/administrative MTU of a route, for advertising the TCP MSS. + * + * Unlike ip_dst_mtu_maybe_forward(), this deliberately ignores the + * ICMP-learned path MTU (rt->rt_pmtu). The advertised MSS bounds what the + * peer may send to us and must reflect our receive capability (the device or + * route-configured MTU), not a path MTU learned on the reverse (send) + * direction, which may not apply to the peer->us path and outlives the fnhe + * for the whole connection. See RFC 2923 section 2.3 and the comment above + * tcp_advertise_mss(). + */ +static inline unsigned int ip_dst_mtu_configured(const struct dst_entry *dst) +{ + unsigned int mtu, res; + + rcu_read_lock(); + mtu = dst_metric_raw(dst, RTAX_MTU); + if (!mtu) + mtu = READ_ONCE(dst_dev_rcu(dst)->mtu); + mtu = min_t(unsigned int, mtu, IP_MAX_MTU); + res = mtu - lwtunnel_headroom(dst->lwtstate, mtu); + rcu_read_unlock(); + + return res; +} + static inline unsigned int ip_skb_dst_mtu(struct sock *sk, const struct sk_buff *skb) { @@ -679,7 +704,8 @@ static inline void ip_ipgre_mc_map(__be32 naddr, const unsigned char *broadcast, static __inline__ void inet_reset_saddr(struct sock *sk) { - inet_sk(sk)->inet_rcv_saddr = inet_sk(sk)->inet_saddr = 0; + inet_sk(sk)->inet_saddr = 0; + WRITE_ONCE(inet_sk(sk)->inet_rcv_saddr, 0); #if IS_ENABLED(CONFIG_IPV6) if (sk->sk_family == PF_INET6) { struct ipv6_pinfo *np = inet6_sk(sk); diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index 09ffe0f13ce75..0a6c4ceb95a7a 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -101,12 +101,12 @@ static inline struct dst_entry *ip6_route_output(struct net *net, } /* Only conditionally release dst if flags indicates - * !RT6_LOOKUP_F_DST_NOREF or dst is in uncached_list. + * !RT6_LOOKUP_F_DST_NOREF or dst is uncached. */ static inline void ip6_rt_put_flags(struct rt6_info *rt, int flags) { if (!(flags & RT6_LOOKUP_F_DST_NOREF) || - !list_empty(&rt->dst.rt_uncached)) + rt->dst.rt_uncached_list) ip6_rt_put(rt); } @@ -382,6 +382,45 @@ static inline unsigned int ip6_dst_mtu_maybe_forward(const struct dst_entry *dst rcu_read_unlock(); out: + mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); + + return mtu - lwtunnel_headroom(dst->lwtstate, mtu); +} + +/* Configured/administrative MTU of a route, for advertising the TCP MSS. + * + * Unlike ip6_dst_mtu_maybe_forward(), this ignores any ICMPv6-learned path + * MTU (which is kept on the RTF_CACHE exception route) and returns the MTU of + * the underlying route (fib6_pmtu) or the egress device. The advertised MSS + * bounds what the peer may send to us and must reflect our receive + * capability, not a path MTU learned on the reverse (send) direction. See + * RFC 2923 section 2.3 and the comment above tcp_advertise_mss(). + */ +static inline unsigned int ip6_dst_mtu_configured(const struct dst_entry *dst) +{ + const struct rt6_info *rt = dst_rt6_info(dst); + const struct fib6_info *from; + struct inet6_dev *idev; + unsigned int mtu = 0; + + rcu_read_lock(); + /* IPv6 keeps the learned PMTU and the configured MTU in the same + * RTAX_MTU slot: the learned value sits on this (possibly RTF_CACHE) + * dst, the configured one on the underlying route. Reach the latter + * via ->from (fib6_pmtu), populated by ip6_route_info_create(). + */ + from = rcu_dereference(rt->from); + if (from) + mtu = from->fib6_pmtu; + if (!mtu) { + mtu = IPV6_MIN_MTU; + idev = __in6_dev_get(dst_dev_rcu(dst)); + if (idev) + mtu = max_t(unsigned int, mtu, READ_ONCE(idev->cnf.mtu6)); + } + rcu_read_unlock(); + + mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); return mtu - lwtunnel_headroom(dst->lwtstate, mtu); } diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index d708b66e55cda..85e3455cea259 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -629,8 +629,7 @@ struct metadata_dst *iptunnel_metadata_reply(struct metadata_dst *md, int skb_tunnel_check_pmtu(struct sk_buff *skb, struct dst_entry *encap_dst, int headroom, bool reply); -static inline void ip_tunnel_adj_headroom(struct net_device *dev, - unsigned int headroom) +static inline unsigned int ip_tunnel_limit_headroom(unsigned int headroom) { /* we must cap headroom to some upperlimit, else pskb_expand_head * will overflow header offsets in skb_headers_offset_update(). @@ -640,6 +639,14 @@ static inline void ip_tunnel_adj_headroom(struct net_device *dev, if (headroom > max_allowed) headroom = max_allowed; + return headroom; +} + +static inline void ip_tunnel_adj_headroom(struct net_device *dev, + unsigned int headroom) +{ + headroom = ip_tunnel_limit_headroom(headroom); + if (headroom > READ_ONCE(dev->needed_headroom)) WRITE_ONCE(dev->needed_headroom, headroom); } diff --git a/include/net/libeth/xsk.h b/include/net/libeth/xsk.h index 5dcc0d7f65b76..a452b7828ce4e 100644 --- a/include/net/libeth/xsk.h +++ b/include/net/libeth/xsk.h @@ -196,7 +196,7 @@ __libeth_xsk_xmit_fill_buf_md(const struct xdp_desc *xdesc, struct libeth_xdp_tx_desc desc; struct xdp_desc_ctx ctx; - ctx = xsk_buff_raw_get_ctx(sq->pool, xdesc->addr); + ctx = xsk_buff_raw_get_ctx(sq->pool, xdesc->addr, xdesc->options); desc = (typeof(desc)){ .addr = ctx.dma, __libeth_xdp_tx_len(xdesc->len), diff --git a/include/net/mac80211.h b/include/net/mac80211.h index 4f95da023746f..36055d40172a1 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -7615,11 +7615,14 @@ bool ieee80211_tx_prepare_skb(struct ieee80211_hw *hw, * * @skb: packet injected by userspace * @dev: the &struct device of this 802.11 device + * @chandef: the channel definition the frame will be transmitted on, or + * %NULL to skip the bandwidth checks * * Return: %true if the radiotap header was parsed, %false otherwise */ bool ieee80211_parse_tx_radiotap(struct sk_buff *skb, - struct net_device *dev); + struct net_device *dev, + const struct cfg80211_chan_def *chandef); /** * struct ieee80211_noa_data - holds temporary data for tracking P2P NoA state diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h index bc42dd0e10e65..c39425e54d87d 100644 --- a/include/net/netfilter/nf_conntrack.h +++ b/include/net/netfilter/nf_conntrack.h @@ -185,6 +185,11 @@ static inline void nf_ct_put(struct nf_conn *ct) nf_ct_destroy(&ct->ct_general); } +static inline bool nf_ct_shared(const struct nf_conn *ct) +{ + return refcount_read(&ct->ct_general.use) > 1; +} + /* load module; enable/disable conntrack in this namespace */ int nf_ct_netns_get(struct net *net, u8 nfproto); void nf_ct_netns_put(struct net *net, u8 nfproto); diff --git a/include/net/netfilter/nf_conntrack_helper.h b/include/net/netfilter/nf_conntrack_helper.h index c761cd8158b25..309f74ea4d098 100644 --- a/include/net/netfilter/nf_conntrack_helper.h +++ b/include/net/netfilter/nf_conntrack_helper.h @@ -108,6 +108,7 @@ void nf_ct_helper_init(struct nf_conntrack_helper *helper, int nf_conntrack_helper_register(struct nf_conntrack_helper *, struct nf_conntrack_helper **); int __nf_conntrack_helper_register(struct nf_conntrack_helper *); void nf_conntrack_helper_unregister(struct nf_conntrack_helper *); +void nf_conntrack_helper_release(struct nf_conntrack_helper *); int nf_conntrack_helpers_register(struct nf_conntrack_helper *, unsigned int, struct nf_conntrack_helper **); diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index 3be612145c130..85de3473da240 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -870,8 +870,6 @@ struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set, const u32 *key, const u32 *key_end, const u32 *data, u64 timeout, u64 expiration, gfp_t gfp); -int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set, - struct nft_expr *expr_array[]); void nft_set_elem_expr_destroy(const struct nft_ctx *ctx, struct nft_set_elem_expr *elem_expr); void nft_set_elem_destroy(const struct nft_set *set, diff --git a/include/net/nfc/hci.h b/include/net/nfc/hci.h index 756c11084f65f..86ed63e5d5334 100644 --- a/include/net/nfc/hci.h +++ b/include/net/nfc/hci.h @@ -144,7 +144,7 @@ struct nfc_hci_dev { data_exchange_cb_t async_cb; void *async_cb_context; - u8 *gb; + u8 gb[NFC_MAX_GT_LEN]; size_t gb_len; unsigned long quirks; diff --git a/include/net/nfc/nfc.h b/include/net/nfc/nfc.h index c54df042db6be..bcafab5c53e51 100644 --- a/include/net/nfc/nfc.h +++ b/include/net/nfc/nfc.h @@ -273,7 +273,8 @@ struct sk_buff *nfc_alloc_recv_skb(unsigned int size, gfp_t gfp); int nfc_set_remote_general_bytes(struct nfc_dev *dev, const u8 *gt, u8 gt_len); -u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, size_t *gb_len); +u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, u8 *out_gb, + size_t gb_max_len, size_t *gb_len); int nfc_fw_download_done(struct nfc_dev *dev, const char *firmware_name, u32 result); diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h index 18a419cd9d949..90d3e7943b19a 100644 --- a/include/net/pkt_sched.h +++ b/include/net/pkt_sched.h @@ -12,6 +12,7 @@ #define DEFAULT_TX_QUEUE_LEN 1000 #define STAB_SIZE_LOG_MAX 30 +#define QDISC_PKT_LEN_MAX (1 << 20) /* 1 MiB */ struct qdisc_walker { int stop; diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index cbc2487765118..f35bd06a6bad6 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -793,7 +793,7 @@ void qdisc_offload_query_caps(struct net_device *dev, struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, struct netlink_ext_ack *extack); -void qdisc_free(struct Qdisc *qdisc); +void qdisc_free_rcu(struct Qdisc *qdisc); struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, u32 parentid, struct netlink_ext_ack *extack); diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index cccc662561aac..b21f23b736fd2 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -2057,7 +2057,7 @@ struct sctp_association { force_delay:1; __u8 strreset_enable; - __u8 strreset_outstanding; /* request param count on the fly */ + __u8 strreset_outstanding; /* request param bitmask on the fly */ __u32 strreset_outseq; /* Update after receiving response */ __u32 strreset_inseq; /* Update after receiving request */ diff --git a/include/net/sock.h b/include/net/sock.h index 51185222aac29..60ea55dc18854 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -2312,6 +2312,8 @@ static inline void sk_gso_disable(struct sock *sk) sk->sk_route_caps &= ~NETIF_F_GSO_MASK; } +bool sk_has_decrypt_user(const struct sock *sk); + static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb, struct iov_iter *from, char *to, int copy, int offset) diff --git a/include/net/tcp.h b/include/net/tcp.h index 2c5b889530b55..4416cdf9bf30e 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -765,8 +765,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, void tcp_send_probe0(struct sock *); int tcp_write_wakeup(struct sock *, int mib); void tcp_send_fin(struct sock *sk); -void tcp_send_active_reset(struct sock *sk, gfp_t priority, - enum sk_rst_reason reason); +void tcp_send_active_reset(struct sock *sk, enum sk_rst_reason reason); int tcp_send_synack(struct sock *); void tcp_push_one(struct sock *, unsigned int mss_now); void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags); @@ -1233,9 +1232,9 @@ static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) static inline bool tcp_skb_can_collapse(const struct sk_buff *to, const struct sk_buff *from) { - /* skb_cmp_decrypted() not needed, use tcp_write_collapse_fence() */ return likely(tcp_skb_can_collapse_to(to) && mptcp_skb_can_collapse(to, from) && + !skb_cmp_decrypted(to, from) && skb_pure_zcopy_same(to, from) && skb_frags_readable(to) == skb_frags_readable(from)); } @@ -1782,6 +1781,11 @@ static inline int tcp_full_space(const struct sock *sk) return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); } +static inline u32 tcp_dst_advmss(const struct dst_entry *dst) +{ + return max_t(u32, dst_metric_advmss(dst), TCP_MIN_MSS); +} + static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh) { int unused_mem = sk_unused_reserved_mem(sk); @@ -2323,7 +2327,7 @@ static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct soc static inline void tcp_write_collapse_fence(struct sock *sk) { - struct sk_buff *skb = tcp_write_queue_tail(sk); + struct sk_buff *skb = tcp_write_queue_tail(sk) ?: tcp_rtx_queue_tail(sk); if (skb) TCP_SKB_CB(skb)->eor = 1; diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index b344789f5df83..d94aeb506379c 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -240,6 +240,7 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) * xsk_buff_raw_get_ctx - get &xdp_desc context * @pool: XSk buff pool desc address belongs to * @addr: desc address (from userspace) + * @options: desc options (from userspace) * * Wrapper for xp_raw_get_ctx() to be used in drivers, see its kdoc for * details. @@ -248,9 +249,9 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) * pointer, if it is present (initialized to %NULL otherwise). */ static inline struct xdp_desc_ctx -xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, u32 options) { - return xp_raw_get_ctx(pool, addr); + return xp_raw_get_ctx(pool, addr, options); } #define XDP_TXMD_FLAGS_VALID ( \ @@ -318,18 +319,20 @@ xsk_tx_metadata_request(const struct xsk_buff_pool *pool, } static inline struct xsk_tx_metadata * -__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data) +__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data, + unsigned int options) { - if (!pool->tx_metadata_len) + if (!pool->tx_metadata_len || !(options & XDP_TX_METADATA)) return NULL; return data - pool->tx_metadata_len; } static inline struct xsk_tx_metadata * -xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr) +xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr, u32 options) { - return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr)); + return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr), + options); } static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp) @@ -510,7 +513,7 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) } static inline struct xdp_desc_ctx -xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, u32 options) { return (struct xdp_desc_ctx){ }; } @@ -530,13 +533,14 @@ xsk_tx_metadata_request(const struct xsk_buff_pool *pool, } static inline struct xsk_tx_metadata * -__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data) +__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data, + unsigned int options) { return NULL; } static inline struct xsk_tx_metadata * -xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr) +xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr, u32 options) { return NULL; } diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index 2bb1d122b1bc6..a7df573784fd6 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -154,7 +154,8 @@ struct xdp_desc_ctx { struct xsk_tx_metadata *meta; }; -struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr); +struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, + u32 options); static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb) { diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h index 24fd362130233..80f3ba6663d06 100644 --- a/include/rdma/uverbs_ioctl.h +++ b/include/rdma/uverbs_ioctl.h @@ -1151,4 +1151,24 @@ static inline int ib_respond_empty_udata(struct ib_udata *udata) return 0; } +/** + * ib_no_udata_io - Ensure no input data and zero fill the response buffer + * @udata: The system call's ib_udata struct + * + * Driver ops which do not accept any input data and do not provide any response + * data may call this at the beginning of their handler to fully adhere to the + * uAPI forward/backward compatibility rules. + * + * Return: Negative failure code if the op should be denied, 0 otherwise. + */ +static inline int ib_no_udata_io(struct ib_udata *udata) +{ + int ret = ib_is_udata_in_empty(udata); + + if (ret) + return ret; + + return ib_respond_empty_udata(udata); +} + #endif diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h index 5a07f9a6dcd1f..6f3622892c0cc 100644 --- a/include/rdma/uverbs_types.h +++ b/include/rdma/uverbs_types.h @@ -180,8 +180,6 @@ struct ib_uverbs_file { struct page *disassociate_page; struct xarray idr; - - struct mutex disassociation_lock; }; extern const struct uverbs_obj_type_class uverbs_idr_class; diff --git a/include/sound/sdca_fdl.h b/include/sound/sdca_fdl.h index dc33927b82bde..bc3600cdd2ec8 100644 --- a/include/sound/sdca_fdl.h +++ b/include/sound/sdca_fdl.h @@ -83,6 +83,10 @@ static inline int sdca_fdl_alloc_state(struct sdca_interrupt *interrupt) return 0; } +static inline void sdca_fdl_free_state(struct sdca_interrupt *interrupt) +{ +} + static inline int sdca_fdl_process(struct sdca_interrupt *interrupt) { return 0; diff --git a/include/sound/soc.h b/include/sound/soc.h index 10ad80f930c2e..1d2772758fb69 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -698,7 +698,8 @@ struct snd_soc_dai_link_component { struct snd_soc_dai_link_ch_map { unsigned int cpu; unsigned int codec; - unsigned int ch_mask; + unsigned int cpu_ch_mask; + unsigned int codec_ch_mask; }; struct snd_soc_dai_link { diff --git a/include/sound/soc_sdw_utils.h b/include/sound/soc_sdw_utils.h index 79c21966220b2..9b28e9aef4f1d 100644 --- a/include/sound/soc_sdw_utils.h +++ b/include/sound/soc_sdw_utils.h @@ -44,6 +44,18 @@ struct asoc_sdw_codec_info; +struct asoc_sdw_mc_private { + struct snd_soc_card card; + struct snd_soc_jack sdw_headset; + struct device *headset_codec_dev; /* only one headset per card */ + struct device *amp_dev1, *amp_dev2; + bool append_dai_type; + bool ignore_internal_dmic; + void *private; + unsigned long mc_quirk; + int codec_info_list_count; +}; + struct asoc_sdw_dai_info { const bool direction[2]; /* playback & capture support */ const char *codec_name; @@ -88,25 +100,13 @@ struct asoc_sdw_codec_info { int (*codec_card_late_probe)(struct snd_soc_card *card); - int (*count_sidecar)(struct snd_soc_card *card, + int (*count_sidecar)(struct asoc_sdw_mc_private *ctx, int *num_dais, int *num_devs); int (*add_sidecar)(struct snd_soc_card *card, struct snd_soc_dai_link **dai_links, struct snd_soc_codec_conf **codec_conf); }; -struct asoc_sdw_mc_private { - struct snd_soc_card card; - struct snd_soc_jack sdw_headset; - struct device *headset_codec_dev; /* only one headset per card */ - struct device *amp_dev1, *amp_dev2; - bool append_dai_type; - bool ignore_internal_dmic; - void *private; - unsigned long mc_quirk; - int codec_info_list_count; -}; - struct asoc_sdw_endpoint { struct list_head list; @@ -182,7 +182,8 @@ struct asoc_sdw_dailink *asoc_sdw_find_dailink(struct asoc_sdw_dailink *dailinks const struct snd_soc_acpi_endpoint *new); int asoc_sdw_get_dai_type(u32 type); -int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card, +int asoc_sdw_parse_sdw_endpoints(struct device *dev, + struct asoc_sdw_mc_private *ctx, struct snd_soc_aux_dev *soc_aux, struct asoc_sdw_dailink *soc_dais, struct asoc_sdw_endpoint *soc_ends, @@ -235,7 +236,7 @@ int asoc_sdw_es9356_amp_init(struct snd_soc_card *card, int asoc_sdw_es9356_exit(struct snd_soc_card *card, struct snd_soc_dai_link *dai_link); /* CS AMP support */ -int asoc_sdw_bridge_cs35l56_count_sidecar(struct snd_soc_card *card, +int asoc_sdw_bridge_cs35l56_count_sidecar(struct asoc_sdw_mc_private *ctx, int *num_dais, int *num_devs); int asoc_sdw_bridge_cs35l56_add_sidecar(struct snd_soc_card *card, struct snd_soc_dai_link **dai_links, diff --git a/include/target/target_core_fabric.h b/include/target/target_core_fabric.h index e9039e73d058a..390ace5bb252b 100644 --- a/include/target/target_core_fabric.h +++ b/include/target/target_core_fabric.h @@ -95,6 +95,8 @@ struct target_core_fabric_ops { struct se_lun *); void (*fabric_pre_unlink)(struct se_portal_group *, struct se_lun *); + void (*fabric_post_unlink)(struct se_portal_group *se_tpg, + struct se_lun *lun); struct se_tpg_np *(*fabric_make_np)(struct se_portal_group *, struct config_group *, const char *); void (*fabric_drop_np)(struct se_tpg_np *); diff --git a/include/trace/events/cachefiles.h b/include/trace/events/cachefiles.h index 6e3b1424eea4d..d002ed508a435 100644 --- a/include/trace/events/cachefiles.h +++ b/include/trace/events/cachefiles.h @@ -380,7 +380,7 @@ TRACE_EVENT(cachefiles_rename, TRACE_EVENT(cachefiles_coherency, TP_PROTO(struct cachefiles_object *obj, ino_t ino, - u64 disk_aux, + const void *disk_aux, enum cachefiles_content content, enum cachefiles_coherency_trace why), @@ -397,12 +397,27 @@ TRACE_EVENT(cachefiles_coherency, ), TP_fast_assign( + union { + __be16 s[4]; + __be64 ll; + } x; + __entry->obj = obj->debug_id; __entry->why = why; __entry->content = content; __entry->ino = ino; __entry->aux = be64_to_cpup((__be64 *)obj->cookie->inline_aux); - __entry->disk_aux = disk_aux; + + /* cachefiles_xattr::data is 2-byte aligned but not 8-byte aligned. */ + if (disk_aux) { + x.s[0] = ((__be16 *)disk_aux)[0]; + x.s[1] = ((__be16 *)disk_aux)[1]; + x.s[2] = ((__be16 *)disk_aux)[2]; + x.s[3] = ((__be16 *)disk_aux)[3]; + __entry->disk_aux = be64_to_cpu(x.ll); + } else { + __entry->disk_aux = 0; + } ), TP_printk("o=%08x %s B=%llx c=%u aux=%llx dsk=%llx", diff --git a/include/trace/events/dma.h b/include/trace/events/dma.h index 31c9ddf72c9d3..daef2c176ecd8 100644 --- a/include/trace/events/dma.h +++ b/include/trace/events/dma.h @@ -133,7 +133,7 @@ DECLARE_EVENT_CLASS(dma_alloc_class, TP_fast_assign( __assign_str(device); __entry->virt_addr = virt_addr; - __entry->dma_addr = dma_addr; + __entry->dma_addr = virt_addr ? dma_addr : 0; __entry->size = size; __entry->flags = flags; __entry->dir = dir; diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index 270c1a2c24c44..1dd9fc5afc46f 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -2114,6 +2114,14 @@ DEFINE_EVENT(f2fs_zip_end, f2fs_decompress_pages_end, ); #ifdef CONFIG_F2FS_IOSTAT +/* + * Number of read folio order buckets emitted by the f2fs_iostat tracepoint. + * TP_printk() cannot loop, so the field count is fixed here and must be >= + * the largest possible NR_PAGE_ORDERS (14 on arm64 with 64K pages). The + * BUILD_BUG_ON() in f2fs_update_read_folio_count() enforces this. + */ +#define F2FS_IOSTAT_RD_FOLIO_ORDERS 16 + TRACE_EVENT(f2fs_iostat, TP_PROTO(struct f2fs_sb_info *sbi, unsigned long long *iostat, @@ -2151,7 +2159,7 @@ TRACE_EVENT(f2fs_iostat, __field(unsigned long long, fs_mrio) __field(unsigned long long, fs_discard) __field(unsigned long long, fs_reset_zone) - __array(unsigned long long, read_folio_count, 11) + __array(unsigned long long, read_folio_count, F2FS_IOSTAT_RD_FOLIO_ORDERS) ), TP_fast_assign( @@ -2186,7 +2194,8 @@ TRACE_EVENT(f2fs_iostat, __entry->fs_reset_zone = iostat[FS_ZONE_RESET_IO]; memset(__entry->read_folio_count, 0, sizeof(__entry->read_folio_count)); memcpy(__entry->read_folio_count, read_folio_count, - sizeof(unsigned long long) * min_t(int, NR_PAGE_ORDERS, 11)); + sizeof(unsigned long long) * + min_t(int, NR_PAGE_ORDERS, F2FS_IOSTAT_RD_FOLIO_ORDERS)); ), TP_printk("dev = (%d,%d), " @@ -2201,7 +2210,8 @@ TRACE_EVENT(f2fs_iostat, "fs [data=%llu, (gc_data=%llu, cdata=%llu), " "node=%llu, meta=%llu], " "read_folio_count [0=%llu, 1=%llu, 2=%llu, 3=%llu, 4=%llu, " - "5=%llu, 6=%llu, 7=%llu, 8=%llu, 9=%llu, 10=%llu]", + "5=%llu, 6=%llu, 7=%llu, 8=%llu, 9=%llu, 10=%llu, 11=%llu, " + "12=%llu, 13=%llu, 14=%llu, 15=%llu]", show_dev(__entry->dev), __entry->app_wio, __entry->app_dio, __entry->app_bio, __entry->app_mio, __entry->app_bcdio, __entry->app_mcdio, __entry->fs_dio, __entry->fs_cdio, @@ -2218,7 +2228,9 @@ TRACE_EVENT(f2fs_iostat, __entry->read_folio_count[4], __entry->read_folio_count[5], __entry->read_folio_count[6], __entry->read_folio_count[7], __entry->read_folio_count[8], __entry->read_folio_count[9], - __entry->read_folio_count[10]) + __entry->read_folio_count[10], __entry->read_folio_count[11], + __entry->read_folio_count[12], __entry->read_folio_count[13], + __entry->read_folio_count[14], __entry->read_folio_count[15]) ); #ifndef __F2FS_IOSTAT_LATENCY_TYPE diff --git a/include/trace/events/icmp.h b/include/trace/events/icmp.h index 31559796949a7..acfcf2b132d60 100644 --- a/include/trace/events/icmp.h +++ b/include/trace/events/icmp.h @@ -27,17 +27,20 @@ TRACE_EVENT(icmp_send, TP_fast_assign( struct iphdr *iph = ip_hdr(skb); - struct udphdr *uh = udp_hdr(skb); - int proto_4 = iph->protocol; + struct udphdr _uh, *uh = NULL; __be32 *p32; __entry->skbaddr = skb; __entry->type = type; __entry->code = code; - if (proto_4 != IPPROTO_UDP || (u8 *)uh < skb->head || - (u8 *)uh + sizeof(struct udphdr) - > skb_tail_pointer(skb)) { + if (iph->protocol == IPPROTO_UDP) + uh = skb_header_pointer(skb, + skb_network_offset(skb) + + (iph->ihl << 2), + sizeof(_uh), &_uh); + + if (!uh) { __entry->sport = 0; __entry->dport = 0; __entry->ulen = 0; diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h index 082cb03c61316..3fec3e8f91c85 100644 --- a/include/trace/events/netfs.h +++ b/include/trace/events/netfs.h @@ -59,6 +59,7 @@ EM(netfs_rreq_trace_free, "FREE ") \ EM(netfs_rreq_trace_intr, "INTR ") \ EM(netfs_rreq_trace_ki_complete, "KI-CMPL") \ + EM(netfs_rreq_trace_ra_put_ref, "RA-PUT ") \ EM(netfs_rreq_trace_recollect, "RECLLCT") \ EM(netfs_rreq_trace_redirty, "REDIRTY") \ EM(netfs_rreq_trace_resubmit, "RESUBMT") \ @@ -70,9 +71,11 @@ EM(netfs_rreq_trace_unpause, "UNPAUSE") \ EM(netfs_rreq_trace_wait_ip, "WAIT-IP") \ EM(netfs_rreq_trace_wait_pause, "--PAUSED--") \ + EM(netfs_rreq_trace_wait_put_ra_refs, "WAIT-P-RA") \ EM(netfs_rreq_trace_wait_quiesce, "WAIT-QUIESCE") \ EM(netfs_rreq_trace_waited_ip, "DONE-IP") \ EM(netfs_rreq_trace_waited_pause, "--UNPAUSED--") \ + EM(netfs_rreq_trace_waited_put_ra_refs, "DONE-P-RA") \ EM(netfs_rreq_trace_waited_quiesce, "DONE-QUIESCE") \ EM(netfs_rreq_trace_wake_ip, "WAKE-IP") \ EM(netfs_rreq_trace_wake_queue, "WAKE-Q ") \ @@ -195,7 +198,6 @@ EM(netfs_folio_trace_clear_cc, "clear-cc") \ EM(netfs_folio_trace_clear_g, "clear-g") \ EM(netfs_folio_trace_clear_s, "clear-s") \ - EM(netfs_folio_trace_copy_to_cache, "mark-copy") \ EM(netfs_folio_trace_end_copy, "end-copy") \ EM(netfs_folio_trace_filled_gaps, "filled-gaps") \ EM(netfs_folio_trace_invalidate_all, "inval-all") \ @@ -206,16 +208,19 @@ EM(netfs_folio_trace_kill_cc, "kill-cc") \ EM(netfs_folio_trace_kill_g, "kill-g") \ EM(netfs_folio_trace_kill_s, "kill-s") \ + EM(netfs_folio_trace_mark_copy, "mark-copy") \ EM(netfs_folio_trace_mkwrite, "mkwrite") \ EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \ - EM(netfs_folio_trace_not_under_wback, "!wback") \ EM(netfs_folio_trace_not_locked, "!locked") \ + EM(netfs_folio_trace_not_under_wback, "!wback") \ + EM(netfs_folio_trace_pgpriv2_copy, "pgpriv2-copy") \ EM(netfs_folio_trace_put, "put") \ EM(netfs_folio_trace_read, "read") \ EM(netfs_folio_trace_read_done, "read-done") \ EM(netfs_folio_trace_read_gaps, "read-gaps") \ EM(netfs_folio_trace_read_unlock, "read-unlock") \ EM(netfs_folio_trace_redirtied, "redirtied") \ + EM(netfs_folio_trace_sched_copy, "sched-copy") \ EM(netfs_folio_trace_store, "store") \ EM(netfs_folio_trace_store_copy, "store-copy") \ EM(netfs_folio_trace_store_plus, "store+") \ @@ -786,6 +791,27 @@ TRACE_EVENT(netfs_folioq, __print_symbolic(__entry->trace, netfs_folioq_traces)) ); +TRACE_EVENT(netfs_read_progress_at, + TP_PROTO(const struct netfs_io_request *rreq), + + TP_ARGS(rreq), + + TP_STRUCT__entry( + __field(unsigned int, rreq) + __field(size_t, progress_at) + __field(size_t, cleaned_to) + ), + + TP_fast_assign( + __entry->rreq = rreq->debug_id; + __entry->cleaned_to = rreq->cleaned_to - rreq->start; + __entry->progress_at = rreq->progress_at; + ), + + TP_printk("R=%08x cln=%zx prg=%zx", + __entry->rreq, __entry->cleaned_to, __entry->progress_at) + ); + #undef EM #undef E_ #endif /* _TRACE_NETFS_H */ diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index 9f3090db2f163..322aef9b54bbe 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -1510,6 +1510,7 @@ struct drm_amdgpu_info_device { __u64 high_va_max; /* gfx10 pa_sc_tile_steering_override */ __u32 pa_sc_tile_steering_override; + __u32 pad; /* disabled TCCs */ __u64 tcc_disabled_mask; __u64 min_engine_clock; @@ -1534,7 +1535,6 @@ struct drm_amdgpu_info_device { __u32 csa_alignment; /* Userq IP mask (1 << AMDGPU_HW_IP_*) */ __u32 userq_ip_mask; - __u32 pad; }; struct drm_amdgpu_info_hw_ip { diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h index 0e455d91e77d4..f857f4530eb2a 100644 --- a/include/uapi/drm/panthor_drm.h +++ b/include/uapi/drm/panthor_drm.h @@ -601,6 +601,18 @@ enum drm_panthor_vm_bind_op_flags { */ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED = 1 << 2, + /** + * @DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE: Sparsely map a virtual memory range + * + * Only valid with DRM_PANTHOR_VM_BIND_OP_TYPE_MAP. + * + * When this flag is set, the whole vm_bind range is mapped over a dummy object in a cyclic + * fashion, and all GPU reads from addresses in the range return undefined values. This flag + * being set means drm_panthor_vm_bind_op::bo_offset and drm_panthor_vm_bind_op::bo_handle + * must both be set to 0. DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC must also be set. + */ + DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE = 1 << 3, + /** * @DRM_PANTHOR_VM_BIND_OP_TYPE_MASK: Mask used to determine the type of operation. */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 48e9f1fdb78dd..50c80af4ad4ef 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -2589,6 +2589,12 @@ enum drm_xe_ras_error_component { DRM_XE_RAS_ERR_COMP_CORE_COMPUTE = 1, /** @DRM_XE_RAS_ERR_COMP_SOC_INTERNAL: SoC Internal Error */ DRM_XE_RAS_ERR_COMP_SOC_INTERNAL, + /** @DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY: Device Memory Error */ + DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY, + /** @DRM_XE_RAS_ERR_COMP_PCIE: PCIe Subsystem Error */ + DRM_XE_RAS_ERR_COMP_PCIE, + /** @DRM_XE_RAS_ERR_COMP_FABRIC: Fabric Subsystem Error */ + DRM_XE_RAS_ERR_COMP_FABRIC, /** @DRM_XE_RAS_ERR_COMP_MAX: Max Error */ DRM_XE_RAS_ERR_COMP_MAX /* non-ABI */ }; @@ -2606,7 +2612,10 @@ enum drm_xe_ras_error_component { */ #define DRM_XE_RAS_ERROR_COMPONENT_NAMES { \ [DRM_XE_RAS_ERR_COMP_CORE_COMPUTE] = "core-compute", \ - [DRM_XE_RAS_ERR_COMP_SOC_INTERNAL] = "soc-internal" \ + [DRM_XE_RAS_ERR_COMP_SOC_INTERNAL] = "soc-internal", \ + [DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY] = "device-memory", \ + [DRM_XE_RAS_ERR_COMP_PCIE] = "pcie", \ + [DRM_XE_RAS_ERR_COMP_FABRIC] = "fabric", \ } #if defined(__cplusplus) diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h index 23a0627814687..50d67df78911f 100644 --- a/include/uapi/linux/if_xdp.h +++ b/include/uapi/linux/if_xdp.h @@ -149,6 +149,7 @@ struct xsk_tx_metadata { __u16 csum_start; /* Offset from csum_start where checksum should be stored. */ __u16 csum_offset; + __u32 reserved; /* XDP_TXMD_FLAGS_LAUNCH_TIME */ /* Launch time in nanosecond against the PTP HW Clock */ diff --git a/include/uapi/linux/landlock.h b/include/uapi/linux/landlock.h index 7ffe2ef127ee7..9c1102ebf06e0 100644 --- a/include/uapi/linux/landlock.h +++ b/include/uapi/linux/landlock.h @@ -351,6 +351,7 @@ struct landlock_net_port_attr { * device. * - %LANDLOCK_ACCESS_FS_MAKE_DIR: Create (or rename) a directory. * - %LANDLOCK_ACCESS_FS_MAKE_REG: Create (or rename or link) a regular file. + * This also guards the creation of whiteout objects as used in OverlayFS. * - %LANDLOCK_ACCESS_FS_MAKE_SOCK: Create (or rename or link) a UNIX domain * socket. * - %LANDLOCK_ACCESS_FS_MAKE_FIFO: Create (or rename or link) a named pipe. diff --git a/include/uapi/linux/nfs4.h b/include/uapi/linux/nfs4.h index 4273e0249fcbb..289205b53a085 100644 --- a/include/uapi/linux/nfs4.h +++ b/include/uapi/linux/nfs4.h @@ -17,11 +17,9 @@ #include #define NFS4_BITMAP_SIZE 3 -#define NFS4_VERIFIER_SIZE 8 #define NFS4_STATEID_SEQID_SIZE 4 #define NFS4_STATEID_OTHER_SIZE 12 #define NFS4_STATEID_SIZE (NFS4_STATEID_SEQID_SIZE + NFS4_STATEID_OTHER_SIZE) -#define NFS4_FHSIZE 128 #define NFS4_MAXPATHLEN PATH_MAX #define NFS4_MAXNAMLEN NAME_MAX #define NFS4_OPAQUE_LIMIT 1024 diff --git a/include/uapi/linux/time_types.h b/include/uapi/linux/time_types.h index bcc0002115d39..03a0d8aaadca5 100644 --- a/include/uapi/linux/time_types.h +++ b/include/uapi/linux/time_types.h @@ -30,7 +30,7 @@ struct __kernel_old_timeval { struct __kernel_old_timespec { __kernel_old_time_t tv_sec; /* seconds */ - long tv_nsec; /* nanoseconds */ + __kernel_long_t tv_nsec; /* nanoseconds */ }; struct __kernel_old_itimerval { diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h index a4599d7b736aa..0a125c2a8cc4e 100644 --- a/include/uapi/rdma/bnxt_re-abi.h +++ b/include/uapi/rdma/bnxt_re-abi.h @@ -247,7 +247,7 @@ struct bnxt_re_query_device_ex_resp { struct bnxt_re_db_region { __u32 dpi; __u32 reserved; - __aligned_u64 umdbr; + __aligned_u64 reserved2; }; enum bnxt_re_obj_dbr_alloc_attrs { diff --git a/init/main.c b/init/main.c index e363232b428b4..dd7b2a6599fca 100644 --- a/init/main.c +++ b/init/main.c @@ -276,7 +276,8 @@ static void * __init get_boot_config_from_initrd(size_t *_size) u8 *hdr; int i; - if (!initrd_end) + if (!initrd_end || initrd_end < initrd_start || + initrd_end - initrd_start < BOOTCONFIG_MAGIC_LEN + 8) return NULL; data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN; @@ -293,16 +294,26 @@ static void * __init get_boot_config_from_initrd(size_t *_size) found: hdr = (u8 *)(data - 8); + if ((unsigned long)hdr < initrd_start) + return NULL; + size = get_unaligned_le32(hdr); csum = get_unaligned_le32(hdr + 4); - data = ((void *)hdr) - size; - if ((unsigned long)data < initrd_start) { - pr_err("bootconfig size %d is greater than initrd size %ld\n", + if (size > XBC_DATA_MAX) { + pr_err("bootconfig size %u is greater than max size %d\n", + size, XBC_DATA_MAX); + return NULL; + } + + if (size > ((unsigned long)hdr - initrd_start)) { + pr_err("bootconfig size %u is greater than initrd size %lu\n", size, initrd_end - initrd_start); return NULL; } + data = ((void *)hdr) - size; + if (xbc_calc_checksum(data, size) != csum) { pr_err("bootconfig checksum failed\n"); return NULL; @@ -405,12 +416,6 @@ static void __init setup_boot_config(void) return; } - if (size >= XBC_DATA_MAX) { - pr_err("bootconfig size %ld greater than max size %d\n", - (long)size, XBC_DATA_MAX); - return; - } - ret = xbc_init(data, size, &msg, &pos); if (ret < 0) { if (pos < 0) @@ -1636,7 +1641,7 @@ static noinline void __init kernel_init_freeable(void) */ set_mems_allowed(node_states[N_MEMORY]); - cad_pid = get_pid(task_pid(current)); + rcu_assign_pointer(cad_pid, get_pid(task_pid(current))); smp_prepare_cpus(setup_max_cpus); diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index d656cc2a0b9ba..63fe6e5d79bad 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -51,9 +51,9 @@ static void io_eventfd_do_signal(struct rcu_head *rcu) /* * Returns true if the caller should put the ev_fd reference, false if not. */ -static bool __io_eventfd_signal(struct io_ev_fd *ev_fd) +static bool __io_eventfd_signal(struct io_ev_fd *ev_fd, bool defer) { - if (eventfd_signal_allowed()) { + if (!defer && eventfd_signal_allowed()) { eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); return true; } @@ -73,7 +73,7 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd) return !ev_fd->eventfd_async || io_wq_current_is_worker(); } -void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) +void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer) { bool skip = false; struct io_ev_fd *ev_fd; @@ -113,7 +113,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) spin_unlock(&ctx->completion_lock); } - if (skip || __io_eventfd_signal(ev_fd)) + if (skip || __io_eventfd_signal(ev_fd, defer)) io_eventfd_put(ev_fd); } diff --git a/io_uring/eventfd.h b/io_uring/eventfd.h index 400eda4a4165a..e965d80d9fdc6 100644 --- a/io_uring/eventfd.h +++ b/io_uring/eventfd.h @@ -5,4 +5,4 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, unsigned int eventfd_async); int io_eventfd_unregister(struct io_ring_ctx *ctx); -void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event); +void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer); diff --git a/io_uring/futex.c b/io_uring/futex.c index 906701b3c5c66..eaee14242a3a3 100644 --- a/io_uring/futex.c +++ b/io_uring/futex.c @@ -149,8 +149,21 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) !futex_validate_input(iof->futex_flags, iof->futex_mask)) return -EINVAL; - /* Mark as inflight, so file exit cancelation will find it */ - io_req_track_inflight(req); + return 0; +} + +int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +{ + struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); + int ret; + + ret = io_futex_prep(req, sqe); + if (unlikely(ret)) + return ret; + + /* inflight tracking only needed for mm private hash */ + if (!(iof->futex_flags & FLAGS_SHARED)) + io_req_track_inflight(req); return 0; } @@ -168,13 +181,14 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) io_req_set_res(req, 0, 0); req->io_task_work.func = io_futexv_complete; - io_req_task_work_add(req); + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); } int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); struct io_futexv_data *ifd; + unsigned int i; int ret; /* No flags or mask supported for waitv */ @@ -199,8 +213,14 @@ int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return ret; } - /* Mark as inflight, so file exit cancelation will find it */ - io_req_track_inflight(req); + /* inflight tracking only needed for mm private hash */ + for (i = 0; i < iof->futex_nr; i++) { + if (!(ifd->futexv[i].w.flags & FLAGS_SHARED)) { + io_req_track_inflight(req); + break; + } + } + iof->futexv_unqueued = 0; req->flags |= REQ_F_ASYNC_DATA; req->async_data = ifd; @@ -217,7 +237,7 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) io_req_set_res(req, 0, 0); req->io_task_work.func = io_futex_complete; - io_req_task_work_add(req); + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); } int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) diff --git a/io_uring/futex.h b/io_uring/futex.h index d789fcf715e38..987db3f2c6d9a 100644 --- a/io_uring/futex.h +++ b/io_uring/futex.h @@ -3,6 +3,7 @@ #include "cancel.h" int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); +int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags); int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags); diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 2e14880eef92a..fa403ed245962 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -211,9 +211,12 @@ static void io_worker_cancel_cb(struct io_worker *worker) struct io_wq *wq = worker->wq; atomic_dec(&acct->nr_running); - raw_spin_lock(&acct->workers_lock); - acct->nr_workers--; - raw_spin_unlock(&acct->workers_lock); + /* create_worker_cb() has not reserved a worker slot yet. */ + if (worker->create_work.func != create_worker_cb) { + raw_spin_lock(&acct->workers_lock); + acct->nr_workers--; + raw_spin_unlock(&acct->workers_lock); + } io_worker_ref_put(wq); clear_bit_unlock(0, &worker->create_state); io_worker_release(worker); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1ea2fca34a36f..6b7db9ab681b6 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -484,7 +484,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED) io_flush_timeouts(ctx); if (ctx->int_flags & IO_RING_F_HAS_EVFD) - io_eventfd_signal(ctx, true); + io_eventfd_signal(ctx, true, false); } static inline void __io_cq_lock(struct io_ring_ctx *ctx) diff --git a/io_uring/net.c b/io_uring/net.c index a74d15f7b7d2b..e6847fb084abc 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -884,7 +884,7 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static inline bool io_recv_finish(struct io_kiocb *req, struct io_async_msghdr *kmsg, struct io_br_sel *sel, bool mshot_finished, - unsigned issue_flags) + unsigned issue_flags, int consumed) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); unsigned int cflags = 0; @@ -908,7 +908,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, if (sr->flags & IORING_RECVSEND_BUNDLE) { size_t this_ret = sel->val - sr->done_io; - cflags |= io_put_kbufs(req, this_ret, sel->buf_list, io_bundle_nbufs(kmsg, this_ret)); + cflags |= io_put_kbufs(req, consumed, sel->buf_list, io_bundle_nbufs(kmsg, consumed)); if (sr->flags & IORING_RECV_RETRY) cflags = req->cqe.flags | (cflags & CQE_F_MASK); if (sr->mshot_len && sel->val >= sr->mshot_len) @@ -930,7 +930,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, return false; } } else { - cflags |= io_put_kbuf(req, sel->val, sel->buf_list); + cflags |= io_put_kbuf(req, consumed, sel->buf_list); } /* @@ -1058,6 +1058,8 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) int ret, min_ret = 0; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; bool mshot_finished = true; + int consumed = 0; + size_t len; sock = sock_from_file(req->file); if (unlikely(!sock)) @@ -1073,9 +1075,8 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) retry_multishot: sel.buf_list = NULL; + len = sr->len; if (io_do_buffer_select(req)) { - size_t len = sr->len; - sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); if (!sel.addr) return -ENOBUFS; @@ -1096,6 +1097,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) if (req->flags & REQ_F_APOLL_MULTISHOT) { ret = io_recvmsg_multishot(sock, sr, kmsg, flags, &mshot_finished); + consumed = ret; } else { /* disable partial retry for recvmsg with cmsg attached */ if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen) @@ -1103,6 +1105,15 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags); + /* + * With MSG_TRUNC, the net layer will return the full size of + * the packet, even if we only filled part of it in the buffers. + * Adjust the returned size to consume only the real part of the + * buffer. + */ + consumed = ret; + if (ret > 0) + consumed = min_t(size_t, ret, len); } if (ret < min_ret) { @@ -1129,7 +1140,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) io_kbuf_recycle(req, sel.buf_list, issue_flags); sel.val = ret; - if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags)) + if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed)) goto retry_multishot; return sel.val; @@ -1139,6 +1150,7 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg struct io_br_sel *sel, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); + size_t len; int ret; /* @@ -1184,13 +1196,14 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg /* special case 1 vec, can be a fast path */ if (ret == 1) { sr->buf = arg.iovs[0].iov_base; - sr->len = arg.iovs[0].iov_len; + len = sr->len = arg.iovs[0].iov_len; goto map_ubuf; } iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret, - arg.out_len); + arg.out_len); + len = arg.out_len; } else { - size_t len = sel->val; + len = sel->val; *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); if (!sel->addr) @@ -1204,7 +1217,7 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg return ret; } - return 0; + return len; } int io_recv(struct io_kiocb *req, unsigned int issue_flags) @@ -1214,9 +1227,10 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) struct io_br_sel sel; struct socket *sock; unsigned flags; - int ret, min_ret = 0; + int ret, min_ret = 0, consumed = 0; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; bool mshot_finished; + size_t len = 0; sock = sock_from_file(req->file); if (unlikely(!sock)) @@ -1244,6 +1258,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) retry_multishot: sel.buf_list = NULL; + len = sr->len; if (io_do_buffer_select(req)) { sel.val = sr->len; ret = io_recv_buf_select(req, kmsg, &sel, issue_flags); @@ -1251,6 +1266,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) kmsg->msg.msg_inq = -1; goto out_free; } + len = ret; sr->buf = NULL; } @@ -1281,6 +1297,17 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) } mshot_finished = ret <= 0; + + /* + * With MSG_TRUNC, the net layer will return the full size of + * the packet, even if we only filled part of it in the buffers. + * Adjust the returned size to consume only the real part of the + * buffer. + */ + consumed = ret; + if (ret > 0) + consumed = min_t(size_t, ret, len); + if (ret > 0) ret += sr->done_io; else if (sr->done_io) @@ -1289,7 +1316,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) io_kbuf_recycle(req, sel.buf_list, issue_flags); sel.val = ret; - if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags)) + if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed)) goto retry_multishot; return sel.val; diff --git a/io_uring/opdef.c b/io_uring/opdef.c index 4e58eb1344eaa..cf3aa2242cd76 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -467,7 +467,7 @@ const struct io_issue_def io_issue_defs[] = { }, [IORING_OP_FUTEX_WAIT] = { #if defined(CONFIG_FUTEX) - .prep = io_futex_prep, + .prep = io_futex_wait_prep, .issue = io_futex_wait, #else .prep = io_eopnotsupp_prep, diff --git a/io_uring/poll.c b/io_uring/poll.c index 0204affdc308c..5447a7c24dce6 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -208,9 +208,9 @@ enum { IOU_POLL_REQUEUE = 4, }; -static void __io_poll_execute(struct io_kiocb *req, int mask) +static void __io_poll_execute(struct io_kiocb *req, int mask, unsigned tw_flags) { - unsigned flags = 0; + unsigned flags = tw_flags; io_req_set_res(req, mask, 0); req->io_task_work.func = io_poll_task_func; @@ -218,14 +218,15 @@ static void __io_poll_execute(struct io_kiocb *req, int mask) trace_io_uring_task_add(req, mask); if (!(req->flags & REQ_F_POLL_NO_LAZY)) - flags = IOU_F_TWQ_LAZY_WAKE; + flags |= IOU_F_TWQ_LAZY_WAKE; __io_req_task_work_add(req, flags); } -static inline void io_poll_execute(struct io_kiocb *req, int res) +static inline void io_poll_execute(struct io_kiocb *req, int res, + unsigned tw_flags) { if (io_poll_get_ownership(req)) - __io_poll_execute(req, res); + __io_poll_execute(req, res, tw_flags); } /* @@ -344,7 +345,7 @@ void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw) if (ret == IOU_POLL_NO_ACTION) { return; } else if (ret == IOU_POLL_REQUEUE) { - __io_poll_execute(req, 0); + __io_poll_execute(req, 0, 0); return; } io_poll_remove_entries(req); @@ -383,7 +384,7 @@ static void io_poll_cancel_req(struct io_kiocb *req) { io_poll_mark_cancelled(req); /* kick tw, which should complete the request */ - io_poll_execute(req, 0); + io_poll_execute(req, 0, 0); } #define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI) @@ -392,7 +393,7 @@ static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) { io_poll_mark_cancelled(req); /* we have to kick tw in case it's not already */ - io_poll_execute(req, 0); + io_poll_execute(req, 0, IOU_F_TWQ_IN_WAKE); io_poll_remove_waitq(poll); return 1; } @@ -430,7 +431,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, else req->flags &= ~REQ_F_SINGLE_POLL; } - __io_poll_execute(req, mask); + __io_poll_execute(req, mask, IOU_F_TWQ_IN_WAKE); } return 1; } @@ -618,7 +619,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, if (mask && (poll->events & EPOLLET) && io_poll_can_finish_inline(req, ipt)) { - __io_poll_execute(req, mask); + __io_poll_execute(req, mask, 0); return 0; } io_napi_add(req); @@ -629,7 +630,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, * poll was waken up, queue up a tw, it'll deal with it. */ if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) - __io_poll_execute(req, 0); + __io_poll_execute(req, 0, 0); } return 0; } diff --git a/io_uring/query.c b/io_uring/query.c index 88a3257369924..f2b5e19e5af67 100644 --- a/io_uring/query.c +++ b/io_uring/query.c @@ -76,6 +76,9 @@ static int io_handle_query_entry(union io_query_data *data, void __user *uhdr, if (copy_from_user(&hdr, uhdr, sizeof(hdr))) return -EFAULT; + /* copy_struct_to_user() zeros up to usize bytes */ + if (hdr.size > PAGE_SIZE) + return -E2BIG; usize = hdr.size; hdr.size = min(hdr.size, IO_MAX_QUERY_SIZE); udata = u64_to_user_ptr(hdr.query_data); diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 98dccefd801b9..174f74cbbf601 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1477,7 +1477,7 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter, struct iovec *iovec, unsigned nr_iovs, struct iou_vec *vec) { - unsigned long folio_size = 1 << imu->folio_shift; + unsigned long folio_size = 1UL << imu->folio_shift; unsigned long folio_mask = folio_size - 1; struct bio_vec *res_bvec = vec->bvec; size_t total_len = 0; diff --git a/io_uring/rw.c b/io_uring/rw.c index 95038cfda6153..45a96e6546fcc 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -531,20 +531,25 @@ static void io_req_end_write(struct io_kiocb *req) } } -/* - * Trigger the notifications after having done some IO, and finish the write - * accounting, if any. - */ -static void io_req_io_end(struct io_kiocb *req) +/* Trigger the notifications after having done some IO. */ +static void io_req_io_notify(struct io_kiocb *req) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); - if (rw->kiocb.ki_flags & IOCB_WRITE) { - io_req_end_write(req); + if (rw->kiocb.ki_flags & IOCB_WRITE) fsnotify_modify(req->file); - } else { + else fsnotify_access(req->file); - } +} + +/* Finish write accounting and notify, for inline completions only. */ +static void io_req_io_end(struct io_kiocb *req) +{ + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); + + if (rw->kiocb.ki_flags & IOCB_WRITE) + io_req_end_write(req); + io_req_io_notify(req); } static void __io_complete_rw_common(struct io_kiocb *req, long res) @@ -577,7 +582,7 @@ void io_req_rw_complete(struct io_tw_req tw_req, io_tw_token_t tw) { struct io_kiocb *req = tw_req.req; - io_req_io_end(req); + io_req_io_notify(req); if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) req->cqe.flags |= io_put_kbuf(req, max(req->cqe.res, 0), NULL); @@ -591,6 +596,10 @@ static void io_complete_rw(struct kiocb *kiocb, long res) struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); struct io_kiocb *req = cmd_to_io_kiocb(rw); + /* ring owner may block in freeze_super() before task_work runs */ + if (kiocb->ki_flags & IOCB_WRITE) + io_req_end_write(req); + __io_complete_rw_common(req, res); io_req_set_res(req, io_fixup_rw_res(req, res), 0); req->io_task_work.func = io_req_rw_complete; diff --git a/io_uring/tw.c b/io_uring/tw.c index e6ee15571e85c..f573bcc3af6a6 100644 --- a/io_uring/tw.c +++ b/io_uring/tw.c @@ -173,7 +173,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) if (mpscq_push(&ctx->work_list, &req->io_task_work.node)) { io_ctx_mark_taskrun(ctx); if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD) - io_eventfd_signal(ctx, false); + io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE); } /* diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index c14c22cff49e3..7c1337f29a5e2 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -38,6 +38,8 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags) if (io_alloc_cache_put(&req->ctx->cmd_cache, ac)) { ioucmd->sqe = NULL; io_req_async_data_clear(req, REQ_F_NEED_CLEANUP); + } else { + io_vec_free(&ac->vec); } } @@ -208,6 +210,8 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ac = io_uring_alloc_async_data(&req->ctx->cmd_cache, req); if (!ac) return -ENOMEM; + if (ac->vec.iovec) + req->flags |= REQ_F_NEED_CLEANUP; ioucmd->sqe = sqe; return 0; } @@ -269,10 +273,6 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) } ret = file->f_op->uring_cmd(ioucmd, issue_flags); - if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) { - if (ret >= 0) - return IOU_ISSUE_SKIP_COMPLETE; - } if (ret == -EAGAIN) { ioucmd->flags |= IORING_URING_CMD_REISSUE; return ret; diff --git a/io_uring/waitid.c b/io_uring/waitid.c index 32f68fd7fcdda..2b756017423f0 100644 --- a/io_uring/waitid.c +++ b/io_uring/waitid.c @@ -125,7 +125,7 @@ static void io_waitid_remove_wq(struct io_kiocb *req) } } -static void io_waitid_complete(struct io_kiocb *req, int ret) +static void io_waitid_complete(struct io_kiocb *req, int ret, bool copy_si) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); @@ -137,13 +137,16 @@ static void io_waitid_complete(struct io_kiocb *req, int ret) hlist_del_init(&req->hash_node); io_waitid_remove_wq(req); - ret = io_waitid_finish(req, ret); + if (copy_si) + ret = io_waitid_finish(req, ret); + else + io_waitid_free(req); if (ret < 0) req_set_fail(req); io_req_set_res(req, ret, 0); } -static bool __io_waitid_cancel(struct io_kiocb *req) +static bool __io_waitid_cancel(struct io_kiocb *req, bool copy_si) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); @@ -159,21 +162,32 @@ static bool __io_waitid_cancel(struct io_kiocb *req) if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK) return false; - io_waitid_complete(req, -ECANCELED); + io_waitid_complete(req, -ECANCELED, copy_si); io_req_queue_tw_complete(req, -ECANCELED); return true; } +static bool io_waitid_cancel_cb(struct io_kiocb *req) +{ + return __io_waitid_cancel(req, true); +} + +static bool io_waitid_cancel_nocopy_cb(struct io_kiocb *req) +{ + return __io_waitid_cancel(req, false); +} + int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, unsigned int issue_flags) { - return io_cancel_remove(ctx, cd, issue_flags, &ctx->waitid_list, __io_waitid_cancel); + return io_cancel_remove(ctx, cd, issue_flags, &ctx->waitid_list, io_waitid_cancel_cb); } bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all) { - return io_cancel_remove_all(ctx, tctx, &ctx->waitid_list, cancel_all, __io_waitid_cancel); + return io_cancel_remove_all(ctx, tctx, &ctx->waitid_list, cancel_all, + tctx ? io_waitid_cancel_cb : io_waitid_cancel_nocopy_cb); } static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req) @@ -202,6 +216,11 @@ static void io_waitid_cb(struct io_tw_req tw_req, io_tw_token_t tw) int ret; io_tw_lock(ctx, tw); + if (unlikely(tw.cancel)) { + io_waitid_complete(req, -ECANCELED, false); + io_req_task_complete(tw_req, tw); + return; + } ret = __do_wait(&iwa->wo); @@ -229,7 +248,7 @@ static void io_waitid_cb(struct io_tw_req tw_req, io_tw_token_t tw) } } - io_waitid_complete(req, ret); + io_waitid_complete(req, ret, true); io_req_task_complete(tw_req, tw); } @@ -253,7 +272,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, return 1; req->io_task_work.func = io_waitid_cb; - io_req_task_work_add(req); + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); return 1; } diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 88c594c6d7fcd..f95859e4e3f09 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -169,6 +169,7 @@ config SCHED_CORE config SCHED_CLASS_EXT bool "Extensible Scheduling Class" depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF + select GENERIC_ALLOCATOR select STACKTRACE if STACKTRACE_SUPPORT help This option enables a new scheduler class sched_ext (SCX), which diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index 7f791afe57917..f526456252148 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c @@ -1023,7 +1023,6 @@ static inline int audit_add_rule(struct audit_entry *entry) int audit_del_rule(struct audit_entry *entry) { struct audit_entry *e; - struct audit_tree *tree = entry->rule.tree; struct list_head *list; int ret = 0; #ifdef CONFIG_AUDITSYSCALL @@ -1071,9 +1070,6 @@ int audit_del_rule(struct audit_entry *entry) out: mutex_unlock(&audit_filter_mutex); - if (tree) - audit_put_tree(tree); /* that's the temporary one */ - return ret; } @@ -1158,6 +1154,8 @@ int audit_rule_change(int type, int seq, void *data, size_t datasz) } if (err || type == AUDIT_DEL_RULE) { + if (type == AUDIT_DEL_RULE && entry->rule.tree) + audit_put_tree(entry->rule.tree); if (entry->rule.exe) audit_remove_mark(entry->rule.exe); audit_free_rule(entry); diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 80b7b8a694464..529c0f6d7d0a8 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -484,8 +484,12 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) kaddr = kbase + (u32)(vmf->address); if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) - /* Make a reasonable effort to address impossible case */ - return VM_FAULT_RETRY; + /* + * A failed lock means a possible deadlock was detected. Don't + * return VM_FAULT_RETRY: this handler never took mmap_lock, but + * the fault path would re-take it on retry and deadlock. Fail. + */ + return VM_FAULT_SIGBUS; page = vmalloc_to_page((void *)kaddr); if (page) { @@ -853,6 +857,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, uaddr &= PAGE_MASK; kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); if (full_uaddr >= uaddr_end) return; diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 248b4818178cd..cc3f8c25a28b5 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -405,7 +405,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, void __percpu *pptr; void *ptr, *val; u32 size; - int cpu; + int cpu, off = 0; if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)) /* unknown flags */ @@ -437,9 +437,10 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, } for_each_possible_cpu(cpu) { ptr = per_cpu_ptr(pptr, cpu); - val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off; copy_map_value(map, ptr, val); bpf_obj_cancel_fields(map, ptr); + off += size; } unlock: rcu_read_unlock(); diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 2e4ae0ef08609..53a1e52d73c5a 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -521,37 +521,34 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -EFAULT; } } else if (opcode == BPF_EXIT) { - bool r0_precise; + bool from_subprog_call, r0_precise; + + /* BPF_EXIT in subprog or callback always returns + * right after the call instruction, so by checking + * whether the instruction at subseq_idx-1 is subprog + * call or not we can distinguish actual exit from + * *subprog* from exit from *callback*. In the former + * case, we need to propagate r0 precision, if + * necessary. In the former we never do that. + */ + from_subprog_call = subseq_idx - 1 >= 0 && + bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + + r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); /* Backtracking to a nested function call, 'idx' is a part of * the inner frame 'subseq_idx' is a part of the outer frame. * In case of a regular function call, instructions giving * precision to registers R1-R5 should have been found already. - * In case of a callback, it is ok to have R1-R5 marked for - * backtracking, as these registers are set by the function - * invoking callback. + * In case of a callback from bpf_loop(), R{1,4} in the calling + * frame would be set as precise and that is correct. */ - if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) - for (i = BPF_REG_1; i <= BPF_REG_5; i++) - bt_clear_reg(bt, i); - if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { + if (from_subprog_call && (bt_reg_mask(bt) & BPF_REGMASK_ARGS)) { verifier_bug(env, "backtracking exit unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } - /* BPF_EXIT in subprog or callback always returns - * right after the call instruction, so by checking - * whether the instruction at subseq_idx-1 is subprog - * call or not we can distinguish actual exit from - * *subprog* from exit from *callback*. In the former - * case, we need to propagate r0 precision, if - * necessary. In the former we never do that. - */ - r0_precise = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && - bt_is_reg_set(bt, BPF_REG_0); - bt_clear_reg(bt, BPF_REG_0); if (bt_subprog_enter(bt)) return -EFAULT; @@ -583,16 +580,29 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ } } else if (class == BPF_LD) { - if (!bt_is_reg_set(bt, dreg)) - return 0; - bt_clear_reg(bt, dreg); /* It's ld_imm64 or ld_abs or ld_ind. * For ld_imm64 no further tracking of precision * into parent is necessary */ - if (mode == BPF_IND || mode == BPF_ABS) - /* to be analyzed */ - return -ENOTSUPP; + if (mode == BPF_IMM) { + bt_clear_reg(bt, dreg); + return 0; + } + /* + * BPF_{IND,ABS} are modelled as two branches: + * - fallthrough; + * - implicit subprogram exit. + * It is necessary to switch current frame if + * implicit subprogram exit branch is backtracked. + */ + if (mode == BPF_IND || mode == BPF_ABS) { + if (bt_is_reg_set(bt, dreg)) + return -ENOTSUPP; + if (subseq_idx != idx + 1) + if (bt_subprog_enter(bt)) + return -EFAULT; + return 0; + } } /* Propagate precision marks to linked registers, to account for * registers marked as precise in this function. diff --git a/kernel/bpf/bloom_filter.c b/kernel/bpf/bloom_filter.c index b73336c976b7c..c6e7559b07de0 100644 --- a/kernel/bpf/bloom_filter.c +++ b/kernel/bpf/bloom_filter.c @@ -41,7 +41,7 @@ static long bloom_map_peek_elem(struct bpf_map *map, void *value) for (i = 0; i < bloom->nr_hash_funcs; i++) { h = hash(bloom, value, map->value_size, i); - if (!test_bit(h, bloom->bitset)) + if (!test_bit(h % BITS_PER_LONG, bloom->bitset + BIT_WORD(h))) return -ENOENT; } @@ -57,9 +57,13 @@ static long bloom_map_push_elem(struct bpf_map *map, void *value, u64 flags) if (flags != BPF_ANY) return -EINVAL; + /* + * On 32-bit architectures, hashes larger than INT_MAX would be + * treated as negative by set_bit(). + */ for (i = 0; i < bloom->nr_hash_funcs; i++) { h = hash(bloom, value, map->value_size, i); - set_bit(h, bloom->bitset); + set_bit(h % BITS_PER_LONG, bloom->bitset + BIT_WORD(h)); } return 0; @@ -94,9 +98,10 @@ static int bloom_map_alloc_check(union bpf_attr *attr) static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) { - u32 bitset_bytes, bitset_mask, nr_hash_funcs, nr_bits; + u32 bitset_mask, nr_hash_funcs, nr_bits; int numa_node = bpf_map_attr_numa_node(attr); struct bpf_bloom_filter *bloom; + u64 bitset_bytes; if (attr->key_size != 0 || attr->value_size == 0 || attr->max_entries == 0 || @@ -127,22 +132,16 @@ static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) if (check_mul_overflow(attr->max_entries, nr_hash_funcs, &nr_bits) || check_mul_overflow(nr_bits / 5, (u32)7, &nr_bits) || nr_bits > (1UL << 31)) { - /* The bit array size is 2^32 bits but to avoid overflowing the - * u32, we use U32_MAX, which will round up to the equivalent - * number of bytes - */ - bitset_bytes = BITS_TO_BYTES(U32_MAX); bitset_mask = U32_MAX; } else { if (nr_bits <= BITS_PER_LONG) nr_bits = BITS_PER_LONG; else nr_bits = roundup_pow_of_two(nr_bits); - bitset_bytes = BITS_TO_BYTES(nr_bits); bitset_mask = nr_bits - 1; } - bitset_bytes = roundup(bitset_bytes, sizeof(unsigned long)); + bitset_bytes = BITS_TO_LONGS((u64)bitset_mask + 1) * sizeof(unsigned long); bloom = bpf_map_area_alloc(sizeof(*bloom) + bitset_bytes, numa_node); if (!bloom) diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index f5eaeb2493d4a..9b4f5236c5ea3 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -754,7 +754,7 @@ const struct bpf_func_proto bpf_loop_proto = { .func = bpf_loop, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_ANYTHING, + .arg1_type = ARG_SCALAR, .arg2_type = ARG_PTR_TO_FUNC, .arg3_type = ARG_PTR_TO_STACK_OR_NULL, .arg4_type = ARG_ANYTHING, diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 1433809bb166a..3983b4ce73c81 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -295,7 +295,6 @@ BTF_ID(func, bpf_lsm_bpf_map_create) BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) -BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_bpf_token_create) BTF_ID(func, bpf_lsm_bpf_token_free) BTF_ID(func, bpf_lsm_bpf_token_cmd) diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 51b16e5f5534e..4e7a48c02be5c 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -445,6 +445,18 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, goto errout; } + /* + * A >8 byte return value is passed back in a register pair, + * which the struct_ops trampoline does not preserve (only + * 8 bytes of the return value are saved and restored). + */ + if (st_ops->func_models[i].ret_size > 8) { + pr_warn("func ptr %s in struct %s has a >8 byte return value, which is not supported\n", + mname, st_ops->name); + err = -EOPNOTSUPP; + goto errout; + } + stub_func_addr = *(void **)(st_ops->cfi_stubs + moff); err = prepare_arg_info(btf, st_ops->name, mname, func_proto, stub_func_addr, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index c4673a54c4baf..c4df013bf0066 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -2912,14 +2912,29 @@ static void btf_modifier_show(const struct btf *btf, else t = btf_type_skip_modifiers(btf, type_id, NULL); - btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); + /* + * A modifier can resolve to void, which has no show op; print a + * placeholder rather than dereferencing NULL. + */ + if (!btf_type_ops(t)) + btf_df_show(btf, t, type_id, data, bits_offset, show); + else + btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); } static void btf_var_show(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offset, struct btf_show *show) { - t = btf_type_id_resolve(btf, &type_id); + /* + * btf_type_id_resolve() dereferences btf->resolved_ids, which is NULL + * for a base BTF (e.g. the vmlinux BTF that bpf_snprintf_btf() uses). + * Resolve the var's type directly in that case. + */ + if (btf->resolved_ids) + t = btf_type_id_resolve(btf, &type_id); + else + t = btf_type_skip_modifiers(btf, t->type, &type_id); btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); } @@ -3669,7 +3684,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, { BPF_RB_NODE, "bpf_rb_node", false }, - { BPF_REFCOUNT, "bpf_refcount", false }, + { BPF_REFCOUNT, "bpf_refcount", true }, }; int type = 0, i; const char *name = __btf_name_by_offset(btf, var_type->name_off); @@ -3751,7 +3766,7 @@ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level); + u32 level, u32 *seen_mask); /* Find special fields in the struct type of a field. * @@ -3762,7 +3777,7 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_nested_struct(const struct btf *btf, const struct btf_type *t, u32 off, u32 nelems, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, err, i; @@ -3770,7 +3785,7 @@ static int btf_find_nested_struct(const struct btf *btf, const struct btf_type * if (level >= MAX_RESOLVE_DEPTH) return -E2BIG; - ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level); + ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level, seen_mask); if (ret <= 0) return ret; @@ -3827,7 +3842,7 @@ static int btf_find_field_one(const struct btf *btf, if (expected_size && expected_size != sz * nelems) return 0; ret = btf_find_nested_struct(btf, var_type, off, nelems, field_mask, - &info[0], info_cnt, level); + &info[0], info_cnt, level, seen_mask); return ret; } @@ -3892,11 +3907,11 @@ static int btf_find_field_one(const struct btf *btf, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level) + u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_member *member; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_member(i, t, member) { const struct btf_type *member_type = btf_type_by_id(btf, @@ -3910,7 +3925,7 @@ static int btf_find_struct_field(const struct btf *btf, ret = btf_find_field_one(btf, t, member_type, i, off, 0, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) return ret; @@ -3921,11 +3936,11 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_var_secinfo *vsi; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_vsi(i, t, vsi) { const struct btf_type *var = btf_type_by_id(btf, vsi->type); @@ -3933,7 +3948,7 @@ static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, off = vsi->offset; ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) @@ -3947,10 +3962,12 @@ static int btf_find_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt) { + u32 seen_mask = 0; + if (__btf_type_is_struct(t)) - return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0); + return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0, &seen_mask); else if (btf_type_is_datasec(t)) - return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0); + return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0, &seen_mask); return -EINVAL; } @@ -4168,7 +4185,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type rec->spin_lock_off = rec->fields[i].offset; break; case BPF_RES_SPIN_LOCK: - WARN_ON_ONCE(rec->spin_lock_off >= 0); + WARN_ON_ONCE(rec->res_spin_lock_off >= 0); /* Cache offset for faster lookup at runtime */ rec->res_spin_lock_off = rec->fields[i].offset; break; @@ -4252,13 +4269,10 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) { int i; - /* There are three types that signify ownership of some other type: - * kptr_ref, bpf_list_head, bpf_rb_root. - * kptr_ref only supports storing kernel types, which can't store - * references to program allocated local types. - * - * Hence we only need to ensure that bpf_{list_head,rb_root} ownership - * does not form cycles. + /* + * Check fields which require the complete BTF and initialize runtime + * metadata. Ownership relationships are validated after every record has + * been fixed up. */ if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & (BPF_GRAPH_ROOT | BPF_UPTR))) return 0; @@ -4289,51 +4303,88 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) if (!meta) return -EFAULT; rec->fields[i].graph_root.value_rec = meta->record; + } + return 0; +} - /* We need to set value_rec for all root types, but no need - * to check ownership cycle for a type unless it's also a - * node type. - */ - if (!(rec->field_mask & BPF_GRAPH_NODE)) +static int btf_owned_type_idx(const struct btf *btf, struct btf_struct_metas *tab, + const struct btf_field *field) +{ + struct btf_struct_meta *meta; + u32 btf_id; + + if (field->type & BPF_GRAPH_ROOT) { + btf_id = field->graph_root.value_btf_id; + } else if (field->type == BPF_KPTR_REF || field->type == BPF_KPTR_PERCPU) { + if (btf_is_kernel(field->kptr.btf)) + return -ENOENT; + btf_id = field->kptr.btf_id; + } else { + return -ENOENT; + } + + meta = btf_find_struct_meta(btf, btf_id); + if (!meta) + return field->type & BPF_GRAPH_ROOT ? -EFAULT : -ENOENT; + return meta - tab->types; +} + +/* + * Each ownership edge adds kernel frames through bpf_obj_free_fields() and + * __bpf_obj_drop_impl(). Keep the bound deliberately small because object + * destruction can itself run below a BPF call chain. A final pointee without + * special fields is not present in the struct metadata table and adds only a + * non-recursing drop. + */ +#define BTF_MAX_OWNERSHIP_DEPTH 8 + +static int btf_ownership_depth(const struct btf *btf, + struct btf_struct_metas *tab, u8 *depth, + int idx, int depth_left) +{ + const struct btf_record *rec = tab->types[idx].record; + int i, ret, max_depth = 0; + + if (!depth_left) + return -ELOOP; + if (depth[idx]) + goto done; + + for (i = 0; i < rec->cnt; i++) { + ret = btf_owned_type_idx(btf, tab, &rec->fields[i]); + if (ret == -ENOENT) continue; + if (ret < 0) + return ret; + ret = btf_ownership_depth(btf, tab, depth, ret, depth_left - 1); + if (ret < 0) + return ret; + max_depth = max(max_depth, ret); + } + depth[idx] = max_depth + 1; +done: + return depth[idx] > depth_left ? -ELOOP : depth[idx]; +} - /* We need to ensure ownership acyclicity among all types. The - * proper way to do it would be to topologically sort all BTF - * IDs based on the ownership edges, since there can be multiple - * bpf_{list_head,rb_node} in a type. Instead, we use the - * following resaoning: - * - * - A type can only be owned by another type in user BTF if it - * has a bpf_{list,rb}_node. Let's call these node types. - * - A type can only _own_ another type in user BTF if it has a - * bpf_{list_head,rb_root}. Let's call these root types. - * - * We ensure that if a type is both a root and node, its - * element types cannot be root types. - * - * To ensure acyclicity: - * - * When A is an root type but not a node, its ownership - * chain can be: - * A -> B -> C - * Where: - * - A is an root, e.g. has bpf_rb_root. - * - B is both a root and node, e.g. has bpf_rb_node and - * bpf_list_head. - * - C is only an root, e.g. has bpf_list_node - * - * When A is both a root and node, some other type already - * owns it in the BTF domain, hence it can not own - * another root type through any of the ownership edges. - * A -> B - * Where: - * - A is both an root and node. - * - B is only an node. - */ - if (meta->record->field_mask & BPF_GRAPH_ROOT) - return -ELOOP; +static int btf_check_ownership_depth(const struct btf *btf, + struct btf_struct_metas *tab) +{ + u8 *depth; + int i, ret = 0; + + depth = kvcalloc(tab->cnt, sizeof(*depth), GFP_KERNEL | __GFP_NOWARN); + if (!depth) + return -ENOMEM; + + for (i = 0; i < tab->cnt; i++) { + ret = btf_ownership_depth(btf, tab, depth, i, + BTF_MAX_OWNERSHIP_DEPTH); + if (ret < 0) + break; + ret = 0; } - return 0; + kvfree(depth); + return ret; } static void __btf_struct_show(const struct btf *btf, const struct btf_type *t, @@ -6028,6 +6079,10 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, if (err < 0) goto errout_meta; } + + err = btf_check_ownership_depth(btf, struct_meta_tab); + if (err < 0) + goto errout_meta; } err = bpf_log_attr_finalize(attr_log, &env->log); @@ -6656,6 +6711,10 @@ struct bpf_raw_tp_null_args { static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { /* sched */ { "sched_pi_setprio", 0x10 }, + /* + * do_wait() passes NULL for wait4(-1) and waitid(P_ALL). + */ + { "sched_process_wait", 0x1 }, /* ... from sched_numa_pair_template event class */ { "sched_stick_numa", 0x100 }, { "sched_swap_numa", 0x100 }, @@ -6723,6 +6782,9 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { { "rxrpc_resend", 0x10 }, { "rxrpc_tq", 0x10 }, { "rxrpc_client", 0x1 }, + /* signal */ + { "signal_generate", 0x20 }, + { "signal_deliver", 0x20 }, /* skb */ {"kfree_skb", 0x1000}, /* sunrpc */ @@ -7108,7 +7170,7 @@ enum bpf_struct_walk_result { static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, const struct btf_type *t, int off, int size, u32 *next_btf_id, enum bpf_type_flag *flag, - const char **field_name) + const char **field_name, bool walk_flex_arrays) { u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; @@ -7135,11 +7197,14 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, *flag |= PTR_UNTRUSTED; if (off + size > t->size) { + struct btf_array *array_elem; + + if (!walk_flex_arrays) + goto error; + /* If the last element is a variable size array, we may * need to relax the rule. */ - struct btf_array *array_elem; - if (vlen == 0) goto error; @@ -7164,7 +7229,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, if (btf_type_is_int(t)) return WALK_SCALAR; - if (!btf_type_is_struct(t)) + if (!btf_type_is_struct(t) || !t->size) goto error; off = (off - moff) % t->size; @@ -7404,7 +7469,8 @@ int btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(btf, id); do { - err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, field_name); + err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, + field_name, !type_is_alloc(reg->type)); switch (err) { case WALK_PTR: @@ -7463,7 +7529,7 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict) + bool strict, bool walk_flex_arrays) { const struct btf_type *type; enum bpf_type_flag flag = 0; @@ -7482,7 +7548,8 @@ bool btf_struct_ids_match(struct bpf_verifier_log *log, type = btf_type_by_id(btf, id); if (!type) return false; - err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL); + err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL, + walk_flex_arrays); if (err != WALK_STRUCT) return false; @@ -8700,6 +8767,7 @@ BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .func = bpf_btf_find_by_name_kind, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE, diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 26d37066465f3..d5fc03f12969b 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -120,6 +120,7 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) /* mark branch target for state pruning */ mark_prune_point(env, w); mark_jmp_point(env, w); + mark_jump_target(env, w); } if (insn_state[w] == 0) { @@ -378,6 +379,7 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) } mark_jmp_point(env, w); + mark_jump_target(env, w); /* EXPLORED || DISCOVERED */ if (insn_state[w]) @@ -539,6 +541,7 @@ static int visit_insn(int t, struct bpf_verifier_env *env) mark_prune_point(env, t + off + 1); mark_jmp_point(env, t + off + 1); + mark_jump_target(env, t + off + 1); return ret; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 4355ccb78a9c6..44068bdd4f759 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -813,8 +813,10 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *old_prog = NULL; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; + struct bpf_cgroup_storage *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_prog *new_prog = prog ? : link->link.prog; enum cgroup_bpf_attach_type atype; + u32 old_flags, old_pl_flags; struct bpf_prog_list *pl; struct hlist_head *progs; int err; @@ -865,6 +867,8 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, if (pl) { old_prog = pl->prog; + old_pl_flags = pl->flags; + bpf_cgroup_storages_assign(old_storage, pl->storage); } else { pl = kmalloc_obj(*pl); if (!pl) { @@ -884,6 +888,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, pl->link = link; pl->flags = flags; bpf_cgroup_storages_assign(pl->storage, storage); + old_flags = cgrp->bpf.flags[atype]; cgrp->bpf.flags[atype] = saved_flags; if (type == BPF_LSM_CGROUP) { @@ -915,12 +920,15 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, if (old_prog) { pl->prog = old_prog; pl->link = NULL; + pl->flags = old_pl_flags; + bpf_cgroup_storages_assign(pl->storage, old_storage); } bpf_cgroup_storages_free(new_storage); if (!old_prog) { hlist_del(&pl->node); kfree(pl); } + cgrp->bpf.flags[atype] = old_flags; return err; } @@ -1018,6 +1026,20 @@ static void replace_effective_prog(struct cgroup *cgrp, } } +static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog, + struct bpf_prog *new_prog) +{ + enum bpf_cgroup_storage_type stype; + + for_each_cgroup_storage_type(stype) { + if (old_prog->aux->cgroup_storage[stype] != + new_prog->aux->cgroup_storage[stype]) + return false; + } + + return true; +} + /** * __cgroup_bpf_replace() - Replace link's program and propagate the change * to descendants @@ -1056,6 +1078,9 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, if (!found) return -ENOENT; + if (!cgroup_bpf_storages_compatible(link->link.prog, new_prog)) + return -EINVAL; + cgrp->bpf.revisions[atype] += 1; old_prog = xchg(&link->link.prog, new_prog); replace_effective_prog(cgrp, atype, pl); @@ -2235,7 +2260,7 @@ int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, if (ret < 0) return ret; - if (ctx.optlen > *optlen) + if (ctx.optlen > *optlen || ctx.optlen < 0) return -EFAULT; /* BPF programs can shrink the buffer, export the modifications. diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index b2a19acadb91e..4cf120c7b2cb4 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -199,14 +199,9 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * ci_out[r] = unknown; break; case BPF_STX: - if (mode != BPF_ATOMIC) - break; - if (insn->imm == BPF_CMPXCHG) - ci_out[BPF_REG_0] = unknown; - else if (insn->imm == BPF_LOAD_ACQ) - *dst = unknown; - else if (insn->imm & BPF_FETCH) - *src = unknown; + r = bpf_atomic_load_reg(insn); + if (r >= 0) + ci_out[r] = unknown; break; } } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6e19a030da6f1..71ee96a77ae26 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -126,6 +127,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag fp->aux->main_prog_aux = aux; fp->aux->prog = fp; fp->jit_requested = ebpf_jit_enabled(); + fp->jit_required = IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON); fp->blinding_requested = bpf_jit_blinding_enabled(fp); #ifdef CONFIG_CGROUP_BPF aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID; @@ -1611,6 +1613,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp * fix it up here on error. */ bpf_jit_prog_release_other(prog, clone); + if (env && fatal_signal_pending(current)) + return ERR_PTR(-EINTR); return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM); } @@ -2640,11 +2644,14 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc orig_prog = prog; prog = bpf_jit_blind_constants(env, prog); /* - * If blinding was requested and we failed during blinding, we must fall - * back to the interpreter. + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(prog)) + if (IS_ERR(prog)) { + if (PTR_ERR(prog) == -EINTR) + return prog; goto out_restore; + } prog = bpf_int_jit_compile(env, prog); if (prog->jited) { @@ -2667,18 +2674,16 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, int *err) { + struct bpf_prog *jit_prog; + /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ - bool jit_needed = false; + bool jit_needed = fp->jit_required; if (fp->bpf_func) goto finalize; - if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) || - bpf_prog_has_kfunc_call(fp)) - jit_needed = true; - if (!bpf_prog_select_interpreter(fp)) jit_needed = true; @@ -2693,7 +2698,12 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct if (*err) return fp; - fp = bpf_prog_jit_compile(env, fp); + jit_prog = bpf_prog_jit_compile(env, fp); + if (IS_ERR(jit_prog)) { + *err = PTR_ERR(jit_prog); + return fp; + } + fp = jit_prog; bpf_prog_jit_attempt_done(fp); if (!fp->jited && jit_needed) { *err = -ENOTSUPP; diff --git a/kernel/bpf/cpumask.c b/kernel/bpf/cpumask.c index b8c805b4b06a0..1336a4efa7553 100644 --- a/kernel/bpf/cpumask.c +++ b/kernel/bpf/cpumask.c @@ -449,12 +449,12 @@ __bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask) * @src__sz: Length of the BPF memory region in bytes. * * Return: - * * 0 if the struct cpumask * instance was populated successfully. + * * 0 if the struct bpf_cpumask * instance was populated successfully. * * -EACCES if the memory region is too small to populate the cpumask. * * -EINVAL if the memory region is not aligned to the size of a long * and the architecture does not support efficient unaligned accesses. */ -__bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) +__bpf_kfunc int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) { unsigned long source = (unsigned long)src; @@ -467,7 +467,7 @@ __bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t !IS_ALIGNED(source, sizeof(long))) return -EINVAL; - bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids); + bitmap_copy(cpumask_bits(&cpumask->cpumask), src, nr_cpu_ids); return 0; } diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c index 51f89cecefb4d..3f3fe2450fc6c 100644 --- a/kernel/bpf/crypto.c +++ b/kernel/bpf/crypto.c @@ -149,8 +149,9 @@ bpf_crypto_ctx_create(const struct bpf_crypto_params *params, u32 params__sz, const struct bpf_crypto_type *type; struct bpf_crypto_ctx *ctx; - if (!params || params->reserved[0] || params->reserved[1] || - params__sz != sizeof(struct bpf_crypto_params)) { + if (!params || + params__sz != sizeof(struct bpf_crypto_params) || + params->reserved[0] || params->reserved[1]) { *err = -EINVAL; return NULL; } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 3cf2cc6e3ab66..19c1b8f1b317c 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -8,15 +8,21 @@ #include #include #include +#include #include #include "disasm.h" #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) +/* + * Matches BPF_PROBE_ATOMIC too: bpf_convert_ctx_accesses() rewrites arena + * atomics before bpf_opt_subreg_zext_lo32_rnd_hi32() runs. + */ static bool is_cmpxchg_insn(const struct bpf_insn *insn) { return BPF_CLASS(insn->code) == BPF_STX && - BPF_MODE(insn->code) == BPF_ATOMIC && + (BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && insn->imm == BPF_CMPXCHG; } @@ -29,16 +35,7 @@ static int insn_def_regno(const struct bpf_insn *insn) case BPF_ST: return -1; case BPF_STX: - if (BPF_MODE(insn->code) == BPF_ATOMIC || - BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { - if (insn->imm == BPF_CMPXCHG) - return BPF_REG_0; - else if (insn->imm == BPF_LOAD_ACQ) - return insn->dst_reg; - else if (insn->imm & BPF_FETCH) - return insn->src_reg; - } - return -1; + return bpf_atomic_load_reg(insn); default: return insn->dst_reg; } @@ -244,12 +241,28 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) } } +/* + * Some post-verification instruction rewriting passes require an + * O(prog->len) operation per instruction. Keep their shared primitives + * killable and preemptible. + */ +static bool bpf_rewrite_must_abort(void) +{ + if (fatal_signal_pending(current)) + return true; + cond_resched(); + return false; +} + struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len) { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + if (bpf_rewrite_must_abort()) + return NULL; + if (len > 1) { new_data = vrealloc(env->insn_aux_data, array_size(env->prog->len + len - 1, @@ -461,6 +474,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) unsigned int orig_prog_len = env->prog->len; int err; + if (bpf_rewrite_must_abort()) + return -EINTR; + if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); @@ -1320,7 +1336,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) } prog = bpf_jit_blind_constants(env, prog); if (IS_ERR(prog)) { - err = -ENOMEM; + err = PTR_ERR(prog); prog = orig_prog; goto out_restore; } @@ -1378,7 +1394,6 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) #ifndef CONFIG_BPF_JIT_ALWAYS_ON struct bpf_prog *prog = env->prog; struct bpf_insn *insn = prog->insnsi; - bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); int depth; #endif int i, err = 0; @@ -1400,12 +1415,12 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) err = bpf_jit_subprogs(env); if (err == 0) return 0; - if (err == -EFAULT) + if (err == -EFAULT || err == -EINTR) return err; } #ifndef CONFIG_BPF_JIT_ALWAYS_ON - if (has_kfunc_call) { - verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); + if (prog->jit_required) { + verbose(env, "program requires BPF JIT compiler but it is not available\n"); return -EINVAL; } for (i = 0; i < env->subprog_cnt; i++) { @@ -1841,8 +1856,10 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } /* Skip inlining the helper call if the JIT does it. */ - if (bpf_jit_inlines_helper_call(insn->imm)) + if (bpf_jit_inlines_helper_call(insn->imm)) { + prog->jit_required = 1; goto next_insn; + } if (insn->imm == BPF_FUNC_get_route_realm) prog->dst_needed = 1; @@ -2007,6 +2024,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) return -EFAULT; } + if (bpf_map_is_percpu_map(map_ptr->map_type)) + prog->jit_required = true; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -2111,6 +2131,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * way, it's fine to back out this inlining logic */ #ifdef CONFIG_SMP + prog->jit_required = true; insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); @@ -2132,6 +2153,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && bpf_verifier_inlines_helper_call(env, insn->imm)) { + prog->jit_required = true; insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 9f394e1aa2e85..61ef6191f79ad 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -530,6 +530,9 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf, { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); + if (btf_type_is_void(key_type)) + return -EINVAL; + if (htab_is_prealloc(htab)) return 0; /* @@ -1026,7 +1029,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, } else { u32 size = round_up(htab->map.value_size, 8); void *val; - int cpu; + int cpu, off = 0; if (map_flags & BPF_F_CPU) { cpu = map_flags >> 32; @@ -1038,9 +1041,10 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, for_each_possible_cpu(cpu) { ptr = per_cpu_ptr(pptr, cpu); - val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off; copy_map_value(&htab->map, ptr, val); bpf_obj_cancel_fields(&htab->map, ptr); + off += size; } } } @@ -1051,14 +1055,17 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, /* When not setting the initial value on all cpus, zero-fill element * values for other cpus. Otherwise, bpf program has no way to ensure * known initial values for cpus other than current one - * (onallcpus=false always when coming from bpf prog). + * (onallcpus=false always when coming from bpf prog, + * map_flags & BPF_F_CPU when coming from syscall but setting + * only one cpu). */ - if (!onallcpus) { - int current_cpu = raw_smp_processor_id(); + if (!onallcpus || (map_flags & BPF_F_CPU)) { + int init_cpu = (map_flags & BPF_F_CPU) ? map_flags >> 32 : + raw_smp_processor_id(); int cpu; for_each_possible_cpu(cpu) { - if (cpu == current_cpu) + if (cpu == init_cpu) copy_map_value(&htab->map, per_cpu_ptr(pptr, cpu), value); else /* Since elem is preallocated, we cannot touch special fields */ zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu)); @@ -1769,6 +1776,12 @@ static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, flags); } +/* + * Max consecutive empty buckets to walk in one RCU + + * instrumentation-disabled section before rescheduling. + */ +#define HTAB_BATCH_EMPTY_RESCHED 64 + static int __htab_map_lookup_and_delete_batch(struct bpf_map *map, const union bpf_attr *attr, @@ -1790,6 +1803,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, unsigned long flags = 0; bool locked = false; struct htab_elem *l; + u32 empty_cnt = 0; struct bucket *b; int ret = 0; @@ -1968,30 +1982,41 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, } next_batch: - /* If we are not copying data, we can go to next bucket and avoid - * unlocking the rcu. + /* + * If we are not copying data, we can go to next bucket and avoid + * unlocking the rcu. Bound the walk though: after + * HTAB_BATCH_EMPTY_RESCHED consecutive empty buckets, fully exit + * the critical section (no locks are held here) and reschedule. */ if (!bucket_cnt && (batch + 1 < htab->n_buckets)) { batch++; - goto again_nocopy; + if (++empty_cnt < HTAB_BATCH_EMPTY_RESCHED) + goto again_nocopy; + empty_cnt = 0; + rcu_read_unlock(); + bpf_enable_instrumentation(); + cond_resched_tasks_rcu_qs(); + goto again; } rcu_read_unlock(); bpf_enable_instrumentation(); - if (bucket_cnt && (copy_to_user(ukeys + total * key_size, keys, - key_size * bucket_cnt) || - copy_to_user(uvalues + total * value_size, values, - value_size * bucket_cnt))) { + if (bucket_cnt && (copy_to_user(ukeys + (size_t)total * key_size, keys, + (size_t)key_size * bucket_cnt) || + copy_to_user(uvalues + (size_t)total * value_size, values, + (size_t)value_size * bucket_cnt))) { ret = -EFAULT; goto after_loop; } total += bucket_cnt; + empty_cnt = 0; batch++; if (batch >= htab->n_buckets) { ret = -ENOENT; goto after_loop; } + cond_resched_tasks_rcu_qs(); goto again; after_loop: @@ -2865,16 +2890,6 @@ static int rhtab_map_alloc_check(union bpf_attr *attr) return htab_map_alloc_check(attr); } -static void rhtab_check_and_free_fields(struct bpf_rhtab *rhtab, - struct rhtab_elem *elem) -{ - if (IS_ERR_OR_NULL(rhtab->map.record)) - return; - - bpf_obj_free_fields(rhtab->map.record, - rhtab_elem_value(elem, rhtab->map.key_size)); -} - static void rhtab_mem_dtor(void *obj, void *ctx) { struct htab_btf_record *hrec = ctx; @@ -2964,8 +2979,8 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v rhtab_read_elem_value(&rhtab->map, copy, elem, flags); check_and_init_map_value(&rhtab->map, copy); } - /* Release internal structs: kptr, bpf_timer, task_work, wq */ - rhtab_check_and_free_fields(rhtab, elem); + bpf_obj_cancel_fields(&rhtab->map, + rhtab_elem_value(elem, rhtab->map.key_size)); bpf_mem_cache_free_rcu(&rhtab->ma, elem); return 0; } @@ -3007,7 +3022,6 @@ static int rhtab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, void static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *elem, void *value, u64 map_flags) { - struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); void *old_val = rhtab_elem_value(elem, map->key_size); if (map_flags & BPF_NOEXIST) @@ -3027,7 +3041,7 @@ static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *el * kptrs/etc. still sit in the slot. Cancel them after the copy * to match arraymap's update semantics. */ - rhtab_check_and_free_fields(rhtab, elem); + bpf_obj_cancel_fields(map, old_val); return 0; } @@ -3068,7 +3082,6 @@ static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u memcpy(elem->data, key, map->key_size); copy_map_value(map, rhtab_elem_value(elem, map->key_size), value); - check_and_init_map_value(map, rhtab_elem_value(elem, map->key_size)); /* Prevent deadlock for NMI programs attempting to take bucket lock */ bpf_disable_instrumentation(); @@ -3112,6 +3125,9 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf, { struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + if (btf_type_is_void(key_type)) + return -EINVAL; + return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor); } diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index c18f1e16edee4..61805d8c95aef 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3395,11 +3395,13 @@ __bpf_kfunc void bpf_throw(u64 cookie) WARN_ON_ONCE(!ctx.aux->exception_boundary); WARN_ON_ONCE(!ctx.bp); WARN_ON_ONCE(!ctx.cnt); - /* Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning + /* + * Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning * deeper stack depths than ctx.sp as we do not return from bpf_throw, - * which skips compiler generated instrumentation to do the same. + * which skips compiler generated instrumentation to do the same. Some + * architectures cannot recover sp while unwinding, so fall back to bp. */ - kasan_unpoison_task_stack_below((void *)(long)ctx.sp); + kasan_unpoison_task_stack_below((void *)(long)(ctx.sp ?: ctx.bp)); ctx.aux->bpf_exception_cb(cookie, ctx.sp + ctx.aux->stack_arg_sp_adjust, ctx.bp, 0, 0); WARN(1, "A call to BPF exception callback should never return\n"); } @@ -4388,7 +4390,7 @@ struct bpf_task_work_ctx { struct bpf_map *map; void *map_val; enum task_work_notify_mode mode; - bpf_task_work_callback_t callback_fn; + bpf_callback_t callback_fn; struct rcu_head rcu; } __aligned(8); @@ -4471,7 +4473,8 @@ static void bpf_task_work_callback(struct callback_head *cb) key = (void *)map_key_from_value(ctx->map, ctx->map_val, &idx); migrate_disable(); - ctx->callback_fn(ctx->map, key, ctx->map_val); + ctx->callback_fn((u64)(long)ctx->map, (u64)(long)key, + (u64)(long)ctx->map_val, 0, 0); migrate_enable(); bpf_task_work_ctx_reset(ctx); @@ -4594,7 +4597,7 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work } static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work *tw, - struct bpf_map *map, bpf_task_work_callback_t callback_fn, + struct bpf_map *map, void *callback_fn, struct bpf_prog_aux *aux, enum task_work_notify_mode mode) { struct bpf_prog *prog; @@ -4619,7 +4622,7 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work } ctx->task = task; - ctx->callback_fn = callback_fn; + ctx->callback_fn = (bpf_callback_t)callback_fn; ctx->prog = prog; ctx->mode = mode; ctx->map = map; diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 0aadfbae0acc5..da4df129f07d0 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1213,12 +1213,9 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, clear_stack_for_all_offs(insn, at_out, insn->dst_reg, at_stack_out, sz); - if (insn->imm == BPF_CMPXCHG) - at_out[BPF_REG_0] = none; - else if (insn->imm == BPF_LOAD_ACQ) - *dst = none; - else if (insn->imm & BPF_FETCH) - *src = none; + r = bpf_atomic_load_reg(insn); + if (r >= 0) + at_out[r] = none; } } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c index 23267213a17fb..83cd527a2542b 100644 --- a/kernel/bpf/local_storage.c +++ b/kernel/bpf/local_storage.c @@ -220,7 +220,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key, struct bpf_cgroup_storage *storage; void *val; u32 size; - int cpu; + int cpu, off = 0; if ((u32)map_flags & ~(BPF_ANY | BPF_EXIST | BPF_F_CPU | BPF_F_ALL_CPUS)) return -EINVAL; @@ -245,8 +245,9 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key, } size = round_up(_map->value_size, 8); for_each_possible_cpu(cpu) { - val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off; copy_map_value(_map, per_cpu_ptr(storage->percpu_buf, cpu), val); + off += size; } unlock: rcu_read_unlock(); diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index e9662db7198fe..8a8f088e83e6f 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -119,6 +119,7 @@ struct bpf_mem_cache { struct llist_head waiting_for_gp_ttrace; struct rcu_head rcu_ttrace; atomic_t call_rcu_ttrace_in_progress; + raw_spinlock_t lock; }; struct bpf_mem_caches { @@ -214,25 +215,24 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic) gfp = __GFP_NOWARN | __GFP_ACCOUNT; gfp |= atomic ? GFP_NOWAIT : GFP_KERNEL; - for (i = 0; i < cnt; i++) { - /* - * For every 'c' llist_del_first(&c->free_by_rcu_ttrace); is - * done only by one CPU == current CPU. Other CPUs might - * llist_add() and llist_del_all() in parallel. - */ - obj = llist_del_first(&c->free_by_rcu_ttrace); - if (!obj) - break; - add_obj_to_free_list(c, obj); - } - if (i >= cnt) - return; + /* + * c->lock serializes concurrent llist_del_first() against + * llist_del_all() in __free_rcu() and do_call_rcu_ttrace(). + */ + scoped_guard(raw_spinlock_irqsave, &c->lock) { + for (i = 0; i < cnt; i++) { + obj = llist_del_first(&c->free_by_rcu_ttrace); + if (!obj) + break; + add_obj_to_free_list(c, obj); + } - for (; i < cnt; i++) { - obj = llist_del_first(&c->waiting_for_gp_ttrace); - if (!obj) - break; - add_obj_to_free_list(c, obj); + for (; i < cnt; i++) { + obj = llist_del_first(&c->waiting_for_gp_ttrace); + if (!obj) + break; + add_obj_to_free_list(c, obj); + } } if (i >= cnt) return; @@ -279,8 +279,12 @@ static int free_all(struct bpf_mem_cache *c, struct llist_node *llnode, bool per static void __free_rcu(struct rcu_head *head) { struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace); + struct llist_node *llnode; + + scoped_guard(raw_spinlock_irqsave, &c->lock) + llnode = llist_del_all(&c->waiting_for_gp_ttrace); - free_all(c, llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size); + free_all(c, llnode, !!c->percpu_size); atomic_set(&c->call_rcu_ttrace_in_progress, 0); } @@ -300,7 +304,8 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c) if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { if (unlikely(READ_ONCE(c->draining))) { - llnode = llist_del_all(&c->free_by_rcu_ttrace); + scoped_guard(raw_spinlock_irqsave, &c->lock) + llnode = llist_del_all(&c->free_by_rcu_ttrace); free_all(c, llnode, !!c->percpu_size); } return; @@ -535,6 +540,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } @@ -557,7 +563,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; - + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } @@ -609,7 +615,7 @@ int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; - + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } diff --git a/kernel/bpf/mmap_unlock_work.h b/kernel/bpf/mmap_unlock_work.h index 5d18d7d85bef9..1834db20b8612 100644 --- a/kernel/bpf/mmap_unlock_work.h +++ b/kernel/bpf/mmap_unlock_work.h @@ -4,12 +4,15 @@ #ifndef __MMAP_UNLOCK_WORK_H__ #define __MMAP_UNLOCK_WORK_H__ +#include +#include #include /* irq_work to run mmap_read_unlock() in irq_work */ struct mmap_unlock_irq_work { struct irq_work irq_work; struct mm_struct *mm; + atomic_t active; }; DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); @@ -18,32 +21,36 @@ DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); * We cannot do mmap_read_unlock() when the irq is disabled, because of * risk to deadlock with rq_lock. To look up vma when the irqs are * disabled, we need to run mmap_read_unlock() in irq_work. We use a - * percpu variable to do the irq_work. If the irq_work is already used - * by another lookup, we fall over. + * percpu variable to do the irq_work. The active flag reserves the slot + * before mmap_read_trylock() and until the irq_work callback consumes mm. */ -static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr) +static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = false; + struct mmap_unlock_irq_work *work; - if (irqs_disabled()) { - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { - work = this_cpu_ptr(&mmap_unlock_work); - if (irq_work_is_busy(&work->irq_work)) { - /* cannot queue more up_read, fallback */ - irq_work_busy = true; - } - } else { - /* - * PREEMPT_RT does not allow to trylock mmap sem in - * interrupt disabled context. Force the fallback code. - */ - irq_work_busy = true; - } - } + if (!irqs_disabled()) + return NULL; + + /* + * PREEMPT_RT does not allow to trylock mmap sem in interrupt + * disabled context. Force the fallback code. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + return ERR_PTR(-EBUSY); + + work = this_cpu_ptr(&mmap_unlock_work); + if (irq_work_is_busy(&work->irq_work) || + atomic_cmpxchg_acquire(&work->active, 0, 1)) + return ERR_PTR(-EBUSY); - *work_ptr = work; - return irq_work_busy; + return work; +} + +static inline void +bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work) +{ + if (work) + atomic_set_release(&work->active, 0); } static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm) diff --git a/kernel/bpf/net_namespace.c b/kernel/bpf/net_namespace.c index 25f30f9edaefd..81006a242618d 100644 --- a/kernel/bpf/net_namespace.c +++ b/kernel/bpf/net_namespace.c @@ -171,33 +171,28 @@ static int bpf_netns_link_update_prog(struct bpf_link *link, struct net *net; int idx, ret; + guard(mutex)(&netns_bpf_mutex); + if (old_prog && old_prog != link->prog) return -EPERM; if (new_prog->type != link->prog->type) return -EINVAL; - mutex_lock(&netns_bpf_mutex); - net = net_link->net; - if (!net || !check_net(net)) { + if (!net || !check_net(net)) /* Link auto-detached or netns dying */ - ret = -ENOLINK; - goto out_unlock; - } + return -ENOLINK; run_array = rcu_dereference_protected(net->bpf.run_array[type], lockdep_is_held(&netns_bpf_mutex)); idx = link_index(net, type, net_link); ret = bpf_prog_array_update_at(run_array, idx, new_prog); if (ret) - goto out_unlock; + return ret; old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); - -out_unlock: - mutex_unlock(&netns_bpf_mutex); - return ret; + return 0; } static int bpf_netns_link_fill_info(const struct bpf_link *link, diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index 0d6f5569588c3..d855399812eec 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -698,6 +698,8 @@ static bool __bpf_offload_dev_match(struct bpf_prog *prog, return false; if (offload->netdev == netdev) return true; + if (!bpf_prog_is_offloaded(prog->aux)) + return false; ondev1 = bpf_offload_find_netdev(offload->netdev); ondev2 = bpf_offload_find_netdev(netdev); diff --git a/kernel/bpf/percpu_freelist.c b/kernel/bpf/percpu_freelist.c index 632762b572998..06ce588d13a39 100644 --- a/kernel/bpf/percpu_freelist.c +++ b/kernel/bpf/percpu_freelist.c @@ -17,6 +17,8 @@ int pcpu_freelist_init(struct pcpu_freelist *s) raw_res_spin_lock_init(&head->lock); head->first = NULL; } + raw_res_spin_lock_init(&s->extralist.lock); + s->extralist.first = NULL; return 0; } @@ -46,22 +48,28 @@ void __pcpu_freelist_push(struct pcpu_freelist *s, struct pcpu_freelist_node *node) { struct pcpu_freelist_head *head; - int cpu; + int cpu, this_cpu; if (___pcpu_freelist_push(this_cpu_ptr(s->freelist), node)) return; + this_cpu = raw_smp_processor_id(); while (true) { - for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) { - if (cpu == raw_smp_processor_id()) + for_each_cpu_wrap(cpu, cpu_possible_mask, this_cpu) { + if (cpu == this_cpu) continue; + head = per_cpu_ptr(s->freelist, cpu); - if (raw_res_spin_lock(&head->lock)) - continue; - pcpu_freelist_push_node(head, node); - raw_res_spin_unlock(&head->lock); - return; + if (___pcpu_freelist_push(head, node)) + return; } + + /* + * Push cannot fail. Use the extra list when none of the + * per-CPU freelists can accept the node. + */ + if (___pcpu_freelist_push(&s->extralist, node)) + return; } } @@ -117,6 +125,17 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s) } raw_res_spin_unlock(&head->lock); } + + /* Per-CPU lists are empty or unavailable, try the extra list. */ + head = &s->extralist; + if (!READ_ONCE(head->first)) + return NULL; + if (raw_res_spin_lock(&head->lock)) + return NULL; + node = head->first; + if (node) + WRITE_ONCE(head->first, node->next); + raw_res_spin_unlock(&head->lock); return node; } diff --git a/kernel/bpf/percpu_freelist.h b/kernel/bpf/percpu_freelist.h index 914798b749672..980cf2884fd27 100644 --- a/kernel/bpf/percpu_freelist.h +++ b/kernel/bpf/percpu_freelist.h @@ -14,6 +14,7 @@ struct pcpu_freelist_head { struct pcpu_freelist { struct pcpu_freelist_head __percpu *freelist; + struct pcpu_freelist_head extralist; }; struct pcpu_freelist_node { diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 9a5f94371e506..c1c9dee4dcdd0 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -99,8 +99,10 @@ static long __queue_map_get(struct bpf_map *map, void *value, bool delete) int err = 0; void *ptr; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); @@ -130,8 +132,10 @@ static long __stack_map_get(struct bpf_map *map, void *value, bool delete) void *ptr; u32 index; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 35ae64ade36b8..06d3cc192601f 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -482,7 +482,7 @@ static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) prod_pos = rb->producer_pos; new_prod_pos = prod_pos + len; - while (pend_pos < prod_pos) { + while (prod_pos - pend_pos > 0) { hdr = (void *)rb->data + (pend_pos & rb->mask); hdr_len = READ_ONCE(hdr->len); if (hdr_len & BPF_RINGBUF_BUSY_BIT) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 41fe87d7302f2..e41fb1c567cd7 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -414,8 +414,7 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, u32 trace_nr, bool user, bool may_fault) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + struct mmap_unlock_irq_work *work; bool has_user_ctx = user && current && current->mm; struct stack_map_build_id_cache cache = {}; struct vm_area_struct *vma; @@ -426,15 +425,16 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, return; } - /* If the irq_work is in use, fall back to report ips. Same - * fallback is used for kernel stack (!user) on a stackmap with - * build_id. - */ - if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) { - /* cannot access current->mm, fall back to ips */ - for (i = 0; i < trace_nr; i++) - stack_map_build_id_set_ip(&id_offs[i]); - return; + if (!has_user_ctx) + goto fallback; + + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) + goto fallback; + + if (!mmap_read_trylock(current->mm)) { + bpf_mmap_unlock_guard_put(work); + goto fallback; } for (i = 0; i < trace_nr; i++) { @@ -465,6 +465,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, vma->vm_pgoff); } bpf_mmap_unlock_mm(work, current->mm); + return; + +fallback: + /* cannot access current->mm, fall back to ips */ + for (i = 0; i < trace_nr; i++) + stack_map_build_id_set_ip(&id_offs[i]); } static struct perf_callchain_entry * @@ -504,78 +510,115 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth) #endif } -static long __bpf_get_stackid(struct bpf_map *map, - struct perf_callchain_entry *trace, u64 flags) +struct stackid { + struct stack_map_bucket *bucket; + u64 *ips; + u32 nr; + u32 len; + u32 hash; + u32 id; + bool hash_matches; +}; + +static int stackid_init(struct stackid *stackid, struct bpf_map *map, + struct perf_callchain_entry *trace, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); - struct stack_map_bucket *bucket, *new_bucket, *old_bucket; - u32 hash, id, trace_nr, trace_len, i, max_depth; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; - bool user = flags & BPF_F_USER_STACK; - u64 *ips; - bool hash_matches; + u32 max_depth; if (trace->nr <= skip) /* skipping more than usable stack trace */ return -EFAULT; max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags); - trace_nr = min_t(u32, trace->nr - skip, max_depth - skip); - trace_len = trace_nr * sizeof(u64); - ips = trace->ip + skip; - hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0); - id = hash & (smap->n_buckets - 1); - bucket = READ_ONCE(smap->buckets[id]); + stackid->nr = min_t(u32, trace->nr - skip, max_depth - skip); + stackid->len = stackid->nr * sizeof(u64); + stackid->ips = trace->ip + skip; + stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0); + stackid->id = stackid->hash & (smap->n_buckets - 1); + stackid->bucket = READ_ONCE(smap->buckets[stackid->id]); + stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; + return 0; +} + +static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map, + struct perf_callchain_entry *trace, u64 flags) +{ + int err; + + err = stackid_init(stackid, map, trace, flags); + if (err) + return err; - hash_matches = bucket && bucket->hash == hash; /* fast cmp */ - if (hash_matches && flags & BPF_F_FAST_STACK_CMP) - return id; + if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP) + return stackid->id; + + if (stack_map_use_build_id(map)) + return -ENOENT; + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) + return stackid->id; + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) + return -EEXIST; + return -ENOENT; +} + +static struct stack_map_bucket * +stackid_new_bucket(struct stackid *stackid, struct bpf_map *map) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + struct bpf_stack_build_id *id_offs; + struct stack_map_bucket *bucket; + u32 i; + + bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); + if (unlikely(!bucket)) + return NULL; + + if (stack_map_use_build_id(map)) { + id_offs = (struct bpf_stack_build_id *)bucket->data; + for (i = 0; i < stackid->nr; i++) + id_offs[i].ip = stackid->ips[i]; + } else { + memcpy(bucket->data, stackid->ips, stackid->len); + } + + bucket->hash = stackid->hash; + bucket->nr = stackid->nr; + return bucket; +} + +static long stackid_install(struct stackid *stackid, struct bpf_map *map, + struct stack_map_bucket *new_bucket, u64 flags) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *old_bucket; + u32 trace_len; if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; - /* for build_id+offset, pop a bucket before slow cmp */ - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - new_bucket->nr = trace_nr; id_offs = (struct bpf_stack_build_id *)new_bucket->data; - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - stack_map_get_build_id_offset(id_offs, trace_nr, user, false /* !may_fault */); - trace_len = trace_nr * sizeof(struct bpf_stack_build_id); - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, new_bucket->data, trace_len) == 0) { + stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */); + trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); - return id; + return stackid->id; } - if (bucket && !(flags & BPF_F_REUSE_STACKID)) { + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); return -EEXIST; } - } else { - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, ips, trace_len) == 0) - return id; - if (bucket && !(flags & BPF_F_REUSE_STACKID)) - return -EEXIST; - - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - memcpy(new_bucket->data, ips, trace_len); } - new_bucket->hash = hash; - new_bucket->nr = trace_nr; - - old_bucket = xchg(&smap->buckets[id], new_bucket); + old_bucket = xchg(&smap->buckets[stackid->id], new_bucket); if (old_bucket) pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); - return id; + return stackid->id; } BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, @@ -583,23 +626,36 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, { u32 elem_size = stack_map_data_size(map); bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *new_bucket; struct perf_callchain_entry *trace; + struct stackid stackid; bool kernel = !user; u32 max_depth; + int err; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) return -EINVAL; max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags); - trace = get_perf_callchain(regs, kernel, user, max_depth, - false, false, 0); - if (unlikely(!trace)) - /* couldn't fetch the stack trace */ - return -EFAULT; + scoped_guard(preempt) { + trace = get_perf_callchain(regs, kernel, user, max_depth, + false, false, 0); + if (unlikely(!trace)) + /* couldn't fetch the stack trace */ + return -EFAULT; - return __bpf_get_stackid(map, trace, flags); + err = stackid_fastpath(&stackid, map, trace, flags); + if (err != -ENOENT) + return err; + + new_bucket = stackid_new_bucket(&stackid, map); + if (!new_bucket) + return -ENOMEM; + } + + return stackid_install(&stackid, map, new_bucket, flags); } const struct bpf_func_proto bpf_get_stackid_proto = { @@ -627,7 +683,9 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, struct bpf_map *, map, u64, flags) { struct perf_event *event = ctx->event; + struct stack_map_bucket *new_bucket; struct perf_callchain_entry *trace; + struct stackid stackid; bool kernel, user; __u64 nr_kernel; int ret; @@ -653,7 +711,6 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, if (kernel) { trace->nr = nr_kernel; - ret = __bpf_get_stackid(map, trace, flags); } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; @@ -662,12 +719,22 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, return -EFAULT; flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - ret = __bpf_get_stackid(map, trace, flags); } + ret = stackid_fastpath(&stackid, map, trace, flags); + if (ret != -ENOENT) + goto out; + + new_bucket = stackid_new_bucket(&stackid, map); + if (new_bucket) { + trace->nr = nr; + return stackid_install(&stackid, map, new_bucket, flags); + } + ret = -ENOMEM; + +out: /* restore nr */ trace->nr = nr; - return ret; } @@ -680,19 +747,55 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = { .arg3_type = ARG_ANYTHING, }; +static u32 callchain_store(struct perf_callchain_entry *trace, void *buf, + u32 elem_size, u64 flags) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + u32 skip = flags & BPF_F_SKIP_FIELD_MASK; + u32 trace_nr, copy_len; + u64 *ips; + + trace_nr = trace->nr - skip; + copy_len = trace_nr * elem_size; + + ips = trace->ip + skip; + if (user_build_id) { + struct bpf_stack_build_id *id_offs = buf; + + for (u32 i = 0; i < trace_nr; i++) + id_offs[i].ip = ips[i]; + } else { + memcpy(buf, ips, copy_len); + } + return trace_nr; +} + +static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size, + u64 flags, bool may_fault) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + bool user = flags & BPF_F_USER_STACK; + u32 copy_len = trace_nr * elem_size; + + if (user_build_id) + stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); + + if (size > copy_len) + memset(buf + copy_len, 0, size - copy_len); + return copy_len; +} + static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, - struct perf_callchain_entry *trace_in, void *buf, u32 size, u64 flags, bool may_fault) { - u32 trace_nr, copy_len, elem_size, max_depth; bool user_build_id = flags & BPF_F_USER_BUILD_ID; bool crosstask = task && task != current; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; bool user = flags & BPF_F_USER_STACK; struct perf_callchain_entry *trace; + u32 trace_nr, elem_size, max_depth; bool kernel = !user; int err = -EINVAL; - u64 *ips; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_USER_BUILD_ID))) @@ -718,13 +821,11 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, max_depth = stack_map_calculate_max_depth(size, elem_size, flags); + preempt_disable(); if (may_fault) rcu_read_lock(); /* need RCU for perf's callchain below */ - if (trace_in) { - trace = trace_in; - trace->nr = min_t(u32, trace->nr, max_depth); - } else if (kernel && task) { + if (kernel && task) { trace = get_callchain_entry_for_task(task, max_depth); } else { trace = get_perf_callchain(regs, kernel, user, max_depth, @@ -734,33 +835,18 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, if (unlikely(!trace) || trace->nr < skip) { if (may_fault) rcu_read_unlock(); + preempt_enable(); goto err_fault; } - trace_nr = trace->nr - skip; - copy_len = trace_nr * elem_size; - - ips = trace->ip + skip; - if (user_build_id) { - struct bpf_stack_build_id *id_offs = buf; - u32 i; + trace_nr = callchain_store(trace, buf, elem_size, flags); - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - } else { - memcpy(buf, ips, copy_len); - } - - /* trace/ips should not be dereferenced after this point */ + /* trace should not be dereferenced after this point */ if (may_fault) rcu_read_unlock(); + preempt_enable(); - if (user_build_id) - stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); - - if (size > copy_len) - memset(buf + copy_len, 0, size - copy_len); - return copy_len; + return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault); err_fault: err = -EFAULT; @@ -772,7 +858,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size, u64, flags) { - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */); } const struct bpf_func_proto bpf_get_stack_proto = { @@ -788,12 +874,13 @@ const struct bpf_func_proto bpf_get_stack_proto = { BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, size, u64, flags) { - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, true /* may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, true /* may_fault */); } const struct bpf_func_proto bpf_get_stack_sleepable_proto = { .func = bpf_get_stack_sleepable, .gpl_only = true, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, @@ -807,14 +894,17 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size, struct pt_regs *regs; long res = -EINVAL; - if (!try_get_task_stack(task)) + if (!try_get_task_stack(task)) { + memset(buf, 0, size); return -EFAULT; + } regs = task_pt_regs(task); if (regs) - res = __bpf_get_stack(regs, task, NULL, buf, size, flags, may_fault); + res = __bpf_get_stack(regs, task, buf, size, flags, may_fault); + else + memset(buf, 0, size); put_task_stack(task); - return res; } @@ -844,6 +934,7 @@ BPF_CALL_4(bpf_get_task_stack_sleepable, struct task_struct *, task, void *, buf const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .func = bpf_get_task_stack_sleepable, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], @@ -852,6 +943,32 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .arg4_type = ARG_ANYTHING, }; +static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32 size, + u64 flags) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + u64 skip = flags & BPF_F_SKIP_FIELD_MASK; + bool user = flags & BPF_F_USER_STACK; + u32 elem_size, max_depth, nr_trace; + bool kernel = !user; + + if (kernel && user_build_id) + return -EINVAL; + + elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64); + if (unlikely(size % elem_size)) + return -EINVAL; + + max_depth = stack_map_calculate_max_depth(size, elem_size, flags); + trace->nr = min_t(u32, trace->nr, max_depth); + + if (trace->nr < skip) + return -EFAULT; + + nr_trace = callchain_store(trace, buf, elem_size, flags); + return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */); +} + BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, void *, buf, u32, size, u64, flags) { @@ -863,7 +980,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, __u64 nr_kernel; if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)) - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */); if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_USER_BUILD_ID))) @@ -883,7 +1000,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, __u64 nr = trace->nr; trace->nr = nr_kernel; - err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */); + err = __bpf_get_stack_pe(trace, buf, size, flags); /* restore nr */ trace->nr = nr; @@ -893,14 +1010,13 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, skip += nr_kernel; if (skip > BPF_F_SKIP_FIELD_MASK) goto clear; - flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */); + err = __bpf_get_stack_pe(trace, buf, size, flags); } - return err; clear: - memset(buf, 0, size); + if (err < 0) + memset(buf, 0, size); return err; } diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index ea2153cf28d0a..6c88ad95b63b7 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -445,22 +445,19 @@ static void __clean_func_state(struct bpf_verifier_env *env, struct bpf_reg_state *spill = &st->stack[i].spilled_ptr; if (lo_live && stype == STACK_SPILL) { - u8 val = STACK_MISC; - if (spill->type != SCALAR_VALUE) continue; - /* - * 8 byte spill of scalar 0 where half slot is dead - * should become STACK_ZERO in lo 4 bytes. + * Can't replace with STACK_ZERO, because + * that requires bpf_mark_chain_precision(). */ if (bpf_register_is_null(spill)) - val = STACK_ZERO; + continue; for (j = 0; j < 4; j++) { u8 *t = &st->stack[i].slot_type[j]; if (*t == STACK_SPILL) - *t = val; + *t = STACK_MISC; } } bpf_mark_reg_not_init(env, spill); @@ -638,6 +635,9 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, /* id relations must be preserved */ if (!check_ids(rold->id, rcur->id, idmap)) return false; + /* Preserve displacements between pointers sharing an ID. */ + if (rold->id && rold->r64.base != rcur->r64.base) + return false; /* new val must satisfy old val knowledge */ return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); @@ -812,7 +812,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * infinite loop check triggering, see * iter_active_depths_differ() */ - if (old_reg->iter.btf != cur_reg->iter.btf || + if (old_reg->type != cur_reg->type || + old_reg->iter.btf != cur_reg->iter.btf || old_reg->iter.btf_id != cur_reg->iter.btf_id || old_reg->iter.state != cur_reg->iter.state || /* ignore {old_reg,cur_reg}->iter.depth, see above */ diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 6db306d23b479..77b3252270f3e 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1657,7 +1657,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_at err = security_bpf_map_create(map, attr, token, uattr.is_kernel); if (err) - goto free_map_sec; + goto free_map; err = bpf_map_alloc_id(map); if (err) @@ -2033,7 +2033,7 @@ int generic_map_delete_batch(struct bpf_map *map, for (cp = 0; cp < max_count; cp++) { err = -EFAULT; - if (copy_from_user(key, keys + cp * map->key_size, + if (copy_from_user(key, keys + (size_t)cp * map->key_size, map->key_size)) break; @@ -2095,9 +2095,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, for (cp = 0; cp < max_count; cp++) { err = -EFAULT; - if (copy_from_user(key, keys + cp * map->key_size, + if (copy_from_user(key, keys + (size_t)cp * map->key_size, map->key_size) || - copy_from_user(value, values + cp * value_size, value_size)) + copy_from_user(value, values + (size_t)cp * value_size, value_size)) break; err = bpf_map_update_value(map, map_file, key, value, @@ -2176,12 +2176,12 @@ int generic_map_lookup_batch(struct bpf_map *map, if (err) goto free_buf; - if (copy_to_user(keys + cp * map->key_size, key, + if (copy_to_user(keys + (size_t)cp * map->key_size, key, map->key_size)) { err = -EFAULT; goto free_buf; } - if (copy_to_user(values + cp * value_size, value, value_size)) { + if (copy_to_user(values + (size_t)cp * value_size, value, value_size)) { err = -EFAULT; goto free_buf; } @@ -3471,9 +3471,10 @@ static const char *bpf_link_type_strs[] = { static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) { const struct bpf_link *link = filp->private_data; - const struct bpf_prog *prog = link->prog; + const struct bpf_prog *prog; enum bpf_link_type type = link->type; char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; + u32 prog_id = 0; if (type < ARRAY_SIZE(bpf_link_type_strs) && bpf_link_type_strs[type]) { if (link->type == BPF_LINK_TYPE_KPROBE_MULTI) @@ -3490,13 +3491,20 @@ static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) } seq_printf(m, "link_id:\t%u\n", link->id); + rcu_read_lock(); + prog = READ_ONCE(link->prog); if (prog) { bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); + prog_id = prog->aux->id; + } + rcu_read_unlock(); + + if (prog) { seq_printf(m, "prog_tag:\t%s\n" "prog_id:\t%u\n", prog_tag, - prog->aux->id); + prog_id); } if (link->ops->show_fdinfo) link->ops->show_fdinfo(link, m); @@ -5535,6 +5543,7 @@ static int bpf_link_get_info_by_fd(struct file *file, { struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); struct bpf_link_info info; + const struct bpf_prog *prog; u32 info_len = attr->info.info_len; int err; @@ -5549,8 +5558,12 @@ static int bpf_link_get_info_by_fd(struct file *file, info.type = link->type; info.id = link->id; - if (link->prog) - info.prog_id = link->prog->aux->id; + + rcu_read_lock(); + prog = READ_ONCE(link->prog); + if (prog) + info.prog_id = prog->aux->id; + rcu_read_unlock(); if (link->ops->fill_link_info) { err = link->ops->fill_link_info(link, &info); @@ -6093,7 +6106,10 @@ struct bpf_link *bpf_link_get_curr_or_next(u32 *id) again: link = idr_get_next(&link_idr, id); if (link) { - link = bpf_link_inc_not_zero(link); + if (link->id) + link = bpf_link_inc_not_zero(link); + else + link = ERR_PTR(-EAGAIN); if (IS_ERR(link)) { (*id)++; goto again; @@ -6620,6 +6636,7 @@ EXPORT_SYMBOL_NS(kern_sys_bpf, "BPF_INTERNAL"); static const struct bpf_func_proto bpf_sys_bpf_proto = { .func = bpf_sys_bpf, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, @@ -6645,6 +6662,7 @@ BPF_CALL_1(bpf_sys_close, u32, fd) static const struct bpf_func_proto bpf_sys_close_proto = { .func = bpf_sys_close, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index e791ae065c39b..13e1aabe6f886 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -753,9 +753,9 @@ static struct bpf_iter_reg task_vma_reg_info = { BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags) { - struct mmap_unlock_irq_work *work = NULL; + struct mmap_unlock_irq_work *work; struct vm_area_struct *vma; - bool irq_work_busy = false; + bool __maybe_unused mmput_needed = false; struct mm_struct *mm; int ret = -ENOENT; @@ -765,14 +765,43 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, if (!task) return -ENOENT; - mm = task->mm; + if (task == current) { + mm = task->mm; + } else { + /* + * Foreign task: pin task->mm against a concurrent exit_mm(). + * Use trylock on alloc_lock instead of get_task_mm()'s + * blocking task_lock() to avoid deadlocking the target task. + */ + if (!IS_ENABLED(CONFIG_MMU)) + return -EOPNOTSUPP; + if (irqs_disabled()) + return -EBUSY; + if (!spin_trylock(&task->alloc_lock)) + return -EBUSY; + mm = task->mm; + if (mm && !(task->flags & PF_KTHREAD)) { + mmget(mm); + mmput_needed = true; + } else { + mm = NULL; + } + spin_unlock(&task->alloc_lock); + } if (!mm) return -ENOENT; - irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) { + ret = PTR_ERR(work); + goto out; + } - if (irq_work_busy || !mmap_read_trylock(mm)) - return -EBUSY; + if (!mmap_read_trylock(mm)) { + bpf_mmap_unlock_guard_put(work); + ret = -EBUSY; + goto out; + } vma = find_vma(mm, start); @@ -782,6 +811,11 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, ret = 0; } bpf_mmap_unlock_mm(work, mm); +out: +#ifdef CONFIG_MMU + if (mmput_needed) + mmput_async(mm); +#endif return ret; } @@ -1161,6 +1195,8 @@ static void do_mmap_read_unlock(struct irq_work *entry) work = container_of(entry, struct mmap_unlock_irq_work, irq_work); mmap_read_unlock_non_owner(work->mm); + work->mm = NULL; + bpf_mmap_unlock_guard_put(work); } static int __init task_iter_init(void) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 1a721fc4bef56..02c208bd2b637 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -670,6 +670,13 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) return ERR_PTR(err); } +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) +{ + trampoline_lock(tr); + tr->flags |= flags; + trampoline_unlock(tr); +} + static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex, const struct bpf_trampoline_ops *ops, void *data) { @@ -1536,6 +1543,7 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); + tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; @@ -1584,7 +1592,17 @@ static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr) static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr) { - if (tr->multi_attach.old_image) + /* + * Only free old_image if it is no longer the active image. + * When bpf_trampoline_update() fails before modify_fentry_multi()/ + * unregister_fentry_multi() is called, cur_image is unchanged + * (cur_image == old_image) and ftrace still points to it. Freeing + * it would cause a UAF when ftrace calls into the freed memory. + * On success, cur_image is either a new image or NULL, so + * old_image != cur_image means the image is stale. + */ + if (tr->multi_attach.old_image && + tr->multi_attach.old_image != tr->cur_image) bpf_tramp_image_put(tr->multi_attach.old_image); tr->multi_attach.old_image = NULL; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fdc5fbb1f78ca..ccf735d74dc82 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -204,6 +204,7 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par static int release_reference_nomark(struct bpf_verifier_state *state, int id); static int release_reference(struct bpf_verifier_env *env, int id); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); +static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); static bool is_tracing_prog_type(enum bpf_prog_type type); static int ref_set_non_owning(struct bpf_verifier_env *env, @@ -366,9 +367,18 @@ static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_stat if (type_may_be_null(type)) return false; + /* + * The types below guarantee a non-NULL base, an unbounded offset can + * still wrap base + offset to zero. + */ + if (reg_smin(reg) <= -BPF_MAX_VAR_OFF || reg_smax(reg) >= BPF_MAX_VAR_OFF) + return false; + type = base_type(type); return type == PTR_TO_SOCKET || type == PTR_TO_TCP_SOCK || + type == PTR_TO_XDP_SOCK || + type == PTR_TO_BUF || type == PTR_TO_MAP_VALUE || type == PTR_TO_MAP_KEY || type == PTR_TO_SOCK_COMMON || @@ -572,7 +582,7 @@ static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_s } off = reg->var_off.value; - if (off % BPF_REG_SIZE) { + if (off >= 0 || off % BPF_REG_SIZE) { verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); return -EINVAL; } @@ -1610,6 +1620,8 @@ static int copy_func_state(struct bpf_func_state *dst, const struct bpf_func_state *src) { memcpy(dst, src, offsetof(struct bpf_func_state, stack)); + /* Instruction accounting is path-local, not part of verifier state. */ + dst->insns_subtotal = 0; return copy_stack_state(dst, src); } @@ -2715,6 +2727,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) prog_aux->kfunc_tab = tab; } + env->prog->jit_required = 1; + /* func_id == 0 is always invalid, but instead of returning an error, be * conservative and wait until the code elimination pass before returning * error, so that invalid calls that get pruned out can be in BPF programs @@ -2769,11 +2783,6 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) return 0; } -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return !!prog->aux->kfunc_tab; -} - static int add_subprog_and_kfunc(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprog = env->subprog_info; @@ -2867,6 +2876,8 @@ static int check_subprogs(struct bpf_verifier_env *env) subprog[cur_subprog].exit_idx = i; goto next; } + if (insn_is_gotox(&insn[i])) + goto next; off = i + bpf_jmp_offset(&insn[i]) + 1; if (off < subprog_start || off >= subprog_end) { verbose(env, "jump out of range from insn %d to %d\n", i, off); @@ -2880,7 +2891,8 @@ static int check_subprogs(struct bpf_verifier_env *env) */ if (code != (BPF_JMP | BPF_EXIT) && code != (BPF_JMP32 | BPF_JA) && - code != (BPF_JMP | BPF_JA)) { + code != (BPF_JMP | BPF_JA) && + !insn_is_gotox(&insn[i])) { verbose(env, "last insn is not an exit or jmp\n"); return -EINVAL; } @@ -4311,6 +4323,13 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, if (type_flag(reg->type) & ~perm_flags) goto bad_type; + /* + * A BPF_KPTR_PERCPU field is read back as MEM_PERCPU, so the value + * stored in it must carry the same flag. + */ + if ((kptr_field->type == BPF_KPTR_PERCPU) != !!(reg->type & MEM_PERCPU)) + goto bad_type; + /* We need to verify reg->type and reg->btf, before accessing reg->btf */ reg_name = btf_type_name(reg->btf, reg->btf_id); @@ -4349,7 +4368,8 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, */ if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, kptr_field->kptr.btf, kptr_field->kptr.btf_id, - kptr_field->type != BPF_KPTR_UNREF)) + kptr_field->type != BPF_KPTR_UNREF, + !type_is_alloc(reg->type))) goto bad_type; return 0; bad_type: @@ -4511,8 +4531,15 @@ static int check_map_kptr_access(struct bpf_verifier_env *env, return ret; } else if (class == BPF_STX) { val_reg = reg_state(env, value_regno); - if (!bpf_register_is_null(val_reg) && - map_kptr_match_type(env, kptr_field, val_reg, value_regno)) + if (bpf_register_is_null(val_reg)) { + /* + * This store is valid only because the scalar is known to be + * zero. Mark it precise so another scalar cannot be pruned + * against this state. + */ + return mark_chain_precision(env, value_regno); + } + if (map_kptr_match_type(env, kptr_field, val_reg, value_regno)) return -EACCES; } else if (class == BPF_ST) { if (insn->imm) { @@ -4844,6 +4871,30 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, + struct bpf_insn *insn) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + /* + * A BPF_LOAD_ACQ is not rewritten to a BPF_PROBE_MEM load by the + * verifier, unlike a regular BPF_LDX. The JIT would emit a plain load + * with no exception table entry, so a fault (e.g. NULL deref) crashes + * the kernel instead of being handled. + * + * Reject the source pointer types that a BPF_LDX would have had that + * fault protection applied to, i.e. the ones bpf_convert_ctx_accesses() + * turns into BPF_PROBE_MEM: a bare PTR_TO_BTF_ID and any PTR_UNTRUSTED + * pointer (untrusted btf ids, untrusted MEM_ALLOC, rdonly untrusted + * memory). A PTR_TRUSTED pointer is not among them, is not converted, + * and stays allowed. Same for the other flagged PTR_TO_BTF_ID variants + * (MEM_ALLOC, MEM_RCU, ...), hence the exact match on the base type. + */ + return insn->imm == BPF_LOAD_ACQ && + (reg->type == PTR_TO_BTF_ID || + (type_flag(reg->type) & PTR_UNTRUSTED)); +} + /* Return false if @regno contains a pointer whose type isn't supported for * atomic instruction @insn. */ @@ -4860,7 +4911,8 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_arena_reg(env, regno)) return bpf_jit_supports_insn(insn, true); - + if (is_load_acq_unsafe(env, regno, insn)) + return false; return true; } @@ -5091,6 +5143,15 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; process_func: + if (subprog[idx].has_ld_abs) { + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { + if (subprog[tmp].is_cb) { + verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n"); + return -EINVAL; + } + } + } + /* protect against potential stack overflow that might happen when * bpf2bpf calls get combined with tailcalls. Limit the caller's stack * depth for such case down to 256 so that the worst case scenario @@ -5213,8 +5274,8 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; - if (subprog[idx].has_tail_call) - tail_call_reachable = true; + /* sync tail_call_reachable with callee state on entry */ + tail_call_reachable = subprog[idx].has_tail_call; frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1; if (frame >= MAX_CALL_FRAMES) { @@ -5789,6 +5850,11 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EACCES; } + if (atype != BPF_READ && (type_flag(reg->type) & PTR_UNTRUSTED)) { + verbose(env, "only read is supported\n"); + return -EACCES; + } + if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { if (!btf_is_kernel(reg->btf)) { verifier_bug(env, "reg->btf must be kernel btf"); @@ -5801,8 +5867,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, reg_arg_name(env, argno), tname, off, size); } else { /* Writes are permitted with default btf_struct_access for - * program allocated objects (which always have id > 0), - * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. + * program allocated objects (which always have id > 0). */ if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) { verbose(env, "only read is supported\n"); @@ -6326,11 +6391,23 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && regs[value_regno].type == SCALAR_VALUE) { - if (!is_ldsx) + if (!is_ldsx) { /* b/h/w load zero-extends, mark upper bits as known 0 */ coerce_reg_to_size(®s[value_regno], size); - else + } else { + /* + * Sign-extension can change the register value relative + * to a scalar it is linked with by id (e.g. a zero- + * extending fill of the same spilled stack slot), thus + * drop the shared id in that case. + */ + bool no_sext = reg_umax(®s[value_regno]) < + (1ULL << (size * BITS_PER_BYTE - 1)); + coerce_reg_to_size_sx(®s[value_regno], size); + if (!no_sext) + clear_scalar_id(®s[value_regno]); + } } return err; } @@ -6467,21 +6544,12 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } - if (insn->imm & BPF_FETCH) { - if (insn->imm == BPF_CMPXCHG) - load_reg = BPF_REG_0; - else - load_reg = insn->src_reg; - + load_reg = bpf_atomic_load_reg(insn); + if (load_reg >= 0) { /* check and record load of old value */ err = check_reg_arg(env, load_reg, DST_OP); if (err) return err; - } else { - /* This instruction accesses a memory location but doesn't - * actually load it into a register. - */ - load_reg = -1; } dst_reg = cur_regs(env) + insn->dst_reg; @@ -6498,11 +6566,9 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, if (err) return err; - if (is_arena_reg(env, insn->dst_reg)) { - err = save_aux_ptr_type(env, PTR_TO_ARENA, false); - if (err) - return err; - } + err = save_aux_ptr_type(env, dst_reg->type, false); + if (err) + return err; /* Check whether we can write into the same memory. */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); @@ -6516,7 +6582,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, { int err; - err = check_load_mem(env, insn, true, false, false, "atomic_load"); + err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err; @@ -6527,7 +6593,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, return -EACCES; } - return 0; + return check_load_mem(env, insn, true, false, false, "atomic_load"); } static int check_atomic_store(struct bpf_verifier_env *env, @@ -7051,6 +7117,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state return err; } } else { + bool was_in_rcu_cs; void *ptr; int type; @@ -7078,10 +7145,13 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state verbose(env, "%s_unlock cannot be out of order\n", lock_str); return -EINVAL; } + was_in_rcu_cs = in_rcu_cs(env); if (release_lock_state(cur, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); return -EINVAL; } + if (was_in_rcu_cs && !in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); invalidate_non_owning_refs(env); } @@ -7810,6 +7880,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_CONST_SIZE] = &scalar_types, [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, + [ARG_SCALAR] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, [ARG_PTR_TO_SOCK_COMMON] = &sock_types, @@ -7947,7 +8018,7 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, btf_vmlinux, *arg_btf_id, - strict_type_match)) { + strict_type_match, !type_is_alloc(reg->type))) { verbose(env, "%s is of type %s but %s is expected\n", reg_arg_name(env, argno), btf_type_name(reg->btf, reg->btf_id), @@ -8250,11 +8321,15 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; } - if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) + if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { /* A NULL register has a SCALAR_VALUE type, so skip * type checking. */ + err = mark_chain_precision(env, regno); + if (err) + return err; goto skip_type_check; + } /* arg_btf_id and arg_size are in a union. */ if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || @@ -9189,6 +9264,16 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, return ret; if (check_mem_reg(env, reg, argno, arg->mem_size)) return -EINVAL; + /* + * PTR_TO_PACKET get passed as PTR_TO_MEM, preventing + * us from adjusting bounds tracking info. + */ + if ((reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) && + sub->changes_pkt_data) { + bpf_log(log, "%s is a packet pointer, but func#%d may change packet data\n", + reg_arg_name(env, argno), subprog); + return -EINVAL; + } if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { bpf_log(log, "%s is expected to be non-NULL\n", @@ -9462,10 +9547,12 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env, callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; + callee->regs[BPF_REG_2].map_uid = caller->regs[BPF_REG_1].map_uid; callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; + callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid; /* pointer to stack or null */ callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; @@ -9543,6 +9630,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, int insn_idx) { struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; + u32 map_uid = caller->regs[BPF_REG_1].map_uid; /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); * callback_fn(struct bpf_map *map, void *key, void *value); @@ -9550,14 +9638,17 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; __mark_reg_known_zero(&callee->regs[BPF_REG_1]); callee->regs[BPF_REG_1].map_ptr = map_ptr; + callee->regs[BPF_REG_1].map_uid = map_uid; callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = map_ptr; + callee->regs[BPF_REG_2].map_uid = map_uid; callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; + callee->regs[BPF_REG_3].map_uid = map_uid; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -9657,6 +9748,7 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, int insn_idx) { struct bpf_map *map_ptr = caller->regs[BPF_REG_3].map_ptr; + u32 map_uid = caller->regs[BPF_REG_3].map_uid; /* * callback_fn(struct bpf_map *map, void *key, void *value); @@ -9664,14 +9756,17 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; __mark_reg_known_zero(&callee->regs[BPF_REG_1]); callee->regs[BPF_REG_1].map_ptr = map_ptr; + callee->regs[BPF_REG_1].map_uid = map_uid; callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = map_ptr; + callee->regs[BPF_REG_2].map_uid = map_uid; callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; + callee->regs[BPF_REG_3].map_uid = map_uid; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -9683,9 +9778,46 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, static bool is_rbtree_lock_required_kfunc(u32 btf_id); -/* Are we currently verifying the callback for a rbtree helper that must - * be called with lock held? If so, no need to complain about unreleased - * lock +static void account_processed_insn(struct bpf_verifier_env *env) +{ + struct bpf_func_state *frame = cur_func(env); + + env->insn_processed++; + frame->insns_subtotal++; + env->subprog_info[frame->subprogno].insns_self++; +} + +static void account_processed_insns(struct bpf_verifier_env *env, + struct bpf_func_state *callee, + struct bpf_func_state *caller) +{ + u32 insns; + + if (!callee) + return; + + insns = callee->insns_subtotal; + + env->subprog_info[callee->subprogno].insns_total += insns; + if (caller) + caller->insns_subtotal += insns; + callee->insns_subtotal = 0; +} + +static void account_current_path(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + int frame; + + for (frame = state->curframe; frame >= 0; frame--) + account_processed_insns(env, state->frame[frame], + frame ? state->frame[frame - 1] : NULL); +} + +/* + * Are we currently verifying the callback for an rbtree kfunc that must + * be called with a lock held, or one of that callback's subprogs? If so, + * no need to complain about an unreleased lock. */ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) { @@ -9693,17 +9825,19 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) struct bpf_insn *insn = env->prog->insnsi; struct bpf_func_state *callee; int kfunc_btf_id; + u32 frame; - if (!state->curframe) - return false; - - callee = state->frame[state->curframe]; + for (frame = state->curframe; frame; frame--) { + callee = state->frame[frame]; + if (!callee->in_callback_fn) + continue; - if (!callee->in_callback_fn) - return false; + kfunc_btf_id = insn[callee->callsite].imm; + if (is_rbtree_lock_required_kfunc(kfunc_btf_id)) + return true; + } - kfunc_btf_id = insn[callee->callsite].imm; - return is_rbtree_lock_required_kfunc(kfunc_btf_id); + return false; } static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg) @@ -9779,6 +9913,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) verbose(env, "to caller at %d:\n", *insn_idx); print_verifier_state(env, state, caller->frameno, true); } + account_processed_insns(env, callee, caller); /* clear everything in the callee. In case of exceptional exits using * bpf_throw, this will be done by copy_verifier_state for extra frames. */ free_func_state(callee); @@ -10067,33 +10202,45 @@ static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env) return &env->insn_aux_data[env->insn_idx]; } -static bool loop_flag_is_zero(struct bpf_verifier_env *env) +/* Returns 1 if R4 is a known zero, 0 if it is not, a negative errno on error. */ +static int loop_flag_is_zero(struct bpf_verifier_env *env) { struct bpf_reg_state *reg = reg_state(env, BPF_REG_4); - bool reg_is_null = bpf_register_is_null(reg); + int err; - if (reg_is_null) - mark_chain_precision(env, BPF_REG_4); + if (!bpf_register_is_null(reg)) + return 0; - return reg_is_null; + err = mark_chain_precision(env, BPF_REG_4); + if (err) + return err; + return 1; } -static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) +static int update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) { struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; + int flag_is_zero; if (!state->initialized) { + flag_is_zero = loop_flag_is_zero(env); + if (flag_is_zero < 0) + return flag_is_zero; state->initialized = 1; - state->fit_for_inline = loop_flag_is_zero(env); + state->fit_for_inline = flag_is_zero; state->callback_subprogno = subprogno; - return; + return 0; } if (!state->fit_for_inline) - return; + return 0; - state->fit_for_inline = (loop_flag_is_zero(env) && + flag_is_zero = loop_flag_is_zero(env); + if (flag_is_zero < 0) + return flag_is_zero; + state->fit_for_inline = (flag_is_zero && state->callback_subprogno == subprogno); + return 0; } /* Returns whether or not the given map can potentially elide @@ -10285,6 +10432,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn verbose(env, "get_local_storage() doesn't support non-zero flags\n"); return -EINVAL; } + err = mark_chain_precision(env, BPF_REG_2); + if (err) + return err; break; case BPF_FUNC_for_each_map_elem: err = push_callback_call(env, insn, insn_idx, meta.subprogno, @@ -10302,7 +10452,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn err = check_bpf_snprintf_call(env, regs); break; case BPF_FUNC_loop: - update_loop_inline_state(env, meta.subprogno); + err = update_loop_inline_state(env, meta.subprogno); + if (err) + return err; /* Verifier relies on R1 value to determine if bpf_loop() iteration * is finished, thus mark it precise. */ @@ -10637,6 +10789,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (env->cur_state->curframe) { struct bpf_verifier_state *branch; + /* + * A taken tail call is modeled as a return from the current + * frame. A callback frame cannot be left that way because + * prepare_func_exit() would apply its return contract to the + * unknown R0 synthesized below. Stack-depth validation rejects + * this construct anyway. + */ + if (cur_func(env)->in_callback_fn) { + verbose(env, "cannot tail call within callback\n"); + return -EINVAL; + } mark_reg_scratched(env, BPF_REG_0); branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) @@ -11414,7 +11577,8 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value, - meta->btf, ref_id, strict_type_match); + meta->btf, ref_id, strict_type_match, + !type_is_alloc(reg->type)); /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. @@ -11861,7 +12025,8 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id); t = btf_type_by_id(reg->btf, reg->btf_id); if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, - field->graph_root.value_btf_id, true)) { + field->graph_root.value_btf_id, true, + !type_is_alloc(reg->type))) { verbose(env, "operation on %s expects arg#1 %s at offset=%d " "in struct %s, but arg is at offset=%d in struct %s\n", btf_field_type_name(head_field_type), @@ -12033,6 +12198,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } meta->r0_size = reg->var_off.value; + if (meta->r0_size > U32_MAX) { + verbose(env, "%s rdonly/rdwr_buf_size exceeds u32 max\n", + reg_arg_name(env, argno)); + return -EINVAL; + } if (regno >= 0) ret = mark_chain_precision(env, regno); else @@ -12419,7 +12589,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ reg_arg_name(env, argno)); return -EINVAL; } - if (!type_is_non_owning_ref(reg->type)) + if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) meta->arg_owning_ref = true; rec = reg_btf_record(reg); @@ -12491,6 +12661,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ { int flags = PROCESS_RES_LOCK; + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); + return -EACCES; + } + if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s doesn't point to map value or allocated object\n", reg_arg_name(env, argno)); @@ -13754,9 +13929,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, !check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) return -EINVAL; - /* pointer types do not carry 32-bit bounds at the moment. */ - __mark_reg32_unbounded(dst_reg); - if (sanitize_needed(opcode)) { ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, &info, false); @@ -13764,6 +13936,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return sanitize_err(env, insn, ret); } + /* + * Pointer types do not carry 32-bit bounds at the moment. Blank r32 + * only after sanitize_ptr_alu() may have snapshotted dst_reg into a + * speculative path: otherwise reg_bounds_sanity_check() might hit some + * constraints violations. + */ + __mark_reg32_unbounded(dst_reg); + switch (opcode) { case BPF_ADD: /* @@ -15455,6 +15635,13 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) { u64 val; + /* + * The low 32 bits of a valid pointer may well be zero, hence + * nothing below applies to a 32-bit comparison. + */ + if (is_jmp32) + return -1; + /* arrange that reg2 is a scalar, and reg1 is a pointer */ if (!is_reg_const(reg2, is_jmp32)) { opcode = flip_opcode(opcode); @@ -16021,6 +16208,16 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, return err; } + /* + * Collect the linked registers before env->{true,false}_reg{1,2} setup, + * otherwise ids dropped by collect_linked_regs() would be resurrected + * when env->{true,false}_reg{1,2} are copied back. + */ + if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) + collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); + if (dst_reg->type == SCALAR_VALUE && dst_reg->id) + collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); + is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; env->false_reg1 = *dst_reg; env->false_reg2 = *src_reg; @@ -16075,10 +16272,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * 'this_branch' and 'other_branch' share this history * if parent state is created. */ - if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) - collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); - if (dst_reg->type == SCALAR_VALUE && dst_reg->id) - collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); if (linked_regs.cnt > 1) { err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs)); if (err) @@ -16128,7 +16321,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, */ if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X && __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && - type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) && base_type(src_reg->type) != PTR_TO_BTF_ID && base_type(dst_reg->type) != PTR_TO_BTF_ID) { eq_branch_regs = NULL; @@ -16144,9 +16336,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, break; } if (eq_branch_regs) { - if (type_may_be_null(src_reg->type)) + /* src == dst && dst != NULL => src != NULL */ + if (reg_not_null(env, dst_reg) && type_may_be_null(src_reg->type)) mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]); - else + /* src == dst && src != NULL => dst != NULL */ + if (reg_not_null(env, src_reg) && type_may_be_null(dst_reg->type)) mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); } } @@ -16161,6 +16355,15 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, type_may_be_null(dst_reg->type) && ((BPF_SRC(insn->code) == BPF_K && insn->imm == 0) || (BPF_SRC(insn->code) == BPF_X && bpf_register_is_null(src_reg)))) { + /* + * For BPF_X the zero is a property of this execution path, + * hence src_reg has to be precise. + */ + if (BPF_SRC(insn->code) == BPF_X) { + err = mark_chain_precision(env, insn->src_reg); + if (err) + return err; + } /* Mark all identical registers in each branch as either * safe or unknown depending R == 0 or R != 0 conditional. */ @@ -16243,6 +16446,15 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) verbose(env, "callback function not static\n"); return -EINVAL; } + /* + * When env->subprog_cnt == 1 this instruction won't be rewritten + * to hold a real function address. Assume that no usable program + * combines e.g. main and timer callback and just reject here. + */ + if (subprogno == 0) { + verbose(env, "callback function cannot be the main program\n"); + return -EINVAL; + } dst_reg->type = PTR_TO_FUNC; dst_reg->subprogno = subprogno; @@ -16305,6 +16517,7 @@ static bool may_access_skb(enum bpf_prog_type type) */ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) { + struct bpf_verifier_state *state = env->cur_state; struct bpf_reg_state *regs = cur_regs(env); static const int ctx_reg = BPF_REG_6; u8 mode = BPF_MODE(insn->code); @@ -16315,6 +16528,13 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return -EINVAL; } + for (i = state->curframe; i; i--) { + if (state->frame[i]->in_callback_fn) { + verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n"); + return -EINVAL; + } + } + if (!env->ops->gen_ld_abs) { verifier_bug(env, "gen_ld_abs is null"); return -EFAULT; @@ -16783,6 +17003,10 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, * r0 = *(u64 *)(r10 - 8); r0 += r1; * r0 += r1; exit; * exit; + * + * Both uses of the marks assume that a pattern is entered at its first + * spill and thus executes as a unit, hence a pattern is not grown past + * an instruction targeted by a jump. */ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, struct bpf_subprog_info *subprog, @@ -16821,6 +17045,10 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, for (i = 1, off = lowest_off; i <= ARRAY_SIZE(caller_saved); ++i, off += BPF_REG_SIZE) { if (insn_idx - i < 0 || insn_idx + i >= env->prog->len) break; + /* stx/ldx/call must not be a jump targets, a jump to the first stx is fine */ + if (bpf_is_jump_target(env, insn_idx - i + 1) || + bpf_is_jump_target(env, insn_idx + i)) + break; stx = &insns[insn_idx - i]; ldx = &insns[insn_idx + i]; /* must be a stack spill/fill pair */ @@ -17364,7 +17592,9 @@ static int do_check(struct bpf_verifier_env *env) insn = &insns[env->insn_idx]; insn_aux = &env->insn_aux_data[env->insn_idx]; - if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { + account_processed_insn(env); + + if (env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { verbose(env, "BPF program is too large. Processed %d insn\n", env->insn_processed); @@ -17504,6 +17734,7 @@ static int do_check(struct bpf_verifier_env *env) "speculation barrier after jump instruction may not have the desired effect")) return -EFAULT; process_bpf_exit: + account_current_path(env); mark_verifier_state_scratched(env); err = bpf_update_branch_counts(env, env->cur_state); if (err) @@ -17830,6 +18061,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return -EOPNOTSUPP; } env->prog->aux->arena = (void *)map; + env->prog->jit_required = true; if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { verbose(env, "arena's user address must be set via map_extra or mmap()\n"); return -EINVAL; @@ -17877,6 +18109,7 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) return err; } env->insn_array_maps[env->insn_array_map_cnt++] = map; + env->prog->jit_required = true; } return env->used_map_cnt - 1; @@ -18496,6 +18729,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) ret = do_check(env); out: + account_current_path(env); if (!ret && pop_log) bpf_vlog_reset(&env->log, 0); free_states(env); @@ -18527,7 +18761,6 @@ static int do_check_subprogs(struct bpf_verifier_env *env) struct bpf_prog_aux *aux = env->prog->aux; struct bpf_func_info_aux *sub_aux; int i, ret, new_cnt; - u32 insn_processed; if (!aux->func_info) return 0; @@ -18542,8 +18775,6 @@ static int do_check_subprogs(struct bpf_verifier_env *env) if (!bpf_subprog_is_global(env, i)) continue; - insn_processed = env->insn_processed; - sub_aux = subprog_aux(env, i); if (!sub_aux->called || sub_aux->verified) continue; @@ -18551,7 +18782,6 @@ static int do_check_subprogs(struct bpf_verifier_env *env) env->insn_idx = env->subprog_info[i].start; WARN_ON_ONCE(env->insn_idx == 0); ret = do_check_common(env, i); - env->subprog_info[i].insn_processed = env->insn_processed - insn_processed; if (ret) { return ret; } else if (env->log.level & BPF_LOG_LEVEL) { @@ -18578,12 +18808,10 @@ static int do_check_subprogs(struct bpf_verifier_env *env) static int do_check_main(struct bpf_verifier_env *env) { - u32 insn_processed = env->insn_processed; int ret; env->insn_idx = 0; ret = do_check_common(env, 0); - env->subprog_info[0].insn_processed = env->insn_processed - insn_processed; if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; @@ -18602,10 +18830,10 @@ static void print_verification_stats(struct bpf_verifier_env *env) for (i = 1; i < subprog_cnt; i++) verbose(env, "+%d", env->subprog_info[i].stack_depth); verbose(env, " max %d\n", env->max_stack_depth); - verbose(env, "insns processed %d", env->subprog_info[0].insn_processed); + verbose(env, "insns processed %d", env->subprog_info[0].insns_total); for (i = 1; i < subprog_cnt; i++) if (bpf_subprog_is_global(env, i)) - verbose(env, "+%d", env->subprog_info[i].insn_processed); + verbose(env, "+%d", env->subprog_info[i].insns_total); verbose(env, "\n"); } verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " @@ -18838,6 +19066,9 @@ static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct b const struct btf_type *t; const char *tname; + if (!btf_is_kernel(btf)) + return -EINVAL; + switch (prog->type) { case BPF_PROG_TYPE_TRACING: t = btf_type_by_id(btf, btf_id); @@ -18922,6 +19153,20 @@ btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id return btf_type_by_id(btf, func->type); } +static bool attach_uses_trampoline_retval(enum bpf_attach_type type) +{ + switch (type) { + case BPF_MODIFY_RETURN: + case BPF_TRACE_FEXIT: + case BPF_TRACE_FEXIT_MULTI: + case BPF_TRACE_FSESSION: + case BPF_TRACE_FSESSION_MULTI: + return true; + default: + return false; + } +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -19186,6 +19431,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, if (ret < 0) return ret; + if (tgt_info->fmodel.ret_size > 8 && + attach_uses_trampoline_retval(prog->expected_attach_type)) { + bpf_log(log, + "Attach to function %s with a >8 byte return value is not supported for this attach type\n", + tname); + return -EOPNOTSUPP; + } + /* * *.multi programs don't need an address during program * verification, we just take the module ref if needed. @@ -19421,7 +19674,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) return -ENOMEM; if (tgt_prog && tgt_prog->aux->tail_call_reachable) - tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX; + bpf_trampoline_set_flags(tr, BPF_TRAMP_F_TAIL_CALL_CTX); prog->aux->dst_trampoline = tr; return 0; @@ -19460,6 +19713,9 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel); if (err < 0) return err; + if (tgt_info->fmodel.ret_size > 8 && + attach_uses_trampoline_retval(prog->expected_attach_type)) + return -EOPNOTSUPP; if (btf_is_module(btf)) { /* The bpf program already holds reference to module. */ if (WARN_ON_ONCE(!prog->aux->mod)) @@ -19483,13 +19739,25 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct btf *bpf_get_btf_vmlinux(void) { - if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { + /* Pairs with the smp_store_release() on the parse path below. */ + struct btf *btf = smp_load_acquire(&btf_vmlinux); + + if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { mutex_lock(&bpf_verifier_lock); - if (!btf_vmlinux) - btf_vmlinux = btf_parse_vmlinux(); + btf = btf_vmlinux; + if (!btf) { + btf = btf_parse_vmlinux(); + /* + * Order the parsed BTF contents and the globals the + * parse populated (e.g. bpf_ctx_convert.t) before + * the pointer publication. Pairs with the acquire + * on the lockless fast path above. + */ + smp_store_release(&btf_vmlinux, btf); + } mutex_unlock(&bpf_verifier_lock); } - return btf_vmlinux; + return btf; } /* diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index b5b461d4418b1..1f0393ad0b18a 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -5228,10 +5228,13 @@ struct task_struct *css_task_iter_next(struct css_task_iter *it) if (it->flags & CSS_TASK_ITER_SKIPPED) css_task_iter_advance(it); - if (it->task_pos) { + while (it->task_pos && !it->cur_task) { it->cur_task = list_entry(it->task_pos, struct task_struct, cg_list); - get_task_struct(it->cur_task); + /* a task on dying_tasks with zero refcount is only valid for + * RCU readers, not even interesting for + * CSS_TASK_ITER_WITH_DEAD, find another one */ + it->cur_task = tryget_task_struct(it->cur_task); css_task_iter_advance(it); } diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index f7aaf01f7cd5e..140700e5e236d 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -165,7 +165,7 @@ struct cpuset { * number of SCHED_DEADLINE tasks attached to this cpuset, so that we * know when to rebuild associated root domain bandwidth information. */ - int nr_deadline_tasks; + atomic_t nr_deadline_tasks; int nr_migrate_dl_tasks; /* DL bandwidth that needs destination reservation for this attach. */ u64 sum_migrate_dl_bw; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 45944b3e31ca4..c43f15c9917c1 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -222,14 +222,14 @@ void inc_dl_tasks_cs(struct task_struct *p) { struct cpuset *cs = task_cs(p); - cs->nr_deadline_tasks++; + atomic_inc(&cs->nr_deadline_tasks); } void dec_dl_tasks_cs(struct task_struct *p) { struct cpuset *cs = task_cs(p); - cs->nr_deadline_tasks--; + atomic_dec(&cs->nr_deadline_tasks); } static inline bool is_partition_valid(const struct cpuset *cs) @@ -918,7 +918,7 @@ static void dl_update_tasks_root_domain(struct cpuset *cs) struct css_task_iter it; struct task_struct *task; - if (cs->nr_deadline_tasks == 0) + if (atomic_read(&cs->nr_deadline_tasks) == 0) return; css_task_iter_start(&cs->css, 0, &it); @@ -1089,12 +1089,35 @@ void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) * @cs: the cpuset the need to recompute the new effective_cpus mask * @parent: the parent cpuset * + * For v2, the parent's effective_cpus is inherited if cpumask is empty. * The result is valid only if the given cpuset isn't a partition root. */ static void compute_effective_cpumask(struct cpumask *new_cpus, struct cpuset *cs, struct cpuset *parent) { - cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + bool has_cpus; + + has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + if (!has_cpus && is_in_v2_mode()) + cpumask_copy(new_cpus, parent->effective_cpus); +} + +/** + * compute_effective_nodemask - Compute the effective nodemask of the cpuset + * @new_mems: the temp variable for the new effective_mems mask + * @cs: the cpuset the need to recompute the new effective_mems mask + * @parent: the parent cpuset + * + * For v2, the parent's effective_mems is inherited if nodemask is empty. + */ +static void compute_effective_nodemask(nodemask_t *new_mems, + struct cpuset *cs, struct cpuset *parent) +{ + bool has_mems; + + has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems); + if (!has_mems && is_in_v2_mode()) + nodes_copy(*new_mems, parent->effective_mems); } /* @@ -1197,6 +1220,28 @@ static void reset_partition_data(struct cpuset *cs) cpumask_copy(cs->effective_cpus, parent->effective_cpus); } +/* Return true if isolated_cpus changes. */ +static bool isolated_cpu_update(int new_prs, int cpu) +{ + lockdep_assert_held(&callback_lock); + lockdep_assert_held(&cpuset_mutex); + + if (new_prs == PRS_ISOLATED) { + if (cpumask_test_cpu(cpu, isolated_cpus)) + return false; + cpumask_set_cpu(cpu, isolated_cpus); + return true; + } + + /* CPUs isolated at boot must remain isolated. */ + if (!cpumask_test_cpu(cpu, + housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)) || + !cpumask_test_cpu(cpu, isolated_cpus)) + return false; + cpumask_clear_cpu(cpu, isolated_cpus); + return true; +} + /* * isolated_cpus_update - Update the isolated_cpus mask * @old_prs: old partition_root_state @@ -1205,19 +1250,16 @@ static void reset_partition_data(struct cpuset *cs) */ static void isolated_cpus_update(int old_prs, int new_prs, struct cpumask *xcpus) { + bool updated = false; + int cpu; + WARN_ON_ONCE(old_prs == new_prs); lockdep_assert_held(&callback_lock); lockdep_assert_held(&cpuset_mutex); - if (new_prs == PRS_ISOLATED) { - if (cpumask_subset(xcpus, isolated_cpus)) - return; - cpumask_or(isolated_cpus, isolated_cpus, xcpus); - } else { - if (!cpumask_intersects(xcpus, isolated_cpus)) - return; - cpumask_andnot(isolated_cpus, isolated_cpus, xcpus); - } - update_housekeeping = true; + for_each_cpu(cpu, xcpus) + updated |= isolated_cpu_update(new_prs, cpu); + if (updated) + update_housekeeping = true; } /* @@ -1510,10 +1552,11 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs, * above it or remote partition root underneath it is not allowed. */ compute_excpus(cs, tmp->new_cpus); - WARN_ON_ONCE(cpumask_intersects(tmp->new_cpus, subpartitions_cpus)); if (!cpumask_intersects(tmp->new_cpus, cpu_active_mask) || cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus)) return PERR_INVCPUS; + if (cpumask_intersects(tmp->new_cpus, subpartitions_cpus)) + return PERR_NOCPUS; if (((new_prs == PRS_ISOLATED) && !isolated_cpus_can_update(tmp->new_cpus, NULL)) || prstate_housekeeping_conflict(new_prs, tmp->new_cpus)) @@ -1525,7 +1568,7 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs, cpumask_copy(cs->effective_xcpus, tmp->new_cpus); spin_unlock_irq(&callback_lock); cpuset_force_rebuild(); - cs->prs_err = 0; + WRITE_ONCE(cs->prs_err, 0); /* * Propagate changes in top_cpuset's effective_cpus down the hierarchy. @@ -1599,7 +1642,7 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus)); if (cpumask_empty(excpus)) { - cs->prs_err = PERR_CPUSEMPTY; + WRITE_ONCE(cs->prs_err, PERR_CPUSEMPTY); goto invalidate; } @@ -1614,13 +1657,13 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, if (adding) { WARN_ON_ONCE(cpumask_intersects(tmp->addmask, subpartitions_cpus)); if (!capable(CAP_SYS_ADMIN)) - cs->prs_err = PERR_ACCESS; + WRITE_ONCE(cs->prs_err, PERR_ACCESS); else if (cpumask_intersects(tmp->addmask, subpartitions_cpus) || cpumask_subset(top_cpuset.effective_cpus, tmp->addmask)) - cs->prs_err = PERR_NOCPUS; + WRITE_ONCE(cs->prs_err, PERR_NOCPUS); else if ((prs == PRS_ISOLATED) && !isolated_cpus_can_update(tmp->addmask, tmp->delmask)) - cs->prs_err = PERR_HKEEPING; + WRITE_ONCE(cs->prs_err, PERR_HKEEPING); if (cs->prs_err) goto invalidate; } @@ -2048,13 +2091,13 @@ static void compute_partition_effective_cpumask(struct cpuset *cs, * partition root. */ WARN_ON_ONCE(is_remote_partition(child)); - child->prs_err = 0; + WRITE_ONCE(child->prs_err, 0); if (!cpumask_subset(child->effective_xcpus, cs->effective_xcpus)) - child->prs_err = PERR_INVCPUS; + WRITE_ONCE(child->prs_err, PERR_INVCPUS); else if (populated && cpumask_subset(new_ecpus, child->effective_xcpus)) - child->prs_err = PERR_NOCPUS; + WRITE_ONCE(child->prs_err, PERR_NOCPUS); if (child->prs_err) { int old_prs = child->partition_root_state; @@ -2143,15 +2186,6 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, goto update_parent_effective; } - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some CPUs unless - * it is a partition root that has explicitly distributed - * out all its CPUs. - */ - if (is_in_v2_mode() && !remote && cpumask_empty(tmp->new_cpus)) - cpumask_copy(tmp->new_cpus, parent->effective_cpus); - /* * Skip the whole subtree if * 1) the cpumask remains the same, @@ -2367,8 +2401,10 @@ static void partition_cpus_change(struct cpuset *cs, struct cpuset *trialcs, return; prs_err = validate_partition(cs, trialcs); - if (prs_err) - trialcs->prs_err = cs->prs_err = prs_err; + if (prs_err) { + WRITE_ONCE(cs->prs_err, prs_err); + trialcs->prs_err = prs_err; + } if (is_remote_partition(cs)) { if (trialcs->prs_err) @@ -2697,14 +2733,7 @@ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) cpuset_for_each_descendant_pre(cp, pos_css, cs) { struct cpuset *parent = parent_cs(cp); - bool has_mems = nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); - - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some MEMs. - */ - if (is_in_v2_mode() && !has_mems) - *new_mems = parent->effective_mems; + compute_effective_nodemask(new_mems, cp, parent); /* Skip the whole subtree if the nodemask remains the same. */ if (nodes_equal(*new_mems, cp->effective_mems)) { @@ -3209,8 +3238,8 @@ static void cpuset_attach(struct cgroup_taskset *tset) cs->old_mems_allowed = cpuset_attach_nodemask_to; if (cs->nr_migrate_dl_tasks) { - cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks; - oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks; + atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); + atomic_sub(cs->nr_migrate_dl_tasks, &oldcs->nr_deadline_tasks); reset_migrate_dl_data(cs); } @@ -3778,7 +3807,7 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) parent = parent_cs(cs); compute_effective_cpumask(&new_cpus, cs, parent); - nodes_and(new_mems, cs->mems_allowed, parent->effective_mems); + compute_effective_nodemask(&new_mems, cs, parent); if (!tmp || !cs->partition_root_state) goto update_tasks; @@ -3794,7 +3823,7 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) if (remote && (cpumask_empty(subpartitions_cpus) || (cpumask_empty(&new_cpus) && partition_is_populated(cs, NULL)))) { - cs->prs_err = PERR_HOTPLUG; + WRITE_ONCE(cs->prs_err, PERR_HOTPLUG); remote_partition_disable(cs, tmp); compute_effective_cpumask(&new_cpus, cs, parent); remote = false; diff --git a/kernel/cgroup/pids.c b/kernel/cgroup/pids.c index ecbb839d2acba..78cdc0558d0c5 100644 --- a/kernel/cgroup/pids.c +++ b/kernel/cgroup/pids.c @@ -253,6 +253,11 @@ static void pids_event(struct pids_cgroup *pids_forking, } if (!cgroup_subsys_on_dfl(pids_cgrp_subsys) || cgrp_dfl_root.flags & CGRP_ROOT_PIDS_LOCAL_EVENTS) { + /* + * pids.events reports the local counter on legacy hierarchies + * and when pids_localevents is enabled. + */ + cgroup_file_notify(&p->events_file); cgroup_file_notify(&p->events_local_file); return; } diff --git a/kernel/crash_dump_dm_crypt.c b/kernel/crash_dump_dm_crypt.c index cb875ddb6ba68..c685497cd470e 100644 --- a/kernel/crash_dump_dm_crypt.c +++ b/kernel/crash_dump_dm_crypt.c @@ -81,7 +81,6 @@ static int add_key_to_keyring(struct dm_crypt_key *dm_key, kexec_dprintk("Error when adding key"); } - key_ref_put(keyring_ref); return r; } @@ -104,6 +103,7 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) struct dm_crypt_key *key; size_t keys_header_size; key_ref_t keyring_ref; + int ret = 0; u64 addr; /* find the target keyring (which must be writable) */ @@ -118,7 +118,8 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) dm_crypt_keys_read((char *)&key_count, sizeof(key_count), &addr); if (key_count > KEY_NUM_MAX) { kexec_dprintk("Failed to read the number of dm-crypt keys\n"); - return -1; + ret = -1; + goto out; } kexec_dprintk("There are %u keys\n", key_count); @@ -126,8 +127,10 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) keys_header_size = get_keys_header_size(key_count); keys_header = kzalloc(keys_header_size, GFP_KERNEL); - if (!keys_header) - return -ENOMEM; + if (!keys_header) { + ret = -ENOMEM; + goto out; + } dm_crypt_keys_read((char *)keys_header, keys_header_size, &addr); @@ -137,7 +140,9 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) add_key_to_keyring(key, keyring_ref); } - return 0; +out: + key_ref_put(keyring_ref); + return ret; } static int read_key_from_user_keyring(struct dm_crypt_key *dm_key) diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c index bcdc0f76d2e80..b6fe902c6a6db 100644 --- a/kernel/dma/coherent.c +++ b/kernel/dma/coherent.c @@ -351,8 +351,7 @@ static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev) min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit)) dev_warn(dev, "reserved memory is beyond device's set DMA address range\n"); - dma_assign_coherent_memory(dev, mem); - return 0; + return dma_assign_coherent_memory(dev, mem); } static void rmem_dma_device_release(struct reserved_mem *rmem, diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index f754079a287d6..66093460584e8 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -140,21 +140,22 @@ static bool numa_cma_configured __initdata; static int __init early_numa_cma(char *p) { int nid, count = 0; - unsigned long tmp; + unsigned long node; + phys_addr_t size; char *s = p; while (*s) { - if (sscanf(s, "%lu%n", &tmp, &count) != 1) + if (sscanf(s, "%lu%n", &node, &count) != 1) break; if (s[count] == ':') { - if (tmp >= MAX_NUMNODES) + if (node >= MAX_NUMNODES) break; - nid = array_index_nospec(tmp, MAX_NUMNODES); + nid = array_index_nospec(node, MAX_NUMNODES); s += count + 1; - tmp = memparse(s, &s); - numa_cma_size[nid] = tmp; + size = memparse(s, &s); + numa_cma_size[nid] = size; if (*s == ',') s++; @@ -207,9 +208,10 @@ static void __init dma_numa_cma_reserve(void) pernuma_size_bytes = cma_get_size(dma_contiguous_default_area); for_each_node(nid) { - int size, ret; + phys_addr_t size; char name[CMA_MAX_NAME]; struct cma **cma; + int ret; if (!node_online(nid)) { if (pernuma_size_bytes || numa_cma_size[nid]) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 4391b797d4db8..b4cb2c03e5d7c 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -164,22 +164,21 @@ static bool dma_direct_use_pool(struct device *dev, gfp_t gfp) return !gfpflags_allow_blocking(gfp) && !is_swiotlb_for_alloc(dev); } -static void *dma_direct_alloc_from_pool(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp) +static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size, + dma_addr_t *dma_handle, void **cpu_addr, gfp_t gfp) { struct page *page; u64 phys_limit; - void *ret; if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))) return NULL; gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit); - page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok); + page = dma_alloc_from_pool(dev, size, cpu_addr, gfp, dma_coherent_ok); if (!page) return NULL; *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); - return ret; + return page; } static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size, @@ -247,8 +246,11 @@ void *dma_direct_alloc(struct device *dev, size_t size, * the atomic pools instead if we aren't allowed block. */ if ((remap || force_dma_unencrypted(dev)) && - dma_direct_use_pool(dev, gfp)) - return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); + dma_direct_use_pool(dev, gfp)) { + page = dma_direct_alloc_from_pool(dev, size, dma_handle, + &ret, gfp); + return page ? ret : NULL; + } /* we always manually zero the memory once we are done */ page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true); @@ -357,7 +359,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size, void *ret; if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp)) - return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); + return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp); page = __dma_direct_alloc_pages(dev, size, gfp, false); if (!page) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 1abd3e6146f45..bd53931337533 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -266,9 +266,9 @@ void __init swiotlb_update_mem_attributes(void) } static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start, - unsigned long nslabs, bool late_alloc, unsigned int nareas) + void *vaddr, unsigned long nslabs, bool late_alloc, + unsigned int nareas) { - void *vaddr = phys_to_virt(start); unsigned long bytes = nslabs << IO_TLB_SHIFT, i; mem->nslabs = nslabs; @@ -409,7 +409,7 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, return; } - swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas); + swiotlb_init_io_tlb_pool(mem, __pa(tlb), tlb, nslabs, false, nareas); add_mem_pool(&io_tlb_default_mem, mem); if (flags & SWIOTLB_VERBOSE) @@ -507,7 +507,7 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, set_memory_decrypted((unsigned long)vstart, (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT); - swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true, + swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), vstart, nslabs, true, nareas); add_mem_pool(&io_tlb_default_mem, mem); @@ -605,25 +605,26 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit) * @bytes: Size of the buffer. * @phys_limit: Maximum allowed physical address of the buffer. * @gfp: GFP flags for the allocation. + * @vaddr: Receives the virtual address for the allocated buffer. * * Return: Allocated pages, or %NULL on allocation failure. */ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes, - u64 phys_limit, gfp_t gfp) + u64 phys_limit, gfp_t gfp, void **vaddr) { struct page *page; + *vaddr = NULL; + /* * Allocate from the atomic pools if memory is encrypted and * the allocation is atomic, because decrypting may block. */ if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) { - void *vaddr; - if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)) return NULL; - return dma_alloc_from_pool(dev, bytes, &vaddr, gfp, + return dma_alloc_from_pool(dev, bytes, vaddr, gfp, dma_coherent_ok); } @@ -645,6 +646,8 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes, return NULL; } + if (page) + *vaddr = phys_to_virt(page_to_phys(page)); return page; } @@ -685,6 +688,7 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, { struct io_tlb_pool *pool; unsigned int slot_order; + void *tlb_vaddr; struct page *tlb; size_t pool_size; size_t tlb_size; @@ -701,7 +705,8 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, pool->areas = (void *)pool + sizeof(*pool); tlb_size = nslabs << IO_TLB_SHIFT; - while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) { + while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp, + &tlb_vaddr))) { if (nslabs <= minslabs) goto error_tlb; nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE); @@ -715,11 +720,12 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, if (!pool->slots) goto error_slots; - swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas); + swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), tlb_vaddr, nslabs, + true, nareas); return pool; error_slots: - swiotlb_free_tlb(page_address(tlb), tlb_size); + swiotlb_free_tlb(tlb_vaddr, tlb_size); error_tlb: kfree(pool); error: @@ -861,7 +867,6 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT; phys_addr_t orig_addr = mem->slots[index].orig_addr; size_t alloc_size = mem->slots[index].alloc_size; - unsigned long pfn = PFN_DOWN(orig_addr); unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start; int tlb_offset; @@ -894,7 +899,8 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size size = alloc_size; } - if (PageHighMem(pfn_to_page(pfn))) { + if (PhysHighMem(orig_addr)) { + unsigned long pfn = PFN_DOWN(orig_addr); unsigned int offset = orig_addr & ~PAGE_MASK; struct page *page; unsigned int sz = 0; @@ -1851,7 +1857,8 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem, set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), rmem->size >> PAGE_SHIFT); - swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs, + swiotlb_init_io_tlb_pool(pool, rmem->base, phys_to_virt(rmem->base), + nslabs, false, nareas); mem->force_bounce = true; mem->for_alloc = true; diff --git a/kernel/events/core.c b/kernel/events/core.c index 1a73ba0747df8..d8f9efaca9893 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3764,6 +3764,9 @@ static void perf_ctx_sched_task_cb(struct perf_event_context *ctx, list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { cpc = this_cpc(pmu_ctx->pmu); + if (cpc->task_epc != pmu_ctx) + continue; + if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task) pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in); } @@ -3914,7 +3917,7 @@ static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc, perf_ctx_lock(cpuctx, cpuctx->task_ctx); perf_pmu_disable(pmu); - pmu->sched_task(cpc->task_epc, task, sched_in); + pmu->sched_task(&cpc->epc, task, sched_in); perf_pmu_enable(pmu); perf_ctx_unlock(cpuctx, cpuctx->task_ctx); @@ -3924,15 +3927,17 @@ static void perf_pmu_sched_task(struct task_struct *prev, struct task_struct *next, bool sched_in) { - struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); - struct perf_cpu_pmu_context *cpc; + struct perf_cpu_pmu_context *cpc, *cpc2; - /* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */ - if (prev == next || cpuctx->task_ctx) + if (prev == next) return; - list_for_each_entry(cpc, this_cpu_ptr(&sched_cb_list), sched_cb_entry) + list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) { + if (cpc->task_epc) + continue; + __perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in); + } } static void perf_event_switch(struct task_struct *task, @@ -5454,6 +5459,8 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache, } if (refcount_inc_not_zero(&old->refcount)) { + if (global) + old->global = true; free_perf_ctx_data(cd); /* unused */ return 0; } @@ -6350,6 +6357,9 @@ static DEFINE_MUTEX(perf_mediated_pmu_mutex); /* !exclude_guest event of PMU with PERF_PMU_CAP_MEDIATED_VPMU */ static inline bool is_include_guest_event(struct perf_event *event) { + if (!event->pmu) + return false; + if ((event->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU) && !event->attr.exclude_guest) return true; @@ -7029,7 +7039,6 @@ static void perf_mmap_close(struct vm_area_struct *vma) mapped_f unmapped = get_mapped(event, event_unmapped); struct perf_buffer *rb = ring_buffer_get(event); struct user_struct *mmap_user = rb->mmap_user; - bool detach_rest = false; /* FIXIES vs perf_pmu_unregister() */ if (unmapped) @@ -7060,17 +7069,18 @@ static void perf_mmap_close(struct vm_area_struct *vma) mutex_unlock(&rb->aux_mutex); } - if (refcount_dec_and_test(&rb->mmap_count)) - detach_rest = true; - - if (!refcount_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) - goto out_put; - - ring_buffer_attach(event, NULL); - mutex_unlock(&event->mmap_mutex); + /* + * Drop references in reverse order of perf_mmap() to prevent + * rb revival after rb->mmap_count reaches zero. + */ + if (refcount_dec_and_mutex_lock(&event->mmap_count, + &event->mmap_mutex)) { + ring_buffer_attach(event, NULL); + mutex_unlock(&event->mmap_mutex); + } /* If there's still other mmap()s of this buffer, we're done. */ - if (!detach_rest) + if (!refcount_dec_and_test(&rb->mmap_count)) goto out_put; /* @@ -12977,6 +12987,7 @@ static void __pmu_detach_event(struct pmu *pmu, struct perf_event *event, exclusive_event_destroy(event); module_put(pmu->module); + mediated_pmu_unaccount_event(event); event->pmu = NULL; /* force fault instead of UAF */ } diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 9fe92161715e0..1b1ffe0533e58 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -509,7 +509,10 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size) /* * Only send RECORD_AUX if we have something useful to communicate * - * Note: the OVERWRITE records by themselves are not considered + * PMU_FORMAT bits identify the PMU type rather than an AUX event + * has occurred, so ignore them for zero-sized records. + * + * The OVERWRITE records by themselves are not considered * useful, as they don't communicate any *new* information, * aside from the short-lived offset, that becomes history at * the next event sched-in and therefore isn't useful. @@ -518,7 +521,9 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size) * offset. So, from now on we don't output AUX records that * have *only* OVERWRITE flag set. */ - if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE)) + if (size || + (handle->aux_flags & ~(u64)(PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK | + PERF_AUX_FLAG_OVERWRITE))) perf_event_aux_event(handle->event, aux_head, size, handle->aux_flags); diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 6300b216012cb..e4f526c9bfbf0 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1806,14 +1806,6 @@ static struct xol_area *get_xol_area(void) return area; } -void __weak arch_uprobe_clear_state(struct mm_struct *mm) -{ -} - -void __weak arch_uprobe_init_state(struct mm_struct *mm) -{ -} - /* * uprobe_clear_state - Free the area allocated for slots. */ @@ -1825,8 +1817,6 @@ void uprobe_clear_state(struct mm_struct *mm) delayed_uprobe_remove(NULL, mm); mutex_unlock(&delayed_uprobe_lock); - arch_uprobe_clear_state(mm); - if (!area) return; diff --git a/kernel/exit.c b/kernel/exit.c index 2c0b1c02920f4..200a936f601a8 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -262,8 +262,11 @@ void release_task(struct task_struct *p) pidfs_exit(p); cgroup_task_release(p); - /* Retrieve @thread_pid before __unhash_process() may set it to NULL. */ - thread_pid = task_pid(p); + /* + * Pin @thread_pid before __unhash_process() clears it. The last + * PIDTYPE detach can otherwise free it before proc_flush_pid(). + */ + thread_pid = get_pid(task_pid(p)); write_lock_irq(&tasklist_lock); ptrace_release_task(p); @@ -292,20 +295,21 @@ void release_task(struct task_struct *p) } write_unlock_irq(&tasklist_lock); - /* @thread_pid can't go away until free_pids() below */ proc_flush_pid(thread_pid); + put_pid(thread_pid); exit_cred_namespaces(p); add_device_randomness(&p->se.sum_exec_runtime, sizeof(p->se.sum_exec_runtime)); free_pids(post.pids); release_thread(p); /* - * This task was already removed from the process/thread/pid lists - * and lock_task_sighand(p) can't succeed. Nobody else can touch - * ->pending or, if group dead, signal->shared_pending. We can call - * flush_sigqueue() lockless. + * This task was already removed from the process/thread/pid lists and + * lock_task_sighand(p) can't succeed. If it's the group leader then + * flush tsk->signal->shared_pending. tsk->pending has been flushed + * already in exit_signals(). Nothing else can touch + * signal->shared_pending anymore, so flush_sigqueue() can be invoked + * lockless. */ - flush_sigqueue(&p->pending); if (thread_group_leader(p)) flush_sigqueue(&p->signal->shared_pending); @@ -555,18 +559,23 @@ void mm_update_next_owner(struct mm_struct *mm) */ static void exit_mm_sched_cache(struct mm_struct *mm) { + struct sched_cache_group *grp; unsigned long fp, sub; if (!current->total_numa_faults) return; /* * No lock protection due to performance considerations. - * Make sure mm->sc_stat.footprint does not become + * Make sure the group footprint does not become * negative. */ - fp = READ_ONCE(mm->sc_stat.footprint); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return; + + fp = READ_ONCE(grp->footprint); sub = min(fp, current->total_numa_faults); - WRITE_ONCE(mm->sc_stat.footprint, fp - sub); + WRITE_ONCE(grp->footprint, fp - sub); } #else static inline void exit_mm_sched_cache(struct mm_struct *mm) diff --git a/kernel/fork.c b/kernel/fork.c index f0e2e131a9a5a..2c88568b0463d 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1069,7 +1069,6 @@ static void mm_init_uprobes_state(struct mm_struct *mm) { #ifdef CONFIG_UPROBES mm->uprobes_state.xol_area = NULL; - arch_uprobe_init_state(mm); #endif } @@ -1978,9 +1977,9 @@ static bool need_futex_hash_allocate_default(u64 clone_flags) { /* * Allocate a default futex hash for any sibling that will - * share the parent's mm, except vfork. + * share the parent's mm. */ - return (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM; + return clone_flags & CLONE_VM; } /* @@ -2115,6 +2114,11 @@ __latent_entropy struct task_struct *copy_process( p = dup_task_struct(current, node); if (!p) goto fork_out; + /* + * Must run before the first fallible op, so error paths never + * free the parent's ret_stack. + */ + ftrace_graph_init_task(p); retval = copy_exec_state(clone_flags, p); if (retval) goto bad_fork_free; @@ -2141,8 +2145,6 @@ __latent_entropy struct task_struct *copy_process( */ p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; - ftrace_graph_init_task(p); - rt_mutex_init_task(p); raw_spin_lock_init(&p->blocked_lock); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 128c5752f225c..be4aeacd4b540 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -45,6 +45,7 @@ #include #include #include +#include #include @@ -143,8 +144,14 @@ static bool futex_private_hash_get(struct futex_private_hash *fph) void futex_private_hash_put(struct futex_private_hash *fph) { - if (fph && futex_ref_put(fph)) - wake_up_var(fph->mm); + struct mm_struct *mm; + + if (!fph) + return; + + mm = fph->mm; + if (futex_ref_put(fph)) + wake_up_var(mm); } static struct futex_hash_bucket * @@ -1524,14 +1531,12 @@ static void futex_cleanup_begin(struct task_struct *tsk) raw_spin_unlock_irq(&tsk->pi_lock); } -static void futex_cleanup_end(struct task_struct *tsk, int state) +static void futex_cleanup_end(struct task_struct *tsk) __releases(&tsk->futex.exit_mutex) { - /* - * Lockless store. The only side effect is that an observer might - * take another loop until it becomes visible. - */ - tsk->futex.state = state; + scoped_guard(raw_spinlock_irq, &tsk->pi_lock) + tsk->futex.state = FUTEX_STATE_DEAD; + /* * Drop the exit protection. This unblocks waiters which observed * FUTEX_STATE_EXITING to reevaluate the state. @@ -1539,29 +1544,49 @@ static void futex_cleanup_end(struct task_struct *tsk, int state) mutex_unlock(&tsk->futex.exit_mutex); } -void futex_exec_release(struct task_struct *tsk) +void futex_exit_release(struct task_struct *tsk) { - /* - * The state handling is done for consistency, but in the case of - * exec() there is no way to prevent further damage as the PID stays - * the same. But for the unlikely and arguably buggy case that a - * futex is held on exec(), this provides at least as much state - * consistency protection which is possible. - */ futex_cleanup_begin(tsk); futex_cleanup(tsk); + futex_cleanup_end(tsk); +} + +void futex_exec_release(struct task_struct *tsk) +{ /* - * Reset the state to FUTEX_STATE_OK. The task is alive and about - * exec a new binary. + * exec() makes it interesting for futexes because the TID of the task + * stays the same, but from a futex perspective the task has to be + * treated like an exiting task. This is especially important for the + * sanity check for private futexes in attach_to_pi_owner() which + * compares the owner's mm with the waiter's mm. + * + * That check would give the wrong answer if futex_cleanup_end() would + * set the state to FUTEX_STATE_OK as long as the task still has the old + * mm. + * + * After the task has switched to the new mm it sets it to + * FUTEX_STATE_OK again in futex_exec_done(). */ - futex_cleanup_end(tsk, FUTEX_STATE_OK); + futex_exit_release(tsk); } -void futex_exit_release(struct task_struct *tsk) +/* + * exec() has switched to the new mm. Futex operations are safe again. + */ +void futex_exec_done(struct task_struct *tsk) { - futex_cleanup_begin(tsk); - futex_cleanup(tsk); - futex_cleanup_end(tsk, FUTEX_STATE_DEAD); + /* + * This store does not have to take tsk::futex::exit_mutex because the + * phase where waiters block on it during state FUTEX_STATE_EXITING has + * been finished when futex_cleanup_end() set the state to + * FUTEX_STATE_DEAD. + * + * This transitions back from FUTEX_STATE_DEAD to FUTEX_STATE_OK. The + * ordering guarantee required here is that the previous store to + * tsk::mm in the calling code cannot be reordered against this store. + */ + guard(raw_spinlock_irq)(&tsk->pi_lock); + tsk->futex.state = FUTEX_STATE_OK; } static void futex_hash_bucket_init(struct futex_hash_bucket *fhb) @@ -1842,14 +1867,18 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) } if (!mm->futex.phash.ref) { + unsigned int __percpu *ref = alloc_percpu(unsigned int); + + if (!ref) + return -ENOMEM; + /* - * This will always be allocated by the first thread and - * therefore requires no locking. + * Tasks sharing the mm can run this concurrently, so take the + * initial reference before publishing the counter. */ - mm->futex.phash.ref = alloc_percpu(unsigned int); - if (!mm->futex.phash.ref) - return -ENOMEM; - this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */ + this_cpu_inc(*ref); /* 0 -> 1 */ + if (cmpxchg(&mm->futex.phash.ref, NULL, ref)) + free_percpu(ref); } fph = kvzalloc(struct_size(fph, queues, hash_slots), @@ -1865,11 +1894,35 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) futex_hash_bucket_init(&fph->queues[i]); if (custom) { + struct wait_bit_queue_entry __wbq_entry; + struct wait_queue_head *__wq_head; + /* * Only let prctl() wait / retry; don't unduly delay clone(). */ again: - wait_var_event(mm, futex_pivot_pending(mm)); + __wq_head = __var_waitqueue(mm); + init_wait_var_entry(&__wbq_entry, mm, 0); + __wbq_entry.wq_entry.func = woken_wake_bit_function; + add_wait_queue(__wq_head, &__wbq_entry.wq_entry); + + /* + * add_wait_queue() futex_ref_put() + * MB (this) MB (implied) + * futex_pivot_pending() wake_up_var() + * waitqueue_active() + * + * Notably, it must not be possible to see + * !futex_pivot_pending() && !waitqueue_active(). + */ + smp_mb(); + + while (!futex_pivot_pending(mm) && + wait_woken(&__wbq_entry.wq_entry, TASK_UNINTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT)) + /* empty */; + + remove_wait_queue(__wq_head, &__wbq_entry.wq_entry); } scoped_guard(mutex, &mm->futex.phash.lock) { diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 795011ea1202f..98f1b962e59a0 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -193,6 +193,58 @@ void put_pi_state(struct futex_pi_state *pi_state) * pi_mutex->wait_lock * p->pi_lock * + * Futex kernel state: + * + * The kernel tracks the task state in p::futex::state to protect against exit() + * and exec(). The states are: + * + * - FUTEX_STATE_OK when the task is alive and waiters can be attached + * + * - FUTEX_STATE_EXITING when the task cleans up the robust list and PI + * state. Concurrent waiters cannot attach anymore and have to wait until the + * cleanup is finished to re-evaluate the potential changes caused by the + * robust list and PI state cleanups. + * + * - FUTEX_STATE_DEAD when the task has cleaned up the robust list. This state + * is set independent of exit() or exec(). In the exit() case the task is + * gone. In the exec() case this ensures that nothing can attach to the task + * after cleaning up the robust list and PI state before it has switched to + * the new mm. From a futex point of view the task is dead until it sets the + * state to FUTEX_STATE_OK again after switching to the new mm. + * + * The valid state transitions for exit(): + * + * FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD + * + * The valid state transitions for exec(): + * + * FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD -> FUTEX_STATE_OK + * + * The state has two related locks: + * + * 1) p::pi_lock + * + * p::pi_lock has to be taken by the waiter when evaluating the state to + * protect against a concurrent exit/exec cleanup by the owner. If the state + * is OK then the waiter can be attached to the owner while still holding + * pi_lock. + * + * The cleanup code has to hold it for all state transitions to ensure that + * the stores to the state cannot be reordered against previous stores on + * which the waiter correctness depends on. + * + * 2) p::futex::exit_mutex + * + * The mutex is acquired when the cleanup starts and released at the end. It + * obviously is not serializing the owner's cleanup against itself. It is + * used to avoid a live lock caused by a waiter preempting the owner's + * cleanup. Such a waiter would busy loop forever waiting for the owner to + * finish the cleanup. + * + * To prevent this, waiters have to drop all locks when observing + * FUTEX_STATE_EXITING and block on the mutex. When the owner releases the + * mutex after finishing the cleanup the waiters make progress and + * re-evaluate the situation. */ /* @@ -318,18 +370,10 @@ static int attach_to_pi_state(u32 __user *uaddr, u32 uval, return ret; } -static int handle_exit_race(u32 __user *uaddr, u32 uval, - struct task_struct *tsk) +static int handle_exit_race(u32 __user *uaddr, u32 uval) { u32 uval2; - /* - * If the futex exit state is not yet FUTEX_STATE_DEAD, tell the - * caller that the alleged owner is busy. - */ - if (tsk && tsk->futex.state != FUTEX_STATE_DEAD) - return -EBUSY; - /* * Reread the user space value to handle the following situation: * @@ -427,7 +471,7 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, return -EAGAIN; p = find_get_task_by_vpid(pid); if (!p) - return handle_exit_race(uaddr, uval, NULL); + return handle_exit_race(uaddr, uval); if (unlikely(p->flags & PF_KTHREAD)) { put_task_struct(p); @@ -435,34 +479,55 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, } /* - * We need to look at the task state to figure out, whether the - * task is exiting. To protect against the change of the task state - * in futex_exit_release(), we do this protected by p->pi_lock: + * We need to look at the task state to figure out whether the task is + * exiting. To protect against the change of the task state from + * FUTEX_STATE_OK to FUTEX_STATE_EXISTING in futex_cleanup_begin() it is + * required to do this protected by p->pi_lock, which prevents the owner + * from concurrently starting the exit cleanup. + * + * If the state is FUTEX_STATE_OK pi_lock must be held until the waiter + * is attached to protect against a concurrent exit()/exec(). */ raw_spin_lock_irq(&p->pi_lock); + + /* Validate that the task is ready for futex operations. */ if (unlikely(p->futex.state != FUTEX_STATE_OK)) { /* - * The task is on the way out. When the futex state is - * FUTEX_STATE_DEAD, we know that the task has finished - * the cleanup: + * The task is on the way out. When state is FUTEX_STATE_EXITING + * the cleanup is in progress. To avoid a live lock when the + * waiter preempted the owner, store the task pointer in + * @exiting and keep the reference on the task. The calling code + * will drop all locks, block on @p::futex::exit_mutex and wait + * for the owner to finish the cleanup. Once the owner released + * the mutex the waiter drops the reference count and + * re-evaluates the situation. */ - int ret = handle_exit_race(uaddr, uval, p); + if (p->futex.state == FUTEX_STATE_EXITING) { + raw_spin_unlock_irq(&p->pi_lock); + *exiting = p; + return -EBUSY; + } + + int ret = handle_exit_race(uaddr, uval); raw_spin_unlock_irq(&p->pi_lock); + put_task_struct(p); + return ret; + } + + if (IS_ENABLED(CONFIG_MMU) && futex_key_is_private(key)) { /* - * If the owner task is between FUTEX_STATE_EXITING and - * FUTEX_STATE_DEAD then store the task pointer and keep - * the reference on the task struct. The calling code will - * drop all locks, wait for the task to reach - * FUTEX_STATE_DEAD and then drop the refcount. This is - * required to prevent a live lock when the current task - * preempted the exiting task between the two states. + * A private futex key holds a pointer to the waiter's mm + * without holding a reference on it. So it must not be attached + * to an owner in a different address space. Otherwise that + * owner's exit cleanup could access the private hash after the + * key's mm is freed. */ - if (ret == -EBUSY) - *exiting = p; - else + if (unlikely(p->mm != key->private.mm)) { + raw_spin_unlock_irq(&p->pi_lock); put_task_struct(p); - return ret; + return -EPERM; + } } __attach_to_pi_owner(p, key, ps); @@ -1005,17 +1070,11 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl * Caution; releasing @hb in-scope. The hb->lock is still locked * while the reference is dropped. The reference can not be dropped * after the unlock because if a user initiated resize is in progress - * then we might need to wake him. This can not be done after the - * rt_mutex_pre_schedule() invocation. The hb will remain valid because - * the thread, performing resize, will block on hb->lock during - * the requeue. + * then we might need to wake him. The hb will remain valid + * because the thread, performing resize, will block on + * hb->lock during the requeue. */ futex_private_hash_put(no_free_ptr(hbr.fph)); - /* - * Must be done before we enqueue the waiter, here is unfortunately - * under the hb lock, but that *should* work because it does nothing. - */ - rt_mutex_pre_schedule(); rt_mutex_init_waiter(&rt_waiter); @@ -1081,10 +1140,6 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl * the */ futex_q_lockptr_lock(&q); - /* - * Waiter is unqueued. - */ - rt_mutex_post_schedule(); no_block: /* * Fixup the pi_state owner and possibly acquire the lock if we diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index 79823ad136830..b3f4a4bccb126 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -154,8 +154,16 @@ static inline void futex_requeue_pi_complete(struct futex_q *q, int locked) } while (!atomic_try_cmpxchg(&q->requeue_state, &old, new)); #ifdef CONFIG_PREEMPT_RT - /* If the waiter interleaved with the requeue let it know */ - if (unlikely(old == Q_REQUEUE_PI_WAIT)) + /* + * The waiter in futex_requeue_pi_wakeup_sync() can interleave with the + * wake below: It will assign Q_REQUEUE_PI_IN_PROGRESS and here it will + * be updated to Q_REQUEUE_PI_LOCKED (locked = 1). The rcuwait_wait_event() + * will already read Q_REQUEUE_PI_LOCKED and skip the schedule() invocation, + * leading to an access of futex_q::requeue_wait after the waiter returned. + * In this case only we skip the wake here and rely on following wake in + * requeue_pi_wake_futex() to perform the wake if needed. + */ + if (unlikely(old == Q_REQUEUE_PI_WAIT) && new != Q_REQUEUE_PI_LOCKED) rcuwait_wake_up(&q->requeue_wait); #endif } diff --git a/kernel/kcov.c b/kernel/kcov.c index 1df373fb562bc..a7514303eff38 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -86,17 +86,12 @@ struct kcov_remote { static DEFINE_SPINLOCK(kcov_remote_lock); static DEFINE_HASHTABLE(kcov_remote_map, 4); -static struct list_head kcov_remote_areas = LIST_HEAD_INIT(kcov_remote_areas); +static struct list_head kcov_remote_areas[2] = { + LIST_HEAD_INIT(kcov_remote_areas[0]), LIST_HEAD_INIT(kcov_remote_areas[1]) +}; struct kcov_percpu_data { - void *irq_area; local_lock_t lock; - - unsigned int saved_mode; - unsigned int saved_size; - void *saved_area; - struct kcov *saved_kcov; - int saved_sequence; }; static DEFINE_PER_CPU(struct kcov_percpu_data, kcov_percpu_data) = { @@ -132,12 +127,13 @@ static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle) } /* Must be called with kcov_remote_lock locked. */ -static struct kcov_remote_area *kcov_remote_area_get(unsigned int size) +static struct kcov_remote_area *kcov_remote_area_get(unsigned int size, bool irq) { struct kcov_remote_area *area; struct list_head *pos; + struct list_head *list = &kcov_remote_areas[irq]; - list_for_each(pos, &kcov_remote_areas) { + list_for_each(pos, list) { area = list_entry(pos, struct kcov_remote_area, list); if (area->size == size) { list_del(&area->list); @@ -149,11 +145,11 @@ static struct kcov_remote_area *kcov_remote_area_get(unsigned int size) /* Must be called with kcov_remote_lock locked. */ static void kcov_remote_area_put(struct kcov_remote_area *area, - unsigned int size) + unsigned int size, bool irq) { INIT_LIST_HEAD(&area->list); area->size = size; - list_add(&area->list, &kcov_remote_areas); + list_add(&area->list, &kcov_remote_areas[irq]); /* * KMSAN doesn't instrument this file, so it may not know area->list * is initialized. Unpoison it explicitly to avoid reports in @@ -390,6 +386,12 @@ void kcov_task_init(struct task_struct *t) kcov_task_reset(t); t->kcov_remote = NULL; t->kcov_handle = current->kcov_handle; + t->kcov_softirq = 0; + t->kcov_saved_mode = 0; + t->kcov_saved_size = 0; + t->kcov_saved_area = NULL; + t->kcov_saved_kcov = NULL; + t->kcov_saved_sequence = 0; } static void kcov_reset(struct kcov *kcov) @@ -836,17 +838,16 @@ static inline bool kcov_mode_enabled(unsigned int mode) static void kcov_remote_softirq_start(struct task_struct *t) __must_hold(&kcov_percpu_data.lock) { - struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data); unsigned int mode; mode = READ_ONCE(t->kcov_mode); barrier(); if (kcov_mode_enabled(mode)) { - data->saved_mode = mode; - data->saved_size = t->kcov_size; - data->saved_area = t->kcov_area; - data->saved_sequence = t->kcov_sequence; - data->saved_kcov = t->kcov; + t->kcov_saved_mode = mode; + t->kcov_saved_size = t->kcov_size; + t->kcov_saved_area = t->kcov_area; + t->kcov_saved_sequence = t->kcov_sequence; + t->kcov_saved_kcov = t->kcov; kcov_stop(t); } } @@ -854,17 +855,15 @@ static void kcov_remote_softirq_start(struct task_struct *t) static void kcov_remote_softirq_stop(struct task_struct *t) __must_hold(&kcov_percpu_data.lock) { - struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data); - - if (data->saved_kcov) { - kcov_start(t, data->saved_kcov, data->saved_size, - data->saved_area, data->saved_mode, - data->saved_sequence); - data->saved_mode = 0; - data->saved_size = 0; - data->saved_area = NULL; - data->saved_sequence = 0; - data->saved_kcov = NULL; + if (t->kcov_saved_kcov) { + kcov_start(t, t->kcov_saved_kcov, t->kcov_saved_size, + t->kcov_saved_area, t->kcov_saved_mode, + t->kcov_saved_sequence); + t->kcov_saved_mode = 0; + t->kcov_saved_size = 0; + t->kcov_saved_area = NULL; + t->kcov_saved_sequence = 0; + t->kcov_saved_kcov = NULL; } } @@ -927,17 +926,17 @@ void kcov_remote_start(u64 handle) sequence = kcov->sequence; if (in_task()) { size = kcov->remote_size; - area = kcov_remote_area_get(size); + area = kcov_remote_area_get(size, false); } else { size = CONFIG_KCOV_IRQ_AREA_SIZE; - area = this_cpu_ptr(&kcov_percpu_data)->irq_area; + area = kcov_remote_area_get(size, true); } spin_unlock(&kcov_remote_lock); - /* Can only happen when in_task(). */ + /* Allocate new buffer if we can sleep. */ if (!area) { local_unlock_irqrestore(&kcov_percpu_data.lock, flags); - area = vmalloc(size * sizeof(unsigned long)); + area = in_task() ? vmalloc(size * sizeof(unsigned long)) : NULL; if (!area) { kcov_put(kcov); return; @@ -1079,11 +1078,9 @@ void kcov_remote_stop(void) kcov_move_area(kcov->mode, kcov->area, kcov->size, area); spin_unlock(&kcov->lock); - if (in_task()) { - spin_lock(&kcov_remote_lock); - kcov_remote_area_put(area, size); - spin_unlock(&kcov_remote_lock); - } + spin_lock(&kcov_remote_lock); + kcov_remote_area_put(area, size, !in_task()); + spin_unlock(&kcov_remote_lock); local_unlock_irqrestore(&kcov_percpu_data.lock, flags); @@ -1129,14 +1126,21 @@ static void __init selftest(void) static int __init kcov_init(void) { - int cpu; + int cpu = num_possible_cpus(); + +#ifdef CONFIG_PREEMPT_RT + /* Allocate some extra buffers in order to prepare for softirq preemption. */ + cpu = cpu >= 4 ? cpu * 2 : cpu + 4; +#endif + while (cpu--) { + void *area = vmalloc(CONFIG_KCOV_IRQ_AREA_SIZE * sizeof(unsigned long)); + unsigned long flags; - for_each_possible_cpu(cpu) { - void *area = vmalloc_node(CONFIG_KCOV_IRQ_AREA_SIZE * - sizeof(unsigned long), cpu_to_node(cpu)); if (!area) return -ENOMEM; - per_cpu_ptr(&kcov_percpu_data, cpu)->irq_area = area; + spin_lock_irqsave(&kcov_remote_lock, flags); + kcov_remote_area_put(area, CONFIG_KCOV_IRQ_AREA_SIZE, true); + spin_unlock_irqrestore(&kcov_remote_lock, flags); } /* diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c index 8a7baf4e332e3..2db82661cd60a 100644 --- a/kernel/kcsan/core.c +++ b/kernel/kcsan/core.c @@ -585,8 +585,14 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned * information is lost if dirtied by KCSAN. */ kcsan_save_irqtrace(current); - if (!interrupt_watcher) + if (!interrupt_watcher) { local_irq_save(irq_flags); + /* + * NMIs can still fire, disable checking for all interrupt + * contexts. + */ + raw_cpu_ptr(&kcsan_cpu_ctx)->disable_count++; + } watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write); if (watchpoint == NULL) { @@ -699,8 +705,10 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]); out_unlock: - if (!interrupt_watcher) + if (!interrupt_watcher) { + raw_cpu_ptr(&kcsan_cpu_ctx)->disable_count--; local_irq_restore(irq_flags); + } kcsan_restore_irqtrace(current); ctx->disable_scoped--; diff --git a/kernel/kprobes.c b/kernel/kprobes.c index bfc89083daa93..4edd8ca5c6578 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -526,7 +527,8 @@ enum { OPTIMIZER_ST_FLUSHING = 2, }; -static DECLARE_COMPLETION(optimizer_completion); +/* Bumped at the end of each kprobe_optimizer() pass, under 'kprobe_mutex' */ +static unsigned long optimizer_passes; #define OPTIMIZE_DELAY 5 @@ -654,9 +656,9 @@ static void kprobe_optimizer(void) do_free_cleaned_kprobes(); } - /* Step 5: Kick optimizer again if needed. But if there is a flush requested, */ - if (completion_done(&optimizer_completion)) - complete(&optimizer_completion); + /* Step 5: Wake up flushers, and kick optimizer again if needed. */ + optimizer_passes++; + wake_up_var_locked(&optimizer_passes, &kprobe_mutex); if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) kick_kprobe_optimizer(); /*normal kick*/ @@ -708,7 +710,8 @@ static void wait_for_kprobe_optimizer_locked(void) lockdep_assert_held(&kprobe_mutex); while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) { - init_completion(&optimizer_completion); + unsigned long passes = optimizer_passes; + /* * Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's * idle. If it's already kicked, it will see the state change. @@ -717,9 +720,12 @@ static void wait_for_kprobe_optimizer_locked(void) OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING) wake_up(&kprobe_optimizer_wait); - mutex_unlock(&kprobe_mutex); - wait_for_completion(&optimizer_completion); - mutex_lock(&kprobe_mutex); + /* + * kprobe_optimizer() holds 'kprobe_mutex' for a whole pass, which + * this drops while sleeping, so a new count means a full pass ran. + */ + wait_var_event_mutex(&optimizer_passes, + optimizer_passes != passes, &kprobe_mutex); } } @@ -1447,8 +1453,14 @@ static bool __within_kprobe_blacklist(unsigned long addr) /* * If 'kprobe_blacklist' is defined, check the address and * reject any probe registration in the prohibited area. + * Note: this can return true during transition period where + * (start_addr, end_addr) in the black list is shrinking + * but old entry has not been removed yet. This is acceptable + * because the worst case is that we reject more probes than + * we should. */ - list_for_each_entry(ent, &kprobe_blacklist, list) { + guard(rcu)(); + list_for_each_entry_rcu(ent, &kprobe_blacklist, list) { if (addr >= ent->start_addr && addr < ent->end_addr) return true; } @@ -2509,7 +2521,7 @@ int kprobe_add_ksym_blacklist(unsigned long entry) ent->start_addr = entry; ent->end_addr = entry + size; INIT_LIST_HEAD(&ent->list); - list_add_tail(&ent->list, &kprobe_blacklist); + list_add_tail_rcu(&ent->list, &kprobe_blacklist); return (int)size; } @@ -2603,8 +2615,8 @@ static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end) list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) { if (ent->start_addr < start || ent->start_addr >= end) continue; - list_del(&ent->list); - kfree(ent); + list_del_rcu(&ent->list); + kfree_rcu(ent, rcu); } } diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 175c08a6e41e3..6fad9152387a9 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -501,7 +501,7 @@ static int __init kho_preserved_memory_reserve(phys_addr_t phys, struct page *page; u64 sz; - sz = 1 << (order + PAGE_SHIFT); + sz = 1UL << (order + PAGE_SHIFT); page = kho_get_preserved_page(phys, order); /* Reserve the memory preserved in KHO in memblock */ diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c index 5c27134ce7ba5..cd715a7c1d992 100644 --- a/kernel/liveupdate/luo_flb.c +++ b/kernel/liveupdate/luo_flb.c @@ -133,7 +133,7 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) return 0; } -static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb) +void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb) { struct luo_flb_private *private = luo_flb_get_private(flb); @@ -168,7 +168,10 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) if (private->incoming.finished) return -ENODATA; - if (private->incoming.retrieved) + if (private->incoming.retrieve_status < 0) + return private->incoming.retrieve_status; + + if (private->incoming.retrieve_status > 0) return 0; if (!fh->active) @@ -194,12 +197,13 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) err = flb->ops->retrieve(&args); if (err) { + private->incoming.retrieve_status = err; module_put(flb->ops->owner); return err; } private->incoming.obj = args.obj; - private->incoming.retrieved = true; + private->incoming.retrieve_status = 1; return 0; } @@ -213,7 +217,7 @@ void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) if (!refcount_dec_and_test(&private->incoming.count)) return; - if (!private->incoming.retrieved) { + if (private->incoming.retrieve_status <= 0) { int err = luo_flb_retrieve_one(flb); if (WARN_ON(err)) @@ -264,7 +268,7 @@ int luo_flb_file_preserve(struct liveupdate_file_handler *fh) exit_err: list_for_each_entry_continue_reverse(iter, flb_list, list) - luo_flb_file_unpreserve_one(iter->flb); + liveupdate_flb_put_outgoing(iter->flb); up_read(&luo_register_rwlock); return err; @@ -289,7 +293,7 @@ void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh) guard(rwsem_read)(&luo_register_rwlock); list_for_each_entry_reverse(iter, flb_list, list) - luo_flb_file_unpreserve_one(iter->flb); + liveupdate_flb_put_outgoing(iter->flb); } /** @@ -544,6 +548,10 @@ int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp) return -EOPNOTSUPP; guard(mutex)(&private->outgoing.lock); + if (!private->outgoing.obj) + return -ENOENT; + + refcount_inc(&private->outgoing.count); *objp = private->outgoing.obj; return 0; diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index f38b5b18f3f81..b4a9f55c44987 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -316,8 +316,12 @@ static int luo_session_finish(struct luo_session *session, struct luo_ucmd *ucmd) { struct liveupdate_session_finish *argp = ucmd->cmd; - int err = luo_session_finish_one(session); + int err; + + if (argp->reserved) + return -EINVAL; + err = luo_session_finish_one(session); if (err) return err; diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 2d4c5bab5af88..06f025abe8875 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -947,6 +947,34 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass) return NULL; } +static __always_inline bool lock_class_cache_is_valid(const struct lockdep_map *lock, + const struct lock_class *class, + unsigned int subclass) +{ + unsigned int class_subclass; + + if (!class) + return false; + + if (unlikely(class < lock_classes || class >= lock_classes + MAX_LOCKDEP_KEYS)) + return false; + + if (unlikely(!arch_test_bit(class - lock_classes, lock_classes_in_use))) + return false; + + if (unlikely(!lock->key)) + return false; + + class_subclass = subclass ? subclass : class->subclass; + if (unlikely(class_subclass >= MAX_LOCKDEP_SUBCLASSES)) + return false; + + if (unlikely(READ_ONCE(class->key) != lock->key->subkeys + class_subclass)) + return false; + + return true; +} + /* * Static locks do not have their class-keys yet - for them the key is * the lock object itself. If the lock is in the per cpu area, the @@ -1379,9 +1407,9 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force) out_set_class_cache: if (!subclass || force) - lock->class_cache[0] = class; + WRITE_ONCE(lock->class_cache[0], class); else if (subclass < NR_LOCKDEP_CACHING_CLASSES) - lock->class_cache[subclass] = class; + WRITE_ONCE(lock->class_cache[subclass], class); /* * Hash collision, did we smoke some? We found a class with a matching @@ -4941,7 +4969,7 @@ void lockdep_init_map_type(struct lockdep_map *lock, const char *name, int i; for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++) - lock->class_cache[i] = NULL; + WRITE_ONCE(lock->class_cache[i], NULL); #ifdef CONFIG_LOCK_STAT lock->cpu = raw_smp_processor_id(); @@ -5006,12 +5034,15 @@ EXPORT_SYMBOL_GPL(__lockdep_no_track__); void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn, lock_print_fn print_fn) { - struct lock_class *class = lock->class_cache[0]; + struct lock_class *class = READ_ONCE(lock->class_cache[0]); unsigned long flags; raw_local_irq_save(flags); lockdep_recursion_inc(); + if (!lock_class_cache_is_valid(lock, class, 0)) + class = NULL; + if (!class) class = register_lock_class(lock, 0, 0); @@ -5103,8 +5134,11 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, if (DEBUG_LOCKS_WARN_ON(subclass >= MAX_LOCKDEP_SUBCLASSES)) return 0; - if (subclass < NR_LOCKDEP_CACHING_CLASSES) - class = lock->class_cache[subclass]; + if (subclass < NR_LOCKDEP_CACHING_CLASSES) { + class = READ_ONCE(lock->class_cache[subclass]); + if (!lock_class_cache_is_valid(lock, class, subclass)) + class = NULL; + } /* * Not cached? */ @@ -5307,9 +5341,9 @@ static noinstr int match_held_lock(const struct held_lock *hlock, return 1; if (hlock->references) { - const struct lock_class *class = lock->class_cache[0]; + const struct lock_class *class = READ_ONCE(lock->class_cache[0]); - if (!class) + if (!lock_class_cache_is_valid(lock, class, 0)) class = look_up_lock_class(lock, 0); /* @@ -5437,6 +5471,8 @@ __lock_set_class(struct lockdep_map *lock, const char *name, lock->wait_type_outer, lock->lock_type); class = register_lock_class(lock, subclass, 0); + if (!class) + return 0; hlock->class_idx = class - lock_classes; curr->lockdep_depth = i; diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index 5d48d64725b12..eb18b094473c9 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -423,6 +423,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, { int ret; + rt_mutex_futex_pre_schedule(); raw_spin_lock_irq(&lock->wait_lock); /* sleep on the mutex */ set_current_state(TASK_INTERRUPTIBLE); @@ -433,6 +434,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, */ fixup_rt_mutex_waiters(lock, true); raw_spin_unlock_irq(&lock->wait_lock); + rt_mutex_futex_post_schedule(); return ret; } diff --git a/kernel/module/dups.c b/kernel/module/dups.c index 1d720a5311ba0..db73772297034 100644 --- a/kernel/module/dups.c +++ b/kernel/module/dups.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "internal.h" @@ -38,27 +39,37 @@ static bool enable_dups_trace = IS_ENABLED(CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS_TRACE); module_param(enable_dups_trace, bool_enable_only, 0644); -/* - * Protects dup_kmod_reqs list, adds / removals with RCU. - */ +/* A mutex-protected list of active kmod requests. */ static DEFINE_MUTEX(kmod_dup_mutex); static LIST_HEAD(dup_kmod_reqs); struct kmod_dup_req { + refcount_t refcount; struct list_head list; char name[MODULE_NAME_LEN]; struct completion first_req_done; - struct work_struct complete_work; struct delayed_work delete_work; int dup_ret; }; +static void get_kmod_req(struct kmod_dup_req *kmod_req) +{ + refcount_inc(&kmod_req->refcount); +} + +static void put_kmod_req(struct kmod_dup_req *kmod_req) +{ + if (refcount_dec_and_test(&kmod_req->refcount)) + kfree(kmod_req); +} + static struct kmod_dup_req *kmod_dup_request_lookup(char *module_name) { struct kmod_dup_req *kmod_req; - list_for_each_entry_rcu(kmod_req, &dup_kmod_reqs, list, - lockdep_is_held(&kmod_dup_mutex)) { + lockdep_assert_held(&kmod_dup_mutex); + + list_for_each_entry(kmod_req, &dup_kmod_reqs, list) { if (strlen(kmod_req->name) == strlen(module_name) && !memcmp(kmod_req->name, module_name, strlen(module_name))) { return kmod_req; @@ -87,33 +98,10 @@ static void kmod_dup_request_delete(struct work_struct *work) * just returning 0. */ mutex_lock(&kmod_dup_mutex); - list_del_rcu(&kmod_req->list); - synchronize_rcu(); + list_del(&kmod_req->list); mutex_unlock(&kmod_dup_mutex); - kfree(kmod_req); -} - -static void kmod_dup_request_complete(struct work_struct *work) -{ - struct kmod_dup_req *kmod_req; - - kmod_req = container_of(work, struct kmod_dup_req, complete_work); - - /* - * This will ensure that the kernel will let all the waiters get - * informed its time to check the return value. It's time to - * go home. - */ - complete_all(&kmod_req->first_req_done); - /* - * Now that we have allowed prior request_module() calls to go on - * with life, let's schedule deleting this entry. We don't have - * to do it right away, but we *eventually* want to do it so to not - * let this linger forever as this is just a boot optimization for - * possible abuses of vmalloc() incurred by finit_module() thrashing. - */ - queue_delayed_work(system_dfl_wq, &kmod_req->delete_work, 60 * HZ); + put_kmod_req(kmod_req); } bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) @@ -129,8 +117,8 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) if (!new_kmod_req) return false; - memcpy(new_kmod_req->name, module_name, strlen(module_name)); - INIT_WORK(&new_kmod_req->complete_work, kmod_dup_request_complete); + refcount_set(&new_kmod_req->refcount, 1); + strscpy(new_kmod_req->name, module_name); INIT_DELAYED_WORK(&new_kmod_req->delete_work, kmod_dup_request_delete); init_completion(&new_kmod_req->first_req_done); @@ -161,10 +149,12 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) * keep tab on duplicates later. */ pr_debug("New request_module() for %s\n", module_name); - list_add_rcu(&new_kmod_req->list, &dup_kmod_reqs); + list_add(&new_kmod_req->list, &dup_kmod_reqs); mutex_unlock(&kmod_dup_mutex); return false; } + + get_kmod_req(kmod_req); mutex_unlock(&kmod_dup_mutex); /* We are dealing with a duplicate request now */ @@ -194,7 +184,7 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) * calls bail out right away. */ *dup_ret = 0; - return true; + goto out; } /* @@ -209,12 +199,14 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) TASK_KILLABLE); if (ret) { *dup_ret = ret; - return true; + goto out; } /* Now the duplicate request has the same exact return value as the first request */ *dup_ret = kmod_req->dup_ret; +out: + put_kmod_req(kmod_req); return true; } @@ -224,24 +216,31 @@ void kmod_dup_request_announce(char *module_name, int ret) mutex_lock(&kmod_dup_mutex); + /* + * Look for a kmod_dup_req previously added in + * kmod_dup_request_exists_wait(). Note that a request_module_nowait() + * without its own kmod_dup_req entry can announce a result of + * a concurrent request_module() call. + */ kmod_req = kmod_dup_request_lookup(module_name); - if (!kmod_req) - goto out; + if (!kmod_req || completion_done(&kmod_req->first_req_done)) { + mutex_unlock(&kmod_dup_mutex); + return; + } kmod_req->dup_ret = ret; - /* - * If we complete() here we may allow duplicate threads - * to continue before the first one that submitted the - * request. We're in no rush also, given that each and - * every bounce back to userspace is slow we avoid that - * with a slight delay here. So queueue up the completion - * and let duplicates suffer, just wait a tad bit longer. - * There is no rush. But we also don't want to hold the - * caller up forever or introduce any boot delays. - */ - queue_work(system_dfl_wq, &kmod_req->complete_work); + /* Inform all duplicate waiters to check the return value. */ + complete_all(&kmod_req->first_req_done); -out: mutex_unlock(&kmod_dup_mutex); + + /* + * Now that we have allowed prior request_module() calls to go on + * with life, let's schedule deleting this entry. We don't have + * to do it right away, but we *eventually* want to do it so to not + * let this linger forever as this is just a boot optimization for + * possible abuses of vmalloc() incurred by finit_module() thrashing. + */ + queue_delayed_work(system_dfl_wq, &kmod_req->delete_work, 60 * HZ); } diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index 0fc11e45df9b9..f23126d804b25 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -258,17 +258,25 @@ static const char *find_kallsyms_symbol(struct module *mod, unsigned int i, best = 0; unsigned long nextval, bestval; struct mod_kallsyms *kallsyms = rcu_dereference(mod->kallsyms); - struct module_memory *mod_mem; + struct module_memory *mod_mem = NULL; - /* At worse, next value is at end of module */ - if (within_module_init(addr, mod)) - mod_mem = &mod->mem[MOD_INIT_TEXT]; - else - mod_mem = &mod->mem[MOD_TEXT]; + for_each_mod_mem_type(type) { +#ifndef CONFIG_KALLSYMS_ALL + if (!mod_mem_type_is_text(type)) + continue; +#endif + if (within_module_mem_type(addr, mod, type)) { + mod_mem = &mod->mem[type]; + break; + } + } - nextval = (unsigned long)mod_mem->base + mod_mem->size; + if (!mod_mem) + return NULL; - bestval = kallsyms_symbol_value(&kallsyms->symtab[best]); + /* Initialize bounds within memory region the address belongs to. */ + nextval = (unsigned long)mod_mem->base + mod_mem->size; + bestval = (unsigned long)mod_mem->base - 1; /* * Scan for closest preceding symbol, and next symbol. (ELF diff --git a/kernel/module/main.c b/kernel/module/main.c index 46dd8d25a6058..7cbc8f0e28c61 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -2011,6 +2011,7 @@ static int elf_validity_cache_sechdrs(struct load_info *info) * Specifically checks: * * * Section name table index is inbounds of section headers + * * Section name table type is SHT_STRTAB * * Section name table is not empty * * Section name table is NUL terminated * * All section name offsets are inbounds of the section @@ -2038,6 +2039,11 @@ static int elf_validity_cache_secstrings(struct load_info *info) strhdr = &info->sechdrs[info->hdr->e_shstrndx]; + if (strhdr->sh_type != SHT_STRTAB) { + pr_err("Invalid ELF section name table type: %u\n", strhdr->sh_type); + return -ENOEXEC; + } + /* * The section name table must be NUL-terminated, as required * by the spec. This makes strcmp and pr_* calls that access @@ -2204,7 +2210,7 @@ static int elf_validity_cache_index_sym(struct load_info *info) * Must have &load_info->index.sym populated. * * Looks at the symbol table's associated string table, makes sure it is - * in-bounds, and caches it. + * in-bounds and of type SHT_STRTAB, and caches it. * * Return: %0 if valid, %-ENOEXEC on failure. */ @@ -2218,6 +2224,12 @@ static int elf_validity_cache_index_str(struct load_info *info) return -ENOEXEC; } + if (info->sechdrs[str_idx].sh_type != SHT_STRTAB) { + pr_err("Invalid ELF symbol string table type: %u\n", + info->sechdrs[str_idx].sh_type); + return -ENOEXEC; + } + info->index.str = str_idx; return 0; } diff --git a/kernel/module/stats.c b/kernel/module/stats.c index 3a9672f93a8ef..78b59bc05387f 100644 --- a/kernel/module/stats.c +++ b/kernel/module/stats.c @@ -253,7 +253,7 @@ int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason) mod_fail = kzalloc_obj(*mod_fail); if (!mod_fail) return -ENOMEM; - memcpy(mod_fail->name, name, strlen(name)); + strscpy(mod_fail->name, name); __set_bit(reason, &mod_fail->dup_fail_mask); atomic_long_inc(&mod_fail->count); list_add_rcu(&mod_fail->list, &dup_failed_modules); diff --git a/kernel/nstree.c b/kernel/nstree.c index 6d12e5900ac01..831f279d174a3 100644 --- a/kernel/nstree.c +++ b/kernel/nstree.c @@ -533,19 +533,13 @@ DEFINE_FREE(ns_put, struct ns_common *, if (!IS_ERR_OR_NULL(_T)) ns_put(_T)) static inline struct ns_common *__must_check legitimize_ns(const struct klistns *kls, struct ns_common *candidate) { - struct ns_common *ns __free(ns_put) = NULL; - if (!ns_requested(kls, candidate)) return NULL; - ns = ns_get_unless_inactive(candidate); - if (!ns) - return NULL; - - if (!may_list_ns(kls, ns)) + if (!may_list_ns(kls, candidate)) return NULL; - return no_free_ptr(ns); + return ns_get_unless_inactive(candidate); } static ssize_t do_listns_userns(struct klistns *kls) diff --git a/kernel/params.c b/kernel/params.c index a668863a4bb63..939ef84a46ffb 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -261,6 +261,7 @@ EXPORT_SYMBOL_GPL(param_set_uint_minmax); int param_set_charp(const char *val, const struct kernel_param *kp) { + char *tmp; size_t len, maxlen = 1024; len = strnlen(val, maxlen + 1); @@ -269,19 +270,20 @@ int param_set_charp(const char *val, const struct kernel_param *kp) return -ENOSPC; } - maybe_kfree_parameter(*(char **)kp->arg); - /* * This is a hack. We can't kmalloc() in early boot, and we * don't need to; this mangled commandline is preserved. */ if (slab_is_available()) { - *(char **)kp->arg = kmalloc_parameter(len + 1); - if (!*(char **)kp->arg) + tmp = kmalloc_parameter(len + 1); + if (!tmp) return -ENOMEM; - strcpy(*(char **)kp->arg, val); + memcpy(tmp, val, len + 1); } else - *(const char **)kp->arg = val; + tmp = (char *)val; + + maybe_kfree_parameter(*(char **)kp->arg); + *(char **)kp->arg = tmp; return 0; } diff --git a/kernel/pid.c b/kernel/pid.c index d01d0dd7114b2..95b8ccfa82690 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -766,29 +766,6 @@ static struct ctl_table_root pid_table_root = { .set_ownership = pid_table_root_set_ownership, }; -static int proc_do_cad_pid(const struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos) -{ - struct pid *new_pid; - pid_t tmp_pid; - int r; - struct ctl_table tmp_table = *table; - - tmp_pid = pid_vnr(cad_pid); - tmp_table.data = &tmp_pid; - - r = proc_dointvec(&tmp_table, write, buffer, lenp, ppos); - if (r || !write) - return r; - - new_pid = find_get_pid(tmp_pid); - if (!new_pid) - return -ESRCH; - - put_pid(xchg(&cad_pid, new_pid)); - return 0; -} - static const struct ctl_table pid_table[] = { { .procname = "pid_max", @@ -799,14 +776,6 @@ static const struct ctl_table pid_table[] = { .extra1 = &pid_max_min, .extra2 = &pid_max_max, }, -#ifdef CONFIG_PROC_SYSCTL - { - .procname = "cad_pid", - .maxlen = sizeof(int), - .mode = 0600, - .proc_handler = proc_do_cad_pid, - }, -#endif }; #endif diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index d2479c69d71a4..c13f68ab7f6e0 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -408,9 +408,18 @@ int hibernation_snapshot(int platform_mode) if (error) goto Close; + error = dpm_prepare(PMSG_FREEZE); + if (error) + goto Complete; + + /* Preallocate image memory before freezing kernel threads and shutting down devices. */ + error = hibernate_preallocate_memory(); + if (error) + goto Complete; + error = freeze_kernel_threads(); if (error) - goto Close; + goto Cleanup; if (hibernation_test(TEST_FREEZER)) { @@ -422,15 +431,6 @@ int hibernation_snapshot(int platform_mode) goto Thaw; } - error = dpm_prepare(PMSG_FREEZE); - if (error) - goto Complete; - - /* Preallocate image memory before shutting down devices. */ - error = hibernate_preallocate_memory(); - if (error) - goto Complete; - console_suspend_all(); pm_restrict_gfp_mask(); @@ -464,10 +464,12 @@ int hibernation_snapshot(int platform_mode) platform_end(platform_mode); return error; - Complete: - dpm_complete(PMSG_RECOVER); Thaw: thaw_kernel_threads(); + Cleanup: + swsusp_free(); + Complete: + dpm_complete(PMSG_RECOVER); goto Close; } diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index d933b5b2c05d4..4a73927cc55b6 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -2797,9 +2797,10 @@ int snapshot_write_next(struct snapshot_handle *handle) return error; error = memory_bm_create(&zero_bm, GFP_ATOMIC, PG_ANY); - if (error) + if (error) { + memory_bm_free(©_bm, PG_UNSAFE_CLEAR); return error; - + } nr_zero_pages = 0; hibernate_restore_protection_begin(); diff --git a/kernel/power/wakelock.c b/kernel/power/wakelock.c index fd763da06a873..a8b6bd5ec46b4 100644 --- a/kernel/power/wakelock.c +++ b/kernel/power/wakelock.c @@ -63,7 +63,7 @@ static unsigned int number_of_wakelocks; static inline bool wakelocks_limit_exceeded(void) { - return number_of_wakelocks > CONFIG_PM_WAKELOCKS_LIMIT; + return number_of_wakelocks >= CONFIG_PM_WAKELOCKS_LIMIT; } static inline void increment_wakelocks_number(void) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 4b03b019cd5ee..ad79e30afd647 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1382,7 +1382,7 @@ bool nbcon_kthread_create(struct console *con) return true; kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index); - if (WARN_ON(IS_ERR(kt))) { + if (IS_ERR(kt)) { con_printk(KERN_ERR, con, "failed to start printing thread\n"); return false; } @@ -1782,7 +1782,7 @@ bool nbcon_alloc(struct console *con) } rcuwait_init(&con->rcuwait); - init_irq_work(&con->irq_work, nbcon_irq_work); + con->irq_work = IRQ_WORK_INIT_LAZY(nbcon_irq_work); atomic_long_set(&ACCESS_PRIVATE(con, nbcon_prev_seq), -1UL); nbcon_state_set(con, &state); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 2fe9a963c823a..96ff9547e888b 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3264,10 +3264,8 @@ static bool console_flush_one_record(bool do_cond_resched, u64 *next_seq, bool * if (flags & CON_NBCON) { progress = nbcon_legacy_emit_next_record(con, handover, cookie, !do_cond_resched); - printk_seq = nbcon_seq_read(con); } else { progress = console_emit_next_record(con, handover, cookie); - printk_seq = con->seq; } /* @@ -3277,6 +3275,15 @@ static bool console_flush_one_record(bool do_cond_resched, u64 *next_seq, bool * if (*handover) goto fail; + /* + * @con can be used here now that it is certain that this + * context is still holding the SRCU read lock. + */ + if (flags & CON_NBCON) + printk_seq = nbcon_seq_read(con); + else + printk_seq = con->seq; + /* Track the next of the highest seq flushed. */ if (printk_seq > *next_seq) *next_seq = printk_seq; @@ -3725,7 +3732,7 @@ static bool legacy_kthread_create(void) lockdep_assert_console_list_lock_held(); kt = kthread_run(legacy_kthread_func, NULL, "pr/legacy"); - if (WARN_ON(IS_ERR(kt))) { + if (IS_ERR(kt)) { pr_err("failed to start legacy printing thread\n"); return false; } diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967adcf3c..608f732866476 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -970,7 +970,7 @@ void rcu_all_qs(void) { unsigned long flags; - if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) + if (!READ_ONCE(*raw_cpu_ptr(&rcu_data.rcu_urgent_qs))) return; preempt_disable(); // For CONFIG_PREEMPT_COUNT=y kernels /* Load rcu_urgent_qs before other flags. */ @@ -978,8 +978,8 @@ void rcu_all_qs(void) preempt_enable(); return; } - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) { local_irq_save(flags); rcu_momentary_eqs(); local_irq_restore(flags); @@ -999,8 +999,8 @@ void rcu_note_context_switch(bool preempt) /* Load rcu_urgent_qs before other flags. */ if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) goto out; - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) rcu_momentary_eqs(); out: rcu_tasks_qs(current, preempt); diff --git a/kernel/reboot.c b/kernel/reboot.c index 695c33e75efd9..d177d89fcc33e 100644 --- a/kernel/reboot.c +++ b/kernel/reboot.c @@ -13,7 +13,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -24,8 +26,7 @@ */ static int C_A_D = 1; -struct pid *cad_pid; -EXPORT_SYMBOL(cad_pid); +struct pid __rcu *cad_pid; #if defined(CONFIG_ARM) #define DEFAULT_REBOOT_MODE = REBOOT_HARD @@ -1366,6 +1367,39 @@ static struct attribute *reboot_attrs[] = { }; #ifdef CONFIG_SYSCTL +static int proc_do_cad_pid(const struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + struct ctl_table tmp_table = *table; + struct pid *new_pid; + struct pid *old_pid; + pid_t tmp_pid; + int r; + + rcu_read_lock(); + tmp_pid = pid_vnr(rcu_dereference(cad_pid)); + rcu_read_unlock(); + + tmp_table.data = &tmp_pid; + + r = proc_dointvec(&tmp_table, write, buffer, lenp, ppos); + if (r || !write) + return r; + + new_pid = find_get_pid(tmp_pid); + if (!new_pid) + return -ESRCH; + + old_pid = unrcu_pointer(xchg(&cad_pid, RCU_INITIALIZER(new_pid))); + /* + * Wait for cad_pid readers before put_pid(). We cannot use + * call_rcu() here because free_pid() already owns pid->rcu. + */ + synchronize_rcu(); + put_pid(old_pid); + return 0; +} + static const struct ctl_table kern_reboot_table[] = { { .procname = "poweroff_cmd", @@ -1381,6 +1415,12 @@ static const struct ctl_table kern_reboot_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "cad_pid", + .maxlen = sizeof(int), + .mode = 0600, + .proc_handler = proc_do_cad_pid, + }, }; static void __init kernel_reboot_sysctls_init(void) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96226707c2f61..c104562638e33 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -443,6 +443,17 @@ static void __sched_core_flip(bool enabled) sched_core_lock(cpu, &flags); + /* + * A core-wide selection may have the shared rq lock temporarily + * released by a lock-dropping ->pick_task(). Flipping would + * rebind rq_lockp() under it. Wait it out. + */ + while (cpu_rq(cpu)->core->core_pick_in_flight) { + sched_core_unlock(cpu, &flags); + cpu_relax(); + sched_core_lock(cpu, &flags); + } + for_each_cpu(t, smt_mask) cpu_rq(t)->core_enabled = enabled; @@ -3341,6 +3352,8 @@ void relax_compatible_cpus_allowed_ptr(struct task_struct *p) void set_task_cpu(struct task_struct *p, unsigned int new_cpu) { unsigned int state = READ_ONCE(p->__state); + bool proxy_migrated = sched_proxy_exec() && p->is_blocked && + task_cpu(p) != p->wake_cpu; /* * We should never call set_task_cpu() on a blocked task, @@ -3376,7 +3389,12 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) */ WARN_ON_ONCE(!cpu_online(new_cpu)); - WARN_ON_ONCE(is_migration_disabled(p)); + /* + * Proxy execution can move a blocked task's scheduling context to any + * CPU without moving its migration-disabled execution context. The + * wakeup path will return the task to a CPU where it can execute. + */ + WARN_ON_ONCE(is_migration_disabled(p) && !proxy_migrated); trace_sched_migrate_task(p, new_cpu); @@ -3732,11 +3750,17 @@ static inline void ttwu_do_wakeup(struct task_struct *p) void update_rq_avg_idle(struct rq *rq) { - u64 delta = rq_clock(rq) - rq->idle_stamp; - u64 max = 2*rq->max_idle_balance_cost; + u64 idle_stamp = rq->idle_stamp; + u64 delta, max; + + if (!idle_stamp) + return; + + delta = rq_clock(rq) - idle_stamp; update_avg(&rq->avg_idle, delta); + max = 2 * rq->max_idle_balance_cost; if (rq->avg_idle > max) rq->avg_idle = max; rq->idle_stamp = 0; @@ -5763,8 +5787,8 @@ void sched_tick(void) { int cpu = smp_processor_id(); struct rq *rq = cpu_rq(cpu); - /* accounting goes to the donor task */ - struct task_struct *donor; + /* scheduler accounting goes to the donor task */ + struct task_struct *curr, *donor; struct rq_flags rf; unsigned long hw_pressure; u64 resched_latency; @@ -5775,9 +5799,10 @@ void sched_tick(void) sched_clock_tick(); rq_lock(rq, &rf); + curr = rq->curr; donor = rq->donor; - psi_account_irqtime(rq, donor, NULL); + psi_account_irqtime(rq, curr, NULL); update_rq_clock(rq); hw_pressure = arch_scale_hw_pressure(cpu_of(rq)); @@ -5800,8 +5825,8 @@ void sched_tick(void) perf_event_task_tick(); - if (donor->flags & PF_WQ_WORKER) - wq_worker_tick(donor); + if (curr->flags & PF_WQ_WORKER) + wq_worker_tick(curr); if (!scx_switched_all()) { rq->idle_balance = idle_cpu(cpu); @@ -6223,7 +6248,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) unsigned long cookie; int i, cpu, occ = 0; struct rq *rq_i; - bool need_sync; + bool need_sync = false; if (!sched_core_enabled(rq)) return __pick_next_task(rq, rf); @@ -6242,6 +6267,8 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) return __pick_next_task(rq, rf); } + rq->core->core_pick_in_flight++; + /* * If there were no {en,de}queues since we picked (IOW, the task * pointers are all still valid), and we haven't scheduled the last @@ -6266,7 +6293,9 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) prev_balance(rq, rf); smt_mask = cpu_smt_mask(cpu); - need_sync = !!rq->core->core_cookie; + +restart: + need_sync |= !!rq->core->core_cookie; /* reset state */ rq->core->core_cookie = 0UL; @@ -6301,10 +6330,15 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) * and there are no cookied tasks running on siblings. */ if (!need_sync) { -restart_single: next = pick_task(rq, rf); - if (unlikely(next == RETRY_TASK)) - goto restart_single; + if (unlikely(next == RETRY_TASK)) { + /* rq lock may have been dropped, clocks invalidated */ + core_clock_updated = false; + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + goto restart; + } + if (!next->core_cookie) { rq->core_pick = NULL; rq->core_dl_server = NULL; @@ -6324,7 +6358,6 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) * * Tie-break prio towards the current CPU */ -restart_multi: max = NULL; for_each_cpu_wrap(i, smt_mask, cpu) { rq_i = cpu_rq(i); @@ -6338,8 +6371,13 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) update_rq_clock(rq_i); p = pick_task(rq_i, rf); - if (unlikely(p == RETRY_TASK)) - goto restart_multi; + if (unlikely(p == RETRY_TASK)) { + /* rq lock may have been dropped, clocks invalidated */ + core_clock_updated = false; + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + goto restart; + } rq_i->core_pick = p; rq_i->core_dl_server = rq_i->dl_server; @@ -6445,6 +6483,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) } out_set_next: + rq->core->core_pick_in_flight--; put_prev_set_next_task(rq, rq->donor, next); if (rq->core->core_forceidle_count && next == rq->idle) queue_core_balance(rq); @@ -6639,6 +6678,13 @@ static void sched_core_cpu_deactivate(unsigned int cpu) core_rq->core_forceidle_seq = rq->core_forceidle_seq; core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; + /* + * A stale leftover would bias the count forever if this CPU later + * returns as its own leader. Move, don't copy. + */ + core_rq->core_pick_in_flight = rq->core_pick_in_flight; + rq->core_pick_in_flight = 0; + /* * Accounting edge for forced idle is handled in pick_next_task(). * Don't need another one here, since the hotplug thread shouldn't @@ -7601,6 +7647,17 @@ void rt_mutex_pre_schedule(void) sched_submit_work(current); } +/* + * Used within the futex syscall context, skips sched_submit_work() because none + * its work will be done. Asserts ensure that it is indeed the case. + */ +void rt_mutex_futex_pre_schedule(void) +{ + lockdep_assert(!(current->flags & (PF_WQ_WORKER | PF_IO_WORKER))); + lockdep_assert(!current->plug); + lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1)); +} + void rt_mutex_schedule(void) { lockdep_assert(current->sched_rt_mutex); @@ -7613,6 +7670,11 @@ void rt_mutex_post_schedule(void) lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); } +void rt_mutex_futex_post_schedule(void) +{ + lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); +} + /* * rt_mutex_setprio - set the current priority of a task * @p: task to boost @@ -9053,6 +9115,7 @@ void __init sched_init(void) rq->core_forceidle_count = 0; rq->core_forceidle_occupation = 0; rq->core_forceidle_start = 0; + rq->core_pick_in_flight = 0; rq->core_cookie = 0UL; #endif diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index dff4ee04694c2..835df03e3d7b1 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -61,6 +61,17 @@ static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu); /************************ Governor internals ***********************/ +static void sugov_update_rate_limit_us(struct sugov_policy *sg_policy) +{ + /* + * Cast rate_limit_us before multiplication to force 64-bit arithmetic. + * Otherwise, on 32-bit platforms, both operands are converted to + * 32-bit unsigned long and the multiplication may overflow. + */ + sg_policy->freq_update_delay_ns = + (s64)sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; +} + static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time) { s64 delta_ns; @@ -314,7 +325,7 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, * A CPU running a task which woken up after an IO operation can have its * utilization boosted to speed up the completion of those IO operations. * The IO boost value is increased each time a task wakes up from IO, in - * sugov_iowait_apply(), and it's instead decreased by this function, + * sugov_iowait_boost(), and it's instead decreased by this function, * each time an increase has not been requested (!iowait_boost_pending). * * A CPU which also appears to have been idle for at least one tick has also @@ -606,7 +617,7 @@ rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf, size_t count tunables->rate_limit_us = rate_limit_us; list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) - sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC; + sugov_update_rate_limit_us(sg_policy); return count; } @@ -848,7 +859,7 @@ static int sugov_start(struct cpufreq_policy *policy) void (*uu)(struct update_util_data *data, u64 time, unsigned int flags); unsigned int cpu; - sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; + sugov_update_rate_limit_us(sg_policy); sg_policy->last_freq_update_time = 0; sg_policy->next_freq = 0; sg_policy->work_in_progress = false; diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 200300043fa54..0663c00c41c04 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2734,15 +2734,17 @@ static int balance_dl(struct rq *rq, struct rq_flags *rf) */ static void wakeup_preempt_dl(struct rq *rq, struct task_struct *p, int flags) { + struct task_struct *donor = rq->donor; /* * Can only get preempted by stop-class, and those should be * few and short lived, doesn't really make sense to push * anything away for that. */ - if (p->sched_class != &dl_sched_class) + if (p->sched_class != &dl_sched_class || + donor->sched_class != &dl_sched_class) return; - if (dl_entity_preempt(&p->dl, &rq->donor->dl)) { + if (dl_entity_preempt(&p->dl, &donor->dl)) { resched_curr(rq); return; } @@ -3026,8 +3028,8 @@ static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root); while (next_node) { i = __node_2_pdl(next_node); - /* make sure task isn't on_cpu (possible with proxy-exec) */ - if (!task_on_cpu(rq, i)) { + /* skip tasks that cannot be migrated */ + if (!task_on_cpu(rq, i) && !is_migration_disabled(i)) { p = i; break; } diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 18183062f751c..2d0ab78b0efa5 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -479,12 +479,27 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); +/* + * Under core scheduling, a pick that releases the rq lock invalidates the + * core-wide selection it is part of. Count the releases so that the core-sched + * pick can tell whether one happened across dispatch. + */ +static void scx_rq_lock_drop(struct rq *rq) +{ + lockdep_assert_rq_held(rq); +#ifdef CONFIG_SCHED_CORE + if (sched_core_enabled(rq)) + rq->scx.lock_drop_seq++; +#endif +} + static void switch_rq_lock(struct rq *from, struct rq *to) { bool tracked = scx_locked_rq() == from; if (tracked) update_locked_rq(NULL); + scx_rq_lock_drop(from); raw_spin_rq_unlock(from); raw_spin_rq_lock(to); if (tracked) @@ -1133,6 +1148,7 @@ static void deferred_irq_workfn(struct irq_work *irq_work) raw_spin_rq_lock(rq); run_deferred(rq); + scx_rq_lock_drop(rq); raw_spin_rq_unlock(rq); } @@ -2021,7 +2037,12 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ int sticky_cpu = p->scx.sticky_cpu; u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; - if (enq_flags & ENQUEUE_WAKEUP) + /* + * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue + * returns. Only the core's wakeup path delivers one. The flags stashed + * for a remote activation may carry the wakeup bit without it. + */ + if (core_enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; /* @@ -2430,12 +2451,14 @@ static bool consume_remote_task(struct rq *this_rq, struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *dsq, struct rq *src_rq) { + scx_rq_lock_drop(this_rq); raw_spin_rq_unlock(this_rq); if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); return true; } else { + scx_rq_lock_drop(src_rq); raw_spin_rq_unlock(src_rq); raw_spin_rq_lock(this_rq); return false; @@ -2737,7 +2760,7 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq, BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); - dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p)); + dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); if (dsq->id == SCX_DSQ_LOCAL) dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); @@ -2774,12 +2797,20 @@ static inline void maybe_queue_balance_callback(struct rq *rq) rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; } +/* what dispatch concluded, consumed by the pick that follows */ +enum scx_dsp_verdict { + SCX_DSP_NONE, /* nothing to run */ + SCX_DSP_LOCAL, /* local DSQ has tasks */ + SCX_DSP_PREV, /* keep running @prev */ + SCX_DSP_RETRY, /* pick helpers only: restart the pick */ +}; + /* * One user of this function is scx_bpf_dispatch() which can be called * recursively as sub-sched dispatches nest. Always inline to reduce stack usage * from the call frame. */ -static __always_inline bool +static __always_inline enum scx_dsp_verdict scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, struct task_struct *prev, bool nested) { @@ -2790,12 +2821,15 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, scx_task_on_sched(sch, prev); if (consume_global_dsq(sch, rq)) - return true; + return SCX_DSP_LOCAL; if (bypass_dsp_enabled(sch)) { /* if @sch is bypassing, only the bypass DSQs are active */ - if (scx_bypassing(sch, cpu)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); + if (scx_bypassing(sch, cpu)) { + if (consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) + return SCX_DSP_LOCAL; + return SCX_DSP_NONE; + } #ifdef CONFIG_EXT_SUB_SCHED /* @@ -2815,13 +2849,13 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) { __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); - return true; + return SCX_DSP_LOCAL; } #endif /* CONFIG_EXT_SUB_SCHED */ } if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) - return false; + return SCX_DSP_NONE; dspc->rq = rq; @@ -2848,14 +2882,12 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, flush_dispatch_buf(sch, rq); - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - return true; - } + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) + return SCX_DSP_PREV; if (rq->scx.local_dsq.nr) - return true; + return SCX_DSP_LOCAL; if (consume_global_dsq(sch, rq)) - return true; + return SCX_DSP_LOCAL; /* * ops.dispatch() can trap us in this loop by repeatedly @@ -2877,20 +2909,20 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * queued. Without this fallback, bypassed tasks could stall if the host * scheduler's ops.dispatch() doesn't yield any tasks. */ - if (bypass_dsp_enabled(sch)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); + if (bypass_dsp_enabled(sch) && consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) + return SCX_DSP_LOCAL; - return false; + return SCX_DSP_NONE; } -static int balance_one(struct rq *rq, struct task_struct *prev) +static enum scx_dsp_verdict balance_one(struct rq *rq, struct task_struct *prev) { struct scx_sched *sch = scx_root; + enum scx_dsp_verdict verdict; s32 cpu = cpu_of(rq); lockdep_assert_rq_held(rq); rq->scx.flags |= SCX_RQ_IN_BALANCE; - rq->scx.flags &= ~SCX_RQ_BAL_KEEP; if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && unlikely(rq->scx.cpu_released)) { @@ -2920,16 +2952,19 @@ static int balance_one(struct rq *rq, struct task_struct *prev) */ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && !scx_bypassing(sch, cpu)) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; + verdict = SCX_DSP_PREV; goto has_tasks; } } /* if there already are tasks to run, nothing to do */ - if (rq->scx.local_dsq.nr) + if (rq->scx.local_dsq.nr) { + verdict = SCX_DSP_LOCAL; goto has_tasks; + } - if (scx_dispatch_sched(sch, rq, prev, false)) + verdict = scx_dispatch_sched(sch, rq, prev, false); + if (verdict != SCX_DSP_NONE) goto has_tasks; /* @@ -2938,12 +2973,12 @@ static int balance_one(struct rq *rq, struct task_struct *prev) */ if ((prev->scx.flags & SCX_TASK_QUEUED) && (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + verdict = SCX_DSP_PREV; goto has_tasks; } rq->scx.flags &= ~SCX_RQ_IN_BALANCE; - return false; + return SCX_DSP_NONE; has_tasks: /* @@ -2960,7 +2995,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev) schedule_reenq_local(rq, 0); rq->scx.flags &= ~SCX_RQ_IN_BALANCE; - return true; + return verdict; } static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) @@ -3113,12 +3148,12 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * ops.enqueue() that @p is the only one available for this cpu, * which should trigger an explicit follow-up scheduling event. * - * Core scheduling can force this CPU idle while @p stays - * runnable. @p's cookie then won't match the core's, so skip - * the warning in that case. + * Under core scheduling, a pick dispatches only when nothing is + * locally runnable and can legitimately go idle with @p still + * runnable (see do_pick_task_scx()). */ if (next && sched_class_above(&ext_sched_class, next->sched_class)) { - WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) && + WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); } else { @@ -3133,11 +3168,26 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, static void kick_sync_wait_bal_cb(struct rq *rq) { - struct scx_kick_syncs __rcu *ks = __this_cpu_read(scx_kick_syncs); - unsigned long *ksyncs = rcu_dereference_sched(ks)->syncs; + struct scx_kick_syncs __rcu *ks; + unsigned long *ksyncs; bool waited; s32 cpu; + /* + * This callback is queued and normally flushed within @rq's own + * scheduling pass. However, dispatch can drop the rq lock while it sits + * queued, and lock takers in that window (the sched class change paths, + * the scx task iterator) flush pending balance callbacks on release, + * running this one on a foreign CPU whose snapshots are unrelated. The + * kicked CPUs are already on their way to advance the kick_syncs being + * waited on. Don't get in the way. + */ + if (unlikely(cpu_of(rq) != smp_processor_id())) + return; + + ks = __this_cpu_read(scx_kick_syncs); + ksyncs = rcu_dereference_sched(ks)->syncs; + /* * Drop rq lock and enable IRQs while waiting. IRQs must be enabled * — a target CPU may be waiting for us to process an IPI (e.g. TLB @@ -3160,6 +3210,7 @@ static void kick_sync_wait_bal_cb(struct rq *rq) continue; } + scx_rq_lock_drop(rq); raw_spin_rq_unlock_irq(rq); while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); @@ -3179,27 +3230,23 @@ static struct task_struct *first_local_task(struct rq *rq) struct task_struct, scx.dsq_list.node); } -static struct task_struct * -do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) +/* + * Run dispatch and queue the follow-up work for a pick. + */ +static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, + struct task_struct *prev) { - struct task_struct *prev = rq->curr; - bool keep_prev; - struct task_struct *p; - - /* see kick_sync_wait_bal_cb() */ - smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); - - rq_modified_begin(rq, &ext_sched_class); + enum scx_dsp_verdict verdict; rq_unpin_lock(rq, rf); - balance_one(rq, prev); + verdict = balance_one(rq, prev); rq_repin_lock(rq, rf); maybe_queue_balance_callback(rq); /* - * Defer to a balance callback which can drop rq lock and enable - * IRQs. Waiting directly in the pick path would deadlock against - * CPUs sending us IPIs (e.g. TLB flushes) while we wait for them. + * Defer to a balance callback which can drop rq lock and enable IRQs. + * Waiting directly in the pick path would deadlock against CPUs sending + * us IPIs (e.g. TLB flushes) while we wait for them. */ if (unlikely(rq->scx.kick_sync_pending)) { rq->scx.kick_sync_pending = false; @@ -3207,10 +3254,102 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) kick_sync_wait_bal_cb); } + if (unlikely(verdict == SCX_DSP_PREV && prev->sched_class != &ext_sched_class)) { + WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); + verdict = SCX_DSP_LOCAL; + } + + return verdict; +} + +#ifdef CONFIG_SCHED_CORE +/* + * Dispatch for a pick when core scheduling is enabled. The selection picks for + * all SMT siblings and the rq_i->core_pick state it builds must stay atomic + * throughout. If the dispatch released the rq lock, anything can have happened + * in between - return %SCX_DSP_RETRY to restart the selection against current + * state. + */ +static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf, + struct task_struct *prev) +{ + enum scx_dsp_verdict verdict; + u32 seq = rq->scx.lock_drop_seq; + + /* another dispatch is in flight on @rq, let that handle it */ + if (rq->scx.flags & SCX_RQ_IN_BALANCE) + return SCX_DSP_NONE; + + rq_unpin_lock(rq, rf); + + verdict = balance_one(rq, prev); + + if (cpu_of(rq) == smp_processor_id()) { + maybe_queue_balance_callback(rq); + + /* see dispatch_pick() */ + if (unlikely(rq->scx.kick_sync_pending)) { + rq->scx.kick_sync_pending = false; + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, + kick_sync_wait_bal_cb); + } + } else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { + /* + * Balance callbacks must run in the context that queued them, + * so they can't be queued on another CPU's rq. Run the deferred + * work directly instead. + */ + rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; + run_deferred(rq); + } + + rq_repin_lock(rq, rf); + + /* if balance_one() released the rq lock, restart the selection */ + if (rq->scx.lock_drop_seq != seq) + return SCX_DSP_RETRY; + + /* see dispatch_pick() */ + if (unlikely(verdict == SCX_DSP_PREV && + prev->sched_class != &ext_sched_class)) { + WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); + verdict = SCX_DSP_LOCAL; + } + + return verdict; +} +#else /* CONFIG_SCHED_CORE */ +static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf, + struct task_struct *prev) +{ + return SCX_DSP_NONE; +} +#endif /* CONFIG_SCHED_CORE */ + +static struct task_struct * +do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) +{ + struct task_struct *prev = rq->curr; + enum scx_dsp_verdict verdict; + struct task_struct *p; + + /* see kick_sync_wait_bal_cb() */ + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); + + rq_modified_begin(rq, &ext_sched_class); + + if (sched_core_enabled(rq)) + verdict = dispatch_core_pick(rq, rf, prev); + else + verdict = dispatch_pick(rq, rf, prev); + + if (verdict == SCX_DSP_RETRY) + return RETRY_TASK; + /* - * If any higher-priority sched class enqueued a runnable task on - * this rq during balance_one(), abort and return RETRY_TASK, so - * that the scheduler loop can restart. + * If any higher-priority sched class enqueued a runnable task on this + * rq during balance_one(), abort and return RETRY_TASK, so that the + * scheduler loop can restart. * * If @force_scx is true, always try to pick a SCHED_EXT task, * regardless of any higher-priority sched classes activity. @@ -3218,19 +3357,11 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) if (!force_scx && rq_modified_above(rq, &ext_sched_class)) return RETRY_TASK; - keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; - if (unlikely(keep_prev && - prev->sched_class != &ext_sched_class)) { - WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); - keep_prev = false; - } - /* - * If balance_one() is telling us to keep running @prev, replenish slice - * if necessary and keep running @prev. Otherwise, pop the first one - * from the local DSQ. + * If we're keeping @prev, replenish slice if necessary and keep running + * @prev. Otherwise, pop the first one from the local DSQ. */ - if (keep_prev) { + if (verdict == SCX_DSP_PREV) { p = prev; if (!p->scx.slice) refill_task_slice_dfl(scx_task_sched(p), p); @@ -3305,6 +3436,8 @@ void ext_server_init(struct rq *rq) * * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to * implement FIFO ordering within each local DSQ. See pick_task_scx(). + * + * Return: %true if @a should run after @b. */ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, bool in_fi) @@ -3313,6 +3446,10 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, struct scx_sched *sch_b = scx_task_sched(b); /* + * scx_prio_less() returns whether @a should run after @b while + * ops.core_sched_before() returns whether its first argument should run + * before the second. Swap the arguments. + * * The const qualifiers are dropped from task_struct pointers when * calling ops.core_sched_before(). Accesses are controlled by the * verifier. @@ -3321,8 +3458,8 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, !scx_bypassing(sch_a, task_cpu(a))) return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before, task_rq(a), - (struct task_struct *)a, - (struct task_struct *)b); + (struct task_struct *)b, + (struct task_struct *)a); else return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); } @@ -3877,7 +4014,7 @@ void scx_post_fork(struct task_struct *p) void scx_cancel_fork(struct task_struct *p) { - if (scx_enabled()) { + if (scx_init_task_enabled) { struct rq *rq; struct rq_flags rf; @@ -4258,8 +4395,10 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag task_rq = task_rq(p); if (locked_rq != task_rq) { - if (locked_rq) + if (locked_rq) { + scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); + } if (unlikely(!raw_spin_rq_trylock(task_rq))) { raw_spin_unlock(&dsq->lock); raw_spin_rq_lock(task_rq); @@ -4285,6 +4424,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { + scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); locked_rq = NULL; cpu_relax(); @@ -4297,8 +4437,10 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag raw_spin_unlock(&dsq->lock); if (locked_rq != rq) { - if (locked_rq) + if (locked_rq) { + scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); + } raw_spin_rq_lock(rq); } } @@ -4309,7 +4451,7 @@ static void process_deferred_reenq_users(struct rq *rq) while (true) { struct scx_dispatch_q *dsq; - u64 reenq_flags; + u64 dsq_id, reenq_flags; scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { struct scx_deferred_reenq_user *dru = @@ -4332,7 +4474,12 @@ static void process_deferred_reenq_users(struct rq *rq) /* see schedule_dsq_reenq() */ smp_mb(); - BUG_ON(dsq->id & SCX_DSQ_FLAG_BUILTIN); + /* destroy_dsq() may have raced and invalidated @dsq, nothing to reenq */ + dsq_id = READ_ONCE(dsq->id); + if (unlikely(dsq_id == SCX_DSQ_INVALID)) + continue; + + BUG_ON(dsq_id & SCX_DSQ_FLAG_BUILTIN); reenq_user(rq, dsq, reenq_flags); } } @@ -5149,6 +5296,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p) /** * handle_lockup - sched_ext common lockup handler + * @exit_cpu: CPU to record in exit_info. Pass the stalled/hung CPU, not current. * @fmt: format string * * Called on system stall or lockup condition and initiates abort of sched_ext @@ -5158,7 +5306,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p) * resolve the lockup. %false if sched_ext is not enabled or abort was already * initiated by someone else. */ -static __printf(1, 2) bool handle_lockup(const char *fmt, ...) +static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...) { struct scx_sched *sch; va_list args; @@ -5174,7 +5322,7 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) case SCX_ENABLING: case SCX_ENABLED: va_start(args, fmt); - ret = scx_verror(sch, fmt, args); + ret = scx_vexit(sch, SCX_EXIT_ERROR, 0, exit_cpu, fmt, args); va_end(args); return ret; default: @@ -5196,7 +5344,7 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) */ bool scx_rcu_cpu_stall(void) { - return handle_lockup("RCU CPU stall detected!"); + return handle_lockup(-1, "RCU CPU stall detected!"); } /** @@ -5211,32 +5359,15 @@ bool scx_rcu_cpu_stall(void) */ void scx_softlockup(u32 dur_s) { - if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) + int cpu = smp_processor_id(); + + if (!handle_lockup(cpu, "soft lockup - CPU %d stuck for %us", cpu, dur_s)) return; printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", - smp_processor_id(), dur_s); -} - -/* - * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(), - * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing - * it from NMI context can lead to deadlocks. Defer via irq_work; the - * disable path runs off irq_work anyway. - */ -static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1); - -static void scx_hardlockup_irq_workfn(struct irq_work *work) -{ - int cpu = atomic_xchg(&scx_hardlockup_cpu, -1); - - if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu)) - printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", - cpu); + cpu, dur_s); } -static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); - /** * scx_hardlockup - sched_ext hardlockup handler * @@ -5245,19 +5376,21 @@ static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); * Try kicking out the current scheduler in an attempt to recover the system to * a good state before taking more drastic actions. * - * Queues an irq_work; the handle_lockup() call happens in IRQ context (see - * scx_hardlockup_irq_workfn). + * Called from NMI. Aborting the scheduler sets ->aborting throughout the + * hierarchy before returning, which is what breaks the dispatch-path live-locks + * that can hard-lock CPUs. * - * Returns %true if sched_ext is enabled and the work was queued, %false - * otherwise. + * Returns %true if sched_ext is enabled and abort was initiated, which may + * resolve the lockup. %false if sched_ext is not enabled or abort was already + * initiated by someone else. */ bool scx_hardlockup(int cpu) { - if (!rcu_access_pointer(scx_root)) + if (!handle_lockup(cpu, "hard lockup - CPU %d", cpu)) return false; - atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu); - irq_work_queue(&scx_hardlockup_irq_work); + printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", + cpu); return true; } @@ -5350,6 +5483,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { list_move_tail(&cursor.node, &n->scx.dsq_list.node); raw_spin_unlock(&donor_dsq->lock); + scx_rq_lock_drop(donor_rq); raw_spin_rq_unlock_irq(donor_rq); cpu_relax(); raw_spin_rq_lock_irq(donor_rq); @@ -5360,6 +5494,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, list_del_init(&cursor.node); raw_spin_unlock(&donor_dsq->lock); + scx_rq_lock_drop(donor_rq); raw_spin_rq_unlock_irq(donor_rq); return nr_balanced; @@ -5573,7 +5708,7 @@ static void disable_bypass_dsp(struct scx_sched *sch) * * - ops.dispatch() is ignored. * - * - balance_one() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice + * - balance_one() does not report %SCX_DSP_PREV on non-zero slice as slice * can't be trusted. Whenever a tick triggers, the running task is rotated to * the tail of the queue with core_sched_at touched. * @@ -5650,6 +5785,7 @@ static void scx_bypass(struct scx_sched *sch, bool bypass) * sees scx_bypassing() before moving tasks to SCX. */ if (!scx_enabled()) { + scx_rq_lock_drop(rq); raw_spin_rq_unlock(rq); continue; } @@ -5676,6 +5812,7 @@ static void scx_bypass(struct scx_sched *sch, bool bypass) if (cpu_online(cpu) || cpu == smp_processor_id()) resched_curr(rq); + scx_rq_lock_drop(rq); raw_spin_rq_unlock(rq); } @@ -6162,12 +6299,9 @@ static void scx_root_disable(struct scx_sched *sch) percpu_up_write(&scx_fork_rwsem); /* - * Invalidate all the rq clocks to prevent getting outdated - * rq clocks from a previous scx scheduler. - * - * Also re-balance the dl_server bandwidth reservations: detach - * ext_server (no more sched_ext tasks) and reinstate fair_server if it - * was previously detached because we were running in full mode. + * Re-balance the dl_server bandwidth reservations: detach ext_server + * (no more sched_ext tasks) and reinstate fair_server if it was + * previously detached because we were running in full mode. * * Unlike the enable path, this runs on a recovery path that cannot * fail, so we use dl_server_swap_bw() to atomically free ext_server's @@ -6180,8 +6314,6 @@ static void scx_root_disable(struct scx_sched *sch) for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); - scx_rq_clock_invalidate(rq); - scoped_guard(rq_lock_irqsave, rq) { update_rq_clock(rq); if (was_switched_all) { @@ -7155,22 +7287,24 @@ static void scx_root_enable_workfn(struct kthread_work *work) #ifdef CONFIG_EXT_SUB_SCHED cgroup_get(cgrp); #endif + /* + * Transition to ENABLING to arm the disable path. Allocation failure + * still unwinds locally. Full disabling on failure applies only after + * scx_alloc_and_add_sched() succeeds. + */ + WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); + WARN_ON_ONCE(scx_root); + sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); if (IS_ERR(sch)) { ret = PTR_ERR(sch); + WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_ENABLING); goto err_free_tid_hash; } if (sch->is_cid_type) static_branch_enable(&__scx_is_cid_type); - /* - * Transition to ENABLING and clear exit info to arm the disable path. - * Failure triggers full disabling from here on. - */ - WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); - WARN_ON_ONCE(scx_root); - atomic_long_set(&scx_nr_rejected, 0); for_each_possible_cpu(cpu) { @@ -7308,7 +7442,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) /* * Enable ops for every task. Fork is excluded by scx_fork_rwsem * preventing new tasks from being added. No need to exclude tasks - * leaving as sched_ext_free() can handle both prepped and enabled + * leaving as sched_ext_dead() can handle both prepped and enabled * tasks. Prep all tasks first and then enable them with preemption * disabled. * @@ -7397,7 +7531,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) /* * We're fully committed and can't fail. The task READY -> ENABLED - * transitions here are synchronized against sched_ext_free() through + * transitions here are synchronized against sched_ext_dead() through * scx_tasks_lock. */ percpu_down_write(&scx_fork_rwsem); @@ -8403,6 +8537,7 @@ static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); } + scx_rq_lock_drop(rq); raw_spin_rq_unlock_irqrestore(rq, flags); return should_wait; @@ -8419,6 +8554,7 @@ static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) (cpu_online(cpu) || cpu == cpu_of(this_rq))) resched_curr(rq); + scx_rq_lock_drop(rq); raw_spin_rq_unlock_irqrestore(rq, flags); } @@ -8456,6 +8592,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) raw_spin_rq_lock(this_rq); this_scx->kick_sync_pending = true; resched_curr(this_rq); + scx_rq_lock_drop(this_rq); raw_spin_rq_unlock(this_rq); } } @@ -8834,10 +8971,17 @@ __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, #ifdef CONFIG_EXT_SUB_SCHED /* * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. + * which scheduler called us, so error out @p's scheduler -- read it + * under RCU as @p's locks aren't necessarily held here. @p may be a + * task past sched_ext_dead() or an idle task, in which case its + * scheduler can't be determined and there is nothing obviously wrong + * to report; just refuse the call. */ if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used"); + struct scx_sched *tsch = scx_task_sched_rcu(p); + + if (tsch) + scx_error(tsch, "__scx_bpf_dsq_insert_vtime() must be used"); return; } #endif @@ -8865,9 +9009,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, { struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; struct scx_sched *sch; - struct rq *this_rq, *src_rq, *locked_rq; + struct rq *p_rq, *src_rq, *locked_rq; bool dispatched = false; - bool in_balance; unsigned long flags; /* @@ -8890,31 +9033,29 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (unlikely(READ_ONCE(sch->aborting))) return false; - if (unlikely(!scx_task_on_sched(sch, p))) { - scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", - p->comm, p->pid); - return false; - } - /* - * Can be called from either ops.dispatch() locking this_rq() or any - * context where no rq lock is held. If latter, lock @p's task_rq which - * we'll likely need anyway. + * Can be called from either ops.dispatch() holding the dispatched rq's + * lock or any context where no rq lock is held. If latter, lock @p's + * task_rq which we'll likely need anyway. */ src_rq = task_rq(p); local_irq_save(flags); - this_rq = this_rq(); - in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; - if (in_balance) { - if (this_rq != src_rq) - switch_rq_lock(this_rq, src_rq); + /* + * Under core scheduling, dispatch can run for a sibling rq, so the + * locked rq is not necessarily this CPU's. + */ + locked_rq = scx_locked_rq(); + + if (locked_rq) { + if (locked_rq != src_rq) + switch_rq_lock(locked_rq, src_rq); } else { raw_spin_rq_lock(src_rq); } - locked_rq = src_rq; + p_rq = src_rq; raw_spin_lock(&src_dsq->lock); /* did someone else get to it while we dropped the locks? */ @@ -8923,8 +9064,19 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, goto out; } + /* + * @p has been on $src_dsq and can't move anymore. If @p is not on @sch, + * the caller didn't have authority over @p at the time of the call. + */ + if (unlikely(!scx_task_on_sched(sch, p))) { + scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", + p->comm, p->pid); + raw_spin_unlock(&src_dsq->lock); + goto out; + } + /* @p is still on $src_dsq and stable, determine the destination */ - dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p)); + dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p)); /* * Apply vtime and slice updates before moving so that the new time is @@ -8937,14 +9089,15 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, p->scx.slice = kit->slice; /* execute move */ - locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); + p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); dispatched = true; out: - if (in_balance) { - if (this_rq != locked_rq) - switch_rq_lock(locked_rq, this_rq); + if (locked_rq) { + if (locked_rq != p_rq) + switch_rq_lock(p_rq, locked_rq); } else { - raw_spin_rq_unlock_irqrestore(locked_rq, flags); + scx_rq_lock_drop(p_rq); + raw_spin_rq_unlock_irqrestore(p_rq, flags); } kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | @@ -9182,7 +9335,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, */ __bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) { - struct rq *this_rq = this_rq(); + struct rq *rq = scx_locked_rq(); struct scx_sched *parent, *child; guard(rcu)(); @@ -9201,8 +9354,8 @@ __bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux * return false; } - return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, - true); + return scx_dispatch_sched(child, rq, rq->scx.sub_dispatch_prev, true) != + SCX_DSP_NONE; } #endif /* CONFIG_EXT_SUB_SCHED */ @@ -9423,9 +9576,11 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) if (raw_spin_rq_trylock(target_rq)) { if (can_skip_idle_kick(target_rq)) { + scx_rq_lock_drop(target_rq); raw_spin_rq_unlock(target_rq); goto out; } + scx_rq_lock_drop(target_rq); raw_spin_rq_unlock(target_rq); } cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); @@ -9496,6 +9651,10 @@ __bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux * * * Return the number of tasks in the DSQ matching @dsq_id. If not found, * -%ENOENT is returned. + * + * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler + * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - + * or the calling CPU's when no rq is locked. */ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux) { @@ -9512,7 +9671,7 @@ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux } if (dsq_id == SCX_DSQ_LOCAL) { - ret = READ_ONCE(this_rq()->scx.local_dsq.nr); + ret = READ_ONCE((scx_locked_rq() ?: this_rq())->scx.local_dsq.nr); goto out; } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); @@ -9691,10 +9850,15 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, * - User DSQs * * Re-enqueues are performed asynchronously. Can be called from anywhere. + * + * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler + * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - + * or the calling CPU's when no rq is locked. */ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, const struct bpf_prog_aux *aux) { + struct rq *locked_rq = scx_locked_rq(); struct scx_sched *sch; struct scx_dispatch_q *dsq; @@ -9713,8 +9877,8 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) reenq_flags |= SCX_REENQ_ANY; - dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, smp_processor_id()); - schedule_dsq_reenq(sch, dsq, reenq_flags, scx_locked_rq()); + dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, smp_processor_id()); + schedule_dsq_reenq(sch, dsq, reenq_flags, locked_rq); } /** diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index 6f93cc32b650e..fff48c164b100 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -1097,10 +1097,17 @@ __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 #ifdef CONFIG_EXT_SUB_SCHED /* * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. + * which scheduler called us, so error out @p's scheduler -- read it + * under RCU as @p's locks aren't necessarily held here. @p may be a + * task past sched_ext_dead() or an idle task, in which case its + * scheduler can't be determined and there is nothing obviously wrong + * to report; just refuse the call. */ if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used"); + struct scx_sched *tsch = scx_task_sched_rcu(p); + + if (tsch) + scx_error(tsch, "__scx_bpf_select_cpu_and() must be used"); return -EINVAL; } #endif diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 673059fa9d728..b295991b9f10c 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1544,8 +1544,6 @@ __printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) -#define scx_verror(sch, fmt, args) \ - scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) /* * Return the rq currently locked from an scx callback, or NULL if no rq is diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f15f5764818eb..db34eb74804d4 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1004,6 +1004,13 @@ static inline void __max_slice_update(struct sched_entity *se, struct rb_node *n } } +static inline void min_vruntime_copy(struct sched_entity *new, struct sched_entity *old) +{ + new->min_vruntime = old->min_vruntime; + new->min_slice = old->min_slice; + new->max_slice = old->max_slice; +} + /* * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) */ @@ -1031,8 +1038,9 @@ static inline bool min_vruntime_update(struct sched_entity *se, bool exit) se->max_slice == old_max_slice; } -RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, - run_node, min_vruntime, min_vruntime_update); + +RB_DECLARE_CALLBACKS_MULTI(static, min_vruntime_cb, struct sched_entity, + run_node, min_vruntime_copy, min_vruntime_update); /* * Enqueue an entity into the rb-tree: @@ -1042,6 +1050,8 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; + se->max_slice = se->slice; + rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, __entity_less, &min_vruntime_cb); } @@ -1363,7 +1373,6 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) se->exec_start = now; if (entity_is_task(se)) { - struct task_struct *donor = task_of(se); struct task_struct *running = rq->curr; /* * If se is a task, we account the time against the running @@ -1376,8 +1385,7 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) account_group_exec_runtime(running, delta_exec); account_mm_sched(rq, running, delta_exec); - /* cgroup time is always accounted against the donor */ - cgroup_account_cputime(donor, delta_exec); + cgroup_account_cputime(running, delta_exec); } else { /* If not task, account the time against donor se */ se->sum_exec_runtime += delta_exec; @@ -1445,12 +1453,17 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) return true; if (static_branch_likely(&sched_numa_balancing)) { + struct sched_cache_group *grp = READ_ONCE(mm->sched_cache_grp); + + if (!grp) + return true; + /* * TBD: RDT exclusive LLC ways reserved should be * excluded. */ llc = sd->llc_bytes; - footprint = READ_ONCE(mm->sc_stat.footprint); + footprint = READ_ONCE(grp->footprint); /* * Scale the LLC size by 256*llc_aggr_tolerance @@ -1482,6 +1495,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, int cpu) { + struct sched_cache_group *grp; int scale; if (get_nr_threads(p) <= 1) @@ -1495,7 +1509,11 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, if (scale == INT_MAX) return false; - return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads), + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return true; + + return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads), (scale * per_cpu(sd_llc_size, cpu))); } @@ -1572,12 +1590,20 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p) } } -void mm_init_sched(struct mm_struct *mm, - struct sched_cache_time __percpu *_pcpu_sched) +int mm_init_sched(struct mm_struct *mm, + struct sched_cache_time __percpu *_pcpu_sched) { + struct sched_cache_group *grp; unsigned long epoch = 0; int i; + grp = kzalloc_obj(*grp); + if (!grp) { + free_percpu(_pcpu_sched); + mm->sched_cache_grp = NULL; + return -ENOMEM; + } + for_each_possible_cpu(i) { struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i); struct rq *rq = cpu_rq(i); @@ -1588,18 +1614,51 @@ void mm_init_sched(struct mm_struct *mm, epoch = rq->cpu_epoch; } - raw_spin_lock_init(&mm->sc_stat.lock); - mm->sc_stat.epoch = epoch; - mm->sc_stat.cpu = -1; - mm->sc_stat.next_scan = jiffies; - mm->sc_stat.nr_running_avg = 0; - mm->sc_stat.footprint = 0; + raw_spin_lock_init(&grp->lock); + grp->epoch = epoch; + grp->cpu = -1; + grp->next_scan = jiffies; + grp->nr_running_avg = 0; + grp->footprint = 0; + refcount_set(&grp->refcnt, 1); /* - * The update to mm->sc_stat should not be reordered - * before initialization to mm's other fields, in case + * The update to grp->pcpu_sched should not be reordered + * before initialization to grp's other fields, in case * the readers may get invalid mm_sched_epoch, etc. */ - smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched); + smp_store_release(&grp->pcpu_sched, _pcpu_sched); + /* + * Publish the group last. Not every reader qualifies it by + * grp->pcpu_sched - can_migrate_llc_task() only checks that the + * pointer is non-NULL before reading grp->footprint and + * grp->nr_running_avg - so a reachable group must already be + * fully initialized. + */ + smp_store_release(&mm->sched_cache_grp, grp); + return 0; +} + +static void sched_cache_group_free_rcu(struct rcu_head *rcu) +{ + struct sched_cache_group *grp = + container_of(rcu, struct sched_cache_group, rcu); + + free_percpu(grp->pcpu_sched); + kfree(grp); +} + +static void sched_cache_group_put(struct sched_cache_group *grp) +{ + if (!grp || !refcount_dec_and_test(&grp->refcnt)) + return; + + call_rcu(&grp->rcu, sched_cache_group_free_rcu); +} + +void mm_destroy_sched(struct mm_struct *mm) +{ + sched_cache_group_put(mm->sched_cache_grp); + mm->sched_cache_grp = NULL; } /* because why would C be fully specified */ @@ -1653,11 +1712,16 @@ static unsigned long fraction_mm_sched(struct rq *rq, static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) { int mm_sched_llc = -1, mm_sched_cpu; + struct sched_cache_group *grp; if (!mm) return -1; - mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return -1; + + mm_sched_cpu = READ_ONCE(grp->cpu); if (mm_sched_cpu != -1) { mm_sched_llc = llc_id(mm_sched_cpu); @@ -1688,6 +1752,7 @@ static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) { struct sched_cache_time *pcpu_sched; + struct sched_cache_group *grp; struct mm_struct *mm = p->mm; int mm_sched_llc = -1; unsigned long epoch; @@ -1701,10 +1766,14 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * init_task, kthreads and user thread created * by user_mode_thread() don't have mm. */ - if (!mm || !mm->sc_stat.pcpu_sched) + if (!mm) + return; + + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp || !grp->pcpu_sched) return; - pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq)); + pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq)); scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) { __update_mm_sched(rq, pcpu_sched); @@ -1717,11 +1786,11 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * If this process hasn't hit task_cache_work() for a while invalidate * its preferred state. */ - if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout || + if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout || invalid_llc_nr(mm, p, cpu_of(rq)) || exceed_llc_capacity(mm, cpu_of(rq))) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); } mm_sched_llc = get_pref_llc(p, mm); @@ -1738,30 +1807,35 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) static void task_tick_cache(struct rq *rq, struct task_struct *p) { struct callback_head *work = &p->cache_work; + struct sched_cache_group *grp; struct mm_struct *mm = p->mm; unsigned long epoch; if (!sched_cache_enabled()) return; - if (!mm || p->flags & PF_KTHREAD || - !mm->sc_stat.pcpu_sched) + if (!mm || p->flags & PF_KTHREAD) + return; + + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp || !grp->pcpu_sched) return; epoch = rq->cpu_epoch; /* avoid moving backwards */ - if (time_after_eq(mm->sc_stat.epoch, epoch)) + if (time_after_eq(grp->epoch, epoch)) return; - guard(raw_spinlock)(&mm->sc_stat.lock); + guard(raw_spinlock)(&grp->lock); if (work->next == work) { task_work_add(p, work, TWA_RESUME); - WRITE_ONCE(mm->sc_stat.epoch, epoch); + WRITE_ONCE(grp->epoch, epoch); } } -static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) +static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p, + struct sched_cache_group *grp) { #ifdef CONFIG_NUMA_BALANCING int cpu, curr_cpu, nid, pref_nid; @@ -1769,7 +1843,7 @@ static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) if (!static_branch_likely(&sched_numa_balancing)) goto out; - cpu = READ_ONCE(p->mm->sc_stat.cpu); + cpu = READ_ONCE(grp->cpu); if (cpu != -1) nid = cpu_to_node(cpu); curr_cpu = task_cpu(p); @@ -1830,6 +1904,7 @@ static void task_cache_work(struct callback_head *work) unsigned long next_scan, now = jiffies; struct task_struct *p = current, *cur; unsigned long curr_m_a_occ = 0; + struct sched_cache_group *grp; struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; cpumask_var_t cpus; @@ -1841,12 +1916,16 @@ static void task_cache_work(struct callback_head *work) if (p->flags & PF_EXITING) return; - next_scan = READ_ONCE(mm->sc_stat.next_scan); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return; + + next_scan = READ_ONCE(grp->next_scan); if (time_before(now, next_scan)) return; /* only 1 thread is allowed to scan */ - if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan, + if (!try_cmpxchg(&grp->next_scan, &next_scan, now + max_t(unsigned long, READ_ONCE(llc_epoch_period), 1))) return; @@ -1854,8 +1933,8 @@ static void task_cache_work(struct callback_head *work) curr_cpu = task_cpu(p); if (invalid_llc_nr(mm, p, curr_cpu) || exceed_llc_capacity(mm, curr_cpu)) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); return; } @@ -1866,7 +1945,7 @@ static void task_cache_work(struct callback_head *work) scoped_guard (cpus_read_lock) { guard(rcu)(); - get_scan_cpumasks(cpus, p); + get_scan_cpumasks(cpus, p, grp); for_each_cpu(cpu, cpus) { /* XXX sched_cluster_active */ @@ -1879,7 +1958,7 @@ static void task_cache_work(struct callback_head *work) for_each_cpu(i, sched_domain_span(sd)) { occ = fraction_mm_sched(cpu_rq(i), - per_cpu_ptr(mm->sc_stat.pcpu_sched, i)); + per_cpu_ptr(grp->pcpu_sched, i)); a_occ += occ; if (occ > m_occ) { m_occ = occ; @@ -1912,7 +1991,7 @@ static void task_cache_work(struct callback_head *work) m_a_cpu = m_cpu; } - if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu))) + if (llc_id(cpu) == llc_id(READ_ONCE(grp->cpu))) curr_m_a_occ = a_occ; cpumask_andnot(cpus, cpus, sched_domain_span(sd)); @@ -1921,7 +2000,7 @@ static void task_cache_work(struct callback_head *work) if (m_a_occ > (2 * curr_m_a_occ)) { /* - * Avoid switching sc_stat.cpu too fast. + * Avoid switching sched_cache_grp->cpu too fast. * The reason to choose 2X is because: * 1. It is better to keep the preferred LLC stable, * rather than changing it frequently and cause migrations @@ -1930,10 +2009,10 @@ static void task_cache_work(struct callback_head *work) * 3. 2X is chosen based on test results, as it delivers * the optimal performance gain so far. */ - WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu); + WRITE_ONCE(grp->cpu, m_a_cpu); } - update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running); + update_avg_scale(&grp->nr_running_avg, nr_running); free_cpumask_var(cpus); } @@ -3639,6 +3718,7 @@ static int preferred_group_nid(struct task_struct *p, int nid) static void task_numa_placement(struct task_struct *p) __context_unsafe(/* conditional locking */) { + struct sched_cache_group __maybe_unused *grp; int seq, nid, max_nid = NUMA_NO_NODE; unsigned long max_faults = 0; unsigned long fault_types[2] = { 0, 0 }; @@ -3731,19 +3811,23 @@ static void task_numa_placement(struct task_struct *p) * heuristic and occasional lost updates are tolerable. * * If a task exits, its corresponding footprint must - * be subtracted from the mm->sc_stat.footprint, otherwise - * the mm->sc_stat.footprint will not converge: - * the exiting thread's footprint remains unchanged/undecayed - * in mm->sc_stat.footprint. See exit_mm(). + * be subtracted from the mm->sched_cache_grp->footprint, + * otherwise the mm->sched_cache_grp->footprint will not + * converge: the exiting thread's footprint remains + * unchanged/undecayed in mm->sched_cache_grp->footprint. + * See exit_mm(). * * Lost updates and unsynchronized subtraction * in exit_mm() can cause footprint + diff to * go negative. Clamp to zero to prevent the * unsigned footprint from wrapping. */ - new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff; - WRITE_ONCE(p->mm->sc_stat.footprint, - max(new_fp, 0L)); + grp = READ_ONCE(p->mm->sched_cache_grp); + if (!grp) + continue; + + new_fp = (long)READ_ONCE(grp->footprint) + diff; + WRITE_ONCE(grp->footprint, max(new_fp, 0L)); #endif } @@ -5174,7 +5258,8 @@ static inline void update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) { long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; - u32 new_sum, divider; + u64 new_sum; + u32 divider; /* Nothing to update */ if (!delta_avg) @@ -5188,7 +5273,7 @@ update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cf /* Set new sched_entity's runnable */ se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; - new_sum = se->avg.runnable_avg * divider; + new_sum = (u64)se->avg.runnable_avg * divider; delta_sum = (long)new_sum - (long)se->avg.runnable_sum; se->avg.runnable_sum = new_sum; @@ -9780,7 +9865,8 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f /* * XXX Getting preempted by higher class, try and find idle CPU? */ - if (p->sched_class != &fair_sched_class) + if (p->sched_class != &fair_sched_class || + donor->sched_class != &fair_sched_class) return; if (unlikely(se == pse)) @@ -10572,6 +10658,7 @@ static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu, static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, struct task_struct *p) { + struct sched_cache_group *grp; struct mm_struct *mm; bool to_pref; int cpu; @@ -10580,15 +10667,19 @@ static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, if (!mm) return mig_unrestricted; - cpu = READ_ONCE(mm->sc_stat.cpu); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return mig_unrestricted; + + cpu = READ_ONCE(grp->cpu); if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu)) return mig_unrestricted; /* skip cache aware load balance for too many threads */ if (invalid_llc_nr(mm, p, dst_cpu) || exceed_llc_capacity(mm, dst_cpu)) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); return mig_unrestricted; } @@ -11865,7 +11956,9 @@ static inline void update_sg_lb_stats(struct lb_env *env, /* Check for a misfit task on the cpu */ if (sgs->group_misfit_task_load < rq->misfit_task_load) { sgs->group_misfit_task_load = rq->misfit_task_load; - *sg_overloaded = 1; + + if (balancing_at_rd) + *sg_overloaded = 1; } } else if (env->idle && sched_reduced_capacity(rq, env->sd)) { /* Check for a task running on a CPU with reduced capacity */ @@ -11944,6 +12037,17 @@ static bool update_sd_pick_busiest(struct lb_env *env, sds->local_stat.group_type != group_has_spare)) return false; + /* + * Candidate sg has no more than one task per CPU and has higher + * per-CPU capacity. Migrating tasks to less capable CPUs may harm + * throughput. Maximize throughput, power/energy consequences are not + * considered. + */ + if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && + (sgs->group_type <= group_fully_busy) && + (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) + return false; + if (sgs->group_type > busiest->group_type) return true; @@ -12050,17 +12154,6 @@ static bool update_sd_pick_busiest(struct lb_env *env, break; } - /* - * Candidate sg has no more than one task per CPU and has higher - * per-CPU capacity. Migrating tasks to less capable CPUs may harm - * throughput. Maximize throughput, power/energy consequences are not - * considered. - */ - if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && - (sgs->group_type <= group_fully_busy) && - (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) - return false; - return true; } diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index ef152d401fe20..156025ef81b75 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -8,6 +8,7 @@ * */ #include +#include #include #include "sched.h" @@ -27,6 +28,7 @@ struct housekeeping { }; static struct housekeeping housekeeping; +static __initdata LLIST_HEAD(memblock_freelist); bool housekeeping_enabled(enum hk_type type) { @@ -189,10 +191,22 @@ void __init housekeeping_init(void) WARN_ON_ONCE(cpumask_empty(omask)); cpumask_copy(nmask, omask); RCU_INIT_POINTER(housekeeping.cpumasks[type], nmask); - memblock_free(omask, cpumask_size()); + __llist_add((struct llist_node *)omask, &memblock_freelist); } } +static int __init housekeeping_late_init(void) +{ + struct llist_node *llnode, *pos, *t; + + /* Free allocated memblock memory, if any */ + llnode = __llist_del_all(&memblock_freelist); + llist_for_each_safe(pos, t, llnode) + memblock_free(pos, cpumask_size()); + return 0; +} +pure_initcall(housekeeping_late_init); + static void __init housekeeping_setup_type(enum hk_type type, cpumask_var_t housekeeping_staging) { diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index e474c31d8fe66..85303add726df 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1629,7 +1629,8 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags) /* * XXX If we're preempted by DL, queue a push? */ - if (p->sched_class != &rt_sched_class) + if (p->sched_class != &rt_sched_class || + donor->sched_class != &rt_sched_class) return; if (p->prio < donor->prio) { @@ -1871,8 +1872,8 @@ static struct task_struct *pick_next_pushable_task(struct rq *rq) return NULL; plist_for_each_entry(i, head, pushable_tasks) { - /* make sure task isn't on_cpu (possible with proxy-exec) */ - if (!task_on_cpu(rq, i)) { + /* skip tasks that cannot be migrated */ + if (!task_on_cpu(rq, i) && !is_migration_disabled(i)) { p = i; break; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 56acf502ba260..f76e6f7f4a014 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -784,7 +784,6 @@ enum scx_rq_flags { */ SCX_RQ_ONLINE = 1 << 0, SCX_RQ_CAN_STOP_TICK = 1 << 1, - SCX_RQ_BAL_KEEP = 1 << 3, /* balance decided to keep current */ SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */ SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */ @@ -804,6 +803,9 @@ struct scx_rq { bool cpu_released; u32 flags; u32 nr_immed; /* ENQ_IMMED tasks on local_dsq */ +#ifdef CONFIG_SCHED_CORE + u32 lock_drop_seq; /* nr dispatch lock releases */ +#endif u64 clock; /* current per-rq clock -- see scx_bpf_now() */ cpumask_var_t cpus_to_kick; cpumask_var_t cpus_to_kick_if_idle; @@ -1358,6 +1360,7 @@ struct rq { unsigned int core_forceidle_seq; unsigned int core_forceidle_occupation; u64 core_forceidle_start; + unsigned int core_pick_in_flight; #endif /* CONFIG_SCHED_CORE */ /* Scratch cpumask to be temporarily used under rq_lock */ diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 622e2e01974c4..70b63224a230d 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -985,8 +985,8 @@ void sched_cache_active_set(void) } /* - * Update the bottom sched_domain's llc_bytes for @cpu and all its - * LLC siblings. Called from cacheinfo_cpu_online() or + * Update the bottom sched_domain's llc_bytes for @cpus sharing a physical + * LLC. Called from cacheinfo_cpu_online() or * cacheinfo_cpu_pre_down() with cpu hotplug lock held. * * Note: get_effective_llc_bytes() returns 0 on PowerPC. @@ -996,17 +996,13 @@ void sched_cache_active_set(void) * and does not populates the per-CPU struct cpu_cacheinfo array * that get_cpu_cacheinfo_llc() reads. */ -void sched_update_llc_bytes(unsigned int cpu) +void sched_update_llc_bytes(const struct cpumask *cpus) { struct sched_domain *sd, *sdp; unsigned int i; sched_domains_mutex_lock(); - sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu)); - if (!sdp) - goto unlock; - /* * ci->shared_cpu_map is built incrementally as CPUs come * online, so the first CPU in an LLC initially sees @@ -1014,14 +1010,22 @@ void sched_update_llc_bytes(unsigned int cpu) * get_effective_llc_bytes(). Re-evaluating every LLC * sibling on each online event corrects this once the full * shared_cpu_map is known. + * + * The departing CPU's domains have already been detached when + * cacheinfo removes it. Use the surviving cache siblings instead. + * They may belong to different cpuset partitions, so use each CPU's + * own LLC domain to scale its share of the physical cache. */ - for_each_cpu(i, sched_domain_span(sdp)) { + for_each_cpu(i, cpus) { + sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, i)); + if (!sdp) + continue; + sd = rcu_dereference_sched_domain(cpu_rq(i)->sd); if (sd) sd->llc_bytes = get_effective_llc_bytes(i, sdp); } -unlock: sched_domains_mutex_unlock(); } diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 20f27e2cf7aec..d033f600f48c6 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -5,6 +5,7 @@ * (C) 2004 Nadia Yvette Chambers, Oracle */ #include "sched.h" +#include void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key) { @@ -463,3 +464,17 @@ int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sy return default_wake_function(wq_entry, mode, sync, key); } EXPORT_SYMBOL(woken_wake_function); + +int woken_wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg) +{ + struct wait_bit_key *key = __var_wake_key(wq_entry, arg); + if (!key) + return 0; + + /* Pairs with the smp_store_mb() in wait_woken(). */ + smp_mb(); /* C */ + wq_entry->flags |= WQ_FLAG_WOKEN; + + return default_wake_function(wq_entry, mode, sync, key); +} +EXPORT_SYMBOL(woken_wake_bit_function); diff --git a/kernel/sched/wait_bit.c b/kernel/sched/wait_bit.c index 1088d3b7012cf..348f7211b4aa4 100644 --- a/kernel/sched/wait_bit.c +++ b/kernel/sched/wait_bit.c @@ -167,9 +167,7 @@ wait_queue_head_t *__var_waitqueue(void *p) } EXPORT_SYMBOL(__var_waitqueue); -static int -var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, - int sync, void *arg) +struct wait_bit_key *__var_wake_key(struct wait_queue_entry *wq_entry, void *arg) { struct wait_bit_key *key = arg; struct wait_bit_queue_entry *wbq_entry = @@ -177,6 +175,16 @@ var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, if (wbq_entry->key.flags != key->flags || wbq_entry->key.bit_nr != key->bit_nr) + return NULL; + + return key; +} + +static int var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, + int sync, void *arg) +{ + struct wait_bit_key *key = __var_wake_key(wq_entry, arg); + if (!key) return 0; return autoremove_wake_function(wq_entry, mode, sync, key); diff --git a/kernel/signal.c b/kernel/signal.c index bbc0fd4cc4d7c..808980d08371d 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -475,18 +475,42 @@ static void __sigqueue_free(struct sigqueue *q) kmem_cache_free(sigqueue_cachep, q); } -void flush_sigqueue(struct sigpending *queue) +/* + * flush_sigqueue_list() can only be invoked without holding sighand::siglock in + * the following cases: + * + * 1) When flushing task::pending _after_ setting task::flags PF_EXITING + * + * All functions which try to send a signal to @task will observe PF_EXITING + * and drop the signal. + * + * 2) When flushing task::signal::shared_pending _after_ the last task in a + * thread group was unhashed and task::sighand is NULL. + * + * Nothing can queue a signal anymore because sighand is NULL. + */ +static void flush_sigqueue_list(struct list_head *head) { - struct sigqueue *q; + struct sigqueue *q, *tmp; - sigemptyset(&queue->signal); - while (!list_empty(&queue->list)) { - q = list_entry(queue->list.next, struct sigqueue , list); + list_for_each_entry_safe(q, tmp, head, list) { list_del_init(&q->list); __sigqueue_free(q); } } +void flush_sigqueue(struct sigpending *queue) +{ + sigemptyset(&queue->signal); + flush_sigqueue_list(&queue->list); +} + +static void sigqueue_dequeue_pending(struct sigpending *queue, struct list_head *head) +{ + sigemptyset(&queue->signal); + list_splice_init(&queue->list, head); +} + /* * Flush all pending signals for this kthread. */ @@ -1037,6 +1061,21 @@ static inline bool legacy_queue(struct sigpending *signals, int sig) return (sig < SIGRTMIN) && sigismember(&signals->signal, sig); } +/* + * When PF_EXITING is set the task is on the way out and has t::pending + * flushed already. Prevent queueing of PIDTYPE_PID signals as they would + * be leaked. + */ +static inline bool task_can_queue_signal(struct task_struct *t, enum pid_type type) +{ + lockdep_assert_held(&t->sighand->siglock); + + if (!(t->flags & PF_EXITING)) + return true; + + return type != PIDTYPE_PID; +} + static int __send_signal_locked(int sig, struct kernel_siginfo *info, struct task_struct *t, enum pid_type type, bool force) { @@ -1048,6 +1087,10 @@ static int __send_signal_locked(int sig, struct kernel_siginfo *info, lockdep_assert_held(&t->sighand->siglock); result = TRACE_SIGNAL_IGNORED; + + if (!task_can_queue_signal(t, type)) + goto ret; + if (!prepare_signal(sig, t, force)) goto ret; @@ -1181,6 +1224,7 @@ static inline bool has_si_pid_and_uid(struct kernel_siginfo *info) int send_signal_locked(int sig, struct kernel_siginfo *info, struct task_struct *t, enum pid_type type) { + struct kernel_siginfo rewritten; /* Should SIGKILL or SIGSTOP be received by a pid namespace init? */ bool force = false; @@ -1194,6 +1238,9 @@ int send_signal_locked(int sig, struct kernel_siginfo *info, /* SIGKILL and SIGSTOP is special or has ids */ struct user_namespace *t_user_ns; + rewritten = *info; + info = &rewritten; + rcu_read_lock(); t_user_ns = task_cred_xxx(t, user_ns); if (current_user_ns() != t_user_ns) { @@ -1899,6 +1946,18 @@ int kill_pid(struct pid *pid, int sig, int priv) } EXPORT_SYMBOL(kill_pid); +int kill_cad_pid(int sig, int priv) +{ + int ret; + + rcu_read_lock(); + ret = kill_pid(rcu_dereference(cad_pid), sig, priv); + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL(kill_cad_pid); + #ifdef CONFIG_POSIX_TIMERS /* * These functions handle POSIX timer signals. POSIX timers use @@ -1975,11 +2034,25 @@ static inline struct task_struct *posixtimer_get_target(struct k_itimer *tmr) struct task_struct *t = pid_task(tmr->it_pid, tmr->it_pid_type); if (t && tmr->it_pid_type != PIDTYPE_PID && - same_thread_group(t, current) && !current->exit_state) + same_thread_group(t, current) && !(current->flags & PF_EXITING)) t = current; return t; } +/* + * Find the target task for the POSIX timer signal and prevent that a + * PIDTYPE_PID signal is queued on a task which has PF_EXITING set. + */ +static inline struct task_struct *posixtimer_get_unignore_target(struct k_itimer *tmr) +{ + struct task_struct *t = posixtimer_get_target(tmr); + + if (t && task_can_queue_signal(t, tmr->it_pid_type)) + return t; + + return NULL; +} + void posixtimer_send_sigqueue(struct k_itimer *tmr) { struct sigqueue *q = &tmr->sigq; @@ -1997,6 +2070,9 @@ void posixtimer_send_sigqueue(struct k_itimer *tmr) if (!likely(lock_task_sighand(t, &flags))) return; + if (!task_can_queue_signal(t, tmr->it_pid_type)) + goto unlock; + /* * Update @tmr::sigqueue_seq for posix timer signals with sighand * locked to prevent a race against dequeue_signal(). @@ -2088,6 +2164,7 @@ void posixtimer_send_sigqueue(struct k_itimer *tmr) result = TRACE_SIGNAL_DELIVERED; out: trace_signal_generate(sig, &q->info, t, tmr->it_pid_type != PIDTYPE_PID, result); +unlock: unlock_task_sighand(t, &flags); } @@ -2143,7 +2220,7 @@ static void posixtimer_sig_unignore(struct task_struct *tsk, int sig) * has exited by now, drop the reference count. */ guard(rcu)(); - target = posixtimer_get_target(tmr); + target = posixtimer_get_unignore_target(tmr); if (target) posixtimer_queue_sigqueue(&tmr->sigq, target, tmr->it_pid_type); else @@ -3127,42 +3204,36 @@ static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which) void exit_signals(struct task_struct *tsk) { + LIST_HEAD(sigq_list); int group_stop = 0; - sigset_t unblocked; /* * @tsk is about to have PF_EXITING set - lock out users which - * expect stable threadgroup. + * expect a stable threadgroup. */ cgroup_threadgroup_change_begin(tsk); - if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) { + scoped_guard(spinlock_irq, &tsk->sighand->siglock) { tsk->flags |= PF_EXITING; - cgroup_threadgroup_change_end(tsk); - return; - } - spin_lock_irq(&tsk->sighand->siglock); - /* - * From now this task is not visible for group-wide signals, - * see wants_signal(), do_signal_stop(). - */ - tsk->flags |= PF_EXITING; + sigqueue_dequeue_pending(&tsk->pending, &sigq_list); - cgroup_threadgroup_change_end(tsk); + if (task_sigpending(tsk) && !thread_group_empty(tsk) && + !(tsk->signal->flags & SIGNAL_GROUP_EXIT)) { + sigset_t unblocked = tsk->blocked; - if (!task_sigpending(tsk)) - goto out; + signotset(&unblocked); + retarget_shared_pending(tsk, &unblocked); - unblocked = tsk->blocked; - signotset(&unblocked); - retarget_shared_pending(tsk, &unblocked); + if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) && + task_participate_group_stop(tsk)) + group_stop = CLD_STOPPED; + } + } - if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) && - task_participate_group_stop(tsk)) - group_stop = CLD_STOPPED; -out: - spin_unlock_irq(&tsk->sighand->siglock); + cgroup_threadgroup_change_end(tsk); + + flush_sigqueue_list(&sigq_list); /* * If group stop has completed, deliver the notification. This diff --git a/kernel/sys.c b/kernel/sys.c index df69bd71de032..35b538ba843c7 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2189,7 +2189,7 @@ static int prctl_set_auxv(struct mm_struct *mm, unsigned long addr, BUILD_BUG_ON(sizeof(user_auxv) != sizeof(mm->saved_auxv)); task_lock(current); - memcpy(mm->saved_auxv, user_auxv, len); + memcpy(mm->saved_auxv, user_auxv, sizeof(user_auxv)); task_unlock(current); return 0; diff --git a/kernel/taskstats.c b/kernel/taskstats.c index 2cd0172d05160..05a905e64fa1e 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c @@ -368,10 +368,9 @@ static int parse(struct nlattr *na, struct cpumask *mask) return -E2BIG; if (len < 1) return -EINVAL; - data = kmalloc(len, GFP_KERNEL); + data = nla_strdup(na, GFP_KERNEL); if (!data) return -ENOMEM; - nla_strscpy(data, na, len); ret = cpulist_parse(data, mask); kfree(data); return ret; diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 0014d163f989b..62ad7c000386e 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -615,33 +615,40 @@ void clockevents_handle_noop(struct clock_event_device *dev) { } -/** - * clockevents_exchange_device - release and request clock devices - * @old: device to release (can be NULL) - * @new: device to request (can be NULL) - * - * Called from various tick functions with clockevents_lock held and - * interrupts disabled. - */ -void clockevents_exchange_device(struct clock_event_device *old, - struct clock_event_device *new) +void __clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new) { /* * Caller releases a clock event device. We queue it into the * released list and do a notify add later. */ if (old) { - module_put(old->owner); clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED); list_move(&old->list, &clockevents_released); } if (new) { - BUG_ON(!clockevent_state_detached(new)); + WARN_ON(!clockevent_state_detached(new)); clockevents_shutdown(new); } } +/** + * clockevents_exchange_device - release and request clock devices + * @old: device to release (can be NULL) + * @new: device to request (can be NULL) + * + * Called from various tick functions with clockevents_lock held and + * interrupts disabled. + */ +void clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new) +{ + __clockevents_exchange_device(old, new); + if (old) + module_put(old->owner); +} + /** * clockevents_suspend - suspend clock devices */ @@ -699,7 +706,7 @@ void tick_offline_cpu(unsigned int cpu) if (cpumask_test_cpu(cpu, dev->cpumask) && cpumask_weight(dev->cpumask) == 1 && !tick_is_broadcast_device(dev)) { - BUG_ON(!clockevent_state_detached(dev)); + WARN_ON(!clockevent_state_detached(dev)); list_del(&dev->list); } } diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index e48c4d379a7ce..5a786b3c778ce 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -1566,8 +1566,12 @@ static int __init init_clocksource_sysfs(void) { int error = subsys_system_register(&clocksource_subsys, NULL); - if (!error) - error = device_register(&device_clocksource); + if (error) + return error; + + error = device_register(&device_clocksource); + if (error) + bus_unregister(&clocksource_subsys); return error; } diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 313dcea127fe4..71015e59f5de6 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1236,13 +1236,23 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b { bool was_first = false; + /* + * Updating the sort key while @timer is queued can temporarily + * make the tree inconsistent. This is safe under cpu_base->lock: + * no other queue operation can observe that state. + * hrtimer_can_update_in_place() either confirms that the new expiry + * fits between the neighbours or timerqueue_linked_del() removes the + * timer without consulting the expiry. + */ + hrtimer_set_expires_range_ns(timer, expires, delta_ns); + expires = hrtimer_get_expires(timer); + /* Remove it from the timer queue if active */ if (timer->is_queued) { was_first = !timerqueue_linked_prev(&timer->node); /* Try to update in place to avoid the de/enqueue dance */ if (hrtimer_can_update_in_place(timer, base, expires)) { - hrtimer_set_expires_range_ns(timer, expires, delta_ns); trace_hrtimer_start(timer, mode, true); if (was_first) base->expires_next = expires; @@ -1253,9 +1263,6 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b timerqueue_linked_del(&base->active, &timer->node); } - /* Set the new expiry time */ - hrtimer_set_expires_range_ns(timer, expires, delta_ns); - debug_activate(timer, mode, timer->is_queued); base->cpu_base->active_bases |= 1 << base->index; @@ -2228,8 +2235,10 @@ void hrtimer_interrupt(struct clock_event_device *dev) expires_next = hrtimer_update_next_event(cpu_base); cpu_base->hang_detected = false; if (expires_next < now) { - if (++retries < 3) + if (++retries < 3) { + cpu_base->nr_retries++; goto retry; + } delta = ktime_sub(now, entry_time); cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta); diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c index 7c6110e964e7e..03a32dffc56cc 100644 --- a/kernel/time/itimer.c +++ b/kernel/time/itimer.c @@ -100,7 +100,7 @@ static int do_getitimer(int which, struct itimerspec64 *value) static int put_itimerval(struct __kernel_old_itimerval __user *o, const struct itimerspec64 *i) { - struct __kernel_old_itimerval v; + struct __kernel_old_itimerval v = {}; v.it_interval.tv_sec = i->it_interval.tv_sec; v.it_interval.tv_usec = i->it_interval.tv_nsec / NSEC_PER_USEC; diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index d51428867a339..f4c43b9716f89 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -181,7 +181,7 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, int *k_ptr, int dir, const struct ctl_table *tbl) { - return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, false, + return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, true, sysctl_u2k_int_conv_ms, sysctl_k2u_int_conv_ms); } diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 5fa0af66cf3f7..3aff27bb0a154 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -293,10 +293,12 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p, return -EINVAL; } - if (off->val.tv_sec > KTIME_SEC_MAX || - off->val.tv_sec < -KTIME_SEC_MAX) + if (off->val.tv_sec > KTIME_SEC_MAX || off->val.tv_sec < -KTIME_SEC_MAX) return -ERANGE; + if (off->val.tv_nsec < 0 || off->val.tv_nsec >= NSEC_PER_SEC) + return -EINVAL; + tp = timespec64_add(tp, off->val); /* * KTIME_SEC_MAX is divided by 2 to be sure that KTIME_MAX is diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index a7d3e8229c4ba..8e3e3917a2f7a 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -408,6 +408,7 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer) new_timer->kclock = &clock_posix_cpu; timerqueue_init(&new_timer->it.cpu.node); + INIT_LIST_HEAD(&new_timer->it.cpu.elist); new_timer->it.cpu.pid = get_pid(pid); rcu_read_unlock(); return 0; @@ -565,6 +566,24 @@ static struct task_struct *timer_lock_sighand(struct k_itimer *timer, unsigned l return NULL; } +/* + * If the timer is queued on the expiry list, then it cannot be dequeued because + * the firing list is not protected by sighand->lock. The delivery path is + * waiting for the timer lock. So go back, unlock and retry. + */ +static bool posix_cpu_timer_on_expiry_list(struct k_itimer *timer) +{ + if (list_empty(&timer->it.cpu.elist)) + return false; + + /* + * Prevent signal delivery as there is no point in delivering a signal + * which is made obsolete right away. + */ + timer->it.cpu.firing = false; + return true; +} + /* * Clean up a CPU-clock timer that is about to be destroyed. * This is called from timer deletion with the timer already locked. @@ -580,18 +599,10 @@ static int posix_cpu_timer_del(struct k_itimer *timer) p = timer_lock_sighand(timer, &flags); if (likely(p)) { - if (timer->it.cpu.firing) { - /* - * Prevent signal delivery. The timer cannot be dequeued - * because it is on the firing list which is not protected - * by sighand->lock. The delivery path is waiting for - * the timer lock. So go back, unlock and retry. - */ - timer->it.cpu.firing = false; + if (posix_cpu_timer_on_expiry_list(timer)) ret = TIMER_RETRY; - } else { + else disarm_timer(timer, p); - } unlock_task_sighand(p, &flags); } @@ -731,14 +742,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, /* Retrieve the current expiry time before disarming the timer */ old_expires = cpu_timer_getexpires(ctmr); - if (unlikely(timer->it.cpu.firing)) { - /* - * Prevent signal delivery. The timer cannot be dequeued - * because it is on the firing list which is not protected - * by sighand->lock. The delivery path is waiting for - * the timer lock. So go back, unlock and retry. - */ - timer->it.cpu.firing = false; + if (posix_cpu_timer_on_expiry_list(timer)) { ret = TIMER_RETRY; } else { cpu_timer_dequeue(ctmr); diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 115e0bf012764..bda3d2391a60b 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -165,23 +165,31 @@ static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev, */ void tick_install_broadcast_device(struct clock_event_device *dev, int cpu) { - struct clock_event_device *cur = tick_broadcast_device.evtdev; + struct clock_event_device *cur; - if (tick_set_oneshot_wakeup_device(dev, cpu)) - return; + scoped_guard(raw_spinlock_irqsave, &tick_broadcast_lock) { - if (!tick_check_broadcast_device(cur, dev)) - return; + if (tick_set_oneshot_wakeup_device(dev, cpu)) + return; - if (!try_module_get(dev->owner)) - return; + cur = tick_broadcast_device.evtdev; + if (!tick_check_broadcast_device(cur, dev)) + return; - clockevents_exchange_device(cur, dev); + if (!try_module_get(dev->owner)) + return; + + __clockevents_exchange_device(cur, dev); + if (cur) + cur->event_handler = clockevents_handle_noop; + WRITE_ONCE(tick_broadcast_device.evtdev, dev); + if (!cpumask_empty(tick_broadcast_mask)) + tick_broadcast_start_periodic(dev); + } + + /* Module release must be outside of the lock */ if (cur) - cur->event_handler = clockevents_handle_noop; - tick_broadcast_device.evtdev = dev; - if (!cpumask_empty(tick_broadcast_mask)) - tick_broadcast_start_periodic(dev); + module_put(cur->owner); if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT)) return; @@ -1218,7 +1226,7 @@ int tick_broadcast_oneshot_active(void) */ bool tick_broadcast_oneshot_available(void) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; + struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev); return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false; } @@ -1226,7 +1234,7 @@ bool tick_broadcast_oneshot_available(void) #else int __tick_broadcast_oneshot_control(enum tick_broadcast_state state) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; + struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev); if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER)) return -EBUSY; diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index 597d816d22e88..ce1c9d1f6b96f 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -54,6 +54,8 @@ static inline void clockevent_set_state(struct clock_event_device *dev, } extern void clockevents_shutdown(struct clock_event_device *dev); +extern void __clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new); extern void clockevents_exchange_device(struct clock_event_device *old, struct clock_event_device *new); extern void clockevents_switch_state(struct clock_event_device *dev, diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index b1b5ec43c0f27..55ebf2703f1bf 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -2390,6 +2390,11 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, * xtime_nsec_2 = xtime_nsec_1 - offset * Which simplifies to: * xtime_nsec -= offset + * + * When subtracting offset from xtime_nsec, the same amount + * (in appropriate units) has to be added to ntp_error, in + * order to correctly track the delta between the time + * reported in xtime_nsec, and the intended time. */ if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { /* NTP adjustment caused clocksource mult overflow */ @@ -2400,6 +2405,7 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, tk->tkr_mono.mult += mult_adj; tk->xtime_interval += interval; tk->tkr_mono.xtime_nsec -= offset; + tk->ntp_error += offset << tk->ntp_error_shift; } /* @@ -2943,10 +2949,12 @@ static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc, return ret; add_device_randomness(txc, sizeof(*txc)); - if (!aux_clock) + if (!aux_clock) { ktime_get_real_ts64(&ts); - else - tk_get_aux_ts64(tkd->timekeeper.id, &ts); + } else { + if (!tk_get_aux_ts64(tkd->timekeeper.id, &ts)) + return -ENODEV; + } add_device_randomness(&ts, sizeof(ts)); @@ -3313,7 +3321,9 @@ static const struct attribute_group aux_clock_enable_attr_group = { static int __init tk_aux_sysfs_init(void) { struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj); + struct kobject *clks[MAX_AUX_CLOCKS]; int ret = -ENOMEM; + int i; if (!tko) return ret; @@ -3322,21 +3332,28 @@ static int __init tk_aux_sysfs_init(void) if (!auxo) goto err_clean; - for (int i = 0; i < MAX_AUX_CLOCKS; i++) { + for (i = 0; i < MAX_AUX_CLOCKS; i++) { char id[2] = { [0] = '0' + i, }; - struct kobject *clk = kobject_create_and_add(id, auxo); + clks[i] = kobject_create_and_add(id, auxo); - if (!clk) { + if (!clks[i]) { ret = -ENOMEM; - goto err_clean; + goto err_clks; } - ret = sysfs_create_group(clk, &aux_clock_enable_attr_group); + ret = sysfs_create_group(clks[i], &aux_clock_enable_attr_group); if (ret) - goto err_clean; + goto err_clk; } return 0; +err_clk: + kobject_put(clks[i]); +err_clks: + while (--i >= 0) { + sysfs_remove_group(clks[i], &aux_clock_enable_attr_group); + kobject_put(clks[i]); + } err_clean: kobject_put(auxo); kobject_put(tko); diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 655a8c6cd84d7..ae9abf14688e0 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -2492,6 +2492,7 @@ static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *h timer = hlist_entry(head->first, struct timer_list, entry); detach_timer(timer, false); timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu; + debug_timer_activate(timer); internal_add_timer(new_base, timer); } } diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 806c23cf71fc9..059d43355e650 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1847,8 +1847,10 @@ static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu, } /* Assert single root without parent */ - if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) + if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) { + kfree(stack); return -EINVAL; + } for (; i >= start_lvl; i--) { group = stack[i]; diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index f681015413b83..2a1d65754b04b 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -171,6 +171,11 @@ static inline bool write_fprobe_header(unsigned long *stack, static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words) { + if (!*stack) { + *fp = NULL; + *size_words = 0; + return; + } *fp = arch_decode_fprobe_header_fp(*stack); *size_words = arch_decode_fprobe_header_size(*stack); } @@ -203,6 +208,12 @@ static inline void read_fprobe_header(unsigned long *stack, { struct __fprobe_header *fph = (struct __fprobe_header *)stack; + if (!*stack) { + *fp = NULL; + *size_words = 0; + return; + } + *fp = fph->fp; *size_words = fph->size_words; } @@ -642,6 +653,10 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops } } + /* Terminate the list, fgraph_reserve_data() does not clear it. */ + if (used && used < reserved_words) + fgraph_data[used] = 0; + /* If any exit_handler is set, data must be used. */ return used != 0; } diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 7c50f8ae5a0c4..db8eeaefeb00b 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -75,6 +75,8 @@ .func_hash = &opsname.local_hash, \ .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), \ .subop_list = LIST_HEAD_INIT(opsname.subop_list), +/* Used only to synchronize the initialization of ftrace_ops */ +static DEFINE_MUTEX(ops_mutex); #else #define INIT_OPS_HASH(opsname) #endif @@ -159,11 +161,18 @@ const struct ftrace_ops ftrace_nop_ops = { static inline void ftrace_ops_init(struct ftrace_ops *ops) { #ifdef CONFIG_DYNAMIC_FTRACE - if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { + unsigned long flags = smp_load_acquire(&ops->flags); + + if (!(flags & FTRACE_OPS_FL_INITIALIZED)) { + guard(mutex)(&ops_mutex); + /* Could have been initialized before lock taken */ + if (unlikely(ops->flags & FTRACE_OPS_FL_INITIALIZED)) + return; mutex_init(&ops->local_hash.regex_lock); INIT_LIST_HEAD(&ops->subop_list); ops->func_hash = &ops->local_hash; - ops->flags |= FTRACE_OPS_FL_INITIALIZED; + flags = ops->flags | FTRACE_OPS_FL_INITIALIZED; + smp_store_release(&ops->flags, flags); } #endif } @@ -4677,7 +4686,8 @@ ftrace_avail_addrs_open(struct inode *inode, struct file *file) /** * ftrace_regex_open - initialize function tracer filter files - * @ops: The ftrace_ops that hold the hash filters + * @tr: The trace_array that holds the ftrace_ops [optional] + * @ops: The ftrace_ops that hold the hash filters [optional] * @flag: The type of filter to process * @inode: The inode, usually passed in to your open routine * @file: The file, usually passed in to your open routine @@ -4691,26 +4701,45 @@ ftrace_avail_addrs_open(struct inode *inode, struct file *file) * tracing_lseek() should be used as the lseek routine, and * release must call ftrace_regex_release(). * + * Note, If @tr is not NULL, its reference has to be taken before + * @ops may be referenced. + * If @ops is NULL and @tr is not, then @tr->ops is used. + * If @tr is NULL and @ops is not then @ops->private is uesd for @tr. + * If both @tr and @ops are NULL, then the &global_ops is + * to be used, and @tr will be the global_ops.private pointer. + * * Returns: 0 on success or a negative errno value on failure */ int -ftrace_regex_open(struct ftrace_ops *ops, int flag, +ftrace_regex_open(struct trace_array *tr, struct ftrace_ops *ops, int flag, struct inode *inode, struct file *file) { - struct ftrace_iterator *iter; + struct ftrace_iterator *iter = NULL; struct ftrace_hash *hash; struct list_head *mod_head; - struct trace_array *tr = ops->private; - int ret = -ENOMEM; - - ftrace_ops_init(ops); + int ret = -ENODEV; if (unlikely(ftrace_disabled)) return -ENODEV; + if (!tr) { + if (!ops) + ops = &global_ops; + tr = ops->private; + } + if (tracing_check_open_get_tr(tr)) return -ENODEV; + if (!ops) + ops = tr->ops; + + if (WARN_ON_ONCE(!ops)) + goto out; + + ftrace_ops_init(ops); + + ret = -ENOMEM; iter = kzalloc_obj(*iter); if (!iter) goto out; @@ -4788,21 +4817,19 @@ ftrace_regex_open(struct ftrace_ops *ops, int flag, static int ftrace_filter_open(struct inode *inode, struct file *file) { - struct ftrace_ops *ops = inode->i_private; + struct trace_array *tr = inode->i_private; - /* Checks for tracefs lockdown */ - return ftrace_regex_open(ops, - FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES, - inode, file); + return ftrace_regex_open(tr, NULL, + FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES, + inode, file); } static int ftrace_notrace_open(struct inode *inode, struct file *file) { - struct ftrace_ops *ops = inode->i_private; + struct trace_array *tr = inode->i_private; - /* Checks for tracefs lockdown */ - return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE, + return ftrace_regex_open(tr, NULL, FTRACE_ITER_NOTRACE, inode, file); } @@ -6648,7 +6675,7 @@ int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, b /* Enable the tmp_ops to have the same functions as the hash object. */ ftrace_ops_init(&tmp_ops); - tmp_ops.func_hash->filter_hash = hash; + rcu_assign_pointer(tmp_ops.func_hash->filter_hash, hash); err = register_ftrace_function_nolock(&tmp_ops); if (err) @@ -7492,15 +7519,15 @@ static const struct file_operations ftrace_graph_notrace_fops = { }; #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ -void ftrace_create_filter_files(struct ftrace_ops *ops, +void ftrace_create_filter_files(struct trace_array *tr, struct dentry *parent) { trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent, - ops, &ftrace_filter_fops); + tr, &ftrace_filter_fops); trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent, - ops, &ftrace_notrace_fops); + tr, &ftrace_notrace_fops); } /* @@ -7525,7 +7552,6 @@ void ftrace_destroy_filter_files(struct ftrace_ops *ops) static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { - trace_create_file("available_filter_functions", TRACE_MODE_READ, d_tracer, NULL, &ftrace_avail_fops); @@ -7538,7 +7564,7 @@ static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) trace_create_file("touched_functions", TRACE_MODE_READ, d_tracer, NULL, &ftrace_touched_fops); - ftrace_create_filter_files(&global_ops, d_tracer); + ftrace_create_filter_files(NULL, d_tracer); #ifdef CONFIG_FUNCTION_GRAPH_TRACER trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer, diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 2667992f0aa29..7c59e8e586196 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -330,6 +330,11 @@ struct buffer_data_read_page { struct buffer_data_page *data; /* actual data, stored in this page */ }; +static __always_inline unsigned int rb_read_page_capacity(struct buffer_data_read_page *rpage) +{ + return (PAGE_SIZE << rpage->order) - BUF_PAGE_HDR_SIZE; +} + /* * Note, the buffer_page list must be first. The buffer pages * are allocated in cache lines, which means that each buffer @@ -391,6 +396,17 @@ static __always_inline unsigned int rb_page_size(struct buffer_page *bpage) return rb_data_page_size(bpage->page); } +/** + * rb_page_capacity - Get the capacity of a buffer page + * @bpage: The buffer page + * + * Return: The maximum size available for events in the given buffer page. + */ +static __always_inline unsigned int rb_page_capacity(struct buffer_page *bpage) +{ + return (PAGE_SIZE << bpage->order) - BUF_PAGE_HDR_SIZE; +} + static void free_buffer_page(struct buffer_page *bpage) { /* Range pages are not to be freed */ @@ -503,11 +519,11 @@ struct ring_buffer_per_cpu { int cpu; atomic_t record_disabled; atomic_t resize_disabled; - struct trace_buffer *buffer; + struct trace_buffer *buffer; raw_spinlock_t reader_lock; /* serialize readers */ arch_spinlock_t lock; struct lock_class_key lock_key; - struct buffer_data_page *free_page; + struct buffer_data_read_page free_page; unsigned long nr_pages; unsigned int current_context; struct list_head *pages; @@ -541,7 +557,6 @@ struct ring_buffer_per_cpu { /* pages removed since last reset */ unsigned long pages_removed; - unsigned int mapped; unsigned int user_mapped; /* user space mapping */ struct mutex mapping_lock; struct buffer_page **subbuf_ids; /* ID to subbuf VA */ @@ -561,7 +576,6 @@ struct ring_buffer_per_cpu { struct trace_buffer { unsigned flags; - int cpus; atomic_t record_disabled; atomic_t resizing; cpumask_var_t cpumask; @@ -586,11 +600,56 @@ struct trace_buffer { struct ring_buffer_meta *meta; - unsigned int subbuf_size; unsigned int subbuf_order; unsigned int max_data_size; }; +static __always_inline unsigned int rb_subbuf_size(struct trace_buffer *buffer) +{ + return PAGE_SIZE << buffer->subbuf_order; +} + +/** + * rb_subbuf_capacity - Get the capacity of a subbuffer + * @buffer: A trace buffer + * + * Unsafe to use without holding trace_buffer::mutex or with resizing enabled. + * Consider rb_page_capacity() instead. + * + * Return: The maximum size available for events in a trace buffer subbuffer. + */ +static __always_inline unsigned int rb_subbuf_capacity(struct trace_buffer *buffer) +{ + return rb_subbuf_size(buffer) - BUF_PAGE_HDR_SIZE; +} + +/** + * rb_subbuf_start - Get the start address of a subbuffer + * @buffer: A trace buffer + * @addr: An address of an event on a subbuffer + * + * Return: The start of the subbuffer for where @addr sits + */ +static __always_inline +unsigned long rb_subbuf_start(struct trace_buffer *buffer, unsigned long addr) +{ + return addr & ~((unsigned long)(rb_subbuf_size(buffer) - 1)); +} + +static bool rb_is_static(struct ring_buffer_per_cpu *cpu_buffer) +{ + return cpu_buffer->user_mapped || cpu_buffer->remote || cpu_buffer->ring_meta; +} + +static unsigned long rb_static_max_pages(void) +{ + /* + * Static ring buffers are using bpage::id and must account for the + * reader page. + */ + return (1UL << 30) - 1; +} + struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; @@ -630,7 +689,7 @@ int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq trace_seq_printf(s, "\tfield: char data;\t" "offset:%u;\tsize:%u;\tsigned:%u;\n", (unsigned int)offsetof(typeof(field), data), - (unsigned int)(buffer ? buffer->subbuf_size : + (unsigned int)(buffer ? rb_subbuf_capacity(buffer) : PAGE_SIZE - BUF_PAGE_HDR_SIZE), (unsigned int)is_signed_type(char)); @@ -826,14 +885,13 @@ static void rb_wake_up_waiters(struct irq_work *work) struct ring_buffer_per_cpu *cpu_buffer = container_of(rbwork, struct ring_buffer_per_cpu, irq_work); - /* Called from interrupt context */ - raw_spin_lock(&cpu_buffer->reader_lock); - rbwork->wakeup_full = false; - rbwork->full_waiters_pending = false; + scoped_guard(raw_spinlock_irqsave, &cpu_buffer->reader_lock) { + rbwork->wakeup_full = false; + rbwork->full_waiters_pending = false; - /* Waking up all waiters, they will reset the shortest full */ - cpu_buffer->shortest_full = 0; - raw_spin_unlock(&cpu_buffer->reader_lock); + /* Waking up all waiters, they will reset the shortest full */ + cpu_buffer->shortest_full = 0; + } wake_up_all(&rbwork->full_waiters); } @@ -1620,7 +1678,7 @@ rb_range_align_subbuf(unsigned long addr, int subbuf_size, int nr_subbufs) */ static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu) { - int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + int subbuf_size = rb_subbuf_size(buffer); struct ring_buffer_cpu_meta *meta; struct ring_buffer_meta *bmeta; unsigned long ptr; @@ -1795,6 +1853,11 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, return false; } + if (meta->nr_subbufs != nr_pages + 1) { + pr_info("Ring buffer boot meta [%d] invalid nr_subbufs\n", cpu); + return false; + } + buffers_start = meta->first_buffer; buffers_end = meta->first_buffer + (subbuf_size * meta->nr_subbufs); @@ -2432,8 +2495,8 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, bpage->id = i + 1; cpu_buffer->subbuf_ids[i + 1] = bpage; } else { - int order = cpu_buffer->buffer->subbuf_order; - bpage->page = alloc_cpu_data(cpu_buffer->cpu, order); + bpage->page = alloc_cpu_data(cpu_buffer->cpu, + cpu_buffer->buffer->subbuf_order); if (!bpage->page) goto free_pages; } @@ -2521,7 +2584,6 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) * Range mapped buffers have the same restrictions as memory * mapped ones do. */ - cpu_buffer->mapped = 1; cpu_buffer->ring_meta = rb_range_meta(buffer, nr_pages, cpu); bpage->page = rb_range_buffer(cpu_buffer, 0); if (!bpage->page) @@ -2556,8 +2618,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) bpage->range = 1; cpu_buffer->subbuf_ids[0] = bpage; } else { - int order = cpu_buffer->buffer->subbuf_order; - bpage->page = alloc_cpu_data(cpu, order); + bpage->page = alloc_cpu_data(cpu, bpage->order); if (!bpage->page) goto fail_free_reader; } @@ -2631,7 +2692,7 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) free_buffer_page(bpage); } - free_page((unsigned long)cpu_buffer->free_page); + free_pages((unsigned long)cpu_buffer->free_page.data, cpu_buffer->free_page.order); kfree(cpu_buffer); } @@ -2731,10 +2792,9 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, buffer->subbuf_order = order; subbuf_size = (PAGE_SIZE << order); - buffer->subbuf_size = subbuf_size - BUF_PAGE_HDR_SIZE; /* Max payload is buffer page size - header (8bytes) */ - buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2); + buffer->max_data_size = rb_subbuf_capacity(buffer) - (sizeof(u32) * 2); buffer->flags = flags; buffer->clock = trace_clock_local; @@ -2743,7 +2803,6 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters); init_waitqueue_head(&buffer->irq_work.waiters); - buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), @@ -2774,6 +2833,8 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, size = end - buffers_start; size = size / nr_cpu_ids; + if (size < sizeof(struct ring_buffer_cpu_meta)) + goto fail_free_buffers; /* * The number of sub-buffers (nr_pages) is determined by the * total size allocated minus the meta data size. @@ -2783,6 +2844,10 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, */ nr_pages = (size - sizeof(struct ring_buffer_cpu_meta)) / (subbuf_size + sizeof(int)); + + if (nr_pages > rb_static_max_pages()) + goto fail_free_buffers; + /* Need at least two pages plus the reader page */ if (nr_pages < 3) goto fail_free_buffers; @@ -2815,12 +2880,15 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, /* The writer is remote. This ring-buffer is read-only */ atomic_inc(&buffer->record_disabled); nr_pages = desc->nr_page_va - 1; + + if (nr_pages > rb_static_max_pages()) + goto fail_free_buffers; + if (nr_pages < 2) goto fail_free_buffers; } else { - /* need at least two pages */ - nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); + nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); if (nr_pages < 2) nr_pages = 2; } @@ -3203,7 +3271,7 @@ static void update_pages_handler(struct work_struct *work) * @size: the new size. * @cpu_id: the cpu buffer to resize * - * Minimum size is 2 * buffer->subbuf_size. + * Minimum size is 2 * rb_subbuf_capacity(buffer). * * Returns 0 on success and < 0 on failure. */ @@ -3225,12 +3293,6 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, !cpumask_test_cpu(cpu_id, buffer->cpumask)) return 0; - nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); - - /* we need a minimum of two pages */ - if (nr_pages < 2) - nr_pages = 2; - /* * Keep CPUs from coming online while resizing to synchronize * with new per CPU buffers being created. @@ -3241,6 +3303,12 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, mutex_lock(&buffer->mutex); atomic_inc(&buffer->resizing); + nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); + + /* we need a minimum of two pages */ + if (nr_pages < 2) + nr_pages = 2; + if (cpu_id == RING_BUFFER_ALL_CPUS) { /* * Don't succeed if resizing is disabled, as a reader might be @@ -3513,7 +3581,7 @@ rb_event_index(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event { unsigned long addr = (unsigned long)event; - addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1; + addr &= (unsigned long)rb_subbuf_size(cpu_buffer->buffer) - 1; return addr - BUF_PAGE_HDR_SIZE; } @@ -3755,8 +3823,8 @@ static inline void rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, unsigned long tail, struct rb_event_info *info) { - unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); struct buffer_page *tail_page = info->tail_page; + unsigned long bsize = rb_page_capacity(tail_page); struct ring_buffer_event *event; unsigned long length = info->length; @@ -4101,8 +4169,7 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, new_index = rb_event_index(cpu_buffer, event); old_index = new_index + rb_event_ts_length(event); - addr = (unsigned long)event; - addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); + addr = rb_subbuf_start(cpu_buffer->buffer, (unsigned long)event); bpage = READ_ONCE(cpu_buffer->tail_page); @@ -4767,7 +4834,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, tail = write - info->length; /* See if we shot pass the end of this buffer page */ - if (unlikely(write > cpu_buffer->buffer->subbuf_size)) { + if (unlikely(write > rb_page_capacity(tail_page))) { check_buffer(cpu_buffer, info, CHECK_FULL_PAGE); return rb_move_tail(cpu_buffer, tail, info); } @@ -5012,7 +5079,7 @@ rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer, struct buffer_page *bpage = cpu_buffer->commit_page; struct buffer_page *start; - addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); + addr = rb_subbuf_start(cpu_buffer->buffer, addr); /* Do the likely case first */ if (likely(bpage->page == (void *)addr)) { @@ -5753,8 +5820,11 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) prev_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id]; - WARN_ON_ONCE(cpu_buffer->remote->swap_reader_page(cpu_buffer->cpu, - cpu_buffer->remote->priv)); + if (cpu_buffer->remote->swap_reader_page(cpu_buffer->cpu, + cpu_buffer->remote->priv)) { + pr_warn_ratelimited("Remote reader page swap failed\n"); + return NULL; + } /* nr_pages doesn't include the reader page */ if (WARN_ON_ONCE(cpu_buffer->meta_page->reader.id > cpu_buffer->nr_pages)) return NULL; @@ -5799,7 +5869,6 @@ static struct buffer_page * __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) { int max_loops = cpu_buffer->ring_meta ? cpu_buffer->nr_pages : 3; - unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); struct buffer_page *reader = NULL; unsigned long overwrite; unsigned long flags; @@ -5947,7 +6016,7 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) #define USECS_WAIT 1000000 for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) { /* If the write is past the end of page, a writer is still updating it */ - if (likely(!reader || rb_page_write(reader) <= bsize)) + if (likely(!reader || rb_page_write(reader) <= rb_page_capacity(reader))) break; udelay(1); @@ -6380,36 +6449,44 @@ EXPORT_SYMBOL_GPL(ring_buffer_consume); struct ring_buffer_iter * ring_buffer_read_start(struct trace_buffer *buffer, int cpu, gfp_t flags) { + struct ring_buffer_iter *iter __free(kfree) = kzalloc_obj(*iter, flags); struct ring_buffer_per_cpu *cpu_buffer; - struct ring_buffer_iter *iter; - - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - return NULL; - iter = kzalloc_obj(*iter, flags); if (!iter) return NULL; - /* Holds the entire event: data and meta data */ - iter->event_size = buffer->subbuf_size; - iter->event = kmalloc(iter->event_size, flags); - if (!iter->event) { - kfree(iter); + if (!cpumask_test_cpu(cpu, buffer->cpumask)) return NULL; - } cpu_buffer = buffer->buffers[cpu]; - iter->cpu_buffer = cpu_buffer; + /* + * Only KDB is using GFP_ATOMIC, for the others, lock the buffer to + * prevent concurrent resizing. + */ + if (gfpflags_allow_blocking(flags)) + mutex_lock(&buffer->mutex); atomic_inc(&cpu_buffer->resize_disabled); + if (gfpflags_allow_blocking(flags)) + mutex_unlock(&buffer->mutex); + + /* Holds the entire event: data and meta data. */ + iter->event_size = rb_page_capacity(READ_ONCE(cpu_buffer->reader_page)); + iter->event = kmalloc(iter->event_size, flags); + if (!iter->event) { + atomic_dec(&cpu_buffer->resize_disabled); + return NULL; + } + iter->cpu_buffer = cpu_buffer; + guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); arch_spin_lock(&cpu_buffer->lock); rb_iter_reset(iter); arch_spin_unlock(&cpu_buffer->lock); - return iter; + return_ptr(iter); } EXPORT_SYMBOL_GPL(ring_buffer_read_start); @@ -6463,7 +6540,7 @@ unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu) if (!cpumask_test_cpu(cpu, buffer->cpumask)) return 0; - return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages; + return rb_subbuf_capacity(buffer) * buffer->buffers[cpu]->nr_pages; } EXPORT_SYMBOL_GPL(ring_buffer_size); @@ -6610,12 +6687,11 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) rb_head_page_activate(cpu_buffer); cpu_buffer->pages_removed = 0; - if (cpu_buffer->mapped) { - rb_update_meta_page(cpu_buffer); - if (cpu_buffer->ring_meta) { - struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; - meta->commit_buffer = meta->head_buffer; - } + rb_update_meta_page(cpu_buffer); + if (cpu_buffer->ring_meta) { + struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; + + meta->commit_buffer = meta->head_buffer; } } @@ -6864,8 +6940,8 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, cpu_buffer_a = buffer_a->buffers[cpu]; cpu_buffer_b = buffer_b->buffers[cpu]; - /* It's up to the callers to not try to swap mapped buffers */ - if (WARN_ON_ONCE(cpu_buffer_a->mapped || cpu_buffer_b->mapped)) + /* It's up to the callers to not try to swap static buffers */ + if (WARN_ON_ONCE(rb_is_static(cpu_buffer_a) || rb_is_static(cpu_buffer_b))) return -EBUSY; /* At least make sure the two buffers are somewhat the same */ @@ -6931,56 +7007,78 @@ EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu); * ring_buffer_alloc_read_page - allocate a page to read from buffer * @buffer: the buffer to allocate for. * @cpu: the cpu buffer to allocate. + * @rpage: pointer to pass in an already allocated page (can be NULL) + * and returns the allocated page. * - * This function is used in conjunction with ring_buffer_read_page. + * This function is used in conjunction with ring_buffer_read_page(). * When reading a full page from the ring buffer, these functions * can be used to speed up the process. The calling function should * allocate a few pages first with this function. Then when it * needs to get pages from the ring buffer, it passes the result - * of this function into ring_buffer_read_page, which will swap + * of this function into ring_buffer_read_page(), which will swap * the page that was allocated, with the read page of the buffer. * + * If @rpage is provided, and it has a different order than the current + * subbuffer order, its payload will be freed and re-allocated. If it + * already matches the order, it is simply returned. + * * Returns: - * The page allocated, or ERR_PTR + * 0 on success, < 0 on error */ -struct buffer_data_read_page * -ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) +int ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu, + struct buffer_data_read_page **rpage) { struct ring_buffer_per_cpu *cpu_buffer; - struct buffer_data_read_page *bpage = NULL; unsigned long flags; + unsigned int order; if (!cpumask_test_cpu(cpu, buffer->cpumask)) - return ERR_PTR(-ENODEV); + return -ENODEV; - bpage = kzalloc_obj(*bpage); - if (!bpage) - return ERR_PTR(-ENOMEM); + if (!rpage) + return -EINVAL; + + order = READ_ONCE(buffer->subbuf_order); - bpage->order = buffer->subbuf_order; + if (*rpage) { + if ((*rpage)->order == order) + return 0; + + /* We can reuse rpage, but we discard the payload */ + free_pages((unsigned long)(*rpage)->data, (*rpage)->order); + (*rpage)->data = NULL; + } else { + *rpage = kzalloc_obj(**rpage); + if (!*rpage) + return -ENOMEM; + } + + (*rpage)->order = order; cpu_buffer = buffer->buffers[cpu]; + local_irq_save(flags); arch_spin_lock(&cpu_buffer->lock); - if (cpu_buffer->free_page) { - bpage->data = cpu_buffer->free_page; - cpu_buffer->free_page = NULL; + if (cpu_buffer->free_page.data) { + **rpage = cpu_buffer->free_page; + cpu_buffer->free_page.data = NULL; } arch_spin_unlock(&cpu_buffer->lock); local_irq_restore(flags); - if (bpage->data) { - rb_init_data_page(bpage->data); + if ((*rpage)->data) { + rb_init_data_page((*rpage)->data); } else { - bpage->data = alloc_cpu_data(cpu, cpu_buffer->buffer->subbuf_order); - if (!bpage->data) { - kfree(bpage); - return ERR_PTR(-ENOMEM); + (*rpage)->data = alloc_cpu_data(cpu, (*rpage)->order); + if (!(*rpage)->data) { + kfree(*rpage); + *rpage = NULL; + return -ENOMEM; } } - return bpage; + return 0; } EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); @@ -6988,21 +7086,30 @@ EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); * ring_buffer_free_read_page - free an allocated read page * @buffer: the buffer the page was allocate for * @cpu: the cpu buffer the page came from - * @data_page: the page to free + * @rpage: the buffer_data_read_page to free * * Free a page allocated from ring_buffer_alloc_read_page. */ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, - struct buffer_data_read_page *data_page) + struct buffer_data_read_page *rpage) { struct ring_buffer_per_cpu *cpu_buffer; - struct buffer_data_page *dpage = data_page->data; - struct page *page = virt_to_page(dpage); + struct buffer_data_page *dpage; unsigned long flags; + struct page *page; if (!buffer || !buffer->buffers || !buffer->buffers[cpu]) return; + if (!rpage) + return; + + dpage = rpage->data; + if (!dpage) + goto out; + + page = virt_to_page(dpage); + cpu_buffer = buffer->buffers[cpu]; /* @@ -7010,14 +7117,14 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, * is different from the subbuffer order of the buffer - * we can't reuse it */ - if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order) + if (page_ref_count(page) > 1 || rpage->order != READ_ONCE(buffer->subbuf_order)) goto out; local_irq_save(flags); arch_spin_lock(&cpu_buffer->lock); - if (!cpu_buffer->free_page) { - cpu_buffer->free_page = dpage; + if (!cpu_buffer->free_page.data) { + cpu_buffer->free_page = *rpage; dpage = NULL; } @@ -7025,8 +7132,8 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, local_irq_restore(flags); out: - free_pages((unsigned long)dpage, data_page->order); - kfree(data_page); + free_pages((unsigned long)dpage, rpage->order); + kfree(rpage); } EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); @@ -7077,7 +7184,6 @@ int ring_buffer_read_page(struct trace_buffer *buffer, unsigned int size; unsigned int read; u64 save_timestamp; - bool force_memcpy; if (!cpumask_test_cpu(cpu, buffer->cpumask)) return -1; @@ -7094,13 +7200,12 @@ int ring_buffer_read_page(struct trace_buffer *buffer, if (!data_page || !data_page->data) return -1; - if (data_page->order != buffer->subbuf_order) - return -1; - dpage = data_page->data; if (!dpage) return -1; + len = min_t(size_t, len, rb_read_page_capacity(data_page)); + guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); reader = rb_get_reader_page(cpu_buffer); @@ -7116,33 +7221,26 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* Check if any events were dropped */ missed_events = cpu_buffer->lost_events; - force_memcpy = cpu_buffer->mapped || cpu_buffer->remote; - - /* - * If this page has been partially read or - * if len is not big enough to read the rest of the page or - * a writer is still on the page, then - * we must copy the data from the page to the buffer. - * Otherwise, we can simply swap the page with the one passed in. - */ + /* + * It is not possible to swap the reader page if: + * - It has been partially read + * - len is not big enough to read it entirely + * - A writer is still on it + * - The ring buffer is static + * - The order doesn't match + */ if (read || (len < (size - read)) || cpu_buffer->reader_page == cpu_buffer->commit_page || - force_memcpy) { + rb_is_static(cpu_buffer) || + data_page->order != reader->order) { struct buffer_data_page *rpage = cpu_buffer->reader_page->page; unsigned int rpos = read; unsigned int pos = 0; unsigned int event_size; unsigned int flags = 0; - /* - * If a full page is expected, this can still be returned - * if there's been a previous partial read and the - * rest of the page can be read and the commit page is off - * the reader page. - */ - if (full && - (!read || (len < (size - read)) || - cpu_buffer->reader_page == cpu_buffer->commit_page)) + /* If a full page is requested, it cannot be the commit page */ + if (full && cpu_buffer->reader_page == cpu_buffer->commit_page) return -1; if (len > (size - read)) @@ -7228,7 +7326,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * missed events, then record it there. */ if (missed_events > 0 && - buffer->subbuf_size - size >= sizeof(missed_events)) { + rb_read_page_capacity(data_page) - size >= sizeof(missed_events)) { memcpy(&dpage->data[size], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); @@ -7248,8 +7346,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* * This page may be off to user land. Zero it out here. */ - if (size < buffer->subbuf_size) - memset(&dpage->data[size], 0, buffer->subbuf_size - size); + if (size < rb_read_page_capacity(data_page)) + memset(&dpage->data[size], 0, rb_read_page_capacity(data_page) - size); return read; } @@ -7267,6 +7365,18 @@ void *ring_buffer_read_page_data(struct buffer_data_read_page *page) } EXPORT_SYMBOL_GPL(ring_buffer_read_page_data); +/** + * ring_buffer_read_page_size - get size of the read page. + * @rpage: the page to get the size from + * + * Returns size of the page in bytes. + */ +unsigned int ring_buffer_read_page_size(struct buffer_data_read_page *rpage) +{ + return rpage ? PAGE_SIZE << rpage->order : 0; +} +EXPORT_SYMBOL_GPL(ring_buffer_read_page_size); + /** * ring_buffer_subbuf_size_get - get size of the sub buffer. * @buffer: the buffer to get the sub buffer size from @@ -7275,7 +7385,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_read_page_data); */ int ring_buffer_subbuf_size_get(struct trace_buffer *buffer) { - return buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + return rb_subbuf_size(buffer); } EXPORT_SYMBOL_GPL(ring_buffer_subbuf_size_get); @@ -7320,7 +7430,8 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) { struct ring_buffer_per_cpu *cpu_buffer; struct buffer_page *bpage, *tmp; - int old_order, old_size; + unsigned int old_capacity; + int old_order; int nr_pages; int psize; int err; @@ -7329,9 +7440,6 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (!buffer || order < 0) return -EINVAL; - if (buffer->subbuf_order == order) - return 0; - psize = (1 << order) * PAGE_SIZE; if (psize <= BUF_PAGE_HDR_SIZE) return -EINVAL; @@ -7340,18 +7448,29 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (psize > RB_WRITE_MASK + 1) return -EINVAL; - old_order = buffer->subbuf_order; - old_size = buffer->subbuf_size; - /* prevent another thread from changing buffer sizes */ guard(mutex)(&buffer->mutex); + + old_order = buffer->subbuf_order; + if (old_order == order) + return 0; + + old_capacity = rb_subbuf_capacity(buffer); + + /* The mmap fast path reads subbuf_order without buffer->mutex. */ + for_each_buffer_cpu(buffer, cpu) { + if (!cpumask_test_cpu(cpu, buffer->cpumask)) + continue; + if (atomic_read(&buffer->buffers[cpu]->resize_disabled)) + return -EBUSY; + } + atomic_inc(&buffer->record_disabled); /* Make sure all commits have finished */ synchronize_rcu(); - buffer->subbuf_order = order; - buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE; + WRITE_ONCE(buffer->subbuf_order, order); /* Make sure all new buffers are allocated, before deleting the old ones */ for_each_buffer_cpu(buffer, cpu) { @@ -7367,8 +7486,8 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) } /* Update the number of pages to match the new size */ - nr_pages = old_size * buffer->buffers[cpu]->nr_pages; - nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size); + nr_pages = old_capacity * buffer->buffers[cpu]->nr_pages; + nr_pages = DIV_ROUND_UP(nr_pages, rb_subbuf_capacity(buffer)); /* we need a minimum of two pages */ if (nr_pages < 2) @@ -7390,7 +7509,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) } for_each_buffer_cpu(buffer, cpu) { - struct buffer_data_page *old_free_data_page; + struct buffer_data_read_page old_free_data_page; struct list_head old_pages; unsigned long flags; @@ -7431,8 +7550,10 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) cpu_buffer->nr_pages = cpu_buffer->nr_pages_to_update; cpu_buffer->nr_pages_to_update = 0; + arch_spin_lock(&cpu_buffer->lock); old_free_data_page = cpu_buffer->free_page; - cpu_buffer->free_page = NULL; + cpu_buffer->free_page.data = NULL; + arch_spin_unlock(&cpu_buffer->lock); rb_head_page_activate(cpu_buffer); @@ -7443,7 +7564,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) list_del_init(&bpage->list); free_buffer_page(bpage); } - free_pages((unsigned long)old_free_data_page, old_order); + free_pages((unsigned long)old_free_data_page.data, old_free_data_page.order); rb_check_pages(cpu_buffer); } @@ -7453,8 +7574,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) return 0; error: - buffer->subbuf_order = old_order; - buffer->subbuf_size = old_size; + WRITE_ONCE(buffer->subbuf_order, old_order); atomic_dec(&buffer->record_disabled); @@ -7532,7 +7652,7 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer, meta->meta_struct_len = sizeof(*meta); meta->nr_subbufs = nr_subbufs; - meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + meta->subbuf_size = rb_subbuf_size(cpu_buffer->buffer); meta->meta_page_size = meta->subbuf_size; rb_update_meta_page(cpu_buffer); @@ -7574,11 +7694,7 @@ static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer, lockdep_assert_held(&cpu_buffer->mapping_lock); - /* mapped is always greater or equal to user_mapped */ - if (WARN_ON(cpu_buffer->mapped < cpu_buffer->user_mapped)) - return -EINVAL; - - if (inc && cpu_buffer->mapped == UINT_MAX) + if (inc && cpu_buffer->user_mapped == UINT_MAX) return -EBUSY; if (WARN_ON(!inc && cpu_buffer->user_mapped == 0)) @@ -7587,13 +7703,10 @@ static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer, mutex_lock(&cpu_buffer->buffer->mutex); raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); - if (inc) { + if (inc) cpu_buffer->user_mapped++; - cpu_buffer->mapped++; - } else { + else cpu_buffer->user_mapped--; - cpu_buffer->mapped--; - } raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); mutex_unlock(&cpu_buffer->buffer->mutex); @@ -7739,6 +7852,9 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, /* prevent another thread from changing buffer/sub-buffer sizes */ guard(mutex)(&buffer->mutex); + if (cpu_buffer->nr_pages > rb_static_max_pages()) + return -E2BIG; + err = rb_alloc_meta_page(cpu_buffer); if (err) return err; @@ -7765,7 +7881,6 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, if (!err) { raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); /* This is the first time it is mapped by user */ - cpu_buffer->mapped++; cpu_buffer->user_mapped = 1; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); } else { @@ -7822,8 +7937,6 @@ int ring_buffer_unmap(struct trace_buffer *buffer, int cpu) raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); /* This is the last user space mapping */ - if (!WARN_ON_ONCE(cpu_buffer->mapped < cpu_buffer->user_mapped)) - cpu_buffer->mapped--; cpu_buffer->user_mapped = 0; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); @@ -7894,7 +8007,7 @@ int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu) * missed events, then record it there. */ commit = rb_page_size(reader); - if (buffer->subbuf_size - commit >= sizeof(missed_events)) { + if (rb_subbuf_capacity(buffer) - commit >= sizeof(missed_events)) { memcpy(&dpage->data[commit], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); @@ -7926,7 +8039,7 @@ int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu) out: /* Some archs do not have data cache coherency between kernel and user-space */ flush_kernel_vmap_range(cpu_buffer->reader_page->page, - buffer->subbuf_size + BUF_PAGE_HDR_SIZE); + rb_subbuf_size(buffer)); rb_update_meta_page(cpu_buffer); diff --git a/kernel/trace/ring_buffer_benchmark.c b/kernel/trace/ring_buffer_benchmark.c index 593e3b59e42e9..c3d34c0e64e28 100644 --- a/kernel/trace/ring_buffer_benchmark.c +++ b/kernel/trace/ring_buffer_benchmark.c @@ -104,7 +104,7 @@ static enum event_status read_event(int cpu) static enum event_status read_page(int cpu) { - struct buffer_data_read_page *bpage; + struct buffer_data_read_page *bpage = NULL; struct ring_buffer_event *event; struct rb_page *rpage; unsigned long commit; @@ -114,8 +114,8 @@ static enum event_status read_page(int cpu) int inc; int i; - bpage = ring_buffer_alloc_read_page(buffer, cpu); - if (IS_ERR(bpage)) + ret = ring_buffer_alloc_read_page(buffer, cpu, &bpage); + if (ret < 0) return EVENT_DROPPED; page_size = ring_buffer_subbuf_size_get(buffer); diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c index f4642f5adda33..49913bb0057a7 100644 --- a/kernel/trace/simple_ring_buffer.c +++ b/kernel/trace/simple_ring_buffer.c @@ -160,8 +160,8 @@ int simple_ring_buffer_swap_reader_page(struct simple_rb_per_cpu *cpu_buffer) overrun = cpu_buffer->meta->overrun; } while (!simple_bpage_unset_head_link(last, reader, SIMPLE_RB_LINK_NORMAL) && retry--); - if (!retry) - return -EINVAL; + if (retry < 0) + return -EBUSY; cpu_buffer->head_page = simple_bpage_from_link(reader->link.next); cpu_buffer->head_page->link.prev = &reader->link; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 395238b2b715c..c4fa38a1f167c 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4669,7 +4669,7 @@ trace_event_update_with_eval_map(struct module *mod, map = start; - trace_event_update_all(map, len); + trace_event_update_all(map, len, mod); if (len <= 0) return; @@ -7080,8 +7080,8 @@ ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, { struct ftrace_buffer_info *info = filp->private_data; struct trace_iterator *iter = &info->iter; + unsigned int spare_size; void *trace_data; - int page_size; ssize_t ret = 0; ssize_t size; @@ -7091,36 +7091,22 @@ ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, if (iter->snapshot && tracer_uses_snapshot(iter->tr->current_trace)) return -EBUSY; - page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); + spare_size = ring_buffer_read_page_size(info->spare); - /* Make sure the spare matches the current sub buffer size */ - if (info->spare) { - if (page_size != info->spare_size) { - ring_buffer_free_read_page(iter->array_buffer->buffer, - info->spare_cpu, info->spare); - info->spare = NULL; - } - } +again: + /* Do we have previous read data to read? */ + if (info->read < spare_size) + goto read; - if (!info->spare) { - info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer, - iter->cpu_file); - if (IS_ERR(info->spare)) { - ret = PTR_ERR(info->spare); - info->spare = NULL; - } else { - info->spare_cpu = iter->cpu_file; - info->spare_size = page_size; - } - } - if (!info->spare) + ret = ring_buffer_alloc_read_page(iter->array_buffer->buffer, iter->cpu_file, + &info->spare); + if (ret) return ret; - /* Do we have previous read data to read? */ - if (info->read < page_size) - goto read; + spare_size = ring_buffer_read_page_size(info->spare); + info->read = spare_size; + info->spare_cpu = iter->cpu_file; - again: trace_access_lock(iter->cpu_file); ret = ring_buffer_read_page(iter->array_buffer->buffer, info->spare, @@ -7146,8 +7132,9 @@ ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, } info->read = 0; + read: - size = page_size - info->read; + size = spare_size - info->read; if (size > count) size = count; trace_data = ring_buffer_read_page_data(info->spare); @@ -7188,26 +7175,24 @@ int tracing_buffers_release(struct inode *inode, struct file *file) __trace_array_put(iter->tr); - if (info->spare) - ring_buffer_free_read_page(iter->array_buffer->buffer, - info->spare_cpu, info->spare); + ring_buffer_free_read_page(iter->array_buffer->buffer, info->spare_cpu, info->spare); kvfree(info); return 0; } struct buffer_ref { - struct trace_buffer *buffer; - void *page; - int cpu; - refcount_t refcount; + struct trace_buffer *buffer; + struct buffer_data_read_page *rpage; + int cpu; + refcount_t refcount; }; static void buffer_ref_release(struct buffer_ref *ref) { if (!refcount_dec_and_test(&ref->refcount)) return; - ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); + ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->rpage); kfree(ref); } @@ -7266,25 +7251,15 @@ ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, .ops = &buffer_pipe_buf_ops, .spd_release = buffer_spd_release, }; + unsigned int page_size = 0; struct buffer_ref *ref; bool woken = false; - int page_size; int entries, i; ssize_t ret = 0; if (iter->snapshot && tracer_uses_snapshot(iter->tr->current_trace)) return -EBUSY; - page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); - if (*ppos & (page_size - 1)) - return -EINVAL; - - if (len & (page_size - 1)) { - if (len < page_size) - return -EINVAL; - len &= (~(page_size - 1)); - } - if (splice_grow_spd(pipe, &spd)) return -ENOMEM; @@ -7304,25 +7279,39 @@ ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, refcount_set(&ref->refcount, 1); ref->buffer = iter->array_buffer->buffer; - ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); - if (IS_ERR(ref->page)) { - ret = PTR_ERR(ref->page); - ref->page = NULL; + + ret = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file, &ref->rpage); + if (ret) { kfree(ref); break; } ref->cpu = iter->cpu_file; - r = ring_buffer_read_page(ref->buffer, ref->page, - len, iter->cpu_file, 1); + page_size = ring_buffer_read_page_size(ref->rpage); + + r = -EINVAL; + if (IS_ALIGNED(*ppos, page_size) && len >= page_size) { + r = ring_buffer_read_page(ref->buffer, ref->rpage, len, iter->cpu_file, 1); + } else if (!i) { + /* + * If this fails to read on the first iteration, it + * means the length was too small and an error should + * be returned to user space. Otherwise, at least + * one sub-buffer was successfully read but this failed + * due to either the length was unaligned or the + * subbuf order changed. Either case, do not report + * an error. + */ + ret = -EINVAL; + } + if (r < 0) { - ring_buffer_free_read_page(ref->buffer, ref->cpu, - ref->page); + ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->rpage); kfree(ref); break; } - page = virt_to_page(ring_buffer_read_page_data(ref->page)); + page = virt_to_page(ring_buffer_read_page_data(ref->rpage)); spd.pages[i] = page; spd.partial[i].len = page_size; @@ -7726,12 +7715,55 @@ trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, return cnt; } +static bool tr_option_match(struct trace_array *tr, void *topt) +{ + for (int i = 0; i < tr->nr_topts; i++) { + struct trace_options *tr_topts = &tr->topts[i]; + + if (topt >= (void *)&tr_topts->topts[0] && + topt < (void *)&tr_topts->topts[tr_topts->nr_topts]) + return true; + } + return false; +} + +/* + * The topt is the address of a trace_array->topts[] element that holds the + * the tracer options descriptor. But since the trace_array reference has not + * been taken yet, it cannot be dereferenced as it could have been freed by + * a rmdir of the instance the trace_array represents. + * + * Search the list of trace_arrays and compare the topt to the address of + * the entire trace_array topts array for each trace_array in the list. + * If one is matched, then take the reference and return it. If not, the + * trace_array no longer exits. + */ +static int trace_array_tracer_options_get(void *topt) +{ + struct trace_array *tr; + int ret; + + ret = security_locked_down(LOCKDOWN_TRACEFS); + if (ret) + return ret; + + if (tracing_disabled) + return -ENODEV; + + guard(mutex)(&trace_types_lock); + list_for_each_entry(tr, &ftrace_trace_arrays, list) { + if (tr_option_match(tr, topt)) + return __trace_array_get(tr); + } + return -ENODEV; +} + static int tracing_open_options(struct inode *inode, struct file *filp) { struct trace_option_dentry *topt = inode->i_private; int ret; - ret = tracing_check_open_get_tr(topt->tr); + ret = trace_array_tracer_options_get(topt); if (ret) return ret; @@ -7840,11 +7872,70 @@ trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, return cnt; } +/* + * The tr_index is the address of a trace_array->trace_flags_index[] + * element that holds the index of the trace flag. But since the + * trace_array reference has not been taken yet, it cannot be referenced + * as it could have been freed by a rmdir of the instance the trace_array + * represents. + * + * Search the list of trace_arrays and compare the tr_index to the + * address of the entire trace_array trace_flags_index array for each + * trace_array in the list. If one is matched, then take the reference + * and return it. If not, the trace_array no longer exits. + */ +static int trace_array_options_get(void *tr_index) +{ + struct trace_array *tr; + int ret; + + ret = security_locked_down(LOCKDOWN_TRACEFS); + if (ret) + return ret; + + if (tracing_disabled) + return -ENODEV; + + guard(mutex)(&trace_types_lock); + list_for_each_entry(tr, &ftrace_trace_arrays, list) { + if (tr_index >= (void *)&tr->trace_flags_index[0] && + tr_index < (void *)&tr->trace_flags_index[TRACE_FLAGS_MAX_SIZE]) + return __trace_array_get(tr); + } + return -ENODEV; +} + +static int trace_options_open(struct inode *inode, struct file *filp) +{ + void *tr_index = inode->i_private; + + if (trace_array_options_get(tr_index) < 0) + return -ENODEV; + + filp->private_data = tr_index; + + return 0; +} + +static int trace_options_release(struct inode *inode, struct file *filp) +{ + void *tr_index = filp->private_data; + struct trace_array *tr; + unsigned int index; + + get_tr_index(tr_index, &tr, &index); + + trace_array_put(tr); + + return 0; +} + static const struct file_operations trace_options_core_fops = { - .open = tracing_open_generic, - .read = trace_options_core_read, - .write = trace_options_core_write, - .llseek = generic_file_llseek, + .open = trace_options_open, + .read = trace_options_core_read, + .write = trace_options_core_write, + .llseek = generic_file_llseek, + .release = trace_options_release, }; struct dentry *trace_create_file(const char *name, @@ -7934,6 +8025,7 @@ create_trace_option_files(struct trace_array *tr, struct tracer *tracer, tr->topts = tr_topts; tr->topts[tr->nr_topts].tracer = tracer; tr->topts[tr->nr_topts].topts = topts; + tr->topts[tr->nr_topts].nr_topts = cnt; tr->nr_topts++; for (cnt = 0; opts[cnt].name; cnt++) { @@ -8214,6 +8306,8 @@ buffer_subbuf_size_write(struct file *filp, const char __user *ubuf, /* Do not allow tracing while changing the order of the ring buffer */ tracing_stop_tr(tr); + trace_access_lock(RING_BUFFER_ALL_CPUS); + old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); if (old_order == order) goto out; @@ -8253,6 +8347,7 @@ buffer_subbuf_size_write(struct file *filp, const char __user *ubuf, #endif (*ppos)++; out: + trace_access_unlock(RING_BUFFER_ALL_CPUS); if (ret) cnt = ret; tracing_start_tr(tr); @@ -9646,6 +9741,11 @@ __init static void enable_instances(void) if (flag_delim) *flag_delim++ = '\0'; + if (trace_array_find(name)) { + pr_warn("Tracing: Instance %s already exists\n", name); + continue; + } + if (backup) { if (backup_instance_area(backup, &addr, &size) < 0) continue; @@ -9726,7 +9826,8 @@ __init static void enable_instances(void) tr = trace_array_create_systems(name, NULL, addr, size); if (IS_ERR(tr)) { - pr_warn("Tracing: Failed to create instance buffer %s\n", curr_str); + pr_warn("Tracing: Failed to create instance buffer '%s' (%ld)\n", name, + PTR_ERR(tr)); continue; } diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index bf77331f56a4d..3749485a7d85f 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -227,6 +227,7 @@ struct array_buffer { struct trace_options { struct tracer *tracer; struct trace_option_dentry *topts; + int nr_topts; }; struct trace_pid_list *trace_pid_list_alloc(void); @@ -745,11 +746,10 @@ static inline int tracing_get_cpu(struct inode *inode) void tracing_reset_cpu(struct array_buffer *buf, int cpu); struct ftrace_buffer_info { - struct trace_iterator iter; - void *spare; - unsigned int spare_cpu; - unsigned int spare_size; - unsigned int read; + struct trace_iterator iter; + struct buffer_data_read_page *spare; + unsigned int spare_cpu; + unsigned int read; }; /** @@ -1340,7 +1340,7 @@ extern void clear_ftrace_function_probes(struct trace_array *tr); int register_ftrace_command(struct ftrace_func_command *cmd); int unregister_ftrace_command(struct ftrace_func_command *cmd); -void ftrace_create_filter_files(struct ftrace_ops *ops, +void ftrace_create_filter_files(struct trace_array *tr, struct dentry *parent); void ftrace_destroy_filter_files(struct ftrace_ops *ops); @@ -1363,11 +1363,12 @@ static inline void clear_ftrace_function_probes(struct trace_array *tr) { } +static inline void ftrace_create_filter_files(struct trace_array *tr, + struct dentry *parent) { } /* * The ops parameter passed in is usually undefined. * This must be a macro. */ -#define ftrace_create_filter_files(ops, parent) do { } while (0) #define ftrace_destroy_filter_files(ops) do { } while (0) #endif /* CONFIG_FUNCTION_TRACER && CONFIG_DYNAMIC_FTRACE */ @@ -1952,6 +1953,7 @@ struct event_trigger_data { struct enable_trigger_data { struct trace_event_file *file; + struct trace_event_call *call; bool enable; bool hist; }; @@ -2285,13 +2287,13 @@ static inline const char *get_syscall_name(int syscall) #ifdef CONFIG_EVENT_TRACING void trace_event_init(void); -void trace_event_update_all(struct trace_eval_map **map, int len); +void trace_event_update_all(struct trace_eval_map **map, int len, struct module *mod); /* Used from boot time tracer */ extern int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set); extern int trigger_process_regex(struct trace_event_file *file, char *buff); #else static inline void __init trace_event_init(void) { } -static inline void trace_event_update_all(struct trace_eval_map **map, int len) { } +static inline void trace_event_update_all(struct trace_eval_map **map, int len, struct module *mod) { } #endif #ifdef CONFIG_TRACER_SNAPSHOT diff --git a/kernel/trace/trace_btf.c b/kernel/trace/trace_btf.c index 00172f301f25a..ee7a04886bf67 100644 --- a/kernel/trace/trace_btf.c +++ b/kernel/trace/trace_btf.c @@ -61,47 +61,50 @@ struct btf_anon_stack { /* * Find a member of data structure/union by name and return it. - * Return NULL if not found, or -EINVAL if parameter is invalid. - * If the member is an member of anonymous union/structure, the offset - * of that anonymous union/structure is stored into @anon_offset. Caller - * can calculate the correct offset from the root data structure by - * adding anon_offset to the member's offset. + * Return NULL if not found, or ERR_PTR(-EINVAL) if parameter is invalid. + * If the member is a member of an anonymous union/structure, the bit offset + * of that anonymous union/structure is stored into @anon_offset. + * If @member_type is non-NULL, the actual containing structure/union type + * of the found member is stored into @member_type. */ const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, - u32 *anon_offset) + u32 *anon_offset, + const struct btf_type **member_type) { struct btf_anon_stack *anon_stack; const struct btf_member *member; + const struct btf_type *mtype; u32 tid, cur_offset = 0; const char *name; int i, top = 0; + if (!btf_type_is_struct(type)) + return ERR_PTR(-EINVAL); + anon_stack = kzalloc_objs(*anon_stack, BTF_ANON_STACK_MAX); if (!anon_stack) return ERR_PTR(-ENOMEM); retry: - if (!btf_type_is_struct(type)) { - member = ERR_PTR(-EINVAL); - goto out; - } - for_each_member(i, type, member) { if (!member->name_off) { /* Anonymous union/struct: push it for later use */ - if (btf_type_skip_modifiers(btf, member->type, &tid) && + mtype = btf_type_skip_modifiers(btf, member->type, &tid); + if (mtype && btf_type_is_struct(mtype) && top < BTF_ANON_STACK_MAX) { anon_stack[top].tid = tid; - anon_stack[top++].offset = - cur_offset + member->offset; + anon_stack[top++].offset = cur_offset + + __btf_member_bit_offset(type, member); } } else { name = btf_name_by_offset(btf, member->name_off); if (name && !strcmp(member_name, name)) { if (anon_offset) *anon_offset = cur_offset; + if (member_type) + *member_type = type; goto out; } } diff --git a/kernel/trace/trace_btf.h b/kernel/trace/trace_btf.h index 4bc44bc261e6f..4bd26bceae23a 100644 --- a/kernel/trace/trace_btf.h +++ b/kernel/trace/trace_btf.h @@ -8,4 +8,5 @@ const struct btf_param *btf_get_func_param(const struct btf_type *func_proto, const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, - u32 *anon_offset); + u32 *anon_offset, + const struct btf_type **member_type); diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 3650d84d4f16c..9c62202b0c97b 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -1462,6 +1462,8 @@ int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set) /* Put back the colon to allow this to be called again */ if (buf) *(buf - 1) = ':'; + if (mod) + *(mod - 5) = ':'; return ret; } @@ -2654,14 +2656,14 @@ static const struct file_operations ftrace_show_event_filters_fops = { .open = ftrace_event_show_filters_open, .read = seq_read, .llseek = seq_lseek, - .release = seq_release, + .release = ftrace_event_release, }; static const struct file_operations ftrace_show_event_triggers_fops = { .open = ftrace_event_show_triggers_open, .read = seq_read, .llseek = seq_lseek, - .release = seq_release, + .release = ftrace_event_release, }; static const struct file_operations ftrace_set_event_pid_fops = { @@ -2819,7 +2821,17 @@ ftrace_event_set_open(struct inode *inode, struct file *file) static int ftrace_event_show_filters_open(struct inode *inode, struct file *file) { - return ftrace_event_open(inode, file, &show_show_event_filters_seq_ops); + struct trace_array *tr = inode->i_private; + int ret; + + ret = tracing_check_open_get_tr(tr); + if (ret) + return ret; + + ret = ftrace_event_open(inode, file, &show_show_event_filters_seq_ops); + if (ret < 0) + trace_array_put(tr); + return ret; } /** @@ -2833,7 +2845,17 @@ ftrace_event_show_filters_open(struct inode *inode, struct file *file) static int ftrace_event_show_triggers_open(struct inode *inode, struct file *file) { - return ftrace_event_open(inode, file, &show_show_event_triggers_seq_ops); + struct trace_array *tr = inode->i_private; + int ret; + + ret = tracing_check_open_get_tr(tr); + if (ret) + return ret; + + ret = ftrace_event_open(inode, file, &show_show_event_triggers_seq_ops); + if (ret < 0) + trace_array_put(tr); + return ret; } static int @@ -3557,7 +3579,7 @@ static void update_event_fields(struct trace_event_call *call, } /* Update all events for replacing eval and sanitizing */ -void trace_event_update_all(struct trace_eval_map **map, int len) +void trace_event_update_all(struct trace_eval_map **map, int len, struct module *mod) { struct trace_event_call *call, *p; const char *last_system = NULL; @@ -3569,6 +3591,10 @@ void trace_event_update_all(struct trace_eval_map **map, int len) mutex_lock(&event_mutex); down_write(&trace_event_sem); list_for_each_entry_safe(call, p, &ftrace_events, list) { + + if (mod && call->module != mod) + continue; + /* events are usually grouped together with systems */ if (!last_system || call->class->system != last_system) { first = true; @@ -4901,6 +4927,8 @@ static __init void event_test_stuff(void) struct task_struct *test_thread; test_thread = kthread_run(event_test_thread, NULL, "test-events"); + if (WARN_ON(IS_ERR(test_thread))) + return; msleep(1); kthread_stop(test_thread); } diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 58d28cd1afa3b..539b5f583af92 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -170,7 +170,6 @@ struct hist_field { struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; struct hist_trigger_data *hist_data; enum hist_field_fn fn_num; - unsigned int ref; unsigned int size; unsigned int offset; unsigned int is_signed; @@ -1906,16 +1905,8 @@ static int contains_operator(char *str, char **sep) return field_op; } -static void get_hist_field(struct hist_field *hist_field) -{ - hist_field->ref++; -} - static void __destroy_hist_field(struct hist_field *hist_field) { - if (--hist_field->ref > 1) - return; - kfree(hist_field->var.name); kfree(hist_field->name); @@ -1962,8 +1953,6 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, if (!hist_field) return NULL; - hist_field->ref = 1; - hist_field->hist_data = hist_data; if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) @@ -2218,10 +2207,8 @@ static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, for (i = 0; i < hist_data->n_var_refs; i++) { ref_field = hist_data->var_refs[i]; if (ref_field->var.idx == var_field->var.idx && - ref_field->var.hist_data == var_field->hist_data) { - get_hist_field(ref_field); + ref_field->var.hist_data == var_field->hist_data) return ref_field; - } } /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */ if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX) @@ -2229,7 +2216,7 @@ static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); if (ref_field) { if (init_var_ref(ref_field, var_field, system, event_name)) { - destroy_hist_field(ref_field, 0); + __destroy_hist_field(ref_field); return NULL; } @@ -2325,6 +2312,7 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, struct ftrace_event_field *field = NULL; char *field_name, *modifier, *str; struct trace_array *tr = file->tr; + bool stack_modifier = false; modifier = str = kstrdup(field_str, GFP_KERNEL); if (!modifier) @@ -2347,9 +2335,10 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, *flags |= HIST_FIELD_FL_EXECNAME; else if (strcmp(modifier, "syscall") == 0) *flags |= HIST_FIELD_FL_SYSCALL; - else if (strcmp(modifier, "stacktrace") == 0) + else if (strcmp(modifier, "stacktrace") == 0) { *flags |= HIST_FIELD_FL_STACKTRACE; - else if (strcmp(modifier, "log2") == 0) + stack_modifier = true; + } else if (strcmp(modifier, "log2") == 0) *flags |= HIST_FIELD_FL_LOG2; else if (strcmp(modifier, "usecs") == 0) *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; @@ -2410,6 +2399,7 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, *flags |= HIST_FIELD_FL_CPU; } else if (field && field->filter_type == FILTER_STACKTRACE) { *flags |= HIST_FIELD_FL_STACKTRACE; + field = NULL; } else if (field && field->filter_type == FILTER_COMM) { *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; } else { @@ -2420,6 +2410,12 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, } } } + + if (stack_modifier && + (!field || field->filter_type != FILTER_STACKTRACE)) { + hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); + field = ERR_PTR(-EINVAL); + } out: kfree(str); @@ -3256,7 +3252,6 @@ static struct hist_field *create_var(struct hist_trigger_data *hist_data, goto out; } - var->ref = 1; var->flags = HIST_FIELD_FL_VAR; var->var.idx = idx; var->var.hist_data = var->hist_data = hist_data; @@ -4293,8 +4288,7 @@ static int __create_val_field(struct hist_trigger_data *hist_data, goto err; } else { /* Value */ - if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | - HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | + if (hist_field->flags & (HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET | HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE)) goto err; @@ -4311,6 +4305,7 @@ static int __create_val_field(struct hist_trigger_data *hist_data, return ret; err: hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); + destroy_hist_field(hist_field, 0); return -EINVAL; } @@ -5670,7 +5665,7 @@ static int print_entries(struct seq_file *m, { struct tracing_map_sort_entry **sort_entries = NULL; struct tracing_map *map = hist_data->map; - int i, j, n_entries; + int i, j, n_entries, ret; struct hist_val_stat *stats = NULL; u64 val; @@ -5680,6 +5675,8 @@ static int print_entries(struct seq_file *m, if (n_entries < 0) return n_entries; + ret = n_entries; + /* Calculate the max and the total for each field if needed. */ for (j = 0; j < hist_data->n_vals; j++) { if (!(hist_data->fields[j]->flags & @@ -5688,7 +5685,7 @@ static int print_entries(struct seq_file *m, if (!stats) { stats = kzalloc_objs(*stats, hist_data->n_vals); if (!stats) { - n_entries = -ENOMEM; + ret = -ENOMEM; goto out; } } @@ -5709,7 +5706,7 @@ static int print_entries(struct seq_file *m, out: tracing_map_destroy_sort_entries(sort_entries, n_entries); - return n_entries; + return ret; } static void hist_trigger_show(struct seq_file *m, @@ -6363,17 +6360,18 @@ static int event_hist_trigger_named_init(struct event_trigger_data *data) { int ret; - data->ref++; - - save_named_trigger(data->named_data->name, data); - ret = event_hist_trigger_init(data->named_data); if (ret < 0) { kfree(data->cmd_ops); data->cmd_ops = &trigger_hist_cmd; + return ret; } - return ret; + data->ref++; + + save_named_trigger(data->named_data->name, data); + + return 0; } static void event_hist_trigger_named_free(struct event_trigger_data *data) @@ -6623,12 +6621,6 @@ static int hist_register_trigger(char *glob, data->cmd_ops = cmd_ops; } - if (data->cmd_ops->init) { - ret = data->cmd_ops->init(data); - if (ret < 0) - goto out; - } - if (hist_data->enable_timestamps) { char *clock = hist_data->attrs->clock; @@ -6641,8 +6633,19 @@ static int hist_register_trigger(char *glob, tracing_set_filter_buffering(file->tr, true); } - if (named_data) + if (data->cmd_ops->init) { + ret = data->cmd_ops->init(data); + if (ret < 0) { + if (hist_data->enable_timestamps) + tracing_set_filter_buffering(file->tr, false); + goto out; + } + } + + if (named_data) { + remove_hist_vars(hist_data); destroy_hist_data(hist_data); + } out: return ret; } @@ -6656,11 +6659,12 @@ static int hist_trigger_enable(struct event_trigger_data *data, update_cond_flag(file); - if (trace_event_trigger_enable_disable(file, 1) < 0) { - list_del_rcu(&data->list); - update_cond_flag(file); + /* + * On failure the caller undoes the registration, and + * hist_unregister_trigger() can only find the trigger here. + */ + if (trace_event_trigger_enable_disable(file, 1) < 0) ret--; - } return ret; } @@ -6738,13 +6742,13 @@ static void hist_unregister_trigger(char *glob, } } - if (test && test->cmd_ops->free) - test->cmd_ops->free(test); - if (hist_data->enable_timestamps) { if (!hist_data->remove || test) tracing_set_filter_buffering(file->tr, false); } + + if (test && test->cmd_ops->free) + test->cmd_ops->free(test); } static bool hist_file_check_refs(struct trace_event_file *file) @@ -6949,6 +6953,8 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, return ret; out_unreg: event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); + /* The unregister frees trigger_data, skip out_free */ + goto out; out_free: remove_hist_vars(hist_data); diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index ad83419cb4201..1954184ab2b9d 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -1728,7 +1728,8 @@ static void enable_trigger_private_data_free(struct event_trigger_data *data) { struct enable_trigger_data *enable_data = data->private_data; - trace_event_put_ref(enable_data->file->event_call); + /* The file may already be freed here, only the call is kept alive */ + trace_event_put_ref(enable_data->call); kfree(enable_data); } @@ -1801,6 +1802,7 @@ int event_enable_trigger_parse(struct event_command *cmd_ops, enable_data->hist = hist; enable_data->enable = enable; enable_data->file = event_enable_file; + enable_data->call = event_enable_file->event_call; trigger_data = trigger_data_alloc(cmd_ops, cmd, param, enable_data); if (!trigger_data) { diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 8c82ecb735f41..e190dad1eb914 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -868,6 +868,9 @@ void user_event_mm_dup(struct task_struct *t, struct user_event_mm *old_mm) struct user_event_mm *mm = user_event_mm_alloc(t); struct user_event_enabler *enabler; + /* On failure, do not free parent's copy */ + t->user_event_mm = NULL; + if (!mm) return; @@ -1119,10 +1122,9 @@ static void user_event_destroy_validators(struct user_event *user) } } -static void user_event_destroy_fields(struct user_event *user) +static void user_event_destroy_fields(struct list_head *head) { struct ftrace_event_field *field, *next; - struct list_head *head = &user->fields; list_for_each_entry_safe(field, next, head, link) { list_del(&field->link); @@ -1499,17 +1501,32 @@ static int user_event_set_call_visible(struct user_event *user, bool visible) static int destroy_user_event(struct user_event *user) { + LIST_HEAD(fields); int ret = 0; lockdep_assert_held(&event_mutex); - /* Must destroy fields before call removal */ - user_event_destroy_fields(user); + /* + * Detach the fields before removing the call. Removing the event + * frees the field list memory (trace_destroy_fields() is run on + * successful removal and kmem_cache_free()s the fields), but the + * fields here are allocated and owned by user_events. Destroy + * them separately once removal has succeeded. + */ + list_splice_init(&user->fields, &fields); ret = user_event_set_call_visible(user, false); - if (ret) + if (ret) { + /* + * Removal failed and the event stays registered, recover + * the fields so it is left in a consistent state. + */ + list_splice(&fields, &user->fields); return ret; + } + + user_event_destroy_fields(&fields); dyn_event_remove(&user->devent); hash_del(&user->node); @@ -2209,7 +2226,7 @@ static int user_event_parse(struct user_event_group *group, char *name, put_user_lock: mutex_unlock(&event_mutex); put_user: - user_event_destroy_fields(user); + user_event_destroy_fields(&user->fields); user_event_destroy_validators(user); kfree(user->call.print_fmt); diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index cd37f20137581..c879d43a5fbb2 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -101,7 +101,7 @@ int ftrace_create_function_files(struct trace_array *tr, return ret; } - ftrace_create_filter_files(tr->ops, parent); + ftrace_create_filter_files(tr, parent); return 0; } diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 0d2d3a2ea7dda..9ce68d207ef93 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -52,6 +52,7 @@ struct fgraph_data { }; struct ftrace_graph_ret_entry ret; int failed; + int ent_size; int cpu; }; @@ -1274,6 +1275,7 @@ print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, if (s->full) { data->failed = 1; data->cpu = cpu; + data->ent_size = iter->ent_size; } else data->failed = 0; } @@ -1457,6 +1459,7 @@ print_graph_function_flags(struct trace_iterator *iter, u32 flags) if (data && data->failed) { field = &data->ent.ent; iter->cpu = data->cpu; + iter->ent_size = data->ent_size; ret = print_graph_entry(field, s, iter, flags); if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) { per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1; diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index c8fd9b946f442..f3c20a093f5dd 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -356,9 +356,8 @@ static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type) { const struct btf_type *real_type; u32 intdata; - s32 tid; - real_type = btf_type_skip_modifiers(btf, type->type, &tid); + real_type = btf_type_skip_modifiers(btf, type->type, NULL); if (!real_type) return false; @@ -375,14 +374,13 @@ static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type) const struct btf_type *real_type; const struct btf_array *array; u32 intdata; - s32 tid; if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY) return false; array = (const struct btf_array *)(type + 1); - real_type = btf_type_skip_modifiers(btf, array->type, &tid); + real_type = btf_type_skip_modifiers(btf, array->type, NULL); intdata = btf_type_int(real_type); return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) @@ -580,12 +578,12 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, { struct fetch_insn *code = *pcode; const struct btf_member *field; + const struct btf_type *mtype; u32 bitoffs, anon_offs; bool is_struct = ctx->struct_btf != NULL; struct btf *btf = ctx_btf(ctx); char *next; int is_ptr; - s32 tid; do { if (!is_struct) { @@ -596,7 +594,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, } /* Convert a struct pointer type to a struct type */ - type = btf_type_skip_modifiers(btf, type->type, &tid); + type = btf_type_skip_modifiers(btf, type->type, NULL); if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -615,7 +613,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, anon_offs = 0; field = btf_find_struct_member(btf, type, fieldname, - &anon_offs); + &anon_offs, &mtype); if (IS_ERR(field)) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return PTR_ERR(field); @@ -628,7 +626,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, bitoffs += anon_offs; /* Accumulate the bit-offsets of the dot-connected fields */ - if (btf_type_kflag(type)) { + if (btf_type_kflag(mtype)) { bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset); ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset); } else { @@ -636,7 +634,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, ctx->last_bitsize = 0; } - type = btf_type_skip_modifiers(btf, field->type, &tid); + type = btf_type_skip_modifiers(btf, field->type, NULL); if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -755,7 +753,7 @@ static int parse_btf_arg(char *varname, return -ENOENT; found: - type = btf_type_skip_modifiers(ctx->btf, tid, &tid); + type = btf_type_skip_modifiers(ctx->btf, tid, NULL); found_type: if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); @@ -2110,19 +2108,60 @@ int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype int traceprobe_define_arg_fields(struct trace_event_call *event_call, size_t offset, struct trace_probe *tp) { + struct trace_probe_event *tpe = trace_probe_event_from_call(event_call); int ret, i; + /* + * A field created by trace_define_field() only stores the name and + * type pointers, it does not copy the strings. Here they point into + * the probe_arg of @tp, which is freed when @tp is removed. For an + * event with multiple probes attached, the field list is defined + * once by the first probe but kept alive by the surviving siblings, + * so removing that first probe would leave the fields referencing + * freed memory. Duplicate the strings and anchor the copies on the + * trace_probe_event, which lives as long as the field list itself. + * + * event_define_fields() ignores the return value of this hook, so + * if a previous attempt failed before creating any field, it may + * call here again. Release duplicates left behind by such an + * attempt before starting over. + */ + for (i = 0; i < tpe->nr_field_strings; i++) + kfree(tpe->field_strings[i]); + kfree(tpe->field_strings); + tpe->field_strings = NULL; + tpe->nr_field_strings = 0; + + if (tp->nr_args) { + tpe->field_strings = kcalloc(tp->nr_args * 2, sizeof(char *), + GFP_KERNEL); + if (!tpe->field_strings) + return -ENOMEM; + } + /* Set argument names as fields */ for (i = 0; i < tp->nr_args; i++) { struct probe_arg *parg = &tp->args[i]; const char *fmt = parg->type->fmttype; int size = parg->type->size; + char *name, *type; if (parg->fmt) fmt = parg->fmt; if (parg->count) size *= parg->count; - ret = trace_define_field(event_call, fmt, parg->name, + + name = kstrdup(parg->name, GFP_KERNEL); + type = kstrdup(fmt, GFP_KERNEL); + if (!name || !type) { + kfree(name); + kfree(type); + return -ENOMEM; + } + tpe->field_strings[tpe->nr_field_strings++] = name; + tpe->field_strings[tpe->nr_field_strings++] = type; + + ret = trace_define_field(event_call, type, name, offset + parg->offset, size, parg->type->is_signed, FILTER_OTHER); @@ -2134,6 +2173,11 @@ int traceprobe_define_arg_fields(struct trace_event_call *event_call, static void trace_probe_event_free(struct trace_probe_event *tpe) { + int i; + + for (i = 0; i < tpe->nr_field_strings; i++) + kfree(tpe->field_strings[i]); + kfree(tpe->field_strings); kfree(tpe->class.system); kfree(tpe->call.name); kfree(tpe->call.print_fmt); diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 0f09f7aaf93fb..480eb6acd883f 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -255,6 +255,8 @@ struct trace_probe_event { struct trace_event_call call; struct list_head files; struct list_head probes; + char **field_strings; + int nr_field_strings; struct trace_uprobe_filter filter[]; }; diff --git a/kernel/trace/trace_remote.c b/kernel/trace/trace_remote.c index e6724f947170d..c79cd21347a03 100644 --- a/kernel/trace/trace_remote.c +++ b/kernel/trace/trace_remote.c @@ -980,9 +980,12 @@ int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, const struct cpumask *cpumask) { size_t min_desc_size = trace_buffer_desc_size(buffer_size, cpumask_weight(cpumask)); - unsigned int nr_pages = max(DIV_ROUND_UP(buffer_size, PAGE_SIZE), 2UL) + 1; struct ring_buffer_desc *rb_desc; int cpu, ret = -ENOMEM; + unsigned int nr_pages; + + if (min_desc_size == SIZE_MAX) + return -E2BIG; if (desc_size < min_desc_size) return -EINVAL; @@ -991,6 +994,7 @@ int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, desc->struct_len = min_desc_size; rb_desc = __first_ring_buffer_desc(desc); + nr_pages = __calc_nr_pages_ring_buffer_desc(buffer_size); for_each_cpu(cpu, cpumask) { unsigned int id; diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c index 0aa2514a6593b..e7f4e523587d0 100644 --- a/kernel/trace/trace_stack.c +++ b/kernel/trace/trace_stack.c @@ -499,7 +499,7 @@ stack_trace_filter_open(struct inode *inode, struct file *file) struct ftrace_ops *ops = inode->i_private; /* Checks for tracefs lockdown */ - return ftrace_regex_open(ops, FTRACE_ITER_FILTER, + return ftrace_regex_open(NULL, ops, FTRACE_ITER_FILTER, inode, file); } diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 78068ae8f28a6..53608ac01fab1 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3844,7 +3844,7 @@ static void check_flush_dependency(struct workqueue_struct *target_wq, WARN_ONCE(current->flags & PF_MEMALLOC, "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps", current->pid, current->comm, target_wq->name, target_func); - WARN_ONCE(worker && ((worker->current_pwq->wq->flags & + WARN_ONCE(worker && worker->current_pwq && ((worker->current_pwq->wq->flags & (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM), "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps", worker->current_pwq->wq->name, worker->current_func, @@ -7689,13 +7689,31 @@ module_param_named(panic_on_stall_time, wq_panic_on_stall_time, uint, 0644); MODULE_PARM_DESC(panic_on_stall_time, "Panic if stall exceeds this many seconds (0=disabled)"); /* - * Show workers that might prevent the processing of pending work items. - * A busy worker that is not running on the CPU (e.g. sleeping in - * wait_event_idle() with PF_WQ_WORKER cleared) can stall the pool just as - * effectively as a CPU-bound one, so dump every in-flight worker. + * Report that a pool has no worker in running state, which is a sign that the + * pool may be stuck. Print pool info. Must be called with pool->lock held and + * inside a printk_deferred_enter/exit region. + */ +static void show_pool_no_running_worker(struct worker_pool *pool) +{ + lockdep_assert_held(&pool->lock); + + printk_deferred_enter(); + pr_info("pool %d: no worker in running state, cpu=%d is %s (nr_workers=%d nr_idle=%d)\n", + pool->id, pool->cpu, + idle_cpu(pool->cpu) ? "idle" : "busy", + pool->nr_workers, pool->nr_idle); + pr_info("The pool might have trouble waking an idle worker.\n"); + printk_deferred_exit(); +} + +/* + * Show running workers that might prevent the processing of pending work items. + * If no running worker is found, the pool may be stuck waiting for an idle + * worker to be woken, so report the pool state. */ static void show_cpu_pool_busy_workers(struct worker_pool *pool) { + bool found_running = false; struct worker *worker; unsigned long irq_flags; int bkt; @@ -7703,6 +7721,11 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) raw_spin_lock_irqsave(&pool->lock, irq_flags); hash_for_each(pool->busy_hash, bkt, worker, hentry) { + /* Skip workers that are not actively running on the CPU. */ + if (!task_is_running(worker->task)) + continue; + + found_running = true; /* * Defer printing to avoid deadlocks in console * drivers that queue work while holding locks @@ -7716,6 +7739,13 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) printk_deferred_exit(); } + /* + * If no running worker was found, the pool is likely stuck. Print pool + * state. + */ + if (!found_running) + show_pool_no_running_worker(pool); + raw_spin_unlock_irqrestore(&pool->lock, irq_flags); } @@ -7893,6 +7923,9 @@ static int wq_watchdog_param_set_thresh(const char *val, if (ret) return ret; + if (thresh > MAX_JIFFY_OFFSET / HZ) + return -ERANGE; + if (system_percpu_wq) wq_watchdog_set_thresh(thresh); else diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 1244dcac2294a..12786379bf1d7 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2247,10 +2247,11 @@ config KCOV_INSTRUMENT_ALL config KCOV_IRQ_AREA_SIZE hex "Size of interrupt coverage collection area in words" depends on KCOV + range 0x80 0x1000000 default 0x40000 help - KCOV uses preallocated per-cpu areas to collect coverage from - soft interrupts. This specifies the size of those areas in the + KCOV uses preallocated areas to collect coverage from soft + interrupts. This specifies the size of those areas in the number of unsigned long words. config KCOV_SELFTEST diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c index eba2d3e289800..b0b07270ce7c3 100644 --- a/lib/interval_tree_test.c +++ b/lib/interval_tree_test.c @@ -140,13 +140,13 @@ static int intersection_range_check(void) intxn1 = bitmap_alloc(nnodes, GFP_KERNEL); if (!intxn1) { - WARN_ON_ONCE("Failed to allocate intxn1\n"); + WARN_ONCE(1, "Failed to allocate intxn1\n"); return -ENOMEM; } intxn2 = bitmap_alloc(nnodes, GFP_KERNEL); if (!intxn2) { - WARN_ON_ONCE("Failed to allocate intxn2\n"); + WARN_ONCE(1, "Failed to allocate intxn2\n"); bitmap_free(intxn1); return -ENOMEM; } diff --git a/lib/iov_iter.c b/lib/iov_iter.c index c2484551a4e86..5238731910917 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1904,6 +1904,8 @@ static unsigned int get_contig_folio_len(struct page **pages, * @max_size: maximum size to extract from @iter * @nr_vecs: number of vectors in @bv (on in and output) * @max_vecs: maximum vectors in @bv, including those filled before calling + * @mem_align_mask: reject with -EINVAL if the source address or + * length is not aligned to this mask * @extraction_flags: flags to qualify request * * Like iov_iter_extract_pages(), but returns physically contiguous ranges @@ -1915,7 +1917,8 @@ static unsigned int get_contig_folio_len(struct page **pages, */ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, size_t max_size, unsigned short *nr_vecs, - unsigned short max_vecs, iov_iter_extraction_t extraction_flags) + unsigned short max_vecs, unsigned mem_align_mask, + iov_iter_extraction_t extraction_flags) { unsigned short entries_left = max_vecs - *nr_vecs; unsigned short nr_pages, i = 0; @@ -1923,6 +1926,24 @@ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, struct page **pages; ssize_t size; + /* + * DMA engines typically have both memory address and length alignment + * requirements, so check these against the alignment mask. For UBUF, + * IOVEC and KVEC, only the current segment will be extracted from; for + * everything else we might extract from multiple segments, so we need + * to check those too. + */ + if (likely(iter_is_ubuf(iter) || + iter_is_iovec(iter) || + iov_iter_is_kvec(iter))) { + unsigned long start = (unsigned long)iter_iov_addr(iter); + + if ((start | iter_iov_len(iter)) & mem_align_mask) + return -EINVAL; + } else if (iov_iter_alignment(iter) & mem_align_mask) { + return -EINVAL; + } + /* * Move page array up in the allocated memory for the bio vecs as far as * possible so that we can start filling biovecs from the beginning diff --git a/lib/maple_tree.c b/lib/maple_tree.c index e52876435b77f..1a3815973add0 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -3793,35 +3793,40 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, void *entry, unsigned long range_lo, unsigned long range_hi, unsigned long *next, gfp_t gfp) { - unsigned long min = range_lo; - int ret = 0; - - range_lo = max(min, *next); - ret = mas_empty_area(mas, range_lo, range_hi, 1); - if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { - mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; - ret = 1; - } - if (ret < 0 && range_lo > min) { - mas_reset(mas); - ret = mas_empty_area(mas, min, range_hi, 1); - if (ret == 0) - ret = 1; - } - if (ret < 0) - return ret; + int ret; + unsigned long min; + min = range_lo; do { + range_lo = max(min, *next); + ret = mas_empty_area(mas, range_lo, range_hi, 1); + if (ret < 0 && range_lo > min) { + mas_reset(mas); + ret = mas_empty_area(mas, min, range_hi, 1); + if (ret == 0) + ret = 1; + } + if (ret < 0) + goto out; + mas_insert(mas, entry); } while (mas_nomem(mas, gfp)); - if (mas_is_err(mas)) - return xa_err(mas->node); + if (mas_is_err(mas)) { + ret = xa_err(mas->node); + goto out; + } + + if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { + mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; + ret = 1; + } *startp = mas->index; *next = *startp + 1; if (*next == 0) mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED; +out: mas_destroy(mas); return ret; } diff --git a/lib/once.c b/lib/once.c index d801bfa945e63..0a0a919156e05 100644 --- a/lib/once.c +++ b/lib/once.c @@ -93,6 +93,6 @@ void __do_once_sleepable_done(bool *done, struct static_key_true *once_key, { *done = true; mutex_unlock(&once_mutex); - static_branch_disable(once_key); + once_disable_jump(once_key, mod); } EXPORT_SYMBOL(__do_once_sleepable_done); diff --git a/lib/string.c b/lib/string.c index a4e8ad23577d8..a3778d5aab4af 100644 --- a/lib/string.c +++ b/lib/string.c @@ -821,7 +821,8 @@ void *memchr_inv(const void *start, int c, size_t bytes) { u8 value = c; u64 value64; - unsigned int words, prefix; + size_t words; + unsigned int prefix; if (bytes <= 16) return check_bytes8(start, value, bytes); diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index 69813c10e6c0b..448c3eb48a4a8 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -392,6 +392,7 @@ static void __init test_bitmap_sg(void) /* Scatter/gather relationship */ bitmap_zero(bmap_tmp, 100); + bitmap_zero(bmap_res, 100); bitmap_gather(bmap_tmp, bmap_scatter, sg_mask, nbits); bitmap_scatter(bmap_res, bmap_tmp, sg_mask, nbits); expect_eq_bitmap(bmap_scatter, bmap_res, 100); diff --git a/lib/test_hmm.c b/lib/test_hmm.c index c4adbf98fac79..45c0cb9922184 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -407,7 +407,7 @@ static int dmirror_fault(struct dmirror *dmirror, unsigned long start, /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) - return 0; + return -EFAULT; for (addr = start; addr < end; addr = range.end) { range.start = addr; diff --git a/lib/ucs2_string.c b/lib/ucs2_string.c index dfb4f2358cabf..ca5c615d5bef2 100644 --- a/lib/ucs2_string.c +++ b/lib/ucs2_string.c @@ -8,7 +8,7 @@ ucs2_strnlen(const ucs2_char_t *s, size_t maxlength) { unsigned long length = 0; - while (*s++ != 0 && length < maxlength) + while (length < maxlength && *s++ != 0) length++; return length; } diff --git a/lib/xarray.c b/lib/xarray.c index 9a8b4916540cf..bfe7bef80f34e 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1053,6 +1053,9 @@ void xas_split_alloc(struct xa_state *xas, void *entry, unsigned int order, if (xas->xa_shift + XA_CHUNK_SHIFT > order) return; + if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT) + gfp |= __GFP_ACCOUNT; + do { struct xa_node *node; diff --git a/mm/backing-dev.c b/mm/backing-dev.c index cecbcf9060a65..18e999053bae0 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -910,8 +910,9 @@ static void cleanup_offline_cgwbs_workfn(struct work_struct *work) continue; spin_unlock_irq(&cgwb_lock); - while (cleanup_offline_cgwb(wb)) - cond_resched(); + do { + cond_resched_tasks_rcu_qs(); + } while (cleanup_offline_cgwb(wb)); spin_lock_irq(&cgwb_lock); wb_put(wb); diff --git a/mm/compaction.c b/mm/compaction.c index f08765ade014c..65fd08aabb2c2 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1381,12 +1381,44 @@ static bool suitable_migration_source(struct compact_control *cc, if (pageblock_skip_persistent(page)) return false; - if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction) + /* + * Background compaction produces blocks for the zone at + * large, with no particular allocation context. Allow all + * block types, including CMA. + */ + if (!cc->direct_compaction) return true; block_mt = get_pageblock_migratetype(page); - if (cc->migratetype == MIGRATE_MOVABLE) + /* + * CMA pages can only be taken by ALLOC_CMA requests. For anybody + * else, vacating a CMA block consumes free pages the caller + * could have used, and produces free pages it cannot. + */ + if (is_migrate_cma(block_mt) && !(cc->alloc_flags & ALLOC_CMA)) + return false; + + /* + * Per default, scans are restricted to blocks compatible with + * the request, to prevent cross-contamination. Once + * compaction priority escalates to synchronous scans, though, + * scan all blocks to try to make forward progress. For + * movable request, this likely helps little: there shouldn't + * be many migratable pages inside non-movable blocks besides + * allocator fallbacks. For non-movable requests, this helps a + * lot, as they can finally scan movable blocks. + */ + if (cc->mode != MIGRATE_ASYNC) + return true; + + /* + * Prevent migratetype == MIGRATE_MOVABLE || cc->order >= pageblock_order) return is_migrate_movable(block_mt); else return block_mt == cc->migratetype; @@ -1974,12 +2006,12 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) return pfn; /* - * Only allow kcompactd and direct requests for movable pages to - * quickly clear out a MOVABLE pageblock for allocation. This - * reduces the risk that a large movable pageblock is freed for - * an unmovable/reclaimable small allocation. + * Prevent direct_compaction && cc->migratetype != MIGRATE_MOVABLE) + if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE && + cc->order < pageblock_order) return pfn; /* @@ -2770,9 +2802,8 @@ compact_zone(struct compact_control *cc, struct capture_control *capc) static enum compact_result compact_zone_order(struct zone *zone, int order, gfp_t gfp_mask, enum compact_priority prio, unsigned int alloc_flags, int highest_zoneidx, - struct page **capture) + struct capture_control *capc) { - enum compact_result ret; struct compact_control cc = { .order = order, .search_order = order, @@ -2787,54 +2818,24 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY), .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY) }; - struct capture_control capc = { - .cc = &cc, - .page = NULL, - }; - /* - * Make sure the structs are really initialized before we expose the - * capture control, in case we are interrupted and the interrupt handler - * frees a page. - */ - barrier(); - WRITE_ONCE(current->capture_control, &capc); - - ret = compact_zone(&cc, &capc); - - /* - * Make sure we hide capture control first before we read the captured - * page pointer, otherwise an interrupt could free and capture a page - * and we would leak it. - */ - WRITE_ONCE(current->capture_control, NULL); - *capture = READ_ONCE(capc.page); - /* - * Technically, it is also possible that compaction is skipped but - * the page is still captured out of luck(IRQ came and freed the page). - * Returning COMPACT_SUCCESS in such cases helps in properly accounting - * the COMPACT[STALL|FAIL] when compaction is skipped. - */ - if (*capture) - ret = COMPACT_SUCCESS; - - return ret; + return compact_zone(&cc, capc); } /** * try_to_compact_pages - Direct compact to satisfy a high-order allocation * @gfp_mask: The GFP mask of the current allocation - * @order: The order of the current allocation + * @order: The order to try to make available * @alloc_flags: The allocation flags of the current allocation * @ac: The context of current allocation * @prio: Determines how hard direct compaction should try to succeed - * @capture: Pointer to free page created by compaction will be stored here + * @capc: Free page capture bypassing the freelist * * This is the main entry point for direct page compaction. */ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, unsigned int alloc_flags, const struct alloc_context *ac, - enum compact_priority prio, struct page **capture) + enum compact_priority prio, struct capture_control *capc) { struct zoneref *z; struct zone *zone; @@ -2861,8 +2862,17 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, continue; } + WRITE_ONCE(capc->zone, zone); + status = compact_zone_order(zone, order, gfp_mask, prio, - alloc_flags, ac->highest_zoneidx, capture); + alloc_flags, ac->highest_zoneidx, capc); + + WRITE_ONCE(capc->zone, NULL); + + /* Stop if a page has been captured */ + if (READ_ONCE(capc->page)) + status = COMPACT_SUCCESS; + rc = max(status, rc); /* The allocation should succeed, stop compacting */ diff --git a/mm/damon/core.c b/mm/damon/core.c index cff932b3317d0..c66fd59eb8ffa 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -579,6 +579,7 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern, INIT_LIST_HEAD(&scheme->ops_filters); scheme->stat = (struct damos_stat){}; scheme->max_nr_snapshots = 0; + scheme->last_applied = NULL; INIT_LIST_HEAD(&scheme->list); scheme->quota = *(damos_quota_init(quota)); @@ -1670,7 +1671,7 @@ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) return sz; } -static void damon_split_region_at(struct damon_target *t, +static int damon_split_region_at(struct damon_target *t, struct damon_region *r, unsigned long sz_r); /* @@ -1696,11 +1697,13 @@ static unsigned long damon_apply_min_nr_regions(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region_safe(r, next, t) { while (damon_sz_region(r) > max_region_sz) { - damon_split_region_at(t, r, max_region_sz); + if (damon_split_region_at(t, r, max_region_sz)) + goto out; r = damon_next_region(r); } } } +out: return max_region_sz; } @@ -2142,36 +2145,39 @@ static bool damos_skip_charged_region(struct damon_target *t, { struct damos_quota *quota = &s->quota; unsigned long sz_to_skip; + bool skip = false; /* Skip previously charged regions */ if (quota->charge_target_from) { if (t != quota->charge_target_from) return true; - if (r == damon_last_region(t)) { - quota->charge_target_from = NULL; - quota->charge_addr_from = 0; - return true; - } if (quota->charge_addr_from && - r->ar.end <= quota->charge_addr_from) - return true; + r->ar.end <= quota->charge_addr_from) { + skip = true; + goto out; + } if (quota->charge_addr_from && r->ar.start < quota->charge_addr_from) { sz_to_skip = ALIGN_DOWN(quota->charge_addr_from - r->ar.start, min_region_sz); if (!sz_to_skip) { - if (damon_sz_region(r) <= min_region_sz) - return true; + if (damon_sz_region(r) <= min_region_sz) { + skip = true; + goto out; + } sz_to_skip = min_region_sz; } damon_split_region_at(t, r, sz_to_skip); - return true; + skip = true; } + } +out: + if (r == damon_last_region(t)) { quota->charge_target_from = NULL; quota->charge_addr_from = 0; } - return false; + return skip; } static void damos_update_stat(struct damos *s, @@ -2890,6 +2896,7 @@ static void damos_set_effective_quota(struct damon_ctx *ctx, struct damos *s) struct damos_quota *quota = &s->quota; unsigned long throughput; unsigned long esz = ULONG_MAX; + unsigned long esz_time; if (!quota->ms && list_empty("a->goals)) { quota->esz = quota->sz; @@ -2910,8 +2917,8 @@ static void damos_set_effective_quota(struct damon_ctx *ctx, struct damos *s) 1000000, quota->total_charged_ns); else throughput = PAGE_SIZE * 1024; - esz = min(throughput * quota->ms, esz); - esz = max(ctx->min_region_sz, esz); + esz_time = max(throughput * quota->ms, ctx->min_region_sz); + esz = min(esz_time, esz); } if (quota->sz && quota->sz < esz) @@ -3040,8 +3047,15 @@ static void kdamond_apply_schemes(struct damon_ctx *c) max_region_sz = damon_region_sz_limit(c); mutex_lock(&c->walk_control_lock); damon_for_each_target(t, c) { - if (c->ops.target_valid && c->ops.target_valid(t) == false) + if (c->ops.target_valid && c->ops.target_valid(t) == false) { + damon_for_each_scheme(s, c) { + if (s->quota.charge_target_from != t) + continue; + s->quota.charge_target_from = NULL; + s->quota.charge_addr_from = 0; + } continue; + } damos_apply_target(c, t, max_region_sz); } @@ -3115,18 +3129,20 @@ static void damon_verify_merge_regions_of(struct damon_region *r) * sz_limit size upper limit of each region */ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, - unsigned long sz_limit) + unsigned long sz_limit, bool count_age) { struct damon_region *r, *prev = NULL, *next; damon_for_each_region_safe(r, next, t) { damon_verify_merge_regions_of(r); - if (abs(r->nr_accesses - r->last_nr_accesses) > thres) - r->age = 0; - else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) - r->age = 0; - else - r->age++; + if (count_age) { + if (abs(r->nr_accesses - r->last_nr_accesses) > thres) + r->age = 0; + else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) + r->age = 0; + else + r->age++; + } if (prev && prev->ar.end == r->ar.start && abs(prev->nr_accesses - r->nr_accesses) <= thres && @@ -3160,18 +3176,26 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, struct damon_target *t; unsigned int nr_regions; unsigned int max_thres; + bool count_age = true; max_thres = c->attrs.aggr_interval / (c->attrs.sample_interval ? c->attrs.sample_interval : 1); - do { + while (true) { nr_regions = 0; damon_for_each_target(t, c) { - damon_merge_regions_of(t, threshold, sz_limit); + damon_merge_regions_of(t, threshold, sz_limit, + count_age); nr_regions += damon_nr_regions(t); } - threshold = max(1, threshold * 2); - } while (nr_regions > c->attrs.max_nr_regions && - threshold / 2 < max_thres); + count_age = false; + if (nr_regions <= c->attrs.max_nr_regions || + max_thres <= threshold) + break; + if (threshold < max_thres / 2) + threshold = max(1, threshold * 2); + else + threshold = max_thres; + } } #ifdef CONFIG_DAMON_DEBUG_SANITY @@ -3194,8 +3218,10 @@ static void damon_verify_split_region_at(struct damon_region *r, * * r the region to be split * sz_r size of the first sub-region that will be made + * + * Return: 0 on success, negative error code otherwise. */ -static void damon_split_region_at(struct damon_target *t, +static int damon_split_region_at(struct damon_target *t, struct damon_region *r, unsigned long sz_r) { struct damon_region *new; @@ -3203,7 +3229,7 @@ static void damon_split_region_at(struct damon_target *t, damon_verify_split_region_at(r, sz_r); new = damon_new_region(r->ar.start + sz_r, r->ar.end); if (!new) - return; + return -ENOMEM; r->ar.end = new->ar.start; @@ -3215,6 +3241,7 @@ static void damon_split_region_at(struct damon_target *t, memcpy(new->probe_hits, r->probe_hits, sizeof(r->probe_hits)); damon_insert_region(new, r, damon_next_region(r), t); + return 0; } /* Split every region in the given target into 'nr_subs' regions */ diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 0bcad6b1e5b9e..58ccb14f80b35 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -61,7 +61,12 @@ void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr * device aspects. */ if (likely(pte_present(pteval))) - young |= ptep_test_and_clear_young(vma, addr, pte); + /* + * Arch implementation of ptep_test_and_clear_young() may + * require aligned @addr + */ + young |= ptep_test_and_clear_young(vma, PAGE_ALIGN_DOWN(addr), + pte); young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); if (young) folio_set_young(folio); @@ -111,8 +116,9 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r, unsigned int age_weight = s->quota.weight_age; int hotness; - freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / - damon_max_nr_accesses(&c->attrs); + freq_subscore = mult_frac(r->nr_accesses_bp / 10000, + DAMON_MAX_SUBSCORE, + damon_max_nr_accesses(&c->attrs)); age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; if (age_in_sec) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index f7613ce279a24..b3507b1f62499 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -65,7 +65,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) } } -static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) +static bool damon_pa_young(phys_addr_t paddr) { struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); bool accessed; @@ -74,7 +74,6 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) return false; accessed = damon_folio_young(folio); - *folio_sz = folio_size(folio); folio_put(folio); return accessed; } @@ -82,23 +81,12 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) static void __damon_pa_check_access(struct damon_region *r, struct damon_attrs *attrs, unsigned long addr_unit) { - static phys_addr_t last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; phys_addr_t sampling_addr = damon_pa_phys_addr( r->sampling_addr, addr_unit); - /* If the region is in the last checked page, reuse the result */ - if (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(sampling_addr, last_folio_sz)) { - damon_update_region_access_rate(r, last_accessed, attrs); - return; - } - - last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); - - last_addr = sampling_addr; + accessed = damon_pa_young(sampling_addr); + damon_update_region_access_rate(r, accessed, attrs); } static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 3cbeccd436e40..e2c8716be6c9c 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -332,6 +332,7 @@ static void damon_sysfs_scheme_regions_rm_dirs( list_for_each_entry_safe(r, next, ®ions->regions_list, list) { damos_sysfs_region_rm_dirs(r); list_del(&r->list); + kobject_del(&r->kobj); kobject_put(&r->kobj); regions->nr_regions--; } @@ -911,8 +912,10 @@ static void damon_sysfs_scheme_filters_rm_dirs( struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; @@ -1460,8 +1463,10 @@ static void damos_sysfs_quota_goals_rm_dirs( struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr; int i; - for (i = 0; i < goals->nr; i++) + for (i = 0; i < goals->nr; i++) { + kobject_del(&goals_arr[i]->kobj); kobject_put(&goals_arr[i]->kobj); + } goals->nr = 0; kfree(goals_arr); goals->goals_arr = NULL; @@ -2138,8 +2143,10 @@ static void damos_sysfs_dests_rm_dirs( struct damos_sysfs_dest **dests_arr = dests->dests_arr; int i; - for (i = 0; i < dests->nr; i++) + for (i = 0; i < dests->nr; i++) { + kobject_del(&dests_arr[i]->kobj); kobject_put(&dests_arr[i]->kobj); + } dests->nr = 0; kfree(dests_arr); dests->dests_arr = NULL; @@ -2681,6 +2688,7 @@ void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes) for (i = 0; i < schemes->nr; i++) { damon_sysfs_scheme_rm_dirs(schemes_arr[i]); + kobject_del(&schemes_arr[i]->kobj); kobject_put(&schemes_arr[i]->kobj); } schemes->nr = 0; @@ -2722,13 +2730,15 @@ static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes, goto out; err = damon_sysfs_scheme_add_dirs(scheme); if (err) - goto out; + goto del_out; schemes_arr[i] = scheme; schemes->nr++; } return 0; +del_out: + kobject_del(&scheme->kobj); out: damon_sysfs_schemes_rm_dirs(schemes); kobject_put(&scheme->kobj); diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index a9e187158067d..26cdad603ad06 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -105,8 +105,10 @@ static void damon_sysfs_regions_rm_dirs(struct damon_sysfs_regions *regions) struct damon_sysfs_region **regions_arr = regions->regions_arr; int i; - for (i = 0; i < regions->nr; i++) + for (i = 0; i < regions->nr; i++) { + kobject_del(®ions_arr[i]->kobj); kobject_put(®ions_arr[i]->kobj); + } regions->nr = 0; kfree(regions_arr); regions->regions_arr = NULL; @@ -331,6 +333,7 @@ static void damon_sysfs_targets_rm_dirs(struct damon_sysfs_targets *targets) for (i = 0; i < targets->nr; i++) { damon_sysfs_target_rm_dirs(targets_arr[i]); + kobject_del(&targets_arr[i]->kobj); kobject_put(&targets_arr[i]->kobj); } targets->nr = 0; @@ -369,13 +372,15 @@ static int damon_sysfs_targets_add_dirs(struct damon_sysfs_targets *targets, err = damon_sysfs_target_add_dirs(target); if (err) - goto out; + goto del_out; targets_arr[i] = target; targets->nr++; } return 0; +del_out: + kobject_del(&target->kobj); out: damon_sysfs_targets_rm_dirs(targets); kobject_put(&target->kobj); @@ -1640,6 +1645,7 @@ static void damon_sysfs_contexts_rm_dirs(struct damon_sysfs_contexts *contexts) for (i = 0; i < contexts->nr; i++) { damon_sysfs_context_rm_dirs(contexts_arr[i]); + kobject_del(&contexts_arr[i]->kobj); kobject_put(&contexts_arr[i]->kobj); } contexts->nr = 0; @@ -1678,13 +1684,15 @@ static int damon_sysfs_contexts_add_dirs(struct damon_sysfs_contexts *contexts, err = damon_sysfs_context_add_dirs(context); if (err) - goto out; + goto del_out; contexts_arr[i] = context; contexts->nr++; } return 0; +del_out: + kobject_del(&context->kobj); out: damon_sysfs_contexts_rm_dirs(contexts); kobject_put(&context->kobj); @@ -2042,16 +2050,18 @@ static inline bool damon_sysfs_kdamond_running( static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, struct damon_sysfs_context *sys_ctx) { + enum damon_ops_id ops_id; int err; - err = damon_select_ops(ctx, sys_ctx->ops_id); + ops_id = READ_ONCE(sys_ctx->ops_id); + err = damon_select_ops(ctx, ops_id); if (err) return err; - ctx->addr_unit = sys_ctx->addr_unit; + ctx->addr_unit = READ_ONCE(sys_ctx->addr_unit); /* addr_unit is respected by only DAMON_OPS_PADDR */ - if (sys_ctx->ops_id == DAMON_OPS_PADDR) + if (ops_id == DAMON_OPS_PADDR) ctx->min_region_sz = max( - DAMON_MIN_REGION_SZ / sys_ctx->addr_unit, 1); + DAMON_MIN_REGION_SZ / ctx->addr_unit, 1); ctx->pause = sys_ctx->pause; err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs); if (err) @@ -2499,6 +2509,7 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds) for (i = 0; i < kdamonds->nr; i++) { damon_sysfs_kdamond_rm_dirs(kdamonds_arr[i]); + kobject_del(&kdamonds_arr[i]->kobj); kobject_put(&kdamonds_arr[i]->kobj); } kdamonds->nr = 0; @@ -2553,13 +2564,15 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds, err = damon_sysfs_kdamond_add_dirs(kdamond); if (err) - goto out; + goto del_out; kdamonds_arr[i] = kdamond; kdamonds->nr++; } return 0; +del_out: + kobject_del(&kdamond->kobj); out: damon_sysfs_kdamonds_rm_dirs(kdamonds); kobject_put(&kdamond->kobj); diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index fcf7c7fadb5fe..e543cfff15909 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -157,6 +157,10 @@ static void damon_test_split_at(struct kunit *test) r->age = 10; damon_add_region(r, t); damon_split_region_at(t, r, 25); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; + KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); @@ -169,6 +173,7 @@ static void damon_test_split_at(struct kunit *test) KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses); KUNIT_EXPECT_EQ(test, r->age, r_new->age); +out: damon_free_target(t); } @@ -256,14 +261,17 @@ static void damon_test_merge_regions_of(struct kunit *test) damon_add_region(r, t); } - damon_merge_regions_of(t, 9, 9999); + damon_merge_regions_of(t, 9, 9999, true); /* 0-112, 114-130, 130-156, 156-170, 170-230, 230-10170 */ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 6u); + if (damon_nr_regions(t) != 6) + goto out; for (i = 0; i < 6; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]); KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]); } +out: damon_free_target(t); } @@ -725,6 +733,7 @@ static void damos_test_commit_quota_goals_for(struct kunit *test, struct damos_quota_goal *goal, *next; bool skip = true; int i; + int nr_dst = 0, nr_src = 0; INIT_LIST_HEAD(&dst.goals); INIT_LIST_HEAD(&src.goals); @@ -747,6 +756,14 @@ static void damos_test_commit_quota_goals_for(struct kunit *test, damos_commit_quota_goals(&dst, &src); + damos_for_each_quota_goal(goal, &dst) + nr_dst++; + damos_for_each_quota_goal(goal, &src) + nr_src++; + KUNIT_EXPECT_EQ(test, nr_dst, nr_src); + if (nr_dst != nr_src) + goto out; + i = 0; damos_for_each_quota_goal(goal, (&dst)) { KUNIT_EXPECT_EQ(test, goal->metric, src_goals[i].metric); @@ -894,6 +911,8 @@ static void damos_test_commit_dests_for(struct kunit *test, skip = false; KUNIT_EXPECT_EQ(test, dst.nr_dests, src_nr_dests); + if (dst.nr_dests != src_nr_dests) + goto out; for (i = 0; i < dst.nr_dests; i++) { KUNIT_EXPECT_EQ(test, dst.node_id_arr[i], src_node_id_arr[i]); KUNIT_EXPECT_EQ(test, dst.weight_arr[i], src_weight_arr[i]); @@ -1152,14 +1171,19 @@ static void damon_test_commit_target_regions_for(struct kunit *test, kunit_skip(test, "src target setup fail"); } damon_commit_target_regions(dst_target, src_target, 1); + + KUNIT_EXPECT_EQ(test, damon_nr_regions(dst_target), nr_expect_regions); + if (damon_nr_regions(dst_target) != nr_expect_regions) + goto out; + i = 0; damon_for_each_region(r, dst_target) { KUNIT_EXPECT_EQ(test, r->ar.start, expect_start_end[i][0]); KUNIT_EXPECT_EQ(test, r->ar.end, expect_start_end[i][1]); i++; } - KUNIT_EXPECT_EQ(test, damon_nr_regions(dst_target), nr_expect_regions); - KUNIT_EXPECT_EQ(test, i, nr_expect_regions); + +out: damon_free_target(dst_target); damon_free_target(src_target); } @@ -1251,6 +1275,8 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 1); KUNIT_EXPECT_EQ(test, r->ar.end, 2); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, 2); KUNIT_EXPECT_EQ(test, r2->ar.end, 4); @@ -1265,11 +1291,14 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 2); KUNIT_EXPECT_EQ(test, r->ar.end, 6); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, 6); KUNIT_EXPECT_EQ(test, r2->ar.end, 8); damon_destroy_region(r2, t); +out: damon_free_target(t); damos_free_filter(f); } diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h index 61f844336ffb5..6a95441d193ae 100644 --- a/mm/damon/tests/vaddr-kunit.h +++ b/mm/damon/tests/vaddr-kunit.h @@ -158,12 +158,17 @@ static void damon_do_test_apply_three_regions(struct kunit *test, kunit_skip(test, "second damon_set_regions() fail"); } + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), nr_expected / 2); + if (damon_nr_regions(t) != nr_expected / 2) + goto out; + for (i = 0; i < nr_expected / 2; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]); KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]); } +out: damon_destroy_target(t, NULL); } diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 2c1c1952c008d..fb4f4911ea70c 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -292,22 +292,29 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, } #ifdef CONFIG_HUGETLB_PAGE +static bool damon_hugetlb_ptep_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, pte_t *entry) +{ + unsigned long psize = huge_page_size(hstate_vma(vma)); + + if (!pte_young(*entry)) + return false; + *entry = huge_ptep_get_and_clear(mm, addr, pte, psize); + *entry = pte_mkold(*entry); + set_huge_pte_at(mm, addr, pte, *entry, psize); + return true; +} + static void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr) { bool referenced = false; pte_t entry = huge_ptep_get(mm, addr, pte); struct folio *folio = pfn_folio(pte_pfn(entry)); - unsigned long psize = huge_page_size(hstate_vma(vma)); folio_get(folio); - if (pte_young(entry)) { - referenced = true; - entry = pte_mkold(entry); - set_huge_pte_at(mm, addr, pte, entry, psize); - } - + referenced = damon_hugetlb_ptep_mkold(pte, mm, vma, addr, &entry); if (mmu_notifier_clear_young(mm, addr, addr + huge_page_size(hstate_vma(vma)))) referenced = true; @@ -382,8 +389,6 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx) } struct damon_young_walk_private { - /* size of the folio for the access checked virtual memory address */ - unsigned long *folio_sz; bool young; }; @@ -410,7 +415,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = HPAGE_PMD_SIZE; huge_out: spin_unlock(ptl); return 0; @@ -429,7 +433,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, if (pte_young(ptent) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = folio_size(folio); out: pte_unmap_unlock(pte, ptl); return 0; @@ -457,7 +460,6 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, if (pte_young(entry) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = huge_page_size(h); folio_put(folio); @@ -469,11 +471,9 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, #define damon_young_hugetlb_entry NULL #endif /* CONFIG_HUGETLB_PAGE */ -static bool damon_va_young(struct mm_struct *mm, unsigned long addr, - unsigned long *folio_sz) +static bool damon_va_young(struct mm_struct *mm, unsigned long addr) { struct damon_young_walk_private arg = { - .folio_sz = folio_sz, .young = false, }; @@ -493,29 +493,18 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr, * r the region to be checked */ static void __damon_va_check_access(struct mm_struct *mm, - struct damon_region *r, bool same_target, + struct damon_region *r, struct damon_attrs *attrs) { - static unsigned long last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; if (!mm) { damon_update_region_access_rate(r, false, attrs); return; } - /* If the region is in the last checked page, reuse the result */ - if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(r->sampling_addr, last_folio_sz))) { - damon_update_region_access_rate(r, last_accessed, attrs); - return; - } - - last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); - - last_addr = r->sampling_addr; + accessed = damon_va_young(mm, r->sampling_addr); + damon_update_region_access_rate(r, accessed, attrs); } static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) @@ -524,16 +513,12 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) struct mm_struct *mm; struct damon_region *r; unsigned int max_nr_accesses = 0; - bool same_target; damon_for_each_target(t, ctx) { mm = damon_get_mm(t); - same_target = false; damon_for_each_region(r, t) { - __damon_va_check_access(mm, r, same_target, - &ctx->attrs); + __damon_va_check_access(mm, r, &ctx->attrs); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); - same_target = true; } if (mm) mmput(mm); diff --git a/mm/filemap.c b/mm/filemap.c index d721986d5f465..ebc51fabfcd8d 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1616,7 +1616,7 @@ static void filemap_end_dropbehind(struct folio *folio) return; if (!folio_test_clear_dropbehind(folio)) return; - if (mapping) + if (mapping && !folio_mapped(folio)) folio_unmap_invalidate(mapping, folio, 0); } diff --git a/mm/gup.c b/mm/gup.c index 0692119b79043..bde05664fe9a9 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2273,6 +2273,7 @@ static unsigned long collect_longterm_unpinnable_folios( for (folio = pofs_get_folio(pofs, i); folio; folio = pofs_next_folio(folio, pofs, &i)) { + const int pin_refs = folio_has_pincount(folio) ? 1 : GUP_PIN_COUNTING_BIAS; if (folio_is_longterm_pinnable(folio)) continue; @@ -2287,15 +2288,20 @@ static unsigned long collect_longterm_unpinnable_folios( continue; } + /* + * We drain not only to make the folio_isolate_lru() succeed, + * but also to remove any other folio references from LRU + * caches. + */ if (drained == 0 && folio_may_be_lru_cached(folio) && folio_ref_count(folio) != - folio_expected_ref_count(folio) + 1) { + folio_expected_ref_count(folio) + pin_refs) { lru_add_drain(); drained = 1; } if (drained == 1 && folio_may_be_lru_cached(folio) && folio_ref_count(folio) != - folio_expected_ref_count(folio) + 1) { + folio_expected_ref_count(folio) + pin_refs) { lru_add_drain_all(); drained = 2; } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 5d94fa4c74fdb..a2ad4a96b1937 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -92,7 +92,7 @@ unsigned long huge_anon_orders_madvise __read_mostly; unsigned long huge_anon_orders_inherit __read_mostly; static bool anon_orders_configured __initdata; -static inline bool file_thp_enabled(struct vm_area_struct *vma) +static inline bool file_thp_enabled(const struct vm_area_struct *vma) { struct inode *inode; @@ -118,6 +118,67 @@ static bool vma_is_special_huge(const struct vm_area_struct *vma) return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT); } +static bool vma_file_bypass_thp_tuneables(const struct vm_area_struct *vma, + enum tva_type type) +{ + const bool has_huge_fault = vma->vm_ops->huge_fault; + + /* MADV_COLLAPSE ignores tuneables. */ + if (type == TVA_FORCED_COLLAPSE) + return true; + /* Huge PFN mappings are uncompactable so the policy doesn't apply. */ + if (vma_test(vma, VMA_PFNMAP_BIT) && has_huge_fault) + return true; + return false; +} + +static bool vma_file_allow_thp_tuneables(vm_flags_t vm_flags) +{ + /* THP=always? */ + if (hugepage_global_always()) + return true; + /* THP=madvise and marked MADV_HUGEPAGE? */ + if (hugepage_global_enabled() && (vm_flags & VM_HUGEPAGE)) + return true; + return false; +} + +static bool vma_file_check_thp_tuneables(const struct vm_area_struct *vma, + vm_flags_t vm_flags, enum tva_type type) +{ + return vma_file_bypass_thp_tuneables(vma, type) || + vma_file_allow_thp_tuneables(vm_flags); +} + +static bool vma_can_map_huge_file(const struct vm_area_struct *vma, + vm_flags_t vm_flags, enum tva_type type) +{ + const bool has_huge_fault = vma->vm_ops->huge_fault; + + /* + * Enforce THP collapse requirements as necessary. Anonymous vmas + * were already handled in thp_vma_allowable_orders(). + */ + if (!vma_file_check_thp_tuneables(vma, vm_flags, type)) + return false; + + switch (type) { + case TVA_PAGEFAULT: + /* + * Trust that ->huge_fault() handlers know what they are doing + * in fault path. + */ + return has_huge_fault; + case TVA_SMAPS: + if (has_huge_fault) + return true; + fallthrough; + default: + /* Only regular file is valid in collapse path. */ + return file_thp_enabled(vma); + } +} + unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type, @@ -190,27 +251,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vma, vma->vm_pgoff, 0, forced_collapse); - if (!vma_is_anonymous(vma)) { - /* - * Enforce THP collapse requirements as necessary. Anonymous vmas - * were already handled in thp_vma_allowable_orders(). - */ - if (!forced_collapse && - (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && - !hugepage_global_always()))) - return 0; - - /* - * Trust that ->huge_fault() handlers know what they are doing - * in fault path. - */ - if (((in_pf || smaps)) && vma->vm_ops->huge_fault) - return orders; - /* Only regular file is valid in collapse path */ - if (((!in_pf || smaps)) && file_thp_enabled(vma)) - return orders; - return 0; - } + if (!vma_is_anonymous(vma)) + return vma_can_map_huge_file(vma, vm_flags, type) ? orders : 0; if (vma_is_temporary_stack(vma)) return 0; @@ -2340,8 +2382,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto out; if (unlikely(!pmd_present(orig_pmd))) { - VM_BUG_ON(thp_migration_supported() && - !pmd_is_migration_entry(orig_pmd)); + VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && + !pmd_is_device_private_entry(orig_pmd)); goto out; } @@ -2414,6 +2456,8 @@ static void zap_huge_pmd_folio(struct mm_struct *mm, struct vm_area_struct *vma, add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); + if (is_present && pmd_dirty(pmdval)) + folio_mark_dirty(folio); if (is_present && pmd_young(pmdval) && likely(vma_has_recency(vma))) folio_mark_accessed(folio); @@ -4027,34 +4071,42 @@ static int __folio_split(struct folio *folio, unsigned int new_order, XA_STATE(xas, &folio->mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); bool is_anon = folio_test_anon(folio); + struct mem_cgroup *memcg, *old_memcg; struct address_space *mapping = NULL; struct anon_vma *anon_vma = NULL; int old_order = folio_order(folio); struct folio *new_folio, *next; int nr_shmem_dropped = 0; enum ttu_flags ttu_flags = 0; - int ret; pgoff_t end = 0; + int ret; VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); if (folio != page_folio(split_at) || folio != page_folio(lock_at)) { ret = -EINVAL; - goto out; + goto out_no_memcg; } if (new_order >= old_order) { ret = -EINVAL; - goto out; + goto out_no_memcg; } ret = folio_check_splittable(folio, new_order, split_type); if (ret) { VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio"); - goto out; + goto out_no_memcg; } + /* + * switch to folio's memcg as xarray node allocation can happen and + * needs to charge to it. + */ + memcg = get_mem_cgroup_from_folio(folio); + old_memcg = set_active_memcg(memcg); + if (is_anon) { /* * The caller does not necessarily hold an mmap_lock that would @@ -4197,6 +4249,10 @@ static int __folio_split(struct folio *folio, unsigned int new_order, if (mapping) i_mmap_unlock_read(mapping); out: + /* restore to caller's old_memcg */ + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); +out_no_memcg: xas_destroy(&xas); if (is_pmd_order(old_order)) count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index e93c4d2456aa4..abb15476eff1d 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1977,6 +1977,15 @@ int dissolve_free_hugetlb_folio(struct folio *folio) struct hstate *h = folio_hstate(folio); bool adjust_surplus = false; + /* + * remove_hugetlb_folio()/update_and_free_hugetlb_folio() bail + * for gigantic hstates without runtime support, so dissolving one + * here would leave it on the free list and, on vmemmap restore + * failure, the add_hugetlb_folio() rollback corrupts that list. + */ + if (hstate_is_gigantic_no_runtime(h)) + goto out; + if (!available_huge_pages(h)) goto out; @@ -2002,7 +2011,8 @@ int dissolve_free_hugetlb_folio(struct folio *folio) if (h->surplus_huge_pages_node[folio_nid(folio)]) adjust_surplus = true; remove_hugetlb_folio(h, folio, adjust_surplus); - h->max_huge_pages--; + if (!adjust_surplus) + h->max_huge_pages--; spin_unlock_irq(&hugetlb_lock); /* @@ -2022,7 +2032,8 @@ int dissolve_free_hugetlb_folio(struct folio *folio) if (rc) { spin_lock_irq(&hugetlb_lock); add_hugetlb_folio(h, folio, adjust_surplus); - h->max_huge_pages++; + if (!adjust_surplus) + h->max_huge_pages++; goto out; } } else { @@ -3353,7 +3364,7 @@ static void __init gather_bootmem_prealloc_parallel(unsigned long start, gather_bootmem_prealloc_node(nid); } -static void __init gather_bootmem_prealloc(void) +void __init hugetlb_bootmem_struct_page_init(void) { struct padata_mt_job job = { .thread_fn = gather_bootmem_prealloc_parallel, @@ -3365,6 +3376,31 @@ static void __init gather_bootmem_prealloc(void) .max_threads = num_node_state(N_MEMORY), .numa_aware = true, }; +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP + struct zone *zone; + + for_each_zone(zone) { + for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { + struct page *tail, *p; + unsigned int order; + + tail = zone->vmemmap_tails[i]; + if (!tail) + continue; + + order = i + VMEMMAP_TAIL_MIN_ORDER; + p = page_to_virt(tail); + /* + * prep_and_add_bootmem_folios() can access pageblock + * flags on bootmem HugeTLB pages, so initialize the + * shared tail struct pages here before bootmem folios + * start using them. + */ + for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) + init_compound_tail(p + j, NULL, order, zone); + } + } +#endif padata_do_multithreaded(&job); } @@ -3557,7 +3593,7 @@ static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h) * - For gigantic pages, this is called early in the boot process and * pages are allocated from memblock allocated or something similar. * Gigantic pages are actually added to pools later with the routine - * gather_bootmem_prealloc. + * hugetlb_bootmem_struct_page_init. * - For non-gigantic pages, this is called later in the boot process after * all of mm is up and functional. Pages are allocated from buddy and * then added to hugetlb pools. @@ -4127,7 +4163,6 @@ static int __init hugetlb_init(void) } hugetlb_init_hstates(); - gather_bootmem_prealloc(); report_hugepages(); hugetlb_sysfs_init(); @@ -5108,18 +5143,21 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, hugetlb_vma_lock_write(vma); i_mmap_lock_write(mapping); for (; old_addr < old_end; old_addr += sz, new_addr += sz) { + const unsigned long offset_to_last_entry = + (old_addr | last_addr_mask) - old_addr; + src_pte = hugetlb_walk(vma, old_addr, sz); if (!src_pte) { - old_addr |= last_addr_mask; - new_addr |= last_addr_mask; + old_addr += offset_to_last_entry; + new_addr += offset_to_last_entry; continue; } if (huge_pte_none(huge_ptep_get(mm, old_addr, src_pte))) continue; if (huge_pmd_unshare(&tlb, vma, old_addr, src_pte)) { - old_addr |= last_addr_mask; - new_addr |= last_addr_mask; + old_addr += offset_to_last_entry; + new_addr += offset_to_last_entry; continue; } @@ -5157,6 +5195,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, bool adjust_reservation; unsigned long last_addr_mask; + i_mmap_assert_write_locked(vma->vm_file->f_mapping); WARN_ON(!is_vm_hugetlb_page(vma)); BUG_ON(start & ~huge_page_mask(h)); BUG_ON(end & ~huge_page_mask(h)); @@ -5248,7 +5287,10 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, /* * Restore the reservation for anonymous page, otherwise the - * backing page could be stolen by someone. + * backing page could be stolen by someone. Restore only on the + * last unmap, otherwise the owner could empty its resv map + * while the folio is still mapped by a child. Note that holding + * i_mmap_lock_write is needed to check the number of mappings. * If there we are freeing a surplus, do not set the restore * reservation bit. */ @@ -5256,7 +5298,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, spin_lock_irq(&hugetlb_lock); if (!h->surplus_huge_pages && __vma_private_lock(vma) && - folio_test_anon(folio)) { + !folio_mapped(folio) && folio_test_anon(folio)) { folio_set_hugetlb_restore_reserve(folio); /* Reservation to be adjusted after the spin lock */ adjust_reservation = true; @@ -7215,14 +7257,14 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int re * There is no need to transfer the per-node surplus state * when we do not cross the node. */ - if (new_nid == old_nid) - return; - spin_lock_irq(&hugetlb_lock); - if (h->surplus_huge_pages_node[old_nid]) { - h->surplus_huge_pages_node[old_nid]--; - h->surplus_huge_pages_node[new_nid]++; + if (new_nid != old_nid) { + spin_lock_irq(&hugetlb_lock); + if (h->surplus_huge_pages_node[old_nid]) { + h->surplus_huge_pages_node[old_nid]--; + h->surplus_huge_pages_node[new_nid]++; + } + spin_unlock_irq(&hugetlb_lock); } - spin_unlock_irq(&hugetlb_lock); } /* diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index e0083de1ca82b..ecb6e0b7819a0 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -97,6 +97,7 @@ static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup, struct page_counter *fault, *fault_parent = NULL; struct page_counter *rsvd, *rsvd_parent = NULL; unsigned long limit; + int ret; if (parent_h_cgroup) { fault_parent = hugetlb_cgroup_counter_from_cgroup( @@ -118,8 +119,10 @@ static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup, limit = round_down(PAGE_COUNTER_MAX, pages_per_huge_page(&hstates[idx])); - VM_BUG_ON(page_counter_set_max(fault, limit)); - VM_BUG_ON(page_counter_set_max(rsvd, limit)); + ret = page_counter_set_max(fault, limit); + VM_WARN_ON_ONCE(ret); + ret = page_counter_set_max(rsvd, limit); + VM_WARN_ON_ONCE(ret); } } diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index 39344d6c78d84..3e22161686d2f 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -30,15 +31,25 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, int node; struct folio *folio; struct page *page = NULL; + const nodemask_t *nmask; + unsigned int cpuset_mems_cookie; if (!hugetlb_cma_size) return NULL; - if (hugetlb_cma[nid]) +retry_cpuset: + if (!nodemask) { + cpuset_mems_cookie = read_mems_allowed_begin(); + nmask = &cpuset_current_mems_allowed; + } else { + nmask = nodemask; + } + + if (hugetlb_cma[nid] && node_isset(nid, *nmask)) page = cma_alloc_frozen_compound(hugetlb_cma[nid], order); if (!page && !(gfp_mask & __GFP_THISNODE)) { - for_each_node_mask(node, *nodemask) { + for_each_node_mask(node, *nmask) { if (node == nid || !hugetlb_cma[node]) continue; @@ -48,8 +59,12 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, } } - if (!page) + if (!page) { + if (!nodemask && + unlikely(read_mems_allowed_retry(cpuset_mems_cookie))) + goto retry_cpuset; return NULL; + } folio = page_folio(page); folio_set_hugetlb_cma(folio); diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 133b46dfb09f5..ea6af85bfec13 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -635,12 +635,12 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, * mirrored tail page structs RO. */ spfn = (unsigned long)&folio->page; - epfn = spfn + pages_per_huge_page(h); + epfn = spfn + hugetlb_vmemmap_size(h); vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio), HUGETLB_VMEMMAP_RESERVE_SIZE); - register_page_bootmem_memmap(pfn_to_section_nr(spfn), + register_page_bootmem_memmap(pfn_to_section_nr(folio_pfn(folio)), &folio->page, - HUGETLB_VMEMMAP_RESERVE_SIZE); + HUGETLB_VMEMMAP_RESERVE_PAGES); continue; } @@ -870,27 +870,10 @@ static const struct ctl_table hugetlb_vmemmap_sysctls[] = { static int __init hugetlb_vmemmap_init(void) { const struct hstate *h; - struct zone *zone; /* HUGETLB_VMEMMAP_RESERVE_SIZE should cover all used struct pages */ BUILD_BUG_ON(__NR_USED_SUBPAGE > HUGETLB_VMEMMAP_RESERVE_PAGES); - for_each_zone(zone) { - for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { - struct page *tail, *p; - unsigned int order; - - tail = zone->vmemmap_tails[i]; - if (!tail) - continue; - - order = i + VMEMMAP_TAIL_MIN_ORDER; - p = page_to_virt(tail); - for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) - init_compound_tail(p + j, NULL, order, zone); - } - } - for_each_hstate(h) { if (hugetlb_vmemmap_optimizable(h)) { register_sysctl_init("vm", hugetlb_vmemmap_sysctls); diff --git a/mm/internal.h b/mm/internal.h index 181e79f1d6a20..5acba3470659d 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1059,7 +1059,15 @@ struct compact_control { * immediately when one is created during the free path. */ struct capture_control { - struct compact_control *cc; + struct zone *zone; + int migratetype; + /* + * Allocation request order. May differ from the compaction + * order: defrag_mode promotes sub-block allocations to + * pageblock-order compaction; capture still matches at the + * original allocation order so prep_new_page() is consistent. + */ + int order; struct page *page; }; diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c index 6958aa713c67e..16f4e67beee82 100644 --- a/mm/kasan/quarantine.c +++ b/mm/kasan/quarantine.c @@ -355,7 +355,12 @@ void kasan_quarantine_remove_cache(struct kmem_cache *cache) */ on_each_cpu(per_cpu_remove_cache, cache, 1); - for_each_online_cpu(cpu) { + /* + * A CPU can go offline after on_each_cpu() returns, leaving cache + * objects on that CPU's shrink list. Scan all possible CPUs to + * drain those lists. + */ + for_each_possible_cpu(cpu) { sq = per_cpu_ptr(&shrink_qlist, cpu); raw_spin_lock_irqsave(&sq->lock, flags); qlist_move_cache(&sq->qlist, &to_free, cache); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 617bca76db49b..2d0aee0312f24 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -452,12 +452,6 @@ int hugepage_madvise(struct vm_area_struct *vma, case MADV_HUGEPAGE: *vm_flags &= ~VM_NOHUGEPAGE; *vm_flags |= VM_HUGEPAGE; - /* - * If the vma become good for khugepaged to scan, - * register it here without waiting a page fault that - * may not happen any time soon. - */ - khugepaged_enter_vma(vma, *vm_flags); break; case MADV_NOHUGEPAGE: *vm_flags &= ~VM_HUGEPAGE; diff --git a/mm/kmemleak.c b/mm/kmemleak.c index e196f53f9b462..e3c9dbde014ba 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1525,22 +1525,25 @@ static int scan_should_stop(void) /* * Scan a memory block (exclusive range) for valid pointers and add those - * found to the gray list. + * found to the gray list. Return non-zero if the scan was interrupted. */ -static void scan_block(void *_start, void *_end, - struct kmemleak_object *scanned) +static int scan_block(void *_start, void *_end, + struct kmemleak_object *scanned) { unsigned long *ptr; unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER); unsigned long *end = _end - (BYTES_PER_POINTER - 1); unsigned long flags; + int stop = 0; raw_spin_lock_irqsave(&kmemleak_lock, flags); for (ptr = start; ptr < end; ptr++) { unsigned long pointer; - if (scan_should_stop()) + if (scan_should_stop()) { + stop = 1; break; + } kasan_disable_current(); pointer = *(unsigned long *)kasan_reset_tag((void *)ptr); @@ -1550,6 +1553,8 @@ static void scan_block(void *_start, void *_end, pointer_update_refs(scanned, pointer, OBJECT_PERCPU); } raw_spin_unlock_irqrestore(&kmemleak_lock, flags); + + return stop; } /* @@ -1564,7 +1569,7 @@ static void scan_large_block(void *start, void *end) next = min(start + MAX_SCAN_SIZE, end); scan_block(start, next, NULL); start = next; - cond_resched(); + cond_resched_tasks_rcu_qs(); } } #endif @@ -1599,7 +1604,7 @@ static void scan_object(struct kmemleak_object *object) scan_block(start, end, object); raw_spin_unlock_irqrestore(&object->lock, flags); - cond_resched(); + cond_resched_tasks_rcu_qs(); raw_spin_lock_irqsave(&object->lock, flags); if (!(object->flags & OBJECT_ALLOCATED)) break; @@ -1621,7 +1626,7 @@ static void scan_object(struct kmemleak_object *object) break; raw_spin_unlock_irqrestore(&object->lock, flags); - cond_resched(); + cond_resched_tasks_rcu_qs(); raw_spin_lock_irqsave(&object->lock, flags); } while (object->flags & OBJECT_ALLOCATED); } else { @@ -1649,7 +1654,7 @@ static void scan_gray_list(void) */ object = list_entry(gray_list.next, typeof(*object), gray_list); while (&object->gray_list != &gray_list) { - cond_resched(); + cond_resched_tasks_rcu_qs(); /* may add new objects to the list */ if (!scan_should_stop()) @@ -1684,7 +1689,7 @@ static void kmemleak_cond_resched(struct kmemleak_object *object) raw_spin_unlock_irq(&kmemleak_lock); rcu_read_unlock(); - cond_resched(); + cond_resched_tasks_rcu_qs(); rcu_read_lock(); raw_spin_lock_irq(&kmemleak_lock); @@ -1696,6 +1701,43 @@ static void kmemleak_cond_resched(struct kmemleak_object *object) put_object(object); } +/* + * Scan all task kernel stacks, rescheduling between tasks. Each task is looked + * up and pinned within its own RCU read-side section, so no lock is held across + * the scan and the walk cannot trip the soft lockup watchdog. + */ +static void kmemleak_scan_task_stacks(void) +{ + struct pid *pid; + int nr = 1; + int stop = 0; + + do { + struct task_struct *p = NULL; + + rcu_read_lock(); + pid = find_ge_pid(nr, &init_pid_ns); + if (pid) { + nr = pid_nr(pid) + 1; + p = pid_task(pid, PIDTYPE_PID); + if (p) + get_task_struct(p); + } + rcu_read_unlock(); + + if (p) { + void *stack = try_get_task_stack(p); + + if (stack) { + stop = scan_block(stack, stack + THREAD_SIZE, NULL); + put_task_stack(p); + } + put_task_struct(p); + } + cond_resched_tasks_rcu_qs(); + } while (pid && !stop); +} + /* * Print one leak inline. The hex dump is gated on OBJECT_ALLOCATED so it * does not touch user memory that was freed concurrently; the rest of the @@ -1866,7 +1908,7 @@ static void kmemleak_scan(void) struct page *page = pfn_to_online_page(pfn); if (!(pfn & 63)) - cond_resched(); + cond_resched_tasks_rcu_qs(); if (!page) continue; @@ -1885,19 +1927,8 @@ static void kmemleak_scan(void) /* * Scanning the task stacks (may introduce false negatives). */ - if (kmemleak_stack_scan) { - struct task_struct *p, *g; - - rcu_read_lock(); - for_each_process_thread(g, p) { - void *stack = try_get_task_stack(p); - if (stack) { - scan_block(stack, stack + THREAD_SIZE, NULL); - put_task_stack(p); - } - } - rcu_read_unlock(); - } + if (kmemleak_stack_scan) + kmemleak_scan_task_stacks(); /* * Scan the objects already referenced from the sections scanned diff --git a/mm/madvise.c b/mm/madvise.c index 77552b03d318d..65e40fc8699db 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -177,6 +177,14 @@ static int madvise_update_vma(vm_flags_t new_flags, /* vm_flags is protected by the mmap_lock held in write mode. */ vma_start_write(vma); vma->flags = new_vma_flags; + /* + * If the vma become good for khugepaged to scan, + * register it here without waiting a page fault that + * may not happen any time soon. + */ + if (vma_flags_test(&new_vma_flags, VMA_HUGEPAGE_BIT)) + khugepaged_enter_vma(vma, vma_flags_to_legacy(new_vma_flags)); + if (set_new_anon_name) return replace_anon_vma_name(vma, anon_name); @@ -388,8 +396,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, goto huge_unlock; if (unlikely(!pmd_present(orig_pmd))) { - VM_BUG_ON(thp_migration_supported() && - !pmd_is_migration_entry(orig_pmd)); + VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && + !pmd_is_device_private_entry(orig_pmd)); goto huge_unlock; } diff --git a/mm/memblock.c b/mm/memblock.c index 6349c48154f4b..a8d1992ba7a13 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -2823,14 +2823,18 @@ static int memblock_debug_show(struct seq_file *m, void *private) else seq_printf(m, "%4c ", 'x'); if (reg->flags) { - for (j = 0; j < count; j++) { - if (reg->flags & (1U << j)) { - seq_printf(m, "%s\n", flagname[j]); - break; - } + unsigned int flags = reg->flags; + bool first = true; + + for (j = 0; flags; j++, flags >>= 1) { + if (!(flags & 1)) + continue; + if (!first) + seq_putc(m, '|'); + seq_puts(m, j < count ? flagname[j] : "UNKNOWN"); + first = false; } - if (j == count) - seq_printf(m, "%s\n", "UNKNOWN"); + seq_putc(m, '\n'); } else { seq_printf(m, "%s\n", "NONE"); } diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 7650692115678..f1a64d0604a59 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -95,7 +95,6 @@ enum { RES_LIMIT, RES_MAX_USAGE, RES_FAILCNT, - RES_SOFT_LIMIT, }; #ifdef CONFIG_LOCKDEP @@ -1181,13 +1180,13 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, event->unregister_event = mem_cgroup_usage_unregister_event; } else if (!strcmp(name, "memory.oom_control")) { pr_warn_once("oom_control is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org" + "Please report your usecase to linux-mm@kvack.org" " if you depend on this functionality.\n"); event->register_event = mem_cgroup_oom_register_event; event->unregister_event = mem_cgroup_oom_unregister_event; } else if (!strcmp(name, "memory.pressure_level")) { pr_warn_once("pressure_level is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org " + "Please report your usecase to linux-mm@kvack.org " "if you depend on this functionality.\n"); event->register_event = vmpressure_register_event; event->unregister_event = vmpressure_unregister_event; @@ -1589,6 +1588,30 @@ static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, return -EINVAL; } +static u64 mem_cgroup_soft_limit_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return (u64)PAGE_COUNTER_MAX * PAGE_SIZE; +} + +static ssize_t mem_cgroup_soft_limit_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + unsigned long nr_pages; + int ret; + + ret = page_counter_memparse(strstrip(buf), "-1", &nr_pages); + if (ret) + return ret; + + pr_warn_once("soft_limit_in_bytes is deprecated and will be removed. " + "Writing any value to this file has no effect. " + "Please report your usecase to linux-mm@kvack.org if you " + "depend on this functionality.\n"); + + return nbytes; +} + static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) { @@ -1625,8 +1648,6 @@ static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, return (u64)counter->watermark * PAGE_SIZE; case RES_FAILCNT: return counter->failcnt; - case RES_SOFT_LIMIT: - return (u64)READ_ONCE(memcg->soft_limit) * PAGE_SIZE; default: BUG(); } @@ -1721,17 +1742,6 @@ static ssize_t mem_cgroup_write(struct kernfs_open_file *of, break; } break; - case RES_SOFT_LIMIT: - if (IS_ENABLED(CONFIG_PREEMPT_RT)) { - ret = -EOPNOTSUPP; - } else { - pr_warn_once("soft_limit_in_bytes is deprecated and will be removed. " - "Please report your usecase to linux-mm@kvack.org if you " - "depend on this functionality.\n"); - WRITE_ONCE(memcg->soft_limit, nr_pages); - ret = 0; - } - break; } return ret ?: nbytes; } @@ -2040,7 +2050,7 @@ static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, struct mem_cgroup *memcg = mem_cgroup_from_css(css); pr_warn_once("oom_control is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org if you " + "Please report your usecase to linux-mm@kvack.org if you " "depend on this functionality.\n"); /* cannot set to root cgroup and only 0 and 1 are allowed */ @@ -2085,9 +2095,8 @@ struct cftype mem_cgroup_legacy_files[] = { }, { .name = "soft_limit_in_bytes", - .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), - .write = mem_cgroup_write, - .read_u64 = mem_cgroup_read_u64, + .write = mem_cgroup_soft_limit_write, + .read_u64 = mem_cgroup_soft_limit_read, }, { .name = "failcnt", diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 6dc4888a90f3f..7138ebc6efc3e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2648,6 +2648,19 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, if (!gfpflags_allow_blocking(gfp_mask)) goto nomem; + /* + * OOM victim still needs to charge memory to exit. OOM reaper should + * help but it might fail on mmap_lock contention. If the victim is a + * large thread group then all exiting threads might compete on oom_lock + * just to learn that there is nothing really killable anymore. Bail + * out early and fail the charge to expedite their exit. They are + * considered fully reclaimed by the oom reaper and they shouldn't + * contribute further charges. + */ + if (tsk_is_oom_victim(current) && + mm_flags_test(MMF_OOM_SKIP, current->signal->oom_mm)) + goto nomem; + __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning); raised_max_event = true; @@ -2907,10 +2920,9 @@ struct mem_cgroup *mem_cgroup_from_virt(void *p) return folio_memcg_check(virt_to_folio(p)); } -static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg) +static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg, + int nid) { - int nid = numa_node_id(); - for (; memcg; memcg = parent_mem_cgroup(memcg)) { struct obj_cgroup *objcg = rcu_dereference(memcg->nodeinfo[nid]->objcg); @@ -2921,12 +2933,13 @@ static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg) return NULL; } -static inline struct obj_cgroup *get_obj_cgroup_from_memcg(struct mem_cgroup *memcg) +static inline struct obj_cgroup *get_obj_cgroup_from_memcg(struct mem_cgroup *memcg, + int nid) { struct obj_cgroup *objcg; rcu_read_lock(); - objcg = __get_obj_cgroup_from_memcg(memcg); + objcg = __get_obj_cgroup_from_memcg(memcg, nid); rcu_read_unlock(); return objcg; @@ -2970,7 +2983,7 @@ static struct obj_cgroup *current_objcg_update(void) rcu_read_lock(); memcg = mem_cgroup_from_task(current); - objcg = __get_obj_cgroup_from_memcg(memcg); + objcg = __get_obj_cgroup_from_memcg(memcg, numa_node_id()); rcu_read_unlock(); /* @@ -3094,7 +3107,7 @@ static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp, memcg = get_mem_cgroup_from_objcg(objcg); - ret = try_charge_memcg(memcg, gfp, nr_pages); + ret = try_charge(memcg, gfp, nr_pages); if (ret) goto out; @@ -4180,9 +4193,11 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) page_counter_init(&memcg->swap, &parent->swap, false); #ifdef CONFIG_MEMCG_V1 memcg->memory.track_failcnt = !memcg_on_dfl; + memcg->memsw.track_failcnt = !memcg_on_dfl; WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable)); page_counter_init(&memcg->kmem, &parent->kmem, false); page_counter_init(&memcg->tcpmem, &parent->tcpmem, false); + memcg->tcpmem.track_failcnt = !memcg_on_dfl; #endif } else { init_memcg_stats(); @@ -4438,6 +4453,7 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(MEMCG_KMEM); memcg->vmstats->state[index] += kmem; + memcg->vmstats->state_local[index] += kmem; if (parent) parent->vmstats->state_pending[index] += kmem; } @@ -4455,9 +4471,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(NR_SLAB_RECLAIMABLE_B); lstats->state[index] += slab; + lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; + memcg->vmstats->state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4466,9 +4484,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B); lstats->state[index] += slab; + lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; + memcg->vmstats->state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4788,6 +4808,9 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, unsigned long nr_pages = page_counter_read(&memcg->memory); unsigned long reclaimed; + if (high != READ_ONCE(memcg->memory.high)) + break; + if (nr_pages <= high) break; @@ -4839,6 +4862,9 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, for (;;) { unsigned long nr_pages = page_counter_read(&memcg->memory); + if (max != READ_ONCE(memcg->memory.max)) + break; + if (nr_pages <= max) break; @@ -5120,7 +5146,7 @@ static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg, int ret = 0; struct obj_cgroup *objcg; - objcg = get_obj_cgroup_from_memcg(memcg); + objcg = get_obj_cgroup_from_memcg(memcg, folio_nid(folio)); /* Do not account at the root objcg level. */ if (!obj_cgroup_is_root(objcg)) ret = try_charge_memcg(memcg, gfp, folio_nr_pages(folio)); @@ -5319,6 +5345,46 @@ void __mem_cgroup_uncharge_folios(struct folio_batch *folios) uncharge_batch(&ug); } +/* + * An LRU folio must hold the objcg belonging to its own node. + * + * memcg_reparent_objcgs() reparents a dying cgroup one node at a time: the + * folios on that node's LRU lists move to the parent and that node's objcg is + * redirected to the parent, atomically under the node's lru_lock. + * folio_lruvec_lock() relies on this to provide a stable folio<->lruvec + * binding. If a folio holds another node's objcg, its list membership and its + * lruvec resolution change in separate lock sections, and an LRU operation in + * between can re-add the folio to, and strand it on, the LRU list of a dead + * memcg. + * + * So when migration transfers the memcg state to a folio on another node, + * re-derive the objcg for the destination node. If the memcg is dying and the + * destination node has already been reparented, the lookup walks up to the + * nearest live ancestor - which is also where that node's LRU lists went. + * + * Returns the objcg to commit to @new, with a reference for the caller. + */ +static struct obj_cgroup *get_migration_objcg(struct folio *old, + struct folio *new) +{ + struct obj_cgroup *old_objcg, *new_objcg; + int new_nid = folio_nid(new); + + old_objcg = get_obj_cgroup_from_folio(old); + + if (folio_nid(old) == new_nid) + return old_objcg; + + rcu_read_lock(); + new_objcg = __get_obj_cgroup_from_memcg(obj_cgroup_memcg(old_objcg), + new_nid); + rcu_read_unlock(); + + obj_cgroup_put(old_objcg); + + return new_objcg; +} + /** * mem_cgroup_replace_folio - Charge a folio's replacement. * @old: Currently circulating folio. @@ -5347,21 +5413,28 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new) if (folio_memcg_charged(new)) return; - objcg = folio_objcg(old); - VM_WARN_ON_ONCE_FOLIO(!objcg, old); - if (!objcg) + VM_WARN_ON_ONCE_FOLIO(!folio_objcg(old), old); + if (!folio_objcg(old)) return; + objcg = get_migration_objcg(old, new); + rcu_read_lock(); memcg = obj_cgroup_memcg(objcg); - /* Force-charge the new page. The old one will be freed soon */ + + /* + * Force-charge the new page. The old one will be freed soon. + * + * The rootness of the committed objcg decides whether the final + * uncharge of @new goes through the page counters (see + * uncharge_folio()); charge them only if the uncharge will. + */ if (!obj_cgroup_is_root(objcg)) { page_counter_charge(&memcg->memory, nr_pages); if (do_memsw_account()) page_counter_charge(&memcg->memsw, nr_pages); } - obj_cgroup_get(objcg); commit_charge(new, objcg); memcg1_commit_charge(new, memcg); rcu_read_unlock(); @@ -5373,14 +5446,15 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new) * @new: Replacement folio. * * Transfer the memcg data from the old folio to the new folio for migration. - * The old folio's data info will be cleared. Note that the memory counters - * will remain unchanged throughout the process. + * The old folio's data info will be cleared. The memory counters remain + * unchanged, unless the charge moves out of a fully reparented ancestry + * and has to be settled (see below). * * Both folios must be locked, @new->mapping must be set up. */ void mem_cgroup_migrate(struct folio *old, struct folio *new) { - struct obj_cgroup *objcg; + struct obj_cgroup *objcg, *new_objcg; VM_BUG_ON_FOLIO(!folio_test_locked(old), old); VM_BUG_ON_FOLIO(!folio_test_locked(new), new); @@ -5401,12 +5475,30 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new) if (!objcg) return; - /* Transfer the charge and the objcg ref */ - commit_charge(new, objcg); + new_objcg = get_migration_objcg(old, new); + + /* + * @old was charged through a non-root objcg, so its charge is in the + * page counters. If the re-derivation walked up to the root objcg - + * @old's entire ancestry is dying and already reparented - the final + * uncharge of @new will skip the page counters (see uncharge_folio()). + * Settle them now: this is @old's eventual uncharge, moved up to the + * point where its charge record ends. + */ + if (obj_cgroup_is_root(new_objcg) && !obj_cgroup_is_root(objcg)) { + rcu_read_lock(); + memcg_uncharge(obj_cgroup_memcg(objcg), folio_nr_pages(old)); + rcu_read_unlock(); + } + + commit_charge(new, new_objcg); /* Warning should never happen, so don't worry about refcount non-0 */ WARN_ON_ONCE(folio_unqueue_deferred_split(old)); old->memcg_data = 0; + + /* @new holds its own reference now, drop @old's */ + obj_cgroup_put(objcg); } DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 7ac19fab22632..8b137328dcf01 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1263,7 +1263,8 @@ static pg_data_t *hotadd_init_pgdat(int nid) pgdat = NODE_DATA(nid); /* init node's zones as empty zones, we don't have any present pages.*/ - free_area_init_core_hotplug(pgdat); + if (free_area_init_core_hotplug(pgdat)) + return NULL; /* * The node we allocated has no zone fallback lists. For avoiding diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 36699fabd3c22..9d7cb63d32732 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -653,12 +653,14 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) { struct folio *folio; struct queue_pages *qp = walk->private; + pmd_t pmdval = pmdp_get(pmd); - if (unlikely(pmd_is_migration_entry(*pmd))) { - qp->nr_failed++; + if (unlikely(!pmd_present(pmdval))) { + if (pmd_is_migration_entry(pmdval)) + qp->nr_failed++; return; } - folio = pmd_folio(*pmd); + folio = pmd_folio(pmdval); if (is_huge_zero_folio(folio)) { walk->action = ACTION_CONTINUE; return; @@ -2057,24 +2059,15 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, bool vma_policy_mof(struct vm_area_struct *vma) { struct mempolicy *pol; + pgoff_t ilx; + bool mof; - if (vma->vm_ops && vma->vm_ops->get_policy) { - bool ret = false; - pgoff_t ilx; /* ignored here */ - - pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx); - if (pol && (pol->flags & MPOL_F_MOF)) - ret = true; - mpol_cond_put(pol); - - return ret; - } - - pol = vma->vm_policy; + pol = __get_vma_policy(vma, vma->vm_start, &ilx); if (!pol) pol = get_task_policy(current); - - return pol->flags & MPOL_F_MOF; + mof = pol->flags & MPOL_F_MOF; + mpol_cond_put(pol); + return mof; } bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone) @@ -2683,7 +2676,7 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, prev_node = node; /* create a local copy of node weights to operate on outside rcu */ - weights = kzalloc(nr_node_ids, GFP_KERNEL); + weights = kmalloc(nr_node_ids, gfp & GFP_RECLAIM_MASK); if (!weights) return total_allocated; diff --git a/mm/migrate.c b/mm/migrate.c index dd15a84b2a52a..35381ce36a8f5 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -371,7 +371,11 @@ static bool remove_migration_pte(struct folio *folio, continue; } #endif - old_pte = ptep_get(pvmw.pte); + if (folio_test_hugetlb(folio)) + old_pte = huge_ptep_get(vma->vm_mm, pvmw.address, + pvmw.pte); + else + old_pte = ptep_get(pvmw.pte); if (rmap_walk_arg->map_unused_to_zeropage && try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx)) continue; @@ -1830,7 +1834,7 @@ static int migrate_pages_batch(struct list_head *from, is_thp = folio_test_pmd_mappable(folio); nr_pages = folio_nr_pages(folio); - cond_resched(); + cond_resched_tasks_rcu_qs(); /* * The rare folio on the deferred split list should diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 908d2d4ec43ac..52ba68f2c9151 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -1183,6 +1183,13 @@ static void __migrate_device_pages(unsigned long *src_pfns, MIGRATE_PFN_COMPOUND); goto next; } + + /* + * reset nr so that only first after-split folio + * is processed below + */ + VM_WARN_ON_ONCE(folio_test_large(folio)); + nr = 1; } else if ((src_pfns[i] & MIGRATE_PFN_MIGRATE) && (dst_pfns[i] & MIGRATE_PFN_COMPOUND) && !(src_pfns[i] & MIGRATE_PFN_COMPOUND)) { @@ -1222,6 +1229,12 @@ static void __migrate_device_pages(unsigned long *src_pfns, folio = page_folio(migrate_pfn_to_page(src_pfns[i+j])); newfolio = page_folio(migrate_pfn_to_page(dst_pfns[i+j])); + /* + * folio_free_swap() removed the folio from the swap + * cache. Refresh the saved mapping before migration. + */ + mapping = folio_mapping(folio); + r = folio_migrate_mapping(mapping, newfolio, folio, extra_cnt); if (r) src_pfns[i+j] &= ~MIGRATE_PFN_MIGRATE; @@ -1400,6 +1413,15 @@ int migrate_device_range(unsigned long *src_pfns, unsigned long start, src_pfns[i] = migrate_device_pfn_lock(pfn); nr = folio_nr_pages(folio); + if (nr > npages - i) { + if (src_pfns[i] & MIGRATE_PFN_MIGRATE) { + folio_unlock(folio); + folio_put(folio); + } + memset(&src_pfns[i], 0, + (npages - i) * sizeof(*src_pfns)); + break; + } if (nr > 1) { src_pfns[i] |= MIGRATE_PFN_COMPOUND; for (j = 1; j < nr; j++) @@ -1434,6 +1456,15 @@ int migrate_device_pfns(unsigned long *src_pfns, unsigned long npages) src_pfns[i] = migrate_device_pfn_lock(src_pfns[i]); nr = folio_nr_pages(folio); + if (nr > npages - i) { + if (src_pfns[i] & MIGRATE_PFN_MIGRATE) { + folio_unlock(folio); + folio_put(folio); + } + memset(&src_pfns[i], 0, + (npages - i) * sizeof(*src_pfns)); + break; + } if (nr > 1) { src_pfns[i] |= MIGRATE_PFN_COMPOUND; for (j = 1; j < nr; j++) diff --git a/mm/mlock.c b/mm/mlock.c index 8c227fefa2df0..2d8f26a1cf263 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -141,7 +141,7 @@ static struct lruvec *__munlock_folio(struct folio *folio, struct lruvec *lruvec munlock: if (folio_test_clear_mlocked(folio)) { - __zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages); + zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages); if (isolated || !folio_test_unevictable(folio)) __count_vm_events(UNEVICTABLE_PGMUNLOCKED, nr_pages); else diff --git a/mm/mm_init.c b/mm/mm_init.c index 498d62c4ece39..d50a54dbb1064 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1338,7 +1338,7 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, unsigned long node_start_pfn, unsigned long node_end_pfn) { - unsigned long realtotalpages = 0, totalpages = 0; + unsigned long realtotalpages = 0; enum zone_type i; for (i = 0; i < MAX_NR_ZONES; i++) { @@ -1368,11 +1368,10 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, zone->present_early_pages = real_size; #endif - totalpages += spanned; realtotalpages += real_size; } - pgdat->node_spanned_pages = totalpages; + pgdat->node_spanned_pages = node_end_pfn - node_start_pfn; pgdat->node_present_pages = realtotalpages; pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); } @@ -1536,7 +1535,7 @@ void __init set_pageblock_order(void) * NOTE: this function is only called during memory hotplug */ #ifdef CONFIG_MEMORY_HOTPLUG -void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat) { int nid = pgdat->node_id; enum zone_type z; @@ -1544,8 +1543,14 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) pgdat_init_internals(pgdat); - if (pgdat->per_cpu_nodestats == &boot_nodestats) - pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); + if (pgdat->per_cpu_nodestats == &boot_nodestats) { + struct per_cpu_nodestat __percpu *p; + + p = alloc_percpu(struct per_cpu_nodestat); + if (!p) + return -ENOMEM; + pgdat->per_cpu_nodestats = p; + } /* * Reset the nr_zones, order and highest_zoneidx before reuse. @@ -1583,6 +1588,8 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) zone->present_pages = 0; zone_init_internals(zone, z, nid, 0); } + + return 0; } #endif @@ -2214,10 +2221,13 @@ bool __init deferred_grow_zone(struct zone *zone, unsigned int order) } /* - * There were no pages to initialize and free which means the zone's - * memory map is completely initialized. + * The loop only tests spfn before entering an iteration, so on exit it + * may point up to a section past the end of the zone. When it does, + * the rest of the zone has already been handed to + * deferred_init_memmap_chunk() and nothing is left to initialize. */ - pgdat->first_deferred_pfn = nr_pages ? spfn : ULONG_MAX; + pgdat->first_deferred_pfn = + spfn < zone_end_pfn(zone) ? spfn : ULONG_MAX; pgdat_resize_unlock(pgdat, &flags); @@ -2330,6 +2340,7 @@ void __init page_alloc_init_late(void) /* Reinit limits that are based on free pages after the kernel is up */ files_maxfiles_init(); #endif + hugetlb_bootmem_struct_page_init(); /* Accounting of total+free memory is stable at this point. */ mem_init_print_info(); diff --git a/mm/mremap.c b/mm/mremap.c index e9c8b1d05832b..ecd9262095605 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -1321,24 +1321,34 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, { unsigned long start = vrm->addr; unsigned long end = vrm->addr + vrm->old_len; - unsigned long old_start = vrm->vma->vm_start; - unsigned long old_end = vrm->vma->vm_end; + struct vm_area_struct *vma = vrm->vma; + unsigned long old_start = vma->vm_start; + unsigned long old_end = vma->vm_end; /* We always clear VM_LOCKED[ONFAULT] on the old VMA. */ - vm_flags_clear(vrm->vma, VM_LOCKED_MASK); + vm_flags_clear(vma, VM_LOCKED_MASK); /* * anon_vma links of the old vma is no longer needed after its page * table has been moved. */ - if (new_vma != vrm->vma && start == old_start && end == old_end) - unlink_anon_vmas(vrm->vma); + if (new_vma != vma && start == old_start && end == old_end) { + const pgoff_t pgoff_unfaulted = vma->vm_start >> PAGE_SHIFT; - /* Because we won't unmap we don't need to touch locked_vm. */ + unlink_anon_vmas(vma); + /* + * The VMA is now unfaulted and it is an invariant that + * unfaulted anonymous VMAs have page offset equal to + * vma->vm_start >> PAGE_SHIFT. + */ + if (vma_is_anonymous(vma) && !vma->vm_file) + vma->vm_pgoff = pgoff_unfaulted; + } } static unsigned long move_vma(struct vma_remap_struct *vrm) { + const bool is_dontunmap = vrm->flags & MREMAP_DONTUNMAP; struct mm_struct *mm = current->mm; struct vm_area_struct *new_vma; unsigned long hiwater_vm; @@ -1379,10 +1389,10 @@ static unsigned long move_vma(struct vma_remap_struct *vrm) */ hiwater_vm = mm->hiwater_vm; - vrm_stat_account(vrm, vrm->new_len); - if (unlikely(!err && (vrm->flags & MREMAP_DONTUNMAP))) + if (unlikely(is_dontunmap && !err)) dontunmap_complete(vrm, new_vma); - else + vrm_stat_account(vrm, vrm->new_len); + if (!is_dontunmap || err) unmap_source_vma(vrm); mm->hiwater_vm = hiwater_vm; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ee902a468c2f5..73645d2eda6f4 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -721,14 +721,14 @@ static inline struct capture_control *task_capc(struct zone *zone) return unlikely(capc) && !(current->flags & PF_KTHREAD) && !capc->page && - capc->cc->zone == zone ? capc : NULL; + capc->zone == zone ? capc : NULL; } static inline bool compaction_capture(struct capture_control *capc, struct page *page, int order, int migratetype) { - if (!capc || order != capc->cc->order) + if (!capc || order != capc->order) return false; /* Do not accidentally pollute CMA or isolated regions*/ @@ -744,12 +744,12 @@ compaction_capture(struct capture_control *capc, struct page *page, * have trouble finding a high-order free page. */ if (order < pageblock_order && migratetype == MIGRATE_MOVABLE && - capc->cc->migratetype != MIGRATE_MOVABLE) + capc->migratetype != MIGRATE_MOVABLE) return false; - if (migratetype != capc->cc->migratetype) - trace_mm_page_alloc_extfrag(page, capc->cc->order, order, - capc->cc->migratetype, migratetype); + if (migratetype != capc->migratetype) + trace_mm_page_alloc_extfrag(page, capc->order, order, + capc->migratetype, migratetype); capc->page = page; return true; @@ -2967,8 +2967,10 @@ static void __free_frozen_pages(struct page *page, unsigned int order, migratetype = MIGRATE_MOVABLE; } - if (unlikely((fpi_flags & FPI_TRYLOCK) && IS_ENABLED(CONFIG_PREEMPT_RT) - && (in_nmi() || in_hardirq()))) { + if (unlikely((fpi_flags & FPI_TRYLOCK) && + ((IS_ENABLED(CONFIG_PREEMPT_RT) && + (in_nmi() || in_hardirq())) || + (!IS_ENABLED(CONFIG_SMP) && in_nmi())))) { add_page_to_zone_llist(zone, page, order); return; } @@ -4146,18 +4148,67 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; unsigned int noreclaim_flag; + struct capture_control capc = { + .zone = NULL, + .migratetype = ac->migratetype, + .order = order, + .page = NULL, + }; + int compact_order = order; - if (!order) + /* + * If fallbacks are not permitted (defrag_mode), we either + * need to reclaim space in a block of matching type, or clear + * out an entire block to allow __rmqueue_claim() to convert. + * + * Reclaim by itself is primarily freeing space in movable + * blocks, since that's where the LRU pages live. So this + * works for movable requests, but not for others. + * + * For those, promote the order to help make blocks, instead + * of spinning in reclaim alone unproductively. + */ + if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE) + compact_order = max(order, pageblock_order); + + if (!compact_order) return NULL; psi_memstall_enter(&pflags); delayacct_compact_start(); + fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); - *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, - prio, &page); + /* + * Make sure the structs are really initialized before we expose the + * capture control, in case we are interrupted and the interrupt handler + * frees a page. + */ + barrier(); + WRITE_ONCE(current->capture_control, &capc); + + *compact_result = try_to_compact_pages(gfp_mask, compact_order, + alloc_flags, ac, prio, &capc); + + /* + * Make sure we hide capture control first before we read the captured + * page pointer, otherwise an interrupt could free and capture a page + * and we would leak it. + */ + WRITE_ONCE(current->capture_control, NULL); + page = READ_ONCE(capc.page); + + /* + * Technically, it is also possible that compaction is skipped but + * the page is still captured out of luck(IRQ came and freed the page). + * Returning COMPACT_SUCCESS in such cases helps in properly accounting + * the COMPACT[STALL|FAIL] when compaction is skipped. + */ + if (page) + *compact_result = COMPACT_SUCCESS; memalloc_noreclaim_restore(noreclaim_flag); + fs_reclaim_release(gfp_mask); psi_memstall_leave(&pflags); delayacct_compact_end(); @@ -4182,7 +4233,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, struct zone *zone = page_zone(page); zone->compact_blockskip_flush = false; - compaction_defer_reset(zone, order, true); + compaction_defer_reset(zone, compact_order, true); count_vm_event(COMPACTSUCCESS); return page; } @@ -4422,9 +4473,14 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; bool drained = false; + int reclaim_order = order; + + /* Match the slowpath compaction promotion in __alloc_pages_direct_compact */ + if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE) + reclaim_order = max(order, pageblock_order); psi_memstall_enter(&pflags); - *did_some_progress = __perform_reclaim(gfp_mask, order, ac); + *did_some_progress = __perform_reclaim(gfp_mask, reclaim_order, ac); if (unlikely(!(*did_some_progress))) goto out; diff --git a/mm/page_owner.c b/mm/page_owner.c index 2dddcb6510aa1..d8d264eb01b01 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -513,6 +513,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, { #ifdef CONFIG_MEMCG unsigned long memcg_data; + struct obj_cgroup *objcg; struct mem_cgroup *memcg; bool online; char name[80]; @@ -522,11 +523,14 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, if (!memcg_data || PageTail(page)) goto out_unlock; - if (memcg_data & MEMCG_DATA_OBJEXTS) + if (memcg_data & MEMCG_DATA_OBJEXTS) { ret += scnprintf(kbuf + ret, count - ret, "Slab cache page\n"); + goto out_unlock; + } - memcg = page_memcg_check(page); + objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); + memcg = objcg ? obj_cgroup_memcg(objcg) : NULL; if (!memcg) goto out_unlock; @@ -534,7 +538,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, cgroup_name(memcg->css.cgroup, name, sizeof(name)); ret += scnprintf(kbuf + ret, count - ret, "Charged %sto %smemcg %s\n", - PageMemcgKmem(page) ? "(via objcg) " : "", + (memcg_data & MEMCG_DATA_KMEM) ? "(via objcg) " : "", online ? "" : "offline ", name); out_unlock: diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index bac2eb5de63d6..d7670ba4147bf 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -107,7 +107,13 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw, pmd_t *pmdvalp, static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) { unsigned long pfn; - pte_t ptent = ptep_get(pvmw->pte); + pte_t ptent; + + if (is_vm_hugetlb_page(pvmw->vma)) + ptent = huge_ptep_get(pvmw->vma->vm_mm, pvmw->address, + pvmw->pte); + else + ptent = ptep_get(pvmw->pte); if (pvmw->flags & PVMW_MIGRATION) { const softleaf_t entry = softleaf_from_pte(ptent); diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 5d87c632a2550..d3bfece319336 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -126,6 +126,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, pmd = pmd_offset(pud, addr); do { again: + walk->action = ACTION_SUBTREE; next = pmd_addr_end(addr, end); if (pmd_none(*pmd)) { if (has_install) @@ -138,8 +139,6 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, continue; } - walk->action = ACTION_SUBTREE; - /* * This implies that each ->pmd_entry() handler * needs to know about pmd_trans_huge() pmds @@ -196,6 +195,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, pud = pud_offset(p4d, addr); do { again: + walk->action = ACTION_SUBTREE; next = pud_addr_end(addr, end); if (pud_none(*pud)) { if (has_install) @@ -208,8 +208,6 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, continue; } - walk->action = ACTION_SUBTREE; - if (ops->pud_entry) err = ops->pud_entry(pud, addr, next, walk); if (err) diff --git a/mm/rmap.c b/mm/rmap.c index 1c77d5dc06e9f..acdadb470279a 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -209,7 +209,11 @@ int __anon_vma_prepare(struct vm_area_struct *vma) /* page_table_lock to protect against threads */ spin_lock(&mm->page_table_lock); if (likely(!vma->anon_vma)) { - vma->anon_vma = anon_vma; + /* + * Make anon_vma fields visible before anon_vma is published. + * Paired with an address dependency in reusable_anon_vma(). + */ + smp_store_release(&vma->anon_vma, anon_vma); anon_vma_chain_assign(vma, avc, anon_vma); anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); anon_vma->num_active_vmas++; @@ -2095,14 +2099,19 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); - /* - * Handle PFN swap PTEs, such as device-exclusive ones, that - * actually map pages. - */ - pteval = ptep_get(pvmw.pte); + address = pvmw.address; + if (folio_test_hugetlb(folio)) { + pteval = huge_ptep_get(mm, address, pvmw.pte); + } else { + pteval = ptep_get(pvmw.pte); + } if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { + /* + * Handle PFN swap PTEs, such as device-exclusive ones, + * that actually map pages. + */ const softleaf_t entry = softleaf_from_pte(pteval); pfn = softleaf_to_pfn(entry); @@ -2110,7 +2119,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); - address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); @@ -2501,14 +2509,18 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); - /* - * Handle PFN swap PTEs, such as device-exclusive ones, that - * actually map pages. - */ - pteval = ptep_get(pvmw.pte); + address = pvmw.address; + if (folio_test_hugetlb(folio)) + pteval = huge_ptep_get(mm, address, pvmw.pte); + else + pteval = ptep_get(pvmw.pte); if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { + /* + * Handle PFN swap PTEs, such as device-exclusive ones, + * that actually map pages. + */ const softleaf_t entry = softleaf_from_pte(pteval); pfn = softleaf_to_pfn(entry); @@ -2516,7 +2528,6 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); - address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); diff --git a/mm/secretmem.c b/mm/secretmem.c index 4877c262cb1f6..5411324fec1ef 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include @@ -47,10 +49,69 @@ bool secretmem_active(void) return !!atomic_read(&secretmem_users); } +struct secretmem_inode_state { + struct user_struct *user; + atomic_long_t nr_pages_accounted; +}; + +static bool __secretmem_account_pages(struct user_struct *user, + unsigned long nr_pages) +{ + unsigned long page_limit, cur_pages, new_pages; + + if (!nr_pages) + return true; + + page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + + cur_pages = atomic_long_read(&user->locked_vm); + do { + new_pages = cur_pages + nr_pages; + if (new_pages > page_limit) + return false; + } while (!atomic_long_try_cmpxchg(&user->locked_vm, + &cur_pages, new_pages)); + return true; +} + +static bool secretmem_account_folio(struct secretmem_inode_state *state, + const struct folio *folio) +{ + const unsigned long nr_pages = folio_nr_pages(folio); + + if (!__secretmem_account_pages(state->user, nr_pages)) + return false; + + atomic_long_add(nr_pages, &state->nr_pages_accounted); + return true; +} + +static void __secretmem_unaccount_pages(struct secretmem_inode_state *state, + unsigned long nr_pages) +{ + atomic_long_sub(nr_pages, &state->user->locked_vm); + atomic_long_sub(nr_pages, &state->nr_pages_accounted); +} + +static void secretmem_unaccount_folio(struct secretmem_inode_state *state, + struct folio *folio) +{ + __secretmem_unaccount_pages(state, folio_nr_pages(folio)); +} + +static void secretmem_unaccount_all_folios(struct secretmem_inode_state *state) +{ + const unsigned long nr_pages_accounted = + atomic_long_read(&state->nr_pages_accounted); + + __secretmem_unaccount_pages(state, nr_pages_accounted); +} + static vm_fault_t secretmem_fault(struct vm_fault *vmf) { struct address_space *mapping = vmf->vma->vm_file->f_mapping; struct inode *inode = file_inode(vmf->vma->vm_file); + struct secretmem_inode_state *state = inode->i_private; pgoff_t offset = vmf->pgoff; gfp_t gfp = vmf->gfp_mask; unsigned long addr; @@ -72,8 +133,15 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf) goto out; } + if (!secretmem_account_folio(state, folio)) { + folio_put(folio); + ret = VM_FAULT_SIGBUS; + goto out; + } + err = set_direct_map_invalid_noflush(folio_page(folio, 0)); if (err) { + secretmem_unaccount_folio(state, folio); folio_put(folio); ret = vmf_error(err); goto out; @@ -82,6 +150,7 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf) __folio_mark_uptodate(folio); err = filemap_add_folio(mapping, folio, offset, gfp); if (unlikely(err)) { + secretmem_unaccount_folio(state, folio); /* * If a split of large page was required, it * already happened when we marked the page invalid @@ -112,22 +181,30 @@ static const struct vm_operations_struct secretmem_vm_ops = { .fault = secretmem_fault, }; +static void secretmem_destroy_inode_priv(struct inode *inode) +{ + struct secretmem_inode_state *state = inode->i_private; + + secretmem_unaccount_all_folios(state); + free_uid(state->user); + kfree(state); + inode->i_private = NULL; +} + static int secretmem_release(struct inode *inode, struct file *file) { atomic_dec(&secretmem_users); + secretmem_destroy_inode_priv(inode); + return 0; } static int secretmem_mmap_prepare(struct vm_area_desc *desc) { - const unsigned long len = vma_desc_size(desc); - if (!vma_desc_test_any(desc, VMA_SHARED_BIT, VMA_MAYSHARE_BIT)) return -EINVAL; - vma_desc_set_flags(desc, VMA_LOCKED_BIT, VMA_DONTDUMP_BIT); - if (!mlock_future_ok(desc->mm, /*is_vma_locked=*/ true, len)) - return -EAGAIN; + vma_desc_set_flags(desc, VMA_DONTDUMP_BIT); desc->vm_ops = &secretmem_vm_ops; return 0; @@ -187,20 +264,40 @@ static const struct inode_operations secretmem_iops = { static struct vfsmount *secretmem_mnt; +static int secretmem_init_inode_priv(struct inode *inode) +{ + struct secretmem_inode_state *state; + + state = kzalloc_obj(*state); + if (!state) + return -ENOMEM; + + state->user = get_uid(current_user()); + inode->i_private = state; + return 0; +} + static struct file *secretmem_file_create(unsigned long flags) { struct file *file; struct inode *inode; const char *anon_name = "[secretmem]"; + int err; inode = anon_inode_make_secure_inode(secretmem_mnt->mnt_sb, anon_name, NULL); if (IS_ERR(inode)) return ERR_CAST(inode); + err = secretmem_init_inode_priv(inode); + if (err) + goto err_free_inode; + file = alloc_file_pseudo(inode, secretmem_mnt, "secretmem", O_RDWR | O_LARGEFILE, &secretmem_fops); - if (IS_ERR(file)) - goto err_free_inode; + if (IS_ERR(file)) { + err = PTR_ERR(file); + goto err_free_priv; + } mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); mapping_set_unevictable(inode->i_mapping); @@ -215,10 +312,11 @@ static struct file *secretmem_file_create(unsigned long flags) atomic_inc(&secretmem_users); return file; - +err_free_priv: + secretmem_destroy_inode_priv(inode); err_free_inode: iput(inode); - return file; + return ERR_PTR(err); } SYSCALL_DEFINE1(memfd_secret, unsigned int, flags) diff --git a/mm/shrinker.c b/mm/shrinker.c index a70aab124a0e7..7ec2a9704f6f2 100644 --- a/mm/shrinker.c +++ b/mm/shrinker.c @@ -227,6 +227,8 @@ static int shrinker_memcg_alloc(struct shrinker *shrinker) { int id; + shrinker->id = -1; + if (mem_cgroup_disabled()) return -ENOSYS; if (mem_cgroup_kmem_disabled() && !(shrinker->flags & SHRINKER_NONSLAB)) diff --git a/mm/slub.c b/mm/slub.c index 0337e60db5ace..3791d69ba6f90 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -333,14 +333,20 @@ enum track_item { TRACK_ALLOC, TRACK_FREE }; #ifdef SLAB_SUPPORTS_SYSFS static int sysfs_slab_add(struct kmem_cache *); +static int __init slab_kset_init(void); +static void __init slab_sysfs_process_aliases(void); #else static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } +static inline int slab_kset_init(void) { return 0; } +static inline void slab_sysfs_process_aliases(void) { } #endif #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) static void debugfs_slab_add(struct kmem_cache *); +static void __init slab_debugfs_root_init(void); #else static inline void debugfs_slab_add(struct kmem_cache *s) { } +static inline void slab_debugfs_root_init(void) { } #endif enum add_mode { @@ -5147,7 +5153,7 @@ void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp, * simply flush and free it. */ if (!barn || data_race(barn->nr_full) >= MAX_FULL_SHEAVES || - refill_sheaf(s, sheaf, gfp)) { + refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) { sheaf_flush_unused(s, sheaf); free_empty_sheaf(s, sheaf); return; @@ -5606,10 +5612,12 @@ static noinline void free_to_partial_list( * * Fail if the slab isn't full anymore due to a concurrent free. */ -static bool __slab_try_return_freelist(struct kmem_cache *s, struct slab *slab, - void *head, int cnt) +static bool __slab_try_return_freelist(struct kmem_cache *s, + struct kmem_cache_node *n, + struct slab *slab, void *head, int cnt) { struct freelist_counters old, new; + unsigned long flags; old.freelist = slab->freelist; old.counters = slab->counters; @@ -5621,9 +5629,15 @@ static bool __slab_try_return_freelist(struct kmem_cache *s, struct slab *slab, new.counters = old.counters; new.inuse -= cnt; - if (!slab_update_freelist(s, slab, &old, &new, "__slab_try_return_freelist")) + spin_lock_irqsave(&n->list_lock, flags); + + if (!slab_update_freelist(s, slab, &old, &new, "__slab_try_return_freelist")) { + spin_unlock_irqrestore(&n->list_lock, flags); return false; + } + add_partial(n, slab, ADD_TO_TAIL); + spin_unlock_irqrestore(&n->list_lock, flags); return true; } @@ -7196,10 +7210,8 @@ __refill_objects_node(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int mi void *head = object; void *tail; - if (__slab_try_return_freelist(s, slab, head, count)) { - list_add(&slab->slab_list, &pc.slabs); + if (__slab_try_return_freelist(s, n, slab, head, count)) break; - } do { tail = object; @@ -7212,7 +7224,7 @@ __refill_objects_node(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int mi break; } - if (!list_empty(&pc.slabs)) { + if (unlikely(!list_empty(&pc.slabs))) { spin_lock_irqsave(&n->list_lock, flags); list_for_each_entry(slab, &pc.slabs, slab_list) @@ -9734,28 +9746,20 @@ int sysfs_slab_alias(struct kmem_cache *s, const char *name) return 0; } -static int __init slab_sysfs_init(void) +static int __init slab_kset_init(void) { - struct kmem_cache *s; - int err; - - mutex_lock(&slab_mutex); - slab_kset = kset_create_and_add("slab", NULL, kernel_kobj); if (!slab_kset) { - mutex_unlock(&slab_mutex); pr_err("Cannot register slab subsystem.\n"); return -ENOMEM; } - slab_state = FULL; + return 0; +} - list_for_each_entry(s, &slab_caches, list) { - err = sysfs_slab_add(s); - if (err) - pr_err("SLUB: Unable to add boot slab %s to sysfs\n", - s->name); - } +static void __init slab_sysfs_process_aliases(void) +{ + int err; while (alias_list) { struct saved_alias *al = alias_list; @@ -9767,12 +9771,41 @@ static int __init slab_sysfs_init(void) al->name); kfree(al); } +} +#endif /* SLAB_SUPPORTS_SYSFS */ + +#if defined(SLAB_SUPPORTS_SYSFS) || \ + (defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)) +static int __init slab_late_init(void) +{ + struct kmem_cache *s; + int err; + + mutex_lock(&slab_mutex); + + err = slab_kset_init(); + if (err) + goto out; + + slab_debugfs_root_init(); + slab_state = FULL; + + list_for_each_entry(s, &slab_caches, list) { + if (sysfs_slab_add(s)) + pr_err("SLUB: Unable to add boot slab %s to sysfs\n", + s->name); + + if (s->flags & SLAB_STORE_USER) + debugfs_slab_add(s); + } + slab_sysfs_process_aliases(); +out: mutex_unlock(&slab_mutex); - return 0; + return err; } -late_initcall(slab_sysfs_init); -#endif /* SLAB_SUPPORTS_SYSFS */ +late_initcall(slab_late_init); +#endif #if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS) static int slab_debugfs_show(struct seq_file *seq, void *v) @@ -9964,23 +9997,16 @@ static void debugfs_slab_add(struct kmem_cache *s) void debugfs_slab_release(struct kmem_cache *s) { + if (unlikely(!slab_debugfs_root)) + return; + debugfs_lookup_and_remove(s->name, slab_debugfs_root); } -static int __init slab_debugfs_init(void) +static void __init slab_debugfs_root_init(void) { - struct kmem_cache *s; - slab_debugfs_root = debugfs_create_dir("slab", NULL); - - list_for_each_entry(s, &slab_caches, list) - if (s->flags & SLAB_STORE_USER) - debugfs_slab_add(s); - - return 0; - } -__initcall(slab_debugfs_init); #endif /* * The /proc/slabinfo ABI diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index ebd3ac997f645..50df73b8f7478 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -342,8 +342,8 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone * * * Any initialization done here will be overwritten by memmap_init(). * - * hugetlb_vmemmap_init() will take care of initialization after - * memmap_init(). + * hugetlb_bootmem_struct_page_init() will take care of initialization + * after memmap_init(). */ p = vmemmap_alloc_block_zero(PAGE_SIZE, node); diff --git a/mm/swap.c b/mm/swap.c index 588f50d8f1a8c..1a87e56bdcc52 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -1153,7 +1153,16 @@ static void lruvec_reparent_lru(struct lruvec *child_lruvec, for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) { unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid); + if (!size) + continue; + + /* + * The folios are accounted to the parent from now on, so the + * size has to be moved, not just copied. Leaving it behind + * makes the dying child describe folios it no longer owns. + */ mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size); + mem_cgroup_update_lru_size(child_lruvec, lru, zid, -(long)size); } } diff --git a/mm/swapfile.c b/mm/swapfile.c index 78b49b0658add..623b30e9c2dfb 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -156,7 +156,7 @@ static struct swap_info_struct *swap_entry_to_info(swp_entry_t entry) * This bit will be set if the device is not on the plist and not * usable, will be cleared if the device is on the plist. */ -#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_t) - 2)) +#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_long_t) - 2)) #define SWAP_USAGE_COUNTER_MASK (~SWAP_USAGE_OFFLIST_BIT) static long swap_usage_in_pages(struct swap_info_struct *si) { @@ -1893,11 +1893,11 @@ struct swap_info_struct *get_swap_device(swp_entry_t entry) return si; bad_nofile: - pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val); + pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_file, entry.val); out: return NULL; put_out: - pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val); + pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_offset, entry.val); percpu_ref_put(&si->users); return NULL; } @@ -2196,7 +2196,14 @@ void swap_free_hibernation_slot(swp_entry_t entry) ci = swap_cluster_lock(si, offset); __swap_cluster_put_entry(ci, offset % SWAPFILE_CLUSTER); - __swap_cluster_free_entries(si, ci, offset % SWAPFILE_CLUSTER, 1); + /* + * A slot with a folio in the swap cache is freed when the folio + * leaves the cache, the same rule swap_put_entries_cluster() follows. + * Readahead can put a folio here, and freeing the slot now would + * leave that folio with no entry behind it. + */ + if (!swp_tb_is_folio(__swap_table_get(ci, offset % SWAPFILE_CLUSTER))) + __swap_cluster_free_entries(si, ci, offset % SWAPFILE_CLUSTER, 1); swap_cluster_unlock(ci); /* In theory readahead might add it to the swap cache by accident */ @@ -3650,6 +3657,13 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) error = -EBUSY; goto bad_swap_unlock_inode; } + if (IS_ENCRYPTED(inode)) { + pr_warn_once( + "Filesystem-level encrypted swapfile '%s' is unsupported. Create a loop device over it, or use dm-crypt\n", + name->name); + error = -EINVAL; + goto bad_swap_unlock_inode; + } /* * The swap subsystem needs a major overhaul to support this. @@ -3857,7 +3871,7 @@ int swap_dup_entry_direct(swp_entry_t entry) si = swap_entry_to_info(entry); if (WARN_ON_ONCE(!si)) { - pr_err("%s%08lx\n", Bad_file, entry.val); + pr_err_ratelimited("%s%08lx\n", Bad_file, entry.val); return -EINVAL; } diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index c3adedaaf7d54..ec5d511bd1791 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -2085,8 +2085,10 @@ static ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, } if (err) { - if (err == -EAGAIN) + if (err == -EAGAIN) { + err = 0; continue; + } break; } diff --git a/mm/vma.c b/mm/vma.c index 9eea2850818a8..638ec4793798e 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -1995,6 +1995,13 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * * acceptable for merging, so we can do all of this optimistically. But * we do that READ_ONCE() to make sure that we never re-load the pointer. * + * The READ_ONCE() establishes an address dependency between anon_vma and + * any access to its fields, which pairs with the assignment to + * vma->anon_vma performed with release semantics in __anon_vma_prepare(). + * + * This is especially important as anon_vma's are SLAB_TYPESAFE_BY_RCU so + * accessing an uninitialised anon_vma's fields may result in a UAF. + * * IOW: that the "list_is_singular()" test on the anon_vma_chain only * matters for the 'stable anon_vma' case (ie the thing we want to avoid * is to return an anon_vma that is "complex" due to having gone through @@ -2009,6 +2016,7 @@ static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *b) { if (anon_vma_compatible(a, b)) { + /* Paired with a memory barrier in __anon_vma_prepare(). */ struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); if (anon_vma && list_is_singular(&old->anon_vma_chain)) @@ -2754,10 +2762,12 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, map.check_ksm_early = can_set_ksm_flags_early(&map); error = __mmap_setup(&map, &desc, uf); - if (!error && have_mmap_prepare) - error = call_mmap_prepare(&map, &desc); if (error) goto abort_munmap; + if (have_mmap_prepare) + error = call_mmap_prepare(&map, &desc); + if (error) + goto unacct_error; if (map.check_ksm_early) update_ksm_flags(&map); diff --git a/mm/vmalloc.c b/mm/vmalloc.c index f4fa227a8d7f0..31c6425b5fdcb 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3360,7 +3360,7 @@ struct vm_struct *remove_vm_area(const void *addr) static inline void set_area_direct_map(const struct vm_struct *area, int (*set_direct_map)(struct page *page)) { - int i; + unsigned long i; /* HUGE_VMALLOC passes small pages to set_direct_map */ for (i = 0; i < area->nr_pages; i++) @@ -3376,7 +3376,7 @@ static void vm_reset_perms(struct vm_struct *area) unsigned long start = ULONG_MAX, end = 0; unsigned int page_order = vm_area_page_order(area); int flush_dmap = 0; - int i; + unsigned long i; /* * Find the start and end range of the direct mappings to make sure that @@ -3449,10 +3449,10 @@ void vfree_atomic(const void *addr) * Caller is responsible for unmapping (vunmap_range) and KASAN * poisoning before calling this. */ -static void vm_area_free_pages(struct vm_struct *vm, unsigned int start_idx, - unsigned int end_idx) +static void vm_area_free_pages(struct vm_struct *vm, unsigned long start_idx, + unsigned long end_idx) { - unsigned int i; + unsigned long i; if (!(vm->flags & VM_MAP_PUT_PAGES)) { for (i = start_idx; i < end_idx; i++) @@ -3664,12 +3664,12 @@ static inline gfp_t vmalloc_gfp_adjust(gfp_t flags, const bool large) return flags; } -static inline unsigned int +static inline unsigned long vm_area_alloc_pages(gfp_t gfp, int nid, - unsigned int order, unsigned int nr_pages, struct page **pages) + unsigned int order, unsigned long nr_pages, struct page **pages) { - unsigned int nr_allocated = 0; - unsigned int nr_remaining = nr_pages; + unsigned long nr_allocated = 0; + unsigned long nr_remaining = nr_pages; unsigned int max_attempt_order = MAX_PAGE_ORDER; struct page *page; int i; @@ -3717,7 +3717,7 @@ vm_area_alloc_pages(gfp_t gfp, int nid, if (!order) { while (nr_allocated < nr_pages) { unsigned int nr, nr_pages_request; - int i; + unsigned long i; /* * A maximum allowed request is hard-coded and is 100 @@ -3725,7 +3725,7 @@ vm_area_alloc_pages(gfp_t gfp, int nid, * long preemption off scenario in the bulk-allocator * so the range is [1:100]. */ - nr_pages_request = min(100U, nr_pages - nr_allocated); + nr_pages_request = min(100UL, nr_pages - nr_allocated); /* memory allocation should consider mempolicy, we can't * wrongly use nearest node when nid == NUMA_NO_NODE, @@ -3871,12 +3871,12 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, unsigned long addr = (unsigned long)area->addr; unsigned long size = get_vm_area_size(area); unsigned long array_size; - unsigned int nr_small_pages = size >> PAGE_SHIFT; + unsigned long nr_small_pages = size >> PAGE_SHIFT; unsigned int page_order; unsigned int flags; int ret; - array_size = (unsigned long)nr_small_pages * sizeof(struct page *); + array_size = nr_small_pages * sizeof(struct page *); /* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */ if (!gfpflags_allow_blocking(gfp_mask)) @@ -4374,7 +4374,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align } if (size <= old_size) { - unsigned int new_nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned long new_nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; /* Zero out "freed" memory, potentially for future realloc. */ if (want_init_on_free() || want_init_on_alloc(flags)) @@ -4403,7 +4403,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align !(vm->flags & (VM_FLUSH_RESET_PERMS | VM_USERMAP)) && gfp_has_io_fs(flags)) { unsigned long addr = (unsigned long)kasan_reset_tag(p); - unsigned int old_nr_pages = vm->nr_pages; + unsigned long old_nr_pages = vm->nr_pages; /* * Use the node lock to synchronize with concurrent @@ -4416,16 +4416,13 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align spin_unlock(&vn->busy.lock); /* Notify kmemleak of the reduced allocation size before unmapping. */ - kmemleak_free_part( - (void *)addr + ((unsigned long)new_nr_pages - << PAGE_SHIFT), - (unsigned long)(old_nr_pages - new_nr_pages) - << PAGE_SHIFT); + kmemleak_free_part((void *)addr + + (new_nr_pages << PAGE_SHIFT), + (old_nr_pages - new_nr_pages) + << PAGE_SHIFT); - vunmap_range(addr + ((unsigned long)new_nr_pages - << PAGE_SHIFT), - addr + ((unsigned long)old_nr_pages - << PAGE_SHIFT)); + vunmap_range(addr + (new_nr_pages << PAGE_SHIFT), + addr + (old_nr_pages << PAGE_SHIFT)); vm_area_free_pages(vm, new_nr_pages, old_nr_pages); } @@ -5250,7 +5247,7 @@ bool vmalloc_dump_obj(void *object) struct vmap_area *va; struct vmap_node *vn; unsigned long addr; - unsigned int nr_pages; + unsigned long nr_pages; addr = PAGE_ALIGN((unsigned long) object); vn = addr_to_node(addr); @@ -5270,7 +5267,7 @@ bool vmalloc_dump_obj(void *object) nr_pages = vm->nr_pages; spin_unlock(&vn->busy.lock); - pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n", + pr_cont(" %lu-page vmalloc region starting at %#lx allocated at %pS\n", nr_pages, addr, caller); return true; @@ -5288,16 +5285,17 @@ bool vmalloc_dump_obj(void *object) static void show_numa_info(struct seq_file *m, struct vm_struct *v, unsigned int *counters) { - unsigned int nr; unsigned int step = 1U << vm_area_page_order(v); + unsigned long i; + unsigned int nr; if (!counters) return; memset(counters, 0, nr_node_ids * sizeof(unsigned int)); - for (nr = 0; nr < v->nr_pages; nr += step) - counters[page_to_nid(v->pages[nr])] += step; + for (i = 0; i < v->nr_pages; i += step) + counters[page_to_nid(v->pages[i])] += step; for_each_node_state(nr, N_HIGH_MEMORY) if (counters[nr]) seq_printf(m, " N%u=%u", nr, counters[nr]); @@ -5355,7 +5353,7 @@ static int vmalloc_info_show(struct seq_file *m, void *p) seq_printf(m, " %pS", v->caller); if (v->nr_pages) - seq_printf(m, " pages=%d", v->nr_pages); + seq_printf(m, " pages=%lu", v->nr_pages); if (v->phys_addr) seq_printf(m, " phys=%pa", &v->phys_addr); diff --git a/mm/vmscan.c b/mm/vmscan.c index 56708d1d2dfd5..8781dbcdb99a6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -198,6 +198,13 @@ struct scan_control { */ int vm_swappiness = 60; +static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) +{ + if (sc->proactive && sc->proactive_swappiness) + return *sc->proactive_swappiness; + return mem_cgroup_swappiness(memcg); +} + #ifdef CONFIG_MEMCG /* Returns true for reclaim through cgroup limits or cgroup interfaces. */ @@ -238,13 +245,6 @@ static bool writeback_throttling_sane(struct scan_control *sc) #endif return false; } - -static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) -{ - if (sc->proactive && sc->proactive_swappiness) - return *sc->proactive_swappiness; - return mem_cgroup_swappiness(memcg); -} #else static bool cgroup_reclaim(struct scan_control *sc) { @@ -260,11 +260,6 @@ static bool writeback_throttling_sane(struct scan_control *sc) { return true; } - -static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) -{ - return READ_ONCE(vm_swappiness); -} #endif static void set_task_reclaim_state(struct task_struct *task, @@ -4559,7 +4554,12 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) { unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid); + if (!size) + continue; + + /* Move the accounting, do not duplicate it. */ mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size); + mem_cgroup_update_lru_size(child_lruvec, lru, zid, -(long)size); } } } @@ -4573,7 +4573,6 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc, int tier_idx) { - bool success; int gen = folio_lru_gen(folio); int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); @@ -4585,15 +4584,9 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); - /* unevictable */ - if (!folio_evictable(folio)) { - success = lru_gen_del_folio(lruvec, folio, true); - VM_WARN_ON_ONCE_FOLIO(!success, folio); - folio_set_unevictable(folio); - lruvec_add_folio(lruvec, folio); - __count_vm_events(UNEVICTABLE_PGCULLED, delta); - return true; - } + /* unevictable: let it through and the generic path will cull it */ + if (!folio_evictable(folio)) + return false; /* promoted */ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { @@ -4843,11 +4836,9 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, list_for_each_entry_safe_reverse(folio, next, &list, lru) { DEFINE_MIN_SEQ(lruvec); - if (!folio_evictable(folio)) { - list_del(&folio->lru); - folio_putback_lru(folio); + /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */ + if (!folio_evictable(folio)) continue; - } /* retry folios that may have missed folio_rotate_reclaimable() */ if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) && @@ -5927,7 +5918,7 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) } } - cond_resched(); + cond_resched_tasks_rcu_qs(); if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) continue; diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 83f5820c45f93..41b03a9ecec25 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -478,6 +478,11 @@ static int get_size_class_index(int size) return min_t(int, ZS_SIZE_CLASSES - 1, idx); } +static struct size_class *lookup_size_class(struct zs_pool *pool, size_t size) +{ + return pool->size_class[get_size_class_index(size + ZS_HANDLE_SIZE)]; +} + static inline void class_stat_add(struct size_class *class, int type, unsigned long cnt) { @@ -1021,7 +1026,7 @@ unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size) { struct size_class *class; - class = pool->size_class[get_size_class_index(size)]; + class = lookup_size_class(pool, size); return class->index; } @@ -1311,9 +1316,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp, if (!handle) return (unsigned long)ERR_PTR(-ENOMEM); - /* extra space in chunk to keep the handle */ - size += ZS_HANDLE_SIZE; - class = pool->size_class[get_size_class_index(size)]; + class = lookup_size_class(pool, size); /* class->lock effectively protects the zpage migration */ spin_lock(&class->lock); diff --git a/mm/zswap.c b/mm/zswap.c index 761cd699e0a3e..0bdb999e46b6c 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1356,11 +1356,12 @@ static void shrink_worker(struct work_struct *w) } while (memcg && !mem_cgroup_tryget_online(memcg)); spin_unlock(&zswap_shrink_lock); - if (!memcg) { - /* - * Continue shrinking without incrementing failures if - * we found candidate memcgs in the last tree walk. - */ + /* + * A NULL memcg ends a full hierarchy pass (except when memcg is + * disabled, where it is always NULL: fall through to the root LRU). + * Count a failure only if the last pass found no candidates. + */ + if (!memcg && !mem_cgroup_disabled()) { if (!attempts && ++failures == MAX_RECLAIM_RETRIES) break; @@ -1379,7 +1380,7 @@ static void shrink_worker(struct work_struct *w) * and failures. */ if (ret == -ENOENT) - continue; + goto resched; ++attempts; if (ret && ++failures == MAX_RECLAIM_RETRIES) diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index ec2569b3f8dac..c949c6a829456 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -55,6 +55,11 @@ static int vlan_dev_hard_header(struct sk_buff *skb, struct net_device *dev, int rc; if (!(vlan->flags & VLAN_FLAG_REORDER_HDR)) { + unsigned int hlen = READ_ONCE(dev->hard_header_len) + + READ_ONCE(dev->needed_headroom); + + if (skb_cow_head(skb, hlen) < 0) + return -ENOMEM; vhdr = skb_push(skb, VLAN_HLEN); vlan_tci = vlan->vlan_id; @@ -502,26 +507,6 @@ static const struct header_ops vlan_header_ops = { .parse_protocol = vlan_parse_protocol, }; -static int vlan_passthru_hard_header(struct sk_buff *skb, struct net_device *dev, - unsigned short type, - const void *daddr, const void *saddr, - unsigned int len) -{ - struct vlan_dev_priv *vlan = vlan_dev_priv(dev); - struct net_device *real_dev = vlan->real_dev; - - if (saddr == NULL) - saddr = dev->dev_addr; - - return dev_hard_header(skb, real_dev, type, daddr, saddr, len); -} - -static const struct header_ops vlan_passthru_header_ops = { - .create = vlan_passthru_hard_header, - .parse = eth_header_parse, - .parse_protocol = vlan_parse_protocol, -}; - static const struct device_type vlan_type = { .name = "vlan", }; @@ -580,14 +565,10 @@ static int vlan_dev_init(struct net_device *dev) dev->fcoe_ddp_xid = real_dev->fcoe_ddp_xid; #endif - dev->needed_headroom = real_dev->needed_headroom; - if (vlan_hw_offload_capable(real_dev->features, vlan->vlan_proto)) { - dev->header_ops = &vlan_passthru_header_ops; - dev->hard_header_len = real_dev->hard_header_len; - } else { - dev->header_ops = &vlan_header_ops; - dev->hard_header_len = real_dev->hard_header_len + VLAN_HLEN; - } + dev->needed_headroom = real_dev->needed_headroom + VLAN_HLEN; + dev->needed_tailroom = real_dev->needed_tailroom; + dev->header_ops = &vlan_header_ops; + dev->hard_header_len = real_dev->hard_header_len; dev->netdev_ops = &vlan_netdev_ops; @@ -1029,10 +1010,9 @@ static void vlan_transfer_features(struct net_device *dev, netif_inherit_tso_max(vlandev, dev); - if (vlan_hw_offload_capable(dev->features, vlan->vlan_proto)) - vlandev->hard_header_len = dev->hard_header_len; - else - vlandev->hard_header_len = dev->hard_header_len + VLAN_HLEN; + vlandev->needed_headroom = dev->needed_headroom + VLAN_HLEN; + vlandev->needed_tailroom = dev->needed_tailroom; + vlandev->hard_header_len = dev->hard_header_len; #if IS_ENABLED(CONFIG_FCOE) vlandev->fcoe_ddp_xid = dev->fcoe_ddp_xid; diff --git a/net/atm/pppoatm.c b/net/atm/pppoatm.c index 6da52d12df68e..5214786e61d11 100644 --- a/net/atm/pppoatm.c +++ b/net/atm/pppoatm.c @@ -292,10 +292,13 @@ static int pppoatm_send(struct ppp_channel *chan, struct sk_buff *skb) struct atm_vcc *vcc; int ret; + if (!pskb_may_pull(skb, 1)) { + kfree_skb(skb); + return DROP_PACKET; + } + ATM_SKB(skb)->vcc = pvcc->atmvcc; pr_debug("(skb=0x%p, vcc=0x%p)\n", skb, pvcc->atmvcc); - if (skb->data[0] == '\0' && (pvcc->flags & SC_COMP_PROT)) - (void) skb_pull(skb, 1); vcc = ATM_SKB(skb)->vcc; bh_lock_sock(sk_atm(vcc)); @@ -317,23 +320,13 @@ static int pppoatm_send(struct ppp_channel *chan, struct sk_buff *skb) switch (pvcc->encaps) { /* LLC encapsulation needed */ case e_llc: - if (skb_headroom(skb) < LLC_LEN) { - struct sk_buff *n; - n = skb_realloc_headroom(skb, LLC_LEN); - if (n != NULL && - !pppoatm_may_send(pvcc, n->truesize)) { - kfree_skb(n); - goto nospace; - } - consume_skb(skb); - skb = n; - if (skb == NULL) { - bh_unlock_sock(sk_atm(vcc)); - return DROP_PACKET; - } - } else if (!pppoatm_may_send(pvcc, skb->truesize)) + if (skb_cow_head(skb, LLC_LEN)) { + bh_unlock_sock(sk_atm(vcc)); + kfree_skb(skb); + return DROP_PACKET; + } + if (!pppoatm_may_send(pvcc, skb->truesize)) goto nospace; - memcpy(skb_push(skb, LLC_LEN), pppllc, LLC_LEN); break; case e_vc: if (!pppoatm_may_send(pvcc, skb->truesize)) @@ -346,6 +339,12 @@ static int pppoatm_send(struct ppp_channel *chan, struct sk_buff *skb) return 1; } + if (skb->data[0] == '\0' && (pvcc->flags & SC_COMP_PROT)) + skb_pull(skb, 1); + + if (pvcc->encaps == e_llc) + memcpy(skb_push(skb, LLC_LEN), pppllc, LLC_LEN); + atm_account_tx(vcc, skb); pr_debug("atm_skb(%p)->vcc(%p)->dev(%p)\n", skb, ATM_SKB(skb)->vcc, ATM_SKB(skb)->vcc->dev); @@ -355,13 +354,6 @@ static int pppoatm_send(struct ppp_channel *chan, struct sk_buff *skb) return ret; nospace: bh_unlock_sock(sk_atm(vcc)); - /* - * We don't have space to send this SKB now, but we might have - * already applied SC_COMP_PROT compression, so may need to undo - */ - if ((pvcc->flags & SC_COMP_PROT) && skb_headroom(skb) > 0 && - skb->data[-1] == '\0') - (void) skb_push(skb, 1); return 0; } diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c index 5c73f6ba16cff..87d5569538436 100644 --- a/net/batman-adv/bridge_loop_avoidance.c +++ b/net/batman-adv/bridge_loop_avoidance.c @@ -324,11 +324,6 @@ batadv_bla_del_backbone_claims(struct batadv_bla_backbone_gw *backbone_gw) } spin_unlock_bh(list_lock); } - - /* all claims gone, initialize CRC */ - spin_lock_bh(&backbone_gw->crc_lock); - backbone_gw->crc = BATADV_BLA_CRC_INIT; - spin_unlock_bh(&backbone_gw->crc_lock); } /** @@ -699,12 +694,14 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, struct batadv_bla_backbone_gw *old_backbone_gw; struct batadv_bla_claim *claim; struct batadv_bla_claim search_claim; - bool remove_crc = false; int hash_added; + u16 claim_crc; + bool changed; ether_addr_copy(search_claim.addr, mac); search_claim.vid = vid; claim = batadv_claim_hash_find(bat_priv, &search_claim); + claim_crc = crc16(0, mac, ETH_ALEN); /* create a new claim entry if it does not exist yet. */ if (!claim) { @@ -736,43 +733,56 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, kfree(claim); return; } + + spin_lock_bh(&backbone_gw->crc_lock); + backbone_gw->crc ^= claim_crc; + spin_unlock_bh(&backbone_gw->crc_lock); + + WRITE_ONCE(backbone_gw->lasttime, jiffies); + + batadv_claim_put(claim); + return; + } + + WRITE_ONCE(claim->lasttime, jiffies); + + /* replace backbone_gw atomically and adjust reference counters */ + spin_lock_bh(&claim->backbone_lock); + if (claim->backbone_gw != backbone_gw) { + changed = true; + + old_backbone_gw = claim->backbone_gw; + kref_get(&backbone_gw->refcount); + claim->backbone_gw = backbone_gw; } else { - WRITE_ONCE(claim->lasttime, jiffies); - if (claim->backbone_gw == backbone_gw) - /* no need to register a new backbone */ - goto claim_free_ref; + old_backbone_gw = NULL; + changed = false; + } + spin_unlock_bh(&claim->backbone_lock); + if (changed) { batadv_dbg(BATADV_DBG_BLA, bat_priv, "%s(): changing ownership for %pM, vid %d to gw %pM\n", __func__, mac, batadv_print_vid(vid), backbone_gw->orig); - remove_crc = true; - } + /* add claim address to new backbone_gw */ + spin_lock_bh(&backbone_gw->crc_lock); + backbone_gw->crc ^= claim_crc; + spin_unlock_bh(&backbone_gw->crc_lock); - /* replace backbone_gw atomically and adjust reference counters */ - spin_lock_bh(&claim->backbone_lock); - old_backbone_gw = claim->backbone_gw; - kref_get(&backbone_gw->refcount); - claim->backbone_gw = backbone_gw; - spin_unlock_bh(&claim->backbone_lock); + WRITE_ONCE(backbone_gw->lasttime, jiffies); + } - if (remove_crc) { + if (old_backbone_gw) { /* remove claim address from old backbone_gw */ spin_lock_bh(&old_backbone_gw->crc_lock); - old_backbone_gw->crc ^= crc16(0, claim->addr, ETH_ALEN); + old_backbone_gw->crc ^= claim_crc; spin_unlock_bh(&old_backbone_gw->crc_lock); - } - batadv_backbone_gw_put(old_backbone_gw); - - /* add claim address to new backbone_gw */ - spin_lock_bh(&backbone_gw->crc_lock); - backbone_gw->crc ^= crc16(0, claim->addr, ETH_ALEN); - spin_unlock_bh(&backbone_gw->crc_lock); - WRITE_ONCE(backbone_gw->lasttime, jiffies); + batadv_backbone_gw_put(old_backbone_gw); + } -claim_free_ref: batadv_claim_put(claim); } @@ -942,26 +952,18 @@ static bool batadv_handle_unclaim(struct batadv_priv *bat_priv, const u8 *backbone_addr, const u8 *claim_addr, unsigned short vid) { - struct batadv_bla_backbone_gw *backbone_gw; - /* unclaim in any case if it is our own */ if (primary_if && batadv_compare_eth(backbone_addr, primary_if->net_dev->dev_addr)) batadv_bla_send_claim(bat_priv, claim_addr, vid, BATADV_CLAIM_TYPE_UNCLAIM); - backbone_gw = batadv_backbone_hash_find(bat_priv, backbone_addr, vid); - - if (!backbone_gw) - return true; - /* this must be an UNCLAIM frame */ batadv_dbg(BATADV_DBG_BLA, bat_priv, "%s(): UNCLAIM %pM on vid %d (sent by %pM)...\n", __func__, - claim_addr, batadv_print_vid(vid), backbone_gw->orig); + claim_addr, batadv_print_vid(vid), backbone_addr); batadv_bla_del_claim(bat_priv, claim_addr, vid); - batadv_backbone_gw_put(backbone_gw); return true; } diff --git a/net/batman-adv/distributed-arp-table.c b/net/batman-adv/distributed-arp-table.c index a6fe4820f65b9..95d8e3a383902 100644 --- a/net/batman-adv/distributed-arp-table.c +++ b/net/batman-adv/distributed-arp-table.c @@ -250,7 +250,10 @@ static u8 *batadv_arp_hw_src(struct sk_buff *skb, int hdr_size) */ static __be32 batadv_arp_ip_src(struct sk_buff *skb, int hdr_size) { - return *(__force __be32 *)(batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN); + u8 *src = batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN; + __be32 *ip = (__force __be32 *)src; + + return get_unaligned(ip); } /** @@ -275,8 +278,9 @@ static u8 *batadv_arp_hw_dst(struct sk_buff *skb, int hdr_size) static __be32 batadv_arp_ip_dst(struct sk_buff *skb, int hdr_size) { u8 *dst = batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN * 2 + 4; + __be32 *ip = (__force __be32 *)dst; - return *(__force __be32 *)dst; + return get_unaligned(ip); } /** @@ -372,18 +376,19 @@ batadv_dat_entry_hash_find(struct batadv_priv *bat_priv, __be32 ip, static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, u8 *mac_addr, unsigned short vid) { + u64 u64_mac = ether_addr_to_u64(mac_addr); struct batadv_dat_entry *dat_entry; int hash_added; dat_entry = batadv_dat_entry_hash_find(bat_priv, ip, vid); /* if this entry is already known, just update it */ if (dat_entry) { - if (!batadv_compare_eth(dat_entry->mac_addr, mac_addr)) - ether_addr_copy(dat_entry->mac_addr, mac_addr); + atomic64_set(&dat_entry->mac_addr, u64_mac); + dat_entry->last_update = jiffies; batadv_dbg(BATADV_DBG_DAT, bat_priv, "Entry updated: %pI4 %pM (vid: %d)\n", - &dat_entry->ip, dat_entry->mac_addr, + &dat_entry->ip, mac_addr, batadv_print_vid(vid)); goto out; } @@ -394,7 +399,7 @@ static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, dat_entry->ip = ip; dat_entry->vid = vid; - ether_addr_copy(dat_entry->mac_addr, mac_addr); + atomic64_set(&dat_entry->mac_addr, u64_mac); dat_entry->last_update = jiffies; kref_init(&dat_entry->refcount); @@ -410,7 +415,7 @@ static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, } batadv_dbg(BATADV_DBG_DAT, bat_priv, "New entry added: %pI4 %pM (vid: %d)\n", - &dat_entry->ip, dat_entry->mac_addr, batadv_print_vid(vid)); + &dat_entry->ip, mac_addr, batadv_print_vid(vid)); out: batadv_dat_entry_put(dat_entry); @@ -864,6 +869,8 @@ batadv_dat_cache_dump_entry(struct sk_buff *msg, u32 portid, struct netlink_callback *cb, struct batadv_dat_entry *dat_entry) { + u8 mac[ETH_ALEN]; + u64 u64_mac; int msecs; void *hdr; @@ -876,11 +883,12 @@ batadv_dat_cache_dump_entry(struct sk_buff *msg, u32 portid, genl_dump_check_consistent(cb, hdr); msecs = jiffies_to_msecs(jiffies - dat_entry->last_update); + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); if (nla_put_in_addr(msg, BATADV_ATTR_DAT_CACHE_IP4ADDRESS, dat_entry->ip) || - nla_put(msg, BATADV_ATTR_DAT_CACHE_HWADDRESS, ETH_ALEN, - dat_entry->mac_addr) || + nla_put(msg, BATADV_ATTR_DAT_CACHE_HWADDRESS, ETH_ALEN, mac) || nla_put_u16(msg, BATADV_ATTR_DAT_CACHE_VID, dat_entry->vid) || nla_put_u32(msg, BATADV_ATTR_LAST_SEEN_MSECS, msecs)) { genlmsg_cancel(msg, hdr); @@ -1147,6 +1155,8 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, struct net_device *mesh_iface = bat_priv->mesh_iface; int hdr_size = 0; unsigned short vid; + u8 mac[ETH_ALEN]; + u64 u64_mac; if (!READ_ONCE(bat_priv->distributed_arp_table)) goto out; @@ -1174,6 +1184,9 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_dst, vid); if (dat_entry) { + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + /* If the ARP request is destined for a local client the local * client will answer itself. DAT would only generate a * duplicate packet. @@ -1182,7 +1195,7 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, * additional DAT answer may trigger kernel warnings about * a packet coming from the wrong port. */ - if (batadv_is_my_client(bat_priv, dat_entry->mac_addr, vid)) { + if (batadv_is_my_client(bat_priv, mac, vid)) { ret = true; goto out; } @@ -1192,18 +1205,16 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, * the backbone gws belonging to our backbone has claimed the * destination. */ - if (!batadv_bla_check_claim(bat_priv, - dat_entry->mac_addr, vid)) { + if (!batadv_bla_check_claim(bat_priv, mac, vid)) { batadv_dbg(BATADV_DBG_DAT, bat_priv, "Device %pM claimed by another backbone gw. Don't send ARP reply!", - dat_entry->mac_addr); + mac); ret = true; goto out; } skb_new = batadv_dat_arp_create_reply(bat_priv, ip_dst, ip_src, - dat_entry->mac_addr, - hw_src, vid); + mac, hw_src, vid); if (!skb_new) goto out; @@ -1245,6 +1256,8 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv, struct batadv_dat_entry *dat_entry = NULL; bool ret = false; unsigned short vid; + u8 mac[ETH_ALEN]; + u64 u64_mac; int err; if (!READ_ONCE(bat_priv->distributed_arp_table)) @@ -1272,8 +1285,11 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv, if (!dat_entry) goto out; + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + skb_new = batadv_dat_arp_create_reply(bat_priv, ip_dst, ip_src, - dat_entry->mac_addr, hw_src, vid); + mac, hw_src, vid); if (!skb_new) goto out; @@ -1364,6 +1380,8 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv, u8 *hw_src, *hw_dst; bool dropped = false; unsigned short vid; + u8 mac[ETH_ALEN]; + u64 u64_mac; if (!READ_ONCE(bat_priv->distributed_arp_table)) goto out; @@ -1392,11 +1410,17 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv, * this frame would lead to doubled receive of an ARP reply. */ dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_src, vid); - if (dat_entry && batadv_compare_eth(hw_src, dat_entry->mac_addr)) { - batadv_dbg(BATADV_DBG_DAT, bat_priv, "Doubled ARP reply removed: ARP MSG = [src: %pM-%pI4 dst: %pM-%pI4]; dat_entry: %pM-%pI4\n", - hw_src, &ip_src, hw_dst, &ip_dst, - dat_entry->mac_addr, &dat_entry->ip); - dropped = true; + if (dat_entry) { + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + + if (batadv_compare_eth(hw_src, mac)) { + batadv_dbg(BATADV_DBG_DAT, bat_priv, + "Doubled ARP reply removed: ARP MSG = [src: %pM-%pI4 dst: %pM-%pI4]; dat_entry: %pM-%pI4\n", + hw_src, &ip_src, hw_dst, &ip_dst, + mac, &dat_entry->ip); + dropped = true; + } } /* Update our internal cache with both the IP addresses the node got diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c index 67bed3ee77e7e..7918886e6f94b 100644 --- a/net/batman-adv/main.c +++ b/net/batman-adv/main.c @@ -450,6 +450,10 @@ int batadv_batman_skb_recv(struct sk_buff *skb, struct net_device *dev, if (!skb) goto err_put; + /* Merged fragments re-enter here with reused skb metadata. */ + skb->dev = dev; + skb->skb_iif = dev->ifindex; + /* packet should hold at least type and version */ if (unlikely(!pskb_may_pull(skb, 2))) goto err_free; diff --git a/net/batman-adv/mesh-interface.c b/net/batman-adv/mesh-interface.c index fbfd99268de47..91913271be3cb 100644 --- a/net/batman-adv/mesh-interface.c +++ b/net/batman-adv/mesh-interface.c @@ -260,6 +260,8 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb, if (batadv_compare_eth(ethhdr->h_dest, ectp_addr)) goto dropped; + batadv_skb_set_priority(skb, 0); + gw_mode = READ_ONCE(bat_priv->gw.mode); if (is_multicast_ether_addr(ethhdr->h_dest)) { /* if gw mode is off, broadcast every packet */ @@ -293,6 +295,9 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb, send: if (do_bcast && !is_broadcast_ether_addr(ethhdr->h_dest)) { + /* WARNING batadv_mcast_forw_mode might add more headers + * in front of the skb. and might even reallocate the skb + */ forw_mode = batadv_mcast_forw_mode(bat_priv, skb, vid, &mcast_is_routable); switch (forw_mode) { @@ -310,8 +315,6 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb, } } - batadv_skb_set_priority(skb, 0); - /* ethernet packet should be broadcasted */ if (do_bcast) { primary_if = batadv_primary_if_get_selected(bat_priv); diff --git a/net/batman-adv/multicast_forw.c b/net/batman-adv/multicast_forw.c index 1404a3b7adfb1..c99579dc1ca98 100644 --- a/net/batman-adv/multicast_forw.c +++ b/net/batman-adv/multicast_forw.c @@ -1100,8 +1100,11 @@ static int batadv_mcast_forw_expand_head(struct batadv_priv *bat_priv, return -EINVAL; } - if (skb_headroom(skb) < hdr_size && - pskb_expand_head(skb, hdr_size, 0, GFP_ATOMIC) < 0) + if (skb_cow(skb, hdr_size) < 0) + return -ENOMEM; + + /* batadv_mcast_forw_scrape() + batadv_mcast_forw_packet() require linearized skb */ + if (skb_linearize(skb) < 0) return -ENOMEM; return 0; diff --git a/net/batman-adv/tvlv.c b/net/batman-adv/tvlv.c index 1c9fb21985f6a..c384db16bcac1 100644 --- a/net/batman-adv/tvlv.c +++ b/net/batman-adv/tvlv.c @@ -433,8 +433,11 @@ static int batadv_tvlv_call_handler(struct batadv_priv *bat_priv, return NET_RX_SUCCESS; tvlv_offset = (unsigned char *)tvlv_value - skb->data; + if (!skb_set_transport_header_careful(skb, + tvlv_offset + tvlv_value_len)) + return -EINVAL; + skb_set_network_header(skb, tvlv_offset); - skb_set_transport_header(skb, tvlv_offset + tvlv_value_len); return tvlv_handler->mcast_handler(bat_priv, skb); } diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h index b1f9f8964c3fd..9ab0dba82f509 100644 --- a/net/batman-adv/types.h +++ b/net/batman-adv/types.h @@ -2176,7 +2176,7 @@ struct batadv_dat_entry { __be32 ip; /** @mac_addr: the MAC address associated to the stored IPv4 */ - u8 mac_addr[ETH_ALEN]; + atomic64_t mac_addr; /** @vid: the vlan ID associated to this entry */ unsigned short vid; diff --git a/net/bluetooth/bnep/core.c b/net/bluetooth/bnep/core.c index f7d88c33e23e4..ad24d2486665a 100644 --- a/net/bluetooth/bnep/core.c +++ b/net/bluetooth/bnep/core.c @@ -270,9 +270,14 @@ static int bnep_rx_extension(struct bnep_session *s, struct sk_buff *skb) BT_DBG("type 0x%x len %u", h->type, h->len); + if (skb->len < h->len) { + err = -EILSEQ; + break; + } + switch (h->type & BNEP_TYPE_MASK) { case BNEP_EXT_CONTROL: - bnep_rx_control(s, skb->data, skb->len); + bnep_rx_control(s, skb->data, h->len); break; default: @@ -373,6 +378,11 @@ static int bnep_rx_frame(struct bnep_session *s, struct sk_buff *skb) goto badframe; } + if ((type & BNEP_TYPE_MASK) == BNEP_CONTROL) { + kfree_skb(skb); + return 0; + } + /* Strip 802.1p header */ if (ntohs(s->eh.h_proto) == ETH_P_8021Q) { if (!skb_pull(skb, 4)) @@ -451,6 +461,11 @@ static int bnep_tx_frame(struct bnep_session *s, struct sk_buff *skb) goto send; } + if (skb->len < ETH_HLEN) { + kfree_skb(skb); + return 0; + } + iv[il++] = (struct kvec) { &type, 1 }; len++; diff --git a/net/bluetooth/bnep/netdev.c b/net/bluetooth/bnep/netdev.c index ee1e39a3daffb..b451ef457741f 100644 --- a/net/bluetooth/bnep/netdev.c +++ b/net/bluetooth/bnep/netdev.c @@ -166,6 +166,12 @@ static netdev_tx_t bnep_net_xmit(struct sk_buff *skb, BT_DBG("skb %p, dev %p", skb, dev); + if (!pskb_may_pull(skb, ETH_HLEN)) { + dev->stats.tx_dropped++; + kfree_skb(skb); + return NETDEV_TX_OK; + } + #ifdef CONFIG_BT_BNEP_MC_FILTER if (bnep_net_mc_filter(skb, s)) { kfree_skb(skb); @@ -218,7 +224,7 @@ void bnep_net_setup(struct net_device *dev) dev->addr_len = ETH_ALEN; ether_setup(dev); - dev->min_mtu = 0; + dev->min_mtu = ETH_MIN_MTU; dev->max_mtu = ETH_MAX_MTU; dev->priv_flags &= ~IFF_TX_SKB_SHARING; dev->netdev_ops = &bnep_netdev_ops; diff --git a/net/bluetooth/coredump.c b/net/bluetooth/coredump.c index 720cb79adf964..b15971ad78a8f 100644 --- a/net/bluetooth/coredump.c +++ b/net/bluetooth/coredump.c @@ -105,6 +105,22 @@ static void hci_devcd_free(struct hci_dev *hdev) hci_devcd_reset(hdev); } +void hci_devcd_shutdown(struct hci_dev *hdev) +{ + unsigned long flags; + + spin_lock_irqsave(&hdev->dump.dump_q.lock, flags); + hdev->dump.supported = false; + spin_unlock_irqrestore(&hdev->dump.dump_q.lock, flags); + + disable_work_sync(&hdev->dump.dump_rx); + disable_delayed_work_sync(&hdev->dump.dump_timeout); + + hci_dev_lock(hdev); + hci_devcd_free(hdev); + hci_dev_unlock(hdev); +} + /* Call with hci_dev_lock only. */ static int hci_devcd_alloc(struct hci_dev *hdev, u32 size) { @@ -444,7 +460,29 @@ EXPORT_SYMBOL(hci_devcd_register); static inline bool hci_devcd_enabled(struct hci_dev *hdev) { - return hdev->dump.supported; + return READ_ONCE(hdev->dump.supported); +} + +static int hci_devcd_queue(struct hci_dev *hdev, struct sk_buff *skb) +{ + unsigned long flags; + int err = 0; + + spin_lock_irqsave(&hdev->dump.dump_q.lock, flags); + if (!hdev->dump.supported) + err = -EOPNOTSUPP; + else + __skb_queue_tail(&hdev->dump.dump_q, skb); + spin_unlock_irqrestore(&hdev->dump.dump_q.lock, flags); + + if (err) { + kfree_skb(skb); + return err; + } + + queue_work(hdev->workqueue, &hdev->dump.dump_rx); + + return 0; } int hci_devcd_init(struct hci_dev *hdev, u32 dump_size) @@ -461,10 +499,7 @@ int hci_devcd_init(struct hci_dev *hdev, u32 dump_size) hci_dmp_cb(skb)->pkt_type = HCI_DEVCOREDUMP_PKT_INIT; put_unaligned_le32(dump_size, skb_put(skb, 4)); - skb_queue_tail(&hdev->dump.dump_q, skb); - queue_work(hdev->workqueue, &hdev->dump.dump_rx); - - return 0; + return hci_devcd_queue(hdev, skb); } EXPORT_SYMBOL(hci_devcd_init); @@ -480,10 +515,7 @@ int hci_devcd_append(struct hci_dev *hdev, struct sk_buff *skb) hci_dmp_cb(skb)->pkt_type = HCI_DEVCOREDUMP_PKT_SKB; - skb_queue_tail(&hdev->dump.dump_q, skb); - queue_work(hdev->workqueue, &hdev->dump.dump_rx); - - return 0; + return hci_devcd_queue(hdev, skb); } EXPORT_SYMBOL(hci_devcd_append); @@ -505,10 +537,7 @@ int hci_devcd_append_pattern(struct hci_dev *hdev, u8 pattern, u32 len) hci_dmp_cb(skb)->pkt_type = HCI_DEVCOREDUMP_PKT_PATTERN; skb_put_data(skb, &p, sizeof(p)); - skb_queue_tail(&hdev->dump.dump_q, skb); - queue_work(hdev->workqueue, &hdev->dump.dump_rx); - - return 0; + return hci_devcd_queue(hdev, skb); } EXPORT_SYMBOL(hci_devcd_append_pattern); @@ -525,10 +554,7 @@ int hci_devcd_complete(struct hci_dev *hdev) hci_dmp_cb(skb)->pkt_type = HCI_DEVCOREDUMP_PKT_COMPLETE; - skb_queue_tail(&hdev->dump.dump_q, skb); - queue_work(hdev->workqueue, &hdev->dump.dump_rx); - - return 0; + return hci_devcd_queue(hdev, skb); } EXPORT_SYMBOL(hci_devcd_complete); @@ -545,9 +571,6 @@ int hci_devcd_abort(struct hci_dev *hdev) hci_dmp_cb(skb)->pkt_type = HCI_DEVCOREDUMP_PKT_ABORT; - skb_queue_tail(&hdev->dump.dump_q, skb); - queue_work(hdev->workqueue, &hdev->dump.dump_rx); - - return 0; + return hci_devcd_queue(hdev, skb); } EXPORT_SYMBOL(hci_devcd_abort); diff --git a/net/bluetooth/eir.c b/net/bluetooth/eir.c index 1de5f9df6eec0..ee0136bfae40b 100644 --- a/net/bluetooth/eir.c +++ b/net/bluetooth/eir.c @@ -369,10 +369,19 @@ u8 eir_create_scan_rsp(struct hci_dev *hdev, u8 instance, u8 *ptr) void *eir_get_service_data(u8 *eir, size_t eir_len, u16 uuid, size_t *len) { + const u8 *eir_end = eir + eir_len; size_t dlen; while ((eir = eir_get_data(eir, eir_len, EIR_SERVICE_DATA, &dlen))) { - u16 value = get_unaligned_le16(eir); + u16 value; + + if (dlen < sizeof(value)) { + eir += dlen; + eir_len = eir_end - eir; + continue; + } + + value = get_unaligned_le16(eir); if (uuid == value) { if (len) @@ -381,7 +390,7 @@ void *eir_get_service_data(u8 *eir, size_t eir_len, u16 uuid, size_t *len) } eir += dlen; - eir_len -= dlen; + eir_len = eir_end - eir; } return NULL; diff --git a/net/bluetooth/hci_codec.c b/net/bluetooth/hci_codec.c index 5bc5003c387c6..7a7e813dcdda5 100644 --- a/net/bluetooth/hci_codec.c +++ b/net/bluetooth/hci_codec.c @@ -145,11 +145,12 @@ void hci_read_supported_codecs(struct hci_dev *hdev) skb_pull(skb, sizeof(rp->status)); - std_codecs = (void *)skb->data; + std_codecs = skb_pull_data(skb, sizeof(*std_codecs)); + if (!std_codecs) + goto error; /* validate codecs length before accessing */ - if (skb->len < flex_array_size(std_codecs, codec, std_codecs->num) - + sizeof(std_codecs->num)) + if (skb->len < flex_array_size(std_codecs, codec, std_codecs->num)) goto error; /* enumerate codec capabilities of standard codecs */ @@ -161,15 +162,14 @@ void hci_read_supported_codecs(struct hci_dev *hdev) LOCAL_CODEC_ACL_MASK | LOCAL_CODEC_SCO_MASK, &caps); } - skb_pull(skb, flex_array_size(std_codecs, codec, std_codecs->num) - + sizeof(std_codecs->num)); + skb_pull(skb, flex_array_size(std_codecs, codec, std_codecs->num)); - vnd_codecs = (void *)skb->data; + vnd_codecs = skb_pull_data(skb, sizeof(*vnd_codecs)); + if (!vnd_codecs) + goto error; /* validate vendor codecs length before accessing */ - if (skb->len < - flex_array_size(vnd_codecs, codec, vnd_codecs->num) - + sizeof(vnd_codecs->num)) + if (skb->len < flex_array_size(vnd_codecs, codec, vnd_codecs->num)) goto error; /* enumerate vendor codec capabilities */ @@ -214,11 +214,12 @@ void hci_read_supported_codecs_v2(struct hci_dev *hdev) skb_pull(skb, sizeof(rp->status)); - std_codecs = (void *)skb->data; + std_codecs = skb_pull_data(skb, sizeof(*std_codecs)); + if (!std_codecs) + goto error; /* check for payload data length before accessing */ - if (skb->len < flex_array_size(std_codecs, codec, std_codecs->num) - + sizeof(std_codecs->num)) + if (skb->len < flex_array_size(std_codecs, codec, std_codecs->num)) goto error; memset(&caps, 0, sizeof(caps)); @@ -229,15 +230,14 @@ void hci_read_supported_codecs_v2(struct hci_dev *hdev) &caps); } - skb_pull(skb, flex_array_size(std_codecs, codec, std_codecs->num) - + sizeof(std_codecs->num)); + skb_pull(skb, flex_array_size(std_codecs, codec, std_codecs->num)); - vnd_codecs = (void *)skb->data; + vnd_codecs = skb_pull_data(skb, sizeof(*vnd_codecs)); + if (!vnd_codecs) + goto error; /* check for payload data length before accessing */ - if (skb->len < - flex_array_size(vnd_codecs, codec, vnd_codecs->num) - + sizeof(vnd_codecs->num)) + if (skb->len < flex_array_size(vnd_codecs, codec, vnd_codecs->num)) goto error; for (i = 0; i < vnd_codecs->num; i++) { diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index b1f911fd4ad6a..96195d2fd10fc 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -283,8 +283,6 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data) struct hci_cp_enhanced_setup_sync_conn cp; const struct sco_param *param; - kfree(conn_handle); - if (!hci_conn_valid(hdev, conn)) return -ECANCELED; @@ -453,6 +451,15 @@ static bool hci_setup_sync_conn(struct hci_conn *conn, __u16 handle) return true; } +static void hci_enhanced_setup_sync_destroy(struct hci_dev *hdev, void *data, + int err) +{ + struct conn_handle_t *conn_handle = data; + + hci_conn_put(conn_handle->conn); + kfree(conn_handle); +} + bool hci_setup_sync(struct hci_conn *conn, __u16 handle) { int result; @@ -464,12 +471,15 @@ bool hci_setup_sync(struct hci_conn *conn, __u16 handle) if (!conn_handle) return false; - conn_handle->conn = conn; + conn_handle->conn = hci_conn_get(conn); conn_handle->handle = handle; result = hci_cmd_sync_queue(conn->hdev, hci_enhanced_setup_sync, - conn_handle, NULL); - if (result < 0) + conn_handle, + hci_enhanced_setup_sync_destroy); + if (result < 0) { + hci_conn_put(conn); kfree(conn_handle); + } return result == 0; } @@ -1013,6 +1023,19 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, if (!hdev->le_mtu && hdev->acl_mtu < HCI_MIN_LE_MTU) return ERR_PTR(-ECONNREFUSED); irk = hci_get_irk(hdev, dst, dst_type); + /* An identity address only reaches a peer advertising an RPA + * if the controller translates it. Unless address resolution + * is enabled and this peer is programmed into the resolving + * list, keep the RPA the peer is on air with; + * le_conn_complete_evt() resolves it back once the link is + * up. + */ + if (irk && + (!hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION) || + !hci_bdaddr_list_lookup_with_irk(&hdev->le_resolv_list, + &irk->bdaddr, + irk->addr_type))) + irk = NULL; break; case SCO_LINK: case ESCO_LINK: @@ -1381,7 +1404,8 @@ static void hci_le_conn_failed(struct hci_conn *conn, u8 status) /* Enable advertising in case this was a failed connection * attempt as a peripheral. */ - hci_enable_advertising(hdev); + if (conn->role == HCI_ROLE_SLAVE) + hci_enable_advertising(hdev); } /* This function requires the caller holds hdev->lock */ @@ -1494,7 +1518,15 @@ struct hci_conn *hci_connect_le(struct hci_dev *hdev, bdaddr_t *dst, } if (conn) { + /* dst may just have been swapped for the peer's RPA above, and + * dst_type describes dst -- it has to travel with it. Leaving + * the identity type behind makes the pair describe a peer that + * does not exist, and nothing downstream repairs it: + * hci_bdaddr_is_rpa() tests the type before the address, so + * the RPA is never treated as one. + */ bacpy(&conn->dst, dst); + conn->dst_type = dst_type; } else { conn = hci_conn_add_unset(hdev, LE_LINK, dst, dst_type, role); if (IS_ERR(conn)) @@ -2047,6 +2079,8 @@ struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst, cis->conn_timeout = timeout; } + hci_conn_hold(cis); + if (cis->state == BT_CONNECTED) return cis; @@ -2088,7 +2122,6 @@ struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst, return ERR_PTR(-EINVAL); } - hci_conn_hold(cis); cis->state = BT_BOUND; return cis; @@ -2342,10 +2375,13 @@ struct hci_conn *hci_bind_bis(struct hci_dev *hdev, bdaddr_t *dst, __u8 sid, parent = hci_conn_hash_lookup_big(hdev, conn->iso_qos.bcast.big); if (parent && parent != conn) { + hci_conn_hold(parent); link = hci_conn_link(parent, conn); hci_conn_drop(conn); - if (!link) + if (!link) { + hci_conn_drop(parent); return ERR_PTR(-ENOLINK); + } } return conn; @@ -2465,6 +2501,12 @@ struct hci_conn *hci_connect_cis(struct hci_dev *hdev, bdaddr_t *dst, return cis; } + /* The existing link already owns the hold on its parent. */ + if (cis->link) { + hci_conn_drop(le); + return cis; + } + link = hci_conn_link(le, cis); hci_conn_drop(cis); if (!link) { diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index 5ba9fe8261ec8..3ea5ea7267d9c 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -2629,11 +2629,11 @@ int hci_register_dev(struct hci_dev *hdev) if (error) BT_WARN("register suspend notifier failed error:%d\n", error); - queue_work(hdev->req_workqueue, &hdev->power_on); - idr_init(&hdev->adv_monitors_idr); msft_register(hdev); + queue_work(hdev->req_workqueue, &hdev->power_on); + return id; err_wqueue: @@ -2670,6 +2670,7 @@ void hci_unregister_dev(struct hci_dev *hdev) disable_work_sync(&hdev->error_reset); disable_delayed_work_sync(&hdev->cmd_timer); disable_delayed_work_sync(&hdev->ncmd_timer); + hci_devcd_shutdown(hdev); hci_cmd_sync_clear(hdev); @@ -3234,6 +3235,17 @@ static void hci_queue_acl(struct hci_chan *chan, struct sk_buff_head *queue, bt_dev_dbg(hdev, "chan %p queued %d", chan, skb_queue_len(queue)); } +/* Queue hdev->tx_work, unless hdev->workqueue is being drained by + * hci_dev_close_sync(), which would otherwise WARN and drop the work. + */ +static void hci_sched_tx(struct hci_dev *hdev) +{ + rcu_read_lock(); + if (!hci_dev_test_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE)) + queue_work(hdev->workqueue, &hdev->tx_work); + rcu_read_unlock(); +} + void hci_send_acl(struct hci_chan *chan, struct sk_buff *skb, __u16 flags) { struct hci_dev *hdev = chan->conn->hdev; @@ -3242,7 +3254,7 @@ void hci_send_acl(struct hci_chan *chan, struct sk_buff *skb, __u16 flags) hci_queue_acl(chan, &chan->data_q, skb, flags); - queue_work(hdev->workqueue, &hdev->tx_work); + hci_sched_tx(hdev); } /* Send SCO data */ @@ -3267,7 +3279,7 @@ void hci_send_sco(struct hci_conn *conn, struct sk_buff *skb) bt_dev_dbg(hdev, "hcon %p queued %d", conn, skb_queue_len(&conn->data_q)); - queue_work(hdev->workqueue, &hdev->tx_work); + hci_sched_tx(hdev); } /* Send ISO data */ @@ -3338,7 +3350,7 @@ void hci_send_iso(struct hci_conn *conn, struct sk_buff *skb) hci_queue_iso(conn, &conn->data_q, skb); - queue_work(hdev->workqueue, &hdev->tx_work); + hci_sched_tx(hdev); } /* ---- HCI TX task (outgoing data) ---- */ @@ -4091,7 +4103,7 @@ static int hci_send_cmd_sync(struct hci_dev *hdev, struct sk_buff *skb) if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND && !hci_dev_test_and_set_flag(hdev, HCI_CMD_PENDING)) { kfree_skb(hdev->req_skb); - hdev->req_skb = skb_clone(hdev->sent_cmd, GFP_KERNEL); + hdev->req_skb = skb_get(hdev->sent_cmd); } return err; diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 741d658e96303..f502be73d8c5e 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -294,8 +294,10 @@ static u8 hci_cc_reset(struct hci_dev *hdev, void *data, struct sk_buff *skb) hdev->ssp_debug_mode = 0; + hci_dev_lock(hdev); hci_bdaddr_list_clear(&hdev->le_accept_list); hci_bdaddr_list_clear(&hdev->le_resolv_list); + hci_dev_unlock(hdev); return rp->status; } @@ -3827,8 +3829,10 @@ static u8 hci_cc_le_set_cig_params(struct hci_dev *hdev, void *data, bt_dev_dbg(hdev, "status 0x%2.2x", rp->status); cp = hci_sent_cmd_data(hdev, HCI_OP_LE_SET_CIG_PARAMS); - if (!rp->status && (!cp || rp->num_handles != cp->num_cis || - rp->cig_id != cp->cig_id)) { + if (!rp->status && + (!cp || rp->num_handles != cp->num_cis || + rp->cig_id != cp->cig_id || + skb->len < array_size(rp->num_handles, sizeof(*rp->handle)))) { bt_dev_err(hdev, "unexpected Set CIG Parameters response data"); status = HCI_ERROR_UNSPECIFIED; } @@ -5720,10 +5724,11 @@ static void le_conn_complete_evt(struct hci_dev *hdev, u8 status, hci_dev_lock(hdev); hci_store_wake_reason(hdev, bdaddr, bdaddr_type); - /* All controllers implicitly stop advertising in the event of a - * connection, so ensure that the state bit is cleared. + /* Advertising stops when a connection is created. On a failed + * connection it keeps running, so leave the state bit alone. */ - hci_dev_clear_flag(hdev, HCI_LE_ADV); + if (!status) + hci_dev_clear_flag(hdev, HCI_LE_ADV); /* Check for existing connection: * diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index 070ca388f9ace..6d56c77741e19 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -164,6 +164,7 @@ static bool is_filtered_packet(struct sock *sk, struct sk_buff *skb) { struct hci_filter *flt; int flt_type, flt_event; + u8 event; /* Apply filter */ flt = &hci_pi(sk)->filter; @@ -177,7 +178,11 @@ static bool is_filtered_packet(struct sock *sk, struct sk_buff *skb) if (hci_skb_pkt_type(skb) != HCI_EVENT_PKT) return false; - flt_event = (*(__u8 *)skb->data & HCI_FLT_EVENT_BITS); + if (skb->len < 1) + return true; + + event = *(__u8 *)skb->data; + flt_event = event & HCI_FLT_EVENT_BITS; if (!hci_test_bit(flt_event, &flt->event_mask)) return true; @@ -186,11 +191,17 @@ static bool is_filtered_packet(struct sock *sk, struct sk_buff *skb) if (!flt->opcode) return false; - if (flt_event == HCI_EV_CMD_COMPLETE && + if (event == HCI_EV_CMD_COMPLETE && skb->len < 5) + return true; + + if (event == HCI_EV_CMD_COMPLETE && flt->opcode != get_unaligned((__le16 *)(skb->data + 3))) return true; - if (flt_event == HCI_EV_CMD_STATUS && + if (event == HCI_EV_CMD_STATUS && skb->len < 6) + return true; + + if (event == HCI_EV_CMD_STATUS && flt->opcode != get_unaligned((__le16 *)(skb->data + 4))) return true; @@ -1881,7 +1892,8 @@ static int hci_sock_sendmsg(struct socket *sock, struct msghdr *msg, u16 ocf = hci_opcode_ocf(opcode); if (((ogf > HCI_SFLT_MAX_OGF) || - !hci_test_bit(ocf & HCI_FLT_OCF_BITS, + (ocf > HCI_FLT_OCF_BITS) || + !hci_test_bit(ocf, &hci_sec_filter.ocf_mask[ogf])) && !capable(CAP_NET_RAW)) { err = -EPERM; diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index c8d14128c363d..9d9e270492f93 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -540,8 +540,6 @@ static int adv_timeout_expire_sync(struct hci_dev *hdev, void *data) { u8 instance = *(u8 *)data; - kfree(data); - hci_clear_adv_instance_sync(hdev, NULL, instance, false); if (list_empty(&hdev->adv_instances)) @@ -550,6 +548,12 @@ static int adv_timeout_expire_sync(struct hci_dev *hdev, void *data) return 0; } +static void adv_timeout_expire_destroy(struct hci_dev *hdev, void *data, + int err) +{ + kfree(data); +} + static void adv_timeout_expire(struct work_struct *work) { u8 *inst_ptr; @@ -570,7 +574,9 @@ static void adv_timeout_expire(struct work_struct *work) goto unlock; *inst_ptr = hdev->cur_adv_instance; - hci_cmd_sync_queue(hdev, adv_timeout_expire_sync, inst_ptr, NULL); + if (hci_cmd_sync_queue(hdev, adv_timeout_expire_sync, inst_ptr, + adv_timeout_expire_destroy) < 0) + kfree(inst_ptr); unlock: hci_dev_unlock(hdev); @@ -1281,6 +1287,7 @@ hci_set_ext_adv_params_sync(struct hci_dev *hdev, u8 instance, } static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { DEFINE_FLEX(struct hci_cp_le_set_ext_adv_data, pdu, data, length, HCI_MAX_EXT_AD_LENGTH); @@ -1369,6 +1376,7 @@ int hci_update_adv_data_sync(struct hci_dev *hdev, u8 instance) } int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { struct hci_cp_le_set_ext_adv_params cp; struct hci_rp_le_set_ext_adv_params rp; @@ -1525,6 +1533,7 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) } static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { DEFINE_FLEX(struct hci_cp_le_set_ext_scan_rsp_data, pdu, data, length, HCI_MAX_EXT_AD_LENGTH); @@ -1578,6 +1587,7 @@ static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) } static int __hci_set_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { struct hci_cp_le_set_scan_rsp_data cp; u8 len; @@ -1719,6 +1729,7 @@ static int hci_set_per_adv_params_sync(struct hci_dev *hdev, u8 instance, } static int hci_set_per_adv_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { DEFINE_FLEX(struct hci_cp_le_set_per_adv_data, pdu, data, length, HCI_MAX_PER_AD_LENGTH); @@ -5374,6 +5385,7 @@ int hci_dev_open_sync(struct hci_dev *hdev) if (hdev->req_skb) { kfree_skb(hdev->req_skb); hdev->req_skb = NULL; + hci_dev_clear_flag(hdev, HCI_CMD_PENDING); } clear_bit(HCI_RUNNING, &hdev->flags); @@ -5558,6 +5570,7 @@ int hci_dev_close_sync(struct hci_dev *hdev) if (hdev->req_skb) { kfree_skb(hdev->req_skb); hdev->req_skb = NULL; + hci_dev_clear_flag(hdev, HCI_CMD_PENDING); } clear_bit(HCI_RUNNING, &hdev->flags); @@ -5574,7 +5587,9 @@ int hci_dev_close_sync(struct hci_dev *hdev) memset(hdev->eir, 0, sizeof(hdev->eir)); memset(hdev->dev_class, 0, sizeof(hdev->dev_class)); bacpy(&hdev->random_addr, BDADDR_ANY); + hci_dev_lock(hdev); hci_codec_list_clear(&hdev->local_codecs); + hci_dev_unlock(hdev); hci_dev_put(hdev); return err; @@ -6313,6 +6328,8 @@ static int hci_pause_discovery_sync(struct hci_dev *hdev) static int hci_update_event_filter_sync(struct hci_dev *hdev) { struct bdaddr_list_with_flags *b; + bdaddr_t *accept_list; + size_t i, num_entries = 0; u8 scan = SCAN_DISABLED; bool scanning = test_bit(HCI_PSCAN, &hdev->flags); int err; @@ -6329,23 +6346,49 @@ static int hci_update_event_filter_sync(struct hci_dev *hdev) /* Always clear event filter when starting */ hci_clear_event_filter_sync(hdev); - list_for_each_entry(b, &hdev->accept_list, list) { - if (!(b->flags & HCI_CONN_FLAG_REMOTE_WAKEUP)) - continue; + hci_dev_lock(hdev); - bt_dev_dbg(hdev, "Adding event filters for %pMR", &b->bdaddr); + list_for_each_entry(b, &hdev->accept_list, list) + if (b->flags & HCI_CONN_FLAG_REMOTE_WAKEUP) + num_entries++; - err = hci_set_event_filter_sync(hdev, HCI_FLT_CONN_SETUP, - HCI_CONN_SETUP_ALLOW_BDADDR, - &b->bdaddr, - HCI_CONN_SETUP_AUTO_ON); + if (!num_entries) { + hci_dev_unlock(hdev); + goto update_scan; + } + + accept_list = kmalloc_array(num_entries, sizeof(*accept_list), + GFP_KERNEL); + if (!accept_list) { + hci_dev_unlock(hdev); + return -ENOMEM; + } + + i = 0; + list_for_each_entry(b, &hdev->accept_list, list) + if (b->flags & HCI_CONN_FLAG_REMOTE_WAKEUP) + bacpy(&accept_list[i++], &b->bdaddr); + + hci_dev_unlock(hdev); + + for (i = 0; i < num_entries; i++) { + bt_dev_dbg(hdev, "Adding event filters for %pMR", + &accept_list[i]); + + err = hci_set_event_filter_sync(hdev, HCI_FLT_CONN_SETUP, + HCI_CONN_SETUP_ALLOW_BDADDR, + &accept_list[i], + HCI_CONN_SETUP_AUTO_ON); if (err) bt_dev_err(hdev, "Failed to set event filter for %pMR", - &b->bdaddr); + &accept_list[i]); else scan = SCAN_PAGE; } + kfree(accept_list); + +update_scan: if (scan && !scanning) hci_write_scan_enable_sync(hdev, scan); else if (!scan && scanning) @@ -7180,8 +7223,13 @@ static void create_le_conn_complete(struct hci_dev *hdev, void *data, int err) goto unlock; } - /* Check if connection is still pending */ - if (conn != hci_lookup_le_connect(hdev)) + /* Check if this connection is still pending. + * + * hci_lookup_le_connect() returns only the first LE connection + * in BT_CONNECT, which is not necessarily this one when two are + * pending at once, so ask the connection itself. + */ + if (conn->state != BT_CONNECT) goto unlock; /* Flush to make sure we send create conn cancel command if needed */ diff --git a/net/bluetooth/hci_sysfs.c b/net/bluetooth/hci_sysfs.c index 8957ce7c21b76..c2065abf753e0 100644 --- a/net/bluetooth/hci_sysfs.c +++ b/net/bluetooth/hci_sysfs.c @@ -13,7 +13,10 @@ static const struct class bt_class = { static void bt_link_release(struct device *dev) { struct hci_conn *conn = to_hci_conn(dev); + struct device *parent = dev->parent; + kfree(conn); + put_device(parent); } static const struct device_type bt_link = { @@ -21,6 +24,16 @@ static const struct device_type bt_link = { .release = bt_link_release, }; +/* + * The rfcomm tty device will possibly retain even when conn + * is down, and sysfs doesn't support move zombie device, + * so we should move the device before conn device is destroyed. + */ +static int __match_tty(struct device *dev, const void *data) +{ + return !strncmp(dev_name(dev), "rfcomm", 6); +} + void hci_conn_init_sysfs(struct hci_conn *conn) { struct hci_dev *hdev = conn->hdev; @@ -29,7 +42,7 @@ void hci_conn_init_sysfs(struct hci_conn *conn) conn->dev.type = &bt_link; conn->dev.class = &bt_class; - conn->dev.parent = &hdev->dev; + conn->dev.parent = get_device(&hdev->dev); device_initialize(&conn->dev); } @@ -69,7 +82,7 @@ void hci_conn_del_sysfs(struct hci_conn *conn) while (1) { struct device *dev; - dev = device_find_any_child(&conn->dev); + dev = device_find_child(&conn->dev, NULL, __match_tty); if (!dev) break; device_move(dev, NULL, DPM_ORDER_DEV_LAST); diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index a461c8a4efed4..7657c2a0abbf7 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -496,6 +496,7 @@ static int iso_connect_cis(struct sock *sk) struct hci_dev *hdev; bdaddr_t src, dst; u8 src_type; + bool already_attached; int err; lock_sock(sk); @@ -568,8 +569,14 @@ static int iso_connect_cis(struct sock *sk) goto unlock; } + iso_conn_lock(conn); + already_attached = iso_pi(sk)->conn == conn && conn->sk == sk; + iso_conn_unlock(conn); + err = iso_chan_add(conn, sk, NULL); iso_conn_put(conn); + if (already_attached || err == -EBUSY) + hci_conn_drop(hcon); if (err) goto unlock; @@ -819,19 +826,24 @@ static void iso_sock_destruct(struct sock *sk) skb_queue_purge(&sk->sk_error_queue); } -static void iso_sock_cleanup_listen(struct sock *parent) +/* Close not yet accepted channels */ +static void iso_sock_flush_accept_q(struct sock *parent) { struct sock *sk; - BT_DBG("parent %p", parent); - - /* Close not yet accepted channels */ while ((sk = bt_accept_dequeue(parent, NULL))) { iso_sock_close(sk); iso_sock_kill(sk); /* Drop the reference handed back by bt_accept_dequeue(). */ sock_put(sk); } +} + +static void iso_sock_cleanup_listen(struct sock *parent) +{ + BT_DBG("parent %p", parent); + + iso_sock_flush_accept_q(parent); /* If listening socket has a hcon, properly disconnect it */ if (iso_pi(parent)->conn && iso_pi(parent)->conn->hcon) { @@ -1536,6 +1548,7 @@ static int iso_sock_getname(struct socket *sock, struct sockaddr *addr, lock_sock(sk); + memset(sa, 0, sizeof(struct sockaddr_iso)); addr->sa_family = AF_BLUETOOTH; if (peer) { @@ -1546,6 +1559,7 @@ static int iso_sock_getname(struct socket *sock, struct sockaddr *addr, sa->iso_bdaddr_type = iso_pi(sk)->dst_type; if (hcon && (hcon->type == BIS_LINK || hcon->type == PA_LINK)) { + memset(sa->iso_bc, 0, sizeof(struct sockaddr_iso_bc)); sa->iso_bc->bc_sid = iso_pi(sk)->bc_sid; sa->iso_bc->bc_num_bis = iso_pi(sk)->bc_num_bis; memcpy(sa->iso_bc->bc_bis, iso_pi(sk)->bc_bis, @@ -1658,9 +1672,9 @@ static void iso_conn_defer_accept(struct hci_conn *conn) hci_send_cmd(hdev, HCI_OP_LE_ACCEPT_CIS, sizeof(cp), &cp); } -static void iso_conn_big_sync(struct sock *sk) +static int iso_conn_big_sync(struct sock *sk) { - int err; + int err = 0; struct hci_dev *hdev; struct iso_conn *conn; bdaddr_t src, dst; @@ -1675,7 +1689,7 @@ static void iso_conn_big_sync(struct sock *sk) hdev = hci_get_route(&dst, &src, src_type); if (!hdev) - return; + return -EHOSTUNREACH; /* hci_le_big_create_sync requires hdev lock to be held, since * it enqueues the HCI LE BIG Create Sync command via @@ -1691,8 +1705,10 @@ static void iso_conn_big_sync(struct sock *sk) * both before dereferencing conn->hcon. */ conn = iso_pi(sk)->conn; - if (!conn || !conn->hcon) + if (!conn || !conn->hcon) { + err = -ENOTCONN; goto unlock; + } if (!test_and_set_bit(BT_SK_BIG_SYNC, &iso_pi(sk)->flags)) { err = hci_conn_big_create_sync(hdev, conn->hcon, @@ -1708,6 +1724,8 @@ static void iso_conn_big_sync(struct sock *sk) release_sock(sk); hci_dev_unlock(hdev); hci_dev_put(hdev); + + return err; } static int iso_sock_recvmsg(struct socket *sock, struct msghdr *msg, @@ -1731,11 +1749,35 @@ static int iso_sock_recvmsg(struct socket *sock, struct msghdr *msg, switch (sk->sk_state) { case BT_CONNECT2: if (test_bit(BT_SK_PA_SYNC, &pi->flags)) { + /* Move to BT_LISTEN before requesting the BIG + * sync: the BIS connections are matched to a + * parent socket in BT_LISTEN state, and they + * may be notified before the request returns. + */ + sk->sk_state = BT_LISTEN; + release_sock(sk); - iso_conn_big_sync(sk); + err = iso_conn_big_sync(sk); lock_sock(sk); - sk->sk_state = BT_LISTEN; + /* The socket lock was dropped, so the + * connection may have been torn down + * meanwhile and iso_chan_del() may have + * already moved the socket to BT_CLOSED. + * Only move back if the BIG sync could not be + * started and nothing else has changed the + * state. + */ + if (err && sk->sk_state == BT_LISTEN) { + /* Discard any child socket that may + * have been queued while the socket + * was in BT_LISTEN, as the cleanup of + * BT_CONNECT2 doesn't drain the + * accept queue. + */ + iso_sock_flush_accept_q(sk); + sk->sk_state = BT_CONNECT2; + } } else { iso_conn_defer_accept(pi->conn->hcon); sk->sk_state = BT_CONFIG; @@ -1745,11 +1787,22 @@ static int iso_sock_recvmsg(struct socket *sock, struct msghdr *msg, break; case BT_CONNECTED: if (test_bit(BT_SK_PA_SYNC, &iso_pi(sk)->flags)) { + /* As above, the BIS connections may be + * notified before the request returns. + */ + sk->sk_state = BT_LISTEN; + release_sock(sk); - iso_conn_big_sync(sk); + err = iso_conn_big_sync(sk); lock_sock(sk); - sk->sk_state = BT_LISTEN; + if (err && sk->sk_state == BT_LISTEN) { + /* As above, don't leave any child + * socket behind in the accept queue. + */ + iso_sock_flush_accept_q(sk); + sk->sk_state = BT_CONNECTED; + } early_ret = true; } @@ -2261,10 +2314,19 @@ static void iso_conn_ready(struct iso_conn *conn) lock_sock(parent); + /* The listener may have been closed concurrently. */ + if (parent->sk_state != BT_LISTEN || + sock_flag(parent, SOCK_ZAPPED)) { + release_sock(parent); + sock_put(parent); + return; + } + sk = iso_sock_alloc(sock_net(parent), NULL, BTPROTO_ISO, GFP_ATOMIC, 0); if (!sk) { release_sock(parent); + sock_put(parent); return; } diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 1156aba4e83c4..9a1b0ae07679e 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -1337,7 +1337,7 @@ static void l2cap_le_connect(struct l2cap_chan *chan) struct l2cap_ecred_conn_data { struct { struct l2cap_ecred_conn_req_hdr req; - __le16 scid[5]; + __le16 scid[L2CAP_ECRED_CONN_SCID_MAX]; } __packed pdu; struct l2cap_chan *chan; struct pid *pid; @@ -1352,7 +1352,7 @@ static void l2cap_ecred_defer_connect(struct l2cap_chan *chan, void *data) if (chan == conn->chan) return; - if (!test_and_clear_bit(FLAG_DEFER_SETUP, &chan->flags)) + if (!test_bit(FLAG_DEFER_SETUP, &chan->flags)) return; pid = chan->ops->get_peer_pid(chan); @@ -1362,9 +1362,16 @@ static void l2cap_ecred_defer_connect(struct l2cap_chan *chan, void *data) chan->mode != L2CAP_MODE_EXT_FLOWCTL || chan->state != BT_CONNECT) return; + if (!test_and_clear_bit(FLAG_DEFER_SETUP, &chan->flags)) + return; + if (test_and_set_bit(FLAG_ECRED_CONN_REQ_SENT, &chan->flags)) return; + /* Unreachable, checked in l2cap_connect (+timer drops it if reached) */ + if (WARN_ON_ONCE(conn->count >= ARRAY_SIZE(conn->pdu.scid))) + return; + l2cap_ecred_init(chan, 0); /* Set the same ident so we can match on the rsp */ @@ -1833,7 +1840,10 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) hci_chan_del(conn->hchan); conn->hchan = NULL; + spin_lock(&hcon->proto_lock); hcon->l2cap_data = NULL; + spin_unlock(&hcon->proto_lock); + mutex_unlock(&conn->lock); l2cap_conn_put(conn); } @@ -3890,6 +3900,9 @@ static void l2cap_ecred_rsp_defer(struct l2cap_chan *chan, void *data) struct l2cap_ecred_conn_rsp *rsp_flex = container_of(&rsp->pdu.rsp, struct l2cap_ecred_conn_rsp, hdr); + if (chan->mode != L2CAP_MODE_EXT_FLOWCTL) + return; + /* Check if channel for outgoing connection or if it wasn't deferred * since in those cases it must be skipped. */ @@ -3900,6 +3913,10 @@ static void l2cap_ecred_rsp_defer(struct l2cap_chan *chan, void *data) /* Reset ident so only one response is sent */ chan->ident = 0; + /* Unreachable, check in l2cap_ecred_conn_req. If reached, drop rest */ + if (WARN_ON_ONCE(rsp->count >= ARRAY_SIZE(rsp->pdu.scid))) + rsp->pdu.rsp.result = cpu_to_le16(L2CAP_CR_LE_NO_MEM); + /* Include all channels pending with the same ident */ if (!rsp->pdu.rsp.result) rsp_flex->dcid[rsp->count++] = cpu_to_le16(chan->scid); @@ -5059,6 +5076,7 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, __set_chan_timer(chan, chan->ops->get_sndtimeo(chan)); chan->ident = cmd->ident; + chan->mode = L2CAP_MODE_LE_FLOWCTL; if (test_bit(FLAG_DEFER_SETUP, &chan->flags)) { l2cap_state_change(chan, BT_CONNECT2); @@ -6683,9 +6701,17 @@ static int l2cap_stream_rx(struct l2cap_chan *chan, struct l2cap_ctrl *control, static int l2cap_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) { struct l2cap_ctrl *control = &bt_cb(skb)->l2cap; - u16 len; + u16 len, min_len; u8 event; + min_len = test_bit(FLAG_EXT_CTRL, &chan->flags) ? + L2CAP_EXT_CTRL_SIZE : L2CAP_ENH_CTRL_SIZE; + if (chan->fcs == L2CAP_FCS_CRC16) + min_len += L2CAP_FCS_SIZE; + + if (skb->len < min_len) + goto drop; + __unpack_control(chan, skb); len = skb->len; @@ -7168,8 +7194,6 @@ static struct l2cap_conn *l2cap_conn_add(struct hci_conn *hcon) } kref_init(&conn->ref); - hcon->l2cap_data = conn; - conn->hcon = hci_conn_get(hcon); conn->hchan = hchan; BT_DBG("hcon %p conn %p hchan %p", hcon, conn, hchan); @@ -7198,6 +7222,11 @@ static struct l2cap_conn *l2cap_conn_add(struct hci_conn *hcon) conn->disc_reason = HCI_ERROR_REMOTE_USER_TERM; + spin_lock(&hcon->proto_lock); + conn->hcon = hci_conn_get(hcon); + hcon->l2cap_data = conn; + spin_unlock(&hcon->proto_lock); + return conn; } @@ -7359,6 +7388,9 @@ int l2cap_chan_connect(struct l2cap_chan *chan, __le16 psm, u16 cid, goto done; } + mutex_lock(&conn->lock); + l2cap_chan_lock(chan); + if (chan->mode == L2CAP_MODE_EXT_FLOWCTL) { struct l2cap_chan_data data; @@ -7366,19 +7398,20 @@ int l2cap_chan_connect(struct l2cap_chan *chan, __le16 psm, u16 cid, data.pid = chan->ops->get_peer_pid(chan); data.count = 1; - l2cap_chan_list(conn, l2cap_chan_by_pid, &data); + __l2cap_chan_list(conn, l2cap_chan_by_pid, &data); + + /* Leave room for non-deferred channel that ends the group. */ + if (test_bit(FLAG_DEFER_SETUP, &chan->flags)) + data.count += 1; /* Check if there isn't too many channels being connected */ if (data.count > L2CAP_ECRED_CONN_SCID_MAX) { hci_conn_drop(hcon); err = -EPROTO; - goto done; + goto chan_unlock; } } - mutex_lock(&conn->lock); - l2cap_chan_lock(chan); - if (cid && __l2cap_get_chan_by_dcid(conn, cid)) { hci_conn_drop(hcon); err = -EBUSY; @@ -7582,13 +7615,18 @@ static void l2cap_connect_cfm(struct hci_conn *hcon, u8 status) int l2cap_disconn_ind(struct hci_conn *hcon) { - struct l2cap_conn *conn = hcon->l2cap_data; + struct l2cap_conn *conn; + int ret = HCI_ERROR_REMOTE_USER_TERM; BT_DBG("hcon %p", hcon); - if (!conn) - return HCI_ERROR_REMOTE_USER_TERM; - return conn->disc_reason; + spin_lock(&hcon->proto_lock); + conn = hcon->l2cap_data; + if (conn) + ret = conn->disc_reason; + spin_unlock(&hcon->proto_lock); + + return ret; } static void l2cap_disconn_cfm(struct hci_conn *hcon, u8 reason) diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 4058ff50cc27a..143654a47989d 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -436,11 +436,26 @@ static int l2cap_get_mode(struct l2cap_chan *chan) return -EINVAL; } +static struct l2cap_conn *l2cap_chan_conn(struct l2cap_chan *chan) +{ + lockdep_assert_held(&chan->lock); + + /* l2cap_conn_del() sets FLAG_DEL while holding chan->lock before + * conn->hcon is deleted. If not set and conn is non-NULL, conn->hcon + * remains alive during this chan->lock critical section. + */ + if (test_bit(FLAG_DEL, &chan->flags)) + return NULL; + + return chan->conn; +} + static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, sockopt_t *sopt) { struct sock *sk = sock->sk; struct l2cap_chan *chan = l2cap_pi(sk)->chan; + struct l2cap_conn *conn; struct l2cap_options opts; struct l2cap_conninfo cinfo; int err = 0; @@ -451,6 +466,7 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, len = sopt->optlen; + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -537,9 +553,15 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, break; } + conn = l2cap_chan_conn(chan); + if (!conn) { + err = -ENOTCONN; + break; + } + memset(&cinfo, 0, sizeof(cinfo)); - cinfo.hci_handle = chan->conn->hcon->handle; - memcpy(cinfo.dev_class, chan->conn->hcon->dev_class, 3); + cinfo.hci_handle = conn->hcon->handle; + memcpy(cinfo.dev_class, conn->hcon->dev_class, 3); len = min(len, sizeof(cinfo)); if (copy_to_iter(&cinfo, len, &sopt->iter_out) != len) @@ -553,6 +575,8 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); + return err; } @@ -561,6 +585,7 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, { struct sock *sk = sock->sk; struct l2cap_chan *chan = l2cap_pi(sk)->chan; + struct l2cap_conn *conn; struct bt_security sec; struct bt_power pwr; int len, mode, err = 0; @@ -578,6 +603,7 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, len = sopt->optlen; + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -589,12 +615,14 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, break; } + conn = l2cap_chan_conn(chan); + memset(&sec, 0, sizeof(sec)); - if (chan->conn) { - sec.level = chan->conn->hcon->sec_level; + if (conn) { + sec.level = conn->hcon->sec_level; if (sk->sk_state == BT_CONNECTED) - sec.key_size = chan->conn->hcon->enc_key_size; + sec.key_size = conn->hcon->enc_key_size; } else { sec.level = chan->sec_level; } @@ -678,12 +706,14 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, break; case BT_PHY: - if (sk->sk_state != BT_CONNECTED) { + conn = l2cap_chan_conn(chan); + + if (sk->sk_state != BT_CONNECTED || !conn) { err = -ENOTCONN; break; } - opt = hci_conn_get_phy(chan->conn->hcon); + opt = hci_conn_get_phy(conn->hcon); if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) != sizeof(opt)) @@ -719,6 +749,7 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); return err; } @@ -749,6 +780,7 @@ static int l2cap_sock_setsockopt_old(struct socket *sock, int optname, BT_DBG("sk %p", sk); + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -850,6 +882,7 @@ static int l2cap_sock_setsockopt_old(struct socket *sock, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); return err; } @@ -913,6 +946,7 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, if (level != SOL_BLUETOOTH) return -ENOPROTOOPT; + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -938,11 +972,10 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, chan->sec_level = sec.level; - if (!chan->conn) + conn = l2cap_chan_conn(chan); + if (!conn) break; - conn = chan->conn; - /* change security for LE channels */ if (chan->scid == L2CAP_CID_ATT) { if (smp_conn_security(conn->hcon, sec.level)) { @@ -997,7 +1030,8 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, } if (opt == BT_FLUSHABLE_OFF) { - conn = chan->conn; + conn = l2cap_chan_conn(chan); + /* proceed further only when we have l2cap_conn and No Flush support in the LM */ if (!conn || !lmp_no_flush_capable(conn->hcon->hdev)) { @@ -1083,7 +1117,8 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, break; case BT_PHY: - if (sk->sk_state != BT_CONNECTED) { + conn = l2cap_chan_conn(chan); + if (sk->sk_state != BT_CONNECTED || !conn) { err = -ENOTCONN; break; } @@ -1093,10 +1128,6 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, if (err) break; - if (!chan->conn) - break; - - conn = chan->conn; err = hci_conn_set_phy(conn->hcon, phys); break; @@ -1139,6 +1170,7 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); return err; } @@ -1312,7 +1344,12 @@ static void l2cap_sock_kill(struct sock *sk) BT_DBG("sk %p state %s", sk, state_to_string(sk->sk_state)); + /* Take lock to synchronize against access without owning sk->sk_socket, + * eg. in l2cap_sock_cleanup_listen(). proto_ops etc. don't need lock. + */ + lock_sock(sk); l2cap_sock_put_chan(sk); + release_sock(sk); /* Kill poor orphan */ sock_set_flag(sk, SOCK_DEAD); @@ -1515,14 +1552,10 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) * establish sk_lock -> conn->lock and invert the established * conn->lock -> chan->lock -> sk_lock order (lockdep deadlock). * - * Instead, briefly take the child sk lock to fetch and pin its chan. - * l2cap_conn_del() reaches the chan free only via - * l2cap_chan_del() -> l2cap_sock_teardown_cb(), which itself takes - * the child sk lock; holding it across l2cap_chan_hold_unless_zero() - * therefore guarantees the chan cannot be freed while we read and - * pin it (hold_unless_zero() additionally skips a chan already past - * its last reference). We then drop the sk lock before taking - * chan->lock, so sk and chan locks are never held together. + * Instead, briefly take the child sk lock to synchronize vs. + * l2cap_sock_kill that puts l2cap_pi(sk)->chan. We then drop the sk + * lock before taking chan->lock, so sk and chan locks are never held + * together. * * Since we cannot call l2cap_chan_close() without conn->lock, * schedule l2cap_chan_timeout to close the channel; it already @@ -1532,10 +1565,12 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) struct l2cap_chan *chan; lock_sock_nested(sk, L2CAP_NESTING_NORMAL); - chan = l2cap_chan_hold_unless_zero(l2cap_pi(sk)->chan); + chan = l2cap_pi(sk)->chan; + if (chan) + l2cap_chan_hold(chan); release_sock(sk); if (!chan) { - /* l2cap_conn_del() already tearing this child down */ + /* Already torn down */ sock_put(sk); continue; } @@ -1567,6 +1602,11 @@ static int l2cap_sock_new_connection_cb(struct l2cap_chan *chan, lock_sock(parent); + if (parent->sk_state != BT_LISTEN) { + release_sock(parent); + return -EINVAL; + } + /* Check for backlog size */ if (sk_acceptq_is_full(parent)) { BT_DBG("backlog full %d", parent->sk_ack_backlog); @@ -1730,10 +1770,14 @@ static void l2cap_sock_state_change_cb(struct l2cap_chan *chan, int state, if (!sk) return; + lock_sock(sk); + sk->sk_state = state; if (err) sk->sk_err = err; + + release_sock(sk); } static struct sk_buff *l2cap_sock_alloc_skb_cb(struct l2cap_chan *chan, @@ -1809,6 +1853,8 @@ static void l2cap_sock_resume_cb(struct l2cap_chan *chan) if (!sk) return; + lock_sock(sk); + if (test_and_clear_bit(FLAG_PENDING_SECURITY, &chan->flags)) { sk->sk_state = BT_CONNECTED; chan->state = BT_CONNECTED; @@ -1816,6 +1862,8 @@ static void l2cap_sock_resume_cb(struct l2cap_chan *chan) clear_bit(BT_SK_SUSPEND, &bt_sk(sk)->flags); sk->sk_state_change(sk); + + release_sock(sk); } static void l2cap_sock_set_shutdown_cb(struct l2cap_chan *chan) diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 167d75e345266..2a177cd17a9ef 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -496,13 +496,9 @@ static int read_unconf_index_list(struct sock *sk, struct hci_dev *hdev, read_lock(&hci_dev_list_lock); - count = 0; - list_for_each_entry(d, &hci_dev_list, list) { - if (hci_dev_test_flag(d, HCI_UNCONFIGURED)) - count++; - } + count = list_count_nodes(&hci_dev_list); - rp_len = sizeof(*rp) + (2 * count); + rp_len = sizeof(*rp) + (sizeof(__le16) * count); rp = kmalloc(rp_len, GFP_ATOMIC); if (!rp) { read_unlock(&hci_dev_list_lock); @@ -2310,6 +2306,8 @@ static void mesh_send_start_complete(struct hci_dev *hdev, void *data, int err) hci_dev_clear_flag(hdev, HCI_MESH_SENDING); /* Send Complete Error Code for handle */ mesh_send_complete(hdev, mesh_tx, false); + if (err != -ECANCELED) + mesh_next(hdev, NULL, 0); return; } @@ -2419,23 +2417,36 @@ static int send_cancel(struct hci_dev *hdev, void *data) do { mesh_tx = mgmt_mesh_next(hdev, cmd->sk); - if (mesh_tx) - mesh_send_complete(hdev, mesh_tx, false); + if (mesh_tx) { + if (!hci_cmd_sync_dequeue(hdev, mesh_send_sync, + mesh_tx, NULL)) + mesh_send_complete(hdev, mesh_tx, false); + } } while (mesh_tx); } else { mesh_tx = mgmt_mesh_find(hdev, cancel->handle); - if (mesh_tx && mesh_tx->sk == cmd->sk) - mesh_send_complete(hdev, mesh_tx, false); + if (mesh_tx && mesh_tx->sk == cmd->sk) { + if (!hci_cmd_sync_dequeue(hdev, mesh_send_sync, + mesh_tx, NULL)) + mesh_send_complete(hdev, mesh_tx, false); + } } mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, 0, NULL, 0); - mgmt_pending_free(cmd); + + if (!hci_dev_test_flag(hdev, HCI_MESH_SENDING)) + mesh_next(hdev, NULL, 0); return 0; } +static void send_cancel_destroy(struct hci_dev *hdev, void *data, int err) +{ + mgmt_pending_free(data); +} + static int mesh_send_cancel(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -2456,7 +2467,8 @@ static int mesh_send_cancel(struct sock *sk, struct hci_dev *hdev, if (!cmd) err = -ENOMEM; else - err = hci_cmd_sync_queue(hdev, send_cancel, cmd, NULL); + err = hci_cmd_sync_queue(hdev, send_cancel, cmd, + send_cancel_destroy); if (err < 0) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, @@ -2642,7 +2654,7 @@ static int send_hci_cmd_sync(struct hci_dev *hdev, void *data) if (IS_ERR(skb)) { mgmt_cmd_status(cmd->sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, mgmt_status(PTR_ERR(skb))); - goto done; + return 0; } mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, 0, @@ -2650,12 +2662,14 @@ static int send_hci_cmd_sync(struct hci_dev *hdev, void *data) kfree_skb(skb); -done: - mgmt_pending_free(cmd); - return 0; } +static void send_hci_cmd_sync_destroy(struct hci_dev *hdev, void *data, int err) +{ + mgmt_pending_free(data); +} + static int mgmt_hci_cmd_sync(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -2668,12 +2682,21 @@ static int mgmt_hci_cmd_sync(struct sock *sk, struct hci_dev *hdev, return mgmt_cmd_status(sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, MGMT_STATUS_INVALID_PARAMS); + /* The HCI command header carries the parameter length in a u8, a + * larger value would be truncated there while the parameters are + * still appended to the frame in full. + */ + if (le16_to_cpu(cp->params_len) > U8_MAX) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, + MGMT_STATUS_INVALID_PARAMS); + hci_dev_lock(hdev); cmd = mgmt_pending_new(sk, MGMT_OP_HCI_CMD_SYNC, hdev, data, len); if (!cmd) err = -ENOMEM; else - err = hci_cmd_sync_queue(hdev, send_hci_cmd_sync, cmd, NULL); + err = hci_cmd_sync_queue(hdev, send_hci_cmd_sync, cmd, + send_hci_cmd_sync_destroy); if (err < 0) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, @@ -6147,6 +6170,7 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, struct mgmt_pending_cmd *cmd; const u16 max_uuid_count = ((U16_MAX - sizeof(*cp)) / 16); u16 uuid_count, expected_len; + u8 (*uuids)[16] = NULL; u8 status; int err; @@ -6223,12 +6247,10 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, hdev->discovery.result_filtering = true; hdev->discovery.type = cp->type; hdev->discovery.rssi = cp->rssi; - hdev->discovery.uuid_count = uuid_count; if (uuid_count > 0) { - hdev->discovery.uuids = kmemdup(cp->uuids, uuid_count * 16, - GFP_KERNEL); - if (!hdev->discovery.uuids) { + uuids = kmemdup(cp->uuids, uuid_count * sizeof(*uuids), GFP_KERNEL); + if (!uuids) { err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_START_SERVICE_DISCOVERY, MGMT_STATUS_FAILED, @@ -6238,6 +6260,11 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, } } + spin_lock(&hdev->discovery.lock); + hdev->discovery.uuids = uuids; + hdev->discovery.uuid_count = uuid_count; + spin_unlock(&hdev->discovery.lock); + err = hci_cmd_sync_queue(hdev, start_discovery_sync, cmd, start_discovery_complete); if (err < 0) { @@ -10367,6 +10394,7 @@ static bool is_filter_match(struct hci_dev *hdev, s8 rssi, u8 *eir, !hci_test_quirk(hdev, HCI_QUIRK_STRICT_DUPLICATE_FILTER)))) return false; + spin_lock(&hdev->discovery.lock); if (hdev->discovery.uuid_count != 0) { /* If a list of UUIDs is provided in filter, results with no * matching UUID should be dropped. @@ -10375,9 +10403,12 @@ static bool is_filter_match(struct hci_dev *hdev, s8 rssi, u8 *eir, hdev->discovery.uuids) && !eir_has_uuids(scan_rsp, scan_rsp_len, hdev->discovery.uuid_count, - hdev->discovery.uuids)) + hdev->discovery.uuids)) { + spin_unlock(&hdev->discovery.lock); return false; + } } + spin_unlock(&hdev->discovery.lock); /* If duplicate filtering does not report RSSI changes, then restart * scanning to ensure updated result with updated RSSI values. diff --git a/net/bluetooth/msft.c b/net/bluetooth/msft.c index d7badce8746ca..d9dd722db3ebd 100644 --- a/net/bluetooth/msft.c +++ b/net/bluetooth/msft.c @@ -165,6 +165,11 @@ static bool read_supported_features(struct hci_dev *hdev, if (rp->sub_opcode != MSFT_OP_READ_SUPPORTED_FEATURES) goto failed; + if (skb->len < sizeof(*rp) + rp->evt_prefix_len) { + bt_dev_err(hdev, "MSFT event prefix length mismatch"); + goto failed; + } + if (rp->evt_prefix_len > 0) { msft->evt_prefix = kmemdup(rp->evt_prefix, rp->evt_prefix_len, GFP_KERNEL); @@ -764,8 +769,8 @@ void msft_register(struct hci_dev *hdev) INIT_LIST_HEAD(&msft->handle_map); INIT_LIST_HEAD(&msft->address_filters); - hdev->msft_data = msft; mutex_init(&msft->filter_lock); + hdev->msft_data = msft; } void msft_release(struct hci_dev *hdev) diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c index 2e8c080b4d9eb..3f757198051a2 100644 --- a/net/bluetooth/rfcomm/core.c +++ b/net/bluetooth/rfcomm/core.c @@ -1331,7 +1331,10 @@ static struct rfcomm_session *rfcomm_recv_disc(struct rfcomm_session *s, return s; } -void rfcomm_dlc_accept(struct rfcomm_dlc *d) +/* Must be called with rfcomm_mutex held, so that the session cannot be + * unlinked from under us. + */ +static void __rfcomm_dlc_accept(struct rfcomm_dlc *d) { struct sock *sk = d->session->sock->sk; struct l2cap_conn *conn = l2cap_pi(sk)->chan->conn; @@ -1353,6 +1356,21 @@ void rfcomm_dlc_accept(struct rfcomm_dlc *d) rfcomm_send_msc(d->session, 1, d->dlci, d->v24_sig); } +void rfcomm_dlc_accept(struct rfcomm_dlc *d) +{ + rfcomm_lock(); + + /* rfcomm_recv_disc() sets the dlc state to BT_CLOSED before calling + * __rfcomm_dlc_close(), so the RFCOMM_DEFER_SETUP handshake there is + * skipped and the session can already be unlinked by the time the + * deferred accept runs from rfcomm_sock_recvmsg(). + */ + if (d->session) + __rfcomm_dlc_accept(d); + + rfcomm_unlock(); +} + static void rfcomm_check_accept(struct rfcomm_dlc *d) { if (rfcomm_check_security(d)) { @@ -1365,7 +1383,7 @@ static void rfcomm_check_accept(struct rfcomm_dlc *d) d->state_change(d, 0); rfcomm_dlc_unlock(d); } else - rfcomm_dlc_accept(d); + __rfcomm_dlc_accept(d); } else { set_bit(RFCOMM_AUTH_PENDING, &d->flags); rfcomm_dlc_set_timer(d, RFCOMM_AUTH_TIMEOUT); @@ -1437,6 +1455,10 @@ static int rfcomm_apply_pn(struct rfcomm_dlc *d, int cr, struct rfcomm_pn *pn) d->mtu = __le16_to_cpu(pn->mtu); + /* MTU 0 causes an infinite loop when fragmenting in sendmsg */ + if (!d->mtu) + d->mtu = RFCOMM_DEFAULT_MTU; + if (cr && d->mtu > s->mtu) d->mtu = s->mtu; @@ -1795,7 +1817,8 @@ static struct rfcomm_session *rfcomm_recv_frame(struct rfcomm_session *s, return s; } - if (skb->len < sizeof(*hdr) + 1) { + if (skb->len < sizeof(*hdr) + 1 || + (!__test_ea(hdr->len) && skb->len < sizeof(*hdr) + 2)) { kfree_skb(skb); return s; } @@ -1958,7 +1981,7 @@ static void rfcomm_process_dlcs(struct rfcomm_session *s) d->state_change(d, 0); rfcomm_dlc_unlock(d); } else - rfcomm_dlc_accept(d); + __rfcomm_dlc_accept(d); } continue; } else if (test_and_clear_bit(RFCOMM_AUTH_REJECT, &d->flags)) { @@ -2195,9 +2218,13 @@ static void rfcomm_security_cfm(struct hci_conn *conn, u8 status, u8 encrypt) BT_DBG("conn %p status 0x%02x encrypt 0x%02x", conn, status, encrypt); + rfcomm_lock(); + s = rfcomm_session_get(&conn->hdev->bdaddr, &conn->dst); - if (!s) + if (!s) { + rfcomm_unlock(); return; + } list_for_each_entry_safe(d, n, &s->dlcs, list) { if (test_and_clear_bit(RFCOMM_SEC_PENDING, &d->flags)) { @@ -2229,6 +2256,8 @@ static void rfcomm_security_cfm(struct hci_conn *conn, u8 status, u8 encrypt) set_bit(RFCOMM_AUTH_REJECT, &d->flags); } + rfcomm_unlock(); + rfcomm_schedule(); } diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c index feb302a491fa0..d2855f9b3abc3 100644 --- a/net/bluetooth/rfcomm/sock.c +++ b/net/bluetooth/rfcomm/sock.c @@ -241,9 +241,7 @@ static void __rfcomm_sock_close(struct sock *sk) */ static void rfcomm_sock_close(struct sock *sk) { - lock_sock(sk); __rfcomm_sock_close(sk); - release_sock(sk); } static void rfcomm_sock_init(struct sock *sk, struct sock *parent) @@ -787,8 +785,10 @@ static int rfcomm_sock_getsockopt_old(struct socket *sock, int optname, break; case RFCOMM_CONNINFO: - if (sk->sk_state != BT_CONNECTED && - !rfcomm_pi(sk)->dlc->defer_setup) { + if ((sk->sk_state != BT_CONNECTED && + !(sk->sk_state == BT_CONNECT2 && + rfcomm_pi(sk)->dlc->defer_setup)) || + !rfcomm_pi(sk)->dlc->session) { err = -ENOTCONN; break; } @@ -904,6 +904,7 @@ static int rfcomm_sock_compat_ioctl(struct socket *sock, unsigned int cmd, unsig static int rfcomm_sock_shutdown(struct socket *sock, int how) { struct sock *sk = sock->sk; + bool cleanup_listen = false; int err = 0; BT_DBG("sock %p, sk %p", sock, sk); @@ -914,9 +915,17 @@ static int rfcomm_sock_shutdown(struct socket *sock, int how) lock_sock(sk); if (!sk->sk_shutdown) { sk->sk_shutdown = SHUTDOWN_MASK; + if (sk->sk_state == BT_LISTEN) { + /* Block new children before cleaning up without sk lock. */ + sk->sk_state = BT_CLOSED; + cleanup_listen = true; + } release_sock(sk); - __rfcomm_sock_close(sk); + if (cleanup_listen) + rfcomm_sock_cleanup_listen(sk); + else + __rfcomm_sock_close(sk); lock_sock(sk); if (sock_flag(sk, SOCK_LINGER) && sk->sk_lingertime && diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index c4470958b0d57..14bb2c82a5624 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -2268,6 +2268,23 @@ static u8 smp_cmd_security_req(struct l2cap_conn *conn, struct sk_buff *skb) bt_dev_dbg(hdev, "conn %p", conn); + /* SMP over BR/EDR only covers cross-transport key derivation; the + * Security Request procedure has no BR/EDR counterpart. Reject it + * here, otherwise smp_ltk_encrypt() finds the peer's LE LTK + * (ADDR_LE_DEV_PUBLIC and BDADDR_BREDR are both 0) and issues + * HCI_OP_LE_START_ENC on the ACL handle, which the controller + * rejects and hci_cs_le_start_enc() turns into a disconnect. Reply + * without smp_failure(): this is not an authentication failure, and + * MGMT_EV_AUTH_FAILED would make bluetoothd drop the device. + */ + if (hcon->type != LE_LINK) { + u8 reason = SMP_CMD_NOTSUPP; + + smp_send_cmd(conn, SMP_CMD_PAIRING_FAIL, sizeof(reason), + &reason); + return 0; + } + if (skb->len < sizeof(*rp)) return SMP_INVALID_PARAMS; diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c index 23eb6931a2b4a..fd65d6af4f35a 100644 --- a/net/bridge/br_arp_nd_proxy.c +++ b/net/bridge/br_arp_nd_proxy.c @@ -195,13 +195,15 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, n = neigh_lookup(&arp_tbl, &tip, vlandev); if (n) { struct net_bridge_fdb_entry *f; + u8 ha[ETH_ALEN] __aligned(2); if (!(READ_ONCE(n->nud_state) & NUD_VALID)) { neigh_release(n); return; } - f = br_fdb_find_rcu(br, n->ha, vid); + neigh_ha_snapshot(ha, n, n->dev); + f = br_fdb_find_rcu(br, ha, vid); if (f) { const struct net_bridge_port *dst = READ_ONCE(f->dst); bool replied = false; @@ -211,10 +213,10 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, br_is_neigh_suppress_enabled(dst, vid)) { if (!vid) br_arp_send(br, p, skb->dev, sip, tip, - sha, n->ha, sha, 0, 0); + sha, ha, sha, 0, 0); else br_arp_send(br, p, skb->dev, sip, tip, - sha, n->ha, sha, + sha, ha, sha, skb->vlan_proto, skb_vlan_tag_get(skb)); replied = true; @@ -252,7 +254,7 @@ struct nd_msg *br_is_nd_neigh_msg(const struct sk_buff *skb, struct nd_msg *msg) } static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, - struct sk_buff *request, struct neighbour *n, + struct sk_buff *request, struct neighbour *n, u8 *ha, __be16 vlan_proto, u16 vlan_tci) { struct net_device *dev = request->dev; @@ -310,7 +312,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, ipv6_eth_mc_map(&in6addr_linklocal_allnodes, eth_hdr(reply)->h_dest); else ether_addr_copy(eth_hdr(reply)->h_dest, daddr); - ether_addr_copy(eth_hdr(reply)->h_source, n->ha); + ether_addr_copy(eth_hdr(reply)->h_source, ha); eth_hdr(reply)->h_proto = htons(ETH_P_IPV6); reply->protocol = htons(ETH_P_IPV6); @@ -340,7 +342,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, na->icmph.icmp6_override = 1; na->icmph.icmp6_solicited = dad ? 0 : 1; na->target = ns->target; - ether_addr_copy(&na->opt[2], n->ha); + ether_addr_copy(&na->opt[2], ha); na->opt[0] = ND_OPT_TARGET_LL_ADDR; na->opt[1] = na_olen >> 3; @@ -369,7 +371,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, __vlan_hwaccel_put_tag(reply, vlan_proto, vlan_tci); netdev_dbg(dev, "nd send dev %s dst %pI6 dst_hw %pM src %pI6 src_hw %pM\n", - dev->name, &pip6->daddr, daddr, &pip6->saddr, n->ha); + dev->name, &pip6->daddr, daddr, &pip6->saddr, ha); if (p) { dev_queue_xmit(reply); @@ -472,24 +474,26 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, n = neigh_lookup(&nd_tbl, &msg->target, vlandev); if (n) { struct net_bridge_fdb_entry *f; + u8 ha[ETH_ALEN] __aligned(2); if (!(READ_ONCE(n->nud_state) & NUD_VALID)) { neigh_release(n); return; } - f = br_fdb_find_rcu(br, n->ha, vid); + neigh_ha_snapshot(ha, n, n->dev); + f = br_fdb_find_rcu(br, ha, vid); if (f) { const struct net_bridge_port *dst = READ_ONCE(f->dst); bool replied = false; if (br_is_neigh_suppress_enabled(dst, vid)) { if (vid != 0) - br_nd_send(br, p, skb, n, + br_nd_send(br, p, skb, n, ha, skb->vlan_proto, skb_vlan_tag_get(skb)); else - br_nd_send(br, p, skb, n, 0, 0); + br_nd_send(br, p, skb, n, ha, 0, 0); replied = true; } diff --git a/net/bridge/br_cfm.c b/net/bridge/br_cfm.c index dea56fffa1c19..9dcc97d63a6fc 100644 --- a/net/bridge/br_cfm.c +++ b/net/bridge/br_cfm.c @@ -367,7 +367,7 @@ static u32 ccm_tlv_extract(struct sk_buff *skb, u32 index, } /* note: already called with rcu_read_lock */ -static int br_cfm_frame_rx(struct net_bridge_port *port, struct sk_buff *skb) +int br_cfm_frame_rx(struct net_bridge_port *port, struct sk_buff *skb) { u32 mdlevel, interval, size, index, max; const struct br_cfm_common_hdr *hdr; @@ -489,11 +489,6 @@ static int br_cfm_frame_rx(struct net_bridge_port *port, struct sk_buff *skb) return 1; } -static struct br_frame_type cfm_frame_type __read_mostly = { - .type = cpu_to_be16(ETH_P_CFM), - .frame_handler = br_cfm_frame_rx, -}; - int br_cfm_mep_create(struct net_bridge *br, const u32 instance, struct br_cfm_mep_create *const create, @@ -559,7 +554,7 @@ int br_cfm_mep_create(struct net_bridge *br, INIT_DELAYED_WORK(&mep->ccm_tx_dwork, ccm_tx_work_expired); if (hlist_empty(&br->mep_list)) - br_add_frame(br, &cfm_frame_type); + br_opt_toggle(br, BROPT_CFM_ENABLED, true); hlist_add_tail_rcu(&mep->head, &br->mep_list); @@ -588,7 +583,7 @@ static void mep_delete_implementation(struct net_bridge *br, kfree_rcu(mep, rcu); if (hlist_empty(&br->mep_list)) - br_del_frame(br, &cfm_frame_type); + br_opt_toggle(br, BROPT_CFM_ENABLED, false); } int br_cfm_mep_delete(struct net_bridge *br, diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index e7f343ab22d37..7b163d2b4e01d 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -503,7 +503,6 @@ void br_dev_setup(struct net_device *dev) spin_lock_init(&br->lock); INIT_LIST_HEAD(&br->port_list); INIT_HLIST_HEAD(&br->fdb_list); - INIT_HLIST_HEAD(&br->frame_type_list); #if IS_ENABLED(CONFIG_BRIDGE_MRP) INIT_HLIST_HEAD(&br->mrp_list); #endif diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c index ddb8f002a40e8..1322543bc9171 100644 --- a/net/bridge/br_input.c +++ b/net/bridge/br_input.c @@ -317,17 +317,25 @@ static int nf_hook_bridge_pre(struct sk_buff *skb, struct sk_buff **pskb) return RX_HANDLER_CONSUMED; } +#define BR_CFM_MRP_OPTS \ + ((IS_ENABLED(CONFIG_BRIDGE_CFM) ? BIT(BROPT_CFM_ENABLED) : 0UL) | \ + (IS_ENABLED(CONFIG_BRIDGE_MRP) ? BIT(BROPT_MRP_ENABLED) : 0UL)) + /* Return 0 if the frame was not processed otherwise 1 * note: already called with rcu_read_lock */ static int br_process_frame_type(struct net_bridge_port *p, struct sk_buff *skb) { - struct br_frame_type *tmp; + struct net_bridge *br = p->br; + + if (skb->protocol == htons(ETH_P_CFM) && + br_opt_get(br, BROPT_CFM_ENABLED)) + return br_cfm_frame_rx(p, skb); - hlist_for_each_entry_rcu(tmp, &p->br->frame_type_list, list) - if (unlikely(tmp->type == skb->protocol)) - return tmp->frame_handler(p, skb); + if (skb->protocol == htons(ETH_P_MRP) && + br_opt_get(br, BROPT_MRP_ENABLED)) + return br_mrp_process(p, skb); return 0; } @@ -425,7 +433,8 @@ static rx_handler_result_t br_handle_frame(struct sk_buff **pskb) } } - if (unlikely(br_process_frame_type(p, skb))) + if (unlikely((READ_ONCE(p->br->options) & BR_CFM_MRP_OPTS) && + br_process_frame_type(p, skb))) return RX_HANDLER_PASS; forward: @@ -467,19 +476,3 @@ rx_handler_func_t *br_get_rx_handler(const struct net_device *dev) return br_handle_frame; } - -void br_add_frame(struct net_bridge *br, struct br_frame_type *ft) -{ - hlist_add_head_rcu(&ft->list, &br->frame_type_list); -} - -void br_del_frame(struct net_bridge *br, struct br_frame_type *ft) -{ - struct br_frame_type *tmp; - - hlist_for_each_entry(tmp, &br->frame_type_list, list) - if (ft == tmp) { - hlist_del_rcu(&ft->list); - return; - } -} diff --git a/net/bridge/br_mdb.c b/net/bridge/br_mdb.c index e0c7020b12f5f..a01bd280c722b 100644 --- a/net/bridge/br_mdb.c +++ b/net/bridge/br_mdb.c @@ -1523,6 +1523,8 @@ static void br_mdb_flush_pgs(struct net_bridge *br, } br_multicast_del_pg(mp, p, pp); + /* br_multicast_del_pg() can remove other groups from this list. */ + pp = &mp->ports; } } diff --git a/net/bridge/br_mrp.c b/net/bridge/br_mrp.c index ef16d07039241..dce6efa96c4c6 100644 --- a/net/bridge/br_mrp.c +++ b/net/bridge/br_mrp.c @@ -6,13 +6,6 @@ static const u8 mrp_test_dmac[ETH_ALEN] = { 0x1, 0x15, 0x4e, 0x0, 0x0, 0x1 }; static const u8 mrp_in_test_dmac[ETH_ALEN] = { 0x1, 0x15, 0x4e, 0x0, 0x0, 0x3 }; -static int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb); - -static struct br_frame_type mrp_frame_type __read_mostly = { - .type = cpu_to_be16(ETH_P_MRP), - .frame_handler = br_mrp_process, -}; - static bool br_mrp_is_ring_port(struct net_bridge_port *p_port, struct net_bridge_port *s_port, struct net_bridge_port *port) @@ -486,7 +479,7 @@ static void br_mrp_del_impl(struct net_bridge *br, struct br_mrp *mrp) kfree_rcu(mrp, rcu); if (hlist_empty(&br->mrp_list)) - br_del_frame(br, &mrp_frame_type); + br_opt_toggle(br, BROPT_MRP_ENABLED, false); } /* Adds a new MRP instance. @@ -536,7 +529,7 @@ int br_mrp_add(struct net_bridge *br, struct br_mrp_instance *instance) rcu_assign_pointer(mrp->s_port, p); if (hlist_empty(&br->mrp_list)) - br_add_frame(br, &mrp_frame_type); + br_opt_toggle(br, BROPT_MRP_ENABLED, true); INIT_DELAYED_WORK(&mrp->test_work, br_mrp_test_work_expired); INIT_DELAYED_WORK(&mrp->in_test_work, br_mrp_in_test_work_expired); @@ -1241,7 +1234,7 @@ static int br_mrp_rcv(struct net_bridge_port *p, * normal forwarding. * note: already called with rcu_read_lock */ -static int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb) +int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb) { /* If there is no MRP instance do normal forwarding */ if (likely(!test_bit(BR_MRP_AWARE_BIT, &p->flags))) diff --git a/net/bridge/br_mst.c b/net/bridge/br_mst.c index 43a300ae6bfaf..1654efd3045b0 100644 --- a/net/bridge/br_mst.c +++ b/net/bridge/br_mst.c @@ -107,21 +107,24 @@ int br_mst_set_state(struct net_bridge_port *p, u16 msti, u8 state, struct net_bridge_vlan *v; int err = 0; - rcu_read_lock(); - vg = nbp_vlan_group_rcu(p); - if (!vg) - goto out; - /* MSTI 0 (CST) state changes are notified via the regular - * SWITCHDEV_ATTR_ID_PORT_STP_STATE. + * SWITCHDEV_ATTR_ID_PORT_STP_STATE. All other MSTIs are handled via + * netlink with RTNL held */ if (msti) { + ASSERT_RTNL(); + err = switchdev_port_attr_set(p->dev, &attr, extack); if (err && err != -EOPNOTSUPP) goto out; + err = 0; } - err = 0; + rcu_read_lock(); + vg = nbp_vlan_group_rcu(p); + if (!vg) + goto out_rcu_unlock; + list_for_each_entry_rcu(v, &vg->vlan_list, vlist) { if (v->brvlan->msti != msti) continue; @@ -129,8 +132,9 @@ int br_mst_set_state(struct net_bridge_port *p, u16 msti, u8 state, br_mst_vlan_set_state(vg, v, state); } -out: +out_rcu_unlock: rcu_read_unlock(); +out: return err; } diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index 00aa9b2879d6e..2191d35d9fa07 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -1435,16 +1435,17 @@ struct net_bridge_port_group *br_multicast_new_port_group( goto free_out; } - rcu_assign_pointer(p->next, next); timer_setup(&p->timer, br_multicast_port_group_expired, 0); timer_setup(&p->rexmit_timer, br_multicast_port_group_rexmit, 0); - hlist_add_head(&p->mglist, &port->mglist); if (src) memcpy(p->eth_addr, src, ETH_ALEN); else eth_broadcast_addr(p->eth_addr); + RCU_INIT_POINTER(p->next, next); + hlist_add_head_rcu(&p->mglist, &port->mglist); + return p; free_out: @@ -1459,11 +1460,11 @@ void br_multicast_del_port_group(struct net_bridge_port_group *p) struct net_bridge_port *port = p->key.port; __u16 vid = p->key.addr.vid; - hlist_del_init(&p->mglist); + hlist_del_init_rcu(&p->mglist); if (!br_multicast_is_star_g(&p->key.addr)) rhashtable_remove_fast(&port->br->sg_port_tbl, &p->rhnode, br_sg_port_rht_params); - kfree(p); + kfree_rcu(p, rcu); br_multicast_port_ngroups_dec(port, vid); } @@ -4312,8 +4313,8 @@ void br_multicast_toggle_one_vlan(struct net_bridge_vlan *vlan, bool on) if (br_vlan_is_master(vlan)) { br = vlan->br; - if (!br_vlan_is_brentry(vlan) || - (on && + if (on && + (!br_vlan_is_brentry(vlan) || br_multicast_ctx_vlan_global_disabled(&vlan->br_mcast_ctx))) return; diff --git a/net/bridge/br_netlink_tunnel.c b/net/bridge/br_netlink_tunnel.c index a713668ea34f0..e7eceab5b515d 100644 --- a/net/bridge/br_netlink_tunnel.c +++ b/net/bridge/br_netlink_tunnel.c @@ -302,7 +302,8 @@ int br_process_vlan_tunnel_info(const struct net_bridge *br, if (!(tinfo_last->flags & BRIDGE_VLAN_INFO_RANGE_BEGIN)) return -EINVAL; - if ((tinfo_curr->vid - tinfo_last->vid) != + if (tinfo_curr->vid < tinfo_last->vid || + (tinfo_curr->vid - tinfo_last->vid) != (tinfo_curr->tunid - tinfo_last->tunid)) return -EINVAL; t = tinfo_last->tunid; diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index d3880f31edc4e..d08e318f04555 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -491,12 +491,13 @@ enum net_bridge_opts { BROPT_MST_ENABLED, BROPT_MDB_OFFLOAD_FAIL_NOTIFICATION, BROPT_FDB_LOCAL_VLAN_0, + BROPT_CFM_ENABLED, + BROPT_MRP_ENABLED, }; struct net_bridge { spinlock_t lock; spinlock_t hash_lock; - struct hlist_head frame_type_list; struct net_device *dev; unsigned long options; /* These fields are accessed on each packet */ @@ -928,16 +929,6 @@ int nbp_backup_change(struct net_bridge_port *p, struct net_device *backup_dev); int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb); rx_handler_func_t *br_get_rx_handler(const struct net_device *dev); -struct br_frame_type { - __be16 type; - int (*frame_handler)(struct net_bridge_port *port, - struct sk_buff *skb); - struct hlist_node list; -}; - -void br_add_frame(struct net_bridge *br, struct br_frame_type *ft); -void br_del_frame(struct net_bridge *br, struct br_frame_type *ft); - static inline bool br_rx_handler_check_rcu(const struct net_device *dev) { return rcu_dereference(dev->rx_handler) == br_get_rx_handler(dev); @@ -2076,6 +2067,7 @@ int br_mrp_parse(struct net_bridge *br, struct net_bridge_port *p, bool br_mrp_enabled(struct net_bridge *br); void br_mrp_port_del(struct net_bridge *br, struct net_bridge_port *p); int br_mrp_fill_info(struct sk_buff *skb, struct net_bridge *br); +int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb); #else static inline int br_mrp_parse(struct net_bridge *br, struct net_bridge_port *p, struct nlattr *attr, int cmd, @@ -2099,6 +2091,11 @@ static inline int br_mrp_fill_info(struct sk_buff *skb, struct net_bridge *br) return 0; } +static inline int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb) +{ + return 0; +} + #endif /* br_cfm.c */ @@ -2107,6 +2104,7 @@ int br_cfm_parse(struct net_bridge *br, struct net_bridge_port *p, struct nlattr *attr, int cmd, struct netlink_ext_ack *extack); bool br_cfm_created(struct net_bridge *br); void br_cfm_port_del(struct net_bridge *br, struct net_bridge_port *p); +int br_cfm_frame_rx(struct net_bridge_port *port, struct sk_buff *skb); int br_cfm_config_fill_info(struct sk_buff *skb, struct net_bridge *br); int br_cfm_status_fill_info(struct sk_buff *skb, struct net_bridge *br, @@ -2131,6 +2129,12 @@ static inline void br_cfm_port_del(struct net_bridge *br, { } +static inline int br_cfm_frame_rx(struct net_bridge_port *port, + struct sk_buff *skb) +{ + return 0; +} + static inline int br_cfm_config_fill_info(struct sk_buff *skb, struct net_bridge *br) { return -EOPNOTSUPP; diff --git a/net/bridge/br_stp_bpdu.c b/net/bridge/br_stp_bpdu.c index 74ec42ba1e7d0..21d092f5acbb8 100644 --- a/net/bridge/br_stp_bpdu.c +++ b/net/bridge/br_stp_bpdu.c @@ -52,7 +52,10 @@ static void br_send_bpdu(struct net_bridge_port *p, LLC_SAP_BSPAN, LLC_PDU_CMD); llc_pdu_init_as_ui_cmd(skb); - llc_mac_hdr_init(skb, p->dev->dev_addr, p->br->group_addr); + if (llc_mac_hdr_init(skb, p->dev->dev_addr, p->br->group_addr)) { + kfree_skb(skb); + return; + } skb_reset_mac_header(skb); diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c index 31c1b2cf75d92..92b3cb621a264 100644 --- a/net/bridge/br_vlan.c +++ b/net/bridge/br_vlan.c @@ -387,12 +387,12 @@ static int __vlan_add(struct net_bridge_vlan *v, u16 flags, goto out; } -static int __vlan_del(struct net_bridge_vlan *v) +static void __vlan_del(struct net_bridge_vlan *v) { struct net_bridge_vlan *masterv = v; struct net_bridge_vlan_group *vg; struct net_bridge_port *p = NULL; - int err = 0; + int err; if (br_vlan_is_master(v)) { vg = br_vlan_group(v->br); @@ -406,12 +406,16 @@ static int __vlan_del(struct net_bridge_vlan *v) if (p) { err = __vlan_vid_del(p->dev, p->br, v); if (err) - goto out; + br_warn(p->br, + "port %u(%s) failed to delete vlan %u from switchdev: %pe\n", + (unsigned int)p->port_no, p->dev->name, + v->vid, ERR_PTR(err)); } else { err = br_switchdev_port_vlan_del(v->br->dev, v->vid); if (err && err != -EOPNOTSUPP) - goto out; - err = 0; + br_warn(v->br, + "failed to delete bridge vlan %u from switchdev: %pe\n", + v->vid, ERR_PTR(err)); } if (br_vlan_should_use(v)) { @@ -431,8 +435,6 @@ static int __vlan_del(struct net_bridge_vlan *v) } br_vlan_put_master(masterv); -out: - return err; } static void __vlan_group_free(struct net_bridge_vlan_group *vg) @@ -449,7 +451,6 @@ static void __vlan_flush(const struct net_bridge *br, { struct net_bridge_vlan *vlan, *tmp; u16 v_start = 0, v_end = 0; - int err; __vlan_delete_pvid(vg, vg->pvid); list_for_each_entry_safe(vlan, tmp, &vg->vlan_list, vlist) { @@ -463,13 +464,7 @@ static void __vlan_flush(const struct net_bridge *br, } v_end = vlan->vid; - err = __vlan_del(vlan); - if (err) { - br_err(br, - "port %u(%s) failed to delete vlan %d: %pe\n", - (unsigned int) p->port_no, p->dev->name, - vlan->vid, ERR_PTR(err)); - } + __vlan_del(vlan); } /* notify about the last/whole vlan range */ @@ -837,8 +832,9 @@ int br_vlan_delete(struct net_bridge *br, u16 vid) br_fdb_delete_by_port(br, NULL, vid, 0); vlan_tunnel_info_del(vg, v); + __vlan_del(v); - return __vlan_del(v); + return 0; } void br_vlan_flush(struct net_bridge *br) @@ -1136,7 +1132,7 @@ int __br_vlan_set_default_pvid(struct net_bridge *br, u16 pvid, if (err) goto out; - if (br_vlan_delete(br, old_pvid)) + if (!br_vlan_delete(br, old_pvid)) br_vlan_notify(br, NULL, old_pvid, 0, RTM_DELVLAN); br_vlan_notify(br, NULL, pvid, 0, RTM_NEWVLAN); __set_bit(0, changed); @@ -1158,7 +1154,7 @@ int __br_vlan_set_default_pvid(struct net_bridge *br, u16 pvid, &vlchange, extack); if (err) goto err_port; - if (nbp_vlan_delete(p, old_pvid)) + if (!nbp_vlan_delete(p, old_pvid)) br_vlan_notify(br, p, old_pvid, 0, RTM_DELVLAN); br_vlan_notify(p->br, p, pvid, 0, RTM_NEWVLAN); __set_bit(p->port_no, changed); @@ -1368,8 +1364,9 @@ int nbp_vlan_delete(struct net_bridge_port *port, u16 vid) return -ENOENT; br_fdb_find_delete_local(port->br, port, port->dev->dev_addr, vid); br_fdb_delete_by_port(port->br, port, vid, 0); + __vlan_del(v); - return __vlan_del(v); + return 0; } void nbp_vlan_flush(struct net_bridge_port *port) diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c index 05f6eea299fc0..b323b61e70239 100644 --- a/net/ceph/messenger_v2.c +++ b/net/ceph/messenger_v2.c @@ -2142,6 +2142,11 @@ static int process_banner_prefix(struct ceph_connection *con) payload_len = ceph_decode_16(&p); dout("%s con %p payload_len %d\n", __func__, con, payload_len); + if (payload_len < sizeof(u64) + sizeof(u64)) { + con->error_msg = "protocol error, bad banner payload len"; + return -EINVAL; + } + return prepare_read_banner_payload(con, payload_len); } diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 28d76c2f6b3e2..f36ce5ae75680 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -5802,6 +5803,31 @@ static inline void convert_extent_map(struct ceph_sparse_read *sr) } #endif +static bool sparse_extent_map_valid(struct ceph_sparse_read *sr) +{ + u64 req_end, pos; + int i; + + if (check_add_overflow(sr->sr_req_off, sr->sr_req_len, &req_end)) + return false; + + pos = sr->sr_req_off; + for (i = 0; i < sr->sr_count; i++) { + struct ceph_sparse_extent *ext = &sr->sr_extent[i]; + u64 end; + + if (ext->off < pos) + return false; + if (check_add_overflow(ext->off, ext->len, &end)) + return false; + if (end > req_end) + return false; + pos = end; + } + + return true; +} + static int osd_sparse_read(struct ceph_connection *con, struct ceph_msg_data_cursor *cursor, char **pbuf) @@ -5852,6 +5878,10 @@ static int osd_sparse_read(struct ceph_connection *con, fallthrough; case CEPH_SPARSE_READ_DATA_LEN: convert_extent_map(sr); + if (!sparse_extent_map_valid(sr)) { + pr_warn_ratelimited("invalid sparse extent map\n"); + return -EREMOTEIO; + } ret = sizeof(sr->sr_datalen); *pbuf = (char *)&sr->sr_datalen; sr->sr_state = CEPH_SPARSE_READ_DATA_PRE; diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c index d6282f0bcff8a..cf34b35c9a901 100644 --- a/net/ceph/osdmap.c +++ b/net/ceph/osdmap.c @@ -517,6 +517,8 @@ static struct crush_map *crush_decode(void *pbyval, void *end) ceph_decode_need(p, end, 4*sizeof(u32), bad); b->id = ceph_decode_32(p); + if (b->id != -1 - i) + goto bad; b->type = ceph_decode_16(p); if (b->type == 0) goto bad; diff --git a/net/core/dev.c b/net/core/dev.c index ece6700536d9b..55e1f4045d2c6 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -742,51 +742,73 @@ EXPORT_SYMBOL_GPL(dev_fill_metadata_dst); static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack) { - int k = stack->num_paths++; - - if (k >= NET_DEVICE_PATH_STACK_MAX) + if (stack->num_paths + 1 > NET_DEVICE_PATH_STACK_MAX) return NULL; - return &stack->path[k]; + return &stack->path[stack->num_paths]; +} + +void dev_fill_forward_path_release(struct net_device_path_stack *stack) +{ + struct net_device_path *path; + int k; + + if (stack->num_paths == 0) + return; + + for (k = stack->num_paths - 1; k >= 0; k--) { + path = &stack->path[k]; + switch (path->type) { + case DEV_PATH_TUN: + dst_release(path->tun.dst); + break; + default: + break; + } + } } +EXPORT_SYMBOL_GPL(dev_fill_forward_path_release); -int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr, +int dev_fill_forward_path(struct net_device_path_ctx *ctx, struct net_device_path_stack *stack) { const struct net_device *last_dev; - struct net_device_path_ctx ctx = { - .dev = dev, - }; struct net_device_path *path; int ret = 0; - memcpy(ctx.daddr, daddr, sizeof(ctx.daddr)); stack->num_paths = 0; - while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) { - last_dev = ctx.dev; + while (ctx->dev && ctx->dev->netdev_ops->ndo_fill_forward_path) { + last_dev = ctx->dev; path = dev_fwd_path(stack); if (!path) - return -1; + goto err_out; memset(path, 0, sizeof(struct net_device_path)); - ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path); + ret = ctx->dev->netdev_ops->ndo_fill_forward_path(ctx, path); if (ret < 0) - return -1; + goto err_out; - if (WARN_ON_ONCE(last_dev == ctx.dev)) - return -1; + stack->num_paths++; + if (last_dev == ctx->dev) + goto err_out; } - if (!ctx.dev) + if (!ctx->dev) return ret; path = dev_fwd_path(stack); if (!path) - return -1; + goto err_out; + path->type = DEV_PATH_ETHERNET; - path->dev = ctx.dev; + path->dev = ctx->dev; + stack->num_paths++; - return ret; + return 0; +err_out: + dev_fill_forward_path_release(stack); + + return -1; } EXPORT_SYMBOL_GPL(dev_fill_forward_path); @@ -2627,11 +2649,13 @@ EXPORT_SYMBOL_GPL(dev_queue_xmit_nit); */ static void netif_setup_tc(struct net_device *dev, unsigned int txq) { + struct netdev_tc_txq res; int i; - struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; + + res.combined = READ_ONCE(dev->tc_to_txq[0].combined); /* If TC0 is invalidated disable TC mapping */ - if (tc->offset + tc->count > txq) { + if (res.offset + res.count > txq) { netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); dev->num_tc = 0; return; @@ -2641,8 +2665,8 @@ static void netif_setup_tc(struct net_device *dev, unsigned int txq) for (i = 1; i < TC_BITMASK + 1; i++) { int q = netdev_get_prio_tc_map(dev, i); - tc = &dev->tc_to_txq[q]; - if (tc->offset + tc->count > txq) { + res.combined = READ_ONCE(dev->tc_to_txq[q].combined); + if (res.offset + res.count > txq) { netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", i, q); netdev_set_prio_tc_map(dev, i, 0); @@ -2658,7 +2682,10 @@ int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) /* walk through the TCs and see if it falls into any of them */ for (i = 0; i < TC_MAX_QUEUE; i++, tc++) { - if ((txq - tc->offset) < tc->count) + struct netdev_tc_txq res; + + res.combined = READ_ONCE(tc->combined); + if ((txq - res.offset) < res.count) return i; } @@ -2870,7 +2897,7 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask, dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; tc = netdev_txq_to_tc(dev, index); - if (tc < 0) + if (tc < 0 || tc >= num_tc) return -EINVAL; } @@ -3078,6 +3105,8 @@ static void netdev_unbind_all_sb_channels(struct net_device *dev) void netdev_reset_tc(struct net_device *dev) { + int i; + #ifdef CONFIG_XPS netif_reset_xps_queues_gt(dev, 0); #endif @@ -3085,21 +3114,26 @@ void netdev_reset_tc(struct net_device *dev) /* Reset TC configuration of device */ dev->num_tc = 0; - memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); + for (i = 0; i < TC_MAX_QUEUE; i++) + WRITE_ONCE(dev->tc_to_txq[i].combined, 0); memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); } EXPORT_SYMBOL(netdev_reset_tc); int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset) { + struct netdev_tc_txq res = { + .count = count, + .offset = offset, + }; + if (tc >= dev->num_tc) return -EINVAL; #ifdef CONFIG_XPS netif_reset_xps_queues(dev, offset, count); #endif - dev->tc_to_txq[tc].count = count; - dev->tc_to_txq[tc].offset = offset; + WRITE_ONCE(dev->tc_to_txq[tc].combined, res.combined); return 0; } EXPORT_SYMBOL(netdev_set_tc_queue); @@ -3123,11 +3157,13 @@ void netdev_unbind_sb_channel(struct net_device *dev, struct net_device *sb_dev) { struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; + int i; #ifdef CONFIG_XPS netif_reset_xps_queues_gt(sb_dev, 0); #endif - memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq)); + for (i = 0; i < TC_MAX_QUEUE; i++) + WRITE_ONCE(sb_dev->tc_to_txq[i].combined, 0); memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map)); while (txq-- != &dev->_tx[0]) { @@ -3150,8 +3186,12 @@ int netdev_bind_sb_channel_queue(struct net_device *dev, return -EINVAL; /* Record the mapping */ - sb_dev->tc_to_txq[tc].count = count; - sb_dev->tc_to_txq[tc].offset = offset; + struct netdev_tc_txq res = { + .count = count, + .offset = offset, + }; + + WRITE_ONCE(sb_dev->tc_to_txq[tc].combined, res.combined); /* Provide a way for Tx queue to find the tc_to_txq map or * XPS map for itself. @@ -3517,9 +3557,11 @@ static u16 skb_tx_hash(const struct net_device *dev, if (dev->num_tc) { u8 tc = netdev_get_prio_tc_map(dev, skb->priority); + struct netdev_tc_txq res; - qoffset = sb_dev->tc_to_txq[tc].offset; - qcount = sb_dev->tc_to_txq[tc].count; + res.combined = READ_ONCE(sb_dev->tc_to_txq[tc].combined); + qoffset = res.offset; + qcount = res.count; if (unlikely(!qcount)) { net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n", sb_dev->name, qoffset, tc); @@ -5328,7 +5370,8 @@ void kick_defer_list_purge(unsigned int cpu) backlog_unlock_irq_restore(sd, flags); } else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) { - smp_call_function_single_async(cpu, &sd->defer_csd); + if (smp_call_function_single_async(cpu, &sd->defer_csd)) + WRITE_ONCE(sd->defer_ipi_scheduled, 0); } } @@ -6852,25 +6895,35 @@ bool napi_complete_done(struct napi_struct *n, int work_done) } EXPORT_SYMBOL(napi_complete_done); -static void skb_defer_free_flush(void) +static void __skb_defer_free_flush(struct skb_defer_node *sdn, int budget) { struct llist_node *free_list; struct sk_buff *skb, *next; + + if (llist_empty(&sdn->defer_list)) + return; + atomic_long_set(&sdn->defer_count, 0); + free_list = llist_del_all(&sdn->defer_list); + + llist_for_each_entry_safe(skb, next, free_list, ll_node) { + prefetch(next); + napi_consume_skb(skb, budget); + } +} + +void skb_defer_node_flush(struct skb_defer_node *sdn) +{ + __skb_defer_free_flush(sdn, 0); +} + +static void skb_defer_free_flush(void) +{ struct skb_defer_node *sdn; int node; for_each_node(node) { sdn = this_cpu_ptr(net_hotdata.skb_defer_nodes) + node; - - if (llist_empty(&sdn->defer_list)) - continue; - atomic_long_set(&sdn->defer_count, 0); - free_list = llist_del_all(&sdn->defer_list); - - llist_for_each_entry_safe(skb, next, free_list, ll_node) { - prefetch(next); - napi_consume_skb(skb, 1); - } + __skb_defer_free_flush(sdn, 1); } } @@ -9932,7 +9985,7 @@ int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len) unsigned int orig_len = dev->tx_queue_len; int res; - if (new_len != (unsigned int)new_len) + if (new_len > S16_MAX) return -ERANGE; if (new_len != orig_len) { @@ -10333,6 +10386,37 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode, netdev_assert_locked_ops_compat(dev); + if (prog) { + enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB + ? XDP_MODE_DRV : XDP_MODE_SKB; + bool offload = mode == XDP_MODE_HW; + + if (!offload && dev_xdp_prog(dev, other_mode)) { + NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time"); + return -EEXIST; + } + if (!offload && bpf_prog_is_offloaded(prog->aux)) { + NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported"); + return -EINVAL; + } + if (bpf_prog_is_dev_bound(prog->aux) && !bpf_offload_dev_match(prog, dev)) { + NL_SET_ERR_MSG(extack, "Program bound to different device"); + return -EINVAL; + } + if (bpf_prog_is_dev_bound(prog->aux) && mode == XDP_MODE_SKB) { + NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); + return -EINVAL; + } + if (prog->expected_attach_type == BPF_XDP_DEVMAP) { + NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device"); + return -EINVAL; + } + if (prog->expected_attach_type == BPF_XDP_CPUMAP) { + NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device"); + return -EINVAL; + } + } + if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && prog && !prog->aux->xdp_has_frags) { NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split"); @@ -10472,38 +10556,10 @@ static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack new_prog = link->link.prog; if (new_prog) { - bool offload = mode == XDP_MODE_HW; - enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB - ? XDP_MODE_DRV : XDP_MODE_SKB; - if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) { NL_SET_ERR_MSG(extack, "XDP program already attached"); return -EBUSY; } - if (!offload && dev_xdp_prog(dev, other_mode)) { - NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time"); - return -EEXIST; - } - if (!offload && bpf_prog_is_offloaded(new_prog->aux)) { - NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported"); - return -EINVAL; - } - if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) { - NL_SET_ERR_MSG(extack, "Program bound to different device"); - return -EINVAL; - } - if (bpf_prog_is_dev_bound(new_prog->aux) && mode == XDP_MODE_SKB) { - NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); - return -EINVAL; - } - if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) { - NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device"); - return -EINVAL; - } - if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) { - NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device"); - return -EINVAL; - } } /* don't call drivers if the effective program didn't change */ @@ -12164,6 +12220,7 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, free_percpu(dev->pcpu_refcnt); free_dev: #endif + ref_tracker_dir_exit(&dev->refcnt_tracker); kvfree(dev); return NULL; } @@ -12537,7 +12594,7 @@ int __dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat, int new_ifindex, struct netlink_ext_ack *extack) { - struct netdev_name_node *name_node; + struct netdev_name_node *name_node, *tmp; struct net *net_old = dev_net(dev); char new_name[IFNAMSIZ] = {}; int err, new_nsid; @@ -12583,13 +12640,19 @@ int __dev_change_net_namespace(struct net_device *dev, struct net *net, } /* Check that none of the altnames conflicts. */ err = -EEXIST; - netdev_for_each_altname(dev, name_node) { - if (netdev_name_in_use(net, name_node->name)) { - NL_SET_ERR_MSG_FMT(extack, - "An interface with the altname %s exists in the target netns", - name_node->name); - goto out; + netdev_for_each_altname_safe(dev, name_node, tmp) { + if (!netdev_name_in_use(net, name_node->name)) + continue; + + if (!check_net(net_old)) { + __netdev_name_node_alt_destroy(name_node); + continue; } + + NL_SET_ERR_MSG_FMT(extack, + "An interface with the altname %s exists in the target netns", + name_node->name); + goto out; } /* Check that new_ifindex isn't used yet. */ @@ -12721,6 +12784,7 @@ static int dev_cpu_dead(unsigned int oldcpu) struct sk_buff **list_skb; struct sk_buff *skb; unsigned int cpu; + int node; struct softnet_data *sd, *oldsd, *remsd = NULL; local_irq_disable(); @@ -12781,6 +12845,17 @@ static int dev_cpu_dead(unsigned int oldcpu) rps_input_queue_head_incr(oldsd); } + for_each_node(node) + skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes, + oldcpu) + node); + node = cpu_to_node(oldcpu); + if (node_possible(node) && + !cpumask_intersects(cpumask_of_node(node), cpu_online_mask)) { + for_each_possible_cpu(cpu) + skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes, + cpu) + node); + } + return 0; } @@ -13040,7 +13115,6 @@ static struct pernet_operations __net_initdata netdev_net_ops = { static void __net_exit default_device_exit_net(struct net *net) { - struct netdev_name_node *name_node, *tmp; struct net_device *dev, *aux; /* * Push all migratable network devices back to the @@ -13064,10 +13138,6 @@ static void __net_exit default_device_exit_net(struct net *net) if (netdev_name_in_use(&init_net, fb_name)) snprintf(fb_name, IFNAMSIZ, "dev%%d"); - netdev_for_each_altname_safe(dev, name_node, tmp) - if (netdev_name_in_use(&init_net, name_node->name)) - __netdev_name_node_alt_destroy(name_node); - err = dev_change_net_namespace(dev, &init_net, fb_name); if (err) { pr_emerg("%s: failed to move %s to init_net: %d\n", diff --git a/net/core/dev.h b/net/core/dev.h index b757faead4d1a..04fb0e9a571e0 100644 --- a/net/core/dev.h +++ b/net/core/dev.h @@ -399,6 +399,8 @@ static inline void napi_assert_will_not_race(const struct napi_struct *napi) WARN_ON(READ_ONCE(napi->list_owner) != -1); } +struct skb_defer_node; +void skb_defer_node_flush(struct skb_defer_node *sdn); void kick_defer_list_purge(unsigned int cpu); int dev_set_hwtstamp_phylib(struct net_device *dev, diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index a320e264eaaf0..164643140a523 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -276,19 +276,18 @@ int dev_get_hwtstamp_phylib(struct net_device *dev, if (phy_is_default_hwtstamp(dev->phydev)) return phy_hwtstamp_get(dev->phydev, cfg); + if (!dev->netdev_ops->ndo_hwtstamp_get) + return -EOPNOTSUPP; + return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg); } static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr) { - const struct net_device_ops *ops = dev->netdev_ops; struct kernel_hwtstamp_config kernel_cfg = {}; struct hwtstamp_config cfg; int err; - if (!ops->ndo_hwtstamp_get) - return -EOPNOTSUPP; - if (!netif_device_present(dev)) return -ENODEV; @@ -359,12 +358,18 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, cfg->source = phy_ts ? HWTSTAMP_SOURCE_PHYLIB : HWTSTAMP_SOURCE_NETDEV; if (phy_ts && dev->see_all_hwtstamp_requests) { + if (!ops->ndo_hwtstamp_get) + return -EOPNOTSUPP; + err = ops->ndo_hwtstamp_get(dev, &old_cfg); if (err) return err; } if (!phy_ts || dev->see_all_hwtstamp_requests) { + if (!ops->ndo_hwtstamp_set) + return -EOPNOTSUPP; + err = ops->ndo_hwtstamp_set(dev, cfg, extack); if (err) { if (extack->_msg) @@ -390,7 +395,6 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr) { - const struct net_device_ops *ops = dev->netdev_ops; struct kernel_hwtstamp_config kernel_cfg = {}; struct netlink_ext_ack extack = {}; struct hwtstamp_config cfg; @@ -413,9 +417,6 @@ static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr) return err; } - if (!ops->ndo_hwtstamp_set) - return -EOPNOTSUPP; - if (!netif_device_present(dev)) return -ENODEV; @@ -441,15 +442,11 @@ static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr) int generic_hwtstamp_get_lower(struct net_device *dev, struct kernel_hwtstamp_config *kernel_cfg) { - const struct net_device_ops *ops = dev->netdev_ops; int err; if (!netif_device_present(dev)) return -ENODEV; - if (!ops->ndo_hwtstamp_get) - return -EOPNOTSUPP; - netdev_lock_ops(dev); err = dev_get_hwtstamp_phylib(dev, kernel_cfg); netdev_unlock_ops(dev); @@ -462,15 +459,11 @@ int generic_hwtstamp_set_lower(struct net_device *dev, struct kernel_hwtstamp_config *kernel_cfg, struct netlink_ext_ack *extack) { - const struct net_device_ops *ops = dev->netdev_ops; int err; if (!netif_device_present(dev)) return -ENODEV; - if (!ops->ndo_hwtstamp_set) - return -EOPNOTSUPP; - netdev_lock_ops(dev); err = dev_set_hwtstamp_phylib(dev, kernel_cfg, extack); netdev_unlock_ops(dev); diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c index abaf108ac4db8..edc660778408e 100644 --- a/net/core/drop_monitor.c +++ b/net/core/drop_monitor.c @@ -141,9 +141,8 @@ static struct sk_buff *reset_per_cpu_data(struct per_cpu_dm_data *data) al = sizeof(struct net_dm_alert_msg); al += dm_hit_limit * sizeof(struct net_dm_drop_point); - al += sizeof(struct nlattr); - skb = genlmsg_new(al, GFP_KERNEL); + skb = genlmsg_new(nla_total_size(al), GFP_KERNEL); if (!skb) goto err; @@ -448,7 +447,7 @@ net_dm_hw_trap_summary_probe(void *ignore, const struct devlink *devlink, if (metadata->trap_type == DEVLINK_TRAP_TYPE_CONTROL) return; - hw_data = this_cpu_ptr(&dm_hw_cpu_data); + hw_data = raw_cpu_ptr(&dm_hw_cpu_data); raw_spin_lock_irqsave(&hw_data->lock, flags); hw_entries = hw_data->hw_entries; @@ -516,7 +515,7 @@ static void net_dm_packet_trace_kfree_skb_hit(void *ignore, */ nskb->tstamp = tstamp; - data = this_cpu_ptr(&dm_cpu_data); + data = raw_cpu_ptr(&dm_cpu_data); spin_lock_irqsave(&data->drop_queue.lock, flags); if (skb_queue_len(&data->drop_queue) < net_dm_queue_len) @@ -983,7 +982,7 @@ net_dm_hw_trap_packet_probe(void *ignore, const struct devlink *devlink, NET_DM_SKB_CB(nskb)->hw_metadata = n_hw_metadata; nskb->tstamp = tstamp; - hw_data = this_cpu_ptr(&dm_hw_cpu_data); + hw_data = raw_cpu_ptr(&dm_hw_cpu_data); spin_lock_irqsave(&hw_data->drop_queue.lock, flags); if (skb_queue_len(&hw_data->drop_queue) < net_dm_queue_len) @@ -1083,7 +1082,7 @@ static int net_dm_hw_monitor_start(struct netlink_ext_ack *extack) struct per_cpu_dm_data *hw_data = &per_cpu(dm_hw_cpu_data, cpu); struct sk_buff *skb; - timer_delete_sync(&hw_data->send_timer); + timer_shutdown_sync(&hw_data->send_timer); cancel_work_sync(&hw_data->dm_alert_work); while ((skb = __skb_dequeue(&hw_data->drop_queue))) { struct devlink_trap_metadata *hw_metadata; @@ -1117,7 +1116,7 @@ static void net_dm_hw_monitor_stop(struct netlink_ext_ack *extack) struct per_cpu_dm_data *hw_data = &per_cpu(dm_hw_cpu_data, cpu); struct sk_buff *skb; - timer_delete_sync(&hw_data->send_timer); + timer_shutdown_sync(&hw_data->send_timer); cancel_work_sync(&hw_data->dm_alert_work); while ((skb = __skb_dequeue(&hw_data->drop_queue))) { struct devlink_trap_metadata *hw_metadata; @@ -1173,12 +1172,13 @@ static int net_dm_trace_on_set(struct netlink_ext_ack *extack) err_unregister_trace: unregister_trace_kfree_skb(ops->kfree_skb_probe, NULL); + tracepoint_synchronize_unregister(); err_module_put: for_each_possible_cpu(cpu) { struct per_cpu_dm_data *data = &per_cpu(dm_cpu_data, cpu); struct sk_buff *skb; - timer_delete_sync(&data->send_timer); + timer_shutdown_sync(&data->send_timer); cancel_work_sync(&data->dm_alert_work); while ((skb = __skb_dequeue(&data->drop_queue))) consume_skb(skb); @@ -1206,7 +1206,7 @@ static void net_dm_trace_off_set(void) struct per_cpu_dm_data *data = &per_cpu(dm_cpu_data, cpu); struct sk_buff *skb; - timer_delete_sync(&data->send_timer); + timer_shutdown_sync(&data->send_timer); cancel_work_sync(&data->dm_alert_work); while ((skb = __skb_dequeue(&data->drop_queue))) consume_skb(skb); diff --git a/net/core/filter.c b/net/core/filter.c index 16845987b2449..1acc53dd04efc 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -3871,12 +3871,6 @@ static u32 __bpf_skb_min_len(const struct sk_buff *skb) if (offset > 0) min_len = offset; } - if (skb->ip_summed == CHECKSUM_PARTIAL) { - offset = skb_checksum_start_offset(skb) + - skb->csum_offset + sizeof(__sum16); - if (offset > 0) - min_len = offset; - } return min_len; } @@ -3893,6 +3887,11 @@ static int bpf_skb_grow_rcsum(struct sk_buff *skb, unsigned int new_len) static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len) { + if (skb->ip_summed == CHECKSUM_PARTIAL && + new_len < skb_checksum_start_offset(skb) + skb->csum_offset + + sizeof(__sum16)) + skb->ip_summed = CHECKSUM_NONE; + return __skb_trim_rcsum(skb, new_len); } @@ -8350,10 +8349,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_CGROUP_UNIX_SENDMSG: case BPF_CGROUP_INET4_GETPEERNAME: case BPF_CGROUP_INET6_GETPEERNAME: - case BPF_CGROUP_UNIX_GETPEERNAME: case BPF_CGROUP_INET4_GETSOCKNAME: case BPF_CGROUP_INET6_GETSOCKNAME: - case BPF_CGROUP_UNIX_GETSOCKNAME: return &bpf_sock_addr_setsockopt_proto; default: return NULL; @@ -8373,10 +8370,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_CGROUP_UNIX_SENDMSG: case BPF_CGROUP_INET4_GETPEERNAME: case BPF_CGROUP_INET6_GETPEERNAME: - case BPF_CGROUP_UNIX_GETPEERNAME: case BPF_CGROUP_INET4_GETSOCKNAME: case BPF_CGROUP_INET6_GETSOCKNAME: - case BPF_CGROUP_UNIX_GETSOCKNAME: return &bpf_sock_addr_getsockopt_proto; default: return NULL; @@ -8886,6 +8881,8 @@ static const struct bpf_func_proto * lwt_seg6local_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { + case BPF_FUNC_skb_pull_data: + return NULL; #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF) case BPF_FUNC_lwt_seg6_store_bytes: return &bpf_lwt_seg6_store_bytes_proto; @@ -10406,11 +10403,12 @@ u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, target_size)); *insn++ = BPF_JMP_IMM(BPF_JNE, si->dst_reg, NO_QUEUE_MAPPING, 1); - *insn++ = BPF_MOV64_IMM(si->dst_reg, -1); + *insn++ = BPF_MOV32_IMM(si->dst_reg, -1); #else - *insn++ = BPF_MOV64_IMM(si->dst_reg, -1); - *target_size = 2; + *insn++ = BPF_MOV32_IMM(si->dst_reg, -1); #endif + *target_size = sizeof_field(struct bpf_sock, rx_queue_mapping); + break; } @@ -10946,18 +10944,7 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, break; case offsetof(struct bpf_sock_ops, rtt_min): - BUILD_BUG_ON(sizeof_field(struct tcp_sock, rtt_min) != - sizeof(struct minmax)); - BUILD_BUG_ON(sizeof(struct minmax) < - sizeof(struct minmax_sample)); - - *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( - struct bpf_sock_ops_kern, sk), - si->dst_reg, si->src_reg, - offsetof(struct bpf_sock_ops_kern, sk)); - *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, - offsetof(struct tcp_sock, rtt_min) + - sizeof_field(struct minmax_sample, t)); + SOCK_OPS_GET_FIELD(rtt_min, rtt_min.s[0].v, struct tcp_sock); break; case offsetof(struct bpf_sock_ops, bpf_sock_ops_cb_flags): @@ -12662,8 +12649,9 @@ __bpf_kfunc_start_defs(); * @sock: Pointer to socket to be destroyed * * Return: - * On error, may return EPROTONOSUPPORT, EINVAL. - * EPROTONOSUPPORT if protocol specific destroy handler is not supported. + * On error, may return EOPNOTSUPP, or whatever the protocol specific + * destroy handler returns. + * EOPNOTSUPP if protocol specific destroy handler is not supported. * 0 otherwise */ __bpf_kfunc int bpf_sock_destroy(struct sock_common *sock) @@ -12675,8 +12663,12 @@ __bpf_kfunc int bpf_sock_destroy(struct sock_common *sock) * Supporting protocols will need to acquire sock lock in the BPF context * prior to invoking this kfunc. */ - if (!sk->sk_prot->diag_destroy || (sk->sk_protocol != IPPROTO_TCP && - sk->sk_protocol != IPPROTO_UDP)) + if (!sk->sk_prot->diag_destroy) + return -EOPNOTSUPP; + + if (sk_fullsock(sk) && + sk->sk_protocol != IPPROTO_TCP && + sk->sk_protocol != IPPROTO_UDP) return -EOPNOTSUPP; return sk->sk_prot->diag_destroy(sk, ECONNABORTED); diff --git a/net/core/gro_cells.c b/net/core/gro_cells.c index 1b84385c04bd9..d8c0a28671201 100644 --- a/net/core/gro_cells.c +++ b/net/core/gro_cells.c @@ -22,6 +22,8 @@ int gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) if (unlikely(!(dev->flags & IFF_UP))) goto drop; + skb_unset_transport_header(skb); + if (!gcells->cells || skb_cloned(skb) || netif_elide_gro(dev)) { res = netif_rx(skb); goto unlock; diff --git a/net/core/lwt_bpf.c b/net/core/lwt_bpf.c index 652952d416f23..da49364ec63de 100644 --- a/net/core/lwt_bpf.c +++ b/net/core/lwt_bpf.c @@ -167,10 +167,10 @@ static int bpf_output(struct net *net, struct sock *sk, struct sk_buff *skb) return dst->lwtstate->orig_output(net, sk, skb); } -static int xmit_check_hhlen(struct sk_buff *skb, int hh_len) +static int xmit_check_headroom(struct sk_buff *skb, int hroom) { - if (skb_headroom(skb) < hh_len) { - int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); + if (skb_headroom(skb) < hroom) { + int nhead = hroom - skb_headroom(skb); if (pskb_expand_head(skb, nhead, 0, GFP_ATOMIC)) return -ENOMEM; @@ -282,7 +282,7 @@ static int bpf_xmit(struct sk_buff *skb) bpf = bpf_lwt_lwtunnel(dst->lwtstate); if (bpf->xmit.prog) { - int hh_len = dst->dev->hard_header_len; + int hroom = LL_RESERVED_SPACE(dst->dev); __be16 proto = skb->protocol; int ret; @@ -298,9 +298,12 @@ static int bpf_xmit(struct sk_buff *skb) return -EINVAL; } /* If the header was expanded, headroom might be too - * small for L2 header to come, expand as needed. + * small for the L2 header to come, expand as needed. + * neigh_hh_output() copies the cached header in + * HH_DATA_MOD aligned chunks, so match the reservation + * made before LWT xmit. */ - ret = xmit_check_hhlen(skb, hh_len); + ret = xmit_check_headroom(skb, hroom); if (unlikely(ret)) return ret; diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 1349c0eedb642..7448320f7ad52 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -2359,6 +2359,13 @@ static const struct nla_policy nl_neightbl_policy[NDTA_MAX+1] = { [NDTA_PARMS] = { .type = NLA_NESTED }, }; +#define NTBL_PARM_MS_MAX (24 * 60 * 60 * MSEC_PER_SEC) + +static const struct netlink_range_validation nl_ntbl_parm_ms_range = { + .min = 1, + .max = NTBL_PARM_MS_MAX, +}; + static const struct nla_policy nl_ntbl_parm_policy[NDTPA_MAX+1] = { [NDTPA_IFINDEX] = { .type = NLA_U32 }, [NDTPA_QUEUE_LEN] = { .type = NLA_U32 }, @@ -2375,7 +2382,8 @@ static const struct nla_policy nl_ntbl_parm_policy[NDTPA_MAX+1] = { [NDTPA_ANYCAST_DELAY] = { .type = NLA_U64 }, [NDTPA_PROXY_DELAY] = { .type = NLA_U64 }, [NDTPA_LOCKTIME] = { .type = NLA_U64 }, - [NDTPA_INTERVAL_PROBE_TIME_MS] = { .type = NLA_U64, .min = 1 }, + [NDTPA_INTERVAL_PROBE_TIME_MS] = NLA_POLICY_FULL_RANGE(NLA_U64, + &nl_ntbl_parm_ms_range), }; static int neightbl_set(struct sk_buff *skb, struct nlmsghdr *nlh, @@ -2579,9 +2587,10 @@ static int neightbl_dump_info(struct sk_buff *skb, struct netlink_callback *cb) { const struct nlmsghdr *nlh = cb->nlh; struct net *net = sock_net(skb->sk); + int default_skip = cb->args[2]; + int neigh_skip = cb->args[1]; int family, tidx, nidx = 0; int tbl_skip = cb->args[0]; - int neigh_skip = cb->args[1]; struct neigh_table *tbl; if (cb->strict_check) { @@ -2605,17 +2614,21 @@ static int neightbl_dump_info(struct sk_buff *skb, struct netlink_callback *cb) if (tidx < tbl_skip || (family && tbl->family != family)) continue; - if (neightbl_fill_info(skb, tbl, NETLINK_CB(cb->skb).portid, + if (!default_skip && + neightbl_fill_info(skb, tbl, NETLINK_CB(cb->skb).portid, nlh->nlmsg_seq, RTM_NEWNEIGHTBL, NLM_F_MULTI) < 0) break; - nidx = 0; - p = list_next_entry(&tbl->parms, list); - list_for_each_entry_from_rcu(p, &tbl->parms_list, list) { + default_skip = 1; + + list_for_each_entry_rcu(p, &tbl->parms_list, list) { if (!net_eq(neigh_parms_net(p), net)) continue; + if (!p->dev || p->dev == blackhole_netdev) + continue; + if (nidx < neigh_skip) goto next; @@ -2630,12 +2643,15 @@ static int neightbl_dump_info(struct sk_buff *skb, struct netlink_callback *cb) } neigh_skip = 0; + nidx = 0; + default_skip = 0; } out: rcu_read_unlock(); cb->args[0] = tidx; cb->args[1] = nidx; + cb->args[2] = default_skip; return skb->len; } @@ -3669,12 +3685,13 @@ static int neigh_proc_dointvec_ms_jiffies_positive(const struct ctl_table *ctl, void *buffer, size_t *lenp, loff_t *ppos) { struct ctl_table tmp = *ctl; - int ret; + int ret, min, max; - int min = msecs_to_jiffies(1); + min = msecs_to_jiffies(1); + max = msecs_to_jiffies(NTBL_PARM_MS_MAX); tmp.extra1 = &min; - tmp.extra2 = NULL; + tmp.extra2 = &max; ret = proc_dointvec_ms_jiffies_minmax(&tmp, write, buffer, lenp, ppos); neigh_proc_update(ctl, write); diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index d9dafe24f57e3..af26bb77991b2 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -399,14 +399,8 @@ static __net_init void preinit_net_sysctl(struct net *net) } /* init code that must occur even if setup_net() is not called. */ -static __net_init int preinit_net(struct net *net, struct user_namespace *user_ns) +static __net_init void preinit_net(struct net *net, struct user_namespace *user_ns) { - int ret; - - ret = ns_common_init(net); - if (ret) - return ret; - refcount_set(&net->passive, 1); ref_tracker_dir_init(&net->refcnt_tracker, 128, "net_refcnt"); ref_tracker_dir_init(&net->notrefcnt_tracker, 128, "net_notrefcnt"); @@ -427,7 +421,6 @@ static __net_init int preinit_net(struct net *net, struct user_namespace *user_n INIT_LIST_HEAD(&net->ptype_all); INIT_LIST_HEAD(&net->ptype_specific); preinit_net_sysctl(net); - return 0; } /* @@ -532,8 +525,12 @@ void net_passive_dec(struct net *net) if (refcount_dec_and_test(&net->passive)) { kfree(rcu_access_pointer(net->gen)); +#ifdef CONFIG_REF_TRACKER /* There should not be any trackers left there. */ ref_tracker_dir_exit(&net->notrefcnt_tracker); + if (!net->refcnt_tracker.dead) + ref_tracker_dir_exit(&net->refcnt_tracker); +#endif /* Wait for an extra rcu_barrier() before final free. */ llist_add(&net->defer_free_list, &defer_free_list); @@ -566,12 +563,14 @@ struct net *copy_net_ns(u64 flags, goto dec_ucounts; } - rv = preinit_net(net, user_ns); - if (rv < 0) - goto dec_ucounts; + preinit_net(net, user_ns); net->ucounts = ucounts; get_user_ns(user_ns); + rv = ns_common_init(net); + if (rv) + goto put_userns_no_common; + rv = down_read_killable(&pernet_ops_rwsem); if (rv < 0) goto put_userns; @@ -583,6 +582,7 @@ struct net *copy_net_ns(u64 flags, if (rv < 0) { put_userns: ns_common_free(net); +put_userns_no_common: #ifdef CONFIG_KEYS key_remove_domain(net->key_domain); #endif @@ -1285,7 +1285,8 @@ void __init net_ns_init(void) * This currently cannot fail as the initial network namespace * has a static inode number. */ - if (preinit_net(&init_net, &init_user_ns)) + preinit_net(&init_net, &init_user_ns); + if (ns_common_init(&init_net)) panic("Could not preinitialize the initial network namespace"); down_write(&pernet_ops_rwsem); diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 21dc4a9c8714d..8f8956fb061be 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -484,6 +484,13 @@ static int page_pool_register_dma_index(struct page_pool *pool, if (unlikely(!PP_DMA_INDEX_BITS)) goto out; + /* + * Drivers request GFP flags according to both the current context and + * the device constraints, but the XArray entry itself is by no mean + * used by the device, so remove zone/policy flags. + */ + gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_COMP); + if (in_softirq()) err = xa_alloc(&pool->dma_mapped, &id, netmem_to_page(netmem), PP_DMA_INDEX_LIMIT, gfp); @@ -500,29 +507,40 @@ static int page_pool_register_dma_index(struct page_pool *pool, return err; } -static int page_pool_release_dma_index(struct page_pool *pool, - netmem_ref netmem) +static void __page_pool_unmap_netmem_dma(struct page_pool *pool, + netmem_ref netmem) { struct page *old, *page = netmem_to_page(netmem); unsigned long id; + dma_addr_t dma; - if (unlikely(!PP_DMA_INDEX_BITS)) - return 0; - - id = netmem_get_dma_index(netmem); - if (!id) - return -1; + if (!pool->dma_map) + return; - if (in_softirq()) - old = xa_cmpxchg(&pool->dma_mapped, id, page, NULL, 0); - else - old = xa_cmpxchg_bh(&pool->dma_mapped, id, page, NULL, 0); - if (old != page) - return -1; + /* Cache dma_addr before xa_cmpxchg. The scrub path holds no page ref; + * the unref path calls put_page() regardless of cmpxchg outcome, so + * after the cmpxchg we cannot safely touch netmem fields. + */ + dma = page_pool_get_dma_addr_netmem(netmem); - netmem_set_dma_index(netmem, 0); + if (likely(PP_DMA_INDEX_BITS)) { + id = netmem_get_dma_index(netmem); + if (!id) + return; + + if (in_softirq()) + old = xa_cmpxchg(&pool->dma_mapped, + id, page, NULL, 0); + else + old = xa_cmpxchg_bh(&pool->dma_mapped, + id, page, NULL, 0); + if (old != page) + return; + } - return 0; + dma_unmap_page_attrs(pool->p.dev, dma, + PAGE_SIZE << pool->p.order, pool->p.dma_dir, + DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); } static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem, gfp_t gfp) @@ -728,24 +746,16 @@ void page_pool_clear_pp_info(netmem_ref netmem) static __always_inline void __page_pool_release_netmem_dma(struct page_pool *pool, netmem_ref netmem) { - dma_addr_t dma; - + /* Caller must hold a page ref: __page_pool_unmap_netmem_dma() is + * safe without a ref, but the field clears below require it. + */ if (!pool->dma_map) - /* Always account for inflight pages, even if we didn't - * map them - */ return; - if (page_pool_release_dma_index(pool, netmem)) - return; - - dma = page_pool_get_dma_addr_netmem(netmem); - - /* When page is unmapped, it cannot be returned to our pool */ - dma_unmap_page_attrs(pool->p.dev, dma, - PAGE_SIZE << pool->p.order, pool->p.dma_dir, - DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); + __page_pool_unmap_netmem_dma(pool, netmem); page_pool_set_dma_addr_netmem(netmem, 0); + if (likely(PP_DMA_INDEX_BITS)) + netmem_set_dma_index(netmem, 0); } /* Disconnects a page (from a page_pool). API users can have a need @@ -1171,8 +1181,9 @@ static void page_pool_scrub(struct page_pool *pool) synchronize_net(); } + /* No page ref, dma-unmap only. */ xa_for_each(&pool->dma_mapped, id, ptr) - __page_pool_release_netmem_dma(pool, page_to_netmem((struct page *)ptr)); + __page_pool_unmap_netmem_dma(pool, page_to_netmem((struct page *)ptr)); } /* No more consumers should exist, but producers could still diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 12aa3aa1688b1..73101ec65c9af 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -2236,6 +2236,11 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, return -EMSGSIZE; } +static const struct netlink_range_validation txqlen_range = { + .min = 0, + .max = S16_MAX, +}; + static const struct nla_policy ifla_policy[IFLA_MAX+1] = { [IFLA_UNSPEC] = { .strict_start_type = IFLA_DPLL_PIN }, [IFLA_IFNAME] = { .type = NLA_STRING, .len = IFNAMSIZ-1 }, @@ -2246,7 +2251,7 @@ static const struct nla_policy ifla_policy[IFLA_MAX+1] = { [IFLA_LINK] = { .type = NLA_U32 }, [IFLA_MASTER] = { .type = NLA_U32 }, [IFLA_CARRIER] = { .type = NLA_U8 }, - [IFLA_TXQLEN] = { .type = NLA_U32 }, + [IFLA_TXQLEN] = NLA_POLICY_FULL_RANGE(NLA_U32, &txqlen_range), [IFLA_WEIGHT] = { .type = NLA_U32 }, [IFLA_OPERSTATE] = { .type = NLA_U8 }, [IFLA_LINKMODE] = { .type = NLA_U8 }, diff --git a/net/core/skbuff.c b/net/core/skbuff.c index ba3dbac80fb49..39102b3660034 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1417,10 +1417,13 @@ EXPORT_SYMBOL(skb_dump); * * Report xmit error if a device callback is tracking this skb. * skb must be freed afterwards. + * + * Does nothing for a cloned skb: the zerocopy state lives in + * skb_shinfo(), which the clones share. */ void skb_tx_error(struct sk_buff *skb) { - if (skb) { + if (skb && !skb_cloned(skb)) { skb_zcopy_downgrade_managed(skb); skb_zcopy_clear(skb, true); } @@ -2001,12 +2004,12 @@ int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) int i, order, psize, new_frags; u32 d_off; - if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) - return -EINVAL; - if (!skb_frags_readable(skb)) return -EFAULT; + if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) + return -EINVAL; + if (!num_frags) goto release; @@ -3907,10 +3910,9 @@ skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) skb_len_add(to, len + plen); - if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { - skb_tx_error(from); + if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) return -ENOMEM; - } + skb_zerocopy_clone(to, from, GFP_ATOMIC); for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { @@ -5963,7 +5965,8 @@ static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) err = skb_maybe_pull_tail(skb, off + sizeof(struct ipv6_opt_hdr), - MAX_IPV6_HDR_LEN); + off + + sizeof(struct ipv6_opt_hdr)); if (err < 0) goto out; @@ -5978,7 +5981,8 @@ static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) err = skb_maybe_pull_tail(skb, off + sizeof(struct ip_auth_hdr), - MAX_IPV6_HDR_LEN); + off + + sizeof(struct ip_auth_hdr)); if (err < 0) goto out; @@ -5993,7 +5997,8 @@ static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) err = skb_maybe_pull_tail(skb, off + sizeof(struct frag_hdr), - MAX_IPV6_HDR_LEN); + off + + sizeof(struct frag_hdr)); if (err < 0) goto out; @@ -6676,6 +6681,13 @@ int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, } skb->protocol = next_proto; + /* The last label is gone, so the inner header recorded by + * skb_mpls_push() no longer describes this packet. Drop it, or a + * later push keeps the stale offset. + */ + if (!eth_p_mpls(next_proto)) + skb->inner_protocol = 0; + return 0; } EXPORT_SYMBOL_GPL(skb_mpls_pop); @@ -6811,6 +6823,34 @@ struct sk_buff *alloc_skb_with_frags(unsigned long header_len, } EXPORT_SYMBOL(alloc_skb_with_frags); +/* pskb_carve_inside_header() and pskb_carve_inside_nonlinear() + * remove the first bytes of a packet and reallocate skb->head. + * + * Whatever headers were present before the operation are gone, + * we must not leave stale offsets, otherwise users of this skb + * (skb_dump(), drop_monitor, taps, ...) would read or pull garbage. + */ +static void skb_carve_reset_headers(struct sk_buff *skb) +{ + skb_unset_mac_header(skb); + skb_unset_transport_header(skb); + skb_reset_network_header(skb); + skb->mac_len = 0; + + /* Inner offsets have no "unset" marker, zero them so that + * skb_inner_network_header_was_set() becomes false and no + * consumer mistakes them for a real (and long gone) header. + */ + skb->inner_mac_header = 0; + skb->inner_network_header = 0; + skb->inner_transport_header = 0; + skb->inner_protocol = 0; + skb->encapsulation = 0; + + if (skb->ip_summed == CHECKSUM_PARTIAL) + skb->ip_summed = CHECKSUM_NONE; +} + /* carve out the first off bytes from skb when off < headlen */ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, const int headlen, gfp_t gfp_mask) @@ -6866,7 +6906,7 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, skb->head_frag = 0; skb_set_end_offset(skb, size); skb_set_tail_pointer(skb, skb_headlen(skb)); - skb_headers_offset_update(skb, 0); + skb_carve_reset_headers(skb); skb->cloned = 0; skb->hdr_len = 0; skb->nohdr = 0; @@ -7006,7 +7046,7 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, skb->data = data; skb_set_end_offset(skb, size); skb_reset_tail_pointer(skb); - skb_headers_offset_update(skb, 0); + skb_carve_reset_headers(skb); skb->cloned = 0; skb->hdr_len = 0; skb->nohdr = 0; @@ -7308,8 +7348,8 @@ void skb_attempt_defer_free(struct sk_buff *skb) struct skb_defer_node *sdn; unsigned long defer_count; unsigned int defer_max; + int cpu, my_cpu; bool kick; - int cpu; if (static_branch_unlikely(&skb_defer_disable_key)) goto nodefer; @@ -7319,7 +7359,8 @@ void skb_attempt_defer_free(struct sk_buff *skb) goto nodefer; cpu = skb->alloc_cpu; - if (cpu == raw_smp_processor_id() || + my_cpu = raw_smp_processor_id(); + if (cpu == my_cpu || WARN_ON_ONCE(cpu >= nr_cpu_ids) || !cpu_online(cpu)) { nodefer: kfree_skb_napi_cache(skb); @@ -7330,7 +7371,7 @@ nodefer: kfree_skb_napi_cache(skb); DEBUG_NET_WARN_ON_ONCE(skb->destructor); DEBUG_NET_WARN_ON_ONCE(skb_nfct(skb)); - sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + numa_node_id(); + sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + cpu_to_node(my_cpu); defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max); defer_count = atomic_long_inc_return(&sdn->defer_count); @@ -7340,6 +7381,11 @@ nodefer: kfree_skb_napi_cache(skb); llist_add(&skb->ll_node, &sdn->defer_list); + if (unlikely(!cpu_online(cpu) || my_cpu != raw_smp_processor_id())) { + skb_defer_node_flush(sdn); + return; + } + /* Send an IPI every time queue reaches half capacity. */ kick = (defer_count - 1) == (defer_max >> 1); diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 2521b643fa05d..df385a5a961e8 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -1000,6 +1000,10 @@ static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb, int err = 0; u32 len, off; + if (verdict == __SK_REDIRECT && skb_bpf_ingress(skb) && + skb_bpf_redirect_fetch(skb) == psock->sk) + verdict = __SK_PASS; + switch (verdict) { case __SK_PASS: err = -EIO; diff --git a/net/core/sock.c b/net/core/sock.c index 1ad41904db25b..4395509ab9dcd 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -142,6 +142,7 @@ #include +#include #include #include #include @@ -2667,6 +2668,12 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst) } EXPORT_SYMBOL_GPL(sk_setup_caps); +bool sk_has_decrypt_user(const struct sock *sk) +{ + return psp_sk_assoc(sk) || + (sk_is_inet(sk) && inet_csk_has_ulp(sk)); /* for tls */ +} + /* * Simple resource managers for sockets. */ @@ -3908,7 +3915,14 @@ int sock_gettstamp(struct socket *sock, void __user *userstamp, struct sock *sk = sock->sk; struct timespec64 ts; - sock_enable_timestamp(sk, SOCK_TIMESTAMP); + /* sk->sk_flags must only be changed under the socket lock, + * because sock_set_flag() uses non atomic operations. + */ + if (!sock_flag(sk, SOCK_TIMESTAMP)) { + lock_sock(sk); + sock_enable_timestamp(sk, SOCK_TIMESTAMP); + release_sock(sk); + } ts = ktime_to_timespec64(sock_read_timestamp(sk)); if (ts.tv_sec == -1) return -ENOENT; diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 9efbd8ca7db83..09d20318a0eae 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -41,6 +41,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr) struct bpf_stab *stab; if (attr->max_entries == 0 || + attr->max_entries > INT_MAX || attr->key_size != 4 || (attr->value_size != sizeof(u32) && attr->value_size != sizeof(u64)) || diff --git a/net/core/xdp.c b/net/core/xdp.c index 0194e69da339a..1d679e8fd6493 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -585,7 +585,7 @@ struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) xdp->data - xdp->data_meta; totsize = xdp->data_end - xdp->data + metasize; - if (sizeof(*xdpf) + totsize > PAGE_SIZE) + if (sizeof(*xdpf) + totsize > SKB_WITH_OVERHEAD(PAGE_SIZE)) return NULL; page = dev_alloc_page(); @@ -602,7 +602,7 @@ struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) xdpf->data = addr + metasize; xdpf->len = totsize - metasize; - xdpf->headroom = 0; + xdpf->headroom = metasize; xdpf->metasize = metasize; xdpf->frame_sz = PAGE_SIZE; xdpf->mem_type = MEM_TYPE_PAGE_ORDER0; diff --git a/net/dsa/tag_brcm.c b/net/dsa/tag_brcm.c index 411e3b57d16af..b7c49822ca888 100644 --- a/net/dsa/tag_brcm.c +++ b/net/dsa/tag_brcm.c @@ -373,6 +373,7 @@ static const struct dsa_device_ops brcm_legacy_fcs_netdev_ops = { .xmit = brcm_leg_fcs_tag_xmit, .rcv = brcm_leg_tag_rcv, .needed_headroom = BRCM_LEG_TAG_LEN, + .needed_tailroom = ETH_FCS_LEN, }; DSA_TAG_DRIVER(brcm_legacy_fcs_netdev_ops); diff --git a/net/ethtool/common.h b/net/ethtool/common.h index 4e5356e26f400..ae32e7fdb563c 100644 --- a/net/ethtool/common.h +++ b/net/ethtool/common.h @@ -163,6 +163,8 @@ ethtool_ioctl_needs_rtnl(const struct net_device *dev, u32 ethcmd) return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_RSS; case ETHTOOL_GLINK: return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_GLINK; + case ETHTOOL_TEST: + return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_TEST; } return false; } diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index 5555b71ab19b5..9c3078dd38c2c 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -820,6 +820,8 @@ int hsr_dev_finalize(struct net_device *hsr_dev, struct net_device *slave[2], hsr_del_ports(hsr); err_add_master: hsr_del_self_node(hsr); + hsr_del_nodes(&hsr->node_db); + hsr_del_nodes(&hsr->proxy_node_db); if (unregister) unregister_netdevice(hsr_dev); diff --git a/net/hsr/hsr_slave.c b/net/hsr/hsr_slave.c index 01c73b4b50ddd..a546f70f9cc8b 100644 --- a/net/hsr/hsr_slave.c +++ b/net/hsr/hsr_slave.c @@ -149,9 +149,12 @@ static int hsr_portdev_setup(struct hsr_priv *hsr, struct net_device *dev, int res; /* Don't use promiscuous mode for offload since L2 frame forward - * happens at the offloaded hardware. + * happens at the offloaded hardware. The interlink port never + * gets forwarding offload (RedBox forwarding to/from it is done + * by this driver), so it still needs promiscuous mode to receive + * frames addressed to hsr_dev's MAC rather than its own. */ - if (!port->hsr->fwd_offloaded) { + if (!port->hsr->fwd_offloaded || port->type == HSR_PT_INTERLINK) { res = dev_set_promiscuity(dev, 1); if (res) return res; @@ -176,7 +179,7 @@ static int hsr_portdev_setup(struct hsr_priv *hsr, struct net_device *dev, fail_rx_handler: netdev_upper_dev_unlink(dev, hsr_dev); fail_upper_dev_link: - if (!port->hsr->fwd_offloaded) + if (!port->hsr->fwd_offloaded || port->type == HSR_PT_INTERLINK) dev_set_promiscuity(dev, -1); return res; @@ -240,7 +243,7 @@ void hsr_del_port(struct hsr_port *port) netdev_update_features(master->dev); dev_set_mtu(master->dev, hsr_get_max_mtu(hsr)); netdev_rx_handler_unregister(port->dev); - if (!port->hsr->fwd_offloaded) + if (!port->hsr->fwd_offloaded || port->type == HSR_PT_INTERLINK) dev_set_promiscuity(port->dev, -1); if (port->type == HSR_PT_SLAVE_A || port->type == HSR_PT_SLAVE_B) vlan_vids_del_by_dev(port->dev, master->dev); diff --git a/net/ieee802154/6lowpan/core.c b/net/ieee802154/6lowpan/core.c index 018929563c6bc..6a8d6852cb930 100644 --- a/net/ieee802154/6lowpan/core.c +++ b/net/ieee802154/6lowpan/core.c @@ -150,7 +150,7 @@ static int lowpan_newlink(struct net_device *ldev, wdev = dev_get_by_index(dev_net(ldev), nla_get_u32(tb[IFLA_LINK])); if (!wdev) return -ENODEV; - if (wdev->type != ARPHRD_IEEE802154) { + if (wdev->type != ARPHRD_IEEE802154 || !wdev->ieee802154_ptr) { dev_put(wdev); return -EINVAL; } diff --git a/net/ife/ife.c b/net/ife/ife.c index 7a75947a31e31..2ddf725d3389e 100644 --- a/net/ife/ife.c +++ b/net/ife/ife.c @@ -37,7 +37,7 @@ void *ife_encode(struct sk_buff *skb, u16 metalen) * where ORIGDATA = original ethernet header ... */ int hdrm = metalen + IFE_METAHDRLEN; - int total_push = hdrm + skb->dev->hard_header_len; + int total_push = hdrm + ETH_HLEN; struct ifeheadr *ifehdr; struct ethhdr *iethh; /* inner ether header */ int skboff = 0; @@ -50,9 +50,9 @@ void *ife_encode(struct sk_buff *skb, u16 metalen) iethh = (struct ethhdr *) skb->data; __skb_push(skb, total_push); - memcpy(skb->data, iethh, skb->dev->hard_header_len); + memcpy(skb->data, iethh, ETH_HLEN); skb_reset_mac_header(skb); - skboff += skb->dev->hard_header_len; + skboff += ETH_HLEN; /* total metadata length */ ifehdr = (struct ifeheadr *) (skb->data + skboff); @@ -69,12 +69,12 @@ void *ife_decode(struct sk_buff *skb, u16 *metalen) int total_pull; u16 ifehdrln; - if (!pskb_may_pull(skb, skb->dev->hard_header_len + IFE_METAHDRLEN)) + if (!pskb_may_pull(skb, ETH_HLEN + IFE_METAHDRLEN)) return NULL; - ifehdr = (struct ifeheadr *) (skb->data + skb->dev->hard_header_len); + ifehdr = (struct ifeheadr *)(skb->data + ETH_HLEN); ifehdrln = ntohs(ifehdr->metalen); - total_pull = skb->dev->hard_header_len + ifehdrln; + total_pull = ETH_HLEN + ifehdrln; if (unlikely(ifehdrln < 2)) return NULL; @@ -82,7 +82,7 @@ void *ife_decode(struct sk_buff *skb, u16 *metalen) if (unlikely(!pskb_may_pull(skb, total_pull + ETH_HLEN))) return NULL; - ifehdr = (struct ifeheadr *)(skb->data + skb->dev->hard_header_len); + ifehdr = (struct ifeheadr *)(skb->data + ETH_HLEN); skb_set_mac_header(skb, total_pull); __skb_pull(skb, total_pull); *metalen = ifehdrln - IFE_METAHDRLEN; diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c index d409f606aec0e..60009d92e0710 100644 --- a/net/ipv4/arp.c +++ b/net/ipv4/arp.c @@ -1278,6 +1278,7 @@ int arp_ioctl(struct net *net, unsigned int cmd, void __user *arg) err = copy_from_user(&r, arg, sizeof(struct arpreq)); if (err) return -EFAULT; + r.arp_dev[IFNAMSIZ - 1] = '\0'; break; default: return -EINVAL; diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index a35b72662e431..a80896647154c 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -2117,9 +2117,10 @@ static int inet_validate_link_af(const struct net_device *dev, return err; if (tb[IFLA_INET_CONF]) { - err = nla_parse_nested(nested_tb, IPV4_DEVCONF_MAX, - tb[IFLA_INET_CONF], inet_devconf_policy, - extack); + err = nla_parse(nested_tb, IPV4_DEVCONF_MAX, + nla_data(tb[IFLA_INET_CONF]), + nla_len(tb[IFLA_INET_CONF]), + inet_devconf_policy, extack); if (err < 0) return err; diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index dfc81ee969ae0..faa48f5b97395 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -441,6 +441,12 @@ int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * esp->inplace = false; + /* Take real page refs and clear SKBFL_MANAGED_FRAG_REFS before + * we mutate the frag array, so the per-frag unref stays balanced + * for zerocopy managed frags (see __ip_append_data()). + */ + skb_zcopy_downgrade_managed(skb); + allocsize = ALIGN(tailen, L1_CACHE_BYTES); spin_lock_bh(&x->lock); diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c index e068a5bace73f..dcbbfdb291ed3 100644 --- a/net/ipv4/fib_rules.c +++ b/net/ipv4/fib_rules.c @@ -214,6 +214,8 @@ INDIRECT_CALLABLE_SCOPE int fib4_rule_match(struct fib_rule *rule, return 1; } +#define FIB_MAX_AUTO_TABLE_ID 4096 + static struct fib_table *fib_empty_table(struct net *net) { u32 id = 1; @@ -222,7 +224,7 @@ static struct fib_table *fib_empty_table(struct net *net) if (!fib_get_table(net, id)) return fib_new_table(net, id); - if (id++ == RT_TABLE_MAX) + if (id++ == FIB_MAX_AUTO_TABLE_ID) break; } return NULL; diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index 0483519b7fb0d..0c25f6dfb8a6f 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -2176,6 +2176,15 @@ static bool fib_good_nh(const struct fib_nh *nh) return !!(state & NUD_VALID); } +static __be32 fib_nh_saddr(struct net *net, const struct fib_info *fi, + struct fib_nh *nh, int genid) +{ + if (READ_ONCE(nh->nh_saddr_genid) == genid) + return READ_ONCE(nh->nh_saddr); + + return fib_info_update_nhc_saddr(net, &nh->nh_common, fi->fib_scope); +} + void fib_select_multipath(struct fib_result *res, int hash, const struct flowi4 *fl4) { @@ -2184,6 +2193,7 @@ void fib_select_multipath(struct fib_result *res, int hash, bool use_neigh; int score = -1; __be32 saddr; + int genid; if (unlikely(res->fi->nh)) { nexthop_path_fib_result(res, hash); @@ -2192,6 +2202,7 @@ void fib_select_multipath(struct fib_result *res, int hash, use_neigh = READ_ONCE(net->ipv4.sysctl_fib_multipath_use_neigh); saddr = fl4 ? fl4->saddr : 0; + genid = saddr ? atomic_read(&net->ipv4.dev_addr_genid) : 0; change_nexthops(fi) { int nh_upper_bound, nh_score = 0; @@ -2204,7 +2215,7 @@ void fib_select_multipath(struct fib_result *res, int hash, (use_neigh && !fib_good_nh(nexthop_nh))) continue; - if (saddr && nexthop_nh->nh_saddr == saddr) + if (saddr && fib_nh_saddr(net, fi, nexthop_nh, genid) == saddr) nh_score += 2; if (hash <= nh_upper_bound) nh_score++; diff --git a/net/ipv4/fou_core.c b/net/ipv4/fou_core.c index ab09dfcdecbd5..9a2999285b4ab 100644 --- a/net/ipv4/fou_core.c +++ b/net/ipv4/fou_core.c @@ -600,6 +600,10 @@ static int fou_create(struct net *net, struct fou_cfg *cfg, /* Initial for fou type */ switch (cfg->type) { case FOU_ENCAP_DIRECT: + if (!cfg->protocol) { + err = -EINVAL; + goto error; + } tunnel_cfg.encap_rcv = fou_udp_recv; tunnel_cfg.gro_receive = fou_gro_receive; tunnel_cfg.gro_complete = fou_gro_complete; diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 0caedfc7ca92f..90c0e22c29bea 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -581,16 +581,19 @@ static struct rtable *icmp_route_lookup(struct net *net, struct flowi4 *fl4, skb_dstref_restore(skb_in, orefdst); /* - * At this point, fl4_dec.daddr should NOT be local (we - * checked fl4_dec.saddr above). However, a race condition - * may occur if the address is added to the interface - * concurrently. In that case, ip_route_input() returns a - * LOCAL route with dst.output=ip_rt_bug, which must not - * be used for output. + * fl4_dec.daddr is not expected to be local here, but it can be + * added to an interface concurrently, in which case + * ip_route_input() returns a LOCAL route. It can also fail to + * build a forwarding route towards fl4_dec.daddr, for example, + * when forwarding is disabled, and return an UNREACHABLE route. + * Both cases will result in a route with dst.output=ip_rt_bug, + * which must not be used for output. */ - if (!err && rt2 && rt2->rt_type == RTN_LOCAL) { + if (!err && rt2 && rt2->rt_type == RTN_LOCAL) net_warn_ratelimited("detected local route for %pI4 during ICMP sending, src %pI4\n", &fl4_dec.daddr, &fl4_dec.saddr); + if (!err && rt2 && + (rt2->rt_type == RTN_LOCAL || rt2->rt_type == RTN_UNREACHABLE)) { dst_release(&rt2->dst); err = -EINVAL; } diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index bb2d4441a4929..0a5b7ded23be2 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -188,6 +188,10 @@ static void ip_ma_put(struct ip_mc_list *im) } } +#define pmc_dereference(e, pmc) \ + rcu_dereference_protected(e, lockdep_is_held(&(pmc)->lock) || \ + lockdep_is_held(&(pmc)->interface->mc_tomb_lock)) + #define for_each_pmc_rcu(in_dev, pmc) \ for (pmc = rcu_dereference(in_dev->mc_list); \ pmc != NULL; \ @@ -198,13 +202,28 @@ static void ip_ma_put(struct ip_mc_list *im) pmc != NULL; \ pmc = rtnl_dereference(pmc->next_rcu)) +#define for_each_psf_mclock(pmc, psf) \ + for (psf = pmc_dereference((pmc)->sources, pmc); \ + psf; \ + psf = pmc_dereference(psf->sf_next, pmc)) + +#define for_each_psf_rcu(im, psf) \ + for (psf = rcu_dereference((im)->sources); \ + psf; \ + psf = rcu_dereference(psf->sf_next)) + +#define for_each_psf_tomb(pmc, psf) \ + for (psf = pmc_dereference((pmc)->tomb, pmc); \ + psf; \ + psf = pmc_dereference(psf->sf_next, pmc)) + static void ip_sf_list_clear_all(struct ip_sf_list *psf) { struct ip_sf_list *next; while (psf) { - next = psf->sf_next; - kfree(psf); + next = rcu_dereference_protected(psf->sf_next, 1); + kfree_rcu(psf, rcu); psf = next; } } @@ -349,7 +368,7 @@ igmp_scount(struct ip_mc_list *pmc, int type, int gdeleted, int sdeleted) struct ip_sf_list *psf; int scount = 0; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (!is_in(pmc, psf, type, gdeleted, sdeleted)) continue; scount++; @@ -494,7 +513,8 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ip_mc_list *pmc, struct net *net = dev_net(dev); struct igmpv3_report *pih; struct igmpv3_grec *pgr = NULL; - struct ip_sf_list *psf, *psf_next, *psf_prev, **psf_list; + struct ip_sf_list *psf, *psf_next, *psf_prev; + struct ip_sf_list __rcu **psf_list; int scount, stotal, first, isquery, truncate; unsigned int mtu; @@ -517,7 +537,7 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ip_mc_list *pmc, psf_list = sdeleted ? &pmc->tomb : &pmc->sources; - if (!*psf_list) + if (!rcu_access_pointer(*psf_list)) goto empty_source; pih = skb ? igmpv3_report_hdr(skb) : NULL; @@ -533,10 +553,12 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ip_mc_list *pmc, } first = 1; psf_prev = NULL; - for (psf = *psf_list; psf; psf = psf_next) { + for (psf = pmc_dereference(*psf_list, pmc); + psf; + psf = psf_next) { __be32 *psrc; - psf_next = psf->sf_next; + psf_next = pmc_dereference(psf->sf_next, pmc); if (!is_in(pmc, psf, type, gdeleted, sdeleted)) { psf_prev = psf; @@ -583,10 +605,12 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ip_mc_list *pmc, psf->sf_crcount--; if ((sdeleted || gdeleted) && psf->sf_crcount == 0) { if (psf_prev) - psf_prev->sf_next = psf->sf_next; + rcu_assign_pointer(psf_prev->sf_next, + psf_next); else - *psf_list = psf->sf_next; - kfree(psf); + rcu_assign_pointer(*psf_list, + psf_next); + kfree_rcu(psf, rcu); continue; } } @@ -655,28 +679,29 @@ static int igmpv3_send_report(struct in_device *in_dev, struct ip_mc_list *pmc) /* * remove zero-count source records from a source filter list */ -static void igmpv3_clear_zeros(struct ip_sf_list **ppsf) +static void igmpv3_clear_zeros(struct ip_sf_list __rcu **ppsf) { struct ip_sf_list *psf_prev, *psf_next, *psf; psf_prev = NULL; - for (psf = *ppsf; psf; psf = psf_next) { - psf_next = psf->sf_next; + for (psf = rcu_dereference_protected(*ppsf, 1); psf; psf = psf_next) { + psf_next = rcu_dereference_protected(psf->sf_next, 1); if (psf->sf_crcount == 0) { if (psf_prev) - psf_prev->sf_next = psf->sf_next; + rcu_assign_pointer(psf_prev->sf_next, psf_next); else - *ppsf = psf->sf_next; - kfree(psf); - } else + rcu_assign_pointer(*ppsf, psf_next); + kfree_rcu(psf, rcu); + } else { psf_prev = psf; + } } } static void kfree_pmc(struct ip_mc_list *pmc) { - ip_sf_list_clear_all(pmc->sources); - ip_sf_list_clear_all(pmc->tomb); + ip_sf_list_clear_all(rcu_dereference_protected(pmc->sources, 1)); + ip_sf_list_clear_all(rcu_dereference_protected(pmc->tomb, 1)); kfree(pmc); } @@ -710,7 +735,8 @@ static void igmpv3_send_cr(struct in_device *in_dev) igmpv3_clear_zeros(&pmc->sources); } } - if (pmc->crcount == 0 && !pmc->tomb && !pmc->sources) { + if (pmc->crcount == 0 && !rcu_access_pointer(pmc->tomb) && + !rcu_access_pointer(pmc->sources)) { if (pmc_prev) pmc_prev->next = pmc_next; else @@ -896,7 +922,7 @@ static int igmp_xmarksources(struct ip_mc_list *pmc, int nsrcs, __be32 *srcs) int i, scount; scount = 0; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (scount == nsrcs) break; for (i = 0; i < nsrcs; i++) { @@ -927,7 +953,7 @@ static int igmp_marksources(struct ip_mc_list *pmc, int nsrcs, __be32 *srcs) /* mark INCLUDE-mode sources */ scount = 0; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (scount == nsrcs) break; for (i = 0; i < nsrcs; i++) @@ -1228,11 +1254,12 @@ static void igmpv3_add_delrec(struct in_device *in_dev, struct ip_mc_list *im, if (pmc->sfmode == MCAST_INCLUDE) { struct ip_sf_list *psf; + for_each_psf_mclock(im, psf) + psf->sf_crcount = pmc->crcount; pmc->tomb = im->tomb; pmc->sources = im->sources; - im->tomb = im->sources = NULL; - for (psf = pmc->sources; psf; psf = psf->sf_next) - psf->sf_crcount = pmc->crcount; + RCU_INIT_POINTER(im->tomb, NULL); + RCU_INIT_POINTER(im->sources, NULL); } spin_unlock_bh(&im->lock); @@ -1271,9 +1298,18 @@ static void igmpv3_del_delrec(struct in_device *in_dev, struct ip_mc_list *im) if (pmc) { im->interface = pmc->interface; if (im->sfmode == MCAST_INCLUDE) { - swap(im->tomb, pmc->tomb); - swap(im->sources, pmc->sources); - for (psf = im->sources; psf; psf = psf->sf_next) + struct ip_sf_list *sources, *tomb; + + tomb = rcu_replace_pointer(im->tomb, + rcu_dereference_protected(pmc->tomb, 1), + lockdep_is_held(&im->lock)); + rcu_assign_pointer(pmc->tomb, tomb); + + sources = rcu_replace_pointer(im->sources, + rcu_dereference_protected(pmc->sources, 1), + lockdep_is_held(&im->lock)); + rcu_assign_pointer(pmc->sources, sources); + for_each_psf_mclock(im, psf) psf->sf_crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); } else { @@ -1310,8 +1346,8 @@ static void igmpv3_clear_delrec(struct in_device *in_dev) struct ip_sf_list *psf; spin_lock_bh(&pmc->lock); - psf = pmc->tomb; - pmc->tomb = NULL; + psf = pmc_dereference(pmc->tomb, pmc); + RCU_INIT_POINTER(pmc->tomb, NULL); spin_unlock_bh(&pmc->lock); ip_sf_list_clear_all(psf); } @@ -1988,7 +2024,7 @@ static int ip_mc_del1_src(struct ip_mc_list *pmc, int sfmode, int rv = 0; psf_prev = NULL; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (psf->sf_inaddr == *psfsrc) break; psf_prev = psf; @@ -1997,7 +2033,7 @@ static int ip_mc_del1_src(struct ip_mc_list *pmc, int sfmode, /* source filter not found, or count wrong => bug */ return -ESRCH; } - psf->sf_count[sfmode]--; + WRITE_ONCE(psf->sf_count[sfmode], psf->sf_count[sfmode] - 1); if (psf->sf_count[sfmode] == 0) { ip_rt_multicast_event(pmc->interface); } @@ -2009,19 +2045,28 @@ static int ip_mc_del1_src(struct ip_mc_list *pmc, int sfmode, /* no more filters for this source */ if (psf_prev) - psf_prev->sf_next = psf->sf_next; + rcu_assign_pointer(psf_prev->sf_next, + pmc_dereference(psf->sf_next, pmc)); else - pmc->sources = psf->sf_next; + rcu_assign_pointer(pmc->sources, + pmc_dereference(psf->sf_next, pmc)); #ifdef CONFIG_IP_MULTICAST if (psf->sf_oldin && !IGMP_V1_SEEN(in_dev) && !IGMP_V2_SEEN(in_dev)) { - psf->sf_crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); - psf->sf_next = pmc->tomb; - pmc->tomb = psf; - rv = 1; - } else + struct ip_sf_list *dpsf = kmalloc_obj(*dpsf, GFP_ATOMIC); + + if (dpsf) { + *dpsf = *psf; + dpsf->sf_crcount = in_dev->mr_qrv ?: + READ_ONCE(net->ipv4.sysctl_igmp_qrv); + rcu_assign_pointer(dpsf->sf_next, + pmc_dereference(pmc->tomb, pmc)); + rcu_assign_pointer(pmc->tomb, dpsf); + rv = 1; + } + } #endif - kfree(psf); + kfree_rcu(psf, rcu); } return rv; } @@ -2058,7 +2103,7 @@ static int ip_mc_del_src(struct in_device *in_dev, __be32 *pmca, int sfmode, err = -EINVAL; if (!pmc->sfcount[sfmode]) goto out_unlock; - pmc->sfcount[sfmode]--; + WRITE_ONCE(pmc->sfcount[sfmode], pmc->sfcount[sfmode] - 1); } err = 0; for (i = 0; i < sfcount; i++) { @@ -2081,7 +2126,7 @@ static int ip_mc_del_src(struct in_device *in_dev, __be32 *pmca, int sfmode, #ifdef CONFIG_IP_MULTICAST pmc->crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); WRITE_ONCE(in_dev->mr_ifc_count, pmc->crcount); - for (psf = pmc->sources; psf; psf = psf->sf_next) + for_each_psf_mclock(pmc, psf) psf->sf_crcount = 0; igmp_ifc_event(pmc->interface); } else if (sf_setstate(pmc) || changerec) { @@ -2102,7 +2147,7 @@ static int ip_mc_add1_src(struct ip_mc_list *pmc, int sfmode, struct ip_sf_list *psf, *psf_prev; psf_prev = NULL; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (psf->sf_inaddr == *psfsrc) break; psf_prev = psf; @@ -2112,12 +2157,12 @@ static int ip_mc_add1_src(struct ip_mc_list *pmc, int sfmode, if (!psf) return -ENOBUFS; psf->sf_inaddr = *psfsrc; - if (psf_prev) { - psf_prev->sf_next = psf; - } else - pmc->sources = psf; + if (psf_prev) + rcu_assign_pointer(psf_prev->sf_next, psf); + else + rcu_assign_pointer(pmc->sources, psf); } - psf->sf_count[sfmode]++; + WRITE_ONCE(psf->sf_count[sfmode], psf->sf_count[sfmode] + 1); if (psf->sf_count[sfmode] == 1) { ip_rt_multicast_event(pmc->interface); } @@ -2130,13 +2175,15 @@ static void sf_markstate(struct ip_mc_list *pmc) struct ip_sf_list *psf; int mca_xcount = pmc->sfcount[MCAST_EXCLUDE]; - for (psf = pmc->sources; psf; psf = psf->sf_next) + for_each_psf_mclock(pmc, psf) { if (pmc->sfcount[MCAST_EXCLUDE]) { psf->sf_oldin = mca_xcount == psf->sf_count[MCAST_EXCLUDE] && !psf->sf_count[MCAST_INCLUDE]; - } else + } else { psf->sf_oldin = psf->sf_count[MCAST_INCLUDE] != 0; + } + } } static int sf_setstate(struct ip_mc_list *pmc) @@ -2147,27 +2194,31 @@ static int sf_setstate(struct ip_mc_list *pmc) int new_in, rv; rv = 0; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (pmc->sfcount[MCAST_EXCLUDE]) { new_in = mca_xcount == psf->sf_count[MCAST_EXCLUDE] && !psf->sf_count[MCAST_INCLUDE]; - } else + } else { new_in = psf->sf_count[MCAST_INCLUDE] != 0; + } if (new_in) { if (!psf->sf_oldin) { struct ip_sf_list *prev = NULL; - for (dpsf = pmc->tomb; dpsf; dpsf = dpsf->sf_next) { + for_each_psf_tomb(pmc, dpsf) { if (dpsf->sf_inaddr == psf->sf_inaddr) break; prev = dpsf; } if (dpsf) { + struct ip_sf_list *dpsf_next; + + dpsf_next = pmc_dereference(dpsf->sf_next, pmc); if (prev) - prev->sf_next = dpsf->sf_next; + rcu_assign_pointer(prev->sf_next, dpsf_next); else - pmc->tomb = dpsf->sf_next; - kfree(dpsf); + rcu_assign_pointer(pmc->tomb, dpsf_next); + kfree_rcu(dpsf, rcu); } psf->sf_crcount = qrv; rv++; @@ -2179,17 +2230,19 @@ static int sf_setstate(struct ip_mc_list *pmc) * add or update "delete" records if an active filter * is now inactive */ - for (dpsf = pmc->tomb; dpsf; dpsf = dpsf->sf_next) + for_each_psf_tomb(pmc, dpsf) { if (dpsf->sf_inaddr == psf->sf_inaddr) break; + } if (!dpsf) { dpsf = kmalloc_obj(*dpsf, GFP_ATOMIC); if (!dpsf) continue; *dpsf = *psf; /* pmc->lock held by callers */ - dpsf->sf_next = pmc->tomb; - pmc->tomb = dpsf; + rcu_assign_pointer(dpsf->sf_next, + pmc_dereference(pmc->tomb, pmc)); + rcu_assign_pointer(pmc->tomb, dpsf); } dpsf->sf_crcount = qrv; rv++; @@ -2229,7 +2282,7 @@ static int ip_mc_add_src(struct in_device *in_dev, __be32 *pmca, int sfmode, #endif isexclude = pmc->sfmode == MCAST_EXCLUDE; if (!delta) - pmc->sfcount[sfmode]++; + WRITE_ONCE(pmc->sfcount[sfmode], pmc->sfcount[sfmode] + 1); err = 0; for (i = 0; i < sfcount; i++) { err = ip_mc_add1_src(pmc, sfmode, &psfsrc[i]); @@ -2240,7 +2293,7 @@ static int ip_mc_add_src(struct in_device *in_dev, __be32 *pmca, int sfmode, int j; if (!delta) - pmc->sfcount[sfmode]--; + WRITE_ONCE(pmc->sfcount[sfmode], pmc->sfcount[sfmode] - 1); for (j = 0; j < i; j++) (void) ip_mc_del1_src(pmc, sfmode, &psfsrc[j]); } else if (isexclude != (pmc->sfcount[MCAST_EXCLUDE] != 0)) { @@ -2260,7 +2313,7 @@ static int ip_mc_add_src(struct in_device *in_dev, __be32 *pmca, int sfmode, pmc->crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); WRITE_ONCE(in_dev->mr_ifc_count, pmc->crcount); - for (psf = pmc->sources; psf; psf = psf->sf_next) + for_each_psf_mclock(pmc, psf) psf->sf_crcount = 0; igmp_ifc_event(in_dev); } else if (sf_setstate(pmc)) { @@ -2276,13 +2329,13 @@ static void ip_mc_clear_src(struct ip_mc_list *pmc) struct ip_sf_list *tomb, *sources; spin_lock_bh(&pmc->lock); - tomb = pmc->tomb; - pmc->tomb = NULL; - sources = pmc->sources; - pmc->sources = NULL; + tomb = pmc_dereference(pmc->tomb, pmc); + RCU_INIT_POINTER(pmc->tomb, NULL); + sources = pmc_dereference(pmc->sources, pmc); + RCU_INIT_POINTER(pmc->sources, NULL); pmc->sfmode = MCAST_EXCLUDE; - pmc->sfcount[MCAST_INCLUDE] = 0; - pmc->sfcount[MCAST_EXCLUDE] = 1; + WRITE_ONCE(pmc->sfcount[MCAST_INCLUDE], 0); + WRITE_ONCE(pmc->sfcount[MCAST_EXCLUDE], 1); spin_unlock_bh(&pmc->lock); ip_sf_list_clear_all(tomb); @@ -2864,20 +2917,19 @@ int ip_check_mc_rcu(struct in_device *in_dev, __be32 mc_addr, __be32 src_addr, u rv = 1; } else if (im) { if (src_addr) { - spin_lock_bh(&im->lock); - for (psf = im->sources; psf; psf = psf->sf_next) { + for_each_psf_rcu(im, psf) { if (psf->sf_inaddr == src_addr) break; } if (psf) - rv = psf->sf_count[MCAST_INCLUDE] || - psf->sf_count[MCAST_EXCLUDE] != - im->sfcount[MCAST_EXCLUDE]; + rv = READ_ONCE(psf->sf_count[MCAST_INCLUDE]) || + READ_ONCE(psf->sf_count[MCAST_EXCLUDE]) != + READ_ONCE(im->sfcount[MCAST_EXCLUDE]); else - rv = im->sfcount[MCAST_EXCLUDE] != 0; - spin_unlock_bh(&im->lock); - } else + rv = READ_ONCE(im->sfcount[MCAST_EXCLUDE]) != 0; + } else { rv = 1; /* unspecified source; tentatively allow */ + } } return rv; } @@ -3041,7 +3093,7 @@ static inline struct ip_sf_list *igmp_mcf_get_first(struct seq_file *seq) im = rcu_dereference(idev->mc_list); if (likely(im)) { spin_lock_bh(&im->lock); - psf = im->sources; + psf = pmc_dereference(im->sources, im); if (likely(psf)) { state->im = im; state->idev = idev; @@ -3057,7 +3109,7 @@ static struct ip_sf_list *igmp_mcf_get_next(struct seq_file *seq, struct ip_sf_l { struct igmp_mcf_iter_state *state = igmp_mcf_seq_private(seq); - psf = psf->sf_next; + psf = pmc_dereference(psf->sf_next, state->im); while (!psf) { spin_unlock_bh(&state->im->lock); state->im = state->im->next; @@ -3073,7 +3125,7 @@ static struct ip_sf_list *igmp_mcf_get_next(struct seq_file *seq, struct ip_sf_l state->im = rcu_dereference(state->idev->mc_list); } spin_lock_bh(&state->im->lock); - psf = state->im->sources; + psf = pmc_dereference(state->im->sources, state->im); } out: return psf; diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 6257459bcee24..6a30f11384547 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -1520,7 +1520,8 @@ void inet_csk_listen_stop(struct sock *sk) local_bh_enable(); sock_put(child); - cond_resched(); + if (!has_current_bpf_ctx()) + cond_resched(); } if (queue->fastopenq.rskq_rst_head) { /* Free all the reqs queued in rskq_rst_head. */ diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c index fc0cb993959f9..b286ee429da8f 100644 --- a/net/ipv4/inet_fragment.c +++ b/net/ipv4/inet_fragment.c @@ -235,6 +235,8 @@ void fqdir_pre_exit(struct fqdir *fqdir) rhashtable_walk_start(&hti); while ((fq = rhashtable_walk_next(&hti))) { + int refs = 0; + if (IS_ERR(fq)) { if (PTR_ERR(fq) != -EAGAIN) break; @@ -242,8 +244,12 @@ void fqdir_pre_exit(struct fqdir *fqdir) } spin_lock_bh(&fq->lock); if (!(fq->flags & INET_FRAG_COMPLETE)) + inet_frag_kill(fq, &refs); + + if (fq->flags & INET_FRAG_HASH_DEAD) inet_frag_queue_flush(fq, 0); spin_unlock_bh(&fq->lock); + inet_frag_putn(fq, refs); } rhashtable_walk_stop(&hti); @@ -435,6 +441,13 @@ int inet_frag_queue_insert(struct inet_frag_queue *q, struct sk_buff *skb, { struct sk_buff *last = q->fragments_tail; + /* An IP fragment is never a GSO packet, but an untrusted source + * (virtio_net_hdr) may have attached GSO metadata to it. Do not let + * that reach the reassembled skb, whose head keeps the first + * fragment's shinfo and whose frag_list is not GRO-shaped. + */ + skb_gso_reset(skb); + /* RFC5722, Section 4, amended by Errata ID : 3089 * When reassembling an IPv6 datagram, if * one or more its constituent fragments is determined to be an diff --git a/net/ipv4/inetpeer.c b/net/ipv4/inetpeer.c index 5b957a831e7c3..adf6dc8a95b4f 100644 --- a/net/ipv4/inetpeer.c +++ b/net/ipv4/inetpeer.c @@ -21,6 +21,7 @@ #include #include #include +#include /* * Theory of operations. @@ -52,6 +53,34 @@ */ static struct kmem_cache *peer_cachep __ro_after_init; +static siphash_aligned_key_t inetpeer_hash_key __read_mostly; + +static u64 inetpeer_addr_hash(const struct inetpeer_addr *a) +{ + net_get_random_once(&inetpeer_hash_key, sizeof(inetpeer_hash_key)); + + if (a->family == AF_INET) + return siphash_2u32((__force u32)a->a4.addr, a->a4.vif, + &inetpeer_hash_key); + + return siphash_4u32((__force u32)a->a6.s6_addr32[0], + (__force u32)a->a6.s6_addr32[1], + (__force u32)a->a6.s6_addr32[2], + (__force u32)a->a6.s6_addr32[3], + &inetpeer_hash_key); +} + +static int inetpeer_entry_cmp(u64 dhash, + const struct inetpeer_addr *daddr, + const struct inet_peer *p) +{ + if (dhash < p->hash) + return -1; + if (dhash > p->hash) + return 1; + + return inetpeer_addr_cmp(daddr, &p->daddr); +} void inet_peer_base_init(struct inet_peer_base *bp) { @@ -84,6 +113,7 @@ void __init inet_initpeers(void) /* Called with rcu_read_lock() or base->lock held */ static struct inet_peer *lookup(const struct inetpeer_addr *daddr, + u64 dhash, struct inet_peer_base *base, unsigned int seq, struct inet_peer *gc_stack[], @@ -105,7 +135,7 @@ static struct inet_peer *lookup(const struct inetpeer_addr *daddr, break; parent = next; p = rb_entry(parent, struct inet_peer, rb_node); - cmp = inetpeer_addr_cmp(daddr, &p->daddr); + cmp = inetpeer_entry_cmp(dhash, daddr, p); if (cmp == 0) { now = jiffies; if (READ_ONCE(p->dtime) != now) @@ -170,6 +200,7 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, const struct inetpeer_addr *daddr) { struct inet_peer *p, *gc_stack[PEER_MAX_GC]; + u64 dhash = inetpeer_addr_hash(daddr); struct rb_node **pp, *parent; unsigned int gc_cnt, seq; @@ -177,7 +208,7 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, * Because of a concurrent writer, we might not find an existing entry. */ seq = read_seqbegin(&base->lock); - p = lookup(daddr, base, seq, NULL, &gc_cnt, &parent, &pp); + p = lookup(daddr, dhash, base, seq, NULL, &gc_cnt, &parent, &pp); /* Make sure tree was not modified during our lookup. */ if (p && !read_seqretry(&base->lock, seq)) @@ -190,11 +221,12 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, write_seqlock_bh(&base->lock); gc_cnt = 0; - p = lookup(daddr, base, seq, gc_stack, &gc_cnt, &parent, &pp); + p = lookup(daddr, dhash, base, seq, gc_stack, &gc_cnt, &parent, &pp); if (!p) { p = kmem_cache_alloc(peer_cachep, GFP_ATOMIC); if (p) { p->daddr = *daddr; + p->hash = dhash; p->dtime = (__u32)jiffies; refcount_set(&p->refcnt, 1); atomic_set(&p->rid, 0); diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index 0ba1e94e9012c..384c02b9700b8 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -1460,6 +1460,12 @@ static int ipgre_changelink(struct net_device *dev, struct nlattr *tb[], if (!rtnl_dev_link_net_capable(dev, t->net)) return -EPERM; + if (data && data[IFLA_GRE_COLLECT_METADATA] && !t->collect_md) { + NL_SET_ERR_MSG(extack, + "Enabling collect_md on an existing device is not supported"); + return -EOPNOTSUPP; + } + err = ipgre_newlink_encap_setup(dev, data); if (err) return err; @@ -1492,6 +1498,12 @@ static int erspan_changelink(struct net_device *dev, struct nlattr *tb[], if (!rtnl_dev_link_net_capable(dev, t->net)) return -EPERM; + if (data && data[IFLA_GRE_COLLECT_METADATA] && !t->collect_md) { + NL_SET_ERR_MSG(extack, + "Enabling collect_md on an existing device is not supported"); + return -EOPNOTSUPP; + } + err = ipgre_newlink_encap_setup(dev, data); if (err) return err; diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index e6dd1e5b8c327..a24cc8ee11d3e 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -790,6 +790,10 @@ int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, */ hlen = iph->ihl * 4; + if (mtu < hlen + 8) { + err = -EMSGSIZE; + goto fail; + } mtu = mtu - hlen; /* Size of data space */ IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE; ll_rs = LL_RESERVED_SPACE(rt->dst.dev); @@ -1299,6 +1303,7 @@ static int ip_setup_cork(struct sock *sk, struct inet_cork *cork, cork->fragsize = ip_sk_use_pmtu(sk) ? dst4_mtu(&rt->dst) : READ_ONCE(rt->dst.dev->mtu); + cork->fragsize = min(cork->fragsize, IP_MAX_MTU); if (!inetdev_valid_mtu(cork->fragsize)) return -ENETUNREACH; diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c index 9d114bd575f92..5b1f180485d42 100644 --- a/net/ipv4/ip_tunnel.c +++ b/net/ipv4/ip_tunnel.c @@ -317,7 +317,7 @@ static int ip_tunnel_bind_dev(struct net_device *dev) mtu = min(tdev->mtu, IP_MAX_MTU); } - dev->needed_headroom = t_hlen + hlen; + dev->needed_headroom = ip_tunnel_limit_headroom(t_hlen + hlen); mtu -= t_hlen + (dev->type == ARPHRD_ETHER ? dev->hard_header_len : 0); if (mtu < IPV4_MIN_MTU) diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c index d3c677e9bff20..bab42b9e277f7 100644 --- a/net/ipv4/ip_tunnel_core.c +++ b/net/ipv4/ip_tunnel_core.c @@ -268,6 +268,9 @@ static int iptunnel_pmtud_build_icmp(struct sk_buff *skb, int mtu) eth_header(skb, skb->dev, ntohs(eh.h_proto), eh.h_source, eh.h_dest, 0); skb_reset_mac_header(skb); + if (skb_valid_dst(skb)) + skb_dst_drop(skb); + return skb->len; } @@ -371,6 +374,9 @@ static int iptunnel_pmtud_build_icmpv6(struct sk_buff *skb, int mtu) eth_header(skb, skb->dev, ntohs(eh.h_proto), eh.h_source, eh.h_dest, 0); skb_reset_mac_header(skb); + if (skb_valid_dst(skb)) + skb_dst_drop(skb); + return skb->len; } @@ -674,8 +680,14 @@ static int ip_tun_get_optlen(struct nlattr *attr, } static int ip_tun_set_opts(struct nlattr *attr, struct ip_tunnel_info *info, - struct netlink_ext_ack *extack) + int opts_len, struct netlink_ext_ack *extack) { + /* `options_len` is the __counted_by() annotation of the `options` + * flexible array, it must be initialized before parsing writes + * into it. + */ + info->options_len = opts_len; + return ip_tun_parse_opts(attr, info, extack); } @@ -706,7 +718,8 @@ static int ip_tun_build_state(struct net *net, struct nlattr *attr, tun_info = lwt_tun_info(new_state); - err = ip_tun_set_opts(tb[LWTUNNEL_IP_OPTS], tun_info, extack); + err = ip_tun_set_opts(tb[LWTUNNEL_IP_OPTS], tun_info, opt_len, + extack); if (err < 0) { lwtstate_free(new_state); return err; @@ -747,7 +760,6 @@ static int ip_tun_build_state(struct net *net, struct nlattr *attr, } tun_info->mode = IP_TUNNEL_INFO_TX; - tun_info->options_len = opt_len; *ts = new_state; @@ -1000,7 +1012,8 @@ static int ip6_tun_build_state(struct net *net, struct nlattr *attr, tun_info = lwt_tun_info(new_state); - err = ip_tun_set_opts(tb[LWTUNNEL_IP6_OPTS], tun_info, extack); + err = ip_tun_set_opts(tb[LWTUNNEL_IP6_OPTS], tun_info, opt_len, + extack); if (err < 0) { lwtstate_free(new_state); return err; @@ -1034,7 +1047,6 @@ static int ip6_tun_build_state(struct net *net, struct nlattr *attr, } tun_info->mode = IP_TUNNEL_INFO_TX | IP_TUNNEL_INFO_IPV6; - tun_info->options_len = opt_len; *ts = new_state; diff --git a/net/ipv4/ipconfig.c b/net/ipv4/ipconfig.c index a35ffedacc7c7..b8a0c98e5e55d 100644 --- a/net/ipv4/ipconfig.c +++ b/net/ipv4/ipconfig.c @@ -676,6 +676,24 @@ static const u8 ic_bootp_cookie[4] = { 99, 130, 83, 99 }; #ifdef IPCONFIG_DHCP +static bool __init +ic_dhcp_add_option(u8 **options, const u8 *end, u8 type, const void *value, + int len) +{ + u8 *e = *options; + + /* leave room for the option header and the END marker */ + if (len > U8_MAX || end - e < len + 3) + return false; + + *e++ = type; + *e++ = len; + memcpy(e, value, len); + *options = e + len; + + return true; +} + static void __init ic_dhcp_init_options(u8 *options, struct ic_device *d) { @@ -691,6 +709,7 @@ ic_dhcp_init_options(u8 *options, struct ic_device *d) 42, /* NTP servers */ }; u8 mt = (ic_servaddr == NONE) ? DHCPDISCOVER : DHCPREQUEST; + u8 *end = options + sizeof(((struct bootp_pkt *)0)->exten); u8 *e = options; int len; @@ -721,31 +740,19 @@ ic_dhcp_init_options(u8 *options, struct ic_device *d) e += sizeof(ic_req_params); if (ic_host_name_set) { - *e++ = 12; /* host-name */ len = strlen(utsname()->nodename); - *e++ = len; - memcpy(e, utsname()->nodename, len); - e += len; + ic_dhcp_add_option(&e, end, 12, utsname()->nodename, len); } if (*vendor_class_identifier) { - pr_info("DHCP: sending class identifier \"%s\"\n", - vendor_class_identifier); - *e++ = 60; /* Class-identifier */ len = strlen(vendor_class_identifier); - *e++ = len; - memcpy(e, vendor_class_identifier, len); - e += len; + if (ic_dhcp_add_option(&e, end, 60, vendor_class_identifier, len)) + pr_info("DHCP: sending class identifier \"%s\"\n", + vendor_class_identifier); } len = strlen(dhcp_client_identifier + 1); - /* the minimum length of identifier is 2, include 1 byte type, - * and can not be larger than the length of options - */ - if (len >= 1 && len < 312 - (e - options) - 1) { - *e++ = 61; - *e++ = len + 1; - memcpy(e, dhcp_client_identifier, len + 1); - e += len + 1; - } + /* the minimum length of identifier is 2, include 1 byte type */ + if (len >= 1) + ic_dhcp_add_option(&e, end, 61, dhcp_client_identifier, len + 1); *e++ = 255; /* End of the list */ } diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index b643194f57d29..ddf62b45566b6 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -248,7 +248,7 @@ static int ipip_tunnel_rcv(struct sk_buff *skb, u8 ipproto) tun_dst = ip_tun_rx_dst(skb, flags, 0, 0); if (!tun_dst) - return 0; + goto drop; ip_tunnel_md_udp_encap(skb, &tun_dst->u.tun_info); } skb_reset_mac_header(skb); diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 1d9a4ac14fcef..b9c544d48c452 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -2213,6 +2213,9 @@ int ip_mr_input(struct sk_buff *skb) if (IPCB(skb)->flags & IPSKB_FORWARDED) goto dont_forward; + if (!local) + skb_orphan(skb); + mrt = ipmr_rt_fib_lookup(net, skb); if (IS_ERR(mrt)) { kfree_skb(skb); @@ -3373,7 +3376,8 @@ int __init ip_mr_init(void) { int err; - mrt_cachep = KMEM_CACHE(mfc_cache, SLAB_HWCACHE_ALIGN | SLAB_PANIC); + mrt_cachep = KMEM_CACHE(mfc_cache, + SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT); err = register_pernet_subsys(&ipmr_net_ops); if (err) diff --git a/net/ipv4/ipmr_base.c b/net/ipv4/ipmr_base.c index 867b24beded11..a0ec6d19a237f 100644 --- a/net/ipv4/ipmr_base.c +++ b/net/ipv4/ipmr_base.c @@ -52,7 +52,7 @@ mr_table_alloc(struct net *net, u32 id, struct mr_table *mrt; int err; - mrt = kzalloc_obj(*mrt); + mrt = kzalloc_obj(*mrt, GFP_KERNEL_ACCOUNT); if (!mrt) return ERR_PTR(-ENOMEM); mrt->id = id; diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 0f1e21a5c812e..901c353196828 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -2029,7 +2029,7 @@ remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, { struct nh_grp_entry *nhges, *new_nhges; struct nexthop *nhp = nhge->nh_parent; - struct netlink_ext_ack extack; + struct netlink_ext_ack extack = {}; struct nexthop *nh = nhge->nh; struct nh_group *nhg, *newg; int i, j, err; diff --git a/net/ipv4/route.c b/net/ipv4/route.c index b82401a6baed7..f07231447883f 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -1358,8 +1358,8 @@ static unsigned int ipv4_default_advmss(const struct dst_entry *dst) rcu_read_lock(); net = dst_dev_net_rcu(dst); - advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size, - net->ipv4.ip_rt_min_advmss); + advmss = max_t(unsigned int, ip_dst_mtu_configured(dst) - header_size, + net->ipv4.ip_rt_min_advmss); rcu_read_unlock(); return min(advmss, IPV4_MAX_PMTU - header_size); diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index ca1180dba1dea..e921cb381b53c 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -51,6 +51,8 @@ static int tcp_ecn_mode_max = 5; static u32 icmp_errors_extension_mask_all = GENMASK_U8(ICMP_ERR_EXT_COUNT - 1, 0); +static int tcp_min_rcvbuf = 4096; + /* obsolete */ static int sysctl_tcp_low_latency __read_mostly; @@ -1462,7 +1464,7 @@ static struct ctl_table ipv4_net_table[] = { .maxlen = sizeof(init_net.ipv4.sysctl_tcp_rmem), .mode = 0644, .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ONE, + .extra1 = &tcp_min_rcvbuf, }, { .procname = "tcp_comp_sack_delay_ns", diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 455441f1b6949..47a34a993972a 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1169,8 +1169,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) zc = MSG_SPLICE_PAGES; } - if (!sockc_err && sockc.dmabuf_id && - (!(flags & MSG_ZEROCOPY) || !sock_flag(sk, SOCK_ZEROCOPY))) { + if (!sockc_err && sockc.dmabuf_id && (zc != MSG_ZEROCOPY || !binding)) { err = -EINVAL; goto out_err; } @@ -1240,7 +1239,8 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) trace_tcp_sendmsg_locked(sk, msg, skb, size_goal); - if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) { + if (copy <= 0 || !tcp_skb_can_collapse_to(skb) || + unlikely(skb_frags_readable(skb) != !binding)) { bool first_skb; new_segment: @@ -3181,8 +3181,7 @@ void __tcp_close(struct sock *sk, long timeout) /* Unread data was tossed, zap the connection. */ NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE); tcp_set_state(sk, TCP_CLOSE); - tcp_send_active_reset(sk, sk->sk_allocation, - SK_RST_REASON_TCP_ABORT_ON_CLOSE); + tcp_send_active_reset(sk, SK_RST_REASON_TCP_ABORT_ON_CLOSE); } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { /* Check zero linger _after_ checking for unread data. */ sk->sk_prot->disconnect(sk, 0); @@ -3256,7 +3255,7 @@ void __tcp_close(struct sock *sk, long timeout) struct tcp_sock *tp = tcp_sk(sk); if (READ_ONCE(tp->linger2) < 0) { tcp_set_state(sk, TCP_CLOSE); - tcp_send_active_reset(sk, GFP_ATOMIC, + tcp_send_active_reset(sk, SK_RST_REASON_TCP_ABORT_ON_LINGER); __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONLINGER); @@ -3275,7 +3274,7 @@ void __tcp_close(struct sock *sk, long timeout) if (sk->sk_state != TCP_CLOSE) { if (tcp_check_oom(sk, 0)) { tcp_set_state(sk, TCP_CLOSE); - tcp_send_active_reset(sk, GFP_ATOMIC, + tcp_send_active_reset(sk, SK_RST_REASON_TCP_ABORT_ON_MEMORY); __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONMEMORY); @@ -3376,14 +3375,14 @@ int tcp_disconnect(struct sock *sk, int flags) } else if (unlikely(tp->repair)) { WRITE_ONCE(sk->sk_err, ECONNABORTED); } else if (tcp_need_reset(old_state)) { - tcp_send_active_reset(sk, gfp_any(), SK_RST_REASON_TCP_STATE); + tcp_send_active_reset(sk, SK_RST_REASON_TCP_STATE); WRITE_ONCE(sk->sk_err, ECONNRESET); } else if (tp->snd_nxt != tp->write_seq && (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) { /* The last check adjusts for discrepancy of Linux wrt. RFC * states */ - tcp_send_active_reset(sk, gfp_any(), + tcp_send_active_reset(sk, SK_RST_REASON_TCP_DISCONNECT_WITH_DATA); WRITE_ONCE(sk->sk_err, ECONNRESET); } else if (old_state == TCP_SYN_SENT) @@ -5146,8 +5145,7 @@ int tcp_abort(struct sock *sk, int err) bh_lock_sock(sk); if (tcp_need_reset(sk->sk_state)) - tcp_send_active_reset(sk, GFP_ATOMIC, - SK_RST_REASON_TCP_STATE); + tcp_send_active_reset(sk, SK_RST_REASON_TCP_STATE); tcp_done_with_error(sk, err); bh_unlock_sock(sk); diff --git a/net/ipv4/tcp_ao.c b/net/ipv4/tcp_ao.c index e4ec60a334963..bb7bbc20ba3fa 100644 --- a/net/ipv4/tcp_ao.c +++ b/net/ipv4/tcp_ao.c @@ -1160,6 +1160,15 @@ void tcp_ao_connect_init(struct sock *sk) l3index = l3mdev_master_ifindex_by_index(sock_net(sk), sk->sk_bound_dev_if); + hlist_for_each_entry(key, &ao_info->head, node) { + if (tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, + family, -1, -1)) { + /* pairs with tcp_inbound_ao_hash() */ + synchronize_rcu(); + break; + } + } + hlist_for_each_entry_safe(key, next, &ao_info->head, node) { if (!tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, family, -1, -1)) continue; @@ -1187,12 +1196,7 @@ void tcp_ao_connect_init(struct sock *sk) ao_info->lisn = htonl(tp->write_seq); ao_info->snd_sne = 0; } else { - /* Can't happen: tcp_connect() verifies that there's - * at least one tcp-ao key that matches the remote peer. - */ - WARN_ON_ONCE(1); - rcu_assign_pointer(tp->ao_info, NULL); - kfree(ao_info); + tcp_ao_destroy_sock(sk, false); } } @@ -1825,6 +1829,9 @@ static int tcp_ao_del_cmd(struct sock *sk, unsigned short int family, if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX)) return -EINVAL; + if (cmd.keyflags & TCP_AO_KEYF_IFINDEX) + l3index = cmd.ifindex; + ao_info = setsockopt_ao_info(sk); if (IS_ERR(ao_info)) return PTR_ERR(ao_info); diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 0f60a1dbf9274..92bc60716f33d 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -6490,6 +6490,7 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, * or pure receivers (this means either the sequence number or the ack * value must stay constant) * - Unexpected TCP option. + * - ACK sequence number is outside [SND.UNA, SND.NXT]. * * When these conditions are not satisfied it drops into a standard * receive procedure patterned after RFC793 to handle all cases. @@ -6539,7 +6540,7 @@ void tcp_rcv_established(struct sock *sk, struct sk_buff *skb) if ((tcp_flag_word(th) & TCP_HP_BITS) == tp->pred_flags && TCP_SKB_CB(skb)->seq == tp->rcv_nxt && - !after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) { + between(TCP_SKB_CB(skb)->ack_seq, tp->snd_una, tp->snd_nxt)) { int tcp_header_len = tp->tcp_header_len; s32 delta = 0; int flag = 0; diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 7f413f509d7dc..497b1a0370cda 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1736,7 +1736,7 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, tcp_ca_openreq_child(newsk, dst); tcp_sync_mss(newsk, dst4_mtu(dst)); - newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); + newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst)); tcp_initialize_rcv_mss(newsk); diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index 6ab3e3a0b4317..fb901b368ecc7 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -440,7 +440,7 @@ void tcp_openreq_init_rwin(struct request_sock *req, u32 rcv_wnd; int mss; - mss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); + mss = tcp_mss_clamp(tp, tcp_dst_advmss(dst)); window_clamp = READ_ONCE(tp->window_clamp); /* Set this up on the first call only */ req->rsk_window_clamp = window_clamp ? : dst_metric(dst, RTAX_WINDOW); diff --git a/net/ipv4/tcp_offload.c b/net/ipv4/tcp_offload.c index 3b1fdcd3cb29b..e74d99ca9face 100644 --- a/net/ipv4/tcp_offload.c +++ b/net/ipv4/tcp_offload.c @@ -332,6 +332,7 @@ struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb, flush |= skb->ip_summed != p->ip_summed; flush |= skb->csum_level != p->csum_level; flush |= NAPI_GRO_CB(p)->count >= 64; + flush |= NAPI_GRO_CB(p)->is_flist != NAPI_GRO_CB(skb)->is_flist; skb_set_network_header(skb, skb_gro_receive_network_offset(skb)); if (flush || skb_gro_receive_list(p, skb)) @@ -395,12 +396,20 @@ static void tcp4_check_fraglist_gro(struct list_head *head, struct sk_buff *skb, struct net *net; int iif, sdif; - if (likely(!(skb->dev->features & NETIF_F_GRO_FRAGLIST))) - return; - p = tcp_gro_lookup(head, th); if (p) { - NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist; + /* flist GRO applies to consecutive non-GSO skbs */ + if (!skb_is_gso(skb) || !NAPI_GRO_CB(p)->is_flist) { + NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist; + return; + } + + /* Fall back to the regular GRO path */ + if (NAPI_GRO_CB(p)->count == 1) + NAPI_GRO_CB(p)->is_flist = 0; + + NAPI_GRO_CB(skb)->is_flist = 0; + return; } @@ -410,7 +419,7 @@ static void tcp4_check_fraglist_gro(struct list_head *head, struct sk_buff *skb, sk = __inet_lookup_established(net, iph->saddr, th->source, iph->daddr, ntohs(th->dest), iif, sdif); - NAPI_GRO_CB(skb)->is_flist = !sk; + NAPI_GRO_CB(skb)->is_flist = !sk && !skb_is_gso(skb); if (sk) sock_gen_put(sk); } @@ -430,7 +439,8 @@ struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb) if (!th) goto flush; - tcp4_check_fraglist_gro(head, skb, th); + if (unlikely(skb->dev->features & NETIF_F_GRO_FRAGLIST)) + tcp4_check_fraglist_gro(head, skb, th); return tcp_gro_receive(head, skb, th); diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index d7c1444b5e304..0b89b624af8ed 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -143,7 +143,7 @@ static __u16 tcp_advertise_mss(struct sock *sk) int mss = tp->advmss; if (dst) { - unsigned int metric = dst_metric_advmss(dst); + unsigned int metric = tcp_dst_advmss(dst); if (metric < mss) { mss = metric; @@ -3603,7 +3603,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) avail_wnd = cur_mss; } - len = cur_mss * segs; + len = cur_mss * (tcp_urg_mode(tp) ? 1 : segs); if (len > avail_wnd) { len = rounddown(avail_wnd, cur_mss); if (!len) @@ -3849,9 +3849,9 @@ void tcp_send_fin(struct sock *sk) * was unread data in the receive queue. This behavior is recommended * by RFC 2525, section 2.17. -DaveM */ -void tcp_send_active_reset(struct sock *sk, gfp_t priority, - enum sk_rst_reason reason) +void tcp_send_active_reset(struct sock *sk, enum sk_rst_reason reason) { + gfp_t priority = sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN); struct sk_buff *skb; TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS); @@ -3886,6 +3886,7 @@ void tcp_send_active_reset(struct sock *sk, gfp_t priority, */ int tcp_send_synack(struct sock *sk) { + struct tcp_sock *tp = tcp_sk(sk); struct sk_buff *skb; skb = tcp_rtx_queue_head(sk); @@ -3903,6 +3904,8 @@ int tcp_send_synack(struct sock *sk) if (!nskb) return -ENOMEM; INIT_LIST_HEAD(&nskb->tcp_tsorted_anchor); + if (skb == tp->retransmit_skb_hint) + tp->retransmit_skb_hint = nskb; tcp_highest_sack_replace(sk, skb, nskb); tcp_rtx_queue_unlink_and_free(skb, sk); __skb_header_release(nskb); @@ -3972,7 +3975,7 @@ struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, } skb_dst_set(skb, dst); - mss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); + mss = tcp_mss_clamp(tp, tcp_dst_advmss(dst)); memset(&opts, 0, sizeof(opts)); now = tcp_clock_ns(); @@ -4127,7 +4130,7 @@ static void tcp_connect_init(struct sock *sk) if (!tp->window_clamp) WRITE_ONCE(tp->window_clamp, dst_metric(dst, RTAX_WINDOW)); - tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); + tp->advmss = tcp_mss_clamp(tp, tcp_dst_advmss(dst)); tcp_initialize_rcv_mss(sk); diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index 4df1c9745c5d5..95296b1cf0ca2 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -126,7 +126,7 @@ static int tcp_out_of_resources(struct sock *sk, bool do_reset) (!tp->snd_wnd && !tp->packets_out)) do_reset = true; if (do_reset) - tcp_send_active_reset(sk, GFP_ATOMIC, + tcp_send_active_reset(sk, SK_RST_REASON_TCP_ABORT_ON_MEMORY); tcp_done(sk); __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONMEMORY); @@ -809,7 +809,7 @@ static void tcp_keepalive_timer(struct timer_list *t) goto out; } } - tcp_send_active_reset(sk, GFP_ATOMIC, SK_RST_REASON_TCP_STATE); + tcp_send_active_reset(sk, SK_RST_REASON_TCP_STATE); goto death; } @@ -836,7 +836,7 @@ static void tcp_keepalive_timer(struct timer_list *t) icsk->icsk_probes_out > 0) || (user_timeout == 0 && icsk->icsk_probes_out >= keepalive_probes(tp))) { - tcp_send_active_reset(sk, GFP_ATOMIC, + tcp_send_active_reset(sk, SK_RST_REASON_TCP_KEEPALIVE_TIMEOUT); tcp_write_err(sk); goto out; diff --git a/net/ipv4/tcp_ulp.c b/net/ipv4/tcp_ulp.c index 2aa442128630e..b58045df101e5 100644 --- a/net/ipv4/tcp_ulp.c +++ b/net/ipv4/tcp_ulp.c @@ -136,6 +136,10 @@ static int __tcp_set_ulp(struct sock *sk, const struct tcp_ulp_ops *ulp_ops) if (icsk->icsk_ulp_ops) goto out_err; + err = -EINVAL; + if (sk_has_decrypt_user(sk)) + goto out_err; + if (sk->sk_socket) clear_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags); diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 70f6cbd4ef73b..59afce196079a 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -616,14 +616,23 @@ void udp_lib_hash4(struct sock *sk, u16 hash) struct net *net = sock_net(sk); struct udp_table *udptable; - /* Connected udp socket can re-connect to another remote address, which - * will be handled by rehash. Thus no need to redo hash4 here. + udptable = net->ipv4.udp_table; + hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash); + + /* A connected socket can re-connect to another address. rehash() + * relocates it, but only runs when the local address changes, so a + * socket bound to a specific address would stay filed under the + * previous peer's hash. Move it here. */ - if (udp_hashed4(sk)) + if (udp_hashed4(sk)) { + if (udp_sk(sk)->udp_lrpa_hash != hash) { + spin_lock_bh(&hslot->lock); + udp_rehash4(udptable, sk, hash); + spin_unlock_bh(&hslot->lock); + } return; + } - udptable = net->ipv4.udp_table; - hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash); hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); hslot4 = udp_hashslot4(udptable, hash); udp_sk(sk)->udp_lrpa_hash = hash; @@ -2164,10 +2173,10 @@ int __udp_disconnect(struct sock *sk, int flags) */ sk->sk_state = TCP_CLOSE; - inet->inet_daddr = 0; + WRITE_ONCE(inet->inet_daddr, 0); inet->inet_dport = 0; sock_rps_reset_rxhash(sk); - sk->sk_bound_dev_if = 0; + WRITE_ONCE(sk->sk_bound_dev_if, 0); if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) { inet_reset_saddr(sk); if (sk->sk_prot->rehash && @@ -2184,9 +2193,31 @@ int __udp_disconnect(struct sock *sk, int flags) } EXPORT_SYMBOL(__udp_disconnect); +/* __udp_disconnect() takes a socket out of the 4-tuple hash table only via + * ->rehash() or ->unhash(), and neither runs for a socket bound to a + * specific address and port. Remove it here, before its peer is cleared. + */ +static void udp_unhash4_on_disconnect(struct sock *sk) +{ + struct net *net = sock_net(sk); + struct udp_table *udptable; + struct udp_hslot *hslot; + + if (!udp_hashed4(sk)) + return; + + udptable = net->ipv4.udp_table; + hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash); + + spin_lock_bh(&hslot->lock); + udp_unhash4(udptable, sk); + spin_unlock_bh(&hslot->lock); +} + int udp_disconnect(struct sock *sk, int flags) { lock_sock(sk); + udp_unhash4_on_disconnect(sk); __udp_disconnect(sk, flags); release_sock(sk); return 0; @@ -3098,6 +3129,7 @@ int udp_abort(struct sock *sk, int err) sk->sk_err = err; sk_error_report(sk); + udp_unhash4_on_disconnect(sk); __udp_disconnect(sk, 0); out: diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c index 76f7a2de9108e..c96f7e0d0a481 100644 --- a/net/ipv6/ah6.c +++ b/net/ipv6/ah6.c @@ -232,26 +232,28 @@ static void ipv6_rearrange_destopt(struct ipv6hdr *iph, struct ipv6_opt_hdr *des * Rearrange the destination address in @iph and the addresses in @rthdr * so that they appear in the order they will at the final destination. * See Appendix A2 of RFC 2402 for details. + * + * Return: 0 on success, -EINVAL if segments_left exceeds the number of + * addresses described by hdrlen. */ -static void ipv6_rearrange_rthdr(struct ipv6hdr *iph, struct ipv6_rt_hdr *rthdr) +static int ipv6_rearrange_rthdr(struct ipv6hdr *iph, struct ipv6_rt_hdr *rthdr) { - int segments, segments_left; + unsigned int segments, segments_left; struct in6_addr *addrs; struct in6_addr final_addr; segments_left = rthdr->segments_left; if (segments_left == 0) - return; - rthdr->segments_left = 0; + return 0; - /* The value of rthdr->hdrlen has been verified either by the system - * call if it is locally generated, or by ipv6_rthdr_rcv() for incoming - * packets. So we can assume that it is even and that segments is - * greater than or equal to segments_left. - * - * For the same reason we can assume that this option is of type 0. + /* Raw locally generated packets can reach AH6 without the invariant + * required by the rt0-style address rearrangement below. */ segments = rthdr->hdrlen >> 1; + if (segments_left > segments) + return -EINVAL; + + rthdr->segments_left = 0; addrs = ((struct rt0_hdr *)rthdr)->addr; final_addr = addrs[segments - 1]; @@ -261,6 +263,8 @@ static void ipv6_rearrange_rthdr(struct ipv6hdr *iph, struct ipv6_rt_hdr *rthdr) addrs[0] = iph->daddr; iph->daddr = final_addr; + + return 0; } static int ipv6_clear_mutable_options(struct ipv6hdr *iph, int len, int dir) @@ -273,6 +277,7 @@ static int ipv6_clear_mutable_options(struct ipv6hdr *iph, int len, int dir) } exthdr = { .iph = iph }; char *end = exthdr.raw + len; int nexthdr = iph->nexthdr; + int err; exthdr.iph++; @@ -292,7 +297,9 @@ static int ipv6_clear_mutable_options(struct ipv6hdr *iph, int len, int dir) break; case NEXTHDR_ROUTING: - ipv6_rearrange_rthdr(iph, exthdr.rth); + err = ipv6_rearrange_rthdr(iph, exthdr.rth); + if (err) + return err; break; default: diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index 296b57926abb9..a3a3857eed98a 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -470,6 +470,12 @@ int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info esp->inplace = false; + /* Take real page refs and clear SKBFL_MANAGED_FRAG_REFS before + * we mutate the frag array, so the per-frag unref stays balanced + * for zerocopy managed frags (see __ip_append_data()). + */ + skb_zcopy_downgrade_managed(skb); + allocsize = ALIGN(tailen, L1_CACHE_BYTES); spin_lock_bh(&x->lock); diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c index 9c677eb1d1a61..09a4552f7f08a 100644 --- a/net/ipv6/exthdrs.c +++ b/net/ipv6/exthdrs.c @@ -368,23 +368,16 @@ static void seg6_update_csum(struct sk_buff *skb) (__be32 *)addr); } -static int ipv6_srh_rcv(struct sk_buff *skb) +static int ipv6_srh_rcv(struct sk_buff *skb, struct inet6_dev *idev) { struct inet6_skb_parm *opt = IP6CB(skb); struct net *net = dev_net(skb->dev); struct ipv6_sr_hdr *hdr; - struct inet6_dev *idev; struct in6_addr *addr; int accept_seg6; hdr = (struct ipv6_sr_hdr *)skb_transport_header(skb); - idev = __in6_dev_get(skb->dev); - if (!idev) { - kfree_skb(skb); - return -1; - } - accept_seg6 = min(READ_ONCE(net->ipv6.devconf_all->seg6_enabled), READ_ONCE(idev->cnf.seg6_enabled)); @@ -452,7 +445,7 @@ static int ipv6_srh_rcv(struct sk_buff *skb) hdr->segments_left--; addr = hdr->segments + hdr->segments_left; - skb_push(skb, sizeof(struct ipv6hdr)); + skb_push(skb, -skb_network_offset(skb)); if (skb->ip_summed == CHECKSUM_COMPLETE) seg6_update_csum(skb); @@ -476,7 +469,7 @@ static int ipv6_srh_rcv(struct sk_buff *skb) } ipv6_hdr(skb)->hop_limit--; - skb_pull(skb, sizeof(struct ipv6hdr)); + skb_pull(skb, skb_transport_offset(skb)); goto looped_back; } @@ -485,12 +478,11 @@ static int ipv6_srh_rcv(struct sk_buff *skb) return -1; } -static int ipv6_rpl_srh_rcv(struct sk_buff *skb) +static int ipv6_rpl_srh_rcv(struct sk_buff *skb, struct inet6_dev *idev) { struct ipv6_rpl_sr_hdr *hdr, *ohdr, *chdr; struct inet6_skb_parm *opt = IP6CB(skb); struct net *net = dev_net(skb->dev); - struct inet6_dev *idev; struct ipv6hdr *oldhdr; unsigned int chdr_len; unsigned char *buf; @@ -499,8 +491,6 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb) u64 n = 0; u32 r; - idev = __in6_dev_get(skb->dev); - accept_rpl_seg = min(READ_ONCE(net->ipv6.devconf_all->rpl_seg_enabled), READ_ONCE(idev->cnf.rpl_seg_enabled)); if (!accept_rpl_seg) { @@ -689,10 +679,14 @@ static int ipv6_rthdr_rcv(struct sk_buff *skb) switch (hdr->type) { case IPV6_SRCRT_TYPE_4: /* segment routing */ - return ipv6_srh_rcv(skb); + if (!idev) + goto disabled; + return ipv6_srh_rcv(skb, idev); case IPV6_SRCRT_TYPE_3: /* rpl segment routing */ - return ipv6_rpl_srh_rcv(skb); + if (!idev) + goto disabled; + return ipv6_rpl_srh_rcv(skb, idev); default: break; } @@ -837,6 +831,10 @@ static int ipv6_rthdr_rcv(struct sk_buff *skb) icmpv6_param_prob(skb, ICMPV6_HDR_FIELD, (&hdr->type) - skb_network_header(skb)); return -1; + +disabled: + kfree_skb_reason(skb, SKB_DROP_REASON_IPV6DISABLED); + return -1; } static const struct inet6_protocol rthdr_protocol = { diff --git a/net/ipv6/exthdrs_core.c b/net/ipv6/exthdrs_core.c index 9d06d487e8b10..4a9748338cf40 100644 --- a/net/ipv6/exthdrs_core.c +++ b/net/ipv6/exthdrs_core.c @@ -278,6 +278,9 @@ int ipv6_find_hdr(const struct sk_buff *skb, unsigned int *offset, hdrlen = ipv6_optlen(hp); if (!found) { + if (skb->len - start < hdrlen) + return -EBADMSG; + nexthdr = hp->nexthdr; start += hdrlen; } diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index e9fc692d4f3b2..fc3da984a9ab0 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -85,7 +85,7 @@ static void fib6_walker_link(struct net *net, struct fib6_walker *w) static void fib6_walker_unlink(struct net *net, struct fib6_walker *w) { write_lock_bh(&net->ipv6.fib6_walker_lock); - list_del(&w->lh); + list_del_init(&w->lh); write_unlock_bh(&net->ipv6.fib6_walker_lock); } @@ -1043,8 +1043,8 @@ static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn, struct fib6_table *table = rt->fib6_table; /* Flush all cached dst in exception table */ - rt6_flush_exceptions(rt); fib6_drop_pcpu_from(rt); + rt6_flush_exceptions(rt); if (rt->nh) { spin_lock(&rt->nh->lock); @@ -2758,7 +2758,7 @@ static void *ipv6_route_seq_start(struct seq_file *seq, loff_t *pos) static bool ipv6_route_iter_active(struct ipv6_route_iter *iter) { struct fib6_walker *w = &iter->w; - return w->node && !(w->state == FWS_U && w->node == w->root); + return w->node && !list_empty(&w->lh); } static void ipv6_route_native_seq_stop(struct seq_file *seq, void *v) diff --git a/net/ipv6/ip6_flowlabel.c b/net/ipv6/ip6_flowlabel.c index 1ab5ad0dcf24f..006585dc8b5c6 100644 --- a/net/ipv6/ip6_flowlabel.c +++ b/net/ipv6/ip6_flowlabel.c @@ -461,6 +461,21 @@ fl_create(struct net *net, struct sock *sk, struct in6_flowlabel_req *freq, return NULL; } +static bool fl_sock_at_lease_limit(const struct sock *sk) +{ + const struct ipv6_fl_socklist *sfl; + int count = 0; + + rcu_read_lock(); + for_each_sk_fl_rcu(sk, sfl) { + if (++count >= FL_MAX_PER_SOCK) + break; + } + rcu_read_unlock(); + + return count >= FL_MAX_PER_SOCK; +} + static int mem_check(struct sock *sk) { const int unpriv_total_limit = FL_MAX_SIZE - (FL_MAX_SIZE / 4); @@ -679,6 +694,10 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq, err = -ENOMEM; if (!sfl1) goto release; + err = -ENOBUFS; + if (fl_sock_at_lease_limit(sk) && + !capable(CAP_NET_ADMIN)) + goto release; if (fl->linger > fl1->linger) fl1->linger = fl->linger; if ((long)(fl->expires - fl1->expires) > 0) diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index b843116e9b703..e6913a7f68811 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -878,6 +878,7 @@ static int ip6gre_xmit_other(struct sk_buff *skb, struct net_device *dev) static netdev_tx_t ip6gre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev) { + struct ip_tunnel_info *tun_info = NULL; struct ip6_tnl *t = netdev_priv(dev); __be16 payload_protocol; int ret; @@ -888,6 +889,9 @@ static netdev_tx_t ip6gre_tunnel_xmit(struct sk_buff *skb, if (!ip6_tnl_xmit_ctl(t, &t->parms.laddr, &t->parms.raddr)) goto tx_err; + if (t->parms.collect_md) + tun_info = skb_tunnel_info_txcheck(skb); + payload_protocol = skb_protocol(skb, true); switch (payload_protocol) { case htons(ETH_P_IP): @@ -907,7 +911,7 @@ static netdev_tx_t ip6gre_tunnel_xmit(struct sk_buff *skb, return NETDEV_TX_OK; tx_err: - if (!t->parms.collect_md || !IS_ERR(skb_tunnel_info_txcheck(skb))) + if (!IS_ERR(tun_info)) DEV_STATS_INC(dev, tx_errors); DEV_STATS_INC(dev, tx_dropped); kfree_skb(skb); @@ -1137,13 +1141,11 @@ static void ip6gre_tnl_link_config_route(struct ip6_tnl *t, int set_mtu, return; if (rt->dst.dev) { - unsigned short dst_len = rt->dst.dev->hard_header_len + - t_hlen; + unsigned int headroom; - if (t->dev->header_ops) - dev->hard_header_len = dst_len; - else - dev->needed_headroom = dst_len; + headroom = rt->dst.dev->hard_header_len + t_hlen; + headroom = ip_tunnel_limit_headroom(headroom); + dev->needed_headroom = headroom; if (set_mtu) { int mtu = rt->dst.dev->mtu - t_hlen; @@ -1171,8 +1173,8 @@ static int ip6gre_calc_hlen(struct ip6_tnl *tunnel) t_hlen = tunnel->hlen + sizeof(struct ipv6hdr); - if (tunnel->dev->header_ops) - tunnel->dev->hard_header_len = LL_MAX_HEADER + t_hlen; + if (tunnel->dev->header_ops && tunnel->dev->type == ARPHRD_IP6GRE) + tunnel->dev->hard_header_len = t_hlen; else tunnel->dev->needed_headroom = LL_MAX_HEADER + t_hlen; @@ -2278,7 +2280,7 @@ static int ip6erspan_changelink(struct net_device *dev, struct nlattr *tb[], return PTR_ERR(t); ip6erspan_set_version(data, &p); - ip6gre_tunnel_unlink_md(ign, t); + ip6erspan_tunnel_unlink_md(ign, t); ip6gre_tunnel_unlink(ign, t); ip6erspan_tnl_change(t, &p, !tb[IFLA_MTU]); ip6erspan_tunnel_link_md(ign, t); diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c index 8972863c93ee5..d332ec60f9150 100644 --- a/net/ipv6/ip6_input.c +++ b/net/ipv6/ip6_input.c @@ -622,6 +622,7 @@ int ip6_mc_input(struct sk_buff *skb) if (deliver) { skb2 = skb_clone(skb, GFP_ATOMIC); } else { + skb_orphan(skb); skb2 = skb; skb = NULL; } diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 2c44e5ed61716..5509650589915 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -116,6 +116,8 @@ static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff * if (res != LWTUNNEL_XMIT_CONTINUE) return res; + hdr = ipv6_hdr(skb); + daddr = &hdr->daddr; } IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUT, skb->len); @@ -1430,6 +1432,8 @@ static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork, if (frag_size && frag_size < mtu) mtu = frag_size; + if (sk_is_udp(sk)) + mtu = min(mtu, IP6_MAX_MTU); cork->base.fragsize = mtu; cork->base.gso_size = ipc6->gso_size; cork->base.tx_flags = 0; diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index ebf83f090376c..cc96bb8b706e0 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -1236,19 +1236,8 @@ int ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield, */ max_headroom += LL_RESERVED_SPACE(tdev); - if (skb_headroom(skb) < max_headroom || skb_shared(skb) || - (skb_cloned(skb) && !skb_clone_writable(skb, 0))) { - struct sk_buff *new_skb; - - new_skb = skb_realloc_headroom(skb, max_headroom); - if (!new_skb) - goto tx_err_dst_release; - - if (skb->sk) - skb_set_owner_w(new_skb, skb->sk); - consume_skb(skb); - skb = new_skb; - } + if (skb_cow_head(skb, max_headroom)) + goto tx_err_dst_release; if (t->parms.collect_md) { if (t->encap.type != TUNNEL_ENCAP_NONE) @@ -1525,8 +1514,11 @@ static void ip6_tnl_link_config(struct ip6_tnl *t) tdev = __dev_get_by_index(t->net, p->link); if (tdev) { - dev->needed_headroom = tdev->hard_header_len + - tdev->needed_headroom + t_hlen; + unsigned int headroom; + + headroom = tdev->hard_header_len + tdev->needed_headroom; + headroom += t_hlen; + dev->needed_headroom = ip_tunnel_limit_headroom(headroom); mtu = min_t(unsigned int, tdev->mtu, IP6_MAX_MTU); mtu = mtu - t_hlen; diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index 604a58838901a..9d8116b5edb17 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -1162,10 +1162,10 @@ static int ip6mr_cache_report(const struct mr_table *mrt, struct sk_buff *pkt, msg->im6_src = ipv6_hdr(pkt)->saddr; msg->im6_dst = ipv6_hdr(pkt)->daddr; - skb_dst_set(skb, dst_clone(skb_dst(pkt))); skb->ip_summed = CHECKSUM_UNNECESSARY; } + skb_dst_drop(skb); mrt6msg_netlink_event(mrt, skb); /* Deliver to user space multicast routing algorithms */ @@ -1246,6 +1246,7 @@ static int ip6mr_cache_unresolved(struct mr_table *mrt, mifi_t mifi, skb->skb_iif = dev->ifindex; } + skb_dst_drop(skb); skb_queue_tail(&c->_c.mfc_un.unres.unresolved, skb); spin_unlock_bh(&mfc_unres_lock); @@ -1426,7 +1427,7 @@ int __init ip6_mr_init(void) { int err; - mrt_cachep = KMEM_CACHE(mfc6_cache, SLAB_HWCACHE_ALIGN); + mrt_cachep = KMEM_CACHE(mfc6_cache, SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT); if (!mrt_cachep) return -ENOMEM; diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c index 4d2b9377ba2de..fad53a5cdfffb 100644 --- a/net/ipv6/mcast.c +++ b/net/ipv6/mcast.c @@ -240,7 +240,8 @@ static int __ipv6_sock_mc_join(struct sock *sk, int ifindex, return err; } - mc_lst->next = np->ipv6_mc_list; + rcu_assign_pointer(mc_lst->next, + sock_dereference(np->ipv6_mc_list, sk)); rcu_assign_pointer(np->ipv6_mc_list, mc_lst); return 0; @@ -300,7 +301,8 @@ int ipv6_sock_mc_drop(struct sock *sk, int ifindex, const struct in6_addr *addr) lnk = &mc_lst->next) { if ((ifindex == 0 || mc_lst->ifindex == ifindex) && ipv6_addr_equal(&mc_lst->addr, addr)) { - *lnk = mc_lst->next; + rcu_assign_pointer(*lnk, + sock_dereference(mc_lst->next, sk)); __ipv6_sock_mc_drop(sk, mc_lst); return 0; } @@ -333,7 +335,8 @@ void __ipv6_sock_mc_close(struct sock *sk) struct ipv6_mc_socklist *mc_lst; while ((mc_lst = sock_dereference(np->ipv6_mc_list, sk)) != NULL) { - np->ipv6_mc_list = mc_lst->next; + rcu_assign_pointer(np->ipv6_mc_list, + sock_dereference(mc_lst->next, sk)); __ipv6_sock_mc_drop(sk, mc_lst); } } @@ -355,12 +358,12 @@ int ip6_mc_source(int add, int omode, struct sock *sk, { struct ipv6_pinfo *inet6 = inet6_sk(sk); struct in6_addr *source, *group; + struct ip6_sf_socklist *newpsl, *psl; struct net *net = sock_net(sk); struct ipv6_mc_socklist *pmc; - struct ip6_sf_socklist *psl; struct inet6_dev *idev; int leavegroup = 0; - int i, j, rv; + int i, j; int err; source = &((struct sockaddr_in6 *)&pgsr->gsr_source)->sin6_addr; @@ -409,13 +412,11 @@ int ip6_mc_source(int add, int omode, struct sock *sk, if (!add) { if (!psl) goto done; /* err = -EADDRNOTAVAIL */ - rv = !0; for (i = 0; i < psl->sl_count; i++) { - rv = !ipv6_addr_equal(&psl->sl_addr[i], source); - if (rv == 0) + if (ipv6_addr_equal(&psl->sl_addr[i], source)) break; } - if (rv) /* source not found */ + if (i == psl->sl_count) /* source not found */ goto done; /* err = -EADDRNOTAVAIL */ /* special case - (INCLUDE, empty) == LEAVE_GROUP */ @@ -424,58 +425,74 @@ int ip6_mc_source(int add, int omode, struct sock *sk, goto done; } + atomic_sub(struct_size(psl, sl_addr, psl->sl_max), + &sk->sk_omem_alloc); + + if (psl->sl_count == 1) { + newpsl = NULL; + } else { + newpsl = sock_kmalloc(sk, struct_size(newpsl, sl_addr, + psl->sl_count - 1), + GFP_KERNEL); + if (!newpsl) { + atomic_add(struct_size(psl, sl_addr, psl->sl_max), + &sk->sk_omem_alloc); + err = -ENOBUFS; + goto done; + } + newpsl->sl_max = psl->sl_count - 1; + newpsl->sl_count = psl->sl_count - 1; + for (j = 0; j < i; j++) + newpsl->sl_addr[j] = psl->sl_addr[j]; + for (j = i + 1; j < psl->sl_count; j++) + newpsl->sl_addr[j - 1] = psl->sl_addr[j]; + } + /* update the interface filter */ ip6_mc_del_src(idev, group, omode, 1, source, 1); - for (j = i+1; j < psl->sl_count; j++) - psl->sl_addr[j-1] = psl->sl_addr[j]; - psl->sl_count--; + rcu_assign_pointer(pmc->sflist, newpsl); + kfree_rcu(psl, rcu); err = 0; goto done; } /* else, add a new source to the filter */ - if (psl && psl->sl_count >= sysctl_mld_max_msf) { + if (psl && psl->sl_count >= READ_ONCE(sysctl_mld_max_msf)) { err = -ENOBUFS; goto done; } - if (!psl || psl->sl_count == psl->sl_max) { - struct ip6_sf_socklist *newpsl; - int count = IP6_SFBLOCK; - - if (psl) - count += psl->sl_max; - newpsl = sock_kmalloc(sk, struct_size(newpsl, sl_addr, count), - GFP_KERNEL); - if (!newpsl) { - err = -ENOBUFS; - goto done; - } - newpsl->sl_max = count; - newpsl->sl_count = count - IP6_SFBLOCK; - if (psl) { - for (i = 0; i < psl->sl_count; i++) - newpsl->sl_addr[i] = psl->sl_addr[i]; - atomic_sub(struct_size(psl, sl_addr, psl->sl_max), - &sk->sk_omem_alloc); + if (psl) { + for (i = 0; i < psl->sl_count; i++) { + if (ipv6_addr_equal(&psl->sl_addr[i], source)) + goto done; /* err = -EADDRNOTAVAIL */ } - rcu_assign_pointer(pmc->sflist, newpsl); - kfree_rcu(psl, rcu); - psl = newpsl; } - rv = 1; /* > 0 for insert logic below if sl_count is 0 */ - for (i = 0; i < psl->sl_count; i++) { - rv = !ipv6_addr_equal(&psl->sl_addr[i], source); - if (rv == 0) /* There is an error in the address. */ - goto done; + + i = psl ? psl->sl_count + 1 : 1; + newpsl = sock_kmalloc(sk, struct_size(newpsl, sl_addr, i), + GFP_KERNEL); + if (!newpsl) { + err = -ENOBUFS; + goto done; } - for (j = psl->sl_count-1; j >= i; j--) - psl->sl_addr[j+1] = psl->sl_addr[j]; - psl->sl_addr[i] = *source; - psl->sl_count++; - err = 0; + newpsl->sl_max = i; + newpsl->sl_count = i; + if (psl) { + for (j = 0; j < psl->sl_count; j++) + newpsl->sl_addr[j] = psl->sl_addr[j]; + } + newpsl->sl_addr[i - 1] = *source; + /* update the interface list */ ip6_mc_add_src(idev, group, omode, 1, source, 1); + + if (psl) + atomic_sub(struct_size(psl, sl_addr, psl->sl_max), + &sk->sk_omem_alloc); + rcu_assign_pointer(pmc->sflist, newpsl); + kfree_rcu(psl, rcu); + err = 0; done: mutex_unlock(&idev->mc_lock); in6_dev_put(idev); @@ -784,9 +801,11 @@ static void mld_del_delrec(struct inet6_dev *idev, struct ifmcaddr6 *im) if (!pmc) return; if (pmc_prev) - rcu_assign_pointer(pmc_prev->next, pmc->next); + rcu_assign_pointer(pmc_prev->next, + mc_dereference(pmc->next, idev)); else - rcu_assign_pointer(idev->mc_tomb, pmc->next); + rcu_assign_pointer(idev->mc_tomb, + mc_dereference(pmc->next, idev)); im->idev = pmc->idev; if (im->mca_sfmode == MCAST_INCLUDE) { @@ -965,7 +984,7 @@ static int __ipv6_dev_mc_inc(struct net_device *dev, return -ENOMEM; } - rcu_assign_pointer(mc->next, idev->mc_list); + rcu_assign_pointer(mc->next, mc_dereference(idev->mc_list, idev)); rcu_assign_pointer(idev->mc_list, mc); mld_del_delrec(idev, mc); @@ -999,7 +1018,8 @@ int __ipv6_dev_mc_dec(struct inet6_dev *idev, const struct in6_addr *addr) WRITE_ONCE(ma->mca_users, new_users); if (new_users == 0) { - *map = ma->next; + rcu_assign_pointer(*map, + mc_dereference(ma->next, idev)); igmp6_group_dropped(ma); inet6_ifmcaddr_notify(idev->dev, ma, @@ -2350,14 +2370,18 @@ static int ip6_mc_del1_src(struct ifmcaddr6 *pmc, int sfmode, if (psf->sf_oldin && !(pmc->mca_flags & MAF_NOREPORT) && !mld_in_v1_mode(idev)) { - psf->sf_crcount = idev->mc_qrv; - rcu_assign_pointer(psf->sf_next, - mc_dereference(pmc->mca_tomb, idev)); - rcu_assign_pointer(pmc->mca_tomb, psf); - rv = 1; - } else { - kfree_rcu(psf, rcu); + struct ip6_sf_list *dpsf = kmalloc_obj(*dpsf); + + if (dpsf) { + *dpsf = *psf; + dpsf->sf_crcount = idev->mc_qrv; + rcu_assign_pointer(dpsf->sf_next, + mc_dereference(pmc->mca_tomb, idev)); + rcu_assign_pointer(pmc->mca_tomb, dpsf); + rv = 1; + } } + kfree_rcu(psf, rcu); } return rv; } @@ -2620,7 +2644,7 @@ static void ip6_mc_clear_src(struct ifmcaddr6 *pmc) static void igmp6_join_group(struct ifmcaddr6 *ma) { - unsigned long delay; + unsigned long delay, interval; mc_assert_locked(ma->idev); @@ -2629,13 +2653,17 @@ static void igmp6_join_group(struct ifmcaddr6 *ma) igmp6_send(&ma->mca_addr, ma->idev->dev, ICMPV6_MGM_REPORT); - delay = get_random_u32_below(unsolicited_report_interval(ma->idev)); + interval = unsolicited_report_interval(ma->idev); + delay = interval; if (cancel_delayed_work(&ma->mca_work)) { refcount_dec(&ma->mca_refcnt); delay = ma->mca_work.timer.expires - jiffies; } + if (delay >= interval) + delay = get_random_u32_below(interval); + if (!mod_delayed_work(mld_wq, &ma->mca_work, delay)) refcount_inc(&ma->mca_refcnt); WRITE_ONCE(ma->mca_flags, ma->mca_flags | @@ -3000,7 +3028,7 @@ static int igmp6_mc_seq_show(struct seq_file *seq, void *v) struct ifmcaddr6 *im = (struct ifmcaddr6 *)v; struct igmp6_mc_iter_state *state = igmp6_mc_seq_private(seq); unsigned int mca_flags = READ_ONCE(im->mca_flags); - unsigned long expires = READ_ONCE(im->mca_work.timer.expires); + long delta = READ_ONCE(im->mca_work.timer.expires) - jiffies; seq_printf(seq, "%-4d %-15s %pi6 %5d %08X %ld\n", @@ -3008,7 +3036,7 @@ static int igmp6_mc_seq_show(struct seq_file *seq, void *v) &im->mca_addr, READ_ONCE(im->mca_users), mca_flags, (mca_flags & MAF_TIMER_RUNNING) ? - jiffies_to_clock_t(expires - jiffies) : 0); + jiffies_delta_to_clock_t(delta) : 0); return 0; } diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c index f42fb96ef64b6..313c4aac377aa 100644 --- a/net/ipv6/netfilter/ip6_tables.c +++ b/net/ipv6/netfilter/ip6_tables.c @@ -647,6 +647,11 @@ check_entry_size_and_hooks(struct ip6t_entry *e, /* Clear counters and comefrom */ e->counters = ((struct xt_counters) { 0, 0 }); e->comefrom = 0; + + /* set F_PROTO, else ip6_packet_match won't do the right thing. */ + if (e->ipv6.proto) + e->ipv6.flags |= IP6T_F_PROTO; + return 0; } diff --git a/net/ipv6/netfilter/ip6t_rpfilter.c b/net/ipv6/netfilter/ip6t_rpfilter.c index 67c87a88cde4f..b5def30c3127e 100644 --- a/net/ipv6/netfilter/ip6t_rpfilter.c +++ b/net/ipv6/netfilter/ip6t_rpfilter.c @@ -61,7 +61,7 @@ static bool rpfilter_lookup_reverse6(struct net *net, const struct sk_buff *skb, fl6.flowi6_oif = dev->ifindex; rt = (void *)ip6_route_lookup(net, &fl6, skb, lookup_flags); - if (rt->dst.error) + if (rt->dst.error || !rt->rt6i_idev) goto out; if (rt->rt6i_flags & (RTF_REJECT|RTF_ANYCAST)) diff --git a/net/ipv6/netfilter/ip6t_rt.c b/net/ipv6/netfilter/ip6t_rt.c index 278b52752f364..2f6261d06a079 100644 --- a/net/ipv6/netfilter/ip6t_rt.c +++ b/net/ipv6/netfilter/ip6t_rt.c @@ -96,7 +96,8 @@ static bool rt_mt6(const struct sk_buff *skb, struct xt_action_param *par) unsigned int i = 0; for (temp = 0; - temp < (unsigned int)((hdrlen - 8) / 16); + temp < (unsigned int)((hdrlen - 8) / 16) && + i < rtinfo->addrnr; temp++) { ap = skb_header_pointer(skb, ptr @@ -112,8 +113,6 @@ static bool rt_mt6(const struct sk_buff *skb, struct xt_action_param *par) if (ipv6_addr_equal(ap, &rtinfo->addrs[i])) i++; - if (i == rtinfo->addrnr) - break; } if (i == rtinfo->addrnr) return ret; @@ -162,6 +161,12 @@ static int rt_mt6_check(const struct xt_mtchk_param *par) pr_debug("too many addresses specified\n"); return -EINVAL; } + + if ((rtinfo->flags & IP6T_RT_FST_MASK) && !rtinfo->addrnr) { + pr_info_ratelimited("address list match requested but addrnr is 0\n"); + return -EINVAL; + } + if ((rtinfo->flags & (IP6T_RT_RES | IP6T_RT_FST_MASK)) && (!(rtinfo->flags & IP6T_RT_TYP) || (rtinfo->rt_type != 0) || diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 5968ce5ad1508..8fddcc631fcb6 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -139,6 +139,7 @@ void rt6_uncached_list_add(struct rt6_info *rt) { struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list); + /* Set once and never cleared: non-NULL marks an uncached route. */ rt->dst.rt_uncached_list = ul; spin_lock_bh(&ul->lock); @@ -1729,6 +1730,11 @@ static int rt6_insert_exception(struct rt6_info *nrt, spin_lock_bh(&rt6_exception_lock); + if (f6i->fib6_destroying) { + err = -ENOENT; + goto out; + } + bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, lockdep_is_held(&rt6_exception_lock)); if (!bucket) { @@ -2721,8 +2727,8 @@ struct dst_entry *ip6_route_output_flags(struct net *net, rcu_read_lock(); dst = ip6_route_output_flags_noref(net, sk, fl6, flags); rt6 = dst_rt6_info(dst); - /* For dst cached in uncached_list, refcnt is already taken. */ - if (list_empty(&rt6->dst.rt_uncached) && !dst_hold_safe(dst)) { + /* For an uncached dst, refcnt is already taken. */ + if (!rt6->dst.rt_uncached_list && !dst_hold_safe(dst)) { dst = &net->ipv6.ip6_null_entry->dst; dst_hold(dst); } @@ -2831,7 +2837,7 @@ INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst, from = rcu_dereference(rt->from); if (from && (rt->rt6i_flags & RTF_PCPU || - unlikely(!list_empty(&rt->dst.rt_uncached)))) + unlikely(rt->dst.rt_uncached_list))) dst_ret = rt6_dst_from_check(rt, from, cookie); else dst_ret = rt6_check(rt, from, cookie); @@ -3261,7 +3267,7 @@ void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk) static unsigned int ip6_default_advmss(const struct dst_entry *dst) { - unsigned int mtu = dst6_mtu(dst); + unsigned int mtu = ip6_dst_mtu_configured(dst); struct net *net; mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr); @@ -4010,6 +4016,7 @@ static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) struct net *net = info->nl_net; struct sk_buff *skb = NULL; struct fib6_table *table; + struct fib6_node *fn; int err = -ENOENT; if (rt == net->ipv6.fib6_null_entry) @@ -4017,9 +4024,13 @@ static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) table = rt->fib6_table; spin_lock_bh(&table->tb6_lock); + fn = rcu_dereference_protected(rt->fib6_node, + lockdep_is_held(&table->tb6_lock)); + if (!fn) + goto out_unlock; + if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) { struct fib6_info *sibling, *next_sibling; - struct fib6_node *fn; /* prefer to send a single notification with all hops */ skb = nlmsg_new(rt6_nlmsg_size(rt), GFP_ATOMIC); @@ -4041,8 +4052,6 @@ static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) * and emit a replace or delete notification, respectively. */ info->skip_notify_kernel = 1; - fn = rcu_dereference_protected(rt->fib6_node, - lockdep_is_held(&table->tb6_lock)); if (rcu_access_pointer(fn->leaf) == rt) { struct fib6_info *last_sibling, *replace_rt; @@ -4840,7 +4849,7 @@ static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total) { int upper_bound = -1; - if (!rt6_is_dead(rt)) { + if (total && !rt6_is_dead(rt)) { *weight += rt->fib6_nh->fib_nh_weight; upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31, total) - 1; @@ -6029,7 +6038,7 @@ static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg) return 0; for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { - hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { + hlist_for_each_entry_rcu(rt6_ex, &bucket->chain, hlist) { if (w->skip) { w->skip--; continue; diff --git a/net/ipv6/seg6.c b/net/ipv6/seg6.c index 62a7eb7792026..8c2b156c227a7 100644 --- a/net/ipv6/seg6.c +++ b/net/ipv6/seg6.c @@ -138,8 +138,8 @@ void seg6_icmp_srh(struct sk_buff *skb, struct inet6_skb_parm *opt) static struct genl_family seg6_genl_family; static const struct nla_policy seg6_genl_policy[SEG6_ATTR_MAX + 1] = { - [SEG6_ATTR_DST] = { .type = NLA_BINARY, - .len = sizeof(struct in6_addr) }, + [SEG6_ATTR_DST] = + NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), [SEG6_ATTR_DSTLEN] = { .type = NLA_S32, }, [SEG6_ATTR_HMACKEYID] = { .type = NLA_U32, }, [SEG6_ATTR_SECRET] = { .type = NLA_BINARY, }, diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c index 2b41e4c0dddd1..d1070aec7b72b 100644 --- a/net/ipv6/seg6_local.c +++ b/net/ipv6/seg6_local.c @@ -256,6 +256,25 @@ static bool decap_and_validate(struct sk_buff *skb, int proto) if (iptunnel_pull_offloads(skb)) return false; + if (proto == IPPROTO_IPIP) { + bool l3slave = ipv6_l3mdev_skb(IP6CB(skb)->flags); + int iif = IP6CB(skb)->iif; + + memset(IPCB(skb), 0, sizeof(*IPCB(skb))); + IPCB(skb)->iif = iif; + if (l3slave) + IPCB(skb)->flags |= IPSKB_L3SLAVE; + } else if (proto == IPPROTO_IPV6) { + bool l3slave = ipv6_l3mdev_skb(IP6CB(skb)->flags); + int iif = IP6CB(skb)->iif; + + memset(IP6CB(skb), 0, sizeof(*IP6CB(skb))); + IP6CB(skb)->iif = iif; + IP6CB(skb)->nhoff = offsetof(struct ipv6hdr, nexthdr); + if (l3slave) + IP6CB(skb)->flags |= IP6SKB_L3SLAVE; + } + return true; } diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index a38b24fb83842..19b7fa8d1a2a0 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -1131,7 +1131,7 @@ static void ipip6_tunnel_bind_dev(struct net_device *dev) WRITE_ONCE(dev->mtu, mtu); hlen = tdev->hard_header_len + tdev->needed_headroom; } - dev->needed_headroom = t_hlen + hlen; + dev->needed_headroom = ip_tunnel_limit_headroom(t_hlen + hlen); } static void ipip6_tunnel_update(struct ip_tunnel *t, diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 9e9155b1b3aa7..7fa4ed2fd4f16 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -1487,7 +1487,7 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * tcp_ca_openreq_child(newsk, dst); tcp_sync_mss(newsk, dst6_mtu(dst)); - newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); + newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst)); tcp_initialize_rcv_mss(newsk); @@ -1604,7 +1604,8 @@ int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) by tcp. Feel free to propose better solution. --ANK (980728) */ - if (np->rxopt.all && sk->sk_state != TCP_LISTEN) + if (np->rxopt.all && + !((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))) opt_skb = skb_clone_and_charge_r(skb, sk); if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */ diff --git a/net/ipv6/tcpv6_offload.c b/net/ipv6/tcpv6_offload.c index f2a659cd6183c..eec3778855eb8 100644 --- a/net/ipv6/tcpv6_offload.c +++ b/net/ipv6/tcpv6_offload.c @@ -26,7 +26,18 @@ static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb, p = tcp_gro_lookup(head, th); if (p) { - NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist; + /* flist GRO applies to consecutive non-GSO skbs */ + if (!skb_is_gso(skb) || !NAPI_GRO_CB(p)->is_flist) { + NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist; + return; + } + + /* Fall back to the regular GRO path */ + if (NAPI_GRO_CB(p)->count == 1) + NAPI_GRO_CB(p)->is_flist = 0; + + NAPI_GRO_CB(skb)->is_flist = 0; + return; } @@ -36,7 +47,7 @@ static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb, sk = __inet6_lookup_established(net, &hdr->saddr, th->source, &hdr->daddr, ntohs(th->dest), iif, sdif); - NAPI_GRO_CB(skb)->is_flist = !sk; + NAPI_GRO_CB(skb)->is_flist = !sk && !skb_is_gso(skb); if (sk) sock_gen_put(sk); #endif /* IS_ENABLED(CONFIG_IPV6) */ diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c index 89d0443b53073..07edef2589844 100644 --- a/net/ipv6/xfrm6_input.c +++ b/net/ipv6/xfrm6_input.c @@ -247,7 +247,7 @@ int xfrm6_input_addr(struct sk_buff *skb, xfrm_address_t *daddr, goto drop; } - if (1 + sp->len == XFRM_MAX_DEPTH) { + if (sp->len >= XFRM_MAX_DEPTH) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); goto drop; } diff --git a/net/ipv6/xfrm6_output.c b/net/ipv6/xfrm6_output.c index 512bdaf136997..44b221a09a0c8 100644 --- a/net/ipv6/xfrm6_output.c +++ b/net/ipv6/xfrm6_output.c @@ -19,7 +19,10 @@ void xfrm6_local_rxpmtu(struct sk_buff *skb, u32 mtu) { struct flowi6 fl6; - struct sock *sk = skb->sk; + struct sock *sk = skb_to_full_sk(skb); + + if (!sk) + return; fl6.flowi6_oif = sk->sk_bound_dev_if; fl6.daddr = ipv6_hdr(skb)->daddr; @@ -31,7 +34,10 @@ void xfrm6_local_error(struct sk_buff *skb, u32 mtu) { struct flowi6 fl6; const struct ipv6hdr *hdr; - struct sock *sk = skb->sk; + struct sock *sk = skb_to_full_sk(skb); + + if (!sk) + return; hdr = skb->encapsulation ? inner_ipv6_hdr(skb) : ipv6_hdr(skb); fl6.fl6_dport = inet_sk(sk)->inet_dport; diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c index ea047bab65e7c..4e5cc9da6e068 100644 --- a/net/iucv/af_iucv.c +++ b/net/iucv/af_iucv.c @@ -2079,6 +2079,8 @@ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev, sk = NULL; read_lock(&iucv_sk_list.lock); sk_for_each(sk, &iucv_sk_list.head) { + if (iucv_sk(sk)->hs_dev != dev) + continue; if (trans_hdr->flags == AF_IUCV_FLAG_SYN) { if ((!memcmp(&iucv_sk(sk)->src_name, trans_hdr->destAppName, 8)) && diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index d469abcd989b4..71af69d442f21 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -5,6 +5,7 @@ * Copyright (c) 2016 Tom Herbert */ +#include #include #include #include @@ -391,7 +392,9 @@ static int kcm_parse_func_strparser(struct strparser *strp, struct sk_buff *skb) struct bpf_prog *prog = psock->bpf_prog; int res; + rcu_read_lock(); res = bpf_prog_run_pin_on_cpu(prog, skb); + rcu_read_unlock(); return res; } diff --git a/net/l2tp/l2tp_netlink.c b/net/l2tp/l2tp_netlink.c index 59457c0c14aab..fbdd3d055dbc7 100644 --- a/net/l2tp/l2tp_netlink.c +++ b/net/l2tp/l2tp_netlink.c @@ -248,8 +248,8 @@ static int l2tp_nl_cmd_tunnel_create(struct sk_buff *skb, struct genl_info *info kfree(tunnel); goto out; } - ret = l2tp_tunnel_notify(&l2tp_nl_family, info, tunnel, - L2TP_CMD_TUNNEL_CREATE); + l2tp_tunnel_notify(&l2tp_nl_family, info, tunnel, + L2TP_CMD_TUNNEL_CREATE); l2tp_tunnel_put(tunnel); out: @@ -305,8 +305,8 @@ static int l2tp_nl_cmd_tunnel_modify(struct sk_buff *skb, struct genl_info *info goto out; } - ret = l2tp_tunnel_notify(&l2tp_nl_family, info, - tunnel, L2TP_CMD_TUNNEL_MODIFY); + l2tp_tunnel_notify(&l2tp_nl_family, info, + tunnel, L2TP_CMD_TUNNEL_MODIFY); l2tp_tunnel_put(tunnel); @@ -645,8 +645,8 @@ static int l2tp_nl_cmd_session_create(struct sk_buff *skb, struct genl_info *inf session = l2tp_session_get(net, tunnel->sock, tunnel->version, tunnel_id, session_id); if (session) { - ret = l2tp_session_notify(&l2tp_nl_family, info, session, - L2TP_CMD_SESSION_CREATE); + l2tp_session_notify(&l2tp_nl_family, info, session, + L2TP_CMD_SESSION_CREATE); l2tp_session_put(session); } } @@ -710,8 +710,8 @@ static int l2tp_nl_cmd_session_modify(struct sk_buff *skb, struct genl_info *inf if (info->attrs[L2TP_ATTR_RECV_TIMEOUT]) session->reorder_timeout = nla_get_msecs(info->attrs[L2TP_ATTR_RECV_TIMEOUT]); - ret = l2tp_session_notify(&l2tp_nl_family, info, - session, L2TP_CMD_SESSION_MODIFY); + l2tp_session_notify(&l2tp_nl_family, info, + session, L2TP_CMD_SESSION_MODIFY); l2tp_session_put(session); diff --git a/net/llc/llc_c_ac.c b/net/llc/llc_c_ac.c index 724ecd741d4cf..1aa7fe28acddb 100644 --- a/net/llc/llc_c_ac.c +++ b/net/llc/llc_c_ac.c @@ -437,7 +437,7 @@ int llc_conn_ac_resend_i_xxx_x_set_0_or_send_rr(struct sock *sk, if (likely(!rc)) llc_conn_send_pdu(sk, nskb); else - kfree_skb(skb); + kfree_skb(nskb); } if (rc) { nr = LLC_I_GET_NR(pdu); diff --git a/net/llc/llc_s_ac.c b/net/llc/llc_s_ac.c index 98deee5603735..831998211b52e 100644 --- a/net/llc/llc_s_ac.c +++ b/net/llc/llc_s_ac.c @@ -121,6 +121,8 @@ int llc_sap_action_send_xid_r(struct llc_sap *sap, struct sk_buff *skb) rc = llc_mac_hdr_init(nskb, mac_sa, mac_da); if (likely(!rc)) rc = dev_queue_xmit(nskb); + else + kfree_skb(nskb); out: return rc; } @@ -170,6 +172,8 @@ int llc_sap_action_send_test_r(struct llc_sap *sap, struct sk_buff *skb) rc = llc_mac_hdr_init(nskb, mac_sa, mac_da); if (likely(!rc)) rc = dev_queue_xmit(nskb); + else + kfree_skb(nskb); out: return rc; } diff --git a/net/llc/llc_sap.c b/net/llc/llc_sap.c index 1bd446a21092f..3904a1b4ba844 100644 --- a/net/llc/llc_sap.c +++ b/net/llc/llc_sap.c @@ -19,12 +19,12 @@ #include #include -static int llc_mac_header_len(unsigned short devtype) +static int llc_mac_header_len(struct net_device *dev) { - switch (devtype) { + switch (dev->type) { case ARPHRD_ETHER: case ARPHRD_LOOPBACK: - return sizeof(struct ethhdr); + return LL_RESERVED_SPACE(dev); } return 0; } @@ -45,7 +45,7 @@ struct sk_buff *llc_alloc_frame(struct sock *sk, struct net_device *dev, int hlen = type == LLC_PDU_TYPE_U ? 3 : 4; struct sk_buff *skb; - hlen += llc_mac_header_len(dev->type); + hlen += llc_mac_header_len(dev); skb = alloc_skb(hlen + data_size, GFP_ATOMIC); if (skb) { diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index 43f142624d33e..33334b978462d 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -115,6 +115,10 @@ static int ieee80211_set_mon_options(struct ieee80211_sub_if_data *sdata, return -EBUSY; } + /* TXQs are reserved in ieee80211_if_add() and cannot be added later */ + if ((params->flags & MONITOR_FLAG_ACTIVE) && !sdata->vif.txq) + return -EOPNOTSUPP; + /* validate whether MU-MIMO can be configured */ if (!ieee80211_hw_check(&local->hw, WANT_MONITOR_VIF) && !ieee80211_hw_check(&local->hw, NO_VIRTUAL_MONITOR) && @@ -1929,6 +1933,9 @@ static int ieee80211_start_ap(struct wiphy *wiphy, struct net_device *dev, return 0; error: + link_conf->enable_beacon = false; + link_conf->beacon_int = prev_beacon_int; + sdata->vif.cfg.ssid_len = 0; ieee80211_link_release_channel(link); return err; @@ -3320,7 +3327,11 @@ static int ieee80211_join_mesh(struct wiphy *wiphy, struct net_device *dev, if (err) return err; - return ieee80211_start_mesh(sdata); + err = ieee80211_start_mesh(sdata); + if (err) + ieee80211_link_release_channel(&sdata->deflink); + + return err; } static int ieee80211_leave_mesh(struct wiphy *wiphy, struct net_device *dev) @@ -3475,7 +3486,7 @@ static int ieee80211_set_txq_params(struct wiphy *wiphy, static int ieee80211_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wowlan) { - return __ieee80211_suspend(wiphy_priv(wiphy), wowlan); + return __ieee80211_suspend(wiphy_priv(wiphy), wowlan, false); } static int ieee80211_resume(struct wiphy *wiphy) @@ -4113,6 +4124,9 @@ static int ieee80211_set_bitrate_mask(struct wiphy *wiphy, if (!ieee80211_sdata_running(sdata)) return -ENETDOWN; + if (!(sdata->flags & IEEE80211_SDATA_IN_DRIVER)) + return -ENETDOWN; + /* * If active validate the setting and reject it if it doesn't leave * at least one basic rate usable, since we really have to be able diff --git a/net/mac80211/debugfs.c b/net/mac80211/debugfs.c index a4d5461f6480f..f36cc25903050 100644 --- a/net/mac80211/debugfs.c +++ b/net/mac80211/debugfs.c @@ -388,7 +388,7 @@ static ssize_t reset_write(struct file *file, const char __user *user_buf, rtnl_lock(); wiphy_lock(local->hw.wiphy); - __ieee80211_suspend(&local->hw, NULL); + __ieee80211_suspend(&local->hw, NULL, true); ret = __ieee80211_resume(&local->hw); wiphy_unlock(local->hw.wiphy); diff --git a/net/mac80211/debugfs_netdev.c b/net/mac80211/debugfs_netdev.c index f3c6a41e49119..6aba224936708 100644 --- a/net/mac80211/debugfs_netdev.c +++ b/net/mac80211/debugfs_netdev.c @@ -657,6 +657,9 @@ static ssize_t ieee80211_if_fmt_tsf( struct ieee80211_local *local = sdata->local; u64 tsf; + if (!ieee80211_sdata_running((struct ieee80211_sub_if_data *)sdata)) + return -ENETDOWN; + tsf = drv_get_tsf(local, (struct ieee80211_sub_if_data *)sdata); return scnprintf(buf, buflen, "0x%016llx\n", (unsigned long long) tsf); @@ -670,6 +673,9 @@ static ssize_t ieee80211_if_parse_tsf( int ret; int tsf_is_delta = 0; + if (!ieee80211_sdata_running(sdata)) + return -ENETDOWN; + if (strncmp(buf, "reset", 5) == 0) { if (local->ops->reset_tsf) { drv_reset_tsf(local, sdata); @@ -729,6 +735,9 @@ static ssize_t ieee80211_if_parse_active_links(struct ieee80211_sub_if_data *sda if (kstrtou16(buf, 0, &active_links) || !active_links) return -EINVAL; + if (!ieee80211_sdata_running(sdata)) + return -ENETDOWN; + return ieee80211_set_active_links(&sdata->vif, active_links) ?: buflen; } IEEE80211_IF_FILE_RW(active_links); diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index 34a9ea8b6f857..0564370419fba 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -2433,7 +2433,7 @@ int ieee80211_reconfig(struct ieee80211_local *local); void ieee80211_stop_device(struct ieee80211_local *local, bool suspend); int __ieee80211_suspend(struct ieee80211_hw *hw, - struct cfg80211_wowlan *wowlan); + struct cfg80211_wowlan *wowlan, bool reset); static inline int __ieee80211_resume(struct ieee80211_hw *hw) { diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index 43460a705a6bd..889c32fd8de19 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -616,6 +616,8 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_do RCU_INIT_POINTER(sdata->vif.bss_conf.chanctx_conf, NULL); /* see comment in the default case below */ ieee80211_free_keys(sdata, true); + /* increased by AP value on ifup, so reset on ifdown */ + sdata->crypto_tx_tailroom_needed_cnt = 0; /* no need to tell driver */ break; case NL80211_IFTYPE_MONITOR: @@ -924,9 +926,33 @@ static void ieee80211_teardown_sdata(struct ieee80211_sub_if_data *sdata) } } +/* + * The netdev can be unregistered without mac80211 doing it, e.g. by the netdev + * core when cfg80211 couldn't move it out of a network namespace that's being + * destroyed. Drop it from the interface list either way. + */ +static void ieee80211_unlist_sdata(struct ieee80211_sub_if_data *sdata) +{ + struct ieee80211_local *local = sdata->local; + struct ieee80211_sub_if_data *iter; + + ASSERT_RTNL(); + + list_for_each_entry(iter, &local->interfaces, list) { + if (iter != sdata) + continue; + guard(mutex)(&local->iflist_mtx); + list_del_rcu(&sdata->list); + return; + } +} + static void ieee80211_uninit(struct net_device *dev) { - ieee80211_teardown_sdata(IEEE80211_DEV_TO_SUB_IF(dev)); + struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); + + ieee80211_unlist_sdata(sdata); + ieee80211_teardown_sdata(sdata); } static int ieee80211_netdev_setup_tc(struct net_device *dev, @@ -935,6 +961,9 @@ static int ieee80211_netdev_setup_tc(struct net_device *dev, struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); struct ieee80211_local *local = sdata->local; + if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN) + return -EOPNOTSUPP; + return drv_net_setup_tc(local, sdata, dev, type, type_data); } @@ -964,7 +993,7 @@ static u16 ieee80211_monitor_select_queue(struct net_device *dev, /* reset flags and info before parsing radiotap header */ memset(info, 0, sizeof(*info)); - if (!ieee80211_parse_tx_radiotap(skb, dev)) + if (!ieee80211_parse_tx_radiotap(skb, dev, NULL)) return 0; /* doesn't matter, frame will be dropped */ len_rthdr = ieee80211_get_radiotap_len(skb->data); @@ -1603,8 +1632,12 @@ int ieee80211_do_open(struct wireless_dev *wdev, bool coming_up) err_del_interface: drv_remove_interface(local, sdata); err_stop: - if (!local->open_count) + if (!local->open_count) { + ieee80211_led_radio(local, false); + ieee80211_mod_tpt_led_trig(local, 0, + IEEE80211_TPT_LEDTRIG_FL_RADIO); drv_stop(local, false); + } if (sdata->vif.type == NL80211_IFTYPE_NAN_DATA) RCU_INIT_POINTER(sdata->u.nan_data.nmi, NULL); if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN) diff --git a/net/mac80211/link.c b/net/mac80211/link.c index dc68144dc363a..931950a105088 100644 --- a/net/mac80211/link.c +++ b/net/mac80211/link.c @@ -360,7 +360,8 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, link = links[link_id]; ieee80211_link_init(sdata, link_id, &link->data, &link->conf); ieee80211_link_setup(&link->data); - ieee80211_set_wmm_default(&link->data, true, non_sta); + if (sdata->vif.type != NL80211_IFTYPE_AP_VLAN) + ieee80211_set_wmm_default(&link->data, true, non_sta); } if (new_links == 0) diff --git a/net/mac80211/main.c b/net/mac80211/main.c index eb1eaaf34612e..8edfef2b57f80 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -1454,6 +1454,10 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) sizeof(struct ieee80211_he_mcs_nss_supp) + IEEE80211_HE_PPE_THRES_MAX_LEN; + if (local->hw.wiphy->bands[NL80211_BAND_6GHZ]) + local->scan_ies_len += + 3 + sizeof(struct ieee80211_he_6ghz_capa); + if (supp_eht) local->scan_ies_len += 3 + sizeof(struct ieee80211_eht_cap_elem) + diff --git a/net/mac80211/mesh.c b/net/mac80211/mesh.c index d4507e4e6ec16..8f88141253758 100644 --- a/net/mac80211/mesh.c +++ b/net/mac80211/mesh.c @@ -1196,6 +1196,21 @@ int ieee80211_start_mesh(struct ieee80211_sub_if_data *sdata) return 0; } +static void ieee80211_mesh_reset_csa(struct ieee80211_sub_if_data *sdata) +{ + struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh; + struct mesh_csa_settings *csa; + + /* Reset the TTL value and Initiator flag */ + ifmsh->csa_role = IEEE80211_MESH_CSA_ROLE_NONE; + ifmsh->chsw_ttl = 0; + + /* Remove the CSA and MCSP elements from the beacon */ + csa = sdata_dereference(ifmsh->csa, sdata); + RCU_INIT_POINTER(ifmsh->csa, NULL); + kfree_rcu(csa, rcu_head); +} + void ieee80211_stop_mesh(struct ieee80211_sub_if_data *sdata) { struct ieee80211_local *local = sdata->local; @@ -1204,6 +1219,11 @@ void ieee80211_stop_mesh(struct ieee80211_sub_if_data *sdata) netif_carrier_off(sdata->dev); + /* abort any running channel switch */ + sdata->vif.bss_conf.csa_active = false; + ieee80211_mesh_reset_csa(sdata); + ieee80211_vif_unblock_queues_csa(sdata); + /* flush STAs and mpaths on this iface */ sta_info_flush(sdata, -1); ieee80211_free_keys(sdata, true); @@ -1510,19 +1530,10 @@ static void ieee80211_mesh_rx_bcn_presp(struct ieee80211_sub_if_data *sdata, int ieee80211_mesh_finish_csa(struct ieee80211_sub_if_data *sdata, u64 *changed) { - struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh; - struct mesh_csa_settings *tmp_csa_settings; - int ret = 0; + int ret; - /* Reset the TTL value and Initiator flag */ - ifmsh->csa_role = IEEE80211_MESH_CSA_ROLE_NONE; - ifmsh->chsw_ttl = 0; + ieee80211_mesh_reset_csa(sdata); - /* Remove the CSA and MCSP elements from the beacon */ - tmp_csa_settings = sdata_dereference(ifmsh->csa, sdata); - RCU_INIT_POINTER(ifmsh->csa, NULL); - if (tmp_csa_settings) - kfree_rcu(tmp_csa_settings, rcu_head); ret = ieee80211_mesh_rebuild_beacon(sdata); if (ret) return -EINVAL; @@ -1555,7 +1566,6 @@ int ieee80211_mesh_csa_beacon(struct ieee80211_sub_if_data *sdata, ret = ieee80211_mesh_rebuild_beacon(sdata); if (ret) { - tmp_csa_settings = rcu_dereference(ifmsh->csa); RCU_INIT_POINTER(ifmsh->csa, NULL); kfree_rcu(tmp_csa_settings, rcu_head); return ret; diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index fa773f3b0541a..1b9d272e7f8e1 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -7977,7 +7977,7 @@ ieee80211_mgd_check_cross_link_csa(struct ieee80211_sub_if_data *sdata, prof = (void *)sta_profiles[link_id]; prof_elems = ieee802_11_parse_elems(prof->variable + (prof->sta_info_len - 1), - len - + len - sizeof(*prof) - (prof->sta_info_len - 1), IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_BEACON, diff --git a/net/mac80211/offchannel.c b/net/mac80211/offchannel.c index 2bceb73717c66..6f8de5d4a20d3 100644 --- a/net/mac80211/offchannel.c +++ b/net/mac80211/offchannel.c @@ -460,6 +460,13 @@ static void __ieee80211_roc_work(struct ieee80211_local *local) return; if (!roc->started) { + /* + * The work can be started by a previous ROC work, but a scan + * can get between things; scan finish will retrigger us. + */ + if (local->scanning) + return; + WARN_ON(!local->emulate_chanctx); _ieee80211_start_next_roc(local); } else { diff --git a/net/mac80211/pm.c b/net/mac80211/pm.c index 5a508d99e84f7..f63676c448536 100644 --- a/net/mac80211/pm.c +++ b/net/mac80211/pm.c @@ -18,7 +18,8 @@ static void ieee80211_sched_scan_cancel(struct ieee80211_local *local) cfg80211_sched_scan_stopped_locked(local->hw.wiphy, 0); } -int __ieee80211_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan) +int __ieee80211_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan, + bool reset) { struct ieee80211_local *local = hw_to_local(hw); struct ieee80211_sub_if_data *sdata; @@ -166,9 +167,10 @@ int __ieee80211_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan) /* * We disconnected on all interfaces before suspend, all channel - * contexts should be released. + * contexts should be released, but on 'reset' debugfs that's + * not true so don't check there. */ - WARN_ON(!list_empty(&local->chanctx_list)); + WARN_ON(!reset && !list_empty(&local->chanctx_list)); /* stop hardware - this must stop RX */ ieee80211_stop_device(local, true); diff --git a/net/mac80211/s1g.c b/net/mac80211/s1g.c index abc338e22e59c..bd7baf4818367 100644 --- a/net/mac80211/s1g.c +++ b/net/mac80211/s1g.c @@ -147,7 +147,7 @@ ieee80211_s1g_tx_twt_setup_fail(struct ieee80211_sub_if_data *sdata, drv_twt_teardown_request(sdata->local, sdata, &sta->sta, flowid); - ieee80211_s1g_send_twt_teardown(sdata, mgmt->sa, sdata->vif.addr, + ieee80211_s1g_send_twt_teardown(sdata, mgmt->da, sdata->vif.addr, flowid); } diff --git a/net/mac80211/scan.c b/net/mac80211/scan.c index eeff230bd909f..8e950ef6d1ead 100644 --- a/net/mac80211/scan.c +++ b/net/mac80211/scan.c @@ -1242,7 +1242,7 @@ int ieee80211_request_ibss_scan(struct ieee80211_sub_if_data *sdata, } } - if (WARN_ON_ONCE(n_ch == 0)) + if (n_ch == 0) return -EINVAL; local->int_scan_req->n_channels = n_ch; diff --git a/net/mac80211/spectmgmt.c b/net/mac80211/spectmgmt.c index ec622750e1c9d..880f4625775dc 100644 --- a/net/mac80211/spectmgmt.c +++ b/net/mac80211/spectmgmt.c @@ -227,7 +227,7 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, { enum nl80211_band new_band = current_band; int new_freq; - u8 new_chan_no = 0, new_op_class = 0; + int new_chan_no = -1, new_op_class = -1; struct ieee80211_channel *new_chan; struct cfg80211_chan_def new_chandef = {}; const struct ieee80211_sec_chan_offs_ie *sec_chan_offs; @@ -256,7 +256,7 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, new_op_class = ext_chansw_elem->new_operating_class; if (!ieee80211_operating_class_to_band(new_op_class, &new_band)) { - new_op_class = 0; + new_op_class = -1; if (!unprot_action) sdata_info(sdata, "cannot understand ECSA IE operating class, %d, ignoring\n", @@ -268,14 +268,14 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, } } - if (!new_op_class && elems->ch_switch_ie) { + if (new_op_class < 0 && elems->ch_switch_ie) { new_chan_no = elems->ch_switch_ie->new_ch_num; csa_ie->count = elems->ch_switch_ie->count; csa_ie->mode = elems->ch_switch_ie->mode; } /* nothing here we understand */ - if (!new_chan_no) + if (new_chan_no < 0) return 1; /* Mesh Channel Switch Parameters Element */ @@ -349,7 +349,8 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, get_unaligned_le16(bwi->info.optional); } else if (!wide_bw_chansw_ie || !wbcs_elem_to_chandef(wide_bw_chansw_ie, &new_chandef)) { - if (!ieee80211_operating_class_to_chandef(new_op_class, new_chan, + if (new_op_class < 0 || + !ieee80211_operating_class_to_chandef(new_op_class, new_chan, &new_chandef)) new_chandef = csa_ie->chanreq.oper; } diff --git a/net/mac80211/tdls.c b/net/mac80211/tdls.c index ffd575a8d188f..9e721d3735e7e 100644 --- a/net/mac80211/tdls.c +++ b/net/mac80211/tdls.c @@ -1142,6 +1142,7 @@ ieee80211_tdls_mgmt_setup(struct wiphy *wiphy, struct net_device *dev, struct ieee80211_local *local = sdata->local; enum ieee80211_smps_mode smps_mode = sdata->deflink.u.mgd.driver_smps_mode; + struct sta_info *sta; int ret; /* don't support setup with forced SMPS mode that's not off */ @@ -1168,14 +1169,10 @@ ieee80211_tdls_mgmt_setup(struct wiphy *wiphy, struct net_device *dev, * Allow error packets to be sent - sometimes we don't even add a STA * before failing the setup. */ - if (status_code == 0) { - rcu_read_lock(); - if (!sta_info_get(sdata, peer)) { - rcu_read_unlock(); - ret = -ENOLINK; - goto out_unlock; - } - rcu_read_unlock(); + sta = sta_info_get(sdata, peer); + if ((status_code == 0 && !sta) || (sta && !sta->sta.tdls)) { + ret = -ENOLINK; + goto out_unlock; } ieee80211_flush_queues(local, sdata, false); @@ -1284,6 +1281,24 @@ int ieee80211_tdls_mgmt(struct wiphy *wiphy, struct net_device *dev, peer_capability, initiator, extra_ies, extra_ies_len); break; + case WLAN_TDLS_SETUP_CONFIRM: { + struct sta_info *sta; + + sta = sta_info_get(sdata, peer); + if (!sta || !sta->sta.tdls) { + ret = -ENOLINK; + break; + } + + ret = ieee80211_tdls_prep_mgmt_packet(wiphy, dev, peer, + link_id, action_code, + dialog_token, + status_code, + peer_capability, + initiator, extra_ies, + extra_ies_len, 0, NULL); + break; + } case WLAN_TDLS_DISCOVERY_REQUEST: /* * Protect the discovery so we can hear the TDLS discovery @@ -1292,7 +1307,6 @@ int ieee80211_tdls_mgmt(struct wiphy *wiphy, struct net_device *dev, */ drv_mgd_protect_tdls_discover(sdata->local, sdata, link_id); fallthrough; - case WLAN_TDLS_SETUP_CONFIRM: case WLAN_PUB_ACTION_TDLS_DISCOVER_RES: /* no special handling */ ret = ieee80211_tdls_prep_mgmt_packet(wiphy, dev, peer, @@ -1442,6 +1456,10 @@ int ieee80211_tdls_oper(struct wiphy *wiphy, struct net_device *dev, */ tdls_dbg(sdata, "TDLS oper %d peer %pM\n", oper, peer); + sta = sta_info_get(sdata, peer); + if (!sta || !sta->sta.tdls) + return -ENOLINK; + switch (oper) { case NL80211_TDLS_ENABLE_LINK: if (sdata->vif.bss_conf.csa_active) { @@ -1449,10 +1467,6 @@ int ieee80211_tdls_oper(struct wiphy *wiphy, struct net_device *dev, return -EBUSY; } - sta = sta_info_get(sdata, peer); - if (!sta || !sta->sta.tdls) - return -ENOLINK; - iee80211_tdls_recalc_chanctx(sdata, sta); iee80211_tdls_recalc_ht_protection(sdata, sta); diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 91b14112e24f0..e264121208736 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -2096,8 +2096,29 @@ static bool ieee80211_validate_radiotap_len(struct sk_buff *skb) return true; } +static bool ieee80211_rate_bw_usable(u16 rate_flags, + const struct cfg80211_chan_def *chandef) +{ + int width; + + if (!chandef) + return true; + + if (rate_flags & IEEE80211_TX_RC_160_MHZ_WIDTH) + width = 160; + else if (rate_flags & IEEE80211_TX_RC_80_MHZ_WIDTH) + width = 80; + else if (rate_flags & IEEE80211_TX_RC_40_MHZ_WIDTH) + width = 40; + else + return true; + + return width <= cfg80211_chandef_get_width(chandef); +} + bool ieee80211_parse_tx_radiotap(struct sk_buff *skb, - struct net_device *dev) + struct net_device *dev, + const struct cfg80211_chan_def *chandef) { struct ieee80211_local *local = wdev_priv(dev->ieee80211_ptr); struct ieee80211_radiotap_iterator iterator; @@ -2271,6 +2292,9 @@ bool ieee80211_parse_tx_radiotap(struct sk_buff *skb, struct ieee80211_supported_band *sband = local->hw.wiphy->bands[info->band]; + if (!ieee80211_rate_bw_usable(rate_flags, chandef)) + return false; + info->control.flags |= IEEE80211_TX_CTRL_RATE_INJECT; for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) { @@ -2470,7 +2494,7 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb, * selected chandef above to accurately set injection rates and * retransmissions. */ - if (!ieee80211_parse_tx_radiotap(skb, dev)) + if (!ieee80211_parse_tx_radiotap(skb, dev, chandef)) goto fail_rcu; /* remove the injection radiotap header */ @@ -2950,10 +2974,23 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, */ skb = skb_share_check(skb, GFP_ATOMIC); if (unlikely(!skb)) { - ret = -ENOMEM; - goto free; + /* skb_share_check() already freed the skb */ + if (info_id) + ieee80211_remove_ack_skb(local, info_id); + return ERR_PTR(-ENOMEM); } + /* set this up so failure paths can clean up ack skb */ + info = IEEE80211_SKB_CB(skb); + memset(info, 0, sizeof(*info)); + + info->flags = info_flags; + if (info_id) { + info->status_data = info_id; + info->status_data_idr = 1; + } + info->band = band; + hdr.frame_control = fc; hdr.duration_id = 0; hdr.seq_ctrl = 0; @@ -2992,10 +3029,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, head_need += local->tx_headroom; head_need = max_t(int, 0, head_need); if (ieee80211_skb_resize(sdata, skb, head_need, ENCRYPT_DATA)) { - ieee80211_free_txskb(&local->hw, skb); - skb = NULL; ret = -ENOMEM; - goto free; + goto free_txskb; } } @@ -3022,16 +3057,6 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, skb_reset_mac_header(skb); - info = IEEE80211_SKB_CB(skb); - memset(info, 0, sizeof(*info)); - - info->flags = info_flags; - if (info_id) { - info->status_data = info_id; - info->status_data_idr = 1; - } - info->band = band; - if (likely(!cookie)) { ctrl_flags |= u32_encode_bits(link_id, IEEE80211_TX_CTRL_MLO_LINK); @@ -3055,16 +3080,17 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, pre_conf_link_id, link_id); #endif ret = -EINVAL; - goto free; + goto free_txskb; } } info->control.flags = ctrl_flags; return skb; + free_txskb: + ieee80211_free_txskb(&local->hw, skb); + return ERR_PTR(ret); free: - if (info_id) - ieee80211_remove_ack_skb(local, info_id); kfree_skb(skb); return ERR_PTR(ret); } @@ -5063,10 +5089,18 @@ static void ieee80211_beacon_add_tim_pvb(struct ps_data *ps, */ static void ieee80211_s1g_beacon_add_tim_pvb(struct ps_data *ps, struct sk_buff *skb, - bool mcast_traffic) + bool mcast_traffic, + bool ucast_traffic) { int blk; + /* + * if no unicast and multicast traffic don't emit a bitmap control + * or pvb + */ + if (!mcast_traffic && !ucast_traffic) + return; + /* * Emit a bitmap control block with a page slice number of 31 and a * page index of 0 which indicates as per IEEE80211-2024 9.4.2.5.1 @@ -5075,6 +5109,10 @@ static void ieee80211_s1g_beacon_add_tim_pvb(struct ps_data *ps, */ skb_put_u8(skb, mcast_traffic | (31 << 1)); + /* If there's no unicast traffic we don't need to include a PVB. */ + if (!ucast_traffic) + return; + /* Emit an encoded block for each non-zero sub-block */ for (blk = 0; blk < IEEE80211_MAX_SUPPORTED_S1G_TIM_BLOCKS; blk++) { u8 blk_bmap = 0; @@ -5156,25 +5194,16 @@ static void __ieee80211_beacon_add_tim(struct ieee80211_sub_if_data *sdata, ps->dtim_bc_mc = mcast_traffic; - if (have_bits) { - if (s1g) - ieee80211_s1g_beacon_add_tim_pvb(ps, skb, - mcast_traffic); - else - ieee80211_beacon_add_tim_pvb(ps, skb, mcast_traffic); + if (s1g) { + ieee80211_s1g_beacon_add_tim_pvb(ps, skb, mcast_traffic, + have_bits); + } else if (have_bits) { + ieee80211_beacon_add_tim_pvb(ps, skb, mcast_traffic); } else { - /* - * If there is no buffered unicast traffic for an S1G - * interface, we can exclude the bitmap control. This is in - * contrast to other phy types as they do include the bitmap - * control and pvb even when there is no buffered traffic. - */ - if (!s1g) { - /* Bitmap control */ - skb_put_u8(skb, mcast_traffic); - /* Part Virt Bitmap */ - skb_put_u8(skb, 0); - } + /* Bitmap control */ + skb_put_u8(skb, mcast_traffic); + /* Part Virt Bitmap */ + skb_put_u8(skb, 0); } tim->datalen = skb_tail_pointer(skb) - tim->data; @@ -5249,7 +5278,8 @@ static void ieee80211_set_beacon_cntdwn(struct ieee80211_sub_if_data *sdata, if (sdata->vif.type == NL80211_IFTYPE_AP && resp) { u16 *resp_offsets = resp->cntdwn_counter_offsets; - resp->data[resp_offsets[i]] = count; + if (resp_offsets[i]) + resp->data[resp_offsets[i]] = count; } } } diff --git a/net/mac802154/ieee802154_i.h b/net/mac802154/ieee802154_i.h index 8f2bff268392b..279ff4e9b11ea 100644 --- a/net/mac802154/ieee802154_i.h +++ b/net/mac802154/ieee802154_i.h @@ -74,6 +74,10 @@ struct ieee802154_local { struct work_struct rx_beacon_work; struct list_head rx_mac_cmd_list; struct work_struct rx_mac_cmd_work; + /* Serializes rx_beacon_list and rx_mac_cmd_list against the RX + * softirq producer, the mac_wq workers and the teardown flush. + */ + spinlock_t rx_lock; /* Association */ struct ieee802154_pan_device *assoc_dev; @@ -300,6 +304,10 @@ static inline bool mac802154_is_beaconing(struct ieee802154_local *local) } void mac802154_rx_mac_cmd_worker(struct work_struct *work); +void mac802154_flush_list(struct list_head *list, + struct ieee802154_sub_if_data *sdata); +void mac802154_flush_queued_pkts(struct ieee802154_local *local, + struct ieee802154_sub_if_data *sdata); int mac802154_perform_association(struct ieee802154_sub_if_data *sdata, struct ieee802154_pan_device *coord, diff --git a/net/mac802154/iface.c b/net/mac802154/iface.c index b823720630e72..31353795fa240 100644 --- a/net/mac802154/iface.c +++ b/net/mac802154/iface.c @@ -694,6 +694,7 @@ void ieee802154_if_remove(struct ieee802154_sub_if_data *sdata) mutex_unlock(&sdata->local->iflist_mtx); synchronize_rcu(); + mac802154_flush_queued_pkts(sdata->local, sdata); unregister_netdevice(sdata->dev); } @@ -705,6 +706,11 @@ void ieee802154_remove_interfaces(struct ieee802154_local *local) list_for_each_entry_safe(sdata, tmp, &local->interfaces, list) { list_del_rcu(&sdata->list); + /* Best-effort: a frame the RX softirq queues for this sdata + * after the flush still pins the netdev, so the + * unregister_netdevice() below waits it out. + */ + mac802154_flush_queued_pkts(local, sdata); unregister_netdevice(sdata->dev); } mutex_unlock(&local->iflist_mtx); diff --git a/net/mac802154/main.c b/net/mac802154/main.c index ea1efef3572ae..386c086c79d17 100644 --- a/net/mac802154/main.c +++ b/net/mac802154/main.c @@ -91,6 +91,7 @@ ieee802154_alloc_hw(size_t priv_data_len, const struct ieee802154_ops *ops) INIT_LIST_HEAD(&local->interfaces); INIT_LIST_HEAD(&local->rx_beacon_list); INIT_LIST_HEAD(&local->rx_mac_cmd_list); + spin_lock_init(&local->rx_lock); mutex_init(&local->iflist_mtx); tasklet_setup(&local->tasklet, ieee802154_tasklet_handler); diff --git a/net/mac802154/rx.c b/net/mac802154/rx.c index cd8f2a11920d0..19b5382e85a84 100644 --- a/net/mac802154/rx.c +++ b/net/mac802154/rx.c @@ -35,16 +35,23 @@ void mac802154_rx_beacon_worker(struct work_struct *work) container_of(work, struct ieee802154_local, rx_beacon_work); struct cfg802154_mac_pkt *mac_pkt; - mac_pkt = list_first_entry_or_null(&local->rx_beacon_list, - struct cfg802154_mac_pkt, node); - if (!mac_pkt) - return; + for (;;) { + spin_lock_bh(&local->rx_lock); + mac_pkt = list_first_entry_or_null(&local->rx_beacon_list, + struct cfg802154_mac_pkt, node); + if (mac_pkt) + list_del(&mac_pkt->node); + spin_unlock_bh(&local->rx_lock); + if (!mac_pkt) + break; - mac802154_process_beacon(local, mac_pkt->skb, mac_pkt->page, mac_pkt->channel); + mac802154_process_beacon(local, mac_pkt->skb, + mac_pkt->page, mac_pkt->channel); - list_del(&mac_pkt->node); - kfree_skb(mac_pkt->skb); - kfree(mac_pkt); + netdev_put(mac_pkt->sdata->dev, &mac_pkt->dev_tracker); + kfree_skb(mac_pkt->skb); + kfree(mac_pkt); + } } static bool mac802154_should_answer_beacon_req(struct ieee802154_local *local) @@ -68,22 +75,15 @@ static bool mac802154_should_answer_beacon_req(struct ieee802154_local *local) return interval == IEEE802154_ACTIVE_SCAN_DURATION; } -void mac802154_rx_mac_cmd_worker(struct work_struct *work) +static void mac802154_rx_mac_cmd(struct ieee802154_local *local, + struct cfg802154_mac_pkt *mac_pkt) { - struct ieee802154_local *local = - container_of(work, struct ieee802154_local, rx_mac_cmd_work); - struct cfg802154_mac_pkt *mac_pkt; u8 mac_cmd; int rc; - mac_pkt = list_first_entry_or_null(&local->rx_mac_cmd_list, - struct cfg802154_mac_pkt, node); - if (!mac_pkt) - return; - rc = ieee802154_get_mac_cmd(mac_pkt->skb, &mac_cmd); if (rc) - goto out; + return; switch (mac_cmd) { case IEEE802154_CMD_BEACON_REQ: @@ -121,11 +121,81 @@ void mac802154_rx_mac_cmd_worker(struct work_struct *work) default: break; } +} -out: - list_del(&mac_pkt->node); - kfree_skb(mac_pkt->skb); - kfree(mac_pkt); +void mac802154_rx_mac_cmd_worker(struct work_struct *work) +{ + struct ieee802154_local *local = + container_of(work, struct ieee802154_local, rx_mac_cmd_work); + struct cfg802154_mac_pkt *mac_pkt; + + for (;;) { + spin_lock_bh(&local->rx_lock); + mac_pkt = list_first_entry_or_null(&local->rx_mac_cmd_list, + struct cfg802154_mac_pkt, node); + if (mac_pkt) + list_del(&mac_pkt->node); + spin_unlock_bh(&local->rx_lock); + if (!mac_pkt) + break; + + /* A stopped interface cannot transmit; skipping avoids a + * needless association response (and the !netif_running() + * warning it would trip) during teardown. The beacon worker + * needs no such check as it never transmits. + */ + if (ieee802154_sdata_running(mac_pkt->sdata)) + mac802154_rx_mac_cmd(local, mac_pkt); + + netdev_put(mac_pkt->sdata->dev, &mac_pkt->dev_tracker); + kfree_skb(mac_pkt->skb); + kfree(mac_pkt); + } +} + +/** + * mac802154_flush_list - free queued RX frames on @list + * @list: rx_beacon_list or rx_mac_cmd_list + * @sdata: only free frames received on this interface, or %NULL for all + * + * Each frame pins the net_device it was received on (via netdev_hold()), + * so release that reference as the frame is dropped. Caller must hold + * local->rx_lock. + */ +void mac802154_flush_list(struct list_head *list, + struct ieee802154_sub_if_data *sdata) +{ + struct cfg802154_mac_pkt *mac_pkt, *tmp; + + list_for_each_entry_safe(mac_pkt, tmp, list, node) { + if (sdata && mac_pkt->sdata != sdata) + continue; + list_del(&mac_pkt->node); + netdev_put(mac_pkt->sdata->dev, &mac_pkt->dev_tracker); + kfree_skb(mac_pkt->skb); + kfree(mac_pkt); + } +} + +/** + * mac802154_flush_queued_pkts - drop queued RX work referencing @sdata + * @local: the mac802154 device + * @sdata: interface being removed + * + * The workers dereference the queued frame's interface directly + * (mac_pkt->sdata) or through skb->dev in mac802154_process_beacon(). Drop + * the not-yet-started entries belonging to @sdata before it is unregistered + * so their netdev reference is released; an entry already dequeued by a + * running worker keeps its own reference until the worker completes, which + * unregister_netdevice() then waits out. + */ +void mac802154_flush_queued_pkts(struct ieee802154_local *local, + struct ieee802154_sub_if_data *sdata) +{ + spin_lock_bh(&local->rx_lock); + mac802154_flush_list(&local->rx_beacon_list, sdata); + mac802154_flush_list(&local->rx_mac_cmd_list, sdata); + spin_unlock_bh(&local->rx_lock); } static int @@ -221,7 +291,10 @@ ieee802154_subif_frame(struct ieee802154_sub_if_data *sdata, mac_pkt->sdata = sdata; mac_pkt->page = sdata->local->scan_page; mac_pkt->channel = sdata->local->scan_channel; + netdev_hold(sdata->dev, &mac_pkt->dev_tracker, GFP_ATOMIC); + spin_lock(&sdata->local->rx_lock); list_add_tail(&mac_pkt->node, &sdata->local->rx_beacon_list); + spin_unlock(&sdata->local->rx_lock); queue_work(sdata->local->mac_wq, &sdata->local->rx_beacon_work); return NET_RX_SUCCESS; @@ -233,7 +306,10 @@ ieee802154_subif_frame(struct ieee802154_sub_if_data *sdata, mac_pkt->skb = skb_get(skb); mac_pkt->sdata = sdata; + netdev_hold(sdata->dev, &mac_pkt->dev_tracker, GFP_ATOMIC); + spin_lock(&sdata->local->rx_lock); list_add_tail(&mac_pkt->node, &sdata->local->rx_mac_cmd_list); + spin_unlock(&sdata->local->rx_lock); queue_work(sdata->local->mac_wq, &sdata->local->rx_mac_cmd_work); return NET_RX_SUCCESS; diff --git a/net/mac802154/scan.c b/net/mac802154/scan.c index 005338f89b75e..ec36eb5716f59 100644 --- a/net/mac802154/scan.c +++ b/net/mac802154/scan.c @@ -104,13 +104,9 @@ static unsigned int mac802154_scan_get_channel_time(u8 duration_order, static void mac802154_flush_queued_beacons(struct ieee802154_local *local) { - struct cfg802154_mac_pkt *mac_pkt, *tmp; - - list_for_each_entry_safe(mac_pkt, tmp, &local->rx_beacon_list, node) { - list_del(&mac_pkt->node); - kfree_skb(mac_pkt->skb); - kfree(mac_pkt); - } + spin_lock_bh(&local->rx_lock); + mac802154_flush_list(&local->rx_beacon_list, NULL); + spin_unlock_bh(&local->rx_lock); } static void diff --git a/net/mctp/route.c b/net/mctp/route.c index 1f3dccbb7aedb..e7c95eeacb486 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -825,7 +825,7 @@ static struct mctp_sk_key *mctp_lookup_prealloc_tag(struct mctp_sock *msk, spin_lock_irqsave(&mns->keys_lock, flags); - hlist_for_each_entry(tmp, &mns->keys, hlist) { + hlist_for_each_entry(tmp, &msk->keys, sklist) { if (tmp->net != netid) continue; @@ -998,14 +998,29 @@ int mctp_route_lookup(struct net *net, unsigned int dnet, mtu = mtu ?: rt->mtu; if (rt->dst_type == MCTP_ROUTE_DIRECT) { - mctp_eid_t saddr = mctp_dev_saddr(rt->dev); + mctp_eid_t saddr; + + /* rt->dev may be going away concurrently: its last + * reference is dropped in mctp_dev_put(), which frees + * mdev->addrs that mctp_dev_saddr() reads, and + * mctp_dst_from_route() takes a reference on it. Pin + * it before use, and skip a device that is already + * dead rather than resurrecting it. + */ + if (!refcount_inc_not_zero(&rt->dev->refs)) + break; + + saddr = mctp_dev_saddr(rt->dev); /* cannot do gateway-ed routes without a src */ - if (saddr == MCTP_ADDR_NULL && depth != 0) + if (saddr == MCTP_ADDR_NULL && depth != 0) { + mctp_dev_put(rt->dev); break; + } if (dst) mctp_dst_from_route(dst, daddr, saddr, mtu, rt); + mctp_dev_put(rt->dev); rc = 0; break; diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index 961be5054a039..17b78dcbf8abc 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -221,6 +221,7 @@ static u32 mpls_multipath_hash(struct mpls_route *rt, struct sk_buff *skb) if (pskb_may_pull(skb, mpls_hdr_len + sizeof(struct iphdr))) { const struct iphdr *v4hdr; + hdr = mpls_hdr(skb) + label_index; v4hdr = (const struct iphdr *)(hdr + 1); if (v4hdr->version == 4) { hash = jhash_3words(ntohl(v4hdr->saddr), @@ -231,6 +232,7 @@ static u32 mpls_multipath_hash(struct mpls_route *rt, struct sk_buff *skb) sizeof(struct ipv6hdr))) { const struct ipv6hdr *v6hdr; + hdr = mpls_hdr(skb) + label_index; v6hdr = (const struct ipv6hdr *)(hdr + 1); hash = __ipv6_addr_jhash(&v6hdr->saddr, hash); hash = __ipv6_addr_jhash(&v6hdr->daddr, hash); diff --git a/net/mptcp/options.c b/net/mptcp/options.c index 1057d500577b0..bf8ea4cccb2f0 100644 --- a/net/mptcp/options.c +++ b/net/mptcp/options.c @@ -93,7 +93,8 @@ static void mptcp_parse_option(const struct sk_buff *skb, * In other words, the only way for checksums not to be used * is if both hosts in their SYNs set A=0." */ - if (flags & MPTCP_CAP_CHECKSUM_REQD) + if ((flags & MPTCP_CAP_CHECKSUM_REQD) && + opsize < TCPOLEN_MPTCP_MPC_ACK_DATA) mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD; mp_opt->deny_join_id0 = !!(flags & MPTCP_CAP_DENY_JOIN_ID0); @@ -529,7 +530,7 @@ static bool mptcp_established_options_mp(struct sock *sk, struct sk_buff *skb, return false; /* MPC/MPJ needed only on 3rd ack packet, DATA_FIN and TCP shutdown take precedence */ - if (READ_ONCE(subflow->fully_established) || snd_data_fin_enable || + if (subflow->fully_established || snd_data_fin_enable || subflow->snd_isn != TCP_SKB_CB(skb)->seq || sk->sk_state != TCP_ESTABLISHED) return false; @@ -611,6 +612,7 @@ static void mptcp_write_data_fin(struct mptcp_subflow_context *subflow, ext->data_seq = data_fin_tx_seq; ext->subflow_seq = 0; ext->data_len = 1; + ext->csum = 0; } else if (ext->data_seq + ext->data_len == data_fin_tx_seq) { /* If there's an existing DSS mapping and it is the * final mapping, DATA_FIN consumes 1 additional byte of @@ -980,7 +982,7 @@ static bool check_fully_established(struct mptcp_sock *msk, struct sock *ssk, /* here we can process OoO, in-window pkts, only in-sequence 4th ack * will make the subflow fully established */ - if (likely(READ_ONCE(subflow->fully_established))) { + if (likely(subflow->fully_established)) { /* on passive sockets, check for 3rd ack retransmission * note that msk is always set by subflow_syn_recv_sock() * for mp_join subflows diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index 64a1236aabee9..79bf88ca033b4 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -462,10 +462,10 @@ bool mptcp_pm_announced_alloc(struct mptcp_sock *msk, add_entry->addr = *addr; add_entry->sock = msk; - add_entry->retrans_times = 0; timer_setup(&add_entry->timer, mptcp_pm_add_addr_timer, 0); reset_timer: + add_entry->retrans_times = 0; add_entry->timer_done = false; timeout = mptcp_adjust_add_addr_timeout(msk); if (timeout) @@ -1065,7 +1065,8 @@ bool mptcp_pm_is_backup(struct mptcp_sock *msk, struct sock_common *skc) return mptcp_pm_nl_is_backup(msk, &skc_local); } -static void mptcp_pm_subflows_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) +static void +mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) { struct mptcp_subflow_context *iter, *subflow = mptcp_subflow_ctx(ssk); struct sock *sk = (struct sock *)msk; @@ -1102,22 +1103,34 @@ static void mptcp_pm_subflows_chk_stale(const struct mptcp_sock *msk, struct soc } } -void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) +void mptcp_pm_chk_stale(const struct mptcp_sock *msk) { - struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); - u32 rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp); - - /* keep track of rtx periods with no progress */ - if (!subflow->stale_count) { - subflow->stale_rcv_tstamp = rcv_tstamp; - subflow->stale_count++; - } else if (subflow->stale_rcv_tstamp == rcv_tstamp) { - if (subflow->stale_count < U8_MAX) + struct mptcp_subflow_context *subflow; + + mptcp_for_each_subflow(msk, subflow) { + struct sock *ssk = mptcp_subflow_tcp_sock(subflow); + u32 rcv_tstamp; + + if (!__mptcp_subflow_active(subflow)) + continue; + + /* No data outstanding at TCP level? not stale */ + if (tcp_rtx_and_write_queues_empty(ssk)) + continue; + + /* keep track of rtx periods with no progress */ + rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp); + if (!subflow->stale_count) { + subflow->stale_rcv_tstamp = rcv_tstamp; subflow->stale_count++; - mptcp_pm_subflows_chk_stale(msk, ssk); - } else { - subflow->stale_count = 0; - mptcp_subflow_set_active(subflow); + } else if (subflow->stale_rcv_tstamp == rcv_tstamp) { + if (subflow->stale_count < U8_MAX) + subflow->stale_count++; + mptcp_pm_subflow_chk_stale(msk, ssk); + } else { + subflow->stale_count = 0; + mptcp_subflow_set_active(subflow); + } } } diff --git a/net/mptcp/pm_kernel.c b/net/mptcp/pm_kernel.c index 424f1a7f92482..1a77508132354 100644 --- a/net/mptcp/pm_kernel.c +++ b/net/mptcp/pm_kernel.c @@ -1137,6 +1137,8 @@ static int mptcp_nl_remove_id_zero_address(struct net *net, while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk; struct mptcp_addr_info msk_local; + struct mptcp_addr_info anno_addr; + bool announced; if (list_empty(&msk->conn_list) || mptcp_pm_is_userspace(msk)) goto next; @@ -1146,7 +1148,13 @@ static int mptcp_nl_remove_id_zero_address(struct net *net, goto next; lock_sock(sk); + /* Drop a possibly pending ADD_ADDR for this address. */ + anno_addr = msk_local; + anno_addr.port = 0; + announced = mptcp_pm_announced_remove(msk, &anno_addr); spin_lock_bh(&msk->pm.lock); + if (announced) + msk->pm.add_addr_signaled--; mptcp_pm_remove_addr(msk, &list); mptcp_pm_rm_subflow(msk, &list); __mark_subflow_endp_available(msk, 0); diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c index 2203cc2d27485..57c13293a5f89 100644 --- a/net/mptcp/pm_userspace.c +++ b/net/mptcp/pm_userspace.c @@ -69,6 +69,19 @@ static int mptcp_userspace_pm_append_new_local_addr(struct mptcp_sock *msk, } if (!addr_match && !id_match) { + unsigned int id; + + if (!entry->addr.id && needs_id) { + id = find_next_zero_bit(id_bitmap, + MPTCP_PM_MAX_ADDR_ID + 1, 1); + if (id > MPTCP_PM_MAX_ADDR_ID) { + ret = -ENOSPC; + goto append_err; + } + } else { + id = entry->addr.id; + } + /* Memory for the entry is allocated from the * sock option buffer. */ @@ -78,10 +91,7 @@ static int mptcp_userspace_pm_append_new_local_addr(struct mptcp_sock *msk, goto append_err; } - if (!e->addr.id && needs_id) - e->addr.id = find_next_zero_bit(id_bitmap, - MPTCP_PM_MAX_ADDR_ID + 1, - 1); + e->addr.id = id; list_add_tail_rcu(&e->list, &msk->pm.userspace_pm_local_addr_list); msk->pm.local_addr_used++; ret = e->addr.id; diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 7c8180d8d5eff..de6f289a6e14e 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -95,6 +95,7 @@ bool __mptcp_try_fallback(struct mptcp_sock *msk, int fb_mib) msk->allow_subflows = false; set_bit(MPTCP_FALLBACK_DONE, &msk->flags); + clear_bit(MPTCP_RTX_ENABLED, &msk->flags); __MPTCP_INC_STATS(net, fb_mib); spin_unlock_bh(&msk->fallback_lock); return true; @@ -772,12 +773,12 @@ static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, mptcp_dss_corruption(msk, ssk); } } else { + sk_eat_skb(ssk, skb); + if (unlikely(!fin)) { DEBUG_NET_WARN_ON_ONCE(1); mptcp_dss_corruption(msk, ssk); } - - sk_eat_skb(ssk, skb); } WRITE_ONCE(tp->copied_seq, seq); @@ -1001,13 +1002,14 @@ static bool mptcp_rtx_timer_pending(struct sock *sk) static void mptcp_reset_rtx_timer(struct sock *sk) { + struct mptcp_sock *msk = mptcp_sk(sk); unsigned long tout; - /* prevent rescheduling on close */ - if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) + /* Prevent rescheduling on close and in case of fallback. */ + if (!test_bit(MPTCP_RTX_ENABLED, &msk->flags)) return; - tout = mptcp_sk(sk)->timer_ival; + tout = msk->timer_ival; sk_reset_timer(sk, &sk->mptcp_retransmit_timer, jiffies + tout); } @@ -1586,7 +1588,9 @@ struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) static void mptcp_push_release(struct sock *ssk, struct mptcp_sendmsg_info *info) { - tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal); + if (info->mss_now) + tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, + info->size_goal); release_sock(ssk); } @@ -1774,7 +1778,8 @@ static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool ret = __subflow_push_pending(sk, ssk, &info); if (ret <= 0) keep_pushing = false; - copied += ret; + else + copied += ret; } mptcp_for_each_subflow(msk, subflow) { @@ -2396,7 +2401,7 @@ static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, mptcp_cleanup_rbuf(msk, copied); err = sk_wait_data(sk, &timeo, last); if (err < 0) { - err = copied ? : err; + copied = copied ? : err; goto out_err; } } @@ -2469,7 +2474,6 @@ struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk) /* still data outstanding at TCP level? skip this */ if (!tcp_rtx_and_write_queues_empty(ssk)) { - mptcp_pm_subflow_chk_stale(msk, ssk); min_stale_count = min_t(int, min_stale_count, subflow->stale_count); continue; } @@ -2801,9 +2805,10 @@ static void __mptcp_retrans(struct sock *sk) int ret, err; u16 len = 0; + mptcp_pm_chk_stale(msk); + mptcp_clean_una_wakeup(sk); - /* first check ssk: need to kick "stale" logic */ err = mptcp_sched_get_retrans(msk); dfrag = mptcp_rtx_head(sk); if (!dfrag) { @@ -2985,8 +2990,7 @@ static void mptcp_do_fastclose(struct sock *sk) */ inet_csk(ssk)->icsk_ack.rcv_mss = TCP_MIN_MSS; - tcp_send_active_reset(ssk, ssk->sk_allocation, - SK_RST_REASON_TCP_ABORT_ON_CLOSE); + tcp_send_active_reset(ssk, SK_RST_REASON_TCP_ABORT_ON_CLOSE); unlock: release_sock(ssk); } @@ -3192,6 +3196,9 @@ void mptcp_set_state(struct sock *sk, int state) * transition from TCP_SYN_RECV to TCP_CLOSE_WAIT. */ break; + case TCP_CLOSE: + clear_bit(MPTCP_RTX_ENABLED, &mptcp_sk(sk)->flags); + fallthrough; default: if (oldstate == TCP_ESTABLISHED || oldstate == TCP_CLOSE_WAIT) MPTCP_DEC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB); @@ -3452,6 +3459,7 @@ static void mptcp_destroy_common(struct mptcp_sock *msk) static int mptcp_disconnect(struct sock *sk, int flags) { + struct inet_connection_sock *icsk = inet_csk(sk); struct mptcp_sock *msk = mptcp_sk(sk); /* We are on the fastopen error path. We can't call straight into the @@ -3464,8 +3472,13 @@ static int mptcp_disconnect(struct sock *sk, int flags) mptcp_check_listen_stop(sk); mptcp_set_state(sk, TCP_CLOSE); - mptcp_stop_rtx_timer(sk); - mptcp_stop_tout_timer(sk); + /* The later subflow close can not kick again the tout timer, + * as the msk is already in closed status. + */ + msk->timer_ival = icsk->icsk_rto_min; + sk_stop_timer_sync(sk, &sk->mptcp_retransmit_timer); + icsk->icsk_mtup.probe_timestamp = 0; + sk_stop_timer_sync(sk, &icsk->mptcp_tout_timer); mptcp_pm_connection_closed(msk); @@ -3744,7 +3757,7 @@ static void schedule_3rdack_retransmission(struct sock *ssk) struct tcp_sock *tp = tcp_sk(ssk); unsigned long timeout; - if (READ_ONCE(mptcp_subflow_ctx(ssk)->fully_established)) + if (mptcp_subflow_ctx(ssk)->fully_established) return; /* reschedule with a timeout above RTT, as we must look only for drop */ @@ -4009,6 +4022,7 @@ static int mptcp_connect(struct sock *sk, struct sockaddr_unsized *uaddr, if (IS_ERR(ssk)) return PTR_ERR(ssk); + set_bit(MPTCP_RTX_ENABLED, &msk->flags); mptcp_set_state(sk, TCP_SYN_SENT); subflow = mptcp_subflow_ctx(ssk); #ifdef CONFIG_TCP_MD5SIG @@ -4156,6 +4170,7 @@ static int mptcp_listen(struct socket *sock, int backlog) goto unlock; } + set_bit(MPTCP_RTX_ENABLED, &msk->flags); mptcp_set_state(sk, TCP_LISTEN); sock_set_flag(sk, SOCK_RCU_FREE); diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 1b80f2d6ec5a2..d44aaa783c22b 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -116,6 +116,7 @@ #define MPTCP_WORK_RTX 1 #define MPTCP_FALLBACK_DONE 2 #define MPTCP_WORK_CLOSE_SUBFLOW 3 +#define MPTCP_RTX_ENABLED 4 /* MPTCP socket release cb flags */ #define MPTCP_PUSH_PENDING 1 @@ -580,11 +581,11 @@ struct mptcp_subflow_context { remote_key_valid : 1, /* received the peer key from */ disposable : 1, /* ctx can be free at ulp release time */ closing : 1, /* must not pass rx data to msk anymore */ - stale : 1, /* unable to snd/rcv data, do not use for xmit */ valid_csum_seen : 1, /* at least one csum validated */ is_mptfo : 1, /* subflow is doing TFO */ close_event_done : 1, /* has done the post-closed part */ mpc_drop : 1, /* the MPC option has been dropped in a rtx */ + resetting : 1, /* subflow is resetting */ __unused : 8; bool data_avail; bool scheduled; @@ -604,7 +605,11 @@ struct mptcp_subflow_context { u8 reset_seen:1; u8 reset_transient:1; u8 reset_reason:4; - u8 stale_count; + u8 stale_count; /* Protected by the msk socket lock */ + u8 stale; /* Protected by the msk socket lock, + * if set the subflow is unable to snd/rcv + * data, the schedule should skip it + */ u32 subflow_id; @@ -687,7 +692,7 @@ mptcp_send_active_reset_reason(struct sock *sk) enum sk_rst_reason reason; reason = sk_rst_convert_mptcp_reason(subflow->reset_reason); - tcp_send_active_reset(sk, GFP_ATOMIC, reason); + tcp_send_active_reset(sk, reason); } /* Made the fwd mem carried by the given skb available to the msk, @@ -1103,7 +1108,7 @@ int mptcp_pm_parse_entry(struct nlattr *attr, struct genl_info *info, bool mptcp_pm_addr_families_match(const struct sock *sk, const struct mptcp_addr_info *loc, const struct mptcp_addr_info *rem); -void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk); +void mptcp_pm_chk_stale(const struct mptcp_sock *msk); void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side); void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk); bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk); diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index e1f20ff8fdb42..edd33dc9367f9 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -436,6 +436,10 @@ void mptcp_subflow_reset(struct sock *ssk) /* must hold: tcp_done() could drop last reference on parent */ sock_hold(sk); + subflow->resetting = 1; + + /* No need to delay the actual close for to-be discarded data. */ + __skb_queue_purge(&ssk->sk_receive_queue); mptcp_send_active_reset_reason(ssk); tcp_done(ssk); if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags)) @@ -1873,6 +1877,13 @@ static void subflow_state_change(struct sock *sk) __subflow_state_change(sk); + /* Rx queue processing is unneeded, error reporting will take place at + * __mptcp_close_ssk() time and subflow reset can't happen in case of + * fallback: subflow_sched_work_if_closed() would be a no-op. + */ + if (subflow->resetting) + return; + /* as recvmsg() does not acquire the subflow socket for ssk selection * a fin packet carrying a DSS can be unnoticed if we don't trigger * the data available machinery here. @@ -2074,7 +2085,6 @@ static void subflow_ulp_clone(const struct request_sock *req, new_ctx->request_bkup = subflow_req->request_bkup; WRITE_ONCE(new_ctx->remote_id, subflow_req->remote_id); new_ctx->token = subflow_req->token; - new_ctx->thmac = subflow_req->thmac; /* the subflow req id is valid, fetched via subflow_check_req() * and subflow_token_join_request() diff --git a/net/mptcp/syncookies.c b/net/mptcp/syncookies.c index 7f22526346a7e..9474706641c11 100644 --- a/net/mptcp/syncookies.c +++ b/net/mptcp/syncookies.c @@ -26,7 +26,8 @@ struct join_entry { u32 local_nonce; u8 join_id; u8 local_id; - u8 backup; + u8 backup:1, + request_bkup:1; u8 valid; }; @@ -63,6 +64,7 @@ static void mptcp_join_store_state(struct join_entry *entry, entry->remote_nonce = subflow_req->remote_nonce; entry->local_nonce = subflow_req->local_nonce; entry->backup = subflow_req->backup; + entry->request_bkup = subflow_req->request_bkup; entry->join_id = subflow_req->remote_id; entry->local_id = subflow_req->local_id; entry->valid = 1; @@ -117,7 +119,9 @@ bool mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subfl subflow_req->remote_nonce = e->remote_nonce; subflow_req->local_nonce = e->local_nonce; subflow_req->backup = e->backup; + subflow_req->request_bkup = e->request_bkup; subflow_req->remote_id = e->join_id; + subflow_req->local_id = e->local_id; subflow_req->token = e->token; subflow_req->msk = msk; spin_unlock_bh(&join_entry_locks[i]); diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index f00c82acd7f05..80ca523f304bb 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -98,14 +99,34 @@ struct htable { #define IPSET_NET_COUNT 1 #endif -/* Book-keeping of the prefixes added to the set */ +/** + * struct net_prefix - Representation of a network prefix. + * @cidr: The CIDR prefix length. + * @count: Number of occurrences. + */ struct net_prefix { - u8 cidr; /* the cidr value */ - u32 count; /* number of elements of this cidr */ + u32 cidr:8; + u32 count:24; }; +#define CIDR_MAX_COUNT ((1 << 24) - 1) + +/** + * struct net_prefixes - A collection of network prefixes. + * @rcu: RCU head + * @seq: Sequence counter guarding in-place reordering of @nets + * @len: Number of entries in the array. + * @nets: Array of net_prefix structures (sorted by CIDR descending). + * + * @nets entries are updated in place under @set's lock. A single entry's + * cidr/count pair is always updated atomically via READ_ONCE()/WRITE_ONCE(), + * but removing an entry also shifts every following entry down by one slot. + * Lockless readers that scan the whole array (i.e. more than a single + * indexed slot) must use @seq to detect and retry across such a shift. + */ struct net_prefixes { struct rcu_head rcu; + seqcount_spinlock_t seq; u8 len; struct net_prefix nets[] __counted_by(len); }; @@ -143,8 +164,11 @@ htable_size(u8 hbits) #endif #define INIT_CIDR(n, host_mask) ({ \ - const struct net_prefixes *__n = rcu_dereference(n); \ - DCIDR_PUT((__n)->len ? (__n)->nets[0].cidr : host_mask);\ + const struct net_prefixes *__n = rcu_dereference(n); \ + struct net_prefix __p = \ + __n->len ? READ_ONCE(__n->nets[0]) \ + : (struct net_prefix){}; \ + DCIDR_PUT(__p.count ? __p.cidr : host_mask); \ }) #endif /* IP_SET_HASH_WITH_NETS */ @@ -318,27 +342,43 @@ struct mtype_resize_ad { }; #ifdef IP_SET_HASH_WITH_NETS -/* Network cidr size book keeping when the hash stores different - * sized networks. cidr == real cidr + 1 to support /0. +/** + * mtype_add_cidr - Add a CIDR entry to hash table bookkeeping + * @set: Pointer to the ip_set + * @h: Pointer to the htype + * @cidr: The CIDR prefix length + * @n: The index of the net_prefix array to add @cidr to + * + * Performs an update if @cidr is found, otherwise performs COW-style + * allocation and replacement via RCU. + * + * Return: 0 on success, negative error code on failure. */ static int mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) { - struct net_prefixes *nets, *tmp; int i, j, found, len = 0, ret = 0; + struct net_prefixes *nets, *tmp; + struct net_prefix np; spin_lock_bh(&set->lock); nets = __ipset_dereference(h->rnets[n]); /* Add in increasing prefix order, so larger cidr first */ for (i = 0, found = -1; i < nets->len; i++) { - if (nets->nets[i].count) + np = READ_ONCE(nets->nets[i]); + if (np.count) len++; if (found != -1) { continue; - } else if (nets->nets[i].cidr < cidr) { + } else if (np.cidr < cidr) { found = i; - } else if (nets->nets[i].cidr == cidr) { - nets->nets[i].count++; + } else if (np.cidr == cidr) { + if (np.count < CIDR_MAX_COUNT) { + np.count++; + WRITE_ONCE(nets->nets[i], np); + } else { + ret = -EOVERFLOW; + } goto unlock; } } @@ -350,6 +390,7 @@ mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) } tmp->len = len; + seqcount_spinlock_init(&tmp->seq, &set->lock); for (i = 0, j = 0; i < nets->len; i++) { if (i == found) { tmp->nets[j].cidr = cidr; @@ -371,42 +412,60 @@ mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) return ret; } +/** + * mtype_del_cidr - Remove CIDR entry and maintain array integrity. + * @set: Pointer to the ip_set. + * @h: Pointer to the htype. + * @cidr: The CIDR prefix length. + * @n: The index of the net_prefix array to remove @cidr from + * + * If CIDR entry count falls to 0, this function performs a "shift-left" + * operation on all following elements. This ensures that the array remains + * contiguous and maintains its descending order by CIDR. The vacated slot + * at the end of the array is zeroed out (cidr=0, count=0). + */ static void mtype_del_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) { - struct net_prefixes *nets, *tmp; - u8 i, j, len = 0; + struct net_prefixes *nets; + struct net_prefix np; int found; + u8 i, j; + + BUILD_BUG_ON(sizeof(struct net_prefix) != sizeof(u32)); spin_lock_bh(&set->lock); nets = __ipset_dereference(h->rnets[n]); for (i = 0, found = -1; i < nets->len; i++) { - if (nets->nets[i].count) - len++; - if (nets->nets[i].cidr == cidr) + np = READ_ONCE(nets->nets[i]); + if (np.count && np.cidr == cidr) { + np.count--; found = i; + break; + } } if (unlikely(found == -1)) goto unlock; - nets->nets[found].count--; - if (nets->nets[found].count) - goto unlock; - len--; - tmp = kzalloc_flex(*tmp, nets, len, GFP_ATOMIC); - if (!tmp) - /* Leave a hole */ + if (np.count) { + WRITE_ONCE(nets->nets[found], np); goto unlock; + } - tmp->len = len; + write_seqcount_begin(&nets->seq); for (i = 0, j = 0; i < nets->len; i++) { - if (!nets->nets[i].count || i == found) + if (i == found) continue; - tmp->nets[j].cidr = nets->nets[i].cidr; - tmp->nets[j++].count = nets->nets[i].count; + + np = READ_ONCE(nets->nets[i]); + if (i != j) + WRITE_ONCE(nets->nets[j], np); + j++; } - rcu_assign_pointer(h->rnets[n], tmp); - kfree_rcu(nets, rcu); + + while (j < nets->len) + WRITE_ONCE(nets->nets[j++], (struct net_prefix){}); + write_seqcount_end(&nets->seq); unlock: spin_unlock_bh(&set->lock); } @@ -451,7 +510,7 @@ mtype_flush(struct ip_set *set) { struct htype *h = set->data; #ifdef IP_SET_HASH_WITH_NETS - struct net_prefixes *nets, *tmp; + struct net_prefixes *nets; #endif struct htable *t; struct hbucket *n; @@ -477,17 +536,15 @@ mtype_flush(struct ip_set *set) } #ifdef IP_SET_HASH_WITH_NETS for (i = 0; i < IPSET_NET_COUNT; i++) { - nets = ipset_dereference_nfnl(h->rnets[i]); - tmp = kzalloc_obj(*tmp, GFP_ATOMIC); - if (!tmp) { - u8 j; + u8 j; - for (j = 0; j < nets->len; j++) - nets->nets[j].count = 0; - } else { - rcu_assign_pointer(h->rnets[i], tmp); - kfree_rcu(nets, rcu); - } + spin_lock_bh(&set->lock); + nets = ipset_dereference_nfnl(h->rnets[i]); + write_seqcount_begin(&nets->seq); + for (j = 0; j < nets->len; j++) + WRITE_ONCE(nets->nets[j], (struct net_prefix){}); + write_seqcount_end(&nets->seq); + spin_unlock_bh(&set->lock); } #endif } @@ -1253,31 +1310,41 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, #if IPSET_NET_COUNT == 2 struct net_prefixes *nets1; struct mtype_elem orig = *d; + unsigned int seq1; int ret, i, j, k; #else int ret, i, j; #endif - u32 key, multi = 0; + unsigned int seq0; + u32 key, multi; u8 pos; pr_debug("test by nets\n"); rcu_read_lock_bh(); +retry: + multi = 0; nets0 = rcu_dereference_bh(h->rnets[0]); + seq0 = read_seqcount_begin(&nets0->seq); #if IPSET_NET_COUNT == 2 nets1 = rcu_dereference_bh(h->rnets[1]); + seq1 = read_seqcount_begin(&nets1->seq); #endif for (j = 0; j < nets0->len && !multi; j++) { - if (!nets0->nets[j].count) + struct net_prefix p0 = READ_ONCE(nets0->nets[j]); + + if (!p0.count) continue; #if IPSET_NET_COUNT == 2 mtype_data_reset_elem(d, &orig); - mtype_data_netmask(d, nets0->nets[j].cidr, false); + mtype_data_netmask(d, p0.cidr, false); for (k = 0; k < nets1->len && !multi; k++) { - if (!nets1->nets[k].count) + struct net_prefix p1 = READ_ONCE(nets1->nets[k]); + + if (!p1.count) continue; - mtype_data_netmask(d, nets1->nets[k].cidr, true); + mtype_data_netmask(d, p1.cidr, true); #else - mtype_data_netmask(d, nets0->nets[j].cidr); + mtype_data_netmask(d, p0.cidr); #endif key = HKEY(d, h->initval, t->htable_bits); n = rcu_dereference_bh(hbucket(t, key)); @@ -1304,6 +1371,12 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, } ret = 0; unlock: + if (read_seqcount_retry(&nets0->seq, seq0)) + goto retry; +#if IPSET_NET_COUNT == 2 + if (read_seqcount_retry(&nets1->seq, seq1)) + goto retry; +#endif rcu_read_unlock_bh(); return ret; } @@ -1707,6 +1780,7 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set, kfree(rcu_dereference_raw(h->rnets[--i])); goto free_hregion; } + seqcount_spinlock_init(&nets->seq, &set->lock); RCU_INIT_POINTER(h->rnets[i], nets); } #endif diff --git a/net/netfilter/ipset/ip_set_hash_netiface.c b/net/netfilter/ipset/ip_set_hash_netiface.c index b44b95f766b7a..b602cc43565dd 100644 --- a/net/netfilter/ipset/ip_set_hash_netiface.c +++ b/net/netfilter/ipset/ip_set_hash_netiface.c @@ -38,7 +38,6 @@ MODULE_ALIAS("ip_set_hash:net,iface"); #define HTYPE hash_netiface #define IP_SET_HASH_WITH_NETS #define IP_SET_HASH_WITH_MULTI -#define IP_SET_HASH_WITH_NET0 #define STRSCPY(a, b) strscpy(a, b, IFNAMSIZ) diff --git a/net/netfilter/ipset/ip_set_hash_netportnet.c b/net/netfilter/ipset/ip_set_hash_netportnet.c index 6291532be7a5a..61af1ce271279 100644 --- a/net/netfilter/ipset/ip_set_hash_netportnet.c +++ b/net/netfilter/ipset/ip_set_hash_netportnet.c @@ -36,7 +36,6 @@ MODULE_ALIAS("ip_set_hash:net,port,net"); #define IP_SET_HASH_WITH_PROTO #define IP_SET_HASH_WITH_NETS #define IPSET_NET_COUNT 2 -#define IP_SET_HASH_WITH_NET0 /* IPv4 variant */ diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index eb806813292ad..6802ce5cb1de4 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -1961,6 +1961,12 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Ensure the IP header is present in headroom */ if (!pskb_may_pull(skb, hlen_orig)) goto ignore_tunnel; + skb_set_transport_header(skb, hlen_orig); + /* Before now we may used ihl from skb frag, revalidate it after + * copying it into skb head to prevent out-of-bounds access + */ + if (ip_hdr(skb)->ihl * 4 != hlen_orig) + goto ignore_tunnel; IP_VS_DBG(12, "Sending ICMP for %pI4->%pI4: t=%u, c=%u, i=%u\n", &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, type, code, ntohl(info)); diff --git a/net/netfilter/ipvs/ip_vs_ftp.c b/net/netfilter/ipvs/ip_vs_ftp.c index b315c608fda43..9e3e005a82635 100644 --- a/net/netfilter/ipvs/ip_vs_ftp.c +++ b/net/netfilter/ipvs/ip_vs_ftp.c @@ -102,7 +102,7 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, char *s, c; unsigned char p[6]; char edelim; - __u16 hport; + __u32 hport; int i = 0; if (data_limit - data < plen) { @@ -144,7 +144,11 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, return -1; c = *data; if (isdigit(c)) { - p[i] = p[i]*10 + c - '0'; + unsigned int val = p[i] * 10 + c - '0'; + + if (val > 255) + return -1; + p[i] = val; } else if (c == ',' && i < 5) { i++; p[i] = 0; @@ -222,6 +226,8 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, if (!isdigit(*s)) break; hport = hport * 10 + *s - '0'; + if (hport > 65535) + return -1; } if (s == data_limit || !hport || *s != edelim) return -1; diff --git a/net/netfilter/ipvs/ip_vs_sync.c b/net/netfilter/ipvs/ip_vs_sync.c index ea5fdd4f4ce76..5383aeafb0aed 100644 --- a/net/netfilter/ipvs/ip_vs_sync.c +++ b/net/netfilter/ipvs/ip_vs_sync.c @@ -747,9 +747,9 @@ void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts) if (cp->flags & IP_VS_CONN_F_SEQ_MASK) { *(p++) = IPVS_OPT_SEQ_DATA; *(p++) = sizeof(struct ip_vs_sync_conn_options); - hton_seq((struct ip_vs_seq *)p, &cp->in_seq); + hton_seq(&cp->in_seq, (struct ip_vs_seq *)p); p += sizeof(struct ip_vs_seq); - hton_seq((struct ip_vs_seq *)p, &cp->out_seq); + hton_seq(&cp->out_seq, (struct ip_vs_seq *)p); p += sizeof(struct ip_vs_seq); } /* Handle pe data */ @@ -999,10 +999,10 @@ static void ip_vs_process_message_v0(struct netns_ipvs *ipvs, const char *buffer pp->name, state); continue; } - } else { - if (state >= IP_VS_CTPL_S_LAST) - IP_VS_DBG(7, "BACKUP v0, Invalid tpl state %u\n", - state); + } else if (state >= IP_VS_CTPL_S_LAST) { + IP_VS_DBG(7, "BACKUP v0, Invalid tpl state %u\n", + state); + continue; } ip_vs_conn_fill_param(ipvs, AF_INET, s->protocol, @@ -1159,10 +1159,10 @@ static inline int ip_vs_proc_sync_conn(struct netns_ipvs *ipvs, __u8 *p, __u8 *m retc = 40; goto out; } - } else { - if (state >= IP_VS_CTPL_S_LAST) - IP_VS_DBG(7, "BACKUP, Invalid tpl state %u\n", - state); + } else if (state >= IP_VS_CTPL_S_LAST) { + IP_VS_DBG(7, "BACKUP, Invalid tpl state %u\n", state); + retc = 40; + goto out; } if (ip_vs_conn_fill_param_sync(ipvs, af, s, ¶m, pe_data, pe_data_len, pe_name, pe_name_len)) { diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c index 500509b17663e..f776cc0009da2 100644 --- a/net/netfilter/nf_conntrack_helper.c +++ b/net/netfilter/nf_conntrack_helper.c @@ -456,6 +456,15 @@ static bool expect_iter_me(struct nf_conntrack_expect *exp, void *data) return this == me; } +void nf_conntrack_helper_release(struct nf_conntrack_helper *me) +{ + nf_ct_expect_iterate_destroy(expect_iter_me, me); + + if (refcount_dec_and_test(&me->ct_refcnt)) + kfree_rcu(me, rcu); +} +EXPORT_SYMBOL_GPL(nf_conntrack_helper_release); + void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me) { mutex_lock(&nf_ct_helper_mutex); @@ -471,10 +480,7 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me) */ synchronize_rcu(); - nf_ct_expect_iterate_destroy(expect_iter_me, me); - - if (refcount_dec_and_test(&me->ct_refcnt)) - kfree_rcu(me, rcu); + nf_conntrack_helper_release(me); } EXPORT_SYMBOL_GPL(nf_conntrack_helper_unregister); diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 31cbb1b55b9e6..6cddfa339d71e 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -1077,6 +1077,8 @@ static int ctnetlink_start(struct netlink_callback *cb) } cb->data = filter; + if (filter) + cb->answer_flags = NLM_F_DUMP_FILTERED; return 0; } @@ -3042,7 +3044,7 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, #endif if (nla_put_be32(skb, CTA_EXPECT_TIMEOUT, htonl(timeout)) || nla_put_be32(skb, CTA_EXPECT_ID, nf_expect_get_id(exp)) || - nla_put_be32(skb, CTA_EXPECT_FLAGS, htonl(exp->flags)) || + nla_put_be32(skb, CTA_EXPECT_FLAGS, htonl(exp->flags & NF_CT_EXPECT_MASK)) || nla_put_be32(skb, CTA_EXPECT_CLASS, htonl(exp->class))) goto nla_put_failure; @@ -3390,7 +3392,8 @@ static bool expect_iter_name(struct nf_conntrack_expect *exp, void *data) struct nf_conntrack_helper *helper; const char *name = data; - helper = rcu_dereference(exp->helper); + helper = rcu_dereference_protected(exp->helper, + lockdep_is_held(&nf_conntrack_expect_lock)); if (!helper) return false; diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index e4a70d1d77b0b..4fb33b5e9a85a 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -429,7 +429,7 @@ static const char *sip_skip_whitespace(const char *dptr, const char *limit) dptr = sip_follow_continuation(dptr, limit); break; } - return dptr; + return dptr < limit ? dptr : NULL; } /* Search within a SIP header value, dealing with continuation lines */ diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c index 8b1165f2b5a49..254063fca0780 100644 --- a/net/netfilter/nf_flow_table_core.c +++ b/net/netfilter/nf_flow_table_core.c @@ -258,6 +258,14 @@ static void flow_offload_route_release(struct flow_offload *flow) nft_flow_dst_release(flow, FLOW_OFFLOAD_DIR_REPLY); } +static void flow_offload_free_rcu(struct rcu_head *rcu_head) +{ + struct flow_offload *flow = container_of(rcu_head, struct flow_offload, rcu_head); + + nf_ct_put(flow->ct); + kfree(flow); +} + void flow_offload_free(struct flow_offload *flow) { switch (flow->type) { @@ -267,8 +275,7 @@ void flow_offload_free(struct flow_offload *flow) default: break; } - nf_ct_put(flow->ct); - kfree_rcu(flow, rcu_head); + call_rcu(&flow->rcu_head, flow_offload_free_rcu); } EXPORT_SYMBOL_GPL(flow_offload_free); @@ -852,6 +859,7 @@ static int __init nf_flow_table_module_init(void) static void __exit nf_flow_table_module_exit(void) { + rcu_barrier(); nf_flow_table_offload_exit(); unregister_pernet_subsys(&nf_flow_table_net_ops); kmem_cache_destroy(flow_offload_cachep); diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c index 801a3dd9ceea3..6757fd89c1f1a 100644 --- a/net/netfilter/nf_flow_table_offload.c +++ b/net/netfilter/nf_flow_table_offload.c @@ -995,7 +995,6 @@ static void flow_offload_work_del(struct flow_offload_work *offload) flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL); if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags)) flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY); - set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags); } static void flow_offload_tuple_stats(struct flow_offload_work *offload, @@ -1059,6 +1058,12 @@ static void flow_offload_work_handler(struct work_struct *work) } clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags); + if (offload->cmd == FLOW_CLS_DESTROY) { + /* Publish after the worker's last flow access. */ + smp_mb__before_atomic(); + set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags); + } + kfree(offload); } diff --git a/net/netfilter/nf_flow_table_path.c b/net/netfilter/nf_flow_table_path.c index 98c03b487f521..b9df7453f7620 100644 --- a/net/netfilter/nf_flow_table_path.c +++ b/net/netfilter/nf_flow_table_path.c @@ -50,6 +50,9 @@ static int nft_dev_fill_forward_path(const struct nf_flow_route *route, { const void *daddr = &ct->tuplehash[!dir].tuple.src.u3; struct net_device *dev = dst_cache->dev; + struct net_device_path_ctx ctx = { + .dev = dev, + }; struct neighbour *n; u8 nud_state; @@ -72,7 +75,9 @@ static int nft_dev_fill_forward_path(const struct nf_flow_route *route, return -1; out: - return dev_fill_forward_path(dev, ha, stack); + ether_addr_copy(ctx.daddr, ha); + + return dev_fill_forward_path(&ctx, stack); } struct nft_forward_info { diff --git a/net/netfilter/nf_log_syslog.c b/net/netfilter/nf_log_syslog.c index 5ffde27d450a7..ae0f9a47556c5 100644 --- a/net/netfilter/nf_log_syslog.c +++ b/net/netfilter/nf_log_syslog.c @@ -1073,12 +1073,12 @@ static int __init nf_log_syslog_init(void) static void __exit nf_log_syslog_exit(void) { - unregister_pernet_subsys(&nf_log_syslog_net_ops); nf_log_unregister(&nf_ip_logger); nf_log_unregister(&nf_arp_logger); nf_log_unregister(&nf_ip6_logger); nf_log_unregister(&nf_netdev_logger); nf_log_unregister(&nf_bridge_logger); + unregister_pernet_subsys(&nf_log_syslog_net_ops); } module_init(nf_log_syslog_init); diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index 63ff6b4d5d214..be021d8ecc41a 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -1230,31 +1230,45 @@ int nf_nat_register_fn(struct net *net, u8 pf, const struct nf_hook_ops *ops, } ret = nf_register_net_hooks(net, nat_ops, ops_count); - if (ret < 0) { - mutex_unlock(&nf_nat_proto_mutex); - for (i = 0; i < ops_count; i++) { - priv = nat_ops[i].priv; - kfree_rcu(priv, rcu_head); - } - kfree_rcu(nat_ops, rcu); - return ret; - } - - nat_proto_net->nat_hook_ops = nat_ops; + if (ret < 0) + goto err_free_hooks; + } else { + nat_ops = nat_proto_net->nat_hook_ops; } - nat_ops = nat_proto_net->nat_hook_ops; priv = nat_ops[hooknum].priv; if (WARN_ON_ONCE(!priv)) { - mutex_unlock(&nf_nat_proto_mutex); - return -EOPNOTSUPP; + ret = -EOPNOTSUPP; + goto err_unregister_hooks; } ret = nf_hook_entries_insert_raw(&priv->entries, ops); - if (ret == 0) - nat_proto_net->users++; + if (ret) + goto err_unregister_hooks; + + if (!nat_proto_net->nat_hook_ops) + nat_proto_net->nat_hook_ops = nat_ops; + + nat_proto_net->users++; mutex_unlock(&nf_nat_proto_mutex); + + return 0; + +err_unregister_hooks: + if (nat_proto_net->nat_hook_ops) { + mutex_unlock(&nf_nat_proto_mutex); + return ret; + } + nf_unregister_net_hooks(net, nat_ops, ops_count); +err_free_hooks: + mutex_unlock(&nf_nat_proto_mutex); + for (i = 0; i < ops_count; i++) { + priv = nat_ops[i].priv; + kfree_rcu(priv, rcu_head); + } + kfree_rcu(nat_ops, rcu); + return ret; } diff --git a/net/netfilter/nf_nat_sip.c b/net/netfilter/nf_nat_sip.c index 133bd713fe0c2..8c412bcf6cff5 100644 --- a/net/netfilter/nf_nat_sip.c +++ b/net/netfilter/nf_nat_sip.c @@ -273,12 +273,17 @@ static unsigned int nf_nat_sip(struct sk_buff *skb, unsigned int protoff, SIP_HDR_CONTACT, &in_header, &matchoff, &matchlen, &addr, &port) > 0) { + int old_len = skb->len, delta; + if (!map_addr(skb, protoff, dataoff, dptr, datalen, matchoff, matchlen, &addr, port)) { nf_ct_helper_log(skb, ct, "cannot mangle contact"); return NF_DROP; } + + delta = (int)skb->len - old_len; + coff += delta; } if (!map_sip_addr(skb, protoff, dataoff, dptr, datalen, SIP_HDR_FROM) || diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index af357f6c5070b..ea15ebd7bce8e 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -2435,11 +2435,14 @@ static struct nft_hook *nft_netdev_hook_alloc(struct net *net, } static struct nft_hook *nft_hook_list_find(struct list_head *hook_list, - const struct nft_hook *this) + const struct nft_hook *this, + bool strict) { struct nft_hook *hook; list_for_each_entry(hook, hook_list, list) { + if (strict && hook->ifnamelen != this->ifnamelen) + continue; if (!strncmp(hook->ifname, this->ifname, min(hook->ifnamelen, this->ifnamelen))) { if (hook->flags & NFT_HOOK_REMOVE) @@ -2481,7 +2484,7 @@ static int nf_tables_parse_netdev_hooks(struct net *net, err = PTR_ERR(hook); goto err_hook; } - if (nft_hook_list_find(hook_list, hook)) { + if (nft_hook_list_find(hook_list, hook, false)) { NL_SET_BAD_ATTR(extack, tmp); nft_netdev_hook_free(hook); err = -EEXIST; @@ -2938,7 +2941,7 @@ static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy, ops->hook = basechain->ops.hook; } - if (nft_hook_list_find(&basechain->hook_list, h)) { + if (nft_hook_list_find(&basechain->hook_list, h, false)) { list_del(&h->list); nft_netdev_hook_free(h); continue; @@ -2951,7 +2954,8 @@ static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy, !nft_trans_chain_update(trans)) continue; - if (nft_hook_list_find(&nft_trans_chain_hooks(trans), h)) { + if (nft_hook_list_find(&nft_trans_chain_hooks(trans), + h, false)) { nft_chain_release_hook(&hook); return -EEXIST; } @@ -3252,7 +3256,7 @@ static int nft_delchain_hook(struct nft_ctx *ctx, return err; list_for_each_entry(this, &chain_hook.list, list) { - hook = nft_hook_list_find(&basechain->hook_list, this); + hook = nft_hook_list_find(&basechain->hook_list, this, true); if (!hook) { err = -ENOENT; goto err_chain_del_hook; @@ -6916,39 +6920,11 @@ static void nft_trans_elems_destroy(const struct nft_ctx *ctx, nf_tables_set_elem_destroy(ctx, te->set, te->elems[i].priv); } -int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set, - struct nft_expr *expr_array[]) -{ - struct nft_expr *expr; - int err, i, k; - - for (i = 0; i < set->num_exprs; i++) { - expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT); - if (!expr) - goto err_expr; - - err = nft_expr_clone(expr, set->exprs[i], GFP_KERNEL_ACCOUNT); - if (err < 0) { - kfree(expr); - goto err_expr; - } - expr_array[i] = expr; - } - - return 0; - -err_expr: - for (k = i - 1; k >= 0; k--) - nft_expr_destroy(ctx, expr_array[k]); - - return -ENOMEM; -} - static int nft_set_elem_expr_setup(struct nft_ctx *ctx, const struct nft_set_ext_tmpl *tmpl, const struct nft_set_ext *ext, struct nft_expr *expr_array[], - u32 num_exprs) + u32 num_exprs, bool override_exprs) { struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext); u32 len = sizeof(struct nft_set_elem_expr); @@ -6971,7 +6947,8 @@ static int nft_set_elem_expr_setup(struct nft_ctx *ctx, goto err_elem_expr_setup; elem_expr->size += expr_array[i]->ops->size; - nft_expr_destroy(ctx, expr_array[i]); + if (override_exprs) + nft_expr_destroy(ctx, expr_array[i]); expr_array[i] = NULL; } @@ -6979,7 +6956,9 @@ static int nft_set_elem_expr_setup(struct nft_ctx *ctx, err_elem_expr_setup: for (; i < num_exprs; i++) { - nft_expr_destroy(ctx, expr_array[i]); + if (override_exprs) + nft_expr_destroy(ctx, expr_array[i]); + expr_array[i] = NULL; } @@ -7011,11 +6990,14 @@ static int nft_setelem_catchall_insert(const struct net *net, { struct nft_set_elem_catchall *catchall; u8 genmask = nft_genmask_next(net); + u64 tstamp = nft_net_tstamp(net); struct nft_set_ext *ext; list_for_each_entry(catchall, &set->catchall_list, list) { ext = nft_set_elem_ext(set, catchall->elem); - if (nft_set_elem_active(ext, genmask)) { + if (nft_set_elem_active(ext, genmask) && + !__nft_set_elem_expired(ext, tstamp) && + !nft_set_elem_is_dead(ext)) { *priv = catchall->elem; return -EEXIST; } @@ -7108,11 +7090,14 @@ static int nft_setelem_catchall_deactivate(const struct net *net, struct nft_set_elem *elem) { struct nft_set_elem_catchall *catchall; + u64 tstamp = nft_net_tstamp(net); struct nft_set_ext *ext; list_for_each_entry(catchall, &set->catchall_list, list) { ext = nft_set_elem_ext(set, catchall->elem); - if (!nft_is_active_next(net, ext)) + if (!nft_is_active_next(net, ext) || + __nft_set_elem_expired(ext, tstamp) || + nft_set_elem_is_dead(ext)) continue; kfree(elem->priv); @@ -7258,6 +7243,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, struct nft_set_binding *binding; struct nft_elem_priv *elem_priv; struct nft_object *obj = NULL; + bool override_exprs = false; struct nft_userdata *udata; struct nft_data_desc desc; enum nft_registers dreg; @@ -7363,6 +7349,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, expr_array[0] = expr; num_exprs = 1; + override_exprs = true; if (set->num_exprs && set->exprs[0]->ops != expr->ops) { err = -EOPNOTSUPP; @@ -7391,6 +7378,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, } expr_array[i] = expr; num_exprs++; + override_exprs = true; if (set->num_exprs && expr->ops != set->exprs[i]->ops) { err = -EOPNOTSUPP; @@ -7404,9 +7392,8 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, } } else if (set->num_exprs > 0 && !(flags & NFT_SET_ELEM_INTERVAL_END)) { - err = nft_set_elem_expr_clone(ctx, set, expr_array); - if (err < 0) - goto err_set_elem_expr_clone; + for (i = 0; i < set->num_exprs; i++) + expr_array[i] = set->exprs[i]; num_exprs = set->num_exprs; } @@ -7545,7 +7532,8 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, udata->len = ulen - 1; nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen); } - err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs); + err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs, + override_exprs); if (err < 0) goto err_elem_free; @@ -7653,9 +7641,11 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, err_parse_key: nft_data_release(&elem.key.val, NFT_DATA_VALUE); err_set_elem_expr: - for (i = 0; i < num_exprs && expr_array[i]; i++) - nft_expr_destroy(ctx, expr_array[i]); -err_set_elem_expr_clone: + if (override_exprs) { + for (i = 0; i < num_exprs && expr_array[i]; i++) + nft_expr_destroy(ctx, expr_array[i]); + } + return err; } @@ -8715,18 +8705,17 @@ static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info, return nft_delobj(&ctx, obj); } -static void -__nft_obj_notify(struct net *net, const struct nft_table *table, - struct nft_object *obj, u32 portid, u32 seq, int event, - u16 flags, int family, int report, gfp_t gfp) +static struct sk_buff * +nft_obj_notify_alloc(struct net *net, const struct nft_table *table, + struct nft_object *obj, u32 portid, u32 seq, int event, + u16 flags, int family, int report, gfp_t gfp) { - struct nftables_pernet *nft_net = nft_pernet(net); struct sk_buff *skb; int err; if (!report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES)) - return; + return NULL; skb = nlmsg_new(NLMSG_GOODSIZE, gfp); if (skb == NULL) @@ -8740,10 +8729,10 @@ __nft_obj_notify(struct net *net, const struct nft_table *table, goto err; } - nft_notify_enqueue(skb, report, &nft_net->notify_list); - return; + return skb; err: nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS); + return NULL; } void nft_obj_notify(struct net *net, const struct nft_table *table, @@ -8752,6 +8741,7 @@ void nft_obj_notify(struct net *net, const struct nft_table *table, { char *buf = kasprintf(gfp, "%s:%u", table->name, nft_base_seq(net)); + struct sk_buff *skb; audit_log_nfcfg(buf, family, @@ -8762,17 +8752,27 @@ void nft_obj_notify(struct net *net, const struct nft_table *table, gfp); kfree(buf); - __nft_obj_notify(net, table, obj, portid, seq, event, - flags, family, report, gfp); + /* Called from the packet path, holding no mutex: notify_list is + * serialised by commit_mutex, so send this notification directly. + */ + skb = nft_obj_notify_alloc(net, table, obj, portid, seq, event, + flags, family, report, gfp); + if (skb) + nfnetlink_send(skb, net, portid, NFNLGRP_NFTABLES, report, gfp); } EXPORT_SYMBOL_GPL(nft_obj_notify); static void nf_tables_obj_notify(const struct nft_ctx *ctx, struct nft_object *obj, int event) { - __nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, - ctx->seq, event, ctx->flags, ctx->family, - ctx->report, GFP_KERNEL); + struct nftables_pernet *nft_net = nft_pernet(ctx->net); + struct sk_buff *skb; + + skb = nft_obj_notify_alloc(ctx->net, ctx->table, obj, ctx->portid, + ctx->seq, event, ctx->flags, ctx->family, + ctx->report, GFP_KERNEL); + if (skb) + nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list); } /* @@ -9041,7 +9041,7 @@ static int nft_register_flowtable_net_hooks(struct net *net, if (!nft_is_active_next(net, ft)) continue; - if (nft_hook_list_find(&ft->hook_list, hook)) { + if (nft_hook_list_find(&ft->hook_list, hook, false)) { err = -EEXIST; goto err_unregister_net_hooks; } @@ -9118,7 +9118,7 @@ static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh, return err; list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) { - if (nft_hook_list_find(&flowtable->hook_list, hook)) { + if (nft_hook_list_find(&flowtable->hook_list, hook, false)) { list_del(&hook->list); nft_netdev_hook_free(hook); continue; @@ -9131,7 +9131,7 @@ static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh, !nft_trans_flowtable_update(trans)) continue; - if (nft_hook_list_find(&nft_trans_flowtable_hooks(trans), hook)) { + if (nft_hook_list_find(&nft_trans_flowtable_hooks(trans), hook, false)) { err = -EEXIST; goto err_flowtable_update_hook; } @@ -9351,7 +9351,7 @@ static int nft_delflowtable_hook(struct nft_ctx *ctx, return err; list_for_each_entry(this, &flowtable_hook.list, list) { - hook = nft_hook_list_find(&flowtable->hook_list, this); + hook = nft_hook_list_find(&flowtable->hook_list, this, true); if (!hook) { err = -ENOENT; goto err_flowtable_del_hook; @@ -10931,10 +10931,6 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) return -EAGAIN; } - err = nft_flow_rule_offload_commit(net); - if (err < 0) - return err; - /* 1. Allocate space for next generation rules_gen_X[] */ list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) { struct nft_table *table = trans->table; @@ -10959,6 +10955,16 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) } } + /* must be last, so audit and chain blob set up does not leave hardware + * in consistent state. + */ + err = nft_flow_rule_offload_commit(net); + if (err < 0) { + nf_tables_commit_chain_prepare_cancel(net); + nf_tables_commit_audit_free(&adl); + return err; + } + /* step 2. Make rules_gen_X visible to packet path */ list_for_each_entry(table, &nft_net->tables, list) { list_for_each_entry(chain, &table->chains, list) diff --git a/net/netfilter/nfnetlink_cttimeout.c b/net/netfilter/nfnetlink_cttimeout.c index 170d3db860c56..659bafcacc856 100644 --- a/net/netfilter/nfnetlink_cttimeout.c +++ b/net/netfilter/nfnetlink_cttimeout.c @@ -652,9 +652,9 @@ static void __exit cttimeout_exit(void) { nfnetlink_subsys_unregister(&cttimeout_subsys); - unregister_pernet_subsys(&cttimeout_ops); RCU_INIT_POINTER(nf_ct_timeout_hook, NULL); synchronize_net(); + unregister_pernet_subsys(&cttimeout_ops); } module_init(cttimeout_init); diff --git a/net/netfilter/nfnetlink_log.c b/net/netfilter/nfnetlink_log.c index 6c7fa2ed34f5c..d923f2cb13980 100644 --- a/net/netfilter/nfnetlink_log.c +++ b/net/netfilter/nfnetlink_log.c @@ -228,13 +228,18 @@ static void __nfulnl_flush(struct nfulnl_instance *inst); static void __instance_destroy(struct nfulnl_instance *inst) { + spin_lock(&inst->lock); + if (inst->copy_mode == NFULNL_COPY_DISABLED) { + /* attempt to UNBIND a queue already pending + * destruction via netlink close event. Ignore. + */ + spin_unlock(&inst->lock); + return; + } + /* first pull it out of the global list */ hlist_del_rcu(&inst->hlist); - /* then flush all pending packets from skb */ - - spin_lock(&inst->lock); - /* lockless readers wont be able to use us */ inst->copy_mode = NFULNL_COPY_DISABLED; @@ -1233,8 +1238,8 @@ static void __exit nfnetlink_log_fini(void) { nfnetlink_subsys_unregister(&nfulnl_subsys); netlink_unregister_notifier(&nfulnl_rtnl_notifier); - unregister_pernet_subsys(&nfnl_log_net_ops); nf_log_unregister(&nfulnl_logger); + unregister_pernet_subsys(&nfnl_log_net_ops); } MODULE_DESCRIPTION("netfilter userspace logging"); diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index b8aaf39cb4d8e..ec5e7716df8b5 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -1527,6 +1527,7 @@ nfqnl_rcv_nl_event(struct notifier_block *this, if (event == NETLINK_URELEASE && n->protocol == NETLINK_NETFILTER) { int i; + nfnl_lock(NFNL_SUBSYS_QUEUE); /* destroy all instances for this portid */ spin_lock(&q->instances_lock); for (i = 0; i < INSTANCE_BUCKETS; i++) { @@ -1540,6 +1541,7 @@ nfqnl_rcv_nl_event(struct notifier_block *this, } } spin_unlock(&q->instances_lock); + nfnl_unlock(NFNL_SUBSYS_QUEUE); } return NOTIFY_DONE; } @@ -1859,9 +1861,9 @@ static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info, /* Lookup queue under RCU. After peer_portid check (or for new queue * in BIND case), the queue is owned by the socket sending this message. - * A socket cannot simultaneously send a message and close, so while - * processing this CONFIG message, nfqnl_rcv_nl_event() (triggered by - * socket close) cannot destroy this queue. Safe to use without RCU. + * nfqnl_rcv_nl_event() will block on the nfnl subsys mutex that is + * held by the caller, so the queue cannot be destroyed in parallel, + * even after we drop the RCU read lock. */ rcu_read_lock(); queue = instance_lookup(q, queue_num); diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index 03a88c77e0f02..9dbf127df9c81 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -1213,6 +1213,8 @@ struct nft_ct_expect_obj { u8 l4proto; u8 size; u32 timeout; + + struct nf_conntrack_helper *helper; }; static int nft_ct_expect_timeout_get(const struct nlattr *attr, u32 *val) @@ -1226,6 +1228,93 @@ static int nft_ct_expect_timeout_get(const struct nlattr *attr, u32 *val) return 0; } +#if IS_ENABLED(CONFIG_NF_NAT) +static void nft_ct_nat_follow_master(struct nf_conn *ct, struct nf_conntrack_expect *this) +{ + const struct nf_ct_helper_expectfn *expfn; + + expfn = nf_ct_helper_expectfn_find_by_name("nat-follow-master"); + if (expfn) + expfn->expectfn(ct, this); +} +#endif + +struct nft_ct_expect_data { + struct nft_ct_expect_obj obj; + enum ip_conntrack_dir dir; +}; + +static int ct_expect_help(struct sk_buff *skb, unsigned int protoff, + struct nf_conn *ct, enum ip_conntrack_info ctinfo) +{ + enum ip_conntrack_dir dir = CTINFO2DIR(ctinfo); + struct nft_ct_expect_data *expect_data; + struct nf_conntrack_expect *exp; + int ret = NF_ACCEPT; + u16 l3num; + + if (nf_ct_is_confirmed(ct)) + return NF_ACCEPT; + + expect_data = nfct_help_data(ct); + if (!expect_data) + return NF_ACCEPT; + + if (expect_data->dir != dir) + return NF_ACCEPT; + + exp = nf_ct_expect_alloc(ct); + if (!exp) + return NF_DROP; + + if (expect_data->obj.l3num == NFPROTO_INET) + l3num = nf_ct_l3num(ct); + else + l3num = expect_data->obj.l3num; + + nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, l3num, + &ct->tuplehash[!dir].tuple.src.u3, + &ct->tuplehash[!dir].tuple.dst.u3, + expect_data->obj.l4proto, NULL, &expect_data->obj.dport); + exp->timeout += expect_data->obj.timeout; + +#if IS_ENABLED(CONFIG_NF_NAT) + if (ct->status & IPS_NAT_MASK) { + exp->saved_proto.tcp.port = expect_data->obj.dport; + exp->dir = !dir; + exp->expectfn = nft_ct_nat_follow_master; + } +#endif + if (nf_ct_expect_related(exp, 0) != 0) + ret = NF_ACCEPT; + + nf_ct_expect_put(exp); + + return ret; +} + +static int nft_ct_expect_helper_alloc(struct nft_ct_expect_obj *priv) +{ + struct nf_conntrack_helper *ct_expect_helper; + + ct_expect_helper = kzalloc_obj(struct nf_conntrack_helper, + GFP_KERNEL_ACCOUNT); + if (!ct_expect_helper) + return -ENOMEM; + + snprintf(ct_expect_helper->name, sizeof(ct_expect_helper->name), "%s", + "nft_ct_expect"); + ct_expect_helper->me = THIS_MODULE; + ct_expect_helper->expect_policy[NF_CT_EXPECT_CLASS_DEFAULT].max_expected = priv->size; + rcu_assign_pointer(ct_expect_helper->help, ct_expect_help); + refcount_set(&ct_expect_helper->ct_refcnt, 1); + + /* No need to register this helper, this is internal. */ + priv->helper = ct_expect_helper; + + return 0; +} + static int nft_ct_expect_obj_init(const struct nft_ctx *ctx, const struct nlattr * const tb[], struct nft_object *obj) @@ -1233,6 +1322,8 @@ static int nft_ct_expect_obj_init(const struct nft_ctx *ctx, struct nft_ct_expect_obj *priv = nft_obj_data(obj); int err; + NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nft_ct_expect_data)); + if (!tb[NFTA_CT_EXPECT_L4PROTO] || !tb[NFTA_CT_EXPECT_DPORT] || !tb[NFTA_CT_EXPECT_TIMEOUT] || @@ -1272,13 +1363,31 @@ static int nft_ct_expect_obj_init(const struct nft_ctx *ctx, priv->dport = nla_get_be16(tb[NFTA_CT_EXPECT_DPORT]); priv->size = nla_get_u8(tb[NFTA_CT_EXPECT_SIZE]); + if (!priv->size) + priv->size = NF_CT_EXPECT_MAX_CNT; + + err = nf_ct_netns_get(ctx->net, ctx->family); + if (err < 0) + return err; - return nf_ct_netns_get(ctx->net, ctx->family); + err = nft_ct_expect_helper_alloc(priv); + if (err < 0) { + nf_ct_netns_put(ctx->net, ctx->family); + return err; + } + + return err; } static void nft_ct_expect_obj_destroy(const struct nft_ctx *ctx, - struct nft_object *obj) + struct nft_object *obj) { + const struct nft_ct_expect_obj *priv = nft_obj_data(obj); + struct nf_conntrack_helper *me = priv->helper; + + /* This helper is going away, disable it. */ + rcu_assign_pointer(me->help, NULL); + nf_conntrack_helper_release(me); nf_ct_netns_put(ctx->net, ctx->family); } @@ -1302,11 +1411,9 @@ static void nft_ct_expect_obj_eval(struct nft_object *obj, const struct nft_pktinfo *pkt) { const struct nft_ct_expect_obj *priv = nft_obj_data(obj); - struct nf_conntrack_expect *exp; + struct nft_ct_expect_data *expect_data; enum ip_conntrack_info ctinfo; struct nf_conn_help *help; - enum ip_conntrack_dir dir; - u16 l3num = priv->l3num; struct nf_conn *ct; ct = nf_ct_get(pkt->skb, &ctinfo); @@ -1314,38 +1421,30 @@ static void nft_ct_expect_obj_eval(struct nft_object *obj, regs->verdict.code = NFT_BREAK; return; } - dir = CTINFO2DIR(ctinfo); help = nfct_help(ct); - if (!help) - help = nf_ct_helper_ext_add(ct, GFP_ATOMIC); - if (!help) { - regs->verdict.code = NF_DROP; - return; - } - - if (help->expecting[NF_CT_EXPECT_CLASS_DEFAULT] >= priv->size) { + if (help) { regs->verdict.code = NFT_BREAK; return; } - if (l3num == NFPROTO_INET) - l3num = nf_ct_l3num(ct); - exp = nf_ct_expect_alloc(ct); - if (exp == NULL) { + help = nf_ct_helper_ext_add(ct, GFP_ATOMIC); + if (!help) { regs->verdict.code = NF_DROP; return; } - nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, l3num, - &ct->tuplehash[!dir].tuple.src.u3, - &ct->tuplehash[!dir].tuple.dst.u3, - priv->l4proto, NULL, &priv->dport); - exp->timeout += priv->timeout; - if (nf_ct_expect_related(exp, 0) != 0) - regs->verdict.code = NF_DROP; + expect_data = nfct_help_data(ct); + if (!expect_data) { + regs->verdict.code = NFT_BREAK; + return; + } + expect_data->obj = *priv; + expect_data->obj.helper = NULL; + expect_data->dir = CTINFO2DIR(ctinfo); - nf_ct_expect_put(exp); + if (help && refcount_inc_not_zero(&priv->helper->ct_refcnt)) + rcu_assign_pointer(help->helper, priv->helper); } static const struct nla_policy nft_ct_expect_policy[NFTA_CT_EXPECT_MAX + 1] = { @@ -1375,6 +1474,13 @@ static struct nft_object_type nft_ct_expect_obj_type __read_mostly = { .owner = THIS_MODULE, }; +#if IS_ENABLED(CONFIG_NF_NAT) +static struct nf_ct_helper_expectfn nft_ct_nat __read_mostly = { + .name = "nft_ct-follow-master", + .expectfn = nft_ct_nat_follow_master, +}; +#endif + static int __init nft_ct_module_init(void) { int err; @@ -1400,6 +1506,9 @@ static int __init nft_ct_module_init(void) err = nft_register_obj(&nft_ct_timeout_obj_type); if (err < 0) goto err4; +#endif +#if IS_ENABLED(CONFIG_NF_NAT) + nf_ct_helper_expectfn_register(&nft_ct_nat); #endif return 0; @@ -1425,6 +1534,13 @@ static void __exit nft_ct_module_exit(void) nft_unregister_obj(&nft_ct_helper_obj_type); nft_unregister_expr(&nft_notrack_type); nft_unregister_expr(&nft_ct_type); + +#if IS_ENABLED(CONFIG_NF_NAT) + nf_ct_helper_expectfn_unregister(&nft_ct_nat); + synchronize_rcu(); + nf_ct_helper_expectfn_destroy(&nft_ct_nat); + synchronize_rcu(); +#endif } module_init(nft_ct_module_init); diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c index ee9d3e7b1ecf1..fa4da694e92d8 100644 --- a/net/netfilter/nft_dynset.c +++ b/net/netfilter/nft_dynset.c @@ -19,7 +19,8 @@ struct nft_dynset { u8 sreg_key; u8 sreg_data; bool invert; - bool expr; + bool expr:1, + override_exprs:1; u8 num_exprs; u64 timeout; struct nft_expr *expr_array[NFT_SET_EXPR_MAX]; @@ -257,6 +258,7 @@ static int nft_dynset_init(const struct nft_ctx *ctx, priv->num_exprs++; priv->expr_array[0] = dynset_expr; + priv->override_exprs = true; if (set->num_exprs > 1 || (set->num_exprs == 1 && @@ -289,6 +291,7 @@ static int nft_dynset_init(const struct nft_ctx *ctx, } priv->expr_array[i] = dynset_expr; priv->num_exprs++; + priv->override_exprs = true; if (set->num_exprs) { if (i >= set->num_exprs) { @@ -307,9 +310,8 @@ static int nft_dynset_init(const struct nft_ctx *ctx, goto err_expr_free; } } else if (set->num_exprs > 0) { - err = nft_set_elem_expr_clone(ctx, set, priv->expr_array); - if (err < 0) - return err; + for (i = 0; i < set->num_exprs; i++) + priv->expr_array[i] = set->exprs[i]; priv->num_exprs = set->num_exprs; } @@ -339,8 +341,10 @@ static int nft_dynset_init(const struct nft_ctx *ctx, return 0; err_expr_free: - for (i = 0; i < priv->num_exprs; i++) - nft_expr_destroy(ctx, priv->expr_array[i]); + if (priv->override_exprs) { + for (i = 0; i < priv->num_exprs; i++) + nft_expr_destroy(ctx, priv->expr_array[i]); + } return err; } @@ -367,9 +371,10 @@ static void nft_dynset_destroy(const struct nft_ctx *ctx, struct nft_dynset *priv = nft_expr_priv(expr); int i; - for (i = 0; i < priv->num_exprs; i++) - nft_expr_destroy(ctx, priv->expr_array[i]); - + if (priv->override_exprs) { + for (i = 0; i < priv->num_exprs; i++) + nft_expr_destroy(ctx, priv->expr_array[i]); + } nf_tables_destroy_set(ctx, priv->set); } @@ -393,7 +398,7 @@ static int nft_dynset_dump(struct sk_buff *skb, nf_jiffies64_to_msecs(priv->timeout), NFTA_DYNSET_PAD)) goto nla_put_failure; - if (priv->set->num_exprs == 0) { + if (priv->set->num_exprs == 0 || priv->override_exprs) { if (priv->num_exprs == 1) { if (nft_expr_dump(skb, NFTA_DYNSET_EXPR, priv->expr_array[0], reset)) diff --git a/net/netfilter/nft_nat.c b/net/netfilter/nft_nat.c index e32cd9fbc7c2e..cdbd800cac969 100644 --- a/net/netfilter/nft_nat.c +++ b/net/netfilter/nft_nat.c @@ -64,8 +64,8 @@ static void nft_nat_setup_netmap(struct nf_nat_range2 *range, const struct nft_pktinfo *pkt, const struct nft_nat *priv) { + union nf_inet_addr new_addr = {}; struct sk_buff *skb = pkt->skb; - union nf_inet_addr new_addr; __be32 netmask; int i, len = 0; diff --git a/net/netfilter/nft_set_pipapo_avx2.c b/net/netfilter/nft_set_pipapo_avx2.c index b3f105520a85b..21f5be68c7036 100644 --- a/net/netfilter/nft_set_pipapo_avx2.c +++ b/net/netfilter/nft_set_pipapo_avx2.c @@ -1134,6 +1134,7 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m, struct nft_pipapo_scratch *scratch; const struct nft_pipapo_field *f; unsigned long *res, *fill, *map; + struct nft_pipapo_elem *e; bool map_index; int ret = 0; int i; @@ -1207,14 +1208,11 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m, next_match: if (ret < 0) { scratch->map_index = map_index; - kernel_fpu_end(); - __local_unlock_nested_bh(&scratch->bh_lock); - return NULL; + e = NULL; + goto out; } if (last) { - struct nft_pipapo_elem *e; - e = f->mt[ret].e; if (unlikely(__nft_set_elem_expired(&e->ext, tstamp) || !nft_set_elem_active(&e->ext, genmask))) { @@ -1224,9 +1222,7 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m, } scratch->map_index = map_index; - kernel_fpu_end(); - __local_unlock_nested_bh(&scratch->bh_lock); - return e; + goto out; } map_index = !map_index; @@ -1234,9 +1230,12 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m, data += NFT_PIPAPO_GROUPS_PADDED_SIZE(f); } + e = NULL; +out: + asm volatile("vzeroupper"); kernel_fpu_end(); __local_unlock_nested_bh(&scratch->bh_lock); - return NULL; + return e; } /** diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c index 9ed288c9d1688..554a96a000f40 100644 --- a/net/netfilter/nft_synproxy.c +++ b/net/netfilter/nft_synproxy.c @@ -118,7 +118,8 @@ static void nft_synproxy_do_eval(const struct nft_synproxy *priv, return; } - if (nf_ip_checksum(skb, nft_hook(pkt), thoff, IPPROTO_TCP)) { + if (nf_checksum(skb, nft_hook(pkt), thoff, IPPROTO_TCP, + nft_pf(pkt))) { regs->verdict.code = NF_DROP; return; } diff --git a/net/netfilter/xt_cgroup.c b/net/netfilter/xt_cgroup.c index 43d2ae2be628d..28e6cd51b2fb0 100644 --- a/net/netfilter/xt_cgroup.c +++ b/net/netfilter/xt_cgroup.c @@ -23,7 +23,7 @@ MODULE_DESCRIPTION("Xtables: process control group matching"); MODULE_ALIAS("ipt_cgroup"); MODULE_ALIAS("ip6t_cgroup"); -#define NET_CLS_CLASSID_INVALID_MSG "xt_cgroup: classid invalid without net_cls cgroups\n" +#define NET_CLS_CLASSID_INVALID_MSG "classid invalid without net_cls cgroups\n" static int cgroup_mt_check_v0(const struct xt_mtchk_param *par) { @@ -33,7 +33,7 @@ static int cgroup_mt_check_v0(const struct xt_mtchk_param *par) return -EINVAL; if (!IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } @@ -49,7 +49,7 @@ static int cgroup_mt_check_v1(const struct xt_mtchk_param *par) return -EINVAL; if (!info->has_path && !info->has_classid) { - pr_info("xt_cgroup: no path or classid specified\n"); + pr_info_ratelimited("no path or classid specified\n"); return -EINVAL; } @@ -59,7 +59,7 @@ static int cgroup_mt_check_v1(const struct xt_mtchk_param *par) } if (info->has_classid && !IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } @@ -89,7 +89,7 @@ static int cgroup_mt_check_v2(const struct xt_mtchk_param *par) return -EINVAL; if (!info->has_path && !info->has_classid) { - pr_info("xt_cgroup: no path or classid specified\n"); + pr_info_ratelimited("no path or classid specified\n"); return -EINVAL; } @@ -99,7 +99,7 @@ static int cgroup_mt_check_v2(const struct xt_mtchk_param *par) } if (info->has_classid && !IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } diff --git a/net/netfilter/xt_hl.c b/net/netfilter/xt_hl.c index 4a12a757ecbf8..59e93d97b5072 100644 --- a/net/netfilter/xt_hl.c +++ b/net/netfilter/xt_hl.c @@ -28,7 +28,7 @@ static int ttl_mt_check(const struct xt_mtchk_param *par) const struct ipt_ttl_info *info = par->matchinfo; if (info->mode > IPT_TTL_GT) { - pr_err("Unknown TTL match mode: %d\n", info->mode); + pr_info_ratelimited("Unknown TTL match mode: %d\n", info->mode); return -EINVAL; } @@ -59,7 +59,7 @@ static int hl_mt6_check(const struct xt_mtchk_param *par) const struct ip6t_hl_info *info = par->matchinfo; if (info->mode > IP6T_HL_GT) { - pr_err("Unknown Hop Limit match mode: %d\n", info->mode); + pr_info_ratelimited("Unknown Hop Limit match mode: %d\n", info->mode); return -EINVAL; } diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 5202fe0b08671..7288de715341a 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -922,9 +922,9 @@ netlink_update_subscriptions(struct sock *sk, unsigned int subscriptions) static int netlink_realloc_groups(struct sock *sk) { + unsigned long *new_groups, *old_groups = NULL; struct netlink_sock *nlk = nlk_sk(sk); unsigned int groups; - unsigned long *new_groups; int err = 0; netlink_table_grab(); @@ -938,18 +938,37 @@ static int netlink_realloc_groups(struct sock *sk) if (nlk->ngroups >= groups) goto out_unlock; - new_groups = krealloc(nlk->groups, NLGRPSZ(groups), GFP_ATOMIC); - if (new_groups == NULL) { + /* Can not use krealloc(), because the old buffer might be freed + * immediately, while lockless readers (netlink diag dump and + * /proc/net/netlink) can still be looking at it. + */ + new_groups = kzalloc(NLGRPSZ(groups), GFP_ATOMIC); + if (!new_groups) { err = -ENOMEM; goto out_unlock; } - memset((char *)new_groups + NLGRPSZ(nlk->ngroups), 0, - NLGRPSZ(groups) - NLGRPSZ(nlk->ngroups)); + old_groups = nlk->groups; + if (old_groups) + memcpy(new_groups, old_groups, NLGRPSZ(nlk->ngroups)); + + /* Publish the new bitmap and its content: pairs with the address + * dependency in lockless readers, which can pick up the new pointer + * while still seeing the old (smaller) nlk->ngroups. + */ + smp_store_release(&nlk->groups, new_groups); + + /* Then publish the new size: pairs with smp_load_acquire() from + * lockless readers, so that they can not read NLGRPSZ(new ngroups) + * bytes from the old buffer. + */ + smp_store_release(&nlk->ngroups, groups); - nlk->groups = new_groups; - nlk->ngroups = groups; out_unlock: netlink_table_ungrab(); + + if (old_groups) + kfree_rcu_mightsleep(old_groups); + return err; } @@ -2705,12 +2724,19 @@ static int netlink_native_seq_show(struct seq_file *seq, void *v) } else { struct sock *s = v; struct netlink_sock *nlk = nlk_sk(s); + const unsigned long *groups; + + /* Lockless read : netlink_realloc_groups() can change + * nlk->groups under us. The old buffer is freed after an + * RCU grace period, and this walk is RCU protected. + */ + groups = READ_ONCE(nlk->groups); seq_printf(seq, "%pK %-3d %-10u %08x %-8d %-8d %-5d %-8d %-8u %-8llu\n", s, s->sk_protocol, nlk->portid, - nlk->groups ? (u32)nlk->groups[0] : 0, + groups ? (u32)groups[0] : 0, sk_rmem_alloc_get(s), sk_wmem_alloc_get(s), READ_ONCE(nlk->cb_running), diff --git a/net/netlink/diag.c b/net/netlink/diag.c index 0b3e021bd0ed2..7979bd9b26060 100644 --- a/net/netlink/diag.c +++ b/net/netlink/diag.c @@ -12,12 +12,24 @@ static int sk_diag_dump_groups(struct sock *sk, struct sk_buff *nlskb) { struct netlink_sock *nlk = nlk_sk(sk); - - if (nlk->groups == NULL) + unsigned long *groups; + unsigned int ngroups; + + /* Hashed sockets are dumped from the rhashtable walk, which only + * holds rcu_read_lock(), while netlink_realloc_groups() can replace + * nlk->groups and nlk->ngroups at any time. + * + * Read nlk->ngroups first : this pairs with smp_store_release() + * from netlink_realloc_groups(), so that we can not use the new + * (bigger) size with the old (smaller) buffer. The old buffer is + * freed after an RCU grace period. + */ + ngroups = smp_load_acquire(&nlk->ngroups); + groups = READ_ONCE(nlk->groups); + if (!groups) return 0; - return nla_put(nlskb, NETLINK_DIAG_GROUPS, NLGRPSZ(nlk->ngroups), - nlk->groups); + return nla_put(nlskb, NETLINK_DIAG_GROUPS, NLGRPSZ(ngroups), groups); } static int sk_diag_put_flags(struct sock *sk, struct sk_buff *skb) diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c index 0da39eaed255f..5cc1037d4917e 100644 --- a/net/netlink/genetlink.c +++ b/net/netlink/genetlink.c @@ -1513,6 +1513,7 @@ struct ctrl_dump_policy_ctx { struct netlink_policy_dump_state *state; const struct genl_family *rt; struct genl_op_iter *op_iter; + struct module *owner; u32 op; u16 fam_id; u8 dump_map:1, @@ -1555,6 +1556,9 @@ static int ctrl_dumppolicy_start(struct netlink_callback *cb) return -ENOENT; ctx->rt = rt; + ctx->owner = rt->module; + if (!try_module_get(ctx->owner)) + return -ENOENT; if (tb[CTRL_ATTR_OP]) { struct genl_split_ops doit, dump; @@ -1565,7 +1569,7 @@ static int ctrl_dumppolicy_start(struct netlink_callback *cb) err = genl_get_cmd_both(ctx->op, rt, &doit, &dump); if (err) { NL_SET_BAD_ATTR(cb->extack, tb[CTRL_ATTR_OP]); - return err; + goto err_put_owner; } if (doit.policy) { @@ -1583,16 +1587,20 @@ static int ctrl_dumppolicy_start(struct netlink_callback *cb) goto err_free_state; } - if (!ctx->state) - return -ENODATA; + if (!ctx->state) { + err = -ENODATA; + goto err_put_owner; + } ctx->dump_map = 1; return 0; } ctx->op_iter = kmalloc_obj(*ctx->op_iter); - if (!ctx->op_iter) - return -ENOMEM; + if (!ctx->op_iter) { + err = -ENOMEM; + goto err_put_owner; + } genl_op_iter_init(rt, ctx->op_iter); ctx->dump_map = genl_op_iter_next(ctx->op_iter); @@ -1624,6 +1632,8 @@ static int ctrl_dumppolicy_start(struct netlink_callback *cb) netlink_policy_dump_free(ctx->state); err_free_op_iter: kfree(ctx->op_iter); +err_put_owner: + module_put(ctx->owner); return err; } @@ -1646,7 +1656,7 @@ static void *ctrl_dumppolicy_prep(struct sk_buff *skb, } static int ctrl_dumppolicy_put_op(struct sk_buff *skb, - struct netlink_callback *cb, + struct netlink_callback *cb, u32 cmd, struct genl_split_ops *doit, struct genl_split_ops *dumpit) { @@ -1667,7 +1677,7 @@ static int ctrl_dumppolicy_put_op(struct sk_buff *skb, if (!nest_pol) goto err; - nest_op = nla_nest_start(skb, doit->cmd); + nest_op = nla_nest_start(skb, cmd); if (!nest_op) goto err; @@ -1711,7 +1721,8 @@ static int ctrl_dumppolicy(struct sk_buff *skb, struct netlink_callback *cb) &doit, &dumpit))) return -ENOENT; - if (ctrl_dumppolicy_put_op(skb, cb, &doit, &dumpit)) + if (ctrl_dumppolicy_put_op(skb, cb, ctx->op, + &doit, &dumpit)) return skb->len; /* done with the per-op policy index list */ @@ -1720,6 +1731,7 @@ static int ctrl_dumppolicy(struct sk_buff *skb, struct netlink_callback *cb) while (ctx->dump_map) { if (ctrl_dumppolicy_put_op(skb, cb, + ctx->op_iter->cmd, &ctx->op_iter->doit, &ctx->op_iter->dumpit)) return skb->len; @@ -1760,6 +1772,7 @@ static int ctrl_dumppolicy_done(struct netlink_callback *cb) kfree(ctx->op_iter); netlink_policy_dump_free(ctx->state); + module_put(ctx->owner); return 0; } diff --git a/net/nfc/core.c b/net/nfc/core.c index a92a6566e6a0d..f521669293f05 100644 --- a/net/nfc/core.c +++ b/net/nfc/core.c @@ -279,10 +279,10 @@ static struct nfc_target *nfc_find_target(struct nfc_dev *dev, u32 target_idx) int nfc_dep_link_up(struct nfc_dev *dev, int target_index, u8 comm_mode) { - int rc = 0; - u8 *gb; - size_t gb_len; struct nfc_target *target; + u8 gb[NFC_MAX_GT_LEN]; + size_t gb_len = 0; + int rc = 0; pr_debug("dev_name=%s comm %d\n", dev_name(&dev->dev), comm_mode); @@ -301,7 +301,7 @@ int nfc_dep_link_up(struct nfc_dev *dev, int target_index, u8 comm_mode) goto error; } - gb = nfc_llcp_general_bytes(dev, &gb_len); + nfc_get_local_general_bytes(dev, gb, sizeof(gb), &gb_len); if (gb_len > NFC_MAX_GT_LEN) { rc = -EINVAL; goto error; @@ -644,11 +644,10 @@ int nfc_set_remote_general_bytes(struct nfc_dev *dev, const u8 *gb, u8 gb_len) } EXPORT_SYMBOL(nfc_set_remote_general_bytes); -u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, size_t *gb_len) +u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, u8 *out_gb, + size_t gb_max_len, size_t *gb_len) { - pr_debug("dev_name=%s\n", dev_name(&dev->dev)); - - return nfc_llcp_general_bytes(dev, gb_len); + return nfc_llcp_general_bytes(dev, out_gb, gb_max_len, gb_len); } EXPORT_SYMBOL(nfc_get_local_general_bytes); diff --git a/net/nfc/digital_core.c b/net/nfc/digital_core.c index 7cb1e6aaae90d..18236221d8988 100644 --- a/net/nfc/digital_core.c +++ b/net/nfc/digital_core.c @@ -127,7 +127,7 @@ static void digital_wq_cmd_complete(struct work_struct *work) mutex_unlock(&ddev->cmd_lock); - if (!IS_ERR(cmd->resp)) + if (!IS_ERR_OR_NULL(cmd->resp)) print_hex_dump_debug("DIGITAL RX: ", DUMP_PREFIX_NONE, 16, 1, cmd->resp->data, cmd->resp->len, false); diff --git a/net/nfc/digital_dep.c b/net/nfc/digital_dep.c index 3982fa084737f..968547c306a5d 100644 --- a/net/nfc/digital_dep.c +++ b/net/nfc/digital_dep.c @@ -1490,14 +1490,14 @@ static int digital_tg_send_atr_res(struct nfc_digital_dev *ddev, struct digital_atr_req *atr_req) { struct digital_atr_res *atr_res; + u8 gb[NFC_MAX_GT_LEN]; struct sk_buff *skb; - u8 *gb, payload_bits; + u8 payload_bits; size_t gb_len; int rc; - gb = nfc_get_local_general_bytes(ddev->nfc_dev, &gb_len); - if (!gb) - gb_len = 0; + nfc_get_local_general_bytes(ddev->nfc_dev, gb, sizeof(gb), + &gb_len); skb = digital_skb_alloc(ddev, sizeof(struct digital_atr_res) + gb_len); if (!skb) diff --git a/net/nfc/digital_technology.c b/net/nfc/digital_technology.c index ae63c5eb06faa..ae6487c10a254 100644 --- a/net/nfc/digital_technology.c +++ b/net/nfc/digital_technology.c @@ -778,6 +778,8 @@ static void digital_in_recv_sensf_res(struct nfc_digital_dev *ddev, void *arg, sensf_res = (struct digital_sensf_res *)resp->data; + resp->len = min_t(unsigned int, resp->len, NFC_SENSF_RES_MAXSIZE); + memcpy(target.sensf_res, sensf_res, resp->len); target.sensf_res_len = resp->len; diff --git a/net/nfc/llcp.h b/net/nfc/llcp.h index d8345ed57c954..23ae7a0112d37 100644 --- a/net/nfc/llcp.h +++ b/net/nfc/llcp.h @@ -91,6 +91,7 @@ struct nfc_llcp_local { struct hlist_head pending_sdreqs; struct timer_list sdreq_timer; struct work_struct sdreq_timeout_work; + struct work_struct release_work; u8 sdreq_next_tid; /* sockets array */ diff --git a/net/nfc/llcp_commands.c b/net/nfc/llcp_commands.c index 291f26facbf3a..80a00938c8696 100644 --- a/net/nfc/llcp_commands.c +++ b/net/nfc/llcp_commands.c @@ -135,7 +135,7 @@ struct nfc_llcp_sdp_tlv *nfc_llcp_build_sdreq_tlv(u8 tid, const char *uri, { struct nfc_llcp_sdp_tlv *sdreq; - pr_debug("uri: %s, len: %zu\n", uri, uri_len); + pr_debug("uri: %.*s, len: %zu\n", (int)uri_len, uri, uri_len); /* sdreq->tlv_len is u8, takes uri_len, + 3 for header, + 1 for NULL */ if (WARN_ON_ONCE(uri_len > U8_MAX - 4)) @@ -193,7 +193,8 @@ int nfc_llcp_parse_gb_tlv(struct nfc_llcp_local *local, const u8 *tlv_array, u16 tlv_array_len) { const u8 *tlv = tlv_array; - u8 type, length, offset = 0; + u8 type, length; + u16 offset = 0; pr_debug("TLV array length %d\n", tlv_array_len); @@ -201,9 +202,15 @@ int nfc_llcp_parse_gb_tlv(struct nfc_llcp_local *local, return -ENODEV; while (offset < tlv_array_len) { + if (offset + 2 > tlv_array_len) + return -EINVAL; + type = tlv[0]; length = tlv[1]; + if (offset + 2 + length > tlv_array_len) + return -EINVAL; + pr_debug("type 0x%x length %d\n", type, length); switch (type) { @@ -243,7 +250,8 @@ int nfc_llcp_parse_connection_tlv(struct nfc_llcp_sock *sock, const u8 *tlv_array, u16 tlv_array_len) { const u8 *tlv = tlv_array; - u8 type, length, offset = 0; + u8 type, length; + u16 offset = 0; pr_debug("TLV array length %d\n", tlv_array_len); @@ -251,9 +259,15 @@ int nfc_llcp_parse_connection_tlv(struct nfc_llcp_sock *sock, return -ENOTCONN; while (offset < tlv_array_len) { + if (offset + 2 > tlv_array_len) + return -EINVAL; + type = tlv[0]; length = tlv[1]; + if (offset + 2 + length > tlv_array_len) + return -EINVAL; + pr_debug("type 0x%x length %d\n", type, length); switch (type) { diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c index dc65c719f35f2..74bf817007cf2 100644 --- a/net/nfc/llcp_core.c +++ b/net/nfc/llcp_core.c @@ -20,6 +20,8 @@ static LIST_HEAD(llcp_devices); /* Protects llcp_devices list */ static DEFINE_SPINLOCK(llcp_devices_lock); +static struct workqueue_struct *llcp_wq; + static void nfc_llcp_rx_skb(struct nfc_llcp_local *local, struct sk_buff *skb); void nfc_llcp_sock_link(struct llcp_sock_list *l, struct sock *sk) @@ -63,21 +65,33 @@ static void nfc_llcp_socket_purge(struct nfc_llcp_sock *sock) } } +static struct sock *nfc_llcp_sock_list_pop(struct llcp_sock_list *l) +{ + struct sock *sk; + + write_lock(&l->lock); + sk = sk_head(&l->head); + if (sk) { + sock_hold(sk); + sk_del_node_init(sk); + } + write_unlock(&l->lock); + + return sk; +} + static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device, int err) { struct sock *sk; - struct hlist_node *tmp; struct nfc_llcp_sock *llcp_sock; skb_queue_purge(&local->tx_queue); - write_lock(&local->sockets.lock); - - sk_for_each_safe(sk, tmp, &local->sockets.head) { + while ((sk = nfc_llcp_sock_list_pop(&local->sockets))) { llcp_sock = nfc_llcp_sock(sk); - bh_lock_sock(sk); + lock_sock(sk); nfc_llcp_socket_purge(llcp_sock); @@ -91,17 +105,27 @@ static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device, list_for_each_entry_safe(lsk, n, &llcp_sock->accept_queue, accept_queue) { - accept_sk = &lsk->sk; - bh_lock_sock(accept_sk); - - nfc_llcp_accept_unlink(accept_sk); + bool put_creation = false; - if (err) - accept_sk->sk_err = err; - accept_sk->sk_state = LLCP_CLOSED; - accept_sk->sk_state_change(sk); + accept_sk = &lsk->sk; + lock_sock_nested(accept_sk, + SINGLE_DEPTH_NESTING); + + if (nfc_llcp_sock(accept_sk)->parent == sk) { + nfc_llcp_accept_unlink(accept_sk); + nfc_llcp_sock_unlink(&local->sockets, accept_sk); + + if (err) + accept_sk->sk_err = err; + accept_sk->sk_state = LLCP_CLOSED; + accept_sk->sk_state_change(accept_sk); + sock_orphan(accept_sk); + put_creation = true; + } - bh_unlock_sock(accept_sk); + release_sock(accept_sk); + if (put_creation) + sock_put(accept_sk); /* creation ref */ } } @@ -110,23 +134,18 @@ static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device, sk->sk_state = LLCP_CLOSED; sk->sk_state_change(sk); - bh_unlock_sock(sk); - - sk_del_node_init(sk); + release_sock(sk); + sock_put(sk); } - write_unlock(&local->sockets.lock); - /* If we still have a device, we keep the RAW sockets alive */ if (device == true) return; - write_lock(&local->raw_sockets.lock); - - sk_for_each_safe(sk, tmp, &local->raw_sockets.head) { + while ((sk = nfc_llcp_sock_list_pop(&local->raw_sockets))) { llcp_sock = nfc_llcp_sock(sk); - bh_lock_sock(sk); + lock_sock(sk); nfc_llcp_socket_purge(llcp_sock); @@ -135,26 +154,20 @@ static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device, sk->sk_state = LLCP_CLOSED; sk->sk_state_change(sk); - bh_unlock_sock(sk); - - sk_del_node_init(sk); + release_sock(sk); + sock_put(sk); } - - write_unlock(&local->raw_sockets.lock); } static struct nfc_llcp_local *nfc_llcp_local_get(struct nfc_llcp_local *local) { - /* Since using nfc_llcp_local may result in usage of nfc_dev, whenever - * we hold a reference to local, we also need to hold a reference to - * the device to avoid UAF. - */ - if (!nfc_get_device(local->dev->idx)) + if (!local) return NULL; - kref_get(&local->ref); + if (kref_get_unless_zero(&local->ref)) + return local; - return local; + return NULL; } static void local_cleanup(struct nfc_llcp_local *local) @@ -172,30 +185,34 @@ static void local_cleanup(struct nfc_llcp_local *local) nfc_llcp_free_sdp_tlv_list(&local->pending_sdreqs); } -static void local_release(struct kref *ref) +static void local_release_work(struct work_struct *work) { struct nfc_llcp_local *local; + struct nfc_dev *dev; - local = container_of(ref, struct nfc_llcp_local, ref); + local = container_of(work, struct nfc_llcp_local, release_work); + dev = local->dev; local_cleanup(local); kfree(local); + nfc_put_device(dev); } -int nfc_llcp_local_put(struct nfc_llcp_local *local) +static void local_release(struct kref *ref) { - struct nfc_dev *dev; - int ret; + struct nfc_llcp_local *local; - if (local == NULL) - return 0; + local = container_of(ref, struct nfc_llcp_local, ref); - dev = local->dev; + queue_work(llcp_wq, &local->release_work); +} - ret = kref_put(&local->ref, local_release); - nfc_put_device(dev); +int nfc_llcp_local_put(struct nfc_llcp_local *local) +{ + if (!local) + return 0; - return ret; + return kref_put(&local->ref, local_release); } static struct nfc_llcp_sock *nfc_llcp_sock_get(struct nfc_llcp_local *local, @@ -341,7 +358,7 @@ static int nfc_llcp_wks_sap(const char *service_name, size_t service_name_len) { int sap, num_wks; - pr_debug("%s\n", service_name); + pr_debug("%.*s\n", (int)service_name_len, service_name); if (service_name == NULL) return -EINVAL; @@ -352,7 +369,8 @@ static int nfc_llcp_wks_sap(const char *service_name, size_t service_name_len) if (wks[sap] == NULL) continue; - if (strncmp(wks[sap], service_name, service_name_len) == 0) + if (strlen(wks[sap]) == service_name_len && + !strncmp(wks[sap], service_name, service_name_len)) return sap; } @@ -635,23 +653,32 @@ static int nfc_llcp_build_gb(struct nfc_llcp_local *local) return ret; } -u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, size_t *general_bytes_len) +u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, u8 *out_gb, size_t gb_max_len, + size_t *general_bytes_len) { struct nfc_llcp_local *local; + if (!out_gb || !general_bytes_len) + return NULL; + local = nfc_llcp_find_local(dev); - if (local == NULL) { + if (!local) { *general_bytes_len = 0; return NULL; } nfc_llcp_build_gb(local); - *general_bytes_len = local->gb_len; + if (local->gb_len) { + *general_bytes_len = min_t(size_t, local->gb_len, gb_max_len); + memcpy(out_gb, local->gb, *general_bytes_len); + } else { + *general_bytes_len = 0; + } nfc_llcp_local_put(local); - return local->gb; + return out_gb; } int nfc_llcp_set_remote_gb(struct nfc_dev *dev, const u8 *gb, u8 gb_len) @@ -849,13 +876,16 @@ static struct nfc_llcp_sock *nfc_llcp_sock_get_sn(struct nfc_llcp_local *local, static const u8 *nfc_llcp_connect_sn(const struct sk_buff *skb, size_t *sn_len) { u8 type, length; - const u8 *tlv = &skb->data[2]; - size_t tlv_array_len = skb->len - LLCP_HEADER_SIZE, offset = 0; + const u8 *tlv = &skb->data[LLCP_HEADER_SIZE]; + const u8 *tlv_end = skb_tail_pointer(skb); - while (offset < tlv_array_len) { + while (tlv + 2 < tlv_end) { type = tlv[0]; length = tlv[1]; + if (tlv + 2 + length > tlv_end) + break; + pr_debug("type 0x%x length %d\n", type, length); if (type == LLCP_TLV_SN) { @@ -863,7 +893,6 @@ static const u8 *nfc_llcp_connect_sn(const struct sk_buff *skb, size_t *sn_len) return &tlv[2]; } - offset += length + 2; tlv += length + 2; } @@ -1072,6 +1101,9 @@ static void nfc_llcp_recv_hdlc(struct nfc_llcp_local *local, struct sock *sk; u8 dsap, ssap, ptype, ns, nr; + if (!pskb_may_pull(skb, LLCP_HEADER_SIZE + LLCP_SEQUENCE_SIZE)) + return; + ptype = nfc_llcp_ptype(skb); dsap = nfc_llcp_dsap(skb); ssap = nfc_llcp_ssap(skb); @@ -1249,6 +1281,7 @@ static void nfc_llcp_recv_dm(struct nfc_llcp_local *local, struct nfc_llcp_sock *llcp_sock; struct sock *sk; u8 dsap, ssap, reason; + bool connecting = false; dsap = nfc_llcp_dsap(skb); ssap = nfc_llcp_ssap(skb); @@ -1260,6 +1293,7 @@ static void nfc_llcp_recv_dm(struct nfc_llcp_local *local, case LLCP_DM_NOBOUND: case LLCP_DM_REJ: llcp_sock = nfc_llcp_connecting_sock_get(local, dsap); + connecting = true; break; default: @@ -1274,10 +1308,33 @@ static void nfc_llcp_recv_dm(struct nfc_llcp_local *local, sk = &llcp_sock->sk; + lock_sock(sk); + + /* Check if socket was destroyed whilst waiting for the lock */ + if (!sk_hashed(sk)) { + release_sock(sk); + nfc_llcp_sock_put(llcp_sock); + return; + } + + /* + * For DM(NOBOUND)/DM(REJ) the socket is still linked on the + * connecting_sockets list. Unlink it here, under the socket lock, + * before moving it to LLCP_CLOSED: llcp_sock_release() selects the + * list to unlink from by sk_state, so leaving a connecting socket + * in the CLOSED state would make it unlink from the wrong list and + * corrupt the connecting_sockets list / desync the socket refcount. + * This mirrors nfc_llcp_recv_cc(). + */ + if (connecting) + nfc_llcp_sock_unlink(&local->connecting_sockets, sk); + sk->sk_err = ENXIO; sk->sk_state = LLCP_CLOSED; sk->sk_state_change(sk); + release_sock(sk); + nfc_llcp_sock_put(llcp_sock); } @@ -1286,10 +1343,9 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, { struct nfc_llcp_sock *llcp_sock; u8 dsap, ssap, type, length, tid, sap; - const u8 *tlv; - u16 tlv_len, offset; + const u8 *tlv, *tlv_end; const char *service_name; - size_t service_name_len; + int service_name_len; struct nfc_llcp_sdp_tlv *sdp; HLIST_HEAD(llc_sdres_list); size_t sdres_tlvs_len; @@ -1305,22 +1361,34 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, return; } + /* + * Walk the SNL TLV list in the linear part of the skb only, + * bounded by skb_tail_pointer(). Each TLV needs a two-byte + * header (type, length) and its declared length must fit before + * the end; this also keeps the walk safe for very short frames. + */ tlv = &skb->data[LLCP_HEADER_SIZE]; - tlv_len = skb->len - LLCP_HEADER_SIZE; - offset = 0; + tlv_end = skb_tail_pointer(skb); sdres_tlvs_len = 0; - while (offset < tlv_len) { + while (tlv + 2 < tlv_end) { type = tlv[0]; length = tlv[1]; + if (tlv + 2 + length > tlv_end) + break; + switch (type) { case LLCP_TLV_SDREQ: + if (length < 1) + break; + tid = tlv[2]; service_name = (char *) &tlv[3]; service_name_len = length - 1; - pr_debug("Looking for %.16s\n", service_name); + pr_debug("Looking for %.*s\n", service_name_len, + service_name); if (service_name_len == strlen("urn:nfc:sn:sdp") && !strncmp(service_name, "urn:nfc:sn:sdp", @@ -1380,6 +1448,9 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, break; case LLCP_TLV_SDRES: + if (length != 2) + break; + mutex_lock(&local->sdreq_lock); pr_debug("LLCP_TLV_SDRES: searching tid %d\n", tlv[2]); @@ -1408,7 +1479,6 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, break; } - offset += length + 2; tlv += length + 2; } @@ -1552,6 +1622,11 @@ static void nfc_llcp_rx_work(struct work_struct *work) static void __nfc_llcp_recv(struct nfc_llcp_local *local, struct sk_buff *skb) { + if (!pskb_may_pull(skb, LLCP_HEADER_SIZE)) { + kfree_skb(skb); + return; + } + local->rx_pending = skb; timer_delete(&local->link_timer); schedule_work(&local->rx_work); @@ -1660,6 +1735,7 @@ int nfc_llcp_register_device(struct nfc_dev *ndev) INIT_WORK(&local->rx_work, nfc_llcp_rx_work); INIT_WORK(&local->timeout_work, nfc_llcp_timeout_work); + INIT_WORK(&local->release_work, local_release_work); rwlock_init(&local->sockets.lock); rwlock_init(&local->connecting_sockets.lock); @@ -1703,10 +1779,23 @@ void nfc_llcp_unregister_device(struct nfc_dev *dev) int __init nfc_llcp_init(void) { - return nfc_llcp_sock_init(); + int ret; + + llcp_wq = alloc_workqueue("nfc_llcp_wq", WQ_UNBOUND, 0); + if (!llcp_wq) + return -ENOMEM; + + ret = nfc_llcp_sock_init(); + if (ret) { + destroy_workqueue(llcp_wq); + return ret; + } + + return 0; } void nfc_llcp_exit(void) { nfc_llcp_sock_exit(); + destroy_workqueue(llcp_wq); } diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c index feab29fc62f44..1e5ee4bcde684 100644 --- a/net/nfc/llcp_sock.c +++ b/net/nfc/llcp_sock.c @@ -319,14 +319,22 @@ static int nfc_llcp_getsockopt(struct socket *sock, int level, int optname, if (get_user(len, optlen)) return -EFAULT; - local = llcp_sock->local; - if (!local) - return -ENODEV; + if (len < 0) + return -EINVAL; + + if (len < sizeof(u32)) + return -EINVAL; len = min_t(u32, len, sizeof(u32)); lock_sock(sk); + local = llcp_sock->local; + if (!local) { + release_sock(sk); + return -ENODEV; + } + switch (optname) { case NFC_LLCP_RW: rw = llcp_sock->rw > LLCP_MAX_RW ? local->rw : llcp_sock->rw; @@ -384,11 +392,12 @@ void nfc_llcp_accept_unlink(struct sock *sk) pr_debug("state %d\n", sk->sk_state); - list_del_init(&llcp_sock->accept_queue); - sk_acceptq_removed(llcp_sock->parent); - llcp_sock->parent = NULL; - - sock_put(sk); + if (llcp_sock->parent) { + list_del_init(&llcp_sock->accept_queue); + sk_acceptq_removed(llcp_sock->parent); + llcp_sock->parent = NULL; + sock_put(sk); + } } void nfc_llcp_accept_enqueue(struct sock *parent, struct sock *sk) @@ -415,12 +424,20 @@ struct sock *nfc_llcp_accept_dequeue(struct sock *parent, list_for_each_entry_safe(lsk, n, &llcp_parent->accept_queue, accept_queue) { + struct nfc_llcp_local *local; + sk = &lsk->sk; - lock_sock(sk); + lock_sock_nested(sk, SINGLE_DEPTH_NESTING); if (sk->sk_state == LLCP_CLOSED) { - release_sock(sk); + local = nfc_llcp_sock(sk)->local; + nfc_llcp_accept_unlink(sk); + if (local) + nfc_llcp_sock_unlink(&local->sockets, sk); + sock_orphan(sk); + release_sock(sk); + sock_put(sk); continue; } @@ -456,7 +473,7 @@ static int llcp_sock_accept(struct socket *sock, struct socket *newsock, pr_debug("parent %p\n", sk); - lock_sock_nested(sk, SINGLE_DEPTH_NESTING); + lock_sock(sk); if (sk->sk_state != LLCP_LISTEN) { ret = -EBADFD; @@ -482,7 +499,12 @@ static int llcp_sock_accept(struct socket *sock, struct socket *newsock, release_sock(sk); timeo = schedule_timeout(timeo); - lock_sock_nested(sk, SINGLE_DEPTH_NESTING); + lock_sock(sk); + + if (sk->sk_state != LLCP_LISTEN) { + ret = -EBADFD; + break; + } } __set_current_state(TASK_RUNNING); remove_wait_queue(sk_sleep(sk), &wait); @@ -621,13 +643,24 @@ static int llcp_sock_release(struct socket *sock) list_for_each_entry_safe(lsk, n, &llcp_sock->accept_queue, accept_queue) { + bool put_creation = false; + accept_sk = &lsk->sk; - lock_sock(accept_sk); + lock_sock_nested(accept_sk, SINGLE_DEPTH_NESTING); - nfc_llcp_send_disconnect(lsk); - nfc_llcp_accept_unlink(accept_sk); + if (nfc_llcp_sock(accept_sk)->parent == sk) { + nfc_llcp_send_disconnect(lsk); + nfc_llcp_accept_unlink(accept_sk); + nfc_llcp_sock_unlink(&local->sockets, accept_sk); + + accept_sk->sk_state = LLCP_CLOSED; + sock_orphan(accept_sk); + put_creation = true; + } release_sock(accept_sk); + if (put_creation) + sock_put(accept_sk); /* creation ref */ } } @@ -726,12 +759,16 @@ static int llcp_sock_connect(struct socket *sock, struct sockaddr_unsized *_addr llcp_sock->service_name_len = min_t(unsigned int, addr->service_name_len, NFC_LLCP_MAX_SERVICE_NAME); - llcp_sock->service_name = kmemdup(addr->service_name, - llcp_sock->service_name_len, - GFP_KERNEL); - if (!llcp_sock->service_name) { - ret = -ENOMEM; - goto sock_llcp_release; + if (llcp_sock->service_name_len == 0) { + llcp_sock->service_name = NULL; + } else { + llcp_sock->service_name = kmemdup(addr->service_name, + llcp_sock->service_name_len, + GFP_KERNEL); + if (!llcp_sock->service_name) { + ret = -ENOMEM; + goto sock_llcp_release; + } } nfc_llcp_sock_link(&local->connecting_sockets, sk); diff --git a/net/nfc/nci/core.c b/net/nfc/nci/core.c index 5f46c4b5720f6..73e3a96470ace 100644 --- a/net/nfc/nci/core.c +++ b/net/nfc/nci/core.c @@ -780,15 +780,15 @@ static int nci_set_local_general_bytes(struct nfc_dev *nfc_dev) { struct nci_dev *ndev = nfc_get_drvdata(nfc_dev); struct nci_set_config_param param; + u8 gb[NFC_MAX_GT_LEN]; int rc; - param.val = nfc_get_local_general_bytes(nfc_dev, ¶m.len); - if ((param.val == NULL) || (param.len == 0)) + nfc_get_local_general_bytes(nfc_dev, gb, sizeof(gb), + ¶m.len); + if (param.len == 0) return 0; - if (param.len > NFC_MAX_GT_LEN) - return -EINVAL; - + param.val = gb; param.id = NCI_PN_ATR_REQ_GEN_BYTES; rc = nci_request(ndev, nci_set_config_req, ¶m, diff --git a/net/nfc/nci/data.c b/net/nfc/nci/data.c index 5f98c73db5afd..4253edea5c8db 100644 --- a/net/nfc/nci/data.c +++ b/net/nfc/nci/data.c @@ -46,11 +46,11 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, timer_delete_sync(&ndev->data_timer); clear_bit(NCI_DATA_EXCHANGE_TO, &ndev->flags); - /* Mark the exchange as done before calling the callback. - * The callback (e.g. rawsock_data_exchange_complete) may - * want to immediately queue another data exchange. - */ - clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); + /* Claim completion atomically -- both close and rx_work may race here */ + if (!test_and_clear_bit(NCI_DATA_EXCHANGE, &ndev->flags)) { + kfree_skb(skb); + return; + } if (cb) { /* forward skb to nfc core */ diff --git a/net/nfc/nci/ntf.c b/net/nfc/nci/ntf.c index c96512bb86531..f5c9a8ab7ec10 100644 --- a/net/nfc/nci/ntf.c +++ b/net/nfc/nci/ntf.c @@ -440,7 +440,7 @@ void nci_clear_target_list(struct nci_dev *ndev) static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, const struct sk_buff *skb) { - struct nci_rf_discover_ntf ntf; + struct nci_rf_discover_ntf ntf = {}; const __u8 *data; bool add_target = true; @@ -525,15 +525,19 @@ static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev, struct nci_rf_intf_activated_ntf *ntf, - const __u8 *data) + const __u8 *data, __u8 data_len) { struct activation_params_nfca_poll_iso_dep *nfca_poll; struct activation_params_nfcb_poll_iso_dep *nfcb_poll; switch (ntf->activation_rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; nfca_poll = &ntf->activation_params.nfca_poll_iso_dep; nfca_poll->rats_res_len = min_t(__u8, *data++, NFC_ATS_MAXSIZE); + data_len--; + nfca_poll->rats_res_len = min_t(__u8, nfca_poll->rats_res_len, data_len); pr_debug("rats_res_len %d\n", nfca_poll->rats_res_len); if (nfca_poll->rats_res_len > 0) { memcpy(nfca_poll->rats_res, @@ -542,8 +546,12 @@ static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev, break; case NCI_NFC_B_PASSIVE_POLL_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; nfcb_poll = &ntf->activation_params.nfcb_poll_iso_dep; nfcb_poll->attrib_res_len = min_t(__u8, *data++, 50); + data_len--; + nfcb_poll->attrib_res_len = min_t(__u8, nfcb_poll->attrib_res_len, data_len); pr_debug("attrib_res_len %d\n", nfcb_poll->attrib_res_len); if (nfcb_poll->attrib_res_len > 0) { memcpy(nfcb_poll->attrib_res, @@ -562,7 +570,7 @@ static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev, static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev, struct nci_rf_intf_activated_ntf *ntf, - const __u8 *data) + const __u8 *data, __u8 data_len) { struct activation_params_poll_nfc_dep *poll; struct activation_params_listen_nfc_dep *listen; @@ -570,9 +578,13 @@ static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev, switch (ntf->activation_rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: case NCI_NFC_F_PASSIVE_POLL_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; poll = &ntf->activation_params.poll_nfc_dep; poll->atr_res_len = min_t(__u8, *data++, NFC_ATR_RES_MAXSIZE - 2); + data_len--; + poll->atr_res_len = min_t(__u8, poll->atr_res_len, data_len); pr_debug("atr_res_len %d\n", poll->atr_res_len); if (poll->atr_res_len > 0) memcpy(poll->atr_res, data, poll->atr_res_len); @@ -580,9 +592,13 @@ static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev, case NCI_NFC_A_PASSIVE_LISTEN_MODE: case NCI_NFC_F_PASSIVE_LISTEN_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; listen = &ntf->activation_params.listen_nfc_dep; listen->atr_req_len = min_t(__u8, *data++, NFC_ATR_REQ_MAXSIZE - 2); + data_len--; + listen->atr_req_len = min_t(__u8, listen->atr_req_len, data_len); pr_debug("atr_req_len %d\n", listen->atr_req_len); if (listen->atr_req_len > 0) memcpy(listen->atr_req, data, listen->atr_req_len); @@ -603,6 +619,12 @@ static void nci_target_auto_activated(struct nci_dev *ndev, struct nfc_target *target; int rc; + /* This is a new target, check if we've enough room */ + if (ndev->n_targets == NCI_MAX_DISCOVERED_TARGETS) { + pr_debug("not enough room, ignoring new target...\n"); + return; + } + target = &ndev->targets[ndev->n_targets]; rc = nci_add_new_protocol(ndev, target, ntf->rf_protocol, @@ -688,7 +710,7 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, const struct sk_buff *skb) { struct nci_conn_info *conn_info; - struct nci_rf_intf_activated_ntf ntf; + struct nci_rf_intf_activated_ntf ntf = {}; const __u8 *data; int err = NCI_STATUS_OK; @@ -806,12 +828,14 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, switch (ntf.rf_interface) { case NCI_RF_INTERFACE_ISO_DEP: err = nci_extract_activation_params_iso_dep(ndev, - &ntf, data); + &ntf, data, + ntf.activation_params_len); break; case NCI_RF_INTERFACE_NFC_DEP: err = nci_extract_activation_params_nfc_dep(ndev, - &ntf, data); + &ntf, data, + ntf.activation_params_len); break; case NCI_RF_INTERFACE_FRAME: diff --git a/net/nfc/nci/rsp.c b/net/nfc/nci/rsp.c index 9eeb862825c5f..b0ab4f5acbce8 100644 --- a/net/nfc/nci/rsp.c +++ b/net/nfc/nci/rsp.c @@ -50,11 +50,27 @@ static u8 nci_core_init_rsp_packet_v1(struct nci_dev *ndev, const struct nci_core_init_rsp_1 *rsp_1 = (void *)skb->data; const struct nci_core_init_rsp_2 *rsp_2; + /* Ensure that the status field can be accessed. */ + if (skb_headlen(skb) < 1) + return NCI_STATUS_SYNTAX_ERROR; + pr_debug("status 0x%x\n", rsp_1->status); if (rsp_1->status != NCI_STATUS_OK) return rsp_1->status; + /* Success response must contain the full fixed-size header */ + if (skb_headlen(skb) < sizeof(*rsp_1)) + return NCI_STATUS_SYNTAX_ERROR; + + /* Ensure the variable-length rf_interfaces array and trailing + * rsp_2 structure are fully contained within the skb. + */ + if (skb_headlen(skb) < sizeof(*rsp_1) + + rsp_1->num_supported_rf_interfaces + + sizeof(*rsp_2)) + return NCI_STATUS_SYNTAX_ERROR; + ndev->nfcc_features = __le32_to_cpu(rsp_1->nfcc_features); ndev->num_supported_rf_interfaces = rsp_1->num_supported_rf_interfaces; @@ -87,15 +103,25 @@ static u8 nci_core_init_rsp_packet_v2(struct nci_dev *ndev, const struct sk_buff *skb) { const struct nci_core_init_rsp_nci_ver2 *rsp = (void *)skb->data; - const u8 *supported_rf_interface = rsp->supported_rf_interfaces; + const u8 *supported_rf_interface; u8 rf_interface_idx = 0; u8 rf_extension_cnt = 0; + /* Ensure that the status field can be accessed. */ + if (skb_headlen(skb) < 1) + return NCI_STATUS_SYNTAX_ERROR; + pr_debug("status %x\n", rsp->status); if (rsp->status != NCI_STATUS_OK) return rsp->status; + /* Success response must contain the full fixed-size header */ + if (skb_headlen(skb) < sizeof(*rsp)) + return NCI_STATUS_SYNTAX_ERROR; + + supported_rf_interface = rsp->supported_rf_interfaces; + ndev->nfcc_features = __le32_to_cpu(rsp->nfcc_features); ndev->num_supported_rf_interfaces = rsp->num_supported_rf_interfaces; @@ -104,13 +130,22 @@ static u8 nci_core_init_rsp_packet_v2(struct nci_dev *ndev, NCI_MAX_SUPPORTED_RF_INTERFACES); while (rf_interface_idx < ndev->num_supported_rf_interfaces) { - ndev->supported_rf_interfaces[rf_interface_idx++] = *supported_rf_interface++; + /* Each entry: [rf_interface_type (1B)] [ext_count (1B)] [ext...] */ + if (supported_rf_interface + 2 > skb_tail_pointer(skb)) + break; + ndev->supported_rf_interfaces[rf_interface_idx] = *supported_rf_interface++; - /* skip rf extension parameters */ rf_extension_cnt = *supported_rf_interface++; + if (supported_rf_interface + rf_extension_cnt > skb_tail_pointer(skb)) + break; + + /* Only count the entry after full validation */ + rf_interface_idx++; supported_rf_interface += rf_extension_cnt; } + ndev->num_supported_rf_interfaces = rf_interface_idx; + ndev->max_logical_connections = rsp->max_logical_connections; ndev->max_routing_table_size = __le16_to_cpu(rsp->max_routing_table_size); @@ -336,6 +371,7 @@ static void nci_core_conn_close_rsp_packet(struct nci_dev *ndev, list_del(&conn_info->list); if (conn_info == ndev->rf_conn_info) ndev->rf_conn_info = NULL; + devm_kfree(&ndev->nfc_dev->dev, conn_info->dest_params); devm_kfree(&ndev->nfc_dev->dev, conn_info); } } diff --git a/net/nfc/netlink.c b/net/nfc/netlink.c index 0c58824cb150d..224bdfa2dd0dc 100644 --- a/net/nfc/netlink.c +++ b/net/nfc/netlink.c @@ -1181,7 +1181,7 @@ static int nfc_genl_llc_sdreq(struct sk_buff *skb, struct genl_info *info) if (rc != 0) { rc = -EINVAL; - goto put_local; + goto free_list; } if (!sdp_attrs[NFC_SDP_ATTR_URI]) @@ -1200,7 +1200,7 @@ static int nfc_genl_llc_sdreq(struct sk_buff *skb, struct genl_info *info) sdreq = nfc_llcp_build_sdreq_tlv(tid, uri, uri_len); if (sdreq == NULL) { rc = -ENOMEM; - goto put_local; + goto free_list; } tlvs_len += sdreq->tlv_len; @@ -1215,6 +1215,9 @@ static int nfc_genl_llc_sdreq(struct sk_buff *skb, struct genl_info *info) rc = nfc_llcp_send_snl_sdreq(local, &sdreq_list, tlvs_len); +free_list: + nfc_llcp_free_sdp_tlv_list(&sdreq_list); + put_local: nfc_llcp_local_put(local); diff --git a/net/nfc/nfc.h b/net/nfc/nfc.h index 0b1e6466f4fbf..82c5dfdad10e2 100644 --- a/net/nfc/nfc.h +++ b/net/nfc/nfc.h @@ -49,7 +49,8 @@ void nfc_llcp_mac_is_up(struct nfc_dev *dev, u32 target_idx, int nfc_llcp_register_device(struct nfc_dev *dev); void nfc_llcp_unregister_device(struct nfc_dev *dev); int nfc_llcp_set_remote_gb(struct nfc_dev *dev, const u8 *gb, u8 gb_len); -u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, size_t *general_bytes_len); +u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, u8 *out_gb, size_t gb_max_len, + size_t *general_bytes_len); int nfc_llcp_data_received(struct nfc_dev *dev, struct sk_buff *skb); struct nfc_llcp_local *nfc_llcp_find_local(struct nfc_dev *dev); int nfc_llcp_local_put(struct nfc_llcp_local *local); diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index 95697d4e16e64..4afb750ad2c87 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -366,7 +366,7 @@ static struct nf_conn_labels *ovs_ct_get_conn_labels(struct nf_conn *ct) struct nf_conn_labels *cl; cl = nf_ct_labels_find(ct); - if (!cl) { + if (!cl && !nf_ct_is_confirmed(ct)) { nf_ct_labels_ext_add(ct); cl = nf_ct_labels_find(ct); } @@ -734,6 +734,18 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key, enum ip_conntrack_info ctinfo; struct nf_conn *ct; + /* If the ct entry is not confirmed and shared with some other skb, + * e.g., a cloned one, we can't just modify it with the commit as we + * must not modify the extension set. Reset. + */ + if (cached && info->commit) { + ct = nf_ct_get(skb, &ctinfo); + if (ct && !nf_ct_is_confirmed(ct) && nf_ct_shared(ct)) { + nf_reset_ct(skb); + cached = false; + } + } + if (!cached) { struct nf_hook_state state = { .hook = NF_INET_PRE_ROUTING, @@ -767,8 +779,6 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key, ct = nf_ct_get(skb, &ctinfo); if (ct) { - bool add_helper = false; - /* Packets starting a new connection must be NATted before the * helper, so that the helper knows about the NAT. We enforce * this by delaying both NAT and helper calls for unconfirmed @@ -800,7 +810,6 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key, GFP_ATOMIC); if (err) return err; - add_helper = true; /* helper installed, add seqadj if NAT is required */ if (info->nat && !nfct_seqadj(ct)) { @@ -809,14 +818,14 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key, } } - /* Call the helper only if: - * - nf_conntrack_in() was executed above ("!cached") or a - * helper was just attached ("add_helper") for a confirmed - * connection, or - * - When committing an unconfirmed connection. + /* Call the helper only if nf_conntrack_in() was executed + * above ("!cached"). + * + * For unconfirmed connections it will be called later during + * commit as we need to have all the other extensions allocated + * before the call. */ - if ((nf_ct_is_confirmed(ct) ? !cached || add_helper : - info->commit)) { + if (nf_ct_is_confirmed(ct) && !cached) { int err = nf_ct_helper(skb, ct, ctinfo, info->family); err = verdict_to_errno(err); @@ -933,10 +942,14 @@ static int ovs_ct_check_limit(struct net *net, const struct ovs_conntrack_info *info) { struct ovs_net *ovs_net = net_generic(net, ovs_net_id); - const struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; + const struct ovs_ct_limit_info *ct_limit_info; u32 per_zone_limit, connections; u32 conncount_key; + ct_limit_info = rcu_dereference(ovs_net->ct_limit_info); + if (!ct_limit_info) + return 0; + conncount_key = info->zone.id; per_zone_limit = ct_limit_get(ct_limit_info, info->zone.id); @@ -1016,6 +1029,14 @@ static int ovs_ct_commit(struct net *net, struct sw_flow_key *key, return err; nf_conn_act_ct_ext_add(skb, ct, ctinfo); + + /* Call the helpers now. We couldn't do this before as + * all the extensions must be allocated before the call. + */ + err = nf_ct_helper(skb, ct, ctinfo, info->family); + err = verdict_to_errno(err); + if (err) + return err; } else if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) && labels_nonzero(&info->labels.mask)) { err = ovs_ct_set_labels(ct, key, &info->labels.value, @@ -1585,40 +1606,55 @@ static void __ovs_ct_free_action(struct ovs_conntrack_info *ct_info) #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) static int ovs_ct_limit_init(struct net *net, struct ovs_net *ovs_net) { + struct ovs_ct_limit_info *info; int i, err; - ovs_net->ct_limit_info = kmalloc_obj(*ovs_net->ct_limit_info); - if (!ovs_net->ct_limit_info) + info = kmalloc_obj(*info); + if (!info) return -ENOMEM; - ovs_net->ct_limit_info->default_limit = OVS_CT_LIMIT_DEFAULT; - ovs_net->ct_limit_info->limits = + info->default_limit = OVS_CT_LIMIT_DEFAULT; + info->limits = kmalloc_objs(struct hlist_head, CT_LIMIT_HASH_BUCKETS); - if (!ovs_net->ct_limit_info->limits) { - kfree(ovs_net->ct_limit_info); + if (!info->limits) { + kfree(info); return -ENOMEM; } for (i = 0; i < CT_LIMIT_HASH_BUCKETS; i++) - INIT_HLIST_HEAD(&ovs_net->ct_limit_info->limits[i]); + INIT_HLIST_HEAD(&info->limits[i]); - ovs_net->ct_limit_info->data = nf_conncount_init(net, sizeof(u32)); + info->data = nf_conncount_init(net, sizeof(u32)); - if (IS_ERR(ovs_net->ct_limit_info->data)) { - err = PTR_ERR(ovs_net->ct_limit_info->data); - kfree(ovs_net->ct_limit_info->limits); - kfree(ovs_net->ct_limit_info); + if (IS_ERR(info->data)) { + err = PTR_ERR(info->data); + kfree(info->limits); + kfree(info); pr_err("openvswitch: failed to init nf_conncount %d\n", err); return err; } + rcu_assign_pointer(ovs_net->ct_limit_info, info); return 0; } -static void ovs_ct_limit_exit(struct net *net, struct ovs_net *ovs_net) +static void *ovs_ct_limit_exit_start(struct ovs_net *ovs_net) { - const struct ovs_ct_limit_info *info = ovs_net->ct_limit_info; + return rcu_replace_pointer(ovs_net->ct_limit_info, NULL, + lockdep_ovsl_is_held()); +} + +/* The CT limit state must be detached by ovs_ct_limit_exit_start() and an + * RCU grace period must elapse before this function runs. The pernet core + * guarantees the grace period between the .pre_exit and .exit callbacks. + */ +static void ovs_ct_limit_exit_finish(struct net *net, void *data) +{ + const struct ovs_ct_limit_info *info = data; int i; + if (!info) + return; + nf_conncount_destroy(net, info->data); for (i = 0; i < CT_LIMIT_HASH_BUCKETS; ++i) { struct hlist_head *head = &info->limits[i]; @@ -1626,7 +1662,7 @@ static void ovs_ct_limit_exit(struct net *net, struct ovs_net *ovs_net) struct hlist_node *next; hlist_for_each_entry_safe(ct_limit, next, head, hlist_node) - kfree_rcu(ct_limit, rcu); + kfree(ct_limit); } kfree(info->limits); kfree(info); @@ -1665,12 +1701,13 @@ static bool check_zone_id(int zone_id, u16 *pzone) return false; } -static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, - struct ovs_ct_limit_info *info) +static int ovs_ct_limit_set_zone_limit(struct ovs_net *ovs_net, + struct nlattr *nla_zone_limit) { struct ovs_zone_limit *zone_limit; - int rem; + struct ovs_ct_limit_info *info; u16 zone; + int rem; rem = NLA_ALIGN(nla_len(nla_zone_limit)); zone_limit = (struct ovs_zone_limit *)nla_data(nla_zone_limit); @@ -1679,6 +1716,7 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, if (unlikely(zone_limit->zone_id == OVS_ZONE_LIMIT_DEFAULT_ZONE)) { ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); info->default_limit = zone_limit->limit; ovs_unlock(); } else if (unlikely(!check_zone_id( @@ -1695,6 +1733,7 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, ct_limit->limit = zone_limit->limit; ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); ct_limit_set(info, ct_limit); ovs_unlock(); } @@ -1709,12 +1748,13 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, return 0; } -static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit, - struct ovs_ct_limit_info *info) +static int ovs_ct_limit_del_zone_limit(struct ovs_net *ovs_net, + struct nlattr *nla_zone_limit) { struct ovs_zone_limit *zone_limit; - int rem; + struct ovs_ct_limit_info *info; u16 zone; + int rem; rem = NLA_ALIGN(nla_len(nla_zone_limit)); zone_limit = (struct ovs_zone_limit *)nla_data(nla_zone_limit); @@ -1723,6 +1763,7 @@ static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit, if (unlikely(zone_limit->zone_id == OVS_ZONE_LIMIT_DEFAULT_ZONE)) { ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); info->default_limit = OVS_CT_LIMIT_DEFAULT; ovs_unlock(); } else if (unlikely(!check_zone_id( @@ -1730,6 +1771,7 @@ static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit, OVS_NLERR(true, "zone id is out of range"); } else { ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); ct_limit_del(info, zone); ovs_unlock(); } @@ -1773,6 +1815,7 @@ static int __ovs_ct_limit_get_zone_limit(struct net *net, return nla_put_nohdr(reply, sizeof(zone_limit), &zone_limit); } +/* Called with RCU read lock held. */ static int ovs_ct_limit_get_zone_limit(struct net *net, struct nlattr *nla_zone_limit, struct ovs_ct_limit_info *info, @@ -1796,12 +1839,10 @@ static int ovs_ct_limit_get_zone_limit(struct net *net, &zone))) { OVS_NLERR(true, "zone id is out of range"); } else { - rcu_read_lock(); limit = ct_limit_get(info, zone); err = __ovs_ct_limit_get_zone_limit( net, info->data, zone, limit, reply); - rcu_read_unlock(); if (err) return err; } @@ -1816,6 +1857,7 @@ static int ovs_ct_limit_get_zone_limit(struct net *net, return 0; } +/* Called with RCU read lock held. */ static int ovs_ct_limit_get_all_zone_limit(struct net *net, struct ovs_ct_limit_info *info, struct sk_buff *reply) @@ -1828,19 +1870,16 @@ static int ovs_ct_limit_get_all_zone_limit(struct net *net, if (err) return err; - rcu_read_lock(); for (i = 0; i < CT_LIMIT_HASH_BUCKETS; ++i) { head = &info->limits[i]; hlist_for_each_entry_rcu(ct_limit, head, hlist_node) { err = __ovs_ct_limit_get_zone_limit(net, info->data, ct_limit->zone, ct_limit->limit, reply); if (err) - goto exit_err; + return err; } } -exit_err: - rcu_read_unlock(); return err; } @@ -1850,7 +1889,6 @@ static int ovs_ct_limit_cmd_set(struct sk_buff *skb, struct genl_info *info) struct sk_buff *reply; struct ovs_header *ovs_reply_header; struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id); - struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_SET, @@ -1863,8 +1901,8 @@ static int ovs_ct_limit_cmd_set(struct sk_buff *skb, struct genl_info *info) goto exit_err; } - err = ovs_ct_limit_set_zone_limit(a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT], - ct_limit_info); + err = ovs_ct_limit_set_zone_limit(ovs_net, + a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]); if (err) goto exit_err; @@ -1884,7 +1922,6 @@ static int ovs_ct_limit_cmd_del(struct sk_buff *skb, struct genl_info *info) struct sk_buff *reply; struct ovs_header *ovs_reply_header; struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id); - struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_DEL, @@ -1897,8 +1934,8 @@ static int ovs_ct_limit_cmd_del(struct sk_buff *skb, struct genl_info *info) goto exit_err; } - err = ovs_ct_limit_del_zone_limit(a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT], - ct_limit_info); + err = ovs_ct_limit_del_zone_limit(ovs_net, + a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]); if (err) goto exit_err; @@ -1918,7 +1955,7 @@ static int ovs_ct_limit_cmd_get(struct sk_buff *skb, struct genl_info *info) struct ovs_header *ovs_reply_header; struct net *net = sock_net(skb->sk); struct ovs_net *ovs_net = net_generic(net, ovs_net_id); - struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; + struct ovs_ct_limit_info *ct_limit_info; int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_GET, @@ -1932,18 +1969,19 @@ static int ovs_ct_limit_cmd_get(struct sk_buff *skb, struct genl_info *info) goto exit_err; } + rcu_read_lock(); + ct_limit_info = rcu_dereference(ovs_net->ct_limit_info); if (a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]) { err = ovs_ct_limit_get_zone_limit( net, a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT], ct_limit_info, reply); - if (err) - goto exit_err; } else { err = ovs_ct_limit_get_all_zone_limit(net, ct_limit_info, reply); - if (err) - goto exit_err; } + rcu_read_unlock(); + if (err) + goto exit_err; nla_nest_end(reply, nla_reply); genlmsg_end(reply, ovs_reply_header); @@ -2001,6 +2039,7 @@ int ovs_ct_init(struct net *net) { unsigned int n_bits = sizeof(struct ovs_key_ct_labels) * BITS_PER_BYTE; struct ovs_net *ovs_net = net_generic(net, ovs_net_id); + int err = 0; if (nf_connlabels_get(net, n_bits - 1)) { ovs_net->xt_label = false; @@ -2010,18 +2049,36 @@ int ovs_ct_init(struct net *net) } #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) - return ovs_ct_limit_init(net, ovs_net); -#else - return 0; + err = ovs_ct_limit_init(net, ovs_net); + if (err && ovs_net->xt_label) + nf_connlabels_put(net); #endif + return err; } -void ovs_ct_exit(struct net *net) +/* Must be called with ovs_mutex held. Detaches the RCU-protected + * ct_limit_info and stores it in ovs_net->ct_limit_exit_data for + * ovs_ct_exit_finish() to complete the teardown after an RCU grace period. + */ +void ovs_ct_exit_start(struct net *net __maybe_unused) +{ +#if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) + struct ovs_net *ovs_net = net_generic(net, ovs_net_id); + + ovs_net->ct_limit_exit_data = ovs_ct_limit_exit_start(ovs_net); +#endif +} + +/* Completes the CT limit teardown. The pernet core guarantees an RCU + * grace period between detaching the state in ovs_ct_exit_start() and + * this call, so no RCU readers remain. + */ +void ovs_ct_exit_finish(struct net *net) { struct ovs_net *ovs_net = net_generic(net, ovs_net_id); #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) - ovs_ct_limit_exit(net, ovs_net); + ovs_ct_limit_exit_finish(net, ovs_net->ct_limit_exit_data); #endif if (ovs_net->xt_label) diff --git a/net/openvswitch/conntrack.h b/net/openvswitch/conntrack.h index 317e525c8a113..ab21d032fde3d 100644 --- a/net/openvswitch/conntrack.h +++ b/net/openvswitch/conntrack.h @@ -14,7 +14,8 @@ enum ovs_key_attr; #if IS_ENABLED(CONFIG_NF_CONNTRACK) int ovs_ct_init(struct net *); -void ovs_ct_exit(struct net *); +void ovs_ct_exit_start(struct net *net); +void ovs_ct_exit_finish(struct net *net); bool ovs_ct_verify(struct net *, enum ovs_key_attr attr); int ovs_ct_copy_action(struct net *, const struct nlattr *, const struct sw_flow_key *, struct sw_flow_actions **, @@ -40,7 +41,8 @@ void ovs_ct_free_action(const struct nlattr *a); static inline int ovs_ct_init(struct net *net) { return 0; } -static inline void ovs_ct_exit(struct net *net) { } +static inline void ovs_ct_exit_start(struct net *net) { } +static inline void ovs_ct_exit_finish(struct net *net) { } static inline bool ovs_ct_verify(struct net *net, int attr) { diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index ae69b2cabab9e..b601e4859de92 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -285,6 +285,7 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key) consume_skb(skb); break; default: + skb_tx_error(skb); kfree_skb(skb); break; } @@ -601,8 +602,6 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid); user_skb = NULL; out: - if (err) - skb_tx_error(skb); consume_skb(user_skb); consume_skb(nskb); @@ -1473,33 +1472,34 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info) goto unlock; } - ovs_flow_tbl_remove(&dp->table, flow); - ovs_unlock(); - - reply = ovs_flow_cmd_alloc_info((const struct sw_flow_actions __force *) flow->sf_acts, + reply = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts), &flow->id, info, false, ufid_flags); - if (likely(reply)) { - if (!IS_ERR(reply)) { - rcu_read_lock(); /*To keep RCU checker happy. */ - err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, - reply, info->snd_portid, - info->snd_seq, 0, - OVS_FLOW_CMD_DEL, - ufid_flags); - rcu_read_unlock(); - if (WARN_ON_ONCE(err < 0)) { - kfree_skb(reply); - goto out_free; - } + if (IS_ERR(reply)) { + netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0, + PTR_ERR(reply)); + reply = NULL; + } - ovs_notify(&dp_flow_genl_family, reply, info); - } else { - netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0, - PTR_ERR(reply)); + if (likely(reply)) { + err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, + reply, info->snd_portid, + info->snd_seq, 0, + OVS_FLOW_CMD_DEL, ufid_flags); + if (WARN_ON_ONCE(err < 0)) { + kfree_skb(reply); + reply = NULL; } } + /* Removal has to happen after ovs_flow_cmd_fill_info(), as it uses + * the flow->mask that can be scheduled to be freed by the + * ovs_flow_tbl_remove() and we're not holding the RCU read lock. + */ + ovs_flow_tbl_remove(&dp->table, flow); + ovs_unlock(); + + if (likely(reply)) + ovs_notify(&dp_flow_genl_family, reply, info); -out_free: ovs_flow_free(flow, true); return 0; unlock: @@ -2756,6 +2756,13 @@ static void __net_exit list_vports_from_net(struct net *net, struct net *dnet, } } +static void __net_exit ovs_pre_exit_net(struct net *dnet) +{ + ovs_lock(); + ovs_ct_exit_start(dnet); + ovs_unlock(); +} + static void __net_exit ovs_exit_net(struct net *dnet) { struct datapath *dp, *dp_next; @@ -2766,7 +2773,7 @@ static void __net_exit ovs_exit_net(struct net *dnet) ovs_lock(); - ovs_ct_exit(dnet); + ovs_ct_exit_finish(dnet); list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node) __dp_destroy(dp); @@ -2790,6 +2797,7 @@ static void __net_exit ovs_exit_net(struct net *dnet) static struct pernet_operations ovs_net_ops = { .init = ovs_init_net, + .pre_exit = ovs_pre_exit_net, .exit = ovs_exit_net, .id = &ovs_net_id, .size = sizeof(struct ovs_net), diff --git a/net/openvswitch/datapath.h b/net/openvswitch/datapath.h index 696640e88fa7e..b2c2b8da12d40 100644 --- a/net/openvswitch/datapath.h +++ b/net/openvswitch/datapath.h @@ -164,7 +164,10 @@ struct dp_upcall_info { * Protected by genl_mutex. * @dp_notify_work: A work notifier to handle port unregistering. * @masks_rebalance: A work to periodically optimize flow table caches. - * @ct_limit_info: A hash table of conntrack zone connection limits. + * @ct_limit_info: Hash table of conntrack zone connection limits. Protected + * by RCU; updates and teardown are serialized by ovs_mutex. May be NULL during + * netns teardown. + * @ct_limit_exit_data: CT limit state detached at .pre_exit, freed at .exit. * @xt_label: Whether connlables are configured for the network or not. */ struct ovs_net { @@ -172,7 +175,8 @@ struct ovs_net { struct work_struct dp_notify_work; struct delayed_work masks_rebalance; #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) - struct ovs_ct_limit_info *ct_limit_info; + struct ovs_ct_limit_info __rcu *ct_limit_info; + struct ovs_ct_limit_info *ct_limit_exit_data; #endif bool xt_label; }; diff --git a/net/openvswitch/flow.c b/net/openvswitch/flow.c index 46c1d66aad8c3..868d9fdf3afdf 100644 --- a/net/openvswitch/flow.c +++ b/net/openvswitch/flow.c @@ -288,7 +288,7 @@ static void get_ipv6_ext_hdrs(struct sk_buff *skb, struct ipv6hdr *nh, if (*ext_hdrs & OFPIEH12_ESP) *ext_hdrs |= OFPIEH12_UNREP; if ((*ext_hdrs & ~(OFPIEH12_HOP | OFPIEH12_DEST | - OFPIEH12_ROUTER | IPPROTO_FRAGMENT | + OFPIEH12_ROUTER | OFPIEH12_FRAG | OFPIEH12_AUTH | OFPIEH12_UNREP)) || dest_options_header_count >= 2) { *ext_hdrs |= OFPIEH12_UNSEQ; @@ -301,7 +301,7 @@ static void get_ipv6_ext_hdrs(struct sk_buff *skb, struct ipv6hdr *nh, *ext_hdrs |= OFPIEH12_UNREP; if ((*ext_hdrs & ~(OFPIEH12_HOP | OFPIEH12_DEST | OFPIEH12_ROUTER | - IPPROTO_FRAGMENT | OFPIEH12_UNREP)) || + OFPIEH12_FRAG | OFPIEH12_UNREP)) || dest_options_header_count >= 2) { *ext_hdrs |= OFPIEH12_UNSEQ; } diff --git a/net/openvswitch/flow_table.c b/net/openvswitch/flow_table.c index 67d5b8c0fe792..1e0f9d193eb06 100644 --- a/net/openvswitch/flow_table.c +++ b/net/openvswitch/flow_table.c @@ -257,11 +257,13 @@ static int tbl_mask_array_realloc(struct flow_table *tbl, int size) if (ovsl_dereference(old->masks[i])) new->masks[new->count++] = old->masks[i]; } - call_rcu(&old->rcu, mask_array_rcu_cb); } rcu_assign_pointer(tbl->mask_array, new); + if (old) + call_rcu(&old->rcu, mask_array_rcu_cb); + return 0; } diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 1168bd6b09cd6..7c83e01526edc 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -88,6 +88,7 @@ #include #include #include +#include #ifdef CONFIG_INET #include #endif @@ -616,7 +617,7 @@ static int prb_calc_retire_blk_tmo(struct packet_sock *po, return DEFAULT_PRB_RETIRE_TOV; div = ecmd.base.speed / 1000; - mbits = (blk_size_in_bytes * 8) / (1024 * 1024); + mbits = (u64)blk_size_in_bytes * 8 / (1024 * 1024); if (div) mbits /= div; @@ -1341,6 +1342,8 @@ static void packet_sock_destruct(struct sock *sk) WARN_ON(atomic_read(&sk->sk_rmem_alloc)); WARN_ON(refcount_read(&sk->sk_wmem_alloc)); + packet_free_pending(pkt_sk(sk)); + if (!sock_flag(sk, SOCK_DEAD)) { pr_err("Attempt to release alive packet socket: %p\n", sk); return; @@ -2381,7 +2384,9 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, virtio_net_hdr_from_skb(skb, h.raw + macoff - sizeof(struct virtio_net_hdr), vio_le(), true, 0)) { - if (po->tp_version == TPACKET_V3) + if (po->tp_version <= TPACKET_V2) + __clear_bit(slot_id, po->rx_ring.rx_owner_map); + else prb_clear_blk_fill_status(&po->rx_ring); goto drop_n_account; } @@ -2525,26 +2530,6 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, goto drop_n_restore; } -static void tpacket_destruct_skb(struct sk_buff *skb) -{ - struct packet_sock *po = pkt_sk(skb->sk); - - if (likely(po->tx_ring.pg_vec)) { - void *ph; - __u32 ts; - - ph = skb_zcopy_get_nouarg(skb); - packet_dec_pending(&po->tx_ring); - - ts = __packet_set_timestamp(po, ph, skb); - __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); - - complete(&po->skb_completion); - } - - sock_wfree(skb); -} - static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len) { if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && @@ -2584,27 +2569,56 @@ static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len, return 0; } +struct tpacket_uarg { + struct ubuf_info ubuf; + struct packet_sock *po; + void *ph; +}; + +static void tpacket_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg, + bool success) +{ + struct tpacket_uarg *tu = container_of(uarg, struct tpacket_uarg, ubuf); + struct packet_sock *po = tu->po; + void *ph = tu->ph; + __u32 ts; + + DEBUG_NET_WARN_ON_ONCE(!skb); + + if (!refcount_dec_and_test(&uarg->refcnt)) + return; + + ts = __packet_set_timestamp(po, ph, skb); + __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); + + packet_dec_pending(&po->tx_ring); + complete(&po->skb_completion); + + kfree(tu); + sk_free(&po->sk); +} + +static const struct ubuf_info_ops tpacket_ubuf_ops = { + .complete = tpacket_ubuf_complete, +}; + static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, - void *frame, struct net_device *dev, void *data, int tp_len, + struct net_device *dev, void *data, int tp_len, __be16 proto, unsigned char *addr, int hlen, int copylen, int hard_header_len, const struct sockcm_cookie *sockc) { - union tpacket_uhdr ph; int to_write, offset, len, nr_frags, len_max; struct socket *sock = po->sk.sk_socket; struct page *page; int err; - ph.raw = frame; - skb->protocol = proto; skb->dev = dev; skb->priority = sockc->priority; skb->mark = sockc->mark; skb_set_delivery_type_by_clockid(skb, sockc->transmit_time, po->sk.sk_clockid); skb_setup_tx_timestamp(skb, sockc); - skb_zcopy_set_nouarg(skb, ph.raw); skb_reserve(skb, hlen); skb_reset_network_header(skb); @@ -2672,7 +2686,8 @@ static int tpacket_parse_header(struct packet_sock *po, void *frame, int size_max, void **data) { union tpacket_uhdr ph; - int tp_len, off; + u32 tp_len; + int off; ph.raw = frame; @@ -2692,7 +2707,7 @@ static int tpacket_parse_header(struct packet_sock *po, void *frame, break; } if (unlikely(tp_len > size_max)) { - pr_err("packet size is too long (%d > %d)\n", tp_len, size_max); + pr_err("packet size is too long (%u > %d)\n", tp_len, size_max); return -EMSGSIZE; } @@ -2743,6 +2758,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) struct virtio_net_hdr vnet_hdr; bool has_vnet_hdr = false; struct sockcm_cookie sockc; + struct tpacket_uarg *uarg; __be16 proto; int err, reserve = 0; void *ph; @@ -2870,7 +2886,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) err = len_sum; goto out_status; } - tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto, + tp_len = tpacket_fill_skb(po, skb, dev, data, tp_len, proto, addr, hlen, copylen, hard_header_len, &sockc); if (likely(tp_len >= 0) && @@ -2902,7 +2918,24 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) virtio_net_hdr_set_proto(skb, &vnet_hdr); } - skb->destructor = tpacket_destruct_skb; + uarg = kmalloc(sizeof(*uarg), GFP_KERNEL); + if (unlikely(!uarg)) { + if (likely(len_sum > 0)) + err = len_sum; + else + err = -ENOMEM; + goto out_status; + } + uarg->po = po; + uarg->ph = ph; + uarg->ubuf.ops = &tpacket_ubuf_ops; + uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG; + refcount_set(&uarg->ubuf.refcnt, 1); + + /* Hold a sk_wmem_alloc reference until completion */ + refcount_inc(&po->sk.sk_wmem_alloc); + skb_zcopy_init(skb, &uarg->ubuf); + __packet_set_status(po, ph, TP_STATUS_SENDING); packet_inc_pending(&po->tx_ring); @@ -3207,7 +3240,6 @@ static int packet_release(struct socket *sock) /* Purge queues */ skb_queue_purge(&sk->sk_receive_queue); - packet_free_pending(po); sock_put(sk); return 0; @@ -4370,11 +4402,26 @@ static const struct vm_operations_struct packet_mmap_ops = { .close = packet_mm_close, }; +struct packet_pg_vec { + struct packet_pg_vec_free *deferred; + unsigned int order; + unsigned int len; + struct pgv pg_vec[] __counted_by(len); +}; + +struct packet_pg_vec_free { + struct delayed_work work; + struct sock *sk; + struct packet_pg_vec *vec; +}; + static void free_pg_vec(struct pgv *pg_vec, unsigned int order, unsigned int len) { + struct packet_pg_vec *vec; int i; + vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]); for (i = 0; i < len; i++) { if (likely(pg_vec[i].buffer)) { if (is_vmalloc_addr(pg_vec[i].buffer)) @@ -4385,7 +4432,46 @@ static void free_pg_vec(struct pgv *pg_vec, unsigned int order, pg_vec[i].buffer = NULL; } } - kfree(pg_vec); + kfree(vec->deferred); + kfree(vec); +} + +static void packet_free_pg_vec_work(struct work_struct *work) +{ + struct packet_pg_vec_free *deferred; + struct packet_pg_vec *vec; + struct sock *sk; + + deferred = container_of_const(to_delayed_work(work), + struct packet_pg_vec_free, work); + vec = deferred->vec; + sk = deferred->sk; + if (sk_wmem_alloc_get(sk)) { + queue_delayed_work(system_long_wq, &deferred->work, 1); + return; + } + + free_pg_vec(vec->pg_vec, vec->order, vec->len); + sock_put(sk); +} + +static void packet_free_tx_ring(struct sock *sk, struct pgv *pg_vec, + unsigned int order, unsigned int len) +{ + struct packet_pg_vec_free *deferred; + struct packet_pg_vec *vec; + + vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]); + deferred = vec->deferred; + if (!deferred || !sk_wmem_alloc_get(sk)) { + free_pg_vec(pg_vec, order, len); + return; + } + + /* A detached ring's pending count can miss late skb destructors. */ + deferred->sk = sk; + sock_hold(sk); + queue_delayed_work(system_long_wq, &deferred->work, 0); } static char *alloc_one_pg_vec_page(unsigned long order) @@ -4413,15 +4499,29 @@ static char *alloc_one_pg_vec_page(unsigned long order) return NULL; } -static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order) +static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order, bool tx_ring) { unsigned int block_nr = req->tp_block_nr; + struct packet_pg_vec *vec; struct pgv *pg_vec; int i; - pg_vec = kzalloc_objs(struct pgv, block_nr, GFP_KERNEL | __GFP_NOWARN); - if (unlikely(!pg_vec)) - goto out; + vec = kzalloc_flex(*vec, pg_vec, block_nr, GFP_KERNEL | __GFP_NOWARN); + if (unlikely(!vec)) + return NULL; + vec->order = order; + vec->len = block_nr; + pg_vec = vec->pg_vec; + + if (tx_ring) { + vec->deferred = kzalloc_obj(*vec->deferred, + GFP_KERNEL | __GFP_NOWARN); + if (!vec->deferred) + goto out_free_pgvec; + vec->deferred->vec = vec; + INIT_DELAYED_WORK(&vec->deferred->work, + packet_free_pg_vec_work); + } for (i = 0; i < block_nr; i++) { pg_vec[i].buffer = alloc_one_pg_vec_page(order); @@ -4509,7 +4609,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, err = -ENOMEM; order = get_order(req->tp_block_size); - pg_vec = alloc_pg_vec(req, order); + pg_vec = alloc_pg_vec(req, order, tx_ring); if (unlikely(!pg_vec)) goto out; switch (po->tp_version) { @@ -4561,6 +4661,9 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, err = -EBUSY; mutex_lock(&po->pg_vec_lock); if (closing || atomic_long_read(&po->mapped) == 0) { + if (tx_ring && !closing && packet_read_pending(rb)) + goto out_unlock; + err = 0; spin_lock_bh(&rb_queue->lock); swap(rb->pg_vec, pg_vec); @@ -4582,6 +4685,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, pr_err("packet_mmap: vma is busy: %ld\n", atomic_long_read(&po->mapped)); } +out_unlock: mutex_unlock(&po->pg_vec_lock); spin_lock(&po->bind_lock); @@ -4603,7 +4707,10 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, out_free_pg_vec: if (pg_vec) { bitmap_free(rx_owner_map); - free_pg_vec(pg_vec, order, req->tp_block_nr); + if (tx_ring && closing) + packet_free_tx_ring(sk, pg_vec, order, req->tp_block_nr); + else + free_pg_vec(pg_vec, order, req->tp_block_nr); } out: return err; diff --git a/net/packet/internal.h b/net/packet/internal.h index b76e645cd78d1..f5c8cd0eed2c1 100644 --- a/net/packet/internal.h +++ b/net/packet/internal.h @@ -21,7 +21,7 @@ struct tpacket_kbdq_core { unsigned int hdrlen; unsigned char reset_pending_on_curr_blk; unsigned short kactive_blk_num; - unsigned short blk_sizeof_priv; + unsigned int blk_sizeof_priv; unsigned short version; diff --git a/net/phonet/pep.c b/net/phonet/pep.c index 31b29e3ca7bc6..e188c563ee5b0 100644 --- a/net/phonet/pep.c +++ b/net/phonet/pep.c @@ -1117,7 +1117,7 @@ static int pep_getsockopt(struct sock *sk, int level, int optname, len = min_t(unsigned int, sizeof(int), len); if (put_user(len, optlen)) return -EFAULT; - if (put_user(val, (int __user *) optval)) + if (copy_to_user(optval, &val, len)) return -EFAULT; return 0; } diff --git a/net/psp/psp_sock.c b/net/psp/psp_sock.c index 07dc4cf741f33..ce423d14ad267 100644 --- a/net/psp/psp_sock.c +++ b/net/psp/psp_sock.c @@ -143,6 +143,10 @@ int psp_sock_assoc_set_rx(struct sock *sk, struct psp_assoc *pas, NL_SET_ERR_MSG(extack, "Socket already has PSP state"); err = -EBUSY; goto exit_unlock; + } else if (sk_has_decrypt_user(sk)) { + NL_SET_ERR_MSG(extack, "Socket has incompatible state"); + err = -EINVAL; + goto exit_unlock; } refcount_inc(&pas->refcnt); diff --git a/net/rds/cong.c b/net/rds/cong.c index 3133b91f9e695..f7634ce3ffc10 100644 --- a/net/rds/cong.c +++ b/net/rds/cong.c @@ -256,9 +256,9 @@ void rds_cong_map_updated(struct rds_cong_map *map, uint64_t portmask) map, &map->m_addr); rds_stats_inc(s_cong_update_received); atomic_inc(&rds_cong_generation); - if (waitqueue_active(&map->m_waitq)) + if (wq_has_sleeper(&map->m_waitq)) wake_up(&map->m_waitq); - if (waitqueue_active(&rds_poll_waitq)) + if (wq_has_sleeper(&rds_poll_waitq)) wake_up_all(&rds_poll_waitq); if (portmask && !list_empty(&rds_cong_monitor)) { diff --git a/net/rds/connection.c b/net/rds/connection.c index 7c8ab8e973e1b..c752a8623cfca 100644 --- a/net/rds/connection.c +++ b/net/rds/connection.c @@ -106,10 +106,12 @@ static struct rds_connection *rds_conn_lookup(struct net *net, } /* - * This is called by transports as they're bringing down a connection. - * It clears partial message state so that the transport can start sending - * and receiving over this connection again in the future. It is up to - * the transport to have serialized this call with its send and recv. + * This is called by rds_conn_shutdown() once the transport has brought + * a path down. It clears partial message state so that the transport + * can start sending and receiving over this path again in the future. + * The caller owns RDS_IN_XMIT and RDS_RECV_REFILL across this call, + * which is what serializes it against the send and receive-refill + * paths. */ static void rds_conn_path_reset(struct rds_conn_path *cp) { @@ -120,7 +122,16 @@ static void rds_conn_path_reset(struct rds_conn_path *cp) rds_stats_inc(s_conn_reset); rds_send_path_reset(cp); - cp->cp_flags = 0; + + /* Clear the bits the reset is responsible for individually: a + * blanket cp_flags = 0 is a plain store that can clobber a + * concurrent atomic read-modify-write on the same word. + * RDS_IN_XMIT and RDS_RECV_REFILL are held as locks by the + * caller, rds_conn_shutdown(), which releases them once the + * teardown is complete. + */ + clear_bit(RDS_LL_SEND_FULL, &cp->cp_flags); + clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); /* Do not clear next_rx_seq here, else we cannot distinguish * retransmitted packets from new packets, and will hand all @@ -265,6 +276,12 @@ static struct rds_connection *__rds_conn_create(struct net *net, conn->c_trans = trans; + /* The transport may just have been swapped for loopback; size the + * set of paths - which is also what rds_conn_destroy() tears down + * again - by the transport the connection actually uses. + */ + npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1); + init_waitqueue_head(&conn->c_hs_waitq); for (i = 0; i < npaths; i++) { __rds_conn_path_init(conn, &conn->c_path[i], @@ -406,28 +423,70 @@ void rds_conn_shutdown(struct rds_conn_path *cp) } mutex_unlock(&cp->cp_cm_lock); + /* Quiesce the transmit and receive-refill paths by + * acquiring their bit locks, not merely waiting for + * them to be released: with a plain wait, either path + * can re-take its lock the instant after we sample it + * clear and then run concurrently with the transport + * shutdown and the path reset below. Holding both + * locks across the teardown makes that structurally + * impossible. + */ wait_event(cp->cp_waitq, - !test_bit(RDS_IN_XMIT, &cp->cp_flags)); + !test_and_set_bit_lock(RDS_IN_XMIT, &cp->cp_flags)); wait_event(cp->cp_waitq, - !test_bit(RDS_RECV_REFILL, &cp->cp_flags)); + !test_and_set_bit(RDS_RECV_REFILL, &cp->cp_flags)); conn->c_trans->conn_path_shutdown(cp); rds_conn_path_reset(cp); + /* Release the two locks and wake any waiter (e.g. + * rds_tcp_reset_callbacks()) that blocked on them while + * we held them. The unlock orders the transport's ring + * re-initialization and the path reset above before + * either bit is seen clear. rds_conn_path_reset() leaves + * both bits alone: ownership ends here, not inside the + * reset. + */ + clear_bit_unlock(RDS_IN_XMIT, &cp->cp_flags); + clear_bit_unlock(RDS_RECV_REFILL, &cp->cp_flags); + wake_up_all(&cp->cp_waitq); + if (!rds_conn_path_transition(cp, RDS_CONN_DISCONNECTING, - RDS_CONN_DOWN) && - !rds_conn_path_transition(cp, RDS_CONN_ERROR, RDS_CONN_DOWN)) { - /* This can happen - eg when we're in the middle of tearing - * down the connection, and someone unloads the rds module. - * Quite reproducible with loopback connections. - * Mostly harmless. + /* The path was dropped again while we tore it + * down: by a socket state-change callback in + * irq context on receipt of a FIN, or by an + * accept that claimed the path just before a + * drop put it back to RDS_CONN_ERROR and then + * installed a fresh socket on it. Unless a + * pending destroy suppressed it, the drop also + * queued another shutdown pass, and that pass + * must run, because it is what tears down + * whatever attached to the path after the + * transport shutdown above sampled its state. + * Consuming the RDS_CONN_ERROR here would turn + * that pass into a no-op: leave the state + * alone, and let the pass finish the job. * - * Note that this also happens with rds-tcp because - * we could have triggered rds_conn_path_drop in irq - * mode from rds_tcp_state change on the receipt of - * a FIN, thus we need to recheck for RDS_CONN_ERROR - * here. + * Quiesce the reconnect timer before bailing + * out, though. When a pending destroy did + * suppress the queue, no later pass runs, and + * rds_conn_path_destroy() is about to flush + * cp_down_w and free the path: it must not + * find cp_conn_w still armed. A successor + * pass, when there is one, re-arms the + * reconnect from its own tail. + */ + cancel_delayed_work_sync(&cp->cp_conn_w); + clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); + + if (rds_conn_path_state(cp) == RDS_CONN_ERROR) + return; + /* No current cp_state writer leaves a + * DISCONNECTING path in any state but + * RDS_CONN_ERROR; report loudly if one ever + * does. */ rds_conn_path_error(cp, "%s: failed to transition " "to state DOWN, current state " diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c index d46146887ba40..2909da8363f39 100644 --- a/net/rds/ib_cm.c +++ b/net/rds/ib_cm.c @@ -115,7 +115,7 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_even &conn->c_laddr, &conn->c_faddr, RDS_PROTOCOL_MAJOR(conn->c_version), RDS_PROTOCOL_MINOR(conn->c_version)); - rds_conn_destroy(conn); + rds_conn_drop(conn); return; } } diff --git a/net/rds/ib_frmr.c b/net/rds/ib_frmr.c index bd861191157b5..8397aa4a17ac8 100644 --- a/net/rds/ib_frmr.c +++ b/net/rds/ib_frmr.c @@ -204,19 +204,16 @@ static int rds_ib_map_frmr(struct rds_ib_device *rds_ibdev, */ rds_ib_teardown_mr(ibmr); - ibmr->sg = sg; - ibmr->sg_len = sg_len; - ibmr->sg_dma_len = 0; frmr->sg_byte_len = 0; - WARN_ON(ibmr->sg_dma_len); - ibmr->sg_dma_len = ib_dma_map_sg(dev, ibmr->sg, ibmr->sg_len, + ibmr->sg_dma_len = ib_dma_map_sg(dev, sg, sg_len, DMA_BIDIRECTIONAL); if (unlikely(!ibmr->sg_dma_len)) { pr_warn("RDS/IB: %s failed!\n", __func__); return -EBUSY; } - frmr->sg_byte_len = 0; + ibmr->sg = sg; + ibmr->sg_len = sg_len; frmr->dma_npages = 0; len = 0; @@ -264,6 +261,8 @@ static int rds_ib_map_frmr(struct rds_ib_device *rds_ibdev, ib_dma_unmap_sg(rds_ibdev->dev, ibmr->sg, ibmr->sg_len, DMA_BIDIRECTIONAL); ibmr->sg_dma_len = 0; + ibmr->sg = NULL; + ibmr->sg_len = 0; return ret; } diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c index 357128d34a546..bd6cb3ffaa571 100644 --- a/net/rds/ib_recv.c +++ b/net/rds/ib_recv.c @@ -363,15 +363,14 @@ static int acquire_refill(struct rds_connection *conn) static void release_refill(struct rds_connection *conn) { - clear_bit(RDS_RECV_REFILL, &conn->c_flags); - smp_mb__after_atomic(); + clear_bit_unlock(RDS_RECV_REFILL, &conn->c_flags); /* We don't use wait_on_bit()/wake_up_bit() because our waking is in a * hot path and finding waiters is very rare. We don't want to walk * the system-wide hashed waitqueue buckets in the fast path only to * almost never find waiters. */ - if (waitqueue_active(&conn->c_waitq)) + if (wq_has_sleeper(&conn->c_waitq)) wake_up_all(&conn->c_waitq); } @@ -392,7 +391,9 @@ void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp) /* the goal here is to just make sure that someone, somewhere * is posting buffers. If we can't get the refill lock, - * let them do their thing + * let them do their thing. The holder may also be + * rds_conn_shutdown() tearing the path down, in which case + * there is nothing to post. */ if (!acquire_refill(conn)) return; diff --git a/net/rds/message.c b/net/rds/message.c index 7feb0eb6537db..9cbf045fe0031 100644 --- a/net/rds/message.c +++ b/net/rds/message.c @@ -405,7 +405,9 @@ struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned in for (i = 0; i < rm->data.op_nents; ++i) { sg_set_page(&rm->data.op_sg[i], virt_to_page((void *)page_addrs[i]), - PAGE_SIZE, 0); + i == rm->data.op_nents - 1 + ? total_len - (i * PAGE_SIZE) + : PAGE_SIZE, 0); } return rm; diff --git a/net/rds/send.c b/net/rds/send.c index 68be1bf0e0adf..ffb3dd9998380 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -114,8 +114,13 @@ static void release_in_xmit(struct rds_conn_path *cp) * hot path and finding waiters is very rare. We don't want to walk * the system-wide hashed waitqueue buckets in the fast path only to * almost never find waiters. + * + * wq_has_sleeper() supplies the full barrier that orders the wait + * queue read after the bit clear; clear_bit_unlock() alone is only + * a release and would let this check read a stale empty queue, + * losing the wake-up. */ - if (waitqueue_active(&cp->cp_waitq)) + if (wq_has_sleeper(&cp->cp_waitq)) wake_up_all(&cp->cp_waitq); } @@ -231,8 +236,11 @@ int rds_send_xmit(struct rds_conn_path *cp) WRITE_ONCE(cp->cp_send_gen, send_gen); /* - * rds_conn_shutdown() sets the conn state and then tests RDS_IN_XMIT, - * we do the opposite to avoid races. + * rds_conn_shutdown() sets the conn state and then acquires + * RDS_IN_XMIT; we take the lock first and then check the state. + * Ownership is decided by the atomic RMW on the cp_flags word: + * if the teardown won the bit we back off here, and if we won + * it the teardown waits until we release it. */ if (!rds_conn_path_up(cp)) { release_in_xmit(cp); diff --git a/net/rds/tcp.c b/net/rds/tcp.c index b263634ac750d..774a71f88d375 100644 --- a/net/rds/tcp.c +++ b/net/rds/tcp.c @@ -115,46 +115,90 @@ void rds_tcp_restore_callbacks(struct socket *sock, } /* - * rds_tcp_reset_callbacks() switches the to the new sock and - * returns the existing tc->t_sock. + * rds_tcp_reset_callbacks() switches a path to a new socket and + * releases the old one it finds in tc->t_sock, resolving a duelling + * SYN. * - * The only functions that set tc->t_sock are rds_tcp_set_callbacks - * and rds_tcp_reset_callbacks. Send and receive trust that - * it is set. The absence of RDS_CONN_UP bit protects those paths - * from being called while it isn't set. + * tc->t_sock is set by rds_tcp_set_callbacks() and cleared by + * rds_tcp_restore_callbacks(). Four paths write it: the active + * connect in rds_tcp_conn_path_connect(), which sets it and clears it + * again on failure; the accept path in rds_tcp_accept_one(), which + * sets it for a path with no socket yet; the teardown in + * rds_tcp_conn_path_shutdown(), which clears it; and the swap done + * here, which does both. The connect and accept paths are serialized + * against each other by t_conn_path_lock. Send and receive trust + * that it is set: the absence of RDS_CONN_UP protects those paths + * from being called while it isn't, and the swap done here runs under + * RDS_IN_XMIT so that it cannot interleave with a sender already + * inside rds_send_xmit(). */ void rds_tcp_reset_callbacks(struct socket *sock, struct rds_conn_path *cp) { struct rds_tcp_connection *tc = cp->cp_transport_data; - struct socket *osock = tc->t_sock; - - if (!osock) - goto newsock; + struct socket *osock; /* Need to resolve a duelling SYN between peers. * We have an outstanding SYN to this peer, which may * potentially have transitioned to the RDS_CONN_UP state, * so we must quiesce any send threads before resetting - * cp_transport_data. We quiesce these threads by setting - * cp_state to something other than RDS_CONN_UP, and then - * waiting for any existing threads in rds_send_xmit to - * complete release_in_xmit(). (Subsequent threads entering - * rds_send_xmit() will bail on !rds_conn_up(). + * cp_transport_data. Setting cp_state to something other + * than RDS_CONN_UP stops new senders, and owning RDS_IN_XMIT + * excludes any thread already inside rds_send_xmit() - or a + * teardown in rds_conn_shutdown(), which holds the same lock + * for the duration of the transport shutdown - for the whole + * socket swap and the rds_send_path_reset() below. * - * However an incoming syn-ack at this point would end up - * marking the conn as RDS_CONN_UP, and would again permit - * rds_send_xmi() threads through, so ideally we would - * synchronize on RDS_CONN_UP after lock_sock(), but cannot - * do that: waiting on !RDS_IN_XMIT after lock_sock() may - * end up deadlocking with tcp_sendmsg(), and the RDS_IN_XMIT - * would not get set. As a result, we set c_state to - * RDS_CONN_RESETTTING, to ensure that rds_tcp_state_change - * cannot mark rds_conn_path_up() in the window before lock_sock() + * An incoming syn-ack at this point would end up marking the + * conn as RDS_CONN_UP, and would again permit rds_send_xmit() + * threads through, so ideally we would synchronize on + * RDS_CONN_UP after lock_sock(), but cannot do that: acquiring + * RDS_IN_XMIT after lock_sock() may end up deadlocking with + * tcp_sendmsg(), which takes the socket lock while holding + * RDS_IN_XMIT. As a result, we set c_state to + * RDS_CONN_RESETTING, to ensure that rds_tcp_state_change + * cannot mark rds_conn_path_up() in the window before + * lock_sock(). + * + * Only make that transition if the path is still connecting + * (or already resetting from an earlier duel). A path in any + * other state - typically RDS_CONN_DISCONNECTING or + * RDS_CONN_ERROR with a shutdown in flight - is dropped + * instead. That still replaces its state, with RDS_CONN_ERROR, + * and, unless a pending destroy is about to reap the whole + * connection anyway, queues one more shutdown pass. A shutdown + * already in flight leaves that RDS_CONN_ERROR alone when it + * finishes; the queued pass then completes the transition to + * RDS_CONN_DOWN and tears down anything that attached to the + * path in the meantime. + */ + if (!rds_conn_path_transition(cp, RDS_CONN_CONNECTING, + RDS_CONN_RESETTING) && + !rds_conn_path_transition(cp, RDS_CONN_RESETTING, + RDS_CONN_RESETTING)) + rds_conn_path_drop(cp, 0); + wait_event(cp->cp_waitq, + !test_and_set_bit_lock(RDS_IN_XMIT, &cp->cp_flags)); + + /* Read t_sock only while owning RDS_IN_XMIT, never before the + * wait: the teardown in rds_conn_shutdown() releases the old + * socket and clears t_sock, so a pointer sampled earlier can + * be stale by the time we wake up. The teardown holds the + * same lock while it does so, so what we read here cannot + * change under us until we release it. + */ + osock = tc->t_sock; + if (!osock) + goto newsock; + + /* reset receive side state for rds_tcp_data_recv() for osock. + * + * The sync cancels while owning RDS_IN_XMIT rely on cp_wq + * being ordered: a teardown blocked on the bit occupies + * cp_wq's only execution slot, so cp_send_w and cp_recv_w are + * pending at most and the cancels never flush. Nothing here + * may flush or wait on cp_wq itself. */ - atomic_set(&cp->cp_state, RDS_CONN_RESETTING); - wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags)); - /* reset receive side state for rds_tcp_data_recv() for osock */ cancel_delayed_work_sync(&cp->cp_send_w); cancel_delayed_work_sync(&cp->cp_recv_w); lock_sock(osock->sk); @@ -172,6 +216,9 @@ void rds_tcp_reset_callbacks(struct socket *sock, lock_sock(sock->sk); rds_tcp_set_callbacks(sock, cp); release_sock(sock->sk); + + clear_bit_unlock(RDS_IN_XMIT, &cp->cp_flags); + wake_up_all(&cp->cp_waitq); } /* Add tc to rds_tcp_tc_list and set tc->t_sock. See comments diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c index a3db9b057084d..13fa60c1985bb 100644 --- a/net/rds/tcp_listen.c +++ b/net/rds/tcp_listen.c @@ -295,7 +295,11 @@ int rds_tcp_accept_one(struct rds_tcp_net *rtn) if (rs_tcp->t_sock) { /* Duelling SYN has been handled in rds_tcp_accept_one() */ rds_tcp_reset_callbacks(new_sock, cp); - /* rds_connect_path_complete() marks RDS_CONN_UP */ + /* rds_connect_path_complete() marks RDS_CONN_UP, or, + * if a concurrent shutdown won the duel, drops the + * path again and the pass that drop queues reaps the + * socket installed above. + */ rds_connect_path_complete(cp, RDS_CONN_RESETTING); } else { rds_tcp_set_callbacks(new_sock, cp); diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 600b7804befd2..e45a63be397ce 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -443,16 +443,28 @@ static size_t tcf_action_shared_attrs_size(const struct tc_action *act) + nla_total_size(IFNAMSIZ) /* TCA_ACT_KIND */ + cookie_len /* TCA_ACT_COOKIE */ + nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_HW_STATS */ + /* TCA_ACT_USED_HW_STATS */ + + nla_total_size(sizeof(struct nla_bitfield32)) + + nla_total_size(sizeof(u32)) /* TCA_ACT_IN_HW_COUNT */ + nla_total_size(0) /* TCA_ACT_STATS nested */ + nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_FLAGS */ /* TCA_STATS_BASIC */ + nla_total_size_64bit(sizeof(struct gnet_stats_basic)) - /* TCA_STATS_PKT64 */ - + nla_total_size_64bit(sizeof(u64)) + /* TCA_STATS_BASIC_HW */ + + nla_total_size_64bit(sizeof(struct gnet_stats_basic)) + /* TCA_STATS_PKT64, emitted by both of the basic copies above */ + + 2 * nla_total_size_64bit(sizeof(u64)) + /* TCA_STATS_RATE_EST */ + + nla_total_size_64bit(sizeof(struct gnet_stats_rate_est)) + /* TCA_STATS_RATE_EST64 */ + + nla_total_size_64bit(sizeof(struct gnet_stats_rate_est64)) /* TCA_STATS_QUEUE */ + nla_total_size_64bit(sizeof(struct gnet_stats_queue)) + nla_total_size(0) /* TCA_ACT_OPTIONS nested */ - + nla_total_size(sizeof(struct tcf_t)); /* TCA_GACT_TM */ + /* TCA_GACT_TM; actions dump their tcf_t with nla_put_64bit(), + * which may emit an extra NLA_PAD attribute. + */ + + nla_total_size_64bit(sizeof(struct tcf_t)); } static size_t tcf_action_full_attrs_size(size_t sz) @@ -746,7 +758,7 @@ static int tcf_idr_delete_index(struct tcf_idrinfo *idrinfo, u32 index) mutex_lock(&idrinfo->lock); p = idr_find(&idrinfo->action_idr, index); - if (!p) { + if (IS_ERR_OR_NULL(p)) { mutex_unlock(&idrinfo->lock); return -ENOENT; } @@ -1188,18 +1200,13 @@ EXPORT_SYMBOL(tcf_action_exec); int tcf_action_destroy(struct tc_action *actions[], int bind) { - const struct tc_action_ops *ops; struct tc_action *a; int ret = 0, i; tcf_act_for_each_action(i, a, actions) { actions[i] = NULL; - ops = a->ops; - ret = __tcf_idr_release(a, bind, true); - if (ret == ACT_P_DELETED) - module_put(ops->owner); - else if (ret < 0) - return ret; + /* Drop our reference even if the action is still bound to a filter. */ + ret = tcf_idr_release(a, bind); } return ret; } @@ -1211,11 +1218,16 @@ static int tcf_action_put(struct tc_action *p) static void tcf_action_put_many(struct tc_action *actions[]) { - struct tc_action *a; int i; - tcf_act_for_each_action(i, a, actions) { - const struct tc_action_ops *ops = a->ops; + /* Deletion may have cleared entries before failing. */ + for (i = 0; i < TCA_ACT_MAX_PRIO; i++) { + struct tc_action *a = actions[i]; + const struct tc_action_ops *ops; + + if (!a) + continue; + ops = a->ops; if (tcf_action_put(a)) module_put(ops->owner); } @@ -1685,12 +1697,12 @@ static int tca_get_fill(struct sk_buff *skb, struct tc_action *actions[], static int tcf_get_notify(struct net *net, u32 portid, struct nlmsghdr *n, - struct tc_action *actions[], int event, + struct tc_action *actions[], size_t attr_size, int event, struct netlink_ext_ack *extack) { struct sk_buff *skb; - skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); + skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); if (!skb) return -ENOBUFS; if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, 0, event, @@ -1855,11 +1867,13 @@ static int tcf_action_delete(struct net *net, struct tc_action *actions[]) static struct sk_buff *tcf_reoffload_del_notify_msg(struct net *net, struct tc_action *action) { - size_t attr_size = tcf_action_fill_size(action); struct tc_action *actions[TCA_ACT_MAX_PRIO] = { [0] = action, }; struct sk_buff *skb; + size_t attr_size; + + attr_size = tcf_action_full_attrs_size(tcf_action_fill_size(action)); skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); if (!skb) @@ -1876,15 +1890,18 @@ static struct sk_buff *tcf_reoffload_del_notify_msg(struct net *net, static int tcf_reoffload_del_notify(struct net *net, struct tc_action *action) { const struct tc_action_ops *ops = action->ops; - struct sk_buff *skb; + struct sk_buff *skb = NULL; int ret; - if (!rtnl_notify_needed(net, 0, RTNLGRP_TC)) { - skb = NULL; - } else { + if (rtnl_notify_needed(net, 0, RTNLGRP_TC)) { skb = tcf_reoffload_del_notify_msg(net, action); + /* The action has already lost its hardware instance and is + * skip_sw, so it must be released whether or not the + * notification can be built. Drop the notification rather + * than leave an action behind that processes no packets. + */ if (IS_ERR(skb)) - return PTR_ERR(skb); + skb = NULL; } ret = tcf_idr_release_unsafe(action); @@ -2041,7 +2058,8 @@ tca_action_gd(struct net *net, struct nlattr *nla, struct nlmsghdr *n, attr_size = tcf_action_full_attrs_size(attr_size); if (event == RTM_GETACTION) - ret = tcf_get_notify(net, portid, n, actions, event, extack); + ret = tcf_get_notify(net, portid, n, actions, attr_size, event, + extack); else { /* delete */ ret = tcf_del_notify(net, n, actions, portid, attr_size, extack); if (ret) diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index 09d46e195e33b..06d8f78b73683 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -389,6 +389,31 @@ static void tcf_bpf_cleanup(struct tc_action *act) tcf_bpf_cfg_cleanup(&tmp); } +static size_t tcf_bpf_get_fill_size(const struct tc_action *act) +{ + struct tcf_bpf *prog = to_bpf(act); + size_t size = nla_total_size(sizeof(struct tc_act_bpf)); + + /* bpf_ops and bpf_num_ops are published as separate stores under + * tcf_lock, so take it here as tcf_bpf_dump() does. + */ + spin_lock_bh(&prog->tcf_lock); + if (tcf_bpf_is_ebpf(prog)) { + /* TCA_ACT_BPF_NAME */ + size += nla_total_size(ACT_BPF_NAME_LEN + 1); + size += nla_total_size(sizeof(u32)); /* TCA_ACT_BPF_ID */ + size += nla_total_size(BPF_TAG_SIZE); /* TCA_ACT_BPF_TAG */ + } else { + size += nla_total_size(sizeof(u16)); /* TCA_ACT_BPF_OPS_LEN */ + /* TCA_ACT_BPF_OPS */ + size += nla_total_size(prog->bpf_num_ops * + sizeof(struct sock_filter)); + } + spin_unlock_bh(&prog->tcf_lock); + + return size; +} + static struct tc_action_ops act_bpf_ops __read_mostly = { .kind = "bpf", .id = TCA_ID_BPF, @@ -397,6 +422,7 @@ static struct tc_action_ops act_bpf_ops __read_mostly = { .dump = tcf_bpf_dump, .cleanup = tcf_bpf_cleanup, .init = tcf_bpf_init, + .get_fill_size = tcf_bpf_get_fill_size, .size = sizeof(struct tcf_bpf), }; MODULE_ALIAS_NET_ACT("bpf"); diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index e250969c84aca..1d62bb654149c 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -432,11 +432,10 @@ static void tcf_ct_flow_table_add(struct tcf_ct_flow_table *ct_ft, if (test_and_set_bit(IPS_OFFLOAD_BIT, &ct->status)) return; + /* NULL if ct is dying (raced flush) or the atomic alloc failed. */ entry = flow_offload_alloc(ct); - if (!entry) { - WARN_ON_ONCE(1); + if (!entry) goto err_alloc; - } if (tcp) { ct->proto.tcp.seen[0].flags |= IP_CT_TCP_FLAG_BE_LIBERAL; @@ -980,11 +979,11 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) { struct net *net = dev_net(skb->dev); + bool cached, commit, clear, nat; enum ip_conntrack_info ctinfo; struct tcf_ct *c = to_ct(a); struct nf_conn *tmpl = NULL; struct nf_hook_state state; - bool cached, commit, clear; int nh_ofs, err, retval; struct tcf_ct_params *p; bool add_helper = false; @@ -999,6 +998,7 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, retval = p->action; commit = p->ct_action & TCA_CT_ACT_COMMIT; clear = p->ct_action & TCA_CT_ACT_CLEAR; + nat = p->ct_action & TCA_CT_ACT_NAT; tmpl = p->tmpl; tcf_lastuse_update(&c->tcf_tm); @@ -1047,6 +1047,19 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, * different zone. */ cached = tcf_ct_skb_nfct_cached(net, skb, p); + + /* If the ct entry is not confirmed and shared with some other skb, + * e.g., a cloned one, we can't just modify it with a commit or nat + * as we must not modify the extension set. Reset. + */ + if (cached && (commit || nat)) { + ct = nf_ct_get(skb, &ctinfo); + if (ct && !nf_ct_is_confirmed(ct) && nf_ct_shared(ct)) { + nf_reset_ct(skb); + cached = false; + } + } + if (!cached) { if (tcf_ct_flow_table_lookup(p, skb, family)) { skip_add = true; @@ -1084,25 +1097,31 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, if (err) goto drop; add_helper = true; - if (p->ct_action & TCA_CT_ACT_NAT && !nfct_seqadj(ct)) { + if (nat && !nfct_seqadj(ct)) { if (!nfct_seqadj_ext_add(ct)) goto drop; } } - if (nf_ct_is_confirmed(ct) ? ((!cached && !skip_add) || add_helper) : commit) { - err = nf_ct_helper(skb, ct, ctinfo, family); - if (err != NF_ACCEPT) - goto nf_error; - } - if (commit) { tcf_ct_act_set_mark(ct, p->mark, p->mark_mask); tcf_ct_act_set_labels(ct, p->labels, p->labels_mask); if (!nf_ct_is_confirmed(ct)) nf_conn_act_ct_ext_add(skb, ct, ctinfo); + } + + /* Run helpers for the connection if nf_conntrack_in() was executed + * or if we're about to commit. This has to be done after all the + * extensions are already added. + */ + if (nf_ct_is_confirmed(ct) ? ((!cached && !skip_add) || add_helper) : commit) { + err = nf_ct_helper(skb, ct, ctinfo, family); + if (err != NF_ACCEPT) + goto nf_error; + } + if (commit) { /* This will take care of sending queued events * even if the connection is already confirmed. */ @@ -1657,6 +1676,51 @@ static int tcf_ct_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_ct_get_fill_size(const struct tc_action *act) +{ + const struct tcf_ct_params *p; + size_t size; + + size = nla_total_size(sizeof(struct tc_ct)) /* TCA_CT_PARMS */ + + nla_total_size(sizeof(u16)); /* TCA_CT_ACTION */ + + rcu_read_lock(); + p = rcu_dereference(to_ct(act)->params); + + if (p->ct_action & TCA_CT_ACT_CLEAR) + goto out; + + /* TCA_CT_MARK, TCA_CT_MARK_MASK */ + if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) + size += nla_total_size(sizeof(p->mark)) + + nla_total_size(sizeof(p->mark_mask)); + + /* TCA_CT_LABELS, TCA_CT_LABELS_MASK */ + if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) + size += nla_total_size(sizeof(p->labels)) + + nla_total_size(sizeof(p->labels_mask)); + + if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) + size += nla_total_size(sizeof(p->zone)); /* TCA_CT_ZONE */ + + if (p->ct_action & TCA_CT_ACT_NAT) + /* TCA_CT_NAT_IPV6_{MIN,MAX}, the larger of the two address + * variants, plus TCA_CT_NAT_PORT_{MIN,MAX}. + */ + size += 2 * nla_total_size(sizeof(struct in6_addr)) + + 2 * nla_total_size(sizeof(__be16)); + + /* TCA_CT_HELPER_{NAME,FAMILY,PROTO} */ + if (p->helper) + size += nla_total_size(NF_CT_HELPER_NAME_LEN) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); +out: + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_ct_ops = { .kind = "ct", .id = TCA_ID_CT, @@ -1666,6 +1730,7 @@ static struct tc_action_ops act_ct_ops = { .init = tcf_ct_init, .cleanup = tcf_ct_cleanup, .stats_update = tcf_stats_update, + .get_fill_size = tcf_ct_get_fill_size, .offload_act_setup = tcf_ct_offload_act_setup, .size = sizeof(struct tcf_ct), }; diff --git a/net/sched/act_ctinfo.c b/net/sched/act_ctinfo.c index 1886ffd2ca956..fced4b1094af8 100644 --- a/net/sched/act_ctinfo.c +++ b/net/sched/act_ctinfo.c @@ -356,6 +356,16 @@ static void tcf_ctinfo_cleanup(struct tc_action *a) kfree_rcu(cp, rcu); } +static size_t tcf_ctinfo_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_ctinfo)) /* TCA_CTINFO_ACT */ + + nla_total_size(sizeof(u16)) /* TCA_CTINFO_ZONE */ + /* TCA_CTINFO_PARMS_{DSCP_MASK,DSCP_STATEMASK,CPMARK_MASK} */ + + 3 * nla_total_size(sizeof(u32)) + /* TCA_CTINFO_STATS_{DSCP_SET,DSCP_ERROR,CPMARK_SET} */ + + 3 * nla_total_size_64bit(sizeof(u64)); +} + static struct tc_action_ops act_ctinfo_ops = { .kind = "ctinfo", .id = TCA_ID_CTINFO, @@ -364,6 +374,7 @@ static struct tc_action_ops act_ctinfo_ops = { .dump = tcf_ctinfo_dump, .init = tcf_ctinfo_init, .cleanup= tcf_ctinfo_cleanup, + .get_fill_size = tcf_ctinfo_get_fill_size, .size = sizeof(struct tcf_ctinfo), }; MODULE_ALIAS_NET_ACT("ctinfo"); diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c index fdbfcaa3e2ab9..a3c96519936ba 100644 --- a/net/sched/act_gate.c +++ b/net/sched/act_gate.c @@ -681,7 +681,35 @@ static void tcf_gate_stats_update(struct tc_action *a, u64 bytes, u64 packets, static size_t tcf_gate_get_fill_size(const struct tc_action *act) { - return nla_total_size(sizeof(struct tc_gate)); + struct tcf_gate *gact = to_gate(act); + const struct tcf_gate_params *p; + struct tcfg_gate_entry *entry; + size_t size = nla_total_size(sizeof(struct tc_gate)) /* TCA_GATE_PARMS */ + + 3 * nla_total_size_64bit(sizeof(u64)) /* TCA_GATE_BASE_TIME + * TCA_GATE_CYCLE_TIME + * TCA_GATE_CYCLE_TIME_EXT + */ + + nla_total_size(sizeof(s32)) /* TCA_GATE_CLOCKID */ + + nla_total_size(sizeof(u32)) /* TCA_GATE_FLAGS */ + + nla_total_size(sizeof(s32)) /* TCA_GATE_PRIORITY */ + + nla_total_size(0); /* TCA_GATE_ENTRY_LIST */ + /* TCA_GATE_TM is budgeted by tcf_action_shared_attrs_size() */ + + rcu_read_lock(); + p = rcu_dereference(gact->param); + if (p) { + list_for_each_entry_rcu(entry, &p->entries, list) + /* TCA_GATE_ONE_ENTRY nest and its attributes */ + size += nla_total_size(0) + + nla_total_size(sizeof(u32)) /* TCA_GATE_ENTRY_INDEX */ + + nla_total_size(0) /* TCA_GATE_ENTRY_GATE */ + + nla_total_size(sizeof(u32)) /* TCA_GATE_ENTRY_INTERVAL */ + + nla_total_size(sizeof(s32)) /* TCA_GATE_ENTRY_MAX_OCTETS */ + + nla_total_size(sizeof(s32)); /* TCA_GATE_ENTRY_IPV */ + } + rcu_read_unlock(); + + return size; } static void tcf_gate_entry_destructor(void *priv) diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 065228026c58e..2afd68983ece4 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -723,7 +724,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, tcf_lastuse_update(&ife->tcf_tm); if (skb_at_tc_ingress(skb)) - skb_push(skb, skb->dev->hard_header_len); + skb_push(skb, ETH_HLEN); tlv_data = ife_decode(skb, &metalen); if (unlikely(!tlv_data)) { @@ -736,6 +737,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, u8 *curr_data; u16 mtype; u16 dlen; + int ret; curr_data = ife_tlv_meta_decode(tlv_data, ifehdr_end, &mtype, &dlen, NULL); @@ -744,13 +746,19 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, return TC_ACT_SHOT; } - if (find_decode_metaid(skb, p, mtype, dlen, curr_data)) { - /* abuse overlimits to count when we receive metadata - * but dont have an ops for it + ret = find_decode_metaid(skb, p, mtype, dlen, curr_data); + if (ret < 0) { + /* abuse overlimits to count metadata we cannot + * decode: no ops for it, or the decoder rejected it */ - pr_info_ratelimited("Unknown metaid %d dlen %d\n", - mtype, dlen); qstats_cpu_overlimit_inc(ife->common.cpu_qstats); + + if (ret == -ENOENT) + pr_info_ratelimited("Unknown metaid %d dlen %d\n", + mtype, dlen); + else + pr_info_ratelimited("Failed to decode metaid %d dlen %d err %d\n", + mtype, dlen, ret); } } @@ -795,7 +803,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, where ORIGDATA = original ethernet header ... */ u16 metalen = ife_get_sz(skb, p); - int hdrm = metalen + skb->dev->hard_header_len + IFE_METAHDRLEN; + int hdrm = metalen + ETH_HLEN + IFE_METAHDRLEN; unsigned int skboff = 0; int new_len = skb->len + hdrm; bool exceed_mtu = false; @@ -826,7 +834,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, } if (skb_at_tc_ingress(skb)) - skb_push(skb, skb->dev->hard_header_len); + skb_push(skb, ETH_HLEN); ife_meta = ife_encode(skb, metalen); if (!ife_meta) @@ -856,11 +864,27 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, oethh->h_proto = htons(p->eth_type); if (skb_at_tc_ingress(skb)) - skb_pull(skb, skb->dev->hard_header_len); + skb_pull(skb, ETH_HLEN); return action; } +/* IFE encapsulates the original Ethernet header and, on decode, expects to + * find one, so it can only ever work on skbs that carry one. Loopback carries + * Ethernet header as well, so it qualifies here. + * At ingress, also verify that the L2 header about to be pushed back really + * is an Ethernet header because the skb could've been redirected with mirred + * from a non-Ethernet device. + */ +static bool tcf_ife_is_eth_skb(const struct sk_buff *skb) +{ + if (skb->dev->type != ARPHRD_ETHER && + skb->dev->type != ARPHRD_LOOPBACK) + return false; + + return !skb_at_tc_ingress(skb) || skb->mac_len == ETH_HLEN; +} + TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) @@ -869,6 +893,13 @@ TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, struct tcf_ife_params *p; int ret; + if (unlikely(!tcf_ife_is_eth_skb(skb))) { + bstats_update(this_cpu_ptr(ife->common.cpu_bstats), skb); + tcf_lastuse_update(&ife->tcf_tm); + qstats_cpu_drop_inc(ife->common.cpu_qstats); + return TC_ACT_SHOT; + } + p = rcu_dereference_bh(ife->params); if (p->flags & IFE_ENCODE) { ret = tcf_ife_encode(skb, a, res, p); @@ -878,6 +909,28 @@ TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, return tcf_ife_decode(skb, a, res); } +static size_t tcf_ife_get_fill_size(const struct tc_action *act) +{ + struct tcf_ife_info *ife = to_ife(act); + const struct tcf_ife_params *p; + struct tcf_meta_info *e; + size_t size = nla_total_size(sizeof(struct tc_ife)) /* TCA_IFE_PARMS */ + + nla_total_size(ETH_ALEN) /* TCA_IFE_DMAC */ + + nla_total_size(ETH_ALEN) /* TCA_IFE_SMAC */ + + nla_total_size(2) /* TCA_IFE_TYPE */ + + nla_total_size(0); /* TCA_IFE_METALST */ + + rcu_read_lock(); + p = rcu_dereference(ife->params); + if (p) { + list_for_each_entry_rcu(e, &p->metalist, metalist) + size += nla_total_size(sizeof(u32)); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_ife_ops = { .kind = "ife", .id = TCA_ID_IFE, @@ -886,6 +939,7 @@ static struct tc_action_ops act_ife_ops = { .dump = tcf_ife_dump, .cleanup = tcf_ife_cleanup, .init = tcf_ife_init, + .get_fill_size = tcf_ife_get_fill_size, .size = sizeof(struct tcf_ife_info), }; MODULE_ALIAS_NET_ACT("ife"); diff --git a/net/sched/act_meta_mark.c b/net/sched/act_meta_mark.c index ea0573cb8b2d6..e2f61b22bf0f8 100644 --- a/net/sched/act_meta_mark.c +++ b/net/sched/act_meta_mark.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -28,9 +29,10 @@ static int skbmark_encode(struct sk_buff *skb, void *skbdata, static int skbmark_decode(struct sk_buff *skb, void *data, u16 len) { - u32 ifemark = *(u32 *)data; + if (len != sizeof(u32)) + return -EINVAL; - skb->mark = ntohl(ifemark); + skb->mark = get_unaligned_be32(data); return 0; } diff --git a/net/sched/act_meta_skbprio.c b/net/sched/act_meta_skbprio.c index 2df3133ce5adc..5cdb57931eab4 100644 --- a/net/sched/act_meta_skbprio.c +++ b/net/sched/act_meta_skbprio.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -33,9 +34,10 @@ static int skbprio_encode(struct sk_buff *skb, void *skbdata, static int skbprio_decode(struct sk_buff *skb, void *data, u16 len) { - u32 ifeprio = *(u32 *)data; + if (len != sizeof(u32)) + return -EINVAL; - skb->priority = ntohl(ifeprio); + skb->priority = get_unaligned_be32(data); return 0; } diff --git a/net/sched/act_meta_skbtcindex.c b/net/sched/act_meta_skbtcindex.c index 44547caead469..8803710c09058 100644 --- a/net/sched/act_meta_skbtcindex.c +++ b/net/sched/act_meta_skbtcindex.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -28,9 +29,10 @@ static int skbtcindex_encode(struct sk_buff *skb, void *skbdata, static int skbtcindex_decode(struct sk_buff *skb, void *data, u16 len) { - u16 ifetc_index = *(u16 *)data; + if (len != sizeof(u16)) + return -EINVAL; - skb->tc_index = ntohs(ifetc_index); + skb->tc_index = get_unaligned_be16(data); return 0; } diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index d4d47a9921f45..99d7e36510bd0 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -626,6 +626,29 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_pedit_get_fill_size(const struct tc_action *act) +{ + const struct tcf_pedit_parms *parms; + size_t size; + + rcu_read_lock(); + parms = rcu_dereference(to_pedit(act)->parms); + size = nla_total_size(struct_size_t(struct tc_pedit, keys, + parms->tcfp_nkeys)); + if (parms->tcfp_keys_ex) { + /* TCA_PEDIT_KEYS_EX, holding one TCA_PEDIT_KEY_EX nest with a + * HTYPE and a CMD attribute per key. + */ + size += nla_total_size(0) + + parms->tcfp_nkeys * (nla_total_size(0) + + nla_total_size(sizeof(u16)) + + nla_total_size(sizeof(u16))); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_pedit_ops = { .kind = "pedit", .id = TCA_ID_PEDIT, @@ -635,6 +658,7 @@ static struct tc_action_ops act_pedit_ops = { .dump = tcf_pedit_dump, .cleanup = tcf_pedit_cleanup, .init = tcf_pedit_init, + .get_fill_size = tcf_pedit_get_fill_size, .offload_act_setup = tcf_pedit_offload_act_setup, .size = sizeof(struct tcf_pedit), }; diff --git a/net/sched/act_police.c b/net/sched/act_police.c index ce08f6840ef7c..3f8147f375493 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -490,6 +490,17 @@ static int tcf_police_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_police_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_police)) /* TCA_POLICE_TBF */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_RATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PEAKRATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PKTRATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PKTBURST64 */ + + nla_total_size(sizeof(u32)) /* TCA_POLICE_RESULT */ + + nla_total_size(sizeof(u32)); /* TCA_POLICE_AVRATE */ +} + MODULE_AUTHOR("Alexey Kuznetsov"); MODULE_DESCRIPTION("Policing actions"); MODULE_LICENSE("GPL"); @@ -503,6 +514,7 @@ static struct tc_action_ops act_police_ops = { .dump = tcf_police_dump, .init = tcf_police_init, .cleanup = tcf_police_cleanup, + .get_fill_size = tcf_police_get_fill_size, .offload_act_setup = tcf_police_offload_act_setup, .size = sizeof(struct tcf_police), }; diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c index 2ceb4d141b713..44319a159b55d 100644 --- a/net/sched/act_sample.c +++ b/net/sched/act_sample.c @@ -315,6 +315,14 @@ static int tcf_sample_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_sample_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_sample)) /* TCA_SAMPLE_PARMS */ + + nla_total_size(sizeof(u32)) /* TCA_SAMPLE_RATE */ + + nla_total_size(sizeof(u32)) /* TCA_SAMPLE_TRUNC_SIZE */ + + nla_total_size(sizeof(u32)); /* TCA_SAMPLE_PSAMPLE_GROUP */ +} + static struct tc_action_ops act_sample_ops = { .kind = "sample", .id = TCA_ID_SAMPLE, @@ -324,6 +332,7 @@ static struct tc_action_ops act_sample_ops = { .dump = tcf_sample_dump, .init = tcf_sample_init, .cleanup = tcf_sample_cleanup, + .get_fill_size = tcf_sample_get_fill_size, .get_psample_group = tcf_sample_get_group, .offload_act_setup = tcf_sample_offload_act_setup, .size = sizeof(struct tcf_sample), diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c index a464b0a3c1b81..7579cf1e0ff37 100644 --- a/net/sched/act_skbmod.c +++ b/net/sched/act_skbmod.c @@ -38,7 +38,6 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, if (unlikely(p->action == TC_ACT_SHOT)) goto drop; - max_edit_len = skb_mac_header_len(skb); flags = p->flags; /* tcf_skbmod_init() guarantees "flags" to be one of the following: @@ -51,14 +50,19 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, if (flags == SKBMOD_F_ECN) { switch (skb_protocol(skb, true)) { case cpu_to_be16(ETH_P_IP): + max_edit_len = sizeof(struct iphdr); + break; case cpu_to_be16(ETH_P_IPV6): - max_edit_len += skb_network_header_len(skb); + max_edit_len = sizeof(struct ipv6hdr); break; default: goto out; } - } else if (!skb->dev || skb->dev->type != ARPHRD_ETHER) { - goto out; + max_edit_len += skb_network_offset(skb); + } else { + if (!skb->dev || skb->dev->type != ARPHRD_ETHER) + goto out; + max_edit_len = ETH_HLEN; } err = skb_ensure_writable(skb, max_edit_len); diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index b14807761d829..ff401ace4f3da 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -835,6 +835,85 @@ static int tcf_tunnel_key_offload_act_setup(struct tc_action *act, return 0; } +static size_t +tunnel_key_geneve_opts_fill_size(const struct ip_tunnel_info *info) +{ + const u8 *src = ip_tunnel_info_opts(info); + int len = info->options_len; + size_t size = 0; + + while (len > 0) { + const struct geneve_opt *opt = (const struct geneve_opt *)src; + + /* TCA_TUNNEL_KEY_ENC_OPT_GENEVE_{CLASS,TYPE,DATA} */ + size += nla_total_size(2) + + nla_total_size(1) + + nla_total_size(opt->length * 4); + + len -= sizeof(struct geneve_opt) + opt->length * 4; + src += sizeof(struct geneve_opt) + opt->length * 4; + } + + return size; +} + +static size_t tunnel_key_opts_fill_size(const struct ip_tunnel_info *info) +{ + size_t size; + + if (!info->options_len) + return 0; + + /* TCA_TUNNEL_KEY_ENC_OPTS and the per-protocol nest inside it */ + size = nla_total_size(0) + nla_total_size(0); + + if (test_bit(IP_TUNNEL_GENEVE_OPT_BIT, info->key.tun_flags)) { + size += tunnel_key_geneve_opts_fill_size(info); + } else if (test_bit(IP_TUNNEL_VXLAN_OPT_BIT, info->key.tun_flags)) { + /* TCA_TUNNEL_KEY_ENC_OPT_VXLAN_GBP */ + size += nla_total_size(sizeof(u32)); + } else if (test_bit(IP_TUNNEL_ERSPAN_OPT_BIT, info->key.tun_flags)) { + /* TCA_TUNNEL_KEY_ENC_OPT_ERSPAN_{VER,INDEX,DIR,HWID} */ + size += nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(__be32)) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); + } + + return size; +} + +static size_t tunnel_key_get_fill_size(const struct tc_action *act) +{ + struct tcf_tunnel_key *t = to_tunnel_key(act); + const struct tcf_tunnel_key_params *params; + /* TCA_TUNNEL_KEY_PARMS */ + size_t size = nla_total_size(sizeof(struct tc_tunnel_key)); + + rcu_read_lock(); + params = rcu_dereference(t->params); + if (params->tcft_action == TCA_TUNNEL_KEY_ACT_SET) { + const struct ip_tunnel_info *info = + ¶ms->tcft_enc_metadata->u.tun_info; + + /* In dump order: TCA_TUNNEL_KEY_ENC_KEY_ID, the IPv6 address + * pair (larger than the IPv4 one), ..._ENC_DST_PORT, + * ..._NO_CSUM, ..._NO_FRAG, the options and ..._ENC_{TOS,TTL}. + */ + size += nla_total_size(sizeof(__be32)) + + 2 * nla_total_size(sizeof(struct in6_addr)) + + nla_total_size(sizeof(__be16)) + + nla_total_size(sizeof(u8)) + + nla_total_size(0) + + tunnel_key_opts_fill_size(info) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_tunnel_key_ops = { .kind = "tunnel_key", .id = TCA_ID_TUNNEL_KEY, @@ -843,6 +922,7 @@ static struct tc_action_ops act_tunnel_key_ops = { .dump = tunnel_key_dump, .init = tunnel_key_init, .cleanup = tunnel_key_release, + .get_fill_size = tunnel_key_get_fill_size, .offload_act_setup = tcf_tunnel_key_offload_act_setup, .size = sizeof(struct tcf_tunnel_key), }; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 4e6a2812a4f32..c47d2ee13641d 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -2248,6 +2248,12 @@ static bool is_ingress_or_clsact(struct tcf_block *block, struct Qdisc *q) return tcf_block_shared(block) || (q && !!(q->flags & TCQ_F_INGRESS)); } +enum tcf_tp_insert_state { + TP_NOT_CREATED = 0, /* did not create and insert a new tp */ + TP_CREATED, /* created and inserted a new tp */ + TP_NOT_OWNED, /* created a proto but failed to insert */ +}; + static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, struct netlink_ext_ack *extack) { @@ -2268,12 +2274,12 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, unsigned long cl; void *fh; int err; - int tp_created; + enum tcf_tp_insert_state tp_state; bool rtnl_held = false; u32 flags; replay: - tp_created = 0; + tp_state = TP_NOT_CREATED; err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); @@ -2395,13 +2401,15 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, goto errout_tp; } - tp_created = 1; + tp_state = TP_CREATED; tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio, rtnl_held); if (IS_ERR(tp)) { err = PTR_ERR(tp); goto errout_tp; } + if (tp != tp_new) + tp_state = TP_NOT_OWNED; } else { mutex_unlock(&chain->filter_chain_lock); } @@ -2455,13 +2463,13 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, } errout: - if (err && tp_created) + if (err && tp_state == TP_CREATED) tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL); errout_tp: if (chain) { if (tp && !IS_ERR(tp)) tcf_proto_put(tp, rtnl_held, NULL); - if (!tp_created) + if (tp_state == TP_NOT_CREATED) tcf_chain_put(chain); } tcf_block_release(q, block, rtnl_held); @@ -3246,7 +3254,7 @@ static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n, tcf_chain_put(chain); errout_block: tcf_block_release(q, block, true); - if (err == -EAGAIN) + if (err == -EAGAIN && n->nlmsg_type == RTM_NEWCHAIN) /* Replay the request. */ goto replay; return err; @@ -3364,7 +3372,8 @@ int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action, * This reference might be taken later from tcf_exts_get_net(). */ exts->net = net; - exts->actions = kzalloc_objs(struct tc_action *, TCA_ACT_MAX_PRIO); + exts->actions = kzalloc_objs(struct tc_action *, TCA_ACT_MAX_PRIO, + GFP_KERNEL_ACCOUNT); if (!exts->actions) return -ENOMEM; #endif diff --git a/net/sched/cls_basic.c b/net/sched/cls_basic.c index 492cd9ce8d46f..e2a94ba9fba76 100644 --- a/net/sched/cls_basic.c +++ b/net/sched/cls_basic.c @@ -193,7 +193,7 @@ static int basic_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; @@ -212,9 +212,11 @@ static int basic_change(struct net *net, struct sk_buff *in_skb, if (err) goto errout; fnew->handle = handle; - fnew->pf = alloc_percpu(struct tc_basic_pcnt); + fnew->pf = alloc_percpu_gfp(struct tc_basic_pcnt, GFP_KERNEL_ACCOUNT); if (!fnew->pf) { err = -ENOMEM; + if (!fold) + idr_remove(&head->handle_idr, fnew->handle); goto errout; } diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index 6d19155becc86..188cf0f949dd4 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -352,7 +352,7 @@ static int cls_bpf_prog_from_ops(struct nlattr **tb, struct cls_bpf_prog *prog) if (bpf_size != nla_len(tb[TCA_BPF_OPS])) return -EINVAL; - bpf_ops = kmemdup(nla_data(tb[TCA_BPF_OPS]), bpf_size, GFP_KERNEL); + bpf_ops = kmemdup(nla_data(tb[TCA_BPF_OPS]), bpf_size, GFP_KERNEL_ACCOUNT); if (bpf_ops == NULL) return -ENOMEM; @@ -403,7 +403,7 @@ static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog, } if (tb[TCA_BPF_NAME]) { - name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL); + name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL_ACCOUNT); if (!name) { bpf_prog_put(fp); return -ENOMEM; @@ -443,7 +443,7 @@ static int cls_bpf_change(struct net *net, struct sk_buff *in_skb, if (ret < 0) return ret; - prog = kzalloc_obj(*prog); + prog = kzalloc_obj(*prog, GFP_KERNEL_ACCOUNT); if (!prog) return -ENOBUFS; diff --git a/net/sched/cls_cgroup.c b/net/sched/cls_cgroup.c index 680a5c308094e..210fd9fd26d83 100644 --- a/net/sched/cls_cgroup.c +++ b/net/sched/cls_cgroup.c @@ -95,7 +95,7 @@ static int cls_cgroup_change(struct net *net, struct sk_buff *in_skb, if (head && handle != head->handle) return -ENOENT; - new = kzalloc_obj(*head); + new = kzalloc_obj(*head, GFP_KERNEL_ACCOUNT); if (!new) return -ENOBUFS; diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index 356c68ebc3895..a9ac3acf6eda0 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -438,7 +438,7 @@ static int flow_change(struct net *net, struct sk_buff *in_skb, return -EOPNOTSUPP; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index 88f8a32fab2b4..0e275b58151c4 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -2233,7 +2233,7 @@ static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head, struct fl_flow_mask *newmask; int err; - newmask = kzalloc_obj(*newmask); + newmask = kzalloc_obj(*newmask, GFP_KERNEL_ACCOUNT); if (!newmask) return ERR_PTR(-ENOMEM); @@ -2394,7 +2394,7 @@ static int fl_change(struct net *net, struct sk_buff *in_skb, goto errout_tb; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) { err = -ENOBUFS; goto errout_tb; diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c index 646a730dca93c..a462b262719c3 100644 --- a/net/sched/cls_fw.c +++ b/net/sched/cls_fw.c @@ -276,7 +276,7 @@ static int fw_change(struct net *net, struct sk_buff *in_skb, if (f->id != handle && handle) return -EINVAL; - fnew = kzalloc_obj(struct fw_filter); + fnew = kzalloc_obj(struct fw_filter, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; @@ -330,7 +330,7 @@ static int fw_change(struct net *net, struct sk_buff *in_skb, rcu_assign_pointer(tp->root, head); } - f = kzalloc_obj(struct fw_filter); + f = kzalloc_obj(struct fw_filter, GFP_KERNEL_ACCOUNT); if (f == NULL) return -ENOBUFS; diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c index 6f126872c14a0..c14899b935bfd 100644 --- a/net/sched/cls_matchall.c +++ b/net/sched/cls_matchall.c @@ -189,7 +189,7 @@ static int mall_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; } - new = kzalloc_obj(*new); + new = kzalloc_obj(*new, GFP_KERNEL_ACCOUNT); if (!new) return -ENOBUFS; @@ -201,7 +201,7 @@ static int mall_change(struct net *net, struct sk_buff *in_skb, handle = 1; new->handle = handle; new->flags = userflags; - new->pf = alloc_percpu(struct tc_matchall_pcnt); + new->pf = alloc_percpu_gfp(struct tc_matchall_pcnt, GFP_KERNEL_ACCOUNT); if (!new->pf) { err = -ENOMEM; goto err_alloc_percpu; diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c index eded7aacd3f70..0f211f030fd9a 100644 --- a/net/sched/cls_route.c +++ b/net/sched/cls_route.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,7 @@ struct route4_head { struct route4_bucket { /* 16 FROM buckets + 16 IIF buckets + 1 wildcard bucket */ struct route4_filter __rcu *ht[16 + 16 + 1]; + refcount_t filters_ref; struct rcu_head rcu; }; @@ -336,7 +338,7 @@ static int route4_delete(struct tcf_proto *tp, void *arg, bool *last, struct route4_filter *nf; struct route4_bucket *b; unsigned int h = 0; - int i, h1; + int h1; if (!head || !f) return -EINVAL; @@ -362,23 +364,14 @@ static int route4_delete(struct tcf_proto *tp, void *arg, bool *last, tcf_exts_get_net(&f->exts); tcf_queue_work(&f->rwork, route4_delete_filter_work); - /* Strip RTNL protected tree */ - for (i = 0; i <= 32; i++) { - struct route4_filter *rt; - - rt = rtnl_dereference(b->ht[i]); - if (rt) - goto out; + if (refcount_dec_and_test(&b->filters_ref)) { + RCU_INIT_POINTER(head->table[to_hash(h)], NULL); + kfree_rcu(b, rcu); } - - /* OK, session has no flows */ - RCU_INIT_POINTER(head->table[to_hash(h)], NULL); - kfree_rcu(b, rcu); break; } } -out: *last = true; for (h1 = 0; h1 <= 256; h1++) { if (rcu_access_pointer(head->table[h1])) { @@ -400,8 +393,9 @@ static const struct nla_policy route4_policy[TCA_ROUTE4_MAX + 1] = { static int route4_set_parms(struct net *net, struct tcf_proto *tp, unsigned long base, struct route4_filter *f, u32 handle, struct route4_head *head, - struct nlattr **tb, struct nlattr *est, int new, - u32 flags, struct netlink_ext_ack *extack) + struct nlattr **tb, struct nlattr *est, + struct route4_filter *fold, u32 flags, + struct netlink_ext_ack *extack) { u32 id = 0, to = 0, nhandle = 0x8000; struct route4_filter *fp; @@ -414,7 +408,7 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, return err; if (tb[TCA_ROUTE4_TO]) { - if (new && handle & 0x8000) { + if (!fold && handle & 0x8000) { NL_SET_ERR_MSG(extack, "Invalid handle"); return -EINVAL; } @@ -437,14 +431,14 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, } else nhandle |= 0xFFFF << 16; - if (handle && new) { + if (handle && (!fold || nhandle == (handle & ~0x7F00))) nhandle |= handle & 0x7F00; - if (nhandle != handle) { - NL_SET_ERR_MSG_FMT(extack, - "Handle mismatch constructed: %x (expected: %x)", - handle, nhandle); - return -EINVAL; - } + + if (handle && !fold && nhandle != handle) { + NL_SET_ERR_MSG_FMT(extack, + "Handle mismatch constructed: %x (expected: %x)", + handle, nhandle); + return -EINVAL; } if (!nhandle) { @@ -455,10 +449,11 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, h1 = to_hash(nhandle); b = rtnl_dereference(head->table[h1]); if (!b) { - b = kzalloc_obj(struct route4_bucket); + b = kzalloc_obj(struct route4_bucket, GFP_KERNEL_ACCOUNT); if (b == NULL) return -ENOBUFS; + refcount_set(&b->filters_ref, 1); rcu_assign_pointer(head->table[h1], b); } else { unsigned int h2 = from_hash(nhandle >> 16); @@ -466,8 +461,14 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, for (fp = rtnl_dereference(b->ht[h2]); fp; fp = rtnl_dereference(fp->next)) - if (fp->handle == f->handle) + if (fp != fold && fp->handle == nhandle) { + NL_SET_ERR_MSG_FMT(extack, + "Handle %x is already in use", + nhandle); return -EEXIST; + } + + refcount_inc(&b->filters_ref); } if (tb[TCA_ROUTE4_TO]) @@ -500,9 +501,8 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, struct route4_filter *fold, *f1, *pfp, *f = NULL; struct route4_bucket *b; struct nlattr *tb[TCA_ROUTE4_MAX + 1]; - unsigned int h, th; + unsigned int h; int err; - bool new = true; if (!handle) { NL_SET_ERR_MSG(extack, "Creating with handle of 0 is invalid"); @@ -524,7 +524,7 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; err = -ENOBUFS; - f = kzalloc_obj(struct route4_filter); + f = kzalloc_obj(struct route4_filter, GFP_KERNEL_ACCOUNT); if (!f) goto errout; @@ -539,11 +539,10 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, f->tp = fold->tp; f->bkt = fold->bkt; - new = false; } err = route4_set_parms(net, tp, base, f, handle, head, tb, - tca[TCA_RATE], new, flags, extack); + tca[TCA_RATE], fold, flags, extack); if (err < 0) goto errout; @@ -560,17 +559,20 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, rcu_assign_pointer(*fp, f); if (fold) { - th = to_hash(fold->handle); + b = fold->bkt; h = from_hash(fold->handle >> 16); - b = rtnl_dereference(head->table[th]); - if (b) { - fp = &b->ht[h]; - for (pfp = rtnl_dereference(*fp); pfp; - fp = &pfp->next, pfp = rtnl_dereference(*fp)) { - if (pfp == fold) { - rcu_assign_pointer(*fp, fold->next); - break; + fp = &b->ht[h]; + for (pfp = rtnl_dereference(*fp); pfp; + fp = &pfp->next, pfp = rtnl_dereference(*fp)) { + if (pfp == fold) { + rcu_assign_pointer(*fp, fold->next); + if (refcount_dec_and_test(&b->filters_ref)) { + unsigned int th = to_hash(fold->handle); + + RCU_INIT_POINTER(head->table[th], NULL); + kfree_rcu(b, rcu); } + break; } } } diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index c297d7dbcf916..76ce2d124079d 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -370,6 +370,10 @@ static int u32_init(struct tcf_proto *tp) refcount_set(&root_ht->refcnt, 1); root_ht->handle = tp_c ? gen_new_htid(tp_c, root_ht) : id2handle(0); + if (root_ht->handle == 0) { + kfree(root_ht); + return -ENOMEM; + } root_ht->prio = tp->prio; root_ht->is_root = true; idr_init(&root_ht->handle_idr); @@ -695,21 +699,33 @@ static int u32_delete(struct tcf_proto *tp, void *arg, bool *last, return ret; } -static u32 gen_new_kid(struct tc_u_hnode *ht, u32 htid) +static u32 gen_new_kid(struct tc_u_hnode *ht, u32 htid, int *err) { u32 index = htid | 0x800; u32 max = htid | 0xFFF; + *err = 0; + if (idr_alloc_u32(&ht->handle_idr, NULL, &index, max, GFP_KERNEL)) { index = htid + 1; - if (idr_alloc_u32(&ht->handle_idr, NULL, &index, max, - GFP_KERNEL)) - index = max; + *err = idr_alloc_u32(&ht->handle_idr, NULL, &index, max, + GFP_KERNEL); + if (*err) + return 0; } return index; } +static int u32_kid_extack(int err, struct netlink_ext_ack *extack) +{ + if (err == -ENOSPC) + NL_SET_ERR_MSG_MOD(extack, "Hash table node ID pool exhausted"); + else + NL_SET_ERR_MSG_MOD(extack, "Failed to allocate node ID"); + return err; +} + static const struct nla_policy u32_policy[TCA_U32_MAX + 1] = { [TCA_U32_CLASSID] = { .type = NLA_U32 }, [TCA_U32_HASH] = { .type = NLA_U32 }, @@ -825,7 +841,7 @@ static struct tc_u_knode *u32_init_knode(struct net *net, struct tcf_proto *tp, struct tc_u32_sel *s = &n->sel; struct tc_u_knode *new; - new = kzalloc_flex(*new, sel.keys, s->nkeys); + new = kzalloc_flex(*new, sel.keys, s->nkeys, GFP_KERNEL_ACCOUNT); if (!new) return NULL; @@ -987,8 +1003,16 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, return -ENOMEM; } } else { - err = idr_alloc_u32(&tp_c->handle_idr, ht, &handle, - handle, GFP_KERNEL); + /* The IDR is keyed on the mapped id, and that is + * what the destroy paths remove. Ask for it here, + * so a manual handle colliding with the + * auto-allocated id space is rejected (-ENOSPC) + * instead of aliasing a future auto id. + */ + u32 id = handle2id(handle); + + err = idr_alloc_u32(&tp_c->handle_idr, ht, &id, id, + GFP_KERNEL); if (err) { kfree(ht); return err; @@ -1079,7 +1103,9 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, * handle which is used to uniquely identify the match entry. */ if (!TC_U32_NODE(handle)) { - handle = gen_new_kid(ht, htid); + handle = gen_new_kid(ht, htid, &err); + if (err) + return u32_kid_extack(err, extack); } else { handle = htid | TC_U32_NODE(handle); err = idr_alloc_u32(&ht->handle_idr, NULL, &handle, @@ -1091,7 +1117,9 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, /* The user did not give us a handle; lets just generate one * from the table's pool of nodeids. */ - handle = gen_new_kid(ht, htid); + handle = gen_new_kid(ht, htid, &err); + if (err) + return u32_kid_extack(err, extack); } if (tb[TCA_U32_SEL] == NULL) { @@ -1114,15 +1142,16 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, goto erridr; } - n = kzalloc_flex(*n, sel.keys, s->nkeys); + n = kzalloc_flex(*n, sel.keys, s->nkeys, GFP_KERNEL_ACCOUNT); if (n == NULL) { err = -ENOBUFS; goto erridr; } #ifdef CONFIG_CLS_U32_PERF - n->pf = __alloc_percpu(struct_size(n->pf, kcnts, s->nkeys), - __alignof__(struct tc_u32_pcnt)); + n->pf = __alloc_percpu_gfp(struct_size(n->pf, kcnts, s->nkeys), + __alignof__(struct tc_u32_pcnt), + GFP_KERNEL_ACCOUNT); if (!n->pf) { err = -ENOBUFS; goto errfree; @@ -1144,7 +1173,7 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, goto errout; #ifdef CONFIG_CLS_U32_MARK - n->pcpu_success = alloc_percpu(u32); + n->pcpu_success = alloc_percpu_gfp(u32, GFP_KERNEL_ACCOUNT); if (!n->pcpu_success) { err = -ENOMEM; goto errout; diff --git a/net/sched/em_text.c b/net/sched/em_text.c index 343f1aebeec2a..4132f8c3c5fc9 100644 --- a/net/sched/em_text.c +++ b/net/sched/em_text.c @@ -113,7 +113,7 @@ static void em_text_destroy(struct tcf_ematch *m) static int em_text_dump(struct sk_buff *skb, struct tcf_ematch *m) { struct text_match *tm = EM_TEXT_PRIV(m); - struct tcf_em_text conf; + struct tcf_em_text conf = {}; strscpy(conf.algo, tm->config->ops->name); conf.from_offset = tm->from_offset; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 65b35528d125f..463ededcdcfe0 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -610,8 +610,11 @@ void __qdisc_calculate_pkt_len(struct sk_buff *skb, pkt_len <<= stab->szopts.size_log; out: - if (unlikely(pkt_len < 1)) - pkt_len = 1; + /* A size table can inflate qdisc_pkt_len() beyond any real packet + * (via overhead, the data table, or size_log); cap it so deficit + * schedulers such as DRR/ETS terminate their refill loops. + */ + pkt_len = clamp_t(int, pkt_len, 1, QDISC_PKT_LEN_MAX); qdisc_skb_cb(skb)->pkt_len = pkt_len; } @@ -1382,7 +1385,7 @@ static struct Qdisc *qdisc_create(struct net_device *dev, err_out3: qdisc_lock_uninit(sch, ops); netdev_put(dev, &sch->dev_tracker); - qdisc_free(sch); + qdisc_free_rcu(sch); err_out2: bpf_module_put(ops, ops->owner); err_out: diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index f25f60978631a..dc93267029e70 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1907,6 +1907,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, ktime_add_ms(q->last_reconfig_time, 250))) { q->config->rate_bps = (q->avg_peak_bandwidth * 15) >> 4; + q->last_reconfig_time = now; cake_reconfigure(sch); } } diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index cacf5244958e8..6aa5829d69612 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -205,7 +205,7 @@ static int codel_init(struct Qdisc *sch, struct nlattr *opt, codel_params_init(&q->params); codel_vars_init(&q->vars); codel_stats_init(&q->stats); - q->params.mtu = psched_mtu(qdisc_dev(sch)); + q->params.mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, 1 << 20); if (opt) { int err = codel_change(sch, opt, extack); diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c index 91b1ef824afaf..8621d057edd9f 100644 --- a/net/sched/sch_drr.c +++ b/net/sched/sch_drr.c @@ -82,8 +82,9 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid, NL_SET_ERR_MSG(extack, "Specified DRR quantum cannot be zero"); return -EINVAL; } + quantum = clamp_t(u32, quantum, 256, 1 << 20); } else - quantum = psched_mtu(qdisc_dev(sch)); + quantum = clamp_t(u32, (u32)psched_mtu(qdisc_dev(sch)), 256, 1 << 20); if (cl != NULL) { if (tca[TCA_RATE]) { diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c index 4f678d4ff10ec..4947def7c49ef 100644 --- a/net/sched/sch_dualpi2.c +++ b/net/sched/sch_dualpi2.c @@ -208,9 +208,11 @@ static void dualpi2_reset_c_protection(struct dualpi2_sched_data *q) static void dualpi2_calculate_c_protection(struct Qdisc *sch, struct dualpi2_sched_data *q, u32 wc) { + u32 mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); + q->c_protection_wc = wc; q->c_protection_wl = MAX_WC - wc; - q->c_protection_init = (s32)psched_mtu(qdisc_dev(sch)) * + q->c_protection_init = (s32)mtu * ((int)q->c_protection_wc - (int)q->c_protection_wl); dualpi2_reset_c_protection(q); } @@ -285,8 +287,9 @@ static bool must_drop(struct Qdisc *sch, struct dualpi2_sched_data *q, u64 local_l_prob; bool overload; u32 prob; + u32 mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); - if (sch->qstats.backlog < 2 * psched_mtu(qdisc_dev(sch))) + if (sch->qstats.backlog < 2 * mtu) return false; prob = READ_ONCE(q->pi2_prob); @@ -712,7 +715,8 @@ static u32 get_memory_limit(struct Qdisc *sch, u32 limit) /* Apply rule of thumb, i.e., doubling the packet length, * to further include per packet overhead in memory_limit. */ - u64 memlim = mul_u32_u32(limit, 2 * psched_mtu(qdisc_dev(sch))); + u64 memlim = mul_u32_u32(limit, 2 * clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 1, 1 << 20)); if (upper_32_bits(memlim)) return U32_MAX; diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index 25fcf4079fece..6cc902a038387 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -83,11 +83,7 @@ static int ets_quantum_parse(struct Qdisc *sch, const struct nlattr *attr, unsigned int *quantum, struct netlink_ext_ack *extack) { - *quantum = nla_get_u32(attr); - if (!*quantum) { - NL_SET_ERR_MSG(extack, "ETS quantum cannot be zero"); - return -EINVAL; - } + *quantum = clamp_t(u32, nla_get_u32(attr), 256, 1 << 20); return 0; } @@ -632,11 +628,13 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt, return err; } /* If there are more bands than strict + quanta provided, the remaining - * ones are ETS with quantum of MTU. Initialize the missing values here. + * ones are ETS with quantum of max(MTU, 256). Initialize the missing + * values here. */ for (i = nstrict; i < nbands; i++) { if (!quanta[i]) - quanta[i] = psched_mtu(qdisc_dev(sch)); + quanta[i] = clamp_t(u32, (u32)psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); } /* Before commit, make sure we can allocate all new qdiscs */ diff --git a/net/sched/sch_fifo.c b/net/sched/sch_fifo.c index e6bfd39ff3396..1b6388d50967f 100644 --- a/net/sched/sch_fifo.c +++ b/net/sched/sch_fifo.c @@ -19,7 +19,7 @@ static int bfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) { - if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= + if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <= READ_ONCE(sch->limit))) return qdisc_enqueue_tail(skb, sch); diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index 7cae082a98472..35f940b2205df 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -828,8 +828,12 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch) * f->time_next_packet was set when prior packet was sent, * and current time (@now) can be too late by tens of us. */ - if (f->time_next_packet) - len -= min(len/2, now - f->time_next_packet); + if (f->time_next_packet) { + s64 drift = now - f->time_next_packet; + + if (drift > 0) + len -= min_t(u64, len / 2, drift); + } f->time_next_packet = now + len; } out: @@ -976,7 +980,7 @@ static int fq_resize(struct Qdisc *sch, u32 log) } static const struct netlink_range_validation iq_range = { - .max = INT_MAX, + .max = 1 << 20, }; static const struct nla_policy fq_policy[TCA_FQ_MAX + 1] = { @@ -1102,14 +1106,10 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt, nla_get_u32(tb[TCA_FQ_FLOW_PLIMIT])); if (tb[TCA_FQ_QUANTUM]) { - u32 quantum = nla_get_u32(tb[TCA_FQ_QUANTUM]); + u32 quantum = clamp_t(u32, nla_get_u32(tb[TCA_FQ_QUANTUM]), + 256, 1 << 20); - if (quantum > 0 && quantum <= (1 << 20)) { - WRITE_ONCE(q->quantum, quantum); - } else { - NL_SET_ERR_MSG_MOD(extack, "invalid quantum"); - err = -EINVAL; - } + WRITE_ONCE(q->quantum, quantum); } if (tb[TCA_FQ_INITIAL_QUANTUM]) @@ -1222,12 +1222,14 @@ static int fq_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct fq_sched_data *q = qdisc_priv(sch); + u32 mtu; int i, err; sch->limit = 10000; q->flow_plimit = 100; - q->quantum = 2 * psched_mtu(qdisc_dev(sch)); - q->initial_quantum = 10 * psched_mtu(qdisc_dev(sch)); + mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); + q->quantum = clamp_t(u32, 2 * mtu, 256, 1 << 20); + q->initial_quantum = min_t(u32, 10 * mtu, 1 << 20); q->flow_refill_delay = msecs_to_jiffies(40); q->flow_max_rate = ~0UL; q->time_next_delayed_flow = ~0ULL; diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 6cce86ba383cf..969b2510b0b86 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -509,6 +509,7 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct fq_codel_sched_data *q = qdisc_priv(sch); + u32 mtu; int i; int err; @@ -516,13 +517,14 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, q->flows_cnt = 1024; q->memory_limit = 32 << 20; /* 32 MBytes */ q->drop_batch_size = 64; - q->quantum = psched_mtu(qdisc_dev(sch)); + mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, FQ_CODEL_QUANTUM_MAX); + q->quantum = mtu; INIT_LIST_HEAD(&q->new_flows); INIT_LIST_HEAD(&q->old_flows); codel_params_init(&q->cparams); codel_stats_init(&q->cstats); q->cparams.ecn = true; - q->cparams.mtu = psched_mtu(qdisc_dev(sch)); + q->cparams.mtu = mtu; if (opt) { err = fq_codel_change(sch, opt, extack); diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 069e1facd4130..5982847df8f85 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -341,7 +341,8 @@ static int fq_pie_change(struct Qdisc *sch, struct nlattr *opt, nla_get_u32(tb[TCA_FQ_PIE_BETA])); if (tb[TCA_FQ_PIE_QUANTUM]) - WRITE_ONCE(q->quantum, nla_get_u32(tb[TCA_FQ_PIE_QUANTUM])); + WRITE_ONCE(q->quantum, + max(256U, nla_get_u32(tb[TCA_FQ_PIE_QUANTUM]))); if (tb[TCA_FQ_PIE_MEMORY_LIMIT]) WRITE_ONCE(q->memory_limit, @@ -427,7 +428,8 @@ static int fq_pie_init(struct Qdisc *sch, struct nlattr *opt, pie_params_init(&q->p_params); sch->limit = 10 * 1024; q->p_params.limit = sch->limit; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); q->sch = sch; q->ecn_prob = 10; q->flows_cnt = 1024; diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index ef2b4bf515641..6f6a6f0d5eb0d 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -1086,21 +1086,21 @@ void qdisc_reset(struct Qdisc *qdisc) } EXPORT_SYMBOL(qdisc_reset); -void qdisc_free(struct Qdisc *qdisc) +static void qdisc_free_cb(struct rcu_head *head) { - if (qdisc_is_percpu_stats(qdisc)) { - free_percpu(qdisc->cpu_bstats); - free_percpu(qdisc->cpu_qstats); + struct Qdisc *q = container_of(head, struct Qdisc, rcu); + + if (qdisc_is_percpu_stats(q)) { + free_percpu(q->cpu_bstats); + free_percpu(q->cpu_qstats); } - kfree(qdisc); + kfree(q); } -static void qdisc_free_cb(struct rcu_head *head) +void qdisc_free_rcu(struct Qdisc *qdisc) { - struct Qdisc *q = container_of(head, struct Qdisc, rcu); - - qdisc_free(q); + call_rcu(&qdisc->rcu, qdisc_free_cb); } static void __qdisc_destroy(struct Qdisc *qdisc) @@ -1127,7 +1127,7 @@ static void __qdisc_destroy(struct Qdisc *qdisc) trace_qdisc_destroy(qdisc); - call_rcu(&qdisc->rcu, qdisc_free_cb); + qdisc_free_rcu(qdisc); } void qdisc_destroy(struct Qdisc *qdisc) @@ -1278,7 +1278,7 @@ static void transition_one_qdisc(struct net_device *dev, rcu_assign_pointer(dev_queue->qdisc, new_qdisc); if (need_watchdog_p) { - WRITE_ONCE(dev_queue->trans_start, 0); + WRITE_ONCE(dev_queue->trans_start, jiffies); *need_watchdog_p = 1; } } diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c index fcc1a4c036362..f04f425c6c449 100644 --- a/net/sched/sch_gred.c +++ b/net/sched/sch_gred.c @@ -179,7 +179,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch, * if no default DP has been configured. This * allows for DP flows to be left untouched. */ - if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= + if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <= sch->limit)) return qdisc_enqueue_tail(skb, sch); else @@ -244,7 +244,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch, break; } - if (gred_backlog(t, q, sch) + qdisc_pkt_len(skb) <= q->limit) { + if ((u64)gred_backlog(t, q, sch) + qdisc_pkt_len(skb) <= q->limit) { q->backlog += qdisc_pkt_len(skb); return qdisc_enqueue_tail(skb, sch); } diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index e87f5021a1995..284490fd6ca91 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -386,6 +386,15 @@ cftree_update(struct hfsc_class *cl) #define SM_MASK ((1ULL << SM_SHIFT) - 1) #define ISM_MASK ((1ULL << ISM_SHIFT) - 1) +/* + * Cap on the non-descending hops a classify walk may take before its + * filter chain is treated as misconfigured. A flowid binding that was + * legal at bind time can become lateral once hfsc_adjust_levels() + * raises a class level; a few such hops are legitimate, an unbounded + * run means the chain cycles. + */ +#define HFSC_CLASSIFY_MAX_DRIFT 8 + static inline u64 seg_x2y(u64 x, u64 sm) { @@ -1133,6 +1142,7 @@ hfsc_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) struct hfsc_class *head, *cl; struct tcf_result res; struct tcf_proto *tcf; + unsigned int drift; int result; if (TC_H_MAJ(skb->priority ^ sch->handle) == 0 && @@ -1142,6 +1152,7 @@ hfsc_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; head = &q->root; + drift = HFSC_CLASSIFY_MAX_DRIFT; tcf = rcu_dereference_bh(q->root.filter_list); while (tcf && (result = tcf_classify_qdisc(skb, tcf, &res, false)) >= 0) { #ifdef CONFIG_NET_CLS_ACT @@ -1167,6 +1178,17 @@ hfsc_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) if (cl->level == 0) return cl; /* hit leaf class */ + /* + * flowid binds skip the level check above (res.class is set + * at bind time and levels drift after), so a walk can follow + * lateral hops without descending; a bounded number of them + * is legal, more means the chain cycles. + */ + if (cl->level >= head->level && drift-- == 0) { + pr_warn_ratelimited("hfsc: classify hop budget exhausted, dropping packet\n"); + return NULL; + } + /* apply inner filter chain */ tcf = rcu_dereference_bh(cl->filter_list); head = cl; diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index d85cb0263b675..5dec1ed969adf 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -527,7 +527,7 @@ static void hhf_destroy(struct Qdisc *sch) static const struct nla_policy hhf_policy[TCA_HHF_MAX + 1] = { [TCA_HHF_BACKLOG_LIMIT] = { .type = NLA_U32 }, [TCA_HHF_QUANTUM] = { .type = NLA_U32 }, - [TCA_HHF_HH_FLOWS_LIMIT] = { .type = NLA_U32 }, + [TCA_HHF_HH_FLOWS_LIMIT] = NLA_POLICY_MAX(NLA_U32, 2 * HH_FLOWS_CNT), [TCA_HHF_RESET_TIMEOUT] = { .type = NLA_U32 }, [TCA_HHF_ADMIT_BYTES] = { .type = NLA_U32 }, [TCA_HHF_EVICT_TIMEOUT] = { .type = NLA_U32 }, @@ -546,12 +546,12 @@ static int hhf_change(struct Qdisc *sch, struct nlattr *opt, u32 new_hhf_non_hh_weight = q->hhf_non_hh_weight; err = nla_parse_nested_deprecated(tb, TCA_HHF_MAX, opt, hhf_policy, - NULL); + extack); if (err < 0) return err; if (tb[TCA_HHF_QUANTUM]) - new_quantum = nla_get_u32(tb[TCA_HHF_QUANTUM]); + new_quantum = max(256U, nla_get_u32(tb[TCA_HHF_QUANTUM])); if (tb[TCA_HHF_NON_HH_WEIGHT]) new_hhf_non_hh_weight = nla_get_u32(tb[TCA_HHF_NON_HH_WEIGHT]); @@ -613,7 +613,7 @@ static int hhf_init(struct Qdisc *sch, struct nlattr *opt, int i; sch->limit = 1000; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, 1 << 20); get_random_bytes(&q->perturbation, sizeof(q->perturbation)); INIT_LIST_HEAD(&q->new_buckets); INIT_LIST_HEAD(&q->old_buckets); @@ -624,6 +624,9 @@ static int hhf_init(struct Qdisc *sch, struct nlattr *opt, q->hhf_evict_timeout = HZ; /* 1 sec */ q->hhf_non_hh_weight = 2; + /* Cap max active HHs at twice len of hh_flows table. */ + q->hh_flows_limit = 2 * HH_FLOWS_CNT; + if (opt) { int err = hhf_change(sch, opt, extack); @@ -639,8 +642,6 @@ static int hhf_init(struct Qdisc *sch, struct nlattr *opt, for (i = 0; i < HH_FLOWS_CNT; i++) INIT_LIST_HEAD(&q->hh_flows[i]); - /* Cap max active HHs at twice len of hh_flows table. */ - q->hh_flows_limit = 2 * HH_FLOWS_CNT; q->hh_flows_overlimit = 0; q->hh_flows_total_cnt = 0; q->hh_flows_current_cnt = 0; diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index fdac0dc8f35a3..1ba67b121de47 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -224,6 +224,7 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, struct htb_class *cl; struct tcf_result res; struct tcf_proto *tcf; + unsigned int hops = 0; int result; /* allow to select class by setting skb->priority to valid classid; @@ -266,6 +267,10 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, if (!cl->level) return cl; /* we hit leaf; return it */ + if (++hops > TC_HTB_MAXDEPTH) { + pr_warn_ratelimited("htb: classify loop detected, dropping packet\n"); + return NULL; + } /* we have got inner class; apply inner filter chain */ tcf = rcu_dereference_bh(cl->filter_list); } @@ -633,13 +638,11 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } else { return qdisc_drop(skb, sch, to_free); } -#ifdef CONFIG_NET_CLS_ACT } else if (!cl) { if (ret & __NET_XMIT_BYPASS) qdisc_qstats_drop(sch); __qdisc_drop(skb, to_free); return ret; -#endif } else if ((ret = qdisc_enqueue(skb, cl->leaf.q, to_free)) != NET_XMIT_SUCCESS) { if (net_xmit_drop_count(ret)) { diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c index ae991fc25b43f..6ced7008ef5c8 100644 --- a/net/sched/sch_mqprio.c +++ b/net/sched/sch_mqprio.c @@ -679,12 +679,14 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, rcu_read_lock(); if (cl >= TC_H_MIN_PRIORITY) { struct net_device *dev = qdisc_dev(sch); - struct netdev_tc_txq tc = dev->tc_to_txq[cl & TC_BITMASK]; + struct netdev_tc_txq tc; struct gnet_stats_queue qstats = {0}; struct gnet_stats_basic_sync bstats; u32 qlen = 0; int i; + tc.combined = READ_ONCE(dev->tc_to_txq[cl & TC_BITMASK].combined); + gnet_stats_basic_sync_init(&bstats); for (i = tc.offset; i < tc.offset + tc.count; i++) { diff --git a/net/sched/sch_mqprio_lib.c b/net/sched/sch_mqprio_lib.c index b3a5572c167b7..b60e130c70781 100644 --- a/net/sched/sch_mqprio_lib.c +++ b/net/sched/sch_mqprio_lib.c @@ -108,8 +108,11 @@ void mqprio_qopt_reconstruct(struct net_device *dev, struct tc_mqprio_qopt *qopt memcpy(qopt->prio_tc_map, dev->prio_tc_map, sizeof(qopt->prio_tc_map)); for (tc = 0; tc < num_tc; tc++) { - qopt->count[tc] = dev->tc_to_txq[tc].count; - qopt->offset[tc] = dev->tc_to_txq[tc].offset; + struct netdev_tc_txq res; + + res.combined = READ_ONCE(dev->tc_to_txq[tc].combined); + qopt->count[tc] = res.count; + qopt->offset[tc] = res.offset; } } EXPORT_SYMBOL_GPL(mqprio_qopt_reconstruct); diff --git a/net/sched/sch_pie.c b/net/sched/sch_pie.c index b41f2def2e2cc..3b7863ffd284c 100644 --- a/net/sched/sch_pie.c +++ b/net/sched/sch_pie.c @@ -35,7 +35,7 @@ bool pie_drop_early(struct Qdisc *sch, struct pie_params *params, { u64 rnd; u64 local_prob = vars->prob; - u32 mtu = psched_mtu(qdisc_dev(sch)); + u32 mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); /* If there is still burst allowance left skip random early drop */ if (vars->burst_time > 0) diff --git a/net/sched/sch_plug.c b/net/sched/sch_plug.c index cefb65201e178..b60ddfee6a68a 100644 --- a/net/sched/sch_plug.c +++ b/net/sched/sch_plug.c @@ -89,7 +89,7 @@ static int plug_enqueue(struct sk_buff *skb, struct Qdisc *sch, { struct plug_sched_data *q = qdisc_priv(sch); - if (likely(sch->qstats.backlog + skb->len <= q->limit)) { + if (likely((u64)sch->qstats.backlog + skb->len <= q->limit)) { if (!q->unplug_indefinite) q->pkts_current_epoch++; return qdisc_enqueue_tail(skb, sch); diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index 77675f9a4c46e..8bbcfc9e85d94 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -660,6 +660,11 @@ static int sfq_change(struct Qdisc *sch, struct nlattr *opt, return -EINVAL; } + if (ctl->quantum > 1 << 20) { + NL_SET_ERR_MSG_MOD(extack, "quantum too large"); + return -EINVAL; + } + if (ctl->perturb_period < 0 || ctl->perturb_period > INT_MAX / HZ) { NL_SET_ERR_MSG_MOD(extack, "invalid perturb period"); @@ -688,7 +693,7 @@ static int sfq_change(struct Qdisc *sch, struct nlattr *opt, /* update and validate configuration */ if (ctl->quantum) - quantum = ctl->quantum; + quantum = max(256U, ctl->quantum); if (ctl->flows) maxflows = min_t(u32, ctl->flows, SFQ_MAX_FLOWS); if (ctl->divisor) { @@ -799,7 +804,8 @@ static int sfq_init(struct Qdisc *sch, struct nlattr *opt, q->tail = NULL; q->divisor = SFQ_DEFAULT_HASH_DIVISOR; q->maxflows = SFQ_DEFAULT_FLOWS; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); q->perturb_period = 0; get_random_bytes(&q->perturbation, sizeof(q->perturbation)); diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index 299234a5f0fe6..7d5fe93a4c124 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -762,12 +762,13 @@ static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq, static void taprio_next_tc_txq(struct net_device *dev, int tc, int *txq) { - int offset = dev->tc_to_txq[tc].offset; - int count = dev->tc_to_txq[tc].count; + struct netdev_tc_txq res; + + res.combined = READ_ONCE(dev->tc_to_txq[tc].combined); (*txq)++; - if (*txq == offset + count) - *txq = offset; + if (*txq == res.offset + res.count) + *txq = res.offset; } /* Prioritize higher traffic classes, and select among TXQs belonging to the @@ -1441,15 +1442,14 @@ static u32 tc_map_to_queue_mask(struct net_device *dev, u32 tc_mask) u32 i, queue_mask = 0; for (i = 0; i < dev->num_tc; i++) { - u32 offset, count; + struct netdev_tc_txq res; if (!(tc_mask & BIT(i))) continue; - offset = dev->tc_to_txq[i].offset; - count = dev->tc_to_txq[i].count; + res.combined = READ_ONCE(dev->tc_to_txq[i].combined); - queue_mask |= GENMASK(offset + count - 1, offset); + queue_mask |= GENMASK(res.offset + res.count - 1, res.offset); } return queue_mask; @@ -1802,10 +1802,14 @@ static int taprio_mqprio_cmp(const struct net_device *dev, if (!mqprio || mqprio->num_tc != dev->num_tc) return -1; - for (i = 0; i < mqprio->num_tc; i++) - if (dev->tc_to_txq[i].count != mqprio->count[i] || - dev->tc_to_txq[i].offset != mqprio->offset[i]) + for (i = 0; i < mqprio->num_tc; i++) { + struct netdev_tc_txq res; + + res.combined = READ_ONCE(dev->tc_to_txq[i].combined); + if (res.count != mqprio->count[i] || + res.offset != mqprio->offset[i]) return -1; + } for (i = 0; i <= TC_BITMASK; i++) if (dev->prio_tc_map[i] != mqprio->prio_tc_map[i]) diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 5c42a29a981cb..409ce50cc0dbc 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -265,14 +265,11 @@ __teql_resolve(struct sk_buff *skb, struct sk_buff *skb_res, } if (neigh_event_send(n, skb_res) == 0) { - int err; char haddr[MAX_ADDR_LEN]; neigh_ha_snapshot(haddr, n, dev); - err = dev_hard_header(skb, dev, ntohs(skb_protocol(skb, false)), - haddr, NULL, skb->len); - - if (err < 0) + if (dev_hard_header(skb, dev, ntohs(skb_protocol(skb, false)), + haddr, NULL, skb->len) < 0) err = -EINVAL; } else { err = (skb_res == NULL) ? -EAGAIN : 1; @@ -377,6 +374,7 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) nores = 1; break; } + skb->dev = dev; __skb_pull(skb, skb_network_offset(skb)); } while ((q = rcu_dereference(NEXT_SLAVE(q))) != start); diff --git a/net/sctp/associola.c b/net/sctp/associola.c index 5be0bed2685ee..1220ff909fbe3 100644 --- a/net/sctp/associola.c +++ b/net/sctp/associola.c @@ -1285,18 +1285,19 @@ void sctp_assoc_update_retran_path(struct sctp_association *asoc) /* Manually skip the head element. */ if (&trans->transports == &asoc->peer.transport_addr_list) continue; - if (trans->state == SCTP_UNCONFIRMED) - continue; - trans_next = sctp_trans_elect_best(trans, trans_next); - /* Active is good enough for immediate return. */ - if (trans_next->state == SCTP_ACTIVE) - break; + if (trans->state != SCTP_UNCONFIRMED) { + trans_next = sctp_trans_elect_best(trans, trans_next); + /* Active is good enough for immediate return. */ + if (trans_next->state == SCTP_ACTIVE) + break; + } /* We've reached the end, time to update path. */ if (trans == asoc->peer.retran_path) break; } - asoc->peer.retran_path = trans_next; + if (trans_next) + asoc->peer.retran_path = trans_next; pr_debug("%s: association:%p updated new path to addr:%pISpc\n", __func__, asoc, &asoc->peer.retran_path->ipaddr.sa); diff --git a/net/sctp/input.c b/net/sctp/input.c index 864741fae4187..9494cfa51106c 100644 --- a/net/sctp/input.c +++ b/net/sctp/input.c @@ -436,9 +436,10 @@ void sctp_icmp_proto_unreachable(struct sock *sk, if (timer_pending(&t->proto_unreach_timer)) return; else { - if (!mod_timer(&t->proto_unreach_timer, - jiffies + (HZ/20))) - sctp_transport_hold(t); + sctp_transport_hold(t); + if (mod_timer(&t->proto_unreach_timer, + jiffies + (HZ / 20))) + sctp_transport_put(t); } } else { struct net *net = sock_net(sk); diff --git a/net/sctp/inqueue.c b/net/sctp/inqueue.c index a024c08432471..5f988b3a8814f 100644 --- a/net/sctp/inqueue.c +++ b/net/sctp/inqueue.c @@ -71,8 +71,11 @@ void sctp_inq_free(struct sctp_inq *queue) */ void sctp_inq_push(struct sctp_inq *q, struct sctp_chunk *chunk) { - /* Directly call the packet handling routine. */ - if (chunk->rcvr->dead) { + /* Directly call the packet handling routine. Drop the chunk if the + * receiver or the transport it was looked up on is gone. + */ + if (chunk->rcvr->dead || + (chunk->transport && chunk->transport->dead)) { sctp_chunk_free(chunk); return; } diff --git a/net/sctp/sm_make_chunk.c b/net/sctp/sm_make_chunk.c index 236e25abc7a42..84a4c97d0f755 100644 --- a/net/sctp/sm_make_chunk.c +++ b/net/sctp/sm_make_chunk.c @@ -3215,6 +3215,9 @@ bool sctp_verify_asconf(const struct sctp_association *asoc, *errp = param.p; switch (param.p->type) { case SCTP_PARAM_ERR_CAUSE: + if (length < sizeof(struct sctp_addip_param) + + sizeof(struct sctp_errhdr)) + return false; break; case SCTP_PARAM_IPV4_ADDRESS: if (length != sizeof(struct sctp_ipv4addr_param)) @@ -3448,20 +3451,15 @@ static __be16 sctp_get_asconf_response(struct sctp_chunk *asconf_ack, case SCTP_PARAM_ERR_CAUSE: length = sizeof(*asconf_ack_param); err_param = (void *)asconf_ack_param + length; - asconf_ack_len -= length; - if (asconf_ack_len > 0) - return err_param->cause; - else - return SCTP_ERROR_INV_PARAM; - break; + return err_param->cause; default: return SCTP_ERROR_INV_PARAM; } } length = ntohs(asconf_ack_param->param_hdr.length); - asconf_ack_param = (void *)asconf_ack_param + length; - asconf_ack_len -= length; + asconf_ack_param = (void *)asconf_ack_param + SCTP_PAD4(length); + asconf_ack_len -= SCTP_PAD4(length); } return err_code; diff --git a/net/sctp/sm_sideeffect.c b/net/sctp/sm_sideeffect.c index 424f10a6fdba9..35f540fb15fc1 100644 --- a/net/sctp/sm_sideeffect.c +++ b/net/sctp/sm_sideeffect.c @@ -244,8 +244,9 @@ void sctp_generate_t3_rtx_event(struct timer_list *t) pr_debug("%s: sock is busy\n", __func__); /* Try again later. */ - if (!mod_timer(&transport->T3_rtx_timer, jiffies + (HZ/20))) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->T3_rtx_timer, jiffies + (HZ / 20))) + sctp_transport_put(transport); goto out_unlock; } @@ -280,8 +281,9 @@ static void sctp_generate_timeout_event(struct sctp_association *asoc, timeout_type); /* Try again later. */ - if (!mod_timer(&asoc->timers[timeout_type], jiffies + (HZ/20))) - sctp_association_hold(asoc); + sctp_association_hold(asoc); + if (mod_timer(&asoc->timers[timeout_type], jiffies + (HZ / 20))) + sctp_association_put(asoc); goto out_unlock; } @@ -378,8 +380,9 @@ void sctp_generate_heartbeat_event(struct timer_list *t) pr_debug("%s: sock is busy\n", __func__); /* Try again later. */ - if (!mod_timer(&transport->hb_timer, jiffies + (HZ/20))) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->hb_timer, jiffies + (HZ / 20))) + sctp_transport_put(transport); goto out_unlock; } @@ -388,8 +391,9 @@ void sctp_generate_heartbeat_event(struct timer_list *t) timeout = sctp_transport_timeout(transport); if (elapsed < timeout) { elapsed = timeout - elapsed; - if (!mod_timer(&transport->hb_timer, jiffies + elapsed)) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->hb_timer, jiffies + elapsed)) + sctp_transport_put(transport); goto out_unlock; } @@ -422,9 +426,10 @@ void sctp_generate_proto_unreach_event(struct timer_list *t) pr_debug("%s: sock is busy\n", __func__); /* Try again later. */ - if (!mod_timer(&transport->proto_unreach_timer, - jiffies + (HZ/20))) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->proto_unreach_timer, + jiffies + (HZ / 20))) + sctp_transport_put(transport); goto out_unlock; } @@ -458,8 +463,9 @@ void sctp_generate_reconf_event(struct timer_list *t) pr_debug("%s: sock is busy\n", __func__); /* Try again later. */ - if (!mod_timer(&transport->reconf_timer, jiffies + (HZ / 20))) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->reconf_timer, jiffies + (HZ / 20))) + sctp_transport_put(transport); goto out_unlock; } @@ -495,8 +501,9 @@ void sctp_generate_probe_event(struct timer_list *t) pr_debug("%s: sock is busy\n", __func__); /* Try again later. */ - if (!mod_timer(&transport->probe_timer, jiffies + (HZ / 20))) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->probe_timer, jiffies + (HZ / 20))) + sctp_transport_put(transport); goto out_unlock; } @@ -1332,6 +1339,10 @@ static int sctp_cmd_interpreter(enum sctp_event_type event_type, sctp_outq_uncork(&asoc->outqueue, gfp); local_cork = 0; } + /* No chunk left in this packet may use this asoc. */ + if (event_type == SCTP_EVENT_T_CHUNK && + chunk->asoc == asoc) + chunk->pdiscard = 1; /* Delete the current association. */ sctp_cmd_delete_tcb(commands, asoc); asoc = NULL; @@ -1541,17 +1552,8 @@ static int sctp_cmd_interpreter(enum sctp_event_type event_type, timeout = asoc->timeouts[cmd->obj.to]; BUG_ON(!timeout); - /* - * SCTP has a hard time with timer starts. Because we process - * timer starts as side effects, it can be hard to tell if we - * have already started a timer or not, which leads to BUG - * halts when we call add_timer. So here, instead of just starting - * a timer, if the timer is already started, and just mod - * the timer with the shorter of the two expiration times - */ - if (!timer_pending(timer)) + if (!timer_reduce(timer, jiffies + timeout)) sctp_association_hold(asoc); - timer_reduce(timer, jiffies + timeout); break; case SCTP_CMD_TIMER_RESTART: diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index 3a8e16b29660c..86f26b8b39934 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -2654,6 +2654,8 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale( sctp_add_cmd_sf(commands, SCTP_CMD_REPLY, SCTP_CHUNK(reply)); + sctp_add_cmd_sf(commands, SCTP_CMD_DISCARD_PACKET, SCTP_NULL()); + return SCTP_DISPOSITION_CONSUME; nomem: diff --git a/net/sctp/stream.c b/net/sctp/stream.c index 34ffe6c945a4b..e4354d58f20b9 100644 --- a/net/sctp/stream.c +++ b/net/sctp/stream.c @@ -22,6 +22,15 @@ #include #include +#define SCTP_STRRESET_MASK(type) \ + BIT(ntohs(type) - ntohs(SCTP_PARAM_RESET_OUT_REQUEST)) +#define SCTP_STRRESET_TEST(asoc, type) \ + ((asoc)->strreset_outstanding & SCTP_STRRESET_MASK(type)) +#define SCTP_STRRESET_SET(asoc, type) \ + ((asoc)->strreset_outstanding |= SCTP_STRRESET_MASK(type)) +#define SCTP_STRRESET_CLEAR(asoc, type) \ + ((asoc)->strreset_outstanding &= ~SCTP_STRRESET_MASK(type)) + static void sctp_stream_shrink_out(struct sctp_stream *stream, __u16 outcnt) { struct sctp_association *asoc; @@ -372,7 +381,10 @@ int sctp_send_reset_streams(struct sctp_association *asoc, goto out; } - asoc->strreset_outstanding = out + in; + if (out) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_OUT_REQUEST); + if (in) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_IN_REQUEST); out: return retval; @@ -417,7 +429,7 @@ int sctp_send_reset_assoc(struct sctp_association *asoc) return retval; } - asoc->strreset_outstanding = 1; + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_TSN_REQUEST); return 0; } @@ -474,7 +486,10 @@ int sctp_send_add_streams(struct sctp_association *asoc, goto out; } - asoc->strreset_outstanding = !!out + !!in; + if (out) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_ADD_OUT_STREAMS); + if (in) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_ADD_IN_STREAMS); out: return retval; @@ -482,13 +497,13 @@ int sctp_send_add_streams(struct sctp_association *asoc, static struct sctp_paramhdr *sctp_chunk_lookup_strreset_param( struct sctp_association *asoc, __be32 resp_seq, - __be16 type) + __be16 type, bool match_seq) { struct sctp_chunk *chunk = asoc->strreset_chunk; struct sctp_reconf_chunk *hdr; union sctp_params param; - if (!chunk) + if (!chunk || !chunk->transport) return NULL; hdr = (struct sctp_reconf_chunk *)chunk->chunk_hdr; @@ -499,7 +514,7 @@ static struct sctp_paramhdr *sctp_chunk_lookup_strreset_param( */ struct sctp_strreset_tsnreq *req = param.v; - if ((!resp_seq || req->request_seq == resp_seq) && + if ((!match_seq || req->request_seq == resp_seq) && (!type || type == req->param_hdr.type)) return param.v; } @@ -564,13 +579,14 @@ struct sctp_chunk *sctp_process_strreset_outreq( if (asoc->strreset_chunk) { if (!sctp_chunk_lookup_strreset_param( asoc, outreq->response_seq, - SCTP_PARAM_RESET_IN_REQUEST)) { + SCTP_PARAM_RESET_IN_REQUEST, true) || + !SCTP_STRRESET_TEST(asoc, SCTP_PARAM_RESET_IN_REQUEST)) { /* same process with outstanding isn't 0 */ result = SCTP_STRRESET_ERR_IN_PROGRESS; goto out; } - asoc->strreset_outstanding--; + SCTP_STRRESET_CLEAR(asoc, SCTP_PARAM_RESET_IN_REQUEST); asoc->strreset_outseq++; if (!asoc->strreset_outstanding) { @@ -669,7 +685,7 @@ struct sctp_chunk *sctp_process_strreset_inreq( SCTP_SO(stream, i)->state = SCTP_STREAM_CLOSED; asoc->strreset_chunk = chunk; - asoc->strreset_outstanding = 1; + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_OUT_REQUEST); sctp_chunk_hold(asoc->strreset_chunk); result = SCTP_STRRESET_PERFORMED; @@ -816,13 +832,14 @@ struct sctp_chunk *sctp_process_strreset_addstrm_out( if (asoc->strreset_chunk) { if (!sctp_chunk_lookup_strreset_param( - asoc, 0, SCTP_PARAM_RESET_ADD_IN_STREAMS)) { + asoc, 0, SCTP_PARAM_RESET_ADD_IN_STREAMS, false) || + !SCTP_STRRESET_TEST(asoc, SCTP_PARAM_RESET_ADD_IN_STREAMS)) { /* same process with outstanding isn't 0 */ result = SCTP_STRRESET_ERR_IN_PROGRESS; goto out; } - asoc->strreset_outstanding--; + SCTP_STRRESET_CLEAR(asoc, SCTP_PARAM_RESET_ADD_IN_STREAMS); asoc->strreset_outseq++; if (!asoc->strreset_outstanding) { @@ -899,7 +916,7 @@ struct sctp_chunk *sctp_process_strreset_addstrm_in( goto out; asoc->strreset_chunk = chunk; - asoc->strreset_outstanding = 1; + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_ADD_OUT_STREAMS); sctp_chunk_hold(asoc->strreset_chunk); stream->outcnt = outcnt; @@ -927,8 +944,9 @@ struct sctp_chunk *sctp_process_strreset_resp( struct sctp_paramhdr *req; __u32 result; - req = sctp_chunk_lookup_strreset_param(asoc, resp->response_seq, 0); - if (!req) + req = sctp_chunk_lookup_strreset_param(asoc, resp->response_seq, 0, + true); + if (!req || !SCTP_STRRESET_TEST(asoc, req->type)) return NULL; result = ntohl(resp->result); @@ -1078,7 +1096,7 @@ struct sctp_chunk *sctp_process_strreset_resp( nums, 0, GFP_ATOMIC); } - asoc->strreset_outstanding--; + SCTP_STRRESET_CLEAR(asoc, req->type); asoc->strreset_outseq++; /* remove everything for this reconf request */ diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 00403175b7406..e9f93b3ab435b 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -409,13 +409,13 @@ void smc_sk_init(struct net *net, struct sock *sk, int protocol) "sk_lock-AF_SMC", &smc_key); spin_lock_init(&smc->accept_q_lock); spin_lock_init(&smc->conn.send_lock); - sk->sk_prot->hash(sk); mutex_init(&smc->clcsock_release_lock); smc_init_saved_callbacks(smc); smc->limit_smc_hs = net->smc.limit_smc_hs; smc->use_fallback = false; /* assume rdma capability first */ smc->fallback_rsn = 0; smc_close_init(smc); + sk->sk_prot->hash(sk); } static struct sock *smc_sock_alloc(struct net *net, struct socket *sock, @@ -3233,7 +3233,8 @@ int smc_ioctl(struct socket *sock, unsigned int cmd, return -EINVAL; } if (smc->sk.sk_state == SMC_INIT || - smc->sk.sk_state == SMC_CLOSED) + smc->sk.sk_state == SMC_CLOSED || + !READ_ONCE(smc->conn.sndbuf_desc)) answ = 0; else answ = smc->conn.sndbuf_desc->len - diff --git a/net/smc/smc.h b/net/smc/smc.h index 52145df83f6e7..427b6d63b9931 100644 --- a/net/smc/smc.h +++ b/net/smc/smc.h @@ -277,9 +277,9 @@ struct smc_connection { * 0 for SMC-R, 32 for SMC-D */ u64 peer_token; /* SMC-D token of peer */ - u8 killed : 1; /* abnormal termination */ - u8 freed : 1; /* normal termination */ - u8 out_of_sync : 1; /* out of sync with peer */ + u8 killed; /* abnormal termination */ + u8 freed; /* normal termination */ + u8 out_of_sync; /* out of sync with peer */ }; struct smc_sock { /* smc sock container */ diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c index b4208cb186c5e..9974149659c2f 100644 --- a/net/smc/smc_core.c +++ b/net/smc/smc_core.c @@ -1148,13 +1148,13 @@ struct smc_link *smc_switch_conns(struct smc_link_group *lgr, read_unlock_bh(&lgr->conns_lock); /* pre-fetch buffer outside of send_lock, might sleep */ rc = smc_cdc_get_free_slot(conn, to_lnk, &wr_buf, NULL, &pend); - if (rc) - goto err_out; - /* avoid race with smcr_tx_sndbuf_nonempty() */ - spin_lock_bh(&conn->send_lock); - smc_switch_link_and_count(conn, to_lnk); - rc = smc_switch_cursor(smc, pend, wr_buf); - spin_unlock_bh(&conn->send_lock); + if (!rc) { + /* avoid race with smcr_tx_sndbuf_nonempty() */ + spin_lock_bh(&conn->send_lock); + smc_switch_link_and_count(conn, to_lnk); + rc = smc_switch_cursor(smc, pend, wr_buf); + spin_unlock_bh(&conn->send_lock); + } sock_put(&smc->sk); if (rc) goto err_out; @@ -1209,14 +1209,16 @@ static void smcd_buf_detach(struct smc_connection *conn) { struct smcd_dev *smcd = conn->lgr->smcd; u64 peer_token = conn->peer_token; + struct smc_buf_desc *buf_desc; if (!conn->sndbuf_desc) return; smc_ism_detach_dmb(smcd, peer_token); - kfree(conn->sndbuf_desc); + buf_desc = conn->sndbuf_desc; conn->sndbuf_desc = NULL; + kfree(buf_desc); } static void smc_buf_unuse(struct smc_connection *conn, @@ -1268,11 +1270,10 @@ void smc_conn_free(struct smc_connection *conn) goto lgr_put; if (lgr->is_smcd) { - if (!list_empty(&lgr->list)) - smc_ism_unset_conn(conn); + smc_ism_unset_conn(conn); + tasklet_kill(&conn->rx_tsklet); if (smc_ism_support_dmb_nocopy(lgr->smcd)) smcd_buf_detach(conn); - tasklet_kill(&conn->rx_tsklet); } else { smc_cdc_wait_pend_tx_wr(conn); if (current_work() != &conn->abort_work) @@ -1525,12 +1526,12 @@ static void smc_conn_kill(struct smc_connection *conn, bool soft) smc_sk_wake_ups(smc); if (conn->lgr->is_smcd) { smc_ism_unset_conn(conn); - if (smc_ism_support_dmb_nocopy(conn->lgr->smcd)) - smcd_buf_detach(conn); if (soft) tasklet_kill(&conn->rx_tsklet); else tasklet_unlock_wait(&conn->rx_tsklet); + if (smc_ism_support_dmb_nocopy(conn->lgr->smcd)) + smcd_buf_detach(conn); } else { smc_cdc_wait_pend_tx_wr(conn); } @@ -1848,6 +1849,7 @@ void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport) struct smc_link_group *lgr, *n; int i; + spin_lock_bh(&smc_lgr_list.lock); list_for_each_entry_safe(lgr, n, &smc_lgr_list.list, list) { if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id, SMC_MAX_PNETID_LEN)) @@ -1862,6 +1864,7 @@ void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport) smcr_link_down_cond_sched(lnk); } } + spin_unlock_bh(&smc_lgr_list.lock); } static void smc_link_down_work(struct work_struct *work) diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c index 9bb495707445e..daaa8a72da90f 100644 --- a/net/smc/smc_ib.c +++ b/net/smc/smc_ib.c @@ -333,6 +333,7 @@ static bool smc_ib_check_link_gid(u8 gid[SMC_GID_SIZE], bool smcrv2, static void smc_ib_gid_check(struct smc_ib_device *smcibdev, u8 ibport) { struct smc_link_group *lgr; + bool stale_gid = false; int i; spin_lock_bh(&smc_lgr_list.lock); @@ -348,11 +349,16 @@ static void smc_ib_gid_check(struct smc_ib_device *smcibdev, u8 ibport) continue; if (!smc_ib_check_link_gid(lgr->lnk[i].gid, lgr->smc_version == SMC_V2, - smcibdev, ibport)) - smcr_port_err(smcibdev, ibport); + smcibdev, ibport)) { + stale_gid = true; + goto out; + } } } +out: spin_unlock_bh(&smc_lgr_list.lock); + if (stale_gid) + smcr_port_err(smcibdev, ibport); } static int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport) diff --git a/net/smc/smc_inet.c b/net/smc/smc_inet.c index a94084b4a498e..520b666fdd8fa 100644 --- a/net/smc/smc_inet.c +++ b/net/smc/smc_inet.c @@ -15,13 +15,16 @@ #include "smc_inet.h" #include "smc.h" +#include "smc_close.h" static int smc_inet_init_sock(struct sock *sk); +static void smc_inet_destroy_sock(struct sock *sk); static struct proto smc_inet_prot = { .name = "INET_SMC", .owner = THIS_MODULE, .init = smc_inet_init_sock, + .destroy = smc_inet_destroy_sock, .hash = smc_hash_sk, .unhash = smc_unhash_sk, .release_cb = smc_release_cb, @@ -68,6 +71,7 @@ static struct proto smc_inet6_prot = { .name = "INET6_SMC", .owner = THIS_MODULE, .init = smc_inet_init_sock, + .destroy = smc_inet_destroy_sock, .hash = smc_hash_sk, .unhash = smc_unhash_sk, .release_cb = smc_release_cb, @@ -116,6 +120,18 @@ static int smc_inet_init_sock(struct sock *sk) return smc_create_clcsk(net, sk, sk->sk_family); } +static void smc_inet_destroy_sock(struct sock *sk) +{ + /* The sock is hashed and smc_diag dumps dereference smc->clcsock + * without clcsock_release_lock, while sk_common_release() calls + * .destroy before .unhash. Unhash first, as __smc_release() does, + * so no dump can observe the clcsock being released; the second + * unhash is a no-op. + */ + sk->sk_prot->unhash(sk); + smc_clcsock_release(smc_sk(sk)); +} + int __init smc_inet_init(void) { int rc; diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c index aa6d83af55eda..c9ccd8480369c 100644 --- a/net/smc/smc_llc.c +++ b/net/smc/smc_llc.c @@ -157,6 +157,7 @@ struct smc_llc_msg_confirm_rkey { /* type 0x06 */ }; #define SMC_LLC_DEL_RKEY_MAX 8 +#define SMC_LLC_DEL_RKEY_V2_INLINE 9 #define SMC_LLC_FLAG_RKEY_RETRY 0x10 #define SMC_LLC_FLAG_RKEY_NEG 0x20 @@ -177,6 +178,15 @@ struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */ __be32 rkey[]; }; +/* the leading rkeys of a DELETE_RKEY_V2 fit into union smc_llc_msg */ +struct smc_llc_msg_delete_rkey_v2_inline { /* type 0x29 */ + struct smc_llc_hdr hd; + u8 num_rkeys; + u8 num_inval_rkeys; + u8 reserved[2]; + __be32 rkey[SMC_LLC_DEL_RKEY_V2_INLINE]; +}; + union smc_llc_msg { struct smc_llc_msg_confirm_link confirm_link; struct smc_llc_msg_add_link add_link; @@ -186,6 +196,7 @@ union smc_llc_msg { struct smc_llc_msg_confirm_rkey confirm_rkey; struct smc_llc_msg_delete_rkey delete_rkey; + struct smc_llc_msg_delete_rkey_v2_inline delete_rkey_v2; struct smc_llc_msg_test_link test_link; struct { @@ -194,15 +205,25 @@ union smc_llc_msg { } raw; }; +static_assert(SMC_LLC_DEL_RKEY_V2_INLINE == + (sizeof(union smc_llc_msg) - + offsetof(struct smc_llc_msg_delete_rkey_v2, rkey)) / + sizeof(__be32)); +static_assert(offsetof(struct smc_llc_msg_delete_rkey_v2_inline, rkey) == + offsetof(struct smc_llc_msg_delete_rkey_v2, rkey)); + #define SMC_LLC_FLAG_RESP 0x80 struct smc_llc_qentry { struct list_head list; struct smc_link *link; + u16 body_len; union smc_llc_msg msg; + u8 body[] __counted_by(body_len); }; -static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); +static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc, + u32 byte_len); struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) { @@ -320,6 +341,7 @@ int smc_llc_flow_initiate(struct smc_link_group *lgr, void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) { spin_lock_bh(&lgr->llc_flow_lock); + smc_llc_flow_qentry_del(flow); memset(flow, 0, sizeof(*flow)); flow->type = SMC_LLC_FLOW_NONE; spin_unlock_bh(&lgr->llc_flow_lock); @@ -998,15 +1020,20 @@ static int smc_llc_cli_conf_link(struct smc_link *link, static void smc_llc_save_add_link_rkeys(struct smc_link *link, struct smc_link *link_new, - u8 *llc_msg) + struct smc_llc_qentry *qentry) { + const u32 rt_off = offsetof(struct smc_llc_msg_add_link_v2_ext, rt); struct smc_llc_msg_add_link_v2_ext *ext; struct smc_link_group *lgr = link->lgr; int max, i; - ext = (struct smc_llc_msg_add_link_v2_ext *)(llc_msg + - SMC_WR_TX_SIZE); + /* the rkey count itself is only there if enough bytes arrived */ + if (qentry->body_len < rt_off) + return; + ext = (struct smc_llc_msg_add_link_v2_ext *)qentry->body; max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); + max = min_t(u32, max, (qentry->body_len - rt_off) / + sizeof(ext->rt[0])); down_write(&lgr->rmbs_lock); for (i = 0; i < max; i++) { smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, @@ -1099,9 +1126,7 @@ int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) if (rc) goto out_clear_lnk; if (lgr->smc_version == SMC_V2) { - u8 *llc_msg = smc_link_shared_v2_rxbuf(link) ? - (u8 *)lgr->wr_rx_buf_v2 : (u8 *)llc; - smc_llc_save_add_link_rkeys(link, lnk_new, llc_msg); + smc_llc_save_add_link_rkeys(link, lnk_new, qentry); } else { rc = smc_llc_cli_rkey_exchange(link, lnk_new); if (rc) { @@ -1481,7 +1506,7 @@ int smc_llc_srv_add_link(struct smc_link *link, } add_llc = &qentry->msg.add_link; if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { - smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); + smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); rc = -ENOLINK; goto out_err; } @@ -1492,7 +1517,8 @@ int smc_llc_srv_add_link(struct smc_link *link, lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; } smc_llc_save_add_link_info(link_new, add_llc); - smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); + /* add_llc still points into qentry, so only detach it here */ + smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); rc = smc_ib_ready_link(link_new); if (rc) @@ -1501,9 +1527,7 @@ int smc_llc_srv_add_link(struct smc_link *link, if (rc) goto out_err; if (lgr->smc_version == SMC_V2) { - u8 *llc_msg = smc_link_shared_v2_rxbuf(link) ? - (u8 *)lgr->wr_rx_buf_v2 : (u8 *)add_llc; - smc_llc_save_add_link_rkeys(link, link_new, llc_msg); + smc_llc_save_add_link_rkeys(link, link_new, qentry); } else { rc = smc_llc_srv_rkey_exchange(link, link_new); if (rc) @@ -1512,14 +1536,14 @@ int smc_llc_srv_add_link(struct smc_link *link, rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); if (rc) goto out_err; - kfree(ini); - return 0; + goto out; out_err: if (link_new) { link_new->state = SMC_LNK_INACTIVE; smcr_link_clear(link_new, false); } out: + kfree(qentry); kfree(ini); if (send_req_add_link_resp) smc_llc_send_req_add_link_response(req_qentry); @@ -1552,7 +1576,8 @@ void smc_llc_add_link_local(struct smc_link *link) add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK; smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc)); /* no dev and port needed */ - smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); + smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc, + sizeof(union smc_llc_msg)); } /* worker to process an add link message */ @@ -1588,7 +1613,8 @@ void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) del_llc.link_num = del_link_id; del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; - smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); + smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc, + sizeof(union smc_llc_msg)); } static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) @@ -1810,19 +1836,28 @@ static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) link = qentry->link; if (lgr->smc_version == SMC_V2) { - struct smc_llc_msg_delete_rkey_v2 *llcv2; + struct smc_llc_msg_delete_rkey_v2_inline *llcv2; - if (smc_link_shared_v2_rxbuf(link)) { - memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc)); - llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2; - } else { - llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)llc; - } + /* The leading SMC_LLC_DEL_RKEY_V2_INLINE rkeys are declared in + * the message itself, any further ones were received into + * qentry->body. + */ + llcv2 = &qentry->msg.delete_rkey_v2; llcv2->num_inval_rkeys = 0; max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); + max = min_t(u32, max, SMC_LLC_DEL_RKEY_V2_INLINE + + qentry->body_len / sizeof(__be32)); for (i = 0; i < max; i++) { - if (smc_rtoken_delete(link, llcv2->rkey[i])) + __be32 rkey; + + if (i < SMC_LLC_DEL_RKEY_V2_INLINE) + rkey = llcv2->rkey[i]; + else + memcpy(&rkey, qentry->body + + (i - SMC_LLC_DEL_RKEY_V2_INLINE) * + sizeof(rkey), sizeof(rkey)); + if (smc_rtoken_delete(link, rkey)) llcv2->num_inval_rkeys++; } memset(&llc->rkey[0], 0, sizeof(llc->rkey)); @@ -1911,6 +1946,8 @@ static void smc_llc_event_handler(struct smc_llc_qentry *qentry) if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK) { /* server started add_link processing */ + /* free any qentry stashed in REQ_ADD_LINK state */ + smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK; smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); @@ -2064,18 +2101,52 @@ static void smc_llc_rx_response(struct smc_link *link, wake_up(&link->lgr->llc_msg_waiter); } -static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) +/* the longest tail either reader of qentry->body can use */ +static u32 smc_llc_max_body_len(union smc_llc_msg *llc) +{ + switch (llc->raw.hdr.common.llc_type) { + case SMC_LLC_ADD_LINK: + return offsetof(struct smc_llc_msg_add_link_v2_ext, rt) + + SMC_LLC_RKEYS_PER_MSG_V2 * + sizeof(struct smc_llc_msg_add_link_cont_rt); + case SMC_LLC_DELETE_RKEY: + return (SMC_LLC_RKEYS_PER_MSG_V2 - + SMC_LLC_DEL_RKEY_V2_INLINE) * sizeof(__be32); + default: + return 0; + } +} + +static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc, + u32 byte_len) { struct smc_link_group *lgr = link->lgr; struct smc_llc_qentry *qentry; unsigned long flags; + u16 body_len = 0; + + /* V2 messages can be longer than the inline union smc_llc_msg. Carry + * the remainder in the qentry itself, so that its lifetime and its + * length match the message the peer actually sent. + */ + if (lgr->smc_version == SMC_V2 && byte_len > SMC_WR_TX_SIZE) + body_len = min_t(u32, byte_len, SMC_WR_BUF_V2_SIZE) - + SMC_WR_TX_SIZE; + body_len = min_t(u32, body_len, smc_llc_max_body_len(llc)); - qentry = kmalloc_obj(*qentry, GFP_ATOMIC); + qentry = kmalloc_flex(*qentry, body, body_len, GFP_ATOMIC); if (!qentry) return; + qentry->body_len = body_len; qentry->link = link; INIT_LIST_HEAD(&qentry->list); memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); + if (body_len) { + u8 *src = smc_link_shared_v2_rxbuf(link) ? + (u8 *)lgr->wr_rx_buf_v2 : (u8 *)llc; + + memcpy(qentry->body, src + SMC_WR_TX_SIZE, body_len); + } /* process responses immediately */ if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) && @@ -2107,7 +2178,7 @@ static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) return; /* invalid message */ } - smc_llc_enqueue(link, llc); + smc_llc_enqueue(link, llc, wc->byte_len); } /***************************** worker, utils *********************************/ diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c index 5c9e4d8b57de4..197fddc6271f0 100644 --- a/net/smc/smc_rx.c +++ b/net/smc/smc_rx.c @@ -115,16 +115,15 @@ static void smc_rx_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { struct smc_spd_priv *priv = (struct smc_spd_priv *)buf->private; + struct smc_connection *conn = &priv->smc->conn; struct smc_sock *smc = priv->smc; - struct smc_connection *conn; struct sock *sk = &smc->sk; - if (sk->sk_state == SMC_CLOSED || - sk->sk_state == SMC_PEERFINCLOSEWAIT || - sk->sk_state == SMC_APPFINCLOSEWAIT) - goto out; - conn = &smc->conn; lock_sock(sk); + if (conn->freed) { + release_sock(sk); + goto out; + } smc_rx_update_cons(smc, priv->len); release_sock(sk); if (atomic_sub_and_test(priv->len, &conn->splice_pending)) diff --git a/net/smc/smc_tx.h b/net/smc/smc_tx.h index a59f370b8b432..610a945aefd69 100644 --- a/net/smc/smc_tx.h +++ b/net/smc/smc_tx.h @@ -20,11 +20,15 @@ static inline int smc_tx_prepared_sends(struct smc_connection *conn) { + struct smc_buf_desc *sndbuf_desc = READ_ONCE(conn->sndbuf_desc); union smc_host_cursor sent, prep; + if (!sndbuf_desc) + return 0; + smc_curs_copy(&sent, &conn->tx_curs_sent, conn); smc_curs_copy(&prep, &conn->tx_curs_prep, conn); - return smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep); + return smc_curs_diff(sndbuf_desc->len, &sent, &prep); } void smc_tx_pending(struct smc_connection *conn); diff --git a/net/smc/smc_wr.c b/net/smc/smc_wr.c index 59c92b46945c5..97ba46893b17f 100644 --- a/net/smc/smc_wr.c +++ b/net/smc/smc_wr.c @@ -602,9 +602,9 @@ static void smc_wr_init_sge(struct smc_link *lnk) /* With SMC-Rv2 there can be messages larger than SMC_WR_TX_SIZE. * Each ib_recv_wr gets 2 sges, the second one is a spillover buffer - * and the same buffer for all sges. When a larger message arrived then - * the content of the first small sge is copied to the beginning of - * the larger spillover buffer, allowing easy data mapping. + * and the same buffer for all sges. The spillover sge starts at + * SMC_WR_TX_SIZE, so the leading bytes of that buffer are never + * written. */ for (i = 0; i < lnk->wr_rx_cnt; i++) { int x = i * lnk->wr_rx_sge_cnt; diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 9d3fb6848f400..8ddc65e894dab 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -2072,7 +2072,11 @@ gss_unwrap_resp_priv(struct rpc_task *task, struct rpc_cred *cred, goto unwrap_failed; opaque_len = be32_to_cpup(p++); offset = (u8 *)(p) - (u8 *)head->iov_base; - if (offset + opaque_len > rcv_buf->len) + if (offset > rcv_buf->len) + goto unwrap_failed; + if (opaque_len > rcv_buf->len - offset) + goto unwrap_failed; + if (opaque_len <= GSS_KRB5_TOK_HDR_LEN) goto unwrap_failed; maj_stat = gss_unwrap(ctx->gc_gss_ctx, offset, diff --git a/net/sunrpc/auth_gss/gss_krb5_unseal.c b/net/sunrpc/auth_gss/gss_krb5_unseal.c index b5fb70419faaa..4d12d49434c28 100644 --- a/net/sunrpc/auth_gss/gss_krb5_unseal.c +++ b/net/sunrpc/auth_gss/gss_krb5_unseal.c @@ -89,6 +89,9 @@ gss_krb5_verify_mic_v2(struct krb5_ctx *ctx, struct xdr_buf *message_buffer, dprintk("RPC: %s\n", __func__); + if (read_token->len < GSS_KRB5_TOK_HDR_LEN + cksum_len) + return GSS_S_DEFECTIVE_TOKEN; + memcpy(&be16_ptr, (char *) ptr, 2); if (be16_to_cpu(be16_ptr) != KG2_TOK_MIC) return GSS_S_DEFECTIVE_TOKEN; diff --git a/net/sunrpc/auth_gss/gss_krb5_wrap.c b/net/sunrpc/auth_gss/gss_krb5_wrap.c index ac4b32df42b96..d3f61c4b5a137 100644 --- a/net/sunrpc/auth_gss/gss_krb5_wrap.c +++ b/net/sunrpc/auth_gss/gss_krb5_wrap.c @@ -73,6 +73,8 @@ static void _rotate_left(struct xdr_buf *buf, unsigned int shift) int shifted = 0; int this_shift; + if (!buf->len) + return; shift %= buf->len; while (shifted < shift) { this_shift = min(shift - shifted, LOCAL_BUF_LEN); @@ -85,6 +87,8 @@ static void rotate_left(u32 base, struct xdr_buf *buf, unsigned int shift) { struct xdr_buf subbuf; + if (buf->len <= base) + return; xdr_buf_subsegment(buf, &subbuf, base, buf->len - base); _rotate_left(&subbuf, shift); } @@ -154,6 +158,9 @@ gss_krb5_unwrap_v2(struct krb5_ctx *kctx, int offset, int len, dprintk("RPC: %s\n", __func__); + if (len - offset <= GSS_KRB5_TOK_HDR_LEN) + return GSS_S_DEFECTIVE_TOKEN; + ptr = buf->head[0].iov_base + offset; if (be16_to_cpu(*((__be16 *)ptr)) != KG2_TOK_WRAP) @@ -220,14 +227,16 @@ gss_krb5_unwrap_v2(struct krb5_ctx *kctx, int offset, int len, * head buffer space rather than that actually occupied. */ movelen = min_t(unsigned int, buf->head[0].iov_len, len); + if (movelen < offset + GSS_KRB5_TOK_HDR_LEN + headskip) + return GSS_S_DEFECTIVE_TOKEN; movelen -= offset + GSS_KRB5_TOK_HDR_LEN + headskip; - BUG_ON(offset + GSS_KRB5_TOK_HDR_LEN + headskip + movelen > - buf->head[0].iov_len); memmove(ptr, ptr + GSS_KRB5_TOK_HDR_LEN + headskip, movelen); buf->head[0].iov_len -= GSS_KRB5_TOK_HDR_LEN + headskip; buf->len = len - (GSS_KRB5_TOK_HDR_LEN + headskip); /* Trim off the trailing "extra count" and checksum blob */ + if (ec + GSS_KRB5_TOK_HDR_LEN + tailskip > buf->len - offset) + return GSS_S_DEFECTIVE_TOKEN; xdr_buf_trim(buf, ec + GSS_KRB5_TOK_HDR_LEN + tailskip); *align = XDR_QUADLEN(GSS_KRB5_TOK_HDR_LEN + headskip); diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.c b/net/sunrpc/auth_gss/gss_rpc_upcall.c index 0fa4778620d9a..b7f70b1adb18a 100644 --- a/net/sunrpc/auth_gss/gss_rpc_upcall.c +++ b/net/sunrpc/auth_gss/gss_rpc_upcall.c @@ -121,12 +121,6 @@ static int gssp_rpc_create(struct net *net, struct rpc_clnt **_clnt) return result; } -void init_gssp_clnt(struct sunrpc_net *sn) -{ - mutex_init(&sn->gssp_lock); - sn->gssp_clnt = NULL; -} - int set_gssp_clnt(struct net *net) { struct sunrpc_net *sn = net_generic(net, sunrpc_net_id); diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.h b/net/sunrpc/auth_gss/gss_rpc_upcall.h index 31e96344167e3..b3c2b2b907985 100644 --- a/net/sunrpc/auth_gss/gss_rpc_upcall.h +++ b/net/sunrpc/auth_gss/gss_rpc_upcall.h @@ -29,7 +29,6 @@ int gssp_accept_sec_context_upcall(struct net *net, struct gssp_upcall_data *data); void gssp_free_upcall_data(struct gssp_upcall_data *data); -void init_gssp_clnt(struct sunrpc_net *); int set_gssp_clnt(struct net *); void clear_gssp_clnt(struct sunrpc_net *); diff --git a/net/sunrpc/auth_gss/gss_rpc_xdr.c b/net/sunrpc/auth_gss/gss_rpc_xdr.c index fceee648d545b..0549edae1ebee 100644 --- a/net/sunrpc/auth_gss/gss_rpc_xdr.c +++ b/net/sunrpc/auth_gss/gss_rpc_xdr.c @@ -222,7 +222,8 @@ static int gssx_dec_linux_creds(struct xdr_stream *xdr, return 0; out_free_groups: - groups_free(creds->cr_group_info); + put_group_info(creds->cr_group_info); + creds->cr_group_info = NULL; return err; } @@ -230,6 +231,7 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, struct gssx_option_array *oa) { struct svc_cred *creds; + bool creds_decoded = false; u32 count, i; __be32 *p; int err; @@ -242,12 +244,12 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, return 0; /* we recognize only 1 currently: CREDS_VALUE */ - oa->count = 1; - oa->data = kmalloc_obj(struct gssx_option); if (!oa->data) return -ENOMEM; + oa->count = 1; + creds = kzalloc_obj(struct svc_cred); if (!creds) { err = -ENOMEM; @@ -280,9 +282,14 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, if (length == sizeof(CREDS_VALUE) && memcmp(p, CREDS_VALUE, sizeof(CREDS_VALUE)) == 0) { /* We have creds here. parse them */ + if (creds_decoded) { + err = -EINVAL; + goto free_creds; + } err = gssx_dec_linux_creds(xdr, creds); if (err) goto free_creds; + creds_decoded = true; oa->data[0].value.len = 1; /* presence */ } else { /* consume uninteresting buffer */ @@ -294,8 +301,10 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, return 0; free_creds: + free_svc_cred(creds); kfree(creds); free_oa: + oa->count = 0; kfree(oa->data); oa->data = NULL; return err; diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index d14209031e180..967e9d53080d7 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -949,6 +949,8 @@ svcauth_gss_unwrap_priv(struct svc_rqst *rqstp, u32 seq, struct gss_ctx *ctx) } if (len > xdr_stream_remaining(xdr)) goto unwrap_failed; + if (len <= GSS_KRB5_TOK_HDR_LEN) + goto unwrap_failed; offset = xdr_stream_pos(xdr); saved_len = buf->len; @@ -1466,7 +1468,6 @@ static int create_use_gss_proxy_proc_entry(struct net *net) &use_gss_proxy_proc_ops, net); if (!*p) return -ENOMEM; - init_gssp_clnt(sn); return 0; } @@ -1573,6 +1574,9 @@ svcauth_gss_decode_credbody(struct xdr_stream *xdr, u32 body_len; __be32 *p; + /* Early-return paths leave deterministic state, not stale residue. */ + memset(gc, 0, sizeof(*gc)); + p = xdr_inline_decode(xdr, XDR_UNIT); if (!p) return false; @@ -1942,6 +1946,8 @@ svcauth_gss_release(struct svc_rqst *rqstp) if (!gsd) goto out; + if (rqstp->rq_auth_stat != rpc_auth_ok) + goto out; gc = &gsd->clcred; if (gc->gc_proc != RPC_GSS_PROC_DATA) goto out; diff --git a/net/sunrpc/backchannel_rqst.c b/net/sunrpc/backchannel_rqst.c index 0ffa4d01a9389..1482b06e0f38b 100644 --- a/net/sunrpc/backchannel_rqst.c +++ b/net/sunrpc/backchannel_rqst.c @@ -25,20 +25,39 @@ unsigned int xprt_bc_max_slots(struct rpc_xprt *xprt) } /* - * Helper function to nullify backchannel server pointer in transport. - * We need to synchronize setting the pointer to NULL (done so after - * the backchannel server is shutdown) with the usage of that pointer - * by the backchannel request processing routines - * xprt_complete_bc_request() and rpcrdma_bc_receive_call(). + * Close the backchannel producer side, drain any requests still + * queued on sv_cb_list, then destroy the callback service. */ void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv) { - spin_lock(&xprt->bc_pa_lock); + struct svc_serv *bc_serv = *serv; + struct rpc_rqst *req; + + xprt_svc_shutdown_bc(xprt); + while ((req = lwq_dequeue(&bc_serv->sv_cb_list, struct rpc_rqst, + rq_bc_list)) != NULL) { + atomic_dec(&req->rq_xprt->bc_slot_count); + xprt_free_bc_request(req); + } svc_destroy(serv); +} +EXPORT_SYMBOL_GPL(xprt_svc_destroy_nullify_bc); + +/* + * Clear the backchannel server pointer in the transport. The NULL + * store is serialized under bc_pa_lock against readers of + * xprt->bc_serv in xprt_complete_bc_request() and + * rpcrdma_bc_receive_call(). Clearing it before the callback service + * is stopped prevents a producer from enqueueing onto a service that + * is being torn down. + */ +void xprt_svc_shutdown_bc(struct rpc_xprt *xprt) +{ + spin_lock(&xprt->bc_pa_lock); xprt->bc_serv = NULL; spin_unlock(&xprt->bc_pa_lock); } -EXPORT_SYMBOL_GPL(xprt_svc_destroy_nullify_bc); +EXPORT_SYMBOL_GPL(xprt_svc_shutdown_bc); /* * Helper routines that track the number of preallocation elements @@ -393,7 +412,12 @@ void xprt_enqueue_bc_request(struct rpc_rqst *req) if (bc_serv) { lwq_enqueue(&req->rq_bc_list, &bc_serv->sv_cb_list); svc_pool_wake_idle_thread(&bc_serv->sv_pools[0]); + spin_unlock(&xprt->bc_pa_lock); + return; } spin_unlock(&xprt->bc_pa_lock); + + atomic_dec(&xprt->bc_slot_count); + xprt_free_bc_request(req); } EXPORT_SYMBOL_GPL(xprt_enqueue_bc_request); diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 391037f152928..1bc04109d213d 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -430,10 +430,9 @@ void sunrpc_destroy_cache_detail(struct cache_detail *cd) list_del_init(&cd->others); spin_unlock(&cd->hash_lock); spin_unlock(&cache_list_lock); - if (list_empty(&cache_list)) { - /* module must be being unloaded so its safe to kill the worker */ - cancel_delayed_work_sync(&cache_cleaner); - } + cancel_delayed_work_sync(&cache_cleaner); + if (!list_empty(&cache_list)) + queue_delayed_work(system_power_efficient_wq, &cache_cleaner, 0); } EXPORT_SYMBOL_GPL(sunrpc_destroy_cache_detail); diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index efa26899bc7dc..6cedc824cf826 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -96,7 +96,10 @@ static void rpc_unregister_client(struct rpc_clnt *clnt) static void __rpc_clnt_remove_pipedir(struct rpc_clnt *clnt) { - rpc_remove_client_dir(clnt); + if (clnt->pipefs_sb) { + rpc_remove_client_dir(clnt); + clnt->pipefs_sb = NULL; + } } static void rpc_clnt_remove_pipedir(struct rpc_clnt *clnt) @@ -177,19 +180,28 @@ static int rpc_clnt_skip_event(struct rpc_clnt *clnt, unsigned long event) } static int __rpc_clnt_handle_event(struct rpc_clnt *clnt, unsigned long event, - struct super_block *sb) + struct super_block *sb) { + int err = 0; + switch (event) { case RPC_PIPEFS_MOUNT: - return rpc_setup_pipedir_sb(sb, clnt); + clnt->pipefs_sb = sb; + err = rpc_setup_pipedir_sb(sb, clnt); + if (err) + clnt->pipefs_sb = NULL; + break; case RPC_PIPEFS_UMOUNT: - __rpc_clnt_remove_pipedir(clnt); + if (clnt->pipefs_sb == sb) { + __rpc_clnt_remove_pipedir(clnt); + clnt->pipefs_sb = NULL; + } break; default: printk(KERN_ERR "%s: unknown event: %ld\n", __func__, event); return -ENOTSUPP; } - return 0; + return err; } static int __rpc_pipefs_event(struct rpc_clnt *clnt, unsigned long event, diff --git a/net/sunrpc/rpcb_clnt.c b/net/sunrpc/rpcb_clnt.c index 6aa372188c86a..4c0b7fefee4e2 100644 --- a/net/sunrpc/rpcb_clnt.c +++ b/net/sunrpc/rpcb_clnt.c @@ -490,6 +490,8 @@ static int rpcb_register_inet4(struct sunrpc_net *sn, int result; map->r_addr = rpc_sockaddr2uaddr(sap, GFP_KERNEL); + if (!map->r_addr) + return -ENOMEM; msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET]; if (port != 0) { @@ -516,6 +518,8 @@ static int rpcb_register_inet6(struct sunrpc_net *sn, int result; map->r_addr = rpc_sockaddr2uaddr(sap, GFP_KERNEL); + if (!map->r_addr) + return -ENOMEM; msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET]; if (port != 0) { diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c index ab88ce46afb55..1a3884a0376a4 100644 --- a/net/sunrpc/sunrpc_syms.c +++ b/net/sunrpc/sunrpc_syms.c @@ -57,6 +57,7 @@ static __net_init int sunrpc_init_net(struct net *net) INIT_LIST_HEAD(&sn->all_clients); spin_lock_init(&sn->rpc_client_lock); spin_lock_init(&sn->rpcb_clnt_lock); + mutex_init(&sn->gssp_lock); return 0; err_pipefs: diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index ae9ec4bf34f72..f85914ccb01a4 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -352,7 +352,7 @@ static int svc_pool_map_get_node(unsigned int pidx) if (m->mode == SVC_POOL_PERNODE) return m->pool_to[pidx]; } - return numa_mem_id(); + return NUMA_NO_NODE; } /* * Set the given thread's cpus_allowed mask so that it @@ -402,6 +402,7 @@ struct svc_pool *svc_pool_for_cpu(struct svc_serv *serv) struct svc_pool_map *m = &svc_pool_map; int cpu = raw_smp_processor_id(); unsigned int pidx = 0; + unsigned int i; if (serv->sv_nrpools <= 1) return serv->sv_pools; @@ -414,8 +415,34 @@ struct svc_pool *svc_pool_for_cpu(struct svc_serv *serv) pidx = m->to_pool[cpu_to_node(cpu)]; break; } + pidx %= serv->sv_nrpools; + + /* + * It's possible to have a pool with no threads. Userland can just set + * things up this way directly. Also, when threads are autodistributed + * they are spread evenly across the pools, but when there are fewer + * threads than pools some pools can end up with none. + * + * A transport enqueued on a threadless pool would never be picked up, + * since each thread only services its own pool. Fall back to the next + * populated pool, trading NUMA locality for a guarantee that the + * transport is serviced. + */ + for (i = 0; i < serv->sv_nrpools; i++) { + struct svc_pool *pool = &serv->sv_pools[pidx]; + + /* This is set under the service mutex and rarely ever + * changes. A data race here is harmless. + */ + if (data_race(pool->sp_nrthreads)) + return pool; + + if (++pidx >= serv->sv_nrpools) + pidx = 0; + } - return &serv->sv_pools[pidx % serv->sv_nrpools]; + /* No pool has any threads; nothing can service the transport. */ + return &serv->sv_pools[pidx]; } static int svc_rpcb_setup(struct svc_serv *serv, struct net *net) @@ -476,6 +503,35 @@ __svc_init_bc(struct svc_serv *serv) } #endif +static int svc_pool_init_counters(struct svc_pool *pool) +{ + int err; + + err = percpu_counter_init(&pool->sp_messages_arrived, 0, GFP_KERNEL); + if (err) + return err; + err = percpu_counter_init(&pool->sp_sockets_queued, 0, GFP_KERNEL); + if (err) + goto err_sockets; + err = percpu_counter_init(&pool->sp_threads_woken, 0, GFP_KERNEL); + if (err) + goto err_threads; + return 0; + +err_threads: + percpu_counter_destroy(&pool->sp_sockets_queued); +err_sockets: + percpu_counter_destroy(&pool->sp_messages_arrived); + return err; +} + +static void svc_pool_destroy_counters(struct svc_pool *pool) +{ + percpu_counter_destroy(&pool->sp_messages_arrived); + percpu_counter_destroy(&pool->sp_sockets_queued); + percpu_counter_destroy(&pool->sp_threads_woken); +} + /* * Create an RPC service */ @@ -540,12 +596,18 @@ __svc_create(struct svc_program *prog, int nprogs, struct svc_stat *stats, INIT_LIST_HEAD(&pool->sp_all_threads); init_llist_head(&pool->sp_idle_threads); - percpu_counter_init(&pool->sp_messages_arrived, 0, GFP_KERNEL); - percpu_counter_init(&pool->sp_sockets_queued, 0, GFP_KERNEL); - percpu_counter_init(&pool->sp_threads_woken, 0, GFP_KERNEL); + if (svc_pool_init_counters(pool)) + goto out_err; } return serv; + +out_err: + while (i--) + svc_pool_destroy_counters(&serv->sv_pools[i]); + kfree(serv->sv_pools); + kfree(serv); + return NULL; } /** @@ -624,9 +686,7 @@ svc_destroy(struct svc_serv **servp) for (i = 0; i < serv->sv_nrpools; i++) { struct svc_pool *pool = &serv->sv_pools[i]; - percpu_counter_destroy(&pool->sp_messages_arrived); - percpu_counter_destroy(&pool->sp_sockets_queued); - percpu_counter_destroy(&pool->sp_threads_woken); + svc_pool_destroy_counters(pool); } kfree(serv->sv_pools); kfree(serv); @@ -683,6 +743,15 @@ svc_release_buffer(struct svc_rqst *rqstp) } } +static void svc_rqst_free_rcu(struct rcu_head *head) +{ + struct svc_rqst *rqstp = container_of(head, struct svc_rqst, rq_rcu_head); + + kfree(rqstp->rq_resp); + kfree(rqstp->rq_argp); + kfree(rqstp); +} + static void svc_rqst_free(struct svc_rqst *rqstp) { @@ -691,10 +760,8 @@ svc_rqst_free(struct svc_rqst *rqstp) svc_release_buffer(rqstp); if (rqstp->rq_scratch_folio) folio_put(rqstp->rq_scratch_folio); - kfree(rqstp->rq_resp); - kfree(rqstp->rq_argp); kfree(rqstp->rq_auth_data); - kfree_rcu(rqstp, rq_rcu_head); + call_rcu(&rqstp->rq_rcu_head, svc_rqst_free_rcu); } static struct svc_rqst * @@ -711,7 +778,9 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) rqstp->rq_server = serv; rqstp->rq_pool = pool; - rqstp->rq_scratch_folio = __folio_alloc_node(GFP_KERNEL, 0, node); + rqstp->rq_scratch_folio = __folio_alloc_node(GFP_KERNEL, 0, + node == NUMA_NO_NODE ? + numa_mem_id() : node); if (!rqstp->rq_scratch_folio) goto out_enomem; diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index fa6a30b5f0463..cb2ef428651f6 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -2049,7 +2049,7 @@ void xdr_buf_trim(struct xdr_buf *buf, unsigned int len) trim -= cur; } fix_len: - buf->len -= (len - trim); + buf->len -= min_t(unsigned int, buf->len, len - trim); } EXPORT_SYMBOL_GPL(xdr_buf_trim); diff --git a/net/sunrpc/xprtrdma/ib_client.c b/net/sunrpc/xprtrdma/ib_client.c index de49ad02053d8..188f7a13397f2 100644 --- a/net/sunrpc/xprtrdma/ib_client.c +++ b/net/sunrpc/xprtrdma/ib_client.c @@ -51,7 +51,11 @@ static struct rpcrdma_device *rpcrdma_get_client_data(struct ib_device *device) * to be invoked when the device is removed, unless this notification * is unregistered first. * - * On failure, a negative errno is returned. + * On failure, a negative errno is returned. rn->rn_done is left + * NULL on every failure path (it is armed before xa_alloc but + * cleared again if xa_alloc fails), so the @rn may safely be + * passed to rpcrdma_rn_unregister() without a separate + * registered/unregistered flag in the caller. */ int rpcrdma_rn_register(struct ib_device *device, struct rpcrdma_notification *rn, @@ -62,10 +66,21 @@ int rpcrdma_rn_register(struct ib_device *device, if (!rd || test_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags)) return -ENETUNREACH; - if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) + /* + * Arm rn_done before xa_alloc() publishes @rn: once @rn is + * visible in rd_xa, a concurrent rpcrdma_remove_one() can + * call rn->rn_done(), so the pointer must already be set. + * + * Restore NULL if xa_alloc() fails. rn_done doubles as the + * registration sentinel for rpcrdma_rn_unregister(); a stale + * value would unregister an @rn that was never inserted. + */ + rn->rn_done = done; + if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) { + rn->rn_done = NULL; return -ENOMEM; + } kref_get(&rd->rd_kref); - rn->rn_done = done; trace_rpcrdma_client_register(device, rn); return 0; } @@ -83,6 +98,10 @@ static void rpcrdma_rn_release(struct kref *kref) * rpcrdma_rn_unregister - stop device removal notifications * @device: monitored device * @rn: notification object that no longer wishes to be notified + * + * It is safe to call this on an @rn whose registration never + * completed or failed; rn_done == NULL is treated as + * never-registered and the call is a no-op. */ void rpcrdma_rn_unregister(struct ib_device *device, struct rpcrdma_notification *rn) @@ -92,6 +111,21 @@ void rpcrdma_rn_unregister(struct ib_device *device, if (!rd) return; + /* + * rn_done is the registration sentinel: rpcrdma_rn_register + * leaves it NULL on every failure path, clearing it again if + * xa_alloc fails, so a non-NULL rn_done marks a completed + * registration. A NULL rn_done means this notification was + * never registered (or its registration failed) or has + * already been unregistered, and the call is a no-op. + * Without this guard, rn_index == 0 from a kzalloc'd + * parent would erase another caller's slot 0 and underflow + * rd_kref. + */ + if (!rn->rn_done) + return; + rn->rn_done = NULL; + trace_rpcrdma_client_unregister(device, rn); xa_erase(&rd->rd_xa, rn->rn_index); kref_put(&rd->rd_kref, rpcrdma_rn_release); diff --git a/net/sunrpc/xprtrdma/svc_rdma_pcl.c b/net/sunrpc/xprtrdma/svc_rdma_pcl.c index 1f8f7dad8b6f8..18d1045799cef 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_pcl.c +++ b/net/sunrpc/xprtrdma/svc_rdma_pcl.c @@ -213,7 +213,6 @@ bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, chunk = pcl_alloc_chunk(segcount, 0); if (!chunk) return false; - list_add_tail(&chunk->ch_list, &pcl->cl_chunks); for (j = 0; j < segcount; j++) { segment = &chunk->ch_segments[j]; @@ -225,6 +224,7 @@ bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, chunk->ch_length += segment->rs_length; chunk->ch_segcount++; } + list_add_tail(&chunk->ch_list, &pcl->cl_chunks); } return true; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index fe9bf0371b6ec..8e71ceec11df9 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -440,11 +440,14 @@ static void svc_rdma_build_arg_xdr(struct svc_rqst *rqstp, * to the first byte past the Read list. rc_read_pcl and * rc_call_pcl cl_count fields are set to the number of * Read segments in the list. - * %false: Read list is corrupt. @rctxt's xdr_stream is left in an - * unknown state. + * %false: Read list is corrupt or exceeds the page budget. @rctxt's + * xdr_stream is left in an unknown state. */ static bool xdr_count_read_segments(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) { + unsigned int maxlen = rctxt->rc_maxpages << PAGE_SHIFT; + unsigned int total_len = 0; + rctxt->rc_call_pcl.cl_count = 0; rctxt->rc_read_pcl.cl_count = 0; while (xdr_item_is_present(p)) { @@ -458,6 +461,11 @@ static bool xdr_count_read_segments(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) xdr_decode_read_segment(p, &position, &handle, &length, &offset); + if (length > maxlen) + return false; + total_len += length; + if (PAGE_ALIGN(total_len) > maxlen) + return false; if (position) { if (position & 3) return false; @@ -508,10 +516,13 @@ static bool xdr_check_write_chunk(struct svc_rdma_recv_ctxt *rctxt) return false; /* Before trusting the segcount value enough to use it in - * a computation, perform a simple range check. This is an - * arbitrary but sensible limit (ie, not architectural). + * a computation, perform a simple range check. A zero + * segcount describes no remote buffer and is rejected so + * downstream consumers never see a degenerate ch_segcount==0 + * chunk. The upper bound is an arbitrary but sensible limit + * (ie, not architectural). */ - if (unlikely(segcount > rctxt->rc_maxpages)) + if (segcount == 0 || unlikely(segcount > rctxt->rc_maxpages)) return false; p = xdr_inline_decode(&rctxt->rc_stream, diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index f7fd22cc4a598..9aaaade99e6e4 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -795,7 +795,7 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp, len -= seg_len; if (len && ((head->rc_curpage + 1) > rqstp->rq_maxpages)) - goto out_overrun; + goto out_put; } ret = svc_rdma_rw_ctx_init(rdma, ctxt, segment->rs_offset, @@ -809,7 +809,8 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp, cc->cc_sqecount += ret; return 0; -out_overrun: +out_put: + svc_rdma_put_rw_ctxt(rdma, ctxt); trace_svcrdma_page_overrun_err(&cc->cc_cid, head->rc_curpage); return -EINVAL; } @@ -847,7 +848,7 @@ static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp, * svc_rdma_copy_inline_range - Copy part of the inline content into pages * @rqstp: RPC transaction context * @head: context for ongoing I/O - * @offset: offset into the Receive buffer of region to copy + * @offset: offset into the inline content of region to copy * @remaining: length of region to copy * * Take a page at a time from rqstp->rq_pages and copy the inline @@ -864,9 +865,13 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp, unsigned int offset, unsigned int remaining) { - unsigned char *dst, *src = head->rc_recv_buf; + unsigned char *dst, *src = head->rc_saved_arg.head[0].iov_base; + unsigned int inline_len = head->rc_saved_arg.head[0].iov_len; unsigned int page_no, numpages; + if (offset > inline_len || remaining > inline_len - offset) + return -EINVAL; + numpages = PAGE_ALIGN(head->rc_pageoff + remaining) >> PAGE_SHIFT; for (page_no = 0; page_no < numpages; page_no++) { unsigned int page_len; @@ -917,9 +922,10 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, { const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; struct svc_rdma_chunk *chunk, *next; - unsigned int start, length; + unsigned int inline_len, start, length; int ret; + inline_len = head->rc_saved_arg.head[0].iov_len; start = 0; chunk = pcl_first_chunk(pcl); length = chunk->ch_position; @@ -937,6 +943,8 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, break; start += length; + if (head->rc_readbytes > next->ch_position) + return -EINVAL; length = next->ch_position - head->rc_readbytes; ret = svc_rdma_copy_inline_range(rqstp, head, start, length); if (ret < 0) @@ -944,7 +952,9 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, } start += length; - length = head->rc_byte_len - start; + if (start > inline_len) + return -EINVAL; + length = inline_len - start; return svc_rdma_copy_inline_range(rqstp, head, start, length); } @@ -969,8 +979,12 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, static int svc_rdma_read_data_item(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head) { - return svc_rdma_build_read_chunk(rqstp, head, - pcl_first_chunk(&head->rc_read_pcl)); + struct svc_rdma_chunk *chunk = pcl_first_chunk(&head->rc_read_pcl); + + if (chunk->ch_position > head->rc_saved_arg.head[0].iov_len) + return -EINVAL; + + return svc_rdma_build_read_chunk(rqstp, head, chunk); } /** @@ -996,17 +1010,20 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp, const struct svc_rdma_segment *segment; int ret; + if (!length) + return 0; + ret = -EINVAL; pcl_for_each_segment(segment, chunk) { struct svc_rdma_segment dummy; - if (offset > segment->rs_length) { + if (offset >= segment->rs_length) { offset -= segment->rs_length; continue; } dummy.rs_handle = segment->rs_handle; - dummy.rs_length = min_t(u32, length, segment->rs_length) - offset; + dummy.rs_length = min_t(u32, length, segment->rs_length - offset); dummy.rs_offset = segment->rs_offset + offset; ret = svc_rdma_build_read_segment(rqstp, head, &dummy); @@ -1015,6 +1032,8 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp, head->rc_readbytes += dummy.rs_length; length -= dummy.rs_length; + if (!length) + break; offset = 0; } return ret; @@ -1039,14 +1058,17 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, pcl_first_chunk(&head->rc_call_pcl); const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; struct svc_rdma_chunk *chunk, *next; - unsigned int start, length; + unsigned int call_len, start, length; int ret; if (pcl_is_empty(pcl)) return svc_rdma_build_read_chunk(rqstp, head, call_chunk); + call_len = call_chunk->ch_length; start = 0; chunk = pcl_first_chunk(pcl); + if (chunk->ch_position > call_len) + return -EINVAL; length = chunk->ch_position; ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); @@ -1063,6 +1085,10 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, break; start += length; + if (next->ch_position > call_len) + return -EINVAL; + if (head->rc_readbytes > next->ch_position) + return -EINVAL; length = next->ch_position - head->rc_readbytes; ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); @@ -1071,7 +1097,9 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, } start += length; - length = call_chunk->ch_length - start; + if (start > call_len) + return -EINVAL; + length = call_len - start; return svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); } diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index 7f6d17bf8c1fd..c09659b17351b 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -825,20 +825,21 @@ static int svc_rdma_xb_count_sges(const struct xdr_buf *xdr, } /** - * svc_rdma_pull_up_needed - Determine whether to use pull-up + * svc_rdma_check_pull_up - Determine whether to use pull-up * @rdma: controlling transport * @sctxt: send_ctxt for the Send WR * @write_pcl: Write chunk list provided by client * @xdr: xdr_buf containing RPC message to transmit * * Returns: - * %true if pull-up must be used - * %false otherwise + * %1 if pull-up must be used + * %0 if pull-up is not needed + * %-E2BIG if the reply is too large to be pulled up */ -static bool svc_rdma_pull_up_needed(const struct svcxprt_rdma *rdma, - const struct svc_rdma_send_ctxt *sctxt, - const struct svc_rdma_pcl *write_pcl, - const struct xdr_buf *xdr) +static int svc_rdma_check_pull_up(const struct svcxprt_rdma *rdma, + const struct svc_rdma_send_ctxt *sctxt, + const struct svc_rdma_pcl *write_pcl, + const struct xdr_buf *xdr) { /* Resources needed for the transport header */ struct svc_rdma_pullup_data args = { @@ -850,11 +851,22 @@ static bool svc_rdma_pull_up_needed(const struct svcxprt_rdma *rdma, ret = pcl_process_nonpayloads(write_pcl, xdr, svc_rdma_xb_count_sges, &args); if (ret < 0) - return false; + return 0; if (args.pd_length < RPCRDMA_PULLUP_THRESH) - return true; - return args.pd_num_sges >= rdma->sc_max_send_sges; + return 1; + if (args.pd_num_sges < rdma->sc_max_send_sges) + return 0; + + /* + * The reply has too many SGEs to Send inline, so it has to be + * linearized into sc_xprt_buf. That buffer holds only + * sc_max_req_size bytes, so a larger reply cannot be pulled up. + * RFC 8166 Section 4.5.3 requires responding with ERR_CHUNK. + */ + if (args.pd_length > rdma->sc_max_req_size) + return -E2BIG; + return 1; } /** @@ -910,7 +922,7 @@ static int svc_rdma_xb_linearize(const struct xdr_buf *xdr, * Assemble the elements of @xdr into the transport header buffer. * * Assumptions: - * pull_up_needed has determined that @xdr will fit in the buffer. + * check_pull_up has determined that @xdr will fit in the buffer. * * Returns: * %0 if pull-up was successful @@ -945,6 +957,7 @@ static int svc_rdma_pull_up_reply_msg(const struct svcxprt_rdma *rdma, * * Returns: * %0 if DMA mapping was successful. + * %-E2BIG if the reply is too large to be pulled up * %-EMSGSIZE if a buffer manipulation problem occurred * %-EIO if DMA mapping failed * @@ -960,6 +973,7 @@ int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma, .md_rdma = rdma, .md_ctxt = sctxt, }; + int ret; /* Set up the (persistently-mapped) transport header SGE. */ sctxt->sc_send_wr.num_sge = 1; @@ -974,7 +988,10 @@ int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma, /* For pull-up, svc_rdma_send() will sync the transport header. * No additional DMA mapping is necessary. */ - if (svc_rdma_pull_up_needed(rdma, sctxt, write_pcl, xdr)) + ret = svc_rdma_check_pull_up(rdma, sctxt, write_pcl, xdr); + if (ret < 0) + return ret; + if (ret) return svc_rdma_pull_up_reply_msg(rdma, sctxt, write_pcl, xdr); return pcl_process_nonpayloads(write_pcl, xdr, @@ -1162,7 +1179,7 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) &rctxt->rc_reply_pcl, sctxt, &rqstp->rq_res); if (ret < 0) - goto reply_chunk; + goto send_err; rc_size = ret; } @@ -1183,10 +1200,10 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) ret = svc_rdma_send_reply_msg(rdma, sctxt, rctxt, rqstp); if (ret < 0) - goto put_ctxt; + goto send_err; return 0; -reply_chunk: +send_err: if (ret != -E2BIG && ret != -EINVAL) goto put_ctxt; diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 7ca71741106b1..093371f9d2459 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -43,6 +43,7 @@ */ #include +#include #include #include #include @@ -245,12 +246,16 @@ svc_rdma_parse_connect_private(struct svcxprt_rdma *newxprt, * structure for the listening endpoint. * * This function creates a new xprt for the new connection and enqueues it on - * the accept queue for the listent xprt. When the listen thread is kicked, it + * the accept queue for the listen xprt. When the listen thread is kicked, it * will call the recvfrom method on the listen xprt which will accept the new * connection. + * + * Return values: + * %0: Do not destroy @new_cma_id + * %1: Destroy @new_cma_id (allocation failure) */ -static void handle_connect_req(struct rdma_cm_id *new_cma_id, - struct rdma_conn_param *param) +static int handle_connect_req(struct rdma_cm_id *new_cma_id, + struct rdma_conn_param *param) { struct svcxprt_rdma *listen_xprt = new_cma_id->context; struct svcxprt_rdma *newxprt; @@ -260,7 +265,7 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id, listen_xprt->sc_xprt.xpt_net, ibdev_to_node(new_cma_id->device)); if (!newxprt) - return; + return 1; newxprt->sc_cm_id = new_cma_id; new_cma_id->context = newxprt; svc_rdma_parse_connect_private(newxprt, param); @@ -294,6 +299,7 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id, set_bit(XPT_CONN, &listen_xprt->sc_xprt.xpt_flags); svc_xprt_enqueue(&listen_xprt->sc_xprt); + return 0; } /** @@ -317,14 +323,14 @@ static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id, switch (event->event) { case RDMA_CM_EVENT_CONNECT_REQUEST: - handle_connect_req(cma_id, &event->param.conn); - break; + return handle_connect_req(cma_id, &event->param.conn); case RDMA_CM_EVENT_ADDR_CHANGE: listen_id = svc_rdma_create_listen_id(cma_rdma->xpt_net, sap, cma_xprt); if (IS_ERR(listen_id)) { pr_err("Listener dead, address change failed for device %s\n", cma_id->device->name); + cma_xprt->sc_cm_id = NULL; } else cma_xprt->sc_cm_id = listen_id; return 1; @@ -387,7 +393,13 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, listen_id = svc_rdma_create_listen_id(net, sa, cma_xprt); if (IS_ERR(listen_id)) { - kfree(cma_xprt); + /* _svc_xprt_create() acquired one module reference and + * puts it on xpo_create failure. svc_xprt_free() puts + * a second one when the kref drops to zero. Take a + * compensating reference so both puts are balanced. + */ + __module_get(cma_xprt->sc_xprt.xpt_class->xcl_owner); + svc_xprt_put(&cma_xprt->sc_xprt); return ERR_CAST(listen_id); } cma_xprt->sc_cm_id = listen_id; @@ -598,13 +610,26 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) return &newxprt->sc_xprt; errout: - /* Take a reference in case the DTO handler runs */ - svc_xprt_get(&newxprt->sc_xprt); - if (newxprt->sc_qp && !IS_ERR(newxprt->sc_qp)) - ib_destroy_qp(newxprt->sc_qp); - rdma_destroy_id(newxprt->sc_cm_id); - rpcrdma_rn_unregister(dev, &newxprt->sc_rn); - /* This call to put will destroy the transport */ + /* + * Drop the kref_init birth reference. svc_xprt_free will + * dispatch xpo_free = svc_rdma_free, which tears down sc_qp, + * sc_sq_cq, sc_rq_cq, and sc_pd under existing IS_ERR/NULL + * guards, and sc_rn under the rn_done sentinel guard inside + * rpcrdma_rn_unregister. + * + * sc_cm_id is destroyed unconditionally by svc_rdma_free; that + * is safe here because sc_cm_id is non-NULL by caller invariant + * on every path that reaches this errout: handle_connect_req + * installs newxprt->sc_cm_id before queueing the new xprt for + * accept, and svc_rdma_accept has already dereferenced it above + * the first goto errout. + * + * svc_handle_xprt() drops its pre-acquired module reference when + * ->xpo_accept() returns NULL. Take a replacement reference before + * freeing @newxprt, because svc_xprt_free() drops the module + * reference associated with @newxprt. + */ + __module_get(newxprt->sc_xprt.xpt_class->xcl_owner); svc_xprt_put(&newxprt->sc_xprt); return NULL; } @@ -614,7 +639,8 @@ static void svc_rdma_detach(struct svc_xprt *xprt) struct svcxprt_rdma *rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt); - rdma_disconnect(rdma->sc_cm_id); + if (rdma->sc_cm_id) + rdma_disconnect(rdma->sc_cm_id); /* * Most close paths go through svc_rdma_xprt_deferred_close(), @@ -634,10 +660,15 @@ static void svc_rdma_free(struct svc_xprt *xprt) { struct svcxprt_rdma *rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt); - struct ib_device *device = rdma->sc_cm_id->device; + struct ib_device *device; might_sleep(); + if (!rdma->sc_cm_id) + goto out_free; + + device = rdma->sc_cm_id->device; + /* This blocks until the Completion Queues are empty */ if (rdma->sc_qp && !IS_ERR(rdma->sc_qp)) ib_drain_qp(rdma->sc_qp); @@ -662,11 +693,13 @@ static void svc_rdma_free(struct svc_xprt *xprt) if (rdma->sc_pd && !IS_ERR(rdma->sc_pd)) ib_dealloc_pd(rdma->sc_pd); + if (!test_bit(XPT_LISTENER, &rdma->sc_xprt.xpt_flags)) + rpcrdma_rn_unregister(device, &rdma->sc_rn); + /* Destroy the CM ID */ rdma_destroy_id(rdma->sc_cm_id); - if (!test_bit(XPT_LISTENER, &rdma->sc_xprt.xpt_flags)) - rpcrdma_rn_unregister(device, &rdma->sc_rn); +out_free: kfree(rdma); } diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 359407aae03e6..7f60723fa64d8 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -1202,9 +1202,9 @@ static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk) static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk) { - sk->sk_data_ready = transport->old_data_ready; + WRITE_ONCE(sk->sk_data_ready, transport->old_data_ready); sk->sk_state_change = transport->old_state_change; - sk->sk_write_space = transport->old_write_space; + WRITE_ONCE(sk->sk_write_space, transport->old_write_space); sk->sk_error_report = transport->old_error_report; } @@ -1664,6 +1664,7 @@ static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) { struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); struct sock *sk = transport->inet; + void (*write_space)(struct sock *sock); if (transport->rcvsize) { sk->sk_userlocks |= SOCK_RCVBUF_LOCK; @@ -1672,7 +1673,8 @@ static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) if (transport->sndsize) { sk->sk_userlocks |= SOCK_SNDBUF_LOCK; sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2; - sk->sk_write_space(sk); + write_space = READ_ONCE(sk->sk_write_space); + write_space(sk); } } @@ -1988,8 +1990,8 @@ static int xs_local_finish_connecting(struct rpc_xprt *xprt, xs_save_old_callbacks(transport, sk); sk->sk_user_data = xprt; - sk->sk_data_ready = xs_data_ready; - sk->sk_write_space = xs_udp_write_space; + WRITE_ONCE(sk->sk_data_ready, xs_data_ready); + WRITE_ONCE(sk->sk_write_space, xs_udp_write_space); sk->sk_state_change = xs_local_state_change; sk->sk_error_report = xs_error_report; sk->sk_use_task_frag = false; @@ -2191,8 +2193,8 @@ static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) xs_save_old_callbacks(transport, sk); sk->sk_user_data = xprt; - sk->sk_data_ready = xs_data_ready; - sk->sk_write_space = xs_udp_write_space; + WRITE_ONCE(sk->sk_data_ready, xs_data_ready); + WRITE_ONCE(sk->sk_write_space, xs_udp_write_space); sk->sk_use_task_frag = false; xprt_set_connected(xprt); @@ -2378,9 +2380,9 @@ static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) xs_save_old_callbacks(transport, sk); sk->sk_user_data = xprt; - sk->sk_data_ready = xs_data_ready; + WRITE_ONCE(sk->sk_data_ready, xs_data_ready); sk->sk_state_change = xs_tcp_state_change; - sk->sk_write_space = xs_tcp_write_space; + WRITE_ONCE(sk->sk_write_space, xs_tcp_write_space); sk->sk_error_report = xs_error_report; sk->sk_use_task_frag = false; @@ -2648,7 +2650,17 @@ static int xs_tls_handshake_sync(struct rpc_xprt *lower_xprt, struct xprtsec_par rc = wait_for_completion_interruptible_timeout(&lower_transport->handshake_done, XS_TLS_HANDSHAKE_TO); if (rc <= 0) { - tls_handshake_cancel(sk); + if (!tls_handshake_cancel(sk)) { + /* + * Cancellation lost to handshake_complete(): the + * callback still owns its xprt reference and is in + * flight. Wait for it to finish before returning. + */ + wait_for_completion(&lower_transport->handshake_done); + if (rc == 0) + rc = -ETIMEDOUT; + goto out; + } if (rc == 0) rc = -ETIMEDOUT; goto out_put_xprt; diff --git a/net/tipc/group.c b/net/tipc/group.c index 14e6732624e28..74f6d3dac0784 100644 --- a/net/tipc/group.c +++ b/net/tipc/group.c @@ -797,10 +797,10 @@ void tipc_group_proto_rcv(struct tipc_group *grp, bool *usr_wakeup, tipc_group_open(m, usr_wakeup); return; case GRP_ACK_MSG: - if (!m) + if (!m || !grp->bc_ackers) return; acked = msg_grp_bc_acked(hdr); - if (less_eq(acked, m->bc_acked)) + if (acked != grp->bc_snd_nxt || m->bc_acked == acked) return; m->bc_acked = acked; if (--grp->bc_ackers) diff --git a/net/tipc/link.c b/net/tipc/link.c index 49dfc098d89b2..6427c69f89294 100644 --- a/net/tipc/link.c +++ b/net/tipc/link.c @@ -504,7 +504,7 @@ bool tipc_link_create(struct net *net, char *if_name, int bearer_id, snprintf(l->name, sizeof(l->name), "%s:%s-%s:unknown", self_str, if_name, peer_str); - strcpy(l->if_name, if_name); + strscpy(l->if_name, if_name); l->addr = peer; l->peer_caps = peer_caps; l->net = net; @@ -574,7 +574,7 @@ bool tipc_link_bc_create(struct net *net, u32 ownnode, u32 peer, u8 *peer_id, snprintf(l->name, sizeof(l->name), "%s:%s", tipc_bclink_name, peer_str); } else { - strcpy(l->name, tipc_bclink_name); + strscpy(l->name, tipc_bclink_name); } trace_tipc_link_reset(l, TIPC_DUMP_ALL, "bclink created!"); tipc_link_reset(l); @@ -1898,7 +1898,7 @@ static void tipc_link_build_proto_msg(struct tipc_link *l, int mtyp, bool probe, msg_set_dest_session(hdr, l->peer_session); } msg_set_max_pkt(hdr, l->advertised_mtu); - strcpy(data, l->if_name); + memcpy(data, l->if_name, TIPC_MAX_IF_NAME); msg_set_size(hdr, INT_H_SIZE + TIPC_MAX_IF_NAME); skb_trim(skb, INT_H_SIZE + TIPC_MAX_IF_NAME); } diff --git a/net/tipc/monitor.c b/net/tipc/monitor.c index a94b9b36a7008..1a438e312d66b 100644 --- a/net/tipc/monitor.c +++ b/net/tipc/monitor.c @@ -632,9 +632,10 @@ static void mon_timeout(struct timer_list *t) { struct tipc_monitor *mon = timer_container_of(mon, t, timer); struct tipc_peer *self; - int best_member_cnt = dom_size(mon->peer_cnt) - 1; + int best_member_cnt; write_lock_bh(&mon->lock); + best_member_cnt = dom_size(mon->peer_cnt) - 1; self = mon->self; if (self && (best_member_cnt != self->applied)) { mon_update_local_domain(mon); diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c index 253c72d1366eb..6fda36ab17669 100644 --- a/net/tipc/name_table.c +++ b/net/tipc/name_table.c @@ -763,21 +763,40 @@ struct publication *tipc_nametbl_publish(struct net *net, struct tipc_uaddr *ua, struct tipc_socket_addr *sk, u32 key) { struct name_table *nt = tipc_name_table(net); + u32 max_user_pub = TIPC_MAX_PUBL - 1; struct tipc_net *tn = tipc_net(net); struct publication *p = NULL; struct sk_buff *skb = NULL; + bool protocol_type = false; u32 rc_dests; + if (ua->sr.type == TIPC_NODE_STATE || ua->sr.type == TIPC_LINK_STATE || + ua->sr.type == TIPC_TOP_SRV) + protocol_type = true; + spin_lock_bh(&tn->nametbl_lock); + if (protocol_type) + goto insert; - if (nt->local_publ_count >= TIPC_MAX_PUBL) { - pr_warn("Bind failed, max limit %u reached\n", TIPC_MAX_PUBL); + /* Reserve one entry for node state service type because it has cluster + * scope and it is distributed in bulk. So, the maximum number of user's + * publications is (TIPC_MAX_PUBL - 1). + */ + if (nt->local_publ_count >= max_user_pub) { + pr_warn("Bind failed, max limit %u reached\n", max_user_pub); goto exit; } +insert: p = tipc_nametbl_insert_publ(net, ua, sk, key); if (p) { - nt->local_publ_count++; + /* Not count node state, link state and topology server types + * so that maximum nt->local_publ_count does not prevent + * protocol service types from being inserted into the name + * table. + */ + if (!protocol_type) + nt->local_publ_count++; skb = tipc_named_publish(net, p); } rc_dests = nt->rc_dests; @@ -810,7 +829,10 @@ void tipc_nametbl_withdraw(struct net *net, struct tipc_uaddr *ua, p = tipc_nametbl_remove_publ(net, ua, sk, key); if (p) { - nt->local_publ_count--; + if (p->sr.type != TIPC_NODE_STATE && + p->sr.type != TIPC_LINK_STATE && + p->sr.type != TIPC_TOP_SRV) + nt->local_publ_count--; skb = tipc_named_withdraw(net, p); list_del_init(&p->binding_sock); kfree_rcu(p, rcu); diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 37bb06a8e8f5b..f11d0528fc431 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -531,7 +531,8 @@ static int tls_push_data(struct sock *sk, if (!size) { last_record: tls_push_record_flags = flags; - if (flags & MSG_MORE) { + if ((flags & MSG_MORE) && + record->num_frags < MAX_SKB_FRAGS - 1) { more = true; break; } diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c index 61b10c697eccb..6cc222008d95c 100644 --- a/net/tls/tls_strp.c +++ b/net/tls/tls_strp.c @@ -430,9 +430,10 @@ static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort) return 0; } -static bool tls_strp_check_queue_ok(struct tls_strparser *strp) +static bool tls_strp_check_queue_ok(struct tls_strparser *strp, + unsigned int len) { - unsigned int len = strp->stm.offset + strp->stm.full_len; + unsigned int remaining = strp->stm.offset + len; struct sk_buff *first, *skb; u32 seq; @@ -443,9 +444,9 @@ static bool tls_strp_check_queue_ok(struct tls_strparser *strp) /* Make sure there's no duplicate data in the queue, * and the decrypted status matches. */ - while (skb->len < len) { + while (skb->len < remaining) { seq += skb->len; - len -= skb->len; + remaining -= skb->len; skb = skb->next; if (TCP_SKB_CB(skb)->seq != seq) @@ -525,6 +526,11 @@ static int tls_strp_read_sock(struct tls_strparser *strp) tls_strp_load_anchor_with_queue(strp, inq); if (!strp->stm.full_len) { + if (inq < TLS_HEADER_SIZE) + return tls_strp_read_copy(strp, true); + if (!tls_strp_check_queue_ok(strp, TLS_HEADER_SIZE)) + return tls_strp_read_copy(strp, false); + sz = tls_rx_msg_size(strp, strp->anchor); if (sz < 0) return sz; @@ -535,7 +541,7 @@ static int tls_strp_read_sock(struct tls_strparser *strp) return tls_strp_read_copy(strp, true); } - if (!tls_strp_check_queue_ok(strp)) + if (!tls_strp_check_queue_ok(strp, strp->stm.full_len)) return tls_strp_read_copy(strp, false); WRITE_ONCE(strp->msg_ready, 1); diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 10ed9421e43aa..c7a03b01119d7 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -2805,8 +2805,8 @@ static int unix_stream_recv_urg(struct unix_stream_read_state *state) return 1; } -static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk, - int flags, int copied) +static struct sk_buff *manage_oob(struct sk_buff *skb, struct sk_buff **last, + struct sock *sk, int flags, int copied) { struct sk_buff *read_skb = NULL, *unread_skb = NULL; struct unix_sock *u = unix_sk(sk); @@ -2820,11 +2820,13 @@ static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk, if (copied && (!u->oob_skb || skb == u->oob_skb)) { skb = NULL; } else if (flags & MSG_PEEK) { + *last = skb; skb = skb_peek_next(skb, &sk->sk_receive_queue); } else { read_skb = skb; skb = skb_peek_next(skb, &sk->sk_receive_queue); __skb_unlink(read_skb, &sk->sk_receive_queue); + *last = skb; } if (!skb) @@ -2843,8 +2845,10 @@ static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk, __skb_unlink(skb, &sk->sk_receive_queue); unread_skb = skb; skb = skb_peek(&sk->sk_receive_queue); + *last = skb; } } else if (!sock_flag(sk, SOCK_URGINLINE)) { + *last = skb; skb = skb_peek_next(skb, &sk->sk_receive_queue); } @@ -2964,8 +2968,8 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state, again: #if IS_ENABLED(CONFIG_AF_UNIX_OOB) if (skb) { - skb = manage_oob(skb, sk, flags, copied); - if (!skb && copied) { + skb = manage_oob(skb, &last, sk, flags, copied); + if (!skb && (copied || !state->size)) { unix_state_unlock(sk); break; } diff --git a/net/unix/garbage.c b/net/unix/garbage.c index 9fcaaf55cba5d..da774f56ca648 100644 --- a/net/unix/garbage.c +++ b/net/unix/garbage.c @@ -374,7 +374,7 @@ static bool unix_vertex_dead(struct unix_vertex *vertex) static LIST_HEAD(unix_visited_vertices); static unsigned long unix_vertex_grouped_index = UNIX_VERTEX_INDEX_MARK2; -static bool unix_scc_dead(struct list_head *scc, bool fast) +static bool unix_scc_dead(struct list_head *scc) { struct unix_vertex *vertex; bool scc_dead = true; @@ -386,10 +386,6 @@ static bool unix_scc_dead(struct list_head *scc, bool fast) /* Don't restart DFS from this vertex. */ list_move_tail(&vertex->entry, &unix_visited_vertices); - /* Mark vertex as off-stack for __unix_walk_scc(). */ - if (!fast) - vertex->index = unix_vertex_grouped_index; - if (scc_dead) scc_dead = unix_vertex_dead(vertex); } @@ -521,6 +517,7 @@ static unsigned long __unix_walk_scc(struct unix_vertex *vertex, } if (vertex->index == vertex->scc_index) { + struct unix_vertex *v; struct list_head scc; /* SCC finalised. @@ -530,7 +527,13 @@ static unsigned long __unix_walk_scc(struct unix_vertex *vertex, */ __list_cut_position(&scc, &vertex_stack, &vertex->scc_entry); - if (unix_scc_dead(&scc, false)) { + list_for_each_entry_reverse(v, &scc, scc_entry) { + /* Mark vertex as off-stack and assign a unique ID. */ + v->index = unix_vertex_grouped_index; + v->scc_index = vertex->scc_index; + } + + if (unix_scc_dead(&scc)) { unix_collect_skb(&scc, hitlist); } else { if (unix_vertex_max_scc_index < vertex->scc_index) @@ -588,7 +591,7 @@ static void unix_walk_scc_fast(struct sk_buff_head *hitlist) vertex = list_first_entry(&unix_unvisited_vertices, typeof(*vertex), entry); list_add(&scc, &vertex->scc_entry); - if (unix_scc_dead(&scc, true)) { + if (unix_scc_dead(&scc)) { cyclic_sccs--; unix_collect_skb(&scc, hitlist); } diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index 622dbd0467994..4a1dfde8414f4 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -1847,12 +1847,10 @@ static int vsock_connect(struct socket *sock, struct sockaddr_unsized *addr, prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); } - if (sk->sk_err) { - err = -sk->sk_err; + err = sock_error(sk); + if (err) { sk->sk_state = TCP_CLOSE; sock->state = SS_UNCONNECTED; - } else { - err = 0; } out_wait: @@ -1893,7 +1891,7 @@ static int vsock_accept(struct socket *sock, struct socket *newsock, timeout = sock_rcvtimeo(listener, arg->flags & O_NONBLOCK); while ((connected = vsock_dequeue_accept(listener)) == NULL && - listener->sk_err == 0 && timeout != 0) { + timeout != 0) { prepare_to_wait(sk_sleep(listener), &wait, TASK_INTERRUPTIBLE); release_sock(listener); timeout = schedule_timeout(timeout); @@ -1906,13 +1904,9 @@ static int vsock_accept(struct socket *sock, struct socket *newsock, } } - if (listener->sk_err) { - err = -listener->sk_err; - } else if (!connected) { + if (!connected) { err = -EAGAIN; - } - - if (connected) { + } else { sk_acceptq_removed(listener); lock_sock_nested(connected, SINGLE_DEPTH_NESTING); @@ -2879,6 +2873,9 @@ static int vsock_net_child_mode_string(const struct ctl_table *table, int write, net = container_of(table->data, struct net, vsock.child_ns_mode); + if (!*lenp) + return 0; + ret = __vsock_net_mode_string(table, write, buffer, lenp, ppos, vsock_net_child_mode(net), &new_mode); if (ret) diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c index 96c9fe8d357c6..4f9aa9c4c3aa5 100644 --- a/net/vmw_vsock/virtio_transport.c +++ b/net/vmw_vsock/virtio_transport.c @@ -872,10 +872,10 @@ static void virtio_vsock_remove(struct virtio_device *vdev) /* Other works can be queued before 'config->del_vqs()', so we flush * all works before to free the vsock object to avoid use after free. */ - flush_work(&vsock->rx_work); flush_work(&vsock->tx_work); flush_work(&vsock->event_work); flush_work(&vsock->send_pkt_work); + flush_work(&vsock->rx_work); mutex_unlock(&the_virtio_vsock_mutex); diff --git a/net/wireless/core.c b/net/wireless/core.c index 610238d723fff..bc7e2c50cc726 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -153,68 +153,93 @@ int cfg80211_dev_rename(struct cfg80211_registered_device *rdev, return 0; } -int cfg80211_switch_netns(struct cfg80211_registered_device *rdev, - struct net *net) +static int cfg80211_switch_wdev_netns(struct wireless_dev *wdev, + struct net *net) { - struct wireless_dev *wdev; - int err = 0; + int err; - if (!(rdev->wiphy.flags & WIPHY_FLAG_NETNS_OK)) - return -EOPNOTSUPP; + if (!wdev->netdev) + return 0; - list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { - if (!wdev->netdev) + wdev->netdev->netns_immutable = false; + err = dev_change_net_namespace(wdev->netdev, net, "wlan%d"); + wdev->netdev->netns_immutable = true; + + return err; +} + +static int __cfg80211_switch_netns(struct cfg80211_registered_device *rdev, + struct net *net, bool force) +{ + struct net *old_net = wiphy_net(&rdev->wiphy); + struct wireless_dev *wdev, *tmp; + int err = 0; + + list_for_each_entry_safe(wdev, tmp, &rdev->wiphy.wdev_list, list) { + err = cfg80211_switch_wdev_netns(wdev, net); + if (!err) continue; - wdev->netdev->netns_immutable = false; - err = dev_change_net_namespace(wdev->netdev, net, "wlan%d"); - if (err) - break; - wdev->netdev->netns_immutable = true; + if (!force) + goto undo; + /* remove interfaces that fail to allow wiphy switching */ + dev_close(wdev->netdev); + scoped_guard(wiphy, &rdev->wiphy) + cfg80211_unregister_wdev(wdev); + err = 0; } - if (err) { - /* failed -- clean up to old netns */ - net = wiphy_net(&rdev->wiphy); - - list_for_each_entry_continue_reverse(wdev, - &rdev->wiphy.wdev_list, - list) { + scoped_guard(wiphy, &rdev->wiphy) { + list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { if (!wdev->netdev) continue; - wdev->netdev->netns_immutable = false; - err = dev_change_net_namespace(wdev->netdev, net, - "wlan%d"); - WARN_ON(err); - wdev->netdev->netns_immutable = true; + nl80211_notify_iface(rdev, wdev, + NL80211_CMD_DEL_INTERFACE); } - return err; - } + nl80211_notify_wiphy(rdev, NL80211_CMD_DEL_WIPHY); - guard(wiphy)(&rdev->wiphy); + wiphy_net_set(&rdev->wiphy, net); - list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { - if (!wdev->netdev) - continue; - nl80211_notify_iface(rdev, wdev, NL80211_CMD_DEL_INTERFACE); - } + /* this only fails on allocation failure */ + err = device_rename(&rdev->wiphy.dev, + dev_name(&rdev->wiphy.dev)); + if (err && !force) + wiphy_net_set(&rdev->wiphy, old_net); - nl80211_notify_wiphy(rdev, NL80211_CMD_DEL_WIPHY); + nl80211_notify_wiphy(rdev, NL80211_CMD_NEW_WIPHY); - wiphy_net_set(&rdev->wiphy, net); + list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { + if (!wdev->netdev) + continue; + nl80211_notify_iface(rdev, wdev, + NL80211_CMD_NEW_INTERFACE); + } + } - err = device_rename(&rdev->wiphy.dev, dev_name(&rdev->wiphy.dev)); - WARN_ON(err); + if (!err || force) + return err; - nl80211_notify_wiphy(rdev, NL80211_CMD_NEW_WIPHY); + /* set to the last one to undo all of them */ + wdev = list_entry(&rdev->wiphy.wdev_list, typeof(*wdev), list); +undo: + /* + * Move back everything, if this fails again (allocation failures) + * then things get stuck in different network namespaces. + */ + list_for_each_entry_continue_reverse(wdev, &rdev->wiphy.wdev_list, + list) + WARN_ON(cfg80211_switch_wdev_netns(wdev, old_net)); - list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) { - if (!wdev->netdev) - continue; - nl80211_notify_iface(rdev, wdev, NL80211_CMD_NEW_INTERFACE); - } + return err; +} - return 0; +int cfg80211_switch_netns(struct cfg80211_registered_device *rdev, + struct net *net) +{ + if (!(rdev->wiphy.flags & WIPHY_FLAG_NETNS_OK)) + return -EOPNOTSUPP; + + return __cfg80211_switch_netns(rdev, net, false); } static void cfg80211_rfkill_poll(struct rfkill *rfkill, void *data) @@ -237,15 +262,15 @@ void cfg80211_stop_p2p_device(struct cfg80211_registered_device *rdev, if (!wdev_running(wdev)) return; + cfg80211_pmsr_wdev_down(wdev); rdev_stop_p2p_device(rdev, wdev); wdev->is_running = false; rdev->opencount--; if (rdev->scan_req && rdev->scan_req->req.wdev == wdev) { - if (WARN_ON(!rdev->scan_req->notified && - (!rdev->int_scan_req || - !rdev->int_scan_req->notified))) + if (!rdev->scan_req->notified && + (!rdev->int_scan_req || !rdev->int_scan_req->notified)) rdev->scan_req->info.aborted = true; ___cfg80211_scan_done(rdev, false); } @@ -264,6 +289,8 @@ void cfg80211_stop_nan(struct cfg80211_registered_device *rdev, if (!wdev_running(wdev)) return; + cfg80211_pmsr_wdev_down(wdev); + /* * If there is a scheduled update pending, mark it as canceled, so the * empty schedule will be accepted @@ -1755,9 +1782,9 @@ static int cfg80211_netdev_notifier_call(struct notifier_block *nb, wiphy_lock(&rdev->wiphy); cfg80211_update_iface_num(rdev, wdev->iftype, -1); if (rdev->scan_req && rdev->scan_req->req.wdev == wdev) { - if (WARN_ON(!rdev->scan_req->notified && - (!rdev->int_scan_req || - !rdev->int_scan_req->notified))) + if (!rdev->scan_req->notified && + (!rdev->int_scan_req || + !rdev->int_scan_req->notified)) rdev->scan_req->info.aborted = true; ___cfg80211_scan_done(rdev, false); } @@ -1865,7 +1892,7 @@ static void __net_exit cfg80211_pernet_exit(struct net *net) rtnl_lock(); for_each_rdev(rdev) { if (net_eq(wiphy_net(&rdev->wiphy), net)) - WARN_ON(cfg80211_switch_netns(rdev, &init_net)); + WARN_ON(__cfg80211_switch_netns(rdev, &init_net, true)); } rtnl_unlock(); } diff --git a/net/wireless/core.h b/net/wireless/core.h index ac6ce9f967ec7..81e595247b932 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -24,6 +24,16 @@ struct cfg80211_scan_request_int { struct cfg80211_scan_info info; bool notified; + /* + * set while the request is handed to the driver, i.e. between + * rdev_scan() and cfg80211_scan_done() + */ + bool driver_owns; + /* + * set when cfg80211 is done with the request but the driver still + * owns it, so that cfg80211_scan_done() knows to just free it + */ + bool stale; /* must be last - variable members */ struct cfg80211_scan_request req; }; @@ -280,8 +290,7 @@ struct cfg80211_event { bool locally_generated; } dc; struct { - u8 bssid[ETH_ALEN]; - struct ieee80211_channel *channel; + struct cfg80211_bss *bss; } ij; struct { u8 peer_addr[ETH_ALEN]; @@ -344,8 +353,7 @@ int __cfg80211_join_ibss(struct cfg80211_registered_device *rdev, void cfg80211_clear_ibss(struct net_device *dev, bool nowext); int cfg80211_leave_ibss(struct cfg80211_registered_device *rdev, struct net_device *dev, bool nowext); -void __cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid, - struct ieee80211_channel *channel); +void __cfg80211_ibss_joined(struct net_device *dev, struct cfg80211_bss *bss); int cfg80211_ibss_wext_join(struct cfg80211_registered_device *rdev, struct wireless_dev *wdev); diff --git a/net/wireless/ibss.c b/net/wireless/ibss.c index b1d748bdb504e..7f6779d326b81 100644 --- a/net/wireless/ibss.c +++ b/net/wireless/ibss.c @@ -16,26 +16,18 @@ #include "rdev-ops.h" -void __cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid, - struct ieee80211_channel *channel) +void __cfg80211_ibss_joined(struct net_device *dev, struct cfg80211_bss *bss) { struct wireless_dev *wdev = dev->ieee80211_ptr; - struct cfg80211_bss *bss; #ifdef CONFIG_CFG80211_WEXT union iwreq_data wrqu; #endif if (WARN_ON(wdev->iftype != NL80211_IFTYPE_ADHOC)) - return; + goto put_bss; if (!wdev->u.ibss.ssid_len) - return; - - bss = cfg80211_get_bss(wdev->wiphy, channel, bssid, NULL, 0, - IEEE80211_BSS_TYPE_IBSS, IEEE80211_PRIVACY_ANY); - - if (WARN_ON(!bss)) - return; + goto put_bss; if (wdev->u.ibss.current_bss) { cfg80211_unhold_bss(wdev->u.ibss.current_bss); @@ -43,17 +35,22 @@ void __cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid, } cfg80211_hold_bss(bss_from_pub(bss)); + /* the reference from the event is transferred to current_bss */ wdev->u.ibss.current_bss = bss_from_pub(bss); cfg80211_upload_connect_keys(wdev); - nl80211_send_ibss_bssid(wiphy_to_rdev(wdev->wiphy), dev, bssid, + nl80211_send_ibss_bssid(wiphy_to_rdev(wdev->wiphy), dev, bss->bssid, GFP_KERNEL); #ifdef CONFIG_CFG80211_WEXT memset(&wrqu, 0, sizeof(wrqu)); - memcpy(wrqu.ap_addr.sa_data, bssid, ETH_ALEN); + memcpy(wrqu.ap_addr.sa_data, bss->bssid, ETH_ALEN); wireless_send_event(dev, SIOCGIWAP, &wrqu, NULL); #endif + return; + +put_bss: + cfg80211_put_bss(wdev->wiphy, bss); } void cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid, @@ -62,6 +59,7 @@ void cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid, struct wireless_dev *wdev = dev->ieee80211_ptr; struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy); struct cfg80211_event *ev; + struct cfg80211_bss *bss; unsigned long flags; trace_cfg80211_ibss_joined(dev, bssid, channel); @@ -69,13 +67,19 @@ void cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid, if (WARN_ON(!channel)) return; + bss = cfg80211_get_bss(wdev->wiphy, channel, bssid, NULL, 0, + IEEE80211_BSS_TYPE_IBSS, IEEE80211_PRIVACY_ANY); + if (WARN_ON(!bss)) + return; + ev = kzalloc_obj(*ev, gfp); - if (!ev) + if (!ev) { + cfg80211_put_bss(wdev->wiphy, bss); return; + } ev->type = EVENT_IBSS_JOINED; - memcpy(ev->ij.bssid, bssid, ETH_ALEN); - ev->ij.channel = channel; + ev->ij.bss = bss; spin_lock_irqsave(&wdev->event_lock, flags); list_add_tail(&ev->list, &wdev->event_list); diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 5adcb6bd0fc56..755f8fe711fb2 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -18750,15 +18750,15 @@ static int nl80211_color_change(struct sk_buff *skb, struct genl_info *info) if (!wdev->links[params.link_id].ap.beacon_interval) return -EINVAL; + tb = kzalloc_objs(*tb, NL80211_ATTR_MAX + 1); + if (!tb) + return -ENOMEM; + err = nl80211_parse_beacon(rdev, info->attrs, ¶ms.beacon_next, wdev->links[params.link_id].ap.chandef.chan, info->extack); if (err) - return err; - - tb = kzalloc_objs(*tb, NL80211_ATTR_MAX + 1); - if (!tb) - return -ENOMEM; + goto out; err = nla_parse_nested(tb, NL80211_ATTR_MAX, info->attrs[NL80211_ATTR_COLOR_CHANGE_ELEMS], diff --git a/net/wireless/rdev-ops.h b/net/wireless/rdev-ops.h index 63c26e8b11395..8f466e1ab1d72 100644 --- a/net/wireless/rdev-ops.h +++ b/net/wireless/rdev-ops.h @@ -464,7 +464,10 @@ static inline int rdev_scan(struct cfg80211_registered_device *rdev, return -EINVAL; trace_rdev_scan(&rdev->wiphy, request); + request->driver_owns = true; ret = rdev->ops->scan(&rdev->wiphy, &request->req); + if (ret) + request->driver_owns = false; trace_rdev_return_int(&rdev->wiphy, ret); return ret; } diff --git a/net/wireless/reg.c b/net/wireless/reg.c index 1e8214d6b6d88..3454afa038bf7 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -2345,7 +2345,7 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev) iftype = wdev->iftype; /* make sure the interface is active */ - if (!wdev->netdev || !netif_running(wdev->netdev)) + if (!wdev_running(wdev)) return true; /* NAN doesn't have links, handle it separately */ diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 071083cc33672..4a1ec7e3547d8 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -1114,6 +1114,21 @@ int cfg80211_scan(struct cfg80211_registered_device *rdev) return 0; } +/* + * Release the scan request, but free it only if the driver is also done, + * e.g. mac80211 may cancel it asynchronously and still use it. + */ +static void cfg80211_put_scan_req(struct cfg80211_scan_request_int *req) +{ + if (!req) + return; + + if (req->driver_owns) + req->stale = true; + else + kfree(req); +} + void ___cfg80211_scan_done(struct cfg80211_registered_device *rdev, bool send_message) { @@ -1173,10 +1188,10 @@ void ___cfg80211_scan_done(struct cfg80211_registered_device *rdev, dev_put(wdev->netdev); - kfree(rdev->int_scan_req); + cfg80211_put_scan_req(rdev->int_scan_req); rdev->int_scan_req = NULL; - kfree(rdev->scan_req); + cfg80211_put_scan_req(rdev->scan_req); rdev->scan_req = NULL; if (!send_message) @@ -1199,6 +1214,18 @@ void cfg80211_scan_done(struct cfg80211_scan_request *request, struct cfg80211_scan_info old_info = intreq->info; trace_cfg80211_scan_done(intreq, info); + + intreq->driver_owns = false; + + if (intreq->stale) { + /* + * The scan is already completed as far as we're concerned, + * it was just kept around for the driver - done now, free it. + */ + kfree(intreq); + return; + } + WARN_ON(intreq != rdev->scan_req && intreq != rdev->int_scan_req); @@ -2022,6 +2049,13 @@ __cfg80211_bss_update(struct cfg80211_registered_device *rdev, if (!hidden) hidden = rb_find_bss(rdev, tmp, BSS_CMP_HIDE_NUL); + /* + * Only group with an entry with beacon data, otherwise + * beacon data can never be filled/updated. + */ + if (hidden && + !rcu_access_pointer(hidden->pub.beacon_ies)) + hidden = NULL; if (hidden) { new->pub.hidden_beacon_bss = &hidden->pub; list_add(&new->hidden_list, @@ -3443,11 +3477,6 @@ void cfg80211_update_assoc_bss_entry(struct wireless_dev *wdev, cbss->pub.channel = chan; list_for_each_entry(bss, &rdev->bss_list, list) { - if (!cfg80211_bss_type_match(bss->pub.capability, - bss->pub.channel->band, - wdev->conn_bss_type)) - continue; - if (bss == cbss) continue; diff --git a/net/wireless/util.c b/net/wireless/util.c index 24527bf321b2a..12285bbd29730 100644 --- a/net/wireless/util.c +++ b/net/wireless/util.c @@ -988,12 +988,30 @@ unsigned int cfg80211_classify8021d(struct sk_buff *skb, } switch (skb->protocol) { - case htons(ETH_P_IP): - dscp = ipv4_get_dsfield(ip_hdr(skb)) & 0xfc; + case htons(ETH_P_IP): { + const struct iphdr *iph; + struct iphdr _iph; + + iph = skb_header_pointer(skb, sizeof(struct ethhdr), + sizeof(*iph), &_iph); + if (!iph) + return 0; + + dscp = ipv4_get_dsfield(iph) & 0xfc; break; - case htons(ETH_P_IPV6): - dscp = ipv6_get_dsfield(ipv6_hdr(skb)) & 0xfc; + } + case htons(ETH_P_IPV6): { + const struct ipv6hdr *ip6h; + struct ipv6hdr _ip6h; + + ip6h = skb_header_pointer(skb, sizeof(struct ethhdr), + sizeof(*ip6h), &_ip6h); + if (!ip6h) + return 0; + + dscp = ipv6_get_dsfield(ip6h) & 0xfc; break; + } case htons(ETH_P_MPLS_UC): case htons(ETH_P_MPLS_MC): { struct mpls_label mpls_tmp, *mpls; @@ -1166,8 +1184,7 @@ void cfg80211_process_wdev_events(struct wireless_dev *wdev) !ev->dc.locally_generated); break; case EVENT_IBSS_JOINED: - __cfg80211_ibss_joined(wdev->netdev, ev->ij.bssid, - ev->ij.channel); + __cfg80211_ibss_joined(wdev->netdev, ev->ij.bss); break; case EVENT_STOPPED: /* @@ -2426,16 +2443,15 @@ static void cfg80211_calculate_bi_data(struct wiphy *wiphy, u32 new_beacon_int, if (wdev->valid_links) continue; + wdev_bi = cfg80211_wdev_bi(wdev); + if (!wdev_bi) + continue; + /* skip wdevs not active on the given wiphy radio */ if (radio_idx >= 0 && !(rdev_get_radio_mask(rdev, wdev->netdev) & BIT(radio_idx))) continue; - wdev_bi = cfg80211_wdev_bi(wdev); - - if (!wdev_bi) - continue; - if (!*beacon_int_gcd) { *beacon_int_gcd = wdev_bi; continue; diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 7855ee09c4b64..33475b180ea6f 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool); void *copy_from = xsk_copy_xdp_start(xdp), *copy_to; u32 from_len, meta_len, rem, num_desc; - struct xdp_buff_xsk *xskb; + struct xdp_buff_xsk *xskb, *tmp; struct xdp_buff *xsk_xdp; + LIST_HEAD(xsk_buffs); skb_frag_t *frag; + u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from; @@ -343,23 +345,45 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0]; } + for (i = 0; i < num_desc; i++) { + xsk_xdp = xsk_buff_alloc(xs->pool); + if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); + if (unlikely(!list_empty(&xskb->list_node))) + goto err_alloc; + + list_add_tail(&xskb->list_node, &xsk_buffs); + } + do { u32 to_len = frame_size + meta_len; u32 copied; - xsk_xdp = xsk_buff_alloc(xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); + list_del_init(&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied = xsk_copy_xdp(copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); __xsk_rcv_zc_safe(xs, xskb, copied - meta_len, rem ? XDP_PKT_CONTD : 0); meta_len = 0; } while (rem); return 0; + +err_alloc: + list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) { + list_del_init(&xskb->list_node); + xsk_buff_free(&xskb->xdp); + } + xs->rx_dropped++; + return -ENOMEM; } static bool xsk_tx_writeable(struct xdp_sock *xs) diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index 78c14f1063957..9d2d94f1fb759 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -763,6 +763,7 @@ EXPORT_SYMBOL(xp_raw_get_dma); * xp_raw_get_ctx - get &xdp_desc context * @pool: XSk buff pool desc address belongs to * @addr: desc address (from userspace) + * @options: desc options (from userspace) * * Helper for getting desc's DMA address and metadata pointer, if present. * Saves one call on hotpath and double calculation of the actual address. @@ -771,14 +772,16 @@ EXPORT_SYMBOL(xp_raw_get_dma); * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata * pointer, if it is present (initialized to %NULL otherwise). */ -struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, + u32 options) { struct xdp_desc_ctx ret; addr = __xp_raw_get_addr(pool, addr); ret.dma = __xp_raw_get_dma(pool, addr); - ret.meta = __xsk_buff_get_metadata(pool, __xp_raw_get_data(pool, addr)); + ret.meta = __xsk_buff_get_metadata(pool, __xp_raw_get_data(pool, addr), + options); return ret; } diff --git a/net/xdp/xskmap.c b/net/xdp/xskmap.c index 3bff346308d0f..bf00d6463c191 100644 --- a/net/xdp/xskmap.c +++ b/net/xdp/xskmap.c @@ -124,7 +124,7 @@ static int xsk_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) struct bpf_insn *insn = insn_buf; *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); - *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); + *insn++ = BPF_JMP32_IMM(BPF_JGE, ret, map->max_entries, 5); *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(sizeof(struct xsk_sock *))); *insn++ = BPF_ALU64_IMM(BPF_ADD, mp, offsetof(struct xsk_map, xsk_map)); *insn++ = BPF_ALU64_REG(BPF_ADD, ret, mp); diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index 374e1b9644383..3e72b9f067b9b 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -30,13 +30,22 @@ static void handle_esp(struct sk_buff *skb, struct sock *sk) { struct tcp_skb_cb *tcp_cb = (struct tcp_skb_cb *)skb->cb; - skb_reset_transport_header(skb); + if (!skb_reset_transport_header_careful(skb)) { + XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR); + kfree_skb(skb); + return; + } /* restore IP CB, we need at least IP6CB->nhoff */ memmove(skb->cb, &tcp_cb->header, sizeof(tcp_cb->header)); rcu_read_lock(); skb->dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif); + if (!skb->dev) { + XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR); + kfree_skb(skb); + goto out; + } local_bh_disable(); #if IS_ENABLED(CONFIG_IPV6) if (sk->sk_family == AF_INET6) @@ -45,6 +54,7 @@ static void handle_esp(struct sk_buff *skb, struct sock *sk) #endif xfrm4_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP); local_bh_enable(); +out: rcu_read_unlock(); } @@ -515,7 +525,8 @@ static void espintcp_close(struct sock *sk, long timeout) strp_stop(&ctx->strp); sk->sk_prot = &tcp_prot; - barrier(); + + synchronize_rcu(); disable_work_sync(&ctx->work); strp_done(&ctx->strp); diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c index eecab337bd0a7..5ed87d51392a5 100644 --- a/net/xfrm/xfrm_input.c +++ b/net/xfrm/xfrm_input.c @@ -474,6 +474,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) struct xfrm_state *x = NULL; xfrm_address_t *daddr; u32 mark = skb->mark; + u8 xfrm_proto = nexthdr; unsigned int family = AF_UNSPEC; int decaps = 0; int async = 0; @@ -485,6 +486,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) if (encap_type < 0 || (xo && (xo->flags & XFRM_GRO || encap_type == 0 || encap_type == UDP_ENCAP_ESPINUDP))) { x = xfrm_input_state(skb); + xfrm_proto = x->type ? x->type->proto : nexthdr; if (unlikely(x->km.state != XFRM_STATE_VALID)) { if (x->km.state == XFRM_STATE_ACQ) @@ -592,11 +594,13 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) x = xfrm_input_state_lookup(net, mark, daddr, spi, nexthdr, family); if (x == NULL) { + xfrm_proto = nexthdr; secpath_reset(skb); XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES); xfrm_audit_state_notfound(skb, family, spi, seq); goto drop; } + xfrm_proto = x->type ? x->type->proto : nexthdr; if (unlikely(x->dir && x->dir != XFRM_SA_DIR_IN)) { secpath_reset(skb); @@ -604,6 +608,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) xfrm_audit_state_notfound(skb, family, spi, seq); xfrm_state_put(x); x = NULL; + xfrm_proto = nexthdr; goto drop; } @@ -728,7 +733,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) } while (!err); rcu_read_lock(); - err = xfrm_rcv_cb(skb, family, x->type->proto, 0); + err = xfrm_rcv_cb(skb, family, xfrm_proto, 0); if (err) { rcu_read_unlock(); goto drop; @@ -753,7 +758,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) xfrm_gro = xo->flags & XFRM_GRO; err = -EAFNOSUPPORT; - afinfo = xfrm_state_afinfo_get_rcu(x->props.family); + afinfo = xfrm_state_afinfo_get_rcu(family); if (likely(afinfo)) err = afinfo->transport_finish(skb, xfrm_gro || async); if (xfrm_gro) { @@ -776,7 +781,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) drop: if (async) dev_put(dev); - xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1); + xfrm_rcv_cb(skb, family, xfrm_proto, -1); kfree_skb(skb); return 0; } @@ -800,12 +805,17 @@ static void xfrm_trans_reinject(struct work_struct *work) spin_unlock_bh(&trans->queue_lock); local_bh_disable(); + rcu_read_lock(); while ((skb = __skb_dequeue(&queue))) { struct net *net = XFRM_TRANS_SKB_CB(skb)->net; + struct net_device *dev = skb->dev; XFRM_TRANS_SKB_CB(skb)->finish(net, NULL, skb); + if (dev) + dev_put(dev); put_net(net); } + rcu_read_unlock(); local_bh_enable(); } @@ -821,12 +831,18 @@ int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb, if (skb_queue_len(&trans->queue) >= READ_ONCE(net_hotdata.max_backlog)) return -ENOBUFS; + if (skb_dst(skb) && !skb_dst_force(skb)) + return -EHOSTUNREACH; + BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb)); hold_net = maybe_get_net(net); if (!hold_net) return -ENODEV; + if (skb->dev) + dev_hold(skb->dev); + XFRM_TRANS_SKB_CB(skb)->finish = finish; XFRM_TRANS_SKB_CB(skb)->net = hold_net; spin_lock_bh(&trans->queue_lock); diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c index 597aedeac26eb..6920940a35b49 100644 --- a/net/xfrm/xfrm_iptfs.c +++ b/net/xfrm/xfrm_iptfs.c @@ -416,6 +416,14 @@ static bool iptfs_skb_can_add_frags(const struct sk_buff *skb, if (skb_has_frag_list(skb) || skb->pp_recycle != walk->pp_recycle) return false; + /* Reject an @offset that is at or beyond the end of the walk's data + * before calling iptfs_skb_reset_frag_walk(), whose fragment-advance + * loop is otherwise unbounded and would index past walk->frags[]. + * This mirrors the guard already present in iptfs_skb_add_frags(). + */ + if (!walk->nr_frags || offset >= walk->total + walk->initial_offset) + return false; + /* Make offset relative to current frag after setting that */ offset = iptfs_skb_reset_frag_walk(walk, offset); @@ -820,8 +828,8 @@ static u32 iptfs_reassem_cont(struct xfrm_iptfs_data *xtfs, u64 seq, * allocate an in progress skb */ ipremain = __iptfs_iplen(xtfs->ra_runt); - if (ipremain < sizeof(xtfs->ra_runt)) { - /* length has to be at least runtsize large */ + if (ipremain < __iptfs_iphlen(xtfs->ra_runt)) { + /* length has to be at least the IP header size */ XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMINIPTFSERROR); goto abandon; diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index eb1b6f67739e1..5cd6d43164dbc 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c @@ -155,25 +155,50 @@ static void nat_keepalive_send(struct nat_keepalive *ka) } } +enum { + NAT_KEEPALIVE_BATCH_SIZE = 16, + NAT_KEEPALIVE_BATCH_FULL = 1, +}; + struct nat_keepalive_work_ctx { + struct xfrm_state *batch[NAT_KEEPALIVE_BATCH_SIZE]; + unsigned int nr; time64_t next_run; time64_t now; }; -static int nat_keepalive_work_single(struct xfrm_state *x, int count, void *ptr) +static int nat_keepalive_work_collect(struct xfrm_state *x, int count, void *ptr) { struct nat_keepalive_work_ctx *ctx = ptr; + + if (!READ_ONCE(x->nat_keepalive_interval)) + return 0; + + if (ctx->nr == ARRAY_SIZE(ctx->batch)) + return NAT_KEEPALIVE_BATCH_FULL; + + xfrm_state_hold(x); + ctx->batch[ctx->nr++] = x; + return 0; +} + +static void nat_keepalive_work_single(struct xfrm_state *x, + struct nat_keepalive_work_ctx *ctx) +{ bool send_keepalive = false; struct nat_keepalive ka; - time64_t next_run; + time64_t next_run = 0; u32 interval; int delta; + spin_lock_bh(&x->lock); + + if (x->km.state == XFRM_STATE_DEAD) + goto out; + interval = x->nat_keepalive_interval; if (!interval) - return 0; - - spin_lock(&x->lock); + goto out; delta = (int)(ctx->now - x->lastused); if (delta < interval) { @@ -187,14 +212,14 @@ static int nat_keepalive_work_single(struct xfrm_state *x, int count, void *ptr) send_keepalive = true; } - spin_unlock(&x->lock); +out: + spin_unlock_bh(&x->lock); if (send_keepalive) nat_keepalive_send(&ka); - if (!ctx->next_run || next_run < ctx->next_run) + if (next_run && (!ctx->next_run || next_run < ctx->next_run)) ctx->next_run = next_run; - return 0; } static void nat_keepalive_work(struct work_struct *work) @@ -202,13 +227,23 @@ static void nat_keepalive_work(struct work_struct *work) struct nat_keepalive_work_ctx ctx; struct xfrm_state_walk walk; struct net *net; + int err, i; ctx.next_run = 0; ctx.now = ktime_get_real_seconds(); net = container_of(work, struct net, xfrm.nat_keepalive_work.work); xfrm_state_walk_init(&walk, IPPROTO_ESP, NULL); - xfrm_state_walk(net, &walk, nat_keepalive_work_single, &ctx); + do { + ctx.nr = 0; + err = xfrm_state_walk(net, &walk, nat_keepalive_work_collect, &ctx); + local_bh_disable(); + for (i = 0; i < ctx.nr; i++) { + nat_keepalive_work_single(ctx.batch[i], &ctx); + xfrm_state_put(ctx.batch[i]); + } + local_bh_enable(); + } while (err == NAT_KEEPALIVE_BATCH_FULL); xfrm_state_walk_done(&walk, net); if (ctx.next_run) schedule_delayed_work(&net->xfrm.nat_keepalive_work, diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index cc35c2fcbbe09..e305ba32e356b 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c @@ -636,10 +636,8 @@ static int xfrm_dev_direct_output(struct sock *sk, struct xfrm_state *x, nf_reset_ct(skb); err = skb_dst(skb)->ops->local_out(net, sk, skb); - if (unlikely(err != 1)) { - kfree_skb(skb); + if (unlikely(err != 1)) return err; - } /* In transport mode, network destination is * directly reachable, while in tunnel mode, diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 932a313b9460a..513c9f2283347 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -2770,9 +2770,12 @@ static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy, xdst0->path = dst; err = -ENODEV; - dev = dst->dev; - if (!dev) + rcu_read_lock(); + dev = dst_dev_rcu(dst); + if (!dev) { + rcu_read_unlock(); goto free_dst; + } xfrm_init_path(xdst0, dst, nfheader_len); xfrm_init_pmtu(bundle, nx); @@ -2780,8 +2783,10 @@ static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy, for (xdst_prev = xdst0; xdst_prev != (struct xfrm_dst *)dst; xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) { err = xfrm_fill_dst(xdst_prev, dev, fl); - if (err) + if (err) { + rcu_read_unlock(); goto free_dst; + } xdst_prev->u.dst.header_len = header_len; xdst_prev->u.dst.trailer_len = trailer_len; @@ -2789,6 +2794,7 @@ static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy, trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len; } + rcu_read_unlock(); return &xdst0->u.dst; put_states: @@ -3058,11 +3064,15 @@ static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net, xfrm_init_path((struct xfrm_dst *)dst1, dst, 0); err = -ENODEV; - dev = dst->dev; - if (!dev) + rcu_read_lock(); + dev = dst_dev_rcu(dst); + if (!dev) { + rcu_read_unlock(); goto free_dst; + } err = xfrm_fill_dst(xdst, dev, fl); + rcu_read_unlock(); if (err) goto free_dst; diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 36a4f6793edef..e45aa1ed5b965 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -226,6 +226,7 @@ static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO]; static DEFINE_SPINLOCK(xfrm_state_gc_lock); static DEFINE_SPINLOCK(xfrm_state_dev_gc_lock); +static DEFINE_MUTEX(xfrm_state_gc_mutex); int __xfrm_state_delete(struct xfrm_state *x); @@ -632,8 +633,10 @@ static void xfrm_state_gc_task(struct work_struct *work) synchronize_rcu(); + mutex_lock(&xfrm_state_gc_mutex); hlist_for_each_entry_safe(x, tmp, &gc_list, gclist) xfrm_state_gc_destroy(x); + mutex_unlock(&xfrm_state_gc_mutex); } static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) @@ -823,9 +826,9 @@ int __xfrm_state_delete(struct xfrm_state *x) if (!hlist_unhashed(&x->byseq)) hlist_del_init_rcu(&x->byseq); if (!hlist_unhashed(&x->state_cache)) - hlist_del_rcu(&x->state_cache); + hlist_del_init_rcu(&x->state_cache); if (!hlist_unhashed(&x->state_cache_input)) - hlist_del_rcu(&x->state_cache_input); + hlist_del_init_rcu(&x->state_cache_input); if (!hlist_unhashed(&x->byspi)) hlist_del_init_rcu(&x->byspi); @@ -1000,6 +1003,7 @@ int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_vali out: spin_unlock_bh(&net->xfrm.xfrm_state_lock); + mutex_lock(&xfrm_state_gc_mutex); spin_lock_bh(&xfrm_state_dev_gc_lock); restart_gc: hlist_for_each_entry_safe(x, tmp, &xfrm_state_dev_gc_list, dev_gclist) { @@ -1014,6 +1018,7 @@ int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_vali } spin_unlock_bh(&xfrm_state_dev_gc_lock); + mutex_unlock(&xfrm_state_gc_mutex); xfrm_flush_gc(); diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index d6db63304ba6b..a2587c7e796b4 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -940,7 +940,7 @@ static struct xfrm_state *xfrm_state_construct(struct net *net, if ((err = attach_auth_trunc(&x->aalg, &x->props.aalgo, attrs[XFRMA_ALG_AUTH_TRUNC], extack))) goto error; - if (!x->props.aalgo) { + if (!x->aalg) { if ((err = attach_auth(&x->aalg, &x->props.aalgo, attrs[XFRMA_ALG_AUTH], extack))) goto error; @@ -1877,7 +1877,6 @@ static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh, struct net *net = sock_net(skb->sk); struct xfrm_state *x; struct xfrm_userspi_info *p; - struct xfrm_translator *xtr; struct sk_buff *resp_skb; xfrm_address_t *daddr; int family; @@ -1943,17 +1942,6 @@ static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh, goto out; } - xtr = xfrm_get_translator(); - if (xtr) { - err = xtr->alloc_compat(skb, nlmsg_hdr(skb)); - - xfrm_put_translator(xtr); - if (err) { - kfree_skb(resp_skb); - goto out; - } - } - err = nlmsg_unicast(xfrm_net_nlsk(net, skb), resp_skb, NETLINK_CB(skb).portid); out: @@ -3337,7 +3325,11 @@ static int xfrm_send_migrate_state(struct net *net, return err; } - return xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_MIGRATE); + rcu_read_lock(); + err = xfrm_nlmsg_multicast(net, skb, 0, XFRMNLGRP_MIGRATE); + rcu_read_unlock(); + + return err; } static int xfrm_do_migrate_state(struct sk_buff *skb, struct nlmsghdr *nlh, diff --git a/rust/Makefile b/rust/Makefile index 627ed79dc6f50..9211d82ac96da 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -350,7 +350,8 @@ rusttestlib-pin_init: $(src)/pin-init/src/lib.rs rusttestlib-macros \ rusttestlib-kernel: private rustc_target_flags = --extern ffi \ --extern build_error --extern macros --extern pin_init \ --extern bindings --extern uapi \ - --extern zerocopy --extern zerocopy_derive + --extern zerocopy=$(objtree)/$(obj)/test/libzerocopy.rlib \ + --extern zerocopy_derive=$(objtree)/$(obj)/test/$(libzerocopy_derive_name) rusttestlib-kernel: $(src)/kernel/lib.rs rusttestlib-bindings rusttestlib-uapi \ rusttestlib-build_error rusttestlib-pin_init $(obj)/$(libmacros_name) \ $(obj)/bindings.o rusttestlib-zerocopy rusttestlib-zerocopy_derive FORCE @@ -644,7 +645,8 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L -Zunstable-options \ $(if $(link_helper),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) \ $(obj)/helpers/helpers$(if $(part-of-module),_module).bc -o $(patsubst %.o,%.m.bc,$@); \ - $(CC) $(CLANG_FLAGS) $(KBUILD_CFLAGS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ + $(CC) $(CLANG_FLAGS) $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS)) \ + $(CC_FLAGS_RUST_INLINE_HELPERS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ $(cmd_ld_single)) \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ $(cmd_objtool) diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 812f8e5a08d53..439ab88a5da13 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -22,11 +22,13 @@ #![feature(cfi_encoding)] #[allow(dead_code)] +#[allow(clippy::as_underscore)] #[allow(clippy::cast_lossless)] #[allow(clippy::ptr_as_ptr)] #[allow(clippy::ref_as_ptr)] #[allow(clippy::undocumented_unsafe_blocks)] -#[cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] +#[cfg_attr(not(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES), allow(unknown_lints))] +#[allow(unnecessary_transmutes)] #[cfg_attr( CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, allow(suspicious_runtime_symbol_definitions) diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e660..4b90a1390ad58 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -58,7 +58,9 @@ #include "cred.c" #include "device.c" #include "dma.c" +#ifdef CONFIG_DMA_SHARED_BUFFER #include "dma-resv.c" +#endif #include "drm.c" #include "drm_gpuvm.c" #include "err.c" diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs index ed943960f8516..3566f0234ca40 100644 --- a/rust/kernel/bug.rs +++ b/rust/kernel/bug.rs @@ -8,6 +8,7 @@ #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] #[cfg(CONFIG_DEBUG_BUGVERBOSE)] macro_rules! warn_flags { @@ -47,12 +48,17 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] #[cfg(not(CONFIG_DEBUG_BUGVERBOSE))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags; + if false { + _ = $file; + } + // SAFETY: // - `flags` and `size` are all compile-time constants, preventing // any invalid memory access. @@ -73,14 +79,19 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, CONFIG_UML))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { + if false { + _ = $file; + } + // SAFETY: It is always safe to call `warn_slowpath_fmt()` // with a valid null-terminated string. unsafe { $crate::bindings::warn_slowpath_fmt( - $crate::c_str!(::core::file!()).as_char_ptr(), + $crate::str::CStrExt::as_char_ptr($crate::c_str!(::core::file!())), line!() as $crate::ffi::c_int, $flags as $crate::ffi::c_uint, ::core::ptr::null(), @@ -91,9 +102,15 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, any(CONFIG_LOONGARCH, CONFIG_ARM)))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + // SAFETY: It is always safe to call `WARN_ON()`. unsafe { $crate::bindings::WARN_ON(true) } }; @@ -101,9 +118,14 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] -#[cfg(not(CONFIG_BUG))] +#[cfg(any(testlib, not(CONFIG_BUG)))] macro_rules! warn_flags { - ($file:expr, $flags:expr) => {}; + ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + }; } #[doc(hidden)] @@ -118,14 +140,14 @@ macro_rules! warn_on { let cond = $cond; #[cfg(CONFIG_DEBUG_BUGVERBOSE_DETAILED)] - const _COND_STR: &str = concat!("[", stringify!($cond), "] ", file!()); + const COND_STR: &str = concat!("[", stringify!($cond), "] ", file!()); #[cfg(not(CONFIG_DEBUG_BUGVERBOSE_DETAILED))] - const _COND_STR: &str = file!(); + const COND_STR: &str = file!(); if cond { const WARN_ON_FLAGS: u32 = $crate::bug::bugflag_taint($crate::bindings::TAINT_WARN); - $crate::warn_flags!(_COND_STR, WARN_ON_FLAGS); + $crate::warn_flags!(COND_STR, WARN_ON_FLAGS); } cond }}; diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 58ac04c650a11..1158ef3a85f2f 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -361,23 +361,28 @@ impl TableBuilder { } } - /// Adds a new entry to the table. - pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + /// Adds a raw frequency-table entry. + fn push(&mut self, frequency: u32, flags: u32, driver_data: u32) -> Result { // Adds the new entry at the end of the vector. Ok(self.entries.push( bindings::cpufreq_frequency_table { flags, driver_data, - frequency: freq.as_khz() as u32, + frequency, }, GFP_KERNEL, )?) } + /// Adds a new entry to the table. + pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + self.push(freq.as_khz() as u32, flags, driver_data) + } + /// Consumes the [`TableBuilder`] and returns [`TableBox`]. pub fn to_table(mut self) -> Result { // Add last entry to the table. - self.add(Hertz(c_ulong::MAX), 0, 0)?; + self.push(bindings::CPUFREQ_TABLE_END as u32, 0, 0)?; TableBox::new(self.entries) } @@ -817,7 +822,9 @@ pub trait Driver { } /// Driver's `bios_limit` callback. - fn bios_limit(_policy: &mut Policy, _limit: &mut u32) -> Result { + /// + /// Returns HW/BIOS max frequency limitations for the CPU. + fn bios_limit(_policy: &mut Policy) -> Result { build_error!(VTABLE_DEFAULT_ERROR) } @@ -1352,9 +1359,12 @@ impl Registration { from_result(|| { let mut policy = PolicyCpu::from_cpu(cpu_id)?; - + let val = T::bios_limit(&mut policy)?; // SAFETY: `limit` is guaranteed by the C code to be valid. - T::bios_limit(&mut policy, &mut (unsafe { *limit })).map(|()| 0) + unsafe { + *limit = val; + } + Ok(0) }) } diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 1a38b3bbdfb7d..09ad8ec31548c 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -511,7 +511,11 @@ pub struct Normal; /// callback it appears in. It is intended to be used for synchronization purposes. Bus device /// implementations can implement methods for [`Device`], such that they can only be called /// from bus callbacks. -pub struct Core<'a>(PhantomData<&'a ()>); +/// +/// The lifetime `'a` is for "lifetime branding" purpose. Callbacks need to polymorphic over this +/// lifetime so the `&'bound Device>` provided to them cannot outlive the scope of the +/// function. For this reason, it needs to be invariant. +pub struct Core<'a>(PhantomData &'a ()>); /// Semantically the same as [`Core`], but reserved for internal usage of the corresponding bus /// abstraction. @@ -522,7 +526,9 @@ pub struct Core<'a>(PhantomData<&'a ()>); /// /// This context mainly exists to share generic [`Device`] infrastructure that should only be called /// from bus callbacks with bus abstractions, but without making them accessible for drivers. -pub struct CoreInternal<'a>(PhantomData<&'a ()>); +/// +/// Lifetime `'a` is invariant for the same reason as [`Core`]. +pub struct CoreInternal<'a>(PhantomData &'a ()>); /// The [`Bound`] context is the [`DeviceContext`] of a bus specific device when it is guaranteed to /// be bound to a driver. diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 8ff8aedf251a8..73ff8575712c3 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -21,7 +21,8 @@ use crate::{ sync::{ aref::ARef, rcu, - Arc, // + Arc, + Completion, // }, types::{ ForeignOwnable, @@ -37,6 +38,8 @@ struct Inner { node: Opaque, #[pin] data: Revocable, + #[pin] + revocation: Completion, } /// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to @@ -53,6 +56,10 @@ struct Inner { /// After the [`Devres`] has been unbound it is not possible to access the encapsulated resource /// anymore. /// +/// When a [`Devres`] is dropped, it is guaranteed that `T` has been fully dropped by the time +/// [`Devres::drop`] returns, even if a concurrent revocation through the release callback is in +/// progress. +/// /// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s /// [`Drop`] implementation. /// @@ -218,6 +225,7 @@ impl Devres { }; }), data <- Revocable::new(data), + revocation <- Completion::new(), }), GFP_KERNEL, )?; @@ -255,7 +263,14 @@ impl Devres { // SAFETY: `inner` is a valid `Inner` pointer. let inner = unsafe { &*inner }; - inner.data.revoke(); + if inner.data.revoke() { + inner.revocation.complete_all(); + } else { + // Devres::drop() is concurrently revoking; wait for it to finish `drop_in_place()` + // before returning to `devres_release_all()`, ensuring `T` is fully torn down before + // the device finishes unbinding. + inner.revocation.wait_for_completion(); + } } #[allow(clippy::missing_safety_doc)] @@ -355,6 +370,8 @@ impl Drop for Devres { // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data // anymore, hence it is safe not to wait for the grace period to finish. if unsafe { self.data().revoke_nosync() } { + self.inner.revocation.complete_all(); + // We revoked `self.data` before devres did, hence try to remove it. if self.remove_node() { // SAFETY: In `Self::new` we have taken an additional reference count of `self.data` @@ -362,6 +379,10 @@ impl Drop for Devres { // this additional reference count. drop(unsafe { Arc::from_raw(Arc::as_ptr(&self.inner)) }); } + } else { + // The release callback is concurrently revoking; wait for it to finish + // `drop_in_place()` of the wrapped object before returning. + self.inner.revocation.wait_for_completion(); } } } diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 200def84fb69e..556424b88fb94 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -1005,7 +1005,11 @@ impl debugfs::BinaryWriter for Coherent { return Ok(0); }; - let count = self.size().saturating_sub(offset_val).min(writer.len()); + if offset_val >= self.size() { + return Ok(0); + } + + let count = (self.size() - offset_val).min(writer.len()); writer.write_dma(self, offset_val, count)?; diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index ae58f6f667c1f..70cf11346ceee 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -72,10 +72,12 @@ pub struct GpuVm { data: UnsafeCell, } -// SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will -// drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait. +// SAFETY: It is safe to send a `GpuVm` to another thread: all data reachable through it +// (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds. unsafe impl Send for GpuVm {} -// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel. +// SAFETY: It is safe to share a `&GpuVm` between threads: `&self` methods only alias data +// that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the +// reference and ultimately drop `T`, which the same bounds make `Send`. unsafe impl Sync for GpuVm {} // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. @@ -116,9 +118,9 @@ impl GpuVm { /// Creates a GPUVM instance. #[expect(clippy::new_ret_no_self)] - pub fn new( + pub fn new( name: &'static CStr, - dev: &drm::Device, + dev: &drm::Device, r_obj: &T::Object, range: Range, reserve_range: Range, @@ -250,18 +252,22 @@ impl GpuVm { } /// The manager for a GPUVM. -pub trait DriverGpuVm: Sized + Send { +pub trait DriverGpuVm: Sized + Send + Sync { /// Parent `Driver` for this object. - type Driver: drm::Driver; + type Driver: drm::Driver; /// The kind of GEM object stored in this GPUVM. - type Object: IntoGEMObject; + type Object: drm::driver::AllocImpl + Send + Sync; /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). - type VaData; + /// + /// Only `Send` is required: the data has a single owner at all times, moving + /// between threads by value (handed back as a [`GpuVaRemoved`]) but never + /// accessed by two threads concurrently. + type VaData: Send; /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). - type VmBoData; + type VmBoData: Send + Sync; /// The private data passed to callbacks. type SmContext<'ctx>; @@ -296,12 +302,10 @@ pub trait DriverGpuVm: Sized + Send { /// # Invariants /// /// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. +// `Send`/`Sync` derive from `ARef>`; the trait bounds make them correct for the unique +// handle's `&mut T` access. pub struct UniqueRefGpuVm(ARef>); -// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and -// concurrent access to `data` is safe due to the `T: Sync` requirement. -unsafe impl Sync for UniqueRefGpuVm {} - impl UniqueRefGpuVm { /// Access the data owned by this `UniqueRefGpuVm` immutably. #[inline] diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index cf328101dde4e..ccf4150d83b69 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -135,6 +135,12 @@ macro_rules! declare_drm_ioctls { // dev/file match the current driver these ioctls are being declared // for, and it's not clear how to enforce this within the type system. let dev = $crate::drm::device::Device::from_raw(raw_dev); + + // Enforce that the handler accepts higher-ranked + // lifetimes, preventing it from requiring 'static + // references that could escape this scope. + let _: for<'a> fn(&'a _, &'a mut _, &'a _) -> _ = $func; + // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we // asserted above matches the size of this type, and all bit patterns of // UAPI structs must be valid. diff --git a/rust/kernel/fmt.rs b/rust/kernel/fmt.rs index 73afbc51ba33a..cd7d9664ff5b9 100644 --- a/rust/kernel/fmt.rs +++ b/rust/kernel/fmt.rs @@ -43,7 +43,14 @@ use core::fmt::{ UpperExp, UpperHex, // }; -impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, Pointer, LowerExp, UpperExp); +impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, LowerExp, UpperExp); + +impl Pointer for Adapter<&T> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(self.0, f) + } +} /// A copy of [`core::fmt::Display`] that allows us to implement it for foreign types. /// diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 406e3a028c558..0f367264ee2eb 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -249,7 +249,7 @@ pub use self::arc_field::{ /// assert_eq!(list.iter().count(), 3); /// } /// -/// // Pop the items from the list using `pop_front()` and verify the content. +/// // Pop the items from the list using `pop_back()` and verify the content. /// { /// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15)); /// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32)); diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 3ca99db5cccf2..b5f1ada8e02e4 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -123,39 +123,37 @@ impl Device { /// Gets the current link state. /// /// It returns true if the link is up. + #[inline] pub fn is_link_up(&self) -> bool { - const LINK_IS_UP: u64 = 1; - // TODO: the code to access to the bit field will be replaced with automatically - // generated code by bindgen when it becomes possible. - // SAFETY: The struct invariant ensures that we may access - // this field without additional synchronization. - let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; - bit_field.get(14, 1) == LINK_IS_UP + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let link = unsafe { bindings::phy_device::link_raw(phydev) }; + link == 1 } /// Gets the current auto-negotiation configuration. /// /// It returns true if auto-negotiation is enabled. + #[inline] pub fn is_autoneg_enabled(&self) -> bool { - // TODO: the code to access to the bit field will be replaced with automatically - // generated code by bindgen when it becomes possible. - // SAFETY: The struct invariant ensures that we may access - // this field without additional synchronization. - let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; - bit_field.get(13, 1) == u64::from(bindings::AUTONEG_ENABLE) + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let autoneg = unsafe { bindings::phy_device::autoneg_raw(phydev) }; + autoneg == bindings::AUTONEG_ENABLE } /// Gets the current auto-negotiation state. /// /// It returns true if auto-negotiation is completed. + #[inline] pub fn is_autoneg_completed(&self) -> bool { - const AUTONEG_COMPLETED: u64 = 1; - // TODO: the code to access to the bit field will be replaced with automatically - // generated code by bindgen when it becomes possible. - // SAFETY: The struct invariant ensures that we may access - // this field without additional synchronization. - let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; - bit_field.get(15, 1) == AUTONEG_COMPLETED + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let completed = unsafe { bindings::phy_device::autoneg_complete_raw(phydev) }; + completed == 1 } /// Sets the speed of the PHY. diff --git a/rust/kernel/num.rs b/rust/kernel/num.rs index 8532b511384cc..ba1eb58aee02f 100644 --- a/rust/kernel/num.rs +++ b/rust/kernel/num.rs @@ -13,9 +13,14 @@ pub enum Unsigned {} /// Designates signed primitive types. pub enum Signed {} +mod private { + pub trait Sealed {} +} + /// Describes core properties of integer types. pub trait Integer: - Sized + private::Sealed + + Sized + Copy + Clone + PartialEq @@ -54,6 +59,8 @@ pub trait Integer: macro_rules! impl_integer { ($($type:ty: $signedness:ty), *) => { $( + impl private::Sealed for $type {} + impl Integer for $type { type Signedness = $signedness; diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index dafe77782d797..6d766c22e61b5 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -13,7 +13,10 @@ use core::{ }; use kernel::{ - num::Integer, + num::{ + Integer, + Unsigned, // + }, prelude::*, // }; @@ -174,13 +177,16 @@ fn fits_within(value: T, num_bits: u32) -> bool { /// // `u8` (regardless of the passed value). /// // let _ = Bounded::::from(10u8); /// -/// // Booleans can be converted into single-bit `Bounded`s. +/// // Booleans can be converted into unsigned `Bounded`s. /// /// let v = Bounded::::from(false); /// assert_eq!(v.get(), 0); /// /// let v = Bounded::::from(true); /// assert_eq!(v.get(), 1); +/// +/// // This does not build because `i8` is signed. +/// // let _ = Bounded::::from(true); /// ``` /// /// Infallible conversions from a [`Bounded`] to a primitive integer are also supported, and @@ -203,12 +209,16 @@ fn fits_within(value: T, num_bits: u32) -> bool { /// let _v = Bounded::::new::<10>(); /// // assert_eq!(u8::from(_v), 10); /// -/// // Single-bit `Bounded`s can be converted into a boolean. +/// // Unsigned single-bit `Bounded`s can be converted into a boolean. /// let v = Bounded::::new::<1>(); /// assert_eq!(bool::from(v), true); /// /// let v = Bounded::::new::<0>(); /// assert_eq!(bool::from(v), false); +/// +/// // This does not build because `i8` is signed. +/// // let v = Bounded::::new::<-1>(); +/// // let _ = bool::from(v); /// ``` /// /// Fallible conversions from any primitive integer to any [`Bounded`] are also supported using the @@ -485,6 +495,7 @@ where /// assert_eq!(v_shifted.get(), 0xff); /// ``` pub fn shr(self) -> Bounded { + const_assert!(SHIFT < T::BITS); const { assert!(RES + SHIFT >= N) } // SAFETY: We shift the value right by `SHIFT`, reducing the number of bits needed to @@ -1077,31 +1088,33 @@ impl_into_primitive!( i8 i16 i32 i64 isize ); -// Single-bit `Bounded`s can be converted from/to a boolean. +// Unsigned single-bit `Bounded`s can be converted to a boolean. impl From> for bool where - T: Integer + Zeroable, + T: Integer + Zeroable, { fn from(value: Bounded) -> Self { value.get() != Zeroable::zeroed() } } +// Booleans can be converted to unsigned `Bounded`s. + impl From for Bounded where - T: Integer + From, + T: Integer + From, { fn from(value: bool) -> Self { - // SAFETY: A boolean can be represented using a single bit, and thus fits within any - // integer type for any `N` > 0. + // SAFETY: A boolean is represented by `0` or `1`, so it fits within any valid unsigned + // `Bounded` width. unsafe { Self::__new(T::from(value)) } } } impl Bounded where - T: Integer + Zeroable, + T: Integer + Zeroable, { /// Converts this [`Bounded`] into a [`bool`]. /// diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index b8463823aed95..54fc46d460b6b 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -441,22 +441,25 @@ impl Delta { /// to the value in the [`Delta`]. #[inline] pub fn as_micros_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. let n = self.as_nanos(); - let n = if n >= 0 { - n.saturating_add(NSEC_PER_USEC - 1) - } else { - n - }; + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; #[cfg(CONFIG_64BIT)] { - n / NSEC_PER_USEC + n / NSEC_PER_USEC + add } #[cfg(not(CONFIG_64BIT))] // SAFETY: It is always safe to call `ktime_to_us()` with any value. unsafe { - bindings::ktime_to_us(n) + bindings::ktime_to_us(n) + add } } diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index fd40c8f244a1e..27a6969fa29b8 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -493,13 +493,7 @@ macro_rules! stack_pin_init { (let $var:ident $(: $t:ty)? = $val:expr) => { let val = $val; let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); - let mut $var = match $crate::__internal::StackInit::init($var, val) { - Ok(res) => res, - Err(x) => { - let x: ::core::convert::Infallible = x; - match x {} - } - }; + let Ok(mut $var) = $crate::__internal::StackInit::init($var, val); }; } diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs index 797ead5b5626c..003e6d4f7c4b7 100644 --- a/rust/uapi/lib.rs +++ b/rust/uapi/lib.rs @@ -10,6 +10,7 @@ #![no_std] #![allow( clippy::all, + clippy::as_underscore, clippy::cast_lossless, clippy::ptr_as_ptr, clippy::ref_as_ptr, @@ -23,7 +24,8 @@ unreachable_pub, unsafe_op_in_unsafe_fn )] -#![cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] +#![cfg_attr(not(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES), allow(unknown_lints))] +#![allow(unnecessary_transmutes)] #![cfg_attr( CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, allow(suspicious_runtime_symbol_definitions) diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 06d7d1a2e8dab..1c4aa4292dcec 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -6,11 +6,11 @@ * Sorted alphabetically. */ -#include #include #include #include #include +#include #include #include #include diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index dae6929f827c2..1c7c9c2b47a0d 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -180,6 +180,7 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote) static int damon_sample_mtier_start(void) { struct damon_ctx *ctx; + int err; ctx = damon_sample_mtier_build_ctx(true); if (!ctx) @@ -191,12 +192,21 @@ static int damon_sample_mtier_start(void) return -ENOMEM; } ctxs[1] = ctx; - return damon_start(ctxs, 2, true); + err = damon_start(ctxs, 2, true); + if (!err) + return 0; + + if (damon_is_running(ctxs[0])) + damon_stop(ctxs, 1); + damon_destroy_ctx(ctxs[0]); + damon_destroy_ctx(ctxs[1]); + return err; } static void damon_sample_mtier_stop(void) { - damon_stop(ctxs, 2); + damon_stop(ctxs, 1); + damon_stop(&ctxs[1], 1); damon_destroy_ctx(ctxs[0]); damon_destroy_ctx(ctxs[1]); } diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c index b7c50f2656ce7..edeae145c4a8a 100644 --- a/samples/damon/prcl.c +++ b/samples/damon/prcl.c @@ -106,11 +106,18 @@ static int damon_sample_prcl_start(void) damon_set_schemes(ctx, &scheme, 1); err = damon_start(&ctx, 1, true); - if (err) + if (err) { + damon_destroy_ctx(ctx); return err; + } repeat_call_control.data = ctx; - return damon_call(ctx, &repeat_call_control); + err = damon_call(ctx, &repeat_call_control); + if (err) { + damon_stop(&ctx, 1); + damon_destroy_ctx(ctx); + } + return err; } static void damon_sample_prcl_stop(void) diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c index 799ad44439434..ff5e8a890f448 100644 --- a/samples/damon/wsse.c +++ b/samples/damon/wsse.c @@ -87,10 +87,17 @@ static int damon_sample_wsse_start(void) target->pid = target_pidp; err = damon_start(&ctx, 1, true); - if (err) + if (err) { + damon_destroy_ctx(ctx); return err; + } repeat_call_control.data = ctx; - return damon_call(ctx, &repeat_call_control); + err = damon_call(ctx, &repeat_call_control); + if (err) { + damon_stop(&ctx, 1); + damon_destroy_ctx(ctx); + } + return err; } static void damon_sample_wsse_stop(void) diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c index 1ba1927b548ee..164d9dd6fd921 100644 --- a/samples/ftrace/ftrace-direct-modify.c +++ b/samples/ftrace/ftrace-direct-modify.c @@ -320,9 +320,15 @@ static int __init ftrace_direct_init(void) ftrace_set_filter_ip(&direct, (unsigned long) my_ip, 0, 0); ret = register_ftrace_direct(&direct, my_tramp); - if (!ret) - simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); - return ret; + if (ret) + return ret; + simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); + if (IS_ERR(simple_tsk)) { + unregister_ftrace_direct(&direct, my_tramp, true); + return PTR_ERR(simple_tsk); + } + + return 0; } static void __exit ftrace_direct_exit(void) diff --git a/samples/ftrace/ftrace-direct-multi-modify.c b/samples/ftrace/ftrace-direct-multi-modify.c index 7a7822dfeb50a..b03766c6217bc 100644 --- a/samples/ftrace/ftrace-direct-multi-modify.c +++ b/samples/ftrace/ftrace-direct-multi-modify.c @@ -364,9 +364,15 @@ static int __init ftrace_direct_multi_init(void) ret = register_ftrace_direct(&direct, my_tramp); - if (!ret) - simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); - return ret; + if (ret) + return ret; + simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); + if (IS_ERR(simple_tsk)) { + unregister_ftrace_direct(&direct, my_tramp, true); + return PTR_ERR(simple_tsk); + } + + return 0; } static void __exit ftrace_direct_multi_exit(void) diff --git a/samples/rust/rust_debugfs.rs b/samples/rust/rust_debugfs.rs index 1f59e08aaa4b0..0b27ad96ecbfa 100644 --- a/samples/rust/rust_debugfs.rs +++ b/samples/rust/rust_debugfs.rs @@ -147,7 +147,9 @@ impl RustDebugFs { dir.read_write_file(c"pair", new_mutex!(Inner { x: 3, y: 10 })) } - fn new<'a>(pdev: &'a platform::Device>) -> impl PinInit + 'a { + fn new<'a, 'b>( + pdev: &'a platform::Device>, + ) -> impl PinInit + use<'a, 'b> { let debugfs = Dir::new(c"sample_debugfs"); let dev = pdev.as_ref(); diff --git a/samples/rust/rust_debugfs_scoped.rs b/samples/rust/rust_debugfs_scoped.rs index 6a575a15a2c2f..ca2b154be3842 100644 --- a/samples/rust/rust_debugfs_scoped.rs +++ b/samples/rust/rust_debugfs_scoped.rs @@ -75,7 +75,10 @@ fn create_file_write( GFP_KERNEL, )?; } - let blob = KBox::pin_init(new_mutex!([0x42; SZ_4K]), GFP_KERNEL)?; + let blob = KBox::pin_init( + new_mutex!(pin_init::init_array_from_fn(|_| 0x42)), + GFP_KERNEL, + )?; let scope = KBox::pin_init( mod_data.device_dir.scope( diff --git a/samples/rust/rust_print_main.rs b/samples/rust/rust_print_main.rs index 682207c81fc2b..01729e87d6b5a 100644 --- a/samples/rust/rust_print_main.rs +++ b/samples/rust/rust_print_main.rs @@ -23,10 +23,10 @@ fn arc_print() -> Result { let b = UniqueArc::new("hello, world", GFP_KERNEL)?; // Prints the value of data in `a`. - pr_info!("{}", a); + pr_info!("{}\n", a); // Uses ":?" to print debug fmt of `b`. - pr_info!("{:?}", b); + pr_info!("{:?}\n", b); let a: Arc<&str> = b.into(); let c = a.clone(); @@ -42,7 +42,7 @@ fn arc_print() -> Result { use kernel::fmt::Display; fn arc_dyn_print(arc: &Arc) { - pr_info!("Arc says {arc}"); + pr_info!("Arc says {arc}\n"); } let a_i32_display: Arc = Arc::new(42i32, GFP_KERNEL)?; @@ -53,7 +53,7 @@ fn arc_print() -> Result { } // Pretty-prints the debug formatting with lower-case hexadecimal integers. - pr_info!("{:#x?}", a); + pr_info!("{:#x?}\n", a); Ok(()) } diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 9117457432463..0b71f759eb5c9 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -346,7 +346,8 @@ quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_o_rs = $(rust_common_cmd) --emit=$(if $(CONFIG_RUST_INLINE_HELPERS),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) $< \ $(if $(CONFIG_RUST_INLINE_HELPERS),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) \ $(objtree)/rust/helpers/helpers$(if $(part-of-module),_module).bc -o $(patsubst %.o,%.m.bc,$@); \ - $(CC) $(CLANG_FLAGS) $(KBUILD_CFLAGS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ + $(CC) $(CLANG_FLAGS) $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS)) \ + $(CC_FLAGS_RUST_INLINE_HELPERS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ $(cmd_ld_single)) \ $(cmd_objtool) diff --git a/scripts/Makefile.vmlinux_o b/scripts/Makefile.vmlinux_o index 527352c222ff6..24a3a4fd271c2 100644 --- a/scripts/Makefile.vmlinux_o +++ b/scripts/Makefile.vmlinux_o @@ -47,6 +47,9 @@ endif vmlinux-objtool-args-$(CONFIG_NOINSTR_VALIDATION) += --noinstr \ $(if $(or $(CONFIG_MITIGATION_UNRET_ENTRY),$(CONFIG_MITIGATION_SRSO)), --unret) +# Only used for builds initiated by klp-build +vmlinux-objtool-args-$(if $(KLP_SYMIDS),y) += --klp-symids + objtool-args = $(vmlinux-objtool-args-y) --link # Link of vmlinux.o used for section mismatch analysis diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index 3bf296581a88e..7687b0dd5474e 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -224,6 +224,11 @@ fn main() { features += ",+harden-sls-ijmp"; features += ",+harden-sls-ret"; } + if cfg.has("X86_NATIVE_CPU") { + // Prevent the backend from generating APX instructions. The kernel is not yet prepared + // for general in-kernel EGPR use. + features += ",-apxf"; + } ts.push("features", features); ts.push("llvm-target", "x86_64-linux-gnu"); ts.push("supported-sanitizers", ["kcfi", "kernel-address"]); diff --git a/scripts/livepatch/init.c b/scripts/livepatch/init.c index f14d8c8fb35fa..16aff8f736eb6 100644 --- a/scripts/livepatch/init.c +++ b/scripts/livepatch/init.c @@ -51,7 +51,7 @@ static int __init livepatch_mod_init(void) if (!funcs) { ret = -ENOMEM; for (int j = 0; j < i; j++) - kfree(objs[i].funcs); + kfree(objs[j].funcs); goto err_free_objs; } diff --git a/scripts/livepatch/klp-build b/scripts/livepatch/klp-build index c4a7acf8edc3f..30cd881ca7ed1 100755 --- a/scripts/livepatch/klp-build +++ b/scripts/livepatch/klp-build @@ -271,6 +271,9 @@ validate_config() { [[ -v CONFIG_GCC_PLUGIN_RANDSTRUCT ]] && \ die "kernel option 'CONFIG_GCC_PLUGIN_RANDSTRUCT' not supported" + [[ -v CONFIG_LD_DEAD_CODE_DATA_ELIMINATION ]] && \ + die "kernel option 'CONFIG_LD_DEAD_CODE_DATA_ELIMINATION' not supported" + [[ -v CONFIG_AS_IS_LLVM ]] && \ [[ "$CONFIG_AS_VERSION" -lt 200000 ]] && \ die "Clang assembler version < 20 not supported" @@ -555,6 +558,8 @@ build_kernel() { # cmd+=("KBUILD_MODPOST_WARN=1") + cmd+=("KLP_SYMIDS=1") + if [[ -v VERBOSE ]]; then cmd+=("V=1") else @@ -605,6 +610,8 @@ copy_orig_objects() { done xtrace_restore + cp -f "$PWD/vmlinux" "$ORIG_DIR" || die "missing vmlinux" + mv -f "$TMP_DIR/build.log" "$ORIG_DIR" touch "$TIMESTAMP" touch "$ORIG_DIR/.complete" @@ -675,6 +682,8 @@ generate_checksums() { "$OBJTOOL" klp checksum "$dest" done + [[ -f "$src_dir/vmlinux" ]] && cp -f "$src_dir/vmlinux" "$dest_dir" + touch "$dest_dir/.complete" } diff --git a/scripts/mksysmap b/scripts/mksysmap index c4531eacde202..856b26ba2ac02 100755 --- a/scripts/mksysmap +++ b/scripts/mksysmap @@ -35,7 +35,7 @@ / __efistub_/d # arm64 local symbols in PIE namespace -/ __pi_\\$/d +/ __pi_\$/d / __pi_\.L/d # arm64 local symbols in non-VHE KVM namespace @@ -83,7 +83,7 @@ / _SDA2_BASE_$/d # MODULE_INFO() -/ __UNIQUE_ID_modinfo[0-9]*$/d +/ __UNIQUE_ID_modinfo_[0-9]*$/d # --------------------------------------------------------------------------- # Ignored patterns diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index a7b72a81d2482..91b35eec69a8b 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -767,6 +767,7 @@ static const char *const section_white_list[] = ".llvm.call-graph-profile", /* call graph */ "__llvm_covfun", "__llvm_covmap", + ".klp.symid", /* objtool --klp-symids */ NULL }; @@ -1591,14 +1592,14 @@ static void read_symbols(const char *modname) struct elf_info info = { }; Elf_Sym *sym; - if (!parse_elf(&info, modname)) - return; - if (!strends(modname, ".o")) { error("%s: filename must be suffixed with .o\n", modname); return; } + if (!parse_elf(&info, modname)) + return; + /* strip trailing .o */ mod = new_module(modname, strlen(modname) - strlen(".o")); diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index 551f1ebd0dcb7..c30983562a2f1 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -208,6 +208,20 @@ if [ "$bindgen_libclang_cversion" -lt "$bindgen_libclang_min_cversion" ]; then exit 1 fi +if [ "$bindgen_libclang_cversion" -ge 2200000 ] && + [ "$rust_bindings_generator_cversion" -lt 7201 ]; then + # Distributions may have patched the issue. + if ! "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang_22.h | grep -q 'pub foo'; then + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' < 0.72.1 together with libclang >= 22" + echo >&2 "*** may not work due to a bug (https://github.com/rust-lang/rust-bindgen/pull/3278)." + echo >&2 "*** Your bindgen version: $rust_bindings_generator_version" + echo >&2 "*** Your libclang version: $bindgen_libclang_version" + echo >&2 "***" + warning=1 + fi +fi + # If the C compiler is Clang, then we can also check whether its version # matches the `libclang` version used by the Rust bindings generator. # diff --git a/scripts/rust_is_available_bindgen_libclang_22.h b/scripts/rust_is_available_bindgen_libclang_22.h new file mode 100644 index 0000000000000..6b33544c14a81 --- /dev/null +++ b/scripts/rust_is_available_bindgen_libclang_22.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +struct S; +struct S { + int foo; +}; diff --git a/scripts/rust_is_available_test.py b/scripts/rust_is_available_test.py index d6d54b7ea42a0..e14cf2c5695e3 100755 --- a/scripts/rust_is_available_test.py +++ b/scripts/rust_is_available_test.py @@ -54,16 +54,23 @@ def generate_rustc(cls, stdout): """) @classmethod - def generate_bindgen(cls, version_stdout, libclang_stderr): + def generate_bindgen(cls, version_stdout, libclang_stderr, libclang_22_patched=False): if libclang_stderr is None: libclang_case = f"raise SystemExit({cls.bindgen_default_bindgen_libclang_failure_exit_code})" else: libclang_case = f"print({repr(libclang_stderr)}, file=sys.stderr)" + if libclang_22_patched: + libclang_22_case = "print('pub foo: ::std::os::raw::c_int,')" + else: + libclang_22_case = "pass" + return cls.generate_executable(f"""#!/usr/bin/env python3 import sys if "rust_is_available_bindgen_libclang.h" in " ".join(sys.argv): {libclang_case} +elif "rust_is_available_bindgen_libclang_22.h" in " ".join(sys.argv): + {libclang_22_case} else: print({repr(version_stdout)}) """) @@ -248,6 +255,27 @@ def test_bindgen_libclang_old_version(self): result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) self.assertIn(f"libclang (used by the Rust bindings generator '{bindgen}') is too old.", result.stderr) + def test_bindgen_bad_libclang_22(self): + for (bindgen_version, libclang_version, expected_not_patched) in ( + ("0.71.1", "21.1.0", self.Expected.SUCCESS), + ("0.71.1", "22.0.0", self.Expected.SUCCESS_WITH_WARNINGS), + ("0.71.1", "22.1.0", self.Expected.SUCCESS_WITH_WARNINGS), + + ("0.72.0", "22.0.0", self.Expected.SUCCESS_WITH_WARNINGS), + + ("0.72.1", "22.0.0", self.Expected.SUCCESS), + ): + with self.subTest(bindgen_version=bindgen_version, libclang_version=libclang_version): + cc = self.generate_clang(f"clang version {libclang_version}") + libclang_stderr = f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {libclang_version} [-W#pragma-messages], err: false" + bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr) + result = self.run_script(expected_not_patched, { "BINDGEN": bindgen, "CC": cc }) + if expected_not_patched == self.Expected.SUCCESS_WITH_WARNINGS: + self.assertIn(f"Rust bindings generator '{bindgen}' < 0.72.1 together with libclang >= 22", result.stderr) + + bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr, libclang_22_patched=True) + result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen, "CC": cc }) + def test_clang_matches_bindgen_libclang_different_bindgen(self): bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version 999.0.0 [-W#pragma-messages], err: false") result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) diff --git a/scripts/sorttable.c b/scripts/sorttable.c index d8dc2a1b7c312..d7b50581c7329 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -116,7 +116,7 @@ static inline void *get_index(void *start, int entsize, int index) } static int extable_ent_size; -static int long_size; +static int long_size __maybe_unused; #define ERRSTR_MAXSZ 256 diff --git a/scripts/tags.sh b/scripts/tags.sh index 243373683f98a..c9dc2763a505e 100755 --- a/scripts/tags.sh +++ b/scripts/tags.sh @@ -100,7 +100,7 @@ all_compiled_sources() { echo include/generated/autoconf.h find $ignore -name "*.cmd" -exec \ - grep -Poh '(?<=^ )\S+|(?<== )\S+[^\\](?=$)' {} \+ | + grep -Poh '(?<=^ )\S+\.([chS]|rs)(?=\s)|(?<== )\S+\.(?1)(?=$)' {} \+ | awk '!a[$0]++' } | xargs realpath -esq $([ -z "$KBUILD_ABS_SRCTREE" ] && echo --relative-to=.) | sort -u diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index d6958eb00e304..e88c19c9b175a 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -874,6 +874,52 @@ static struct aa_label *label_merge_wrap(struct aa_label *a, struct aa_label *b, return label; } +static bool is_profile_priv_restricted_to_stack(const struct cred *subj_cred, + struct aa_profile *profile) +{ + if (profile_unconfined(profile) && profile == profile->ns->unconfined && + aa_unprivileged_unconfined_restricted && + /* cap_capable returns false (0) if true, hence true here means + * doesn't have capability and the stack will be restricted + */ + cap_capable(current_cred(), &init_user_ns, CAP_MAC_OVERRIDE, + CAP_OPT_NOAUDIT)) + return true; + return false; +} + +static const char *stack_msg = "change_profile unprivileged unconfined converted to stacking"; + +static struct aa_label *priv_restricted_transition(const struct cred *subj_cred, + struct aa_profile *profile, + const char *op, u32 request, + const char *name, + struct aa_label *transition, + gfp_t gfp) +{ + if (!is_profile_priv_restricted_to_stack(subj_cred, profile)) + return aa_get_newest_label(transition); + + /* transition allowed but only via stack */ + struct aa_label *target = label_merge_wrap(&profile->label, + transition, gfp); + if (IS_ERR_OR_NULL(target)) + return target; + + /* doing this here is less than optimal but good enough until the + * fs mediation rework lands + */ + struct aa_perms perms = { + .allow = request, + .audit = request, + }; + aa_audit_file(subj_cred, profile, &perms, op, + request, name, NULL, target, + subj_cred->euid, stack_msg, 0); + + return target; +} + static struct aa_label *handle_onexec(const struct cred *subj_cred, struct aa_label *label, struct aa_label *onexec, bool stack, @@ -902,7 +948,10 @@ static struct aa_label *handle_onexec(const struct cred *subj_cred, new = fn_label_build_in_scope(label, profile, GFP_KERNEL, stack ? label_merge_wrap(&profile->label, onexec, GFP_KERNEL) - : aa_get_newest_label(onexec), + : priv_restricted_transition(subj_cred, profile, + OP_CHANGE_ONEXEC, AA_MAY_ONEXEC, + bprm->filename, onexec, + GFP_KERNEL), profile_transition(subj_cred, profile, bprm, buffer, cond, unsafe)); AA_BUG(!new); @@ -1084,8 +1133,8 @@ static struct aa_label *build_change_hat(const struct cred *subj_cred, if (!hat) { error = -ENOENT; if (COMPLAIN_MODE(profile)) { - hat = aa_new_learning_profile(profile, true, name, - GFP_KERNEL); + hat = __aa_new_learning_profile(profile, true, name, + GFP_KERNEL); if (!hat) { info = "failed null profile create"; error = -ENOMEM; @@ -1123,6 +1172,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, bool sibling = false; const char *name, *info = NULL; int i, error; + bool needput = false; AA_BUG(!label); AA_BUG(!hats); @@ -1135,7 +1185,6 @@ static struct aa_label *change_hat(const struct cred *subj_cred, * the profiles and label, we can rely on the namespaces being live * and avoid incrementing their refcounts while grabbing the lock. */ - label = aa_get_label(label); ns = labels_ns(label); retry: @@ -1143,15 +1192,19 @@ static struct aa_label *change_hat(const struct cred *subj_cred, if (label_is_stale(label)) { new = aa_get_newest_label(label); new_ns = labels_ns(new); + + if (needput) + /* aa_put_label() is safe to call when under lock */ + aa_put_label(label); + label = new; + needput = true; + /* check if replaced with label in parent ns, and lock there */ if (new_ns != ns) { - aa_put_label(new); mutex_unlock(&ns->lock); ns = new_ns; - label = new; + /* retry will bottom out at the root of the tree */ goto retry; } - aa_put_label(label); - label = new; } if (PROFILE_IS_HAT(labels_profile(label))) @@ -1162,7 +1215,8 @@ static struct aa_label *change_hat(const struct cred *subj_cred, name = hats[i]; label_for_each_in_scope(it, labels_ns(label), label, profile) { if (sibling && PROFILE_IS_HAT(profile)) { - root = aa_get_profile(profile->parent); + root = aa_get_profile(rcu_dereference_protected(profile->parent, + mutex_is_locked(&ns->lock))); } else if (!sibling && !PROFILE_IS_HAT(profile)) { root = aa_get_profile(profile); } else { /* conflicting change type */ @@ -1223,6 +1277,8 @@ static struct aa_label *change_hat(const struct cred *subj_cred, } } mutex_unlock(&ns->lock); + if (needput) + aa_put_label(label); return ERR_PTR(error); build: @@ -1233,7 +1289,8 @@ static struct aa_label *change_hat(const struct cred *subj_cred, mutex_unlock(&ns->lock); AA_BUG(!new); /* return new label or error ptr */ - + if (needput) + aa_put_label(label); return new; } @@ -1406,8 +1463,6 @@ static int change_profile_perms_wrapper(const char *op, const char *name, return error; } -static const char *stack_msg = "change_profile unprivileged unconfined converted to stacking"; - /** * aa_change_profile - perform a one-way profile transition * @fqname: name of profile may include namespace (NOT NULL) @@ -1467,28 +1522,6 @@ int aa_change_profile(const char *fqname, int flags) op = OP_CHANGE_PROFILE; } - /* This should move to a per profile test. Requires pushing build - * into callback - */ - if (!stack && unconfined(label) && - label == &labels_ns(label)->unconfined->label && - aa_unprivileged_unconfined_restricted && - /* TODO: refactor so this check is a fn */ - cap_capable(current_cred(), &init_user_ns, CAP_MAC_OVERRIDE, - CAP_OPT_NOAUDIT)) { - /* regardless of the request in this case apparmor - * stacks against unconfined so admin set policy can't be - * by-passed - */ - stack = true; - perms.audit = request; - (void) fn_for_each_in_scope(label, profile, - aa_audit_file(subj_cred, profile, &perms, op, - request, auditname, NULL, target, - GLOBAL_ROOT_UID, stack_msg, 0)); - perms.audit = 0; - } - if (*fqname == '&') { stack = true; /* don't have label_parse() do stacking */ @@ -1559,7 +1592,10 @@ int aa_change_profile(const char *fqname, int flags) /* stacking is always a subset, so only check the nonstack case */ if (!stack) { new = fn_label_build_in_scope(label, profile, GFP_KERNEL, - aa_get_label(target), + priv_restricted_transition(subj_cred, profile, + op, request, + auditname, target, + GFP_KERNEL), aa_get_label(&profile->label)); AA_BUG(!new); if (IS_ERR(new)) diff --git a/security/apparmor/include/cred.h b/security/apparmor/include/cred.h index 2b6098149b15e..0e8b67159f56d 100644 --- a/security/apparmor/include/cred.h +++ b/security/apparmor/include/cred.h @@ -222,13 +222,9 @@ static inline struct aa_label *begin_current_label_crit_section(void) { struct aa_label *label = aa_current_raw_label(); - might_sleep(); - if (label_is_stale(label)) { label = aa_get_newest_label(label); - if (aa_replace_current_label(label) == 0) - /* task cred will keep the reference */ - aa_put_label(label); + aa_schedule_stale_label_replacement(); } return label; diff --git a/security/apparmor/include/label.h b/security/apparmor/include/label.h index b5a722a47fd2c..37cb135de3232 100644 --- a/security/apparmor/include/label.h +++ b/security/apparmor/include/label.h @@ -23,7 +23,7 @@ struct aa_ruleset; #define LOCAL_VEC_ENTRIES 8 #define DEFINE_VEC(T, V) \ - struct aa_ ## T *(_ ## V ## _localtmp)[LOCAL_VEC_ENTRIES]; \ + struct aa_ ## T *(_ ## V ## _localtmp)[LOCAL_VEC_ENTRIES + 1]; \ struct aa_ ## T **(V) #define vec_setup(T, V, N, GFP) \ @@ -31,10 +31,10 @@ struct aa_ruleset; if ((N) <= LOCAL_VEC_ENTRIES) { \ typeof(N) i; \ (V) = (_ ## V ## _localtmp); \ - for (i = 0; i < (N); i++) \ + for (i = 0; i <= (N); i++) \ (V)[i] = NULL; \ } else \ - (V) = kzalloc(sizeof(struct aa_ ## T *) * (N), (GFP)); \ + (V) = kzalloc_objs(struct aa_ ## T *, (N) + 1, (GFP)); \ (V) ? 0 : -ENOMEM; \ }) diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index 3895f8774a3f9..c6439ce6e436a 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -295,6 +295,9 @@ struct aa_profile *aa_alloc_profile(const char *name, struct aa_proxy *proxy, gfp_t gfp); struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, gfp_t gfp); +struct aa_profile *__aa_new_learning_profile(struct aa_profile *parent, + bool hat, const char *base, + gfp_t gfp); struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, const char *base, gfp_t gfp); void aa_free_profile(struct aa_profile *profile); diff --git a/security/apparmor/include/task.h b/security/apparmor/include/task.h index b1aaaf60fa8b8..6f26758ca10f0 100644 --- a/security/apparmor/include/task.h +++ b/security/apparmor/include/task.h @@ -21,15 +21,22 @@ static inline struct aa_task_ctx *task_ctx(struct task_struct *task) * @onexec: profile to transition to on next exec (MAY BE NULL) * @previous: profile the task may return to (MAY BE NULL) * @token: magic value the task must know for returning to @previous_profile + * @label_replacement_tw: for aa_schedule_stale_label_replacement() + * @label_replacement_pending: is @label_replacement_tw pending? + * + * When changing this, check if aa_dup_task_ctx() needs to be updated. */ struct aa_task_ctx { struct aa_label *nnp; struct aa_label *onexec; struct aa_label *previous; u64 token; + struct callback_head label_replacement_tw; + bool label_replacement_pending; }; int aa_replace_current_label(struct aa_label *label); +void aa_schedule_stale_label_replacement(void); void aa_set_current_onexec(struct aa_label *label, bool stack); int aa_set_current_hat(struct aa_label *label, u64 token); int aa_restore_previous_label(u64 cookie); @@ -56,10 +63,10 @@ static inline void aa_free_task_ctx(struct aa_task_ctx *ctx) static inline void aa_dup_task_ctx(struct aa_task_ctx *new, const struct aa_task_ctx *old) { - *new = *old; - aa_get_label(new->nnp); - aa_get_label(new->previous); - aa_get_label(new->onexec); + new->nnp = aa_get_label(old->nnp); + new->onexec = aa_get_label(old->onexec); + new->previous = aa_get_label(old->previous); + new->token = old->token; } /** diff --git a/security/apparmor/lib.c b/security/apparmor/lib.c index e41ff57798b25..b284cfcc5d349 100644 --- a/security/apparmor/lib.c +++ b/security/apparmor/lib.c @@ -482,6 +482,8 @@ bool aa_policy_init(struct aa_policy *policy, const char *prefix, char *hname; size_t hname_sz; + INIT_LIST_HEAD(&policy->list); + INIT_LIST_HEAD(&policy->profiles); hname_sz = (prefix ? strlen(prefix) + 2 : 0) + strlen(name) + 1; /* freed by policy_free */ hname = aa_str_alloc(hname_sz, gfp); @@ -494,8 +496,6 @@ bool aa_policy_init(struct aa_policy *policy, const char *prefix, policy->hname = hname; /* base.name is a substring of fqname */ policy->name = basename(policy->hname); - INIT_LIST_HEAD(&policy->list); - INIT_LIST_HEAD(&policy->profiles); return true; } diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index 94b4a7e727cc1..cd87370578dda 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -719,7 +719,7 @@ struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, } /** - * aa_new_learning_profile - create or find a null-X learning profile + * __aa_new_learning_profile - create or find a null-X learning profile * @parent: profile that caused this profile to be created (NOT NULL) * @hat: true if the null- learning profile is a hat * @base: name to base the null profile off of @@ -736,8 +736,9 @@ struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, * * Returns: new refcounted profile else NULL on failure */ -struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, - const char *base, gfp_t gfp) +struct aa_profile *__aa_new_learning_profile(struct aa_profile *parent, + bool hat, const char *base, + gfp_t gfp) { struct aa_profile *p, *profile; const char *bname; @@ -745,6 +746,7 @@ struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, size_t name_sz; AA_BUG(!parent); + AA_BUG(!mutex_is_locked(&parent->ns->lock)); if (base) { name_sz = strlen(parent->base.hname) + 8 + strlen(base); @@ -778,7 +780,6 @@ struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, if (hat) profile->label.flags |= FLAG_HAT; - mutex_lock_nested(&profile->ns->lock, profile->ns->level); p = __find_child(&parent->base.profiles, bname); if (p) { aa_free_profile(profile); @@ -786,7 +787,6 @@ struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, } else { __add_profile(&parent->base.profiles, profile); } - mutex_unlock(&profile->ns->lock); /* refcount released by caller */ out: @@ -800,6 +800,18 @@ struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, return NULL; } +struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, + const char *base, gfp_t gfp) +{ + struct aa_profile *profile; + + mutex_lock_nested(&parent->ns->lock, parent->ns->level); + profile = __aa_new_learning_profile(parent, hat, base, gfp); + mutex_unlock(&parent->ns->lock); + + return profile; +} + /** * replacement_allowed - test to see if replacement is allowed * @profile: profile to test if it can be replaced (MAYBE NULL) diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index d9dcff167c480..15b5e9270c0fa 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -730,7 +730,7 @@ static bool verify_tags(struct aa_tags_struct *tags, const char **info) /* count followed by count indexes into hdrs */ u32 cnt = tags->sets.table[i]; - if (i+cnt >= tags->sets.size) { + if ((u64)i + cnt >= tags->sets.size) { AA_DEBUG(DEBUG_UNPACK, "tagset too large %d+%d > sets.table[%d]", i, cnt, tags->sets.size); diff --git a/security/apparmor/task.c b/security/apparmor/task.c index b9fb3738124e0..e16ff4130bc2f 100644 --- a/security/apparmor/task.c +++ b/security/apparmor/task.c @@ -14,6 +14,7 @@ #include #include +#include #include "include/path.h" #include "include/audit.h" @@ -89,6 +90,32 @@ int aa_replace_current_label(struct aa_label *label) return 0; } +static void aa_replace_stale_label_tw_func(struct callback_head *tw) +{ + struct aa_task_ctx *ctx = task_ctx(current); + struct aa_label *label; + + ctx->label_replacement_pending = false; + label = aa_current_raw_label(); + if (!label_is_stale(label)) + return; + label = aa_get_newest_label(label); + aa_replace_current_label(label); + aa_put_label(label); +} + +/* replace the current task's stale label on syscall return */ +void aa_schedule_stale_label_replacement(void) +{ + struct aa_task_ctx *ctx = task_ctx(current); + + if (ctx->label_replacement_pending) + return; + init_task_work(&ctx->label_replacement_tw, aa_replace_stale_label_tw_func); + if (task_work_add(current, &ctx->label_replacement_tw, TWA_RESUME) == 0) + ctx->label_replacement_pending = true; +} + /** * aa_set_current_onexec - set the tasks change_profile to happen onexec diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index ced2e131b061b..b280488e15fcb 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -748,6 +748,8 @@ static int validate_hash_algo(struct dentry *dentry, return -EACCES; path = dentry_path(dentry, pathbuf, PATH_MAX); + if (IS_ERR(path)) + path = NULL; integrity_audit_msg(AUDIT_INTEGRITY_DATA, d_inode(dentry), path, "set_data", errmsg, -EACCES, 0); diff --git a/security/ipe/eval.c b/security/ipe/eval.c index 21439c5be3364..4a7be96c84f03 100644 --- a/security/ipe/eval.c +++ b/security/ipe/eval.c @@ -134,10 +134,14 @@ static bool evaluate_boot_verified(const struct ipe_eval_ctx *const ctx) static bool evaluate_dmv_roothash(const struct ipe_eval_ctx *const ctx, struct ipe_prop *p) { - return !!ctx->ipe_bdev && - !!ctx->ipe_bdev->root_hash && - ipe_digest_eval(p->value, - ctx->ipe_bdev->root_hash); + const struct digest_info *root_hash; + + if (!ctx->ipe_bdev) + return false; + + root_hash = rcu_dereference(ctx->ipe_bdev->root_hash); + + return root_hash && ipe_digest_eval(p->value, root_hash); } #else static bool evaluate_dmv_roothash(const struct ipe_eval_ctx *const ctx, diff --git a/security/ipe/eval.h b/security/ipe/eval.h index fef65a36468cb..b26d1147d0708 100644 --- a/security/ipe/eval.h +++ b/security/ipe/eval.h @@ -27,7 +27,7 @@ struct ipe_bdev { #ifdef CONFIG_IPE_PROP_DM_VERITY_SIGNATURE bool dm_verity_signed; #endif /* CONFIG_IPE_PROP_DM_VERITY_SIGNATURE */ - struct digest_info *root_hash; + struct digest_info __rcu *root_hash; }; #endif /* CONFIG_IPE_PROP_DM_VERITY */ diff --git a/security/ipe/fs.c b/security/ipe/fs.c index 076c111c85c86..847a76afb93d7 100644 --- a/security/ipe/fs.c +++ b/security/ipe/fs.c @@ -159,18 +159,16 @@ static ssize_t new_policy(struct file *f, const char __user *data, } rc = ipe_new_policyfs_node(p); - if (rc) - goto out; out: kfree(copy); if (rc < 0) { ipe_free_policy(p); ipe_audit_policy_load(ERR_PTR(rc)); - } else { - ipe_audit_policy_load(p); + return rc; } - return (rc < 0) ? rc : len; + + return len; } static const struct file_operations np_fops = { diff --git a/security/ipe/hooks.c b/security/ipe/hooks.c index 0ae54a880405a..d878b52ceec84 100644 --- a/security/ipe/hooks.c +++ b/security/ipe/hooks.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "ipe.h" #include "hooks.h" @@ -232,7 +233,20 @@ void ipe_bdev_free_security(struct block_device *bdev) { struct ipe_bdev *blob = ipe_bdev(bdev); - ipe_digest_free(blob->root_hash); + ipe_digest_free(rcu_access_pointer(blob->root_hash)); +} + +static void ipe_set_dmverity_roothash(struct ipe_bdev *blob, + struct digest_info *info) +{ + struct digest_info *old; + + /* Protected by device-mapper's md->suspend_lock */ + old = rcu_replace_pointer(blob->root_hash, info, true); + if (old) { + synchronize_rcu(); + ipe_digest_free(old); + } } #ifdef CONFIG_IPE_PROP_DM_VERITY_SIGNATURE @@ -280,8 +294,7 @@ int ipe_bdev_setintegrity(struct block_device *bdev, enum lsm_integrity_type typ return -EINVAL; if (!value) { - ipe_digest_free(blob->root_hash); - blob->root_hash = NULL; + ipe_set_dmverity_roothash(blob, NULL); return 0; } @@ -301,8 +314,7 @@ int ipe_bdev_setintegrity(struct block_device *bdev, enum lsm_integrity_type typ info->digest_len = digest->digest_len; - ipe_digest_free(blob->root_hash); - blob->root_hash = info; + ipe_set_dmverity_roothash(blob, info); return 0; err: diff --git a/security/ipe/policy_fs.c b/security/ipe/policy_fs.c index 9d92d8a14b13d..a7aeb57483c6a 100644 --- a/security/ipe/policy_fs.c +++ b/security/ipe/policy_fs.c @@ -481,6 +481,9 @@ int ipe_new_policyfs_node(struct ipe_policy *p) inode_lock(root); p->policyfs = policyfs; root->i_private = p; + /* Only audit signed policies from userspace */ + if (p->pkcs7) + ipe_audit_policy_load(p); inode_unlock(root); return 0; diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c index 59cb77b237b36..e07092ea301ac 100644 --- a/security/keys/encrypted-keys/encrypted.c +++ b/security/keys/encrypted-keys/encrypted.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -579,6 +580,7 @@ static struct encrypted_key_payload *encrypted_key_alloc(struct key *key, { struct encrypted_key_payload *epayload = NULL; unsigned short datablob_len; + unsigned short payload_totallen; unsigned short decrypted_datalen; unsigned short payload_datalen; unsigned int encrypted_datalen; @@ -632,16 +634,22 @@ static struct encrypted_key_payload *encrypted_key_alloc(struct key *key, encrypted_datalen = roundup(decrypted_datalen, blksize); - datablob_len = format_len + 1 + strlen(master_desc) + 1 - + strlen(datalen) + 1 + ivsize + 1 + encrypted_datalen; + if (check_add_overflow(format_len + 1 + strlen(master_desc) + 1 + + strlen(datalen) + 1 + ivsize + 1, + encrypted_datalen, &datablob_len)) + return ERR_PTR(-EINVAL); + + if (check_add_overflow(datablob_len, + payload_datalen + HASH_SIZE + 1, + &payload_totallen)) + return ERR_PTR(-EINVAL); - ret = key_payload_reserve(key, payload_datalen + datablob_len - + HASH_SIZE + 1); + ret = key_payload_reserve(key, payload_totallen); if (ret < 0) return ERR_PTR(ret); - epayload = kzalloc(sizeof(*epayload) + payload_datalen + - datablob_len + HASH_SIZE + 1, GFP_KERNEL); + epayload = kzalloc_flex(*epayload, payload_data, payload_totallen, + GFP_KERNEL); if (!epayload) return ERR_PTR(-ENOMEM); diff --git a/security/keys/gc.c b/security/keys/gc.c index 748e83818a760..eda445f815d47 100644 --- a/security/keys/gc.c +++ b/security/keys/gc.c @@ -318,9 +318,7 @@ static void key_garbage_collector(struct work_struct *work) if (unlikely(gc_state & KEY_GC_REAPING_DEAD_3)) { kdebug("dead wake"); - smp_mb(); - clear_bit(KEY_GC_REAPING_KEYTYPE, &key_gc_flags); - wake_up_bit(&key_gc_flags, KEY_GC_REAPING_KEYTYPE); + clear_and_wake_up_bit(KEY_GC_REAPING_KEYTYPE, &key_gc_flags); } if (gc_state & KEY_GC_REAP_AGAIN) diff --git a/security/keys/request_key_auth.c b/security/keys/request_key_auth.c index 282e09d8fa46c..ed6f55b9cdd93 100644 --- a/security/keys/request_key_auth.c +++ b/security/keys/request_key_auth.c @@ -9,6 +9,8 @@ #include #include +#include +#include #include #include #include @@ -73,7 +75,10 @@ static void request_key_auth_describe(const struct key *key, seq_puts(m, "key:"); seq_puts(m, key->description); if (key_is_positive(key)) - seq_printf(m, " pid:%d ci:%zu", rka->pid, rka->callout_len); + seq_printf(m, " pid:%d ci:%zu", + pid_nr_ns(rka->pid, + proc_pid_ns(file_inode(m->file)->i_sb)), + rka->callout_len); } /* @@ -113,6 +118,7 @@ static void free_request_key_auth(struct request_key_auth *rka) if (rka->cred) put_cred(rka->cred); kfree(rka->callout_info); + put_pid(rka->pid); kfree(rka); } @@ -226,14 +232,14 @@ struct key *request_key_auth_new(struct key *target, const char *op, irka = cred->request_key_auth->payload.data[0]; rka->cred = get_cred(irka->cred); - rka->pid = irka->pid; + rka->pid = get_pid(irka->pid); up_read(&cred->request_key_auth->sem); } else { /* it isn't - use this process as the context */ rka->cred = get_cred(cred); - rka->pid = current->pid; + rka->pid = get_pid(task_pid(current)); } rka->target_key = key_get(target); diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c index 13513819991e7..8f57c6111e7e4 100644 --- a/security/keys/trusted-keys/trusted_tpm1.c +++ b/security/keys/trusted-keys/trusted_tpm1.c @@ -987,9 +987,9 @@ static int __init trusted_tpm_init(void) static void trusted_tpm_exit(void) { if (chip) { + unregister_key_type(&key_type_trusted); put_device(&chip->dev); kfree(digests); - unregister_key_type(&key_type_trusted); } } diff --git a/security/keys/trusted-keys/trusted_tpm2.c b/security/keys/trusted-keys/trusted_tpm2.c index 6340823f8b53c..e8c129396b8b8 100644 --- a/security/keys/trusted-keys/trusted_tpm2.c +++ b/security/keys/trusted-keys/trusted_tpm2.c @@ -100,7 +100,7 @@ struct tpm2_key_context { static int tpm2_key_decode(struct trusted_key_payload *payload, struct trusted_key_options *options, - u8 **buf) + u8 **buf, unsigned int *blob_len) { int ret; struct tpm2_key_context ctx; @@ -121,6 +121,7 @@ static int tpm2_key_decode(struct trusted_key_payload *payload, return -ENOMEM; *buf = blob; + *blob_len = ctx.priv_len + ctx.pub_len; options->keyhandle = ctx.parent; memcpy(blob, ctx.priv, ctx.priv_len); @@ -381,10 +382,11 @@ static int tpm2_load_cmd(struct tpm_chip *chip, int rc; u32 attrs; - rc = tpm2_key_decode(payload, options, &blob); + rc = tpm2_key_decode(payload, options, &blob, &blob_len); if (rc) { /* old form */ blob = payload->blob; + blob_len = payload->blob_len; payload->old_format = 1; } else { /* Bind for cleanup: */ @@ -396,17 +398,17 @@ static int tpm2_load_cmd(struct tpm_chip *chip, return -EINVAL; /* must be big enough for at least the two be16 size counts */ - if (payload->blob_len < 4) + if (blob_len < 4) return -EINVAL; private_len = get_unaligned_be16(blob); /* must be big enough for following public_len */ - if (private_len + 2 + 2 > (payload->blob_len)) + if (private_len + 2 + 2 > blob_len) return -E2BIG; public_len = get_unaligned_be16(blob + 2 + private_len); - if (private_len + 2 + public_len + 2 > payload->blob_len) + if (private_len + 2 + public_len + 2 > blob_len) return -E2BIG; pub = blob + 2 + private_len + 2; diff --git a/security/landlock/access.h b/security/landlock/access.h index d926078bf0a5f..a0fa1009ccdb5 100644 --- a/security/landlock/access.h +++ b/security/landlock/access.h @@ -61,6 +61,10 @@ union access_masks_all { static_assert(sizeof(typeof_member(union access_masks_all, masks)) == sizeof(typeof_member(union access_masks_all, all))); +#define _LANDLOCK_LAYER_MASK_PADDING \ + (BITS_PER_TYPE(access_mask_t) - LANDLOCK_NUM_ACCESS_MAX - \ + IS_ENABLED(CONFIG_AUDIT)) + /** * struct layer_mask - The access rights and rule flags for a layer. * @@ -81,6 +85,10 @@ struct layer_mask { */ access_mask_t quiet : 1; #endif /* CONFIG_AUDIT */ + /** + * @__pad: Padding for the compiler's bitfield initialization. + */ + access_mask_t __pad : _LANDLOCK_LAYER_MASK_PADDING; } __packed __aligned(sizeof(access_mask_t)); /* diff --git a/security/landlock/errata/abi-1.h b/security/landlock/errata/abi-1.h index 3f099555f0596..315ea7e0fe504 100644 --- a/security/landlock/errata/abi-1.h +++ b/security/landlock/errata/abi-1.h @@ -22,3 +22,26 @@ * from their original mount points. */ LANDLOCK_ERRATUM(3) + +/** + * DOC: erratum_4 + * + * Erratum 4: Creation of whiteout objects + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * + * This fix changes the access rights required for the creation of whiteout + * objects through :manpage:`mknod(2)`, :manpage:`renameat2(2)`, or + * :manpage:`link(2)`. Creating whiteout objects is now guarded by + * ``LANDLOCK_ACCESS_FS_MAKE_REG`` instead of ``LANDLOCK_ACCESS_FS_MAKE_CHAR``. + * + * Whiteout objects are used in OverlayFS to mark the absence of a file in an + * upper file system. Despite being created with ``S_IFCHR``, whiteout objects + * do not count as character devices. + * + * Impact: + * + * Sandboxed programs that create OverlayFS whiteouts (such as fuse-overlayfs) + * now require ``LANDLOCK_ACCESS_FS_MAKE_REG`` instead of + * ``LANDLOCK_ACCESS_FS_MAKE_CHAR``. + */ +LANDLOCK_ERRATUM(4) diff --git a/security/landlock/fs.c b/security/landlock/fs.c index f7e5e4ef9eac3..f1c9effc93fe4 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -983,7 +984,8 @@ static int current_check_access_path(const struct path *const path, return -EACCES; } -static __attribute_const__ access_mask_t get_mode_access(const umode_t mode) +static __attribute_const__ access_mask_t get_mode_access(const umode_t mode, + const dev_t dev) { switch (mode & S_IFMT) { case S_IFLNK: @@ -991,6 +993,9 @@ static __attribute_const__ access_mask_t get_mode_access(const umode_t mode) case S_IFDIR: return LANDLOCK_ACCESS_FS_MAKE_DIR; case S_IFCHR: + /* Whiteout objects are guarded with MAKE_REG. */ + if (dev == WHITEOUT_DEV) + return LANDLOCK_ACCESS_FS_MAKE_REG; return LANDLOCK_ACCESS_FS_MAKE_CHAR; case S_IFBLK: return LANDLOCK_ACCESS_FS_MAKE_BLOCK; @@ -1007,6 +1012,13 @@ static __attribute_const__ access_mask_t get_mode_access(const umode_t mode) } } +static access_mask_t get_dentry_access(const struct dentry *const dentry) +{ + const struct inode *const inode = d_backing_inode(dentry); + + return get_mode_access(inode->i_mode, inode->i_rdev); +} + static access_mask_t maybe_remove(const struct dentry *const dentry) { if (d_is_negative(dentry)) @@ -1093,6 +1105,7 @@ static bool collect_domain_accesses(const struct landlock_ruleset *const domain, * @new_dentry: Destination file or directory. * @removable: Sets to true if it is a rename operation. * @exchange: Sets to true if it is a rename operation with RENAME_EXCHANGE. + * @whiteout: Sets to true if it is a rename operation with RENAME_WHITEOUT. * * Because of its unprivileged constraints, Landlock relies on file hierarchies * (and not only inodes) to tie access rights to files. Being able to link or @@ -1140,7 +1153,8 @@ static bool collect_domain_accesses(const struct landlock_ruleset *const domain, static int current_check_refer_path(struct dentry *const old_dentry, const struct path *const new_dir, struct dentry *const new_dentry, - const bool removable, const bool exchange) + const bool removable, const bool exchange, + const bool whiteout) { const struct landlock_cred_security *const subject = landlock_get_applicable_subject(current_cred(), any_fs, NULL); @@ -1159,18 +1173,25 @@ static int current_check_refer_path(struct dentry *const old_dentry, if (exchange) { if (unlikely(d_is_negative(new_dentry))) return -ENOENT; - access_request_parent1 = - get_mode_access(d_backing_inode(new_dentry)->i_mode); + access_request_parent1 = get_dentry_access(new_dentry); } else { access_request_parent1 = 0; } - access_request_parent2 = - get_mode_access(d_backing_inode(old_dentry)->i_mode); + access_request_parent2 = get_dentry_access(old_dentry); if (removable) { access_request_parent1 |= maybe_remove(old_dentry); access_request_parent2 |= maybe_remove(new_dentry); } + /* + * In case of renameat2(2) with RENAME_WHITEOUT, a whiteout object is + * created in the source location, so we require an additional access + * right there. + */ + if (whiteout) + access_request_parent1 |= + get_mode_access(S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); + /* The mount points are the same for old and new paths, cf. EXDEV. */ if (old_dentry->d_parent == new_dir->dentry) { /* @@ -1201,11 +1222,12 @@ static int current_check_refer_path(struct dentry *const old_dentry, /* * old_dentry may be the root of the common mount point and * !IS_ROOT(old_dentry) at the same time (e.g. with open_tree() and - * OPEN_TREE_CLONE). We do not need to call dget(old_parent) because - * we keep a reference to old_dentry. + * OPEN_TREE_CLONE). Pin the dentry used as old_parent in either case. + * Otherwise, dget_parent() safely fetches and pins the current parent + * against a concurrent rename(2). */ - old_parent = (old_dentry == mnt_dir.dentry) ? old_dentry : - old_dentry->d_parent; + old_parent = (old_dentry == mnt_dir.dentry) ? dget(old_dentry) : + dget_parent(old_dentry); /* new_dir->dentry is equal to new_dentry->d_parent */ allow_parent1 = collect_domain_accesses(subject->domain, mnt_dir.dentry, @@ -1214,8 +1236,10 @@ static int current_check_refer_path(struct dentry *const old_dentry, allow_parent2 = collect_domain_accesses(subject->domain, mnt_dir.dentry, new_dir->dentry, &layer_masks_parent2); - if (allow_parent1 && allow_parent2) + if (allow_parent1 && allow_parent2) { + dput(old_parent); return 0; + } /* * To be able to compare source and destination domain access rights, @@ -1227,8 +1251,10 @@ static int current_check_refer_path(struct dentry *const old_dentry, subject->domain, &mnt_dir, access_request_parent1, &layer_masks_parent1, &request1, old_dentry, access_request_parent2, &layer_masks_parent2, &request2, - exchange ? new_dentry : NULL)) + exchange ? new_dentry : NULL)) { + dput(old_parent); return 0; + } if (request1.access) { request1.audit.u.path.dentry = old_parent; @@ -1238,6 +1264,7 @@ static int current_check_refer_path(struct dentry *const old_dentry, request2.audit.u.path.dentry = new_dir->dentry; landlock_log_denial(subject, &request2); } + dput(old_parent); /* * This prioritizes EACCES over EXDEV for all actions, including @@ -1520,7 +1547,7 @@ static int hook_path_link(struct dentry *const old_dentry, struct dentry *const new_dentry) { return current_check_refer_path(old_dentry, new_dir, new_dentry, false, - false); + false, false); } static int hook_path_rename(const struct path *const old_dir, @@ -1531,7 +1558,8 @@ static int hook_path_rename(const struct path *const old_dir, { /* old_dir refers to old_dentry->d_parent and new_dir->mnt */ return current_check_refer_path(old_dentry, new_dir, new_dentry, true, - !!(flags & RENAME_EXCHANGE)); + !!(flags & RENAME_EXCHANGE), + !!(flags & RENAME_WHITEOUT)); } static int hook_path_mkdir(const struct path *const dir, @@ -1544,7 +1572,8 @@ static int hook_path_mknod(const struct path *const dir, struct dentry *const dentry, const umode_t mode, const unsigned int dev) { - return current_check_access_path(dir, get_mode_access(mode)); + return current_check_access_path( + dir, get_mode_access(mode, new_decode_dev(dev))); } static int hook_path_symlink(const struct path *const dir, diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 813e82bcfc278..221c41978dc50 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -1150,8 +1150,11 @@ inline int avc_has_perm_noaudit(u32 ssid, u32 tsid, u32 denied; struct avc_node *node; - if (WARN_ON(!requested)) + if (WARN_ON(!requested)) { + /* Provide a deny-all, audit-all decision to the caller. */ + *avd = (struct av_decision){ .auditdeny = 0xffffffff }; return -EACCES; + } rcu_read_lock(); node = avc_lookup(ssid, tsid, tclass); diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 18dd28b2bb13d..5832131f29c47 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1672,26 +1672,32 @@ static int cred_has_capability(const struct cred *cred, return rc; } -/* Check whether a task has a particular permission to an inode. - The 'adp' parameter is optional and allows other audit - data to be passed (e.g. the dentry). */ -static int inode_has_perm(const struct cred *cred, - struct inode *inode, - u32 perms, - struct common_audit_data *adp) +/* + * Check whether a SID has a particular permission to an inode. The 'adp' + * parameter is optional and allows other audit data to be passed (e.g. the + * dentry). + */ +static int inode_sid_has_perm(u32 sid, struct inode *inode, u32 perms, + struct common_audit_data *adp) { struct inode_security_struct *isec; - u32 sid; if (unlikely(IS_PRIVATE(inode))) return 0; - sid = cred_sid(cred); isec = selinux_inode(inode); return avc_has_perm(sid, isec->sid, isec->sclass, perms, adp); } +static int inode_has_perm(const struct cred *cred, + struct inode *inode, + u32 perms, + struct common_audit_data *adp) +{ + return inode_sid_has_perm(cred_sid(cred), inode, perms, adp); +} + /* Same as inode_has_perm, but pass explicit audit data containing the dentry to help the auditing code to more easily generate the pathname if needed. */ @@ -3841,17 +3847,74 @@ static int selinux_file_alloc_security(struct file *file) return 0; } +static inline u32 selinux_file_user_sid(const struct file *file) +{ + if (unlikely(file->f_mode & FMODE_BACKING)) + return selinux_backing_file(file)->uf_sid; + return selinux_file(file)->sid; +} + static int selinux_backing_file_alloc(struct file *backing_file, const struct file *user_file) { struct backing_file_security_struct *bfsec; + const struct backing_file_security_struct *ubfsec; + struct backing_file_security_layer *layer; + u32 i; bfsec = selinux_backing_file(backing_file); - bfsec->uf_sid = selinux_file(user_file)->sid; + bfsec->uf_sid = selinux_file_user_sid(user_file); + if (!(user_file->f_mode & FMODE_BACKING)) + return 0; + + ubfsec = selinux_backing_file(user_file); + /* a wrapped count would make kmalloc_array() return ZERO_SIZE_PTR */ + if (unlikely(ubfsec->layer_count == U32_MAX)) + return -EOVERFLOW; + + /* + * The final VMA only retains the lowest backing file, so record the + * whole chain here rather than in the mmap hook, where concurrent + * mappings would have to be serialized. Size it dynamically: erofs + * inode sharing adds a backing file without bumping s_stack_depth. + */ + bfsec->layers = kmalloc_array(ubfsec->layer_count + 1, + sizeof(*bfsec->layers), GFP_KERNEL); + if (!bfsec->layers) + return -ENOMEM; + + for (i = 0; i < ubfsec->layer_count; i++) { + layer = &bfsec->layers[i]; + *layer = ubfsec->layers[i]; + path_get(&layer->path); + } + + /* f_path, not file_user_path(): this layer, not the top-level file */ + layer = &bfsec->layers[i]; + layer->path = user_file->f_path; + layer->mounter_sid = cred_sid(user_file->f_cred); + layer->fd_sid = selinux_file(user_file)->sid; + path_get(&layer->path); + bfsec->layer_count = ubfsec->layer_count + 1; return 0; } +static void selinux_backing_file_free(struct file *backing_file) +{ + struct backing_file_security_struct *bfsec; + + /* security_backing_file_free() may be called twice after an error */ + if (!backing_file_security(backing_file)) + return; + + bfsec = selinux_backing_file(backing_file); + while (bfsec->layer_count) + path_put(&bfsec->layers[--bfsec->layer_count].path); + kfree(bfsec->layers); + bfsec->layers = NULL; +} + /* * Check whether a task has the ioctl permission and cmd * operation to an inode. @@ -3969,6 +4032,53 @@ static int selinux_file_ioctl_compat(struct file *file, unsigned int cmd, static int default_noexec __ro_after_init; +static u32 file_map_prot_to_av(unsigned long prot, bool shared) +{ + u32 av = FILE__READ; + + if (shared && (prot & PROT_WRITE)) + av |= FILE__WRITE; + if (prot & PROT_EXEC) + av |= FILE__EXECUTE; + + return av; +} + +static int backing_mounters_has_perm(const struct file *file, u32 av) +{ + const struct backing_file_security_struct *bfsec; + const struct backing_file_security_layer *layer; + struct common_audit_data ad; + struct inode *inode; + u32 i; + int rc; + + if (WARN_ON_ONCE(!(file->f_mode & FMODE_BACKING))) + return -EIO; + + bfsec = selinux_backing_file(file); + for (i = 0; i < bfsec->layer_count; i++) { + layer = &bfsec->layers[i]; + inode = d_inode(layer->path.dentry); + + ad.type = LSM_AUDIT_DATA_PATH; + ad.u.path = layer->path; + + if (layer->mounter_sid != layer->fd_sid) { + rc = avc_has_perm(layer->mounter_sid, layer->fd_sid, + SECCLASS_FD, FD__USE, &ad); + if (rc) + return rc; + } + + rc = inode_sid_has_perm(layer->mounter_sid, inode, av, &ad); + if (rc) + return rc; + } + + return 0; +} + static int __file_map_prot_check(const struct file *file, unsigned long prot, bool shared, bool mounter_check, bool bf_user_file) @@ -4002,14 +4112,10 @@ static int __file_map_prot_check(const struct file *file, unsigned long prot, if (file) { const struct cred *cred = mounter_check ? file->f_cred : current_cred(); - /* "read" always possible, "write" only if shared */ - u32 av = FILE__READ; - if (shared && prot_write) - av |= FILE__WRITE; - if (prot_exec) - av |= FILE__EXECUTE; - return __file_has_perm(cred, file, av, bf_user_file); + return __file_has_perm(cred, file, + file_map_prot_to_av(prot, shared), + bf_user_file); } return 0; @@ -4104,6 +4210,7 @@ static int selinux_file_mprotect(struct vm_area_struct *vma, int rc; const struct cred *cred = current_cred(); u32 sid = cred_sid(cred); + u32 av; const struct file *file = vma->vm_file; bool backing_file; bool shared = vma->vm_flags & VM_SHARED; @@ -4147,6 +4254,10 @@ static int selinux_file_mprotect(struct vm_area_struct *vma, if (rc) return rc; if (backing_file) { + rc = backing_mounters_has_perm(file, + FILE__EXECMOD); + if (rc) + return rc; rc = file_has_perm(file->f_cred, file, FILE__EXECMOD); if (rc) @@ -4159,6 +4270,10 @@ static int selinux_file_mprotect(struct vm_area_struct *vma, if (rc) return rc; if (backing_file) { + av = file_map_prot_to_av(prot, shared); + rc = backing_mounters_has_perm(file, av); + if (rc) + return rc; rc = file_map_prot_check(file, prot, shared, true); if (rc) return rc; @@ -7628,6 +7743,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(file_permission, selinux_file_permission), LSM_HOOK_INIT(file_alloc_security, selinux_file_alloc_security), LSM_HOOK_INIT(backing_file_alloc, selinux_backing_file_alloc), + LSM_HOOK_INIT(backing_file_free, selinux_backing_file_free), LSM_HOOK_INIT(file_ioctl, selinux_file_ioctl), LSM_HOOK_INIT(file_ioctl_compat, selinux_file_ioctl_compat), LSM_HOOK_INIT(mmap_file, selinux_mmap_file), diff --git a/security/selinux/include/objsec.h b/security/selinux/include/objsec.h index 3c0a16ec978b0..2f21568251ffe 100644 --- a/security/selinux/include/objsec.h +++ b/security/selinux/include/objsec.h @@ -86,8 +86,16 @@ struct file_security_struct { u32 pseqno; /* Policy seqno at the time of file open */ }; +struct backing_file_security_layer { + struct path path; /* this layer's real path */ + u32 mounter_sid; /* SID of the mounter that opened it */ + u32 fd_sid; /* SID of its open file description */ +}; + struct backing_file_security_struct { - u32 uf_sid; /* associated user file fsec->sid */ + u32 uf_sid; /* top-level user file fsec->sid */ + u32 layer_count; /* number of intermediate backing files */ + struct backing_file_security_layer *layers; }; struct superblock_security_struct { diff --git a/security/smack/smack.h b/security/smack/smack.h index 9b9eb262fe33e..fe6a498200143 100644 --- a/security/smack/smack.h +++ b/security/smack/smack.h @@ -317,8 +317,9 @@ int smack_populate_secattr(struct smack_known *skp); * Shared data. */ extern int smack_enabled __initdata; -extern int smack_cipso_direct; -extern int smack_cipso_mapped; +extern u8 smack_cipso_auto_level[2]; +#define smack_cipso_direct (+smack_cipso_auto_level[0]) +#define smack_cipso_mapped (+smack_cipso_auto_level[1]) extern struct smack_known *smack_net_ambient; extern struct smack_known *smack_syslog_label; #ifdef CONFIG_SECURITY_SMACK_BRINGUP diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index ff115068c5c06..9a706f37df36b 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -130,12 +130,13 @@ static int smk_bu_note(char *note, struct smack_known *sskp, #define smk_bu_note(note, sskp, oskp, mode, RC) (RC) #endif -#ifdef CONFIG_SECURITY_SMACK_BRINGUP -static int smk_bu_current(char *note, struct smack_known *oskp, - int mode, int rc) +static int +smk_bu_tsk_to_obj(struct task_struct *tsk, const struct task_smack *tsp, + char *note, struct smack_known *oskp, int mode, int rc) { - struct task_smack *tsp = smack_cred(current_cred()); +#ifdef CONFIG_SECURITY_SMACK_BRINGUP char acc[SMK_NUM_ACCESS_TYPE + 1]; + char comm[TASK_COMM_LEN]; if (rc <= 0) return rc; @@ -143,14 +144,22 @@ static int smk_bu_current(char *note, struct smack_known *oskp, rc = 0; smk_bu_mode(mode, acc); + pr_info("Smack %s: (%s %s %s) %s %s\n", smk_bu_mess[rc], - tsp->smk_task->smk_known, oskp->smk_known, - acc, current->comm, note); + smk_of_task(tsp)->smk_known, oskp->smk_known, + acc, get_task_comm(comm, tsk), note); return 0; -} #else -#define smk_bu_current(note, oskp, mode, RC) (RC) + return rc; #endif +} + +static int smk_bu_current(char *note, struct smack_known *oskp, + int mode, int rc) +{ + return smk_bu_tsk_to_obj(current, smack_cred(current_cred()), + note, oskp, mode, rc); +} #ifdef CONFIG_SECURITY_SMACK_BRINGUP static int smk_bu_task(struct task_struct *otp, int mode, int rc) @@ -1312,7 +1321,7 @@ static int smack_inode_getattr(const struct path *path) */ static int smack_inode_xattr_skipcap(const char *name) { - if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX))) + if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX)) == 0) return 0; if (strcmp(name, XATTR_NAME_SMACK) == 0 || @@ -1964,7 +1973,7 @@ static int smack_file_send_sigiotask(struct task_struct *tsk, { struct smack_known **blob; struct smack_known *skp; - struct smack_known *tkp = smk_of_task(smack_cred(tsk->cred)); + struct smack_known *tkp = smk_of_task_struct_obj(tsk); const struct cred *tcred; struct file *file; int rc; @@ -3348,14 +3357,20 @@ static int smack_sem_semop(struct kern_ipc_perm *isp, struct sembuf *sops, } /** - * smk_curacc_msq : helper to check if current has access on msq - * @isp : the msq + * smk_tskacc_msq : helper to check if tsk has access on msq + * @tsk: the task that requests access + * @isp : the sysv msg queue permissions * @access : access requested * - * return 0 if current has access, error otherwise + * return 0 if tsk has access, error otherwise */ -static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) +static int +smk_tskacc_msq(struct task_struct *tsk, struct kern_ipc_perm *isp, int access) { + const bool tsk_is_current = (tsk == current); + const struct cred * const tsk_cred = + (tsk_is_current ? current_cred() : get_task_cred(tsk)); + struct task_smack * const tsp = smack_cred(tsk_cred); struct smack_known *msp = smack_of_ipc(isp); struct smk_audit_info ad; int rc; @@ -3364,11 +3379,25 @@ static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) smk_ad_init(&ad, __func__, LSM_AUDIT_DATA_IPC); ad.a.u.ipc_id = isp->id; #endif - rc = smk_curacc(msp, access, &ad); - rc = smk_bu_current("msq", msp, access, rc); + rc = smk_tskacc(tsp, msp, access, &ad); + rc = smk_bu_tsk_to_obj(tsk, tsp, "msq", msp, access, rc); + if (!tsk_is_current) + put_cred(tsk_cred); return rc; } +/** + * smk_curacc_msq : helper to check if current has access on msq + * @isp : the sysv msg queue permissions + * @access : access requested + * + * return 0 if current has access, error otherwise + */ +static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) +{ + return smk_tskacc_msq(current, isp, access); +} + /** * smack_msg_queue_associate - Smack access check for msg_queue * @isp: the object @@ -3436,21 +3465,21 @@ static int smack_msg_queue_msgsnd(struct kern_ipc_perm *isp, struct msg_msg *msg } /** - * smack_msg_queue_msgrcv - Smack access check for msg_queue + * smack_msg_queue_msgrcv - check it target has r/w access to msg_queue * @isp: the object * @msg: unused - * @target: unused + * @target: the task that msgrcv() from the queue * @type: unused * @mode: unused * - * Returns 0 if current has read and write access, error code otherwise + * Returns 0 if target has read and write access, error code otherwise */ static int smack_msg_queue_msgrcv(struct kern_ipc_perm *isp, struct msg_msg *msg, struct task_struct *target, long type, int mode) { - return smk_curacc_msq(isp, MAY_READWRITE); + return smk_tskacc_msq(target, isp, MAY_READWRITE); } /** diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c index 6e62dcb36f74f..c7eae7c6427ff 100644 --- a/security/smack/smackfs.c +++ b/security/smack/smackfs.c @@ -83,18 +83,27 @@ static DEFINE_MUTEX(smk_net6addr_lock); struct smack_known *smack_net_ambient; /* - * This is the level in a CIPSO header that indicates a + * Sensitivity levels for automatically created CIPSO labels. + * See smack_access.c`smack_populate_secattr() + * + * [0] "direct" labeling, label length < SMK_CIPSOLEN(24): * smack label is contained directly in the category set. * It can be reset via smackfs/direct - */ -int smack_cipso_direct = SMACK_CIPSO_DIRECT_DEFAULT; - -/* - * This is the level in a CIPSO header that indicates a + * + * [1] "mapped" labeling, label length >= SMK_CIPSOLEN(24): * secid is contained directly in the category set. * It can be reset via smackfs/mapped */ -int smack_cipso_mapped = SMACK_CIPSO_MAPPED_DEFAULT; +u8 smack_cipso_auto_level[2] = { + SMACK_CIPSO_DIRECT_DEFAULT, + SMACK_CIPSO_MAPPED_DEFAULT, +}; + +static int +smk_cipso_auto_level_idx(const struct file *file) +{ + return (file_inode(file)->i_ino != SMK_DIRECT); +} #ifdef CONFIG_SECURITY_SMACK_BRINGUP /* @@ -1598,24 +1607,17 @@ static ssize_t smk_read_doi(struct file *filp, char __user *buf, static ssize_t smk_write_doi(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[80]; - unsigned long u; + int ret; + u32 u; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; + ret = kstrtou32_from_user(buf, count, 10, &u); + if (unlikely(ret)) + return ret; - temp[count] = '\0'; - - if (kstrtoul(temp, 10, &u)) - return -EINVAL; - - if (u == CIPSO_V4_DOI_UNKNOWN || u > U32_MAX) + if (u == CIPSO_V4_DOI_UNKNOWN) return -EINVAL; return smk_cipso_doi(u, GFP_KERNEL) ? : count; @@ -1628,158 +1630,80 @@ static const struct file_operations smk_doi_ops = { }; /** - * smk_read_direct - read() for /smack/direct - * @filp: file pointer, not actually used + * smk_read_cipso_auto_level - read() for smackfs/direct and smackfs/mapped + * @filp: file pointer * @buf: where to put the result * @count: maximum to send along * @ppos: where to start * * Returns number of bytes read or error code, as appropriate */ -static ssize_t smk_read_direct(struct file *filp, char __user *buf, +static ssize_t smk_read_cipso_auto_level(struct file *filp, char __user *buf, size_t count, loff_t *ppos) { - char temp[80]; - ssize_t rc; + char temp[sizeof "255"]; + int n; if (*ppos != 0) return 0; - sprintf(temp, "%d", smack_cipso_direct); - rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); - - return rc; + n = sprintf(temp, "%u", (unsigned int)smack_cipso_auto_level[ + smk_cipso_auto_level_idx(filp)]); + return simple_read_from_buffer(buf, count, ppos, temp, n); } /** - * smk_write_direct - write() for /smack/direct - * @file: file pointer, not actually used + * smk_write_cipso_auto_level - write() for smackfs/direct and smackfs/mapped + * @filp: file pointer * @buf: where to get the data from * @count: bytes sent * @ppos: where to start * * Returns number of bytes written or error code, as appropriate */ -static ssize_t smk_write_direct(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) +static ssize_t +smk_write_cipso_auto_level(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) { - struct smack_known *skp; - char temp[80]; - int i; + int ret, idx; + u8 i, old_lvl; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; - /* - * Don't do anything if the value hasn't actually changed. - * If it is changing reset the level on entries that were - * set up to be direct when they were created. + * draft-ietf-cipso-ipsecurity-01 (CIPSO 2.2), 3.4.2.4: + * "Sensitivity Level is 1 octet in length. Its value is from 0 to 255" */ - if (smack_cipso_direct != i) { - mutex_lock(&smack_known_lock); - list_for_each_entry_rcu(skp, &smack_known_list, list) - if (skp->smk_netlabel.attr.mls.lvl == - smack_cipso_direct) - skp->smk_netlabel.attr.mls.lvl = i; - smack_cipso_direct = i; - mutex_unlock(&smack_known_lock); - } - - return count; -} - -static const struct file_operations smk_direct_ops = { - .read = smk_read_direct, - .write = smk_write_direct, - .llseek = default_llseek, -}; - -/** - * smk_read_mapped - read() for /smack/mapped - * @filp: file pointer, not actually used - * @buf: where to put the result - * @count: maximum to send along - * @ppos: where to start - * - * Returns number of bytes read or error code, as appropriate - */ -static ssize_t smk_read_mapped(struct file *filp, char __user *buf, - size_t count, loff_t *ppos) -{ - char temp[80]; - ssize_t rc; - - if (*ppos != 0) - return 0; - - sprintf(temp, "%d", smack_cipso_mapped); - rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); - - return rc; -} - -/** - * smk_write_mapped - write() for /smack/mapped - * @file: file pointer, not actually used - * @buf: where to get the data from - * @count: bytes sent - * @ppos: where to start - * - * Returns number of bytes written or error code, as appropriate - */ -static ssize_t smk_write_mapped(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct smack_known *skp; - char temp[80]; - int i; - - if (!smack_privileged(CAP_MAC_ADMIN)) - return -EPERM; - - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; + ret = kstrtou8_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; /* * Don't do anything if the value hasn't actually changed. * If it is changing reset the level on entries that were - * set up to be mapped when they were created. + * set up to be "auto" level when they were created. */ - if (smack_cipso_mapped != i) { + idx = smk_cipso_auto_level_idx(filp); + old_lvl = smack_cipso_auto_level[idx]; + + if (old_lvl != i) { + struct smack_known *skp; mutex_lock(&smack_known_lock); list_for_each_entry_rcu(skp, &smack_known_list, list) if (skp->smk_netlabel.attr.mls.lvl == - smack_cipso_mapped) + old_lvl) skp->smk_netlabel.attr.mls.lvl = i; - smack_cipso_mapped = i; + smack_cipso_auto_level[idx] = i; mutex_unlock(&smack_known_lock); } return count; } -static const struct file_operations smk_mapped_ops = { - .read = smk_read_mapped, - .write = smk_write_mapped, +static const struct file_operations +smk_cipso_auto_level_ops = { + .read = smk_read_cipso_auto_level, + .write = smk_write_cipso_auto_level, .llseek = default_llseek, }; @@ -2179,22 +2103,15 @@ static ssize_t smk_read_logging(struct file *filp, char __user *buf, static ssize_t smk_write_logging(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[32]; - int i; + int i, ret; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (count >= sizeof(temp) || count == 0) - return -EINVAL; + ret = kstrtos32_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; if (i < 0 || i > 3) return -EINVAL; log_policy = i; @@ -2838,22 +2755,15 @@ static ssize_t smk_read_ptrace(struct file *filp, char __user *buf, static ssize_t smk_write_ptrace(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[32]; - int i; + int i, ret; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (*ppos != 0 || count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; + ret = kstrtos32_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; if (i < SMACK_PTRACE_DEFAULT || i > SMACK_PTRACE_MAX) return -EINVAL; smack_ptrace_rule = i; @@ -2888,7 +2798,7 @@ static int smk_fill_super(struct super_block *sb, struct fs_context *fc) [SMK_DOI] = { "doi", &smk_doi_ops, S_IRUGO|S_IWUSR}, [SMK_DIRECT] = { - "direct", &smk_direct_ops, S_IRUGO|S_IWUSR}, + "direct", &smk_cipso_auto_level_ops, 0644}, [SMK_AMBIENT] = { "ambient", &smk_ambient_ops, S_IRUGO|S_IWUSR}, [SMK_NET4ADDR] = { @@ -2904,7 +2814,7 @@ static int smk_fill_super(struct super_block *sb, struct fs_context *fc) [SMK_ACCESSES] = { "access", &smk_access_ops, S_IRUGO|S_IWUGO}, [SMK_MAPPED] = { - "mapped", &smk_mapped_ops, S_IRUGO|S_IWUSR}, + "mapped", &smk_cipso_auto_level_ops, 0644}, [SMK_LOAD2] = { "load2", &smk_load2_ops, S_IRUGO|S_IWUSR}, [SMK_LOAD_SELF2] = { diff --git a/sound/core/control_led.c b/sound/core/control_led.c index 8cbacee57ce70..3d13bbec1c544 100644 --- a/sound/core/control_led.c +++ b/sound/core/control_led.c @@ -255,6 +255,8 @@ static int snd_ctl_led_set_id(int card_number, struct snd_ctl_elem_id *id, kctl = snd_ctl_find_id(card, id); if (!kctl) return -ENOENT; + if (!kctl->info || !kctl->get) + return -EINVAL; ioff = snd_ctl_get_ioff(kctl, id); vd = &kctl->vd[ioff]; access = vd->access & SNDRV_CTL_ELEM_ACCESS_LED_MASK; diff --git a/sound/core/init.c b/sound/core/init.c index 56dde5bd73c4e..cd21c6f520e98 100644 --- a/sound/core/init.c +++ b/sound/core/init.c @@ -310,7 +310,7 @@ static int snd_card_init(struct snd_card *card, struct device *parent, kfree(card); /* manually free here, as no destructor called */ return err; } - card->dev = parent; + card->dev = get_device(parent); card->number = idx; WARN_ON(IS_MODULE(CONFIG_SND) && !module); card->module = module; @@ -584,6 +584,8 @@ EXPORT_SYMBOL_GPL(snd_card_disconnect_sync); static int snd_card_do_free(struct snd_card *card) { + bool managed = card->managed; + card->releasing = true; #if IS_ENABLED(CONFIG_SND_MIXER_OSS) if (snd_mixer_oss_notify_callback) @@ -599,9 +601,10 @@ static int snd_card_do_free(struct snd_card *card) dev_warn(card->dev, "unable to free card info\n"); /* Not fatal error */ } + put_device(card->dev); if (card->release_completion) complete(card->release_completion); - if (!card->managed) + if (!managed) kfree(card); return 0; } diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index c9d0c2bb55b26..2fcdde09a0305 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -1468,6 +1468,8 @@ static int snd_pcm_pre_start(struct snd_pcm_substream *substream, struct snd_pcm_runtime *runtime = substream->runtime; if (runtime->state != SNDRV_PCM_STATE_PREPARED) return -EBADFD; + if (atomic_read(&runtime->buffer_accessing) < 0) + return -EBADFD; /* during hw_params, hw_free or prepare */ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && !snd_pcm_playback_data(substream)) return -EPIPE; @@ -4017,20 +4019,33 @@ int snd_pcm_mmap_data(struct snd_pcm_substream *substream, struct file *file, return -EINVAL; } runtime = substream->runtime; - if (runtime->state == SNDRV_PCM_STATE_OPEN) - return -EBADFD; - if (!(runtime->info & SNDRV_PCM_INFO_MMAP)) - return -ENXIO; + /* don't race with buffer reallocation in hw_params/hw_free */ + if (!atomic_inc_unless_negative(&runtime->buffer_accessing)) + return -EBUSY; + if (runtime->state == SNDRV_PCM_STATE_OPEN) { + err = -EBADFD; + goto out; + } + if (!(runtime->info & SNDRV_PCM_INFO_MMAP)) { + err = -ENXIO; + goto out; + } if (runtime->access == SNDRV_PCM_ACCESS_RW_INTERLEAVED || - runtime->access == SNDRV_PCM_ACCESS_RW_NONINTERLEAVED) - return -EINVAL; + runtime->access == SNDRV_PCM_ACCESS_RW_NONINTERLEAVED) { + err = -EINVAL; + goto out; + } size = area->vm_end - area->vm_start; offset = area->vm_pgoff << PAGE_SHIFT; dma_bytes = PAGE_ALIGN(runtime->dma_bytes); - if ((size_t)size > dma_bytes) - return -EINVAL; - if (offset > dma_bytes - size) - return -EINVAL; + if ((size_t)size > dma_bytes) { + err = -EINVAL; + goto out; + } + if (offset > dma_bytes - size) { + err = -EINVAL; + goto out; + } area->vm_ops = &snd_pcm_vm_ops_data; area->vm_private_data = substream; @@ -4040,6 +4055,8 @@ int snd_pcm_mmap_data(struct snd_pcm_substream *substream, struct file *file, err = snd_pcm_lib_default_mmap(substream, area); if (!err) atomic_inc(&substream->mmap_count); +out: + atomic_dec(&runtime->buffer_accessing); return err; } EXPORT_SYMBOL(snd_pcm_mmap_data); diff --git a/sound/core/pcm_timer.c b/sound/core/pcm_timer.c index ab0e5bd70f8fa..18bedd66435dc 100644 --- a/sound/core/pcm_timer.c +++ b/sound/core/pcm_timer.c @@ -111,12 +111,15 @@ void snd_pcm_timer_init(struct snd_pcm_substream *substream) snd_pcm_direction_name(substream->stream), tid.card, tid.device, tid.subdevice); timer->hw = snd_pcm_timer; + /* Set before registering: a concurrent reader can invoke our hw + * callbacks as soon as the timer is on the global list. + */ + timer->private_data = substream; + timer->private_free = snd_pcm_timer_free; if (snd_device_register(timer->card, timer) < 0) { snd_device_free(timer->card, timer); return; } - timer->private_data = substream; - timer->private_free = snd_pcm_timer_free; substream->timer = timer; } diff --git a/sound/core/rawmidi.c b/sound/core/rawmidi.c index 4dfd9d53e6d3a..62addab22efe9 100644 --- a/sound/core/rawmidi.c +++ b/sound/core/rawmidi.c @@ -782,7 +782,7 @@ int snd_rawmidi_input_params(struct snd_rawmidi_substream *substream, substream->framing = framing; substream->clock_type = clock_type; } - return 0; + return err; } EXPORT_SYMBOL(snd_rawmidi_input_params); diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index 28782e1776fa8..9ec6c1c0cabc5 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -528,7 +528,7 @@ static int bounce_error_event(struct snd_seq_client *client, struct snd_seq_event *event, int err, int atomic, int hop) { - struct snd_seq_event bounce_ev; + struct snd_seq_event bounce_ev, quoted; int result; if (client == NULL || @@ -548,15 +548,19 @@ static int bounce_error_event(struct snd_seq_client *client, * For user clients, send SNDRV_SEQ_EVENT_BOUNCE with the * original event embedded as variable-length data. This * avoids exposing data.quote.event (a kernel pointer) to - * userspace. The variable-length path in snd_seq_event_dup() - * copies the event data from data.ext.ptr into chained cells, - * and snd_seq_expand_var_event() copies only the data content - * -- never the pointer -- to userspace. + * userspace. Sanitise the embedded copy too - a queued + * variable-length event carries the address of its own + * extension cell, and the payload goes out verbatim. */ + quoted = *event; + if (snd_seq_ev_is_variable("ed)) { + quoted.data.ext.len &= ~SNDRV_SEQ_EXT_MASK; + quoted.data.ext.ptr = NULL; + } bounce_ev.type = SNDRV_SEQ_EVENT_BOUNCE; bounce_ev.flags = SNDRV_SEQ_EVENT_LENGTH_VARIABLE; bounce_ev.data.ext.len = sizeof(struct snd_seq_event); - bounce_ev.data.ext.ptr = (char *)event; + bounce_ev.data.ext.ptr = (char *)"ed; } else { /* * For kernel clients, quote the event pointer directly. diff --git a/sound/core/seq/seq_midi.c b/sound/core/seq/seq_midi.c index 2eb12199c92f9..a16a5debf3391 100644 --- a/sound/core/seq/seq_midi.c +++ b/sound/core/seq/seq_midi.c @@ -42,9 +42,11 @@ struct seq_midisynth { struct snd_rawmidi *rmidi; int device; int subdevice; + struct snd_rawmidi_substream __rcu *input_substream; + snd_use_lock_t input_use_lock; /* in-flight event_input users */ struct snd_rawmidi_file input_rfile; - spinlock_t output_lock; /* protects output_rfile publication */ snd_use_lock_t output_use_lock; /* in-flight event_input users */ + struct snd_rawmidi_substream __rcu *output_substream; struct snd_rawmidi_file output_rfile; int seq_client; int seq_port; @@ -76,6 +78,14 @@ static void snd_midi_input_event(struct snd_rawmidi_substream *substream) msynth = runtime->private_data; if (msynth == NULL) return; + + scoped_guard(rcu) { + if (rcu_dereference(msynth->input_substream) != substream) + return; + + snd_use_lock_use(&msynth->input_use_lock); + } + memset(&ev, 0, sizeof(ev)); while (runtime->avail > 0) { res = snd_rawmidi_kernel_read(substream, buf, sizeof(buf)); @@ -95,6 +105,8 @@ static void snd_midi_input_event(struct snd_rawmidi_substream *substream) memset(&ev, 0, sizeof(ev)); } } + + snd_use_lock_free(&msynth->input_use_lock); } static int dump_midi(struct snd_rawmidi_substream *substream, const char *buf, int count) @@ -134,8 +146,8 @@ static int event_process_midi(struct snd_seq_event *ev, int direct, if (snd_BUG_ON(!msynth)) return -EINVAL; - scoped_guard(spinlock_irqsave, &msynth->output_lock) { - substream = msynth->output_rfile.output; + scoped_guard(rcu) { + substream = rcu_dereference(msynth->output_substream); if (!substream) return -ENODEV; snd_use_lock_use(&msynth->output_use_lock); @@ -177,7 +189,7 @@ static int snd_seq_midisynth_new(struct seq_midisynth *msynth, msynth->card = card; msynth->device = device; msynth->subdevice = subdevice; - spin_lock_init(&msynth->output_lock); + snd_use_lock_init(&msynth->input_use_lock); snd_use_lock_init(&msynth->output_use_lock); return 0; } @@ -188,28 +200,31 @@ static int midisynth_subscribe(void *private_data, struct snd_seq_port_subscribe int err; struct seq_midisynth *msynth = private_data; struct snd_rawmidi_runtime *runtime; + struct snd_rawmidi_file rfile = {}; struct snd_rawmidi_params params; /* open midi port */ err = snd_rawmidi_kernel_open(msynth->rmidi, msynth->subdevice, SNDRV_RAWMIDI_LFLG_INPUT, - &msynth->input_rfile); + &rfile); if (err < 0) { pr_debug("ALSA: seq_midi: midi input open failed!!!\n"); return err; } - runtime = msynth->input_rfile.input->runtime; + runtime = rfile.input->runtime; memset(¶ms, 0, sizeof(params)); params.avail_min = 1; params.buffer_size = input_buffer_size; - err = snd_rawmidi_input_params(msynth->input_rfile.input, ¶ms); + err = snd_rawmidi_input_params(rfile.input, ¶ms); if (err < 0) { - snd_rawmidi_kernel_release(&msynth->input_rfile); + snd_rawmidi_kernel_release(&rfile); return err; } snd_midi_event_reset_encode(msynth->parser); runtime->event = snd_midi_input_event; runtime->private_data = msynth; + msynth->input_rfile = rfile; + rcu_assign_pointer(msynth->input_substream, rfile.input); snd_rawmidi_kernel_read(msynth->input_rfile.input, NULL, 0); return 0; } @@ -219,10 +234,19 @@ static int midisynth_unsubscribe(void *private_data, struct snd_seq_port_subscri { int err; struct seq_midisynth *msynth = private_data; + struct snd_rawmidi_file rfile; - if (snd_BUG_ON(!msynth->input_rfile.input)) + rcu_assign_pointer(msynth->input_substream, NULL); + synchronize_rcu(); + snd_use_lock_sync(&msynth->input_use_lock); + + rfile = msynth->input_rfile; + msynth->input_rfile = (struct snd_rawmidi_file){}; + + if (snd_BUG_ON(!rfile.input)) return -EINVAL; - err = snd_rawmidi_kernel_release(&msynth->input_rfile); + + err = snd_rawmidi_kernel_release(&rfile); return err; } @@ -252,8 +276,8 @@ static int midisynth_use(void *private_data, struct snd_seq_port_subscribe *info return err; } snd_midi_event_reset_decode(msynth->parser); - scoped_guard(spinlock_irqsave, &msynth->output_lock) - msynth->output_rfile = rfile; + msynth->output_rfile = rfile; + rcu_assign_pointer(msynth->output_substream, rfile.output); return 0; } @@ -261,17 +285,16 @@ static int midisynth_use(void *private_data, struct snd_seq_port_subscribe *info static int midisynth_unuse(void *private_data, struct snd_seq_port_subscribe *info) { struct seq_midisynth *msynth = private_data; - struct snd_rawmidi_file rfile = {}; + struct snd_rawmidi_file rfile; - scoped_guard(spinlock_irqsave, &msynth->output_lock) { - rfile = msynth->output_rfile; - msynth->output_rfile = (struct snd_rawmidi_file){}; - } + rcu_assign_pointer(msynth->output_substream, NULL); + synchronize_rcu(); + snd_use_lock_sync(&msynth->output_use_lock); + rfile = msynth->output_rfile; + msynth->output_rfile = (struct snd_rawmidi_file){}; if (snd_BUG_ON(!rfile.output)) return -EINVAL; - - snd_use_lock_sync(&msynth->output_use_lock); snd_rawmidi_drain_output(rfile.output); return snd_rawmidi_kernel_release(&rfile); } diff --git a/sound/core/ump.c b/sound/core/ump.c index 632c13baf21e1..a85d1006e7a51 100644 --- a/sound/core/ump.c +++ b/sound/core/ump.c @@ -1333,6 +1333,8 @@ static void update_legacy_names(struct snd_ump_endpoint *ump) { struct snd_rawmidi *rmidi = ump->legacy_rmidi; + if (!rmidi) + return; update_legacy_substreams(ump, rmidi, SNDRV_RAWMIDI_STREAM_INPUT); update_legacy_substreams(ump, rmidi, SNDRV_RAWMIDI_STREAM_OUTPUT); } @@ -1341,6 +1343,8 @@ static void ump_legacy_set_rawmidi_name(struct snd_ump_endpoint *ump) { struct snd_rawmidi *rmidi = ump->legacy_rmidi; + if (!rmidi) + return; snprintf(rmidi->name, sizeof(rmidi->name), "%.68s (MIDI 1.0)", ump->core.name); } diff --git a/sound/core/ump_convert.c b/sound/core/ump_convert.c index 0fe13d0316568..85cc67de63306 100644 --- a/sound/core/ump_convert.c +++ b/sound/core/ump_convert.c @@ -258,6 +258,7 @@ static int cvt_legacy_sysex_to_ump(struct ump_cvt_to_ump *cvt, else status = UMP_SYSEX_STATUS_CONTINUE; *data = ump_compose(UMP_MSG_TYPE_DATA, group, status, cvt->len); + data[1] = 0; offset = 8; for (i = 0; i < cvt->len; i++) { *data |= cvt->buf[i] << offset; diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c index 06bfe09eae1a2..fd5bbf4615e20 100644 --- a/sound/drivers/aloop.c +++ b/sound/drivers/aloop.c @@ -1802,6 +1802,12 @@ static int loopback_probe(struct platform_device *devptr) int dev = devptr->id; int err; + if (dev < 0 || dev >= SNDRV_CARDS) { + dev_warn(&devptr->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + err = snd_devm_card_new(&devptr->dev, index[dev], id[dev], THIS_MODULE, sizeof(struct loopback), &card); if (err < 0) diff --git a/sound/drivers/dummy.c b/sound/drivers/dummy.c index 7283f0f18813d..21eb0d76d6f8d 100644 --- a/sound/drivers/dummy.c +++ b/sound/drivers/dummy.c @@ -782,7 +782,7 @@ static int snd_dummy_capsrc_put(struct snd_kcontrol *kcontrol, struct snd_ctl_el left = ucontrol->value.integer.value[0] & 1; right = ucontrol->value.integer.value[1] & 1; guard(spinlock_irq)(&dummy->mixer_lock); - change = dummy->capture_source[addr][0] != left && + change = dummy->capture_source[addr][0] != left || dummy->capture_source[addr][1] != right; dummy->capture_source[addr][0] = left; dummy->capture_source[addr][1] = right; @@ -1017,6 +1017,12 @@ static int snd_dummy_probe(struct platform_device *devptr) int idx, err; int dev = devptr->id; + if (dev < 0 || dev >= SNDRV_CARDS) { + dev_warn(&devptr->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + err = snd_devm_card_new(&devptr->dev, index[dev], id[dev], THIS_MODULE, sizeof(struct snd_dummy), &card); if (err < 0) diff --git a/sound/drivers/mpu401/mpu401.c b/sound/drivers/mpu401/mpu401.c index c217c427bf1e0..b93291ad067ac 100644 --- a/sound/drivers/mpu401/mpu401.c +++ b/sound/drivers/mpu401/mpu401.c @@ -89,6 +89,12 @@ static int snd_mpu401_probe(struct platform_device *devptr) int err; struct snd_card *card; + if (dev < 0 || dev >= SNDRV_CARDS) { + dev_warn(&devptr->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + if (port[dev] == SNDRV_AUTO_PORT) { dev_err(&devptr->dev, "specify port\n"); return -EINVAL; diff --git a/sound/drivers/mtpav.c b/sound/drivers/mtpav.c index d31eadf4be5f8..1ed6d4f0cb625 100644 --- a/sound/drivers/mtpav.c +++ b/sound/drivers/mtpav.c @@ -642,9 +642,7 @@ static void snd_mtpav_free(struct snd_card *card) { struct mtpav *crd = card->private_data; - guard(spinlock_irqsave)(&crd->spinlock); - if (crd->istimer > 0) - snd_mtpav_remove_output_timer(crd); + timer_shutdown_sync(&crd->timer); } /* diff --git a/sound/drivers/mts64.c b/sound/drivers/mts64.c index 36e9eab204cac..cefaa00b83e70 100644 --- a/sound/drivers/mts64.c +++ b/sound/drivers/mts64.c @@ -900,6 +900,12 @@ static int snd_mts64_probe(struct platform_device *pdev) p = platform_get_drvdata(pdev); platform_set_drvdata(pdev, NULL); + if (dev < 0) { + dev_warn(&pdev->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + if (dev >= SNDRV_CARDS) return -ENODEV; if (!enable[dev]) diff --git a/sound/drivers/portman2x4.c b/sound/drivers/portman2x4.c index dcc0899cfb99a..03ed3c3c27fb7 100644 --- a/sound/drivers/portman2x4.c +++ b/sound/drivers/portman2x4.c @@ -697,6 +697,12 @@ static int snd_portman_probe(struct platform_device *pdev) p = platform_get_drvdata(pdev); platform_set_drvdata(pdev, NULL); + if (dev < 0) { + dev_warn(&pdev->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + if (dev >= SNDRV_CARDS) return -ENODEV; if (!enable[dev]) diff --git a/sound/drivers/serial-u16550.c b/sound/drivers/serial-u16550.c index 3c28961091b12..cc9c325df910b 100644 --- a/sound/drivers/serial-u16550.c +++ b/sound/drivers/serial-u16550.c @@ -846,6 +846,12 @@ static int snd_serial_probe(struct platform_device *devptr) int err; int dev = devptr->id; + if (dev < 0 || dev >= SNDRV_CARDS) { + dev_warn(&devptr->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + switch (adaptor[dev]) { case SNDRV_SERIAL_SOUNDCANVAS: ins[dev] = 1; diff --git a/sound/drivers/virmidi.c b/sound/drivers/virmidi.c index a204f42d10265..70e16105f1c4a 100644 --- a/sound/drivers/virmidi.c +++ b/sound/drivers/virmidi.c @@ -75,6 +75,12 @@ static int snd_virmidi_probe(struct platform_device *devptr) int idx, err; int dev = devptr->id; + if (dev < 0 || dev >= SNDRV_CARDS) { + dev_warn(&devptr->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + err = snd_devm_card_new(&devptr->dev, index[dev], id[dev], THIS_MODULE, sizeof(struct snd_card_virmidi), &card); if (err < 0) diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index 48a2e7f2ebb15..e37945aee2bf2 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -3524,6 +3524,16 @@ static void alc287_fixup_yoga9_14iap7_bass_spk_pin(struct hda_codec *codec, } } +static void alc_fixup_yoga_pro7_audio(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + /* Reuse the DAC routing selected for ThinkPad X1 Gen7 */ + alc285_fixup_thinkpad_x1_gen7(codec, fix, action); + + /* Limit the internal mic boost to 0 or 1 to avoid noise */ + alc269_fixup_limit_int_mic_boost(codec, fix, action); +} + static void alc295_fixup_dell_inspiron_top_speakers(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -3932,6 +3942,7 @@ enum { ALC269VB_FIXUP_ASUS_ZENBOOK, ALC269VB_FIXUP_ASUS_ZENBOOK_UX31A, ALC269VB_FIXUP_ASUS_MIC_NO_PRESENCE, + ALC269VB_FIXUP_AYANEO_SPKR_PIN_FIX, ALC269_FIXUP_LIMIT_INT_MIC_BOOST_MUTE_LED, ALC269VB_FIXUP_ORDISSIMO_EVE2, ALC283_FIXUP_CHROME_BOOK, @@ -4199,6 +4210,7 @@ enum { ALC245_FIXUP_CLEVO_NOISY_MIC, ALC269_FIXUP_VAIO_VJFH52_MIC_NO_PRESENCE, ALC233_FIXUP_MEDION_MTL_SPK, + ALC269_FIXUP_STARLABS_LIMIT_INT_MIC_BOOST, ALC233_FIXUP_STARLABS_STARFIGHTER, ALC294_FIXUP_BASS_SPEAKER_15, ALC283_FIXUP_DELL_HP_RESUME, @@ -4734,6 +4746,13 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC269_FIXUP_HEADSET_MIC }, + [ALC269VB_FIXUP_AYANEO_SPKR_PIN_FIX] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x1a, 0x90170110 }, + { } + }, + }, [ALC269_FIXUP_LIMIT_INT_MIC_BOOST_MUTE_LED] = { .type = HDA_FIXUP_FUNC, .v.func = alc269_fixup_limit_int_mic_boost, @@ -6209,8 +6228,7 @@ static const struct hda_fixup alc269_fixups[] = { }, [ALC287_FIXUP_LENOVO_YOGA_PRO7] = { .type = HDA_FIXUP_FUNC, - /* Reuse the DAC routing selected for ThinkPad X1 Gen7 */ - .v.func = alc285_fixup_thinkpad_x1_gen7, + .v.func = alc_fixup_yoga_pro7_audio, .chained = true, .chain_id = ALC269_FIXUP_LENOVO_XPAD_ACPI, }, @@ -6746,9 +6764,15 @@ static const struct hda_fixup alc269_fixups[] = { { } }, }, + [ALC269_FIXUP_STARLABS_LIMIT_INT_MIC_BOOST] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc269_fixup_limit_int_mic_boost, + }, [ALC233_FIXUP_STARLABS_STARFIGHTER] = { .type = HDA_FIXUP_FUNC, .v.func = alc233_fixup_starlabs_starfighter, + .chained = true, + .chain_id = ALC269_FIXUP_STARLABS_LIMIT_INT_MIC_BOOST, }, [ALC294_FIXUP_BASS_SPEAKER_15] = { .type = HDA_FIXUP_FUNC, @@ -7442,6 +7466,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8dfc, "HP EliteBook 645 G12", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8dfd, "HP EliteBook 6 G1a 16", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8dfe, "HP EliteBook 665 G12", ALC236_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8e0d, "HP EliteBook 6 G1a 14 (AD3Q9ET#UUG)", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8e11, "HP Trekker", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e12, "HP Trekker", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e13, "HP Trekker", ALC287_FIXUP_CS35L41_I2C_2), @@ -7949,6 +7974,10 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x38a8, "Y780P AMD VECO dual", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38a9, "Thinkbook 16P", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), SND_PCI_QUIRK(0x17aa, 0x38ab, "Thinkbook 16P", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), + /* Lenovo Yoga Slim 7 14AKP10 shares PCI SSID 17aa:38b4 with Legion Slim 7 + * 16IRH8; use codec SSID to distinguish them + */ + HDA_CODEC_QUIRK(0x17aa, 0x391a, "Lenovo Yoga Slim 7 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x38b4, "Legion Slim 7 16IRH8", ALC287_FIXUP_CS35L41_I2C_2), HDA_CODEC_QUIRK(0x17aa, 0x391c, "Lenovo Yoga 7 2-in-1 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), HDA_CODEC_QUIRK(0x17aa, 0x391d, "Lenovo Yoga 7 2-in-1 16AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), @@ -7989,7 +8018,6 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3911, "Lenovo Yoga Pro 7 14IAH10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3912, "Lenovo Xiaoxin 14 GT", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3913, "Lenovo 145", ALC236_FIXUP_LENOVO_INV_DMIC), - SND_PCI_QUIRK(0x17aa, 0x391a, "Lenovo Yoga Slim 7 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x391f, "Yoga S990-16 pro Quad YC Quad", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3920, "Yoga S990-16 pro Quad VECO Quad", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3929, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), @@ -8055,6 +8083,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1d05, 0x3031, "TongFang X6AR55xU", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d05, 0x3033, "TongFang X6SP45xU", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d05, 0x3034, "TongFang X6KK45xU", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1d05, 0x30ba, "TongFang XxAF5xxx", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d17, 0x3288, "Haier Boyue G42", ALC269VC_FIXUP_ACER_VCOPPERBOX_PINS), SND_PCI_QUIRK(0x1d72, 0x1602, "RedmiBook", ALC255_FIXUP_XIAOMI_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1701, "XiaomiNotebook Pro", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE), @@ -8068,9 +8097,11 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1ee7, 0x2078, "HONOR BRB-X M1010", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1ee7, 0x2081, "HONOR MRB-XXX M1020", ALC256_FIXUP_HONOR_MRB_XXX_M1020_AUDIO), SND_PCI_QUIRK(0x1f4c, 0xe001, "Minisforum V3 (SE)", ALC245_FIXUP_BASS_HP_DAC), + SND_PCI_QUIRK(0x1f66, 0x0103, "AYANEO AIR 1S", ALC269VB_FIXUP_AYANEO_SPKR_PIN_FIX), SND_PCI_QUIRK(0x1f66, 0x0105, "Ayaneo Portable Game Player", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x2014, 0x800a, "Positivo ARN50", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x2039, 0x0001, "Inspur S14-G1", ALC295_FIXUP_CHROME_BOOK), + SND_PCI_QUIRK(0x2145, 0x0001, "Star Labs StarFighter", ALC233_FIXUP_STARLABS_STARFIGHTER), SND_PCI_QUIRK(0x2782, 0x0214, "VAIO VJFE-CL", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x2782, 0x0228, "Infinix ZERO BOOK 13", ALC269VB_FIXUP_INFINIX_ZERO_BOOK_13), SND_PCI_QUIRK(0x2782, 0x0232, "CHUWI CoreBook XPro", ALC269VB_FIXUP_CHUWI_COREBOOK_XPRO), @@ -8154,6 +8185,7 @@ static const struct hda_quirk alc269_fixup_vendor_tbl[] = { SND_PCI_QUIRK_VENDOR(0x104d, "Sony VAIO", ALC269_FIXUP_SONY_VAIO), SND_PCI_QUIRK_VENDOR(0x17aa, "Lenovo XPAD", ALC269_FIXUP_LENOVO_XPAD_ACPI), SND_PCI_QUIRK_VENDOR(0x19e5, "Huawei Matebook", ALC255_FIXUP_MIC_MUTE_LED), + SND_PCI_QUIRK_VENDOR(0x2145, "Star Labs", ALC269_FIXUP_STARLABS_LIMIT_INT_MIC_BOOST), {} }; @@ -8213,6 +8245,7 @@ static const struct hda_model_fixup alc269_fixup_models[] = { {.id = ALC269VB_FIXUP_ASUS_ZENBOOK, .name = "asus-zenbook"}, {.id = ALC269VB_FIXUP_ASUS_ZENBOOK_UX31A, .name = "asus-zenbook-ux31a"}, {.id = ALC269VB_FIXUP_ORDISSIMO_EVE2, .name = "ordissimo"}, + {.id = ALC269VB_FIXUP_AYANEO_SPKR_PIN_FIX, .name = "ayaneo-speaker-pin-fix"}, {.id = ALC282_FIXUP_ASUS_TX300, .name = "asus-tx300"}, {.id = ALC283_FIXUP_INT_MIC, .name = "alc283-int-mic"}, {.id = ALC290_FIXUP_MONO_SPEAKERS_HSJACK, .name = "mono-speakers"}, diff --git a/sound/hda/codecs/realtek/alc662.c b/sound/hda/codecs/realtek/alc662.c index 2cf3664a35ff0..a640292c4f100 100644 --- a/sound/hda/codecs/realtek/alc662.c +++ b/sound/hda/codecs/realtek/alc662.c @@ -852,6 +852,7 @@ static const struct hda_quirk alc662_fixup_tbl[] = { SND_PCI_QUIRK(0x1b35, 0x1234, "CZC ET26", ALC662_FIXUP_CZC_ET26), SND_PCI_QUIRK(0x1b35, 0x2206, "CZC P10T", ALC662_FIXUP_CZC_P10T), SND_PCI_QUIRK(0x1c6c, 0x1239, "Compaq N14JP6-V2", ALC897_FIXUP_HP_HSMIC_VERB), + SND_PCI_QUIRK(0x1e63, 0x6d9a, "F+ FLAPTOP r", ALC897_FIXUP_HP_HSMIC_VERB), #if 0 /* Below is a quirk table taken from the old code. diff --git a/sound/hda/common/codec.c b/sound/hda/common/codec.c index ef533770179b4..33dbe21fa165c 100644 --- a/sound/hda/common/codec.c +++ b/sound/hda/common/codec.c @@ -2271,6 +2271,7 @@ static int snd_hda_spdif_default_put(struct snd_kcontrol *kcontrol, int idx = kcontrol->private_value; struct hda_spdif_out *spdif; hda_nid_t nid; + unsigned int old_status; unsigned short val; int change; @@ -2279,6 +2280,7 @@ static int snd_hda_spdif_default_put(struct snd_kcontrol *kcontrol, guard(mutex)(&codec->spdif_mutex); spdif = snd_array_elem(&codec->spdif_out, idx); nid = spdif->nid; + old_status = spdif->status; spdif->status = ucontrol->value.iec958.status[0] | ((unsigned int)ucontrol->value.iec958.status[1] << 8) | ((unsigned int)ucontrol->value.iec958.status[2] << 16) | @@ -2289,7 +2291,7 @@ static int snd_hda_spdif_default_put(struct snd_kcontrol *kcontrol, spdif->ctls = val; if (change && nid != (u16)-1) set_dig_out_convert(codec, nid, val & 0xff, (val >> 8) & 0xff); - return change; + return change || spdif->status != old_status; } #define snd_hda_spdif_out_switch_info snd_ctl_boolean_mono_info diff --git a/sound/hda/common/controller.c b/sound/hda/common/controller.c index afec5c5546ec7..18dae022b324f 100644 --- a/sound/hda/common/controller.c +++ b/sound/hda/common/controller.c @@ -586,11 +586,11 @@ static int azx_pcm_open(struct snd_pcm_substream *substream) snd_hda_codec_pcm_get(apcm->info); mutex_lock(&chip->open_mutex); azx_dev = azx_assign_device(chip, substream); - trace_azx_pcm_open(chip, azx_dev); if (azx_dev == NULL) { err = -EBUSY; goto unlock; } + trace_azx_pcm_open(chip, azx_dev); runtime->private_data = azx_dev; runtime->hw = azx_pcm_hw; diff --git a/sound/hda/common/proc.c b/sound/hda/common/proc.c index c83796b13d3de..3dabdb418c7bf 100644 --- a/sound/hda/common/proc.c +++ b/sound/hda/common/proc.c @@ -624,7 +624,7 @@ static void print_conn_list(struct snd_info_buffer *buffer, /* Get Cache connections info */ cache_len = snd_hda_get_conn_list(codec, nid, &list); if (cache_len >= 0 && (cache_len != conn_len || - memcmp(list, conn, conn_len) != 0)) { + memcmp(list, conn, conn_len * sizeof(*conn)) != 0)) { snd_iprintf(buffer, " In-driver Connection: %d\n", cache_len); if (cache_len > 0) { snd_iprintf(buffer, " "); diff --git a/sound/hda/controllers/Kconfig b/sound/hda/controllers/Kconfig index 5d6a77e68588e..26ca69b09a9c3 100644 --- a/sound/hda/controllers/Kconfig +++ b/sound/hda/controllers/Kconfig @@ -33,6 +33,7 @@ config SND_HDA_TEGRA config SND_HDA_CIX_IPBLOQ tristate "CIX IPBLOQ HD Audio" depends on ARCH_CIX || COMPILE_TEST + depends on ARCH_DMA_ADDR_T_64BIT select SND_HDA select SND_HDA_ALIGNED_MMIO help diff --git a/sound/hda/core/device.c b/sound/hda/core/device.c index 160c8d0453b0e..ebaa106d0f018 100644 --- a/sound/hda/core/device.c +++ b/sound/hda/core/device.c @@ -404,6 +404,7 @@ static void setup_fg_nodes(struct hdac_device *codec) */ int snd_hdac_refresh_widgets(struct hdac_device *codec) { + hda_nid_t fg = codec->afg ? codec->afg : codec->mfg; hda_nid_t start_nid; int nums, err = 0; @@ -412,10 +413,10 @@ int snd_hdac_refresh_widgets(struct hdac_device *codec) * widgets array. */ guard(mutex)(&codec->widget_lock); - nums = snd_hdac_get_sub_nodes(codec, codec->afg, &start_nid); + nums = snd_hdac_get_sub_nodes(codec, fg, &start_nid); if (!start_nid || nums <= 0 || nums >= 0xff) { dev_err(&codec->dev, "cannot read sub nodes for FG 0x%02x\n", - codec->afg); + fg); return -EINVAL; } diff --git a/sound/hda/core/ext/stream.c b/sound/hda/core/ext/stream.c index 4c7506d49f55a..517bd151fcc38 100644 --- a/sound/hda/core/ext/stream.c +++ b/sound/hda/core/ext/stream.c @@ -210,8 +210,8 @@ void snd_hdac_ext_stream_reset(struct hdac_ext_stream *hext_stream) break; udelay(3); } while (--timeout); - val &= ~AZX_PPLCCTL_STRST; - writel(val, hext_stream->pplc_addr + AZX_REG_PPLCCTL); + snd_hdac_updatel(hext_stream->pplc_addr, AZX_REG_PPLCCTL, + AZX_PPLCCTL_STRST, 0); udelay(3); timeout = 50; diff --git a/sound/parisc/harmony.c b/sound/parisc/harmony.c index fb40476c6c91d..a9625aedf2a47 100644 --- a/sound/parisc/harmony.c +++ b/sound/parisc/harmony.c @@ -868,6 +868,9 @@ snd_harmony_create(struct snd_card *card, goto free_and_ret; } + spin_lock_init(&h->mixer_lock); + spin_lock_init(&h->lock); + err = request_irq(padev->irq, snd_harmony_interrupt, 0, "harmony", h); if (err) { @@ -877,9 +880,6 @@ snd_harmony_create(struct snd_card *card, } h->irq = padev->irq; - spin_lock_init(&h->mixer_lock); - spin_lock_init(&h->lock); - err = snd_device_new(card, SNDRV_DEV_LOWLEVEL, h, &ops); if (err < 0) goto free_and_ret; diff --git a/sound/pci/asihpi/hpi6000.c b/sound/pci/asihpi/hpi6000.c index c8d1518ee3e74..fd7fe9dba0b80 100644 --- a/sound/pci/asihpi/hpi6000.c +++ b/sound/pci/asihpi/hpi6000.c @@ -537,6 +537,11 @@ static short create_adapter_obj(struct hpi_adapter_obj *pao, hr1.size = sizeof(hr1); error = hpi6000_message_response_sequence(pao, 0, &hm, &hr0); + if (error) { + HPI_DEBUG_LOG(ERROR, "message transport error %d\n", + error); + return error; + } if (hr0.error) { HPI_DEBUG_LOG(DEBUG, "message error %d\n", hr0.error); return hr0.error; diff --git a/sound/pci/ctxfi/cthw20k2.c b/sound/pci/ctxfi/cthw20k2.c index 07e1490a6d174..36066ffe8784c 100644 --- a/sound/pci/ctxfi/cthw20k2.c +++ b/sound/pci/ctxfi/cthw20k2.c @@ -994,6 +994,7 @@ static int daio_mgr_dao_init(struct hw *hw, void *blk, unsigned int idx, unsigne /* S/PDIF output */ switch ((conf & 0xf)) { case 1: + case 9: set_field(&ctl->txctl[idx], ATXCTL_NUC, 0); break; case 2: diff --git a/sound/pci/ice1712/ice1712.c b/sound/pci/ice1712/ice1712.c index 7d1a357ed90dc..d6bfd8fb20e65 100644 --- a/sound/pci/ice1712/ice1712.c +++ b/sound/pci/ice1712/ice1712.c @@ -2523,7 +2523,7 @@ static int snd_ice1712_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) { static int dev; - struct snd_card *card; + struct snd_card *card __free(snd_card_unref) = NULL; struct snd_ice1712 *ice; int pcm_dev = 0, err; const struct snd_ice1712_card_info * const *tbl, *c; @@ -2535,8 +2535,8 @@ static int snd_ice1712_probe(struct pci_dev *pci, return -ENOENT; } - err = snd_card_new(&pci->dev, index[dev], id[dev], THIS_MODULE, - sizeof(*ice), &card); + err = snd_devm_card_new(&pci->dev, index[dev], id[dev], THIS_MODULE, + sizeof(*ice), &card); if (err < 0) return err; ice = card->private_data; @@ -2640,6 +2640,7 @@ static int snd_ice1712_probe(struct pci_dev *pci, if (err < 0) return err; pci_set_drvdata(pci, card); + card = NULL; /* probe succeeded, don't release as error */ dev++; return 0; } diff --git a/sound/pci/pcxhr/pcxhr.c b/sound/pci/pcxhr/pcxhr.c index 25a6403044305..60cc0721d0a6f 100644 --- a/sound/pci/pcxhr/pcxhr.c +++ b/sound/pci/pcxhr/pcxhr.c @@ -1535,6 +1535,13 @@ static int pcxhr_probe(struct pci_dev *pci, mgr->pci = pci; mgr->irq = -1; + /* ISR lock */ + mutex_init(&mgr->lock); + mutex_init(&mgr->msg_lock); + + /* init setup mutex*/ + mutex_init(&mgr->setup_mutex); + if (request_threaded_irq(pci->irq, pcxhr_interrupt, pcxhr_threaded_irq, IRQF_SHARED, KBUILD_MODNAME, mgr)) { @@ -1548,13 +1555,6 @@ static int pcxhr_probe(struct pci_dev *pci, "Digigram at 0x%lx & 0x%lx, 0x%lx irq %i", mgr->port[0], mgr->port[1], mgr->port[2], mgr->irq); - /* ISR lock */ - mutex_init(&mgr->lock); - mutex_init(&mgr->msg_lock); - - /* init setup mutex*/ - mutex_init(&mgr->setup_mutex); - mgr->prmh = kmalloc(sizeof(*mgr->prmh) + sizeof(u32) * (PCXHR_SIZE_MAX_LONG_STATUS - PCXHR_SIZE_MAX_STATUS), diff --git a/sound/pci/via82xx_modem.c b/sound/pci/via82xx_modem.c index 9b84d3fb9eaf5..b32f84ac17cc1 100644 --- a/sound/pci/via82xx_modem.c +++ b/sound/pci/via82xx_modem.c @@ -573,24 +573,18 @@ static inline unsigned int calc_linear_pos(struct via82xx_modem *chip, viadev->bufsize2, viadev->idx_table[idx].offset, viadev->idx_table[idx].size, count); #endif - if (count && size < count) { + if (! count) + /* bogus count 0 on the DMA boundary? */ + res = viadev->idx_table[idx].offset; + else + /* count register returns full size + * when end of buffer is reached + */ + res = viadev->idx_table[idx].offset + size; + if (check_invalid_pos(viadev, res)) { dev_dbg(chip->card->dev, - "invalid via82xx_cur_ptr, using last valid pointer\n"); + "invalid via82xx_cur_ptr (2), using last valid pointer\n"); res = viadev->lastpos; - } else { - if (! count) - /* bogus count 0 on the DMA boundary? */ - res = viadev->idx_table[idx].offset; - else - /* count register returns full size - * when end of buffer is reached - */ - res = viadev->idx_table[idx].offset + size; - if (check_invalid_pos(viadev, res)) { - dev_dbg(chip->card->dev, - "invalid via82xx_cur_ptr (2), using last valid pointer\n"); - res = viadev->lastpos; - } } } viadev->lastpos = res; /* remember the last position */ diff --git a/sound/soc/amd/acp-config.c b/sound/soc/amd/acp-config.c index 88e4230d66c3d..8e1563824a97b 100644 --- a/sound/soc/amd/acp-config.c +++ b/sound/soc/amd/acp-config.c @@ -54,7 +54,7 @@ static const struct dmi_system_id acp70_acpi_flag_override_table[] = { { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC"), - DMI_MATCH(DMI_PRODUCT_NAME, "TUF Gaming A14 FA401EA"), + DMI_MATCH(DMI_PRODUCT_NAME, "FA401EA"), }, }, { diff --git a/sound/soc/amd/acp-da7219-max98357a.c b/sound/soc/amd/acp-da7219-max98357a.c index af559653e6255..1ac729a58bb45 100644 --- a/sound/soc/amd/acp-da7219-max98357a.c +++ b/sound/soc/amd/acp-da7219-max98357a.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "acp.h" #include "../codecs/da7219.h" @@ -742,6 +743,18 @@ static const struct regulator_desc acp_da7219_desc = { .n_voltages = 1, }; +/* + * The ACP3.x+ (Raven/Picasso and later) audio coprocessor is a dedicated PCI + * function. Carrizo/Stoney - the only platforms handled by this driver - reach + * the ACP through the GPU driver and have no such device. + */ +#define ACP3X_PCI_DEV_ID 0x15e2 + +static const struct pci_device_id acp3x_pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_AMD, ACP3X_PCI_DEV_ID) }, + { 0, }, +}; + static int cz_probe(struct platform_device *pdev) { int ret; @@ -750,6 +763,16 @@ static int cz_probe(struct platform_device *pdev) struct regulator_dev *rdev; struct device *dev = &pdev->dev; + /* + * AMDI5682 is also matched by acp3x-alc5682-max98357 (Raven/Picasso). + * If the ACP3.x PCI function is present this is such a board; return + * -ENODEV so that driver binds instead. + */ + if (pci_dev_present(acp3x_pci_ids)) { + dev_info(dev, "ACP3.x PCI device present, deferring to acp3x-alc5682-max98357\n"); + return -ENODEV; + } + card = (struct snd_soc_card *)acp_soc_is_rltk_max(dev); if (!card) return -ENODEV; diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c index e8b6819cc4b45..9e3d5d3ed1d75 100644 --- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c +++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c @@ -205,9 +205,19 @@ static int create_sdw_dailink(struct snd_soc_card *card, return -EINVAL; } + if (!soc_end->link_mask) { + dev_err(dev, "invalid zero link_mask\n"); + return -EINVAL; + } + if ((ffs(soc_end->link_mask) - 1) >= amd_ctx->max_sdw_links) { + dev_err(dev, "link_id %d exceeds max_sdw_links %d\n", + ffs(soc_end->link_mask) - 1, amd_ctx->max_sdw_links); + return -EINVAL; + } + switch (amd_ctx->acp_rev) { case ACP63_PCI_REV: - ret = get_acp63_cpu_pin_id(ffs(soc_end->link_mask - 1), + ret = get_acp63_cpu_pin_id(ffs(soc_end->link_mask) - 1, *be_id, &cpu_pin_id, dev); if (ret) return ret; @@ -215,7 +225,7 @@ static int create_sdw_dailink(struct snd_soc_card *card, case ACP70_PCI_REV: case ACP71_PCI_REV: case ACP72_PCI_REV: - ret = get_acp70_cpu_pin_id(ffs(soc_end->link_mask - 1), + ret = get_acp70_cpu_pin_id(ffs(soc_end->link_mask) - 1, *be_id, &cpu_pin_id, dev); if (ret) return ret; @@ -432,7 +442,7 @@ static int soc_card_dai_links_create(struct snd_soc_card *card) if (!soc_aux) return -ENOMEM; - ret = asoc_sdw_parse_sdw_endpoints(card, soc_aux, soc_dais, soc_ends, &num_confs); + ret = asoc_sdw_parse_sdw_endpoints(dev, ctx, soc_aux, soc_dais, soc_ends, &num_confs); if (ret < 0) return ret; diff --git a/sound/soc/amd/acp/acp-sdw-sof-mach.c b/sound/soc/amd/acp/acp-sdw-sof-mach.c index a423853f3a97d..13885ca20168b 100644 --- a/sound/soc/amd/acp/acp-sdw-sof-mach.c +++ b/sound/soc/amd/acp/acp-sdw-sof-mach.c @@ -121,9 +121,18 @@ static int create_sdw_dailink(struct snd_soc_card *card, return -EINVAL; } + if (!sof_end->link_mask) { + dev_err(dev, "invalid zero link_mask\n"); + return -EINVAL; + } + if ((ffs(sof_end->link_mask) - 1) >= amd_ctx->max_sdw_links) { + dev_err(dev, "link_id %d exceeds max_sdw_links %d\n", + ffs(sof_end->link_mask) - 1, amd_ctx->max_sdw_links); + return -EINVAL; + } switch (amd_ctx->acp_rev) { case ACP63_PCI_REV: - ret = get_acp63_cpu_pin_id(ffs(sof_end->link_mask - 1), + ret = get_acp63_cpu_pin_id(ffs(sof_end->link_mask) - 1, *be_id, &cpu_pin_id, dev); if (ret) return ret; @@ -131,7 +140,7 @@ static int create_sdw_dailink(struct snd_soc_card *card, case ACP70_PCI_REV: case ACP71_PCI_REV: case ACP72_PCI_REV: - ret = get_acp70_cpu_pin_id(ffs(sof_end->link_mask - 1), + ret = get_acp70_cpu_pin_id(ffs(sof_end->link_mask) - 1, *be_id, &cpu_pin_id, dev); if (ret) return ret; @@ -277,6 +286,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card) int num_devs = 0; int num_ends = 0; int num_aux = 0; + int num_confs; int num_links; int be_id = 0; int ret; @@ -287,6 +297,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card) return ret; } + num_confs = num_ends; /* One per DAI link, worst case is a DAI link for every endpoint */ struct asoc_sdw_dailink *sof_dais __free(kfree) = kzalloc_objs(*sof_dais, num_ends); @@ -303,7 +314,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card) if (!sof_aux) return -ENOMEM; - ret = asoc_sdw_parse_sdw_endpoints(card, sof_aux, sof_dais, sof_ends, &num_devs); + ret = asoc_sdw_parse_sdw_endpoints(dev, ctx, sof_aux, sof_dais, sof_ends, &num_confs); if (ret < 0) return ret; @@ -315,7 +326,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card) dev_dbg(dev, "sdw %d, dmic %d", sdw_be_num, dmic_num); - codec_conf = devm_kcalloc(dev, num_devs, sizeof(*codec_conf), GFP_KERNEL); + codec_conf = devm_kcalloc(dev, num_confs, sizeof(*codec_conf), GFP_KERNEL); if (!codec_conf) return -ENOMEM; @@ -326,7 +337,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card) return -ENOMEM; card->codec_conf = codec_conf; - card->num_configs = num_devs; + card->num_configs = num_confs; card->dai_link = dai_links; card->num_links = num_links; card->aux_dev = sof_aux; diff --git a/sound/soc/amd/renoir/acp3x-pdm-dma.c b/sound/soc/amd/renoir/acp3x-pdm-dma.c index e60e3821703cc..3f59b753243d5 100644 --- a/sound/soc/amd/renoir/acp3x-pdm-dma.c +++ b/sound/soc/amd/renoir/acp3x-pdm-dma.c @@ -104,7 +104,7 @@ static void disable_pdm_interrupts(void __iomem *acp_base) u32 ext_int_ctrl; ext_int_ctrl = rn_readl(acp_base + ACP_EXTERNAL_INTR_CNTL); - ext_int_ctrl |= ~PDM_DMA_INTR_MASK; + ext_int_ctrl &= ~PDM_DMA_INTR_MASK; rn_writel(ext_int_ctrl, acp_base + ACP_EXTERNAL_INTR_CNTL); } diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index ae90e9f513d15..79858fc948065 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -843,6 +843,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Thin A15 B7VE"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Micro-Star International Co., Ltd."), + DMI_MATCH(DMI_PRODUCT_NAME, "Thin A15 B7UC"), + } + }, { .driver_data = &acp6x_card, .matches = { diff --git a/sound/soc/amd/yc/acp6x-pdm-dma.c b/sound/soc/amd/yc/acp6x-pdm-dma.c index 710db721ffa48..40c4d833f4ed3 100644 --- a/sound/soc/amd/yc/acp6x-pdm-dma.c +++ b/sound/soc/amd/yc/acp6x-pdm-dma.c @@ -275,9 +275,11 @@ static int acp6x_pdm_dma_close(struct snd_soc_component *component, struct snd_pcm_substream *substream) { struct pdm_dev_data *adata = dev_get_drvdata(component->dev); + struct snd_pcm_runtime *runtime = substream->runtime; acp6x_disable_pdm_interrupts(adata->acp6x_base); adata->capture_stream = NULL; + kfree(runtime->private_data); return 0; } diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index 492165c0e1ea9..ebe116f32661c 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -210,10 +210,10 @@ static void mca_fe_early_trigger(struct snd_pcm_substream *substream, int cmd, SERDES_STATUS_EN | SERDES_STATUS_RST, SERDES_STATUS_RST); /* - * Experiments suggest that it takes at most ~1 us - * for the bit to clear, so wait 2 us for good measure. + * The SERDES cluster needs a bit of time to reset itself + * and settle before we start poking it. This is... slow... */ - udelay(2); + udelay(25); WARN_ON(readl_relaxed(cl->base + serdes_unit + REG_SERDES_STATUS) & SERDES_STATUS_RST); mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL, diff --git a/sound/soc/bcm/bcm63xx-i2s-whistler.c b/sound/soc/bcm/bcm63xx-i2s-whistler.c index c47ed1e6ea2b6..14d111fe29d83 100644 --- a/sound/soc/bcm/bcm63xx-i2s-whistler.c +++ b/sound/soc/bcm/bcm63xx-i2s-whistler.c @@ -285,6 +285,7 @@ static const struct of_device_id snd_soc_bcm_audio_match[] = { {.compatible = "brcm,bcm63xx-i2s"}, { } }; +MODULE_DEVICE_TABLE(of, snd_soc_bcm_audio_match); #endif static struct platform_driver bcm63xx_i2s_driver = { diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig index e4d139108d720..19bcaed004600 100644 --- a/sound/soc/codecs/Kconfig +++ b/sound/soc/codecs/Kconfig @@ -2920,4 +2920,6 @@ config SND_SOC_LPASS_TX_MACRO select SND_SOC_LPASS_MACRO_COMMON tristate "Qualcomm TX Macro in LPASS(Low Power Audio SubSystem)" +source "sound/soc/codecs/aw87xxx/Kconfig" + endmenu diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile index aa0396e5b575b..14516b38ab676 100644 --- a/sound/soc/codecs/Makefile +++ b/sound/soc/codecs/Makefile @@ -879,6 +879,7 @@ obj-$(CONFIG_SND_SOC_WSA884X) += snd-soc-wsa884x.o obj-$(CONFIG_SND_SOC_ZL38060) += snd-soc-zl38060.o # Amp +obj-$(CONFIG_SND_SOC_AW87XXX) += aw87xxx/ obj-$(CONFIG_SND_SOC_MAX9877) += snd-soc-max9877.o obj-$(CONFIG_SND_SOC_MAX98504) += snd-soc-max98504.o obj-$(CONFIG_SND_SOC_SIMPLE_AMPLIFIER) += snd-soc-simple-amplifier.o diff --git a/sound/soc/codecs/ab8500-codec.c b/sound/soc/codecs/ab8500-codec.c index 6e8ef9cd1b31a..e1a0e35836e6c 100644 --- a/sound/soc/codecs/ab8500-codec.c +++ b/sound/soc/codecs/ab8500-codec.c @@ -20,10 +20,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -53,32 +51,9 @@ /* Macrocell register definitions */ #define AB8500_GPIO_DIR4_REG 0x13 /* Bank AB8500_MISC */ -/* Nr of FIR/IIR-coeff banks in ANC-block */ -#define AB8500_NR_OF_ANC_COEFF_BANKS 2 - -/* Minimum duration to keep ANC IIR Init bit high or -low before proceeding with the configuration sequence */ -#define AB8500_ANC_SM_DELAY 2000 - -/* Sidetone states */ -static const char * const enum_sid_state[] = { - "Unconfigured", - "Apply FIR", - "FIR is configured", -}; -enum sid_state { - SID_UNCONFIGURED = 0, - SID_APPLY_FIR = 1, - SID_FIR_CONFIGURED = 2, -}; - /* Private data for AB8500 device-driver */ struct ab8500_codec_drvdata { struct regmap *regmap; - struct mutex ctrl_lock; - - /* Sidetone */ - enum sid_state sid_status; }; static inline const char *amic_micbias_str(enum amic_micbias micbias) @@ -258,7 +233,7 @@ static const struct snd_kcontrol_new dapm_anc_in_select[] = { /* ANC - Enable/Disable */ static const struct snd_kcontrol_new dapm_anc_enable[] = { SOC_DAPM_SINGLE("Switch", AB8500_ANCCONF1, - AB8500_ANCCONF1_ENANC, 0, 0), + AB8500_ANCCONF1_ENANC, 1, 0), }; /* ANC to Earpiece - Mute */ @@ -340,12 +315,6 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = { /* DA/AD */ - SND_SOC_DAPM_INPUT("ADC Input"), - SND_SOC_DAPM_ADC("ADC", "ab8500_0c", SND_SOC_NOPM, 0, 0), - - SND_SOC_DAPM_DAC("DAC", NULL, SND_SOC_NOPM, 0, 0), - SND_SOC_DAPM_OUTPUT("DAC Output"), - SND_SOC_DAPM_AIF_IN("DA_IN1", NULL, 0, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_AIF_IN("DA_IN2", NULL, 0, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_AIF_IN("DA_IN3", NULL, 0, SND_SOC_NOPM, 0, 0), @@ -537,9 +506,8 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = { SND_SOC_DAPM_MIXER("AD3 Channel Volume", SND_SOC_NOPM, 0, 0, NULL, 0), - SND_SOC_DAPM_MIXER("AD3 Enable", - AB8500_ADPATHENA, AB8500_ADPATHENA_ENAD34, 0, - NULL, 0), + SND_SOC_DAPM_SUPPLY("AD34 Enable", AB8500_ADPATHENA, + AB8500_ADPATHENA_ENAD34, 0, NULL, 0), /* Mic 2 */ @@ -598,9 +566,8 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = { SND_SOC_NOPM, 0, 0, NULL, 0), - SND_SOC_DAPM_MIXER("AD12 Enable", - AB8500_ADPATHENA, AB8500_ADPATHENA_ENAD12, 0, - NULL, 0), + SND_SOC_DAPM_SUPPLY("AD12 Enable", AB8500_ADPATHENA, + AB8500_ADPATHENA_ENAD12, 0, NULL, 0), /* HD Capture path */ @@ -614,12 +581,8 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = { SND_SOC_DAPM_MIXER("AD6 Channel Volume", SND_SOC_NOPM, 0, 0, NULL, 0), - SND_SOC_DAPM_MIXER("AD57 Enable", - AB8500_ADPATHENA, AB8500_ADPATHENA_ENAD5768, 0, - NULL, 0), - SND_SOC_DAPM_MIXER("AD68 Enable", - AB8500_ADPATHENA, AB8500_ADPATHENA_ENAD5768, 0, - NULL, 0), + SND_SOC_DAPM_SUPPLY("AD5768 Enable", AB8500_ADPATHENA, + AB8500_ADPATHENA_ENAD5768, 0, NULL, 0), /* Digital Microphone path */ @@ -651,15 +614,8 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = { SND_SOC_DAPM_MIXER("AD4 Channel Volume", SND_SOC_NOPM, 0, 0, NULL, 0), - SND_SOC_DAPM_MIXER("AD4 Enable", - AB8500_ADPATHENA, AB8500_ADPATHENA_ENAD34, - 0, NULL, 0), - /* Acoustical Noise Cancellation path */ - SND_SOC_DAPM_INPUT("ANC Configure Input"), - SND_SOC_DAPM_OUTPUT("ANC Configure Output"), - SND_SOC_DAPM_MUX("ANC Source", SND_SOC_NOPM, 0, 0, dapm_anc_in_select), @@ -702,24 +658,13 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { {"Main Supply", NULL, "Audio Power"}, {"Main Supply", NULL, "Audio Analog Power"}, - {"DAC", NULL, "ab8500_0p"}, - {"DAC", NULL, "Main Supply"}, - {"ADC", NULL, "ab8500_0c"}, - {"ADC", NULL, "Main Supply"}, - - /* ANC Configure */ - {"ANC Configure Input", NULL, "Main Supply"}, - {"ANC Configure Output", NULL, "ANC Configure Input"}, - - /* AD/DA */ - {"ADC", NULL, "ADC Input"}, - {"DAC Output", NULL, "DAC"}, - /* Powerup charge pump if DA1/2 is in use */ {"DA_IN1", NULL, "ab8500_0p"}, + {"DA_IN1", NULL, "Main Supply"}, {"DA_IN1", NULL, "Charge Pump"}, {"DA_IN2", NULL, "ab8500_0p"}, + {"DA_IN2", NULL, "Main Supply"}, {"DA_IN2", NULL, "Charge Pump"}, /* Headset path */ @@ -754,8 +699,10 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { /* HF or LineOut path */ {"DA_IN3", NULL, "ab8500_0p"}, + {"DA_IN3", NULL, "Main Supply"}, {"DA3 Channel Volume", NULL, "DA_IN3"}, {"DA_IN4", NULL, "ab8500_0p"}, + {"DA_IN4", NULL, "Main Supply"}, {"DA4 Channel Volume", NULL, "DA_IN4"}, {"Speaker Left Source", "Audio Path", "DA3 Channel Volume"}, @@ -813,8 +760,10 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { /* Vibrator path */ {"DA_IN5", NULL, "ab8500_0p"}, + {"DA_IN5", NULL, "Main Supply"}, {"DA5 Channel Volume", NULL, "DA_IN5"}, {"DA_IN6", NULL, "ab8500_0p"}, + {"DA_IN6", NULL, "Main Supply"}, {"DA6 Channel Volume", NULL, "DA_IN6"}, {"VIB1 DAC", NULL, "DA5 Channel Volume"}, @@ -856,13 +805,15 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { {"AD1 Channel Volume", NULL, "AD1 Source Select"}, {"AD2 Channel Volume", NULL, "AD2 Source Select"}, - {"AD12 Enable", NULL, "AD1 Channel Volume"}, - {"AD12 Enable", NULL, "AD2 Channel Volume"}, + {"AD1 Channel Volume", NULL, "AD12 Enable"}, + {"AD2 Channel Volume", NULL, "AD12 Enable"}, - {"AD_OUT1", NULL, "ab8500_0c"}, - {"AD_OUT1", NULL, "AD12 Enable"}, - {"AD_OUT2", NULL, "ab8500_0c"}, - {"AD_OUT2", NULL, "AD12 Enable"}, + {"ab8500_0c", NULL, "AD_OUT1"}, + {"AD_OUT1", NULL, "Main Supply"}, + {"AD_OUT1", NULL, "AD1 Channel Volume"}, + {"ab8500_0c", NULL, "AD_OUT2"}, + {"AD_OUT2", NULL, "Main Supply"}, + {"AD_OUT2", NULL, "AD2 Channel Volume"}, /* Mic 1 */ @@ -879,11 +830,11 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { {"AD3 Source Select", "Mic 1", "MIC1 ADC"}, {"AD3 Channel Volume", NULL, "AD3 Source Select"}, + {"AD3 Channel Volume", NULL, "AD34 Enable"}, - {"AD3 Enable", NULL, "AD3 Channel Volume"}, - - {"AD_OUT3", NULL, "ab8500_0c"}, - {"AD_OUT3", NULL, "AD3 Enable"}, + {"ab8500_0c", NULL, "AD_OUT3"}, + {"AD_OUT3", NULL, "Main Supply"}, + {"AD_OUT3", NULL, "AD3 Channel Volume"}, /* HD Capture path */ @@ -892,14 +843,15 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { {"AD5 Channel Volume", NULL, "AD5 Source Select"}, {"AD6 Channel Volume", NULL, "AD6 Source Select"}, + {"AD5 Channel Volume", NULL, "AD5768 Enable"}, + {"AD6 Channel Volume", NULL, "AD5768 Enable"}, - {"AD57 Enable", NULL, "AD5 Channel Volume"}, - {"AD68 Enable", NULL, "AD6 Channel Volume"}, - - {"AD_OUT57", NULL, "ab8500_0c"}, - {"AD_OUT57", NULL, "AD57 Enable"}, - {"AD_OUT68", NULL, "ab8500_0c"}, - {"AD_OUT68", NULL, "AD68 Enable"}, + {"ab8500_0c", NULL, "AD_OUT57"}, + {"AD_OUT57", NULL, "Main Supply"}, + {"AD_OUT57", NULL, "AD5 Channel Volume"}, + {"ab8500_0c", NULL, "AD_OUT68"}, + {"AD_OUT68", NULL, "Main Supply"}, + {"AD_OUT68", NULL, "AD6 Channel Volume"}, /* Digital Microphone path */ @@ -910,17 +862,25 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { {"DMic 5", NULL, "V-DMIC"}, {"DMic 6", NULL, "V-DMIC"}, - {"AD1 Source Select", NULL, "DMic 1"}, - {"AD2 Source Select", NULL, "DMic 2"}, - {"AD3 Source Select", NULL, "DMic 3"}, - {"AD5 Source Select", NULL, "DMic 5"}, - {"AD6 Source Select", NULL, "DMic 6"}, + {"DMIC1", NULL, "DMic 1"}, + {"DMIC2", NULL, "DMic 2"}, + {"DMIC3", NULL, "DMic 3"}, + {"DMIC4", NULL, "DMic 4"}, + {"DMIC5", NULL, "DMic 5"}, + {"DMIC6", NULL, "DMic 6"}, - {"AD4 Channel Volume", NULL, "DMic 4"}, - {"AD4 Enable", NULL, "AD4 Channel Volume"}, + {"AD1 Source Select", "DMic 1", "DMIC1"}, + {"AD2 Source Select", "DMic 2", "DMIC2"}, + {"AD3 Source Select", "DMic 3", "DMIC3"}, + {"AD5 Source Select", "DMic 5", "DMIC5"}, + {"AD6 Source Select", "DMic 6", "DMIC6"}, - {"AD_OUT4", NULL, "ab8500_0c"}, - {"AD_OUT4", NULL, "AD4 Enable"}, + {"AD4 Channel Volume", NULL, "DMIC4"}, + {"AD4 Channel Volume", NULL, "AD34 Enable"}, + + {"ab8500_0c", NULL, "AD_OUT4"}, + {"AD_OUT4", NULL, "Main Supply"}, + {"AD_OUT4", NULL, "AD4 Channel Volume"}, /* LineIn Bypass path */ @@ -945,13 +905,13 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { /* Sidetone Filter path */ - {"Sidetone Left Source", "LineIn Left", "AD12 Enable"}, - {"Sidetone Left Source", "LineIn Right", "AD12 Enable"}, - {"Sidetone Left Source", "Mic 1", "AD3 Enable"}, + {"Sidetone Left Source", "LineIn Left", "AD1 Channel Volume"}, + {"Sidetone Left Source", "LineIn Right", "AD2 Channel Volume"}, + {"Sidetone Left Source", "Mic 1", "AD3 Channel Volume"}, {"Sidetone Left Source", "Headset Left", "DA_IN1"}, - {"Sidetone Right Source", "LineIn Right", "AD12 Enable"}, - {"Sidetone Right Source", "Mic 1", "AD3 Enable"}, - {"Sidetone Right Source", "DMic 4", "AD4 Enable"}, + {"Sidetone Right Source", "LineIn Right", "AD2 Channel Volume"}, + {"Sidetone Right Source", "Mic 1", "AD3 Channel Volume"}, + {"Sidetone Right Source", "DMic 4", "AD4 Channel Volume"}, {"Sidetone Right Source", "Headset Right", "DA_IN2"}, {"STFIR1 Control", NULL, "Sidetone Left Source"}, @@ -979,78 +939,6 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes_mic2_vamicx[] = { {"MIC2 V-AMICx Enable", NULL, "V-AMIC2"}, }; -/* - * Control-events - */ - -static int sid_status_control_get(struct snd_kcontrol *kcontrol, - struct snd_ctl_elem_value *ucontrol) -{ - struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); - struct ab8500_codec_drvdata *drvdata = dev_get_drvdata(component->dev); - - mutex_lock(&drvdata->ctrl_lock); - ucontrol->value.enumerated.item[0] = drvdata->sid_status; - mutex_unlock(&drvdata->ctrl_lock); - - return 0; -} - -/* Write sidetone FIR-coefficients configuration sequence */ -static int sid_status_control_put(struct snd_kcontrol *kcontrol, - struct snd_ctl_elem_value *ucontrol) -{ - struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); - struct ab8500_codec_drvdata *drvdata = dev_get_drvdata(component->dev); - unsigned int param, sidconf; - int status = 1; - - dev_dbg(component->dev, "%s: Enter\n", __func__); - - if (ucontrol->value.enumerated.item[0] != SID_APPLY_FIR) { - dev_err(component->dev, - "%s: ERROR: This control supports '%s' only!\n", - __func__, enum_sid_state[SID_APPLY_FIR]); - return -EIO; - } - - mutex_lock(&drvdata->ctrl_lock); - - sidconf = snd_soc_component_read(component, AB8500_SIDFIRCONF); - if (((sidconf & BIT(AB8500_SIDFIRCONF_FIRSIDBUSY)) != 0)) { - if ((sidconf & BIT(AB8500_SIDFIRCONF_ENFIRSIDS)) == 0) { - dev_err(component->dev, "%s: Sidetone busy while off!\n", - __func__); - status = -EPERM; - } else { - status = -EBUSY; - } - goto out; - } - - snd_soc_component_write(component, AB8500_SIDFIRADR, 0); - - for (param = 0; param < AB8500_SID_FIR_COEFFS; param++) { - snd_soc_component_write(component, AB8500_SIDFIRCOEF1, 0); - snd_soc_component_write(component, AB8500_SIDFIRCOEF2, 0); - } - - snd_soc_component_update_bits(component, AB8500_SIDFIRADR, - BIT(AB8500_SIDFIRADR_FIRSIDSET), - BIT(AB8500_SIDFIRADR_FIRSIDSET)); - snd_soc_component_update_bits(component, AB8500_SIDFIRADR, - BIT(AB8500_SIDFIRADR_FIRSIDSET), 0); - - drvdata->sid_status = SID_FIR_CONFIGURED; - -out: - mutex_unlock(&drvdata->ctrl_lock); - - dev_dbg(component->dev, "%s: Exit\n", __func__); - - return status; -} - /* * Controls - Non-DAPM ASoC */ @@ -1334,9 +1222,6 @@ static SOC_ENUM_SINGLE_DECL(soc_enum_bfifomast, AB8500_FIFOCONF3, AB8500_FIFOCONF3_BFIFOMAST_SHIFT, enum_slavemaster); -/* Sidetone */ -static SOC_ENUM_SINGLE_EXT_DECL(soc_enum_sidstate, enum_sid_state); - /* ANC */ static struct snd_kcontrol_new ab8500_ctrls[] = { @@ -1627,8 +1512,6 @@ static struct snd_kcontrol_new ab8500_ctrls[] = { AB8500_ANC_WARP_DELAY_MIN, AB8500_ANC_WARP_DELAY_MAX, 0), /* Sidetone */ - SOC_ENUM_EXT("Sidetone Status", soc_enum_sidstate, - sid_status_control_get, sid_status_control_put), SOC_SINGLE_STROBE("Sidetone Reset", AB8500_SIDFIRADR, AB8500_SIDFIRADR_FIRSIDSET, 0), }; @@ -1640,23 +1523,25 @@ static struct snd_kcontrol_new ab8500_ctrls[] = { static int ab8500_audio_init_audioblock(struct snd_soc_component *component) { int status; + u8 mask = AB8500_STW4500CTRL3_CLK32KOUT2DIS | + AB8500_STW4500CTRL3_RESETAUDN; dev_dbg(component->dev, "%s: Enter.\n", __func__); - /* Reset audio-registers and disable 32kHz-clock output 2 */ - status = ab8500_sysctrl_write(AB8500_STW4500CTRL3, - AB8500_STW4500CTRL3_CLK32KOUT2DIS | - AB8500_STW4500CTRL3_RESETAUDN, - AB8500_STW4500CTRL3_RESETAUDN); + /* Reset the audio registers and disable the unused 32 kHz output. */ + status = ab8500_sysctrl_write(AB8500_STW4500CTRL3, mask, + AB8500_STW4500CTRL3_CLK32KOUT2DIS); if (status < 0) return status; - return 0; + return ab8500_sysctrl_write(AB8500_STW4500CTRL3, mask, mask); } static int ab8500_audio_setup_mics(struct snd_soc_component *component, struct amic_settings *amics) { + struct device *dev = component->dev; + struct ab8500 *ab8500 = dev_get_drvdata(dev->parent); struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component); u8 value8; unsigned int value; @@ -1665,20 +1550,21 @@ static int ab8500_audio_setup_mics(struct snd_soc_component *component, dev_dbg(component->dev, "%s: Enter.\n", __func__); - /* Set DMic-clocks to outputs */ - status = abx500_get_register_interruptible(component->dev, AB8500_MISC, - AB8500_GPIO_DIR4_REG, - &value8); - if (status < 0) - return status; - value = value8 | GPIO27_DIR_OUTPUT | GPIO29_DIR_OUTPUT | - GPIO31_DIR_OUTPUT; - status = abx500_set_register_interruptible(component->dev, - AB8500_MISC, - AB8500_GPIO_DIR4_REG, - value); - if (status < 0) - return status; + /* Set DMic-clocks to outputs; these GPIOs do not exist on AB8505. */ + if (!is_ab8505(ab8500)) { + status = abx500_get_register_interruptible(dev, AB8500_MISC, + AB8500_GPIO_DIR4_REG, + &value8); + if (status < 0) + return status; + value = value8 | GPIO27_DIR_OUTPUT | GPIO29_DIR_OUTPUT | + GPIO31_DIR_OUTPUT; + status = abx500_set_register_interruptible(dev, AB8500_MISC, + AB8500_GPIO_DIR4_REG, + value); + if (status < 0) + return status; + } /* Attach regulators to AMic DAPM-paths */ dev_dbg(component->dev, "%s: Mic 1a regulator: %s\n", __func__, @@ -1747,149 +1633,91 @@ static int ab8500_audio_set_ear_cmv(struct snd_soc_component *component, return 0; } -static int ab8500_audio_set_bit_delay(struct snd_soc_dai *dai, - unsigned int delay) -{ - unsigned int mask, val; - struct snd_soc_component *component = dai->component; - - mask = BIT(AB8500_DIGIFCONF2_IF0DEL); - val = 0; - - switch (delay) { - case 0: - break; - case 1: - val |= BIT(AB8500_DIGIFCONF2_IF0DEL); - break; - default: - dev_err(dai->component->dev, - "%s: ERROR: Unsupported bit-delay (0x%x)!\n", - __func__, delay); - return -EINVAL; - } - - dev_dbg(dai->component->dev, "%s: IF0 Bit-delay: %d bits.\n", - __func__, delay); - snd_soc_component_update_bits(component, AB8500_DIGIFCONF2, mask, val); - - return 0; -} - -/* Gates clocking according format mask */ -static int ab8500_codec_set_dai_clock_gate(struct snd_soc_component *component, - unsigned int fmt) -{ - unsigned int mask; - unsigned int val; - - mask = BIT(AB8500_DIGIFCONF1_ENMASTGEN) | - BIT(AB8500_DIGIFCONF1_ENFSBITCLK0); - - val = BIT(AB8500_DIGIFCONF1_ENMASTGEN); - - switch (fmt & SND_SOC_DAIFMT_CLOCK_MASK) { - case SND_SOC_DAIFMT_CONT: /* continuous clock */ - dev_dbg(component->dev, "%s: IF0 Clock is continuous.\n", - __func__); - val |= BIT(AB8500_DIGIFCONF1_ENFSBITCLK0); - break; - case SND_SOC_DAIFMT_GATED: /* clock is gated */ - dev_dbg(component->dev, "%s: IF0 Clock is gated.\n", - __func__); - break; - default: - dev_err(component->dev, - "%s: ERROR: Unsupported clock mask (0x%x)!\n", - __func__, fmt & SND_SOC_DAIFMT_CLOCK_MASK); - return -EINVAL; - } - - snd_soc_component_update_bits(component, AB8500_DIGIFCONF1, mask, val); - - return 0; -} - static int ab8500_codec_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt) { - unsigned int mask; - unsigned int val; struct snd_soc_component *component = dai->component; - int status; + unsigned int conf1_mask, conf1_val = 0; + unsigned int conf2_mask, conf2_val = 0; + unsigned int conf3_mask, conf3_val = 0; + bool provider = false; + int ret; dev_dbg(component->dev, "%s: Enter (fmt = 0x%x)\n", __func__, fmt); - mask = BIT(AB8500_DIGIFCONF3_IF1DATOIF0AD) | + conf3_mask = BIT(AB8500_DIGIFCONF3_IF1DATOIF0AD) | BIT(AB8500_DIGIFCONF3_IF1CLKTOIF0CLK) | BIT(AB8500_DIGIFCONF3_IF0BFIFOEN) | BIT(AB8500_DIGIFCONF3_IF0MASTER); - val = 0; switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { case SND_SOC_DAIFMT_CBP_CFP: - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0 Master-mode: AB8500 provider.\n", __func__); - val |= BIT(AB8500_DIGIFCONF3_IF0MASTER); + conf3_val |= BIT(AB8500_DIGIFCONF3_IF0MASTER); + provider = true; break; case SND_SOC_DAIFMT_CBC_CFC: - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0 Master-mode: AB8500 consumer.\n", __func__); break; case SND_SOC_DAIFMT_CBC_CFP: case SND_SOC_DAIFMT_CBP_CFC: - dev_err(dai->component->dev, + dev_err(component->dev, "%s: ERROR: The device is either a provider or a consumer.\n", __func__); fallthrough; default: - dev_err(dai->component->dev, - "%s: ERROR: Unsupporter clocking mask 0x%x\n", + dev_err(component->dev, + "%s: ERROR: Unsupported clocking mask 0x%x\n", __func__, fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK); return -EINVAL; } - snd_soc_component_update_bits(component, AB8500_DIGIFCONF3, mask, val); - - /* Set clock gating */ - status = ab8500_codec_set_dai_clock_gate(component, fmt); - if (status) { - dev_err(dai->component->dev, - "%s: ERROR: Failed to set clock gate (%d).\n", - __func__, status); - return status; + conf1_mask = BIT(AB8500_DIGIFCONF1_ENMASTGEN) | + BIT(AB8500_DIGIFCONF1_ENFSBITCLK0); + switch (fmt & SND_SOC_DAIFMT_CLOCK_MASK) { + case SND_SOC_DAIFMT_CONT: + if (provider) + conf1_val = conf1_mask; + break; + case SND_SOC_DAIFMT_GATED: + if (provider) + conf1_val = BIT(AB8500_DIGIFCONF1_ENMASTGEN); + break; + default: + dev_err(component->dev, "%s: Unsupported clock mask 0x%x\n", + __func__, fmt & SND_SOC_DAIFMT_CLOCK_MASK); + return -EINVAL; } - /* Setting data transfer format */ - - mask = BIT(AB8500_DIGIFCONF2_IF0FORMAT0) | - BIT(AB8500_DIGIFCONF2_IF0FORMAT1) | - BIT(AB8500_DIGIFCONF2_FSYNC0P) | - BIT(AB8500_DIGIFCONF2_BITCLK0P); - val = 0; + conf2_mask = BIT(AB8500_DIGIFCONF2_IF0FORMAT0) | + BIT(AB8500_DIGIFCONF2_IF0FORMAT1) | + BIT(AB8500_DIGIFCONF2_IF0DEL) | + BIT(AB8500_DIGIFCONF2_FSYNC0P) | + BIT(AB8500_DIGIFCONF2_BITCLK0P); switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { case SND_SOC_DAIFMT_I2S: /* I2S mode */ - dev_dbg(dai->component->dev, "%s: IF0 Protocol: I2S\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT1); - ab8500_audio_set_bit_delay(dai, 0); + dev_dbg(component->dev, "%s: IF0 Protocol: I2S\n", __func__); + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT1) | + BIT(AB8500_DIGIFCONF2_IF0DEL); break; case SND_SOC_DAIFMT_DSP_A: /* L data MSB after FRM LRC */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0 Protocol: DSP A (TDM)\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT0); - ab8500_audio_set_bit_delay(dai, 1); + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT0) | + BIT(AB8500_DIGIFCONF2_IF0DEL); break; case SND_SOC_DAIFMT_DSP_B: /* L data MSB during FRM LRC */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0 Protocol: DSP B (TDM)\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT0); - ab8500_audio_set_bit_delay(dai, 0); + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT0); break; default: - dev_err(dai->component->dev, + dev_err(component->dev, "%s: ERROR: Unsupported format (0x%x)!\n", __func__, fmt & SND_SOC_DAIFMT_FORMAT_MASK); return -EINVAL; @@ -1897,39 +1725,50 @@ static int ab8500_codec_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt) switch (fmt & SND_SOC_DAIFMT_INV_MASK) { case SND_SOC_DAIFMT_NB_NF: /* normal bit clock + frame */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0: Normal bit clock, normal frame\n", __func__); break; case SND_SOC_DAIFMT_NB_IF: /* normal BCLK + inv FRM */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0: Normal bit clock, inverted frame\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_FSYNC0P); + conf2_val |= BIT(AB8500_DIGIFCONF2_FSYNC0P); break; case SND_SOC_DAIFMT_IB_NF: /* invert BCLK + nor FRM */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0: Inverted bit clock, normal frame\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_BITCLK0P); + conf2_val |= BIT(AB8500_DIGIFCONF2_BITCLK0P); break; case SND_SOC_DAIFMT_IB_IF: /* invert BCLK + FRM */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0: Inverted bit clock, inverted frame\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_FSYNC0P); - val |= BIT(AB8500_DIGIFCONF2_BITCLK0P); + conf2_val |= BIT(AB8500_DIGIFCONF2_FSYNC0P) | + BIT(AB8500_DIGIFCONF2_BITCLK0P); break; default: - dev_err(dai->component->dev, + dev_err(component->dev, "%s: ERROR: Unsupported INV mask 0x%x\n", __func__, fmt & SND_SOC_DAIFMT_INV_MASK); return -EINVAL; } - snd_soc_component_update_bits(component, AB8500_DIGIFCONF2, mask, val); + ret = snd_soc_component_update_bits(component, AB8500_DIGIFCONF3, + conf3_mask, conf3_val); + if (ret < 0) + return ret; - return 0; + ret = snd_soc_component_update_bits(component, AB8500_DIGIFCONF1, + conf1_mask, conf1_val); + if (ret < 0) + return ret; + + ret = snd_soc_component_update_bits(component, AB8500_DIGIFCONF2, + conf2_mask, conf2_val); + + return ret < 0 ? ret : 0; } static int ab8500_codec_set_dai_tdm_slot(struct snd_soc_dai *dai, @@ -1937,23 +1776,27 @@ static int ab8500_codec_set_dai_tdm_slot(struct snd_soc_dai *dai, int slots, int slot_width) { struct snd_soc_component *component = dai->component; - unsigned int val, mask, slot, slots_active; + unsigned int active_mask, clock_ratio, slot, value, ad_out, reg; + unsigned int tx_active, rx_active; + unsigned int conf1_val, conf2_val; + unsigned int mask; + int channel, ret; mask = BIT(AB8500_DIGIFCONF2_IF0WL0) | BIT(AB8500_DIGIFCONF2_IF0WL1); - val = 0; + conf2_val = 0; switch (slot_width) { case 16: break; case 20: - val |= BIT(AB8500_DIGIFCONF2_IF0WL0); + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0WL0); break; case 24: - val |= BIT(AB8500_DIGIFCONF2_IF0WL1); + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0WL1); break; case 32: - val |= BIT(AB8500_DIGIFCONF2_IF0WL1) | + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0WL1) | BIT(AB8500_DIGIFCONF2_IF0WL0); break; default: @@ -1962,27 +1805,11 @@ static int ab8500_codec_set_dai_tdm_slot(struct snd_soc_dai *dai, return -EINVAL; } - dev_dbg(dai->component->dev, "%s: IF0 slot-width: %d bits.\n", - __func__, slot_width); - snd_soc_component_update_bits(component, AB8500_DIGIFCONF2, mask, val); - - /* Setup TDM clocking according to slot count */ - dev_dbg(dai->component->dev, "%s: Slots, total: %d\n", __func__, slots); - mask = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0) | - BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); switch (slots) { case 2: - val = AB8500_MASK_NONE; - break; case 4: - val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0); - break; case 8: - val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); - break; case 16: - val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0) | - BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); break; default: dev_err(dai->component->dev, @@ -1990,92 +1817,134 @@ static int ab8500_codec_set_dai_tdm_slot(struct snd_soc_dai *dai, __func__, slots); return -EINVAL; } - snd_soc_component_update_bits(component, AB8500_DIGIFCONF1, mask, val); - - /* Setup TDM DA according to active tx slots */ - if (tx_mask & ~0xff) - return -EINVAL; - - mask = AB8500_DASLOTCONFX_SLTODAX_MASK; - tx_mask = tx_mask << AB8500_DA_DATA0_OFFSET; - slots_active = hweight32(tx_mask); - - dev_dbg(dai->component->dev, "%s: Slots, active, TX: %d\n", __func__, - slots_active); - - switch (slots_active) { - case 0: + clock_ratio = slots * slot_width; + switch (clock_ratio) { + case 32: + conf1_val = 0; break; - case 1: - slot = ffs(tx_mask); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF1, mask, slot); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF3, mask, slot); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF2, mask, slot); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF4, mask, slot); + case 64: + conf1_val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0); break; - case 2: - slot = ffs(tx_mask); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF1, mask, slot); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF3, mask, slot); - slot = fls(tx_mask); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF2, mask, slot); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF4, mask, slot); + case 128: + conf1_val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); break; - case 8: - dev_dbg(dai->component->dev, - "%s: In 8-channel mode DA-from-slot mapping is set manually.", - __func__); + case 256: + conf1_val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0) | + BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); break; default: - dev_err(dai->component->dev, - "%s: Unsupported number of active TX-slots (%d)!\n", - __func__, slots_active); + dev_err(component->dev, "%s: Unsupported BCLK ratio (%u)!\n", + __func__, clock_ratio); return -EINVAL; } - /* Setup TDM AD according to active RX-slots */ + active_mask = GENMASK(min(slots, 8) - 1, 0); + if ((tx_mask | rx_mask) & ~active_mask) { + dev_err(component->dev, "%s: Slot mask exceeds slot count\n", + __func__); + return -EINVAL; + } - if (rx_mask & ~0xff) + tx_active = hweight32(tx_mask); + rx_active = hweight32(rx_mask); + if (tx_active != 0 && tx_active != 1 && tx_active != 2 && + tx_active != 8) { + dev_err(component->dev, "%s: Unsupported active TX slots (%u)!\n", + __func__, tx_active); return -EINVAL; + } + if (rx_active != 0 && rx_active != 1 && rx_active != 2 && + rx_active != 8) { + dev_err(component->dev, "%s: Unsupported active RX slots (%u)!\n", + __func__, rx_active); + return -EINVAL; + } - rx_mask = rx_mask << AB8500_AD_DATA0_OFFSET; - slots_active = hweight32(rx_mask); + dev_dbg(component->dev, + "%s: %d slots of %d bits, TX active: %u, RX active: %u\n", + __func__, slots, slot_width, tx_active, rx_active); - dev_dbg(dai->component->dev, "%s: Slots, active, RX: %d\n", __func__, - slots_active); + ret = snd_soc_component_update_bits(component, AB8500_DIGIFCONF2, + mask, conf2_val); + if (ret < 0) + return ret; - switch (slots_active) { - case 0: - break; - case 1: - slot = ffs(rx_mask); - snd_soc_component_update_bits(component, AB8500_ADSLOTSEL(slot), - AB8500_MASK_SLOT(slot), - AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(AB8500_AD_OUT3, slot)); - break; - case 2: - slot = ffs(rx_mask); - snd_soc_component_update_bits(component, - AB8500_ADSLOTSEL(slot), - AB8500_MASK_SLOT(slot), - AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(AB8500_AD_OUT3, slot)); - slot = fls(rx_mask); - snd_soc_component_update_bits(component, - AB8500_ADSLOTSEL(slot), - AB8500_MASK_SLOT(slot), - AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(AB8500_AD_OUT2, slot)); - break; - case 8: - dev_dbg(dai->component->dev, - "%s: In 8-channel mode AD-to-slot mapping is set manually.", - __func__); - break; - default: - dev_err(dai->component->dev, - "%s: Unsupported number of active RX-slots (%d)!\n", - __func__, slots_active); - return -EINVAL; + mask = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0) | + BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); + ret = snd_soc_component_update_bits(component, AB8500_DIGIFCONF1, + mask, conf1_val); + if (ret < 0) + return ret; + + mask = AB8500_DASLOTCONFX_SLTODAX_MASK; + if (tx_active == 1 || tx_active == 2) { + slot = __ffs(tx_mask) + AB8500_DA_DATA0_OFFSET; + reg = AB8500_DASLOTCONF1; + ret = snd_soc_component_update_bits(component, reg, mask, slot); + if (ret < 0) + return ret; + reg = AB8500_DASLOTCONF3; + ret = snd_soc_component_update_bits(component, reg, mask, slot); + if (ret < 0) + return ret; + + if (tx_active == 2) + slot = __fls(tx_mask) + AB8500_DA_DATA0_OFFSET; + reg = AB8500_DASLOTCONF2; + ret = snd_soc_component_update_bits(component, reg, mask, slot); + if (ret < 0) + return ret; + reg = AB8500_DASLOTCONF4; + ret = snd_soc_component_update_bits(component, reg, mask, slot); + if (ret < 0) + return ret; + } else if (tx_active == 8) { + channel = 0; + for (slot = 0; slot < 8; slot++) { + if (!(tx_mask & BIT(slot))) + continue; + reg = AB8500_DASLOTCONF1 + channel++; + value = slot + AB8500_DA_DATA0_OFFSET; + ret = snd_soc_component_update_bits(component, reg, mask, value); + if (ret < 0) + return ret; + } + } + + if (rx_active == 1 || rx_active == 2) { + slot = __ffs(rx_mask) + AB8500_AD_DATA0_OFFSET; + value = AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(AB8500_AD_OUT3, + slot); + reg = AB8500_ADSLOTSEL(slot); + mask = AB8500_MASK_SLOT(slot); + ret = snd_soc_component_update_bits(component, reg, mask, value); + if (ret < 0) + return ret; + + if (rx_active == 2) { + slot = __fls(rx_mask) + AB8500_AD_DATA0_OFFSET; + value = AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(AB8500_AD_OUT2, + slot); + reg = AB8500_ADSLOTSEL(slot); + mask = AB8500_MASK_SLOT(slot); + ret = snd_soc_component_update_bits(component, reg, mask, value); + if (ret < 0) + return ret; + } + } else if (rx_active == 8) { + channel = 0; + for (slot = 0; slot < 8; slot++) { + if (!(rx_mask & BIT(slot))) + continue; + ad_out = AB8500_AD_OUT1 + channel++; + value = AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(ad_out, slot); + reg = AB8500_ADSLOTSEL(slot); + mask = AB8500_MASK_SLOT(slot); + ret = snd_soc_component_update_bits(component, reg, mask, value); + if (ret < 0) + return ret; + } } return 0; @@ -2172,10 +2041,8 @@ static void ab8500_codec_of_probe(struct device *dev, struct device_node *np, static int ab8500_codec_probe(struct snd_soc_component *component) { - struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component); struct device *dev = component->dev; struct device_node *np = dev->of_node; - struct ab8500_codec_drvdata *drvdata = dev_get_drvdata(dev); struct ab8500_codec_platform_data codec_pdata; int status; @@ -2183,6 +2050,13 @@ static int ab8500_codec_probe(struct snd_soc_component *component) ab8500_codec_of_probe(dev, np, &codec_pdata); + status = ab8500_audio_init_audioblock(component); + if (status < 0) { + dev_err(dev, "%s: failed to init audio-block (%d)!\n", + __func__, status); + return status; + } + status = ab8500_audio_setup_mics(component, &codec_pdata.amics); if (status < 0) { pr_err("%s: Failed to setup mics (%d)!\n", __func__, status); @@ -2195,23 +2069,12 @@ static int ab8500_codec_probe(struct snd_soc_component *component) return status; } - status = ab8500_audio_init_audioblock(component); - if (status < 0) { - dev_err(dev, "%s: failed to init audio-block (%d)!\n", - __func__, status); - return status; - } - /* Override HW-defaults */ snd_soc_component_write(component, AB8500_ANACONF5, BIT(AB8500_ANACONF5_HSAUTOEN)); snd_soc_component_write(component, AB8500_SHORTCIRCONF, BIT(AB8500_SHORTCIRCONF_HSZCDDIS)); - snd_soc_dapm_disable_pin(dapm, "ANC Configure Input"); - - mutex_init(&drvdata->ctrl_lock); - return status; } @@ -2240,7 +2103,6 @@ static int ab8500_codec_driver_probe(struct platform_device *pdev) GFP_KERNEL); if (!drvdata) return -ENOMEM; - drvdata->sid_status = SID_UNCONFIGURED; dev_set_drvdata(&pdev->dev, drvdata); drvdata->regmap = devm_regmap_init(&pdev->dev, NULL, &pdev->dev, diff --git a/sound/soc/codecs/adau1761.c b/sound/soc/codecs/adau1761.c index a70c46dd5d76f..27ce03b0fb91a 100644 --- a/sound/soc/codecs/adau1761.c +++ b/sound/soc/codecs/adau1761.c @@ -68,23 +68,33 @@ #define ADAU1761_FIRMWARE "adau1761.bin" static const struct reg_default adau1761_reg_defaults[] = { - { ADAU1761_DEJITTER, 0x03 }, + { ADAU17X1_CLOCK_CONTROL, 0x00 }, + { ADAU17X1_PLL_CONTROL, 0x00 }, { ADAU1761_DIGMIC_JACKDETECT, 0x00 }, + { ADAU17X1_REC_POWER_MGMT, 0x00 }, { ADAU1761_REC_MIXER_LEFT0, 0x00 }, { ADAU1761_REC_MIXER_LEFT1, 0x00 }, { ADAU1761_REC_MIXER_RIGHT0, 0x00 }, { ADAU1761_REC_MIXER_RIGHT1, 0x00 }, { ADAU1761_LEFT_DIFF_INPUT_VOL, 0x00 }, + { ADAU1761_RIGHT_DIFF_INPUT_VOL, 0x00 }, + { ADAU17X1_MICBIAS, 0x00 }, { ADAU1761_ALC_CTRL0, 0x00 }, { ADAU1761_ALC_CTRL1, 0x00 }, { ADAU1761_ALC_CTRL2, 0x00 }, { ADAU1761_ALC_CTRL3, 0x00 }, - { ADAU1761_RIGHT_DIFF_INPUT_VOL, 0x00 }, - { ADAU1761_PLAY_LR_MIXER_LEFT, 0x00 }, + { ADAU17X1_SERIAL_PORT0, 0x00 }, + { ADAU17X1_SERIAL_PORT1, 0x00 }, + { ADAU17X1_CONVERTER0, 0x00 }, + { ADAU17X1_CONVERTER1, 0x00 }, + { ADAU17X1_ADC_CONTROL, 0x00 }, + { ADAU17X1_LEFT_INPUT_DIGITAL_VOL, 0x00 }, + { ADAU17X1_RIGHT_INPUT_DIGITAL_VOL, 0x00 }, { ADAU1761_PLAY_MIXER_LEFT0, 0x00 }, { ADAU1761_PLAY_MIXER_LEFT1, 0x00 }, { ADAU1761_PLAY_MIXER_RIGHT0, 0x00 }, { ADAU1761_PLAY_MIXER_RIGHT1, 0x00 }, + { ADAU1761_PLAY_LR_MIXER_LEFT, 0x00 }, { ADAU1761_PLAY_LR_MIXER_RIGHT, 0x00 }, { ADAU1761_PLAY_MIXER_MONO, 0x00 }, { ADAU1761_PLAY_HP_LEFT_VOL, 0x00 }, @@ -93,20 +103,6 @@ static const struct reg_default adau1761_reg_defaults[] = { { ADAU1761_PLAY_LINE_RIGHT_VOL, 0x00 }, { ADAU1761_PLAY_MONO_OUTPUT_VOL, 0x00 }, { ADAU1761_POP_CLICK_SUPPRESS, 0x00 }, - { ADAU1761_JACK_DETECT_PIN, 0x00 }, - { ADAU1761_CLK_ENABLE0, 0x00 }, - { ADAU1761_CLK_ENABLE1, 0x00 }, - { ADAU17X1_CLOCK_CONTROL, 0x00 }, - { ADAU17X1_PLL_CONTROL, 0x00 }, - { ADAU17X1_REC_POWER_MGMT, 0x00 }, - { ADAU17X1_MICBIAS, 0x00 }, - { ADAU17X1_SERIAL_PORT0, 0x00 }, - { ADAU17X1_SERIAL_PORT1, 0x00 }, - { ADAU17X1_CONVERTER0, 0x00 }, - { ADAU17X1_CONVERTER1, 0x00 }, - { ADAU17X1_LEFT_INPUT_DIGITAL_VOL, 0x00 }, - { ADAU17X1_RIGHT_INPUT_DIGITAL_VOL, 0x00 }, - { ADAU17X1_ADC_CONTROL, 0x00 }, { ADAU17X1_PLAY_POWER_MGMT, 0x00 }, { ADAU17X1_DAC_CONTROL0, 0x00 }, { ADAU17X1_DAC_CONTROL1, 0x00 }, @@ -114,12 +110,16 @@ static const struct reg_default adau1761_reg_defaults[] = { { ADAU17X1_SERIAL_PORT_PAD, 0xaa }, { ADAU17X1_CONTROL_PORT_PAD0, 0xaa }, { ADAU17X1_CONTROL_PORT_PAD1, 0x00 }, + { ADAU1761_JACK_DETECT_PIN, 0x00 }, + { ADAU1761_DEJITTER, 0x03 }, { ADAU17X1_DSP_SAMPLING_RATE, 0x01 }, { ADAU17X1_SERIAL_INPUT_ROUTE, 0x00 }, { ADAU17X1_SERIAL_OUTPUT_ROUTE, 0x00 }, { ADAU17X1_DSP_ENABLE, 0x00 }, { ADAU17X1_DSP_RUN, 0x00 }, { ADAU17X1_SERIAL_SAMPLING_RATE, 0x00 }, + { ADAU1761_CLK_ENABLE0, 0x00 }, + { ADAU1761_CLK_ENABLE1, 0x00 }, }; static const DECLARE_TLV_DB_SCALE(adau1761_sing_in_tlv, -1500, 300, 1); diff --git a/sound/soc/codecs/aw87xxx/Kconfig b/sound/soc/codecs/aw87xxx/Kconfig new file mode 100644 index 0000000000000..bd0f208e2cfed --- /dev/null +++ b/sound/soc/codecs/aw87xxx/Kconfig @@ -0,0 +1,5 @@ +config SND_SOC_AW87XXX + tristate "SoC Audio for awinic AW87XXX Smart K PA" + depends on I2C + help + This option enables support for AW87XXX Smart K PA. diff --git a/sound/soc/codecs/aw87xxx/Makefile b/sound/soc/codecs/aw87xxx/Makefile new file mode 100644 index 0000000000000..d32f319a5b014 --- /dev/null +++ b/sound/soc/codecs/aw87xxx/Makefile @@ -0,0 +1,4 @@ +#for AWINIC AW87XXX Smart K PA +snd-soc-aw87xxx-objs := aw87xxx.o aw87xxx_device.o aw87xxx_monitor.o aw87xxx_bin_parse.o aw87xxx_dsp.o aw87xxx_acf_bin.o +obj-$(CONFIG_SND_SOC_AW87XXX) += snd-soc-aw87xxx.o + diff --git a/sound/soc/codecs/aw87xxx/aw87xxx.c b/sound/soc/codecs/aw87xxx/aw87xxx.c new file mode 100644 index 0000000000000..0699ec1c9f32a --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx.c @@ -0,0 +1,1603 @@ +/* + * aw87xxx.c aw87xxx pa module + * + * Copyright (c) 2021 AWINIC Technology CO., LTD + * + * Author: Barry + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aw87xxx.h" +#include "aw87xxx_device.h" +#include "aw87xxx_log.h" +#include "aw87xxx_monitor.h" +#include "aw87xxx_acf_bin.h" +#include "aw87xxx_bin_parse.h" +#include "aw87xxx_dsp.h" + +/***************************************************************** +* aw87xxx marco +******************************************************************/ +#define AW87XXX_I2C_NAME "aw87xxx_pa" +#define AW87XXX_DRIVER_VERSION "v2.7.0" +#define AW87XXX_FW_BIN_NAME "aw87xxx_acf.bin" +#define AW87XXX_PROF_MUSIC "Music" +/************************************************************************* + * aw87xxx variable + ************************************************************************/ +static LIST_HEAD(g_aw87xxx_list); +static DEFINE_MUTEX(g_aw87xxx_mutex_lock); +unsigned int g_aw87xxx_dev_cnt = 0; + +static const char *const aw87xxx_monitor_switch[] = {"Disable", "Enable"}; +static const char *const aw87xxx_spin_switch[] = {"spin_0", "spin_90", + "spin_180", "spin_270"}; +#if defined(AW_KERNEL_VER_OVER_6_16_1) +static void new_snd_soc_unregister_component(struct device *dev) +{ + return snd_soc_unregister_component_by_driver(dev, NULL); +} +static struct aw_componet_codec_ops aw_componet_codec_ops = { + .add_codec_controls = snd_soc_add_component_controls, + .unregister_codec = new_snd_soc_unregister_component, +}; +#elif defined(AW_KERNEL_VER_OVER_4_19_1) +static struct aw_componet_codec_ops aw_componet_codec_ops = { + .add_codec_controls = snd_soc_add_component_controls, + .unregister_codec = snd_soc_unregister_component, +}; +#else +static struct aw_componet_codec_ops aw_componet_codec_ops = { + .add_codec_controls = snd_soc_add_codec_controls, + .unregister_codec = snd_soc_unregister_codec, +}; +#endif + +enum smi_bus_type { + SMI_I2C, + SMI_SPI, + SMI_AUTO_DETECT, +}; + +struct smi_instance { + const char *type; + unsigned int flags; + int irq_idx; +}; + +struct smi_node { + enum smi_bus_type bus_type; + struct smi_instance instances[]; +}; + +/************************************************************************ + * + * aw87xxx device update profile + * + ************************************************************************/ +static int aw87xxx_power_down(struct aw87xxx *aw87xxx, char *profile) +{ + int ret = 0; + struct aw_prof_desc *prof_desc = NULL; + struct aw_prof_info *prof_info = &aw87xxx->acf_info.prof_info; + struct aw_data_container *data_container = NULL; + struct aw_device *aw_dev = &aw87xxx->aw_dev; + + AW_DEV_LOGD(aw87xxx->dev, "enter"); + + if (!prof_info->status) { + AW_DEV_LOGE(aw87xxx->dev, "profile_cfg not load"); + return -EINVAL; + } + + prof_desc = aw87xxx_acf_get_prof_desc_form_name(aw87xxx->dev, &aw87xxx->acf_info, profile); + if (prof_desc == NULL) + goto no_bin_pwr_off; + + if (!prof_desc->prof_st) + goto no_bin_pwr_off; + + + data_container = &prof_desc->data_container; + AW_DEV_LOGD(aw87xxx->dev, "get profile[%s] data len [%d]", + profile, data_container->len); + + if (aw_dev->hwen_status == AW_DEV_HWEN_OFF) { + AW_DEV_LOGI(aw87xxx->dev, "profile[%s] has already load ", profile); + } else { + if (aw_dev->ops.pwr_off_func) { + ret = aw_dev->ops.pwr_off_func(aw_dev, data_container); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "load profile[%s] failed ", profile); + goto pwr_off_failed; + } + } else { + ret = aw87xxx_dev_default_pwr_off(aw_dev, data_container); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "load profile[%s] failed ", profile); + goto pwr_off_failed; + } + } + } + + aw87xxx->current_profile = prof_desc->prof_name; + return 0; + +pwr_off_failed: +no_bin_pwr_off: + aw87xxx_dev_hw_pwr_ctrl(&aw87xxx->aw_dev, false); + aw87xxx->current_profile = aw87xxx->prof_off_name; + return ret; +} + +static int aw87xxx_power_on(struct aw87xxx *aw87xxx, char *profile) +{ + int ret = -EINVAL; + struct aw_prof_desc *prof_desc = NULL; + struct aw_prof_info *prof_info = &aw87xxx->acf_info.prof_info; + struct aw_data_container *data_container = NULL; + struct aw_device *aw_dev = &aw87xxx->aw_dev; + + AW_DEV_LOGD(aw87xxx->dev, "enter"); + + if (!prof_info->status) { + AW_DEV_LOGE(aw87xxx->dev, "profile_cfg not load"); + return -EINVAL; + } + + if (0 == strncmp(profile, aw87xxx->prof_off_name, AW_PROFILE_STR_MAX)) + return aw87xxx_power_down(aw87xxx, profile); + + prof_desc = aw87xxx_acf_get_prof_desc_form_name(aw87xxx->dev, &aw87xxx->acf_info, profile); + if (prof_desc == NULL) { + AW_DEV_LOGE(aw87xxx->dev, "not found [%s] parameter", profile); + return -EINVAL; + } + + if (!prof_desc->prof_st) { + AW_DEV_LOGE(aw87xxx->dev, "not found data container"); + return -EINVAL; + } + + data_container = &prof_desc->data_container; + AW_DEV_LOGD(aw87xxx->dev, "get profile[%s] data len [%d]", + profile, data_container->len); + + if (aw_dev->ops.pwr_on_func) { + ret = aw_dev->ops.pwr_on_func(aw_dev, data_container); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "load profile[%s] failed ", + profile); + return aw87xxx_power_down(aw87xxx, aw87xxx->prof_off_name); + } + } else { + ret = aw87xxx_dev_default_pwr_on(aw_dev, data_container); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "load profile[%s] failed ", + profile); + return aw87xxx_power_down(aw87xxx, aw87xxx->prof_off_name); + } + } + + aw87xxx->current_profile = prof_desc->prof_name; + AW_DEV_LOGD(aw87xxx->dev, "load profile[%s] succeed", profile); + + return 0; +} + + + +int aw87xxx_update_profile(struct aw87xxx *aw87xxx, char *profile) +{ + int ret = -1; + + AW_DEV_LOGD(aw87xxx->dev, "load profile[%s] enter", profile); + mutex_lock(&aw87xxx->reg_lock); + aw87xxx_monitor_stop(&aw87xxx->monitor); + if (0 == strncmp(profile, aw87xxx->prof_off_name, AW_PROFILE_STR_MAX)) { + ret = aw87xxx_power_down(aw87xxx, profile); + } else { + ret = aw87xxx_power_on(aw87xxx, profile); + if (!ret) + aw87xxx_monitor_start(&aw87xxx->monitor); + } + mutex_unlock(&aw87xxx->reg_lock); + + return ret; +} + +int aw87xxx_update_profile_esd(struct aw87xxx *aw87xxx, char *profile) +{ + int ret = -1; + + if (0 == strncmp(profile, aw87xxx->prof_off_name, AW_PROFILE_STR_MAX)) + ret = aw87xxx_power_down(aw87xxx, profile); + else + ret = aw87xxx_power_on(aw87xxx, profile); + + return ret; +} + +char *aw87xxx_show_current_profile(int dev_index) +{ + struct list_head *pos = NULL; + struct aw87xxx *aw87xxx = NULL; + + list_for_each(pos, &g_aw87xxx_list) { + aw87xxx = list_entry(pos, struct aw87xxx, list); + if (aw87xxx->dev_index == dev_index) { + AW_DEV_LOGI(aw87xxx->dev, "current profile is [%s]", + aw87xxx->current_profile); + return aw87xxx->current_profile; + } + } + + AW_LOGE("not found struct aw87xxx, dev_index = [%d]", dev_index); + return NULL; +} +EXPORT_SYMBOL(aw87xxx_show_current_profile); + +int aw87xxx_set_profile(int dev_index, char *profile) +{ + struct list_head *pos = NULL; + struct aw87xxx *aw87xxx = NULL; + + list_for_each(pos, &g_aw87xxx_list) { + aw87xxx = list_entry(pos, struct aw87xxx, list); + if (profile && aw87xxx->dev_index == dev_index) { + AW_DEV_LOGD(aw87xxx->dev, "set dev_index = %d, profile = %s", + dev_index, profile); + return aw87xxx_update_profile(aw87xxx, profile); + } + } + + AW_LOGE("not found struct aw87xxx, dev_index = [%d]", dev_index); + return -EINVAL; +} +EXPORT_SYMBOL(aw87xxx_set_profile); + +int aw87xxx_set_profile_by_id(int dev_index, int profile_id) +{ + char *profile = NULL; + + profile = aw87xxx_ctos_get_prof_name(profile_id); + if (profile == NULL) { + AW_LOGE("aw87xxx, dev_index[%d] profile[%d] not support!", + dev_index, profile_id); + return -EINVAL; + } + + AW_LOGI("aw87xxx, dev_index[%d] set profile[%s] by id[%d]", + dev_index, profile, profile_id); + return aw87xxx_set_profile(dev_index, profile); +} +EXPORT_SYMBOL(aw87xxx_set_profile_by_id); + +/**************************************************************************** + * + * aw87xxx Kcontrols + * + ****************************************************************************/ +static int aw87xxx_profile_switch_info(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_info *uinfo) +{ + int count = 0; + char *name = NULL; + char *profile_name = NULL; + struct aw87xxx *aw87xxx = (struct aw87xxx *)kcontrol->private_value; + + if (aw87xxx == NULL) { + AW_LOGE("get struct aw87xxx failed"); + return -EINVAL; + } + + uinfo->type = SNDRV_CTL_ELEM_TYPE_ENUMERATED; + uinfo->count = 1; + + /*make sure have prof */ + count = aw87xxx_acf_get_profile_count(aw87xxx->dev, &aw87xxx->acf_info); + if (count <= 0) { + uinfo->value.enumerated.items = 0; + AW_DEV_LOGE(aw87xxx->dev, "get count[%d] failed", count); + return 0; + } + + uinfo->value.enumerated.items = count; + if (uinfo->value.enumerated.item >= count) + uinfo->value.enumerated.item = count - 1; + + name = uinfo->value.enumerated.name; + count = uinfo->value.enumerated.item; + profile_name = aw87xxx_acf_get_prof_name_form_index(aw87xxx->dev, + &aw87xxx->acf_info, count); + if (profile_name == NULL) { + strscpy(uinfo->value.enumerated.name, "NULL", + strlen("NULL") + 1); + return 0; + } + + strscpy(name, profile_name, sizeof(uinfo->value.enumerated.name)); + + return 0; +} + +static int aw87xxx_profile_switch_put(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + int ret = -1; + char *profile_name = NULL; + int index = ucontrol->value.integer.value[0]; + struct aw87xxx *aw87xxx = (struct aw87xxx *)kcontrol->private_value; + struct acf_bin_info *acf_info = NULL; + + if (aw87xxx == NULL) { + AW_LOGE("get struct aw87xxx failed"); + return -EINVAL; + } + + acf_info = &aw87xxx->acf_info; + + profile_name = aw87xxx_acf_get_prof_name_form_index(aw87xxx->dev, acf_info, index); + if (!profile_name) { + AW_DEV_LOGE(aw87xxx->dev, "not found profile name,index=[%d]", + index); + return -EINVAL; + } + + AW_DEV_LOGI(aw87xxx->dev, "set profile [%s]", profile_name); + + ret = aw87xxx_update_profile(aw87xxx, profile_name); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "set dev_index[%d] profile failed, profile = %s", + aw87xxx->dev_index, profile_name); + return ret; + } + + return 0; +} + +static int aw87xxx_profile_switch_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + int index = 0; + char *profile; + struct aw87xxx *aw87xxx = (struct aw87xxx *)kcontrol->private_value; + + if (aw87xxx == NULL) { + AW_LOGE("get struct aw87xxx failed"); + return -EINVAL; + } + + if (!aw87xxx->current_profile) { + AW_DEV_LOGE(aw87xxx->dev, "profile not init"); + return -EINVAL; + } + + profile = aw87xxx->current_profile; + AW_DEV_LOGI(aw87xxx->dev, "current profile:[%s]", + aw87xxx->current_profile); + + + index = aw87xxx_acf_get_prof_index_form_name(aw87xxx->dev, + &aw87xxx->acf_info, aw87xxx->current_profile); + if (index < 0) { + AW_DEV_LOGE(aw87xxx->dev, "get profile index failed"); + return index; + } + + ucontrol->value.integer.value[0] = index; + + return 0; +} + +static int aw87xxx_vmax_get_info(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.min = INT_MIN; + uinfo->value.integer.max = AW_VMAX_MAX; + + return 0; +} + +static int aw87xxx_vmax_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + int ret = -1; + int vmax_val = 0; + struct aw87xxx *aw87xxx = (struct aw87xxx *)kcontrol->private_value; + + if (aw87xxx == NULL) { + AW_LOGE("get struct aw87xxx failed"); + return -EINVAL; + } + + ret = aw87xxx_monitor_no_dsp_get_vmax(&aw87xxx->monitor, &vmax_val); + if (ret < 0) + return ret; + + ucontrol->value.integer.value[0] = vmax_val; + AW_DEV_LOGI(aw87xxx->dev, "get vmax = [0x%x]", vmax_val); + + return 0; +} + +static int aw87xxx_monitor_switch_info(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_info *uinfo) +{ + int count; + + uinfo->type = SNDRV_CTL_ELEM_TYPE_ENUMERATED; + uinfo->count = 1; + count = ARRAY_SIZE(aw87xxx_monitor_switch); + + uinfo->value.enumerated.items = count; + + if (uinfo->value.enumerated.item >= count) + uinfo->value.enumerated.item = count - 1; + + strscpy(uinfo->value.enumerated.name, + aw87xxx_monitor_switch[uinfo->value.enumerated.item], + strlen(aw87xxx_monitor_switch[uinfo->value.enumerated.item]) + 1); + + return 0; +} + +static int aw87xxx_monitor_switch_put(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + uint32_t ctrl_value = ucontrol->value.integer.value[0]; + struct aw87xxx *aw87xxx = (struct aw87xxx *)kcontrol->private_value; + struct aw_monitor *aw_monitor = &aw87xxx->monitor; + int ret = -1; + + ret = aw87xxx_dev_monitor_switch_set(aw_monitor, ctrl_value); + if (ret) + return ret; + + return 0; +} + +static int aw87xxx_monitor_switch_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct aw87xxx *aw87xxx = (struct aw87xxx *)kcontrol->private_value; + struct aw_monitor *aw_monitor = &aw87xxx->monitor; + + ucontrol->value.integer.value[0] = aw_monitor->monitor_hdr.monitor_switch; + + AW_DEV_LOGI(aw87xxx->dev, "monitor switch is %ld", ucontrol->value.integer.value[0]); + return 0; +} + +static int aw87xxx_spin_switch_info(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_info *uinfo) +{ + int count; + + uinfo->type = SNDRV_CTL_ELEM_TYPE_ENUMERATED; + uinfo->count = 1; + count = ARRAY_SIZE(aw87xxx_spin_switch); + + uinfo->value.enumerated.items = count; + + if (uinfo->value.enumerated.item >= count) + uinfo->value.enumerated.item = count - 1; + + strscpy(uinfo->value.enumerated.name, + aw87xxx_spin_switch[uinfo->value.enumerated.item], + strlen(aw87xxx_spin_switch[uinfo->value.enumerated.item]) + 1); + + return 0; +} + +static int aw87xxx_spin_switch_put(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + uint32_t ctrl_value = 0; + int ret = 0; + struct aw87xxx *aw87xxx = (struct aw87xxx *)kcontrol->private_value; + ctrl_value = ucontrol->value.integer.value[0]; + + ret = aw87xxx_dsp_set_spin(ctrl_value); + if (ret) { + AW_DEV_LOGE(aw87xxx->dev, "write spin failed"); + return ret; + } + AW_DEV_LOGD(aw87xxx->dev, "write spin done ctrl_value=%d", ctrl_value); + return 0; +} + +static int aw87xxx_spin_switch_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct aw87xxx *aw87xxx = (struct aw87xxx *)kcontrol->private_value; + + ucontrol->value.integer.value[0] = aw87xxx_dsp_get_spin(); + AW_DEV_LOGD(aw87xxx->dev, "current spin is %ld", ucontrol->value.integer.value[0]); + + return 0; +} + + +static int aw87xxx_kcontrol_dynamic_create(struct aw87xxx *aw87xxx, + void *codec) +{ + struct snd_kcontrol_new *aw87xxx_kcontrol = NULL; + aw_snd_soc_codec_t *soc_codec = (aw_snd_soc_codec_t *)codec; + char *kctl_name[AW87XXX_PRIVATE_KCONTROL_NUM]; + int kcontrol_num = AW87XXX_PRIVATE_KCONTROL_NUM; + int ret = -1; + + AW_DEV_LOGD(aw87xxx->dev, "enter"); + aw87xxx->codec = soc_codec; + + aw87xxx_kcontrol = devm_kzalloc(aw87xxx->dev, + sizeof(struct snd_kcontrol_new) * kcontrol_num, + GFP_KERNEL); + if (aw87xxx_kcontrol == NULL) { + AW_DEV_LOGE(aw87xxx->dev, "aw87xxx_kcontrol devm_kzalloc failed"); + return -ENOMEM; + } + + kctl_name[0] = devm_kzalloc(aw87xxx->dev, AW_NAME_BUF_MAX, + GFP_KERNEL); + if (kctl_name[0] == NULL) + return -ENOMEM; + + snprintf(kctl_name[0], AW_NAME_BUF_MAX, "aw87xxx_profile_switch_%d", + aw87xxx->dev_index); + + aw87xxx_kcontrol[0].name = kctl_name[0]; + aw87xxx_kcontrol[0].iface = SNDRV_CTL_ELEM_IFACE_MIXER; + aw87xxx_kcontrol[0].info = aw87xxx_profile_switch_info; + aw87xxx_kcontrol[0].get = aw87xxx_profile_switch_get; + aw87xxx_kcontrol[0].put = aw87xxx_profile_switch_put; + aw87xxx_kcontrol[0].private_value = (unsigned long)aw87xxx; + + kctl_name[1] = devm_kzalloc(aw87xxx->codec->dev, AW_NAME_BUF_MAX, + GFP_KERNEL); + if (kctl_name[1] == NULL) + return -ENOMEM; + + snprintf(kctl_name[1], AW_NAME_BUF_MAX, "aw87xxx_vmax_get_%d", + aw87xxx->dev_index); + + aw87xxx_kcontrol[1].name = kctl_name[1]; + aw87xxx_kcontrol[1].iface = SNDRV_CTL_ELEM_IFACE_MIXER; + aw87xxx_kcontrol[1].access = SNDRV_CTL_ELEM_ACCESS_READ; + aw87xxx_kcontrol[1].info = aw87xxx_vmax_get_info; + aw87xxx_kcontrol[1].get = aw87xxx_vmax_get; + aw87xxx_kcontrol[1].private_value = (unsigned long)aw87xxx; + + kctl_name[2] = devm_kzalloc(aw87xxx->codec->dev, AW_NAME_BUF_MAX, + GFP_KERNEL); + if (kctl_name[2] == NULL) + return -ENOMEM; + + snprintf(kctl_name[2], AW_NAME_BUF_MAX, "aw87xxx_monitor_switch_%d", + aw87xxx->dev_index); + + aw87xxx_kcontrol[2].name = kctl_name[2]; + aw87xxx_kcontrol[2].iface = SNDRV_CTL_ELEM_IFACE_MIXER; + aw87xxx_kcontrol[2].info = aw87xxx_monitor_switch_info; + aw87xxx_kcontrol[2].get = aw87xxx_monitor_switch_get; + aw87xxx_kcontrol[2].put = aw87xxx_monitor_switch_put; + aw87xxx_kcontrol[2].private_value = (unsigned long)aw87xxx; + + ret = aw_componet_codec_ops.add_codec_controls(aw87xxx->codec, + aw87xxx_kcontrol, kcontrol_num); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "add codec controls failed, ret = %d", + ret); + return ret; + } + + AW_DEV_LOGI(aw87xxx->dev, "add codec controls[%s,%s,%s]", + aw87xxx_kcontrol[0].name, + aw87xxx_kcontrol[1].name, + aw87xxx_kcontrol[2].name); + + return 0; +} + +static int aw87xxx_public_kcontrol_create(struct aw87xxx *aw87xxx, + void *codec) +{ + struct snd_kcontrol_new *aw87xxx_kcontrol = NULL; + aw_snd_soc_codec_t *soc_codec = (aw_snd_soc_codec_t *)codec; + char *kctl_name[AW87XXX_PUBLIC_KCONTROL_NUM]; + int kcontrol_num = AW87XXX_PUBLIC_KCONTROL_NUM; + int ret = -1; + + AW_DEV_LOGD(aw87xxx->dev, "enter"); + aw87xxx->codec = soc_codec; + + aw87xxx_kcontrol = devm_kzalloc(aw87xxx->dev, + sizeof(struct snd_kcontrol_new) * kcontrol_num, + GFP_KERNEL); + if (aw87xxx_kcontrol == NULL) { + AW_DEV_LOGE(aw87xxx->dev, "aw87xxx_kcontrol devm_kzalloc failed"); + return -ENOMEM; + } + + kctl_name[0] = devm_kzalloc(aw87xxx->dev, AW_NAME_BUF_MAX, + GFP_KERNEL); + if (kctl_name[0] == NULL) + return -ENOMEM; + + snprintf(kctl_name[0], AW_NAME_BUF_MAX, "aw87xxx_spin_switch"); + + aw87xxx_kcontrol[0].name = kctl_name[0]; + aw87xxx_kcontrol[0].iface = SNDRV_CTL_ELEM_IFACE_MIXER; + aw87xxx_kcontrol[0].info = aw87xxx_spin_switch_info; + aw87xxx_kcontrol[0].get = aw87xxx_spin_switch_get; + aw87xxx_kcontrol[0].put = aw87xxx_spin_switch_put; + aw87xxx_kcontrol[0].private_value = (unsigned long)aw87xxx; + + ret = aw_componet_codec_ops.add_codec_controls(aw87xxx->codec, + aw87xxx_kcontrol, kcontrol_num); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "add codec controls failed, ret = %d", + ret); + return ret; + } + + AW_DEV_LOGI(aw87xxx->dev, "add public codec controls[%s]", + aw87xxx_kcontrol[0].name); + + return 0; +} + +/**************************************************************************** + * + *aw87xxx kcontrol create + * + ****************************************************************************/ +int aw87xxx_add_codec_controls(void *codec) +{ + struct list_head *pos = NULL; + struct aw87xxx *aw87xxx = NULL; + int ret = -1; + + list_for_each(pos, &g_aw87xxx_list) { + aw87xxx = list_entry(pos, struct aw87xxx, list); + ret = aw87xxx_kcontrol_dynamic_create(aw87xxx, codec); + if (ret < 0) + return ret; + + if (aw87xxx->dev_index == 0) { + ret = aw87xxx_public_kcontrol_create(aw87xxx, codec); + if (ret < 0) + return ret; + } + } + + return 0; +} +EXPORT_SYMBOL(aw87xxx_add_codec_controls); + + +/**************************************************************************** + * + * aw87xxx firmware cfg load + * + ***************************************************************************/ +static void aw87xxx_fw_cfg_free(struct aw87xxx *aw87xxx) +{ + AW_DEV_LOGD(aw87xxx->dev, "enter"); + aw87xxx_acf_profile_free(aw87xxx->dev, &aw87xxx->acf_info); + aw87xxx_monitor_cfg_free(&aw87xxx->monitor); +} + +static int aw87xxx_init_default_prof(struct aw87xxx *aw87xxx) +{ + char *profile = NULL; + + profile = aw87xxx_acf_get_prof_off_name(aw87xxx->dev, &aw87xxx->acf_info); + if (profile == NULL) { + AW_DEV_LOGE(aw87xxx->dev, "get profile off name failed"); + return -EINVAL; + } + + snprintf(aw87xxx->prof_off_name, AW_PROFILE_STR_MAX, "%s", profile); + aw87xxx->current_profile = profile; + AW_DEV_LOGI(aw87xxx->dev, "init profile name [%s]", + aw87xxx->current_profile); + + return 0; +} + +static void aw87xxx_fw_load_retry(struct aw87xxx *aw87xxx) +{ + struct acf_bin_info *acf_info = &aw87xxx->acf_info; + int ram_timer_val = 2000; + + AW_DEV_LOGD(aw87xxx->dev, "failed to read [%s]", + aw87xxx->fw_name); + + if (acf_info->load_count < AW_LOAD_FW_RETRIES) { + AW_DEV_LOGD(aw87xxx->dev, + "restart hrtimer to load firmware"); + schedule_delayed_work(&aw87xxx->fw_load_work, + msecs_to_jiffies(ram_timer_val)); + } else { + acf_info->load_count = 0; + AW_DEV_LOGE(aw87xxx->dev, + "can not load firmware,please check name or file exists"); + return; + } + acf_info->load_count++; +} + +static void aw87xxx_fw_load(const struct firmware *fw, void *context) +{ + int ret = -1; + struct aw87xxx *aw87xxx = context; + struct acf_bin_info *acf_info = &aw87xxx->acf_info; + + AW_DEV_LOGD(aw87xxx->dev, "enter"); + + if (!fw) { + aw87xxx_fw_load_retry(aw87xxx); + return; + } + + AW_DEV_LOGD(aw87xxx->dev, "loaded %s - size: %ld", + aw87xxx->fw_name, (u_long)(fw ? fw->size : 0)); + + mutex_lock(&aw87xxx->reg_lock); + acf_info->fw_data = vmalloc(fw->size); + if (!acf_info->fw_data) { + AW_DEV_LOGE(aw87xxx->dev, "fw_data kzalloc memory failed"); + goto exit_vmalloc_failed; + } + memset(acf_info->fw_data, 0, fw->size); + memcpy(acf_info->fw_data, fw->data, fw->size); + acf_info->fw_size = fw->size; + + ret = aw87xxx_acf_parse(aw87xxx->dev, &aw87xxx->acf_info); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "fw_data parse failed"); + goto exit_acf_parse_failed; + } + + ret = aw87xxx_init_default_prof(aw87xxx); + if (ret < 0) { + aw87xxx_fw_cfg_free(aw87xxx); + goto exit_acf_parse_failed; + } + + AW_DEV_LOGI(aw87xxx->dev, "acf parse succeed"); + mutex_unlock(&aw87xxx->reg_lock); + release_firmware(fw); + // Updating profile to "Music" because the firmware is set to "off" during init + aw87xxx_update_profile(aw87xxx, AW87XXX_PROF_MUSIC); + + return; + +exit_acf_parse_failed: +exit_vmalloc_failed: + release_firmware(fw); + mutex_unlock(&aw87xxx->reg_lock); +} + +static void aw87xxx_fw_load_work_routine(struct work_struct *work) +{ + struct aw87xxx *aw87xxx = container_of(work, + struct aw87xxx, fw_load_work.work); + struct aw_prof_info *prof_info = &aw87xxx->acf_info.prof_info; + + AW_DEV_LOGD(aw87xxx->dev, "enter"); + + if (prof_info->status == AW_ACF_WAIT) { + request_firmware_nowait(THIS_MODULE, +// FW_ACTION_HOTPLUG, + FW_ACTION_UEVENT, + aw87xxx->fw_name, + aw87xxx->dev, + GFP_KERNEL, aw87xxx, + aw87xxx_fw_load); + } +} + +static const struct dmi_system_id firmware_names[] = { + { + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "AYANEO"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "KUN"), + }, + .driver_data = (void *)"aw87xxx_acf_kun.bin", + }, + { + .matches = { + DMI_MATCH(DMI_PRODUCT_NAME, "Loki MiniPro"), + }, + .driver_data = (void *)"aw87xxx_acf_minipro.bin", + }, + { + .matches = { + DMI_MATCH(DMI_PRODUCT_NAME, "AIR 1S"), + }, + .driver_data = (void *)"aw87xxx_acf_air1s.bin", + }, + { + .matches = { + DMI_MATCH(DMI_PRODUCT_NAME, "AIR Plus"), + }, + .driver_data = (void *)"aw87xxx_acf_airplus.bin", + }, + { + .matches = { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "AYANEO"), + DMI_MATCH(DMI_PRODUCT_NAME, "FLIP"), + }, + .driver_data = (void *)"aw87xxx_acf_flip.bin", + }, + { + .matches = { + DMI_MATCH(DMI_PRODUCT_NAME, "NEO-01"), + }, + .driver_data = (void *)"aw87xxx_acf_orangepi.bin", + }, + {} +}; + +static void aw87xxx_fw_load_init(struct aw87xxx *aw87xxx) +{ +#ifdef AW_CFG_UPDATE_DELAY + int cfg_timer_val = AW_CFG_UPDATE_DELAY_TIMER; +#else + int cfg_timer_val = 0; +#endif + const struct dmi_system_id *fwname_sysid = dmi_first_match(firmware_names); + const char *fwname = fwname_sysid ? + fwname_sysid->driver_data : AW87XXX_FW_BIN_NAME; + + AW_DEV_LOGI(aw87xxx->dev, "loading firmware name: [%s]", fwname); + snprintf(aw87xxx->fw_name, AW87XXX_FW_NAME_MAX, "%s", fwname); + aw87xxx_acf_init(&aw87xxx->aw_dev, &aw87xxx->acf_info, aw87xxx->dev_index); + + INIT_DELAYED_WORK(&aw87xxx->fw_load_work, aw87xxx_fw_load_work_routine); + schedule_delayed_work(&aw87xxx->fw_load_work, + msecs_to_jiffies(cfg_timer_val)); +} + +/**************************************************************************** + * + *aw87xxx attribute node + * + ****************************************************************************/ +static ssize_t aw87xxx_attr_get_reg(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + int ret = 0; + unsigned int i = 0; + unsigned char reg_val = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_device *aw_dev = &aw87xxx->aw_dev; + + mutex_lock(&aw87xxx->reg_lock); + for (i = 0; i < aw_dev->reg_max_addr; i++) { + if (!(aw_dev->reg_access[i] & AW_DEV_REG_RD_ACCESS)) + continue; + ret = aw87xxx_dev_i2c_read_byte(&aw87xxx->aw_dev, i, ®_val); + if (ret < 0) { + len += snprintf(buf + len, PAGE_SIZE - len, + "read reg [0x%x] failed\n", i); + AW_DEV_LOGE(aw87xxx->dev, "read reg [0x%x] failed", i); + } else { + len += snprintf(buf + len, PAGE_SIZE - len, + "reg:0x%02X=0x%02X\n", i, reg_val); + AW_DEV_LOGD(aw87xxx->dev, "reg:0x%02X=0x%02X", + i, reg_val); + } + } + mutex_unlock(&aw87xxx->reg_lock); + + return len; +} + +static ssize_t aw87xxx_attr_set_reg(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t len) +{ + unsigned int databuf[2] = { 0 }; + int ret = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + + mutex_lock(&aw87xxx->reg_lock); + if (sscanf(buf, "0x%x 0x%x", &databuf[0], &databuf[1]) == 2) { + if (databuf[0] >= aw87xxx->aw_dev.reg_max_addr) { + AW_DEV_LOGE(aw87xxx->dev, "set reg[0x%x] error,is out of reg_addr_max[0x%x]", + databuf[0], aw87xxx->aw_dev.reg_max_addr); + mutex_unlock(&aw87xxx->reg_lock); + return -EINVAL; + } + + ret = aw87xxx_dev_i2c_write_byte(&aw87xxx->aw_dev, + databuf[0], databuf[1]); + if (ret < 0) + AW_DEV_LOGE(aw87xxx->dev, "set [0x%x]=0x%x failed", + databuf[0], databuf[1]); + else + AW_DEV_LOGD(aw87xxx->dev, "set [0x%x]=0x%x succeed", + databuf[0], databuf[1]); + } else { + AW_DEV_LOGE(aw87xxx->dev, "i2c write cmd input error"); + } + mutex_unlock(&aw87xxx->reg_lock); + + return len; +} + +static ssize_t aw87xxx_attr_get_profile(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + unsigned int i = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_prof_info *prof_info = &aw87xxx->acf_info.prof_info; + + if (!prof_info->status) { + len += snprintf(buf + len, PAGE_SIZE - len, + "profile_cfg not load\n"); + return len; + } + + AW_DEV_LOGI(aw87xxx->dev, "current profile:[%s]", aw87xxx->current_profile); + + for (i = 0; i < prof_info->count; i++) { + if (!strncmp(aw87xxx->current_profile, prof_info->prof_name_list[i], + AW_PROFILE_STR_MAX)) + len += snprintf(buf + len, PAGE_SIZE - len, + ">%s\n", prof_info->prof_name_list[i]); + else + len += snprintf(buf + len, PAGE_SIZE - len, + " %s\n", prof_info->prof_name_list[i]); + } + + return len; +} + +static ssize_t aw87xxx_attr_set_profile(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t len) +{ + char profile[AW_PROFILE_STR_MAX] = {0}; + int ret = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + + if (strlen(buf) > AW_PROFILE_STR_MAX) { + AW_DEV_LOGE(aw87xxx->dev, "input profile_str_len is out of max[%d]", + AW_PROFILE_STR_MAX); + return -EINVAL; + } + + if (sscanf(buf, "%s", profile) == 1) { + AW_DEV_LOGD(aw87xxx->dev, "set profile [%s]", profile); + ret = aw87xxx_update_profile(aw87xxx, profile); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "set profile[%s] failed", + profile); + return ret; + } + } + + return len; +} + +static ssize_t aw87xxx_attr_get_hwen(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + int hwen = aw87xxx->aw_dev.hwen_status; + + if (hwen >= AW_DEV_HWEN_INVALID) + len += snprintf(buf + len, PAGE_SIZE - len, "hwen_status: invalid\n"); + else if (hwen == AW_DEV_HWEN_ON) + len += snprintf(buf + len, PAGE_SIZE - len, "hwen_status: on\n"); + else if (hwen == AW_DEV_HWEN_OFF) + len += snprintf(buf + len, PAGE_SIZE - len, "hwen_status: off\n"); + + return len; +} + +static ssize_t aw87xxx_attr_set_hwen(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t len) +{ + int ret = -1; + unsigned int state; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + + ret = kstrtouint(buf, 0, &state); + if (ret) { + AW_DEV_LOGE(aw87xxx->dev, "fail to channelge str to int"); + return ret; + } + + mutex_lock(&aw87xxx->reg_lock); + if (state == AW_DEV_HWEN_OFF) + aw87xxx_dev_hw_pwr_ctrl(&aw87xxx->aw_dev, false); /*OFF*/ + else if (state == AW_DEV_HWEN_ON) + aw87xxx_dev_hw_pwr_ctrl(&aw87xxx->aw_dev, true); /*ON*/ + else + AW_DEV_LOGE(aw87xxx->dev, "input [%d] error, hwen_on=[%d],hwen_off=[%d]", + state, AW_DEV_HWEN_ON, AW_DEV_HWEN_OFF); + mutex_unlock(&aw87xxx->reg_lock); + return len; +} + +int aw87xxx_awrw_write(struct aw87xxx *aw87xxx, + const char *buf, size_t count) +{ + int i = 0, ret = -1; + char *data_buf = NULL; + int buf_len = 0; + int temp_data = 0; + int data_str_size = 0; + char *reg_data; + struct aw_i2c_packet *packet = &aw87xxx->i2c_packet; + + AW_DEV_LOGD(aw87xxx->dev, "enter"); + /* one addr or one data string Composition of Contains two bytes of symbol(0X)*/ + /* and two byte of hexadecimal data*/ + data_str_size = 2 + 2 * AWRW_DATA_BYTES; + + /* The buf includes the first address of the register to be written and all data */ + buf_len = AWRW_ADDR_BYTES + packet->reg_num * AWRW_DATA_BYTES; + AW_DEV_LOGI(aw87xxx->dev, "buf_len = %d,reg_num = %d", buf_len, packet->reg_num); + data_buf = vmalloc(buf_len); + if (data_buf == NULL) { + AW_DEV_LOGE(aw87xxx->dev, "alloc memory failed"); + return -ENOMEM; + } + memset(data_buf, 0, buf_len); + + data_buf[0] = packet->reg_addr; + reg_data = data_buf + 1; + + AW_DEV_LOGD(aw87xxx->dev, "reg_addr: 0x%02x", data_buf[0]); + + /*ag:0x00 0x01 0x01 0x01 0x01 0x00\x0a*/ + for (i = 0; i < packet->reg_num; i++) { + ret = sscanf(buf + AWRW_HDR_LEN + 1 + i * (data_str_size + 1), + "0x%x", &temp_data); + if (ret != 1) { + AW_DEV_LOGE(aw87xxx->dev, "sscanf failed,ret=%d", ret); + vfree(data_buf); + data_buf = NULL; + return ret; + } + reg_data[i] = temp_data; + AW_DEV_LOGD(aw87xxx->dev, "[%d] : 0x%02x", i, reg_data[i]); + } + + mutex_lock(&aw87xxx->reg_lock); + ret = i2c_master_send(aw87xxx->aw_dev.i2c, data_buf, buf_len); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "write failed"); + vfree(data_buf); + data_buf = NULL; + return -EFAULT; + } + mutex_unlock(&aw87xxx->reg_lock); + + vfree(data_buf); + data_buf = NULL; + + AW_DEV_LOGD(aw87xxx->dev, "down"); + return 0; +} + +static int aw87xxx_awrw_data_check(struct aw87xxx *aw87xxx, + int *data, size_t count) +{ + struct aw_i2c_packet *packet = &aw87xxx->i2c_packet; + int req_data_len = 0; + int act_data_len = 0; + int data_str_size = 0; + + if ((data[AWRW_HDR_ADDR_BYTES] != AWRW_ADDR_BYTES) || + (data[AWRW_HDR_DATA_BYTES] != AWRW_DATA_BYTES)) { + AW_DEV_LOGE(aw87xxx->dev, "addr_bytes [%d] or data_bytes [%d] unsupport", + data[AWRW_HDR_ADDR_BYTES], data[AWRW_HDR_DATA_BYTES]); + return -EINVAL; + } + + /* one data string Composition of Contains two bytes of symbol(0x)*/ + /* and two byte of hexadecimal data*/ + data_str_size = 2 + 2 * AWRW_DATA_BYTES; + act_data_len = count - AWRW_HDR_LEN - 1; + + /* There is a comma(,) or space between each piece of data */ + if (data[AWRW_HDR_WR_FLAG] == AWRW_FLAG_WRITE) { + /*ag:0x00 0x01 0x01 0x01 0x01 0x00\x0a*/ + req_data_len = (data_str_size + 1) * packet->reg_num; + if (req_data_len > act_data_len) { + AW_DEV_LOGE(aw87xxx->dev, "data_len checkfailed,requeset data_len [%d],actaul data_len [%d]", + req_data_len, act_data_len); + return -EINVAL; + } + } + + return 0; +} + +/* flag addr_bytes data_bytes reg_num reg_addr*/ +static int aw87xxx_awrw_parse_buf(struct aw87xxx *aw87xxx, + const char *buf, size_t count, int *wr_status) +{ + int data[AWRW_HDR_MAX] = {0}; + struct aw_i2c_packet *packet = &aw87xxx->i2c_packet; + int ret = -1; + + if (sscanf(buf, "0x%02x 0x%02x 0x%02x 0x%02x 0x%02x", + &data[AWRW_HDR_WR_FLAG], &data[AWRW_HDR_ADDR_BYTES], + &data[AWRW_HDR_DATA_BYTES], &data[AWRW_HDR_REG_NUM], + &data[AWRW_HDR_REG_ADDR]) == 5) { + + packet->reg_addr = data[AWRW_HDR_REG_ADDR]; + packet->reg_num = data[AWRW_HDR_REG_NUM]; + *wr_status = data[AWRW_HDR_WR_FLAG]; + ret = aw87xxx_awrw_data_check(aw87xxx, data, count); + if (ret < 0) + return ret; + + return 0; + } + + return -EINVAL; +} + +static ssize_t aw87xxx_attr_awrw_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_i2c_packet *packet = &aw87xxx->i2c_packet; + int wr_status = 0; + int ret = -1; + + if (count < AWRW_HDR_LEN) { + AW_DEV_LOGE(aw87xxx->dev, "data count too smaller, please check write format"); + AW_DEV_LOGE(aw87xxx->dev, "string %s,count=%ld", + buf, (u_long)count); + return -EINVAL; + } + + AW_DEV_LOGI(aw87xxx->dev, "string:[%s],count=%ld", buf, (u_long)count); + ret = aw87xxx_awrw_parse_buf(aw87xxx, buf, count, &wr_status); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "can not parse string"); + return ret; + } + + if (wr_status == AWRW_FLAG_WRITE) { + ret = aw87xxx_awrw_write(aw87xxx, buf, count); + if (ret < 0) + return ret; + } else if (wr_status == AWRW_FLAG_READ) { + packet->status = AWRW_I2C_ST_READ; + AW_DEV_LOGI(aw87xxx->dev, "read_cmd:reg_addr[0x%02x], reg_num[%d]", + packet->reg_addr, packet->reg_num); + } else { + AW_DEV_LOGE(aw87xxx->dev, "please check str format, unsupport read_write_status: %d", + wr_status); + return -EINVAL; + } + + return count; +} + +static ssize_t aw87xxx_attr_awrw_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_i2c_packet *packet = &aw87xxx->i2c_packet; + int data_len = 0; + size_t len = 0; + int ret = -1, i = 0; + char *reg_data = NULL; + + if (packet->status != AWRW_I2C_ST_READ) { + AW_DEV_LOGE(aw87xxx->dev, "please write read cmd first"); + return -EINVAL; + } + + data_len = AWRW_DATA_BYTES * packet->reg_num; + reg_data = (char *)vmalloc(data_len); + if (reg_data == NULL) { + AW_DEV_LOGE(aw87xxx->dev, "memory alloc failed"); + ret = -EINVAL; + goto exit; + } + + mutex_lock(&aw87xxx->reg_lock); + ret = aw87xxx_dev_i2c_read_msg(&aw87xxx->aw_dev, packet->reg_addr, + (char *)reg_data, data_len); + if (ret < 0) { + ret = -EFAULT; + mutex_unlock(&aw87xxx->reg_lock); + goto exit; + } + mutex_unlock(&aw87xxx->reg_lock); + + AW_DEV_LOGI(aw87xxx->dev, "reg_addr 0x%02x, reg_num %d", + packet->reg_addr, packet->reg_num); + + for (i = 0; i < data_len; i++) { + len += snprintf(buf + len, PAGE_SIZE - len, + "0x%02x,", reg_data[i]); + AW_DEV_LOGI(aw87xxx->dev, "0x%02x", reg_data[i]); + } + + ret = len; + +exit: + if (reg_data) { + vfree(reg_data); + reg_data = NULL; + } + packet->status = AWRW_I2C_ST_NONE; + return ret; +} + +static ssize_t aw87xxx_drv_ver_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + + len += snprintf(buf + len, PAGE_SIZE - len, + "driver_ver: %s \n", AW87XXX_DRIVER_VERSION); + + return len; +} + +static DEVICE_ATTR(reg, S_IWUSR | S_IRUGO, + aw87xxx_attr_get_reg, aw87xxx_attr_set_reg); +static DEVICE_ATTR(profile, S_IWUSR | S_IRUGO, + aw87xxx_attr_get_profile, aw87xxx_attr_set_profile); +static DEVICE_ATTR(hwen, S_IWUSR | S_IRUGO, + aw87xxx_attr_get_hwen, aw87xxx_attr_set_hwen); +static DEVICE_ATTR(awrw, S_IWUSR | S_IRUGO, + aw87xxx_attr_awrw_show, aw87xxx_attr_awrw_store); +static DEVICE_ATTR(drv_ver, S_IRUGO, aw87xxx_drv_ver_show, NULL); + +static struct attribute *aw87xxx_attributes[] = { + &dev_attr_reg.attr, + &dev_attr_profile.attr, + &dev_attr_hwen.attr, + &dev_attr_awrw.attr, + &dev_attr_drv_ver.attr, + NULL +}; + +static struct attribute_group aw87xxx_attribute_group = { + .attrs = aw87xxx_attributes +}; + +/**************************************************************************** + * + *aw87xxx device probe + * + ****************************************************************************/ +static const struct acpi_gpio_params reset_gpio = { 0, 0, false }; +static const struct acpi_gpio_mapping reset_acpi_gpios[] = { + { "reset-gpios", &reset_gpio, 1 }, + { } +}; + +static struct aw87xxx *aw87xxx_malloc_init(struct i2c_client *client) +{ + struct aw87xxx *aw87xxx = NULL; + + aw87xxx = devm_kzalloc(&client->dev, sizeof(struct aw87xxx), + GFP_KERNEL); + if (aw87xxx == NULL) { + AW_DEV_LOGE(&client->dev, "failed to devm_kzalloc aw87xxx"); + return NULL; + } + memset(aw87xxx, 0, sizeof(struct aw87xxx)); + + aw87xxx->dev = &client->dev; + aw87xxx->aw_dev.dev = &client->dev; + aw87xxx->aw_dev.i2c_bus = client->adapter->nr; + aw87xxx->aw_dev.i2c_addr = client->addr; + aw87xxx->aw_dev.i2c = client; + aw87xxx->aw_dev.hwen_status = false; + aw87xxx->aw_dev.reg_access = NULL; + aw87xxx->aw_dev.hwen_status = AW_DEV_HWEN_INVALID; + aw87xxx->off_bin_status = AW87XXX_NO_OFF_BIN; + aw87xxx->codec = NULL; + aw87xxx->current_profile = aw87xxx->prof_off_name; + + mutex_init(&aw87xxx->reg_lock); + + AW_DEV_LOGI(&client->dev, "Driver struct alloc and mutex init done, devinfo: i2c_bus=%u, i2c_addr=%x", client->adapter->nr, client->addr); + return aw87xxx; +} + +static int aw87xxx_i2c_probe(struct i2c_client *client) +{ + struct device_node *dev_node = client->dev.of_node; + const struct smi_node *node; + struct acpi_device *adev = ACPI_COMPANION(&client->dev); + struct aw87xxx *aw87xxx = NULL; + struct gpio_desc *gpiod = NULL; + struct i2c_board_info board_info = {}; + char i2c_name[32]; + int ret = -1; + int acpi_dev_count = 0; + + /* aw87xxx Get APCI I2C device count */ + if(g_aw87xxx_dev_cnt == 0){ + acpi_dev_count = i2c_acpi_client_count(adev); + AW_DEV_LOGI(&client->dev, "I2C_ACPI_CLIENT_COUNT returned [%d]", acpi_dev_count); + } + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { + AW_DEV_LOGE(&client->dev, "check_functionality failed"); + ret = -ENODEV; + goto exit_check_functionality_failed; + } + + /* aw87xxx i2c_dev struct init */ + aw87xxx = aw87xxx_malloc_init(client); + if (aw87xxx == NULL) + goto exit_malloc_init_failed; + + i2c_set_clientdata(client, aw87xxx); + + aw87xxx_device_parse_port_id_dt(&aw87xxx->aw_dev); + aw87xxx_device_parse_topo_id_dt(&aw87xxx->aw_dev); + + /* aw87xxx Get ACPI GPIO */ + + if (g_aw87xxx_dev_cnt == 0){ + ret = devm_acpi_dev_add_driver_gpios(aw87xxx->dev, reset_acpi_gpios); + if(ret){ + AW_DEV_LOGE(aw87xxx->dev, "Unable to add GPIO mapping table"); + goto exit_device_init_failed; + } + + gpiod = devm_gpiod_get(aw87xxx->dev, "reset", GPIOD_OUT_LOW); + if (gpiod == NULL){ + AW_DEV_LOGE(aw87xxx->dev, "Gpiod returned NULL failing gracefully."); + goto exit_device_init_failed; + } + + if (IS_ERR(gpiod)){ + AW_DEV_LOGE(aw87xxx->dev, "Get gpiod failed."); + goto exit_device_init_failed; + } + + aw87xxx->aw_dev.rst_gpio = desc_to_gpio(gpiod); + aw87xxx->aw_dev.hwen_status = AW_DEV_HWEN_OFF; + AW_DEV_LOGI(aw87xxx->dev, "reset gpio[%x] parse succeed", aw87xxx->aw_dev.rst_gpio); + + if (gpio_is_valid(aw87xxx->aw_dev.rst_gpio)) { + ret = devm_gpio_request_one(aw87xxx->dev, aw87xxx->aw_dev.rst_gpio, GPIOF_OUT_INIT_LOW, "aw87xxx_reset"); + if ((ret < 0) && (ret != -EBUSY)) { + AW_DEV_LOGE(aw87xxx->dev, "reset request failed, returned [%d]", ret); + goto exit_device_init_failed; + } + }else{ + /*Disabling RESET GPIO*/ + AW_DEV_LOGI(aw87xxx->dev, "no reset gpio provided, hardware reset unavailable"); + aw87xxx->aw_dev.rst_gpio = AW_NO_RESET_GPIO; + aw87xxx->aw_dev.hwen_status = AW_DEV_HWEN_INVALID; + } + + } + + /*hw power on PA*/ + if(g_aw87xxx_dev_cnt == 0) { + aw87xxx_dev_hw_pwr_ctrl(&aw87xxx->aw_dev, true); + } + + /* aw87xxx devices private attributes init */ + ret = aw87xxx_dev_init(&aw87xxx->aw_dev); + if (ret < 0) + goto exit_device_init_failed; + + /*product register reset */ + aw87xxx_dev_soft_reset(&aw87xxx->aw_dev); + + /*hw power off */ + if(g_aw87xxx_dev_cnt == 0) { + aw87xxx_dev_hw_pwr_ctrl(&aw87xxx->aw_dev, false); + } + + /* create debug attrbute nodes */ + ret = sysfs_create_group(&aw87xxx->dev->kobj, &aw87xxx_attribute_group); + if (ret < 0) + AW_DEV_LOGE(aw87xxx->dev, "failed to create sysfs nodes, will not allowed to use"); + + /* cfg_load init */ + aw87xxx_fw_load_init(aw87xxx); + + /*monitor init*/ + aw87xxx_monitor_init(aw87xxx->dev, &aw87xxx->monitor, dev_node); + + /*add device to total list */ + mutex_lock(&g_aw87xxx_mutex_lock); + g_aw87xxx_dev_cnt++; + list_add(&aw87xxx->list, &g_aw87xxx_list); + aw87xxx->dev_index = g_aw87xxx_dev_cnt; + + mutex_unlock(&g_aw87xxx_mutex_lock); + AW_DEV_LOGI(aw87xxx->dev, "succeed, dev_index=[%d], g_aw87xxx_dev_cnt= [%d]", + aw87xxx->dev_index, g_aw87xxx_dev_cnt); + + AW_DEV_LOGI(aw87xxx->dev, "acpi_c=[%d] dev_c=[%d]", acpi_dev_count, g_aw87xxx_dev_cnt); + + /* Attempt to add other I2C AMPs */ + if ((acpi_dev_count > 1) && (g_aw87xxx_dev_cnt == 1)){ + /* power on the chip */ + aw87xxx_dev_hw_pwr_ctrl(&aw87xxx->aw_dev, true); + + node = device_get_match_data(aw87xxx->dev); + memset(&board_info, 0, sizeof(board_info)); + strscpy(board_info.type, client->name, I2C_NAME_SIZE); + snprintf(i2c_name, sizeof(i2c_name), "%s.%d", client->name, 1); + board_info.dev_name = i2c_name; + + aw87xxx_i2c_probe(i2c_acpi_new_device_by_fwnode(acpi_fwnode_handle(adev), 1, &board_info)); + + } + + return 0; + +exit_device_init_failed: + AW_DEV_LOGE(aw87xxx->dev, "pa init failed"); + + devm_kfree(&client->dev, aw87xxx); + aw87xxx = NULL; +exit_malloc_init_failed: +exit_check_functionality_failed: + return ret; +} + +static void aw87xxx_i2c_remove(struct i2c_client *client) +{ + struct aw87xxx *aw87xxx = i2c_get_clientdata(client); + + aw87xxx_monitor_exit(&aw87xxx->monitor); + + /*rm attr node*/ + sysfs_remove_group(&aw87xxx->dev->kobj, &aw87xxx_attribute_group); + + aw87xxx_fw_cfg_free(aw87xxx); + + mutex_lock(&g_aw87xxx_mutex_lock); + g_aw87xxx_dev_cnt--; + list_del(&aw87xxx->list); + mutex_unlock(&g_aw87xxx_mutex_lock); + + devm_kfree(&client->dev, aw87xxx); + aw87xxx = NULL; + +// return 0; +} + +static void aw87xxx_i2c_shutdown(struct i2c_client *client) +{ + struct aw87xxx *aw87xxx = i2c_get_clientdata(client); + + AW_DEV_LOGI(&client->dev, "enter"); + + /*soft and hw power off*/ + aw87xxx_update_profile(aw87xxx, aw87xxx->prof_off_name); +} + + +static int aw87xxx_runtime_suspend(struct device *dev) +{ + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + + AW_DEV_LOGI(aw87xxx->dev, "Suspending..."); + + // soft and hw power off + aw87xxx_update_profile(aw87xxx, aw87xxx->prof_off_name); + + return 0; +} + +static int aw87xxx_runtime_resume(struct device *dev) +{ + struct list_head *pos = NULL; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + + // Power on PA + if (aw87xxx->dev_index == 1) + aw87xxx_dev_hw_pwr_ctrl(&aw87xxx->aw_dev, true); + + // Set profile to Music + list_for_each_prev(pos, &g_aw87xxx_list) { + aw87xxx = list_entry(pos, struct aw87xxx, list); + AW_DEV_LOGI(aw87xxx->dev, "Resuming..."); + + mutex_lock(&aw87xxx->reg_lock); + aw87xxx_power_on(aw87xxx, AW87XXX_PROF_MUSIC); + mutex_unlock(&aw87xxx->reg_lock); + + } + + return 0; +} + +static SIMPLE_DEV_PM_OPS(aw87xxx_pm_ops, aw87xxx_runtime_suspend, aw87xxx_runtime_resume); + +static const struct acpi_device_id aw87xxx_acpi_match[] = { + { "AWDZ8830", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, aw87xxx_acpi_match); + +// This is not necessary if the acpi match probes correctly. This is needed for userspace `new_device() functionality +static const struct i2c_device_id aw87xxx_i2c_id[] = { + {AW87XXX_I2C_NAME, 0}, + {}, +}; + +static struct i2c_driver aw87xxx_i2c_driver = { + .driver = { + .owner = THIS_MODULE, + .name = AW87XXX_I2C_NAME, + .acpi_match_table = aw87xxx_acpi_match, + .pm = &aw87xxx_pm_ops, + }, + .probe = aw87xxx_i2c_probe, + .remove = aw87xxx_i2c_remove, + .shutdown = aw87xxx_i2c_shutdown, + .id_table = aw87xxx_i2c_id, +}; + +module_i2c_driver(aw87xxx_i2c_driver) + +MODULE_AUTHOR(""); +MODULE_DESCRIPTION("awinic aw87xxx pa driver"); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/codecs/aw87xxx/aw87xxx.h b/sound/soc/codecs/aw87xxx/aw87xxx.h new file mode 100644 index 0000000000000..9638819cdc7ef --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx.h @@ -0,0 +1,130 @@ +#ifndef __AW87XXX_H__ +#define __AW87XXX_H__ +#include +#include +#include +#include + +#include "aw87xxx_device.h" +#include "aw87xxx_monitor.h" +#include "aw87xxx_acf_bin.h" + +#define AW_CFG_UPDATE_DELAY +#define AW_CFG_UPDATE_DELAY_TIMER (3000) + +#define AW87XXX_NO_OFF_BIN (0) +#define AW87XXX_OFF_BIN_OK (1) + +#define AW87XXX_PRIVATE_KCONTROL_NUM (3) +#define AW87XXX_PUBLIC_KCONTROL_NUM (1) + +#define AW_I2C_RETRIES (5) +#define AW_I2C_RETRY_DELAY (2) +#define AW_I2C_READ_MSG_NUM (2) + +#define AW87XXX_FW_NAME_MAX (64) +#define AW_NAME_BUF_MAX (64) +#define AW_LOAD_FW_RETRIES (3) + +#define AW_DEV_REG_RD_ACCESS (1 << 0) +#define AW_DEV_REG_WR_ACCESS (1 << 1) + +#define AWRW_ADDR_BYTES (1) +#define AWRW_DATA_BYTES (1) +#define AWRW_HDR_LEN (24) + +/*********************************************************** + * + * aw87xxx codec control compatible with kernel 4.19 + * + ***********************************************************/ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 16, 1) +#define AW_KERNEL_VER_OVER_6_16_1 +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 1) +#define AW_KERNEL_VER_OVER_4_19_1 +#endif + +#ifdef AW_KERNEL_VER_OVER_4_19_1 +typedef struct snd_soc_component aw_snd_soc_codec_t; +#else +typedef struct snd_soc_codec aw_snd_soc_codec_t; +#endif + +struct aw_componet_codec_ops { + int (*add_codec_controls)(aw_snd_soc_codec_t *codec, + const struct snd_kcontrol_new *controls, unsigned int num_controls); + void (*unregister_codec)(struct device *dev); +}; + + +/******************************************** + * + * aw87xxx devices attributes + * + *******************************************/ +enum { + AWRW_FLAG_WRITE = 0, + AWRW_FLAG_READ, +}; + +enum { + AWRW_I2C_ST_NONE = 0, + AWRW_I2C_ST_READ, + AWRW_I2C_ST_WRITE, +}; + +enum { + AWRW_HDR_WR_FLAG = 0, + AWRW_HDR_ADDR_BYTES, + AWRW_HDR_DATA_BYTES, + AWRW_HDR_REG_NUM, + AWRW_HDR_REG_ADDR, + AWRW_HDR_MAX, +}; + +struct aw_i2c_packet { + char status; + unsigned int reg_num; + unsigned int reg_addr; + char *reg_data; +}; + + +/******************************************** + * + * aw87xxx device struct + * + *******************************************/ +struct aw87xxx { + char fw_name[AW87XXX_FW_NAME_MAX]; + int32_t dev_index; + char *current_profile; + char prof_off_name[AW_PROFILE_STR_MAX]; + uint32_t off_bin_status; + struct device *dev; + + struct mutex reg_lock; + struct aw_device aw_dev; + struct aw_i2c_packet i2c_packet; + + struct delayed_work fw_load_work; + struct acf_bin_info acf_info; + + aw_snd_soc_codec_t *codec; + + struct list_head list; + + struct aw_monitor monitor; +}; + +int aw87xxx_update_profile(struct aw87xxx *aw87xxx, char *profile); +int aw87xxx_update_profile_esd(struct aw87xxx *aw87xxx, char *profile); + +char *aw87xxx_show_current_profile(int dev_index); +int aw87xxx_set_profile(int dev_index, char *profile); +int aw87xxx_set_profile_by_id(int dev_index, int profile_id); +int aw87xxx_add_codec_controls(void *codec); +int aw87xxx_awrw_write(struct aw87xxx *aw87xxx, const char *buf, size_t count); +#endif diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_acf_bin.c b/sound/soc/codecs/aw87xxx/aw87xxx_acf_bin.c new file mode 100644 index 0000000000000..0de7085cb3456 --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_acf_bin.c @@ -0,0 +1,1558 @@ +/* + * aw87xxx_acf_bin.c + * + * Copyright (c) 2021 AWINIC Technology CO., LTD + * + * Author: Barry + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aw87xxx.h" +#include "aw87xxx_acf_bin.h" +#include "aw87xxx_monitor.h" +#include "aw87xxx_log.h" +#include "aw87xxx_bin_parse.h" + +/************************************************************************* + * + *Table corresponding to customized profile ids to profile names + * + *************************************************************************/ +enum aw_customers_profile_id { + AW_CTOS_PROFILE_OFF = 0, + AW_CTOS_PROFILE_MUSIC, + AW_CTOS_PROFILE_VOICE, + AW_CTOS_PROFILE_VOIP, + AW_CTOS_PROFILE_RINGTONE, + AW_CTOS_PROFILE_RINGTONE_HS, + AW_CTOS_PROFILE_LOWPOWER, + AW_CTOS_PROFILE_BYPASS, + AW_CTOS_PROFILE_MMI, + AW_CTOS_PROFILE_FM, + AW_CTOS_PROFILE_NOTIFICATION, + AW_CTOS_PROFILE_RECEIVER, + AW_CTOS_PROFILE_MAX, +}; + +static char *g_ctos_profile_name[AW_PROFILE_MAX] = { + [AW_CTOS_PROFILE_OFF] = "Off", + [AW_CTOS_PROFILE_MUSIC] = "Music", + [AW_CTOS_PROFILE_VOICE] = "Voice", + [AW_CTOS_PROFILE_VOIP] = "Voip", + [AW_CTOS_PROFILE_RINGTONE] = "Ringtone", + [AW_CTOS_PROFILE_RINGTONE_HS] = "Ringtone_hs", + [AW_CTOS_PROFILE_LOWPOWER] = "Lowpower", + [AW_CTOS_PROFILE_BYPASS] = "Bypass", + [AW_CTOS_PROFILE_MMI] = "Mmi", + [AW_CTOS_PROFILE_FM] = "Fm", + [AW_CTOS_PROFILE_NOTIFICATION] = "Notification", + [AW_CTOS_PROFILE_RECEIVER] = "Receiver", +}; + + +char *aw87xxx_ctos_get_prof_name(int profile_id) +{ + if (profile_id < 0 || profile_id >= AW_CTOS_PROFILE_MAX) + return NULL; + else + return g_ctos_profile_name[profile_id]; +} + + +static char *g_profile_name[] = {"Music", "Voice", "Voip", + "Ringtone", "Ringtone_hs", "Lowpower", "Bypass", "Mmi", + "Fm", "Notification", "Receiver", "Off"}; + +static char *g_power_off_name[] = {"Off", "OFF", "off", "oFF", "power_down"}; + +static char *aw_get_prof_name(int profile) +{ + if (profile < 0 || profile >= AW_PROFILE_MAX) + return "NULL"; + else + return g_profile_name[profile]; +} + +/************************************************************************* + * + *acf check + * + *************************************************************************/ +static int aw_crc8_check(const unsigned char *data, unsigned int data_size) + +{ + unsigned char crc_value = 0x00; + unsigned char *pdata; + int i; + unsigned char pdatabuf = 0; + + pdata = (unsigned char *)data; + + while (data_size--) { + pdatabuf = *pdata++; + for (i = 0; i < 8; i++) { + if ((crc_value ^ (pdatabuf)) & 0x01) { + crc_value ^= 0x18; + crc_value >>= 1; + crc_value |= 0x80; + } else { + crc_value >>= 1; + } + pdatabuf >>= 1; + } + } + + return (int)crc_value; +} + +static int aw_check_file_id(struct device *dev, + char *fw_data, int32_t file_id) +{ + int32_t *acf_file_id = NULL; + + acf_file_id = (int32_t *)fw_data; + if (*acf_file_id != file_id) { + AW_DEV_LOGE(dev, "file id [%x] check failed", *acf_file_id); + return -ENFILE; + } + + return 0; +} + +static int aw_check_header_size(struct device *dev, + char *fw_data, size_t fw_size) +{ + if (fw_size < sizeof(struct aw_acf_hdr)) { + AW_DEV_LOGE(dev, "acf size check failed,size less-than aw_acf_hdr"); + return -ENOEXEC; + } + + return 0; +} + +/*************************************************************************** + * V0.0.0.1 version acf check + **************************************************************************/ +static int aw_check_ddt_size_v_0_0_0_1(struct device *dev, char *fw_data) +{ + struct aw_acf_hdr *acf_hdr = (struct aw_acf_hdr *)fw_data; + struct aw_acf_dde *acf_dde = NULL; + + acf_dde = (struct aw_acf_dde *)(fw_data + acf_hdr->ddt_offset); + + /* check ddt_size in acf_header is aqual to ddt_num multiply by dde_size */ + if (acf_hdr->ddt_size != acf_hdr->dde_num * sizeof(struct aw_acf_dde)) { + AW_DEV_LOGE(dev, "acf ddt size check failed"); + return -EINVAL; + } + + return 0; +} + +static int aw_check_data_size_v_0_0_0_1(struct device *dev, + char *fw_data, size_t fw_size) +{ + int i = 0; + size_t data_size = 0; + struct aw_acf_hdr *acf_hdr = NULL; + struct aw_acf_dde *acf_dde = NULL; + + acf_hdr = (struct aw_acf_hdr *)fw_data; + acf_dde = (struct aw_acf_dde *)(fw_data + acf_hdr->ddt_offset); + + for (i = 0; i < acf_hdr->dde_num; ++i) { + if (acf_dde[i].data_size % 2) { + AW_DEV_LOGE(dev, "acf dde[%d].data_size[%d],dev_name[%s],data_type[%d], data_size check failed", + i, acf_dde[i].data_size, acf_dde[i].dev_name, + acf_dde[i].data_type); + return -EINVAL; + } + data_size += acf_dde[i].data_size; + } + + /* Verify that the file size is equal to the header size plus */ + /* the table size and data size */ + if (fw_size != data_size + sizeof(struct aw_acf_hdr) + acf_hdr->ddt_size) { + AW_DEV_LOGE(dev, "acf size check failed"); + AW_DEV_LOGE(dev, "fw_size=%ld,hdr_size and ddt size and data size =%ld", + (u_long)fw_size, (u_long)(data_size + sizeof(struct aw_acf_hdr) + + acf_hdr->ddt_size)); + return -EINVAL; + } + + return 0; +} + +static int aw_check_data_crc_v_0_0_0_1(struct device *dev, char *fw_data) +{ + int i = 0; + size_t crc_val = 0; + char *data = NULL; + struct aw_acf_hdr *acf_hdr = NULL; + struct aw_acf_dde *acf_dde = NULL; + + acf_hdr = (struct aw_acf_hdr *)fw_data; + acf_dde = (struct aw_acf_dde *)(fw_data + acf_hdr->ddt_offset); + + for (i = 0; i < acf_hdr->dde_num; ++i) { + data = fw_data + acf_dde[i].data_offset; + crc_val = aw_crc8_check(data, acf_dde[i].data_size); + if (crc_val != acf_dde[i].data_crc) { + AW_DEV_LOGE(dev, "acf dde_crc check failed"); + return -EINVAL; + } + } + + return 0; +} + +static int aw_check_profile_id_v_0_0_0_1(struct device *dev, char *fw_data) +{ + int i = 0; + struct aw_acf_hdr *acf_hdr = NULL; + struct aw_acf_dde *acf_dde = NULL; + + acf_hdr = (struct aw_acf_hdr *)fw_data; + acf_dde = (struct aw_acf_dde *)(fw_data + acf_hdr->ddt_offset); + + for (i = 0; i < acf_hdr->dde_num; ++i) { + if (acf_dde[i].data_type == AW_MONITOR) + continue; + if (acf_dde[i].dev_profile > AW_PROFILE_MAX) { + AW_DEV_LOGE(dev, "parse profile_id[%d] failed", acf_dde[i].dev_profile); + return -EINVAL; + } + } + + return 0; +} +static int aw_check_data_v_0_0_0_1(struct device *dev, + char *fw_data, size_t size) +{ + int ret = -1; + + /* check file type id is awinic acf file */ + ret = aw_check_file_id(dev, fw_data, AW_ACF_FILE_ID); + if (ret < 0) + return ret; + + /* check ddt_size in header is equal to all ddt aize */ + ret = aw_check_ddt_size_v_0_0_0_1(dev, fw_data); + if (ret < 0) + return ret; + + /* Verify that the file size is equal to the header size plus */ + /* the table size and data size */ + ret = aw_check_data_size_v_0_0_0_1(dev, fw_data, size); + if (ret < 0) + return ret; + + /* check crc in is equal to dde data crc */ + ret = aw_check_data_crc_v_0_0_0_1(dev, fw_data); + if (ret < 0) + return ret; + + /* check profile id is in profile_id_max */ + ret = aw_check_profile_id_v_0_0_0_1(dev, fw_data); + if (ret < 0) + return ret; + + AW_DEV_LOGI(dev, "acf fimware check succeed"); + + return 0; +} + +/*************************************************************************** + * V1.0.0.0 version acf chack + **************************************************************************/ +static int aw_check_ddt_size_v_1_0_0_0(struct device *dev, char *fw_data) +{ + struct aw_acf_hdr *acf_hdr = (struct aw_acf_hdr *)fw_data; + struct aw_acf_dde_v_1_0_0_0 *acf_dde = NULL; + + acf_dde = (struct aw_acf_dde_v_1_0_0_0 *)(fw_data + acf_hdr->ddt_offset); + + /* check ddt_size in acf_header is aqual to ddt_num multiply by dde_size */ + if (acf_hdr->ddt_size != acf_hdr->dde_num * sizeof(struct aw_acf_dde_v_1_0_0_0)) { + AW_DEV_LOGE(dev, "acf ddt size check failed"); + return -EINVAL; + } + + return 0; +} + +static int aw_check_data_size_v_1_0_0_0(struct device *dev, + char *fw_data, size_t fw_size) +{ + int i = 0; + size_t data_size = 0; + struct aw_acf_hdr *acf_hdr = NULL; + struct aw_acf_dde_v_1_0_0_0 *acf_dde = NULL; + + acf_hdr = (struct aw_acf_hdr *)fw_data; + acf_dde = (struct aw_acf_dde_v_1_0_0_0 *)(fw_data + acf_hdr->ddt_offset); + + for (i = 0; i < acf_hdr->dde_num; ++i) { + if (acf_dde[i].data_size % 2) { + AW_DEV_LOGE(dev, "acf dde[%d].data_size[%d],dev_name[%s],data_type[%d], data_size check failed", + i, acf_dde[i].data_size, acf_dde[i].dev_name, + acf_dde[i].data_type); + return -EINVAL; + } + data_size += acf_dde[i].data_size; + } + + /* Verify that the file size is equal to the header size plus */ + /* the table size and data size */ + if (fw_size != data_size + sizeof(struct aw_acf_hdr) + acf_hdr->ddt_size) { + AW_DEV_LOGE(dev, "acf size check failed"); + AW_DEV_LOGE(dev, "fw_size=%ld,hdr_size and ddt size and data size =%ld", + (u_long)fw_size, (u_long)(data_size + sizeof(struct aw_acf_hdr) + + acf_hdr->ddt_size)); + return -EINVAL; + } + + return 0; +} + +static int aw_check_data_crc_v_1_0_0_0(struct device *dev, char *fw_data) +{ + int i = 0; + size_t crc_val = 0; + char *data = NULL; + struct aw_acf_hdr *acf_hdr = NULL; + struct aw_acf_dde_v_1_0_0_0 *acf_dde = NULL; + + acf_hdr = (struct aw_acf_hdr *)fw_data; + acf_dde = (struct aw_acf_dde_v_1_0_0_0 *)(fw_data + acf_hdr->ddt_offset); + + for (i = 0; i < acf_hdr->dde_num; ++i) { + data = fw_data + acf_dde[i].data_offset; + crc_val = aw_crc8_check(data, acf_dde[i].data_size); + if (crc_val != acf_dde[i].data_crc) { + AW_DEV_LOGE(dev, "acf dde_crc check failed"); + return -EINVAL; + } + } + + return 0; +} + +static int aw_check_data_v_1_0_0_0(struct device *dev, + char *fw_data, size_t size) +{ + int ret = -1; + + /* check file type id is awinic acf file */ + ret = aw_check_file_id(dev, fw_data, AW_ACF_FILE_ID); + if (ret < 0) + return ret; + + /* check ddt_size in header is equal to all ddt aize */ + ret = aw_check_ddt_size_v_1_0_0_0(dev, fw_data); + if (ret < 0) + return ret; + + /* Verify that the file size is equal to the header size plus */ + /* the table size and data size */ + ret = aw_check_data_size_v_1_0_0_0(dev, fw_data, size); + if (ret < 0) + return ret; + + /* check crc in is equal to dde data crc */ + ret = aw_check_data_crc_v_1_0_0_0(dev, fw_data); + if (ret < 0) + return ret; + + AW_DEV_LOGI(dev, "acf fimware check succeed"); + + return 0; +} + +/*************************************************************************** + * acf chack API + **************************************************************************/ +static int aw_check_acf_firmware(struct device *dev, + char *fw_data, size_t size) +{ + int ret = -1; + struct aw_acf_hdr *acf_hdr = NULL; + + if (fw_data == NULL) { + AW_DEV_LOGE(dev, "fw_data is NULL,fw_data check failed"); + return -ENODATA; + } + + /* check file size is less-than header size */ + ret = aw_check_header_size(dev, fw_data, size); + if (ret < 0) + return ret; + + acf_hdr = (struct aw_acf_hdr *)fw_data; + AW_DEV_LOGI(dev, "project name: [%s]", acf_hdr->project); + AW_DEV_LOGI(dev, "custom name: [%s]", acf_hdr->custom); + AW_DEV_LOGI(dev, "version name: [%s]", acf_hdr->version); + AW_DEV_LOGI(dev, "author_id: [%d]", acf_hdr->author_id); + + switch (acf_hdr->hdr_version) { + case AW_ACF_HDR_VER_0_0_0_1: + return aw_check_data_v_0_0_0_1(dev, fw_data, size); + case AW_ACF_HDR_VER_1_0_0_0: + return aw_check_data_v_1_0_0_0(dev, fw_data, size); + default: + AW_DEV_LOGE(dev, "unsupported hdr_version [0x%x]", + acf_hdr->hdr_version); + return -EINVAL; + } + + return ret; +} + + + +/************************************************************************* + * + *acf parse + * + *************************************************************************/ +static int aw_parse_raw_reg(struct device *dev, uint8_t *data, + uint32_t data_len, struct aw_prof_desc *prof_desc) +{ + AW_DEV_LOGD(dev, "data_size:%d enter", data_len); + + prof_desc->data_container.data = data; + prof_desc->data_container.len = data_len; + + prof_desc->prof_st = AW_PROFILE_OK; + + return 0; +} + +static int aw_parse_reg_with_hdr(struct device *dev, uint8_t *data, + uint32_t data_len, struct aw_prof_desc *prof_desc) +{ + struct aw_bin *aw_bin = NULL; + int ret = -1; + + AW_DEV_LOGD(dev, "data_size:%d enter", data_len); + + aw_bin = kzalloc(data_len + sizeof(struct aw_bin), GFP_KERNEL); + if (aw_bin == NULL) { + AW_DEV_LOGE(dev, "devm_kzalloc aw_bin failed"); + return -ENOMEM; + } + + aw_bin->info.len = data_len; + memcpy(aw_bin->info.data, data, data_len); + + ret = aw87xxx_parsing_bin_file(aw_bin); + if (ret < 0) { + AW_DEV_LOGE(dev, "parse bin failed"); + goto parse_bin_failed; + } + + if ((aw_bin->all_bin_parse_num != 1) || + (aw_bin->header_info[0].bin_data_type != DATA_TYPE_REGISTER)) { + AW_DEV_LOGE(dev, "bin num or type error"); + goto parse_bin_failed; + } + + prof_desc->data_container.data = + data + aw_bin->header_info[0].valid_data_addr; + prof_desc->data_container.len = aw_bin->header_info[0].valid_data_len; + prof_desc->prof_st = AW_PROFILE_OK; + + kfree(aw_bin); + aw_bin = NULL; + + return 0; + +parse_bin_failed: + kfree(aw_bin); + aw_bin = NULL; + return ret; +} + +static int aw_parse_monitor_config(struct device *dev, + char *monitor_data, uint32_t data_len) +{ + int ret = -1; + + if (monitor_data == NULL || data_len == 0) { + AW_DEV_LOGE(dev, "no data to parse"); + return -EBFONT; + } + + ret = aw87xxx_monitor_bin_parse(dev, monitor_data, data_len); + if (ret < 0) { + AW_DEV_LOGE(dev, "monitor_config parse failed"); + return ret; + } + + AW_DEV_LOGI(dev, "monitor_bin parse succeed"); + + return 0; +} + +static int aw_check_prof_str_is_off(char *profile_name) +{ + int i = 0; + + for (i = 0; i < AW_POWER_OFF_NAME_SUPPORT_COUNT; i++) { + if (strnstr(profile_name, g_power_off_name[i], + strlen(profile_name) + 1)) + return 0; + } + + return -EINVAL; +} + +/*************************************************************************** + * V0.0.0.1 version acf paese + **************************************************************************/ +static int aw_check_product_name_v_0_0_0_1(struct device *dev, + struct acf_bin_info *acf_info, + struct aw_acf_dde *prof_hdr) +{ + int i = 0; + + for (i = 0; i < acf_info->product_cnt; i++) { + if (0 == strcmp(acf_info->product_tab[i], prof_hdr->dev_name)) { + AW_DEV_LOGD(dev, "bin_dev_name:%s", + prof_hdr->dev_name); + return 0; + } + } + + return -ENXIO; +} + +static int aw_check_data_type_is_monitor_v_0_0_0_1(struct device *dev, + struct aw_acf_dde *prof_hdr) +{ + if (prof_hdr->data_type == AW_MONITOR) { + AW_DEV_LOGD(dev, "bin data is monitor"); + return 0; + } + + return -ENXIO; +} + +static int aw_parse_data_by_sec_type_v_0_0_0_1(struct device *dev, + struct acf_bin_info *acf_info, + struct aw_acf_dde *prof_hdr, + struct aw_prof_desc *profile_prof_desc) +{ + int ret = -1; + char *cfg_data = acf_info->fw_data + prof_hdr->data_offset; + + switch (prof_hdr->data_type) { + case AW_BIN_TYPE_REG: + snprintf(profile_prof_desc->dev_name, sizeof(prof_hdr->dev_name), + "%s", prof_hdr->dev_name); + profile_prof_desc->prof_name = aw_get_prof_name(prof_hdr->dev_profile); + AW_DEV_LOGD(dev, "parse reg type data enter,profile=%s", + aw_get_prof_name(prof_hdr->dev_profile)); + ret = aw_parse_raw_reg(dev, cfg_data, prof_hdr->data_size, + profile_prof_desc); + break; + case AW_BIN_TYPE_HDR_REG: + snprintf(profile_prof_desc->dev_name, sizeof(prof_hdr->dev_name), + "%s", prof_hdr->dev_name); + profile_prof_desc->prof_name = aw_get_prof_name(prof_hdr->dev_profile); + AW_DEV_LOGD(dev, "parse hdr_reg type data enter,profile=%s", + aw_get_prof_name(prof_hdr->dev_profile)); + ret = aw_parse_reg_with_hdr(dev, cfg_data, + prof_hdr->data_size, + profile_prof_desc); + break; + } + + return ret; +} + +static int aw_parse_dev_type_v_0_0_0_1(struct device *dev, + struct acf_bin_info *acf_info, struct aw_all_prof_info *all_prof_info) +{ + int i = 0; + int ret = -1; + int sec_num = 0; + char *cfg_data = NULL; + struct aw_prof_desc *prof_desc = NULL; + struct aw_acf_dde *acf_dde = + (struct aw_acf_dde *)(acf_info->fw_data + acf_info->acf_hdr.ddt_offset); + + AW_DEV_LOGD(dev, "enter"); + + for (i = 0; i < acf_info->acf_hdr.dde_num; i++) { + if ((acf_info->aw_dev->i2c_bus == acf_dde[i].dev_bus) && + (acf_info->aw_dev->i2c_addr == acf_dde[i].dev_addr) && + (acf_dde[i].type == AW_DDE_DEV_TYPE_ID)) { + + ret = aw_check_product_name_v_0_0_0_1(dev, acf_info, &acf_dde[i]); + if (ret < 0) + continue; + + ret = aw_check_data_type_is_monitor_v_0_0_0_1(dev, &acf_dde[i]); + if (ret == 0) { + cfg_data = acf_info->fw_data + acf_dde[i].data_offset; + ret = aw_parse_monitor_config(dev, cfg_data, acf_dde[i].data_size); + if (ret < 0) + return ret; + continue; + } + + prof_desc = &all_prof_info->prof_desc[acf_dde[i].dev_profile]; + ret = aw_parse_data_by_sec_type_v_0_0_0_1(dev, acf_info, &acf_dde[i], + prof_desc); + if (ret < 0) { + AW_DEV_LOGE(dev, "parse dev type data failed"); + return ret; + } + sec_num++; + } + } + + if (sec_num == 0) { + AW_DEV_LOGD(dev, "get dev type num is %d, please use default", + sec_num); + return AW_DEV_TYPE_NONE; + } + + return AW_DEV_TYPE_OK; +} + +static int aw_parse_default_type_v_0_0_0_1(struct device *dev, + struct acf_bin_info *acf_info, struct aw_all_prof_info *all_prof_info) +{ + int i = 0; + int ret = -1; + int sec_num = 0; + char *cfg_data = NULL; + struct aw_prof_desc *prof_desc = NULL; + struct aw_acf_dde *acf_dde = + (struct aw_acf_dde *)(acf_info->fw_data + acf_info->acf_hdr.ddt_offset); + + AW_DEV_LOGD(dev, "enter"); + + for (i = 0; i < acf_info->acf_hdr.dde_num; i++) { + if ((acf_info->dev_index == acf_dde[i].dev_index) && + (acf_dde[i].type == AW_DDE_DEV_DEFAULT_TYPE_ID)) { + + ret = aw_check_product_name_v_0_0_0_1(dev, acf_info, &acf_dde[i]); + if (ret < 0) + continue; + + ret = aw_check_data_type_is_monitor_v_0_0_0_1(dev, &acf_dde[i]); + if (ret == 0) { + cfg_data = acf_info->fw_data + acf_dde[i].data_offset; + ret = aw_parse_monitor_config(dev, cfg_data, acf_dde[i].data_size); + if (ret < 0) + return ret; + continue; + } + + prof_desc = &all_prof_info->prof_desc[acf_dde[i].dev_profile]; + ret = aw_parse_data_by_sec_type_v_0_0_0_1(dev, acf_info, &acf_dde[i], + prof_desc); + if (ret < 0) { + AW_DEV_LOGE(dev, "parse default type data failed"); + return ret; + } + sec_num++; + } + } + + if (sec_num == 0) { + AW_DEV_LOGE(dev, "get dev default type failed, get num[%d]", + sec_num); + return -EINVAL; + } + + return 0; +} + +static int aw_get_prof_count_v_0_0_0_1(struct device *dev, + struct acf_bin_info *acf_info, + struct aw_all_prof_info *all_prof_info) +{ + int i = 0; + int prof_count = 0; + struct aw_prof_desc *prof_desc = all_prof_info->prof_desc; + + for (i = 0; i < AW_PROFILE_MAX; i++) { + if (prof_desc[i].prof_st == AW_PROFILE_OK) { + prof_count++; + } else if (i == AW_PROFILE_OFF) { + prof_count++; + AW_DEV_LOGI(dev, "not found profile [Off], set default"); + } + } + + AW_DEV_LOGI(dev, "get profile count=[%d]", prof_count); + return prof_count; +} + +static int aw_set_prof_off_info_v_0_0_0_1(struct device *dev, + struct acf_bin_info *acf_info, + struct aw_all_prof_info *all_prof_info, + int index) +{ + struct aw_prof_desc *prof_desc = all_prof_info->prof_desc; + struct aw_prof_info *prof_info = &acf_info->prof_info; + + if (index >= prof_info->count) { + AW_DEV_LOGE(dev, "index[%d] is out of table,profile count[%d]", + index, prof_info->count); + return -EINVAL; + } + + if (prof_desc[AW_PROFILE_OFF].prof_st == AW_PROFILE_OK) { + prof_info->prof_desc[index] = prof_desc[AW_PROFILE_OFF]; + AW_DEV_LOGI(dev, "product=[%s]----profile=[%s]", + prof_info->prof_desc[index].dev_name, + aw_get_prof_name(AW_PROFILE_OFF)); + } else { + memset(&prof_info->prof_desc[index].data_container, 0, + sizeof(struct aw_data_container)); + prof_info->prof_desc[index].prof_st = AW_PROFILE_WAIT; + prof_info->prof_desc[index].prof_name = aw_get_prof_name(AW_PROFILE_OFF); + AW_DEV_LOGI(dev, "set default power_off with no data to profile"); + } + + return 0; +} + + +static int aw_get_vaild_prof_v_0_0_0_1(struct device *dev, + struct acf_bin_info *acf_info, + struct aw_all_prof_info *all_prof_info) +{ + int i = 0; + int ret = 0; + int index = 0; + struct aw_prof_desc *prof_desc = all_prof_info->prof_desc; + struct aw_prof_info *prof_info = &acf_info->prof_info; + + prof_info->count = 0; + ret = aw_get_prof_count_v_0_0_0_1(dev, acf_info, all_prof_info); + if (ret < 0) + return ret; + prof_info->count = ret; + prof_info->prof_desc = devm_kzalloc(dev, + prof_info->count * sizeof(struct aw_prof_desc), + GFP_KERNEL); + if (prof_info->prof_desc == NULL) { + AW_DEV_LOGE(dev, "prof_desc kzalloc failed"); + return -ENOMEM; + } + + for (i = 0; i < AW_PROFILE_MAX; i++) { + if (i != AW_PROFILE_OFF && prof_desc[i].prof_st == AW_PROFILE_OK) { + if (index >= prof_info->count) { + AW_DEV_LOGE(dev, "get profile index[%d] overflow count[%d]", + index, prof_info->count); + return -ENOMEM; + } + prof_info->prof_desc[index] = prof_desc[i]; + AW_DEV_LOGI(dev, "product=[%s]----profile=[%s]", + prof_info->prof_desc[index].dev_name, + aw_get_prof_name(i)); + index++; + } + } + + ret = aw_set_prof_off_info_v_0_0_0_1(dev, acf_info, all_prof_info, index); + if (ret < 0) + return ret; + + AW_DEV_LOGD(dev, "get vaild profile succeed"); + return 0; +} + +static int aw_set_prof_name_list_v_0_0_0_1(struct device *dev, + struct acf_bin_info *acf_info) +{ + int i = 0; + int count = acf_info->prof_info.count; + struct aw_prof_info *prof_info = &acf_info->prof_info; + + prof_info->prof_name_list = (char (*)[AW_PROFILE_STR_MAX])devm_kzalloc(dev, + count * (AW_PROFILE_STR_MAX), GFP_KERNEL); + if (prof_info->prof_name_list == NULL) { + AW_DEV_LOGE(dev, "prof_name_list devm_kzalloc failed"); + return -ENOMEM; + } + + for (i = 0; i < count; ++i) { + snprintf(prof_info->prof_name_list[i], AW_PROFILE_STR_MAX, "%s", + prof_info->prof_desc[i].prof_name); + AW_DEV_LOGI(dev, "index=[%d], profile_name=[%s]", + i, prof_info->prof_name_list[i]); + } + + return 0; +} + +static int aw_parse_acf_v_0_0_0_1(struct device *dev, + struct acf_bin_info *acf_info) + +{ + int ret = 0; + struct aw_all_prof_info all_prof_info; + + AW_DEV_LOGD(dev, "enter"); + acf_info->prof_info.status = AW_ACF_WAIT; + + memset(&all_prof_info, 0, sizeof(struct aw_all_prof_info)); + + ret = aw_parse_dev_type_v_0_0_0_1(dev, acf_info, &all_prof_info); + if (ret < 0) { + return ret; + } else if (ret == AW_DEV_TYPE_NONE) { + AW_DEV_LOGD(dev, "get dev type num is 0, parse default dev type"); + ret = aw_parse_default_type_v_0_0_0_1(dev, acf_info, &all_prof_info); + if (ret < 0) + return ret; + } + + ret = aw_get_vaild_prof_v_0_0_0_1(dev, acf_info, &all_prof_info); + if (ret < 0) { + aw87xxx_acf_profile_free(dev, acf_info); + AW_DEV_LOGE(dev, "hdr_cersion[0x%x] parse failed", + acf_info->acf_hdr.hdr_version); + return ret; + } + + ret = aw_set_prof_name_list_v_0_0_0_1(dev, acf_info); + if (ret < 0) { + aw87xxx_acf_profile_free(dev, acf_info); + AW_DEV_LOGE(dev, "creat prof_id_and_name_list failed"); + return ret; + } + + acf_info->prof_info.status = AW_ACF_UPDATE; + AW_DEV_LOGI(dev, "acf parse success"); + return 0; +} + +/*************************************************************************** + * V1.0.0.0 version acf paese + **************************************************************************/ +static int aw_check_product_name_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info, + struct aw_acf_dde_v_1_0_0_0 *prof_hdr) +{ + int i = 0; + + for (i = 0; i < acf_info->product_cnt; i++) { + if (0 == strcmp(acf_info->product_tab[i], prof_hdr->dev_name)) { + AW_DEV_LOGI(dev, "bin_dev_name:%s", prof_hdr->dev_name); + return 0; + } + } + + return -ENXIO; +} + +static int aw_get_dde_type_info_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + int i; + int dev_num = 0; + int default_num = 0; + struct aw_acf_hdr *acf_hdr = (struct aw_acf_hdr *)acf_info->fw_data; + struct aw_prof_info *prof_info = &acf_info->prof_info; + struct aw_acf_dde_v_1_0_0_0 *acf_dde = + (struct aw_acf_dde_v_1_0_0_0 *)(acf_info->fw_data + acf_hdr->ddt_offset); + + prof_info->prof_type = AW_DEV_NONE_TYPE_ID; + for (i = 0; i < acf_hdr->dde_num; i++) { + if (acf_dde[i].type == AW_DDE_DEV_TYPE_ID) + dev_num++; + if (acf_dde[i].type == AW_DDE_DEV_DEFAULT_TYPE_ID) + default_num++; + } + + if (!(dev_num || default_num)) { + AW_DEV_LOGE(dev, "can't find scene"); + return -EINVAL; + } + + if (dev_num != 0) + prof_info->prof_type = AW_DDE_DEV_TYPE_ID; + else if (default_num != 0) + prof_info->prof_type = AW_DDE_DEV_DEFAULT_TYPE_ID; + + return 0; +} + + +static int aw_parse_get_dev_type_prof_count_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + struct aw_acf_hdr *acf_hdr = (struct aw_acf_hdr *)acf_info->fw_data; + struct aw_acf_dde_v_1_0_0_0 *acf_dde = + (struct aw_acf_dde_v_1_0_0_0 *)(acf_info->fw_data + acf_hdr->ddt_offset); + int i = 0; + int ret = 0; + int found_off_prof_flag = 0; + int count = acf_info->prof_info.count; + + for (i = 0; i < acf_hdr->dde_num; ++i) { + if (((acf_dde[i].data_type == AW_BIN_TYPE_REG) || + (acf_dde[i].data_type == AW_BIN_TYPE_HDR_REG)) && + ((acf_info->aw_dev->i2c_bus == acf_dde[i].dev_bus) && + (acf_info->aw_dev->i2c_addr == acf_dde[i].dev_addr)) && + (acf_info->aw_dev->chipid == acf_dde[i].chip_id)) { + + ret = aw_check_product_name_v_1_0_0_0(dev, acf_info, &acf_dde[i]); + if (ret < 0) + continue; + + ret = aw_check_prof_str_is_off(acf_dde[i].dev_profile_str); + if (ret == 0) { + found_off_prof_flag = AW_PROFILE_OK; + } + count++; + } + } + + if (count == 0) { + AW_DEV_LOGE(dev, "can't find profile"); + return -EINVAL; + } + + if (!found_off_prof_flag) { + count++; + AW_DEV_LOGD(dev, "set no config power off profile in count"); + } + + acf_info->prof_info.count = count; + AW_DEV_LOGI(dev, "profile dev_type profile count is %d", acf_info->prof_info.count); + return 0; +} + +static int aw_parse_get_default_type_prof_count_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + struct aw_acf_hdr *acf_hdr = (struct aw_acf_hdr *)acf_info->fw_data; + struct aw_acf_dde_v_1_0_0_0 *acf_dde = + (struct aw_acf_dde_v_1_0_0_0 *)(acf_info->fw_data + acf_hdr->ddt_offset); + int i = 0; + int ret = 0; + int found_off_prof_flag = 0; + int count = acf_info->prof_info.count; + + for (i = 0; i < acf_hdr->dde_num; ++i) { + if (((acf_dde[i].data_type == AW_BIN_TYPE_REG) || + (acf_dde[i].data_type == AW_BIN_TYPE_HDR_REG)) && + (acf_info->dev_index == acf_dde[i].dev_index) && + (acf_info->aw_dev->chipid == acf_dde[i].chip_id)) { + + ret = aw_check_product_name_v_1_0_0_0(dev, acf_info, &acf_dde[i]); + if (ret < 0) + continue; + + ret = aw_check_prof_str_is_off(acf_dde[i].dev_profile_str); + if (ret == 0) { + found_off_prof_flag = AW_PROFILE_OK; + } + count++; + } + } + + if (count == 0) { + AW_DEV_LOGE(dev, "can't find profile"); + return -EINVAL; + } + + if (!found_off_prof_flag) { + count++; + AW_DEV_LOGD(dev, "set no config power off profile in count"); + } + + acf_info->prof_info.count = count; + AW_DEV_LOGI(dev, "profile default_type profile count is %d", acf_info->prof_info.count); + return 0; +} + +static int aw_parse_get_profile_count_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + int ret = 0; + + ret = aw_get_dde_type_info_v_1_0_0_0(dev, acf_info); + if (ret < 0) + return ret; + + if (acf_info->prof_info.prof_type == AW_DDE_DEV_TYPE_ID) { + ret = aw_parse_get_dev_type_prof_count_v_1_0_0_0(dev, acf_info); + if (ret < 0) { + AW_DEV_LOGE(dev, "parse dev_type profile count failed"); + return ret; + } + } else if (acf_info->prof_info.prof_type == AW_DDE_DEV_DEFAULT_TYPE_ID) { + ret = aw_parse_get_default_type_prof_count_v_1_0_0_0(dev, acf_info); + if (ret < 0) { + AW_DEV_LOGE(dev, "parse default_type profile count failed"); + return ret; + } + } else { + AW_DEV_LOGE(dev, "unsupport prof_type[0x%x]", + acf_info->prof_info.prof_type); + return -EINVAL; + } + + AW_DEV_LOGI(dev, "profile count is %d", acf_info->prof_info.count); + return 0; +} + +static int aw_parse_dev_type_prof_name_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + struct aw_acf_hdr *acf_hdr = (struct aw_acf_hdr *)acf_info->fw_data; + struct aw_acf_dde_v_1_0_0_0 *acf_dde = + (struct aw_acf_dde_v_1_0_0_0 *)(acf_info->fw_data + acf_hdr->ddt_offset); + struct aw_prof_info *prof_info = &acf_info->prof_info; + int i, ret, list_index = 0; + + for (i = 0; i < acf_hdr->dde_num; ++i) { + if (((acf_dde[i].data_type == AW_BIN_TYPE_REG) || + (acf_dde[i].data_type == AW_BIN_TYPE_HDR_REG)) && + (acf_info->aw_dev->i2c_bus == acf_dde[i].dev_bus) && + (acf_info->aw_dev->i2c_addr == acf_dde[i].dev_addr) && + (acf_info->aw_dev->chipid == acf_dde[i].chip_id)) { + if (list_index > prof_info->count) { + AW_DEV_LOGE(dev, "%s:Alrealdy set list_index [%d], redundant profile [%s]exist\n", + __func__, list_index, + acf_dde[i].dev_profile_str); + return -EINVAL; + } + + ret = aw_check_product_name_v_1_0_0_0(dev, acf_info, &acf_dde[i]); + if (ret < 0) + continue; + + snprintf(prof_info->prof_name_list[list_index], AW_PROFILE_STR_MAX, "%s", + acf_dde[i].dev_profile_str); + AW_DEV_LOGI(dev, "profile_name=[%s]", + prof_info->prof_name_list[list_index]); + list_index++; + } + } + + return 0; +} + +static int aw_parse_default_type_prof_name_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + struct aw_acf_hdr *acf_hdr = (struct aw_acf_hdr *)acf_info->fw_data; + struct aw_acf_dde_v_1_0_0_0 *acf_dde = + (struct aw_acf_dde_v_1_0_0_0 *)(acf_info->fw_data + acf_hdr->ddt_offset); + struct aw_prof_info *prof_info = &acf_info->prof_info; + int i, ret, list_index = 0; + + for (i = 0; i < acf_hdr->dde_num; ++i) { + if (((acf_dde[i].data_type == AW_BIN_TYPE_REG) || + (acf_dde[i].data_type == AW_BIN_TYPE_HDR_REG)) && + (acf_info->dev_index == acf_dde[i].dev_index) && + (acf_info->aw_dev->chipid == acf_dde[i].chip_id)) { + if (list_index > prof_info->count) { + AW_DEV_LOGE(dev, "%s:Alrealdy set list_index [%d], redundant profile [%s]exist\n", + __func__, list_index, + acf_dde[i].dev_profile_str); + return -EINVAL; + } + + ret = aw_check_product_name_v_1_0_0_0(dev, acf_info, &acf_dde[i]); + if (ret < 0) + continue; + + snprintf(prof_info->prof_name_list[list_index], AW_PROFILE_STR_MAX, "%s", + acf_dde[i].dev_profile_str); + AW_DEV_LOGI(dev, "profile_name=[%s]", + prof_info->prof_name_list[list_index]); + list_index++; + } + } + + return 0; +} + +static int aw_parse_prof_name_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + int ret = 0; + int count = acf_info->prof_info.count; + struct aw_prof_info *prof_info = &acf_info->prof_info; + + prof_info->prof_name_list = (char (*)[AW_PROFILE_STR_MAX])devm_kzalloc(dev, + count * (AW_PROFILE_STR_MAX), GFP_KERNEL); + if (prof_info->prof_name_list == NULL) { + AW_DEV_LOGE(dev, "prof_name_list devm_kzalloc failed"); + return -ENOMEM; + } + + if (acf_info->prof_info.prof_type == AW_DDE_DEV_TYPE_ID) { + ret = aw_parse_dev_type_prof_name_v_1_0_0_0(dev, acf_info); + if (ret < 0) { + AW_DEV_LOGE(dev, "parse dev_type profile count failed"); + return ret; + } + } else if (acf_info->prof_info.prof_type == AW_DDE_DEV_DEFAULT_TYPE_ID) { + ret = aw_parse_default_type_prof_name_v_1_0_0_0(dev, acf_info); + if (ret < 0) { + AW_DEV_LOGE(dev, "parse default_type profile count failed"); + return ret; + } + } else { + AW_DEV_LOGE(dev, "unsupport prof_type[0x%x]", + acf_info->prof_info.prof_type); + return -EINVAL; + } + + AW_DEV_LOGI(dev, "profile name parse succeed"); + return 0; +} + + +static int aw_search_prof_index_from_list_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info, + struct aw_prof_desc **prof_desc, + struct aw_acf_dde_v_1_0_0_0 *prof_hdr) +{ + int i = 0; + int count = acf_info->prof_info.count; + char (*prof_name_list)[AW_PROFILE_STR_MAX] = acf_info->prof_info.prof_name_list; + + for (i = 0; i < count; i++) { + if (!strncmp(prof_name_list[i], prof_hdr->dev_profile_str, AW_PROFILE_STR_MAX)) { + *prof_desc = &(acf_info->prof_info.prof_desc[i]); + return 0; + } + } + + if (i == count) + AW_DEV_LOGE(dev, "not find prof_id and prof_name in list"); + + return -EINVAL; +} + +static int aw_parse_data_by_sec_type_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info, + struct aw_acf_dde_v_1_0_0_0 *prof_hdr) +{ + int ret = -1; + char *cfg_data = acf_info->fw_data + prof_hdr->data_offset; + struct aw_prof_desc *prof_desc = NULL; + + ret = aw_search_prof_index_from_list_v_1_0_0_0(dev, acf_info, &prof_desc, prof_hdr); + if (ret < 0) + return ret; + + switch (prof_hdr->data_type) { + case AW_BIN_TYPE_REG: + snprintf(prof_desc->dev_name, sizeof(prof_hdr->dev_name), + "%s", prof_hdr->dev_name); + AW_DEV_LOGI(dev, "parse reg type data enter,product=[%s],prof_id=[%d],prof_name=[%s]", + prof_hdr->dev_name, prof_hdr->dev_profile, + prof_hdr->dev_profile_str); + prof_desc->prof_name = prof_hdr->dev_profile_str; + ret = aw_parse_raw_reg(dev, cfg_data, prof_hdr->data_size, + prof_desc); + break; + case AW_BIN_TYPE_HDR_REG: + snprintf(prof_desc->dev_name, sizeof(prof_hdr->dev_name), + "%s", prof_hdr->dev_name); + AW_DEV_LOGI(dev, "parse hdr_reg type data enter,product=[%s],prof_id=[%d],prof_name=[%s]", + prof_hdr->dev_name, prof_hdr->dev_profile, + prof_hdr->dev_profile_str); + prof_desc->prof_name = prof_hdr->dev_profile_str; + ret = aw_parse_reg_with_hdr(dev, cfg_data, + prof_hdr->data_size, prof_desc); + break; + } + + return ret; +} + +static int aw_parse_dev_type_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + int i = 0; + int ret; + int parse_prof_count = 0; + char *cfg_data = NULL; + struct aw_acf_dde_v_1_0_0_0 *acf_dde = + (struct aw_acf_dde_v_1_0_0_0 *)(acf_info->fw_data + acf_info->acf_hdr.ddt_offset); + + AW_DEV_LOGD(dev, "enter"); + + for (i = 0; i < acf_info->acf_hdr.dde_num; i++) { + if ((acf_dde[i].type == AW_DDE_DEV_TYPE_ID) && + (acf_info->aw_dev->i2c_bus == acf_dde[i].dev_bus) && + (acf_info->aw_dev->i2c_addr == acf_dde[i].dev_addr) && + (acf_info->aw_dev->chipid == acf_dde[i].chip_id)) { + ret = aw_check_product_name_v_1_0_0_0(dev, acf_info, &acf_dde[i]); + if (ret < 0) + continue; + + if (acf_dde[i].data_type == AW_MONITOR) { + cfg_data = acf_info->fw_data + acf_dde[i].data_offset; + AW_DEV_LOGD(dev, "parse monitor type data enter"); + ret = aw_parse_monitor_config(dev, cfg_data, + acf_dde[i].data_size); + } else { + ret = aw_parse_data_by_sec_type_v_1_0_0_0(dev, acf_info, + &acf_dde[i]); + if (ret < 0) + AW_DEV_LOGE(dev, "parse dev type data failed"); + else + parse_prof_count++; + } + } + } + + if (parse_prof_count == 0) { + AW_DEV_LOGE(dev, "get dev type num is %d, parse failed", parse_prof_count); + return -EINVAL; + } + + return AW_DEV_TYPE_OK; +} + +static int aw_parse_default_type_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + int i = 0; + int ret; + int parse_prof_count = 0; + char *cfg_data = NULL; + struct aw_acf_dde_v_1_0_0_0 *acf_dde = + (struct aw_acf_dde_v_1_0_0_0 *)(acf_info->fw_data + acf_info->acf_hdr.ddt_offset); + + AW_DEV_LOGD(dev, "enter"); + + for (i = 0; i < acf_info->acf_hdr.dde_num; i++) { + if ((acf_dde[i].type == AW_DDE_DEV_DEFAULT_TYPE_ID) && + (acf_info->dev_index == acf_dde[i].dev_index) && + (acf_info->aw_dev->chipid == acf_dde[i].chip_id)) { + ret = aw_check_product_name_v_1_0_0_0(dev, acf_info, &acf_dde[i]); + if (ret < 0) + continue; + + if (acf_dde[i].data_type == AW_MONITOR) { + cfg_data = acf_info->fw_data + acf_dde[i].data_offset; + AW_DEV_LOGD(dev, "parse monitor type data enter"); + ret = aw_parse_monitor_config(dev, cfg_data, + acf_dde[i].data_size); + } else { + ret = aw_parse_data_by_sec_type_v_1_0_0_0(dev, acf_info, + &acf_dde[i]); + if (ret < 0) + AW_DEV_LOGE(dev, "parse default type data failed"); + else + parse_prof_count++; + } + } + } + + if (parse_prof_count == 0) { + AW_DEV_LOGE(dev, "get default type num is %d,parse failed", parse_prof_count); + return -EINVAL; + } + + return AW_DEV_TYPE_OK; +} + +static int aw_parse_by_hdr_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + int ret; + + if (acf_info->prof_info.prof_type == AW_DDE_DEV_TYPE_ID) { + ret = aw_parse_dev_type_v_1_0_0_0(dev, acf_info); + if (ret < 0) + return ret; + } else if (acf_info->prof_info.prof_type == AW_DDE_DEV_DEFAULT_TYPE_ID) { + ret = aw_parse_default_type_v_1_0_0_0(dev, acf_info); + if (ret < 0) + return ret; + } + + return 0; +} + +static int aw_set_prof_off_info_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) +{ + struct aw_prof_info *prof_info = &acf_info->prof_info; + int i = 0; + int ret = 0; + + for (i = 0; i < prof_info->count; ++i) { + if (!(prof_info->prof_desc[i].prof_st)) { + snprintf(prof_info->prof_name_list[i], AW_PROFILE_STR_MAX, "%s", + g_power_off_name[0]); + prof_info->prof_desc[i].prof_name = prof_info->prof_name_list[i]; + prof_info->prof_desc[i].prof_st = AW_PROFILE_WAIT; + memset(&prof_info->prof_desc[i].data_container, 0, + sizeof(struct aw_data_container)); + return 0; + } + + ret = aw_check_prof_str_is_off(prof_info->prof_name_list[i]); + if (ret == 0) { + AW_DEV_LOGD(dev, "found profile off,data_len=[%d]", + prof_info->prof_desc[i].data_container.len); + return 0; + } + } + + AW_DEV_LOGE(dev, "index[%d] is out of table,profile count[%d]", + i, prof_info->count); + return -EINVAL; +} + +static int aw_parse_acf_v_1_0_0_0(struct device *dev, + struct acf_bin_info *acf_info) + +{ + struct aw_prof_info *prof_info = &acf_info->prof_info; + int ret; + + ret = aw_parse_get_profile_count_v_1_0_0_0(dev, acf_info); + if (ret < 0) { + AW_DEV_LOGE(dev, "get profile count failed"); + return ret; + } + + ret = aw_parse_prof_name_v_1_0_0_0(dev, acf_info); + if (ret < 0) { + AW_DEV_LOGE(dev, "get profile count failed"); + return ret; + } + + acf_info->prof_info.prof_desc = devm_kzalloc(dev, + prof_info->count * sizeof(struct aw_prof_desc), GFP_KERNEL); + if (acf_info->prof_info.prof_desc == NULL) { + AW_DEV_LOGE(dev, "prof_desc devm_kzalloc failed"); + return -ENOMEM; + } + + ret = aw_parse_by_hdr_v_1_0_0_0(dev, acf_info); + if (ret < 0) { + AW_DEV_LOGE(dev, "parse data failed"); + return ret; + } + + ret = aw_set_prof_off_info_v_1_0_0_0(dev, acf_info); + if (ret < 0) { + AW_DEV_LOGE(dev, "set profile off info failed"); + return ret; + } + + prof_info->status = AW_ACF_UPDATE; + AW_DEV_LOGI(dev, "acf paese succeed"); + return 0; +} + + +/************************************************************************* + * + *acf parse API + * + *************************************************************************/ +void aw87xxx_acf_profile_free(struct device *dev, struct acf_bin_info *acf_info) +{ + struct aw_prof_info *prof_info = &acf_info->prof_info; + + prof_info->count = 0; + prof_info->status = AW_ACF_WAIT; + memset(&acf_info->acf_hdr, 0, sizeof(struct aw_acf_hdr)); + + if (prof_info->prof_desc) { + devm_kfree(dev, prof_info->prof_desc); + prof_info->prof_desc = NULL; + } + + if (prof_info->prof_name_list) { + devm_kfree(dev, prof_info->prof_name_list); + prof_info->prof_name_list = NULL; + } + + if (acf_info->fw_data) { + vfree(acf_info->fw_data); + acf_info->fw_data = NULL; + } +} + +int aw87xxx_acf_parse(struct device *dev, struct acf_bin_info *acf_info) +{ + int ret = 0; + + AW_DEV_LOGD(dev, "enter"); + acf_info->prof_info.status = AW_ACF_WAIT; + ret = aw_check_acf_firmware(dev, acf_info->fw_data, + acf_info->fw_size); + if (ret < 0) { + AW_DEV_LOGE(dev, "load firmware check failed"); + return -EINVAL; + } + + memcpy(&acf_info->acf_hdr, acf_info->fw_data, + sizeof(struct aw_acf_hdr)); + + switch (acf_info->acf_hdr.hdr_version) { + case AW_ACF_HDR_VER_0_0_0_1: + return aw_parse_acf_v_0_0_0_1(dev, acf_info); + case AW_ACF_HDR_VER_1_0_0_0: + return aw_parse_acf_v_1_0_0_0(dev, acf_info); + default: + AW_DEV_LOGE(dev, "unsupported hdr_version [0x%x]", + acf_info->acf_hdr.hdr_version); + return -EINVAL; + } + + return ret; +} + +struct aw_prof_desc *aw87xxx_acf_get_prof_desc_form_name(struct device *dev, + struct acf_bin_info *acf_info, char *profile_name) +{ + int i = 0; + struct aw_prof_desc *prof_desc = NULL; + struct aw_prof_info *prof_info = &acf_info->prof_info; + + AW_DEV_LOGD(dev, "enter"); + + if (!acf_info->prof_info.status) { + AW_DEV_LOGE(dev, "profile_cfg not load"); + return NULL; + } + + for (i = 0; i < prof_info->count; i++) { + if (!strncmp(profile_name, prof_info->prof_desc[i].prof_name, + AW_PROFILE_STR_MAX)) { + prof_desc = &prof_info->prof_desc[i]; + break; + } + } + + if (i == prof_info->count) { + AW_DEV_LOGE(dev, "profile not found"); + return NULL; + } + + AW_DEV_LOGI(dev, "get prof desc down"); + return prof_desc; +} + +int aw87xxx_acf_get_prof_index_form_name(struct device *dev, + struct acf_bin_info *acf_info, char *profile_name) +{ + int i = 0; + struct aw_prof_info *prof_info = &acf_info->prof_info; + + if (!acf_info->prof_info.status) { + AW_DEV_LOGE(dev, "profile_cfg not load"); + return -EINVAL; + } + + for (i = 0; i < prof_info->count; i++) { + if (!strncmp(profile_name, prof_info->prof_name_list[i], + AW_PROFILE_STR_MAX)) { + return i; + } + } + + AW_DEV_LOGE(dev, "profile_index not found"); + return -EINVAL; +} + +char *aw87xxx_acf_get_prof_name_form_index(struct device *dev, + struct acf_bin_info *acf_info, int index) +{ + struct aw_prof_info *prof_info = &acf_info->prof_info; + + if (!acf_info->prof_info.status) { + AW_DEV_LOGE(dev, "profile_cfg not load"); + return NULL; + } + + if (index >= prof_info->count || index < 0) { + AW_DEV_LOGE(dev, "profile_index out of table"); + return NULL; + } + + return prof_info->prof_desc[index].prof_name; +} + + +int aw87xxx_acf_get_profile_count(struct device *dev, + struct acf_bin_info *acf_info) +{ + struct aw_prof_info *prof_info = &acf_info->prof_info; + + if (!acf_info->prof_info.status) { + AW_DEV_LOGE(dev, "profile_cfg not load"); + return -EINVAL; + } + + if (prof_info->count > 0) { + return prof_info->count; + } + + return -EINVAL; +} + +char *aw87xxx_acf_get_prof_off_name(struct device *dev, + struct acf_bin_info *acf_info) +{ + int i = 0; + int ret = 0; + struct aw_prof_info *prof_info = &acf_info->prof_info; + + if (!acf_info->prof_info.status) { + AW_DEV_LOGE(dev, "profile_cfg not load"); + return NULL; + } + + for (i = 0; i < prof_info->count; i++) { + ret = aw_check_prof_str_is_off(prof_info->prof_name_list[i]); + if (ret == 0) + return prof_info->prof_name_list[i]; + } + + return NULL; +} + +void aw87xxx_acf_init(struct aw_device *aw_dev, struct acf_bin_info *acf_info, int index) +{ + + acf_info->load_count = 0; + acf_info->prof_info.status = AW_ACF_WAIT; + acf_info->dev_index = index; + acf_info->aw_dev = aw_dev; + acf_info->product_cnt = aw_dev->product_cnt; + acf_info->product_tab = aw_dev->product_tab; + acf_info->prof_info.prof_desc = NULL; + acf_info->fw_data = NULL; + acf_info->fw_size = 0; +} + diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_acf_bin.h b/sound/soc/codecs/aw87xxx/aw87xxx_acf_bin.h new file mode 100644 index 0000000000000..3a14d9b8af9bc --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_acf_bin.h @@ -0,0 +1,191 @@ +#ifndef __AW87XXX_ACF_BIN_H__ +#define __AW87XXX_ACF_BIN_H__ + +#include "aw87xxx_device.h" + +#define AW_PROJECT_NAME_MAX (24) +#define AW_CUSTOMER_NAME_MAX (16) +#define AW_CFG_VERSION_MAX (4) +#define AW_TBL_VERSION_MAX (4) +#define AW_DDE_DEVICE_TYPE (0) +#define AW_DDE_SKT_TYPE (1) +#define AW_DDE_DEFAULT_TYPE (2) + +#define AW_REG_ADDR_BYTE (1) +#define AW_REG_DATA_BYTE (1) + +#define AW_ACF_FILE_ID (0xa15f908) +#define AW_PROFILE_STR_MAX (32) +#define AW_POWER_OFF_NAME_SUPPORT_COUNT (5) + +enum aw_cfg_hdr_version { + AW_ACF_HDR_VER_0_0_0_1 = 0x00000001, + AW_ACF_HDR_VER_1_0_0_0 = 0x01000000, +}; + +enum aw_acf_dde_type_id { + AW_DEV_NONE_TYPE_ID = 0xFFFFFFFF, + AW_DDE_DEV_TYPE_ID = 0x00000000, + AW_DDE_SKT_TYPE_ID = 0x00000001, + AW_DDE_DEV_DEFAULT_TYPE_ID = 0x00000002, + AW_DDE_TYPE_MAX, +}; + +enum aw_raw_data_type_id { + AW_BIN_TYPE_REG = 0x00000000, + AW_BIN_TYPE_DSP, + AW_BIN_TYPE_DSP_CFG, + AW_BIN_TYPE_DSP_FW, + AW_BIN_TYPE_HDR_REG, + AW_BIN_TYPE_HDR_DSP_CFG, + AW_BIN_TYPE_HDR_DSP_FW, + AW_BIN_TYPE_MUTLBIN, + AW_SKT_UI_PROJECT, + AW_DSP_CFG, + AW_MONITOR, + AW_BIN_TYPE_MAX, +}; + +enum { + AW_DEV_TYPE_OK = 0, + AW_DEV_TYPE_NONE = 1, +}; + +enum aw_profile_status { + AW_PROFILE_WAIT = 0, + AW_PROFILE_OK, +}; + +enum aw_acf_load_status { + AW_ACF_WAIT = 0, + AW_ACF_UPDATE, +}; + +enum aw_bin_dev_profile_id { + AW_PROFILE_MUSIC = 0x0000, + AW_PROFILE_VOICE, + AW_PROFILE_VOIP, + AW_PROFILE_RINGTONE, + AW_PROFILE_RINGTONE_HS, + AW_PROFILE_LOWPOWER, + AW_PROFILE_BYPASS, + AW_PROFILE_MMI, + AW_PROFILE_FM, + AW_PROFILE_NOTIFICATION, + AW_PROFILE_RECEIVER, + AW_PROFILE_OFF, + AW_PROFILE_MAX, +}; + +struct aw_acf_hdr { + int32_t a_id; /* acf file ID 0xa15f908 */ + char project[AW_PROJECT_NAME_MAX]; /* project name */ + char custom[AW_CUSTOMER_NAME_MAX]; /* custom name :huawei xiaomi vivo oppo */ + uint8_t version[AW_CFG_VERSION_MAX]; /* author update version */ + int32_t author_id; /* author id */ + int32_t ddt_size; /* sub section table entry size */ + int32_t dde_num; /* sub section table entry num */ + int32_t ddt_offset; /* sub section table offset in file */ + int32_t hdr_version; /* sub section table version */ + int32_t reserve[3]; /* Reserved Bits */ +}; + +struct aw_acf_dde { + int32_t type; /* dde type id */ + char dev_name[AW_CUSTOMER_NAME_MAX]; /* customer dev name */ + int16_t dev_index; /* dev id */ + int16_t dev_bus; /* dev bus id */ + int16_t dev_addr; /* dev addr id */ + int16_t dev_profile; /* dev profile id */ + int32_t data_type; /* data type id */ + int32_t data_size; /* dde data size in block */ + int32_t data_offset; /* dde data offset in block */ + int32_t data_crc; /* dde data crc checkout */ + int32_t reserve[5]; /* Reserved Bits */ +}; + +struct aw_acf_dde_v_1_0_0_0 { + uint32_t type; /* DDE type id */ + char dev_name[AW_CUSTOMER_NAME_MAX]; /* customer dev name */ + uint16_t dev_index; /* dev id */ + uint16_t dev_bus; /* dev bus id */ + uint16_t dev_addr; /* dev addr id */ + uint16_t dev_profile; /* dev profile id*/ + uint32_t data_type; /* data type id */ + uint32_t data_size; /* dde data size in block */ + uint32_t data_offset; /* dde data offset in block */ + uint32_t data_crc; /* dde data crc checkout */ + char dev_profile_str[AW_PROFILE_STR_MAX]; /* dde custom profile name */ + uint32_t chip_id; /* dde custom product chip id */ + uint32_t reserve[4]; +}; + +struct aw_data_with_header { + uint32_t check_sum; + uint32_t header_ver; + uint32_t bin_data_type; + uint32_t bin_data_ver; + uint32_t bin_data_size; + uint32_t ui_ver; + char product[8]; + uint32_t addr_byte_len; + uint32_t data_byte_len; + uint32_t device_addr; + uint32_t reserve[4]; +}; + +struct aw_data_container { + uint32_t len; + uint8_t *data; +}; + +struct aw_prof_desc { + uint32_t prof_st; + char *prof_name; + char dev_name[AW_CUSTOMER_NAME_MAX]; + struct aw_data_container data_container; +}; + +struct aw_all_prof_info { + struct aw_prof_desc prof_desc[AW_PROFILE_MAX]; +}; + +struct aw_prof_info { + int count; + int status; + int prof_type; + char (*prof_name_list)[AW_PROFILE_STR_MAX]; + struct aw_prof_desc *prof_desc; +}; + +struct acf_bin_info { + int load_count; + int fw_size; + int16_t dev_index; + char *fw_data; + int product_cnt; + const char **product_tab; + struct aw_device *aw_dev; + + struct aw_acf_hdr acf_hdr; + struct aw_prof_info prof_info; +}; + +char *aw87xxx_ctos_get_prof_name(int profile_id); +void aw87xxx_acf_profile_free(struct device *dev, + struct acf_bin_info *acf_info); +int aw87xxx_acf_parse(struct device *dev, struct acf_bin_info *acf_info); +struct aw_prof_desc *aw87xxx_acf_get_prof_desc_form_name(struct device *dev, + struct acf_bin_info *acf_info, char *profile_name); +int aw87xxx_acf_get_prof_index_form_name(struct device *dev, + struct acf_bin_info *acf_info, char *profile_name); +char *aw87xxx_acf_get_prof_name_form_index(struct device *dev, + struct acf_bin_info *acf_info, int index); +int aw87xxx_acf_get_profile_count(struct device *dev, + struct acf_bin_info *acf_info); +char *aw87xxx_acf_get_prof_off_name(struct device *dev, + struct acf_bin_info *acf_info); +void aw87xxx_acf_init(struct aw_device *aw_dev, struct acf_bin_info *acf_info, int index); + + +#endif diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_bin_parse.c b/sound/soc/codecs/aw87xxx/aw87xxx_bin_parse.c new file mode 100644 index 0000000000000..89984a6076b25 --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_bin_parse.c @@ -0,0 +1,514 @@ +/* +* aw87xxx_bin_parse.c +* +* Copyright (c) 2020 AWINIC Technology CO., LTD +* +* This program is free software; you can redistribute it and/or modify it +* under the terms of the GNU General Public License as published by the +* Free Software Foundation; either version 2 of the License, or (at your +* option) any later version. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aw87xxx_bin_parse.h" + +#define AWINIC_CODE_VERSION "V0.0.7-V1.0.4" /* "code version"-"excel version" */ + +#define DEBUG_LOG_LEVEL +#ifdef DEBUG_LOG_LEVEL +#define DBG(fmt, arg...) do {\ +printk("AWINIC_BIN %s,line= %d,"fmt, __func__, __LINE__, ##arg);\ +} while (0) +#define DBG_ERR(fmt, arg...) do {\ +printk("AWINIC_BIN_ERR %s,line= %d,"fmt, __func__, __LINE__, ##arg);\ +} while (0) +#else +#define DBG(fmt, arg...) do {} while (0) +#define DBG_ERR(fmt, arg...) do {} while (0) +#endif + +#define printing_data_code + +typedef unsigned short int aw_uint16; +typedef unsigned long int aw_uint32; + +#define BigLittleSwap16(A) ((((aw_uint16)(A) & 0xff00) >> 8) | \ + (((aw_uint16)(A) & 0x00ff) << 8)) + +#define BigLittleSwap32(A) ((((aw_uint32)(A) & 0xff000000) >> 24) | \ + (((aw_uint32)(A) & 0x00ff0000) >> 8) | \ + (((aw_uint32)(A) & 0x0000ff00) << 8) | \ + (((aw_uint32)(A) & 0x000000ff) << 24)) + + +static int aw_parse_bin_header_1_0_0(struct aw_bin *bin); + +/** +* +* Interface function +* +* return value: +* value = 0 :success; +* value = -1 :check bin header version +* value = -2 :check bin data type +* value = -3 :check sum or check bin data len error +* value = -4 :check data version +* value = -5 :check register num +* value = -6 :check dsp reg num +* value = -7 :check soc app num +* value = -8 :bin is NULL point +* +**/ + +/******************************************************** +* +* check sum data +* +********************************************************/ +static int aw_check_sum(struct aw_bin *bin, int bin_num) +{ + unsigned int i = 0; + unsigned int sum_data = 0; + unsigned int check_sum = 0; + unsigned char *p_check_sum = NULL; + + DBG("enter\n"); + + p_check_sum = + &(bin->info.data[(bin->header_info[bin_num].valid_data_addr - + bin->header_info[bin_num].header_len)]); + DBG("aw_bin_parse p_check_sum = %p\n", p_check_sum); + check_sum = GET_32_DATA(*(p_check_sum + 3), + *(p_check_sum + 2), + *(p_check_sum + 1), *(p_check_sum)); + + for (i = 4; + i < + bin->header_info[bin_num].bin_data_len + + bin->header_info[bin_num].header_len; i++) { + sum_data += *(p_check_sum + i); + } + DBG("aw_bin_parse bin_num=%d, check_sum = 0x%x, sum_data = 0x%x\n", + bin_num, check_sum, sum_data); + if (sum_data != check_sum) { + p_check_sum = NULL; + DBG_ERR("aw_bin_parse check sum or check bin data len error\n"); + DBG_ERR("aw_bin_parse bin_num=%d, check_sum = 0x%x, sum_data = 0x%x\n", bin_num, check_sum, sum_data); + return -3; + } + p_check_sum = NULL; + + return 0; +} + +static int aw_check_data_version(struct aw_bin *bin, int bin_num) +{ + int i = 0; + DBG("enter\n"); + + for (i = DATA_VERSION_V1; i < DATA_VERSION_MAX; i++) { + if (bin->header_info[bin_num].bin_data_ver == i) { + return 0; + } + } + DBG_ERR("aw_bin_parse Unrecognized this bin data version\n"); + return -4; +} + +static int aw_check_register_num_v1(struct aw_bin *bin, int bin_num) +{ + unsigned int check_register_num = 0; + unsigned int parse_register_num = 0; + unsigned char *p_check_sum = NULL; + + DBG("enter\n"); + + p_check_sum = + &(bin->info.data[(bin->header_info[bin_num].valid_data_addr)]); + DBG("aw_bin_parse p_check_sum = %p\n", p_check_sum); + parse_register_num = GET_32_DATA(*(p_check_sum + 3), + *(p_check_sum + 2), + *(p_check_sum + 1), *(p_check_sum)); + check_register_num = (bin->header_info[bin_num].bin_data_len - 4) / + (bin->header_info[bin_num].reg_byte_len + + bin->header_info[bin_num].data_byte_len); + DBG + ("aw_bin_parse bin_num=%d, parse_register_num = 0x%x, check_register_num = 0x%x\n", + bin_num, parse_register_num, check_register_num); + if (parse_register_num != check_register_num) { + p_check_sum = NULL; + DBG_ERR("aw_bin_parse register num is error\n"); + DBG_ERR("aw_bin_parse bin_num=%d, parse_register_num = 0x%x, check_register_num = 0x%x\n", bin_num, parse_register_num, check_register_num); + return -5; + } + bin->header_info[bin_num].reg_num = parse_register_num; + bin->header_info[bin_num].valid_data_len = + bin->header_info[bin_num].bin_data_len - 4; + p_check_sum = NULL; + bin->header_info[bin_num].valid_data_addr = + bin->header_info[bin_num].valid_data_addr + 4; + return 0; +} + +static int aw_check_dsp_reg_num_v1(struct aw_bin *bin, int bin_num) +{ + unsigned int check_dsp_reg_num = 0; + unsigned int parse_dsp_reg_num = 0; + unsigned char *p_check_sum = NULL; + + DBG("enter\n"); + + p_check_sum = + &(bin->info.data[(bin->header_info[bin_num].valid_data_addr)]); + DBG("aw_bin_parse p_check_sum = %p\n", p_check_sum); + parse_dsp_reg_num = GET_32_DATA(*(p_check_sum + 7), + *(p_check_sum + 6), + *(p_check_sum + 5), *(p_check_sum + 4)); + bin->header_info[bin_num].reg_data_byte_len = + GET_32_DATA(*(p_check_sum + 11), *(p_check_sum + 10), + *(p_check_sum + 9), *(p_check_sum + 8)); + check_dsp_reg_num = + (bin->header_info[bin_num].bin_data_len - + 12) / bin->header_info[bin_num].reg_data_byte_len; + DBG + ("aw_bin_parse bin_num=%d, parse_dsp_reg_num = 0x%x, check_dsp_reg_num = 0x%x\n", + bin_num, parse_dsp_reg_num, check_dsp_reg_num); + if (parse_dsp_reg_num != check_dsp_reg_num) { + p_check_sum = NULL; + DBG_ERR("aw_bin_parse dsp reg num is error\n"); + DBG_ERR("aw_bin_parse bin_num=%d, parse_dsp_reg_num = 0x%x, check_dsp_reg_num = 0x%x\n", bin_num, parse_dsp_reg_num, check_dsp_reg_num); + return -6; + } + bin->header_info[bin_num].download_addr = + GET_32_DATA(*(p_check_sum + 3), *(p_check_sum + 2), + *(p_check_sum + 1), *(p_check_sum)); + bin->header_info[bin_num].reg_num = parse_dsp_reg_num; + bin->header_info[bin_num].valid_data_len = + bin->header_info[bin_num].bin_data_len - 12; + p_check_sum = NULL; + bin->header_info[bin_num].valid_data_addr = + bin->header_info[bin_num].valid_data_addr + 12; + return 0; +} + +static int aw_check_soc_app_num_v1(struct aw_bin *bin, int bin_num) +{ + unsigned int check_soc_app_num = 0; + unsigned int parse_soc_app_num = 0; + unsigned char *p_check_sum = NULL; + + DBG("enter\n"); + + p_check_sum = + &(bin->info.data[(bin->header_info[bin_num].valid_data_addr)]); + DBG("aw_bin_parse p_check_sum = %p\n", p_check_sum); + bin->header_info[bin_num].app_version = GET_32_DATA(*(p_check_sum + 3), + *(p_check_sum + 2), + *(p_check_sum + 1), + *(p_check_sum)); + parse_soc_app_num = GET_32_DATA(*(p_check_sum + 11), + *(p_check_sum + 10), + *(p_check_sum + 9), *(p_check_sum + 8)); + check_soc_app_num = bin->header_info[bin_num].bin_data_len - 12; + DBG + ("aw_bin_parse bin_num=%d, parse_soc_app_num = 0x%x, check_soc_app_num = 0x%x\n", + bin_num, parse_soc_app_num, check_soc_app_num); + if (parse_soc_app_num != check_soc_app_num) { + p_check_sum = NULL; + DBG_ERR("aw_bin_parse soc app num is error\n"); + DBG_ERR("aw_bin_parse bin_num=%d, parse_soc_app_num = 0x%x, check_soc_app_num = 0x%x\n", bin_num, parse_soc_app_num, check_soc_app_num); + return -7; + } + bin->header_info[bin_num].reg_num = parse_soc_app_num; + bin->header_info[bin_num].download_addr = + GET_32_DATA(*(p_check_sum + 7), *(p_check_sum + 6), + *(p_check_sum + 5), *(p_check_sum + 4)); + bin->header_info[bin_num].valid_data_len = + bin->header_info[bin_num].bin_data_len - 12; + p_check_sum = NULL; + bin->header_info[bin_num].valid_data_addr = + bin->header_info[bin_num].valid_data_addr + 12; + return 0; +} + +/************************ +*** +***bin header 1_0_0 +*** +************************/ +static void aw_get_single_bin_header_1_0_0(struct aw_bin *bin) +{ + int i; + DBG("enter %s\n", __func__); + bin->header_info[bin->all_bin_parse_num].header_len = 60; + bin->header_info[bin->all_bin_parse_num].check_sum = + GET_32_DATA(*(bin->p_addr + 3), *(bin->p_addr + 2), + *(bin->p_addr + 1), *(bin->p_addr)); + bin->header_info[bin->all_bin_parse_num].header_ver = + GET_32_DATA(*(bin->p_addr + 7), *(bin->p_addr + 6), + *(bin->p_addr + 5), *(bin->p_addr + 4)); + bin->header_info[bin->all_bin_parse_num].bin_data_type = + GET_32_DATA(*(bin->p_addr + 11), *(bin->p_addr + 10), + *(bin->p_addr + 9), *(bin->p_addr + 8)); + bin->header_info[bin->all_bin_parse_num].bin_data_ver = + GET_32_DATA(*(bin->p_addr + 15), *(bin->p_addr + 14), + *(bin->p_addr + 13), *(bin->p_addr + 12)); + bin->header_info[bin->all_bin_parse_num].bin_data_len = + GET_32_DATA(*(bin->p_addr + 19), *(bin->p_addr + 18), + *(bin->p_addr + 17), *(bin->p_addr + 16)); + bin->header_info[bin->all_bin_parse_num].ui_ver = + GET_32_DATA(*(bin->p_addr + 23), *(bin->p_addr + 22), + *(bin->p_addr + 21), *(bin->p_addr + 20)); + bin->header_info[bin->all_bin_parse_num].reg_byte_len = + GET_32_DATA(*(bin->p_addr + 35), *(bin->p_addr + 34), + *(bin->p_addr + 33), *(bin->p_addr + 32)); + bin->header_info[bin->all_bin_parse_num].data_byte_len = + GET_32_DATA(*(bin->p_addr + 39), *(bin->p_addr + 38), + *(bin->p_addr + 37), *(bin->p_addr + 36)); + bin->header_info[bin->all_bin_parse_num].device_addr = + GET_32_DATA(*(bin->p_addr + 43), *(bin->p_addr + 42), + *(bin->p_addr + 41), *(bin->p_addr + 40)); + for (i = 0; i < 8; i++) { + bin->header_info[bin->all_bin_parse_num].chip_type[i] = + *(bin->p_addr + 24 + i); + } + bin->header_info[bin->all_bin_parse_num].reg_num = 0x00000000; + bin->header_info[bin->all_bin_parse_num].reg_data_byte_len = 0x00000000; + bin->header_info[bin->all_bin_parse_num].download_addr = 0x00000000; + bin->header_info[bin->all_bin_parse_num].app_version = 0x00000000; + bin->header_info[bin->all_bin_parse_num].valid_data_len = 0x00000000; + bin->all_bin_parse_num += 1; +} + +static int aw_parse_each_of_multi_bins_1_0_0(unsigned int bin_num, int bin_serial_num, + struct aw_bin *bin) +{ + int ret = 0; + unsigned int bin_start_addr = 0; + unsigned int valid_data_len = 0; + DBG("aw_bin_parse enter multi bin branch -- %s\n", __func__); + if (!bin_serial_num) { + bin_start_addr = GET_32_DATA(*(bin->p_addr + 67), + *(bin->p_addr + 66), + *(bin->p_addr + 65), + *(bin->p_addr + 64)); + bin->p_addr += (60 + bin_start_addr); + bin->header_info[bin->all_bin_parse_num].valid_data_addr = + bin->header_info[bin->all_bin_parse_num - + 1].valid_data_addr + 4 + 8 * bin_num + 60; + } else { + valid_data_len = + bin->header_info[bin->all_bin_parse_num - 1].bin_data_len; + bin->p_addr += (60 + valid_data_len); + bin->header_info[bin->all_bin_parse_num].valid_data_addr = + bin->header_info[bin->all_bin_parse_num - + 1].valid_data_addr + + bin->header_info[bin->all_bin_parse_num - 1].bin_data_len + + 60; + } + + ret = aw_parse_bin_header_1_0_0(bin); + return ret; +} + +/* Get the number of bins in multi bins, and set a for loop, loop processing each bin data */ +static int aw_get_multi_bin_header_1_0_0(struct aw_bin *bin) +{ + int i = 0; + int ret = 0; + unsigned int bin_num = 0; + DBG("aw_bin_parse enter multi bin branch -- %s\n", __func__); + bin_num = GET_32_DATA(*(bin->p_addr + 63), + *(bin->p_addr + 62), + *(bin->p_addr + 61), *(bin->p_addr + 60)); + if (bin->multi_bin_parse_num == 1) { + bin->header_info[bin->all_bin_parse_num].valid_data_addr = 60; + } + aw_get_single_bin_header_1_0_0(bin); + + for (i = 0; i < bin_num; i++) { + DBG("aw_bin_parse enter multi bin for is %d\n", i); + ret = aw_parse_each_of_multi_bins_1_0_0(bin_num, i, bin); + if (ret < 0) { + return ret; + } + } + return 0; +} + +/******************************************************** +* +* If the bin framework header version is 1.0.0, + determine the data type of bin, and then perform different processing + according to the data type + If it is a single bin data type, write the data directly into the structure array + If it is a multi-bin data type, first obtain the number of bins, + and then recursively call the bin frame header processing function + according to the bin number to process the frame header information of each bin separately +* +********************************************************/ +static int aw_parse_bin_header_1_0_0(struct aw_bin *bin) +{ + int ret = 0; + unsigned int bin_data_type; + DBG("enter %s\n", __func__); + bin_data_type = GET_32_DATA(*(bin->p_addr + 11), + *(bin->p_addr + 10), + *(bin->p_addr + 9), *(bin->p_addr + 8)); + DBG("aw_bin_parse bin_data_type 0x%x\n", bin_data_type); + switch (bin_data_type) { + case DATA_TYPE_REGISTER: + case DATA_TYPE_DSP_REG: + case DATA_TYPE_SOC_APP: + /* Divided into two processing methods, + one is single bin processing, + and the other is single bin processing in multi bin */ + DBG("aw_bin_parse enter single bin branch\n"); + bin->single_bin_parse_num += 1; + DBG("%s bin->single_bin_parse_num is %d\n", __func__, + bin->single_bin_parse_num); + if (!bin->multi_bin_parse_num) { + bin->header_info[bin-> + all_bin_parse_num].valid_data_addr = + 60; + } + aw_get_single_bin_header_1_0_0(bin); + break; + case DATA_TYPE_MULTI_BINS: + /* Get the number of times to enter multi bins */ + DBG("aw_bin_parse enter multi bin branch\n"); + bin->multi_bin_parse_num += 1; + DBG("%s bin->multi_bin_parse_num is %d\n", __func__, + bin->multi_bin_parse_num); + ret = aw_get_multi_bin_header_1_0_0(bin); + if (ret < 0) { + return ret; + } + break; + default: + DBG_ERR("aw_bin_parse Unrecognized this bin data type\n"); + return -2; + } + return 0; +} + +/* get the bin's header version */ +static int aw_check_bin_header_version(struct aw_bin *bin) +{ + int ret = 0; + unsigned int header_version = 0; + + header_version = GET_32_DATA(*(bin->p_addr + 7), + *(bin->p_addr + 6), + *(bin->p_addr + 5), *(bin->p_addr + 4)); + + DBG("aw_bin_parse header_version 0x%x\n", header_version); + + /* Write data to the corresponding structure array + according to different formats of the bin frame header version */ + switch (header_version) { + case HEADER_VERSION_1_0_0: + ret = aw_parse_bin_header_1_0_0(bin); + return ret; + default: + DBG_ERR("aw_bin_parse Unrecognized this bin header version \n"); + return -1; + } +} + +int aw87xxx_parsing_bin_file(struct aw_bin *bin) +{ + int i = 0; + int ret = 0; + + DBG("aw_bin_parse code version:%s\n", AWINIC_CODE_VERSION); + if (!bin) { + DBG_ERR("aw_bin_parse bin is NULL\n"); + return -8; + } + bin->p_addr = bin->info.data; + bin->all_bin_parse_num = 0; + bin->multi_bin_parse_num = 0; + bin->single_bin_parse_num = 0; + + /* filling bins header info */ + ret = aw_check_bin_header_version(bin); + if (ret < 0) { + DBG_ERR("aw_bin_parse check bin header version error\n"); + return ret; + } + bin->p_addr = NULL; + + /* check bin header info */ + for (i = 0; i < bin->all_bin_parse_num; i++) { + /* check sum */ + ret = aw_check_sum(bin, i); + if (ret < 0) { + DBG_ERR("aw_bin_parse check sum data error\n"); + return ret; + } + /* check bin data version */ + ret = aw_check_data_version(bin, i); + if (ret < 0) { + DBG_ERR("aw_bin_parse check data version error\n"); + return ret; + } + /* check valid data */ + if (bin->header_info[i].bin_data_ver == DATA_VERSION_V1) { + /* check register num */ + if (bin->header_info[i].bin_data_type == + DATA_TYPE_REGISTER) { + ret = aw_check_register_num_v1(bin, i); + if (ret < 0) { + DBG_ERR + ("aw_bin_parse check register num error\n"); + return ret; + } + /* check dsp reg num */ + } else if (bin->header_info[i].bin_data_type == + DATA_TYPE_DSP_REG) { + ret = aw_check_dsp_reg_num_v1(bin, i); + if (ret < 0) { + DBG_ERR + ("aw_bin_parse check dsp reg num error\n"); + return ret; + } + /* check soc app num */ + } else if (bin->header_info[i].bin_data_type == + DATA_TYPE_SOC_APP) { + ret = aw_check_soc_app_num_v1(bin, i); + if (ret < 0) { + DBG_ERR + ("aw_bin_parse check soc app num error\n"); + return ret; + } + } else { + bin->header_info[i].valid_data_len = + bin->header_info[i].bin_data_len; + } + } + } + DBG("aw_bin_parse parsing success\n"); + + return 0; +} diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_bin_parse.h b/sound/soc/codecs/aw87xxx/aw87xxx_bin_parse.h new file mode 100644 index 0000000000000..a99c2409e6133 --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_bin_parse.h @@ -0,0 +1,73 @@ +#ifndef __AW87XXX_BIN_PARSE_H__ +#define __AW87XXX_BIN_PARSE_H__ + +#define NULL ((void *)0) +#define GET_32_DATA(w, x, y, z) ((unsigned int)(((w) << 24) | ((x) << 16) | ((y) << 8) | (z))) +#define BIN_NUM_MAX 100 +#define HEADER_LEN 60 +/********************************************************* + * + * header information + * + ********************************************************/ +enum bin_header_version_enum { + HEADER_VERSION_1_0_0 = 0x01000000, +}; + +enum data_type_enum { + DATA_TYPE_REGISTER = 0x00000000, + DATA_TYPE_DSP_REG = 0x00000010, + DATA_TYPE_DSP_CFG = 0x00000011, + DATA_TYPE_SOC_REG = 0x00000020, + DATA_TYPE_SOC_APP = 0x00000021, + DATA_TYPE_MULTI_BINS = 0x00002000, + DATA_TYPE_MONITOR_ANALOG = 0x00020000, +}; + +enum data_version_enum { + DATA_VERSION_V1 = 0X00000001, /*default little edian */ + DATA_VERSION_MAX, +}; + +struct bin_header_info { + unsigned int header_len; /* Frame header length */ + unsigned int check_sum; /* Frame header information-Checksum */ + unsigned int header_ver; /* Frame header information-Frame header version */ + unsigned int bin_data_type; /* Frame header information-Data type */ + unsigned int bin_data_ver; /* Frame header information-Data version */ + unsigned int bin_data_len; /* Frame header information-Data length */ + unsigned int ui_ver; /* Frame header information-ui version */ + unsigned char chip_type[8]; /* Frame header information-chip type */ + unsigned int reg_byte_len; /* Frame header information-reg byte len */ + unsigned int data_byte_len; /* Frame header information-data byte len */ + unsigned int device_addr; /* Frame header information-device addr */ + unsigned int valid_data_len; /* Length of valid data obtained after parsing */ + unsigned int valid_data_addr; /* The offset address of the valid data obtained after parsing relative to info */ + + unsigned int reg_num; /* The number of registers obtained after parsing */ + unsigned int reg_data_byte_len; /* The byte length of the register obtained after parsing */ + unsigned int download_addr; /* The starting address or download address obtained after parsing */ + unsigned int app_version; /* The software version number obtained after parsing */ +}; + +/************************************************************ +* +* function define +* +************************************************************/ +struct bin_container { + unsigned int len; /* The size of the bin file obtained from the firmware */ + unsigned char data[]; /* Store the bin file obtained from the firmware */ +}; + +struct aw_bin { + unsigned char *p_addr; /* Offset pointer (backward offset pointer to obtain frame header information and important information) */ + unsigned int all_bin_parse_num; /* The number of all bin files */ + unsigned int multi_bin_parse_num; /* The number of single bin files */ + unsigned int single_bin_parse_num; /* The number of multiple bin files */ + struct bin_header_info header_info[BIN_NUM_MAX]; /* Frame header information and other important data obtained after parsing */ + struct bin_container info; /* Obtained bin file data that needs to be parsed */ +}; + +extern int aw87xxx_parsing_bin_file(struct aw_bin *bin); +#endif diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_device.c b/sound/soc/codecs/aw87xxx/aw87xxx_device.c new file mode 100644 index 0000000000000..8619aacc2ccfe --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_device.c @@ -0,0 +1,976 @@ +/* + * aw87xxx_device.c aw87xxx pa module + * + * Copyright (c) 2021 AWINIC Technology CO., LTD + * + * Author: Barry + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aw87xxx.h" +#include "aw87xxx_device.h" +#include "aw87xxx_log.h" +#include "aw87xxx_pid_9b_reg.h" +#include "aw87xxx_pid_18_reg.h" +#include "aw87xxx_pid_39_reg.h" +#include "aw87xxx_pid_59_3x9_reg.h" +#include "aw87xxx_pid_59_5x9_reg.h" +#include "aw87xxx_pid_5a_reg.h" +#include "aw87xxx_pid_76_reg.h" +#include "aw87xxx_pid_60_reg.h" + +/************************************************************************* + * aw87xxx variable + ************************************************************************/ +const char *g_aw_pid_9b_product[] = { + "aw87319", +}; +const char *g_aw_pid_18_product[] = { + "aw87418", +}; + +const char *g_aw_pid_39_product[] = { + "aw87329", + "aw87339", + "aw87349", +}; + +const char *g_aw_pid_59_3x9_product[] = { + "aw87359", + "aw87389", +}; + +const char *g_aw_pid_59_5x9_product[] = { + "aw87509", + "aw87519", + "aw87529", + "aw87539", +}; + +const char *g_aw_pid_5a_product[] = { + "aw87549", + "aw87559", + "aw87569", + "aw87579", + "aw81509", +}; + +const char *g_aw_pid_76_product[] = { + "aw87390", + "aw87320", + "aw87401", + "aw87360", +}; + +const char *g_aw_pid_60_product[] = { + "aw87560", + "aw87561", + "aw87562", + "aw87501", + "aw87550", +}; + +static int aw87xxx_dev_get_chipid(struct aw_device *aw_dev); + +/*************************************************************************** + * + * reading and writing of I2C bus + * + ***************************************************************************/ +int aw87xxx_dev_i2c_write_byte(struct aw_device *aw_dev, + uint8_t reg_addr, uint8_t reg_data) +{ + int ret = -1; + unsigned char cnt = 0; + + while (cnt < AW_I2C_RETRIES) { + ret = i2c_smbus_write_byte_data(aw_dev->i2c, reg_addr, reg_data); + if (ret < 0) + AW_DEV_LOGE(aw_dev->dev, "i2c_write cnt=%d error=%d i2c_bus=%u i2c_addr=%X chipid=%X", + cnt, ret, aw_dev->i2c_bus, aw_dev->i2c_addr, aw_dev->chipid); + else + break; + + cnt++; + msleep(AW_I2C_RETRY_DELAY); + } + + return ret; +} + +int aw87xxx_dev_i2c_read_byte(struct aw_device *aw_dev, + uint8_t reg_addr, uint8_t *reg_data) +{ + int ret = -1; + unsigned char cnt = 0; + + while (cnt < AW_I2C_RETRIES) { + ret = i2c_smbus_read_byte_data(aw_dev->i2c, reg_addr); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "i2c_read cnt=%d error=%d i2c_bus=%u i2c_addr=%X chipid=%X", + cnt, ret, aw_dev->i2c_bus, aw_dev->i2c_addr, aw_dev->chipid); + } else { + *reg_data = ret; + break; + } + cnt++; + msleep(AW_I2C_RETRY_DELAY); + } + + return ret; +} + +int aw87xxx_dev_i2c_read_msg(struct aw_device *aw_dev, + uint8_t reg_addr, uint8_t *data_buf, uint32_t data_len) +{ + int ret = -1; + + struct i2c_msg msg[] = { + [0] = { + .addr = aw_dev->i2c_addr, + .flags = 0, + .len = sizeof(uint8_t), + .buf = ®_addr, + }, + [1] = { + .addr = aw_dev->i2c_addr, + .flags = I2C_M_RD, + .len = data_len, + .buf = data_buf, + }, + }; + + ret = i2c_transfer(aw_dev->i2c->adapter, msg, ARRAY_SIZE(msg)); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "transfer failed"); + return ret; + } else if (ret != AW_I2C_READ_MSG_NUM) { + AW_DEV_LOGE(aw_dev->dev, "transfer failed(size error)"); + return -ENXIO; + } + + return 0; +} + +int aw87xxx_dev_i2c_write_bits(struct aw_device *aw_dev, + uint8_t reg_addr, uint8_t mask, uint8_t reg_data) +{ + int ret = -1; + unsigned char reg_val = 0; + + ret = aw87xxx_dev_i2c_read_byte(aw_dev, reg_addr, ®_val); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "i2c read error, ret=%d", ret); + return ret; + } + reg_val &= mask; + reg_val |= (reg_data & (~mask)); + ret = aw87xxx_dev_i2c_write_byte(aw_dev, reg_addr, reg_val); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "i2c write error, ret=%d", ret); + return ret; + } + + return 0; +} + +/************************************************************************ + * + * aw87xxx device update profile data to registers + * + ************************************************************************/ +static int aw87xxx_dev_reg_update(struct aw_device *aw_dev, + struct aw_data_container *profile_data) +{ + int i = 0; + int ret = -1; + + if (profile_data == NULL) + return -EINVAL; + + if (aw_dev->hwen_status == AW_DEV_HWEN_OFF) { + AW_DEV_LOGE(aw_dev->dev, "dev is pwr_off,can not update reg"); + return -EINVAL; + } + + for (i = 0; i < profile_data->len; i = i + 2) { + AW_DEV_LOGI(aw_dev->dev, "reg=0x%02x, val = 0x%02x", + profile_data->data[i], profile_data->data[i + 1]); + + ret = aw87xxx_dev_i2c_write_byte(aw_dev, profile_data->data[i], + profile_data->data[i + 1]); + if (ret < 0) + return ret; + } + + return 0; +} + +static void aw87xxx_dev_reg_mute_bits_set(struct aw_device *aw_dev, + uint8_t *reg_val, bool enable) +{ + if (enable) { + *reg_val &= aw_dev->mute_desc.mask; + *reg_val |= aw_dev->mute_desc.enable; + } else { + *reg_val &= aw_dev->mute_desc.mask; + *reg_val |= aw_dev->mute_desc.disable; + } +} + +static int aw87xxx_dev_reg_update_mute(struct aw_device *aw_dev, + struct aw_data_container *profile_data) +{ + int i = 0; + int ret = -1; + uint8_t reg_val = 0; + + if (profile_data == NULL) + return -EINVAL; + + if (aw_dev->hwen_status == AW_DEV_HWEN_OFF) { + AW_DEV_LOGE(aw_dev->dev, "hwen is off,can not update reg"); + return -EINVAL; + } + + if (aw_dev->mute_desc.mask == AW_DEV_REG_INVALID_MASK) { + AW_DEV_LOGE(aw_dev->dev, "mute ctrl mask invalid"); + return -EINVAL; + } + + for (i = 0; i < profile_data->len; i = i + 2) { + AW_DEV_LOGI(aw_dev->dev, "reg=0x%02x, val = 0x%02x", + profile_data->data[i], profile_data->data[i + 1]); + + reg_val = profile_data->data[i + 1]; + if (profile_data->data[i] == aw_dev->mute_desc.addr) { + aw87xxx_dev_reg_mute_bits_set(aw_dev, ®_val, true); + AW_DEV_LOGD(aw_dev->dev, "change mute_mask, val = 0x%02x", + reg_val); + } + + ret = aw87xxx_dev_i2c_write_byte(aw_dev, profile_data->data[i], reg_val); + if (ret < 0) + return ret; + } + + return 0; +} + +/************************************************************************ + * + * aw87xxx device hadware and soft contols + * + ************************************************************************/ +static bool aw87xxx_dev_gpio_is_valid(struct aw_device *aw_dev) +{ + if (gpio_is_valid(aw_dev->rst_gpio)) + return true; + else + return false; +} + +void aw87xxx_dev_hw_pwr_ctrl(struct aw_device *aw_dev, bool enable) +{ + if (aw_dev->hwen_status == AW_DEV_HWEN_INVALID) { + AW_DEV_LOGD(aw_dev->dev, "product not have reset-pin,hardware pwd control invalid"); + return; + } + if (enable) { + if (aw87xxx_dev_gpio_is_valid(aw_dev)) { + gpio_set_value_cansleep(aw_dev->rst_gpio, AW_GPIO_LOW_LEVEL); + mdelay(2); + gpio_set_value_cansleep(aw_dev->rst_gpio, AW_GPIO_HIGHT_LEVEL); + mdelay(2); + aw_dev->hwen_status = AW_DEV_HWEN_ON; + AW_DEV_LOGI(aw_dev->dev, "hw power on"); + } else { + AW_DEV_LOGI(aw_dev->dev, "hw already power on"); + } + } else { + if (aw87xxx_dev_gpio_is_valid(aw_dev)) { + gpio_set_value_cansleep(aw_dev->rst_gpio, AW_GPIO_LOW_LEVEL); + mdelay(2); + aw_dev->hwen_status = AW_DEV_HWEN_OFF; + AW_DEV_LOGI(aw_dev->dev, "hw power off"); + } else { + AW_DEV_LOGI(aw_dev->dev, "hw already power off"); + } + } +} + +static int aw87xxx_dev_mute_ctrl(struct aw_device *aw_dev, bool enable) +{ + int ret = 0; + + if (enable) { + ret = aw87xxx_dev_i2c_write_bits(aw_dev, aw_dev->mute_desc.addr, + aw_dev->mute_desc.mask, aw_dev->mute_desc.enable); + if (ret < 0) + return ret; + AW_DEV_LOGI(aw_dev->dev, "set mute down"); + } else { + ret = aw87xxx_dev_i2c_write_bits(aw_dev, aw_dev->mute_desc.addr, + aw_dev->mute_desc.mask, aw_dev->mute_desc.disable); + if (ret < 0) + return ret; + AW_DEV_LOGI(aw_dev->dev, "close mute down"); + } + + return 0; +} + +void aw87xxx_dev_soft_reset(struct aw_device *aw_dev) +{ + int i = 0; + int ret = -1; + struct aw_soft_rst_desc *soft_rst = &aw_dev->soft_rst_desc; + + AW_DEV_LOGD(aw_dev->dev, "enter"); + + if (aw_dev->hwen_status == AW_DEV_HWEN_OFF) { + AW_DEV_LOGE(aw_dev->dev, "hw is off,can not softrst"); + return; + } + + if (aw_dev->soft_rst_enable == AW_DEV_SOFT_RST_DISENABLE) { + AW_DEV_LOGD(aw_dev->dev, "softrst is disenable"); + return; + } + + if (soft_rst->access == NULL || soft_rst->len == 0) { + AW_DEV_LOGE(aw_dev->dev, "softrst_info not init"); + return; + } + + if (soft_rst->len % 2) { + AW_DEV_LOGE(aw_dev->dev, "softrst data_len[%d] is odd number,data not available", + aw_dev->soft_rst_desc.len); + return; + } + + for (i = 0; i < soft_rst->len; i += 2) { + AW_DEV_LOGD(aw_dev->dev, "softrst_reg=0x%02x, val = 0x%02x", + soft_rst->access[i], soft_rst->access[i + 1]); + + ret = aw87xxx_dev_i2c_write_byte(aw_dev, soft_rst->access[i], + soft_rst->access[i + 1]); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "write failed,ret = %d,cnt=%d", + ret, i); + return; + } + } + AW_DEV_LOGD(aw_dev->dev, "down"); +} + + +int aw87xxx_dev_default_pwr_off(struct aw_device *aw_dev, + struct aw_data_container *profile_data) +{ + int ret = 0; + + AW_DEV_LOGD(aw_dev->dev, "enter"); + if (aw_dev->hwen_status == AW_DEV_HWEN_OFF) { + AW_DEV_LOGE(aw_dev->dev, "hwen is already off"); + return 0; + } + + if (aw_dev->soft_off_enable && profile_data) { + ret = aw87xxx_dev_reg_update(aw_dev, profile_data); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "update profile[Off] fw config failed"); + goto reg_off_update_failed; + } + } + + aw87xxx_dev_hw_pwr_ctrl(aw_dev, false); + AW_DEV_LOGD(aw_dev->dev, "down"); + return 0; + +reg_off_update_failed: + aw87xxx_dev_hw_pwr_ctrl(aw_dev, false); + return ret; +} + + +/************************************************************************ + * + * aw87xxx device power on process function + * + ************************************************************************/ + +int aw87xxx_dev_default_pwr_on(struct aw_device *aw_dev, + struct aw_data_container *profile_data) +{ + int ret = 0; + + /*hw power on*/ + aw87xxx_dev_hw_pwr_ctrl(aw_dev, true); + + ret = aw87xxx_dev_reg_update(aw_dev, profile_data); + if (ret < 0) + return ret; + + return 0; +} + +/**************************************************************************** + * + * aw87xxx chip esd status check + * + ****************************************************************************/ +int aw87xxx_dev_esd_reg_status_check(struct aw_device *aw_dev) +{ + int ret; + unsigned char reg_val = 0; + struct aw_esd_check_desc *esd_desc = &aw_dev->esd_desc; + + AW_DEV_LOGD(aw_dev->dev, "enter"); + + if (!esd_desc->first_update_reg_addr) { + AW_DEV_LOGE(aw_dev->dev, "esd check info if not init,please check"); + return -EINVAL; + } + + ret = aw87xxx_dev_i2c_read_byte(aw_dev, esd_desc->first_update_reg_addr, + ®_val); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "read reg 0x%02x failed", + esd_desc->first_update_reg_addr); + return ret; + } + + AW_DEV_LOGD(aw_dev->dev, "0x%02x:default val=0x%02x real val=0x%02x", + esd_desc->first_update_reg_addr, + esd_desc->first_update_reg_val, reg_val); + + if (reg_val == esd_desc->first_update_reg_val) { + AW_DEV_LOGE(aw_dev->dev, "reg status check failed"); + return -EINVAL; + } + return 0; +} + +int aw87xxx_dev_check_reg_is_rec_mode(struct aw_device *aw_dev) +{ + int ret; + unsigned char reg_val = 0; + struct aw_rec_mode_desc *rec_desc = &aw_dev->rec_desc; + + if (!rec_desc->addr) { + AW_DEV_LOGE(aw_dev->dev, "rec check info if not init,please check"); + return -EINVAL; + } + + ret = aw87xxx_dev_i2c_read_byte(aw_dev, rec_desc->addr, ®_val); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "read reg 0x%02x failed", + rec_desc->addr); + return ret; + } + + if (rec_desc->enable) { + if (reg_val & ~(rec_desc->mask)) { + AW_DEV_LOGI(aw_dev->dev, "reg status is receiver mode"); + aw_dev->is_rec_mode = AW_IS_REC_MODE; + } else { + aw_dev->is_rec_mode = AW_NOT_REC_MODE; + } + } else { + if (!(reg_val & ~(rec_desc->mask))) { + AW_DEV_LOGI(aw_dev->dev, "reg status is receiver mode"); + aw_dev->is_rec_mode = AW_IS_REC_MODE; + } else { + aw_dev->is_rec_mode = AW_NOT_REC_MODE; + } + } + return 0; +} + + +/**************************************************************************** + * + * aw87xxx product attributes init info + * + ****************************************************************************/ + +/********************** aw87xxx_pid_9A attributes ***************************/ + +static int aw_dev_pid_9b_reg_update(struct aw_device *aw_dev, + struct aw_data_container *profile_data) +{ + int i = 0; + int ret = -1; + uint8_t reg_val = 0; + + if (profile_data == NULL) + return -EINVAL; + + if (aw_dev->hwen_status == AW_DEV_HWEN_OFF) { + AW_DEV_LOGE(aw_dev->dev, "dev is pwr_off,can not update reg"); + return -EINVAL; + } + + if (profile_data->len != AW_PID_9B_BIN_REG_CFG_COUNT) { + AW_DEV_LOGE(aw_dev->dev, "reg_config count of bin is error,can not update reg"); + return -EINVAL; + } + ret = aw87xxx_dev_i2c_write_byte(aw_dev, AW87XXX_PID_9B_ENCRYPTION_REG, + AW87XXX_PID_9B_ENCRYPTION_BOOST_OUTPUT_SET); + if (ret < 0) + return ret; + + for (i = 1; i < AW_PID_9B_BIN_REG_CFG_COUNT; i++) { + AW_DEV_LOGI(aw_dev->dev, "reg=0x%02x, val = 0x%02x", + i, profile_data->data[i]); + reg_val = profile_data->data[i]; + if (i == AW87XXX_PID_9B_SYSCTRL_REG) { + aw87xxx_dev_reg_mute_bits_set(aw_dev, ®_val, true); + AW_DEV_LOGD(aw_dev->dev, "change mute_mask, val = 0x%02x", + reg_val); + } + + ret = aw87xxx_dev_i2c_write_byte(aw_dev, i, reg_val); + if (ret < 0) + return ret; + } + + return 0; +} + +static int aw_dev_pid_9b_pwr_on(struct aw_device *aw_dev, struct aw_data_container *data) +{ + int ret = 0; + + /*hw power on*/ + aw87xxx_dev_hw_pwr_ctrl(aw_dev, true); + + /* open the mute */ + ret = aw87xxx_dev_mute_ctrl(aw_dev, true); + if (ret < 0) + return ret; + + /* Update scene parameters in mute mode */ + ret = aw_dev_pid_9b_reg_update(aw_dev, data); + if (ret < 0) + return ret; + + /* close the mute */ + ret = aw87xxx_dev_mute_ctrl(aw_dev, false); + if (ret < 0) + return ret; + + return 0; +} + +static void aw_dev_pid_9b_init(struct aw_device *aw_dev) +{ + /* Product register permission info */ + aw_dev->reg_max_addr = AW87XXX_PID_9B_REG_MAX; + aw_dev->reg_access = aw87xxx_pid_9b_reg_access; + + aw_dev->mute_desc.addr = AW87XXX_PID_9B_SYSCTRL_REG; + aw_dev->mute_desc.mask = AW87XXX_PID_9B_REG_EN_SW_MASK; + aw_dev->mute_desc.enable = AW87XXX_PID_9B_REG_EN_SW_DISABLE_VALUE; + aw_dev->mute_desc.disable = AW87XXX_PID_9B_REG_EN_SW_ENABLE_VALUE; + aw_dev->ops.pwr_on_func = aw_dev_pid_9b_pwr_on; + + /* software reset control info */ + aw_dev->soft_rst_desc.len = sizeof(aw87xxx_pid_9b_softrst_access); + aw_dev->soft_rst_desc.access = aw87xxx_pid_9b_softrst_access; + aw_dev->soft_rst_enable = AW_DEV_SOFT_RST_ENABLE; + + /* Whether to allow register operation to power off */ + aw_dev->soft_off_enable = AW_DEV_SOFT_OFF_DISENABLE; + + aw_dev->product_tab = g_aw_pid_9b_product; + aw_dev->product_cnt = AW87XXX_PID_9B_PRODUCT_MAX; + + aw_dev->rec_desc.addr = AW87XXX_PID_9B_SYSCTRL_REG; + aw_dev->rec_desc.disable = AW87XXX_PID_9B_SPK_MODE_ENABLE; + aw_dev->rec_desc.enable = AW87XXX_PID_9B_SPK_MODE_DISABLE; + aw_dev->rec_desc.mask = AW87XXX_PID_9B_SPK_MODE_MASK; + + /* esd reg info */ + aw_dev->esd_desc.first_update_reg_addr = AW87XXX_PID_9B_SYSCTRL_REG; + aw_dev->esd_desc.first_update_reg_val = AW87XXX_PID_9B_SYSCTRL_DEFAULT; +} + +static int aw_dev_pid_9a_init(struct aw_device *aw_dev) +{ + int ret = 0; + + ret = aw87xxx_dev_i2c_write_byte(aw_dev, AW87XXX_PID_9B_ENCRYPTION_REG, + AW87XXX_PID_9B_ENCRYPTION_BOOST_OUTPUT_SET); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "write 0x64=0x2C error"); + return -EINVAL; + } + + ret = aw87xxx_dev_get_chipid(aw_dev); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "read chipid is failed,ret=%d", ret); + return ret; + } + + if (aw_dev->chipid == AW_DEV_CHIPID_9B) { + AW_DEV_LOGI(aw_dev->dev, "product is pid_9B class"); + aw_dev_pid_9b_init(aw_dev); + } else { + AW_DEV_LOGE(aw_dev->dev, "product is not pid_9B class,not support"); + return -EINVAL; + } + + return 0; +} + +/********************** aw87xxx_pid_9b attributes end ***********************/ + +/********************** aw87xxx_pid_18 attributes ***************************/ +static int aw_dev_pid_18_pwr_on(struct aw_device *aw_dev, struct aw_data_container *data) +{ + int ret = 0; + + /*hw power on*/ + aw87xxx_dev_hw_pwr_ctrl(aw_dev, true); + + /* open the mute */ + ret = aw87xxx_dev_mute_ctrl(aw_dev, true); + if (ret < 0) + return ret; + + /* Update scene parameters in mute mode */ + ret = aw87xxx_dev_reg_update_mute(aw_dev, data); + if (ret < 0) + return ret; + + /* close the mute */ + ret = aw87xxx_dev_mute_ctrl(aw_dev, false); + if (ret < 0) + return ret; + + return 0; +} + +static void aw_dev_chipid_18_init(struct aw_device *aw_dev) +{ + /* Product register permission info */ + aw_dev->reg_max_addr = AW87XXX_PID_18_REG_MAX; + aw_dev->reg_access = aw87xxx_pid_18_reg_access; + + aw_dev->mute_desc.addr = AW87XXX_PID_18_SYSCTRL_REG; + aw_dev->mute_desc.mask = AW87XXX_PID_18_REG_EN_SW_MASK; + aw_dev->mute_desc.enable = AW87XXX_PID_18_REG_EN_SW_DISABLE_VALUE; + aw_dev->mute_desc.disable = AW87XXX_PID_18_REG_EN_SW_ENABLE_VALUE; + aw_dev->ops.pwr_on_func = aw_dev_pid_18_pwr_on; + + /* software reset control info */ + aw_dev->soft_rst_desc.len = sizeof(aw87xxx_pid_18_softrst_access); + aw_dev->soft_rst_desc.access = aw87xxx_pid_18_softrst_access; + aw_dev->soft_rst_enable = AW_DEV_SOFT_RST_ENABLE; + + /* Whether to allow register operation to power off */ + aw_dev->soft_off_enable = AW_DEV_SOFT_OFF_ENABLE; + + aw_dev->product_tab = g_aw_pid_18_product; + aw_dev->product_cnt = AW87XXX_PID_18_PRODUCT_MAX; + + aw_dev->rec_desc.addr = AW87XXX_PID_18_SYSCTRL_REG; + aw_dev->rec_desc.disable = AW87XXX_PID_18_REG_REC_MODE_DISABLE; + aw_dev->rec_desc.enable = AW87XXX_PID_18_REG_REC_MODE_ENABLE; + aw_dev->rec_desc.mask = AW87XXX_PID_18_REG_REC_MODE_MASK; + + /* esd reg info */ + aw_dev->esd_desc.first_update_reg_addr = AW87XXX_PID_18_CLASSD_REG; + aw_dev->esd_desc.first_update_reg_val = AW87XXX_PID_18_CLASSD_DEFAULT; +} +/********************** aw87xxx_pid_18 attributes end ***********************/ + +/********************** aw87xxx_pid_39 attributes ***************************/ +static void aw_dev_chipid_39_init(struct aw_device *aw_dev) +{ + /* Product register permission info */ + aw_dev->reg_max_addr = AW87XXX_PID_39_REG_MAX; + aw_dev->reg_access = aw87xxx_pid_39_reg_access; + + /* software reset control info */ + aw_dev->soft_rst_desc.len = sizeof(aw87xxx_pid_39_softrst_access); + aw_dev->soft_rst_desc.access = aw87xxx_pid_39_softrst_access; + aw_dev->soft_rst_enable = AW_DEV_SOFT_RST_ENABLE; + + /* Whether to allow register operation to power off */ + aw_dev->soft_off_enable = AW_DEV_SOFT_OFF_ENABLE; + + aw_dev->product_tab = g_aw_pid_39_product; + aw_dev->product_cnt = AW87XXX_PID_39_PRODUCT_MAX; + + aw_dev->rec_desc.addr = AW87XXX_PID_39_REG_MODECTRL; + aw_dev->rec_desc.disable = AW87XXX_PID_39_REC_MODE_DISABLE; + aw_dev->rec_desc.enable = AW87XXX_PID_39_REC_MODE_ENABLE; + aw_dev->rec_desc.mask = AW87XXX_PID_39_REC_MODE_MASK; + + /* esd reg info */ + aw_dev->esd_desc.first_update_reg_addr = AW87XXX_PID_39_REG_MODECTRL; + aw_dev->esd_desc.first_update_reg_val = AW87XXX_PID_39_MODECTRL_DEFAULT; +} +/********************* aw87xxx_pid_39 attributes end *************************/ + + +/********************* aw87xxx_pid_59_5x9 attributes *************************/ +static void aw_dev_chipid_59_5x9_init(struct aw_device *aw_dev) +{ + /* Product register permission info */ + aw_dev->reg_max_addr = AW87XXX_PID_59_5X9_REG_MAX; + aw_dev->reg_access = aw87xxx_pid_59_5x9_reg_access; + + /* software reset control info */ + aw_dev->soft_rst_desc.len = sizeof(aw87xxx_pid_59_5x9_softrst_access); + aw_dev->soft_rst_desc.access = aw87xxx_pid_59_5x9_softrst_access; + aw_dev->soft_rst_enable = AW_DEV_SOFT_RST_ENABLE; + + /* Whether to allow register operation to power off */ + aw_dev->soft_off_enable = AW_DEV_SOFT_OFF_ENABLE; + + aw_dev->product_tab = g_aw_pid_59_5x9_product; + aw_dev->product_cnt = AW87XXX_PID_59_5X9_PRODUCT_MAX; + + aw_dev->rec_desc.addr = AW87XXX_PID_59_5X9_REG_SYSCTRL; + aw_dev->rec_desc.disable = AW87XXX_PID_59_5X9_REC_MODE_DISABLE; + aw_dev->rec_desc.enable = AW87XXX_PID_59_5X9_REC_MODE_ENABLE; + aw_dev->rec_desc.mask = AW87XXX_PID_59_5X9_REC_MODE_MASK; + + /* esd reg info */ + aw_dev->esd_desc.first_update_reg_addr = AW87XXX_PID_59_5X9_REG_ENCR; + aw_dev->esd_desc.first_update_reg_val = AW87XXX_PID_59_5X9_ENCRY_DEFAULT; +} +/******************* aw87xxx_pid_59_5x9 attributes end ***********************/ + +/********************* aw87xxx_pid_59_3x9 attributes *************************/ +static void aw_dev_chipid_59_3x9_init(struct aw_device *aw_dev) +{ + /* Product register permission info */ + aw_dev->reg_max_addr = AW87XXX_PID_59_3X9_REG_MAX; + aw_dev->reg_access = aw87xxx_pid_59_3x9_reg_access; + + /* software reset control info */ + aw_dev->soft_rst_desc.len = sizeof(aw87xxx_pid_59_3x9_softrst_access); + aw_dev->soft_rst_desc.access = aw87xxx_pid_59_3x9_softrst_access; + aw_dev->soft_rst_enable = AW_DEV_SOFT_RST_ENABLE; + + /* Whether to allow register operation to power off */ + aw_dev->soft_off_enable = AW_DEV_SOFT_OFF_ENABLE; + + aw_dev->product_tab = g_aw_pid_59_3x9_product; + aw_dev->product_cnt = AW87XXX_PID_59_3X9_PRODUCT_MAX; + + aw_dev->rec_desc.addr = AW87XXX_PID_59_3X9_REG_MDCRTL; + aw_dev->rec_desc.disable = AW87XXX_PID_59_3X9_SPK_MODE_ENABLE; + aw_dev->rec_desc.enable = AW87XXX_PID_59_3X9_SPK_MODE_DISABLE; + aw_dev->rec_desc.mask = AW87XXX_PID_59_3X9_SPK_MODE_MASK; + + /* esd reg info */ + aw_dev->esd_desc.first_update_reg_addr = AW87XXX_PID_59_3X9_REG_ENCR; + aw_dev->esd_desc.first_update_reg_val = AW87XXX_PID_59_3X9_ENCR_DEFAULT; +} +/******************* aw87xxx_pid_59_3x9 attributes end ***********************/ + +/********************** aw87xxx_pid_5a attributes ****************************/ +static void aw_dev_chipid_5a_init(struct aw_device *aw_dev) +{ + /* Product register permission info */ + aw_dev->reg_max_addr = AW87XXX_PID_5A_REG_MAX; + aw_dev->reg_access = aw87xxx_pid_5a_reg_access; + + /* software reset control info */ + aw_dev->soft_rst_desc.len = sizeof(aw87xxx_pid_5a_softrst_access); + aw_dev->soft_rst_desc.access = aw87xxx_pid_5a_softrst_access; + aw_dev->soft_rst_enable = AW_DEV_SOFT_RST_ENABLE; + + /* Whether to allow register operation to power off */ + aw_dev->soft_off_enable = AW_DEV_SOFT_OFF_ENABLE; + + aw_dev->product_tab = g_aw_pid_5a_product; + aw_dev->product_cnt = AW87XXX_PID_5A_PRODUCT_MAX; + + aw_dev->rec_desc.addr = AW87XXX_PID_5A_REG_SYSCTRL_REG; + aw_dev->rec_desc.disable = AW87XXX_PID_5A_REG_RCV_MODE_DISABLE; + aw_dev->rec_desc.enable = AW87XXX_PID_5A_REG_RCV_MODE_ENABLE; + aw_dev->rec_desc.mask = AW87XXX_PID_5A_REG_RCV_MODE_MASK; + + /* esd reg info */ + aw_dev->esd_desc.first_update_reg_addr = AW87XXX_PID_5A_REG_DFT3R_REG; + aw_dev->esd_desc.first_update_reg_val = AW87XXX_PID_5A_DFT3R_DEFAULT; +} +/********************** aw87xxx_pid_5a attributes end ************************/ + +/********************** aw87xxx_pid_76 attributes ****************************/ +static void aw_dev_chipid_76_init(struct aw_device *aw_dev) +{ + /* Product register permission info */ + aw_dev->reg_max_addr = AW87XXX_PID_76_REG_MAX; + aw_dev->reg_access = aw87xxx_pid_76_reg_access; + + /* software reset control info */ + aw_dev->soft_rst_desc.len = sizeof(aw87xxx_pid_76_softrst_access); + aw_dev->soft_rst_desc.access = aw87xxx_pid_76_softrst_access; + aw_dev->soft_rst_enable = AW_DEV_SOFT_RST_ENABLE; + + /* software power off control info */ + aw_dev->soft_off_enable = AW_DEV_SOFT_OFF_ENABLE; + + aw_dev->product_tab = g_aw_pid_76_product; + aw_dev->product_cnt = AW87XXX_PID_76_PROFUCT_MAX; + + aw_dev->rec_desc.addr = AW87XXX_PID_76_MDCTRL_REG; + aw_dev->rec_desc.disable = AW87XXX_PID_76_EN_SPK_ENABLE; + aw_dev->rec_desc.enable = AW87XXX_PID_76_EN_SPK_DISABLE; + aw_dev->rec_desc.mask = AW87XXX_PID_76_EN_SPK_MASK; + + /* esd reg info */ + aw_dev->esd_desc.first_update_reg_addr = AW87XXX_PID_76_DFT_ADP1_REG; + aw_dev->esd_desc.first_update_reg_val = AW87XXX_PID_76_DFT_ADP1_CHECK; +} +/********************** aw87xxx_pid_76 attributes end ************************/ + +/********************** aw87xxx_pid_60 attributes ****************************/ +static void aw_dev_chipid_60_init(struct aw_device *aw_dev) +{ + /* Product register permission info */ + aw_dev->reg_max_addr = AW87XXX_PID_60_REG_MAX; + aw_dev->reg_access = aw87xxx_pid_60_reg_access; + + /* software reset control info */ + aw_dev->soft_rst_desc.len = sizeof(aw87xxx_pid_60_softrst_access); + aw_dev->soft_rst_desc.access = aw87xxx_pid_60_softrst_access; + aw_dev->soft_rst_enable = AW_DEV_SOFT_RST_ENABLE; + + /* software power off control info */ + aw_dev->soft_off_enable = AW_DEV_SOFT_OFF_ENABLE; + + aw_dev->product_tab = g_aw_pid_60_product; + aw_dev->product_cnt = AW87XXX_PID_60_PROFUCT_MAX; + + aw_dev->rec_desc.addr = AW87XXX_PID_60_SYSCTRL_REG; + aw_dev->rec_desc.disable = AW87XXX_PID_60_RCV_MODE_DISABLE; + aw_dev->rec_desc.enable = AW87XXX_PID_60_RCV_MODE_ENABLE; + aw_dev->rec_desc.mask = AW87XXX_PID_60_RCV_MODE_MASK; + + /* esd reg info */ + aw_dev->esd_desc.first_update_reg_addr = AW87XXX_PID_60_NG3_REG; + aw_dev->esd_desc.first_update_reg_val = AW87XXX_PID_60_ESD_REG_VAL; +} +/********************** aw87xxx_pid_60 attributes end ************************/ + +static int aw_dev_chip_init(struct aw_device *aw_dev) +{ + int ret = 0; + + /*get info by chipid*/ + switch (aw_dev->chipid) { + case AW_DEV_CHIPID_9A: + ret = aw_dev_pid_9a_init(aw_dev); + if (ret < 0) + AW_DEV_LOGE(aw_dev->dev, "product is pid_9B init failed"); + break; + case AW_DEV_CHIPID_9B: + aw_dev_pid_9b_init(aw_dev); + AW_DEV_LOGI(aw_dev->dev, "product is pid_9B class"); + break; + case AW_DEV_CHIPID_18: + aw_dev_chipid_18_init(aw_dev); + AW_DEV_LOGI(aw_dev->dev, "product is pid_18 class"); + break; + case AW_DEV_CHIPID_39: + aw_dev_chipid_39_init(aw_dev); + AW_DEV_LOGI(aw_dev->dev, "product is pid_39 class"); + break; + case AW_DEV_CHIPID_59: + if (aw87xxx_dev_gpio_is_valid(aw_dev)) { + aw_dev_chipid_59_5x9_init(aw_dev); + AW_DEV_LOGI(aw_dev->dev, "product is pid_59_5x9 class"); + } else { + aw_dev_chipid_59_3x9_init(aw_dev); + AW_DEV_LOGI(aw_dev->dev, "product is pid_59_3x9 class"); + } + break; + case AW_DEV_CHIPID_5A: + aw_dev_chipid_5a_init(aw_dev); + AW_DEV_LOGI(aw_dev->dev, "product is pid_5A class"); + break; + case AW_DEV_CHIPID_76: + aw_dev_chipid_76_init(aw_dev); + AW_DEV_LOGI(aw_dev->dev, "product is pid_76 class"); + break; + case AW_DEV_CHIPID_60: + aw_dev_chipid_60_init(aw_dev); + AW_DEV_LOGI(aw_dev->dev, "product is pid_60 class"); + break; + default: + AW_DEV_LOGE(aw_dev->dev, "unsupported device revision [0x%x]", + aw_dev->chipid); + return -EINVAL; + } + + return 0; +} + +static int aw87xxx_dev_get_chipid(struct aw_device *aw_dev) +{ + int ret = -1; + unsigned int cnt = 0; + unsigned char reg_val = 0; + + for (cnt = 0; cnt < AW_READ_CHIPID_RETRIES; cnt++) { + ret = aw87xxx_dev_i2c_read_byte(aw_dev, AW_DEV_REG_CHIPID, ®_val); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "[%d] read chip is failed, ret=%d", + cnt, ret); + continue; + } + break; + } + + + if (cnt == AW_READ_CHIPID_RETRIES) { + AW_DEV_LOGE(aw_dev->dev, "read chip is failed,cnt=%d", cnt); + return -EINVAL; + } + + AW_DEV_LOGI(aw_dev->dev, "read chipid[0x%x] succeed", reg_val); + aw_dev->chipid = reg_val; + + return 0; +} + +int aw87xxx_dev_init(struct aw_device *aw_dev) +{ + int ret = -1; + + ret = aw87xxx_dev_get_chipid(aw_dev); + if (ret < 0) { + AW_DEV_LOGE(aw_dev->dev, "read chipid is failed,ret=%d", ret); + return ret; + } + + ret = aw_dev_chip_init(aw_dev); + + return ret; +} + + diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_device.h b/sound/soc/codecs/aw87xxx/aw87xxx_device.h new file mode 100644 index 0000000000000..7c85f80a958e6 --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_device.h @@ -0,0 +1,149 @@ +#ifndef __AW87XXX_DEVICE_H__ +#define __AW87XXX_DEVICE_H__ +#include +#include +#include +#include +#include "aw87xxx_acf_bin.h" + +#define AW87XXX_PID_9B_PRODUCT_MAX (1) +#define AW87XXX_PID_18_PRODUCT_MAX (1) +#define AW87XXX_PID_39_PRODUCT_MAX (3) +#define AW87XXX_PID_59_3X9_PRODUCT_MAX (2) +#define AW87XXX_PID_59_5X9_PRODUCT_MAX (4) +#define AW87XXX_PID_5A_PRODUCT_MAX (5) +#define AW87XXX_PID_76_PROFUCT_MAX (4) +#define AW87XXX_PID_60_PROFUCT_MAX (5) +#define AW_PRODUCT_NAME_LEN (8) + +#define AW_GPIO_HIGHT_LEVEL (1) +#define AW_GPIO_LOW_LEVEL (0) + +#define AW_I2C_RETRIES (5) +#define AW_I2C_RETRY_DELAY (2) +#define AW_I2C_READ_MSG_NUM (2) + +#define AW_READ_CHIPID_RETRIES (5) +#define AW_READ_CHIPID_RETRY_DELAY (2) +#define AW_DEV_REG_CHIPID (0x00) + +#define AW_DEV_REG_INVALID_MASK (0xff) + +#define AW_NO_RESET_GPIO (-1) + +#define AW_PID_9B_BIN_REG_CFG_COUNT (10) + +/******************************************** + * + * aw87xxx devices attributes + * + *******************************************/ +struct aw_device; + +struct aw_device_ops { + int (*pwr_on_func)(struct aw_device *aw_dev, struct aw_data_container *data); + int (*pwr_off_func)(struct aw_device *aw_dev, struct aw_data_container *data); +}; + +enum aw_dev_chipid { + AW_DEV_CHIPID_18 = 0x18, + AW_DEV_CHIPID_39 = 0x39, + AW_DEV_CHIPID_59 = 0x59, + AW_DEV_CHIPID_69 = 0x69, + AW_DEV_CHIPID_5A = 0x5A, + AW_DEV_CHIPID_9A = 0x9A, + AW_DEV_CHIPID_9B = 0x9B, + AW_DEV_CHIPID_76 = 0x76, + AW_DEV_CHIPID_60 = 0x60, +}; + +enum aw_dev_hw_status { + AW_DEV_HWEN_OFF = 0, + AW_DEV_HWEN_ON, + AW_DEV_HWEN_INVALID, + AW_DEV_HWEN_STATUS_MAX, +}; + +enum aw_dev_soft_off_enable { + AW_DEV_SOFT_OFF_DISENABLE = 0, + AW_DEV_SOFT_OFF_ENABLE = 1, +}; + +enum aw_dev_soft_rst_enable { + AW_DEV_SOFT_RST_DISENABLE = 0, + AW_DEV_SOFT_RST_ENABLE = 1, +}; + +enum aw_reg_receiver_mode { + AW_NOT_REC_MODE = 0, + AW_IS_REC_MODE = 1, +}; + +struct aw_mute_desc { + uint8_t addr; + uint8_t enable; + uint8_t disable; + uint16_t mask; +}; + +struct aw_soft_rst_desc { + int len; + unsigned char *access; +}; + +struct aw_esd_check_desc { + uint8_t first_update_reg_addr; + uint8_t first_update_reg_val; +}; + +struct aw_rec_mode_desc { + uint8_t addr; + uint8_t enable; + uint8_t disable; + uint8_t mask; +}; + +struct aw_device { + uint8_t i2c_addr; + uint8_t chipid; + uint8_t soft_rst_enable; + uint8_t soft_off_enable; + uint8_t is_rec_mode; + int hwen_status; + int i2c_bus; + int rst_gpio; + int reg_max_addr; + int product_cnt; + const char **product_tab; + const unsigned char *reg_access; + + struct device *dev; + struct i2c_client *i2c; + struct aw_mute_desc mute_desc; + struct aw_soft_rst_desc soft_rst_desc; + struct aw_esd_check_desc esd_desc; + struct aw_rec_mode_desc rec_desc; + + struct aw_device_ops ops; +}; + + +int aw87xxx_dev_i2c_write_byte(struct aw_device *aw_dev, + uint8_t reg_addr, uint8_t reg_data); +int aw87xxx_dev_i2c_read_byte(struct aw_device *aw_dev, + uint8_t reg_addr, uint8_t *reg_data); +int aw87xxx_dev_i2c_read_msg(struct aw_device *aw_dev, + uint8_t reg_addr, uint8_t *data_buf, uint32_t data_len); +int aw87xxx_dev_i2c_write_bits(struct aw_device *aw_dev, + uint8_t reg_addr, uint8_t mask, uint8_t reg_data); +void aw87xxx_dev_soft_reset(struct aw_device *aw_dev); +void aw87xxx_dev_hw_pwr_ctrl(struct aw_device *aw_dev, bool enable); +int aw87xxx_dev_default_pwr_on(struct aw_device *aw_dev, + struct aw_data_container *profile_data); +int aw87xxx_dev_default_pwr_off(struct aw_device *aw_dev, + struct aw_data_container *profile_data); +int aw87xxx_dev_esd_reg_status_check(struct aw_device *aw_dev); +int aw87xxx_dev_check_reg_is_rec_mode(struct aw_device *aw_dev); +int aw87xxx_dev_init(struct aw_device *aw_dev); + +#endif diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_dsp.c b/sound/soc/codecs/aw87xxx/aw87xxx_dsp.c new file mode 100644 index 0000000000000..93b02e30122d8 --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_dsp.c @@ -0,0 +1,355 @@ +/* + * aw87xxx_dsp.c + * + * Copyright (c) 2021 AWINIC Technology CO., LTD + * + * Author: Barry + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aw87xxx_log.h" +#include "aw87xxx_dsp.h" + +static DEFINE_MUTEX(g_dsp_lock); +static unsigned int g_spin_value = 0; + +static int g_rx_topo_id = AW_RX_DEFAULT_TOPO_ID; +static int g_rx_port_id = AW_RX_DEFAULT_PORT_ID; + +#ifdef AW_MTK_OPEN_DSP_PLATFORM +extern int mtk_spk_send_ipi_buf_to_dsp(void *data_buffer, + uint32_t data_size); +extern int mtk_spk_recv_ipi_buf_from_dsp(int8_t *buffer, + int16_t size, uint32_t *buf_len); +/* +static int mtk_spk_send_ipi_buf_to_dsp(void *data_buffer, + uint32_t data_size) +{ + AW_LOGI("enter"); + return 0; +} + +static int mtk_spk_recv_ipi_buf_from_dsp(int8_t *buffer, + int16_t size, uint32_t *buf_len) +{ + AW_LOGI("enter"); + return 0; +} +*/ +#elif defined AW_QCOM_OPEN_DSP_PLATFORM +extern int afe_get_topology(int port_id); +extern int aw_send_afe_cal_apr(uint32_t param_id, + void *buf, int cmd_size, bool write); +/* +static int afe_get_topology(int port_id) +{ + return -EPERM; +} + +static int aw_send_afe_cal_apr(uint32_t param_id, + void *buf, int cmd_size, bool write) +{ + AW_LOGI("enter, no define AWINIC_ADSP_ENABLE", __func__); + return 0; +} +*/ +#endif + +#ifdef AW_QCOM_OPEN_DSP_PLATFORM +extern void aw_set_port_id(int rx_port_id); +#else +static void aw_set_port_id(int rx_port_id) +{ + return; +} +#endif + +uint8_t aw87xxx_dsp_isEnable(void) +{ +#if (defined AW_QCOM_OPEN_DSP_PLATFORM) || (defined AW_MTK_OPEN_DSP_PLATFORM) + return true; +#else + return false; +#endif +} + +/*****************mtk dsp communication function start**********************/ +#ifdef AW_MTK_OPEN_DSP_PLATFORM +static int aw_mtk_write_data_to_dsp(int32_t param_id, + void *data, int size) +{ + int32_t *dsp_data = NULL; + mtk_dsp_hdr_t *hdr = NULL; + int ret; + + dsp_data = kzalloc(sizeof(mtk_dsp_hdr_t) + size, GFP_KERNEL); + if (!dsp_data) { + AW_LOGE("kzalloc dsp_msg error"); + return -ENOMEM; + } + + hdr = (mtk_dsp_hdr_t *)dsp_data; + hdr->type = DSP_MSG_TYPE_DATA; + hdr->opcode_id = param_id; + hdr->version = AW_DSP_MSG_HDR_VER; + + memcpy(((char *)dsp_data) + sizeof(mtk_dsp_hdr_t), + data, size); + + ret = mtk_spk_send_ipi_buf_to_dsp(dsp_data, + sizeof(mtk_dsp_hdr_t) + size); + if (ret < 0) { + AW_LOGE("write data failed"); + kfree(dsp_data); + dsp_data = NULL; + return ret; + } + + kfree(dsp_data); + dsp_data = NULL; + return 0; +} + +static int aw_mtk_read_data_from_dsp(int32_t param_id, void *data, + int data_size) +{ + int ret; + mtk_dsp_hdr_t hdr; + + mutex_lock(&g_dsp_lock); + hdr.type = DSP_MSG_TYPE_CMD; + hdr.opcode_id = param_id; + hdr.version = AW_DSP_MSG_HDR_VER; + + ret = mtk_spk_send_ipi_buf_to_dsp(&hdr, sizeof(mtk_dsp_hdr_t)); + if (ret < 0) + goto failed; + + ret = mtk_spk_recv_ipi_buf_from_dsp(data, data_size, &data_size); + if (ret < 0) + goto failed; + + mutex_unlock(&g_dsp_lock); + return 0; + +failed: + mutex_unlock(&g_dsp_lock); + return ret; +} + +#endif +/********************mtk dsp communication function end***********************/ + +/******************qcom dsp communication function start**********************/ +#ifdef AW_QCOM_OPEN_DSP_PLATFORM +static void aw_check_dsp_ready(void) +{ + int ret; + + ret = afe_get_topology(g_rx_port_id); + AW_LOGD("topo_id 0x%x", ret); + + if (ret != g_rx_topo_id) + AW_LOGE("topo id 0x%x", ret); + +} + +static int aw_qcom_write_data_to_dsp(int32_t param_id, + void *data, int data_size) +{ + int ret = 0; + + AW_LOGI("enter"); + mutex_lock(&g_dsp_lock); + aw_check_dsp_ready(); + ret = aw_send_afe_cal_apr(param_id, data, + data_size, true); + mutex_unlock(&g_dsp_lock); + return ret; +} + +static int aw_qcom_read_data_from_dsp(int32_t param_id, + void *data, int data_size) +{ + int ret = 0; + + AW_LOGI("enter"); + mutex_lock(&g_dsp_lock); + aw_check_dsp_ready(); + ret = aw_send_afe_cal_apr(param_id, data, + data_size, false); + mutex_unlock(&g_dsp_lock); + return ret; +} + +#endif +/*****************qcom dsp communication function end*********************/ + +/*****************read/write msg communication function*********************/ +static int aw_write_data_to_dsp(int32_t param_id, void *data, int data_size) +{ +#if defined AW_QCOM_OPEN_DSP_PLATFORM + return aw_qcom_write_data_to_dsp(param_id, data, data_size); +#elif defined AW_MTK_OPEN_DSP_PLATFORM + return aw_mtk_write_data_to_dsp(param_id, data, data_size); +#else + return -EINVAL; +#endif +} + +static int aw_read_data_from_dsp(int32_t param_id, void *data, int data_size) +{ +#if defined AW_QCOM_OPEN_DSP_PLATFORM + return aw_qcom_read_data_from_dsp(param_id, data, data_size); +#elif defined AW_MTK_OPEN_DSP_PLATFORM + return aw_mtk_read_data_from_dsp(param_id, data, data_size); +#else + return -EINVAL; +#endif +} + +/***************read/write msg communication function end*******************/ + +int aw87xxx_dsp_get_rx_module_enable(int *enable) +{ + if (!enable) { + AW_LOGE("enable is NULL"); + return -EINVAL; + } + + return aw_read_data_from_dsp(AWDSP_RX_SET_ENABLE, + (void *)enable, sizeof(uint32_t)); +} + +int aw87xxx_dsp_set_rx_module_enable(int enable) +{ + switch (enable) { + case AW_RX_MODULE_DISENABLE: + AW_LOGD("set enable=%d", enable); + break; + case AW_RX_MODULE_ENABLE: + AW_LOGD("set enable=%d", enable); + break; + default: + AW_LOGE("unsupport enable=%d", enable); + return -EINVAL; + } + + return aw_write_data_to_dsp(AWDSP_RX_SET_ENABLE, + &enable, sizeof(uint32_t)); +} + + +int aw87xxx_dsp_get_vmax(uint32_t *vmax, int dev_index) +{ + int32_t param_id = 0; + + switch (dev_index % AW_DSP_CHANNEL_MAX) { + case AW_DSP_CHANNEL_0: + param_id = AWDSP_RX_VMAX_0; + break; + case AW_DSP_CHANNEL_1: + param_id = AWDSP_RX_VMAX_1; + break; + default: + AW_LOGE("algo only support double PA channel:%d unsupport", + dev_index); + return -EINVAL; + } + + return aw_read_data_from_dsp(param_id, + (void *)vmax, sizeof(uint32_t)); +} + +int aw87xxx_dsp_set_vmax(uint32_t vmax, int dev_index) +{ + int32_t param_id = 0; + + switch (dev_index % AW_DSP_CHANNEL_MAX) { + case AW_DSP_CHANNEL_0: + param_id = AWDSP_RX_VMAX_0; + break; + case AW_DSP_CHANNEL_1: + param_id = AWDSP_RX_VMAX_1; + break; + default: + AW_LOGE("algo only support double PA channel:%d unsupport", + dev_index); + return -EINVAL; + } + + return aw_write_data_to_dsp(param_id, &vmax, sizeof(uint32_t)); +} + +int aw87xxx_dsp_set_spin(uint32_t ctrl_value) +{ + int ret = 0; + + if (ctrl_value >= AW_SPIN_MAX) { + AW_LOGE("spin [%d] unsupported ", ctrl_value); + return -EINVAL; + } + ret = aw_write_data_to_dsp(AW_MSG_ID_SPIN, &ctrl_value, + sizeof(uint32_t)); + if (ret) { + AW_LOGE("spin [%d] set failed ", ctrl_value); + return ret; + } + + g_spin_value = ctrl_value; + return 0; +} + +int aw87xxx_dsp_get_spin(void) +{ + return g_spin_value; +} + +int aw87xxx_spin_set_record_val(void) +{ + AW_LOGD("record write spin enter"); + + return aw87xxx_dsp_set_spin(g_spin_value); +} +EXPORT_SYMBOL(aw87xxx_spin_set_record_val); + +void aw87xxx_device_parse_topo_id_dt(struct aw_device *aw_dev) +{ + int ret; + + ret = of_property_read_u32(aw_dev->dev->of_node, "aw-rx-topo-id", &g_rx_topo_id); + if (ret < 0) { + g_rx_topo_id = AW_RX_DEFAULT_TOPO_ID; + AW_DEV_LOGI(aw_dev->dev, "read aw-rx-topo-id failed,use default"); + } + + AW_DEV_LOGI(aw_dev->dev, "rx-topo-id: 0x%x", g_rx_topo_id); +} + +void aw87xxx_device_parse_port_id_dt(struct aw_device *aw_dev) +{ + int ret; + + ret = of_property_read_u32(aw_dev->dev->of_node, "aw-rx-port-id", &g_rx_port_id); + if (ret < 0) { + g_rx_port_id = AW_RX_DEFAULT_PORT_ID; + AW_DEV_LOGI(aw_dev->dev, "read aw-rx-port-id failed,use default"); + } + + aw_set_port_id(g_rx_port_id); + AW_DEV_LOGI(aw_dev->dev, "rx-port-id: 0x%x", g_rx_port_id); +} + diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_dsp.h b/sound/soc/codecs/aw87xxx/aw87xxx_dsp.h new file mode 100644 index 0000000000000..7acc4dc0dfd9b --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_dsp.h @@ -0,0 +1,65 @@ +#ifndef __AW87XXX_DSP_H__ +#define __AW87XXX_DSP_H__ + +#include "aw87xxx_device.h" + +/*#define AW_MTK_OPEN_DSP_PLATFORM*/ +/*#define AW_QCOM_OPEN_DSP_PLATFORM*/ + +/*Note: The pord_ID is configured according to different platforms*/ +#define AW_DSP_SLEEP_TIME (10) + +#define AW_DSP_MSG_HDR_VER (1) + +#define AW_RX_DEFAULT_TOPO_ID (0x1000FF01) +#define AW_RX_DEFAULT_PORT_ID (0x4000) + +#define AWDSP_RX_SET_ENABLE (0x10013D11) +#define AWDSP_RX_PARAMS (0x10013D12) +#define AWDSP_RX_VMAX_0 (0X10013D17) +#define AWDSP_RX_VMAX_1 (0X10013D18) +#define AW_MSG_ID_SPIN (0x10013D2E) + +enum { + AW_SPIN_0 = 0, + AW_SPIN_90, + AW_SPIN_180, + AW_SPIN_270, + AW_SPIN_MAX, +}; + +typedef struct mtk_dsp_msg_header { + int32_t type; + int32_t opcode_id; + int32_t version; + int32_t reserver[3]; +} mtk_dsp_hdr_t; + +enum aw_rx_module_enable { + AW_RX_MODULE_DISENABLE = 0, + AW_RX_MODULE_ENABLE, +}; + +enum aw_dsp_msg_type { + DSP_MSG_TYPE_DATA = 0, + DSP_MSG_TYPE_CMD = 1, +}; + +enum aw_dsp_channel { + AW_DSP_CHANNEL_0 = 0, + AW_DSP_CHANNEL_1, + AW_DSP_CHANNEL_MAX, +}; + +uint8_t aw87xxx_dsp_isEnable(void); +int aw87xxx_dsp_get_rx_module_enable(int *enable); +int aw87xxx_dsp_set_rx_module_enable(int enable); +int aw87xxx_dsp_get_vmax(uint32_t *vmax, int channel); +int aw87xxx_dsp_set_vmax(uint32_t vmax, int channel); +int aw87xxx_dsp_set_spin(uint32_t ctrl_value); +int aw87xxx_dsp_get_spin(void); +int aw87xxx_spin_set_record_val(void); +void aw87xxx_device_parse_port_id_dt(struct aw_device *aw_dev); +void aw87xxx_device_parse_topo_id_dt(struct aw_device *aw_dev); + +#endif diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_log.h b/sound/soc/codecs/aw87xxx/aw87xxx_log.h new file mode 100644 index 0000000000000..b3bde38a23c67 --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_log.h @@ -0,0 +1,33 @@ +#ifndef __AW87XXX_LOG_H__ +#define __AW87XXX_LOG_H__ + +#include + + +/******************************************** + * + * print information control + * + *******************************************/ +#define AW_LOGI(fmt, ...)\ + pr_info("[Awinic] %s:" fmt "\n", __func__, ##__VA_ARGS__) + +#define AW_LOGD(fmt, ...)\ + pr_debug("[Awinic] %s:" fmt "\n", __func__, ##__VA_ARGS__) + +#define AW_LOGE(fmt, ...)\ + pr_err("[Awinic] %s:" fmt "\n", __func__, ##__VA_ARGS__) + + +#define AW_DEV_LOGI(dev, fmt, ...)\ + pr_info("[Awinic] [%s]%s: " fmt "\n", dev_name(dev), __func__, ##__VA_ARGS__) + +#define AW_DEV_LOGD(dev, fmt, ...)\ + pr_debug("[Awinic] [%s]%s: " fmt "\n", dev_name(dev), __func__, ##__VA_ARGS__) + +#define AW_DEV_LOGE(dev, fmt, ...)\ + pr_err("[Awinic] [%s]%s: " fmt "\n", dev_name(dev), __func__, ##__VA_ARGS__) + + + +#endif diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_monitor.c b/sound/soc/codecs/aw87xxx/aw87xxx_monitor.c new file mode 100644 index 0000000000000..61adba6b6169e --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_monitor.c @@ -0,0 +1,1207 @@ +/* + * aw87xxx_monitor.c + * + * Copyright (c) 2021 AWINIC Technology CO., LTD + * + * Author: Barry + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aw87xxx.h" +#include "aw87xxx_log.h" +#include "aw87xxx_monitor.h" +#include "aw87xxx_dsp.h" +#include "aw87xxx_bin_parse.h" +#include "aw87xxx_device.h" + +#define AW_MONITOT_BIN_PARSE_VERSION "V0.1.0" + +#define AW_GET_32_DATA(w, x, y, z) \ + ((uint32_t)((((uint8_t)w) << 24) | (((uint8_t)x) << 16) | \ + (((uint8_t)y) << 8) | ((uint8_t)z))) + +/**************************************************************************** + * + * aw87xxx monitor bin check + * + ****************************************************************************/ +static int aw_monitor_check_header_v_1_0_0(struct device *dev, + char *data, uint32_t data_len) +{ + int i = 0; + struct aw_bin_header *header = (struct aw_bin_header *)data; + + if (header->bin_data_type != DATA_TYPE_MONITOR_ANALOG) { + AW_DEV_LOGE(dev, "monitor data_type check error!"); + return -EINVAL; + } + + if (header->bin_data_size != AW_MONITOR_HDR_DATA_SIZE) { + AW_DEV_LOGE(dev, "monitor data_size error!"); + return -EINVAL; + } + + if (header->data_byte_len != AW_MONITOR_HDR_DATA_BYTE_LEN) { + AW_DEV_LOGE(dev, "monitor data_byte_len error!"); + return -EINVAL; + } + + for (i = 0; i < AW_MONITOR_DATA_VER_MAX; i++) { + if (header->bin_data_ver == i) { + AW_LOGD("monitor bin_data_ver[0x%x]", i); + break; + } + } + if (i == AW_MONITOR_DATA_VER_MAX) + return -EINVAL; + + return 0; +} + +static int aw_monitor_check_data_v1_size(struct device *dev, + char *data, int32_t data_len) +{ + int32_t bin_header_len = sizeof(struct aw_bin_header); + int32_t monitor_header_len = sizeof(struct aw_monitor_header); + int32_t monitor_data_len = sizeof(struct vmax_step_config); + int32_t len = 0; + struct aw_monitor_header *monitor_header = NULL; + + AW_DEV_LOGD(dev, "enter"); + + if (data_len < bin_header_len + monitor_header_len) { + AW_DEV_LOGE(dev, "bin len is less than aw_bin_header and monitoor_header,check failed"); + return -EINVAL; + } + + monitor_header = (struct aw_monitor_header *)(data + bin_header_len); + len = data_len - bin_header_len - monitor_header_len; + if (len < monitor_header->step_count * monitor_data_len) { + AW_DEV_LOGE(dev, "bin data len is not enough,check failed"); + return -EINVAL; + } + + AW_DEV_LOGD(dev, "succeed"); + + return 0; +} + +static int aw_monitor_check_data_size(struct device *dev, + char *data, int32_t data_len) +{ + int ret = -1; + struct aw_bin_header *header = (struct aw_bin_header *)data; + + switch (header->bin_data_ver) { + case AW_MONITOR_DATA_VER: + ret = aw_monitor_check_data_v1_size(dev, data, data_len); + if (ret < 0) + return ret; + break; + default: + AW_DEV_LOGE(dev, "bin data_ver[0x%x] non support", + header->bin_data_ver); + return -EINVAL; + } + + return 0; +} + + +static int aw_monitor_check_bin_header(struct device *dev, + char *data, int32_t data_len) +{ + int ret = -1; + struct aw_bin_header *header = NULL; + + if (data_len < sizeof(struct aw_bin_header)) { + AW_DEV_LOGE(dev, "bin len is less than aw_bin_header,check failed"); + return -EINVAL; + } + header = (struct aw_bin_header *)data; + + switch (header->header_ver) { + case HEADER_VERSION_1_0_0: + ret = aw_monitor_check_header_v_1_0_0(dev, data, data_len); + if (ret < 0) { + AW_DEV_LOGE(dev, "monitor bin haeder info check error!"); + return ret; + } + break; + default: + AW_DEV_LOGE(dev, "bin version[0x%x] non support", + header->header_ver); + return -EINVAL; + } + + return 0; +} + +static int aw_monitor_bin_check_sum(struct device *dev, + char *data, int32_t data_len) +{ + int i, data_sum = 0; + uint32_t *check_sum = (uint32_t *)data; + + for (i = 4; i < data_len; i++) + data_sum += data[i]; + + if (*check_sum != data_sum) { + AW_DEV_LOGE(dev, "check_sum[%d] is not equal to data_sum[%d]", + *check_sum, data_sum); + return -ENOMEM; + } + + AW_DEV_LOGD(dev, "succeed"); + + return 0; +} + +static int aw_monitor_bin_check(struct device *dev, + char *monitor_data, uint32_t data_len) +{ + int ret = -1; + + if (monitor_data == NULL || data_len == 0) { + AW_DEV_LOGE(dev, "none data to parse"); + return -EINVAL; + } + + ret = aw_monitor_bin_check_sum(dev, monitor_data, data_len); + if (ret < 0) { + AW_DEV_LOGE(dev, "bin data check sum failed"); + return ret; + } + + ret = aw_monitor_check_bin_header(dev, monitor_data, data_len); + if (ret < 0) { + AW_DEV_LOGE(dev, "bin data len check failed"); + return ret; + } + + ret = aw_monitor_check_data_size(dev, monitor_data, data_len); + if (ret < 0) { + AW_DEV_LOGE(dev, "bin header info check failed"); + return ret; + } + + return 0; +} + +/***************************************************************************** + * + * aw87xxx monitor header bin parse + * + *****************************************************************************/ +static void aw_monitor_write_to_table_v1(struct device *dev, + struct vmax_step_config *vmax_step, + char *vmax_data, uint32_t step_count) +{ + int i = 0; + int index = 0; + int vmax_step_size = (int)sizeof(struct vmax_step_config); + + for (i = 0; i < step_count; i++) { + index = vmax_step_size * i; + vmax_step[i].vbat_min = + AW_GET_32_DATA(vmax_data[index + 3], + vmax_data[index + 2], + vmax_data[index + 1], + vmax_data[index + 0]); + vmax_step[i].vbat_max = + AW_GET_32_DATA(vmax_data[index + 7], + vmax_data[index + 6], + vmax_data[index + 5], + vmax_data[index + 4]); + vmax_step[i].vmax_vol = + AW_GET_32_DATA(vmax_data[index + 11], + vmax_data[index + 10], + vmax_data[index + 9], + vmax_data[index + 8]); + } + + for (i = 0; i < step_count; i++) + AW_DEV_LOGI(dev, "vbat_min:%d, vbat_max%d, vmax_vol:0x%x", + vmax_step[i].vbat_min, + vmax_step[i].vbat_max, + vmax_step[i].vmax_vol); +} + +static int aw_monitor_parse_vol_data_v1(struct device *dev, + struct aw_monitor *monitor, char *monitor_data) +{ + uint32_t step_count = 0; + char *vmax_data = NULL; + struct vmax_step_config *vmax_step = NULL; + + AW_DEV_LOGD(dev, "enter"); + + step_count = monitor->monitor_hdr.step_count; + if (step_count) { + vmax_step = devm_kzalloc(dev, sizeof(struct vmax_step_config) * step_count, + GFP_KERNEL); + if (vmax_step == NULL) { + AW_DEV_LOGE(dev, "vmax_cfg vmalloc failed"); + return -ENOMEM; + } + memset(vmax_step, 0, + sizeof(struct vmax_step_config) * step_count); + } + + vmax_data = monitor_data + sizeof(struct aw_bin_header) + + sizeof(struct aw_monitor_header); + aw_monitor_write_to_table_v1(dev, vmax_step, vmax_data, step_count); + monitor->vmax_cfg = vmax_step; + + AW_DEV_LOGI(dev, "vmax_data parse succeed"); + + return 0; +} + +static int aw_monitor_parse_data_v1(struct device *dev, + struct aw_monitor *monitor, char *monitor_data) +{ + int ret = -1; + int header_len = 0; + struct aw_monitor_header *monitor_hdr = &monitor->monitor_hdr; + + header_len = sizeof(struct aw_bin_header); + memcpy(monitor_hdr, monitor_data + header_len, + sizeof(struct aw_monitor_header)); + + AW_DEV_LOGI(dev, "monitor_switch:%d, monitor_time:%d (ms), monitor_count:%d, step_count:%d", + monitor_hdr->monitor_switch, monitor_hdr->monitor_time, + monitor_hdr->monitor_count, monitor_hdr->step_count); + + ret = aw_monitor_parse_vol_data_v1(dev, monitor, monitor_data); + if (ret < 0) { + AW_DEV_LOGE(dev, "vmax_data parse failed"); + return ret; + } + + monitor->bin_status = AW_MONITOR_CFG_OK; + + return 0; +} + + +static int aw_monitor_parse_v_1_0_0(struct device *dev, + struct aw_monitor *monitor, char *monitor_data) +{ + int ret = -1; + struct aw_bin_header *header = (struct aw_bin_header *)monitor_data; + + switch (header->bin_data_ver) { + case AW_MONITOR_DATA_VER: + ret = aw_monitor_parse_data_v1(dev, monitor, monitor_data); + if (ret < 0) + return ret; + break; + default: + return -EINVAL; + } + + return 0; +} + +void aw87xxx_monitor_cfg_free(struct aw_monitor *monitor) +{ + struct aw87xxx *aw87xxx = + container_of(monitor, struct aw87xxx, monitor); + + monitor->bin_status = AW_MONITOR_CFG_WAIT; + memset(&monitor->monitor_hdr, 0, + sizeof(struct aw_monitor_header)); + if (monitor->vmax_cfg) { + devm_kfree(aw87xxx->dev, monitor->vmax_cfg); + monitor->vmax_cfg = NULL; + } +} + +int aw87xxx_monitor_bin_parse(struct device *dev, + char *monitor_data, uint32_t data_len) +{ + int ret = -1; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = NULL; + struct aw_bin_header *bin_header = NULL; + + if (aw87xxx == NULL) { + AW_DEV_LOGE(dev, "get struct aw87xxx failed"); + return -EINVAL; + } + + monitor = &aw87xxx->monitor; + monitor->bin_status = AW_MONITOR_CFG_WAIT; + + AW_DEV_LOGI(dev, "monitor bin parse version: %s", + AW_MONITOT_BIN_PARSE_VERSION); + + ret = aw_monitor_bin_check(dev, monitor_data, data_len); + if (ret < 0) { + AW_DEV_LOGE(dev, "monitor bin check failed"); + return ret; + } + + bin_header = (struct aw_bin_header *)monitor_data; + switch (bin_header->bin_data_ver) { + case DATA_VERSION_V1: + ret = aw_monitor_parse_v_1_0_0(dev, monitor, + monitor_data); + if (ret < 0) { + aw87xxx_monitor_cfg_free(monitor); + return ret; + } + break; + default: + AW_DEV_LOGE(dev, "Unrecognized this bin data version[0x%x]", + bin_header->bin_data_ver); + } + + return 0; +} + +/*************************************************************************** + * + * aw87xxx monitor get adjustment vmax of power + * + ***************************************************************************/ +static int aw_monitor_get_battery_capacity(struct device *dev, + struct aw_monitor *monitor, int *vbat_capacity) +{ + char name[] = "battery"; + int ret = -1; + union power_supply_propval prop = { 0 }; + struct power_supply *psy = NULL; + + psy = power_supply_get_by_name(name); + if (psy == NULL) { + AW_DEV_LOGE(dev, "no struct power supply name:%s", name); + return -EINVAL; + } + + ret = power_supply_get_property(psy, POWER_SUPPLY_PROP_CAPACITY, &prop); + if (ret < 0) { + AW_DEV_LOGE(dev, "get vbat capacity failed"); + return -EINVAL; + } + *vbat_capacity = prop.intval; + AW_DEV_LOGI(dev, "The percentage is %d", + *vbat_capacity); + + return 0; +} + +static int aw_search_vmax_from_table(struct device *dev, + struct aw_monitor *monitor, + const int vbat_vol, int *vmax_vol) +{ + int i = 0; + int vmax_set = 0; + uint32_t vmax_flag = 0; + struct aw_monitor_header *monitor_hdr = &monitor->monitor_hdr; + struct vmax_step_config *vmax_cfg = monitor->vmax_cfg; + + if (monitor->bin_status == AW_MONITOR_CFG_WAIT) { + AW_DEV_LOGE(dev, "vmax_cfg not loaded or parse failed"); + return -ENODATA; + } + + for (i = 0; i < monitor_hdr->step_count; i++) { + if (vbat_vol == AW_VBAT_MAX) { + vmax_set = AW_VMAX_MAX; + vmax_flag = 1; + AW_DEV_LOGD(dev, "vbat=%d, setting vmax=0x%x", + vbat_vol, vmax_set); + break; + } + + if (vbat_vol >= vmax_cfg[i].vbat_min && + vbat_vol < vmax_cfg[i].vbat_max) { + vmax_set = vmax_cfg[i].vmax_vol; + vmax_flag = 1; + AW_DEV_LOGD(dev, "read setting vmax=0x%x, step[%d]: vbat_min=%d,vbat_max=%d", + vmax_set, i, + vmax_cfg[i].vbat_min, + vmax_cfg[i].vbat_max); + break; + } + } + + if (!vmax_flag) { + AW_DEV_LOGE(dev, "vmax_cfg not found"); + return -ENODATA; + } + + *vmax_vol = vmax_set; + return 0; +} + + +/*************************************************************************** + * + *monitor_esd_func + * + ***************************************************************************/ +static int aw_chip_status_recover(struct aw87xxx *aw87xxx) +{ + int ret = -1; + struct aw_monitor *monitor = &aw87xxx->monitor; + char *profile = aw87xxx->current_profile; + + AW_DEV_LOGD(aw87xxx->dev, "enter"); + + ret = aw87xxx_update_profile_esd(aw87xxx, profile); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "load profile[%s] failed ", + profile); + return ret; + } + + AW_DEV_LOGI(aw87xxx->dev, "current prof[%s], dev_index[%d] ", + profile, aw87xxx->dev_index); + + monitor->pre_vmax = AW_VMAX_INIT_VAL; + monitor->first_entry = AW_FIRST_ENTRY; + monitor->timer_cnt = 0; + monitor->vbat_sum = 0; + + return 0; +} + +static int aw_monitor_chip_esd_check_work(struct aw87xxx *aw87xxx) +{ + int ret = 0; + int i = 0; + + for (i = 0; i < REG_STATUS_CHECK_MAX; i++) { + AW_DEV_LOGD(aw87xxx->dev, "reg_status_check[%d]", i); + + ret = aw87xxx_dev_esd_reg_status_check(&aw87xxx->aw_dev); + if (ret < 0) { + aw_chip_status_recover(aw87xxx); + } else { + AW_DEV_LOGD(aw87xxx->dev, "chip status check succeed"); + break; + } + msleep(AW_ESD_CHECK_DELAY); + } + + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "chip status recover failed,chip off"); + aw87xxx_update_profile_esd(aw87xxx, aw87xxx->prof_off_name); + return ret; + } + + return 0; +} + + +/*************************************************************************** + * + * aw87xxx monitor work with dsp + * + ***************************************************************************/ +static int aw_monitor_update_vmax_to_dsp(struct device *dev, + struct aw_monitor *monitor, int vmax_set) +{ + int ret = -1; + uint32_t enable = 0; + + if (monitor->pre_vmax != vmax_set) { + ret = aw87xxx_dsp_get_rx_module_enable(&enable); + if (!enable || ret < 0) { + AW_DEV_LOGE(dev, "get rx failed or rx disable, ret=%d, enable=%d", + ret, enable); + return -EPERM; + } + + ret = aw87xxx_dsp_set_vmax(vmax_set, monitor->dev_index); + if (ret) { + AW_DEV_LOGE(dev, "set dsp msg fail, ret=%d", ret); + return ret; + } + + AW_DEV_LOGI(dev, "set dsp vmax=0x%x sucess", vmax_set); + monitor->pre_vmax = vmax_set; + } else { + AW_DEV_LOGI(dev, "vmax=0x%x no change", vmax_set); + } + + return 0; +} + +static void aw_monitor_with_dsp_vmax_work(struct device *dev, + struct aw_monitor *monitor) +{ + int ret = -1; + int vmax_set = 0; + int vbat_capacity = 0; + int ave_capacity = 0; + struct aw_monitor_header *monitor_hdr = &monitor->monitor_hdr; + + AW_DEV_LOGD(dev, "enter with dsp monitor"); + + ret = aw_monitor_get_battery_capacity(dev, monitor, &vbat_capacity); + if (ret < 0) + return; + + if (monitor->timer_cnt < monitor_hdr->monitor_count) { + monitor->timer_cnt++; + monitor->vbat_sum += vbat_capacity; + AW_DEV_LOGI(dev, "timer_cnt = %d", + monitor->timer_cnt); + } + if ((monitor->timer_cnt >= monitor_hdr->monitor_count) || + (monitor->first_entry == AW_FIRST_ENTRY)) { + if (monitor->first_entry == AW_FIRST_ENTRY) + monitor->first_entry = AW_NOT_FIRST_ENTRY; + ave_capacity = monitor->vbat_sum / monitor->timer_cnt; + + if (monitor->custom_capacity) + ave_capacity = monitor->custom_capacity; + + AW_DEV_LOGI(dev, "get average capacity = %d", ave_capacity); + + ret = aw_search_vmax_from_table(dev, monitor, + ave_capacity, &vmax_set); + if (ret < 0) + AW_DEV_LOGE(dev, "not find vmax_vol"); + else + aw_monitor_update_vmax_to_dsp(dev, monitor, vmax_set); + + monitor->timer_cnt = 0; + monitor->vbat_sum = 0; + } +} + +static void aw_monitor_work_func(struct work_struct *work) +{ + int ret = 0; + struct aw87xxx *aw87xxx = container_of(work, + struct aw87xxx, monitor.with_dsp_work.work); + struct device *dev = aw87xxx->dev; + struct aw_monitor *monitor = &aw87xxx->monitor; + struct aw_monitor_header *monitor_hdr = &monitor->monitor_hdr; + + AW_DEV_LOGD(dev, "enter"); + + if (monitor->esd_enable) { + ret = aw_monitor_chip_esd_check_work(aw87xxx); + if (ret < 0) + return; + } + + if (monitor_hdr->monitor_switch && !(aw87xxx->aw_dev.is_rec_mode) && + monitor->open_dsp_en && monitor->bin_status == AW_ACF_UPDATE) { + AW_DEV_LOGD(dev, "start low power protection"); + aw_monitor_with_dsp_vmax_work(dev, monitor); + } + + if (monitor->esd_enable || (monitor_hdr->monitor_switch && + !(aw87xxx->aw_dev.is_rec_mode) && monitor->open_dsp_en && + monitor->bin_status == AW_ACF_UPDATE)) { + schedule_delayed_work(&monitor->with_dsp_work, + msecs_to_jiffies(monitor_hdr->monitor_time)); + } +} + +void aw87xxx_monitor_stop(struct aw_monitor *monitor) +{ + struct aw87xxx *aw87xxx = + container_of(monitor, struct aw87xxx, monitor); + + AW_DEV_LOGD(aw87xxx->dev, "enter"); + cancel_delayed_work_sync(&monitor->with_dsp_work); +} + +void aw87xxx_monitor_start(struct aw_monitor *monitor) +{ + struct aw87xxx *aw87xxx = + container_of(monitor, struct aw87xxx, monitor); + int ret = 0; + + ret = aw87xxx_dev_check_reg_is_rec_mode(&aw87xxx->aw_dev); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "get reg current mode failed"); + return; + } + + if (monitor->esd_enable || (monitor->monitor_hdr.monitor_switch && + !(aw87xxx->aw_dev.is_rec_mode) && monitor->open_dsp_en + && monitor->bin_status == AW_ACF_UPDATE)) { + + AW_DEV_LOGD(aw87xxx->dev, "enter"); + monitor->pre_vmax = AW_VMAX_INIT_VAL; + monitor->first_entry = AW_FIRST_ENTRY; + monitor->timer_cnt = 0; + monitor->vbat_sum = 0; + + schedule_delayed_work(&monitor->with_dsp_work, + msecs_to_jiffies(monitor->monitor_hdr.monitor_time)); + } +} +/*************************************************************************** + * + * aw87xxx no dsp monitor func + * + ***************************************************************************/ +int aw87xxx_monitor_no_dsp_get_vmax(struct aw_monitor *monitor, int32_t *vmax) +{ + int vbat_capacity = 0; + int ret = -1; + int vmax_vol = 0; + struct aw87xxx *aw87xxx = + container_of(monitor, struct aw87xxx, monitor); + struct device *dev = aw87xxx->dev; + + ret = aw_monitor_get_battery_capacity(dev, monitor, &vbat_capacity); + if (ret < 0) + return ret; + + if (monitor->custom_capacity) + vbat_capacity = monitor->custom_capacity; + AW_DEV_LOGI(dev, "get_battery_capacity is[%d]", vbat_capacity); + + ret = aw_search_vmax_from_table(dev, monitor, + vbat_capacity, &vmax_vol); + if (ret < 0) { + AW_DEV_LOGE(dev, "not find vmax_vol"); + return ret; + } + + *vmax = vmax_vol; + return 0; +} + + +/*************************************************************************** + * + * aw87xxx monitor sysfs nodes + * + ***************************************************************************/ +static ssize_t aw_attr_get_esd_enable(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + + if (monitor->esd_enable) { + AW_DEV_LOGI(aw87xxx->dev, "esd-enable=true"); + len += snprintf(buf + len, PAGE_SIZE - len, + "esd-enable=true\n"); + } else { + AW_DEV_LOGI(aw87xxx->dev, "esd-enable=false"); + len += snprintf(buf + len, PAGE_SIZE - len, + "esd-enable=false\n"); + } + + return len; +} + +static ssize_t aw_attr_set_esd_enable(struct device *dev, + struct device_attribute *attr, const char *buf, size_t len) +{ + char esd_enable[AW_ESD_ENABLE_STRLEN] = {0}; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + + if (strlen(buf) > AW_ESD_ENABLE_STRLEN) { + AW_DEV_LOGE(aw87xxx->dev, "input esd_enable_str_len is out of max[%d]", + AW_ESD_ENABLE_STRLEN); + return -EINVAL; + } + + if (sscanf(buf, "%s", esd_enable) == 1) { + AW_DEV_LOGD(aw87xxx->dev, "input esd-enable=[%s]", esd_enable); + if (!strcmp(esd_enable, "true")) + monitor->esd_enable = AW_ESD_ENABLE; + else + monitor->esd_enable = AW_ESD_DISABLE; + AW_DEV_LOGI(dev, "set esd-enable=[%s]", + monitor->esd_enable ? "true" : "false"); + } else { + AW_DEV_LOGE(aw87xxx->dev, "input esd-enable error"); + return -EINVAL; + } + + return len; +} + +static ssize_t aw_attr_get_vbat(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + int ret = -1; + int vbat_capacity = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + + if (monitor->custom_capacity == 0) { + ret = aw_monitor_get_battery_capacity(dev, monitor, + &vbat_capacity); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "get battery_capacity failed"); + return ret; + } + len += snprintf(buf + len, PAGE_SIZE - len, + "vbat capacity=%d\n", vbat_capacity); + } else { + len += snprintf(buf + len, PAGE_SIZE - len, + "vbat capacity=%d\n", + monitor->custom_capacity); + } + + return len; +} + +static ssize_t aw_attr_set_vbat(struct device *dev, + struct device_attribute *attr, const char *buf, size_t len) +{ + int ret = -1; + int capacity = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + + ret = kstrtouint(buf, 0, &capacity); + if (ret < 0) + return ret; + AW_DEV_LOGI(aw87xxx->dev, "set capacity = %d", capacity); + if (capacity >= AW_VBAT_CAPACITY_MIN && + capacity <= AW_VBAT_CAPACITY_MAX){ + monitor->custom_capacity = capacity; + } else { + AW_DEV_LOGE(aw87xxx->dev, "vbat_set=invalid,please input value [%d-%d]", + AW_VBAT_CAPACITY_MIN, AW_VBAT_CAPACITY_MAX); + return -EINVAL; + } + + return len; +} + +static ssize_t aw_attr_get_vmax(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + int ret = -1; + int vbat_capacity = 0; + int vmax_get = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + + if (monitor->open_dsp_en) { + ret = aw87xxx_dsp_get_vmax(&vmax_get, aw87xxx->dev_index); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, + "get dsp vmax fail, ret=%d", ret); + return ret; + } + len += snprintf(buf + len, PAGE_SIZE - len, + "get_vmax=%d\n", vmax_get); + } else { + ret = aw_monitor_get_battery_capacity(dev, monitor, + &vbat_capacity); + if (ret < 0) + return ret; + AW_DEV_LOGI(aw87xxx->dev, "get_battery_capacity is [%d]", + vbat_capacity); + + if (monitor->custom_capacity) { + vbat_capacity = monitor->custom_capacity; + AW_DEV_LOGI(aw87xxx->dev, "get custom_capacity is [%d]", + vbat_capacity); + } + + ret = aw_search_vmax_from_table(aw87xxx->dev, monitor, + vbat_capacity, &vmax_get); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "not find vmax_vol"); + len += snprintf(buf + len, PAGE_SIZE - len, + "not_find_vmax_vol\n"); + return len; + } + len += snprintf(buf + len, PAGE_SIZE - len, + "0x%x\n", vmax_get); + AW_DEV_LOGI(aw87xxx->dev, "0x%x", vmax_get); + } + + return len; +} + +static ssize_t aw_attr_set_vmax(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + uint32_t vmax_set = 0; + int ret = -1; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + + ret = kstrtouint(buf, 0, &vmax_set); + if (ret < 0) + return ret; + + AW_DEV_LOGI(aw87xxx->dev, "vmax_set=0x%x", vmax_set); + + if (monitor->open_dsp_en) { + ret = aw87xxx_dsp_set_vmax(vmax_set, aw87xxx->dev_index); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "send dsp_msg error, ret = %d", + ret); + return ret; + } + msleep(2); + } else { + AW_DEV_LOGE(aw87xxx->dev, "no_dsp system,vmax_set invalid"); + return -EINVAL; + } + + return count; +} + +static ssize_t aw_attr_get_monitor_switch(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + struct aw_monitor_header *monitor_hdr = &monitor->monitor_hdr; + + len += snprintf(buf + len, PAGE_SIZE - len, + "aw87xxx monitor switch: %u\n", + monitor_hdr->monitor_switch); + return len; +} + + +int aw87xxx_dev_monitor_switch_set(struct aw_monitor *monitor, uint32_t enable) +{ + struct aw87xxx *aw87xxx = + container_of(monitor, struct aw87xxx, monitor); + struct aw_monitor_header *monitor_hdr = &monitor->monitor_hdr; + + AW_DEV_LOGI(aw87xxx->dev, "monitor switch set =%d", enable); + + if (!monitor->bin_status) { + AW_DEV_LOGE(aw87xxx->dev, "bin parse faile or not loaded,set invalid"); + return -EINVAL; + } + + if (monitor_hdr->monitor_switch == enable) + return 0; + + if (enable > 0) { + monitor_hdr->monitor_switch = 1; + if (monitor->open_dsp_en) { + monitor->pre_vmax = AW_VMAX_INIT_VAL; + monitor->first_entry = AW_FIRST_ENTRY; + monitor->timer_cnt = 0; + monitor->vbat_sum = 0; + } + } else { + monitor_hdr->monitor_switch = 0; + } + + return 0; +} + +static ssize_t aw_attr_set_monitor_switch(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + uint32_t enable = 0; + int ret = -1; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + + ret = kstrtouint(buf, 0, &enable); + if (ret < 0) + return ret; + + ret = aw87xxx_dev_monitor_switch_set(monitor, enable); + if (ret) + return ret; + + return count; +} + +static ssize_t aw_attr_get_monitor_time(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + struct aw_monitor_header *monitor_hdr = &monitor->monitor_hdr; + + len += snprintf(buf + len, PAGE_SIZE - len, + "aw_monitor_timer = %u(ms)\n", + monitor_hdr->monitor_time); + return len; +} + +static ssize_t aw_attr_set_monitor_time(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + unsigned int timer_val = 0; + int ret = -1; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + struct aw_monitor_header *monitor_hdr = &monitor->monitor_hdr; + + ret = kstrtouint(buf, 0, &timer_val); + if (ret < 0) + return ret; + + AW_DEV_LOGI(aw87xxx->dev, "input monitor timer=%d(ms)", timer_val); + + if (!monitor->bin_status) { + AW_DEV_LOGE(aw87xxx->dev, "bin parse faile or not loaded,set invalid"); + return -EINVAL; + } + + if (timer_val != monitor_hdr->monitor_time) + monitor_hdr->monitor_time = timer_val; + else + AW_DEV_LOGI(aw87xxx->dev, "no_change monitor_time"); + + return count; +} + +static ssize_t aw_attr_get_monitor_count(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + struct aw_monitor_header *monitor_hdr = &monitor->monitor_hdr; + + len += snprintf(buf + len, PAGE_SIZE - len, + "aw_monitor_count = %u\n", + monitor_hdr->monitor_count); + return len; +} + +static ssize_t aw_attr_set_monitor_count(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + unsigned int monitor_count = 0; + int ret = -1; + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + struct aw_monitor_header *monitor_hdr = &monitor->monitor_hdr; + + ret = kstrtouint(buf, 0, &monitor_count); + if (ret < 0) + return ret; + AW_DEV_LOGI(aw87xxx->dev, "input monitor count=%d", monitor_count); + + if (!monitor->bin_status) { + AW_DEV_LOGE(aw87xxx->dev, "bin parse faile or not loaded,set invalid"); + return -EINVAL; + } + + if (monitor_count != monitor_hdr->monitor_count) + monitor_hdr->monitor_count = monitor_count; + else + AW_DEV_LOGI(aw87xxx->dev, "no_change monitor_count"); + + return count; +} + + +static ssize_t aw_attr_get_rx(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + ssize_t len = 0; + int ret = -1; + uint32_t enable = 0; + + if (monitor->open_dsp_en) { + ret = aw87xxx_dsp_get_rx_module_enable(&enable); + if (ret) { + AW_DEV_LOGE(aw87xxx->dev, "dsp_msg error, ret=%d", ret); + return ret; + } + len += snprintf(buf + len, PAGE_SIZE - len, + "aw87xxx rx: %u\n", enable); + } else { + len += snprintf(buf + len, PAGE_SIZE - len, + "command is invalid\n"); + } + + return len; +} + +static ssize_t aw_attr_set_rx(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct aw87xxx *aw87xxx = dev_get_drvdata(dev); + struct aw_monitor *monitor = &aw87xxx->monitor; + int ret = -1; + uint32_t enable; + + ret = kstrtouint(buf, 0, &enable); + if (ret < 0) + return ret; + + if (monitor->open_dsp_en) { + AW_DEV_LOGI(aw87xxx->dev, "set rx enable=%d", enable); + + ret = aw87xxx_dsp_set_rx_module_enable(enable); + if (ret < 0) { + AW_DEV_LOGE(aw87xxx->dev, "dsp_msg error, ret=%d", + ret); + return ret; + } + } else { + AW_DEV_LOGE(aw87xxx->dev, "command is invalid"); + return -EINVAL; + } + + return count; +} + + +static DEVICE_ATTR(esd_enable, S_IWUSR | S_IRUGO, + aw_attr_get_esd_enable, aw_attr_set_esd_enable); +static DEVICE_ATTR(vbat, S_IWUSR | S_IRUGO, + aw_attr_get_vbat, aw_attr_set_vbat); +static DEVICE_ATTR(vmax, S_IWUSR | S_IRUGO, + aw_attr_get_vmax, aw_attr_set_vmax); + +static DEVICE_ATTR(monitor_switch, S_IWUSR | S_IRUGO, + aw_attr_get_monitor_switch, aw_attr_set_monitor_switch); +static DEVICE_ATTR(monitor_time, S_IWUSR | S_IRUGO, + aw_attr_get_monitor_time, aw_attr_set_monitor_time); +static DEVICE_ATTR(monitor_count, S_IWUSR | S_IRUGO, + aw_attr_get_monitor_count, aw_attr_set_monitor_count); +static DEVICE_ATTR(rx, S_IWUSR | S_IRUGO, + aw_attr_get_rx, aw_attr_set_rx); + +static struct attribute *aw_monitor_vol_adjust[] = { + &dev_attr_esd_enable.attr, + &dev_attr_vbat.attr, + &dev_attr_vmax.attr, + NULL +}; + +static struct attribute_group aw_monitor_vol_adjust_group = { + .attrs = aw_monitor_vol_adjust, +}; + +static struct attribute *aw_monitor_control[] = { + &dev_attr_monitor_switch.attr, + &dev_attr_monitor_time.attr, + &dev_attr_monitor_count.attr, + &dev_attr_rx.attr, + NULL +}; + +static struct attribute_group aw_monitor_control_group = { + .attrs = aw_monitor_control, +}; + +/*************************************************************************** + * + * aw87xxx monitor init + * + ***************************************************************************/ +static void aw_monitor_dtsi_parse(struct device *dev, + struct aw_monitor *monitor, + struct device_node *dev_node) +{ + int ret = -1; + const char *esd_enable; + + ret = of_property_read_string(dev_node, "esd-enable", &esd_enable); + if (ret < 0) { + AW_DEV_LOGI(dev, "esd_enable parse failed, user default[disable]"); + monitor->esd_enable = AW_ESD_DISABLE; + } else { + if (!strcmp(esd_enable, "true")) + monitor->esd_enable = AW_ESD_ENABLE; + else + monitor->esd_enable = AW_ESD_DISABLE; + + AW_DEV_LOGI(dev, "parse esd-enable=[%s]", + monitor->esd_enable ? "true" : "false"); + } +} + +void aw87xxx_monitor_init(struct device *dev, struct aw_monitor *monitor, + struct device_node *dev_node) +{ + int ret = -1; + struct aw87xxx *aw87xxx = + container_of(monitor, struct aw87xxx, monitor); + + monitor->dev_index = aw87xxx->dev_index; + monitor->monitor_hdr.monitor_time = AW_DEFAULT_MONITOR_TIME; + + aw_monitor_dtsi_parse(dev, monitor, dev_node); + + /* get platform open dsp type */ + monitor->open_dsp_en = aw87xxx_dsp_isEnable(); + + ret = sysfs_create_group(&dev->kobj, &aw_monitor_vol_adjust_group); + if (ret < 0) + AW_DEV_LOGE(dev, "failed to create monitor vol_adjust sysfs nodes"); + + INIT_DELAYED_WORK(&monitor->with_dsp_work, aw_monitor_work_func); + + if (monitor->open_dsp_en) { + ret = sysfs_create_group(&dev->kobj, &aw_monitor_control_group); + if (ret < 0) + AW_DEV_LOGE(dev, "failed to create monitor dsp control sysfs nodes"); + } + + if (!ret) + AW_DEV_LOGI(dev, "monitor init succeed"); +} + +void aw87xxx_monitor_exit(struct aw_monitor *monitor) +{ + struct aw87xxx *aw87xxx = + container_of(monitor, struct aw87xxx, monitor); + /*rm attr node*/ + sysfs_remove_group(&aw87xxx->dev->kobj, + &aw_monitor_vol_adjust_group); + + aw87xxx_monitor_stop(monitor); + + if (monitor->open_dsp_en) { + sysfs_remove_group(&aw87xxx->dev->kobj, + &aw_monitor_control_group); + } +} + diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_monitor.h b/sound/soc/codecs/aw87xxx/aw87xxx_monitor.h new file mode 100644 index 0000000000000..daf9f2bfa09f5 --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_monitor.h @@ -0,0 +1,96 @@ +#ifndef __AW87XXX_MONITOR_H__ +#define __AW87XXX_MONITOR_H__ + +#define AW_WAIT_DSP_OPEN_TIME (3000) +#define AW_VBAT_CAPACITY_MIN (0) +#define AW_VBAT_CAPACITY_MAX (100) +#define AW_VMAX_INIT_VAL (0xFFFFFFFF) +#define AW_VBAT_MAX (100) +#define AW_VMAX_MAX (0) +#define AW_DEFAULT_MONITOR_TIME (3000) +#define AW_WAIT_TIME (3000) +#define REG_STATUS_CHECK_MAX (10) +#define AW_ESD_CHECK_DELAY (1) + +#define AW_ESD_ENABLE (true) +#define AW_ESD_DISABLE (false) +#define AW_ESD_ENABLE_STRLEN (16) + +enum aw_monitor_init { + AW_MONITOR_CFG_WAIT = 0, + AW_MONITOR_CFG_OK = 1, +}; + +enum aw_monitor_hdr_info { + AW_MONITOR_HDR_DATA_SIZE = 0x00000004, + AW_MONITOR_HDR_DATA_BYTE_LEN = 0x00000004, +}; + +enum aw_monitor_data_ver { + AW_MONITOR_DATA_VER = 0x00000001, + AW_MONITOR_DATA_VER_MAX, +}; + +enum aw_monitor_first_enter { + AW_FIRST_ENTRY = 0, + AW_NOT_FIRST_ENTRY = 1, +}; + +struct aw_bin_header { + uint32_t check_sum; + uint32_t header_ver; + uint32_t bin_data_type; + uint32_t bin_data_ver; + uint32_t bin_data_size; + uint32_t ui_ver; + char product[8]; + uint32_t addr_byte_len; + uint32_t data_byte_len; + uint32_t device_addr; + uint32_t reserve[4]; +}; + +struct aw_monitor_header { + uint32_t monitor_switch; + uint32_t monitor_time; + uint32_t monitor_count; + uint32_t step_count; + uint32_t reserve[4]; +}; + +struct vmax_step_config { + uint32_t vbat_min; + uint32_t vbat_max; + int vmax_vol; +}; + +struct aw_monitor { + bool open_dsp_en; + bool esd_enable; + int32_t dev_index; + uint8_t first_entry; + uint8_t timer_cnt; + uint32_t vbat_sum; + int32_t custom_capacity; + uint32_t pre_vmax; + + int bin_status; + struct aw_monitor_header monitor_hdr; + struct vmax_step_config *vmax_cfg; + + struct delayed_work with_dsp_work; +}; + +void aw87xxx_monitor_cfg_free(struct aw_monitor *monitor); +int aw87xxx_monitor_bin_parse(struct device *dev, + char *monitor_data, uint32_t data_len); +void aw87xxx_monitor_stop(struct aw_monitor *monitor); +void aw87xxx_monitor_start(struct aw_monitor *monitor); +int aw87xxx_monitor_no_dsp_get_vmax(struct aw_monitor *monitor, + int32_t *vmax); +void aw87xxx_monitor_init(struct device *dev, struct aw_monitor *monitor, + struct device_node *dev_node); +void aw87xxx_monitor_exit(struct aw_monitor *monitor); +int aw87xxx_dev_monitor_switch_set(struct aw_monitor *monitor, uint32_t enable); + +#endif diff --git a/sound/soc/codecs/aw87xxx/aw87xxx_pid_18_reg.h b/sound/soc/codecs/aw87xxx/aw87xxx_pid_18_reg.h new file mode 100644 index 0000000000000..74d6548db91e3 --- /dev/null +++ b/sound/soc/codecs/aw87xxx/aw87xxx_pid_18_reg.h @@ -0,0 +1,2315 @@ +#ifndef __AW87XXX_PID_18_REG_H__ +#define __AW87XXX_PID_18_REG_H__ + +/* registers list */ +#define AW87XXX_PID_18_CHIPID_REG (0x00) +#define AW87XXX_PID_18_SYSST_REG (0x01) +#define AW87XXX_PID_18_SYSINT_REG (0x02) +#define AW87XXX_PID_18_SYSCTRL_REG (0x03) +#define AW87XXX_PID_18_CPOC_REG (0x04) +#define AW87XXX_PID_18_CLASSD_REG (0x05) +#define AW87XXX_PID_18_MADPVTH_REG (0x06) +#define AW87XXX_PID_18_A3PARAM_REG (0x07) +#define AW87XXX_PID_18_A3A2PO_REG (0x08) +#define AW87XXX_PID_18_A2PARAM_REG (0x09) +#define AW87XXX_PID_18_A1PARAM_REG (0x0A) +#define AW87XXX_PID_18_POPCLK_REG (0x0B) +#define AW87XXX_PID_18_GTDRCPSS_REG (0x0C) +#define AW87XXX_PID_18_MULTI_REG (0x0D) +#define AW87XXX_PID_18_DFT1_REG (0x61) +#define AW87XXX_PID_18_DFT2_REG (0x62) +#define AW87XXX_PID_18_DFT3_REG (0x63) +#define AW87XXX_PID_18_DFT4_REG (0x64) +#define AW87XXX_PID_18_DFT5_REG (0x65) +#define AW87XXX_PID_18_DFT6_REG (0x66) + +#define AW87XXX_PID_18_CLASSD_DEFAULT (0x10) + +/******************************************** + * soft control info + * If you need to update this file, add this information manually + *******************************************/ +unsigned char aw87xxx_pid_18_softrst_access[2] = {0x00, 0xaa}; + +/******************************************** + * Register Access + *******************************************/ +#define AW87XXX_PID_18_REG_MAX (0x67) + +#define REG_NONE_ACCESS (0) +#define REG_RD_ACCESS (1 << 0) +#define REG_WR_ACCESS (1 << 1) + +const unsigned char aw87xxx_pid_18_reg_access[AW87XXX_PID_18_REG_MAX] = { + [AW87XXX_PID_18_CHIPID_REG] = (REG_RD_ACCESS), + [AW87XXX_PID_18_SYSST_REG] = (REG_RD_ACCESS), + [AW87XXX_PID_18_SYSINT_REG] = (REG_RD_ACCESS), + [AW87XXX_PID_18_SYSCTRL_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_CPOC_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_CLASSD_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_MADPVTH_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_A3PARAM_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_A3A2PO_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_A2PARAM_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_A1PARAM_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_POPCLK_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_GTDRCPSS_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_MULTI_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_DFT1_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_DFT2_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_DFT3_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_DFT4_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_DFT5_REG] = (REG_RD_ACCESS | REG_WR_ACCESS), + [AW87XXX_PID_18_DFT6_REG] = (REG_RD_ACCESS), +}; + +/* detail information of registers begin */ +/* CHIPID (0x00) detail */ +/* IDCODE bit 7:0 (CHIPID 0x00) */ +#define AW87XXX_PID_18_IDCODE_START_BIT (0) +#define AW87XXX_PID_18_IDCODE_BITS_LEN (8) +#define AW87XXX_PID_18_IDCODE_MASK \ + (~(((1<dev, "check sysst fail, reg_val=0x%04x, check:0x%x", - reg_val, AW88261_BIT_SYSST_CHECK); + reg_val, AW88261_BIT_PLL_CHECK); usleep_range(AW88261_2000_US, AW88261_2000_US + 10); } else { return 0; diff --git a/sound/soc/codecs/aw88261.h b/sound/soc/codecs/aw88261.h index 270ccf375f367..f16f9f8c40a2f 100644 --- a/sound/soc/codecs/aw88261.h +++ b/sound/soc/codecs/aw88261.h @@ -194,12 +194,6 @@ AW88261_OTHS_OT_VALUE | \ AW88261_PLLS_LOCKED_VALUE)) -#define AW88261_BIT_SYSST_CHECK \ - (AW88261_BSTS_FINISHED_VALUE | \ - AW88261_SWS_SWITCHING_VALUE | \ - AW88261_CLKS_STABLE_VALUE | \ - AW88261_PLLS_LOCKED_VALUE) - #define AW88261_ULS_HMUTE_START_BIT (14) #define AW88261_ULS_HMUTE_BITS_LEN (1) #define AW88261_ULS_HMUTE_MASK \ diff --git a/sound/soc/codecs/cs-amp-lib.c b/sound/soc/codecs/cs-amp-lib.c index 371e99205b58e..58b06e6878b97 100644 --- a/sound/soc/codecs/cs-amp-lib.c +++ b/sound/soc/codecs/cs-amp-lib.c @@ -314,6 +314,8 @@ static void *cs_amp_alloc_get_efi_variable(efi_char16_t *name, unsigned long size = 0; status = cs_amp_get_efi_variable(name, guid, NULL, &size, NULL); + if (status == EFI_SUCCESS) + return ERR_PTR(-ENOENT); if (status != EFI_BUFFER_TOO_SMALL) return ERR_PTR(cs_amp_convert_efi_status(status)); diff --git a/sound/soc/codecs/cs35l33.c b/sound/soc/codecs/cs35l33.c index f49edb9ea1f2c..e217da124549a 100644 --- a/sound/soc/codecs/cs35l33.c +++ b/sound/soc/codecs/cs35l33.c @@ -40,6 +40,7 @@ struct cs35l33_private { struct regmap *regmap; struct gpio_desc *reset_gpio; bool amp_cal; + bool irq_requested; int mclk_int; struct regulator_bulk_data core_supplies[2]; int num_core_supplies; @@ -881,6 +882,9 @@ static int cs35l33_runtime_resume(struct device *dev) goto err; } + if (cs35l33->irq_requested) + enable_irq(to_i2c_client(dev)->irq); + return 0; err: @@ -900,6 +904,10 @@ static int cs35l33_runtime_suspend(struct device *dev) /* redo the calibration in next power up */ cs35l33->amp_cal = false; + /* Drain and block the threaded IRQ before cache_only/power-off. */ + if (cs35l33->irq_requested) + disable_irq(to_i2c_client(dev)->irq); + regcache_cache_only(cs35l33->regmap, true); regcache_mark_dirty(cs35l33->regmap); regulator_bulk_disable(cs35l33->num_core_supplies, @@ -1154,10 +1162,12 @@ static int cs35l33_i2c_probe(struct i2c_client *i2c_client) } ret = devm_request_threaded_irq(&i2c_client->dev, i2c_client->irq, NULL, - cs35l33_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW, - "cs35l33", cs35l33); + cs35l33_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW, + "cs35l33", cs35l33); if (ret != 0) dev_warn(&i2c_client->dev, "Failed to request IRQ: %d\n", ret); + else + cs35l33->irq_requested = true; /* We could issue !RST or skip it based on AMP topology */ cs35l33->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev, diff --git a/sound/soc/codecs/cs35l34.c b/sound/soc/codecs/cs35l34.c index e80984b221593..3f265bd760f7c 100644 --- a/sound/soc/codecs/cs35l34.c +++ b/sound/soc/codecs/cs35l34.c @@ -45,6 +45,7 @@ struct cs35l34_private { int num_core_supplies; int mclk_int; bool tdm_mode; + bool irq_requested; struct gpio_desc *reset_gpio; /* Active-low reset GPIO */ }; @@ -1032,10 +1033,12 @@ static int cs35l34_i2c_probe(struct i2c_client *i2c_client) } ret = devm_request_threaded_irq(&i2c_client->dev, i2c_client->irq, NULL, - cs35l34_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW, - "cs35l34", cs35l34); + cs35l34_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW, + "cs35l34", cs35l34); if (ret != 0) dev_err(&i2c_client->dev, "Failed to request IRQ: %d\n", ret); + else + cs35l34->irq_requested = true; cs35l34->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev, "reset", GPIOD_OUT_LOW); @@ -1140,6 +1143,9 @@ static int cs35l34_runtime_resume(struct device *dev) dev_err(dev, "Failed to restore register cache\n"); goto err; } + + if (cs35l34->irq_requested) + enable_irq(to_i2c_client(dev)->irq); return 0; err: regcache_cache_only(cs35l34->regmap, true); @@ -1153,6 +1159,10 @@ static int cs35l34_runtime_suspend(struct device *dev) { struct cs35l34_private *cs35l34 = dev_get_drvdata(dev); + /* Drain and block the threaded IRQ before cache_only/power-off. */ + if (cs35l34->irq_requested) + disable_irq(to_i2c_client(dev)->irq); + regcache_cache_only(cs35l34->regmap, true); regcache_mark_dirty(cs35l34->regmap); diff --git a/sound/soc/codecs/cs35l56-i2c.c b/sound/soc/codecs/cs35l56-i2c.c index 4f6ddf1c5a3f6..5e69ddbe342a1 100644 --- a/sound/soc/codecs/cs35l56-i2c.c +++ b/sound/soc/codecs/cs35l56-i2c.c @@ -51,15 +51,7 @@ static int cs35l56_i2c_probe(struct i2c_client *client) return dev_err_probe(cs35l56->base.dev, ret, "Failed to allocate register map\n"); } - ret = cs35l56_common_probe(cs35l56); - if (ret != 0) - return ret; - - ret = cs35l56_irq_request(&cs35l56->base, client->irq); - if (ret < 0) - cs35l56_remove(cs35l56); - - return ret; + return cs35l56_common_probe(cs35l56, client->irq); } static void cs35l56_i2c_remove(struct i2c_client *client) diff --git a/sound/soc/codecs/cs35l56-sdw.c b/sound/soc/codecs/cs35l56-sdw.c index 0a55b93b96f96..a3812ab4c0227 100644 --- a/sound/soc/codecs/cs35l56-sdw.c +++ b/sound/soc/codecs/cs35l56-sdw.c @@ -483,11 +483,7 @@ static int cs35l56_sdw_probe(struct sdw_slave *peripheral, const struct sdw_devi /* Start in cache-only until device is enumerated */ regcache_cache_only(cs35l56->base.regmap, true); - ret = cs35l56_common_probe(cs35l56); - if (ret != 0) - return ret; - - return 0; + return cs35l56_common_probe(cs35l56, -EINVAL); } static void cs35l56_sdw_remove(struct sdw_slave *peripheral) diff --git a/sound/soc/codecs/cs35l56-spi.c b/sound/soc/codecs/cs35l56-spi.c index b1eb924a5b6cc..21b18da9e73d1 100644 --- a/sound/soc/codecs/cs35l56-spi.c +++ b/sound/soc/codecs/cs35l56-spi.c @@ -40,15 +40,7 @@ static int cs35l56_spi_probe(struct spi_device *spi) if (ret) return ret; - ret = cs35l56_common_probe(cs35l56); - if (ret != 0) - return ret; - - ret = cs35l56_irq_request(&cs35l56->base, spi->irq); - if (ret < 0) - cs35l56_remove(cs35l56); - - return ret; + return cs35l56_common_probe(cs35l56, spi->irq); } static void cs35l56_spi_remove(struct spi_device *spi) diff --git a/sound/soc/codecs/cs35l56.c b/sound/soc/codecs/cs35l56.c index 063ef7a70de03..8beef93ac4676 100644 --- a/sound/soc/codecs/cs35l56.c +++ b/sound/soc/codecs/cs35l56.c @@ -18,9 +18,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -37,6 +39,13 @@ #include "wm_adsp.h" #include "cs35l56.h" +/* + * snd_soc_register_component() can call component_probe() on all instances + * in a card, so deferred registration must be protected across all instances. + */ +static DEFINE_MUTEX(cs35l56_component_register_lock); +static bool cs35l56_shutting_down; + void cs35l56_mask_soundwire_interrupts(struct sdw_slave *peripheral) { /* @@ -1368,12 +1377,6 @@ static int _cs35l56_component_probe(struct snd_soc_component *component) BUILD_BUG_ON(ARRAY_SIZE(cs35l56_tx_input_texts) != ARRAY_SIZE(cs35l56_tx_input_values)); - if (!wait_for_completion_timeout(&cs35l56->init_completion, - msecs_to_jiffies(5000))) { - dev_err(cs35l56->base.dev, "%s: init_completion timed out\n", __func__); - return -ENODEV; - } - cs35l56->dsp.part = kasprintf(GFP_KERNEL, "cs35l%02x", cs35l56->base.type); if (!cs35l56->dsp.part) return -ENOMEM; @@ -1941,7 +1944,46 @@ static int cs35l56_try_get_broken_sdca_spkid_gpio(struct cs35l56_private *cs35l5 return ret; } -int cs35l56_common_probe(struct cs35l56_private *cs35l56) +static int cs35l56_component_register(struct cs35l56_private *cs35l56) +{ + int ret; + + ret = snd_soc_register_component(cs35l56->base.dev, + &soc_component_dev_cs35l56, + cs35l56_dai, ARRAY_SIZE(cs35l56_dai)); + if (ret < 0) { + dev_err(cs35l56->base.dev, "Register codec failed: %d\n", ret); + return ret; + } + + cs35l56->component_registered = true; + + return 0; +} + +static void cs35l56_component_register_work(struct work_struct *work) +{ + struct cs35l56_private *cs35l56 = container_of(work, + struct cs35l56_private, + component_register_work); + int ret; + + guard(mutex)(&cs35l56_component_register_lock); + + if (cs35l56_shutting_down) + return; + + PM_RUNTIME_ACQUIRE_AUTOSUSPEND(cs35l56->base.dev, pm_err); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm_err); + if (ret) { + dev_err(cs35l56->base.dev, "register_work failed to get pm_runtime: %d\n", ret); + return; + } + + cs35l56_component_register(cs35l56); +} + +int cs35l56_common_probe(struct cs35l56_private *cs35l56, int irq) { int ret; @@ -1949,6 +1991,7 @@ int cs35l56_common_probe(struct cs35l56_private *cs35l56) mutex_init(&cs35l56->base.irq_lock); cs35l56->base.cal_index = -1; cs35l56->speaker_id = -ENOENT; + INIT_WORK(&cs35l56->component_register_work, cs35l56_component_register_work); dev_set_drvdata(cs35l56->base.dev, cs35l56); @@ -2018,16 +2061,29 @@ int cs35l56_common_probe(struct cs35l56_private *cs35l56) goto err_remove_wm_adsp; } - ret = snd_soc_register_component(cs35l56->base.dev, - &soc_component_dev_cs35l56, - cs35l56_dai, ARRAY_SIZE(cs35l56_dai)); - if (ret < 0) { - dev_err_probe(cs35l56->base.dev, ret, "Register codec failed\n"); + ret = cs35l56_irq_request(&cs35l56->base, irq); + if (ret) goto err_remove_wm_adsp; + + /* + * Defer calling snd_soc_register_component() on SoundWire to prevent + * a deadlock where it calls our component_probe(), which requires the + * SoundWire enumeration to complete, but because we are still in probe() + * the SoundWire core will not call the update_status() callback. At time + * of writing snd_soc_register_component() never returns EPROBE_DEFER. + */ + if (!cs35l56->sdw_peripheral) { + ret = cs35l56_component_register(cs35l56); + if (ret < 0) + goto err_free_irq; } return 0; +err_free_irq: + if (cs35l56->base.irq) + devm_free_irq(cs35l56->base.dev, cs35l56->base.irq, &cs35l56->base); + err_remove_wm_adsp: wm_adsp2_remove(&cs35l56->dsp); @@ -2049,6 +2105,7 @@ EXPORT_SYMBOL_NS_GPL(cs35l56_common_probe, "SND_SOC_CS35L56_CORE"); int cs35l56_init(struct cs35l56_private *cs35l56) { + bool first_time_init = !cs35l56->base.init_done; int ret; /* @@ -2125,13 +2182,23 @@ int cs35l56_init(struct cs35l56_private *cs35l56) cs35l56->base.init_done = true; complete_all(&cs35l56->init_completion); + if (cs35l56->sdw_peripheral && first_time_init) { + /* + * Hardware now accessible, queue work to call + * snd_soc_register_component(). + */ + queue_work(system_freezable_wq, &cs35l56->component_register_work); + } + return 0; } EXPORT_SYMBOL_NS_GPL(cs35l56_init, "SND_SOC_CS35L56_CORE"); void cs35l56_remove(struct cs35l56_private *cs35l56) { - snd_soc_unregister_component(cs35l56->base.dev); + cancel_work_sync(&cs35l56->component_register_work); + if (cs35l56->component_registered) + snd_soc_unregister_component(cs35l56->base.dev); cs35l56->base.init_done = false; @@ -2166,6 +2233,37 @@ EXPORT_NS_GPL_DEV_PM_OPS(cs35l56_pm_ops_i2c_spi, SND_SOC_CS35L56_CORE) = { }; #endif +static int cs35l56_reboot_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + guard(mutex)(&cs35l56_component_register_lock); + cs35l56_shutting_down = true; + + return NOTIFY_DONE; +} + +static struct notifier_block cs35l56_reboot_notifier = { + .notifier_call = cs35l56_reboot_notify, +}; + +static int __init cs35l56_modinit(void) +{ + /* + * Use reboot notifier to prevent race between shutdown and + * snd_soc_register_component(). Driver shutdown() callback would + * run too late, after device_shutdown() is already walking the + * device list that component registration can modify. + */ + return register_reboot_notifier(&cs35l56_reboot_notifier); +} +module_init(cs35l56_modinit); + +static void __exit cs35l56_modexit(void) +{ + unregister_reboot_notifier(&cs35l56_reboot_notifier); +} +module_exit(cs35l56_modexit); + MODULE_DESCRIPTION("ASoC CS35L56 driver"); MODULE_IMPORT_NS("SND_SOC_CS35L56_SHARED"); MODULE_IMPORT_NS("SND_SOC_CS_AMP_LIB"); diff --git a/sound/soc/codecs/cs35l56.h b/sound/soc/codecs/cs35l56.h index 9acd2e7e17c93..6adba4d2da1f8 100644 --- a/sound/soc/codecs/cs35l56.h +++ b/sound/soc/codecs/cs35l56.h @@ -32,6 +32,7 @@ struct sdw_slave; struct cs35l56_private { struct wm_adsp dsp; /* must be first member */ struct cs35l56_base base; + struct work_struct component_register_work; struct work_struct dsp_work; struct workqueue_struct *dsp_wq; struct snd_soc_component *component; @@ -43,6 +44,7 @@ struct cs35l56_private { bool sdw_irq_no_unmask; bool soft_resetting; bool sdw_attached; + bool component_registered; struct completion init_completion; int speaker_id; @@ -78,7 +80,7 @@ int cs35l56_system_resume_early(struct device *dev); int cs35l56_system_resume(struct device *dev); irqreturn_t cs35l56_irq(int irq, void *data); int cs35l56_irq_request(struct cs35l56_base *cs35l56_base, int irq); -int cs35l56_common_probe(struct cs35l56_private *cs35l56); +int cs35l56_common_probe(struct cs35l56_private *cs35l56, int irq); int cs35l56_init(struct cs35l56_private *cs35l56); void cs35l56_remove(struct cs35l56_private *cs35l56); diff --git a/sound/soc/codecs/cx2072x.c b/sound/soc/codecs/cx2072x.c index 83c6cbd40804f..ffd6a5119f5eb 100644 --- a/sound/soc/codecs/cx2072x.c +++ b/sound/soc/codecs/cx2072x.c @@ -96,51 +96,60 @@ static const struct reg_default cx2072x_reg_defaults[] = { { CX2072X_GPIO_WAKE, 0x00000000 }, { CX2072X_GPIO_UM_ENABLE, 0x00000000 }, { CX2072X_GPIO_STICKY_MASK, 0x00000000 }, - { CX2072X_DAC1_CONVERTER_FORMAT, 0x00000031 }, - { CX2072X_DAC1_AMP_GAIN_RIGHT, 0x0000004a }, - { CX2072X_DAC1_AMP_GAIN_LEFT, 0x0000004a }, { CX2072X_DAC1_POWER_STATE, 0x00000433 }, { CX2072X_DAC1_CONVERTER_STREAM_CHANNEL, 0x00000000 }, { CX2072X_DAC1_EAPD_ENABLE, 0x00000000 }, - { CX2072X_DAC2_CONVERTER_FORMAT, 0x00000031 }, - { CX2072X_DAC2_AMP_GAIN_RIGHT, 0x0000004a }, - { CX2072X_DAC2_AMP_GAIN_LEFT, 0x0000004a }, + { CX2072X_DAC1_AMP_GAIN_RIGHT, 0x0000004a }, + { CX2072X_DAC1_AMP_GAIN_LEFT, 0x0000004a }, + { CX2072X_DAC1_CONVERTER_FORMAT, 0x00000031 }, { CX2072X_DAC2_POWER_STATE, 0x00000433 }, { CX2072X_DAC2_CONVERTER_STREAM_CHANNEL, 0x00000000 }, - { CX2072X_ADC1_CONVERTER_FORMAT, 0x00000031 }, + { CX2072X_DAC2_AMP_GAIN_RIGHT, 0x0000004a }, + { CX2072X_DAC2_AMP_GAIN_LEFT, 0x0000004a }, + { CX2072X_DAC2_CONVERTER_FORMAT, 0x00000031 }, + { CX2072X_ADC1_CONNECTION_SELECT_CONTROL, 0x00000000 }, + { CX2072X_ADC1_POWER_STATE, 0x00000433 }, + { CX2072X_ADC1_CONVERTER_STREAM_CHANNEL, 0x00000000 }, { CX2072X_ADC1_AMP_GAIN_RIGHT_0, 0x0000004a }, - { CX2072X_ADC1_AMP_GAIN_LEFT_0, 0x0000004a }, { CX2072X_ADC1_AMP_GAIN_RIGHT_1, 0x0000004a }, - { CX2072X_ADC1_AMP_GAIN_LEFT_1, 0x0000004a }, { CX2072X_ADC1_AMP_GAIN_RIGHT_2, 0x0000004a }, - { CX2072X_ADC1_AMP_GAIN_LEFT_2, 0x0000004a }, { CX2072X_ADC1_AMP_GAIN_RIGHT_3, 0x0000004a }, - { CX2072X_ADC1_AMP_GAIN_LEFT_3, 0x0000004a }, { CX2072X_ADC1_AMP_GAIN_RIGHT_4, 0x0000004a }, - { CX2072X_ADC1_AMP_GAIN_LEFT_4, 0x0000004a }, { CX2072X_ADC1_AMP_GAIN_RIGHT_5, 0x0000004a }, - { CX2072X_ADC1_AMP_GAIN_LEFT_5, 0x0000004a }, { CX2072X_ADC1_AMP_GAIN_RIGHT_6, 0x0000004a }, + { CX2072X_ADC1_AMP_GAIN_LEFT_0, 0x0000004a }, + { CX2072X_ADC1_AMP_GAIN_LEFT_1, 0x0000004a }, + { CX2072X_ADC1_AMP_GAIN_LEFT_2, 0x0000004a }, + { CX2072X_ADC1_AMP_GAIN_LEFT_3, 0x0000004a }, + { CX2072X_ADC1_AMP_GAIN_LEFT_4, 0x0000004a }, + { CX2072X_ADC1_AMP_GAIN_LEFT_5, 0x0000004a }, { CX2072X_ADC1_AMP_GAIN_LEFT_6, 0x0000004a }, - { CX2072X_ADC1_CONNECTION_SELECT_CONTROL, 0x00000000 }, - { CX2072X_ADC1_POWER_STATE, 0x00000433 }, - { CX2072X_ADC1_CONVERTER_STREAM_CHANNEL, 0x00000000 }, - { CX2072X_ADC2_CONVERTER_FORMAT, 0x00000031 }, + { CX2072X_ADC1_CONVERTER_FORMAT, 0x00000031 }, + { CX2072X_ADC2_CONNECTION_SELECT_CONTROL, 0x00000000 }, + { CX2072X_ADC2_POWER_STATE, 0x00000433 }, + { CX2072X_ADC2_CONVERTER_STREAM_CHANNEL, 0x00000000 }, { CX2072X_ADC2_AMP_GAIN_RIGHT_0, 0x0000004a }, - { CX2072X_ADC2_AMP_GAIN_LEFT_0, 0x0000004a }, { CX2072X_ADC2_AMP_GAIN_RIGHT_1, 0x0000004a }, - { CX2072X_ADC2_AMP_GAIN_LEFT_1, 0x0000004a }, { CX2072X_ADC2_AMP_GAIN_RIGHT_2, 0x0000004a }, + { CX2072X_ADC2_AMP_GAIN_LEFT_0, 0x0000004a }, + { CX2072X_ADC2_AMP_GAIN_LEFT_1, 0x0000004a }, { CX2072X_ADC2_AMP_GAIN_LEFT_2, 0x0000004a }, - { CX2072X_ADC2_CONNECTION_SELECT_CONTROL, 0x00000000 }, - { CX2072X_ADC2_POWER_STATE, 0x00000433 }, - { CX2072X_ADC2_CONVERTER_STREAM_CHANNEL, 0x00000000 }, + { CX2072X_ADC2_CONVERTER_FORMAT, 0x00000031 }, + { CX2072X_MIXER_POWER_STATE, 0x00000433 }, + { CX2072X_MIXER_GAIN_RIGHT_0, 0x0000004a }, + { CX2072X_MIXER_GAIN_RIGHT_1, 0x0000004a }, + { CX2072X_MIXER_GAIN_LEFT_0, 0x0000004a }, + { CX2072X_MIXER_GAIN_LEFT_1, 0x0000004a }, { CX2072X_PORTA_CONNECTION_SELECT_CTRL, 0x00000000 }, { CX2072X_PORTA_POWER_STATE, 0x00000433 }, { CX2072X_PORTA_PIN_CTRL, 0x000000c0 }, { CX2072X_PORTA_UNSOLICITED_RESPONSE, 0x00000000 }, { CX2072X_PORTA_PIN_SENSE, 0x00000000 }, { CX2072X_PORTA_EAPD_BTL, 0x00000002 }, + { CX2072X_PORTG_CONNECTION_SELECT_CTRL, 0x00000000 }, + { CX2072X_PORTG_POWER_STATE, 0x00000433 }, + { CX2072X_PORTG_PIN_CTRL, 0x00000040 }, + { CX2072X_PORTG_EAPD_BTL, 0x00000002 }, { CX2072X_PORTB_POWER_STATE, 0x00000433 }, { CX2072X_PORTB_PIN_CTRL, 0x00000000 }, { CX2072X_PORTB_UNSOLICITED_RESPONSE, 0x00000000 }, @@ -148,43 +157,16 @@ static const struct reg_default cx2072x_reg_defaults[] = { { CX2072X_PORTB_EAPD_BTL, 0x00000002 }, { CX2072X_PORTB_GAIN_RIGHT, 0x00000000 }, { CX2072X_PORTB_GAIN_LEFT, 0x00000000 }, - { CX2072X_PORTC_POWER_STATE, 0x00000433 }, - { CX2072X_PORTC_PIN_CTRL, 0x00000000 }, - { CX2072X_PORTC_GAIN_RIGHT, 0x00000000 }, - { CX2072X_PORTC_GAIN_LEFT, 0x00000000 }, { CX2072X_PORTD_POWER_STATE, 0x00000433 }, { CX2072X_PORTD_PIN_CTRL, 0x00000020 }, { CX2072X_PORTD_UNSOLICITED_RESPONSE, 0x00000000 }, { CX2072X_PORTD_PIN_SENSE, 0x00000000 }, { CX2072X_PORTD_GAIN_RIGHT, 0x00000000 }, { CX2072X_PORTD_GAIN_LEFT, 0x00000000 }, - { CX2072X_PORTE_CONNECTION_SELECT_CTRL, 0x00000000 }, - { CX2072X_PORTE_POWER_STATE, 0x00000433 }, - { CX2072X_PORTE_PIN_CTRL, 0x00000040 }, - { CX2072X_PORTE_UNSOLICITED_RESPONSE, 0x00000000 }, - { CX2072X_PORTE_PIN_SENSE, 0x00000000 }, - { CX2072X_PORTE_EAPD_BTL, 0x00000002 }, - { CX2072X_PORTE_GAIN_RIGHT, 0x00000000 }, - { CX2072X_PORTE_GAIN_LEFT, 0x00000000 }, - { CX2072X_PORTF_POWER_STATE, 0x00000433 }, - { CX2072X_PORTF_PIN_CTRL, 0x00000000 }, - { CX2072X_PORTF_UNSOLICITED_RESPONSE, 0x00000000 }, - { CX2072X_PORTF_PIN_SENSE, 0x00000000 }, - { CX2072X_PORTF_GAIN_RIGHT, 0x00000000 }, - { CX2072X_PORTF_GAIN_LEFT, 0x00000000 }, - { CX2072X_PORTG_POWER_STATE, 0x00000433 }, - { CX2072X_PORTG_PIN_CTRL, 0x00000040 }, - { CX2072X_PORTG_CONNECTION_SELECT_CTRL, 0x00000000 }, - { CX2072X_PORTG_EAPD_BTL, 0x00000002 }, - { CX2072X_PORTM_POWER_STATE, 0x00000433 }, - { CX2072X_PORTM_PIN_CTRL, 0x00000000 }, - { CX2072X_PORTM_CONNECTION_SELECT_CTRL, 0x00000000 }, - { CX2072X_PORTM_EAPD_BTL, 0x00000002 }, - { CX2072X_MIXER_POWER_STATE, 0x00000433 }, - { CX2072X_MIXER_GAIN_RIGHT_0, 0x0000004a }, - { CX2072X_MIXER_GAIN_LEFT_0, 0x0000004a }, - { CX2072X_MIXER_GAIN_RIGHT_1, 0x0000004a }, - { CX2072X_MIXER_GAIN_LEFT_1, 0x0000004a }, + { CX2072X_PORTC_POWER_STATE, 0x00000433 }, + { CX2072X_PORTC_PIN_CTRL, 0x00000000 }, + { CX2072X_PORTC_GAIN_RIGHT, 0x00000000 }, + { CX2072X_PORTC_GAIN_LEFT, 0x00000000 }, { CX2072X_SPKR_DRC_ENABLE_STEP, 0x040065a4 }, { CX2072X_SPKR_DRC_CONTROL, 0x007b0024 }, { CX2072X_SPKR_DRC_TEST, 0x00000000 }, @@ -195,12 +177,15 @@ static const struct reg_default cx2072x_reg_defaults[] = { { CX2072X_I2SPCM_CONTROL3, 0x00000000 }, { CX2072X_I2SPCM_CONTROL4, 0x00000000 }, { CX2072X_I2SPCM_CONTROL5, 0x00000000 }, - { CX2072X_I2SPCM_CONTROL6, 0x00000000 }, { CX2072X_UM_INTERRUPT_CRTL_E, 0x00000000 }, + { CX2072X_I2SPCM_CONTROL6, 0x00000000 }, + { CX2072X_DIGITAL_TEST16, 0x00000021 }, + { CX2072X_DIGITAL_TEST17, 0x00000018 }, + { CX2072X_DIGITAL_TEST18, 0x00000024 }, + { CX2072X_DIGITAL_TEST19, 0x00000001 }, + { CX2072X_DIGITAL_TEST20, 0x00000002 }, { CX2072X_CODEC_TEST2, 0x00000000 }, { CX2072X_CODEC_TEST9, 0x00000004 }, - { CX2072X_CODEC_TEST20, 0x00000600 }, - { CX2072X_CODEC_TEST26, 0x00000208 }, { CX2072X_ANALOG_TEST4, 0x00000000 }, { CX2072X_ANALOG_TEST5, 0x00000000 }, { CX2072X_ANALOG_TEST6, 0x0000059a }, @@ -215,11 +200,26 @@ static const struct reg_default cx2072x_reg_defaults[] = { { CX2072X_DIGITAL_TEST11, 0x00000000 }, { CX2072X_DIGITAL_TEST12, 0x00000084 }, { CX2072X_DIGITAL_TEST15, 0x00000077 }, - { CX2072X_DIGITAL_TEST16, 0x00000021 }, - { CX2072X_DIGITAL_TEST17, 0x00000018 }, - { CX2072X_DIGITAL_TEST18, 0x00000024 }, - { CX2072X_DIGITAL_TEST19, 0x00000001 }, - { CX2072X_DIGITAL_TEST20, 0x00000002 }, + { CX2072X_CODEC_TEST20, 0x00000600 }, + { CX2072X_CODEC_TEST26, 0x00000208 }, + { CX2072X_PORTE_CONNECTION_SELECT_CTRL, 0x00000000 }, + { CX2072X_PORTE_POWER_STATE, 0x00000433 }, + { CX2072X_PORTE_PIN_CTRL, 0x00000040 }, + { CX2072X_PORTE_UNSOLICITED_RESPONSE, 0x00000000 }, + { CX2072X_PORTE_PIN_SENSE, 0x00000000 }, + { CX2072X_PORTE_EAPD_BTL, 0x00000002 }, + { CX2072X_PORTE_GAIN_RIGHT, 0x00000000 }, + { CX2072X_PORTE_GAIN_LEFT, 0x00000000 }, + { CX2072X_PORTF_POWER_STATE, 0x00000433 }, + { CX2072X_PORTF_PIN_CTRL, 0x00000000 }, + { CX2072X_PORTF_UNSOLICITED_RESPONSE, 0x00000000 }, + { CX2072X_PORTF_PIN_SENSE, 0x00000000 }, + { CX2072X_PORTF_GAIN_RIGHT, 0x00000000 }, + { CX2072X_PORTF_GAIN_LEFT, 0x00000000 }, + { CX2072X_PORTM_CONNECTION_SELECT_CTRL, 0x00000000 }, + { CX2072X_PORTM_POWER_STATE, 0x00000433 }, + { CX2072X_PORTM_PIN_CTRL, 0x00000000 }, + { CX2072X_PORTM_EAPD_BTL, 0x00000002 }, }; /* diff --git a/sound/soc/codecs/hdac_hda.c b/sound/soc/codecs/hdac_hda.c index 680e341aa7f1b..1ab5f8a26e03b 100644 --- a/sound/soc/codecs/hdac_hda.c +++ b/sound/soc/codecs/hdac_hda.c @@ -642,10 +642,8 @@ static int hdac_hda_dev_probe(struct hdac_device *hdev) &hdac_hda_codec, hdac_hda_dais, ARRAY_SIZE(hdac_hda_dais)); - if (ret < 0) { + if (ret < 0) dev_err(&hdev->dev, "%s: failed to register HDA codec %d\n", __func__, ret); - return ret; - } snd_hdac_ext_bus_link_put(hdev->bus, hlink); diff --git a/sound/soc/codecs/hdmi-codec.c b/sound/soc/codecs/hdmi-codec.c index 13ae9e83bc21f..2db9cf8ec6e96 100644 --- a/sound/soc/codecs/hdmi-codec.c +++ b/sound/soc/codecs/hdmi-codec.c @@ -425,10 +425,14 @@ static int hdmi_codec_iec958_default_put(struct snd_kcontrol *kcontrol, struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); struct hdmi_codec_priv *hcp = snd_soc_component_get_drvdata(component); + if (!memcmp(hcp->iec_status, ucontrol->value.iec958.status, + sizeof(hcp->iec_status))) + return 0; + memcpy(hcp->iec_status, ucontrol->value.iec958.status, sizeof(hcp->iec_status)); - return 0; + return 1; } static int hdmi_codec_iec958_mask_get(struct snd_kcontrol *kcontrol, diff --git a/sound/soc/codecs/max9860.c b/sound/soc/codecs/max9860.c index 0d7ac37850bba..20e288e38669b 100644 --- a/sound/soc/codecs/max9860.c +++ b/sound/soc/codecs/max9860.c @@ -48,7 +48,6 @@ static int max9860_dvddio_event(struct notifier_block *nb, } static const struct reg_default max9860_reg_defaults[] = { - { MAX9860_PWRMAN, 0x00 }, { MAX9860_INTEN, 0x00 }, { MAX9860_SYSCLK, 0x00 }, { MAX9860_AUDIOCLKHIGH, 0x00 }, @@ -62,6 +61,7 @@ static const struct reg_default max9860_reg_defaults[] = { { MAX9860_MICGAIN, 0x00 }, { MAX9860_MICADC, 0x00 }, { MAX9860_NOISEGATE, 0x00 }, + { MAX9860_PWRMAN, 0x00 }, }; static bool max9860_readable(struct device *dev, unsigned int reg) diff --git a/sound/soc/codecs/ml26124.c b/sound/soc/codecs/ml26124.c index 8a14626d43ffd..7209b47c20291 100644 --- a/sound/soc/codecs/ml26124.c +++ b/sound/soc/codecs/ml26124.c @@ -224,7 +224,6 @@ static const struct reg_default ml26124_reg[] = { /* Analog Path Control Register */ {0x54, 0x00}, /* Speaker AMP Output Control */ {0x5a, 0x00}, /* Mic IF Control */ - {0xe8, 0x01}, /* Mic Select Control */ /* Audio Interface Control Register */ {0x60, 0x00}, /* SAI-Trans Control */ @@ -287,6 +286,9 @@ static const struct reg_default ml26124_reg[] = { {0xd0, 0x01}, /* VIDEO AMP Gain Control */ {0xd2, 0x01}, /* VIDEO AMP Setup 1 */ {0xd4, 0x01}, /* VIDEO AMP Control2 */ + + /* Analog Path Control Register */ + {0xe8, 0x01}, /* Mic Select Control */ }; /* Get sampling rate value of sampling rate setting register (0x0) */ diff --git a/sound/soc/codecs/mt6351.c b/sound/soc/codecs/mt6351.c index 1768c249650d8..3be0491a2588c 100644 --- a/sound/soc/codecs/mt6351.c +++ b/sound/soc/codecs/mt6351.c @@ -1478,6 +1478,7 @@ static const struct of_device_id mt6351_of_match[] = { {.compatible = "mediatek,mt6351-sound",}, {} }; +MODULE_DEVICE_TABLE(of, mt6351_of_match); static struct platform_driver mt6351_codec_driver = { .driver = { diff --git a/sound/soc/codecs/pcm512x.c b/sound/soc/codecs/pcm512x.c index fe3b5011fa167..03b1fcb1bf96e 100644 --- a/sound/soc/codecs/pcm512x.c +++ b/sound/soc/codecs/pcm512x.c @@ -78,28 +78,10 @@ static const struct reg_default pcm512x_reg_defaults[] = { { PCM512x_POWER, 0x00 }, { PCM512x_MUTE, 0x00 }, { PCM512x_DSP, 0x00 }, - { PCM512x_PLL_REF, 0x00 }, - { PCM512x_DAC_REF, 0x00 }, - { PCM512x_DAC_ROUTING, 0x11 }, - { PCM512x_DSP_PROGRAM, 0x01 }, - { PCM512x_CLKDET, 0x00 }, - { PCM512x_AUTO_MUTE, 0x00 }, - { PCM512x_ERROR_DETECT, 0x00 }, - { PCM512x_DIGITAL_VOLUME_1, 0x00 }, - { PCM512x_DIGITAL_VOLUME_2, 0x30 }, - { PCM512x_DIGITAL_VOLUME_3, 0x30 }, - { PCM512x_DIGITAL_MUTE_1, 0x22 }, - { PCM512x_DIGITAL_MUTE_2, 0x00 }, - { PCM512x_DIGITAL_MUTE_3, 0x07 }, - { PCM512x_OUTPUT_AMPLITUDE, 0x00 }, - { PCM512x_ANALOG_GAIN_CTRL, 0x00 }, - { PCM512x_UNDERVOLTAGE_PROT, 0x00 }, - { PCM512x_ANALOG_MUTE_CTRL, 0x00 }, - { PCM512x_ANALOG_GAIN_BOOST, 0x00 }, - { PCM512x_VCOM_CTRL_1, 0x00 }, - { PCM512x_VCOM_CTRL_2, 0x01 }, { PCM512x_BCLK_LRCLK_CFG, 0x00 }, { PCM512x_MASTER_MODE, 0x7c }, + { PCM512x_PLL_REF, 0x00 }, + { PCM512x_DAC_REF, 0x00 }, { PCM512x_GPIO_DACIN, 0x00 }, { PCM512x_GPIO_PLLIN, 0x00 }, { PCM512x_SYNCHRONIZE, 0x10 }, @@ -117,8 +99,26 @@ static const struct reg_default pcm512x_reg_defaults[] = { { PCM512x_FS_SPEED_MODE, 0x00 }, { PCM512x_IDAC_1, 0x01 }, { PCM512x_IDAC_2, 0x00 }, + { PCM512x_ERROR_DETECT, 0x00 }, { PCM512x_I2S_1, 0x02 }, { PCM512x_I2S_2, 0x00 }, + { PCM512x_DAC_ROUTING, 0x11 }, + { PCM512x_DSP_PROGRAM, 0x01 }, + { PCM512x_CLKDET, 0x00 }, + { PCM512x_AUTO_MUTE, 0x00 }, + { PCM512x_DIGITAL_VOLUME_1, 0x00 }, + { PCM512x_DIGITAL_VOLUME_2, 0x30 }, + { PCM512x_DIGITAL_VOLUME_3, 0x30 }, + { PCM512x_DIGITAL_MUTE_1, 0x22 }, + { PCM512x_DIGITAL_MUTE_2, 0x00 }, + { PCM512x_DIGITAL_MUTE_3, 0x07 }, + { PCM512x_OUTPUT_AMPLITUDE, 0x00 }, + { PCM512x_ANALOG_GAIN_CTRL, 0x00 }, + { PCM512x_UNDERVOLTAGE_PROT, 0x00 }, + { PCM512x_ANALOG_MUTE_CTRL, 0x00 }, + { PCM512x_ANALOG_GAIN_BOOST, 0x00 }, + { PCM512x_VCOM_CTRL_1, 0x00 }, + { PCM512x_VCOM_CTRL_2, 0x01 }, }; static bool pcm512x_readable(struct device *dev, unsigned int reg) diff --git a/sound/soc/codecs/pm4125-sdw.c b/sound/soc/codecs/pm4125-sdw.c index 1c612ae4a4b27..ee4cde4d8686c 100644 --- a/sound/soc/codecs/pm4125-sdw.c +++ b/sound/soc/codecs/pm4125-sdw.c @@ -126,8 +126,8 @@ static const struct reg_default pm4125_defaults[] = { { PM4125_ANA_HPHPA_FSM_CLK, 0x12 }, { PM4125_ANA_HPHPA_L_GAIN, 0x00 }, { PM4125_ANA_HPHPA_R_GAIN, 0x00 }, - { PM4125_SWR_HPHPA_HD2, 0x1B }, { PM4125_ANA_HPHPA_SPARE_CTL, 0x02 }, + { PM4125_SWR_HPHPA_HD2, 0x1B }, { PM4125_ANA_SURGE_EN, 0x38 }, { PM4125_ANA_COMBOPA_CTL, 0x35 }, { PM4125_ANA_COMBOPA_CTL_4, 0x84 }, diff --git a/sound/soc/codecs/rt1017-sdca-sdw.h b/sound/soc/codecs/rt1017-sdca-sdw.h index 4932b5dbe3c0b..1604e1cfd85eb 100644 --- a/sound/soc/codecs/rt1017-sdca-sdw.h +++ b/sound/soc/codecs/rt1017-sdca-sdw.h @@ -165,19 +165,19 @@ static const struct reg_default rt1017_sdca_reg_defaults[] = { { 0xdb09, 0x0f }, { 0xdb0a, 0xff }, { 0xdb14, 0x00 }, - - { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1017_SDCA_ENT_UDMPU21, - RT1017_SDCA_CTL_UDMPU_CLUSTER, 0), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1017_SDCA_ENT_FU, RT1017_SDCA_CTL_FU_MUTE, 0x01), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1017_SDCA_ENT_XU22, RT1017_SDCA_CTL_BYPASS, 0), 0x01 }, - { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1017_SDCA_ENT_CS21, - RT1017_SDCA_CTL_FS_INDEX, 0), 0x09 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1017_SDCA_ENT_PDE23, RT1017_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1017_SDCA_ENT_PDE22, RT1017_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, + + { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1017_SDCA_ENT_UDMPU21, + RT1017_SDCA_CTL_UDMPU_CLUSTER, 0), 0x00 }, + { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1017_SDCA_ENT_CS21, + RT1017_SDCA_CTL_FS_INDEX, 0), 0x09 }, }; #endif /* __RT1017_SDW_H__ */ diff --git a/sound/soc/codecs/rt1316-sdw.c b/sound/soc/codecs/rt1316-sdw.c index ca318dbd946ee..0f6fc36e50f0f 100644 --- a/sound/soc/codecs/rt1316-sdw.c +++ b/sound/soc/codecs/rt1316-sdw.c @@ -59,13 +59,13 @@ static const struct reg_default rt1316_reg_defaults[] = { { 0xd101, 0x00 }, { 0xd102, 0x30 }, { 0xd103, 0x00 }, - { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1316_SDCA_ENT_UDMPU21, RT1316_SDCA_CTL_UDMPU_CLUSTER, 0), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1316_SDCA_ENT_FU21, RT1316_SDCA_CTL_FU_MUTE, CH_L), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1316_SDCA_ENT_FU21, RT1316_SDCA_CTL_FU_MUTE, CH_R), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1316_SDCA_ENT_XU24, RT1316_SDCA_CTL_BYPASS, 0), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1316_SDCA_ENT_PDE23, RT1316_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1316_SDCA_ENT_PDE22, RT1316_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1316_SDCA_ENT_PDE24, RT1316_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, + { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1316_SDCA_ENT_UDMPU21, RT1316_SDCA_CTL_UDMPU_CLUSTER, 0), 0x00 }, }; static const struct reg_sequence rt1316_blind_write[] = { diff --git a/sound/soc/codecs/rt1318-sdw.c b/sound/soc/codecs/rt1318-sdw.c index c038ac0e3b769..2f1a776585fcb 100644 --- a/sound/soc/codecs/rt1318-sdw.c +++ b/sound/soc/codecs/rt1318-sdw.c @@ -235,10 +235,10 @@ static const struct reg_default rt1318_reg_defaults[] = { { 0xf805, 0x00 }, { 0xf806, 0x07 }, { 0xf807, 0xff }, - { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1318_SDCA_ENT_UDMPU21, RT1318_SDCA_CTL_UDMPU_CLUSTER, 0), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1318_SDCA_ENT_FU21, RT1318_SDCA_CTL_FU_MUTE, CH_L), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1318_SDCA_ENT_FU21, RT1318_SDCA_CTL_FU_MUTE, CH_R), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1318_SDCA_ENT_PDE23, RT1318_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, + { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1318_SDCA_ENT_UDMPU21, RT1318_SDCA_CTL_UDMPU_CLUSTER, 0), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_SMART_AMP, RT1318_SDCA_ENT_CS21, RT1318_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, }; diff --git a/sound/soc/codecs/rt1318.c b/sound/soc/codecs/rt1318.c index d13fd0e141252..55607f1b9a1ed 100644 --- a/sound/soc/codecs/rt1318.c +++ b/sound/soc/codecs/rt1318.c @@ -337,6 +337,8 @@ static const struct reg_default rt1318_reg[] = { { 0xdd08, 0x40 }, { 0xdd12, 0x00 }, { 0xdd35, 0x00 }, + { 0xdd93, 0x00 }, + { 0xdd94, 0x64 }, { 0xddb5, 0x00 }, { 0xddb6, 0x40 }, { 0xddb7, 0x00 }, @@ -345,8 +347,6 @@ static const struct reg_default rt1318_reg[] = { { 0xddc6, 0x00 }, { 0xddc7, 0x00 }, { 0xddc8, 0x00 }, - { 0xdd93, 0x00 }, - { 0xdd94, 0x64 }, { 0xdf00, 0x00 }, { 0xdf5f, 0x00 }, { 0xdf60, 0x00 }, diff --git a/sound/soc/codecs/rt274.c b/sound/soc/codecs/rt274.c index 63b5fc439773d..f8c370106504c 100644 --- a/sound/soc/codecs/rt274.c +++ b/sound/soc/codecs/rt274.c @@ -184,8 +184,10 @@ static const struct reg_default rt274_reg[] = { { 0x0023a000, 0x00000057 }, { 0x00270500, 0x00000400 }, { 0x00370500, 0x00000400 }, + { 0x00830000, 0x00000097 }, { 0x00870500, 0x00000400 }, { 0x00920000, 0x00000031 }, + { 0x00930000, 0x00000097 }, { 0x00935000, 0x00000097 }, { 0x00936000, 0x00000097 }, { 0x00970500, 0x00000400 }, @@ -195,10 +197,12 @@ static const struct reg_default rt274_reg[] = { { 0x00c37000, 0x00000400 }, { 0x00c37100, 0x00000400 }, { 0x01270500, 0x00000400 }, + { 0x01270700, 0x00000000 }, { 0x01370500, 0x00000400 }, { 0x01371f00, 0x411111f0 }, { 0x01937000, 0x00000000 }, { 0x01970500, 0x00000400 }, + { 0x01970700, 0x00000020 }, { 0x02050000, 0x0000001b }, { 0x02139000, 0x00000080 }, { 0x0213a000, 0x00000080 }, @@ -207,10 +211,6 @@ static const struct reg_default rt274_reg[] = { { 0x02170700, 0x00000000 }, { 0x02270100, 0x00000000 }, { 0x02370100, 0x00000000 }, - { 0x01970700, 0x00000020 }, - { 0x00830000, 0x00000097 }, - { 0x00930000, 0x00000097 }, - { 0x01270700, 0x00000000 }, }; static bool rt274_volatile_register(struct device *dev, unsigned int reg) diff --git a/sound/soc/codecs/rt286.c b/sound/soc/codecs/rt286.c index ded0ea3324804..4217467904d6f 100644 --- a/sound/soc/codecs/rt286.c +++ b/sound/soc/codecs/rt286.c @@ -77,8 +77,10 @@ static const struct reg_default rt286_reg[] = { { 0x0023a000, 0x0000007f }, { 0x00270500, 0x00000400 }, { 0x00370500, 0x00000400 }, + { 0x00830000, 0x000000c3 }, { 0x00870500, 0x00000400 }, { 0x00920000, 0x00000031 }, + { 0x00930000, 0x000000c3 }, { 0x00935000, 0x000000c3 }, { 0x00936000, 0x000000c3 }, { 0x00970500, 0x00000400 }, @@ -88,16 +90,18 @@ static const struct reg_default rt286_reg[] = { { 0x00c37000, 0x00000000 }, { 0x00c37100, 0x00000080 }, { 0x01270500, 0x00000400 }, + { 0x01270700, 0x00000000 }, { 0x01370500, 0x00000400 }, { 0x01371f00, 0x411111f0 }, { 0x01439000, 0x00000080 }, { 0x0143a000, 0x00000080 }, - { 0x01470700, 0x00000000 }, + { 0x01470100, 0x00000000 }, { 0x01470500, 0x00000400 }, + { 0x01470700, 0x00000000 }, { 0x01470c00, 0x00000000 }, - { 0x01470100, 0x00000000 }, { 0x01837000, 0x00000000 }, { 0x01870500, 0x00000400 }, + { 0x01870700, 0x00000020 }, { 0x02050000, 0x00000000 }, { 0x02139000, 0x00000080 }, { 0x0213a000, 0x00000080 }, @@ -106,10 +110,6 @@ static const struct reg_default rt286_reg[] = { { 0x02170700, 0x00000000 }, { 0x02270100, 0x00000000 }, { 0x02370100, 0x00000000 }, - { 0x01870700, 0x00000020 }, - { 0x00830000, 0x000000c3 }, - { 0x00930000, 0x000000c3 }, - { 0x01270700, 0x00000000 }, }; static bool rt286_volatile_register(struct device *dev, unsigned int reg) diff --git a/sound/soc/codecs/rt298.c b/sound/soc/codecs/rt298.c index 5414a1712b571..09aed08b5b798 100644 --- a/sound/soc/codecs/rt298.c +++ b/sound/soc/codecs/rt298.c @@ -78,8 +78,10 @@ static const struct reg_default rt298_reg[] = { { 0x0023a000, 0x0000007f }, { 0x00270500, 0x00000400 }, { 0x00370500, 0x00000400 }, + { 0x00830000, 0x000000c3 }, { 0x00870500, 0x00000400 }, { 0x00920000, 0x00000031 }, + { 0x00930000, 0x000000c3 }, { 0x00935000, 0x000000c3 }, { 0x00936000, 0x000000c3 }, { 0x00970500, 0x00000400 }, @@ -89,16 +91,18 @@ static const struct reg_default rt298_reg[] = { { 0x00c37000, 0x00000000 }, { 0x00c37100, 0x00000080 }, { 0x01270500, 0x00000400 }, + { 0x01270700, 0x00000000 }, { 0x01370500, 0x00000400 }, { 0x01371f00, 0x411111f0 }, { 0x01439000, 0x00000080 }, { 0x0143a000, 0x00000080 }, - { 0x01470700, 0x00000000 }, + { 0x01470100, 0x00000000 }, { 0x01470500, 0x00000400 }, + { 0x01470700, 0x00000000 }, { 0x01470c00, 0x00000000 }, - { 0x01470100, 0x00000000 }, { 0x01837000, 0x00000000 }, { 0x01870500, 0x00000400 }, + { 0x01870700, 0x00000020 }, { 0x02050000, 0x00000000 }, { 0x02139000, 0x00000080 }, { 0x0213a000, 0x00000080 }, @@ -107,10 +111,6 @@ static const struct reg_default rt298_reg[] = { { 0x02170700, 0x00000000 }, { 0x02270100, 0x00000000 }, { 0x02370100, 0x00000000 }, - { 0x01870700, 0x00000020 }, - { 0x00830000, 0x000000c3 }, - { 0x00930000, 0x000000c3 }, - { 0x01270700, 0x00000000 }, }; static bool rt298_volatile_register(struct device *dev, unsigned int reg) diff --git a/sound/soc/codecs/rt700-sdw.c b/sound/soc/codecs/rt700-sdw.c index a451d5d1f8ab5..bb449f08e30cf 100644 --- a/sound/soc/codecs/rt700-sdw.c +++ b/sound/soc/codecs/rt700-sdw.c @@ -458,10 +458,8 @@ static void rt700_sdw_remove(struct sdw_slave *slave) { struct rt700_priv *rt700 = dev_get_drvdata(&slave->dev); - if (rt700->hw_init) { - cancel_delayed_work_sync(&rt700->jack_detect_work); - cancel_delayed_work_sync(&rt700->jack_btn_check_work); - } + cancel_delayed_work_sync(&rt700->jack_detect_work); + cancel_delayed_work_sync(&rt700->jack_btn_check_work); pm_runtime_disable(&slave->dev); } diff --git a/sound/soc/codecs/rt700-sdw.h b/sound/soc/codecs/rt700-sdw.h index 4ad0dcfd16fdb..002f94493b01a 100644 --- a/sound/soc/codecs/rt700-sdw.h +++ b/sound/soc/codecs/rt700-sdw.h @@ -313,17 +313,16 @@ static const struct reg_default rt700_reg_defaults[] = { { 0x3122, 0x0000 }, { 0x3123, 0x0000 }, { 0x7303, 0x0057 }, - { 0x7303, 0x0057 }, - { 0x8383, 0x0057 }, { 0x7308, 0x0097 }, - { 0x8388, 0x0097 }, { 0x7309, 0x0097 }, - { 0x8389, 0x0097 }, { 0x7312, 0x0000 }, - { 0x8392, 0x0000 }, { 0x7313, 0x0000 }, - { 0x8393, 0x0000 }, { 0x7319, 0x0000 }, + { 0x8383, 0x0057 }, + { 0x8388, 0x0097 }, + { 0x8389, 0x0097 }, + { 0x8392, 0x0000 }, + { 0x8393, 0x0000 }, { 0x8399, 0x0000 }, { 0x75201a, 0x8003 }, { 0x752045, 0x5289 }, diff --git a/sound/soc/codecs/rt711-sdca-sdw.h b/sound/soc/codecs/rt711-sdca-sdw.h index 0d774e473ab94..9aa22b52d556f 100644 --- a/sound/soc/codecs/rt711-sdca-sdw.h +++ b/sound/soc/codecs/rt711-sdca-sdw.h @@ -58,12 +58,12 @@ static const struct reg_default rt711_sdca_reg_defaults[] = { { 0x2f0f, 0x00 }, { 0x2f50, 0x03 }, { 0x2f5a, 0x00 }, - { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_CS01, RT711_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_USER_FU05, RT711_SDCA_CTL_FU_MUTE, CH_L), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_USER_FU05, RT711_SDCA_CTL_FU_MUTE, CH_R), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_USER_FU0F, RT711_SDCA_CTL_FU_MUTE, CH_L), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_USER_FU0F, RT711_SDCA_CTL_FU_MUTE, CH_R), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_PDE28, RT711_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, + { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_CS01, RT711_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT711_SDCA_ENT_USER_FU1E, RT711_SDCA_CTL_FU_MUTE, CH_L), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT711_SDCA_ENT_USER_FU1E, RT711_SDCA_CTL_FU_MUTE, CH_R), 0x01 }, }; @@ -86,14 +86,14 @@ static const struct reg_default rt711_sdca_mbq_defaults[] = { { 0x610003f, 0xff12 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_USER_FU05, RT711_SDCA_CTL_FU_VOLUME, CH_L), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_USER_FU05, RT711_SDCA_CTL_FU_VOLUME, CH_R), 0x00 }, - { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT711_SDCA_ENT_USER_FU1E, RT711_SDCA_CTL_FU_VOLUME, CH_L), 0x00 }, - { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT711_SDCA_ENT_USER_FU1E, RT711_SDCA_CTL_FU_VOLUME, CH_R), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_USER_FU0F, RT711_SDCA_CTL_FU_VOLUME, CH_L), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_USER_FU0F, RT711_SDCA_CTL_FU_VOLUME, CH_R), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_PLATFORM_FU44, RT711_SDCA_CTL_FU_CH_GAIN, CH_L), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT711_SDCA_ENT_PLATFORM_FU44, RT711_SDCA_CTL_FU_CH_GAIN, CH_R), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT711_SDCA_ENT_PLATFORM_FU15, RT711_SDCA_CTL_FU_CH_GAIN, CH_L), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT711_SDCA_ENT_PLATFORM_FU15, RT711_SDCA_CTL_FU_CH_GAIN, CH_R), 0x00 }, + { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT711_SDCA_ENT_USER_FU1E, RT711_SDCA_CTL_FU_VOLUME, CH_L), 0x00 }, + { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT711_SDCA_ENT_USER_FU1E, RT711_SDCA_CTL_FU_VOLUME, CH_R), 0x00 }, }; #endif /* __RT711_SDW_SDCA_H__ */ diff --git a/sound/soc/codecs/rt711-sdw.h b/sound/soc/codecs/rt711-sdw.h index 6acf9858330d7..82da0e62d9d7d 100644 --- a/sound/soc/codecs/rt711-sdw.h +++ b/sound/soc/codecs/rt711-sdw.h @@ -256,16 +256,16 @@ static const struct reg_default rt711_reg_defaults[] = { { 0x3122, 0x00 }, { 0x3123, 0x00 }, { 0x7303, 0x57 }, - { 0x8383, 0x57 }, { 0x7308, 0x97 }, - { 0x8388, 0x97 }, { 0x7309, 0x97 }, - { 0x8389, 0x97 }, { 0x7312, 0x00 }, - { 0x8392, 0x00 }, { 0x7313, 0x00 }, - { 0x8393, 0x00 }, { 0x7319, 0x00 }, + { 0x8383, 0x57 }, + { 0x8388, 0x97 }, + { 0x8389, 0x97 }, + { 0x8392, 0x00 }, + { 0x8393, 0x00 }, { 0x8399, 0x00 }, { 0x752008, 0xa807 }, { 0x752009, 0x1029 }, diff --git a/sound/soc/codecs/rt712-sdca-dmic.c b/sound/soc/codecs/rt712-sdca-dmic.c index 85779547653e1..ccc5e54f5fd7f 100644 --- a/sound/soc/codecs/rt712-sdca-dmic.c +++ b/sound/soc/codecs/rt712-sdca-dmic.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "rt712-sdca.h" #include "rt712-sdca-dmic.h" @@ -632,10 +633,10 @@ static int rt712_sdca_dmic_hw_params(struct snd_pcm_substream *substream, { struct snd_soc_component *component = dai->component; struct rt712_sdca_dmic_priv *rt712 = snd_soc_component_get_drvdata(component); - struct sdw_stream_config stream_config; + struct sdw_stream_config stream_config = {0}; struct sdw_port_config port_config; struct sdw_stream_runtime *sdw_stream; - int retval, num_channels; + int retval; unsigned int sampling_rate; dev_dbg(dai->dev, "%s %s", __func__, dai->name); @@ -647,13 +648,8 @@ static int rt712_sdca_dmic_hw_params(struct snd_pcm_substream *substream, if (!rt712->slave) return -EINVAL; - stream_config.frame_rate = params_rate(params); - stream_config.ch_count = params_channels(params); - stream_config.bps = snd_pcm_format_width(params_format(params)); - stream_config.direction = SDW_DATA_DIR_TX; - - num_channels = params_channels(params); - port_config.ch_mask = GENMASK(num_channels - 1, 0); + /* SoundWire specific configuration */ + snd_sdw_params_to_config(substream, params, &stream_config, &port_config); port_config.num = 2; retval = sdw_stream_add_slave(rt712->slave, &stream_config, diff --git a/sound/soc/codecs/rt712-sdca-dmic.h b/sound/soc/codecs/rt712-sdca-dmic.h index 110154e74efed..2f08e58ed519b 100644 --- a/sound/soc/codecs/rt712-sdca-dmic.h +++ b/sound/soc/codecs/rt712-sdca-dmic.h @@ -36,6 +36,7 @@ struct rt712_sdca_dmic_kctrl_priv { #define CH_03 0x03 #define CH_04 0x04 +/* must stay sorted by register address, regcache_lookup_reg() does a bsearch() */ static const struct reg_default rt712_sdca_dmic_reg_defaults[] = { { 0x201a, 0x00 }, { 0x201b, 0x00 }, @@ -72,15 +73,16 @@ static const struct reg_default rt712_sdca_dmic_reg_defaults[] = { { 0x2f59, 0x07 }, { 0x3201, 0x01 }, { 0x320c, 0x00 }, - { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_IT26, RT712_SDCA_CTL_VENDOR_DEF, 0), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_USER_FU1E, RT712_SDCA_CTL_FU_MUTE, CH_01), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_USER_FU1E, RT712_SDCA_CTL_FU_MUTE, CH_02), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_USER_FU1E, RT712_SDCA_CTL_FU_MUTE, CH_03), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_USER_FU1E, RT712_SDCA_CTL_FU_MUTE, CH_04), 0x01 }, - { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_CS1F, RT712_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_CS1C, RT712_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, + { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_CS1F, RT712_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, + { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_IT26, RT712_SDCA_CTL_VENDOR_DEF, 0), 0x00 }, }; +/* must stay sorted by register address, regcache_lookup_reg() does a bsearch() */ static const struct reg_default rt712_sdca_dmic_mbq_defaults[] = { { 0x0590001e, 0x0020 }, { 0x06100000, 0x0010 }, diff --git a/sound/soc/codecs/rt712-sdca-sdw.h b/sound/soc/codecs/rt712-sdca-sdw.h index 99fd2d67f04db..7ad25cc27f624 100644 --- a/sound/soc/codecs/rt712-sdca-sdw.h +++ b/sound/soc/codecs/rt712-sdca-sdw.h @@ -11,21 +11,21 @@ #include #include +/* must stay sorted by register address, regcache_lookup_reg() does a bsearch() */ static const struct reg_default rt712_sdca_reg_defaults[] = { - - { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_CS01, RT712_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, - { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_CS11, RT712_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_USER_FU05, RT712_SDCA_CTL_FU_MUTE, CH_01), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_USER_FU05, RT712_SDCA_CTL_FU_MUTE, CH_02), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_USER_FU0F, RT712_SDCA_CTL_FU_MUTE, CH_01), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_USER_FU0F, RT712_SDCA_CTL_FU_MUTE, CH_02), 0x01 }, - { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_PDE40, RT712_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_PDE12, RT712_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, - { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_CS1C, RT712_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, + { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_CS01, RT712_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, + { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_CS11, RT712_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, + { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT712_SDCA_ENT_PDE40, RT712_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_USER_FU1E, RT712_SDCA_CTL_FU_MUTE, CH_01), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_USER_FU1E, RT712_SDCA_CTL_FU_MUTE, CH_02), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_USER_FU1E, RT712_SDCA_CTL_FU_MUTE, CH_03), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_USER_FU1E, RT712_SDCA_CTL_FU_MUTE, CH_04), 0x01 }, + { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_CS1C, RT712_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT712_SDCA_ENT_CS1F, RT712_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, { SDW_SDCA_CTL(FUNC_NUM_AMP, RT712_SDCA_ENT_USER_FU06, RT712_SDCA_CTL_FU_MUTE, CH_01), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_AMP, RT712_SDCA_ENT_USER_FU06, RT712_SDCA_CTL_FU_MUTE, CH_02), 0x01 }, @@ -34,6 +34,7 @@ static const struct reg_default rt712_sdca_reg_defaults[] = { { SDW_SDCA_CTL(FUNC_NUM_AMP, RT712_SDCA_ENT_OT23, RT712_SDCA_CTL_VENDOR_DEF, 0), 0x00 }, }; +/* must stay sorted by register address, regcache_lookup_reg() does a bsearch() */ static const struct reg_default rt712_sdca_mbq_defaults[] = { { 0x2000004, 0xaa01 }, { 0x200000e, 0x21e0 }, diff --git a/sound/soc/codecs/rt715-sdca-sdw.h b/sound/soc/codecs/rt715-sdca-sdw.h index 0cbc14844f8c2..d408f2d7d0f28 100644 --- a/sound/soc/codecs/rt715-sdca-sdw.h +++ b/sound/soc/codecs/rt715-sdca-sdw.h @@ -76,8 +76,6 @@ static const struct reg_default rt715_reg_defaults_sdca[] = { { 0x2f52, 0x01 }, { 0x2f5a, 0x02 }, { 0x2f5b, 0x05 }, - { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_CX_CLK_SEL_EN, - RT715_SDCA_CX_CLK_SEL_CTRL, CH_00), 0x1 }, { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_ADC8_9_VOL, RT715_SDCA_FU_MUTE_CTRL, CH_01), 0x01 }, { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_ADC8_9_VOL, @@ -98,14 +96,12 @@ static const struct reg_default rt715_reg_defaults_sdca[] = { RT715_SDCA_FU_MUTE_CTRL, CH_01), 0x01 }, { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_ADC7_27_VOL, RT715_SDCA_FU_MUTE_CTRL, CH_02), 0x01 }, + { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_CX_CLK_SEL_EN, + RT715_SDCA_CX_CLK_SEL_CTRL, CH_00), 0x1 }, { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_SMPU_TRIG_ST_EN, RT715_SDCA_SMPU_TRIG_EN_CTRL, CH_00), 0x02 }, { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_SMPU_TRIG_ST_EN, RT715_SDCA_SMPU_TRIG_ST_CTRL, CH_00), 0x00 }, - { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_ADC7_27_VOL, - RT715_SDCA_FU_MUTE_CTRL, CH_01), 0x01 }, - { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_ADC7_27_VOL, - RT715_SDCA_FU_MUTE_CTRL, CH_02), 0x01 }, }; static const struct reg_default rt715_mbq_reg_defaults_sdca[] = { @@ -149,8 +145,6 @@ static const struct reg_default rt715_mbq_reg_defaults_sdca[] = { RT715_SDCA_FU_DMIC_GAIN_CTRL, CH_06), 0x00 }, { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_AMIC_GAIN_EN, RT715_SDCA_FU_DMIC_GAIN_CTRL, CH_07), 0x00 }, - { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_AMIC_GAIN_EN, - RT715_SDCA_FU_DMIC_GAIN_CTRL, CH_08), 0x00 }, { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_DMIC_GAIN_EN, RT715_SDCA_FU_DMIC_GAIN_CTRL, CH_01), 0x00 }, { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_DMIC_GAIN_EN, @@ -165,6 +159,8 @@ static const struct reg_default rt715_mbq_reg_defaults_sdca[] = { RT715_SDCA_FU_DMIC_GAIN_CTRL, CH_06), 0x00 }, { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_DMIC_GAIN_EN, RT715_SDCA_FU_DMIC_GAIN_CTRL, CH_07), 0x00 }, + { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_AMIC_GAIN_EN, + RT715_SDCA_FU_DMIC_GAIN_CTRL, CH_08), 0x00 }, { SDW_SDCA_CTL(FUN_MIC_ARRAY, RT715_SDCA_FU_DMIC_GAIN_EN, RT715_SDCA_FU_DMIC_GAIN_CTRL, CH_08), 0x00 }, }; diff --git a/sound/soc/codecs/rt715-sdw.h b/sound/soc/codecs/rt715-sdw.h index 5d7661e335ae5..f19fafb913f00 100644 --- a/sound/soc/codecs/rt715-sdw.h +++ b/sound/soc/codecs/rt715-sdw.h @@ -281,8 +281,8 @@ static const struct reg_default rt715_reg_defaults[] = { { 0x371b, 0x00 }, { 0x371d, 0x00 }, { 0x3729, 0x00 }, - { 0x385e, 0x00 }, { 0x3859, 0x00 }, + { 0x385e, 0x00 }, { 0x4c12, 0x411111f0 }, { 0x4c13, 0x411111f0 }, { 0x4c1d, 0x411111f0 }, @@ -300,36 +300,36 @@ static const struct reg_default rt715_reg_defaults[] = { { 0x4f1d, 0x411111f0 }, { 0x4f29, 0x411111f0 }, { 0x7207, 0x00 }, - { 0x8287, 0x00 }, { 0x7208, 0x00 }, - { 0x8288, 0x00 }, { 0x7209, 0x00 }, - { 0x8289, 0x00 }, { 0x7227, 0x00 }, - { 0x82a7, 0x00 }, { 0x7307, 0x97 }, - { 0x8387, 0x97 }, { 0x7308, 0x97 }, - { 0x8388, 0x97 }, { 0x7309, 0x97 }, - { 0x8389, 0x97 }, { 0x7312, 0x00 }, - { 0x8392, 0x00 }, { 0x7313, 0x00 }, - { 0x8393, 0x00 }, { 0x7318, 0x00 }, - { 0x8398, 0x00 }, { 0x7319, 0x00 }, - { 0x8399, 0x00 }, { 0x731a, 0x00 }, - { 0x839a, 0x00 }, { 0x731b, 0x00 }, - { 0x839b, 0x00 }, { 0x731d, 0x00 }, - { 0x839d, 0x00 }, { 0x7327, 0x97 }, - { 0x83a7, 0x97 }, { 0x7329, 0x00 }, + { 0x8287, 0x00 }, + { 0x8288, 0x00 }, + { 0x8289, 0x00 }, + { 0x82a7, 0x00 }, + { 0x8387, 0x97 }, + { 0x8388, 0x97 }, + { 0x8389, 0x97 }, + { 0x8392, 0x00 }, + { 0x8393, 0x00 }, + { 0x8398, 0x00 }, + { 0x8399, 0x00 }, + { 0x839a, 0x00 }, + { 0x839b, 0x00 }, + { 0x839d, 0x00 }, + { 0x83a7, 0x97 }, { 0x83a9, 0x00 }, { 0x752039, 0xa500 }, }; diff --git a/sound/soc/codecs/rt721-sdca-sdw.h b/sound/soc/codecs/rt721-sdca-sdw.h index 214b31b825833..13d6227f3392a 100644 --- a/sound/soc/codecs/rt721-sdca-sdw.h +++ b/sound/soc/codecs/rt721-sdca-sdw.h @@ -29,14 +29,6 @@ static const struct reg_default rt721_sdca_reg_defaults[] = { { 0x2f5b, 0x07 }, { 0x2f5c, 0x27 }, { 0x2f5d, 0x07 }, - { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_CS01, - RT721_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, - { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_CS11, - RT721_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, - { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_PDE12, - RT721_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, - { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_PDE40, - RT721_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_USER_FU05, RT721_SDCA_CTL_FU_MUTE, CH_L), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_USER_FU05, @@ -45,6 +37,14 @@ static const struct reg_default rt721_sdca_reg_defaults[] = { RT721_SDCA_CTL_FU_MUTE, CH_L), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_USER_FU0F, RT721_SDCA_CTL_FU_MUTE, CH_R), 0x01 }, + { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_PDE12, + RT721_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, + { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_CS01, + RT721_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, + { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_CS11, + RT721_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, + { SDW_SDCA_CTL(FUNC_NUM_JACK_CODEC, RT721_SDCA_ENT_PDE40, + RT721_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT721_SDCA_ENT_USER_FU1E, RT721_SDCA_CTL_FU_MUTE, CH_01), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT721_SDCA_ENT_USER_FU1E, @@ -53,30 +53,30 @@ static const struct reg_default rt721_sdca_reg_defaults[] = { RT721_SDCA_CTL_FU_MUTE, CH_03), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT721_SDCA_ENT_USER_FU1E, RT721_SDCA_CTL_FU_MUTE, CH_04), 0x01 }, + { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT721_SDCA_ENT_PDE2A, + RT721_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT721_SDCA_ENT_CS1F, RT721_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT721_SDCA_ENT_IT26, RT721_SDCA_CTL_VENDOR_DEF, 0), 0x00 }, - { SDW_SDCA_CTL(FUNC_NUM_MIC_ARRAY, RT721_SDCA_ENT_PDE2A, - RT721_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, - { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_CS31, - RT721_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_USER_FU06, RT721_SDCA_CTL_FU_MUTE, CH_L), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_USER_FU06, RT721_SDCA_CTL_FU_MUTE, CH_R), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_PDE23, RT721_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, - { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_OT23, - RT721_SDCA_CTL_VENDOR_DEF, 0), 0x00 }, { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_PDE23, RT721_SDCA_CTL_FU_MUTE, CH_01), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_PDE23, RT721_SDCA_CTL_FU_MUTE, CH_02), 0x01 }, + { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_CS31, + RT721_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), 0x09 }, { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_FU55, RT721_SDCA_CTL_FU_MUTE, CH_01), 0x01 }, { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_FU55, RT721_SDCA_CTL_FU_MUTE, CH_02), 0x01 }, + { SDW_SDCA_CTL(FUNC_NUM_AMP, RT721_SDCA_ENT_OT23, + RT721_SDCA_CTL_VENDOR_DEF, 0), 0x00 }, }; static const struct reg_default rt721_sdca_mbq_defaults[] = { diff --git a/sound/soc/codecs/sgtl5000.c b/sound/soc/codecs/sgtl5000.c index 59642673b4cbe..35df1a8c4c8c6 100644 --- a/sound/soc/codecs/sgtl5000.c +++ b/sound/soc/codecs/sgtl5000.c @@ -56,8 +56,8 @@ static const struct reg_default sgtl5000_reg_defaults[] = { { SGTL5000_CHIP_PLL_CTRL, 0x5000 }, { SGTL5000_CHIP_CLK_TOP_CTRL, 0x0000 }, { SGTL5000_CHIP_ANA_STATUS, 0x0000 }, - { SGTL5000_CHIP_SHORT_CTRL, 0x0000 }, { SGTL5000_CHIP_ANA_TEST2, 0x0000 }, + { SGTL5000_CHIP_SHORT_CTRL, 0x0000 }, { SGTL5000_DAP_CTRL, 0x0000 }, { SGTL5000_DAP_PEQ, 0x0000 }, { SGTL5000_DAP_BASS_ENHANCE, 0x0040 }, diff --git a/sound/soc/codecs/sti-sas.c b/sound/soc/codecs/sti-sas.c index 4ab15be69f3a4..b7eaf4795d8ba 100644 --- a/sound/soc/codecs/sti-sas.c +++ b/sound/soc/codecs/sti-sas.c @@ -44,8 +44,8 @@ enum { }; static const struct reg_default stih407_sas_reg_defaults[] = { - { STIH407_AUDIO_DAC_CTRL, 0x000000000 }, { STIH407_AUDIO_GLUE_CTRL, 0x00000040 }, + { STIH407_AUDIO_DAC_CTRL, 0x000000000 }, }; struct sti_dac_audio { diff --git a/sound/soc/codecs/tas2552.c b/sound/soc/codecs/tas2552.c index 1a7650b9b2a7b..ea083d1cec2b4 100644 --- a/sound/soc/codecs/tas2552.c +++ b/sound/soc/codecs/tas2552.c @@ -31,25 +31,25 @@ static const struct reg_default tas2552_reg_defs[] = { {TAS2552_CFG_1, 0x22}, + {TAS2552_CFG_2, 0xef}, {TAS2552_CFG_3, 0x80}, {TAS2552_DOUT, 0x00}, - {TAS2552_OUTPUT_DATA, 0xc0}, - {TAS2552_PDM_CFG, 0x01}, - {TAS2552_PGA_GAIN, 0x00}, - {TAS2552_BOOST_APT_CTRL, 0x0f}, - {TAS2552_RESERVED_0D, 0xbe}, - {TAS2552_LIMIT_RATE_HYS, 0x08}, - {TAS2552_CFG_2, 0xef}, {TAS2552_SER_CTRL_1, 0x00}, {TAS2552_SER_CTRL_2, 0x00}, + {TAS2552_OUTPUT_DATA, 0xc0}, {TAS2552_PLL_CTRL_1, 0x10}, {TAS2552_PLL_CTRL_2, 0x00}, {TAS2552_PLL_CTRL_3, 0x00}, {TAS2552_BTIP, 0x8f}, {TAS2552_BTS_CTRL, 0x80}, + {TAS2552_RESERVED_0D, 0xbe}, + {TAS2552_LIMIT_RATE_HYS, 0x08}, {TAS2552_LIMIT_RELEASE, 0x04}, {TAS2552_LIMIT_INT_COUNT, 0x00}, + {TAS2552_PDM_CFG, 0x01}, + {TAS2552_PGA_GAIN, 0x00}, {TAS2552_EDGE_RATE_CTRL, 0x40}, + {TAS2552_BOOST_APT_CTRL, 0x0f}, {TAS2552_VBAT_DATA, 0x00}, }; diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index b11b998aa32a6..f9c65c9f6d0a4 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -897,13 +897,13 @@ static const struct reg_default tas2764_reg_defaults[] = { { TAS2764_PAGE, 0x00 }, { TAS2764_SW_RST, 0x00 }, { TAS2764_PWR_CTRL, 0x1a }, - { TAS2764_DVC, 0x00 }, { TAS2764_CHNL_0, 0x28 }, { TAS2764_TDM_CFG0, 0x09 }, { TAS2764_TDM_CFG1, 0x02 }, { TAS2764_TDM_CFG2, 0x0a }, { TAS2764_TDM_CFG3, 0x10 }, { TAS2764_TDM_CFG5, 0x42 }, + { TAS2764_DVC, 0x00 }, { TAS2764_INT_CLK_CFG, 0x19 }, }; diff --git a/sound/soc/codecs/tas2780.c b/sound/soc/codecs/tas2780.c index 1ec1c076204fb..f1fce4305fa11 100644 --- a/sound/soc/codecs/tas2780.c +++ b/sound/soc/codecs/tas2780.c @@ -527,13 +527,13 @@ static const struct reg_default tas2780_reg_defaults[] = { { TAS2780_PAGE, 0x00 }, { TAS2780_SW_RST, 0x00 }, { TAS2780_PWR_CTRL, 0x1a }, - { TAS2780_DVC, 0x00 }, { TAS2780_CHNL_0, 0x00 }, { TAS2780_TDM_CFG0, 0x09 }, { TAS2780_TDM_CFG1, 0x02 }, { TAS2780_TDM_CFG2, 0x0a }, { TAS2780_TDM_CFG3, 0x10 }, { TAS2780_TDM_CFG5, 0x42 }, + { TAS2780_DVC, 0x00 }, }; static const struct regmap_range_cfg tas2780_regmap_ranges[] = { diff --git a/sound/soc/codecs/tas2783-sdw.c b/sound/soc/codecs/tas2783-sdw.c index 3d0b116544cc4..4fbbf2ca5bf9a 100644 --- a/sound/soc/codecs/tas2783-sdw.c +++ b/sound/soc/codecs/tas2783-sdw.c @@ -106,7 +106,6 @@ struct tas2783_prv { static const struct reg_default tas2783_reg_default[] = { {TAS2783_AMP_LEVEL, 0x28}, - {TASDEV_REG_SDW(0, 0, 0x03), 0x28}, {TASDEV_REG_SDW(0, 0, 0x04), 0x21}, {TASDEV_REG_SDW(0, 0, 0x05), 0x41}, {TASDEV_REG_SDW(0, 0, 0x06), 0x00}, @@ -142,6 +141,7 @@ static const struct reg_default tas2783_reg_default[] = { {TASDEV_REG_SDW(0, 0, 0x41), 0x14}, {TASDEV_REG_SDW(0, 0, 0x5c), 0x19}, {TASDEV_REG_SDW(0, 0, 0x5d), 0x80}, + {TASDEV_REG_SDW(0, 0, 0x60), 0x21}, {TASDEV_REG_SDW(0, 0, 0x63), 0x48}, {TASDEV_REG_SDW(0, 0, 0x65), 0x08}, {TASDEV_REG_SDW(0, 0, 0x66), 0xb2}, @@ -157,7 +157,6 @@ static const struct reg_default tas2783_reg_default[] = { {TASDEV_REG_SDW(0, 0, 0x73), 0x08}, {TASDEV_REG_SDW(0, 0, 0x75), 0xe0}, {TASDEV_REG_SDW(0, 0, 0x7a), 0x60}, - {TASDEV_REG_SDW(0, 0, 0x60), 0x21}, {TASDEV_REG_SDW(0, 1, 0x02), 0x00}, {TASDEV_REG_SDW(0, 1, 0x17), 0xc0}, {TASDEV_REG_SDW(0, 1, 0x19), 0x60}, @@ -176,63 +175,44 @@ static const struct reg_default tas2783_reg_default[] = { {TASDEV_REG_SDW(0, 0xfd, 0x39), 0x00}, {TASDEV_REG_SDW(0, 0xfd, 0x3e), 0x00}, {TASDEV_REG_SDW(0, 0xfd, 0x45), 0x00}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS21, 0x02, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS21, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS24, 0x02, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS24, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS26, 0x02, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS26, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS28, 0x02, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS28, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS127, 0x02, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS127, 0x10, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU21, 0x01, 1), 0x1}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU21, 0x02, 1), 0x9c00}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU23, 0x01, 0), 0x1}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU23, 0x01, 1), 0x1}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU23, 0x0b, 1), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU23, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU26, 0x01, 1), 0x1}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU26, 0x01, 0), 0x1}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU26, 0x01, 1), 0x1}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU26, 0x0b, 1), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU26, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x01, 0), 0x1}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x01, 1), 0x1}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x01, 2), 0x1}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x0b, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x0b, 1), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x0b, 2), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x01, 0), 0x1}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x06, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x07, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x08, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x09, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x0a, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS24, 0x02, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS21, 0x02, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS26, 0x02, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS28, 0x02, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PDE23, 0x1, 0), 0x3}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_SAPU29, 0x05, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x06, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU26, 0x06, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT21, 0x04, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT21, 0x08, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT21, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT21, 0x11, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT29, 0x04, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT29, 0x08, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT26, 0x04, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT26, 0x08, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT26, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT26, 0x11, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT28, 0x04, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT28, 0x08, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT28, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT28, 0x11, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT29, 0x04, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT29, 0x08, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT29, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT29, 0x11, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x01, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x04, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x05, 0), 0x1}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x08, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x11, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x12, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x01, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x04, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x05, 0), 0x1}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x08, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x11, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x12, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT24, 0x04, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT24, 0x08, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT23, 0x04, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT23, 0x08, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT25, 0x04, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT25, 0x08, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT28, 0x04, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT28, 0x08, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MU26, 0x01, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MU26, 0x01, 1), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MU26, 0x01, 2), 0x0}, @@ -242,19 +222,60 @@ static const struct reg_default tas2783_reg_default[] = { {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MU26, 0x01, 6), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MU26, 0x01, 7), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MU26, 0x06, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT23, 0x04, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT23, 0x08, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT24, 0x04, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT24, 0x08, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x04, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x08, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x01, 0), 0x1}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x01, 1), 0x1}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x01, 2), 0x1}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x0b, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x0b, 1), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x0b, 2), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS127, 0x02, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x01, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x04, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x05, 0), 0x1}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x08, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x01, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x04, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x05, 0), 0x1}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x08, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU23, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU26, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x10, 0), 0x1}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x12, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x13, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x14, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x15, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x16, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS24, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS21, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS26, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS28, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PDE23, 0x10, 0), 0x3}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_UDMPU23, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_SAPU29, 0x10, 0), 0x1}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_SAPU29, 0x11, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_SAPU29, 0x12, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x11, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x12, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x13, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU26, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU26, 0x11, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU26, 0x12, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU26, 0x13, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_TG23, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT21, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT21, 0x11, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT29, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT29, 0x11, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT26, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT26, 0x11, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT28, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_IT28, 0x11, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT24, 0x11, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT25, 0x04, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT25, 0x08, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT25, 0x11, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT28, 0x04, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT28, 0x08, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT28, 0x11, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x04, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x08, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x11, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 1), 0x0}, @@ -264,6 +285,14 @@ static const struct reg_default tas2783_reg_default[] = { {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 5), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 6), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 7), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU127, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_CS127, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x11, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU21, 0x12, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x10, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x11, 0), 0x0}, + {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_MFPU26, 0x12, 0), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 8), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 9), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 0xa), 0x0}, @@ -272,36 +301,6 @@ static const struct reg_default tas2783_reg_default[] = { {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 0xd), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 0xe), 0x0}, {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_OT127, 0x12, 0xf), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PDE23, 0x1, 0), 0x3}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PDE23, 0x10, 0), 0x3}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x06, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x11, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x12, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x13, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU26, 0x06, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU26, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU26, 0x11, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU26, 0x12, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU26, 0x13, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_SAPU29, 0x05, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_SAPU29, 0x10, 0), 0x1}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_SAPU29, 0x11, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_SAPU29, 0x12, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_TG23, 0x10, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x01, 0), 0x1}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x06, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x07, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x08, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x09, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x0a, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x10, 0), 0x1}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x12, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x13, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x14, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x15, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_XU22, 0x16, 0), 0x0}, - {SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_UDMPU23, 0x10, 0), 0x0}, }; static const struct reg_sequence tas2783_init_seq[] = { @@ -1099,7 +1098,13 @@ static s32 tas2783_sdca_dev_resume(struct device *dev) } regcache_cache_only(tas_dev->regmap, false); - regcache_sync(tas_dev->regmap); + ret = regcache_sync(tas_dev->regmap); + if (ret) { + regcache_cache_only(tas_dev->regmap, true); + regcache_mark_dirty(tas_dev->regmap); + return ret; + } + return 0; } @@ -1225,9 +1230,23 @@ static s32 tas_update_status(struct sdw_slave *slave, if (tas_dev->hw_init || tas_dev->status != SDW_SLAVE_ATTACHED) return 0; - /* updated the cache data to device */ regcache_cache_only(tas_dev->regmap, false); - regcache_sync(tas_dev->regmap); + + /* + * The device is attaching uninitialized: either this is the first + * attach, or it lost power (and with it all register and DSP state) + * while the controller was power-gated during system suspend. The + * cache still holds the pre-suspend values, and tas_io_init() below + * resets the device via TAS2783_SW_RESET anyway, so syncing it back + * is both useless and harmful: later read-modify-write updates would + * compare against stale data and skip the hardware write. + * + * Drop the cache instead, so that subsequent accesses see the real + * hardware state. Syncing after the reset is not an option either: + * the cache accepts registers for which tas2783_sdca_mbq_size() + * returns 0, and writing those back fails with -EINVAL. + */ + regcache_drop_region(tas_dev->regmap, 0, UINT_MAX); /* perform I/O transfers required for Slave initialization */ return tas_io_init(&slave->dev, slave); diff --git a/sound/soc/codecs/tas675x.c b/sound/soc/codecs/tas675x.c index 82526362de7be..404706b621567 100644 --- a/sound/soc/codecs/tas675x.c +++ b/sound/soc/codecs/tas675x.c @@ -1924,15 +1924,6 @@ static const struct reg_default tas675x_reg_defaults[] = { { TAS675X_PWM_PHASE_M_CTRL_CH2_REG, 0x00 }, { TAS675X_PWM_PHASE_M_CTRL_CH3_REG, 0x00 }, { TAS675X_PWM_PHASE_M_CTRL_CH4_REG, 0x00 }, - { TAS675X_DC_LDG_CTRL_REG, 0x00 }, - { TAS675X_DC_LDG_LO_CTRL_REG, 0x00 }, - { TAS675X_DC_LDG_TIME_CTRL_REG, 0x00 }, - { TAS675X_DC_LDG_SL_CH1_CH2_CTRL_REG, 0x11 }, - { TAS675X_DC_LDG_SL_CH3_CH4_CTRL_REG, 0x11 }, - { TAS675X_AC_LDG_CTRL_REG, 0x10 }, - { TAS675X_TWEETER_DETECT_CTRL_REG, 0x08 }, - { TAS675X_TWEETER_DETECT_THRESH_REG, 0x00 }, - { TAS675X_AC_LDG_FREQ_CTRL_REG, 0xC8 }, { TAS675X_REPORT_ROUTING_1_REG, 0x00 }, { TAS675X_OTSD_RECOVERY_EN_REG, 0x00 }, { TAS675X_REPORT_ROUTING_2_REG, 0xA2 }, @@ -1943,6 +1934,15 @@ static const struct reg_default tas675x_reg_defaults[] = { { TAS675X_GPIO1_OUTPUT_SEL_REG, 0x00 }, { TAS675X_GPIO2_OUTPUT_SEL_REG, 0x00 }, { TAS675X_GPIO_CTRL_REG, TAS675X_GPIO_CTRL_RSTVAL }, + { TAS675X_DC_LDG_CTRL_REG, 0x00 }, + { TAS675X_DC_LDG_LO_CTRL_REG, 0x00 }, + { TAS675X_DC_LDG_TIME_CTRL_REG, 0x00 }, + { TAS675X_DC_LDG_SL_CH1_CH2_CTRL_REG, 0x11 }, + { TAS675X_DC_LDG_SL_CH3_CH4_CTRL_REG, 0x11 }, + { TAS675X_AC_LDG_CTRL_REG, 0x10 }, + { TAS675X_TWEETER_DETECT_CTRL_REG, 0x08 }, + { TAS675X_TWEETER_DETECT_THRESH_REG, 0x00 }, + { TAS675X_AC_LDG_FREQ_CTRL_REG, 0xC8 }, { TAS675X_OTW_CTRL_CH1_CH2_REG, 0x11 }, { TAS675X_OTW_CTRL_CH3_CH4_REG, 0x11 }, }; diff --git a/sound/soc/fsl/Kconfig b/sound/soc/fsl/Kconfig index 8ae59c0948788..04940879dfd80 100644 --- a/sound/soc/fsl/Kconfig +++ b/sound/soc/fsl/Kconfig @@ -244,7 +244,7 @@ config SND_SOC_MPC5200_AC97 config SND_MPC52xx_SOC_PCM030 tristate "SoC AC97 Audio support for Phytec pcm030 and WM9712" depends on PPC_MPC5200_SIMPLE - select SND_SOC_MPC5200_AC97 + depends on SND_SOC_MPC5200_AC97 select SND_SOC_WM9712 help Say Y if you want to add support for sound on the Phytec pcm030 @@ -253,7 +253,7 @@ config SND_MPC52xx_SOC_PCM030 config SND_MPC52xx_SOC_EFIKA tristate "SoC AC97 Audio support for bbplan Efika and STAC9766" depends on PPC_EFIKA - select SND_SOC_MPC5200_AC97 + depends on SND_SOC_MPC5200_AC97 select SND_SOC_STAC9766 help Say Y if you want to add support for sound on the Efika. diff --git a/sound/soc/fsl/efika-audio-fabric.c b/sound/soc/fsl/efika-audio-fabric.c index 4d3266d9cbb79..e9c728bae8b97 100644 --- a/sound/soc/fsl/efika-audio-fabric.c +++ b/sound/soc/fsl/efika-audio-fabric.c @@ -22,8 +22,6 @@ #include #include -#include "mpc5200_dma.h" - #define DRV_NAME "efika-audio-fabric" SND_SOC_DAILINK_DEFS(analog, diff --git a/sound/soc/fsl/fsl-asoc-card.c b/sound/soc/fsl/fsl-asoc-card.c index 709543308fe9e..eff46666c8add 100644 --- a/sound/soc/fsl/fsl-asoc-card.c +++ b/sound/soc/fsl/fsl-asoc-card.c @@ -728,8 +728,8 @@ static int fsl_asoc_card_probe(struct platform_device *pdev) cpu_pdev = of_find_device_by_node(cpu_np); if (!cpu_pdev) { - dev_err(&pdev->dev, "failed to find CPU DAI device\n"); - ret = -EINVAL; + ret = dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "failed to find CPU DAI device\n"); goto fail; } diff --git a/sound/soc/fsl/fsl_audmix.c b/sound/soc/fsl/fsl_audmix.c index f819f33ec46b8..2885cc10b02d4 100644 --- a/sound/soc/fsl/fsl_audmix.c +++ b/sound/soc/fsl/fsl_audmix.c @@ -457,6 +457,9 @@ static const struct of_device_id fsl_audmix_ids[] = { }; MODULE_DEVICE_TABLE(of, fsl_audmix_ids); +static int fsl_audmix_runtime_resume(struct device *dev); +static int fsl_audmix_runtime_suspend(struct device *dev); + static int fsl_audmix_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -488,13 +491,25 @@ static int fsl_audmix_probe(struct platform_device *pdev) spin_lock_init(&priv->lock); platform_set_drvdata(pdev, priv); pm_runtime_enable(dev); + if (!pm_runtime_enabled(dev)) { + ret = fsl_audmix_runtime_resume(dev); + if (ret) + goto err_disable_pm; + } + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + goto err_pm_get_sync; + + /* To enable regmap cache only when runtime PM enabled */ + pm_runtime_put(dev); ret = devm_snd_soc_register_component(dev, &fsl_audmix_component, fsl_audmix_dai, ARRAY_SIZE(fsl_audmix_dai)); if (ret) { dev_err(dev, "failed to register ASoC DAI\n"); - goto err_disable_pm; + goto err_pm_get_sync; } /* @@ -506,12 +521,15 @@ static int fsl_audmix_probe(struct platform_device *pdev) if (IS_ERR(priv->pdev)) { ret = PTR_ERR(priv->pdev); dev_err(dev, "failed to register platform: %d\n", ret); - goto err_disable_pm; + goto err_pm_get_sync; } } return 0; +err_pm_get_sync: + if (!pm_runtime_status_suspended(dev)) + fsl_audmix_runtime_suspend(dev); err_disable_pm: pm_runtime_disable(dev); return ret; @@ -522,6 +540,8 @@ static void fsl_audmix_remove(struct platform_device *pdev) struct fsl_audmix *priv = dev_get_drvdata(&pdev->dev); pm_runtime_disable(&pdev->dev); + if (!pm_runtime_status_suspended(&pdev->dev)) + fsl_audmix_runtime_suspend(&pdev->dev); if (priv->pdev) platform_device_unregister(priv->pdev); diff --git a/sound/soc/fsl/fsl_easrc.c b/sound/soc/fsl/fsl_easrc.c index 8535ef844ce03..be59c138e3ea4 100644 --- a/sound/soc/fsl/fsl_easrc.c +++ b/sound/soc/fsl/fsl_easrc.c @@ -1711,12 +1711,12 @@ static const struct reg_default fsl_easrc_reg_defaults[] = { {REG_EASRC_SFS(2), 0x00000000}, {REG_EASRC_SFS(3), 0x00000000}, {REG_EASRC_RRL(0), 0x00000000}, - {REG_EASRC_RRL(1), 0x00000000}, - {REG_EASRC_RRL(2), 0x00000000}, - {REG_EASRC_RRL(3), 0x00000000}, {REG_EASRC_RRH(0), 0x00000000}, + {REG_EASRC_RRL(1), 0x00000000}, {REG_EASRC_RRH(1), 0x00000000}, + {REG_EASRC_RRL(2), 0x00000000}, {REG_EASRC_RRH(2), 0x00000000}, + {REG_EASRC_RRL(3), 0x00000000}, {REG_EASRC_RRH(3), 0x00000000}, {REG_EASRC_RUC(0), 0x00000000}, {REG_EASRC_RUC(1), 0x00000000}, @@ -2058,7 +2058,7 @@ static int fsl_easrc_m2m_calc_out_len(struct fsl_asrc_pair *pair, int input_buff /* right shift 12 bit to make ratio in 32bit space */ val2 = (u64)in_samples << (frac_bits - 12); val1 = val1 >> 12; - do_div(val2, val1); + val2 = div64_u64(val2, val1); out_samples = val2; out_length = out_samples * out_width * channels; diff --git a/sound/soc/fsl/fsl_micfil.c b/sound/soc/fsl/fsl_micfil.c index 60ac8eabab9da..5d8f0f76ab46f 100644 --- a/sound/soc/fsl/fsl_micfil.c +++ b/sound/soc/fsl/fsl_micfil.c @@ -953,12 +953,17 @@ static int fsl_micfil_reparent_rootclk(struct fsl_micfil *micfil, unsigned int s /* Get root clock */ clk = micfil->mclk; - /* Disable clock first, for it was enabled by pm_runtime */ + /* Reparent root clock to the PLL matching this sample rate */ fsl_asoc_reparent_pll_clocks(dev, clk, micfil->pll8k_clk, micfil->pll11k_clk, ratio); - ret = clk_prepare_enable(clk); - if (ret) - return ret; + + /* Enable only once; hw_params can be called multiple times */ + if (!micfil->mclk_flag) { + ret = clk_prepare_enable(clk); + if (ret) + return ret; + micfil->mclk_flag = true; + } return 0; } @@ -991,8 +996,6 @@ static int fsl_micfil_hw_params(struct snd_pcm_substream *substream, if (ret) return ret; - micfil->mclk_flag = true; - /* floor(K * CLKDIV) */ switch (micfil->quality) { case QUALITY_HIGH: @@ -1068,8 +1071,10 @@ static int fsl_micfil_hw_free(struct snd_pcm_substream *substream, { struct fsl_micfil *micfil = snd_soc_dai_get_drvdata(dai); - clk_disable_unprepare(micfil->mclk); - micfil->mclk_flag = false; + if (micfil->mclk_flag) { + clk_disable_unprepare(micfil->mclk); + micfil->mclk_flag = false; + } return 0; } diff --git a/sound/soc/fsl/mpc5200_psc_i2s.c b/sound/soc/fsl/mpc5200_psc_i2s.c index 9ad44eeed6ad8..7831136f4f129 100644 --- a/sound/soc/fsl/mpc5200_psc_i2s.c +++ b/sound/soc/fsl/mpc5200_psc_i2s.c @@ -170,6 +170,7 @@ static int psc_i2s_of_probe(struct platform_device *op) psc_i2s_dai, ARRAY_SIZE(psc_i2s_dai)); if (rc != 0) { pr_err("Failed to register DAI\n"); + mpc5200_audio_dma_destroy(op); return rc; } diff --git a/sound/soc/fsl/pcm030-audio-fabric.c b/sound/soc/fsl/pcm030-audio-fabric.c index 5542c4ee6d129..a7a5dba83a2f8 100644 --- a/sound/soc/fsl/pcm030-audio-fabric.c +++ b/sound/soc/fsl/pcm030-audio-fabric.c @@ -13,8 +13,6 @@ #include -#include "mpc5200_dma.h" - #define DRV_NAME "pcm030-audio-fabric" struct pcm030_audio_data { diff --git a/sound/soc/intel/atom/sst/sst_pci.c b/sound/soc/intel/atom/sst/sst_pci.c index 22ae2d22f121a..1a53c993b57fc 100644 --- a/sound/soc/intel/atom/sst/sst_pci.c +++ b/sound/soc/intel/atom/sst/sst_pci.c @@ -130,13 +130,15 @@ static int intel_sst_probe(struct pci_dev *pci, sst_drv_ctx->pci = pci_dev_get(pci); ret = sst_platform_get_resources(sst_drv_ctx); if (ret < 0) - goto do_free_drv_ctx; + goto do_put_pci; pci_set_drvdata(pci, sst_drv_ctx); sst_configure_runtime_pm(sst_drv_ctx); return ret; +do_put_pci: + pci_dev_put(sst_drv_ctx->pci); do_free_drv_ctx: sst_context_cleanup(sst_drv_ctx); dev_err(sst_drv_ctx->dev, "Probe failed with %d\n", ret); @@ -165,6 +167,7 @@ static const struct pci_device_id intel_sst_ids[] = { { PCI_DEVICE_DATA(INTEL, SST_TNG, 0) }, { 0, } }; +MODULE_DEVICE_TABLE(pci, intel_sst_ids); static struct pci_driver sst_driver = { .name = SST_DRV_NAME, diff --git a/sound/soc/intel/avs/core.c b/sound/soc/intel/avs/core.c index 1a53856c2ffbe..a6b877f41e8b2 100644 --- a/sound/soc/intel/avs/core.c +++ b/sound/soc/intel/avs/core.c @@ -473,8 +473,13 @@ static int avs_pci_probe(struct pci_dev *pci, const struct pci_device_id *id) } snd_hdac_bus_parse_capabilities(bus); - if (bus->mlcap) - snd_hdac_ext_bus_get_ml_capabilities(bus); + if (bus->mlcap) { + ret = snd_hdac_ext_bus_get_ml_capabilities(bus); + if (ret < 0) { + dev_err(dev, "failed to get ml capabilities: %d\n", ret); + goto err_ml_cap; + } + } if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); @@ -516,6 +521,8 @@ static int avs_pci_probe(struct pci_dev *pci, const struct pci_device_id *id) snd_hdac_bus_free_stream_pages(bus); snd_hdac_ext_stream_free_all(bus); err_init_streams: + snd_hdac_ext_link_free_all(bus); +err_ml_cap: iounmap(adev->dsp_ba); err_remap_bar4: iounmap(bus->remap_addr); diff --git a/sound/soc/intel/avs/debugfs.c b/sound/soc/intel/avs/debugfs.c index 701c247227bf9..fc321f61fc929 100644 --- a/sound/soc/intel/avs/debugfs.c +++ b/sound/soc/intel/avs/debugfs.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -235,15 +236,20 @@ static int strace_open(struct inode *inode, struct file *file) if (!try_module_get(adev->dev->driver->owner)) return -ENODEV; - if (kfifo_initialized(&adev->trace_fifo)) - return -EBUSY; + if (kfifo_initialized(&adev->trace_fifo)) { + ret = -EBUSY; + goto err; + } ret = kfifo_alloc(&adev->trace_fifo, PAGE_SIZE, GFP_KERNEL); if (ret < 0) - return ret; + goto err; file->private_data = adev; return 0; +err: + module_put(adev->dev->driver->owner); + return ret; } static int strace_release(struct inode *inode, struct file *file) diff --git a/sound/soc/intel/avs/path.c b/sound/soc/intel/avs/path.c index 2291f9728a54f..de699e23c9e22 100644 --- a/sound/soc/intel/avs/path.c +++ b/sound/soc/intel/avs/path.c @@ -838,15 +838,10 @@ static int avs_path_module_type_create(struct avs_dev *adev, struct avs_path_mod static int avs_path_module_send_init_configs(struct avs_dev *adev, struct avs_path_module *mod) { - struct avs_soc_component *acomp; - - acomp = to_avs_soc_component(mod->template->owner->owner->owner->owner->comp); - - u32 num_ids = mod->template->num_config_ids; - u32 *ids = mod->template->config_ids; + struct avs_tplg_module *template = mod->template; - for (int i = 0; i < num_ids; i++) { - struct avs_tplg_init_config *config = &acomp->tplg->init_configs[ids[i]]; + for (int i = 0; i < template->num_init_configs; i++) { + struct avs_tplg_init_config *config = template->init_configs[i]; size_t len = config->length; void *data = config->data; u32 param = config->param; diff --git a/sound/soc/intel/avs/pcm.c b/sound/soc/intel/avs/pcm.c index 797b9c9163b49..094ea1366ea8f 100644 --- a/sound/soc/intel/avs/pcm.c +++ b/sound/soc/intel/avs/pcm.c @@ -6,6 +6,7 @@ // Amadeusz Slawinski // +#include #include #include #include @@ -987,13 +988,25 @@ static int avs_component_load_libraries(struct avs_soc_component *acomp) return ret; } +static int avs_request_topology(struct snd_soc_component *component, const char *name, + const struct firmware **fw) +{ + char *fullname __free(kfree) = NULL; + + fullname = kasprintf(GFP_KERNEL, "%s/%s", component->driver->topology_name_prefix, name); + if (!fullname) + return -ENOMEM; + + return request_firmware(fw, fullname, component->dev); +} + static int avs_component_probe(struct snd_soc_component *component) { struct snd_soc_card *card = component->card; struct snd_soc_acpi_mach *mach; struct avs_soc_component *acomp; + const struct firmware *fw; struct avs_dev *adev; - char *filename; int ret; dev_dbg(card->dev, "probing %s card %s\n", component->name, card->name); @@ -1009,13 +1022,7 @@ static int avs_component_probe(struct snd_soc_component *component) goto finalize; /* Load specified topology and create debugfs for it. */ - filename = kasprintf(GFP_KERNEL, "%s/%s", component->driver->topology_name_prefix, - mach->tplg_filename); - if (!filename) - return -ENOMEM; - - ret = avs_load_topology(component, filename); - kfree(filename); + ret = avs_request_topology(component, mach->tplg_filename, &fw); if (ret == -ENOENT && !strncmp(mach->tplg_filename, "hda-", 4)) { unsigned int vendor_id; @@ -1030,18 +1037,17 @@ static int avs_component_probe(struct snd_soc_component *component) "hda-generic-tplg.bin"); if (!mach->tplg_filename) return -ENOMEM; - filename = kasprintf(GFP_KERNEL, "%s/%s", component->driver->topology_name_prefix, - mach->tplg_filename); - if (!filename) - return -ENOMEM; dev_info(card->dev, "trying to load fallback topology %s\n", mach->tplg_filename); - ret = avs_load_topology(component, filename); - kfree(filename); + ret = avs_request_topology(component, mach->tplg_filename, &fw); } if (ret < 0) return ret; + ret = snd_soc_tplg_component_load(component, &avs_tplg_ops, fw); + if (ret) + return ret; + ret = avs_component_load_libraries(acomp); if (ret < 0) { dev_err(card->dev, "libraries loading failed: %d\n", ret); diff --git a/sound/soc/intel/avs/topology.c b/sound/soc/intel/avs/topology.c index 9033f683393c4..c9837d78e6234 100644 --- a/sound/soc/intel/avs/topology.c +++ b/sound/soc/intel/avs/topology.c @@ -350,6 +350,7 @@ AVS_DEFINE_PTR_PARSER(modcfg_base, struct avs_tplg_modcfg_base, modcfgs_base); AVS_DEFINE_PTR_PARSER(modcfg_ext, struct avs_tplg_modcfg_ext, modcfgs_ext); AVS_DEFINE_PTR_PARSER(pplcfg, struct avs_tplg_pplcfg, pplcfgs); AVS_DEFINE_PTR_PARSER(binding, struct avs_tplg_binding, bindings); +AVS_DEFINE_PTR_PARSER(init_config, struct avs_tplg_init_config, init_configs); AVS_DEFINE_PTR_PARSER(nhlt_config, struct avs_tplg_nhlt_config, nhlt_configs); static int @@ -1198,7 +1199,7 @@ static const struct avs_tplg_token_parser module_parsers[] = { { .token = AVS_TKN_MOD_INIT_CONFIG_NUM_IDS_U32, .type = SND_SOC_TPLG_TUPLE_TYPE_WORD, - .offset = offsetof(struct avs_tplg_module, num_config_ids), + .offset = offsetof(struct avs_tplg_module, num_init_configs), .parse = avs_parse_byte_token, }, { @@ -1214,10 +1215,32 @@ static const struct avs_tplg_token_parser init_config_parsers[] = { .token = AVS_TKN_MOD_INIT_CONFIG_ID_U32, .type = SND_SOC_TPLG_TUPLE_TYPE_WORD, .offset = 0, - .parse = avs_parse_word_token, + .parse = avs_parse_init_config_ptr, }, }; +static int avs_tplg_module_init_configs(struct snd_soc_component *comp, + struct avs_tplg_module *module, + struct snd_soc_tplg_vendor_array *tuples, u32 block_size) +{ + struct avs_tplg_init_config **cfgs; + int ret; + + if (!module->num_init_configs) + return -EINVAL; + + cfgs = devm_kcalloc(comp->card->dev, module->num_init_configs, sizeof(*cfgs), GFP_KERNEL); + if (!cfgs) + return -ENOMEM; + + ret = parse_dictionary_entries(comp, tuples, block_size, cfgs, module->num_init_configs, + sizeof(*cfgs), AVS_TKN_MOD_INIT_CONFIG_ID_U32, + init_config_parsers, ARRAY_SIZE(init_config_parsers)); + if (!ret) + module->init_configs = cfgs; + return ret; +} + static struct avs_tplg_module * avs_tplg_module_create(struct snd_soc_component *comp, struct avs_tplg_pipeline *owner, struct snd_soc_tplg_vendor_array *tuples, u32 block_size) @@ -1244,27 +1267,11 @@ avs_tplg_module_create(struct snd_soc_component *comp, struct avs_tplg_pipeline block_size -= esize; /* Parse trailing config ids if any. */ if (block_size) { - u32 num_config_ids = module->num_config_ids; - u32 *config_ids; - - if (!num_config_ids) - return ERR_PTR(-EINVAL); - - config_ids = devm_kcalloc(comp->card->dev, num_config_ids, sizeof(*config_ids), - GFP_KERNEL); - if (!config_ids) - return ERR_PTR(-ENOMEM); - tuples = avs_tplg_vendor_array_at(tuples, esize); - ret = parse_dictionary_entries(comp, tuples, block_size, - config_ids, num_config_ids, sizeof(*config_ids), - AVS_TKN_MOD_INIT_CONFIG_ID_U32, - init_config_parsers, - ARRAY_SIZE(init_config_parsers)); + + ret = avs_tplg_module_init_configs(comp, module, tuples, block_size); if (ret) return ERR_PTR(ret); - - module->config_ids = config_ids; } module->owner = owner; @@ -2194,7 +2201,7 @@ avs_control_load(struct snd_soc_component *comp, int index, struct snd_kcontrol_ return 0; } -static const struct snd_soc_tplg_ops avs_tplg_ops = { +const struct snd_soc_tplg_ops avs_tplg_ops = { .io_ops = avs_control_ops, .io_ops_count = ARRAY_SIZE(avs_control_ops), .control_load = avs_control_load, diff --git a/sound/soc/intel/avs/topology.h b/sound/soc/intel/avs/topology.h index 1cf7455b6c010..189984ce7b51e 100644 --- a/sound/soc/intel/avs/topology.h +++ b/sound/soc/intel/avs/topology.h @@ -221,8 +221,8 @@ struct avs_tplg_module { u8 domain; struct avs_tplg_modcfg_ext *cfg_ext; u32 ctl_id; - u32 num_config_ids; - u32 *config_ids; + u32 num_init_configs; + struct avs_tplg_init_config **init_configs; struct avs_tplg_nhlt_config *nhlt_config; struct avs_tplg_pipeline *owner; @@ -230,6 +230,7 @@ struct avs_tplg_module { struct list_head node; }; +extern const struct snd_soc_tplg_ops avs_tplg_ops; struct avs_tplg *avs_tplg_new(struct snd_soc_component *comp); int avs_load_topology(struct snd_soc_component *comp, const char *filename); diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c index c527d575d1ed6..1013e07052a37 100644 --- a/sound/soc/intel/boards/sof_sdw.c +++ b/sound/soc/intel/boards/sof_sdw.c @@ -1288,7 +1288,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card) goto err_dai; } - ret = asoc_sdw_parse_sdw_endpoints(card, sof_aux, sof_dais, sof_ends, &num_confs); + ret = asoc_sdw_parse_sdw_endpoints(dev, ctx, sof_aux, sof_dais, sof_ends, &num_confs); if (ret < 0) goto err_end; diff --git a/sound/soc/loongson/loongson_card.c b/sound/soc/loongson/loongson_card.c index 7910d5d9ac4ff..ea895fe6b5e97 100644 --- a/sound/soc/loongson/loongson_card.c +++ b/sound/soc/loongson/loongson_card.c @@ -91,7 +91,7 @@ static int loongson_card_parse_acpi(struct loongson_card_data *data) const char *codec_dai_name; struct acpi_device *adev; struct device *phy_dev; - int i; + int i, ret; /* fixup platform name based on reference node */ adev = loongson_card_acpi_find_device(card, "cpu"); @@ -108,7 +108,9 @@ static int loongson_card_parse_acpi(struct loongson_card_data *data) return -ENOENT; snprintf(codec_name, sizeof(codec_name), "i2c-%s", acpi_dev_name(adev)); - device_property_read_string(card->dev, "codec-dai-name", &codec_dai_name); + ret = device_property_read_string(card->dev, "codec-dai-name", &codec_dai_name); + if (ret) + return ret; for (i = 0; i < card->num_links; i++) { loongson_dai_links[i].platforms->name = dev_name(phy_dev); diff --git a/sound/soc/meson/meson-card-utils.c b/sound/soc/meson/meson-card-utils.c index cdb759b466ad4..8617a4661a339 100644 --- a/sound/soc/meson/meson-card-utils.c +++ b/sound/soc/meson/meson-card-utils.c @@ -50,25 +50,20 @@ int meson_card_reallocate_links(struct snd_soc_card *card, num_links * sizeof(*priv->card.dai_link), GFP_KERNEL | __GFP_ZERO); if (!links) - goto err_links; + return -ENOMEM; + + priv->card.dai_link = links; + priv->card.num_links = num_links; ldata = krealloc(priv->link_data, num_links * sizeof(*priv->link_data), GFP_KERNEL | __GFP_ZERO); + /* meson_card_clean_references() will free the links on this error path */ if (!ldata) - goto err_ldata; + return -ENOMEM; - priv->card.dai_link = links; priv->link_data = ldata; - priv->card.num_links = num_links; return 0; - -err_ldata: - kfree(links); -err_links: - dev_err(priv->card.dev, "failed to allocate links\n"); - return -ENOMEM; - } EXPORT_SYMBOL_GPL(meson_card_reallocate_links); diff --git a/sound/soc/pxa/pxa-ssp.c b/sound/soc/pxa/pxa-ssp.c index f8054c1c59fae..3a0abcb0bfbdf 100644 --- a/sound/soc/pxa/pxa-ssp.c +++ b/sound/soc/pxa/pxa-ssp.c @@ -769,13 +769,10 @@ static int pxa_ssp_probe(struct snd_soc_dai *dai) goto err_priv; } - priv->extclk = devm_clk_get(dev, "extclk"); + priv->extclk = devm_clk_get_optional(dev, "extclk"); if (IS_ERR(priv->extclk)) { ret = PTR_ERR(priv->extclk); - if (ret == -EPROBE_DEFER) - goto err_priv; - - priv->extclk = NULL; + goto err_priv; } } else { priv->ssp = pxa_ssp_request(dai->id + 1, "SoC audio"); diff --git a/sound/soc/qcom/qdsp6/q6apm.c b/sound/soc/qcom/qdsp6/q6apm.c index 641d6d2432299..f167b9dae3fa1 100644 --- a/sound/soc/qcom/qdsp6/q6apm.c +++ b/sound/soc/qcom/qdsp6/q6apm.c @@ -802,14 +802,17 @@ EXPORT_SYMBOL_GPL(q6apm_graph_prepare); int q6apm_graph_start(struct q6apm_graph *graph) { struct audioreach_graph *ar_graph = graph->ar_graph; - int ret = 0; + int ret; - if (ar_graph->start_count == 0) + if (ar_graph->start_count == 0) { ret = audioreach_graph_mgmt_cmd(ar_graph, APM_CMD_GRAPH_START); + if (ret) + return ret; + } ar_graph->start_count++; - return ret; + return 0; } EXPORT_SYMBOL_GPL(q6apm_graph_start); @@ -817,6 +820,9 @@ int q6apm_graph_stop(struct q6apm_graph *graph) { struct audioreach_graph *ar_graph = graph->ar_graph; + if (ar_graph->start_count == 0) + return 0; + if (--ar_graph->start_count > 0) return 0; diff --git a/sound/soc/samsung/aries_wm8994.c b/sound/soc/samsung/aries_wm8994.c index 48ccc1d1854bc..6db91b73f25c3 100644 --- a/sound/soc/samsung/aries_wm8994.c +++ b/sound/soc/samsung/aries_wm8994.c @@ -658,6 +658,7 @@ static int aries_audio_probe(struct platform_device *pdev) goto out; } + of_node_get(aries_dai[0].cpus->of_node); aries_dai[0].platforms->of_node = aries_dai[0].cpus->of_node; /* Set CPU of_node for BT DAI */ diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c index f80f697a5d55d..f80e8d4981565 100644 --- a/sound/soc/samsung/i2s.c +++ b/sound/soc/samsung/i2s.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -512,7 +513,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, u32 mod, mask, val = 0; int ret = 0; - pm_runtime_get_sync(dai->dev); + guard(pm_runtime_active)(dai->dev); scoped_guard(spinlock_irqsave, &priv->lock) mod = readl(priv->addr + I2SMOD); @@ -537,8 +538,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, && (mod & cdcon_mask))))) { dev_err(&i2s->pdev->dev, "%s:%d Other DAI busy\n", __func__, __LINE__); - ret = -EAGAIN; - goto err; + return -EAGAIN; } if (dir == SND_SOC_CLOCK_IN) @@ -566,7 +566,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, } else { priv->rclk_srcrate = clk_get_rate(priv->op_clk); - goto done; + return 0; } } @@ -580,14 +580,14 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, if (WARN_ON(IS_ERR(priv->op_clk))) { ret = PTR_ERR(priv->op_clk); priv->op_clk = NULL; - goto err; + return ret; } ret = clk_prepare_enable(priv->op_clk); if (ret) { clk_put(priv->op_clk); priv->op_clk = NULL; - goto err; + return ret; } priv->rclk_srcrate = clk_get_rate(priv->op_clk); @@ -595,11 +595,10 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, || (clk_id && !(mod & rsrc_mask))) { dev_err(&i2s->pdev->dev, "%s:%d Other DAI busy\n", __func__, __LINE__); - ret = -EAGAIN; - goto err; + return -EAGAIN; } else { /* Call can't be on the active DAI */ - goto done; + return 0; } if (clk_id == 1) @@ -607,8 +606,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, break; default: dev_err(&i2s->pdev->dev, "We don't serve that!\n"); - ret = -EINVAL; - goto err; + return -EINVAL; } scoped_guard(spinlock_irqsave, &priv->lock) { @@ -616,13 +614,8 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, mod = (mod & ~mask) | val; writel(mod, priv->addr + I2SMOD); } -done: - pm_runtime_put(dai->dev); return 0; -err: - pm_runtime_put(dai->dev); - return ret; } static int i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) diff --git a/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c b/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c index e0e32a279787c..129a437ae397e 100644 --- a/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c +++ b/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c @@ -99,11 +99,9 @@ static const struct snd_soc_dai_link bridge_dai_template = { SND_SOC_DAILINK_REG(asoc_sdw_bridge_dai), }; -int asoc_sdw_bridge_cs35l56_count_sidecar(struct snd_soc_card *card, +int asoc_sdw_bridge_cs35l56_count_sidecar(struct asoc_sdw_mc_private *ctx, int *num_dais, int *num_devs) { - struct asoc_sdw_mc_private *ctx = snd_soc_card_get_drvdata(card); - if (ctx->mc_quirk & SOC_SDW_SIDECAR_AMPS) { (*num_dais)++; (*num_devs) += ARRAY_SIZE(bridge_cs35l56_name_prefixes); diff --git a/sound/soc/sdw_utils/soc_sdw_utils.c b/sound/soc/sdw_utils/soc_sdw_utils.c index d8db8fc5313ee..befcad08135e7 100644 --- a/sound/soc/sdw_utils/soc_sdw_utils.c +++ b/sound/soc/sdw_utils/soc_sdw_utils.c @@ -1541,7 +1541,7 @@ int asoc_sdw_hw_params(struct snd_pcm_substream *substream, * ASoC will set the corresponding channel numbers for each cpu dai. */ for_each_link_ch_maps(rtd->dai_link, i, ch_maps) - ch_maps->ch_mask = ch_mask << (i * step); + ch_maps->cpu_ch_mask = ch_mask << (i * step); return 0; } @@ -1976,14 +1976,13 @@ static int is_sdca_endpoint_present(struct device *dev, return ret; } -int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card, +int asoc_sdw_parse_sdw_endpoints(struct device *dev, + struct asoc_sdw_mc_private *ctx, struct snd_soc_aux_dev *soc_aux, struct asoc_sdw_dailink *soc_dais, struct asoc_sdw_endpoint *soc_ends, int *num_devs) { - struct device *dev = card->dev; - struct asoc_sdw_mc_private *ctx = snd_soc_card_get_drvdata(card); struct snd_soc_acpi_mach *mach = dev_get_platdata(dev); struct snd_soc_acpi_mach_params *mach_params = &mach->mach_params; const struct snd_soc_acpi_link_adr *adr_link; @@ -2045,7 +2044,7 @@ int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card, ctx->ignore_internal_dmic |= codec_info->ignore_internal_dmic; if (codec_info->count_sidecar && codec_info->add_sidecar) { - ret = codec_info->count_sidecar(card, &num_dais, num_devs); + ret = codec_info->count_sidecar(ctx, &num_dais, num_devs); if (ret) return ret; diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index 4ad126bd4f705..646f26b7b4153 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -3608,7 +3608,7 @@ int snd_soc_dapm_put_enum_double(struct snd_kcontrol *kcontrol, val = snd_soc_enum_item_to_val(e, item[0]) << e->shift_l; mask = e->mask << e->shift_l; if (e->shift_l != e->shift_r) { - if (item[1] > e->items) + if (item[1] >= e->items) return -EINVAL; val |= snd_soc_enum_item_to_val(e, item[1]) << e->shift_r; mask |= e->mask << e->shift_r; diff --git a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c index 0e49290a8c903..3137c091bdb88 100644 --- a/sound/soc/soc-pcm.c +++ b/sound/soc/soc-pcm.c @@ -1206,7 +1206,9 @@ static int __soc_pcm_hw_params(struct snd_pcm_substream *substream, goto out; for_each_rtd_codec_dais(rtd, i, codec_dai) { - unsigned int tdm_mask = snd_soc_dai_tdm_mask_get(codec_dai, substream->stream); + unsigned int ch_mask = snd_soc_dai_tdm_mask_get(codec_dai, substream->stream); + struct snd_soc_dai_link_ch_map *ch_maps; + int j; /* * Skip CODECs which don't support the current stream type, @@ -1228,9 +1230,15 @@ static int __soc_pcm_hw_params(struct snd_pcm_substream *substream, /* copy params for each codec */ tmp_params = *params; - /* fixup params based on TDM slot masks */ - if (tdm_mask) - soc_pcm_codec_params_fixup(&tmp_params, tdm_mask); + /* fixup params based on TDM or ch_map masks */ + if (!ch_mask) { + for_each_rtd_ch_maps(rtd, j, ch_maps) + if (ch_maps->codec == i) + ch_mask |= ch_maps->codec_ch_mask; + } + + if (ch_mask) + soc_pcm_codec_params_fixup(&tmp_params, ch_mask); ret = snd_soc_dai_hw_params(codec_dai, substream, &tmp_params); @@ -1264,7 +1272,7 @@ static int __soc_pcm_hw_params(struct snd_pcm_substream *substream, */ for_each_rtd_ch_maps(rtd, j, ch_maps) if (ch_maps->cpu == i) - ch_mask |= ch_maps->ch_mask; + ch_mask |= ch_maps->cpu_ch_mask; /* fixup cpu channel number */ if (ch_mask) diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index 6fdfb667cce8d..8f37fbdd3f7fa 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -1127,6 +1127,7 @@ static int sof_ipc4_widget_setup_comp_src(struct snd_sof_widget *swidget) "Invalid number of formats: input: %d, output: %d\n", src->available_fmt.num_input_formats, src->available_fmt.num_output_formats); + ret = -EINVAL; goto err; } @@ -1179,6 +1180,7 @@ static int sof_ipc4_widget_setup_comp_asrc(struct snd_sof_widget *swidget) "Invalid number of formats: input: %d, output: %d\n", asrc->available_fmt.num_input_formats, asrc->available_fmt.num_output_formats); + ret = -EINVAL; goto err; } diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index 42a2d90bb705d..31dd7a66a9cf9 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -846,6 +846,7 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, struct snd_soc_tplg_mixer_control *mc = container_of(hdr, struct snd_soc_tplg_mixer_control, hdr); int tlv[SOF_TLV_ITEMS]; + u32 min, max; unsigned int mask; int ret; @@ -853,6 +854,11 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, if (le32_to_cpu(mc->num_channels) > SND_SOC_TPLG_MAX_CHAN) return -EINVAL; + min = le32_to_cpu(mc->min); + max = le32_to_cpu(mc->max); + if (min > max || max >= INT_MAX) + return -EINVAL; + /* * If control has more than 2 channels we need to override the info. This is because even if * ASoC layer has defined topology's max channel count to SND_SOC_TPLG_MAX_CHAN = 8, the @@ -863,12 +869,12 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, kc->info = snd_sof_volume_info; scontrol->comp_id = sdev->next_comp_id; - scontrol->min_volume_step = le32_to_cpu(mc->min); - scontrol->max_volume_step = le32_to_cpu(mc->max); + scontrol->min_volume_step = min; + scontrol->max_volume_step = max; scontrol->num_channels = le32_to_cpu(mc->num_channels); - scontrol->max = le32_to_cpu(mc->max); - if (le32_to_cpu(mc->max) == 1) + scontrol->max = max; + if (max == 1) goto skip; /* extract tlv data */ @@ -878,7 +884,7 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, } /* set up volume table */ - ret = set_up_volume_table(scontrol, tlv, le32_to_cpu(mc->max) + 1); + ret = set_up_volume_table(scontrol, tlv, max + 1); if (ret < 0) { dev_err(scomp->dev, "error: setting up volume table\n"); return ret; @@ -911,7 +917,7 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, return 0; err: - if (le32_to_cpu(mc->max) > 1) + if (max > 1) kfree(scontrol->volume_table); return ret; diff --git a/sound/soc/spacemit/k1_i2s.c b/sound/soc/spacemit/k1_i2s.c index 8871fc15b29cc..67c03b8a5e0c4 100644 --- a/sound/soc/spacemit/k1_i2s.c +++ b/sound/soc/spacemit/k1_i2s.c @@ -52,7 +52,7 @@ struct spacemit_i2s_dev { struct clk *sysclk; struct clk *bclk; - struct clk *sspa_clk; + struct clk *func_clk; struct clk *sysclk_div; struct clk *c_sysclk; struct clk *c_bclk; @@ -221,7 +221,7 @@ static int spacemit_i2s_hw_params(struct snd_pcm_substream *substream, if (ret) return ret; - return clk_set_rate(i2s->sspa_clk, bclk_rate); + return clk_set_rate(i2s->func_clk, bclk_rate); } static int spacemit_i2s_set_sysclk(struct snd_soc_dai *cpu_dai, int clk_id, @@ -347,22 +347,6 @@ static const struct snd_soc_dai_ops spacemit_i2s_dai_ops = { static struct snd_soc_dai_driver spacemit_i2s_dai = { .ops = &spacemit_i2s_dai_ops, - .playback = { - .channels_min = 1, - .channels_max = 2, - .rates = SPACEMIT_PCM_RATES, - .rate_min = SNDRV_PCM_RATE_8000, - .rate_max = SNDRV_PCM_RATE_48000, - .formats = SPACEMIT_PCM_FORMATS, - }, - .capture = { - .channels_min = 1, - .channels_max = 2, - .rates = SPACEMIT_PCM_RATES, - .rate_min = SNDRV_PCM_RATE_8000, - .rate_max = SNDRV_PCM_RATE_48000, - .formats = SPACEMIT_PCM_FORMATS, - }, .symmetric_rate = 1, }; @@ -392,6 +376,8 @@ static int spacemit_i2s_init_dai(struct spacemit_i2s_dev *i2s, dai->playback.channels_min = 1; dai->playback.channels_max = 2; dai->playback.rates = SPACEMIT_PCM_RATES; + dai->playback.rate_min = SNDRV_PCM_RATE_8000; + dai->playback.rate_max = SNDRV_PCM_RATE_48000; dai->playback.formats = SPACEMIT_PCM_FORMATS; i2s->playback_dma_data.addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; @@ -404,6 +390,8 @@ static int spacemit_i2s_init_dai(struct spacemit_i2s_dev *i2s, dai->capture.channels_min = 1; dai->capture.channels_max = 2; dai->capture.rates = SPACEMIT_PCM_RATES; + dai->capture.rate_min = SNDRV_PCM_RATE_8000; + dai->capture.rate_max = SNDRV_PCM_RATE_48000; dai->capture.formats = SPACEMIT_PCM_FORMATS; i2s->capture_dma_data.addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; @@ -445,14 +433,14 @@ static int spacemit_i2s_probe(struct platform_device *pdev) if (IS_ERR(i2s->bclk)) return dev_err_probe(i2s->dev, PTR_ERR(i2s->bclk), "failed to enable bit clock\n"); - clk = devm_clk_get_enabled(i2s->dev, "sspa_bus"); + clk = devm_clk_get_enabled(i2s->dev, "bus"); if (IS_ERR(clk)) - return dev_err_probe(i2s->dev, PTR_ERR(clk), "failed to enable sspa_bus clock\n"); + return dev_err_probe(i2s->dev, PTR_ERR(clk), "failed to enable bus clock\n"); - i2s->sspa_clk = devm_clk_get_enabled(i2s->dev, "sspa"); - if (IS_ERR(i2s->sspa_clk)) - return dev_err_probe(i2s->dev, PTR_ERR(i2s->sspa_clk), - "failed to enable sspa clock\n"); + i2s->func_clk = devm_clk_get_enabled(i2s->dev, "func"); + if (IS_ERR(i2s->func_clk)) + return dev_err_probe(i2s->dev, PTR_ERR(i2s->func_clk), + "failed to enable func clock\n"); i2s->sysclk_div = devm_clk_get_optional_enabled(i2s->dev, "sysclk_div"); if (IS_ERR(i2s->sysclk_div)) diff --git a/sound/soc/sprd/sprd-pcm-compress.c b/sound/soc/sprd/sprd-pcm-compress.c index a7d437b49fbf5..e5249924b54d9 100644 --- a/sound/soc/sprd/sprd-pcm-compress.c +++ b/sound/soc/sprd/sprd-pcm-compress.c @@ -17,7 +17,7 @@ /* Default values if userspace does not set */ #define SPRD_COMPR_MIN_FRAGMENT_SIZE SZ_8K -#define SPRD_COMPR_MAX_FRAGMENT_SIZE SZ_128K +#define SPRD_COMPR_MAX_FRAGMENT_SIZE SZ_32K #define SPRD_COMPR_MIN_NUM_FRAGMENTS 4 #define SPRD_COMPR_MAX_NUM_FRAGMENTS 64 @@ -271,6 +271,19 @@ static int sprd_platform_compr_set_params(struct snd_soc_component *component, struct sprd_compr_params compr_params = { }; int ret; + /* + * The stage 0 IRAM buffer and the stage 1 DDR buffer are allocated + * with fixed sizes at open time, so the requested fragment size and + * fragments must fit into them, otherwise sprd_platform_compr_copy() + * would overflow the buffers. Note the compress core only checks the + * fragment size and fragments against an u32 overflow, not against + * the buffer sizes advertised by get_caps. + */ + if (params->buffer.fragment_size > SPRD_COMPR_IRAM_BUF_SIZE || + (u64)params->buffer.fragment_size * params->buffer.fragments > + SPRD_COMPR_AREA_BUF_SIZE) + return -EINVAL; + /* * Configure the DMA engine 2-stage transfer mode. Channel 1 set as the * destination channel, and channel 0 set as the source channel, that diff --git a/sound/soc/sti/uniperif_reader.c b/sound/soc/sti/uniperif_reader.c index 45d7613f595cc..5347f9620f255 100644 --- a/sound/soc/sti/uniperif_reader.c +++ b/sound/soc/sti/uniperif_reader.c @@ -416,6 +416,8 @@ int uni_reader_init(struct platform_device *pdev, else reader->hw = &uni_reader_pcm_hw; + spin_lock_init(&reader->irq_lock); + ret = devm_request_irq(&pdev->dev, reader->irq, uni_reader_irq_handler, IRQF_SHARED, dev_name(&pdev->dev), reader); @@ -424,8 +426,6 @@ int uni_reader_init(struct platform_device *pdev, return -EBUSY; } - spin_lock_init(&reader->irq_lock); - return 0; } EXPORT_SYMBOL_GPL(uni_reader_init); diff --git a/sound/soc/tegra/tegra210_admaif.c b/sound/soc/tegra/tegra210_admaif.c index 7299c6bfcf158..87e898cc35e42 100644 --- a/sound/soc/tegra/tegra210_admaif.c +++ b/sound/soc/tegra/tegra210_admaif.c @@ -26,94 +26,162 @@ #define CH_RX_REG(reg, id) CH_REG(admaif->soc_data->rx_base, reg, id) -#define REG_DEFAULTS(id, rx_ctrl, tx_ctrl, tx_base, rx_base, cif_ctrl) \ +#define RX_REG_DEFAULTS(id, rx_ctrl, rx_base, cif_ctrl) \ { CH_REG(rx_base, TEGRA_ADMAIF_RX_INT_MASK, id), 0x00000001 }, \ { CH_REG(rx_base, TEGRA_ADMAIF_CH_ACIF_RX_CTRL, id), cif_ctrl }, \ - { CH_REG(rx_base, TEGRA_ADMAIF_RX_FIFO_CTRL, id), rx_ctrl }, \ + { CH_REG(rx_base, TEGRA_ADMAIF_RX_FIFO_CTRL, id), rx_ctrl } + +#define TX_REG_DEFAULTS(id, tx_ctrl, tx_base, cif_ctrl) \ { CH_REG(tx_base, TEGRA_ADMAIF_TX_INT_MASK, id), 0x00000001 }, \ { CH_REG(tx_base, TEGRA_ADMAIF_CH_ACIF_TX_CTRL, id), cif_ctrl }, \ { CH_REG(tx_base, TEGRA_ADMAIF_TX_FIFO_CTRL, id), tx_ctrl } -#define ADMAIF_REG_DEFAULTS(id, chip) \ - REG_DEFAULTS((id) - 1, \ +#define ADMAIF_RX_REG_DEFAULTS(id, chip) \ + RX_REG_DEFAULTS((id) - 1, \ chip ## _ADMAIF_RX ## id ## _FIFO_CTRL_REG_DEFAULT, \ + chip ## _ADMAIF_RX_BASE, \ + chip ## _ADMAIF_CIF_REG_DEFAULT) + +#define ADMAIF_TX_REG_DEFAULTS(id, chip) \ + TX_REG_DEFAULTS((id) - 1, \ chip ## _ADMAIF_TX ## id ## _FIFO_CTRL_REG_DEFAULT, \ chip ## _ADMAIF_TX_BASE, \ - chip ## _ADMAIF_RX_BASE, \ chip ## _ADMAIF_CIF_REG_DEFAULT) static const struct reg_default tegra186_admaif_reg_defaults[] = { - {(TEGRA_ADMAIF_GLOBAL_CG_0 + TEGRA186_ADMAIF_GLOBAL_BASE), 0x00000003}, - ADMAIF_REG_DEFAULTS(1, TEGRA186), - ADMAIF_REG_DEFAULTS(2, TEGRA186), - ADMAIF_REG_DEFAULTS(3, TEGRA186), - ADMAIF_REG_DEFAULTS(4, TEGRA186), - ADMAIF_REG_DEFAULTS(5, TEGRA186), - ADMAIF_REG_DEFAULTS(6, TEGRA186), - ADMAIF_REG_DEFAULTS(7, TEGRA186), - ADMAIF_REG_DEFAULTS(8, TEGRA186), - ADMAIF_REG_DEFAULTS(9, TEGRA186), - ADMAIF_REG_DEFAULTS(10, TEGRA186), - ADMAIF_REG_DEFAULTS(11, TEGRA186), - ADMAIF_REG_DEFAULTS(12, TEGRA186), - ADMAIF_REG_DEFAULTS(13, TEGRA186), - ADMAIF_REG_DEFAULTS(14, TEGRA186), - ADMAIF_REG_DEFAULTS(15, TEGRA186), - ADMAIF_REG_DEFAULTS(16, TEGRA186), - ADMAIF_REG_DEFAULTS(17, TEGRA186), - ADMAIF_REG_DEFAULTS(18, TEGRA186), - ADMAIF_REG_DEFAULTS(19, TEGRA186), - ADMAIF_REG_DEFAULTS(20, TEGRA186) + ADMAIF_RX_REG_DEFAULTS(1, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(2, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(3, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(4, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(5, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(6, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(7, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(8, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(9, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(10, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(11, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(12, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(13, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(14, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(15, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(16, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(17, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(18, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(19, TEGRA186), + ADMAIF_RX_REG_DEFAULTS(20, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(1, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(2, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(3, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(4, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(5, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(6, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(7, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(8, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(9, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(10, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(11, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(12, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(13, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(14, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(15, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(16, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(17, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(18, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(19, TEGRA186), + ADMAIF_TX_REG_DEFAULTS(20, TEGRA186), + {(TEGRA_ADMAIF_GLOBAL_CG_0 + TEGRA186_ADMAIF_GLOBAL_BASE), 0x00000003} }; static const struct reg_default tegra210_admaif_reg_defaults[] = { - {(TEGRA_ADMAIF_GLOBAL_CG_0 + TEGRA210_ADMAIF_GLOBAL_BASE), 0x00000003}, - ADMAIF_REG_DEFAULTS(1, TEGRA210), - ADMAIF_REG_DEFAULTS(2, TEGRA210), - ADMAIF_REG_DEFAULTS(3, TEGRA210), - ADMAIF_REG_DEFAULTS(4, TEGRA210), - ADMAIF_REG_DEFAULTS(5, TEGRA210), - ADMAIF_REG_DEFAULTS(6, TEGRA210), - ADMAIF_REG_DEFAULTS(7, TEGRA210), - ADMAIF_REG_DEFAULTS(8, TEGRA210), - ADMAIF_REG_DEFAULTS(9, TEGRA210), - ADMAIF_REG_DEFAULTS(10, TEGRA210) + ADMAIF_RX_REG_DEFAULTS(1, TEGRA210), + ADMAIF_RX_REG_DEFAULTS(2, TEGRA210), + ADMAIF_RX_REG_DEFAULTS(3, TEGRA210), + ADMAIF_RX_REG_DEFAULTS(4, TEGRA210), + ADMAIF_RX_REG_DEFAULTS(5, TEGRA210), + ADMAIF_RX_REG_DEFAULTS(6, TEGRA210), + ADMAIF_RX_REG_DEFAULTS(7, TEGRA210), + ADMAIF_RX_REG_DEFAULTS(8, TEGRA210), + ADMAIF_RX_REG_DEFAULTS(9, TEGRA210), + ADMAIF_RX_REG_DEFAULTS(10, TEGRA210), + ADMAIF_TX_REG_DEFAULTS(1, TEGRA210), + ADMAIF_TX_REG_DEFAULTS(2, TEGRA210), + ADMAIF_TX_REG_DEFAULTS(3, TEGRA210), + ADMAIF_TX_REG_DEFAULTS(4, TEGRA210), + ADMAIF_TX_REG_DEFAULTS(5, TEGRA210), + ADMAIF_TX_REG_DEFAULTS(6, TEGRA210), + ADMAIF_TX_REG_DEFAULTS(7, TEGRA210), + ADMAIF_TX_REG_DEFAULTS(8, TEGRA210), + ADMAIF_TX_REG_DEFAULTS(9, TEGRA210), + ADMAIF_TX_REG_DEFAULTS(10, TEGRA210), + {(TEGRA_ADMAIF_GLOBAL_CG_0 + TEGRA210_ADMAIF_GLOBAL_BASE), 0x00000003} }; static const struct reg_default tegra264_admaif_reg_defaults[] = { - {(TEGRA_ADMAIF_GLOBAL_CG_0 + TEGRA264_ADMAIF_GLOBAL_BASE), 0x00000003}, - ADMAIF_REG_DEFAULTS(1, TEGRA264), - ADMAIF_REG_DEFAULTS(2, TEGRA264), - ADMAIF_REG_DEFAULTS(3, TEGRA264), - ADMAIF_REG_DEFAULTS(4, TEGRA264), - ADMAIF_REG_DEFAULTS(5, TEGRA264), - ADMAIF_REG_DEFAULTS(6, TEGRA264), - ADMAIF_REG_DEFAULTS(7, TEGRA264), - ADMAIF_REG_DEFAULTS(8, TEGRA264), - ADMAIF_REG_DEFAULTS(9, TEGRA264), - ADMAIF_REG_DEFAULTS(10, TEGRA264), - ADMAIF_REG_DEFAULTS(11, TEGRA264), - ADMAIF_REG_DEFAULTS(12, TEGRA264), - ADMAIF_REG_DEFAULTS(13, TEGRA264), - ADMAIF_REG_DEFAULTS(14, TEGRA264), - ADMAIF_REG_DEFAULTS(15, TEGRA264), - ADMAIF_REG_DEFAULTS(16, TEGRA264), - ADMAIF_REG_DEFAULTS(17, TEGRA264), - ADMAIF_REG_DEFAULTS(18, TEGRA264), - ADMAIF_REG_DEFAULTS(19, TEGRA264), - ADMAIF_REG_DEFAULTS(20, TEGRA264), - ADMAIF_REG_DEFAULTS(21, TEGRA264), - ADMAIF_REG_DEFAULTS(22, TEGRA264), - ADMAIF_REG_DEFAULTS(23, TEGRA264), - ADMAIF_REG_DEFAULTS(24, TEGRA264), - ADMAIF_REG_DEFAULTS(25, TEGRA264), - ADMAIF_REG_DEFAULTS(26, TEGRA264), - ADMAIF_REG_DEFAULTS(27, TEGRA264), - ADMAIF_REG_DEFAULTS(28, TEGRA264), - ADMAIF_REG_DEFAULTS(29, TEGRA264), - ADMAIF_REG_DEFAULTS(30, TEGRA264), - ADMAIF_REG_DEFAULTS(31, TEGRA264), - ADMAIF_REG_DEFAULTS(32, TEGRA264) + ADMAIF_RX_REG_DEFAULTS(1, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(2, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(3, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(4, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(5, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(6, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(7, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(8, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(9, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(10, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(11, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(12, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(13, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(14, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(15, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(16, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(17, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(18, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(19, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(20, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(21, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(22, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(23, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(24, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(25, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(26, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(27, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(28, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(29, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(30, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(31, TEGRA264), + ADMAIF_RX_REG_DEFAULTS(32, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(1, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(2, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(3, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(4, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(5, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(6, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(7, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(8, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(9, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(10, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(11, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(12, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(13, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(14, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(15, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(16, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(17, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(18, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(19, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(20, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(21, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(22, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(23, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(24, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(25, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(26, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(27, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(28, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(29, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(30, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(31, TEGRA264), + ADMAIF_TX_REG_DEFAULTS(32, TEGRA264), + {(TEGRA_ADMAIF_GLOBAL_CG_0 + TEGRA264_ADMAIF_GLOBAL_BASE), 0x00000003} }; static bool tegra_admaif_wr_reg(struct device *dev, unsigned int reg) diff --git a/sound/soc/tegra/tegra210_i2s.c b/sound/soc/tegra/tegra210_i2s.c index ff8c72fc38c5c..fdcc5d62b720c 100644 --- a/sound/soc/tegra/tegra210_i2s.c +++ b/sound/soc/tegra/tegra210_i2s.c @@ -23,9 +23,9 @@ static const struct reg_default tegra210_i2s_reg_defaults[] = { { TEGRA210_I2S_RX_CIF_CTRL, 0x00007700 }, { TEGRA210_I2S_TX_INT_MASK, 0x00000003 }, { TEGRA210_I2S_TX_CIF_CTRL, 0x00007700 }, + { TEGRA210_I2S_ENABLE, 0x0 }, { TEGRA210_I2S_CG, 0x1 }, { TEGRA210_I2S_TIMING, 0x0000001f }, - { TEGRA210_I2S_ENABLE, 0x1 }, /* * Below update does not have any effect on Tegra186 and Tegra194. * On Tegra210, I2S4 has "i2s4a" and "i2s4b" pins and below update @@ -38,13 +38,13 @@ static const struct reg_default tegra210_i2s_reg_defaults[] = { static const struct reg_default tegra264_i2s_reg_defaults[] = { { TEGRA210_I2S_RX_INT_MASK, 0x00000003 }, { TEGRA210_I2S_RX_CIF_CTRL, 0x00003f00 }, + { TEGRA264_I2S_RX_FIFO_WR_ACCESS_MODE, 0x1 }, { TEGRA264_I2S_TX_INT_MASK, 0x00000003 }, { TEGRA264_I2S_TX_CIF_CTRL, 0x00003f00 }, + { TEGRA264_I2S_TX_FIFO_RD_ACCESS_MODE, 0x1 }, + { TEGRA264_I2S_ENABLE, 0x0 }, { TEGRA264_I2S_CG, 0x1 }, { TEGRA264_I2S_TIMING, 0x0000001f }, - { TEGRA264_I2S_ENABLE, 0x1 }, - { TEGRA264_I2S_RX_FIFO_WR_ACCESS_MODE, 0x1 }, - { TEGRA264_I2S_TX_FIFO_RD_ACCESS_MODE, 0x1 }, }; static void tegra210_i2s_set_slot_ctrl(struct tegra210_i2s *i2s, @@ -201,9 +201,21 @@ static int tegra210_i2s_runtime_resume(struct device *dev) } regcache_cache_only(i2s->regmap, false); - regcache_sync(i2s->regmap); + err = regcache_sync(i2s->regmap); + if (err) + goto err; + + err = regmap_write(i2s->regmap, i2s->soc_data->enable_reg, I2S_EN); + if (err) + goto err; return 0; + +err: + regcache_cache_only(i2s->regmap, true); + clk_disable_unprepare(i2s->clk_i2s); + + return err; } static void tegra210_i2s_set_data_offset(struct tegra210_i2s *i2s, @@ -1133,6 +1145,7 @@ static const struct tegra_i2s_soc_data soc_data_tegra210 = { .regmap_conf = &tegra210_regmap_conf, .i2s_cmpnt = &tegra210_i2s_cmpnt, .max_ch = TEGRA210_I2S_MAX_CHANNEL, + .enable_reg = TEGRA210_I2S_ENABLE, .tx_offset = TEGRA210_I2S_TX_OFFSET, .i2s_ctrl_offset = TEGRA210_I2S_CTRL_OFFSET, .fsync_width_mask = I2S_CTRL_FSYNC_WIDTH_MASK, @@ -1144,6 +1157,7 @@ static const struct tegra_i2s_soc_data soc_data_tegra264 = { .regmap_conf = &tegra264_regmap_conf, .i2s_cmpnt = &tegra264_i2s_cmpnt, .max_ch = TEGRA264_I2S_MAX_CHANNEL, + .enable_reg = TEGRA264_I2S_ENABLE, .tx_offset = TEGRA264_I2S_TX_OFFSET, .i2s_ctrl_offset = TEGRA264_I2S_CTRL_OFFSET, .fsync_width_mask = TEGRA264_I2S_CTRL_FSYNC_WIDTH_MASK, diff --git a/sound/soc/tegra/tegra210_i2s.h b/sound/soc/tegra/tegra210_i2s.h index 42be2137342c4..82292f96ab367 100644 --- a/sound/soc/tegra/tegra210_i2s.h +++ b/sound/soc/tegra/tegra210_i2s.h @@ -150,6 +150,7 @@ struct tegra_i2s_soc_data { const struct regmap_config *regmap_conf; const struct snd_soc_component_driver *i2s_cmpnt; unsigned int max_ch; + unsigned int enable_reg; unsigned int tx_offset; unsigned int i2s_ctrl_offset; unsigned int fsync_width_mask; diff --git a/sound/soc/tegra/tegra210_mbdrc.c b/sound/soc/tegra/tegra210_mbdrc.c index f5d4a93dd6f16..1d78b4c1cc885 100644 --- a/sound/soc/tegra/tegra210_mbdrc.c +++ b/sound/soc/tegra/tegra210_mbdrc.c @@ -20,33 +20,33 @@ #define MBDRC_FILTER_REG(reg, id) \ ((reg) + ((id) * TEGRA210_MBDRC_FILTER_PARAM_STRIDE)) -#define MBDRC_FILTER_REG_DEFAULTS(id) \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_IIR_CFG, id), 0x00000005}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_IN_ATTACK, id), 0x3e48590c}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_IN_RELEASE, id), 0x08414e9f}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_FAST_ATTACK, id), 0x7fffffff}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_IN_THRESHOLD, id), 0x06145082}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_OUT_THRESHOLD, id), 0x060d379b}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_RATIO_1ST, id), 0x0000a000}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_RATIO_2ND, id), 0x00002000}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_RATIO_3RD, id), 0x00000b33}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_RATIO_4TH, id), 0x00000800}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_RATIO_5TH, id), 0x0000019a}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_MAKEUP_GAIN, id), 0x00000002}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_INIT_GAIN, id), 0x00066666}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_GAIN_ATTACK, id), 0x00d9ba0e}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_GAIN_RELEASE, id), 0x3e48590c}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_FAST_RELEASE, id), 0x7ffff26a}, \ - { MBDRC_FILTER_REG(TEGRA210_MBDRC_CFG_RAM_CTRL, id), 0x4000} +#define MBDRC_FILTER_PARAM_REG_DEFAULTS(reg, val) \ + { MBDRC_FILTER_REG(reg, 0), val}, \ + { MBDRC_FILTER_REG(reg, 1), val}, \ + { MBDRC_FILTER_REG(reg, 2), val} static const struct reg_default tegra210_mbdrc_reg_defaults[] = { { TEGRA210_MBDRC_CFG, 0x0030de51}, { TEGRA210_MBDRC_CHANNEL_MASK, 0x00000003}, { TEGRA210_MBDRC_FAST_FACTOR, 0x30000800}, - MBDRC_FILTER_REG_DEFAULTS(0), - MBDRC_FILTER_REG_DEFAULTS(1), - MBDRC_FILTER_REG_DEFAULTS(2), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_IIR_CFG, 0x00000005), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_IN_ATTACK, 0x3e48590c), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_IN_RELEASE, 0x08414e9f), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_FAST_ATTACK, 0x7fffffff), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_IN_THRESHOLD, 0x06145082), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_OUT_THRESHOLD, 0x060d379b), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_RATIO_1ST, 0x0000a000), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_RATIO_2ND, 0x00002000), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_RATIO_3RD, 0x00000b33), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_RATIO_4TH, 0x00000800), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_RATIO_5TH, 0x0000019a), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_MAKEUP_GAIN, 0x00000002), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_INIT_GAIN, 0x00066666), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_GAIN_ATTACK, 0x00d9ba0e), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_GAIN_RELEASE, 0x3e48590c), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_FAST_RELEASE, 0x7ffff26a), + MBDRC_FILTER_PARAM_REG_DEFAULTS(TEGRA210_MBDRC_CFG_RAM_CTRL, 0x4000), }; /* Default MBDRC parameters */ diff --git a/sound/soc/tegra/tegra210_mixer.c b/sound/soc/tegra/tegra210_mixer.c index a69774578d69f..96259a6cbc59b 100644 --- a/sound/soc/tegra/tegra210_mixer.c +++ b/sound/soc/tegra/tegra210_mixer.c @@ -57,10 +57,10 @@ static const struct reg_default tegra210_mixer_reg_defaults[] = { MIXER_TX_REG_DEFAULTS(3), MIXER_TX_REG_DEFAULTS(4), + { TEGRA210_MIXER_ENABLE, 0x0 }, { TEGRA210_MIXER_CG, 0x00000001}, { TEGRA210_MIXER_GAIN_CFG_RAM_CTRL, 0x00004000}, { TEGRA210_MIXER_PEAKM_RAM_CTRL, 0x00004000}, - { TEGRA210_MIXER_ENABLE, 0x1 }, }; /* Default gain parameters */ @@ -86,11 +86,15 @@ static int tegra210_mixer_runtime_suspend(struct device *dev) static int tegra210_mixer_runtime_resume(struct device *dev) { struct tegra210_mixer *mixer = dev_get_drvdata(dev); + int err; regcache_cache_only(mixer->regmap, false); - regcache_sync(mixer->regmap); + err = regcache_sync(mixer->regmap); + if (err) + return err; - return 0; + return regmap_write(mixer->regmap, TEGRA210_MIXER_ENABLE, + TEGRA210_MIXER_EN); } static int tegra210_mixer_write_ram(struct tegra210_mixer *mixer, diff --git a/sound/soc/ti/j721e-evm.c b/sound/soc/ti/j721e-evm.c index c214ae0d7b95e..312298e0b0049 100644 --- a/sound/soc/ti/j721e-evm.c +++ b/sound/soc/ti/j721e-evm.c @@ -4,6 +4,7 @@ * Author: Peter Ujfalusi */ +#include #include #include #include @@ -263,7 +264,7 @@ static int j721e_audio_startup(struct snd_pcm_substream *substream) int ret = 0; int i; - guard(mutex)(&priv->mutex); + mutex_lock(&priv->mutex); domain->active++; @@ -303,6 +304,7 @@ static int j721e_audio_startup(struct snd_pcm_substream *substream) out: if (ret) domain->active--; + mutex_unlock(&priv->mutex); return ret; } diff --git a/sound/soc/ux500/ux500_msp_dai.c b/sound/soc/ux500/ux500_msp_dai.c index 7798957c6504d..37c48cc70394e 100644 --- a/sound/soc/ux500/ux500_msp_dai.c +++ b/sound/soc/ux500/ux500_msp_dai.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include @@ -34,8 +34,10 @@ static int setup_pcm_multichan(struct snd_soc_dai *dai, if (drvdata->slots > 1) { msp_config->multichannel_configured = 1; - multi->tx_multichannel_enable = true; - multi->rx_multichannel_enable = true; + multi->tx_multichannel_enable = + msp_config->direction & MSP_DIR_TX; + multi->rx_multichannel_enable = + msp_config->direction & MSP_DIR_RX; multi->rx_comparison_enable_mode = MSP_COMPARISON_DISABLED; multi->tx_channel_0_enable = drvdata->tx_mask; @@ -57,72 +59,21 @@ static int setup_pcm_multichan(struct snd_soc_dai *dai, return 0; } -static int setup_frameper(struct snd_soc_dai *dai, unsigned int rate, - struct msp_protdesc *prot_desc) +static void setup_frameper(struct snd_soc_dai *dai, + struct msp_protdesc *prot_desc) { struct ux500_msp_i2s_drvdata *drvdata = dev_get_drvdata(dai->dev); - switch (drvdata->slots) { - case 1: - switch (rate) { - case 8000: - prot_desc->frame_period = - FRAME_PER_SINGLE_SLOT_8_KHZ; - break; - - case 16000: - prot_desc->frame_period = - FRAME_PER_SINGLE_SLOT_16_KHZ; - break; - - case 44100: - prot_desc->frame_period = - FRAME_PER_SINGLE_SLOT_44_1_KHZ; - break; - - case 48000: - prot_desc->frame_period = - FRAME_PER_SINGLE_SLOT_48_KHZ; - break; - - default: - dev_err(dai->dev, - "%s: Error: Unsupported sample-rate (freq = %d)!\n", - __func__, rate); - return -EINVAL; - } - break; - - case 2: - prot_desc->frame_period = FRAME_PER_2_SLOTS; - break; - - case 8: - prot_desc->frame_period = FRAME_PER_8_SLOTS; - break; - - case 16: - prot_desc->frame_period = FRAME_PER_16_SLOTS; - break; - default: - dev_err(dai->dev, - "%s: Error: Unsupported slot-count (slots = %d)!\n", - __func__, drvdata->slots); - return -EINVAL; - } - - prot_desc->clocks_per_frame = - prot_desc->frame_period+1; + prot_desc->clocks_per_frame = drvdata->slots * drvdata->slot_width; + prot_desc->frame_period = prot_desc->clocks_per_frame - 1; dev_dbg(dai->dev, "%s: Clocks per frame: %u\n", __func__, prot_desc->clocks_per_frame); - - return 0; } -static int setup_pcm_framing(struct snd_soc_dai *dai, unsigned int rate, - struct msp_protdesc *prot_desc) +static int setup_pcm_framing(struct snd_soc_dai *dai, + struct msp_protdesc *prot_desc) { struct ux500_msp_i2s_drvdata *drvdata = dev_get_drvdata(dai->dev); @@ -163,7 +114,9 @@ static int setup_pcm_framing(struct snd_soc_dai *dai, unsigned int rate, prot_desc->tx_elem_len_2 = MSP_ELEM_LEN_16; prot_desc->rx_elem_len_2 = MSP_ELEM_LEN_16; - return setup_frameper(dai, rate, prot_desc); + setup_frameper(dai, prot_desc); + + return 0; } static int setup_clocking(struct snd_soc_dai *dai, @@ -177,7 +130,16 @@ static int setup_clocking(struct snd_soc_dai *dai, case SND_SOC_DAIFMT_NB_IF: msp_config->tx_fsync_pol ^= 1 << TFSPOL_SHIFT; msp_config->rx_fsync_pol ^= 1 << RFSPOL_SHIFT; + break; + + case SND_SOC_DAIFMT_IB_NF: + msp_config->bclk_inverted = true; + break; + case SND_SOC_DAIFMT_IB_IF: + msp_config->bclk_inverted = true; + msp_config->tx_fsync_pol ^= 1 << TFSPOL_SHIFT; + msp_config->rx_fsync_pol ^= 1 << RFSPOL_SHIFT; break; default: @@ -192,6 +154,7 @@ static int setup_clocking(struct snd_soc_dai *dai, case SND_SOC_DAIFMT_BC_FC: dev_dbg(dai->dev, "%s: Codec is master.\n", __func__); + msp_config->clock_provider = false; msp_config->iodelay = 0x20; msp_config->rx_fsync_sel = 0; msp_config->tx_fsync_sel = 1 << TFSSEL_SHIFT; @@ -204,6 +167,7 @@ static int setup_clocking(struct snd_soc_dai *dai, case SND_SOC_DAIFMT_BP_FP: dev_dbg(dai->dev, "%s: Codec is slave.\n", __func__); + msp_config->clock_provider = true; msp_config->tx_clk_sel = TX_CLK_SEL_SRG; msp_config->tx_fsync_sel = TX_SYNC_SRG_PROG; msp_config->rx_clk_sel = RX_CLK_SEL_SRG; @@ -362,7 +326,7 @@ static int setup_msp_config(struct snd_pcm_substream *substream, if (ret < 0) return ret; - ret = setup_pcm_framing(dai, runtime->rate, prot_desc); + ret = setup_pcm_framing(dai, prot_desc); if (ret < 0) return ret; @@ -424,21 +388,21 @@ static void ux500_msp_dai_shutdown(struct snd_pcm_substream *substream, int ret; struct ux500_msp_i2s_drvdata *drvdata = dev_get_drvdata(dai->dev); bool is_playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK); + unsigned int configured = is_playback ? PLAYBACK_CONFIGURED : + CAPTURE_CONFIGURED; + unsigned int dir = is_playback ? MSP_DIR_TX : MSP_DIR_RX; dev_dbg(dai->dev, "%s: MSP %d (%s): Enter.\n", __func__, dai->id, snd_pcm_stream_str(substream)); - if (drvdata->vape_opp_constraint == 1) { - prcmu_qos_update_requirement(PRCMU_QOS_APE_OPP, - "ux500_msp_i2s", 50); - drvdata->vape_opp_constraint = 0; - } - - if (ux500_msp_i2s_close(drvdata->msp, - is_playback ? MSP_DIR_TX : MSP_DIR_RX)) { - dev_err(dai->dev, - "%s: Error: MSP %d (%s): Unable to close i2s.\n", - __func__, dai->id, snd_pcm_stream_str(substream)); + if (drvdata->configured & configured) { + if (ux500_msp_i2s_close(drvdata->msp, dir)) { + dev_err(dai->dev, + "%s: Error: MSP %d (%s): Unable to close i2s.\n", + __func__, dai->id, + snd_pcm_stream_str(substream)); + } + drvdata->configured &= ~configured; } /* Disable and unprepare clocks */ @@ -456,15 +420,23 @@ static void ux500_msp_dai_shutdown(struct snd_pcm_substream *substream, static int ux500_msp_dai_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { - int ret = 0; struct ux500_msp_i2s_drvdata *drvdata = dev_get_drvdata(dai->dev); struct snd_pcm_runtime *runtime = substream->runtime; struct ux500_msp_config msp_config; + bool is_playback = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; + unsigned int configured = is_playback ? PLAYBACK_CONFIGURED : + CAPTURE_CONFIGURED; + int ret; dev_dbg(dai->dev, "%s: MSP %d (%s): Enter (rate = %d).\n", __func__, dai->id, snd_pcm_stream_str(substream), runtime->rate); - setup_msp_config(substream, dai, &msp_config); + if (drvdata->configured & configured) + return 0; + + ret = setup_msp_config(substream, dai, &msp_config); + if (ret) + return ret; ret = ux500_msp_i2s_open(drvdata->msp, &msp_config); if (ret < 0) { @@ -473,22 +445,9 @@ static int ux500_msp_dai_prepare(struct snd_pcm_substream *substream, return ret; } - /* Set OPP-level */ - if ((drvdata->fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) && - (drvdata->msp->f_bitclk > 19200000)) { - /* If the bit-clock is higher than 19.2MHz, Vape should be - * run in 100% OPP. Only when bit-clock is used (MSP master) - */ - prcmu_qos_update_requirement(PRCMU_QOS_APE_OPP, - "ux500-msp-i2s", 100); - drvdata->vape_opp_constraint = 1; - } else { - prcmu_qos_update_requirement(PRCMU_QOS_APE_OPP, - "ux500-msp-i2s", 50); - drvdata->vape_opp_constraint = 0; - } + drvdata->configured |= configured; - return ret; + return 0; } static int ux500_msp_dai_hw_params(struct snd_pcm_substream *substream, @@ -496,7 +455,6 @@ static int ux500_msp_dai_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { unsigned int mask, slots_active; - struct snd_pcm_runtime *runtime = substream->runtime; struct ux500_msp_i2s_drvdata *drvdata = dev_get_drvdata(dai->dev); dev_dbg(dai->dev, "%s: MSP %d (%s): Enter.\n", @@ -504,9 +462,8 @@ static int ux500_msp_dai_hw_params(struct snd_pcm_substream *substream, switch (drvdata->fmt & SND_SOC_DAIFMT_FORMAT_MASK) { case SND_SOC_DAIFMT_I2S: - snd_pcm_hw_constraint_minmax(runtime, - SNDRV_PCM_HW_PARAM_CHANNELS, - 1, 2); + if (params_channels(params) < 1 || params_channels(params) > 2) + return -EINVAL; break; case SND_SOC_DAIFMT_DSP_B: @@ -518,9 +475,8 @@ static int ux500_msp_dai_hw_params(struct snd_pcm_substream *substream, slots_active = hweight32(mask); dev_dbg(dai->dev, "TDM-slots active: %d", slots_active); - snd_pcm_hw_constraint_single(runtime, - SNDRV_PCM_HW_PARAM_CHANNELS, - slots_active); + if (!slots_active || params_channels(params) != slots_active) + return -EINVAL; break; default: @@ -553,20 +509,21 @@ static int ux500_msp_dai_set_dai_fmt(struct snd_soc_dai *dai, default: dev_err(dai->dev, "%s: Error: Unsupported protocol/master (fmt = 0x%x)!\n", - __func__, drvdata->fmt); + __func__, fmt); return -EINVAL; } switch (fmt & SND_SOC_DAIFMT_INV_MASK) { case SND_SOC_DAIFMT_NB_NF: case SND_SOC_DAIFMT_NB_IF: + case SND_SOC_DAIFMT_IB_NF: case SND_SOC_DAIFMT_IB_IF: break; default: dev_err(dai->dev, "%s: Error: Unsupported inversion (fmt = 0x%x)!\n", - __func__, drvdata->fmt); + __func__, fmt); return -EINVAL; } @@ -600,17 +557,23 @@ static int ux500_msp_dai_set_tdm_slot(struct snd_soc_dai *dai, __func__, slots); return -EINVAL; } - drvdata->slots = slots; - if (!(slot_width == 16)) { + if (slot_width != 16) { dev_err(dai->dev, "%s: Error: Unsupported slot-width (%d)!\n", __func__, slot_width); return -EINVAL; } - drvdata->slot_width = slot_width; - drvdata->tx_mask = tx_mask & cap; - drvdata->rx_mask = rx_mask & cap; + if ((tx_mask | rx_mask) & ~cap) { + dev_err(dai->dev, "%s: Slot mask exceeds %d slots\n", + __func__, slots); + return -EINVAL; + } + + drvdata->slots = slots; + drvdata->slot_width = slot_width; + drvdata->tx_mask = tx_mask; + drvdata->rx_mask = rx_mask; return 0; } @@ -716,6 +679,7 @@ static const struct snd_soc_component_driver ux500_msp_component = { static int ux500_msp_drv_probe(struct platform_device *pdev) { struct ux500_msp_i2s_drvdata *drvdata; + struct reset_control *reset; int ret = 0; drvdata = devm_kzalloc(&pdev->dev, @@ -729,7 +693,6 @@ static int ux500_msp_drv_probe(struct platform_device *pdev) drvdata->tx_mask = 0x01; drvdata->rx_mask = 0x01; drvdata->slot_width = 16; - drvdata->master_clk = MSP_INPUT_FREQ_APB; drvdata->reg_vape = devm_regulator_get(&pdev->dev, "v-ape"); if (IS_ERR(drvdata->reg_vape)) { @@ -739,8 +702,6 @@ static int ux500_msp_drv_probe(struct platform_device *pdev) __func__, ret); return ret; } - prcmu_qos_add_requirement(PRCMU_QOS_APE_OPP, (char *)pdev->name, 50); - drvdata->pclk = devm_clk_get(&pdev->dev, "apb_pclk"); if (IS_ERR(drvdata->pclk)) { ret = PTR_ERR(drvdata->pclk); @@ -758,9 +719,19 @@ static int ux500_msp_drv_probe(struct platform_device *pdev) __func__, ret); return ret; } + drvdata->master_clk = clk_get_rate(drvdata->clk); + if (!drvdata->master_clk) { + dev_err(&pdev->dev, "MSP clock has no rate\n"); + return -EINVAL; + } + + reset = devm_reset_control_get_exclusive_deasserted(&pdev->dev, NULL); + if (IS_ERR(reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(reset), + "Failed to deassert MSP reset\n"); ret = ux500_msp_i2s_init_msp(pdev, &drvdata->msp); - if (!drvdata->msp) { + if (ret) { dev_err(&pdev->dev, "%s: ERROR: Failed to init MSP-struct (%d)!", __func__, ret); @@ -799,8 +770,6 @@ static void ux500_msp_drv_remove(struct platform_device *pdev) snd_soc_unregister_component(&pdev->dev); - prcmu_qos_remove_requirement(PRCMU_QOS_APE_OPP, "ux500_msp_i2s"); - ux500_msp_i2s_cleanup_msp(pdev, drvdata->msp); } diff --git a/sound/soc/ux500/ux500_msp_dai.h b/sound/soc/ux500/ux500_msp_dai.h index 30bf708381961..aae582030d959 100644 --- a/sound/soc/ux500/ux500_msp_dai.h +++ b/sound/soc/ux500/ux500_msp_dai.h @@ -22,17 +22,6 @@ #define UX500_I2S_FORMATS (SNDRV_PCM_FMTBIT_S16_LE) -#define FRAME_PER_SINGLE_SLOT_8_KHZ 31 -#define FRAME_PER_SINGLE_SLOT_16_KHZ 124 -#define FRAME_PER_SINGLE_SLOT_44_1_KHZ 63 -#define FRAME_PER_SINGLE_SLOT_48_KHZ 49 -#define FRAME_PER_2_SLOTS 31 -#define FRAME_PER_8_SLOTS 138 -#define FRAME_PER_16_SLOTS 277 - -#define UX500_MSP_INTERNAL_CLOCK_FREQ 40000000 -#define UX500_MSP1_INTERNAL_CLOCK_FREQ UX500_MSP_INTERNAL_CLOCK_FREQ - #define UX500_MSP_MIN_CHANNELS 1 #define UX500_MSP_MAX_CHANNELS 8 @@ -47,6 +36,7 @@ struct ux500_msp_i2s_drvdata { struct ux500_msp *msp; struct regulator *reg_vape; unsigned int fmt; + unsigned int configured; unsigned int tx_mask; unsigned int rx_mask; int slots; @@ -57,8 +47,6 @@ struct ux500_msp_i2s_drvdata { struct clk *clk; struct clk *pclk; - /* Regulators */ - int vape_opp_constraint; }; int ux500_msp_dai_set_data_delay(struct snd_soc_dai *dai, int delay); diff --git a/sound/soc/ux500/ux500_msp_i2s.c b/sound/soc/ux500/ux500_msp_i2s.c index fbfeefa418ca7..683b485fb5708 100644 --- a/sound/soc/ux500/ux500_msp_i2s.c +++ b/sound/soc/ux500/ux500_msp_i2s.c @@ -201,10 +201,12 @@ static int configure_protocol(struct ux500_msp *msp, /* The code below should not be separated. */ temp_reg = readl(msp->registers + MSP_GCR) & ~TX_CLK_POL_RISING; - temp_reg |= MSP_TX_CLKPOL_BIT(~protdesc->tx_clk_pol); + temp_reg |= MSP_TX_CLKPOL_BIT(!protdesc->tx_clk_pol ^ + config->bclk_inverted); writel(temp_reg, msp->registers + MSP_GCR); temp_reg = readl(msp->registers + MSP_GCR) & ~RX_CLK_POL_RISING; - temp_reg |= MSP_RX_CLKPOL_BIT(protdesc->rx_clk_pol); + temp_reg |= MSP_RX_CLKPOL_BIT(protdesc->rx_clk_pol ^ + config->bclk_inverted); writel(temp_reg, msp->registers + MSP_GCR); return 0; @@ -212,35 +214,20 @@ static int configure_protocol(struct ux500_msp *msp, static int setup_bitclk(struct ux500_msp *msp, struct ux500_msp_config *config) { + struct msp_protdesc *protdesc; + u64 desired_bitclk; + unsigned int bitclk; u32 reg_val_GCR; - u32 frame_per = 0; - u32 sck_div = 0; - u32 frame_width = 0; - u32 temp_reg = 0; - struct msp_protdesc *protdesc = NULL; + u32 sck_div; + u32 temp_reg; reg_val_GCR = readl(msp->registers + MSP_GCR); writel(reg_val_GCR & ~SRG_ENABLE, msp->registers + MSP_GCR); - if (config->default_protdesc) - protdesc = - (struct msp_protdesc *)&prot_descs[config->protocol]; - else - protdesc = (struct msp_protdesc *)&config->protdesc; - switch (config->protocol) { case MSP_PCM_PROTOCOL: case MSP_PCM_COMPAND_PROTOCOL: - frame_width = protdesc->frame_width; - sck_div = config->f_inputclk / (config->frame_freq * - (protdesc->clocks_per_frame)); - frame_per = protdesc->frame_period; - break; case MSP_I2S_PROTOCOL: - frame_width = protdesc->frame_width; - sck_div = config->f_inputclk / (config->frame_freq * - (protdesc->clocks_per_frame)); - frame_per = protdesc->frame_period; break; default: dev_err(msp->dev, "%s: ERROR: Unknown protocol (%d)!\n", @@ -249,12 +236,35 @@ static int setup_bitclk(struct ux500_msp *msp, struct ux500_msp_config *config) return -EINVAL; } + if (config->default_protdesc) + protdesc = (struct msp_protdesc *)&prot_descs[config->protocol]; + else + protdesc = &config->protdesc; + + if (!config->frame_freq || !protdesc->clocks_per_frame) + return -EINVAL; + + desired_bitclk = (u64)config->frame_freq * protdesc->clocks_per_frame; + if (desired_bitclk > config->f_inputclk) + return -EINVAL; + bitclk = desired_bitclk; + if (config->f_inputclk % bitclk) { + dev_err(msp->dev, + "Input clock %u cannot generate bit clock %u\n", + config->f_inputclk, bitclk); + return -EINVAL; + } + + sck_div = config->f_inputclk / bitclk; + if (!sck_div || sck_div > SCK_DIV_MASK + 1) + return -EINVAL; + temp_reg = (sck_div - 1) & SCK_DIV_MASK; - temp_reg |= FRAME_WIDTH_BITS(frame_width); - temp_reg |= FRAME_PERIOD_BITS(frame_per); + temp_reg |= FRAME_WIDTH_BITS(protdesc->frame_width); + temp_reg |= FRAME_PERIOD_BITS(protdesc->frame_period); writel(temp_reg, msp->registers + MSP_SRG); - msp->f_bitclk = (config->f_inputclk)/(sck_div + 1); + msp->f_bitclk = config->f_inputclk / sck_div; /* Enable bit-clock */ udelay(100); @@ -344,20 +354,27 @@ static int configure_multichannel(struct ux500_msp *msp, return 0; } -static int enable_msp(struct ux500_msp *msp, struct ux500_msp_config *config) +static int enable_msp(struct ux500_msp *msp, struct ux500_msp_config *config, + bool first) { - int status = 0; - u32 reg_val_DMACR, reg_val_GCR; + int status; + u32 reg_val_DMACR; /* Configure msp with protocol dependent settings */ - configure_protocol(msp, config); - setup_bitclk(msp, config); + status = configure_protocol(msp, config); + if (status) + return status; + + if (first && config->clock_provider) { + status = setup_bitclk(msp, config); + if (status) + return status; + } + if (config->multichannel_configured == 1) { status = configure_multichannel(msp, config); if (status) - dev_warn(msp->dev, - "%s: WARN: configure_multichannel failed (%d)!\n", - __func__, status); + return status; } reg_val_DMACR = readl(msp->registers + MSP_DMACR); @@ -369,11 +386,7 @@ static int enable_msp(struct ux500_msp *msp, struct ux500_msp_config *config) writel(config->iodelay, msp->registers + MSP_IODLY); - /* Enable frame generation logic */ - reg_val_GCR = readl(msp->registers + MSP_GCR); - writel(reg_val_GCR | FRAME_GEN_ENABLE, msp->registers + MSP_GCR); - - return status; + return 0; } static void flush_fifo_rx(struct ux500_msp *msp) @@ -411,12 +424,37 @@ static void flush_fifo_tx(struct ux500_msp *msp) writel(reg_val_GCR, msp->registers + MSP_GCR); } +static bool ux500_msp_config_compatible(struct ux500_msp *msp, + struct ux500_msp_config *config) +{ + struct ux500_msp_config *active = &msp->config; + + return active->f_inputclk == config->f_inputclk && + active->tx_clk_sel == config->tx_clk_sel && + active->rx_clk_sel == config->rx_clk_sel && + active->srg_clk_sel == config->srg_clk_sel && + active->rx_fsync_pol == config->rx_fsync_pol && + active->tx_fsync_pol == config->tx_fsync_pol && + active->rx_fsync_sel == config->rx_fsync_sel && + active->tx_fsync_sel == config->tx_fsync_sel && + active->default_protdesc == config->default_protdesc && + active->protocol == config->protocol && + active->frame_freq == config->frame_freq && + active->data_size == config->data_size && + active->def_elem_len == config->def_elem_len && + active->clock_provider == config->clock_provider && + active->bclk_inverted == config->bclk_inverted && + !memcmp(&active->protdesc, &config->protdesc, + sizeof(active->protdesc)); +} + int ux500_msp_i2s_open(struct ux500_msp *msp, struct ux500_msp_config *config) { u32 old_reg, new_reg, mask; int res; unsigned int tx_sel, rx_sel, tx_busy, rx_busy; + bool first; if (in_interrupt()) { dev_err(msp->dev, @@ -444,40 +482,68 @@ int ux500_msp_i2s_open(struct ux500_msp *msp, return -EBUSY; } - msp->dir_busy |= (tx_sel ? MSP_DIR_TX : 0) | (rx_sel ? MSP_DIR_RX : 0); - - /* First do the global config register */ - mask = RX_CLK_SEL_MASK | TX_CLK_SEL_MASK | RX_FSYNC_MASK | - TX_FSYNC_MASK | RX_SYNC_SEL_MASK | TX_SYNC_SEL_MASK | - RX_FIFO_ENABLE_MASK | TX_FIFO_ENABLE_MASK | SRG_CLK_SEL_MASK | - LOOPBACK_MASK | TX_EXTRA_DELAY_MASK; - - new_reg = (config->tx_clk_sel | config->rx_clk_sel | - config->rx_fsync_pol | config->tx_fsync_pol | - config->rx_fsync_sel | config->tx_fsync_sel | - config->rx_fifo_config | config->tx_fifo_config | - config->srg_clk_sel | config->loopback_enable | - config->tx_data_enable); + first = !msp->dir_busy; + if (!first && !ux500_msp_config_compatible(msp, config)) { + dev_err(msp->dev, "%s: Incompatible duplex configuration\n", + __func__); + return -EBUSY; + } - old_reg = readl(msp->registers + MSP_GCR); - old_reg &= ~mask; - new_reg |= old_reg; - writel(new_reg, msp->registers + MSP_GCR); + if (first) { + /* First do the global config register */ + mask = RX_CLK_SEL_MASK | TX_CLK_SEL_MASK | RX_FSYNC_MASK | + TX_FSYNC_MASK | RX_SYNC_SEL_MASK | TX_SYNC_SEL_MASK | + RX_FIFO_ENABLE_MASK | TX_FIFO_ENABLE_MASK | + SRG_CLK_SEL_MASK | LOOPBACK_MASK | TX_EXTRA_DELAY_MASK; + + new_reg = config->tx_clk_sel | config->rx_clk_sel | + config->rx_fsync_pol | config->tx_fsync_pol | + config->rx_fsync_sel | config->tx_fsync_sel | + config->rx_fifo_config | config->tx_fifo_config | + config->srg_clk_sel | config->loopback_enable | + config->tx_data_enable; + + old_reg = readl(msp->registers + MSP_GCR); + old_reg &= ~mask; + new_reg |= old_reg; + writel(new_reg, msp->registers + MSP_GCR); + writel(MSP_WMRK_TX_4_ELEMENTS | MSP_WMRK_RX_4_ELEMENTS, + msp->registers + MSP_WMRK); + } - res = enable_msp(msp, config); + res = enable_msp(msp, config, first); if (res < 0) { dev_err(msp->dev, "%s: ERROR: enable_msp failed (%d)!\n", __func__, res); - return -EBUSY; + if (tx_sel) + writel(0, msp->registers + MSP_TCF); + if (rx_sel) + writel(0, msp->registers + MSP_RCF); + if (first) { + writel(0, msp->registers + MSP_GCR); + writel(0, msp->registers + MSP_DMACR); + writel(0, msp->registers + MSP_SRG); + writel(0, msp->registers + MSP_MCR); + } + return res; + } + + msp->dir_busy |= config->direction; + if (first) { + msp->config = *config; + msp->clock_provider = config->clock_provider; } if (config->loopback_enable & 0x80) msp->loopback_enable = 1; /* Flush FIFOs */ - flush_fifo_tx(msp); - flush_fifo_rx(msp); + if (tx_sel) + flush_fifo_tx(msp); + if (rx_sel) + flush_fifo_rx(msp); - msp->msp_state = MSP_STATE_CONFIGURED; + if (!msp->dir_running) + msp->msp_state = MSP_STATE_CONFIGURED; return 0; } @@ -494,7 +560,6 @@ static void disable_msp_rx(struct ux500_msp *msp) ~(RX_SERVICE_INT | RX_OVERRUN_ERROR_INT), msp->registers + MSP_IMSC); - msp->dir_busy &= ~MSP_DIR_RX; } static void disable_msp_tx(struct ux500_msp *msp) @@ -510,7 +575,6 @@ static void disable_msp_tx(struct ux500_msp *msp) ~(TX_SERVICE_INT | TX_UNDERRUN_ERR_INT), msp->registers + MSP_IMSC); - msp->dir_busy &= ~MSP_DIR_TX; } static int disable_msp(struct ux500_msp *msp, unsigned int dir) @@ -520,7 +584,7 @@ static int disable_msp(struct ux500_msp *msp, unsigned int dir) reg_val_GCR = readl(msp->registers + MSP_GCR); disable_tx = dir & MSP_DIR_TX; - disable_rx = dir & MSP_DIR_TX; + disable_rx = dir & MSP_DIR_RX; if (disable_tx && disable_rx) { reg_val_GCR = readl(msp->registers + MSP_GCR); writel(reg_val_GCR | LOOPBACK_MASK, @@ -553,7 +617,15 @@ static int disable_msp(struct ux500_msp *msp, unsigned int dir) int ux500_msp_i2s_trigger(struct ux500_msp *msp, int cmd, int direction) { - u32 reg_val_GCR, enable_bit; + u32 reg_val_DMACR, reg_val_GCR, dma_enable_bit, enable_bit; + unsigned int dir; + + if (direction == SNDRV_PCM_STREAM_PLAYBACK) + dir = MSP_DIR_TX; + else if (direction == SNDRV_PCM_STREAM_CAPTURE) + dir = MSP_DIR_RX; + else + return -EINVAL; if (msp->msp_state == MSP_STATE_IDLE) { dev_err(msp->dev, "%s: ERROR: MSP is not configured!\n", @@ -565,21 +637,44 @@ int ux500_msp_i2s_trigger(struct ux500_msp *msp, int cmd, int direction) case SNDRV_PCM_TRIGGER_START: case SNDRV_PCM_TRIGGER_RESUME: case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: - if (direction == SNDRV_PCM_STREAM_PLAYBACK) + if (direction == SNDRV_PCM_STREAM_PLAYBACK) { enable_bit = TX_ENABLE; - else + dma_enable_bit = TX_DMA_ENABLE; + } else { enable_bit = RX_ENABLE; + dma_enable_bit = RX_DMA_ENABLE; + } + if (!(msp->dir_busy & dir)) + return -EINVAL; + reg_val_DMACR = readl(msp->registers + MSP_DMACR); + writel(reg_val_DMACR | dma_enable_bit, + msp->registers + MSP_DMACR); reg_val_GCR = readl(msp->registers + MSP_GCR); + if (msp->clock_provider) + enable_bit |= FRAME_GEN_ENABLE; writel(reg_val_GCR | enable_bit, msp->registers + MSP_GCR); + msp->dir_running |= dir; + msp->msp_state = MSP_STATE_RUNNING; break; case SNDRV_PCM_TRIGGER_STOP: case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_PAUSE_PUSH: - if (direction == SNDRV_PCM_STREAM_PLAYBACK) + if (!(msp->dir_busy & dir)) + return -EINVAL; + if (direction == SNDRV_PCM_STREAM_PLAYBACK) { disable_msp_tx(msp); - else + msp->dir_running &= ~MSP_DIR_TX; + } else { disable_msp_rx(msp); + msp->dir_running &= ~MSP_DIR_RX; + } + if (!msp->dir_running) { + reg_val_GCR = readl(msp->registers + MSP_GCR); + writel(reg_val_GCR & ~FRAME_GEN_ENABLE, + msp->registers + MSP_GCR); + msp->msp_state = MSP_STATE_CONFIGURED; + } break; default: return -EINVAL; @@ -594,7 +689,18 @@ int ux500_msp_i2s_close(struct ux500_msp *msp, unsigned int dir) dev_dbg(msp->dev, "%s: Enter (dir = 0x%01x).\n", __func__, dir); + if (!dir || dir & ~(MSP_DIR_TX | MSP_DIR_RX) || + (msp->dir_busy & dir) != dir) + return -EINVAL; + status = disable_msp(msp, dir); + msp->dir_busy &= ~dir; + msp->dir_running &= ~dir; + if (msp->dir_busy && !msp->dir_running) { + writel(readl(msp->registers + MSP_GCR) & ~FRAME_GEN_ENABLE, + msp->registers + MSP_GCR); + msp->msp_state = MSP_STATE_CONFIGURED; + } if (msp->dir_busy == 0) { /* disable sample rate and frame generators */ msp->msp_state = MSP_STATE_IDLE; @@ -618,6 +724,8 @@ int ux500_msp_i2s_close(struct ux500_msp *msp, unsigned int dir) writel(0, msp->registers + MSP_RCE1); writel(0, msp->registers + MSP_RCE2); writel(0, msp->registers + MSP_RCE3); + memset(&msp->config, 0, sizeof(msp->config)); + msp->clock_provider = false; } return status; @@ -627,7 +735,7 @@ int ux500_msp_i2s_close(struct ux500_msp *msp, unsigned int dir) int ux500_msp_i2s_init_msp(struct platform_device *pdev, struct ux500_msp **msp_p) { - struct resource *res = NULL; + struct resource *res; struct ux500_msp *msp; *msp_p = devm_kzalloc(&pdev->dev, sizeof(struct ux500_msp), GFP_KERNEL); @@ -637,20 +745,10 @@ int ux500_msp_i2s_init_msp(struct platform_device *pdev, msp->dev = &pdev->dev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res == NULL) { - dev_err(&pdev->dev, "%s: ERROR: Unable to get resource!\n", - __func__); - return -ENOMEM; - } - + msp->registers = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(msp->registers)) + return PTR_ERR(msp->registers); msp->tx_rx_addr = res->start + MSP_DR; - msp->registers = devm_ioremap(&pdev->dev, res->start, - resource_size(res)); - if (msp->registers == NULL) { - dev_err(&pdev->dev, "%s: ERROR: ioremap failed!\n", __func__); - return -ENOMEM; - } msp->msp_state = MSP_STATE_IDLE; msp->loopback_enable = 0; diff --git a/sound/soc/ux500/ux500_msp_i2s.h b/sound/soc/ux500/ux500_msp_i2s.h index 69d4ebc409fc1..c66ef455e1380 100644 --- a/sound/soc/ux500/ux500_msp_i2s.h +++ b/sound/soc/ux500/ux500_msp_i2s.h @@ -12,8 +12,6 @@ #include -#define MSP_INPUT_FREQ_APB 48000000 - /*** Stereo mode. Used for APB data accesses as 16 bits accesses (mono), * 32 bits accesses (stereo). ***/ @@ -64,6 +62,7 @@ enum msp_direction { #define MSP_SRG 0x10 #define MSP_FLR 0x14 #define MSP_DMACR 0x18 +#define MSP_WMRK 0x1c #define MSP_IMSC 0x20 #define MSP_RIS 0x24 @@ -148,8 +147,8 @@ enum msp_direction { #define RCKPOL_MASK BIT(0) #define TCKPOL_MASK BIT(0) #define SPICKM_MASK (BIT(1) | BIT(0)) -#define MSP_RX_CLKPOL_BIT(n) ((n & RCKPOL_MASK) << RCKPOL_SHIFT) -#define MSP_TX_CLKPOL_BIT(n) ((n & TCKPOL_MASK) << TCKPOL_SHIFT) +#define MSP_RX_CLKPOL_BIT(n) (((n) & RCKPOL_MASK) << RCKPOL_SHIFT) +#define MSP_TX_CLKPOL_BIT(n) (((n) & TCKPOL_MASK) << TCKPOL_SHIFT) #define P1ELEN_SHIFT 0 #define P1FLEN_SHIFT 3 @@ -230,6 +229,10 @@ enum msp_direction { #define RDMAE_SHIFT 0 #define TDMAE_SHIFT 1 +/* FIFO watermark register */ +#define MSP_WMRK_RX_4_ELEMENTS BIT(0) +#define MSP_WMRK_TX_4_ELEMENTS BIT(3) + /* Interrupt Register */ #define RX_SERVICE_INT BIT(0) #define RX_OVERRUN_ERROR_INT BIT(1) @@ -460,6 +463,8 @@ struct ux500_msp_config { enum msp_data_size data_size; unsigned int def_elem_len; unsigned int iodelay; + bool clock_provider; + bool bclk_inverted; }; struct ux500_msp { @@ -470,8 +475,11 @@ struct ux500_msp { enum msp_state msp_state; int def_elem_len; unsigned int dir_busy; + unsigned int dir_running; int loopback_enable; unsigned int f_bitclk; + bool clock_provider; + struct ux500_msp_config config; }; int ux500_msp_i2s_init_msp(struct platform_device *pdev, diff --git a/sound/soc/xilinx/xlnx_formatter_pcm.c b/sound/soc/xilinx/xlnx_formatter_pcm.c index f15336197ed31..798f17a6960ac 100644 --- a/sound/soc/xilinx/xlnx_formatter_pcm.c +++ b/sound/soc/xilinx/xlnx_formatter_pcm.c @@ -384,7 +384,7 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, if (err) { dev_err(component->dev, "Unable to set constraint on period bytes\n"); - return err; + goto error; } /* Resize the buffer bytes as divisible by 64 */ @@ -394,7 +394,7 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, if (err) { dev_err(component->dev, "Unable to set constraint on buffer bytes\n"); - return err; + goto error; } /* Set periods as integer multiple */ @@ -403,7 +403,7 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, if (err < 0) { dev_err(component->dev, "Unable to set constraint on periods to be integer\n"); - return err; + goto error; } /* enable DMA IOC irq */ @@ -412,6 +412,14 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, writel(val, stream_data->mmio + XLNX_AUD_CTRL); return 0; + +error: + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + adata->play_stream = NULL; + else + adata->capture_stream = NULL; + kfree(stream_data); + return err; } static int xlnx_formatter_pcm_close(struct snd_soc_component *component, diff --git a/sound/usb/6fire/comm.c b/sound/usb/6fire/comm.c index 9a0b85653c618..510d310824e91 100644 --- a/sound/usb/6fire/comm.c +++ b/sound/usb/6fire/comm.c @@ -21,7 +21,6 @@ enum { static void usb6fire_comm_init_urb(struct comm_runtime *rt, struct urb *urb, u8 *buffer, void *context, void(*handler)(struct urb *urb)) { - usb_init_urb(urb); urb->transfer_buffer = buffer; urb->pipe = usb_sndintpipe(rt->chip->dev, COMM_EP); urb->complete = handler; @@ -36,11 +35,14 @@ static void usb6fire_comm_receiver_handler(struct urb *urb) struct midi_runtime *midi_rt = rt->chip->midi; if (!urb->status) { - if (rt->receiver_buffer[0] == 0x10) /* midi in event */ + u8 len = rt->receiver_buffer[1]; + + if (rt->receiver_buffer[0] == 0x10 && /* midi in event */ + len <= COMM_RECEIVER_BUFSIZE - 2 && + urb->actual_length >= len + 2) if (midi_rt) midi_rt->in_received(midi_rt, - rt->receiver_buffer + 2, - rt->receiver_buffer[1]); + rt->receiver_buffer + 2, len); } if (!rt->chip->shutdown) { @@ -139,6 +141,19 @@ static int usb6fire_comm_write16(struct comm_runtime *rt, u8 request, return ret; } +static void usb6fire_comm_free(struct comm_runtime *rt) +{ + if (!rt) + return; + + if (rt->chip) + rt->chip->comm = NULL; + + usb_free_urb(rt->receiver); + kfree(rt->receiver_buffer); + kfree(rt); +} + int usb6fire_comm_init(struct sfire_chip *chip) { struct comm_runtime *rt = kzalloc_obj(struct comm_runtime); @@ -150,14 +165,18 @@ int usb6fire_comm_init(struct sfire_chip *chip) rt->receiver_buffer = kzalloc(COMM_RECEIVER_BUFSIZE, GFP_KERNEL); if (!rt->receiver_buffer) { - kfree(rt); - return -ENOMEM; + ret = -ENOMEM; + goto error; } - urb = &rt->receiver; + urb = usb_alloc_urb(0, GFP_KERNEL); + if (!urb) { + ret = -ENOMEM; + goto error; + } + rt->receiver = urb; rt->serial = 1; rt->chip = chip; - usb_init_urb(urb); rt->init_urb = usb6fire_comm_init_urb; rt->write8 = usb6fire_comm_write8; rt->write16 = usb6fire_comm_write16; @@ -172,13 +191,15 @@ int usb6fire_comm_init(struct sfire_chip *chip) urb->interval = 1; ret = usb_submit_urb(urb, GFP_KERNEL); if (ret < 0) { - kfree(rt->receiver_buffer); - kfree(rt); dev_err(&chip->dev->dev, "cannot create comm data receiver."); - return ret; + goto error; } chip->comm = rt; return 0; + + error: + usb6fire_comm_free(rt); + return ret; } void usb6fire_comm_abort(struct sfire_chip *chip) @@ -186,14 +207,10 @@ void usb6fire_comm_abort(struct sfire_chip *chip) struct comm_runtime *rt = chip->comm; if (rt) - usb_poison_urb(&rt->receiver); + usb_poison_urb(rt->receiver); } void usb6fire_comm_destroy(struct sfire_chip *chip) { - struct comm_runtime *rt = chip->comm; - - kfree(rt->receiver_buffer); - kfree(rt); - chip->comm = NULL; + usb6fire_comm_free(chip->comm); } diff --git a/sound/usb/6fire/comm.h b/sound/usb/6fire/comm.h index 2447d7ecf1798..89976f510f6c2 100644 --- a/sound/usb/6fire/comm.h +++ b/sound/usb/6fire/comm.h @@ -19,7 +19,7 @@ enum /* settings for comm */ struct comm_runtime { struct sfire_chip *chip; - struct urb receiver; + struct urb *receiver; u8 *receiver_buffer; u8 serial; /* urb serial */ diff --git a/sound/usb/6fire/midi.c b/sound/usb/6fire/midi.c index 6b0bb096f27a1..279b449936e75 100644 --- a/sound/usb/6fire/midi.c +++ b/sound/usb/6fire/midi.c @@ -66,7 +66,7 @@ static void usb6fire_midi_out_trigger( struct snd_rawmidi_substream *alsa_sub, int up) { struct midi_runtime *rt = alsa_sub->rmidi->private_data; - struct urb *urb = &rt->out_urb; + struct urb *urb = rt->out_urb; __s8 ret; guard(spinlock_irqsave)(&rt->out_lock); @@ -137,6 +137,19 @@ static const struct snd_rawmidi_ops in_ops = { .trigger = usb6fire_midi_in_trigger }; +static void usb6fire_midi_free(struct midi_runtime *rt) +{ + if (!rt) + return; + + if (rt->chip) + rt->chip->midi = NULL; + + usb_free_urb(rt->out_urb); + kfree(rt->out_buffer); + kfree(rt); +} + int usb6fire_midi_init(struct sfire_chip *chip) { int ret; @@ -148,8 +161,14 @@ int usb6fire_midi_init(struct sfire_chip *chip) rt->out_buffer = kzalloc(MIDI_BUFSIZE, GFP_KERNEL); if (!rt->out_buffer) { - kfree(rt); - return -ENOMEM; + ret = -ENOMEM; + goto error; + } + + rt->out_urb = usb_alloc_urb(0, GFP_KERNEL); + if (!rt->out_urb) { + ret = -ENOMEM; + goto error; } rt->chip = chip; @@ -160,15 +179,13 @@ int usb6fire_midi_init(struct sfire_chip *chip) spin_lock_init(&rt->in_lock); spin_lock_init(&rt->out_lock); - comm_rt->init_urb(comm_rt, &rt->out_urb, rt->out_buffer, rt, + comm_rt->init_urb(comm_rt, rt->out_urb, rt->out_buffer, rt, usb6fire_midi_out_handler); ret = snd_rawmidi_new(chip->card, "6FireUSB", 0, 1, 1, &rt->instance); if (ret < 0) { - kfree(rt->out_buffer); - kfree(rt); dev_err(&chip->dev->dev, "unable to create midi.\n"); - return ret; + goto error; } rt->instance->private_data = rt; strscpy(rt->instance->name, "DMX6FireUSB MIDI"); @@ -182,6 +199,10 @@ int usb6fire_midi_init(struct sfire_chip *chip) chip->midi = rt; return 0; + + error: + usb6fire_midi_free(rt); + return ret; } void usb6fire_midi_abort(struct sfire_chip *chip) @@ -189,14 +210,10 @@ void usb6fire_midi_abort(struct sfire_chip *chip) struct midi_runtime *rt = chip->midi; if (rt) - usb_poison_urb(&rt->out_urb); + usb_poison_urb(rt->out_urb); } void usb6fire_midi_destroy(struct sfire_chip *chip) { - struct midi_runtime *rt = chip->midi; - - kfree(rt->out_buffer); - kfree(rt); - chip->midi = NULL; + usb6fire_midi_free(chip->midi); } diff --git a/sound/usb/6fire/midi.h b/sound/usb/6fire/midi.h index 47640c845903b..8716ab8a863ae 100644 --- a/sound/usb/6fire/midi.h +++ b/sound/usb/6fire/midi.h @@ -22,7 +22,7 @@ struct midi_runtime { spinlock_t in_lock; spinlock_t out_lock; struct snd_rawmidi_substream *out; - struct urb out_urb; + struct urb *out_urb; u8 out_serial; /* serial number of out packet */ u8 *out_buffer; int buffer_offset; diff --git a/sound/usb/6fire/pcm.c b/sound/usb/6fire/pcm.c index d2e274b731fe5..0285d79ace0fc 100644 --- a/sound/usb/6fire/pcm.c +++ b/sound/usb/6fire/pcm.c @@ -138,8 +138,8 @@ static void usb6fire_pcm_stream_stop(struct pcm_runtime *rt) rt->stream_state = STREAM_STOPPING; for (i = 0; i < PCM_N_URBS; i++) { - usb_kill_urb(&rt->in_urbs[i].instance); - usb_kill_urb(&rt->out_urbs[i].instance); + usb_kill_urb(rt->in_urbs[i].instance); + usb_kill_urb(rt->out_urbs[i].instance); } ctrl_rt->usb_streaming = false; ctrl_rt->update_streaming(ctrl_rt); @@ -161,13 +161,13 @@ static int usb6fire_pcm_stream_start(struct pcm_runtime *rt) rt->stream_state = STREAM_STARTING; for (i = 0; i < PCM_N_URBS; i++) { for (k = 0; k < PCM_N_PACKETS_PER_URB; k++) { - packet = &rt->in_urbs[i].packets[k]; + packet = &rt->in_urbs[i].instance->iso_frame_desc[k]; packet->offset = k * rt->in_packet_size; packet->length = rt->in_packet_size; packet->actual_length = 0; packet->status = 0; } - ret = usb_submit_urb(&rt->in_urbs[i].instance, + ret = usb_submit_urb(rt->in_urbs[i].instance, GFP_ATOMIC); if (ret) { usb6fire_pcm_stream_stop(rt); @@ -197,6 +197,7 @@ static void usb6fire_pcm_capture(struct pcm_substream *sub, struct pcm_urb *urb) unsigned int total_length = 0; struct pcm_runtime *rt = snd_pcm_substream_chip(sub->instance); struct snd_pcm_runtime *alsa_rt = sub->instance->runtime; + struct usb_iso_packet_descriptor *isoc; u32 *src = NULL; u32 *dest = (u32 *) (alsa_rt->dma_area + sub->dma_off * (alsa_rt->frame_bits >> 3)); @@ -207,8 +208,9 @@ static void usb6fire_pcm_capture(struct pcm_substream *sub, struct pcm_urb *urb) for (i = 0; i < PCM_N_PACKETS_PER_URB; i++) { /* at least 4 header bytes for valid packet. * after that: 32 bits per sample for analog channels */ - if (urb->packets[i].actual_length > 4) - frame_count = (urb->packets[i].actual_length - 4) + isoc = &urb->instance->iso_frame_desc[i]; + if (isoc->actual_length > 4) + frame_count = (isoc->actual_length - 4) / (rt->in_n_analog << 2); else frame_count = 0; @@ -220,7 +222,7 @@ static void usb6fire_pcm_capture(struct pcm_substream *sub, struct pcm_urb *urb) else return; src++; /* skip leading 4 bytes of every packet */ - total_length += urb->packets[i].length; + total_length += isoc->length; for (frame = 0; frame < frame_count; frame++) { memcpy(dest, src, bytes_per_frame); dest += alsa_rt->channels; @@ -244,6 +246,7 @@ static void usb6fire_pcm_playback(struct pcm_substream *sub, int frame_count; struct pcm_runtime *rt = snd_pcm_substream_chip(sub->instance); struct snd_pcm_runtime *alsa_rt = sub->instance->runtime; + struct usb_iso_packet_descriptor *isoc; u32 *src = (u32 *) (alsa_rt->dma_area + sub->dma_off * (alsa_rt->frame_bits >> 3)); u32 *src_end = (u32 *) (alsa_rt->dma_area + alsa_rt->buffer_size @@ -263,8 +266,9 @@ static void usb6fire_pcm_playback(struct pcm_substream *sub, for (i = 0; i < PCM_N_PACKETS_PER_URB; i++) { /* at least 4 header bytes for valid packet. * after that: 32 bits per sample for analog channels */ - if (urb->packets[i].length > 4) - frame_count = (urb->packets[i].length - 4) + isoc = &urb->instance->iso_frame_desc[i]; + if (isoc->length > 4) + frame_count = (isoc->length - 4) / (rt->out_n_analog << 2); else frame_count = 0; @@ -289,6 +293,7 @@ static void usb6fire_pcm_in_urb_handler(struct urb *usb_urb) struct pcm_urb *out_urb = in_urb->peer; struct pcm_runtime *rt = in_urb->chip->pcm; struct pcm_substream *sub; + struct usb_iso_packet_descriptor *isoc_out, *isoc_in; bool period_elapsed; int total_length = 0; int frame_count; @@ -299,11 +304,13 @@ static void usb6fire_pcm_in_urb_handler(struct urb *usb_urb) if (usb_urb->status || rt->panic || rt->stream_state == STREAM_STOPPING) return; - for (i = 0; i < PCM_N_PACKETS_PER_URB; i++) - if (in_urb->packets[i].status) { + for (i = 0; i < PCM_N_PACKETS_PER_URB; i++) { + isoc_in = &in_urb->instance->iso_frame_desc[i]; + if (isoc_in->status) { rt->panic = true; return; } + } if (rt->stream_state == STREAM_DISABLED) { dev_err(&rt->chip->dev->dev, @@ -328,12 +335,21 @@ static void usb6fire_pcm_in_urb_handler(struct urb *usb_urb) /* setup out urb structure */ for (i = 0; i < PCM_N_PACKETS_PER_URB; i++) { - out_urb->packets[i].offset = total_length; - out_urb->packets[i].length = (in_urb->packets[i].actual_length - - 4) / (rt->in_n_analog << 2) - * (rt->out_n_analog << 2) + 4; - out_urb->packets[i].status = 0; - total_length += out_urb->packets[i].length; + unsigned int frames = 0; + + isoc_out = &out_urb->instance->iso_frame_desc[i]; + isoc_in = &in_urb->instance->iso_frame_desc[i]; + if (isoc_in->actual_length > 4) + frames = (isoc_in->actual_length - 4) + / (rt->in_n_analog << 2); + frames = min_t(unsigned int, frames, + (rt->out_packet_size - 4) + / (rt->out_n_analog << 2)); + + isoc_out->offset = total_length; + isoc_out->length = frames * (rt->out_n_analog << 2) + 4; + isoc_out->status = 0; + total_length += isoc_out->length; } memset(out_urb->buffer, 0, total_length); @@ -354,9 +370,10 @@ static void usb6fire_pcm_in_urb_handler(struct urb *usb_urb) /* setup the 4th byte of each sample (0x40 for analog channels) */ dest = out_urb->buffer; - for (i = 0; i < PCM_N_PACKETS_PER_URB; i++) - if (out_urb->packets[i].length >= 4) { - frame_count = (out_urb->packets[i].length - 4) + for (i = 0; i < PCM_N_PACKETS_PER_URB; i++) { + isoc_out = &out_urb->instance->iso_frame_desc[i]; + if (isoc_out->length >= 4) { + frame_count = (isoc_out->length - 4) / (rt->out_n_analog << 2); *(dest++) = 0xaa; *(dest++) = 0xaa; @@ -370,8 +387,10 @@ static void usb6fire_pcm_in_urb_handler(struct urb *usb_urb) *(dest++) = 0x40; } } - usb_submit_urb(&out_urb->instance, GFP_ATOMIC); - usb_submit_urb(&in_urb->instance, GFP_ATOMIC); + } + + usb_submit_urb(out_urb->instance, GFP_ATOMIC); + usb_submit_urb(in_urb->instance, GFP_ATOMIC); } static void usb6fire_pcm_out_urb_handler(struct urb *usb_urb) @@ -534,22 +553,25 @@ static const struct snd_pcm_ops pcm_ops = { .pointer = usb6fire_pcm_pointer, }; -static void usb6fire_pcm_init_urb(struct pcm_urb *urb, - struct sfire_chip *chip, bool in, int ep, - void (*handler)(struct urb *)) +static int usb6fire_pcm_init_urb(struct pcm_urb *urb, + struct sfire_chip *chip, bool in, int ep, + void (*handler)(struct urb *)) { urb->chip = chip; - usb_init_urb(&urb->instance); - urb->instance.transfer_buffer = urb->buffer; - urb->instance.transfer_buffer_length = + urb->instance = usb_alloc_urb(PCM_N_PACKETS_PER_URB, GFP_KERNEL); + if (!urb->instance) + return -ENOMEM; + urb->instance->transfer_buffer = urb->buffer; + urb->instance->transfer_buffer_length = PCM_N_PACKETS_PER_URB * PCM_MAX_PACKET_SIZE; - urb->instance.dev = chip->dev; - urb->instance.pipe = in ? usb_rcvisocpipe(chip->dev, ep) + urb->instance->dev = chip->dev; + urb->instance->pipe = in ? usb_rcvisocpipe(chip->dev, ep) : usb_sndisocpipe(chip->dev, ep); - urb->instance.interval = 1; - urb->instance.complete = handler; - urb->instance.context = urb; - urb->instance.number_of_packets = PCM_N_PACKETS_PER_URB; + urb->instance->interval = 1; + urb->instance->complete = handler; + urb->instance->context = urb; + urb->instance->number_of_packets = PCM_N_PACKETS_PER_URB; + return 0; } static int usb6fire_pcm_buffers_init(struct pcm_runtime *rt) @@ -571,14 +593,23 @@ static int usb6fire_pcm_buffers_init(struct pcm_runtime *rt) return 0; } -static void usb6fire_pcm_buffers_destroy(struct pcm_runtime *rt) +static void usb6fire_pcm_free(struct pcm_runtime *rt) { int i; + if (!rt) + return; + + if (rt->chip) + rt->chip->pcm = NULL; + for (i = 0; i < PCM_N_URBS; i++) { + usb_free_urb(rt->out_urbs[i].instance); kfree(rt->out_urbs[i].buffer); + usb_free_urb(rt->in_urbs[i].instance); kfree(rt->in_urbs[i].buffer); } + kfree(rt); } int usb6fire_pcm_init(struct sfire_chip *chip) @@ -593,11 +624,8 @@ int usb6fire_pcm_init(struct sfire_chip *chip) return -ENOMEM; ret = usb6fire_pcm_buffers_init(rt); - if (ret) { - usb6fire_pcm_buffers_destroy(rt); - kfree(rt); - return ret; - } + if (ret) + goto error; rt->chip = chip; rt->stream_state = STREAM_DISABLED; @@ -609,10 +637,14 @@ int usb6fire_pcm_init(struct sfire_chip *chip) spin_lock_init(&rt->capture.lock); for (i = 0; i < PCM_N_URBS; i++) { - usb6fire_pcm_init_urb(&rt->in_urbs[i], chip, true, IN_EP, - usb6fire_pcm_in_urb_handler); - usb6fire_pcm_init_urb(&rt->out_urbs[i], chip, false, OUT_EP, - usb6fire_pcm_out_urb_handler); + ret = usb6fire_pcm_init_urb(&rt->in_urbs[i], chip, true, IN_EP, + usb6fire_pcm_in_urb_handler); + if (ret < 0) + goto error; + ret = usb6fire_pcm_init_urb(&rt->out_urbs[i], chip, false, OUT_EP, + usb6fire_pcm_out_urb_handler); + if (ret < 0) + goto error; rt->in_urbs[i].peer = &rt->out_urbs[i]; rt->out_urbs[i].peer = &rt->in_urbs[i]; @@ -620,10 +652,8 @@ int usb6fire_pcm_init(struct sfire_chip *chip) ret = snd_pcm_new(chip->card, "DMX6FireUSB", 0, 1, 1, &pcm); if (ret < 0) { - usb6fire_pcm_buffers_destroy(rt); - kfree(rt); dev_err(&chip->dev->dev, "cannot create pcm instance.\n"); - return ret; + goto error; } pcm->private_data = rt; @@ -636,6 +666,10 @@ int usb6fire_pcm_init(struct sfire_chip *chip) chip->pcm = rt; return 0; + + error: + usb6fire_pcm_free(rt); + return ret; } void usb6fire_pcm_abort(struct sfire_chip *chip) @@ -653,8 +687,8 @@ void usb6fire_pcm_abort(struct sfire_chip *chip) snd_pcm_stop_xrun(rt->capture.instance); for (i = 0; i < PCM_N_URBS; i++) { - usb_poison_urb(&rt->in_urbs[i].instance); - usb_poison_urb(&rt->out_urbs[i].instance); + usb_poison_urb(rt->in_urbs[i].instance); + usb_poison_urb(rt->out_urbs[i].instance); } } @@ -662,9 +696,5 @@ void usb6fire_pcm_abort(struct sfire_chip *chip) void usb6fire_pcm_destroy(struct sfire_chip *chip) { - struct pcm_runtime *rt = chip->pcm; - - usb6fire_pcm_buffers_destroy(rt); - kfree(rt); - chip->pcm = NULL; + usb6fire_pcm_free(chip->pcm); } diff --git a/sound/usb/6fire/pcm.h b/sound/usb/6fire/pcm.h index 5a092dfd69f5a..b586fe220fd11 100644 --- a/sound/usb/6fire/pcm.h +++ b/sound/usb/6fire/pcm.h @@ -24,10 +24,7 @@ enum /* settings for pcm */ struct pcm_urb { struct sfire_chip *chip; - /* BEGIN DO NOT SEPARATE */ - struct urb instance; - struct usb_iso_packet_descriptor packets[PCM_N_PACKETS_PER_URB]; - /* END DO NOT SEPARATE */ + struct urb *instance; u8 *buffer; struct pcm_urb *peer; diff --git a/sound/usb/bcd2000/bcd2000.c b/sound/usb/bcd2000/bcd2000.c index bebb48cb9abc8..2bd49bf827489 100644 --- a/sound/usb/bcd2000/bcd2000.c +++ b/sound/usb/bcd2000/bcd2000.c @@ -43,6 +43,7 @@ struct bcd2000 { struct usb_interface *intf; int card_index; + spinlock_t midi_lock; int midi_out_active; struct snd_rawmidi *rmidi; struct snd_rawmidi_substream *midi_receive_substream; @@ -90,6 +91,8 @@ static void bcd2000_midi_input_trigger(struct snd_rawmidi_substream *substream, int up) { struct bcd2000 *bcd2k = substream->rmidi->private_data; + + guard(spinlock_irqsave)(&bcd2k->midi_lock); bcd2k->midi_receive_substream = up ? substream : NULL; } @@ -134,6 +137,9 @@ static void bcd2000_midi_send(struct bcd2000 *bcd2k) if (!midi_out_substream) return; + if (!bcd2k->midi_out_urb) + return; + /* copy command prefix bytes */ memcpy(bcd2k->midi_out_buf, device_cmd_prefix, sizeof(device_cmd_prefix)); @@ -178,7 +184,7 @@ static int bcd2000_midi_output_close(struct snd_rawmidi_substream *substream) { struct bcd2000 *bcd2k = substream->rmidi->private_data; - if (bcd2k->midi_out_active) { + if (bcd2k->midi_out_active && bcd2k->midi_out_urb) { usb_kill_urb(bcd2k->midi_out_urb); bcd2k->midi_out_active = 0; } @@ -192,6 +198,8 @@ static void bcd2000_midi_output_trigger(struct snd_rawmidi_substream *substream, { struct bcd2000 *bcd2k = substream->rmidi->private_data; + guard(spinlock_irqsave)(&bcd2k->midi_lock); + if (up) { bcd2k->midi_out_substream = substream; /* check if there is data userspace wants to send */ @@ -216,6 +224,7 @@ static void bcd2000_output_complete(struct urb *urb) return; /* check if there is more data userspace wants to send */ + guard(spinlock_irqsave)(&bcd2k->midi_lock); bcd2000_midi_send(bcd2k); } @@ -231,6 +240,8 @@ static void bcd2000_input_complete(struct urb *urb) if (!bcd2k || urb->status == -ESHUTDOWN) return; + guard(spinlock_irqsave)(&bcd2k->midi_lock); + if (urb->actual_length > 0) bcd2000_midi_handle_input(bcd2k, urb->transfer_buffer, urb->actual_length); @@ -345,14 +356,26 @@ static int bcd2000_init_midi(struct bcd2000 *bcd2k) return 0; } +static void bcd2000_midi_free(struct bcd2000 *bcd2k, + struct urb **urb_p) +{ + struct urb *urb = *urb_p; + + if (!urb) + return; + + usb_poison_urb(urb); + scoped_guard(spinlock_irq, &bcd2k->midi_lock) + *urb_p = NULL; + + usb_free_urb(urb); +} + static void bcd2000_free_usb_related_resources(struct bcd2000 *bcd2k, struct usb_interface *interface) { - usb_kill_urb(bcd2k->midi_out_urb); - usb_kill_urb(bcd2k->midi_in_urb); - - usb_free_urb(bcd2k->midi_out_urb); - usb_free_urb(bcd2k->midi_in_urb); + bcd2000_midi_free(bcd2k, &bcd2k->midi_out_urb); + bcd2000_midi_free(bcd2k, &bcd2k->midi_in_urb); if (bcd2k->intf) { usb_set_intfdata(bcd2k->intf, NULL); @@ -388,6 +411,7 @@ static int bcd2000_probe(struct usb_interface *interface, bcd2k->card = card; bcd2k->card_index = card_index; bcd2k->intf = interface; + spin_lock_init(&bcd2k->midi_lock); snd_card_set_dev(card, &interface->dev); diff --git a/sound/usb/caiaq/audio.c b/sound/usb/caiaq/audio.c index ba3f73455ebee..bb6280aa35334 100644 --- a/sound/usb/caiaq/audio.c +++ b/sound/usb/caiaq/audio.c @@ -828,16 +828,13 @@ int snd_usb_caiaq_audio_init(struct snd_usb_caiaqdev *cdev) cdev->data_urbs_in = alloc_urbs(cdev, SNDRV_PCM_STREAM_CAPTURE, &ret); if (ret < 0) { - kfree(cdev->data_cb_info); - free_urbs(cdev->data_urbs_in); + snd_usb_caiaq_audio_free(cdev); return ret; } cdev->data_urbs_out = alloc_urbs(cdev, SNDRV_PCM_STREAM_PLAYBACK, &ret); if (ret < 0) { - kfree(cdev->data_cb_info); - free_urbs(cdev->data_urbs_in); - free_urbs(cdev->data_urbs_out); + snd_usb_caiaq_audio_free(cdev); return ret; } @@ -858,6 +855,9 @@ void snd_usb_caiaq_audio_free(struct snd_usb_caiaqdev *cdev) dev_dbg(dev, "%s(%p)\n", __func__, cdev); free_urbs(cdev->data_urbs_in); + cdev->data_urbs_in = NULL; free_urbs(cdev->data_urbs_out); + cdev->data_urbs_out = NULL; kfree(cdev->data_cb_info); + cdev->data_cb_info = NULL; } diff --git a/sound/usb/caiaq/device.c b/sound/usb/caiaq/device.c index a16e592484803..3d821fde45820 100644 --- a/sound/usb/caiaq/device.c +++ b/sound/usb/caiaq/device.c @@ -192,8 +192,8 @@ static void usb_ep1_command_reply_dispatch (struct urb* urb) break; } - cdev->ep1_in_urb.actual_length = 0; - ret = usb_submit_urb(&cdev->ep1_in_urb, GFP_ATOMIC); + cdev->ep1_in_urb->actual_length = 0; + ret = usb_submit_urb(cdev->ep1_in_urb, GFP_ATOMIC); if (ret < 0) dev_err(dev, "unable to submit urb. OOM!?\n"); } @@ -408,6 +408,10 @@ static void card_free(struct snd_card *card) #endif snd_usb_caiaq_audio_free(cdev); usb_put_dev(cdev->chip.dev); + usb_free_urb(cdev->ep1_in_urb); + cdev->ep1_in_urb = NULL; + usb_free_urb(cdev->midi_out_urb); + cdev->midi_out_urb = NULL; } static int create_card(struct usb_device *usb_dev, @@ -457,22 +461,30 @@ static int init_card(struct snd_usb_caiaqdev *cdev) return -EIO; } - usb_init_urb(&cdev->ep1_in_urb); - usb_init_urb(&cdev->midi_out_urb); + cdev->ep1_in_urb = usb_alloc_urb(0, GFP_KERNEL); + if (!cdev->ep1_in_urb) + return -ENOMEM; - usb_fill_bulk_urb(&cdev->ep1_in_urb, usb_dev, + cdev->midi_out_urb = usb_alloc_urb(0, GFP_KERNEL); + if (!cdev->midi_out_urb) { + usb_free_urb(cdev->ep1_in_urb); + cdev->ep1_in_urb = NULL; + return -ENOMEM; + } + + usb_fill_bulk_urb(cdev->ep1_in_urb, usb_dev, usb_rcvbulkpipe(usb_dev, 0x1), cdev->ep1_in_buf, EP1_BUFSIZE, usb_ep1_command_reply_dispatch, cdev); - usb_fill_bulk_urb(&cdev->midi_out_urb, usb_dev, + usb_fill_bulk_urb(cdev->midi_out_urb, usb_dev, usb_sndbulkpipe(usb_dev, 0x1), cdev->midi_out_buf, EP1_BUFSIZE, snd_usb_caiaq_midi_output_done, cdev); /* sanity checks of EPs before actually submitting */ - if (usb_urb_ep_type_check(&cdev->ep1_in_urb) || - usb_urb_ep_type_check(&cdev->midi_out_urb)) { + if (usb_urb_ep_type_check(cdev->ep1_in_urb) || + usb_urb_ep_type_check(cdev->midi_out_urb)) { dev_err(dev, "invalid EPs\n"); return -EINVAL; } @@ -480,7 +492,7 @@ static int init_card(struct snd_usb_caiaqdev *cdev) init_waitqueue_head(&cdev->ep1_wait_queue); init_waitqueue_head(&cdev->prepare_wait_queue); - if (usb_submit_urb(&cdev->ep1_in_urb, GFP_KERNEL) != 0) + if (usb_submit_urb(cdev->ep1_in_urb, GFP_KERNEL) != 0) return -EIO; err = snd_usb_caiaq_send_command(cdev, EP1_CMD_GET_DEVICE_INFO, NULL, 0); @@ -530,7 +542,7 @@ static int init_card(struct snd_usb_caiaqdev *cdev) return 0; err_kill_urb: - usb_kill_urb(&cdev->ep1_in_urb); + usb_kill_urb(cdev->ep1_in_urb); return err; } @@ -576,8 +588,8 @@ static void snd_disconnect(struct usb_interface *intf) #endif snd_usb_caiaq_audio_disconnect(cdev); - usb_kill_urb(&cdev->ep1_in_urb); - usb_kill_urb(&cdev->midi_out_urb); + usb_kill_urb(cdev->ep1_in_urb); + usb_kill_urb(cdev->midi_out_urb); snd_card_free_when_closed(card); } diff --git a/sound/usb/caiaq/device.h b/sound/usb/caiaq/device.h index 743eb0387b5fb..1c6f34693fa81 100644 --- a/sound/usb/caiaq/device.h +++ b/sound/usb/caiaq/device.h @@ -60,8 +60,8 @@ struct snd_usb_caiaq_cb_info; struct snd_usb_caiaqdev { struct snd_usb_audio chip; - struct urb ep1_in_urb; - struct urb midi_out_urb; + struct urb *ep1_in_urb; + struct urb *midi_out_urb; struct urb **data_urbs_in; struct urb **data_urbs_out; struct snd_usb_caiaq_cb_info *data_cb_info; diff --git a/sound/usb/caiaq/midi.c b/sound/usb/caiaq/midi.c index c656d01624321..18529484c8dcc 100644 --- a/sound/usb/caiaq/midi.c +++ b/sound/usb/caiaq/midi.c @@ -43,7 +43,7 @@ static int snd_usb_caiaq_midi_output_close(struct snd_rawmidi_substream *substre { struct snd_usb_caiaqdev *cdev = substream->rmidi->private_data; if (cdev->midi_out_active) { - usb_kill_urb(&cdev->midi_out_urb); + usb_kill_urb(cdev->midi_out_urb); cdev->midi_out_active = 0; } return 0; @@ -64,9 +64,9 @@ static void snd_usb_caiaq_midi_send(struct snd_usb_caiaqdev *cdev, return; cdev->midi_out_buf[2] = len; - cdev->midi_out_urb.transfer_buffer_length = len+3; + cdev->midi_out_urb->transfer_buffer_length = len+3; - ret = usb_submit_urb(&cdev->midi_out_urb, GFP_ATOMIC); + ret = usb_submit_urb(cdev->midi_out_urb, GFP_ATOMIC); if (ret < 0) dev_err(dev, "snd_usb_caiaq_midi_send(%p): usb_submit_urb() failed," diff --git a/sound/usb/card.c b/sound/usb/card.c index 6a3b576fb0679..8b9d2ebf1809c 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -1276,8 +1276,11 @@ static int usb_audio_resume(struct usb_interface *intf) list_for_each_entry(as, &chip->pcm_list, list) { err = snd_usb_pcm_resume(as); - if (err < 0) - goto err_out; + if (err < 0) { + if (!chip->system_suspend) + goto err_out; + goto out; + } } /* @@ -1286,8 +1289,11 @@ static int usb_audio_resume(struct usb_interface *intf) */ list_for_each_entry(mixer, &chip->mixer_list, list) { err = snd_usb_mixer_resume(mixer); - if (err < 0) - goto err_out; + if (err < 0) { + if (!chip->system_suspend) + goto err_out; + goto out; + } } list_for_each(p, &chip->midi_list) { diff --git a/sound/usb/card.h b/sound/usb/card.h index e34d92d576a27..8299ac241c603 100644 --- a/sound/usb/card.h +++ b/sound/usb/card.h @@ -116,6 +116,7 @@ struct snd_usb_endpoint { unsigned int phase; /* phase accumulator */ unsigned int maxpacksize; /* max packet size in bytes */ unsigned int maxframesize; /* max packet size in frames */ + unsigned int max_urb_packs; /* packets allocated per data URB */ unsigned int max_urb_frames; /* max URB size in frames */ unsigned int curpacksize; /* current packet size in bytes (for capture) */ unsigned int curframesize; /* current packet size in frames (for capture) */ diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index a1d449f2a3423..a0ec51cc56ace 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -492,9 +492,10 @@ int snd_usb_queue_pending_output_urbs(struct snd_usb_endpoint *ep, /* copy over the length information */ if (implicit_fb) { - ctx->packets = packet->packets; + ctx->packets = min_t(int, packet->packets, + ep->max_urb_packs); memcpy(ctx->packet_size, packet->packet_size, - packet->packets * sizeof(packet->packet_size[0])); + ctx->packets * sizeof(packet->packet_size[0])); } /* call the data handler to fill in playback data */ @@ -1239,15 +1240,16 @@ static int data_ep_set_params(struct snd_usb_endpoint *ep) ep->nurbs = min(max_urbs, urbs_per_period * ep->cur_buffer_periods); } + if (fmt->fmt_type == UAC_FORMAT_TYPE_II) + urb_packs++; /* for transfer delimiter */ + ep->max_urb_packs = urb_packs; + /* allocate and initialize data urbs */ for (i = 0; i < ep->nurbs; i++) { struct snd_urb_ctx *u = &ep->urb[i]; u->index = i; u->ep = ep; u->packets = urb_packs; - - if (fmt->fmt_type == UAC_FORMAT_TYPE_II) - u->packets++; /* for transfer delimiter */ u->buffer_size = maxsize * u->packets; u->urb = usb_alloc_urb(u->packets, GFP_KERNEL); if (!u->urb) diff --git a/sound/usb/fcp.c b/sound/usb/fcp.c index 2bf572c6fdc40..5fc2131b45617 100644 --- a/sound/usb/fcp.c +++ b/sound/usb/fcp.c @@ -82,6 +82,7 @@ struct fcp_data { struct mutex mutex; /* serialise access to the device */ struct completion cmd_done; /* wait for command completion */ struct file *file; /* hwdep file */ + struct urb *urb; /* FCP notification endpoint */ struct fcp_notify notify; @@ -190,7 +191,7 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode, const int max_retries = 5; int err; - if (!mixer->urb) + if (!private->urb) return -ENODEV; struct fcp_usb_packet *req __free(kfree) = NULL; @@ -305,7 +306,7 @@ static int fcp_reinit(struct usb_mixer_interface *mixer) { struct fcp_data *private = mixer->private_data; - if (mixer->urb) + if (private->urb) return 0; void *step0_resp __free(kfree) = @@ -493,7 +494,7 @@ static int fcp_ioctl_init(struct usb_mixer_interface *mixer, buf_size = init.step0_resp_size + init.step2_resp_size; void *resp __free(kfree) = - kmalloc(buf_size, GFP_KERNEL); + kzalloc(buf_size, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -901,13 +902,15 @@ static int fcp_hwdep_init(struct usb_mixer_interface *mixer) static void fcp_cleanup_urb(struct usb_mixer_interface *mixer) { - if (!mixer->urb) + struct fcp_data *private = mixer->private_data; + + if (!private->urb) return; - usb_kill_urb(mixer->urb); - kfree(mixer->urb->transfer_buffer); - usb_free_urb(mixer->urb); - mixer->urb = NULL; + usb_kill_urb(private->urb); + kfree(private->urb->transfer_buffer); + usb_free_urb(private->urb); + private->urb = NULL; } static void fcp_private_free(struct usb_mixer_interface *mixer) @@ -978,37 +981,37 @@ static int fcp_init_notify(struct usb_mixer_interface *mixer) int err; /* Already set up */ - if (mixer->urb) + if (private->urb) return 0; if (usb_pipe_type_check(dev, pipe)) return -EINVAL; - mixer->urb = usb_alloc_urb(0, GFP_KERNEL); - if (!mixer->urb) + private->urb = usb_alloc_urb(0, GFP_KERNEL); + if (!private->urb) return -ENOMEM; transfer_buffer = kmalloc(private->wMaxPacketSize, GFP_KERNEL); if (!transfer_buffer) { - usb_free_urb(mixer->urb); - mixer->urb = NULL; + usb_free_urb(private->urb); + private->urb = NULL; return -ENOMEM; } - usb_fill_int_urb(mixer->urb, dev, pipe, + usb_fill_int_urb(private->urb, dev, pipe, transfer_buffer, private->wMaxPacketSize, fcp_notify, mixer, private->bInterval); - init_completion(&private->cmd_done); + reinit_completion(&private->cmd_done); - err = usb_submit_urb(mixer->urb, GFP_KERNEL); + err = usb_submit_urb(private->urb, GFP_KERNEL); if (err) { usb_audio_err(mixer->chip, "%s: usb_submit_urb failed: %d\n", __func__, err); kfree(transfer_buffer); - usb_free_urb(mixer->urb); - mixer->urb = NULL; + usb_free_urb(private->urb); + private->urb = NULL; } return err; @@ -1030,6 +1033,8 @@ static int fcp_init(struct usb_mixer_interface *mixer, step0_resp, private->step0_resp_size); if (err < 0) return err; + if (err != private->step0_resp_size) + return -EIO; err = fcp_init_notify(mixer); if (err < 0) @@ -1059,6 +1064,7 @@ static int fcp_init_private(struct usb_mixer_interface *mixer) return -ENOMEM; mutex_init(&private->mutex); + init_completion(&private->cmd_done); init_waitqueue_head(&private->notify.queue); spin_lock_init(&private->notify.lock); diff --git a/sound/usb/midi.c b/sound/usb/midi.c index f8996416c3be1..7e5b1b13360f0 100644 --- a/sound/usb/midi.c +++ b/sound/usb/midi.c @@ -875,6 +875,8 @@ static void snd_usbmidi_novation_output(struct snd_usb_midi_out_endpoint *ep, if (!ep->ports[0].active) return; + if (ep->max_transfer < 3) + return; transfer_buffer = urb->transfer_buffer; count = snd_rawmidi_transmit(ep->ports[0].substream, &transfer_buffer[2], @@ -969,6 +971,8 @@ static void snd_usbmidi_us122l_output(struct snd_usb_midi_out_endpoint *ep, default: count = 2; } + if (ep->max_transfer < count) + return; count = snd_rawmidi_transmit(ep->ports[0].substream, urb->transfer_buffer, count); diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index 703c118f9d4ed..5de182181edee 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -3935,6 +3935,12 @@ int snd_usb_mixer_resume(struct usb_mixer_interface *mixer) struct usb_mixer_elem_list *list; int id, err; + if (mixer->private_resume) { + err = mixer->private_resume(mixer); + if (err < 0) + return err; + } + /* restore cached mixer values */ for (id = 0; id < MAX_ID_ELEMS; id++) { for_each_mixer_elem(list, mixer, id) { diff --git a/sound/usb/mixer.h b/sound/usb/mixer.h index 3fa1bd96f8588..037b446d8b6f5 100644 --- a/sound/usb/mixer.h +++ b/sound/usb/mixer.h @@ -18,6 +18,7 @@ struct usb_mixer_interface { struct usb_host_interface *hostif; struct list_head list; unsigned int ignore_ctl_error; + /* UAC2 status interrupt endpoint; owned by mixer.c */ struct urb *urb; /* array[MAX_ID_ELEMS], indexed by unit id */ struct usb_mixer_elem_list **id_elems; @@ -42,6 +43,7 @@ struct usb_mixer_interface { void *private_data; void (*private_free)(struct usb_mixer_interface *mixer); void (*private_suspend)(struct usb_mixer_interface *mixer); + int (*private_resume)(struct usb_mixer_interface *mixer); }; #define MAX_CHANNELS 64 /* max logical channels */ diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c index 78fb72e626cac..502854cc9f9fa 100644 --- a/sound/usb/mixer_scarlett2.c +++ b/sound/usb/mixer_scarlett2.c @@ -1403,6 +1403,7 @@ struct scarlett2_data { struct usb_mixer_interface *mixer; struct mutex usb_mutex; /* prevent sending concurrent USB requests */ struct completion cmd_done; + struct urb *urb; /* notification endpoint */ struct mutex data_mutex; /* lock access to this data */ u8 running; u8 hwdep_in_use; @@ -8565,13 +8566,70 @@ static void scarlett2_notify(struct urb *urb) } } -/*** Cleanup/Suspend Callbacks ***/ +/*** Notification URB and Cleanup/Suspend Callbacks ***/ + +/* Submit a URB to receive notifications from the device */ +static int scarlett2_init_notify(struct usb_mixer_interface *mixer) +{ + struct usb_device *dev = mixer->chip->dev; + struct scarlett2_data *private = mixer->private_data; + unsigned int pipe = usb_rcvintpipe(dev, private->bEndpointAddress); + void *transfer_buffer; + int err; + + /* Already set up */ + if (private->urb) + return 0; + + if (usb_pipe_type_check(dev, pipe)) + return -EINVAL; + + private->urb = usb_alloc_urb(0, GFP_KERNEL); + if (!private->urb) + return -ENOMEM; + + transfer_buffer = kmalloc(private->wMaxPacketSize, GFP_KERNEL); + if (!transfer_buffer) { + usb_free_urb(private->urb); + private->urb = NULL; + return -ENOMEM; + } + + usb_fill_int_urb(private->urb, dev, pipe, + transfer_buffer, private->wMaxPacketSize, + scarlett2_notify, mixer, private->bInterval); + + reinit_completion(&private->cmd_done); + + err = usb_submit_urb(private->urb, GFP_KERNEL); + if (err) { + kfree(transfer_buffer); + usb_free_urb(private->urb); + private->urb = NULL; + } + + return err; +} + +static void scarlett2_cleanup_urb(struct usb_mixer_interface *mixer) +{ + struct scarlett2_data *private = mixer->private_data; + + if (!private->urb) + return; + + usb_kill_urb(private->urb); + kfree(private->urb->transfer_buffer); + usb_free_urb(private->urb); + private->urb = NULL; +} static void scarlett2_private_free(struct usb_mixer_interface *mixer) { struct scarlett2_data *private = mixer->private_data; cancel_delayed_work_sync(&private->work); + scarlett2_cleanup_urb(mixer); kfree(private); mixer->private_data = NULL; } @@ -8582,6 +8640,8 @@ static void scarlett2_private_suspend(struct usb_mixer_interface *mixer) if (cancel_delayed_work_sync(&private->work)) scarlett2_config_save(private->mixer); + + scarlett2_cleanup_urb(mixer); } /*** Initialisation ***/ @@ -8701,11 +8761,13 @@ static int scarlett2_init_private(struct usb_mixer_interface *mixer, mutex_init(&private->usb_mutex); mutex_init(&private->data_mutex); + init_completion(&private->cmd_done); INIT_DELAYED_WORK(&private->work, scarlett2_config_save_work); mixer->private_data = private; mixer->private_free = scarlett2_private_free; mixer->private_suspend = scarlett2_private_suspend; + mixer->private_resume = scarlett2_init_notify; private->info = entry->info; @@ -8722,40 +8784,6 @@ static int scarlett2_init_private(struct usb_mixer_interface *mixer, return scarlett2_find_fc_interface(mixer->chip->dev, private); } -/* Submit a URB to receive notifications from the device */ -static int scarlett2_init_notify(struct usb_mixer_interface *mixer) -{ - struct usb_device *dev = mixer->chip->dev; - struct scarlett2_data *private = mixer->private_data; - unsigned int pipe = usb_rcvintpipe(dev, private->bEndpointAddress); - void *transfer_buffer; - - if (mixer->urb) { - usb_audio_err(mixer->chip, - "%s: mixer urb already in use!\n", __func__); - return 0; - } - - if (usb_pipe_type_check(dev, pipe)) - return -EINVAL; - - mixer->urb = usb_alloc_urb(0, GFP_KERNEL); - if (!mixer->urb) - return -ENOMEM; - - transfer_buffer = kmalloc(private->wMaxPacketSize, GFP_KERNEL); - if (!transfer_buffer) - return -ENOMEM; - - usb_fill_int_urb(mixer->urb, dev, pipe, - transfer_buffer, private->wMaxPacketSize, - scarlett2_notify, mixer, private->bInterval); - - init_completion(&private->cmd_done); - - return usb_submit_urb(mixer->urb, GFP_KERNEL); -} - /* Cargo cult proprietary initialisation sequence */ static int scarlett2_usb_init(struct usb_mixer_interface *mixer) { diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h index 71444c2898b4b..9a28d7c6d923a 100644 --- a/sound/usb/quirks-table.h +++ b/sound/usb/quirks-table.h @@ -2692,6 +2692,7 @@ YAMAHA_DEVICE(0x7010, "UB99"), { QUIRK_DATA_AUDIOFORMAT(2) { .formats = SNDRV_PCM_FMTBIT_S24_3LE, + .fmt_bits = 24, .channels = 2, .iface = 2, .altsetting = 1, @@ -2703,11 +2704,16 @@ YAMAHA_DEVICE(0x7010, "UB99"), SNDRV_PCM_RATE_48000, .rate_min = 44100, .rate_max = 48000, + .nr_rates = 2, + .rate_table = (unsigned int[]) { + 44100, 48000 + }, } }, { QUIRK_DATA_AUDIOFORMAT(3) { .formats = SNDRV_PCM_FMTBIT_S24_3LE, + .fmt_bits = 24, .channels = 2, .iface = 3, .altsetting = 1, @@ -2719,6 +2725,10 @@ YAMAHA_DEVICE(0x7010, "UB99"), SNDRV_PCM_RATE_48000, .rate_min = 44100, .rate_max = 48000, + .nr_rates = 2, + .rate_table = (unsigned int[]) { + 44100, 48000 + }, } }, QUIRK_COMPOSITE_END diff --git a/sound/usb/usx2y/us122l.c b/sound/usb/usx2y/us122l.c index f00b53346abd3..a5db0d044ef96 100644 --- a/sound/usb/usx2y/us122l.c +++ b/sound/usb/usx2y/us122l.c @@ -180,8 +180,11 @@ static int usb_stream_hwdep_mmap(struct snd_hwdep *hw, guard(mutex)(&us122l->mutex); s = us122l->sk.s; read = offset < s->read_size; - if (read && area->vm_flags & VM_WRITE) - return -EPERM; + if (read) { + if (area->vm_flags & VM_WRITE) + return -EPERM; + vm_flags_clear(area, VM_MAYWRITE); + } /* if userspace tries to mmap beyond end of our buffer, fail */ if (size > PAGE_ALIGN(read ? s->read_size : s->write_size)) { dev_warn(hw->card->dev, "%s: size %lu > %u\n", __func__, diff --git a/sound/usb/usx2y/usbusx2y.c b/sound/usb/usx2y/usbusx2y.c index 4190227c5a2a5..108f9bddf9aa3 100644 --- a/sound/usb/usx2y/usbusx2y.c +++ b/sound/usb/usx2y/usbusx2y.c @@ -189,6 +189,9 @@ static void i_usx2y_in04_int(struct urb *urb) return; } + if (urb->actual_length < USX2Y_IN04_SIZE) + goto resubmit; + if (us428ctls) { diff = -1; if (us428ctls->ctl_snapshot_last == -2) { @@ -196,7 +199,7 @@ static void i_usx2y_in04_int(struct urb *urb) memcpy(usx2y->in04_last, usx2y->in04_buf, sizeof(usx2y->in04_last)); us428ctls->ctl_snapshot_last = -1; } else { - for (i = 0; i < 21; i++) { + for (i = 0; i < USX2Y_IN04_SIZE; i++) { if (usx2y->in04_last[i] != ((char *)usx2y->in04_buf)[i]) { if (diff < 0) diff = i; @@ -253,6 +256,7 @@ static void i_usx2y_in04_int(struct urb *urb) if (err) dev_err(&urb->dev->dev, "in04_int() usb_submit_urb err=%i\n", err); +resubmit: urb->dev = usx2y->dev; usb_submit_urb(urb, GFP_ATOMIC); } @@ -305,7 +309,7 @@ int usx2y_in04_init(struct usx2ydev *usx2y) goto error; } - usx2y->in04_buf = kmalloc(21, GFP_KERNEL); + usx2y->in04_buf = kzalloc(USX2Y_IN04_SIZE, GFP_KERNEL); if (!usx2y->in04_buf) { err = -ENOMEM; goto error; @@ -313,7 +317,7 @@ int usx2y_in04_init(struct usx2ydev *usx2y) init_waitqueue_head(&usx2y->in04_wait_queue); usb_fill_int_urb(usx2y->in04_urb, usx2y->dev, usb_rcvintpipe(usx2y->dev, 0x4), - usx2y->in04_buf, 21, + usx2y->in04_buf, USX2Y_IN04_SIZE, i_usx2y_in04_int, usx2y, 10); if (usb_urb_ep_type_check(usx2y->in04_urb)) { diff --git a/sound/usb/usx2y/usbusx2y.h b/sound/usb/usx2y/usbusx2y.h index 6a76d04bf1c7d..7b6deed17d4b1 100644 --- a/sound/usb/usx2y/usbusx2y.h +++ b/sound/usb/usx2y/usbusx2y.h @@ -5,6 +5,8 @@ #include "../midi.h" #include "usbus428ctldefs.h" +#define USX2Y_IN04_SIZE sizeof(struct us428_ctls) + #define NRURBS 2 /* Default value used for nr of packs per urb. @@ -55,7 +57,7 @@ struct usx2ydev { int stride; struct urb *in04_urb; void *in04_buf; - char in04_last[24]; + char in04_last[USX2Y_IN04_SIZE]; unsigned int in04_int_calls; struct snd_usx2y_urb_seq *us04; wait_queue_head_t in04_wait_queue; diff --git a/sound/virtio/virtio_card.c b/sound/virtio/virtio_card.c index 647190f4d5afc..6f35276416fed 100644 --- a/sound/virtio/virtio_card.c +++ b/sound/virtio/virtio_card.c @@ -354,8 +354,8 @@ static void virtsnd_remove(struct virtio_device *vdev) if (snd->card) snd_card_free(snd->card); - vdev->config->del_vqs(vdev); virtio_reset_device(vdev); + vdev->config->del_vqs(vdev); for (i = 0; snd->substreams && i < snd->nsubstreams; ++i) { struct virtio_pcm_substream *vss = &snd->substreams[i]; @@ -383,8 +383,8 @@ static int virtsnd_freeze(struct virtio_device *vdev) virtsnd_disable_event_vq(snd); virtsnd_ctl_msg_cancel_all(snd); - vdev->config->del_vqs(vdev); virtio_reset_device(vdev); + vdev->config->del_vqs(vdev); for (i = 0; i < snd->nsubstreams; ++i) cancel_work_sync(&snd->substreams[i].elapsed_period); diff --git a/tools/arch/riscv/include/asm/csr.h b/tools/arch/riscv/include/asm/csr.h index 21d8cee046383..8df64314d6131 100644 --- a/tools/arch/riscv/include/asm/csr.h +++ b/tools/arch/riscv/include/asm/csr.h @@ -163,12 +163,24 @@ #define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT #endif -/* VSIP & HVIP relation */ +/* + * VSIP & HVIP relation + * + * The bit positions are same between VSIP and HVIP for interrupt + * numbers 13-63, where there's a shift for the SSI, STI and SEI. + */ #define VSIP_TO_HVIP_SHIFT (IRQ_VS_SOFT - IRQ_S_SOFT) -#define VSIP_VALID_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ +#define VSIP_BIAS_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ (_AC(1, UL) << IRQ_S_TIMER) | \ - (_AC(1, UL) << IRQ_S_EXT) | \ - (_AC(1, UL) << IRQ_PMU_OVF)) + (_AC(1, UL) << IRQ_S_EXT)) +#define VSIP_NO_BIAS_MASK (_AC(1, UL) << IRQ_PMU_OVF) +#define VSIP_VALID_MASK (VSIP_BIAS_MASK | VSIP_NO_BIAS_MASK) +#define vsip_to_hvip(_vsip) ((((_vsip) & VSIP_BIAS_MASK) << \ + VSIP_TO_HVIP_SHIFT) | \ + ((_vsip) & VSIP_NO_BIAS_MASK)) +#define hvip_to_vsip(_hvip) ((((_hvip) >> VSIP_TO_HVIP_SHIFT) & \ + VSIP_BIAS_MASK) | \ + ((_hvip) & VSIP_NO_BIAS_MASK)) /* AIA CSR bits */ #define TOPI_IID_SHIFT 16 diff --git a/tools/bootconfig/main.c b/tools/bootconfig/main.c index 7dc9fff9b6370..17d971d47f879 100644 --- a/tools/bootconfig/main.c +++ b/tools/bootconfig/main.c @@ -140,6 +140,9 @@ static int load_xbc_fd(int fd, char **buf, int size) { int ret; + if (size < 0 || size > XBC_DATA_MAX) + return -EINVAL; + *buf = malloc(size + 1); if (!*buf) return -ENOMEM; @@ -168,6 +171,13 @@ static int load_xbc_file(const char *path, char **buf) return ret; } + if (stat.st_size > XBC_DATA_MAX) { + pr_err("%s size is too big\n", path); + ret = -E2BIG; + close(fd); + return ret; + } + ret = load_xbc_fd(fd, buf, stat.st_size); close(fd); @@ -218,7 +228,8 @@ static int load_xbc_from_initrd(int fd, char **buf) csum = le32toh(csum); /* Wrong size error */ - if (stat.st_size < size + BOOTCONFIG_FOOTER_SIZE) { + if (size > XBC_DATA_MAX || + size > stat.st_size - BOOTCONFIG_FOOTER_SIZE) { pr_err("bootconfig size is too big\n"); return -E2BIG; } diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 271a7dc772730..b0f7168e79432 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -99,7 +99,7 @@ endif HOST_LDFLAGS := $(LDFLAGS) # Remove warnings for libbpf bootstrap build -LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat -Wformat-signedness,$(HOST_CFLAGS)) +LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat%,$(HOST_CFLAGS)) INSTALL ?= install RM ?= rm -f diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 71a45d96617ed..684a8fb724146 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -659,8 +659,6 @@ static int do_show_subset(int argc, char **argv) show_map_close_json(fds[i], &info); else show_map_close_plain(fds[i], &info); - - close(fds[i]); } if (json_output && nb_fds > 1) jsonw_end_array(json_wtr); /* root array */ @@ -790,6 +788,12 @@ static int maps_have_btf(int *fds, int nb_fds) static struct btf *btf_vmlinux; +static void free_btf_vmlinux(void) +{ + btf__free(btf_vmlinux); + btf_vmlinux = NULL; +} + static int get_map_kv_btf(const struct bpf_map_info *info, struct btf **btf) { int err = 0; @@ -889,7 +893,6 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, exit_free: free(key); free(value); - close(fd); free_map_kv_btf(btf); return err; @@ -938,6 +941,7 @@ static int do_dump(int argc, char **argv) for (i = 0; i < nb_fds; i++) { if (bpf_map_get_info_by_fd(fds[i], &info, &len)) { p_err("can't get map info: %s", strerror(errno)); + err = -1; break; } err = map_dump(fds[i], &info, wtr, nb_fds > 1); @@ -958,7 +962,7 @@ static int do_dump(int argc, char **argv) close(fds[i]); exit_free: free(fds); - btf__free(btf_vmlinux); + free_btf_vmlinux(); return err; } @@ -1049,7 +1053,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, btf_wtr = get_btf_writer(); if (!btf_wtr) { p_info("failed to create json writer for btf. falling back to plain output"); - btf__free(btf); + free_map_kv_btf(btf); btf = NULL; print_entry_plain(info, key, value); } else { @@ -1065,7 +1069,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, } else { print_entry_plain(info, key, value); } - btf__free(btf); + free_map_kv_btf(btf); } static int do_lookup(int argc, char **argv) diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c index f9b742f4bb104..1257dba8ef2fd 100644 --- a/tools/bpf/bpftool/sign.c +++ b/tools/bpf/bpftool/sign.c @@ -175,8 +175,11 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } - EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash, - &opts->excl_prog_hash_sz, EVP_sha256(), NULL); + if (EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash, + &opts->excl_prog_hash_sz, EVP_sha256(), NULL) != 1) { + err = -EIO; + goto cleanup; + } bd_out = BIO_new(BIO_s_mem()); if (!bd_out) { diff --git a/tools/bpf/bpftool/struct_ops.c b/tools/bpf/bpftool/struct_ops.c index aa43dead249cb..835e5e561f7fc 100644 --- a/tools/bpf/bpftool/struct_ops.c +++ b/tools/bpf/bpftool/struct_ops.c @@ -643,6 +643,10 @@ int do_struct_ops(int argc, char **argv) err = cmd_select(cmds, argc, argv, do_help); btf__free(btf_vmlinux); + btf_vmlinux = NULL; + map_info_type = NULL; + map_info_alloc_len = 0; + map_info_type_id = 0; return err; } diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index ed1374af31c16..99eb0ea095375 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -123,9 +123,7 @@ FEATURE_TESTS_EXTRA := \ libbfd-liberty-z \ libopencsd \ libperl \ - cxx \ llvm \ - clang \ libbpf \ libpfm4 \ libdebuginfod \ diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 62909a9c799d3..7d165018116a5 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -55,7 +55,6 @@ FILES= \ test-bpf.bin \ test-libbpf.bin \ test-sdt.bin \ - test-cxx.bin \ test-gettid.bin \ test-jvmti.bin \ test-jvmti-cmlr.bin \ @@ -63,7 +62,6 @@ FILES= \ test-sched_getcpu.bin \ test-setns.bin \ test-libopencsd.bin \ - test-clang.bin \ test-llvm.bin \ test-llvm-perf.bin \ test-libaio.bin \ @@ -335,9 +333,6 @@ $(OUTPUT)test-libbpf.bin: $(OUTPUT)test-sdt.bin: $(BUILD) -$(OUTPUT)test-cxx.bin: - $(BUILDXX) -std=gnu++11 - $(OUTPUT)test-gettid.bin: $(BUILD) @@ -363,15 +358,6 @@ $(OUTPUT)test-llvm-perf.bin: $(shell $(LLVM_CONFIG) --system-libs) \ > $(@:.bin=.make.output) 2>&1 -$(OUTPUT)test-clang.bin: - $(BUILDXX) -std=gnu++17 \ - -I$(shell $(LLVM_CONFIG) --includedir) \ - -L$(shell $(LLVM_CONFIG) --libdir) \ - -Wl,--start-group -lclang-cpp -Wl,--end-group \ - $(shell $(LLVM_CONFIG) --libs Core option) \ - $(shell $(LLVM_CONFIG) --system-libs) \ - > $(@:.bin=.make.output) 2>&1 - -include $(OUTPUT)*.d $(OUTPUT)test-libaio.bin: diff --git a/tools/include/linux/compiler.h b/tools/include/linux/compiler.h index f40bd2b04c298..f2f54b0381680 100644 --- a/tools/include/linux/compiler.h +++ b/tools/include/linux/compiler.h @@ -119,7 +119,7 @@ #define __read_mostly #ifndef __attribute_const__ -# define __attribute_const__ +# define __attribute_const__ __attribute__ ((__const__)) #endif #ifndef __maybe_unused diff --git a/tools/include/linux/filter.h b/tools/include/linux/filter.h index bcc6df79301ae..4ead4e72097c1 100644 --- a/tools/include/linux/filter.h +++ b/tools/include/linux/filter.h @@ -6,6 +6,7 @@ #define __TOOLS_LINUX_FILTER_H #include +#include /* ArgX, context and stack frame pointer register positions. Note, * Arg1, Arg2, Arg3, etc are used as argument mappings of function diff --git a/tools/include/nolibc/arch-powerpc.h b/tools/include/nolibc/arch-powerpc.h index a1ab91d553845..dbe2e5205aaaf 100644 --- a/tools/include/nolibc/arch-powerpc.h +++ b/tools/include/nolibc/arch-powerpc.h @@ -26,7 +26,7 @@ */ #define _NOLIBC_SYSCALL_CLOBBERLIST \ - "memory", "cr0", "r12", "r11", "r10", "r9" + "memory", "cr0", "ctr", "xer", "r12", "r11", "r10", "r9" #define __nolibc_syscall0(num) \ ({ \ diff --git a/tools/include/nolibc/arch-sparc.h b/tools/include/nolibc/arch-sparc.h index ddae9bc10dfe3..23fab40accfad 100644 --- a/tools/include/nolibc/arch-sparc.h +++ b/tools/include/nolibc/arch-sparc.h @@ -45,7 +45,7 @@ \ __asm__ volatile ( \ _NOLIBC_SYSCALL \ - : "+r"(_arg1) \ + : "=r"(_arg1) \ : "r"(_num) \ : "memory", "cc" \ ); \ diff --git a/tools/include/uapi/linux/if_xdp.h b/tools/include/uapi/linux/if_xdp.h index 23a0627814687..50d67df78911f 100644 --- a/tools/include/uapi/linux/if_xdp.h +++ b/tools/include/uapi/linux/if_xdp.h @@ -149,6 +149,7 @@ struct xsk_tx_metadata { __u16 csum_start; /* Offset from csum_start where checksum should be stored. */ __u16 csum_offset; + __u32 reserved; /* XDP_TXMD_FLAGS_LAUNCH_TIME */ /* Launch time in nanosecond against the PTP HW Clock */ diff --git a/tools/lib/api/fd/array.c b/tools/lib/api/fd/array.c index f0f195207fca9..16a047f1906ee 100644 --- a/tools/lib/api/fd/array.c +++ b/tools/lib/api/fd/array.c @@ -115,6 +115,9 @@ int fdarray__filter(struct fdarray *fda, short revents, return 0; for (fd = 0; fd < fda->nr; ++fd) { + if (fda->priv[fd].flags & fdarray_flag__nonfilterable) + continue; + if (!fda->entries[fd].events) continue; @@ -122,12 +125,17 @@ int fdarray__filter(struct fdarray *fda, short revents, if (entry_destructor) entry_destructor(fda, fd, arg); + /* + * Set fd to -1 so poll() ignores this entry; otherwise + * POLLHUP/POLLERR are still reported for events=0 fds + * (POSIX: always checked), causing a poll storm. + */ + fda->entries[fd].fd = -1; fda->entries[fd].revents = fda->entries[fd].events = 0; continue; } - if (!(fda->priv[fd].flags & fdarray_flag__nonfilterable)) - ++nr; + ++nr; } return nr; diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index b7e388f99d0bb..e5641fa601637 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -577,10 +577,10 @@ static int probe_ldimm64_full_range_off(int token_fd) static int probe_uprobe_syscall(int token_fd) { /* - * If kernel supports uprobe() syscall, it will return -ENXIO when called + * If kernel supports uprobe() syscall, it will return -EPROTO when called * from the outside of a kernel-generated uprobe trampoline. */ - return syscall(__NR_uprobe) < 0 && errno == ENXIO; + return syscall(__NR_uprobe) < 0 && errno == EPROTO; } #else static int probe_uprobe_syscall(int token_fd) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 1368752aa13c3..4f9ce4ff005b6 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -6162,6 +6162,13 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) return -EINVAL; insn = &prog->insns[insn_idx]; + if (is_ldimm64_insn(insn) && (size_t)insn_idx + 1 >= prog->insns_cnt) { + pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) is truncated\n", + prog->name, i, insn_idx); + err = -EINVAL; + goto out; + } + err = record_relo_core(prog, rec, insn_idx); if (err) { pr_warn("prog '%s': relo #%d: failed to record relocation: %s\n", @@ -12973,13 +12980,14 @@ static const char *arch_specific_lib_paths(void) /* Get full path to program/shared library. */ static int resolve_full_path(const char *file, char *result, size_t result_sz) { - const char *search_paths[3] = {}; + const char *search_paths[4] = {}; int i, perm; if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { search_paths[0] = getenv("LD_LIBRARY_PATH"); search_paths[1] = "/usr/lib64:/usr/lib"; search_paths[2] = arch_specific_lib_paths(); + search_paths[3] = "/lib64:/lib"; perm = R_OK; } else { search_paths[0] = getenv("PATH"); diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index 6ae3f2a15ad0c..4755de3f99957 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -980,23 +980,30 @@ static int bpf_core_calc_relo(const char *prog_name, } /* - * Turn instruction for which CO_RE relocation failed into invalid one with + * Turn instruction for which CO-RE relocation failed into invalid one with * distinct signature. */ -static void bpf_core_poison_insn(const char *prog_name, int relo_idx, - int insn_idx, struct bpf_insn *insn) +static int bpf_core_poison_insn(const char *prog_name, int relo_idx, + struct bpf_insn *insn, int insn_idx) { - pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n", - prog_name, relo_idx, insn_idx); - insn->code = BPF_JMP | BPF_CALL; - insn->dst_reg = 0; - insn->src_reg = 0; - insn->off = 0; - /* if this instruction is reachable (not a dead code), - * verifier will complain with the following message: - * invalid func unknown#195896080 - */ - insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */ + int insn_cnt = is_ldimm64_insn(insn) ? 2 : 1; + int i; + + for (i = 0; i < insn_cnt; i++) { + pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n", + prog_name, relo_idx, insn_idx + i); + insn[i].code = BPF_JMP | BPF_CALL; + insn[i].dst_reg = 0; + insn[i].src_reg = 0; + insn[i].off = 0; + /* + * If this instruction is reachable (not dead code), the verifier + * will complain with "invalid func unknown#195896080". + */ + insn[i].imm = 195896080; /* => 0xbad2310 => "bad relo" */ + } + + return 0; } static int insn_bpf_size_to_bytes(struct bpf_insn *insn) @@ -1047,17 +1054,6 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, class = BPF_CLASS(insn->code); - if (res->poison) { -poison: - /* poison second part of ldimm64 to avoid confusing error from - * verifier about "unknown opcode 00" - */ - if (is_ldimm64_insn(insn)) - bpf_core_poison_insn(prog_name, relo_idx, insn_idx + 1, insn + 1); - bpf_core_poison_insn(prog_name, relo_idx, insn_idx, insn); - return 0; - } - orig_val = res->orig_val; new_val = res->new_val; @@ -1065,7 +1061,9 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, case BPF_ALU: case BPF_ALU64: if (BPF_SRC(insn->code) != BPF_K) - return -EINVAL; + goto bad_insn; + if (res->poison) + return bpf_core_poison_insn(prog_name, relo_idx, insn, insn_idx); if (res->validate && insn->imm != orig_val) { pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n", prog_name, relo_idx, @@ -1082,6 +1080,8 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, case BPF_LDX: case BPF_ST: case BPF_STX: + if (res->poison) + return bpf_core_poison_insn(prog_name, relo_idx, insn, insn_idx); if (res->validate && insn->off != orig_val) { pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val, @@ -1097,7 +1097,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. " "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n", prog_name, relo_idx, insn_idx); - goto poison; + return bpf_core_poison_insn(prog_name, relo_idx, insn, insn_idx); } orig_val = insn->off; @@ -1140,6 +1140,9 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, return -EINVAL; } + if (res->poison) + return bpf_core_poison_insn(prog_name, relo_idx, insn, insn_idx); + imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32); if (res->validate && imm != orig_val) { pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n", @@ -1157,6 +1160,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, break; } default: +bad_insn: pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n", prog_name, relo_idx, insn_idx, insn->code, insn->src_reg, insn->dst_reg, insn->off, insn->imm); diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index 57fb82bb81b58..d2ecd3daab961 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -305,7 +305,7 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj) /* * Detect kernel support for uprobe() syscall, it's presence means we can - * take advantage of faster nop5 uprobe handling. + * take advantage of faster nop10 uprobe handling. * Added in: 56101b69c919 ("uprobes/x86: Add uprobe syscall to speed up uprobe") */ man->has_uprobe_syscall = kernel_supports(obj, FEAT_UPROBE_SYSCALL); @@ -604,14 +604,14 @@ static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, #if defined(__x86_64__) static bool has_nop_combo(int fd, long off) { - unsigned char nop_combo[6] = { - 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 /* nop,nop5 */ + unsigned char nop_combo[11] = { + 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, }; - unsigned char buf[6]; + unsigned char buf[11]; - if (pread(fd, buf, 6, off) != 6) + if (pread(fd, buf, 11, off) != 11) return false; - return memcmp(buf, nop_combo, 6) == 0; + return memcmp(buf, nop_combo, 11) == 0; } #else static bool has_nop_combo(int fd, long off) @@ -822,8 +822,8 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, memset(target, 0, sizeof(*target)); /* - * We have uprobe syscall and usdt with nop,nop5 instructions combo, - * so we can place the uprobe directly on nop5 (+1) and get this probe + * We have uprobe syscall and usdt with nop,nop10 instructions combo, + * so we can place the uprobe directly on nop10 (+1) and get this probe * optimized. */ if (man->has_uprobe_syscall && has_nop_combo(elf_fd->fd, usdt_rel_ip)) { diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 index c7ae506076bb4..d1405c7c5354a 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 @@ -1,3 +1,4 @@ {# SPDX-License-Identifier: GPL-2.0 #} + bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ name }} *ptr); bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, {{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 index cf65b71eaef39..7ddc2bf3edacc 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length string) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 index 1d477c2d197aa..5bf00070ae959 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length opaque) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 index cf65b71eaef39..7ddc2bf3edacc 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length string) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 index 1d477c2d197aa..5bf00070ae959 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length opaque) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 index 5fc1937ba7744..19ee759d70c6a 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 @@ -1,9 +1,3 @@ {# SPDX-License-Identifier: GPL-2.0 #} } u; }; -{%- if name in public_apis %} - - -bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr); -bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *ptr); -{%- endif -%} diff --git a/tools/objtool/Build b/tools/objtool/Build index 93a37b0dfd313..59f9486280981 100644 --- a/tools/objtool/Build +++ b/tools/objtool/Build @@ -6,13 +6,15 @@ objtool-y += check.o objtool-y += special.o objtool-y += builtin-check.o objtool-y += elf.o +objtool-y += klp-symid.o objtool-y += objtool.o objtool-$(BUILD_DISAS) += disas.o objtool-$(BUILD_DISAS) += trace.o objtool-$(BUILD_ORC) += orc_gen.o orc_dump.o -objtool-$(BUILD_KLP) += builtin-klp.o klp-checksum.o klp-diff.o klp-post-link.o +objtool-$(BUILD_KLP) += builtin-klp.o klp-checksum.o klp-diff.o \ + klp-post-link.o klp-sympos.o objtool-y += libstring.o objtool-y += libctype.o diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index a4484fd22a96d..4cc2e756af840 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -89,9 +89,11 @@ LIBOPCODES_LIBS := $(shell \ "-lopcodes -lbfd" \ "-lopcodes -lbfd -liberty" \ "-lopcodes -lbfd -liberty -lz"; do \ - echo 'extern void disassemble_init_for_target(void *);' \ - 'int main(void) { disassemble_init_for_target(0); return 0; }' | \ - $(HOSTCC) -xc - -o /dev/null $$libs 2>/dev/null && \ + printf '%s\n' \ + '$(pound)include ' \ + '$(pound)include ' \ + 'int main(void) { disassemble_init_for_target(0); return 0; }' | \ + $(HOSTCC) $(HOSTCFLAGS) -DPACKAGE='"objtool"' -xc - -o /dev/null $$libs 2>/dev/null && \ echo "$$libs" && break; \ done) diff --git a/tools/objtool/arch/x86/special.c b/tools/objtool/arch/x86/special.c index e817a3fff4491..1e84c81bfcd81 100644 --- a/tools/objtool/arch/x86/special.c +++ b/tools/objtool/arch/x86/special.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include #include @@ -9,6 +10,32 @@ /* cpu feature name array generated from cpufeatures.h */ #include "cpu-feature-names.c" +/* + * An alternative with an empty replacement, e.g. the second entry of + * + * ALTERNATIVE_2("orig", "repl", ft1, "", ft2) + * + * still gets a relocation for its replacement offset. But the label it points + * at is the end of the previous entry's replacement, which is also the + * beginning of the *next* entry's replacement. The value is meaningless: it's + * only ever used with a length of zero. + */ +bool arch_alt_ignore_new_reloc(struct section *sec, unsigned long offset) +{ + unsigned long entry_off; + + if (strcmp(sec->name, ".altinstructions")) + return false; + + entry_off = offset - (offset % ALT_ENTRY_SIZE); + + if (offset - entry_off != ALT_NEW_OFFSET) + return false; + + return !*(unsigned char *)(sec->data->d_buf + entry_off + + ALT_NEW_LEN_OFFSET); +} + void arch_handle_alternative(struct special_alt *alt) { static struct special_alt *group, *prev; diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c index 118c3de2f293e..75b11dc85010e 100644 --- a/tools/objtool/builtin-check.c +++ b/tools/objtool/builtin-check.c @@ -76,6 +76,7 @@ static const struct option check_options[] = { OPT_STRING_OPTARG('d', "disas", &opts.disas, "function-pattern", "disassemble functions", "*"), OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr,skylake", "patch toolchain bugs/limitations", parse_hacks), OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"), + OPT_BOOLEAN(0, "klp-symids", &opts.klp_symids, "generate .klp.symids for duplicate symbol disambiguation"), OPT_BOOLEAN('m', "mcount", &opts.mcount, "annotate mcount/fentry calls for ftrace"), OPT_BOOLEAN(0, "noabs", &opts.noabs, "reject absolute references in allocatable sections"), OPT_BOOLEAN('n', "noinstr", &opts.noinstr, "validate noinstr rules"), @@ -174,10 +175,16 @@ static bool opts_valid(void) return false; } + if (opts.klp_symids && !opts.link) { + ERROR("--klp-symids requires --link"); + return false; + } + if (opts.disas || opts.hack_jump_label || opts.hack_noinstr || opts.ibt || + opts.klp_symids || opts.mcount || opts.noabs || opts.noinstr || diff --git a/tools/objtool/check.c b/tools/objtool/check.c index f03dd59e7fca1..28cc2fed6f15e 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -194,6 +195,8 @@ static bool is_rust_noreturn(const struct symbol *func) */ return str_ends_with(func->name, "_4core3num20from_str_radix_panic") || str_ends_with(func->name, "_4core3num22from_ascii_radix_panic") || + str_ends_with(func->name, "_4core3num28from_ascii_bytes_radix_panic") || + str_ends_with(func->name, "_4core3str16slice_error_fail") || str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || str_ends_with(func->name, "_4core6option13expect_failed") || str_ends_with(func->name, "_4core6option13unwrap_failed") || @@ -4923,6 +4926,12 @@ int check(struct objtool_file *file) goto out; } + if (opts.klp_symids) { + ret = klp_create_symid_sections(file); + if (ret) + goto out; + } + if (opts.noabs) warnings += check_abs_references(file); diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index 33c95a74a51bd..a791f4ea6ec19 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -23,6 +23,7 @@ #include #include #include +#include #include static ssize_t demangled_name_len(const char *name); @@ -626,6 +627,18 @@ static int read_symbols(struct elf *elf) return -1; } + /* + * "klp diff" renames the placeholder symbols of KLP relocs to + * hide them from modpost. Hide the prefix from the rest of + * objtool so its many name-based heuristics (noreturns, + * uaccess safe list, ...) still see the original symbol name. + * + * st_name is left alone, so the renamed symbol is preserved in + * the output file. + */ + if (strstarts(sym->name, KLP_TOMBSTONE_PREFIX)) + sym->name += strlen(KLP_TOMBSTONE_PREFIX); + if ((sym->sym.st_shndx > SHN_UNDEF && sym->sym.st_shndx < SHN_LORESERVE) || (shndx_data && sym->sym.st_shndx == SHN_XINDEX)) { diff --git a/tools/objtool/include/objtool/builtin.h b/tools/objtool/include/objtool/builtin.h index e844e9c82b7b2..349690bb1c50e 100644 --- a/tools/objtool/include/objtool/builtin.h +++ b/tools/objtool/include/objtool/builtin.h @@ -16,6 +16,7 @@ struct opts { bool hack_noinstr; bool hack_skylake; bool ibt; + bool klp_symids; bool mcount; bool noabs; bool noinstr; diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index d9c44df9cc76a..a82517a76a0f6 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -97,6 +97,7 @@ struct symbol { u8 included : 1; u8 klp : 1; u8 dont_correlate : 1; + u8 fake : 1; struct list_head pv_target; struct reloc *relocs; struct section *group_sec; diff --git a/tools/objtool/include/objtool/klp.h b/tools/objtool/include/objtool/klp.h index 6f60cf05db864..c57775d78c71e 100644 --- a/tools/objtool/include/objtool/klp.h +++ b/tools/objtool/include/objtool/klp.h @@ -14,21 +14,48 @@ #define KLP_FUNCS_SEC ".init.klp_funcs" /* - * __klp_relocs is an intermediate section which are created by klp diff and - * converted into KLP symbols/relas by "objtool klp post-link". This is needed - * to work around the linker, which doesn't preserve SHN_LIVEPATCH or + * __klp_relocs. are intermediate sections which are created by klp + * diff and converted into KLP symbols/relas by "objtool klp post-link". This + * is needed to work around the linker, which doesn't preserve SHN_LIVEPATCH or * SHF_RELA_LIVEPATCH, nor does it support having two RELA sections for a * single PROGBITS section. + * + * "objname" is the object whose loading gates the relocation: "vmlinux" for + * references to vmlinux symbols, otherwise the name of the module being + * patched. post-link uses it to name the resulting + * .klp.rela.objname.section_name sections. */ #define KLP_RELOCS_SEC "__klp_relocs" #define KLP_STRINGS_SEC ".rodata.klp.str1.1" +#define KLP_TOMBSTONE_PREFIX ".klp.tombstone." + struct klp_reloc { void *offset; void *sym; u32 type; }; +/* + * .klp.symid is used to correlate symbols between vmlinux.o and vmlinux, for + * calculating sympos to disambiguate duplicately-named symbols. + */ +#define KLP_SYMID_SEC ".klp.symid" + +struct klp_symid { + u64 id; + u64 addr; +}; + +struct objtool_file; +struct elf; +struct symbol; + +int klp_create_symid_sections(struct objtool_file *file); + +int klp_sympos_init(struct elf *orig); +unsigned long klp_find_sympos(struct elf *elf, struct symbol *sym); + int cmd_klp_checksum(int argc, const char **argv); int cmd_klp_diff(int argc, const char **argv); int cmd_klp_post_link(int argc, const char **argv); diff --git a/tools/objtool/include/objtool/special.h b/tools/objtool/include/objtool/special.h index 121c3761899c1..620dbf6cb0e58 100644 --- a/tools/objtool/include/objtool/special.h +++ b/tools/objtool/include/objtool/special.h @@ -32,6 +32,13 @@ int special_get_alts(struct elf *elf, struct list_head *alts); void arch_handle_alternative(struct special_alt *alt); +/* + * Should the reloc at @offset -- the "new" (replacement) field of a special + * section group entry -- be ignored? The meaning of a zero-length replacement + * is arch specific, so the arch decides. + */ +bool arch_alt_ignore_new_reloc(struct section *sec, unsigned long offset); + bool arch_support_alt_relocation(struct special_alt *special_alt, struct instruction *insn, struct reloc *reloc); diff --git a/tools/objtool/klp-checksum.c b/tools/objtool/klp-checksum.c index b8e47f28997e9..ebe25f9c5260a 100644 --- a/tools/objtool/klp-checksum.c +++ b/tools/objtool/klp-checksum.c @@ -54,6 +54,19 @@ static int checksum_debug_init(struct objtool_file *file) return 0; } +/* + * Detect a reference to anonymous constant pool data which the compiler places + * in .rodata.cst and which either has an .LC symbol associated with + * it or (with Clang) no symbol at all. These are typically initializers for + * local function stack data, so they're considered part of the function rather + * than data per se. + */ +static bool is_anonymous_const_data(struct symbol *sym) +{ + return strstarts(sym->sec->name, ".rodata.cst") && + (is_sec_sym(sym) || strstarts(sym->name, ".LC")); +} + static void checksum_update_insn(struct objtool_file *file, struct symbol *func, struct instruction *insn) { @@ -129,6 +142,14 @@ static void checksum_update_insn(struct objtool_file *file, struct symbol *func, goto alts; } + if (is_anonymous_const_data(sym)) { + void *cst; + + cst = sym->sec->data->d_buf + sym->offset + offset; + __checksum_update_insn(func, insn, cst, sym->sec->sh.sh_entsize); + goto alts; + } + if (is_sec_sym(sym)) { sym = find_symbol_containing(reloc->sym->sec, offset); if (!sym) diff --git a/tools/objtool/klp-diff.c b/tools/objtool/klp-diff.c index f8787d7d14547..291ada0aa288d 100644 --- a/tools/objtool/klp-diff.c +++ b/tools/objtool/klp-diff.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include @@ -30,7 +30,9 @@ struct elfs { struct export { struct hlist_node hash; - char *mod, *sym; + char *mod; + char *sym; + bool mod_ns; }; bool debug, debug_correlate, debug_clone; @@ -83,6 +85,35 @@ static char *escape_str(const char *orig) return new; } +/* + * Convert a build-tree object path to a runtime module name: strip + * directory components, replace '-' with '_', and remove file + * extensions. Examples: + * + * "arch/x86/kvm/kvm" -> "kvm" + * "arch/x86/kvm/kvm-intel" -> "kvm_intel". + * + * Used by read_exports() to normalize Module.symvers entries and by + * __find_modname() as a fallback when .modinfo lacks a "name=" tag. + */ +static char *normalize_modname(char *name) +{ + char *slash = strrchr(name, '/'); + + if (slash) + name = slash + 1; + + for (char *c = name; *c; c++) { + if (*c == '-') + *c = '_'; + else if (*c == '.') { + *c = '\0'; + break; + } + } + return name; +} + static int read_exports(void) { const char *symvers = "Module.symvers"; @@ -106,7 +137,7 @@ static int read_exports(void) } while (fgets(line, 1024, file)) { - char *sym, *mod, *type; + char *sym, *mod, *type, *namespace; struct export *export; sym = strchr(line, '\t'); @@ -133,6 +164,14 @@ static int read_exports(void) *type++ = '\0'; + namespace = strchr(type, '\t'); + if (!namespace) { + ERROR("malformed Module.symvers (namespace) at line %d", line_num); + return -1; + } + + *namespace++ = '\0'; + if (*sym == '\0' || *mod == '\0') { ERROR("malformed Module.symvers at line %d", line_num); return -1; @@ -150,12 +189,18 @@ static int read_exports(void) return -1; } + if (strcmp(export->mod, "vmlinux")) + export->mod = normalize_modname(export->mod); + export->sym = strdup(sym); if (!export->sym) { ERROR_GLIBC("strdup"); return -1; } + /* EXPORT_SYMBOL_FOR_MODULES() */ + export->mod_ns = strstarts(namespace, "module:"); + hash_add(exports, &export->hash, str_hash(sym)); } @@ -866,65 +911,6 @@ static int correlate_symbols(struct elfs *e) return 0; } -/* "sympos" is used by livepatch to disambiguate duplicate symbol names */ -static unsigned long find_sympos(struct elf *elf, struct symbol *sym) -{ - bool vmlinux = str_ends_with(objname, "vmlinux.o"); - unsigned long sympos = 0, nr_matches = 0; - bool has_dup = false; - struct symbol *s; - - if (sym->bind != STB_LOCAL) - return 0; - - if (vmlinux && is_func_sym(sym)) { - /* - * HACK: Unfortunately, symbol ordering can differ between - * vmlinux.o and vmlinux due to the linker script emitting - * .text.unlikely* before .text*. Count .text.unlikely* first. - * - * TODO: Disambiguate symbols more reliably (checksums?) - */ - for_each_sym(elf, s) { - if (strstarts(s->sec->name, ".text.unlikely") && - !strcmp(s->name, sym->name)) { - nr_matches++; - if (s == sym) - sympos = nr_matches; - else - has_dup = true; - } - } - for_each_sym(elf, s) { - if (!strstarts(s->sec->name, ".text.unlikely") && - !strcmp(s->name, sym->name)) { - nr_matches++; - if (s == sym) - sympos = nr_matches; - else - has_dup = true; - } - } - } else { - for_each_sym(elf, s) { - if (!strcmp(s->name, sym->name)) { - nr_matches++; - if (s == sym) - sympos = nr_matches; - else - has_dup = true; - } - } - } - - if (!sympos) { - ERROR("can't find sympos for %s", sym->name); - return ULONG_MAX; - } - - return has_dup ? sympos : 0; -} - static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym); static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym, @@ -1129,6 +1115,9 @@ static struct export *find_export(struct symbol *sym) { struct export *export; + if (is_local_sym(sym)) + return NULL; + hash_for_each_possible(exports, export, hash, str_hash(sym->name)) { if (!strcmp(export->sym, sym->name)) return export; @@ -1158,18 +1147,7 @@ static const char *__find_modname(struct elfs *e) return NULL; } - for (char *c = name; *c; c++) { - if (*c == '/') - name = c + 1; - else if (*c == '-') - *c = '_'; - else if (*c == '.') { - *c = '\0'; - break; - } - } - - return name; + return normalize_modname(name); } /* Get the object's module name as defined by the kernel (and klp_object) */ @@ -1210,11 +1188,16 @@ static bool klp_reloc_needed(struct reloc *patched_reloc) * clusterfunk that is late module patching, the patch module is * allowed to be loaded before any modules it depends on. * - * If exported by vmlinux, a normal reloc will do. + * If exported by vmlinux to all modules, a normal reloc will do. */ export = find_export(patched_sym); - if (export) - return strcmp(export->mod, "vmlinux"); + if (export) { + if (strcmp(export->mod, "vmlinux")) + return true; + + /* EXPORT_SYMBOL_FOR_MODULES() gets a klp reloc */ + return export->mod_ns; + } if (!patched_sym->twin) { /* @@ -1337,12 +1320,14 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, struct section *sec, unsigned long offset, struct export *export) { + const char *sym_modname, *sym_orig_name, *sec_objname; struct symbol *patched_sym = patched_reloc->sym; s64 addend = reloc_addend(patched_reloc); - const char *sym_modname, *sym_orig_name; - static struct section *klp_relocs; + char tombstone_name[SYM_NAME_LEN]; struct symbol *sym, *klp_sym; unsigned long klp_reloc_off; + struct section *klp_relocs; + char sec_name[SEC_NAME_LEN]; char sym_name[SYM_NAME_LEN]; struct klp_reloc klp_reloc; unsigned long sympos; @@ -1355,15 +1340,22 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, /* * Keep the original reloc intact for now to avoid breaking objtool run * which relies on proper relocations for many of its features. This - * will be disabled later by "objtool klp post-link". + * reloc now targets a functionally dead tombstone symbol and will be + * disabled later by "objtool klp post-link". * - * Convert it to UNDEF (and WEAK to avoid modpost warnings). + * Convert the symbol to UNDEF/WEAK and rename to + * .klp.tombstone.sym_name to prevent modpost from printing warnings or + * creating false module dependencies. The prefix is hidden from the + * objtool run itself by read_symbols(). */ sym = patched_sym->clone; if (!sym) { - /* STB_WEAK: avoid modpost undefined symbol warnings */ - sym = elf_create_symbol(e->out, patched_sym->name, NULL, + if (snprintf_check(tombstone_name, SYM_NAME_LEN, + KLP_TOMBSTONE_PREFIX "%s", patched_sym->name)) + return -1; + + sym = elf_create_symbol(e->out, tombstone_name, NULL, STB_WEAK, patched_sym->type, 0, 0); if (!sym) return -1; @@ -1389,7 +1381,7 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, return -1; sym_orig_name = patched_sym->twin->name; - sympos = find_sympos(e->orig, patched_sym->twin); + sympos = klp_find_sympos(e->orig, patched_sym->twin); if (sympos == ULONG_MAX) return -1; } @@ -1411,16 +1403,35 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, } /* - * Create the __klp_relocs entry. This will be converted to an actual - * KLP rela by "objtool klp post-link". + * Create the __klp_relocs. entry. This will be converted to + * an actual KLP rela by "objtool klp post-link". * * This intermediate step is necessary to prevent corruption by the * linker, which doesn't know how to properly handle two rela sections * applying to the same base section. + * + * The objname decides when the reloc gets applied. A reference to a + * vmlinux symbol goes in the vmlinux section so it gets applied when + * the patch module loads. Everything else goes in the patched + * object's section, applied when the patched module is loaded. */ + if (!strcmp(sym_modname, "vmlinux")) { + sec_objname = "vmlinux"; + } else { + sec_objname = find_modname(e); + if (!sec_objname) + return -1; + } + + /* section format: __klp_relocs.objname */ + if (snprintf_check(sec_name, SEC_NAME_LEN, + KLP_RELOCS_SEC ".%s", sec_objname)) + return -1; + + klp_relocs = find_section_by_name(e->out, sec_name); if (!klp_relocs) { - klp_relocs = elf_create_section(e->out, KLP_RELOCS_SEC, 0, + klp_relocs = elf_create_section(e->out, sec_name, 0, 0, SHT_PROGBITS, 8, SHF_ALLOC); if (!klp_relocs) return -1; @@ -1568,6 +1579,10 @@ static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym) !strcmp(patched_reloc->sym->sec->name, ".altinstr_aux")) continue; + if (arch_alt_ignore_new_reloc(patched_sym->sec, + reloc_offset(patched_reloc))) + continue; + ret = convert_reloc_sym(e->patched, patched_reloc); if (ret < 0) { ERROR_FUNC(patched_rsec->base, reloc_offset(patched_reloc), @@ -1591,6 +1606,7 @@ static int create_fake_symbol(struct elf *elf, struct section *sec, unsigned long offset, size_t size) { char name[SYM_NAME_LEN]; + struct symbol *sym; unsigned int type; static int ctr; char *c; @@ -1607,7 +1623,24 @@ static int create_fake_symbol(struct elf *elf, struct section *sec, * while still allowing objdump to disassemble it. */ type = is_text_sec(sec) ? STT_NOTYPE : STT_OBJECT; - return elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size) ? 0 : -1; + + sym = elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size); + if (!sym) + return -1; + + sym->fake = 1; + return 0; +} + +static bool has_fake_symbols(struct section *sec) +{ + struct symbol *sym; + + sec_for_each_sym(sec, sym) + if (sym->fake) + return true; + + return false; } /* @@ -1658,13 +1691,17 @@ static int create_fake_symbols(struct elf *elf) for_each_reloc(sec->rsec, reloc) { unsigned long offset, size; struct reloc *next_reloc; + bool last = true; if (annotype(elf, sec, reloc) != ANNOTYPE_DATA_SPECIAL) continue; offset = reloc_addend(reloc); - size = 0; + /* + * Find the start of the next entry so the fake symbol size can + * be calculated. + */ next_reloc = reloc; for_each_reloc_continue(sec->rsec, next_reloc) { if (annotype(elf, sec, next_reloc) != ANNOTYPE_DATA_SPECIAL || @@ -1672,10 +1709,15 @@ static int create_fake_symbols(struct elf *elf) continue; size = reloc_addend(next_reloc) - offset; + last = false; break; } - if (!size) + /* + * If no next entry found, this is the last entry, so its size + * is from the current offset to the end of the section. + */ + if (last) size = sec_size(reloc->sym->sec) - offset; if (create_fake_symbol(elf, reloc->sym->sec, offset, size)) @@ -1690,7 +1732,11 @@ static int create_fake_symbols(struct elf *elf) unsigned int entry_size; unsigned long offset; - if (!is_special_section(sec) || find_symbol_by_offset(sec, 0)) + if (!is_special_section(sec)) + continue; + + /* Skip sections already handled by step 1 above */ + if (has_fake_symbols(sec)) continue; if (!sec->rsec) { @@ -2007,7 +2053,7 @@ static int create_klp_sections(struct elfs *e) /* klp_func_ext.sympos */ BUILD_BUG_ON(sizeof(sympos) != sizeof_field(struct klp_func_ext, sympos)); - sympos = find_sympos(e->orig, sym->clone->twin); + sympos = klp_find_sympos(e->orig, sym->clone->twin); if (sympos == ULONG_MAX) return -1; memcpy(func_data + offsetof(struct klp_func_ext, sympos), &sympos, @@ -2161,6 +2207,9 @@ int cmd_klp_diff(int argc, const char **argv) if (!e.orig || !e.patched) return -1; + if (klp_sympos_init(e.orig)) + return -1; + if (read_exports()) return -1; diff --git a/tools/objtool/klp-post-link.c b/tools/objtool/klp-post-link.c index c013e39957b11..350d20495897b 100644 --- a/tools/objtool/klp-post-link.c +++ b/tools/objtool/klp-post-link.c @@ -19,19 +19,11 @@ #include #include -static int fix_klp_relocs(struct elf *elf) +static int fix_klp_reloc_sec(struct elf *elf, struct section *symtab, + struct section *klp_relocs) { - struct section *symtab, *klp_relocs; - - klp_relocs = find_section_by_name(elf, KLP_RELOCS_SEC); - if (!klp_relocs) - return 0; - - symtab = find_section_by_name(elf, ".symtab"); - if (!symtab) { - ERROR("missing .symtab"); - return -1; - } + /* section format: __klp_relocs.sec_objname */ + const char *sec_objname = klp_relocs->name + strlen(KLP_RELOCS_SEC "."); for (int i = 0; i < sec_size(klp_relocs) / sizeof(struct klp_reloc); i++) { struct klp_reloc *klp_reloc; @@ -39,7 +31,6 @@ static int fix_klp_relocs(struct elf *elf) struct section *sec, *tmp, *klp_rsec; unsigned long offset; struct reloc *reloc; - char sym_modname[64]; char rsec_name[SEC_NAME_LEN]; u64 addend; struct symbol *sym, *klp_sym; @@ -55,7 +46,7 @@ static int fix_klp_relocs(struct elf *elf) reloc = find_reloc_by_dest(elf, klp_relocs, klp_reloc_off + offsetof(struct klp_reloc, offset)); if (!reloc) { - ERROR("malformed " KLP_RELOCS_SEC " section"); + ERROR("malformed %s section", klp_relocs->name); return -1; } @@ -66,17 +57,13 @@ static int fix_klp_relocs(struct elf *elf) reloc = find_reloc_by_dest(elf, klp_relocs, klp_reloc_off + offsetof(struct klp_reloc, sym)); if (!reloc) { - ERROR("malformed " KLP_RELOCS_SEC " section"); + ERROR("malformed %s section", klp_relocs->name); return -1; } klp_sym = reloc->sym; addend = reloc_addend(reloc); - /* symbol format: .klp.sym.modname.sym_name,sympos */ - if (sscanf(klp_sym->name + strlen(KLP_SYM_PREFIX), "%55[^.]", sym_modname) != 1) - ERROR("can't find modname in klp symbol '%s'", klp_sym->name); - /* * Create the KLP rela: */ @@ -84,7 +71,7 @@ static int fix_klp_relocs(struct elf *elf) /* section format: .klp.rela.sec_objname.section_name */ if (snprintf_check(rsec_name, SEC_NAME_LEN, KLP_RELOC_SEC_PREFIX "%s.%s", - sym_modname, sec->name)) + sec_objname, sec->name)) return -1; klp_rsec = find_section_by_name(elf, rsec_name); @@ -134,10 +121,32 @@ static int fix_klp_relocs(struct elf *elf) return 0; } +static int fix_klp_relocs(struct elf *elf) +{ + struct section *symtab, *sec; + + symtab = find_section_by_name(elf, ".symtab"); + if (!symtab) { + ERROR("missing .symtab"); + return -1; + } + + for_each_sec(elf, sec) { + if (strncmp(sec->name, KLP_RELOCS_SEC ".", + strlen(KLP_RELOCS_SEC "."))) + continue; + + if (fix_klp_reloc_sec(elf, symtab, sec)) + return -1; + } + + return 0; +} + /* * This runs on the livepatch module after all other linking has been done. It - * converts the intermediate __klp_relocs section into proper KLP relocs to be - * processed by livepatch. This needs to run last to avoid linker wreckage. + * converts the intermediate __klp_relocs.* sections into proper KLP relocs to + * be processed by livepatch. This needs to run last to avoid linker wreckage. * Linkers don't tend to handle the "two rela sections for a single base * section" case very well, nor do they appreciate SHN_LIVEPATCH. */ diff --git a/tools/objtool/klp-symid.c b/tools/objtool/klp-symid.c new file mode 100644 index 0000000000000..b19f76dff13e9 --- /dev/null +++ b/tools/objtool/klp-symid.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Emit the .klp.symid table which allows "objtool klp diff" to reliably + * disambiguate duplicate-named local symbols in vmlinux. + * + * Livepatch identifies a duplicate-named symbol by its position (sympos) + * among the same-named kallsyms entries, counted in ascending address order + * in the final linked vmlinux. That order can't be derived from vmlinux.o + * alone: the final link reorders sub-sections (.text.unlikely*, .data..*, + * etc). + * + * Bridge the gap with a table which survives the final link: a single + * non-alloc section containing an array of { id, addr } entries, where + * 'id' is a unique counter identifier and 'addr' has a relocation to the + * symbol. The linker copies 'id' verbatim and resolves 'addr' to the symbol's + * final address. + * + * The table is only emitted for vmlinux.o, and only when klp-build asks for it + * with KLP_SYMIDS=1, which adds --klp-symids to the vmlinux.o objtool run. + * + * It can't survive --gc-sections, which sweeps the whole section; klp-build + * rejects CONFIG_LD_DEAD_CODE_DATA_ELIMINATION. + */ +#include + +#include +#include +#include +#include + +static const char * const discarded_secs[] = { + ".discard", + ".exitcall.exit", + ".modinfo", + ".no_trim_symbol", + "__tracepoint_check", +}; + +static bool discarded_sec(struct section *sec) +{ + if (!(sec->sh.sh_flags & SHF_ALLOC)) + return true; + + for (int i = 0; i < ARRAY_SIZE(discarded_secs); i++) + if (strstarts(sec->name, discarded_secs[i])) + return true; + + return false; +} + +static bool symid_needed(struct elf *elf, struct symbol *sym) +{ + struct symbol *s; + + if (!is_local_sym(sym) || is_undef_sym(sym)) + return false; + + if (!is_func_sym(sym) && !is_object_sym(sym)) + return false; + + if (is_prefix_func(sym)) + return false; + + if (discarded_sec(sym->sec)) + return false; + + for_each_sym_by_name(elf, sym->name, s) { + if (s == sym || is_sec_sym(s) || is_file_sym(s) || is_undef_sym(s)) + continue; + return true; + } + + return false; +} + +int klp_create_symid_sections(struct objtool_file *file) +{ + struct elf *elf = file->elf; + struct klp_symid *symids; + struct section *sec; + struct symbol *sym; + u64 nr = 0, i = 0; + + if (!str_ends_with(objname, "vmlinux.o")) + return 0; + + for_each_sym(elf, sym) + if (symid_needed(elf, sym)) + nr++; + + if (!nr) + return 0; + + sec = elf_create_section(elf, KLP_SYMID_SEC, 0, sizeof(struct klp_symid), + SHT_PROGBITS, 8, 0); + if (!sec) + return -1; + + symids = elf_add_data(elf, sec, NULL, nr * sizeof(struct klp_symid)); + if (!symids) + return -1; + + for_each_sym(elf, sym) { + if (!symid_needed(elf, sym)) + continue; + + symids[i].id = bswap_if_needed(elf, i); + + if (!elf_create_reloc(elf, sec, + i * sizeof(struct klp_symid) + + offsetof(struct klp_symid, addr), + sym, 0, R_ABS64)) + return -1; + + i++; + } + + return 0; +} diff --git a/tools/objtool/klp-sympos.c b/tools/objtool/klp-sympos.c new file mode 100644 index 0000000000000..bbfae516d3395 --- /dev/null +++ b/tools/objtool/klp-sympos.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Compute "sympos", the position used by livepatch to disambiguate + * duplicate symbol names in the patched object. + */ +#include +#include +#include + +#include +#include +#include +#include + +#include + +struct vmlinux_sym { + struct hlist_node hash; + const char *name; + u64 addr; +}; + +struct vmlinux_symid { + struct hlist_node hash; + u64 id; + u64 addr; +}; + +struct vmlinux_o_symid { + struct hlist_node hash; + u64 id; + unsigned int sym_idx; +}; + +static DEFINE_HASHTABLE(vmlinux_o_symids, 16); + +/* + * The original linked kernel, found next to the orig vmlinux.o. Read with raw + * libelf rather than elf_open_read(): only the symbol table and the resolved + * .klp.symid table are needed, not the (huge) instruction/reloc machinery. + * + * Both tables are built once by read_orig_vmlinux(). The Elf handle stays + * open because the hashed names point into its mmapped string table. + */ +static struct { + Elf *elf; + DECLARE_HASHTABLE(syms, 16); /* name -> address */ + DECLARE_HASHTABLE(symids, 16); /* .klp.symid id -> address */ +} vmlinux; + +/* + * Would the symbol be visible to the runtime's kallsyms-based symbol lookup? + */ +static bool vmlinux_sym_in_kallsyms(Elf *elf, GElf_Sym *sym) +{ + unsigned int type = GELF_ST_TYPE(sym->st_info); + GElf_Shdr shdr; + Elf_Scn *scn; + + if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) + return false; + + if (type == STT_SECTION || type == STT_FILE) + return false; + + scn = elf_getscn(elf, sym->st_shndx); + if (!scn || !gelf_getshdr(scn, &shdr)) + return false; + + return shdr.sh_flags & SHF_ALLOC; +} + +static int read_orig_vmlinux(const char *filename) +{ + size_t shstrndx, nr_syms = 0, nr_symids = 0, strtab_idx = 0; + Elf_Data *symtab_data = NULL, *symid_data = NULL; + struct klp_symid *symids; + Elf_Scn *scn = NULL; + GElf_Ehdr ehdr; + int fd; + + fd = open(filename, O_RDONLY); + if (fd == -1) { + ERROR_GLIBC("can't open '%s'", filename); + return -1; + } + + if (elf_version(EV_CURRENT) == EV_NONE) { + ERROR_ELF("elf_version"); + return -1; + } + + vmlinux.elf = elf_begin(fd, ELF_C_READ_MMAP, NULL); + if (!vmlinux.elf) { + ERROR_ELF("elf_begin"); + return -1; + } + + if (!gelf_getehdr(vmlinux.elf, &ehdr)) { + ERROR_ELF("gelf_getehdr"); + return -1; + } + + if (elf_getshdrstrndx(vmlinux.elf, &shstrndx)) { + ERROR_ELF("elf_getshdrstrndx"); + return -1; + } + + while ((scn = elf_nextscn(vmlinux.elf, scn))) { + const char *name; + GElf_Shdr shdr; + + if (!gelf_getshdr(scn, &shdr)) { + ERROR_ELF("gelf_getshdr"); + return -1; + } + + if (shdr.sh_type == SHT_SYMTAB) { + symtab_data = elf_getdata(scn, NULL); + if (!symtab_data) { + ERROR_ELF("elf_getdata"); + return -1; + } + nr_syms = shdr.sh_size / shdr.sh_entsize; + strtab_idx = shdr.sh_link; + continue; + } + + name = elf_strptr(vmlinux.elf, shstrndx, shdr.sh_name); + if (name && !strcmp(name, KLP_SYMID_SEC)) { + if (shdr.sh_size % sizeof(struct klp_symid)) { + ERROR("%s: %s: struct klp_symid size mismatch", + filename, KLP_SYMID_SEC); + return -1; + } + symid_data = elf_getdata(scn, NULL); + if (!symid_data) { + ERROR_ELF("elf_getdata"); + return -1; + } + nr_symids = shdr.sh_size / sizeof(struct klp_symid); + } + } + + if (!symtab_data) { + ERROR("%s: missing symbol table", filename); + return -1; + } + + if (!symid_data) { + ERROR("%s: missing %s section, kernel not built with CONFIG_KLP_BUILD?", + filename, KLP_SYMID_SEC); + return -1; + } + + for (size_t i = 0; i < nr_syms; i++) { + struct vmlinux_sym *vsym; + const char *name; + GElf_Sym s; + + if (!gelf_getsym(symtab_data, i, &s)) { + ERROR_ELF("gelf_getsym"); + return -1; + } + + if (!vmlinux_sym_in_kallsyms(vmlinux.elf, &s)) + continue; + + name = elf_strptr(vmlinux.elf, strtab_idx, s.st_name); + if (!name) + continue; + + vsym = calloc(1, sizeof(*vsym)); + if (!vsym) { + ERROR_GLIBC("calloc"); + return -1; + } + + vsym->name = name; + vsym->addr = s.st_value; + hash_add(vmlinux.syms, &vsym->hash, str_hash(name)); + } + + symids = symid_data->d_buf; + + for (size_t i = 0; i < nr_symids; i++) { + struct vmlinux_symid *vsymid; + + vsymid = calloc(1, sizeof(*vsymid)); + if (!vsymid) { + ERROR_GLIBC("calloc"); + return -1; + } + + vsymid->id = __bswap_if_needed(&ehdr, symids[i].id); + vsymid->addr = __bswap_if_needed(&ehdr, symids[i].addr); + hash_add(vmlinux.symids, &vsymid->hash, vsymid->id); + } + + /* the fd and Elf handle stay open, the hashed names live in the mmap */ + return 0; +} + +/* + * Read the orig vmlinux.o's .klp.symid table, an array of entries whose 'addr' + * fields have relocs to the symbols they describe. + */ +static int read_vmlinux_o_symids(struct elf *vmlinux_o) +{ + struct section *sec; + + for_each_sec(vmlinux_o, sec) { + unsigned long nr; + + if (strcmp(sec->name, KLP_SYMID_SEC)) + continue; + + if (sec_size(sec) % sizeof(struct klp_symid)) { + ERROR("%s: %s: struct klp_symid size mismatch", + vmlinux_o->name, KLP_SYMID_SEC); + return -1; + } + + nr = sec_size(sec) / sizeof(struct klp_symid); + + for (unsigned long i = 0; i < nr; i++) { + unsigned long offset = i * sizeof(struct klp_symid); + struct vmlinux_o_symid *entry; + struct klp_symid *symid; + struct reloc *reloc; + + entry = calloc(1, sizeof(*entry)); + if (!entry) { + ERROR_GLIBC("calloc"); + return -1; + } + + symid = sec->data->d_buf + offset; + entry->id = bswap_if_needed(vmlinux_o, symid->id); + + reloc = find_reloc_by_dest(vmlinux_o, sec, + offset + offsetof(struct klp_symid, addr)); + if (!reloc) { + ERROR("%s: missing reloc for %s entry", + vmlinux_o->name, KLP_SYMID_SEC); + return -1; + } + entry->sym_idx = reloc->sym->idx; + + hash_add(vmlinux_o_symids, &entry->hash, entry->sym_idx); + } + } + + return 0; +} + +int klp_sympos_init(struct elf *orig) +{ + char *filename; + int ret; + + if (!str_ends_with(objname, "vmlinux.o")) + return 0; + + if (read_vmlinux_o_symids(orig)) + return -1; + + filename = strndup(objname, strlen(objname) - 2); + if (!filename) { + ERROR_GLIBC("strndup"); + return -1; + } + + ret = read_orig_vmlinux(filename); + free(filename); + + return ret; +} + +/* Find the symbol's id in the orig vmlinux.o's .klp.symid table */ +static int find_vmlinux_o_symid(struct symbol *sym, u64 *id) +{ + struct vmlinux_o_symid *entry; + + hash_for_each_possible(vmlinux_o_symids, entry, hash, sym->idx) { + if (entry->sym_idx == sym->idx) { + *id = entry->id; + return 0; + } + } + + ERROR("no %s entry for symbol %s in orig vmlinux.o", KLP_SYMID_SEC, + sym->name); + return -1; +} + +/* Find the symbol's final address in the orig vmlinux's .klp.symid table */ +static int find_vmlinux_symid_addr(u64 id, u64 *addr) +{ + struct vmlinux_symid *symid; + + hash_for_each_possible(vmlinux.symids, symid, hash, id) { + if (symid->id == id) { + *addr = symid->addr; + return 0; + } + } + + return -1; +} + +/* + * Find the sympos of a vmlinux-local symbol by ranking its final address + * among the duplicately named symbols in the linked orig vmlinux, replicating + * the order in which kallsyms_on_each_match_symbol() counts them. + */ +static unsigned long find_vmlinux_sympos(struct symbol *sym) +{ + unsigned long nr_matches = 0, sympos = 1; + u32 key = str_hash(sym->name); + struct vmlinux_sym *vsym; + bool found = false; + u64 id, addr; + + hash_for_each_possible(vmlinux.syms, vsym, hash, key) + if (!strcmp(vsym->name, sym->name)) + nr_matches++; + + if (!nr_matches) { + ERROR("can't find symbol %s in orig vmlinux", sym->name); + return ULONG_MAX; + } + + /* + * Unique symbols don't need disambiguating. They also have no + * .klp.symid entry, which is only emitted for names duplicated in + * vmlinux.o, so the lookups below would fail. + */ + if (nr_matches == 1) + return 0; + + if (find_vmlinux_o_symid(sym, &id)) + return ULONG_MAX; + + if (find_vmlinux_symid_addr(id, &addr)) { + ERROR("no %s entry for symbol %s in orig vmlinux", KLP_SYMID_SEC, + sym->name); + return ULONG_MAX; + } + + hash_for_each_possible(vmlinux.syms, vsym, hash, key) { + if (strcmp(vsym->name, sym->name)) + continue; + + if (vsym->addr < addr) + sympos++; + else if (vsym->addr == addr) + found = true; + } + + if (!found) { + ERROR("%s address mismatch for symbol %s, stale orig vmlinux?", + KLP_SYMID_SEC, sym->name); + return ULONG_MAX; + } + + return sympos; +} + +/* + * "sympos" is used by livepatch to disambiguate duplicate symbol names. + */ +unsigned long klp_find_sympos(struct elf *elf, struct symbol *sym) +{ + unsigned long sympos = 0, nr_matches = 0; + bool has_dup = false; + struct symbol *s; + + if (sym->bind != STB_LOCAL) + return 0; + + /* + * vmlinux: the final link reorders symbols relative to vmlinux.o, + * so the position needs to be derived from the linked orig vmlinux via + * the .klp.symid table. + */ + if (vmlinux.elf) + return find_vmlinux_sympos(sym); + + /* + * modules: the final .ko preserves symbol table order, so a + * symtab-order count here matches the runtime count done by + * module_kallsyms_on_each_symbol(). + */ + for_each_sym(elf, s) { + if (!strcmp(s->name, sym->name)) { + nr_matches++; + if (s == sym) + sympos = nr_matches; + else + has_dup = true; + } + } + + if (!sympos) { + ERROR("can't find sympos for %s", sym->name); + return ULONG_MAX; + } + + return has_dup ? sympos : 0; +} diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 0ba307e78fe1f..4d5993da9f945 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -27,6 +27,7 @@ CFLAGS += -fno-strict-aliasing ifeq ($(CC_NO_CLANG), 0) CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu + CLANG_TARGET_FLAGS_loongarch := loongarch64-linux-gnu CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu @@ -1142,6 +1143,7 @@ ifndef NO_RUST ifneq ($(CROSS_COMPILE),) RUST_TARGET_FLAGS_arm := arm-unknown-linux-gnueabi RUST_TARGET_FLAGS_arm64 := aarch64-unknown-linux-gnu + RUST_TARGET_FLAGS_loongarch := loongarch64-unknown-linux-gnu RUST_TARGET_FLAGS_m68k := m68k-unknown-linux-gnu RUST_TARGET_FLAGS_mips := mipsel-unknown-linux-gnu RUST_TARGET_FLAGS_powerpc := powerpc64le-unknown-linux-gnu diff --git a/tools/perf/arch/arm/util/cs-etm.c b/tools/perf/arch/arm/util/cs-etm.c index cdf8e3e606067..d2861d66a6612 100644 --- a/tools/perf/arch/arm/util/cs-etm.c +++ b/tools/perf/arch/arm/util/cs-etm.c @@ -201,7 +201,7 @@ static int cs_etm_validate_config(struct perf_pmu *cs_etm_pmu, { unsigned int idx; int err = 0; - struct perf_cpu_map *event_cpus = evsel->evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(evsel->evlist)->user_requested_cpus; struct perf_cpu_map *intersect_cpus; struct perf_cpu cpu; @@ -325,7 +325,7 @@ static int cs_etm_recording_options(struct auxtrace_record *itr, container_of(itr, struct cs_etm_recording, itr); struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu; struct evsel *evsel, *cs_etm_evsel = NULL; - struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); int err = 0; @@ -551,7 +551,7 @@ cs_etm_info_priv_size(struct auxtrace_record *itr, { unsigned int idx; int etmv3 = 0, etmv4 = 0, ete = 0; - struct perf_cpu_map *event_cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(evlist)->user_requested_cpus; struct perf_cpu_map *intersect_cpus; struct perf_cpu cpu; struct perf_pmu *cs_etm_pmu = cs_etm_get_pmu(itr); @@ -790,7 +790,7 @@ static int cs_etm_info_fill(struct auxtrace_record *itr, u32 offset; u64 nr_cpu, type; struct perf_cpu_map *cpu_map; - struct perf_cpu_map *event_cpus = session->evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(session->evlist)->user_requested_cpus; struct perf_cpu_map *online_cpus = perf_cpu_map__new_online_cpus(); struct cs_etm_recording *ptr = container_of(itr, struct cs_etm_recording, itr); @@ -800,7 +800,7 @@ static int cs_etm_info_fill(struct auxtrace_record *itr, if (priv_size != cs_etm_info_priv_size(itr, session->evlist)) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; /* If the cpu_map has the "any" CPU all online CPUs are involved */ diff --git a/tools/perf/arch/arm64/util/arm-spe.c b/tools/perf/arch/arm64/util/arm-spe.c index 91bb28cad79a5..1ba803a8d9b45 100644 --- a/tools/perf/arch/arm64/util/arm-spe.c +++ b/tools/perf/arch/arm64/util/arm-spe.c @@ -60,7 +60,7 @@ static bool arm_spe_is_set_freq(struct evsel *evsel) */ static struct perf_cpu_map *arm_spe_find_cpus(struct evlist *evlist) { - struct perf_cpu_map *event_cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(evlist)->user_requested_cpus; struct perf_cpu_map *online_cpus = perf_cpu_map__new_online_cpus(); struct perf_cpu_map *intersect_cpus; @@ -157,7 +157,7 @@ static int arm_spe_info_fill(struct auxtrace_record *itr, if (priv_size != arm_spe_info_priv_size(itr, session->evlist)) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; cpu_map = arm_spe_find_cpus(session->evlist); @@ -363,7 +363,7 @@ static int arm_spe_setup_tracking_event(struct evlist *evlist, { int err; struct evsel *tracking_evsel; - struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; /* Add dummy event to keep tracking */ err = parse_event(evlist, "dummy:u"); @@ -396,7 +396,7 @@ static int arm_spe_recording_options(struct auxtrace_record *itr, struct arm_spe_recording *sper = container_of(itr, struct arm_spe_recording, itr); struct evsel *evsel, *tmp; - struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool discard = false; int err; u64 discard_bit; diff --git a/tools/perf/arch/arm64/util/hisi-ptt.c b/tools/perf/arch/arm64/util/hisi-ptt.c index fe457fd58c9e8..52257715d2b74 100644 --- a/tools/perf/arch/arm64/util/hisi-ptt.c +++ b/tools/perf/arch/arm64/util/hisi-ptt.c @@ -53,7 +53,7 @@ static int hisi_ptt_info_fill(struct auxtrace_record *itr, if (priv_size != HISI_PTT_AUXTRACE_PRIV_SIZE) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; auxtrace_info->type = PERF_AUXTRACE_HISI_PTT; diff --git a/tools/perf/arch/x86/tests/amd-ibs-period.c b/tools/perf/arch/x86/tests/amd-ibs-period.c index 6a92b3a23ed7a..32713f8fcd5c8 100644 --- a/tools/perf/arch/x86/tests/amd-ibs-period.c +++ b/tools/perf/arch/x86/tests/amd-ibs-period.c @@ -46,7 +46,6 @@ static int dummy_workload_1(unsigned long count) { int (*func)(void); int ret = 0; - char *p; char insn1[] = { 0xb8, 0x01, 0x00, 0x00, 0x00, /* mov 1,%eax */ 0xc3, /* ret */ @@ -59,18 +58,11 @@ static int dummy_workload_1(unsigned long count) 0xcc, /* int 3 */ }; - p = calloc(2, page_size); - if (!p) { - printf("malloc() failed. %m"); - return 1; - } - - func = (void *)((unsigned long)(p + page_size - 1) & ~(page_size - 1)); - - ret = mprotect(func, page_size, PROT_READ | PROT_WRITE | PROT_EXEC); - if (ret) { - printf("mprotect() failed. %m"); - goto out; + func = mmap(NULL, page_size, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (func == MAP_FAILED) { + pr_debug("mmap() failed. %m\n"); + return -1; } if (count < 100000) @@ -93,7 +85,7 @@ static int dummy_workload_1(unsigned long count) } out: - free(p); + munmap(func, page_size); return ret; } diff --git a/tools/perf/arch/x86/tests/hybrid.c b/tools/perf/arch/x86/tests/hybrid.c index e221ea1041740..0477e17b8e53d 100644 --- a/tools/perf/arch/x86/tests/hybrid.c +++ b/tools/perf/arch/x86/tests/hybrid.c @@ -26,7 +26,7 @@ static int test__hybrid_hw_event_with_pmu(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -38,7 +38,7 @@ static int test__hybrid_hw_group_event(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -57,7 +57,7 @@ static int test__hybrid_sw_hw_group_event(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_SOFTWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong leader", evsel__has_leader(evsel, leader)); @@ -74,7 +74,7 @@ static int test__hybrid_hw_sw_group_event(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -91,7 +91,7 @@ static int test__hybrid_group_modifier1(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -113,7 +113,7 @@ static int test__hybrid_raw1(struct evlist *evlist) { struct perf_evsel *evsel; - perf_evlist__for_each_evsel(&evlist->core, evsel) { + perf_evlist__for_each_evsel(evlist__core(evlist), evsel) { struct perf_pmu *pmu = perf_pmus__find_by_type(evsel->attr.type); TEST_ASSERT_VAL("missing pmu", pmu); @@ -127,7 +127,7 @@ static int test__hybrid_raw2(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->core.attr.type); TEST_ASSERT_VAL("wrong config", test_config(evsel, 0x1a)); return TEST_OK; @@ -137,7 +137,7 @@ static int test__hybrid_cache_event(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HW_CACHE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong config", 0x2 == (evsel->core.attr.config & 0xffffffff)); return TEST_OK; @@ -148,7 +148,7 @@ static int test__checkevent_pmu(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->core.attr.type); TEST_ASSERT_VAL("wrong config", 10 == evsel->core.attr.config); TEST_ASSERT_VAL("wrong config1", 1 == evsel->core.attr.config1); @@ -168,7 +168,7 @@ static int test__hybrid_hw_group_event_2(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -268,7 +268,7 @@ static int test_event(const struct evlist_test *e) ret = e->check(evlist); } parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/arch/x86/tests/topdown.c b/tools/perf/arch/x86/tests/topdown.c index 221f2c4bbb615..2b6f47ce49324 100644 --- a/tools/perf/arch/x86/tests/topdown.c +++ b/tools/perf/arch/x86/tests/topdown.c @@ -56,7 +56,7 @@ static int event_cb(void *state, struct pmu_event_info *info) *ret = TEST_FAIL; } } - evlist__delete(evlist); + evlist__put(evlist); return 0; } @@ -174,7 +174,7 @@ static int test_sort(const char *str, int expected_slots_group_size, CHECK_COND(slots_seen, "slots seen"); ret = TEST_OK; out_err: - evlist__delete(evlist); + evlist__put(evlist); parse_events_error__exit(&err); return ret; } diff --git a/tools/perf/arch/x86/util/auxtrace.c b/tools/perf/arch/x86/util/auxtrace.c index ecbf61a7eb3a3..84fce0b51ccf7 100644 --- a/tools/perf/arch/x86/util/auxtrace.c +++ b/tools/perf/arch/x86/util/auxtrace.c @@ -55,7 +55,7 @@ struct auxtrace_record *auxtrace_record__init(struct evlist *evlist, int *err) { char buffer[64]; - struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus); + struct perf_cpu cpu = perf_cpu_map__min(evlist__core(evlist)->all_cpus); int ret; *err = 0; diff --git a/tools/perf/arch/x86/util/intel-bts.c b/tools/perf/arch/x86/util/intel-bts.c index 100a23d27998f..d44d568a6d210 100644 --- a/tools/perf/arch/x86/util/intel-bts.c +++ b/tools/perf/arch/x86/util/intel-bts.c @@ -79,10 +79,10 @@ static int intel_bts_info_fill(struct auxtrace_record *itr, if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; - pc = session->evlist->mmap[0].core.base; + pc = evlist__mmap(session->evlist)[0].core.base; if (pc) { err = perf_read_tsc_conversion(pc, &tc); if (err) { @@ -114,7 +114,7 @@ static int intel_bts_recording_options(struct auxtrace_record *itr, container_of(itr, struct intel_bts_recording, itr); struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu; struct evsel *evsel, *intel_bts_evsel = NULL; - const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + const struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); if (opts->auxtrace_sample_mode) { diff --git a/tools/perf/arch/x86/util/intel-pt.c b/tools/perf/arch/x86/util/intel-pt.c index 0307ff15d9fc9..a533114c0048c 100644 --- a/tools/perf/arch/x86/util/intel-pt.c +++ b/tools/perf/arch/x86/util/intel-pt.c @@ -360,10 +360,10 @@ static int intel_pt_info_fill(struct auxtrace_record *itr, filter = intel_pt_find_filter(session->evlist, ptr->intel_pt_pmu); filter_str_len = filter ? strlen(filter) : 0; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; - pc = session->evlist->mmap[0].core.base; + pc = evlist__mmap(session->evlist)[0].core.base; if (pc) { err = perf_read_tsc_conversion(pc, &tc); if (err) { @@ -376,7 +376,8 @@ static int intel_pt_info_fill(struct auxtrace_record *itr, ui__warning("Intel Processor Trace: TSC not available\n"); } - per_cpu_mmaps = !perf_cpu_map__is_any_cpu_or_is_empty(session->evlist->core.user_requested_cpus); + per_cpu_mmaps = !perf_cpu_map__is_any_cpu_or_is_empty( + evlist__core(session->evlist)->user_requested_cpus); auxtrace_info->type = PERF_AUXTRACE_INTEL_PT; auxtrace_info->priv[INTEL_PT_PMU_TYPE] = intel_pt_pmu->type; @@ -621,7 +622,7 @@ static int intel_pt_recording_options(struct auxtrace_record *itr, struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; bool have_timing_info, need_immediate = false; struct evsel *evsel, *intel_pt_evsel = NULL; - const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + const struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); u64 tsc_bit; int err; diff --git a/tools/perf/arch/x86/util/iostat.c b/tools/perf/arch/x86/util/iostat.c index 7442a2cd87eda..b13abea3a6f48 100644 --- a/tools/perf/arch/x86/util/iostat.c +++ b/tools/perf/arch/x86/util/iostat.c @@ -332,13 +332,15 @@ static int iostat_event_group(struct evlist *evl, return ret; } -int iostat_prepare(struct evlist *evlist, struct perf_stat_config *config) +int iostat_prepare(struct evlist **evlist_ptr, struct perf_stat_config *config) { - if (evlist->core.nr_entries > 0) { + struct evlist *evlist = *evlist_ptr; + + if (evlist__nr_entries(evlist) > 0) { pr_warning("The -e and -M options are not supported." "All chosen events/metrics will be dropped\n"); - evlist__delete(evlist); - evlist = evlist__new(); + evlist__put(evlist); + *evlist_ptr = evlist = evlist__new(); if (!evlist) return -ENOMEM; } @@ -400,7 +402,7 @@ void iostat_prefix(struct evlist *evlist, struct perf_stat_config *config, char *prefix, struct timespec *ts) { - struct iio_root_port *rp = evlist->selected->priv; + struct iio_root_port *rp = evlist__selected(evlist)->priv; if (rp) { /* @@ -463,7 +465,7 @@ void iostat_print_counters(struct evlist *evlist, iostat_prefix(evlist, config, prefix, ts); fprintf(config->output, "%s", prefix); evlist__for_each_entry(evlist, counter) { - perf_device = evlist->selected->priv; + perf_device = evlist__selected(evlist)->priv; if (perf_device && perf_device != counter->priv) { evlist__set_selected(evlist, counter); iostat_prefix(evlist, config, prefix, ts); diff --git a/tools/perf/bench/evlist-open-close.c b/tools/perf/bench/evlist-open-close.c index faf9c34b4a5dc..748ebbe458f49 100644 --- a/tools/perf/bench/evlist-open-close.c +++ b/tools/perf/bench/evlist-open-close.c @@ -76,7 +76,7 @@ static struct evlist *bench__create_evlist(char *evstr, const char *uid_str) parse_events_error__exit(&err); pr_err("Run 'perf list' for a list of valid events\n"); ret = 1; - goto out_delete_evlist; + goto out_put_evlist; } parse_events_error__exit(&err); if (uid_str) { @@ -85,24 +85,24 @@ static struct evlist *bench__create_evlist(char *evstr, const char *uid_str) if (uid == UINT_MAX) { pr_err("Invalid User: %s", uid_str); ret = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } ret = parse_uid_filter(evlist, uid); if (ret) - goto out_delete_evlist; + goto out_put_evlist; } ret = evlist__create_maps(evlist, &opts.target); if (ret < 0) { pr_err("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } evlist__config(evlist, &opts, NULL); return evlist; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); return NULL; } @@ -116,7 +116,7 @@ static int bench__do_evlist_open_close(struct evlist *evlist) return err; } - err = evlist__mmap(evlist, opts.mmap_pages); + err = evlist__do_mmap(evlist, opts.mmap_pages); if (err < 0) { pr_err("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); return err; @@ -124,7 +124,7 @@ static int bench__do_evlist_open_close(struct evlist *evlist) evlist__enable(evlist); evlist__disable(evlist); - evlist__munmap(evlist); + evlist__do_munmap(evlist); evlist__close(evlist); return 0; @@ -145,13 +145,14 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str) init_stats(&time_stats); - printf(" Number of cpus:\t%d\n", perf_cpu_map__nr(evlist->core.user_requested_cpus)); - printf(" Number of threads:\t%d\n", evlist->core.threads->nr); + printf(" Number of cpus:\t%d\n", + perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus)); + printf(" Number of threads:\t%d\n", evlist__core(evlist)->threads->nr); printf(" Number of events:\t%d (%d fds)\n", - evlist->core.nr_entries, evlist__count_evsel_fds(evlist)); + evlist__nr_entries(evlist), evlist__count_evsel_fds(evlist)); printf(" Number of iterations:\t%d\n", iterations); - evlist__delete(evlist); + evlist__put(evlist); for (i = 0; i < iterations; i++) { pr_debug("Started iteration %d\n", i); @@ -162,7 +163,7 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str) gettimeofday(&start, NULL); err = bench__do_evlist_open_close(evlist); if (err) { - evlist__delete(evlist); + evlist__put(evlist); return err; } @@ -171,7 +172,7 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str) runtime_us = timeval2usec(&diff); update_stats(&time_stats, runtime_us); - evlist__delete(evlist); + evlist__put(evlist); pr_debug("Iteration %d took:\t%" PRIu64 "us\n", i, runtime_us); } diff --git a/tools/perf/builtin-annotate.c b/tools/perf/builtin-annotate.c index 8a0eb30eac24f..69cb72b2082a6 100644 --- a/tools/perf/builtin-annotate.c +++ b/tools/perf/builtin-annotate.c @@ -562,7 +562,7 @@ static int __cmd_annotate(struct perf_annotate *ann) goto out; if ((use_browser == 1 || ann->use_stdio2) && ann->has_br_stack) - if (session->evlist->nr_br_cntr > 0) + if (evlist__nr_br_cntr(session->evlist) > 0) annotate_opts.show_br_cntr = true; if (dump_trace) { @@ -928,8 +928,11 @@ int cmd_annotate(int argc, const char **argv) * branch counters, if the corresponding branch info is available * in the perf data in the TUI mode. */ - if ((use_browser == 1 || annotate.use_stdio2) && annotate.has_br_stack) + if ((use_browser == 1 || annotate.use_stdio2) && annotate.has_br_stack) { sort__mode = SORT_MODE__BRANCH; + if (evlist__nr_br_cntr(annotate.session->evlist) > 0) + annotate_opts.show_br_cntr = true; + } if (setup_sorting(/*evlist=*/NULL, perf_session__env(annotate.session)) < 0) usage_with_options(annotate_usage, options); diff --git a/tools/perf/builtin-c2c.c b/tools/perf/builtin-c2c.c index c9584dbedf77a..bc16a57e09274 100644 --- a/tools/perf/builtin-c2c.c +++ b/tools/perf/builtin-c2c.c @@ -12,11 +12,14 @@ */ #include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -226,7 +229,6 @@ he__get_c2c_hists(struct hist_entry *he, ret = c2c_hists__init(hists, sort, nr_header_lines, env); if (ret) { - perf_hpp__reset_output_field(&hists->list); c2c_he->hists = NULL; free(hists); return NULL; @@ -2063,26 +2065,38 @@ static int c2c_hists__init_sort(struct perf_hpp_list *hpp_list, char *name, stru return 0; } -#define PARSE_LIST(_list, _fn) \ - do { \ - char *tmp, *tok; \ - ret = 0; \ - \ - if (!_list) \ - break; \ - \ - for (tok = strtok_r((char *)_list, ", ", &tmp); \ - tok; tok = strtok_r(NULL, ", ", &tmp)) { \ - ret = _fn(hpp_list, tok, env); \ - if (ret == -EINVAL) { \ - pr_err("Invalid --fields key: `%s'", tok); \ - break; \ - } else if (ret == -ESRCH) { \ - pr_err("Unknown --fields key: `%s'", tok); \ - break; \ - } \ - } \ - } while (0) +static int __hpp_list__parse(struct perf_hpp_list *hpp_list, char *_list, struct perf_env *env, + int (*_fn)(struct perf_hpp_list *hpp_list, char *name, struct perf_env *env)) +{ + char *tmp, *tok; + int ret = 0; + + if (!_list) + return 0; + + for (tok = strtok_r(_list, ", ", &tmp); tok; tok = strtok_r(NULL, ", ", &tmp)) { + ret = _fn(hpp_list, tok, env); + switch (ret) { + case 0: + continue; + case -EINVAL: + pr_err("Invalid --fields key: `%s'", tok); + goto out; + case -ESRCH: + pr_err("Unknown --fields key: `%s'", tok); + goto out; + default: { + char buf[STRERR_BUFSIZE]; + + pr_err("%s for --fields key: `%s'", + str_error_r(-ret, buf, sizeof(buf)), tok); + goto out; + } + } + } +out: + return ret; +} static int hpp_list__parse(struct perf_hpp_list *hpp_list, const char *output_, @@ -2093,8 +2107,18 @@ static int hpp_list__parse(struct perf_hpp_list *hpp_list, char *sort = sort_ ? strdup(sort_) : NULL; int ret; - PARSE_LIST(output, c2c_hists__init_output); - PARSE_LIST(sort, c2c_hists__init_sort); + /* strdup() returns NULL on OOM, don't silently treat as empty */ + if ((output_ && !output) || (sort_ && !sort)) { + ret = -ENOMEM; + goto out; + } + + ret = __hpp_list__parse(hpp_list, output, env, c2c_hists__init_output); + if (ret) + goto out; + ret = __hpp_list__parse(hpp_list, sort, env, c2c_hists__init_sort); + if (ret) + goto out; /* copy sort keys to output fields */ perf_hpp__setup_output_field(hpp_list); @@ -2111,6 +2135,7 @@ static int hpp_list__parse(struct perf_hpp_list *hpp_list, perf_hpp__append_sort_keys(&hists->list); #endif +out: free(output); free(sort); return ret; @@ -2121,6 +2146,8 @@ static int c2c_hists__init(struct c2c_hists *hists, int nr_header_lines, struct perf_env *env) { + int ret; + __hists__init(&hists->hists, &hists->list); /* @@ -2133,7 +2160,13 @@ static int c2c_hists__init(struct c2c_hists *hists, /* Overload number of header lines.*/ hists->list.nr_header_lines = nr_header_lines; - return hpp_list__parse(&hists->list, /*output=*/NULL, sort, env); + ret = hpp_list__parse(&hists->list, /*output=*/NULL, sort, env); + + /* Unregister any formats added before the failure point */ + if (ret) + perf_hpp__reset_output_field(&hists->list); + + return ret; } static int c2c_hists__reinit(struct c2c_hists *c2c_hists, @@ -2141,8 +2174,16 @@ static int c2c_hists__reinit(struct c2c_hists *c2c_hists, const char *sort, struct perf_env *env) { + int ret; + perf_hpp__reset_output_field(&c2c_hists->list); - return hpp_list__parse(&c2c_hists->list, output, sort, env); + ret = hpp_list__parse(&c2c_hists->list, output, sort, env); + + /* Unregister any formats added before the failure point */ + if (ret) + perf_hpp__reset_output_field(&c2c_hists->list); + + return ret; } #define DISPLAY_LINE_LIMIT 0.001 @@ -2281,6 +2322,7 @@ static int resort_cl_cb(struct hist_entry *he, void *arg) struct c2c_hist_entry *c2c_he; struct c2c_hists *c2c_hists; bool display = he__display(he, &c2c.shared_clines_stats); + int ret; c2c_he = container_of(he, struct c2c_hist_entry, he); c2c_hists = c2c_he->hists; @@ -2291,7 +2333,9 @@ static int resort_cl_cb(struct hist_entry *he, void *arg) c2c_he->cacheline_idx = idx++; calc_width(c2c_he); - c2c_hists__reinit(c2c_hists, c2c.cl_output, c2c.cl_resort, env); + ret = c2c_hists__reinit(c2c_hists, c2c.cl_output, c2c.cl_resort, env); + if (ret) + return ret; hists__collapse_resort(&c2c_hists->hists, NULL); hists__output_resort_cb(&c2c_hists->hists, NULL, filter_cb); @@ -3356,13 +3400,19 @@ static int perf_c2c__report(int argc, const char **argv) else if (c2c.display == DISPLAY_SNP_PEER) sort_str = "tot_peer"; - c2c_hists__reinit(&c2c.hists, output_str, sort_str, perf_session__env(session)); + err = c2c_hists__reinit(&c2c.hists, output_str, sort_str, perf_session__env(session)); + if (err) { + pr_err("Failed to reinitialize hists\n"); + goto out_mem2node; + } ui_progress__init(&prog, c2c.hists.hists.nr_entries, "Sorting..."); hists__collapse_resort(&c2c.hists.hists, NULL); hists__output_resort_cb(&c2c.hists.hists, &prog, resort_shared_cl_cb); - hists__iterate_cb(&c2c.hists.hists, resort_cl_cb, perf_session__env(session)); + err = hists__iterate_cb(&c2c.hists.hists, resort_cl_cb, perf_session__env(session)); + if (err) + goto out_mem2node; ui_progress__finish(); diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c index 8a7dbfb14535e..f7126196b0921 100644 --- a/tools/perf/builtin-ftrace.c +++ b/tools/perf/builtin-ftrace.c @@ -72,18 +72,11 @@ static void ftrace__workload_exec_failed_signal(int signo __maybe_unused, static bool check_ftrace_capable(void) { - bool used_root; - - if (perf_cap__capable(CAP_PERFMON, &used_root)) - return true; - - if (!used_root && perf_cap__capable(CAP_SYS_ADMIN, &used_root)) + if (perf_cap__capable(CAP_PERFMON) || + perf_cap__capable(CAP_SYS_ADMIN)) return true; - pr_err("ftrace only works for %s!\n", - used_root ? "root" - : "users with the CAP_PERFMON or CAP_SYS_ADMIN capability" - ); + pr_err("ftrace only works for users with the CAP_PERFMON or CAP_SYS_ADMIN capability!\n"); return false; } @@ -377,9 +370,9 @@ static int set_tracing_pid(struct perf_ftrace *ftrace) if (target__has_cpu(&ftrace->target)) return 0; - for (i = 0; i < perf_thread_map__nr(ftrace->evlist->core.threads); i++) { + for (i = 0; i < perf_thread_map__nr(evlist__core(ftrace->evlist)->threads); i++) { scnprintf(buf, sizeof(buf), "%d", - perf_thread_map__pid(ftrace->evlist->core.threads, i)); + perf_thread_map__pid(evlist__core(ftrace->evlist)->threads, i)); if (append_tracing_file("set_ftrace_pid", buf) < 0) return -1; } @@ -413,7 +406,7 @@ static int set_tracing_cpumask(struct perf_cpu_map *cpumap) static int set_tracing_cpu(struct perf_ftrace *ftrace) { - struct perf_cpu_map *cpumap = ftrace->evlist->core.user_requested_cpus; + struct perf_cpu_map *cpumap = evlist__core(ftrace->evlist)->user_requested_cpus; if (!target__has_cpu(&ftrace->target)) return 0; @@ -1999,20 +1992,20 @@ int cmd_ftrace(int argc, const char **argv) ret = evlist__create_maps(ftrace.evlist, &ftrace.target); if (ret < 0) - goto out_delete_evlist; + goto out_put_evlist; if (argc) { ret = evlist__prepare_workload(ftrace.evlist, &ftrace.target, argv, false, ftrace__workload_exec_failed_signal); if (ret < 0) - goto out_delete_evlist; + goto out_put_evlist; } ret = cmd_func(&ftrace); -out_delete_evlist: - evlist__delete(ftrace.evlist); +out_put_evlist: + evlist__put(ftrace.evlist); out_delete_filters: delete_filter_func(&ftrace.filters); diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c index 6d6cce4765a7f..b13ce4caf8098 100644 --- a/tools/perf/builtin-inject.c +++ b/tools/perf/builtin-inject.c @@ -1520,7 +1520,7 @@ static int synthesize_id_index(struct perf_inject *inject, size_t new_cnt) struct perf_session *session = inject->session; struct evlist *evlist = session->evlist; struct machine *machine = &session->machines.host; - size_t from = evlist->core.nr_entries - new_cnt; + size_t from = evlist__nr_entries(evlist) - new_cnt; return __perf_event__synthesize_id_index(&inject->tool, perf_event__repipe, evlist, machine, from); @@ -2055,7 +2055,7 @@ static int host__finished_init(const struct perf_tool *tool, struct perf_session if (ret) return ret; - ret = synthesize_id_index(inject, gs->session->evlist->core.nr_entries); + ret = synthesize_id_index(inject, evlist__nr_entries(gs->session->evlist)); if (ret) { pr_err("Failed to synthesize id_index\n"); return ret; diff --git a/tools/perf/builtin-kvm.c b/tools/perf/builtin-kvm.c index 394302ebdb161..2c6aef1e13a04 100644 --- a/tools/perf/builtin-kvm.c +++ b/tools/perf/builtin-kvm.c @@ -1221,7 +1221,7 @@ static s64 perf_kvm__mmap_read_idx(struct perf_kvm_stat *kvm, int idx, int err; *mmap_time = ULLONG_MAX; - md = &evlist->mmap[idx]; + md = &evlist__mmap(evlist)[idx]; err = perf_mmap__read_init(&md->core); if (err < 0) return (err == -EAGAIN) ? 0 : -1; @@ -1266,7 +1266,7 @@ static int perf_kvm__mmap_read(struct perf_kvm_stat *kvm) s64 n, ntotal = 0; u64 flush_time = ULLONG_MAX, mmap_time; - for (i = 0; i < kvm->evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(kvm->evlist)->nr_mmaps; i++) { n = perf_kvm__mmap_read_idx(kvm, i, &mmap_time); if (n < 0) return -1; @@ -1449,7 +1449,7 @@ static int kvm_events_live_report(struct perf_kvm_stat *kvm) evlist__enable(kvm->evlist); while (!done) { - struct fdarray *fda = &kvm->evlist->core.pollfd; + struct fdarray *fda = &evlist__core(kvm->evlist)->pollfd; int rc; rc = perf_kvm__mmap_read(kvm); @@ -1531,7 +1531,7 @@ static int kvm_live_open_events(struct perf_kvm_stat *kvm) goto out; } - if (evlist__mmap(evlist, kvm->opts.mmap_pages) < 0) { + if (evlist__do_mmap(evlist, kvm->opts.mmap_pages) < 0) { ui__error("Failed to mmap the events: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__close(evlist); @@ -1810,7 +1810,7 @@ static struct evlist *kvm_live_event_list(void) out: if (err) { - evlist__delete(evlist); + evlist__put(evlist); evlist = NULL; } @@ -1931,7 +1931,7 @@ static int kvm_events_live(struct perf_kvm_stat *kvm, perf_session__set_id_hdr_size(kvm->session); ordered_events__set_copy_on_queue(&kvm->session->ordered_events, true); machine__synthesize_threads(&kvm->session->machines.host, &kvm->opts.target, - kvm->evlist->core.threads, true, false, 1); + evlist__core(kvm->evlist)->threads, true, false, 1); err = kvm_live_open_events(kvm); if (err) goto out; @@ -1941,7 +1941,7 @@ static int kvm_events_live(struct perf_kvm_stat *kvm, out: perf_session__delete(kvm->session); kvm->session = NULL; - evlist__delete(kvm->evlist); + evlist__put(kvm->evlist); return err; } diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 7b61168e01e9d..fce588441e499 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -1814,7 +1814,7 @@ static int perf_kwork__check_config(struct perf_kwork *kwork, } } - list_for_each_entry(evsel, &session->evlist->core.entries, core.node) { + list_for_each_entry(evsel, &evlist__core(session->evlist)->entries, core.node) { if (kwork->show_callchain && !evsel__has_callchain(evsel)) { pr_debug("Samples do not have callchains\n"); kwork->show_callchain = 0; @@ -1864,9 +1864,9 @@ static int perf_kwork__read_events(struct perf_kwork *kwork) goto out_delete; } - kwork->nr_events = session->evlist->stats.nr_events[0]; - kwork->nr_lost_events = session->evlist->stats.total_lost; - kwork->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST]; + kwork->nr_events = evlist__stats(session->evlist)->nr_events[0]; + kwork->nr_lost_events = evlist__stats(session->evlist)->total_lost; + kwork->nr_lost_chunks = evlist__stats(session->evlist)->nr_events[PERF_RECORD_LOST]; out_delete: perf_session__delete(session); diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index 5841d43be9718..d5c0d55cb82d9 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -2129,7 +2129,7 @@ static int __cmd_contention(int argc, const char **argv) evlist__start_workload(con.evlist); while (!done) { - if (argc && waitpid(con.evlist->workload.pid, NULL, WNOHANG) > 0) + if (argc && waitpid(evlist__workload_pid(con.evlist), NULL, WNOHANG) > 0) break; sleep(1); } @@ -2149,7 +2149,7 @@ static int __cmd_contention(int argc, const char **argv) out_delete: lock_filter_finish(); - evlist__delete(con.evlist); + evlist__put(con.evlist); lock_contention_finish(&con); perf_session__delete(session); perf_env__exit(&host_env); diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index e915390556752..a57987851cf03 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -64,6 +64,7 @@ #include #include #include +#include #ifndef HAVE_GETTID #include #endif @@ -502,12 +503,12 @@ static void record__aio_mmap_read_sync(struct record *rec) { int i; struct evlist *evlist = rec->evlist; - struct mmap *maps = evlist->mmap; + struct mmap *maps = evlist__mmap(evlist); if (!record__aio_enabled(rec)) return; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { struct mmap *map = &maps[i]; if (map->core.base) @@ -653,27 +654,14 @@ static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size) struct record *rec = to; if (record__comp_enabled(rec)) { - struct perf_record_compressed2 *event = map->data; - size_t padding = 0; - u8 pad[8] = {0}; ssize_t compressed = zstd_compress(rec->session, map, map->data, mmap__mmap_len(map), bf, size); if (compressed < 0) return (int)compressed; - bf = event; thread->samples++; - - /* - * The record from `zstd_compress` is not 8 bytes aligned, which would cause asan - * error. We make it aligned here. - */ - event->data_size = compressed - sizeof(struct perf_record_compressed2); - event->header.size = PERF_ALIGN(compressed, sizeof(u64)); - padding = event->header.size - compressed; - return record__write(rec, map, bf, compressed) || - record__write(rec, map, &pad, padding); + return record__write(rec, map, map->data, compressed); } thread->samples++; @@ -811,8 +799,8 @@ static int record__auxtrace_read_snapshot_all(struct record *rec) int i; int rc = 0; - for (i = 0; i < rec->evlist->core.nr_mmaps; i++) { - struct mmap *map = &rec->evlist->mmap[i]; + for (i = 0; i < evlist__core(rec->evlist)->nr_mmaps; i++) { + struct mmap *map = &evlist__mmap(rec->evlist)[i]; if (!map->auxtrace_mmap.base) continue; @@ -1055,15 +1043,15 @@ static void record__thread_data_close_pipes(struct record_thread *thread_data) static bool evlist__per_thread(struct evlist *evlist) { - return cpu_map__is_dummy(evlist->core.user_requested_cpus); + return cpu_map__is_dummy(evlist__core(evlist)->user_requested_cpus); } static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist) { - int m, tm, nr_mmaps = evlist->core.nr_mmaps; - struct mmap *mmap = evlist->mmap; - struct mmap *overwrite_mmap = evlist->overwrite_mmap; - struct perf_cpu_map *cpus = evlist->core.all_cpus; + int m, tm, nr_mmaps = evlist__core(evlist)->nr_mmaps; + struct mmap *mmap = evlist__mmap(evlist); + struct mmap *overwrite_mmap = evlist__overwrite_mmap(evlist); + struct perf_cpu_map *cpus = evlist__core(evlist)->all_cpus; bool per_thread = evlist__per_thread(evlist); if (per_thread) @@ -1118,16 +1106,17 @@ static int record__thread_data_init_pollfd(struct record_thread *thread_data, st overwrite_map = thread_data->overwrite_maps ? thread_data->overwrite_maps[tm] : NULL; - for (f = 0; f < evlist->core.pollfd.nr; f++) { - void *ptr = evlist->core.pollfd.priv[f].ptr; + for (f = 0; f < evlist__core(evlist)->pollfd.nr; f++) { + void *ptr = evlist__core(evlist)->pollfd.priv[f].ptr; if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) { pos = fdarray__dup_entry_from(&thread_data->pollfd, f, - &evlist->core.pollfd); + &evlist__core(evlist)->pollfd); if (pos < 0) return pos; pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n", - thread_data, pos, evlist->core.pollfd.entries[f].fd); + thread_data, pos, + evlist__core(evlist)->pollfd.entries[f].fd); } } } @@ -1171,7 +1160,7 @@ static int record__update_evlist_pollfd_from_thread(struct record *rec, struct evlist *evlist, struct record_thread *thread_data) { - struct pollfd *e_entries = evlist->core.pollfd.entries; + struct pollfd *e_entries = evlist__core(evlist)->pollfd.entries; struct pollfd *t_entries = thread_data->pollfd.entries; int err = 0; size_t i; @@ -1195,7 +1184,7 @@ static int record__dup_non_perf_events(struct record *rec, struct evlist *evlist, struct record_thread *thread_data) { - struct fdarray *fda = &evlist->core.pollfd; + struct fdarray *fda = &evlist__core(evlist)->pollfd; int i, ret; for (i = 0; i < fda->nr; i++) { @@ -1322,17 +1311,17 @@ static int record__mmap_evlist(struct record *rec, return ret; if (record__threads_enabled(rec)) { - ret = perf_data__create_dir(&rec->data, evlist->core.nr_mmaps); + ret = perf_data__create_dir(&rec->data, evlist__core(evlist)->nr_mmaps); if (ret) { errno = -ret; pr_err("Failed to create data directory: %m\n"); return ret; } - for (i = 0; i < evlist->core.nr_mmaps; i++) { - if (evlist->mmap) - evlist->mmap[i].file = &rec->data.dir.files[i]; - if (evlist->overwrite_mmap) - evlist->overwrite_mmap[i].file = &rec->data.dir.files[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + if (evlist__mmap(evlist)) + evlist__mmap(evlist)[i].file = &rec->data.dir.files[i]; + if (evlist__overwrite_mmap(evlist)) + evlist__overwrite_mmap(evlist)[i].file = &rec->data.dir.files[i]; } } @@ -1481,11 +1470,11 @@ static int record__open(struct record *rec) static void set_timestamp_boundary(struct record *rec, u64 sample_time) { - if (rec->evlist->first_sample_time == 0) - rec->evlist->first_sample_time = sample_time; + if (evlist__first_sample_time(rec->evlist) == 0) + evlist__set_first_sample_time(rec->evlist, sample_time); if (sample_time) - rec->evlist->last_sample_time = sample_time; + evlist__set_last_sample_time(rec->evlist, sample_time); } static int process_sample_event(const struct perf_tool *tool, @@ -1591,18 +1580,36 @@ static void record__adjust_affinity(struct record *rec, struct mmap *map) } } -static size_t process_comp_header(void *record, size_t increment) +/* + * Called once with data_size == 0 to start a record, then once with + * data_size == compressed payload size to finalize and 8-byte-pad it + * (unaligned records trip ASan in the reader). + * Returns the bytes written, or -1 if it won't fit. + */ +static ssize_t process_comp_header(void *record, size_t dst_size, + size_t data_size) { struct perf_record_compressed2 *event = record; size_t size = sizeof(*event); - if (increment) { - event->header.size += increment; - return increment; + if (data_size) { + size_t padding; + + event->data_size = data_size; + event->header.size = PERF_ALIGN(size + data_size, sizeof(u64)); + padding = event->header.size - size - data_size; + if (padding > dst_size) + return -1; + memset(record + size + data_size, 0, padding); + return padding; } + if (size > dst_size) + return -1; + event->header.type = PERF_RECORD_COMPRESSED2; event->header.size = size; + event->data_size = 0; return size; } @@ -1611,7 +1618,12 @@ static ssize_t zstd_compress(struct perf_session *session, struct mmap *map, void *dst, size_t dst_size, void *src, size_t src_size) { ssize_t compressed; - size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed2) - 1; + /* + * Reserve space so per-record PERF_ALIGN() padding keeps header.size + * within u16. + */ + size_t max_record_size = PERF_SAMPLE_MAX_SIZE + - sizeof(struct perf_record_compressed2) - sizeof(u64); struct zstd_data *zstd_data = &session->zstd_data; if (map && map->file) @@ -1653,7 +1665,7 @@ static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist, if (!maps) return 0; - if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING) + if (overwrite && evlist__bkw_mmap_state(evlist) != BKW_MMAP_DATA_PENDING) return 0; if (record__aio_enabled(rec)) @@ -1808,7 +1820,7 @@ static void record__init_features(struct record *rec) if (rec->no_buildid) perf_header__clear_feat(&session->header, HEADER_BUILD_ID); - if (!have_tracepoints(&rec->evlist->core.entries)) + if (!have_tracepoints(&evlist__core(rec->evlist)->entries)) perf_header__clear_feat(&session->header, HEADER_TRACING_DATA); if (!rec->opts.branch_stack) @@ -1874,7 +1886,7 @@ static int record__synthesize_workload(struct record *rec, bool tail) if (rec->opts.tail_synthesize != tail) return 0; - thread_map = thread_map__new_by_tid(rec->evlist->workload.pid); + thread_map = thread_map__new_by_tid(evlist__workload_pid(rec->evlist)); if (thread_map == NULL) return -1; @@ -2067,10 +2079,10 @@ static void alarm_sig_handler(int sig); static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist) { if (evlist) { - if (evlist->mmap && evlist->mmap[0].core.base) - return evlist->mmap[0].core.base; - if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base) - return evlist->overwrite_mmap[0].core.base; + if (evlist__mmap(evlist) && evlist__mmap(evlist)[0].core.base) + return evlist__mmap(evlist)[0].core.base; + if (evlist__overwrite_mmap(evlist) && evlist__overwrite_mmap(evlist)[0].core.base) + return evlist__overwrite_mmap(evlist)[0].core.base; } return NULL; } @@ -2150,7 +2162,7 @@ static int record__synthesize(struct record *rec, bool tail) if (err) goto out; - err = perf_event__synthesize_thread_map2(&rec->tool, rec->evlist->core.threads, + err = perf_event__synthesize_thread_map2(&rec->tool, evlist__core(rec->evlist)->threads, process_synthesized_event, NULL); if (err < 0) { @@ -2158,7 +2170,7 @@ static int record__synthesize(struct record *rec, bool tail) return err; } - err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.all_cpus, + err = perf_event__synthesize_cpu_map(&rec->tool, evlist__core(rec->evlist)->all_cpus, process_synthesized_event, NULL); if (err < 0) { pr_err("Couldn't synthesize cpu map.\n"); @@ -2191,7 +2203,7 @@ static int record__synthesize(struct record *rec, bool tail) bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP; err = __machine__synthesize_threads(machine, tool, &opts->target, - rec->evlist->core.threads, + evlist__core(rec->evlist)->threads, f, needs_mmap, opts->record_data_mmap, rec->opts.nr_threads_synthesize); } @@ -2246,6 +2258,8 @@ static int record__setup_sb_evlist(struct record *rec) if (evlist__add_bpf_sb_event(rec->sb_evlist, perf_session__env(rec->session))) { pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n."); + evlist__put(rec->sb_evlist); + rec->sb_evlist = NULL; return -1; } } @@ -2544,7 +2558,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * because we synthesize event name through the pipe * and need the id for that. */ - if (data->is_pipe && rec->evlist->core.nr_entries == 1) + if (data->is_pipe && evlist__nr_entries(rec->evlist) == 1) rec->opts.sample_id = true; if (rec->timestamp_filename && perf_data__is_pipe(data)) { @@ -2568,7 +2582,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } /* Debug message used by test scripts */ pr_debug3("perf record done opening and mmapping events\n"); - env->comp_mmap_len = session->evlist->core.mmap_len; + env->comp_mmap_len = evlist__core(session->evlist)->mmap_len; if (rec->opts.kcore) { err = record__kcore_copy(&session->machines.host, data); @@ -2669,7 +2683,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * Synthesize COMM event to prevent it. */ tgid = perf_event__synthesize_comm(tool, event, - rec->evlist->workload.pid, + evlist__workload_pid(rec->evlist), process_synthesized_event, machine); free(event); @@ -2689,7 +2703,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * Synthesize NAMESPACES event for the command specified. */ perf_event__synthesize_namespaces(tool, event, - rec->evlist->workload.pid, + evlist__workload_pid(rec->evlist), tgid, process_synthesized_event, machine); free(event); @@ -2706,7 +2720,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } } - err = event_enable_timer__start(rec->evlist->eet); + err = event_enable_timer__start(evlist__event_enable_timer(rec->evlist)); if (err) goto out_child; @@ -2768,7 +2782,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * record__mmap_read_all() didn't collect data from * overwritable ring buffer. Read again. */ - if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING) + if (evlist__bkw_mmap_state(rec->evlist) == BKW_MMAP_RUNNING) continue; trigger_ready(&switch_output_trigger); @@ -2837,7 +2851,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } } - err = event_enable_timer__process(rec->evlist->eet); + err = event_enable_timer__process(evlist__event_enable_timer(rec->evlist)); if (err < 0) goto out_child; if (err) { @@ -2889,11 +2903,13 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) record__synthesize_workload(rec, true); out_child: + evlist__disable(rec->evlist); record__stop_threads(rec); record__mmap_read_all(rec, true); goto out_free_threads; out_child_no_flush: /* mmap read already failed — retrying would just fail again */ + evlist__disable(rec->evlist); record__stop_threads(rec); out_free_threads: record__free_thread_data(rec); @@ -2909,7 +2925,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) int exit_status; if (!child_finished) - kill(rec->evlist->workload.pid, SIGTERM); + kill(evlist__workload_pid(rec->evlist), SIGTERM); wait(&exit_status); @@ -4032,7 +4048,7 @@ static int record__init_thread_default_masks(struct record *rec, struct perf_cpu static int record__init_thread_masks(struct record *rec) { int ret = 0; - struct perf_cpu_map *cpus = rec->evlist->core.all_cpus; + struct perf_cpu_map *cpus = evlist__core(rec->evlist)->all_cpus; if (!record__threads_enabled(rec)) return record__init_thread_default_masks(rec, cpus); @@ -4283,15 +4299,15 @@ int cmd_record(int argc, const char **argv) if (record.opts.overwrite) record.opts.tail_synthesize = true; - if (rec->evlist->core.nr_entries == 0) { + if (evlist__nr_entries(rec->evlist) == 0) { struct evlist *def_evlist = evlist__new_default(&rec->opts.target, callchain_param.enabled); if (!def_evlist) goto out; - evlist__splice_list_tail(rec->evlist, &def_evlist->core.entries); - evlist__delete(def_evlist); + evlist__splice_list_tail(rec->evlist, &evlist__core(def_evlist)->entries); + evlist__put(def_evlist); } if (rec->opts.target.tid && !rec->opts.no_inherit_set) @@ -4401,7 +4417,7 @@ int cmd_record(int argc, const char **argv) auxtrace_record__free(rec->itr); out_opts: evlist__close_control(rec->opts.ctl_fd, rec->opts.ctl_fd_ack, &rec->opts.ctl_fd_close); - evlist__delete(rec->evlist); + evlist__put(rec->evlist); return err; } diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c index dd1309c320943..10db1e5f1e6c4 100644 --- a/tools/perf/builtin-report.c +++ b/tools/perf/builtin-report.c @@ -561,7 +561,7 @@ static int evlist__tty_browse_hists(struct evlist *evlist, struct report *rep, c if (!quiet) { fprintf(stdout, "#\n# Total Lost Samples: %" PRIu64 "\n#\n", - evlist->stats.total_lost_samples); + evlist__stats(evlist)->total_lost_samples); } evlist__for_each_entry(evlist, pos) { @@ -1156,7 +1156,7 @@ static int __cmd_report(struct report *rep) PERF_HPP_REPORT__BLOCK_AVG_CYCLES, }; - if (session->evlist->nr_br_cntr > 0) + if (evlist__nr_br_cntr(session->evlist) > 0) block_hpps[nr_hpps++] = PERF_HPP_REPORT__BLOCK_BRANCH_COUNTER; block_hpps[nr_hpps++] = PERF_HPP_REPORT__BLOCK_RANGE; @@ -1291,7 +1291,7 @@ static int process_attr(const struct perf_tool *tool __maybe_unused, * on events sample_type. */ sample_type = evlist__combined_sample_type(*pevlist); - session = (*pevlist)->session; + session = evlist__session(*pevlist); callchain_param_setup(sample_type, perf_session__e_machine(session, /*e_flags=*/NULL)); return 0; } diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c index 7fd63a9db4574..d0921bd0ce758 100644 --- a/tools/perf/builtin-sched.c +++ b/tools/perf/builtin-sched.c @@ -1581,12 +1581,6 @@ static int process_sched_wakeup_event(const struct perf_tool *tool, return 0; } -static int process_sched_wakeup_ignore(const struct perf_tool *tool __maybe_unused, - struct perf_sample *sample __maybe_unused, - struct machine *machine __maybe_unused) -{ - return 0; -} static bool thread__has_color(struct thread *thread) { @@ -1833,7 +1827,7 @@ static int map_switch_event(struct perf_sched *sched, struct perf_sample *sampl sched_out: if (sched->map.task_name) { tr = thread__get_runtime(sched->curr_out_thread[this_cpu.cpu]); - if (strcmp(tr->shortname, "") == 0) + if (tr == NULL || strcmp(tr->shortname, "") == 0) goto out; if (proceed == 1) @@ -1938,6 +1932,22 @@ typedef int (*tracepoint_handler)(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine); +static struct evsel_str_handler latency_handlers[] = { + { "sched:sched_switch", process_sched_switch_event, }, + { "sched:sched_stat_runtime", process_sched_runtime_event, }, + { "sched:sched_wakeup", process_sched_wakeup_event, }, + { "sched:sched_waking", process_sched_wakeup_event, }, + { "sched:sched_wakeup_new", process_sched_wakeup_event, }, + { "sched:sched_migrate_task", process_sched_migrate_task_event, }, +}; + +static int process_sched_ignore(const struct perf_tool *tool __maybe_unused, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + return 0; +} + static int perf_sched__process_tracepoint_sample(const struct perf_tool *tool __maybe_unused, union perf_event *event __maybe_unused, struct perf_sample *sample, @@ -1946,7 +1956,23 @@ static int perf_sched__process_tracepoint_sample(const struct perf_tool *tool __ struct evsel *evsel = sample->evsel; int err = 0; - if (evsel->handler != NULL) { + if (evsel->handler == NULL) { + evsel->handler = process_sched_ignore; + for (size_t i = 0; i < ARRAY_SIZE(latency_handlers); i++) { + if (!evsel__name_is(evsel, latency_handlers[i].name)) + continue; + + if (!strcmp(latency_handlers[i].name, "sched:sched_wakeup") && + sample->evsel->evlist && + evlist__find_tracepoint_by_name(sample->evsel->evlist, "sched:sched_waking")) + break; + + evsel->handler = latency_handlers[i].handler; + break; + } + } + + if (evsel->handler != process_sched_ignore) { tracepoint_handler f = evsel->handler; err = f(tool, sample, machine); } @@ -1987,21 +2013,13 @@ static int perf_sched__process_comm(const struct perf_tool *tool __maybe_unused, static int perf_sched__read_events(struct perf_sched *sched) { - struct evsel_str_handler handlers[] = { - { "sched:sched_switch", process_sched_switch_event, }, - { "sched:sched_stat_runtime", process_sched_runtime_event, }, - { "sched:sched_wakeup", process_sched_wakeup_event, }, - { "sched:sched_waking", process_sched_wakeup_event, }, - { "sched:sched_wakeup_new", process_sched_wakeup_event, }, - { "sched:sched_migrate_task", process_sched_migrate_task_event, }, - }; struct perf_session *session; struct perf_data data = { .path = input_name, .mode = PERF_DATA_MODE_READ, .force = sched->force, }; - int rc = -1; + int rc = -1, err; session = perf_session__new(&data, &sched->tool); if (IS_ERR(session)) { @@ -2011,25 +2029,34 @@ static int perf_sched__read_events(struct perf_sched *sched) symbol__init(perf_session__env(session)); - /* prefer sched_waking if it is captured */ - if (evlist__find_tracepoint_by_name(session->evlist, "sched:sched_waking")) - handlers[2].handler = process_sched_wakeup_ignore; + if (!perf_data__is_pipe(session->data)) { + /* prefer sched_waking if it is captured */ + if (evlist__find_tracepoint_by_name(session->evlist, "sched:sched_waking")) + latency_handlers[2].handler = process_sched_ignore; - if (perf_session__set_tracepoints_handlers(session, handlers)) + if (perf_session__set_tracepoints_handlers(session, latency_handlers)) + goto out_delete; + } + + if (!perf_data__is_pipe(session->data) && + !perf_session__has_traces(session, "record -R")) goto out_delete; - if (perf_session__has_traces(session, "record -R")) { - int err = perf_session__process_events(session); - if (err) { - pr_err("Failed to process events, error %d", err); - goto out_delete; - } + err = perf_session__process_events(session); + if (err) { + pr_err("Failed to process events, error %d", err); + goto out_delete; + } - sched->nr_events = session->evlist->stats.nr_events[0]; - sched->nr_lost_events = session->evlist->stats.total_lost; - sched->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST]; + if (perf_data__is_pipe(session->data) && + !perf_session__has_traces(session, "record -R")) { + goto out_delete; } + sched->nr_events = evlist__stats(session->evlist)->nr_events[0]; + sched->nr_lost_events = evlist__stats(session->evlist)->total_lost; + sched->nr_lost_chunks = evlist__stats(session->evlist)->nr_events[PERF_RECORD_LOST]; + rc = 0; out_delete: perf_session__delete(session); @@ -3303,7 +3330,7 @@ static int timehist_check_attr(struct perf_sched *sched, struct evsel *evsel; struct evsel_runtime *er; - list_for_each_entry(evsel, &evlist->core.entries, core.node) { + list_for_each_entry(evsel, &evlist__core(evlist)->entries, core.node) { er = evsel__get_runtime(evsel); if (er == NULL) { pr_err("Failed to allocate memory for evsel runtime data\n"); @@ -3475,9 +3502,9 @@ static int perf_sched__timehist(struct perf_sched *sched) goto out; } - sched->nr_events = evlist->stats.nr_events[0]; - sched->nr_lost_events = evlist->stats.total_lost; - sched->nr_lost_chunks = evlist->stats.nr_events[PERF_RECORD_LOST]; + sched->nr_events = evlist__stats(evlist)->nr_events[0]; + sched->nr_lost_events = evlist__stats(evlist)->total_lost; + sched->nr_lost_chunks = evlist__stats(evlist)->nr_events[PERF_RECORD_LOST]; if (sched->summary) timehist_print_summary(sched, session); @@ -3924,7 +3951,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, session = perf_session__new(&data, &sched->tool); if (IS_ERR(session)) { pr_err("Perf session creation failed.\n"); - evlist__delete(evlist); + evlist__put(evlist); return PTR_ERR(session); } @@ -3982,7 +4009,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, if (err < 0) goto out; - user_requested_cpus = evlist->core.user_requested_cpus; + user_requested_cpus = evlist__core(evlist)->user_requested_cpus; err = perf_event__synthesize_schedstat(&(sched->tool), process_synthesized_schedstat_event, @@ -3998,7 +4025,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, evlist__start_workload(evlist); while (!done) { - if (argc && waitpid(evlist->workload.pid, NULL, WNOHANG) > 0) + if (argc && waitpid(evlist__workload_pid(evlist), NULL, WNOHANG) > 0) break; sleep(1); } @@ -4023,8 +4050,8 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, else fprintf(stderr, "[ perf sched stats: Failed !! ]\n"); - evlist__delete(evlist); - close(fd); + perf_session__delete(session); + evlist__put(evlist); return err; } @@ -4627,6 +4654,8 @@ static int perf_sched__process_schedstat(const struct perf_tool *tool __maybe_un domain_second_pass = list_first_entry(&cpu_second_pass->domain_head, struct schedstat_domain, domain_list); store_schedstat_cpu_diff(temp); + free(temp->cpu_data); + free(temp); } } else if (event->header.type == PERF_RECORD_SCHEDSTAT_DOMAIN) { struct schedstat_cpu *cpu_tail; @@ -4647,6 +4676,8 @@ static int perf_sched__process_schedstat(const struct perf_tool *tool __maybe_un } else { store_schedstat_domain_diff(temp); domain_second_pass = list_next_entry(domain_second_pass, domain_list); + free(temp->domain_data); + free(temp); } } @@ -4699,7 +4730,7 @@ static int perf_sched__schedstat_report(struct perf_sched *sched) if (err < 0) goto out; - user_requested_cpus = session->evlist->core.user_requested_cpus; + user_requested_cpus = evlist__core(session->evlist)->user_requested_cpus; err = perf_session__process_events(session); @@ -4875,7 +4906,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched, if (err < 0) goto out; - user_requested_cpus = evlist->core.user_requested_cpus; + user_requested_cpus = evlist__core(evlist)->user_requested_cpus; err = perf_event__synthesize_schedstat(&(sched->tool), process_synthesized_event_live, @@ -4891,7 +4922,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched, evlist__start_workload(evlist); while (!done) { - if (argc && waitpid(evlist->workload.pid, NULL, WNOHANG) > 0) + if (argc && waitpid(evlist__workload_pid(evlist), NULL, WNOHANG) > 0) break; sleep(1); } @@ -4927,7 +4958,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched, free_cpu_domain_info(cd_map, sv, nr); out: free_schedstat(&cpu_head); - evlist__delete(evlist); + evlist__put(evlist); return err; } @@ -5167,6 +5198,10 @@ int cmd_sched(int argc, const char **argv) sched.tool.namespaces = perf_event__process_namespaces; sched.tool.lost = perf_event__process_lost; sched.tool.fork = perf_sched__process_fork_event; + sched.tool.attr = perf_event__process_attr; + sched.tool.tracing_data = perf_event__process_tracing_data; + sched.tool.build_id = perf_event__process_build_id; + sched.tool.feature = perf_event__process_feature; argc = parse_options_subcommand(argc, argv, sched_options, sched_subcommands, sched_usage, PARSE_OPT_STOP_AT_NON_OPTION); @@ -5251,19 +5286,20 @@ int cmd_sched(int argc, const char **argv) if (argc) argc = parse_options(argc, argv, stats_options, stats_usage, 0); - return perf_sched__schedstat_record(&sched, argc, argv); + ret = perf_sched__schedstat_record(&sched, argc, argv); } else if (argv[0] && !strcmp(argv[0], "report")) { if (argc) argc = parse_options(argc, argv, stats_options, stats_usage, 0); - return perf_sched__schedstat_report(&sched); + ret = perf_sched__schedstat_report(&sched); } else if (argv[0] && !strcmp(argv[0], "diff")) { if (argc) argc = parse_options(argc, argv, stats_options, stats_usage, 0); - return perf_sched__schedstat_diff(&sched, argc, argv); + ret = perf_sched__schedstat_diff(&sched, argc, argv); + } else { + ret = perf_sched__schedstat_live(&sched, argc, argv); } - return perf_sched__schedstat_live(&sched, argc, argv); } else { usage_with_options(sched_usage, sched_options); } diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c index 9ac29bdc3cd54..ad8ca08ceb5f2 100644 --- a/tools/perf/builtin-script.c +++ b/tools/perf/builtin-script.c @@ -2174,6 +2174,7 @@ static int script_find_metrics(const struct pmu_metric *pm, struct evsel *metric_evsel; int ret = metricgroup__parse_groups(metric_evlist, /*pmu=*/"all", + /*cputype_filter=*/false, pm->metric_name, /*metric_no_group=*/false, /*metric_no_merge=*/false, @@ -2229,9 +2230,10 @@ static int script_find_metrics(const struct pmu_metric *pm, evlist__for_each_entry(metric_evlist, metric_evsel) { struct evsel *script_evsel = map_metric_evsel_to_script_evsel(script_evlist, metric_evsel); - struct metric_event *metric_me = metricgroup__lookup(&metric_evlist->metric_events, - metric_evsel, - /*create=*/false); + struct metric_event *metric_me = + metricgroup__lookup(evlist__metric_events(metric_evlist), + metric_evsel, + /*create=*/false); if (script_evsel->metric_id == NULL) { script_evsel->metric_id = metric_evsel->metric_id; @@ -2251,7 +2253,7 @@ static int script_find_metrics(const struct pmu_metric *pm, if (metric_me) { struct metric_expr *expr; struct metric_event *script_me = - metricgroup__lookup(&script_evlist->metric_events, + metricgroup__lookup(evlist__metric_events(script_evlist), script_evsel, /*create=*/true); @@ -2274,8 +2276,8 @@ static int script_find_metrics(const struct pmu_metric *pm, } pr_debug("Found metric '%s' whose evsels match those of in the perf data\n", pm->metric_name); - evlist__delete(metric_evlist); out: + evlist__put(metric_evlist); return 0; } @@ -2321,7 +2323,7 @@ static void perf_sample__fprint_metric(struct thread *thread, assert(stat_config.aggr_mode == AGGR_GLOBAL); stat_config.aggr_get_id = script_aggr_cpu_id_get; stat_config.aggr_map = - cpu_aggr_map__new(evsel->evlist->core.user_requested_cpus, + cpu_aggr_map__new(evlist__core(evsel->evlist)->user_requested_cpus, aggr_cpu_id__global, /*data=*/NULL, /*needs_sort=*/false); } @@ -3909,7 +3911,7 @@ static int set_maps(struct perf_script *script) if (WARN_ONCE(script->allocated, "stats double allocation\n")) return -EINVAL; - perf_evlist__set_maps(&evlist->core, script->cpus, script->threads); + perf_evlist__set_maps(evlist__core(evlist), script->cpus, script->threads); if (evlist__alloc_stats(&stat_config, evlist, /*alloc_raw=*/true)) return -ENOMEM; diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index a04466ea3b0a0..10681a4880086 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -321,7 +321,7 @@ static int read_single_counter(struct evsel *counter, int cpu_map_idx, int threa */ static int read_counter_cpu(struct evsel *counter, int cpu_map_idx) { - int nthreads = perf_thread_map__nr(evsel_list->core.threads); + int nthreads = perf_thread_map__nr(evlist__core(evsel_list)->threads); int thread; if (!counter->supported) @@ -628,11 +628,12 @@ static int dispatch_events(bool forks, int timeout, int interval, int *times) time_to_sleep = sleep_time; while (!done) { - if (forks) + if (forks) { child_exited = waitpid(child_pid, &status, WNOHANG); - else - child_exited = !is_target_alive(&target, evsel_list->core.threads) ? 1 : 0; - + } else { + child_exited = !is_target_alive(&target, + evlist__core(evsel_list)->threads) ? 1 : 0; + } if (child_exited) break; @@ -681,14 +682,15 @@ static enum counter_recovery stat_handle_error(struct evsel *counter, int err) return COUNTER_RETRY; } if (target__has_per_thread(&target) && err != EOPNOTSUPP && - evsel_list->core.threads && evsel_list->core.threads->err_thread != -1) { + evlist__core(evsel_list)->threads && + evlist__core(evsel_list)->threads->err_thread != -1) { /* * For global --per-thread case, skip current * error thread. */ - if (!thread_map__remove(evsel_list->core.threads, - evsel_list->core.threads->err_thread)) { - evsel_list->core.threads->err_thread = -1; + if (!thread_map__remove(evlist__core(evsel_list)->threads, + evlist__core(evsel_list)->threads->err_thread)) { + evlist__core(evsel_list)->threads->err_thread = -1; counter->supported = true; return COUNTER_RETRY; } @@ -787,11 +789,12 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx) bool second_pass = false, has_supported_counters; if (forks) { - if (evlist__prepare_workload(evsel_list, &target, argv, is_pipe, workload_exec_failed_signal) < 0) { + if (evlist__prepare_workload(evsel_list, &target, argv, is_pipe, + workload_exec_failed_signal) < 0) { perror("failed to prepare workload"); return -1; } - child_pid = evsel_list->workload.pid; + child_pid = evlist__workload_pid(evsel_list); } evlist__for_each_entry(evsel_list, counter) { @@ -1199,7 +1202,7 @@ static int parse_cputype(const struct option *opt, const struct perf_pmu *pmu; struct evlist *evlist = *(struct evlist **)opt->value; - if (!list_empty(&evlist->core.entries)) { + if (!list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "Must define cputype before events/metrics\n"); return -1; } @@ -1210,6 +1213,7 @@ static int parse_cputype(const struct option *opt, return -1; } parse_events_option_args.pmu_filter = pmu->name; + parse_events_option_args.cputype_filter = true; return 0; } @@ -1220,12 +1224,13 @@ static int parse_pmu_filter(const struct option *opt, { struct evlist *evlist = *(struct evlist **)opt->value; - if (!list_empty(&evlist->core.entries)) { + if (!list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "Must define pmu-filter before events/metrics\n"); return -1; } parse_events_option_args.pmu_filter = str; + parse_events_option_args.cputype_filter = false; return 0; } @@ -1586,8 +1591,9 @@ static int perf_stat_init_aggr_mode(void) if (get_id) { bool needs_sort = stat_config.aggr_mode != AGGR_NONE; - stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.user_requested_cpus, - get_id, /*data=*/NULL, needs_sort); + stat_config.aggr_map = cpu_aggr_map__new( + evlist__core(evsel_list)->user_requested_cpus, + get_id, /*data=*/NULL, needs_sort); if (!stat_config.aggr_map) { pr_err("cannot build %s map\n", aggr_mode__string[stat_config.aggr_mode]); return -1; @@ -1596,7 +1602,7 @@ static int perf_stat_init_aggr_mode(void) } if (stat_config.aggr_mode == AGGR_THREAD) { - nr = perf_thread_map__nr(evsel_list->core.threads); + nr = perf_thread_map__nr(evlist__core(evsel_list)->threads); stat_config.aggr_map = cpu_aggr_map__empty_new(nr); if (stat_config.aggr_map == NULL) return -ENOMEM; @@ -1615,7 +1621,7 @@ static int perf_stat_init_aggr_mode(void) * taking the highest cpu number to be the size of * the aggregation translate cpumap. */ - nr = perf_cpu_map__max(evsel_list->core.all_cpus).cpu + 1; + nr = perf_cpu_map__max(evlist__core(evsel_list)->all_cpus).cpu + 1; stat_config.cpus_aggr_map = cpu_aggr_map__empty_new(nr); return stat_config.cpus_aggr_map ? 0 : -ENOMEM; } @@ -1902,7 +1908,7 @@ static int perf_stat_init_aggr_mode_file(struct perf_stat *st) bool needs_sort = stat_config.aggr_mode != AGGR_NONE; if (stat_config.aggr_mode == AGGR_THREAD) { - int nr = perf_thread_map__nr(evsel_list->core.threads); + int nr = perf_thread_map__nr(evlist__core(evsel_list)->threads); stat_config.aggr_map = cpu_aggr_map__empty_new(nr); if (stat_config.aggr_map == NULL) @@ -1920,7 +1926,7 @@ static int perf_stat_init_aggr_mode_file(struct perf_stat *st) if (!get_id) return 0; - stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.user_requested_cpus, + stat_config.aggr_map = cpu_aggr_map__new(evlist__core(evsel_list)->user_requested_cpus, get_id, env, needs_sort); if (!stat_config.aggr_map) { pr_err("cannot build %s map\n", aggr_mode__string[stat_config.aggr_mode]); @@ -1999,7 +2005,9 @@ static int add_default_events(void) ret = -1; goto out; } - ret = metricgroup__parse_groups(evlist, pmu, "transaction", + ret = metricgroup__parse_groups(evlist, pmu, + parse_events_option_args.cputype_filter, + "transaction", stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -2036,7 +2044,9 @@ static int add_default_events(void) if (!force_metric_only) stat_config.metric_only = true; - ret = metricgroup__parse_groups(evlist, pmu, "smi", + ret = metricgroup__parse_groups(evlist, pmu, + parse_events_option_args.cputype_filter, + "smi", stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -2073,7 +2083,7 @@ static int add_default_events(void) } str[8] = stat_config.topdown_level + '0'; if (metricgroup__parse_groups(evlist, - pmu, str, + pmu, parse_events_option_args.cputype_filter, str, /*metric_no_group=*/false, /*metric_no_merge=*/false, /*metric_no_threshold=*/true, @@ -2088,7 +2098,7 @@ static int add_default_events(void) if (!stat_config.topdown_level) stat_config.topdown_level = 1; - if (!evlist->core.nr_entries && !evsel_list->core.nr_entries) { + if (!evlist__nr_entries(evlist) && !evlist__nr_entries(evsel_list)) { /* * Add Default metrics. To minimize multiplexing, don't request * threshold computation, but it will be computed if the events @@ -2112,14 +2122,16 @@ static int add_default_events(void) ret = -ENOMEM; break; } - if (metricgroup__parse_groups(metric_evlist, pmu, default_metricgroup_names[i], + if (metricgroup__parse_groups(metric_evlist, pmu, + parse_events_option_args.cputype_filter, + default_metricgroup_names[i], /*metric_no_group=*/false, /*metric_no_merge=*/false, /*metric_no_threshold=*/true, stat_config.user_requested_cpu_list, stat_config.system_wide, stat_config.hardware_aware_grouping) < 0) { - evlist__delete(metric_evlist); + evlist__put(metric_evlist); ret = -1; break; } @@ -2127,13 +2139,13 @@ static int add_default_events(void) evlist__for_each_entry(metric_evlist, evsel) evsel->default_metricgroup = true; - evlist__splice_list_tail(evlist, &metric_evlist->core.entries); + evlist__splice_list_tail(evlist, &evlist__core(metric_evlist)->entries); metricgroup__copy_metric_events(evlist, /*cgrp=*/NULL, - &evlist->metric_events, - &metric_evlist->metric_events); - evlist__delete(metric_evlist); + evlist__metric_events(evlist), + evlist__metric_events(metric_evlist)); + evlist__put(metric_evlist); } - list_sort(/*priv=*/NULL, &evlist->core.entries, default_evlist_evsel_cmp); + list_sort(/*priv=*/NULL, &evlist__core(evlist)->entries, default_evlist_evsel_cmp); } out: @@ -2148,11 +2160,11 @@ static int add_default_events(void) } } parse_events_error__exit(&err); - evlist__splice_list_tail(evsel_list, &evlist->core.entries); + evlist__splice_list_tail(evsel_list, &evlist__core(evlist)->entries); metricgroup__copy_metric_events(evsel_list, /*cgrp=*/NULL, - &evsel_list->metric_events, - &evlist->metric_events); - evlist__delete(evlist); + evlist__metric_events(evsel_list), + evlist__metric_events(evlist)); + evlist__put(evlist); return ret; } @@ -2272,7 +2284,7 @@ static int set_maps(struct perf_stat *st) if (WARN_ONCE(st->maps_allocated, "stats double allocation\n")) return -EINVAL; - perf_evlist__set_maps(&evsel_list->core, st->cpus, st->threads); + perf_evlist__set_maps(evlist__core(evsel_list), st->cpus, st->threads); if (evlist__alloc_stats(&stat_config, evsel_list, /*alloc_raw=*/true)) return -ENOMEM; @@ -2387,7 +2399,7 @@ static int __cmd_report(int argc, const char **argv) perf_stat.session = session; stat_config.output = stderr; - evlist__delete(evsel_list); + evlist__put(evsel_list); evsel_list = session->evlist; ret = perf_session__process_events(session); @@ -2424,7 +2436,7 @@ static void setup_system_wide(int forks) } } - if (evsel_list->core.nr_entries) + if (evlist__nr_entries(evsel_list)) target.system_wide = true; } } @@ -2651,7 +2663,7 @@ int cmd_stat(int argc, const char **argv) stat_config.csv_sep = DEFAULT_SEPARATOR; if (affinity_set) - evsel_list->no_affinity = !affinity; + evlist__set_no_affinity(evsel_list, !affinity); if (argc && strlen(argv[0]) > 2 && strstarts("record", argv[0])) { argc = __cmd_record(stat_options, &opt_mode, argc, argv); @@ -2714,7 +2726,8 @@ int cmd_stat(int argc, const char **argv) output = fopen(output_name, mode); if (!output) { perror("failed to create output file"); - return -1; + status = -1; + goto out; } if (!stat_config.json_output) { clock_gettime(CLOCK_REALTIME, &tm); @@ -2725,7 +2738,8 @@ int cmd_stat(int argc, const char **argv) output = fdopen(output_fd, mode); if (!output) { perror("Failed opening logfd"); - return -errno; + status = -errno; + goto out; } } @@ -2734,7 +2748,8 @@ int cmd_stat(int argc, const char **argv) parse_options_usage(stat_usage, stat_options, "o", 1); parse_options_usage(NULL, stat_options, "log-fd", 0); parse_options_usage(NULL, stat_options, "interval-clear", 0); - return -1; + status = -1; + goto out; } stat_config.output = output; @@ -2818,7 +2833,7 @@ int cmd_stat(int argc, const char **argv) } if (stat_config.iostat_run) { - status = iostat_prepare(evsel_list, &stat_config); + status = iostat_prepare(&evsel_list, &stat_config); if (status) goto out; if (iostat_mode == IOSTAT_LIST) { @@ -2848,7 +2863,9 @@ int cmd_stat(int argc, const char **argv) */ if (metrics) { const char *pmu = parse_events_option_args.pmu_filter ?: "all"; - int ret = metricgroup__parse_groups(evsel_list, pmu, metrics, + int ret = metricgroup__parse_groups(evsel_list, pmu, + parse_events_option_args.cputype_filter, + metrics, stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -2882,9 +2899,10 @@ int cmd_stat(int argc, const char **argv) } #ifdef HAVE_BPF_SKEL if (target.use_bpf && nr_cgroups && - (evsel_list->core.nr_entries / nr_cgroups) > BPERF_CGROUP__MAX_EVENTS) { + (evlist__nr_entries(evsel_list) / nr_cgroups) > BPERF_CGROUP__MAX_EVENTS) { pr_warning("Disabling BPF counters due to more events (%d) than the max (%d)\n", - evsel_list->core.nr_entries / nr_cgroups, BPERF_CGROUP__MAX_EVENTS); + evlist__nr_entries(evsel_list) / nr_cgroups, + BPERF_CGROUP__MAX_EVENTS); target.use_bpf = false; } #endif // HAVE_BPF_SKEL @@ -2922,7 +2940,7 @@ int cmd_stat(int argc, const char **argv) * so we could print it out on output. */ if (stat_config.aggr_mode == AGGR_THREAD) { - thread_map__read_comms(evsel_list->core.threads); + thread_map__read_comms(evlist__core(evsel_list)->threads); } if (stat_config.aggr_mode == AGGR_NODE) @@ -3066,7 +3084,7 @@ int cmd_stat(int argc, const char **argv) if (smi_cost && smi_reset) sysfs__write_int(FREEZE_ON_SMI_PATH, 0); - evlist__delete(evsel_list); + evlist__put(evsel_list); evlist__close_control(stat_config.ctl_fd, stat_config.ctl_fd_ack, &stat_config.ctl_fd_close); diff --git a/tools/perf/builtin-top.c b/tools/perf/builtin-top.c index 1211401616ee3..570410599f1b3 100644 --- a/tools/perf/builtin-top.c +++ b/tools/perf/builtin-top.c @@ -141,7 +141,7 @@ static int perf_top__parse_source(struct perf_top *top, struct hist_entry *he) notes = symbol__annotation(sym); annotation__lock(notes); - if (!symbol__hists(sym, top->evlist->core.nr_entries)) { + if (!symbol__hists(sym, evlist__nr_entries(top->evlist))) { annotation__unlock(notes); pr_err("Not enough memory for annotating '%s' symbol!\n", sym->name); @@ -267,7 +267,7 @@ static void perf_top__show_details(struct perf_top *top) more = hist_entry__annotate_printf(he, top->sym_evsel); - if (top->evlist->enabled) { + if (evlist__enabled(top->evlist)) { if (top->zero) symbol__annotate_zero_histogram(symbol, top->sym_evsel); else @@ -293,7 +293,7 @@ static void perf_top__resort_hists(struct perf_top *t) */ hists__unlink(hists); - if (evlist->enabled) { + if (evlist__enabled(evlist)) { if (t->zero) { hists__delete_entries(hists); } else { @@ -334,13 +334,13 @@ static void perf_top__print_sym_table(struct perf_top *top) printf("%-*.*s\n", win_width, win_width, graph_dotted_line); if (!top->record_opts.overwrite && - (top->evlist->stats.nr_lost_warned != - top->evlist->stats.nr_events[PERF_RECORD_LOST])) { - top->evlist->stats.nr_lost_warned = - top->evlist->stats.nr_events[PERF_RECORD_LOST]; + (evlist__stats(top->evlist)->nr_lost_warned != + evlist__stats(top->evlist)->nr_events[PERF_RECORD_LOST])) { + evlist__stats(top->evlist)->nr_lost_warned = + evlist__stats(top->evlist)->nr_events[PERF_RECORD_LOST]; color_fprintf(stdout, PERF_COLOR_RED, "WARNING: LOST %d chunks, Check IO/CPU overload", - top->evlist->stats.nr_lost_warned); + evlist__stats(top->evlist)->nr_lost_warned); ++printed; } @@ -447,7 +447,7 @@ static void perf_top__print_mapped_keys(struct perf_top *top) fprintf(stdout, "\t[d] display refresh delay. \t(%d)\n", top->delay_secs); fprintf(stdout, "\t[e] display entries (lines). \t(%d)\n", top->print_entries); - if (top->evlist->core.nr_entries > 1) + if (evlist__nr_entries(top->evlist) > 1) fprintf(stdout, "\t[E] active event counter. \t(%s)\n", evsel__name(top->sym_evsel)); fprintf(stdout, "\t[f] profile display filter (count). \t(%d)\n", top->count_filter); @@ -482,7 +482,7 @@ static int perf_top__key_mapped(struct perf_top *top, int c) case 'S': return 1; case 'E': - return top->evlist->core.nr_entries > 1 ? 1 : 0; + return evlist__nr_entries(top->evlist) > 1 ? 1 : 0; default: break; } @@ -528,7 +528,7 @@ static bool perf_top__handle_keypress(struct perf_top *top, int c) } break; case 'E': - if (top->evlist->core.nr_entries > 1) { + if (evlist__nr_entries(top->evlist) > 1) { /* Select 0 as the default event: */ int counter = 0; @@ -539,7 +539,7 @@ static bool perf_top__handle_keypress(struct perf_top *top, int c) prompt_integer(&counter, "Enter details event counter"); - if (counter >= top->evlist->core.nr_entries) { + if (counter >= evlist__nr_entries(top->evlist)) { top->sym_evsel = evlist__first(top->evlist); fprintf(stderr, "Sorry, no such event, using %s.\n", evsel__name(top->sym_evsel)); sleep(1); @@ -598,8 +598,8 @@ static void perf_top__sort_new_samples(void *arg) { struct perf_top *t = arg; - if (t->evlist->selected != NULL) - t->sym_evsel = t->evlist->selected; + if (evlist__selected(t->evlist) != NULL) + t->sym_evsel = evlist__selected(t->evlist); perf_top__resort_hists(t); @@ -766,7 +766,7 @@ static void perf_event__process_sample(const struct perf_tool *tool, if (!machine) { pr_err("%u unprocessable samples recorded.\r", - top->session->evlist->stats.nr_unprocessable_samples++); + evlist__stats(top->session->evlist)->nr_unprocessable_samples++); return; } @@ -859,7 +859,7 @@ perf_top__process_lost(struct perf_top *top, union perf_event *event, { top->lost += event->lost.lost; top->lost_total += event->lost.lost; - evsel->evlist->stats.total_lost += event->lost.lost; + evlist__stats(evsel->evlist)->total_lost += event->lost.lost; } static void @@ -869,7 +869,7 @@ perf_top__process_lost_samples(struct perf_top *top, { top->lost += event->lost_samples.lost; top->lost_total += event->lost_samples.lost; - evsel->evlist->stats.total_lost_samples += event->lost_samples.lost; + evlist__stats(evsel->evlist)->total_lost_samples += event->lost_samples.lost; } static u64 last_timestamp; @@ -881,7 +881,7 @@ static void perf_top__mmap_read_idx(struct perf_top *top, int idx) struct mmap *md; union perf_event *event; - md = opts->overwrite ? &evlist->overwrite_mmap[idx] : &evlist->mmap[idx]; + md = opts->overwrite ? &evlist__overwrite_mmap(evlist)[idx] : &evlist__mmap(evlist)[idx]; if (perf_mmap__read_init(&md->core) < 0) return; @@ -918,7 +918,7 @@ static void perf_top__mmap_read(struct perf_top *top) if (overwrite) evlist__toggle_bkw_mmap(evlist, BKW_MMAP_DATA_PENDING); - for (i = 0; i < top->evlist->core.nr_mmaps; i++) + for (i = 0; i < evlist__core(top->evlist)->nr_mmaps; i++) perf_top__mmap_read_idx(top, i); if (overwrite) { @@ -1063,7 +1063,7 @@ static int perf_top__start_counters(struct perf_top *top) goto out_err; } - if (evlist__mmap(evlist, opts->mmap_pages) < 0) { + if (evlist__do_mmap(evlist, opts->mmap_pages) < 0) { ui__error("Failed to mmap with %d (%s)\n", errno, str_error_r(errno, msg, sizeof(msg))); goto out_err; @@ -1218,10 +1218,10 @@ static int deliver_event(struct ordered_events *qe, } else if (event->header.type == PERF_RECORD_LOST_SAMPLES) { perf_top__process_lost_samples(top, event, evsel); } else if (event->header.type < PERF_RECORD_MAX) { - events_stats__inc(&session->evlist->stats, event->header.type); + events_stats__inc(evlist__stats(session->evlist), event->header.type); machine__process_event(machine, event, &sample); } else - ++session->evlist->stats.nr_unknown_events; + ++evlist__stats(session->evlist)->nr_unknown_events; ret = 0; next_event: @@ -1296,7 +1296,7 @@ static int __cmd_top(struct perf_top *top) pr_debug("Couldn't synthesize cgroup events.\n"); machine__synthesize_threads(&top->session->machines.host, &opts->target, - top->evlist->core.threads, true, false, + evlist__core(top->evlist)->threads, true, false, top->nr_threads_synthesize); perf_set_multithreaded(); @@ -1652,14 +1652,14 @@ int cmd_top(int argc, const char **argv) perf_env__init(&host_env); status = perf_config(perf_top_config, &top); if (status) - goto out_delete_evlist; + goto out_put_evlist; /* * Since the per arch annotation init routine may need the cpuid, read * it here, since we are not getting this from the perf.data header. */ status = perf_env__set_cmdline(&host_env, argc, argv); if (status) - goto out_delete_evlist; + goto out_put_evlist; status = perf_env__read_cpuid(&host_env); if (status) { @@ -1680,30 +1680,30 @@ int cmd_top(int argc, const char **argv) annotate_opts.disassembler_style = strdup(disassembler_style); if (!annotate_opts.disassembler_style) { status = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } } if (objdump_path) { annotate_opts.objdump_path = strdup(objdump_path); if (!annotate_opts.objdump_path) { status = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } } if (addr2line_path) { symbol_conf.addr2line_path = strdup(addr2line_path); if (!symbol_conf.addr2line_path) { status = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } } status = symbol__validate_sym_arguments(); if (status) - goto out_delete_evlist; + goto out_put_evlist; if (annotate_check_args() < 0) - goto out_delete_evlist; + goto out_put_evlist; status = target__validate(target); if (status) { @@ -1714,19 +1714,19 @@ int cmd_top(int argc, const char **argv) if (target__none(target)) target->system_wide = true; - if (!top.evlist->core.nr_entries) { + if (!evlist__nr_entries(top.evlist)) { struct evlist *def_evlist = evlist__new_default(target, callchain_param.enabled); if (!def_evlist) - goto out_delete_evlist; + goto out_put_evlist; - evlist__splice_list_tail(top.evlist, &def_evlist->core.entries); - evlist__delete(def_evlist); + evlist__splice_list_tail(top.evlist, &evlist__core(def_evlist)->entries); + evlist__put(def_evlist); } status = evswitch__init(&top.evswitch, top.evlist, stderr); if (status) - goto out_delete_evlist; + goto out_put_evlist; if (symbol_conf.report_hierarchy) { /* disable incompatible options */ @@ -1737,18 +1737,18 @@ int cmd_top(int argc, const char **argv) pr_err("Error: --hierarchy and --fields options cannot be used together\n"); parse_options_usage(top_usage, options, "fields", 0); parse_options_usage(NULL, options, "hierarchy", 0); - goto out_delete_evlist; + goto out_put_evlist; } } if (top.stitch_lbr && !(callchain_param.record_mode == CALLCHAIN_LBR)) { pr_err("Error: --stitch-lbr must be used with --call-graph lbr\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (nr_cgroups > 0 && opts->record_cgroup) { pr_err("--cgroup and --all-cgroups cannot be used together\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (branch_call_mode) { @@ -1772,7 +1772,7 @@ int cmd_top(int argc, const char **argv) status = perf_env__read_core_pmu_caps(&host_env); if (status) { pr_err("PMU capability data is not available\n"); - goto out_delete_evlist; + goto out_put_evlist; } } @@ -1795,9 +1795,9 @@ int cmd_top(int argc, const char **argv) if (IS_ERR(top.session)) { status = PTR_ERR(top.session); top.session = NULL; - goto out_delete_evlist; + goto out_put_evlist; } - top.evlist->session = top.session; + evlist__set_session(top.evlist, top.session); if (setup_sorting(top.evlist, perf_session__env(top.session)) < 0) { if (sort_order) @@ -1805,7 +1805,7 @@ int cmd_top(int argc, const char **argv) if (field_order) parse_options_usage(sort_order ? NULL : top_usage, options, "fields", 0); - goto out_delete_evlist; + goto out_put_evlist; } if (top.uid_str) { @@ -1814,18 +1814,18 @@ int cmd_top(int argc, const char **argv) if (uid == UINT_MAX) { ui__error("Invalid User: %s", top.uid_str); status = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } status = parse_uid_filter(top.evlist, uid); if (status) - goto out_delete_evlist; + goto out_put_evlist; } if (evlist__create_maps(top.evlist, target) < 0) { ui__error("Couldn't create thread/CPU maps: %s\n", errno == ENOENT ? "No such process" : str_error_r(errno, errbuf, sizeof(errbuf))); status = -errno; - goto out_delete_evlist; + goto out_put_evlist; } if (top.delay_secs < 1) @@ -1833,7 +1833,7 @@ int cmd_top(int argc, const char **argv) if (record_opts__config(opts)) { status = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } top.sym_evsel = evlist__first(top.evlist); @@ -1848,14 +1848,14 @@ int cmd_top(int argc, const char **argv) status = symbol__annotation_init(); if (status < 0) - goto out_delete_evlist; + goto out_put_evlist; annotation_config__init(); symbol_conf.try_vmlinux_path = (symbol_conf.vmlinux_name == NULL); status = symbol__init(NULL); if (status < 0) - goto out_delete_evlist; + goto out_put_evlist; sort__setup_elide(stdout); @@ -1875,13 +1875,15 @@ int cmd_top(int argc, const char **argv) if (top.sb_evlist == NULL) { pr_err("Couldn't create side band evlist.\n."); status = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } if (evlist__add_bpf_sb_event(top.sb_evlist, &host_env)) { pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n."); status = -EINVAL; - goto out_delete_evlist; + evlist__put(top.sb_evlist); + top.sb_evlist = NULL; + goto out_put_evlist; } } #endif @@ -1896,8 +1898,8 @@ int cmd_top(int argc, const char **argv) if (!opts->no_bpf_event) evlist__stop_sb_thread(top.sb_evlist); -out_delete_evlist: - evlist__delete(top.evlist); +out_put_evlist: + evlist__put(top.evlist); perf_session__delete(top.session); annotation_options__exit(); perf_env__exit(&host_env); diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index ba0f8749fc7d7..0418808dbc4d2 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -460,10 +460,10 @@ static int evsel__init_tp_ptr_field(struct evsel *evsel, struct tp_field *field, ({ struct syscall_tp *sc = __evsel__syscall_tp(evsel);\ evsel__init_tp_ptr_field(evsel, &sc->name, #name); }) -static void evsel__delete_priv(struct evsel *evsel) +static void evsel__put_and_free_priv(struct evsel *evsel) { zfree(&evsel->priv); - evsel__delete(evsel); + evsel__put(evsel); } static int evsel__init_syscall_tp(struct evsel *evsel) @@ -543,7 +543,7 @@ static struct evsel *perf_evsel__raw_syscall_newtp(const char *direction, void * return evsel; out_delete: - evsel__delete_priv(evsel); + evsel__put_and_free_priv(evsel); return NULL; } @@ -2023,7 +2023,7 @@ static int trace__symbols_init(struct trace *trace, int argc, const char **argv, goto out; err = __machine__synthesize_threads(trace->host, &trace->tool, &trace->opts.target, - evlist->core.threads, trace__tool_process, + evlist__core(evlist)->threads, trace__tool_process, /*needs_mmap=*/callchain_param.enabled && !trace->summary_only, /*mmap_data=*/false, @@ -3207,6 +3207,22 @@ static void bpf_output__fprintf(struct trace *trace, ++trace->nr_events_printed; } +static unsigned char bitmap_byte(const unsigned long *mask, int byte_idx) +{ + unsigned char b_val = 0; + int bit_in_byte; + + for (bit_in_byte = 0; bit_in_byte < 8; bit_in_byte++) { + int b_idx = byte_idx * 8 + bit_in_byte; + int host_w_idx = b_idx / BITS_PER_LONG; + int host_bit_in_word = b_idx % BITS_PER_LONG; + + if (mask[host_w_idx] & (1UL << host_bit_in_word)) + b_val |= (1 << bit_in_byte); + } + return b_val; +} + static size_t trace__fprintf_tp_fields(struct trace *trace, struct perf_sample *sample, struct thread *thread, void *augmented_args, int augmented_args_size) { @@ -3238,17 +3254,54 @@ static size_t trace__fprintf_tp_fields(struct trace *trace, struct perf_sample * syscall_arg.len = 0; syscall_arg.fmt = arg; if (field->flags & TEP_FIELD_IS_ARRAY) { - int offset = field->offset; - - if (field->flags & TEP_FIELD_IS_DYNAMIC) { - offset = format_field__intval(field, sample, evsel->needs_swap); - syscall_arg.len = offset >> 16; - offset &= 0xffff; - if (tep_field_is_relative(field->flags)) - offset += field->offset + field->size; + void *ptr = format_field__get_raw_data(field, sample, + evsel->needs_swap, + &syscall_arg.len); + + if (!ptr) { + pr_err("Problem processing %s field, skipping...\n", field->name); + continue; + } + val = (uintptr_t)ptr; + } else if ((field->flags & TEP_FIELD_IS_DYNAMIC) && + strstr(field->type, "cpumask")) { + unsigned long *mask = format_field__get_cpumask(field, sample, + evsel->needs_swap, + &syscall_arg.len); + + if (!mask) { + pr_err("Problem processing %s field, skipping...\n", field->name); + continue; } - val = (uintptr_t)(sample->raw_data + offset); + printed += scnprintf(bf + printed, size - printed, "%s", printed ? ", " : ""); + if (trace->show_arg_names) + printed += scnprintf(bf + printed, size - printed, "%s: ", field->name); + + if (syscall_arg.len == 0) { + printed += scnprintf(bf + printed, size - printed, "0"); + } else { + int i; + bool skip_zero = true; + + printed += scnprintf(bf + printed, size - printed, "0x"); + /* Print bytes from most significant to least significant */ + for (i = syscall_arg.len - 1; i >= 0; i--) { + unsigned char b_val = bitmap_byte(mask, i); + + if (skip_zero && b_val == 0 && i > 0) + continue; + + if (skip_zero) { + printed += scnprintf(bf + printed, size - printed, "%x", b_val); + skip_zero = false; + } else { + printed += scnprintf(bf + printed, size - printed, "%02x", b_val); + } + } + } + free(mask); + continue; } else val = format_field__intval(field, sample, evsel->needs_swap); /* @@ -3633,7 +3686,7 @@ static bool evlist__add_vfs_getname(struct evlist *evlist) list_del_init(&evsel->core.node); evsel->evlist = NULL; - evsel__delete(evsel); + evsel__put(evsel); } return found; @@ -3749,9 +3802,9 @@ static int trace__add_syscall_newtp(struct trace *trace) return ret; out_delete_sys_exit: - evsel__delete_priv(sys_exit); + evsel__put_and_free_priv(sys_exit); out_delete_sys_enter: - evsel__delete_priv(sys_enter); + evsel__put_and_free_priv(sys_enter); goto out; } @@ -4216,7 +4269,7 @@ static int trace__set_filter_pids(struct trace *trace) err = augmented_syscalls__set_filter_pids(trace->filter_pids.nr, trace->filter_pids.entries); } - } else if (perf_thread_map__pid(trace->evlist->core.threads, 0) == -1) { + } else if (perf_thread_map__pid(evlist__core(trace->evlist)->threads, 0) == -1) { err = trace__set_filter_loop_pids(trace); } @@ -4439,7 +4492,7 @@ static int trace__run(struct trace *trace, int argc, const char **argv) if (trace->summary_bpf) { if (trace_prepare_bpf_summary(trace->summary_mode) < 0) - goto out_delete_evlist; + goto out_put_evlist; if (trace->summary_only) goto create_maps; @@ -4507,19 +4560,19 @@ static int trace__run(struct trace *trace, int argc, const char **argv) err = evlist__create_maps(evlist, &trace->opts.target); if (err < 0) { fprintf(trace->output, "Problems parsing the target to trace, check your options!\n"); - goto out_delete_evlist; + goto out_put_evlist; } err = trace__symbols_init(trace, argc, argv, evlist); if (err < 0) { fprintf(trace->output, "Problems initializing symbol libraries!\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (trace->summary_mode == SUMMARY__BY_TOTAL && !trace->summary_bpf) { trace->syscall_stats = alloc_syscall_stats(); if (!trace->syscall_stats) - goto out_delete_evlist; + goto out_put_evlist; } evlist__config(evlist, &trace->opts, &callchain_param); @@ -4528,9 +4581,9 @@ static int trace__run(struct trace *trace, int argc, const char **argv) err = evlist__prepare_workload(evlist, &trace->opts.target, argv, false, NULL); if (err < 0) { fprintf(trace->output, "Couldn't run the workload!\n"); - goto out_delete_evlist; + goto out_put_evlist; } - workload_pid = evlist->workload.pid; + workload_pid = evlist__workload_pid(evlist); } err = evlist__open(evlist); @@ -4576,13 +4629,13 @@ static int trace__run(struct trace *trace, int argc, const char **argv) err = trace__expand_filters(trace, &evsel); if (err) - goto out_delete_evlist; + goto out_put_evlist; err = evlist__apply_filters(evlist, &evsel, &trace->opts.target); if (err < 0) goto out_error_apply_filters; if (!trace->summary_only || !trace->summary_bpf) { - err = evlist__mmap(evlist, trace->opts.mmap_pages); + err = evlist__do_mmap(evlist, trace->opts.mmap_pages); if (err < 0) goto out_error_mmap; } @@ -4601,8 +4654,8 @@ static int trace__run(struct trace *trace, int argc, const char **argv) if (trace->summary_bpf) trace_start_bpf_summary(); - trace->multiple_threads = perf_thread_map__pid(evlist->core.threads, 0) == -1 || - perf_thread_map__nr(evlist->core.threads) > 1 || + trace->multiple_threads = perf_thread_map__pid(evlist__core(evlist)->threads, 0) == -1 || + perf_thread_map__nr(evlist__core(evlist)->threads) > 1 || evlist__first(evlist)->core.attr.inherit; /* @@ -4619,11 +4672,11 @@ static int trace__run(struct trace *trace, int argc, const char **argv) again: before = trace->nr_events; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { union perf_event *event; struct mmap *md; - md = &evlist->mmap[i]; + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -4693,12 +4746,12 @@ static int trace__run(struct trace *trace, int argc, const char **argv) } } -out_delete_evlist: +out_put_evlist: trace_cleanup_bpf_summary(); delete_syscall_stats(trace->syscall_stats); trace__symbols__exit(trace); evlist__free_syscall_tp_fields(evlist); - evlist__delete(evlist); + evlist__put(evlist); cgroup__put(trace->cgroup); trace->evlist = NULL; trace->live = false; @@ -4723,21 +4776,21 @@ static int trace__run(struct trace *trace, int argc, const char **argv) out_error: fprintf(trace->output, "%s\n", errbuf); - goto out_delete_evlist; + goto out_put_evlist; out_error_apply_filters: fprintf(trace->output, "Failed to set filter \"%s\" on event %s: %m\n", evsel->filter, evsel__name(evsel)); - goto out_delete_evlist; + goto out_put_evlist; } out_error_mem: fprintf(trace->output, "Not enough memory to run!\n"); - goto out_delete_evlist; + goto out_put_evlist; out_errno: fprintf(trace->output, "%m\n"); - goto out_delete_evlist; + goto out_put_evlist; } static int trace__replay(struct trace *trace) @@ -5325,7 +5378,7 @@ static int trace__parse_cgroups(const struct option *opt, const char *str, int u { struct trace *trace = opt->value; - if (!list_empty(&trace->evlist->core.entries)) { + if (!list_empty(&evlist__core(trace->evlist)->entries)) { struct option o = { .value = &trace->evlist, }; @@ -5417,7 +5470,7 @@ static void trace__exit(struct trace *trace) zfree(&trace->syscalls.table); } zfree(&trace->perfconfig_events); - evlist__delete(trace->evlist); + evlist__put(trace->evlist); trace->evlist = NULL; ordered_events__free(&trace->oe.data); #ifdef HAVE_LIBBPF_SUPPORT @@ -5599,7 +5652,7 @@ int cmd_trace(int argc, const char **argv) * .perfconfig trace.add_events, and filter those out. */ if (!trace.trace_syscalls && !trace.trace_pgfaults && - trace.evlist->core.nr_entries == 0 /* Was --events used? */) { + evlist__nr_entries(trace.evlist) == 0 /* Was --events used? */) { trace.trace_syscalls = true; } /* @@ -5685,7 +5738,7 @@ int cmd_trace(int argc, const char **argv) symbol_conf.use_callchain = true; } - if (trace.evlist->core.nr_entries > 0) { + if (evlist__nr_entries(trace.evlist) > 0) { bool use_btf = false; evlist__set_default_evsel_handler(trace.evlist, trace__event_handler); diff --git a/tools/perf/pmu-events/arch/arm64/nvidia/t410/metrics.json b/tools/perf/pmu-events/arch/arm64/nvidia/t410/metrics.json index b825ede03f544..59474ccb7417f 100644 --- a/tools/perf/pmu-events/arch/arm64/nvidia/t410/metrics.json +++ b/tools/perf/pmu-events/arch/arm64/nvidia/t410/metrics.json @@ -346,42 +346,42 @@ "MetricExpr": "l1d_demand_misses / l1d_demand_accesses", "BriefDescription": "This metric measures the ratio of L1 D-cache Read accesses missed to the total number of L1 D-cache accesses. This gives an indication of the effectiveness of the L1 D-cache for demand Load or Store traffic.", "ScaleUnit": "1per cache access", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_demand_accesses", "MetricExpr": "L1D_CACHE_RW", "BriefDescription": "This metric measures the count of L1 D-cache accesses incurred on Load or Store by the instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_demand_misses", "MetricExpr": "L1D_CACHE_REFILL_RW", "BriefDescription": "This metric measures the count of L1 D-cache misses incurred on a Load or Store by the instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_prf_accuracy", "MetricExpr": "100 * (l1d_useful_prf / l1d_refilled_prf)", "BriefDescription": "This metric measures the fraction of prefetched memory addresses that are used by the instruction stream.", "ScaleUnit": "1percent of prefetch", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_prf_coverage", "MetricExpr": "100 * (l1d_useful_prf / (l1d_demand_misses + l1d_refilled_prf))", "BriefDescription": "This metric measures the baseline demand cache misses which the prefetcher brings into the cache.", "ScaleUnit": "1percent of cache access", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_refilled_prf", "MetricExpr": "L1D_CACHE_REFILL_HWPRF + L1D_CACHE_REFILL_PRFM + L1D_LFB_HIT_RW_FHWPRF + L1D_LFB_HIT_RW_FPRFM", "BriefDescription": "This metric measures the count of cache lines refilled by L1 data prefetcher (hardware prefetches or software preload) into L1 D-cache.", "ScaleUnit": "1count", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1d_tlb_miss_ratio", @@ -402,7 +402,7 @@ "MetricExpr": "L1D_CACHE_HIT_RW_FPRF + L1D_LFB_HIT_RW_FHWPRF + L1D_LFB_HIT_RW_FPRFM", "BriefDescription": "This metric measures the count of cache lines refilled by L1 data prefetcher (hardware prefetches or software preload) into L1 D-cache which are further used by Load or Store from the instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1I_Prefetcher_Effectiveness" + "MetricGroup": "L1D_Prefetcher_Effectiveness" }, { "MetricName": "l1i_cache_miss_ratio", @@ -423,42 +423,42 @@ "MetricExpr": "l1i_demand_misses / l1i_demand_accesses", "BriefDescription": "This metric measures the ratio of L1 I-cache Read accesses missed to the total number of L1 I-cache accesses. This gives an indication of the effectiveness of the L1 I-cache for demand instruction fetch traffic. Note that cache accesses in this cache are demand instruction fetch.", "ScaleUnit": "1per cache access", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_demand_accesses", "MetricExpr": "L1I_CACHE_RD", "BriefDescription": "This metric measures the count of L1 I-cache accesses caused by an instruction fetch by the instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_demand_misses", "MetricExpr": "L1I_CACHE_REFILL_RD", "BriefDescription": "This metric measures the count of L1 I-cache misses caused by an instruction fetch by the instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_prf_accuracy", "MetricExpr": "100 * (l1i_useful_prf / l1i_refilled_prf)", "BriefDescription": "This metric measures the fraction of prefetched memory addresses that are used by the instruction stream.", "ScaleUnit": "1percent of prefetch", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_prf_coverage", "MetricExpr": "100 * (l1i_useful_prf / (l1i_demand_misses + l1i_refilled_prf))", "BriefDescription": "This metric measures the baseline demand cache misses which the prefetcher brings into the cache.", "ScaleUnit": "1percent of cache access", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_refilled_prf", "MetricExpr": "L1I_CACHE_REFILL_HWPRF + L1I_CACHE_REFILL_PRFM", "BriefDescription": "This metric measures the count of cache lines refilled by L1 instruction prefetcher (hardware prefetches or software preload) into L1 I-cache.", "ScaleUnit": "1count", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l1i_tlb_miss_ratio", @@ -479,7 +479,7 @@ "MetricExpr": "L1I_CACHE_HIT_RD_FPRF", "BriefDescription": "This metric measures the count of cache lines refilled by L1 instruction prefetcher (hardware prefetches or software preload) into L1 I-cache which are further used by instruction stream of the program.", "ScaleUnit": "1count", - "MetricGroup": "L1D_Prefetcher_Effectiveness" + "MetricGroup": "L1I_Prefetcher_Effectiveness" }, { "MetricName": "l2_cache_miss_ratio", diff --git a/tools/perf/pmu-events/arch/arm64/nvidia/t410/misc.json b/tools/perf/pmu-events/arch/arm64/nvidia/t410/misc.json index 8ff87d844e521..e1b45cc33b180 100644 --- a/tools/perf/pmu-events/arch/arm64/nvidia/t410/misc.json +++ b/tools/perf/pmu-events/arch/arm64/nvidia/t410/misc.json @@ -316,8 +316,8 @@ }, { "EventCode": "0x0197", - "EventName": "TXREQ_LIMIT_1QUARTER_CYCLES", - "PublicDescription": "Number of cycles in which the dynamic TXREQ limit is between 1/4 of the L2_TQ_SIZE and 1/2 of the L2_TQ_SIZE." + "EventName": "TXREQ_LIMIT_BELOW_HALF_CYCLES", + "PublicDescription": "Number of cycles in which the dynamic TXREQ limit is between 0 and 1/2 of the L2_TQ_SIZE." }, { "EventCode": "0x019d", @@ -517,7 +517,7 @@ { "EventCode": "0x01cc", "EventName": "TXREQ_LIMIT_COUNT_CYCLES", - "PublicDescription": "This event increments by the dynamic TXREQ value, in each cycle.\nThis is a companion event of TXREQ_LIMIT_MAX_CYCLES, TXREQ_LIMIT_3QUARTER_CYCLES, TXREQ_LIMIT_HALF_CYCLES, and TXREQ_LIMIT_1QUARTER_CYCLES." + "PublicDescription": "This event increments by the dynamic TXREQ value, in each cycle.\nThis is a companion event of TXREQ_LIMIT_MAX_CYCLES, TXREQ_LIMIT_3QUARTER_CYCLES, TXREQ_LIMIT_HALF_CYCLES, and TXREQ_LIMIT_BELOW_HALF_CYCLES." }, { "EventCode": "0x01ce", diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json b/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json index 9204bfb1d69e0..50d38434f8d3d 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json @@ -179,6 +179,30 @@ "BriefDescription": "Retired scalar floating-point blend ops.", "UMask": "0x09" }, + { + "EventName": "fp_ops_retired_by_type.scalar_mov", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point MOV ops.", + "UMask": "0x0a" + }, + { + "EventName": "fp_ops_retired_by_type.scalar_shuffle", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", + "UMask": "0x0b" + }, + { + "EventName": "fp_ops_retired_by_type.scalar_bfloat", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point bfloat ops.", + "UMask": "0x0c" + }, + { + "EventName": "fp_ops_retired_by_type.scalar_logical", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point logical ops.", + "UMask": "0x0d" + }, { "EventName": "fp_ops_retired_by_type.scalar_other", "EventCode": "0x0a", @@ -245,12 +269,24 @@ "BriefDescription": "Retired vector floating-point blend ops.", "UMask": "0x90" }, + { + "EventName": "fp_ops_retired_by_type.vector_mov", + "EventCode": "0x0a", + "BriefDescription": "Retired vector floating-point MOV ops.", + "UMask": "0xa0" + }, { "EventName": "fp_ops_retired_by_type.vector_shuffle", "EventCode": "0x0a", "BriefDescription": "Retired vector floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "fp_ops_retired_by_type.vector_bfloat", + "EventCode": "0x0a", + "BriefDescription": "Retired vector floating-point bfloat ops.", + "UMask": "0xc0" + }, { "EventName": "fp_ops_retired_by_type.vector_logical", "EventCode": "0x0a", @@ -278,7 +314,7 @@ { "EventName": "sse_avx_ops_retired.mmx_add", "EventCode": "0x0b", - "BriefDescription": "Retired MMX integer add.", + "BriefDescription": "Retired MMX integer add ops.", "UMask": "0x01" }, { @@ -299,16 +335,34 @@ "BriefDescription": "Retired MMX integer multiply-accumulate ops.", "UMask": "0x04" }, + { + "EventName": "sse_avx_ops_retired.mmx_aes", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer AES ops.", + "UMask": "0x05" + }, + { + "EventName": "sse_avx_ops_retired.mmx_sha", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer SHA ops.", + "UMask": "0x06" + }, { "EventName": "sse_avx_ops_retired.mmx_cmp", "EventCode": "0x0b", "BriefDescription": "Retired MMX integer compare ops.", "UMask": "0x07" }, + { + "EventName": "sse_avx_ops_retired.mmx_cvt", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer convert or pack ops.", + "UMask": "0x08" + }, { "EventName": "sse_avx_ops_retired.mmx_shift", "EventCode": "0x0b", - "BriefDescription": "Retired MMX integer shift ops.", + "BriefDescription": "Retired MMX integer shift or rotate ops.", "UMask": "0x09" }, { @@ -323,10 +377,17 @@ "BriefDescription": "Retired MMX integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0x0b" }, + { + "EventName": "sse_avx_ops_retired.mmx_vnni", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer VNNI ops.", + "UMask": "0x0c" + }, { "EventName": "sse_avx_ops_retired.mmx_pack", "EventCode": "0x0b", - "BriefDescription": "Retired MMX integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.mmx_vnni", + "Deprecated": "1", "UMask": "0x0c" }, { @@ -389,16 +450,23 @@ "BriefDescription": "Retired SSE and AVX integer compare ops.", "UMask": "0x70" }, + { + "EventName": "sse_avx_ops_retired.sse_avx_cvt", + "EventCode": "0x0b", + "BriefDescription": "Retired SSE and AVX integer convert or pack ops.", + "UMask": "0x80" + }, { "EventName": "sse_avx_ops_retired.sse_avx_clm", "EventCode": "0x0b", - "BriefDescription": "Retired SSE and AVX integer CLM ops.", + "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.sse_avx_cvt", + "Deprecated": "1", "UMask": "0x80" }, { "EventName": "sse_avx_ops_retired.sse_avx_shift", "EventCode": "0x0b", - "BriefDescription": "Retired SSE and AVX integer shift ops.", + "BriefDescription": "Retired SSE and AVX integer shift or rotate ops.", "UMask": "0x90" }, { @@ -413,10 +481,17 @@ "BriefDescription": "Retired SSE and AVX integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "sse_avx_ops_retired.sse_avx_vnni", + "EventCode": "0x0b", + "BriefDescription": "Retired SSE and AVX integer VNNI ops.", + "UMask": "0xc0" + }, { "EventName": "sse_avx_ops_retired.sse_avx_pack", "EventCode": "0x0b", - "BriefDescription": "Retired SSE and AVX integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.sse_avx_vnni", + "Deprecated": "1", "UMask": "0xc0" }, { @@ -497,12 +572,24 @@ "BriefDescription": "Retired 128-bit packed floating-point blend ops.", "UMask": "0x09" }, + { + "EventName": "fp_pack_ops_retired.fp128_mov", + "EventCode": "0x0c", + "BriefDescription": "Retired 128-bit packed floating-point MOV ops.", + "UMask": "0x0a" + }, { "EventName": "fp_pack_ops_retired.fp128_shuffle", "EventCode": "0x0c", "BriefDescription": "Retired 128-bit packed floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0x0b" }, + { + "EventName": "fp_pack_ops_retired.fp128_bfloat", + "EventCode": "0x0c", + "BriefDescription": "Retired 128-bit packed floating-point bfloat ops.", + "UMask": "0x0c" + }, { "EventName": "fp_pack_ops_retired.fp128_logical", "EventCode": "0x0c", @@ -575,12 +662,24 @@ "BriefDescription": "Retired 256-bit packed floating-point blend ops.", "UMask": "0x90" }, + { + "EventName": "fp_pack_ops_retired.fp256_mov", + "EventCode": "0x0c", + "BriefDescription": "Retired 256-bit packed floating-point MOV ops.", + "UMask": "0xa0" + }, { "EventName": "fp_pack_ops_retired.fp256_shuffle", "EventCode": "0x0c", "BriefDescription": "Retired 256-bit packed floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "fp_pack_ops_retired.fp256_bfloat", + "EventCode": "0x0c", + "BriefDescription": "Retired 256-bit packed floating-point bfloat ops.", + "UMask": "0xc0" + }, { "EventName": "fp_pack_ops_retired.fp256_logical", "EventCode": "0x0c", @@ -647,16 +746,23 @@ "BriefDescription": "Retired 128-bit packed integer compare ops.", "UMask": "0x07" }, + { + "EventName": "packed_int_op_type.int128_cvt", + "EventCode": "0x0d", + "BriefDescription": "Retired 128-bit packed integer convert or pack ops.", + "UMask": "0x08" + }, { "EventName": "packed_int_op_type.int128_clm", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer CLM ops.", + "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int128_cvt", + "Deprecated": "1", "UMask": "0x08" }, { "EventName": "packed_int_op_type.int128_shift", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer shift ops.", + "BriefDescription": "Retired 128-bit packed integer shift or rotate ops.", "UMask": "0x09" }, { @@ -671,10 +777,17 @@ "BriefDescription": "Retired 128-bit packed integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0x0b" }, + { + "EventName": "packed_int_op_type.int128_vnni", + "EventCode": "0x0d", + "BriefDescription": "Retired 128-bit packed integer VNNI ops.", + "UMask": "0x0c" + }, { "EventName": "packed_int_op_type.int128_pack", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int128_vnni", + "Deprecated": "1", "UMask": "0x0c" }, { @@ -719,16 +832,34 @@ "BriefDescription": "Retired 256-bit packed integer multiply-accumulate ops.", "UMask": "0x40" }, + { + "EventName": "packed_int_op_type.int256_aes", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer AES ops.", + "UMask": "0x50" + }, + { + "EventName": "packed_int_op_type.int256_sha", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer SHA ops.", + "UMask": "0x60" + }, { "EventName": "packed_int_op_type.int256_cmp", "EventCode": "0x0d", "BriefDescription": "Retired 256-bit packed integer compare ops.", "UMask": "0x70" }, + { + "EventName": "packed_int_op_type.int256_cvt", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer convert or pack ops.", + "UMask": "0x80" + }, { "EventName": "packed_int_op_type.int256_shift", "EventCode": "0x0d", - "BriefDescription": "Retired 256-bit packed integer shift ops.", + "BriefDescription": "Retired 256-bit packed integer shift or rotate ops.", "UMask": "0x90" }, { @@ -743,10 +874,17 @@ "BriefDescription": "Retired 256-bit packed integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "packed_int_op_type.int256_vnni", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer VNNI ops.", + "UMask": "0xc0" + }, { "EventName": "packed_int_op_type.int256_pack", "EventCode": "0x0d", - "BriefDescription": "Retired 256-bit packed integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int256_vnni", + "Deprecated": "1", "UMask": "0xc0" }, { diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json index 06bbaea159259..b1994539ece82 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json @@ -8,9 +8,15 @@ { "EventName": "ls_locks.bus_lock", "EventCode": "0x25", - "BriefDescription": "Retired Lock instructions which caused a bus lock.", + "BriefDescription": "Retired lock instructions which caused a bus lock.", "UMask": "0x01" }, + { + "EventName": "ls_locks.all", + "EventCode": "0x25", + "BriefDescription": "Retired lock instructions of all types.", + "UMask": "0x1f" + }, { "EventName": "ls_ret_cl_flush", "EventCode": "0x26", diff --git a/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json b/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json index 03cb039434dee..71b883776f37b 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json @@ -212,7 +212,7 @@ { "EventName": "fp_ops_ret_by_type.scalar_logical", "EventCode": "0x0a", - "BriefDescription": "Retired scalar floating-point move uops.", + "BriefDescription": "Retired scalar floating-point logical uops.", "UMask": "0x0d" }, { @@ -665,6 +665,12 @@ "BriefDescription": "Retired 256-bit packed floating-point shuffle uops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "fp_pack_ops_ret.fp256_bfloat", + "EventCode": "0x0c", + "BriefDescription": "Retired 256-bit packed floating-point bfloat uops.", + "UMask": "0xc0" + }, { "EventName": "fp_pack_ops_ret.fp256_logical", "EventCode": "0x0c", @@ -758,7 +764,7 @@ { "EventName": "fp_pack_int_ops_ret.int128_vnni", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer VNNI ops.", + "BriefDescription": "Retired 128-bit packed integer VNNI uops.", "UMask": "0x0c" }, { @@ -803,12 +809,30 @@ "BriefDescription": "Retired 256-bit packed integer multiply-accumulate uops.", "UMask": "0x40" }, + { + "EventName": "fp_pack_int_ops_ret.int256_aes", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer AES uops.", + "UMask": "0x50" + }, + { + "EventName": "fp_pack_int_ops_ret.int256_sha", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer SHA uops.", + "UMask": "0x60" + }, { "EventName": "fp_pack_int_ops_ret.int256_cmp", "EventCode": "0x0d", "BriefDescription": "Retired 256-bit packed integer compare uops.", "UMask": "0x70" }, + { + "EventName": "fp_pack_int_ops_ret.int256_cvt", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer convert or pack uops.", + "UMask": "0x80" + }, { "EventName": "fp_pack_int_ops_ret.int256_shift", "EventCode": "0x0d", @@ -1083,19 +1107,19 @@ "EventName": "fp_nsq_read_stalls.fp_prf", "EventCode": "0x13", "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free physical register file (FP-PRF) entries.", - "UMask": "0x0e" + "UMask": "0x02" }, { "EventName": "fp_nsq_read_stalls.k_prf", "EventCode": "0x13", "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free mask physical register file (K-PRF) entries.", - "UMask": "0x0e" + "UMask": "0x04" }, { "EventName": "fp_nsq_read_stalls.fp_sq", "EventCode": "0x13", "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free scheduler entries.", - "UMask": "0x0e" + "UMask": "0x08" }, { "EventName": "fp_nsq_read_stalls.all", diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index 376dc2d241621..3c6cfeefbd5dc 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -570,13 +570,14 @@ def fix_none(s: Optional[str]) -> str: def print_pending_metrics() -> None: """Optionally close metrics table.""" - def metric_cmp_key(j: JsonEvent) -> Tuple[bool, str, str]: + def metric_cmp_key(j: JsonEvent) -> Tuple[str, str, str, str]: def fix_none(s: Optional[str]) -> str: if s is None: return '' return s - return (j.desc is not None, fix_none(j.pmu), fix_none(j.metric_name)) + return (fix_none(j.pmu), fix_none(j.metric_name), j.metric_expr.ToPerfJson(), + fix_none(j.desc)) global _pending_metrics if not _pending_metrics: diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py index a91ccb5977f08..11c7162825f4b 100644 --- a/tools/perf/pmu-events/metric.py +++ b/tools/perf/pmu-events/metric.py @@ -623,7 +623,11 @@ def __init__(self, def __lt__(self, other): """Sort order.""" - return self.name < other.name + if self.name != other.name: + return self.name < other.name + if not self.expr.Equals(other.expr): + return self.expr.ToPerfJson() < other.expr.ToPerfJson() + return self.description < other.description def AddToMetricGroup(self, group): """Callback used when being added to a MetricGroup.""" diff --git a/tools/perf/scripts/python/arm-cs-trace-disasm.py b/tools/perf/scripts/python/arm-cs-trace-disasm.py index 8f6fa4a007b42..42579f8586842 100755 --- a/tools/perf/scripts/python/arm-cs-trace-disasm.py +++ b/tools/perf/scripts/python/arm-cs-trace-disasm.py @@ -31,18 +31,19 @@ # # Output disassembly with objdump and auto detect vmlinux # (when running on same machine.): -# perf script -s scripts/python/arm-cs-trace-disasm.py -d +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \ +# -- -d # # Output disassembly with llvm-objdump: -# perf script -s scripts/python/arm-cs-trace-disasm.py \ +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \ # -- -d llvm-objdump-11 -k path/to/vmlinux # # Output accurate disassembly by passing kcore to script: -# perf script -s scripts/python/arm-cs-trace-disasm.py \ +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \ # -- -d -k perf.data/kcore_dir/kcore # # Output only source line and symbols: -# perf script -s scripts/python/arm-cs-trace-disasm.py +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py def default_objdump(): config = perf_config_get("annotate.objdump") diff --git a/tools/perf/tests/backward-ring-buffer.c b/tools/perf/tests/backward-ring-buffer.c index c5e7999f28177..2735cc26d7eec 100644 --- a/tools/perf/tests/backward-ring-buffer.c +++ b/tools/perf/tests/backward-ring-buffer.c @@ -34,8 +34,8 @@ static int count_samples(struct evlist *evlist, int *sample_count, { int i; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - struct mmap *map = &evlist->overwrite_mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + struct mmap *map = &evlist__overwrite_mmap(evlist)[i]; union perf_event *event; perf_mmap__read_init(&map->core); @@ -65,7 +65,7 @@ static int do_test(struct evlist *evlist, int mmap_pages, int err; char sbuf[STRERR_BUFSIZE]; - err = evlist__mmap(evlist, mmap_pages); + err = evlist__do_mmap(evlist, mmap_pages); if (err < 0) { pr_debug("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); @@ -77,7 +77,7 @@ static int do_test(struct evlist *evlist, int mmap_pages, evlist__disable(evlist); err = count_samples(evlist, sample_count, comm_count); - evlist__munmap(evlist); + evlist__do_munmap(evlist); return err; } @@ -111,7 +111,7 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in err = evlist__create_maps(evlist, &opts.target); if (err < 0) { pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } parse_events_error__init(&parse_error); @@ -124,7 +124,7 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in if (err) { pr_debug("Failed to parse tracepoint event, try use root\n"); ret = TEST_SKIP; - goto out_delete_evlist; + goto out_put_evlist; } evlist__config(evlist, &opts, NULL); @@ -133,19 +133,19 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in if (err < 0) { pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } ret = TEST_FAIL; err = do_test(evlist, opts.mmap_pages, &sample_count, &comm_count); if (err != TEST_OK) - goto out_delete_evlist; + goto out_put_evlist; if ((sample_count != NR_ITERS) || (comm_count != NR_ITERS)) { pr_err("Unexpected counter: sample_count=%d, comm_count=%d\n", sample_count, comm_count); - goto out_delete_evlist; + goto out_put_evlist; } evlist__close(evlist); @@ -154,16 +154,16 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in if (err < 0) { pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } err = do_test(evlist, 1, &sample_count, &comm_count); if (err != TEST_OK) - goto out_delete_evlist; + goto out_put_evlist; ret = TEST_OK; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c index 7e75f590f225e..9662223274325 100644 --- a/tools/perf/tests/builtin-test.c +++ b/tools/perf/tests/builtin-test.c @@ -10,37 +10,40 @@ #ifdef HAVE_BACKTRACE_SUPPORT #include #endif -#include -#include #include -#include #include -#include +#include + #include -#include +#include "util/term.h" +#include +#include +#include +#include +#include #include #include -#include -#include "util/term.h" +#include +#include +#include + +#include +#include +#include + #include "builtin.h" +#include "color.h" #include "config.h" +#include "debug.h" #include "hist.h" #include "intlist.h" -#include "tests.h" -#include "debug.h" -#include "color.h" -#include -#include #include "string2.h" #include "symbol.h" +#include "tests-scripts.h" +#include "tests.h" #include "util/rlimit.h" #include "util/strbuf.h" -#include -#include -#include -#include - -#include "tests-scripts.h" +#include "util/term.h" static const char *junit_filename; static struct strbuf junit_xml_buf = STRBUF_INIT; @@ -415,73 +418,73 @@ static char *xml_escape(const char *str) return res ? res : strdup(""); } -static const char *format_test_description(const char *desc, int max_desc_width, - char *buf, size_t buf_sz) +static int get_term_width(void) { - int len = strlen(desc); + struct winsize ws; + int cols = 80; + int term_width; /* - * Clamp to buf_sz to prevent GCC format-truncation warnings - * when terminal width is very large. + * If output is redirected to a file or piped, we don't need to wrap + * or truncate at all. Use a massive virtually infinite terminal width + * so descriptions are printed in full. */ - if (max_desc_width >= (int)buf_sz) - max_desc_width = buf_sz - 1; + if (!isatty(fileno(debug_file()))) + return 10000; - if (len > max_desc_width) { - snprintf(buf, buf_sz, "%.*s...", max_desc_width - 3, desc); - return buf; - } - return desc; + get_term_dimensions(&ws); + if (ws.ws_col > 0) + cols = ws.ws_col; + + /* + * Limit description width to fit on a single line. We subtract 35 + * columns of headroom to allocate space for: + * - The suite index prefix: e.g. " 10.100:" (8 characters) plus 1 space separator. + * - The trailing colon (1 character) and space before status (1 character). + * - The longest status results: e.g. "Skip (some metrics failed)" (26 characters) + * or "Running (XX active)" (20 characters). + * + * A minimum description width of 10 is enforced to ensure names are + * legible even on very narrow consoles. + */ + term_width = cols - 35; + if (term_width < 10) + term_width = 10; + + return term_width; +} + +static int get_max_desc_width(int width) +{ + int term_width = get_term_width(); + + return width > term_width ? term_width : width; } static int print_test_result(struct test_suite *t, int curr_suite, int curr_test_case, int result, int width, int running, const char *err_output, double elapsed) { - char desc_buf[256]; - const char *desc = test_description(t, curr_test_case); - struct winsize ws; - int max_desc_area_width; - int target_desc_area_width; - int desc_padding; - - get_term_dimensions(&ws); - /* - * Total terminal columns minus space for status e.g. " Running (12 active)" - * which is 20 chars, plus a margin of 3 chars = 23 chars. - */ - max_desc_area_width = ws.ws_col - 23; - if (max_desc_area_width < 40) - max_desc_area_width = 40; - - /* Standard test has prefix "%3d: " which is 5 chars */ - target_desc_area_width = width + 5; - if (target_desc_area_width > max_desc_area_width) - target_desc_area_width = max_desc_area_width; + int pad_width = get_max_desc_width(width); + int term_width = get_term_width(); if (test_suite__num_test_cases(t) > 1) { char prefix[32]; int len = snprintf(prefix, sizeof(prefix), "%3d.%1d:", curr_suite + 1, curr_test_case + 1); + int pad = len >= 4 ? pad_width + 4 - len : pad_width; + int trunc = len >= 4 ? term_width + 4 - len : term_width; - desc_padding = target_desc_area_width - (len + 1); - if (desc_padding < 20) - desc_padding = 20; - - desc = format_test_description(desc, desc_padding, desc_buf, sizeof(desc_buf)); - pr_info("%s %-*s:", prefix, desc_padding, desc); + pr_info("%s %-*.*s:", prefix, pad, trunc, + test_description(t, curr_test_case)); } else { - desc_padding = target_desc_area_width - 5; - if (desc_padding < 20) - desc_padding = 20; - - desc = format_test_description(desc, desc_padding, desc_buf, sizeof(desc_buf)); - pr_info("%3d: %-*s:", curr_suite + 1, desc_padding, desc); + pr_info("%3d: %-*.*s:", curr_suite + 1, pad_width, term_width, + test_description(t, curr_test_case)); } switch (result) { case TEST_RUNNING: - color_fprintf(stderr, PERF_COLOR_YELLOW, " Running (%d active)\n", running); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Running (%d active)\n", running); break; case TEST_OK: if (test_suite__num_test_cases(t) > 1) @@ -495,9 +498,9 @@ static int print_test_result(struct test_suite *t, int curr_suite, int curr_test summary_tests_skipped++; if (reason) - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n", reason); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip (%s)\n", reason); else - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n"); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip\n"); } break; case TEST_FAIL: @@ -511,7 +514,7 @@ static int print_test_result(struct test_suite *t, int curr_suite, int curr_test strbuf_addf_safe(&summary_failed_tests_buf, " %3d: %s\n", curr_suite + 1, test_description(t, curr_test_case)); - color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n"); + color_fprintf(debug_file(), PERF_COLOR_RED, " FAILED!\n"); break; } @@ -747,6 +750,7 @@ static void finish_test(struct child_test **child_tests, int running_test, int c int ret; struct timespec end_time; double elapsed; + width = get_max_desc_width(width); if (child_test == NULL) { /* Test wasn't started. */ @@ -761,7 +765,8 @@ static void finish_test(struct child_test **child_tests, int running_test, int c * sub test names. */ if (test_suite__num_test_cases(t) > 1 && curr_test_case == 0) - pr_info("%3d: %s:\n", curr_suite + 1, test_description(t, -1)); + pr_info("%3d: %-*.*s:\n", curr_suite + 1, width, width, + test_description(t, -1)); /* * Busy loop reading from the child's stdout/stderr that are set to be @@ -969,6 +974,8 @@ static int finish_tests_parallel(struct child_test **child_tests, size_t num_tes int last_suite_printed = -1; sigset_t set, oldset; + width = get_max_desc_width(width); + sigemptyset(&set); sigaddset(&set, SIGINT); sigaddset(&set, SIGTERM); @@ -1037,8 +1044,11 @@ static int finish_tests_parallel(struct child_test **child_tests, size_t num_tes if (next_child) { if (test_suite__num_test_cases(next_child->test) > 1 && last_suite_printed != next_child->suite_num) { - pr_info("%3d: %s:\n", next_child->suite_num + 1, - test_description(next_child->test, -1)); + pr_info("%3d: %-*.*s:\n", + next_child->suite_num + 1, + width, width, + test_description( + next_child->test, -1)); last_suite_printed = next_child->suite_num; } print_test_result(next_child->test, next_child->suite_num, @@ -1101,7 +1111,8 @@ static int finish_tests_parallel(struct child_test **child_tests, size_t num_tes if (test_suite__num_test_cases(child->test) > 1 && last_suite_printed != child->suite_num) { - pr_info("%3d: %s:\n", child->suite_num + 1, + pr_info("%3d: %-*.*s:\n", child->suite_num + 1, + width, width, test_description(child->test, -1)); last_suite_printed = child->suite_num; } @@ -1225,12 +1236,12 @@ static void print_tests_summary(void) pr_info("Passed subtests : %u\n", summary_subtests_passed); pr_info("Skipped tests : %u\n", summary_tests_skipped); if (summary_tests_failed > 0) { - color_fprintf(stderr, PERF_COLOR_RED, "Failed tests : %u\n", + color_fprintf(debug_file(), PERF_COLOR_RED, "Failed tests : %u\n", summary_tests_failed); pr_info("List of failed tests:\n"); pr_info("%s", summary_failed_tests_buf.buf); } else { - color_fprintf(stderr, PERF_COLOR_GREEN, "Failed tests : 0\n"); + color_fprintf(debug_file(), PERF_COLOR_GREEN, "Failed tests : 0\n"); } if (junit_filename) { @@ -1348,9 +1359,13 @@ static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], if (intlist__find(skiplist, curr_suite + 1)) { if (pass == 1) { - pr_info("%3d: %-*s:", curr_suite + 1, width, + int pad_width = get_max_desc_width(width); + int term_width = get_term_width(); + + pr_info("%3d: %-*.*s:", curr_suite + 1, + pad_width, term_width, test_description(*t, -1)); - color_fprintf(stderr, PERF_COLOR_YELLOW, + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip (user override)\n"); summary_tests_skipped++; if (junit_filename) { @@ -1781,9 +1796,14 @@ int cmd_test(int argc, const char **argv) rlimit__bump_memlock(); suites = build_suites(); - if (!suites) - return errno ? -errno : -ENOMEM; + if (!suites) { + int err = errno; + + intlist__delete(skiplist); + return err ? -err : -ENOMEM; + } ret = __cmd_test(suites, argc, argv, skiplist); free(suites); + intlist__delete(skiplist); return ret; } diff --git a/tools/perf/tests/code-reading.c b/tools/perf/tests/code-reading.c index e82ecdc957778..f0e8ea8754ef5 100644 --- a/tools/perf/tests/code-reading.c +++ b/tools/perf/tests/code-reading.c @@ -592,8 +592,8 @@ static int process_events(struct machine *machine, struct evlist *evlist, struct mmap *md; int i, ret; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -781,7 +781,7 @@ static int do_test_code_reading(bool try_kcore) goto out_put; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); str = events[evidx]; pr_debug("Parsing event '%s'\n", str); @@ -809,8 +809,8 @@ static int do_test_code_reading(bool try_kcore) pr_debug("perf_evlist__open() failed!\n%s\n", errbuf); } - perf_evlist__set_maps(&evlist->core, NULL, NULL); - evlist__delete(evlist); + perf_evlist__set_maps(evlist__core(evlist), NULL, NULL); + evlist__put(evlist); evlist = NULL; continue; } @@ -820,7 +820,7 @@ static int do_test_code_reading(bool try_kcore) if (events[evidx] == NULL) goto out_put; - ret = evlist__mmap(evlist, UINT_MAX); + ret = evlist__do_mmap(evlist, UINT_MAX); if (ret < 0) { pr_debug("evlist__mmap failed\n"); goto out_put; @@ -847,7 +847,7 @@ static int do_test_code_reading(bool try_kcore) out_put: thread__put(thread); out_err: - evlist__delete(evlist); + evlist__put(evlist); perf_cpu_map__put(cpus); perf_thread_map__put(threads); machine__delete(machine); diff --git a/tools/perf/tests/event-times.c b/tools/perf/tests/event-times.c index ae3b98bb42cf8..56dd37ca760ec 100644 --- a/tools/perf/tests/event-times.c +++ b/tools/perf/tests/event-times.c @@ -50,7 +50,7 @@ static int attach__enable_on_exec(struct evlist *evlist) static int detach__enable_on_exec(struct evlist *evlist) { - waitpid(evlist->workload.pid, NULL, 0); + waitpid(evlist__workload_pid(evlist), NULL, 0); return 0; } @@ -186,7 +186,7 @@ static int test_times(int (attach)(struct evlist *), err = attach(evlist); if (err == TEST_SKIP) { pr_debug(" SKIP : not enough rights\n"); - evlist__delete(evlist); + evlist__put(evlist); return err; } @@ -205,7 +205,7 @@ static int test_times(int (attach)(struct evlist *), count.ena, count.run); out_err: - evlist__delete(evlist); + evlist__put(evlist); return !err ? TEST_OK : TEST_FAIL; } diff --git a/tools/perf/tests/event_update.c b/tools/perf/tests/event_update.c index facc65e29f20c..220cc0347747d 100644 --- a/tools/perf/tests/event_update.c +++ b/tools/perf/tests/event_update.c @@ -92,7 +92,7 @@ static int test__event_update(struct test_suite *test __maybe_unused, int subtes TEST_ASSERT_VAL("failed to allocate ids", !perf_evsel__alloc_id(&evsel->core, 1, 1)); - perf_evlist__id_add(&evlist->core, &evsel->core, 0, 0, 123); + perf_evlist__id_add(evlist__core(evlist), &evsel->core, 0, 0, 123); free((char *)evsel->unit); evsel->unit = strdup("KRAVA"); @@ -117,7 +117,7 @@ static int test__event_update(struct test_suite *test __maybe_unused, int subtes TEST_ASSERT_VAL("failed to synthesize attr update cpus", !perf_event__synthesize_event_update_cpus(&tmp.tool, evsel, process_event_cpus)); - evlist__delete(evlist); + evlist__put(evlist); return 0; } diff --git a/tools/perf/tests/evsel-roundtrip-name.c b/tools/perf/tests/evsel-roundtrip-name.c index 1922cac13a245..6a220634c52f6 100644 --- a/tools/perf/tests/evsel-roundtrip-name.c +++ b/tools/perf/tests/evsel-roundtrip-name.c @@ -33,7 +33,7 @@ static int perf_evsel__roundtrip_cache_name_test(void) if (err) { pr_debug("Failure to parse cache event '%s' possibly as PMUs don't support it", name); - evlist__delete(evlist); + evlist__put(evlist); continue; } evlist__for_each_entry(evlist, evsel) { @@ -42,7 +42,7 @@ static int perf_evsel__roundtrip_cache_name_test(void) ret = TEST_FAIL; } } - evlist__delete(evlist); + evlist__put(evlist); } } } @@ -66,7 +66,7 @@ static int perf_evsel__name_array_test(const char *const names[], int nr_names) if (err) { pr_debug("failed to parse event '%s', err %d\n", names[i], err); - evlist__delete(evlist); + evlist__put(evlist); ret = TEST_FAIL; continue; } @@ -76,7 +76,7 @@ static int perf_evsel__name_array_test(const char *const names[], int nr_names) ret = TEST_FAIL; } } - evlist__delete(evlist); + evlist__put(evlist); } return ret; } diff --git a/tools/perf/tests/evsel-tp-sched.c b/tools/perf/tests/evsel-tp-sched.c index 226196fb96779..9e456f88a13ac 100644 --- a/tools/perf/tests/evsel-tp-sched.c +++ b/tools/perf/tests/evsel-tp-sched.c @@ -64,7 +64,7 @@ static int test__perf_evsel__tp_sched_test(struct test_suite *test __maybe_unuse if (evsel__test_field(evsel, "next_prio", 4, true)) ret = TEST_FAIL; - evsel__delete(evsel); + evsel__put(evsel); evsel = evsel__newtp("sched", "sched_wakeup"); @@ -85,7 +85,7 @@ static int test__perf_evsel__tp_sched_test(struct test_suite *test __maybe_unuse if (evsel__test_field(evsel, "target_cpu", 4, true)) ret = TEST_FAIL; - evsel__delete(evsel); + evsel__put(evsel); return ret; } diff --git a/tools/perf/tests/expand-cgroup.c b/tools/perf/tests/expand-cgroup.c index dd547f2f77cc8..04d62611766ad 100644 --- a/tools/perf/tests/expand-cgroup.c +++ b/tools/perf/tests/expand-cgroup.c @@ -28,7 +28,7 @@ static int test_expand_events(struct evlist *evlist) TEST_ASSERT_VAL("evlist is empty", !evlist__empty(evlist)); - nr_events = evlist->core.nr_entries; + nr_events = evlist__nr_entries(evlist); ev_name = calloc(nr_events, sizeof(*ev_name)); if (ev_name == NULL) { pr_debug("memory allocation failure\n"); @@ -54,7 +54,7 @@ static int test_expand_events(struct evlist *evlist) } ret = TEST_FAIL; - if (evlist->core.nr_entries != nr_events * nr_cgrps) { + if (evlist__nr_entries(evlist) != nr_events * nr_cgrps) { pr_debug("event count doesn't match\n"); goto out; } @@ -106,7 +106,7 @@ static int expand_default_events(void) TEST_ASSERT_VAL("failed to get evlist", evlist); ret = test_expand_events(evlist); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -133,7 +133,7 @@ static int expand_group_events(void) ret = test_expand_events(evlist); out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -164,7 +164,7 @@ static int expand_libpfm_events(void) ret = test_expand_events(evlist); out: - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -179,7 +179,8 @@ static int expand_metric_events(void) TEST_ASSERT_VAL("failed to get evlist", evlist); pme_test = find_core_metrics_table("testarch", "testcpu"); - ret = metricgroup__parse_groups_test(evlist, pme_test, metric_str); + ret = metricgroup__parse_groups_test(evlist, pme_test, metric_str, + /*cputype_filter=*/false); if (ret < 0) { pr_debug("failed to parse '%s' metric\n", metric_str); goto out; @@ -188,7 +189,7 @@ static int expand_metric_events(void) ret = test_expand_events(evlist); out: - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/hists_cumulate.c b/tools/perf/tests/hists_cumulate.c index 09ee08085b06b..9356451a172ea 100644 --- a/tools/perf/tests/hists_cumulate.c +++ b/tools/perf/tests/hists_cumulate.c @@ -744,7 +744,7 @@ static int test__hists_cumulate(struct test_suite *test __maybe_unused, int subt out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hists_filter.c b/tools/perf/tests/hists_filter.c index ac5affb7afff1..f9eaa511487b4 100644 --- a/tools/perf/tests/hists_filter.c +++ b/tools/perf/tests/hists_filter.c @@ -332,7 +332,7 @@ static int test__hists_filter(struct test_suite *test __maybe_unused, int subtes out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); reset_output_field(); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hists_link.c b/tools/perf/tests/hists_link.c index e55990163865e..d88591bcbe508 100644 --- a/tools/perf/tests/hists_link.c +++ b/tools/perf/tests/hists_link.c @@ -353,7 +353,7 @@ static int test__hists_link(struct test_suite *test __maybe_unused, int subtest out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); reset_output_field(); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hists_output.c b/tools/perf/tests/hists_output.c index 5e59dba92e813..f58c8d18fe330 100644 --- a/tools/perf/tests/hists_output.c +++ b/tools/perf/tests/hists_output.c @@ -631,7 +631,7 @@ static int test__hists_output(struct test_suite *test __maybe_unused, int subtes out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hwmon_pmu.c b/tools/perf/tests/hwmon_pmu.c index 62e0841a6c310..e26b3fe3fab15 100644 --- a/tools/perf/tests/hwmon_pmu.c +++ b/tools/perf/tests/hwmon_pmu.c @@ -184,9 +184,10 @@ static int do_test(size_t i, bool with_pmu, bool with_alias) } ret = TEST_OK; - if (with_pmu ? (evlist->core.nr_entries != 1) : (evlist->core.nr_entries < 1)) { + if (with_pmu ? (evlist__nr_entries(evlist) != 1) + : (evlist__nr_entries(evlist) < 1)) { pr_debug("FAILED %s:%d Unexpected number of events for '%s' of %d\n", - __FILE__, __LINE__, str, evlist->core.nr_entries); + __FILE__, __LINE__, str, evlist__nr_entries(evlist)); ret = TEST_FAIL; goto out; } @@ -215,7 +216,7 @@ static int do_test(size_t i, bool with_pmu, bool with_alias) out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/keep-tracking.c b/tools/perf/tests/keep-tracking.c index 729cc9cc1cb77..b760041bed307 100644 --- a/tools/perf/tests/keep-tracking.c +++ b/tools/perf/tests/keep-tracking.c @@ -37,8 +37,8 @@ static int find_comm(struct evlist *evlist, const char *comm) int i, found; found = 0; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; while ((event = perf_mmap__read_event(&md->core)) != NULL) { @@ -87,7 +87,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte evlist = evlist__new(); CHECK_NOT_NULL__(evlist); - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); CHECK__(parse_event(evlist, "dummy:u")); CHECK__(parse_event(evlist, "cpu-cycles:u")); @@ -106,7 +106,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte goto out_err; } - CHECK__(evlist__mmap(evlist, UINT_MAX)); + CHECK__(evlist__do_mmap(evlist, UINT_MAX)); /* * First, test that a 'comm' event can be found when the event is @@ -153,7 +153,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte out_err: if (evlist) { evlist__disable(evlist); - evlist__delete(evlist); + evlist__put(evlist); } perf_cpu_map__put(cpus); perf_thread_map__put(threads); diff --git a/tools/perf/tests/mmap-basic.c b/tools/perf/tests/mmap-basic.c index a69cd1046e9aa..5cec7644952c7 100644 --- a/tools/perf/tests/mmap-basic.c +++ b/tools/perf/tests/mmap-basic.c @@ -81,7 +81,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest goto out_free_cpus; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); for (i = 0; i < nsyscalls; ++i) { char name[64]; @@ -94,7 +94,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest /* Permissions failure, flag the failure as a skip. */ err = TEST_SKIP; } - goto out_delete_evlist; + goto out_put_evlist; } evsels[i]->core.attr.wakeup_events = 1; @@ -106,17 +106,17 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest pr_debug("failed to open counter: %s, " "tweak /proc/sys/kernel/perf_event_paranoid?\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } nr_events[i] = 0; expected_nr_events[i] = 1 + rand() % 127; } - if (evlist__mmap(evlist, 128) < 0) { + if (evlist__do_mmap(evlist, 128) < 0) { pr_debug("failed to mmap events: %d (%s)\n", errno, str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } for (i = 0; i < nsyscalls; ++i) @@ -124,7 +124,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest syscalls[i](); } - md = &evlist->mmap[0]; + md = &evlist__mmap(evlist)[0]; if (perf_mmap__read_init(&md->core) < 0) goto out_init; @@ -134,7 +134,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest if (event->header.type != PERF_RECORD_SAMPLE) { pr_debug("unexpected %s event\n", perf_event__name(event->header.type)); - goto out_delete_evlist; + goto out_put_evlist; } perf_sample__init(&sample, /*all=*/false); @@ -142,7 +142,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest if (err) { pr_err("Can't parse sample, err = %d\n", err); perf_sample__exit(&sample); - goto out_delete_evlist; + goto out_put_evlist; } err = -1; @@ -153,7 +153,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest if (evsel == NULL) { pr_debug("event with id %" PRIu64 " doesn't map to an evsel\n", sample.id); - goto out_delete_evlist; + goto out_put_evlist; } nr_events[evsel->core.idx]++; perf_mmap__consume(&md->core); @@ -168,12 +168,12 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest expected_nr_events[evsel->core.idx], evsel__name(evsel), nr_events[evsel->core.idx]); err = -1; - goto out_delete_evlist; + goto out_put_evlist; } } -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); out_free_cpus: perf_cpu_map__put(cpus); out_free_threads: diff --git a/tools/perf/tests/openat-syscall-all-cpus.c b/tools/perf/tests/openat-syscall-all-cpus.c index 0be43f8db3bda..cc63df2b3bc53 100644 --- a/tools/perf/tests/openat-syscall-all-cpus.c +++ b/tools/perf/tests/openat-syscall-all-cpus.c @@ -59,7 +59,7 @@ static int test__openat_syscall_event_on_all_cpus(struct test_suite *test __mayb "tweak /proc/sys/kernel/perf_event_paranoid?\n", str_error_r(errno, sbuf, sizeof(sbuf))); err = TEST_SKIP; - goto out_evsel_delete; + goto out_evsel_put; } perf_cpu_map__for_each_cpu(cpu, idx, cpus) { @@ -116,8 +116,8 @@ static int test__openat_syscall_event_on_all_cpus(struct test_suite *test __mayb evsel__free_counts(evsel); out_close_fd: perf_evsel__close_fd(&evsel->core); -out_evsel_delete: - evsel__delete(evsel); +out_evsel_put: + evsel__put(evsel); out_cpu_map_delete: perf_cpu_map__put(cpus); out_thread_map_delete: diff --git a/tools/perf/tests/openat-syscall-tp-fields.c b/tools/perf/tests/openat-syscall-tp-fields.c index 9ff8caff98c3a..5365889d326f8 100644 --- a/tools/perf/tests/openat-syscall-tp-fields.c +++ b/tools/perf/tests/openat-syscall-tp-fields.c @@ -51,7 +51,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused if (IS_ERR(evsel)) { pr_debug("%s: evsel__newtp\n", __func__); ret = PTR_ERR(evsel) == -EACCES ? TEST_SKIP : TEST_FAIL; - goto out_delete_evlist; + goto out_put_evlist; } evlist__add(evlist, evsel); @@ -59,25 +59,25 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused err = evlist__create_maps(evlist, &opts.target); if (err < 0) { pr_debug("%s: evlist__create_maps\n", __func__); - goto out_delete_evlist; + goto out_put_evlist; } evsel__config(evsel, &opts, NULL); - perf_thread_map__set_pid(evlist->core.threads, 0, getpid()); + perf_thread_map__set_pid(evlist__core(evlist)->threads, 0, getpid()); err = evlist__open(evlist); if (err < 0) { pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } - err = evlist__mmap(evlist, UINT_MAX); + err = evlist__do_mmap(evlist, UINT_MAX); if (err < 0) { pr_debug("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } evlist__enable(evlist); @@ -90,11 +90,11 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused while (1) { int before = nr_events; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { union perf_event *event; struct mmap *md; - md = &evlist->mmap[i]; + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -115,7 +115,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused if (err) { pr_debug("Can't parse sample, err = %d\n", err); perf_sample__exit(&sample); - goto out_delete_evlist; + goto out_put_evlist; } tp_flags = perf_sample__intval(&sample, "flags"); @@ -126,7 +126,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused (tp_flags & flags) != flags) { pr_debug("%s: Expected flags=%#x, got %#x\n", __func__, flags, tp_flags); - goto out_delete_evlist; + goto out_put_evlist; } goto out_ok; @@ -139,13 +139,13 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused if (++nr_polls > 5) { pr_debug("%s: no events!\n", __func__); - goto out_delete_evlist; + goto out_put_evlist; } } out_ok: ret = TEST_OK; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); out: return ret; } diff --git a/tools/perf/tests/openat-syscall.c b/tools/perf/tests/openat-syscall.c index b54cbe5f18085..9f16f0dd3a295 100644 --- a/tools/perf/tests/openat-syscall.c +++ b/tools/perf/tests/openat-syscall.c @@ -42,7 +42,7 @@ static int test__openat_syscall_event(struct test_suite *test __maybe_unused, "tweak /proc/sys/kernel/perf_event_paranoid?\n", str_error_r(errno, sbuf, sizeof(sbuf))); err = TEST_SKIP; - goto out_evsel_delete; + goto out_evsel_put; } for (i = 0; i < nr_openat_calls; ++i) { @@ -64,8 +64,8 @@ static int test__openat_syscall_event(struct test_suite *test __maybe_unused, err = TEST_OK; out_close_fd: perf_evsel__close_fd(&evsel->core); -out_evsel_delete: - evsel__delete(evsel); +out_evsel_put: + evsel__put(evsel); out_thread_map_delete: perf_thread_map__put(threads); return err; diff --git a/tools/perf/tests/parse-events.c b/tools/perf/tests/parse-events.c index 05c3e899b4251..5f6f95c8a5b40 100644 --- a/tools/perf/tests/parse-events.c +++ b/tools/perf/tests/parse-events.c @@ -109,7 +109,7 @@ static int test__checkevent_tracepoint(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVLIST("wrong number of groups", 0 == evlist__nr_groups(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_TRACEPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong sample_type", @@ -122,7 +122,7 @@ static int test__checkevent_tracepoint_multi(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries > 1, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", evlist__nr_entries(evlist) > 1, evlist); TEST_ASSERT_EVLIST("wrong number of groups", 0 == evlist__nr_groups(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -144,7 +144,7 @@ static int test__checkevent_raw(struct evlist *evlist) struct evsel *evsel; bool raw_type_match = false; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { struct perf_pmu *pmu __maybe_unused = NULL; @@ -182,7 +182,7 @@ static int test__checkevent_numeric(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", 1 == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel); return TEST_OK; @@ -193,7 +193,7 @@ static int test__checkevent_symbolic_name(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("unexpected event", @@ -207,7 +207,7 @@ static int test__checkevent_symbolic_name_config(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("unexpected event", @@ -228,7 +228,7 @@ static int test__checkevent_symbolic_alias(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type/config", evsel__match(evsel, SOFTWARE, SW_PAGE_FAULTS), evsel); return TEST_OK; @@ -238,7 +238,7 @@ static int test__checkevent_genhw(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_HW_CACHE == evsel->core.attr.type, evsel); @@ -251,7 +251,7 @@ static int test__checkevent_breakpoint(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", @@ -265,7 +265,7 @@ static int test__checkevent_breakpoint_x(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_X == evsel->core.attr.bp_type, evsel); @@ -278,7 +278,7 @@ static int test__checkevent_breakpoint_r(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_R == evsel->core.attr.bp_type, evsel); @@ -290,7 +290,7 @@ static int test__checkevent_breakpoint_w(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_W == evsel->core.attr.bp_type, evsel); @@ -302,7 +302,7 @@ static int test__checkevent_breakpoint_rw(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", @@ -316,7 +316,7 @@ static int test__checkevent_tracepoint_modifier(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel, evsel); TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel); @@ -330,7 +330,7 @@ test__checkevent_tracepoint_multi_modifier(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries > 1, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", evlist__nr_entries(evlist) > 1, evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); @@ -346,7 +346,7 @@ static int test__checkevent_raw_modifier(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); @@ -361,7 +361,7 @@ static int test__checkevent_numeric_modifier(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); @@ -377,7 +377,7 @@ static int test__checkevent_symbolic_name_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -394,7 +394,7 @@ static int test__checkevent_exclude_host_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -409,7 +409,7 @@ static int test__checkevent_exclude_guest_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -423,7 +423,8 @@ static int test__checkevent_symbolic_alias_modifier(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", + 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel); @@ -437,7 +438,7 @@ static int test__checkevent_genhw_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -454,7 +455,7 @@ static int test__checkevent_exclude_idle_modifier(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude idle", evsel->core.attr.exclude_idle, evsel); @@ -473,7 +474,7 @@ static int test__checkevent_exclude_idle_modifier_1(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude idle", evsel->core.attr.exclude_idle, evsel); @@ -622,7 +623,7 @@ static int test__checkevent_breakpoint_2_events(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist__nr_entries(evlist), evsel); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong name", evsel__name_is(evsel, "breakpoint1"), evsel); @@ -641,7 +642,7 @@ static int test__checkevent_pmu(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); struct perf_pmu *core_pmu = perf_pmus__find_core_pmu(); - TEST_ASSERT_EVSEL("wrong number of entries", 1 == evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 1 == evlist__nr_entries(evlist), evsel); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", test_hw_config(evsel, 10), evsel); TEST_ASSERT_EVSEL("wrong config1", 1 == evsel->core.attr.config1, evsel); @@ -661,7 +662,7 @@ static int test__checkevent_list(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVSEL("wrong number of entries", 3 <= evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 3 <= evlist__nr_entries(evlist), evsel); /* r1 */ TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_TRACEPOINT != evsel->core.attr.type, evsel); @@ -707,14 +708,15 @@ static int test__checkevent_pmu_name(struct evlist *evlist) char buf[256]; /* default_core/config=1,name=krava/u */ - TEST_ASSERT_EVLIST("wrong number of entries", 2 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", + 2 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong name", evsel__name_is(evsel, "krava"), evsel); /* default_core/config=2/u" */ evsel = evsel__next(evsel); - TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist__nr_entries(evlist), evsel); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 2 == evsel->core.attr.config, evsel); snprintf(buf, sizeof(buf), "%s/config=2/u", core_pmu->name); @@ -729,7 +731,8 @@ static int test__checkevent_pmu_partial_time_callgraph(struct evlist *evlist) struct perf_pmu *core_pmu = perf_pmus__find_core_pmu(); /* default_core/config=1,call-graph=fp,time,period=100000/ */ - TEST_ASSERT_EVLIST("wrong number of entries", 2 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", + 2 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel); /* @@ -760,7 +763,7 @@ static int test__checkevent_pmu_events(struct evlist *evlist) struct evsel *evsel; struct perf_pmu *core_pmu = perf_pmus__find_core_pmu(); - TEST_ASSERT_EVLIST("wrong number of entries", 1 <= evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 <= evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong type", @@ -787,8 +790,9 @@ static int test__checkevent_pmu_events_mix(struct evlist *evlist) * The wild card event will be opened at least once, but it may be * opened on each core PMU. */ - TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries >= 2, evlist); - for (int i = 0; i < evlist->core.nr_entries - 1; i++) { + TEST_ASSERT_EVLIST("wrong number of entries", + evlist__nr_entries(evlist) >= 2, evlist); + for (int i = 0; i < evlist__nr_entries(evlist) - 1; i++) { evsel = (i == 0 ? evlist__first(evlist) : evsel__next(evsel)); /* pmu-event:u */ TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); @@ -905,7 +909,7 @@ static int test__group1(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (num_core_entries(evlist) * 2), + evlist__nr_entries(evlist) == (num_core_entries(evlist) * 2), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -950,7 +954,7 @@ static int test__group2(struct evlist *evlist) struct evsel *evsel, *leader = NULL; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist) + 1), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist) + 1), evlist); /* * TODO: Currently the software event won't be grouped with the hardware @@ -1018,7 +1022,7 @@ static int test__group3(struct evlist *evlist __maybe_unused) struct evsel *evsel, *group1_leader = NULL, *group2_leader = NULL; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (3 * perf_pmus__num_core_pmus() + 2), + evlist__nr_entries(evlist) == (3 * perf_pmus__num_core_pmus() + 2), evlist); /* * Currently the software event won't be grouped with the hardware event @@ -1144,7 +1148,7 @@ static int test__group4(struct evlist *evlist __maybe_unused) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (num_core_entries(evlist) * 2), + evlist__nr_entries(evlist) == (num_core_entries(evlist) * 2), evlist); TEST_ASSERT_EVLIST("wrong number of groups", num_core_entries(evlist) == evlist__nr_groups(evlist), @@ -1191,7 +1195,7 @@ static int test__group5(struct evlist *evlist __maybe_unused) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (5 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (5 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == (2 * num_core_entries(evlist)), @@ -1284,7 +1288,7 @@ static int test__group_gh1(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1329,7 +1333,7 @@ static int test__group_gh2(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1374,7 +1378,7 @@ static int test__group_gh3(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1419,7 +1423,7 @@ static int test__group_gh4(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1464,7 +1468,7 @@ static int test__leader_sample1(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (3 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (3 * num_core_entries(evlist)), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1520,7 +1524,7 @@ static int test__leader_sample2(struct evlist *evlist __maybe_unused) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1562,7 +1566,7 @@ static int test__checkevent_pinned_modifier(struct evlist *evlist) struct evsel *evsel = NULL; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1581,7 +1585,7 @@ static int test__pinned_group(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (3 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (3 * num_core_entries(evlist)), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1618,7 +1622,7 @@ static int test__checkevent_exclusive_modifier(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); @@ -1634,7 +1638,7 @@ static int test__exclusive_group(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == 3 * num_core_entries(evlist), + evlist__nr_entries(evlist) == 3 * num_core_entries(evlist), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1669,7 +1673,7 @@ static int test__checkevent_breakpoint_len(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", @@ -1684,7 +1688,7 @@ static int test__checkevent_breakpoint_len_w(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_W == evsel->core.attr.bp_type, evsel); @@ -1698,7 +1702,7 @@ test__checkevent_breakpoint_len_rw_modifier(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel); @@ -1712,7 +1716,7 @@ static int test__checkevent_precise_max_modifier(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == 1 + num_core_entries(evlist), + evlist__nr_entries(evlist) == 1 + num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type/config", evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK), evsel); return TEST_OK; @@ -1723,7 +1727,7 @@ static int test__checkevent_config_symbol(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "insn"), evsel); return TEST_OK; @@ -1733,7 +1737,7 @@ static int test__checkevent_config_raw(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "rawpmu"), evsel); return TEST_OK; } @@ -1742,7 +1746,7 @@ static int test__checkevent_config_num(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "numpmu"), evsel); return TEST_OK; } @@ -1752,7 +1756,7 @@ static int test__checkevent_config_cache(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "cachepmu"), evsel); return test__checkevent_genhw(evlist); @@ -1777,7 +1781,7 @@ static int test__intel_pt(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "intel_pt//u"), evsel); return TEST_OK; } @@ -1798,7 +1802,8 @@ static int test__ratio_to_prev(struct evlist *evlist) { struct evsel *evsel, *leader; - TEST_ASSERT_VAL("wrong number of entries", 2 * perf_pmus__num_core_pmus() == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", + 2 * perf_pmus__num_core_pmus() == evlist__nr_entries(evlist)); evlist__for_each_entry(evlist, evsel) { if (evsel != evsel__leader(evsel) || @@ -1842,7 +1847,7 @@ static int test__checkevent_complex_name(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong complex name parsing", evsel__name_is(evsel, @@ -1855,7 +1860,7 @@ static int test__checkevent_raw_pmu(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_SOFTWARE == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0x1a == evsel->core.attr.config, evsel); return TEST_OK; @@ -1866,7 +1871,7 @@ static int test__sym_event_slash(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); @@ -1878,7 +1883,7 @@ static int test__sym_event_dc(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); @@ -1890,7 +1895,7 @@ static int test__term_equal_term(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong name setting", strcmp(evsel->name, "name") == 0, evsel); @@ -1902,7 +1907,7 @@ static int test__term_equal_legacy(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong name setting", strcmp(evsel->name, "l1d") == 0, evsel); @@ -1958,7 +1963,7 @@ static int count_tracepoints(void) static int test__all_tracepoints(struct evlist *evlist) { TEST_ASSERT_VAL("wrong events count", - count_tracepoints() == evlist->core.nr_entries); + count_tracepoints() == evlist__nr_entries(evlist)); return test__checkevent_tracepoint_multi(evlist); } @@ -2556,8 +2561,10 @@ static int test_event(const struct evlist_test *e) return TEST_FAIL; } parse_events_error__init(&err); - ret = __parse_events(evlist, e->name, /*pmu_filter=*/NULL, &err, /*fake_pmu=*/false, - /*warn_if_reordered=*/true, /*fake_tp=*/true); + ret = __parse_events(evlist, e->name, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, &err, /*fake_pmu=*/false, + /*warn_if_reordered=*/true, + /*fake_tp=*/true); if (ret) { pr_debug("failed to parse event '%s', err %d\n", e->name, ret); parse_events_error__print(&err, e->name); @@ -2568,7 +2575,7 @@ static int test_event(const struct evlist_test *e) ret = e->check(evlist); } parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -2584,8 +2591,9 @@ static int test_event_fake_pmu(const char *str) return -ENOMEM; parse_events_error__init(&err); - ret = __parse_events(evlist, str, /*pmu_filter=*/NULL, &err, - /*fake_pmu=*/true, /*warn_if_reordered=*/true, + ret = __parse_events(evlist, str, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, &err, /*fake_pmu=*/true, + /*warn_if_reordered=*/true, /*fake_tp=*/true); if (ret) { pr_debug("failed to parse event '%s', err %d\n", @@ -2594,7 +2602,7 @@ static int test_event_fake_pmu(const char *str) } parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/parse-metric.c b/tools/perf/tests/parse-metric.c index 7c7f489a5eb0a..872d9a7aa72f6 100644 --- a/tools/perf/tests/parse-metric.c +++ b/tools/perf/tests/parse-metric.c @@ -53,7 +53,7 @@ static double compute_single(struct evlist *evlist, const char *name) struct evsel *evsel; evlist__for_each_entry(evlist, evsel) { - me = metricgroup__lookup(&evlist->metric_events, evsel, false); + me = metricgroup__lookup(evlist__metric_events(evlist), evsel, false); if (me != NULL) { list_for_each_entry (mexp, &me->head, nd) { if (strcmp(mexp->metric_name, name)) @@ -84,15 +84,16 @@ static int __compute_metric(const char *name, struct value *vals, cpus = perf_cpu_map__new("0"); if (!cpus) { - evlist__delete(evlist); + evlist__put(evlist); return -ENOMEM; } - perf_evlist__set_maps(&evlist->core, cpus, NULL); + perf_evlist__set_maps(evlist__core(evlist), cpus, NULL); /* Parse the metric into metric_events list. */ pme_test = find_core_metrics_table("testarch", "testcpu"); - err = metricgroup__parse_groups_test(evlist, pme_test, name); + err = metricgroup__parse_groups_test(evlist, pme_test, name, + /*cputype_filter=*/false); if (err) goto out; @@ -113,7 +114,7 @@ static int __compute_metric(const char *name, struct value *vals, /* ... cleanup. */ evlist__free_stats(evlist); perf_cpu_map__put(cpus); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/parse-no-sample-id-all.c b/tools/perf/tests/parse-no-sample-id-all.c index 8ac862c94879f..78bb8db192404 100644 --- a/tools/perf/tests/parse-no-sample-id-all.c +++ b/tools/perf/tests/parse-no-sample-id-all.c @@ -49,7 +49,7 @@ static int process_events(union perf_event **events, size_t count) for (i = 0; i < count && !err; i++) err = process_event(&evlist, events[i]); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/perf-record.c b/tools/perf/tests/perf-record.c index ad44cc68820b3..0cac6ae1a1fc0 100644 --- a/tools/perf/tests/perf-record.c +++ b/tools/perf/tests/perf-record.c @@ -105,7 +105,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest err = evlist__create_maps(evlist, &opts.target); if (err < 0) { pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -117,7 +117,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest err = evlist__prepare_workload(evlist, &opts.target, argv, false, NULL); if (err < 0) { pr_debug("Couldn't run the workload!\n"); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -129,12 +129,12 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest evsel__set_sample_bit(evsel, TIME); evlist__config(evlist, &opts, NULL); - err = sched__get_first_possible_cpu(evlist->workload.pid, cpu_mask); + err = sched__get_first_possible_cpu(evlist__workload_pid(evlist), cpu_mask); if (err < 0) { pr_debug("sched__get_first_possible_cpu: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } cpu = err; @@ -142,11 +142,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest /* * So that we can check perf_sample.cpu on all the samples. */ - if (sched_setaffinity(evlist->workload.pid, cpu_mask_size, cpu_mask) < 0) { + if (sched_setaffinity(evlist__workload_pid(evlist), cpu_mask_size, cpu_mask) < 0) { pr_debug("sched_setaffinity: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -158,7 +158,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -166,12 +166,12 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest * fds in the same CPU to be injected in the same mmap ring buffer * (using ioctl(PERF_EVENT_IOC_SET_OUTPUT)). */ - err = evlist__mmap(evlist, opts.mmap_pages); + err = evlist__do_mmap(evlist, opts.mmap_pages); if (err < 0) { pr_debug("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -188,11 +188,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest while (1) { int before = total_events; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { union perf_event *event; struct mmap *md; - md = &evlist->mmap[i]; + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -209,7 +209,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest if (verbose > 0) perf_event__fprintf(event, NULL, stderr); pr_debug("Couldn't parse sample\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (verbose > 0) { @@ -231,15 +231,15 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest ++errs; } - if ((pid_t)sample.pid != evlist->workload.pid) { + if ((pid_t)sample.pid != evlist__workload_pid(evlist)) { pr_debug("%s with unexpected pid, expected %d, got %d\n", - name, evlist->workload.pid, sample.pid); + name, evlist__workload_pid(evlist), sample.pid); ++errs; } - if ((pid_t)sample.tid != evlist->workload.pid) { + if ((pid_t)sample.tid != evlist__workload_pid(evlist)) { pr_debug("%s with unexpected tid, expected %d, got %d\n", - name, evlist->workload.pid, sample.tid); + name, evlist__workload_pid(evlist), sample.tid); ++errs; } @@ -248,7 +248,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest type == PERF_RECORD_MMAP2 || type == PERF_RECORD_FORK || type == PERF_RECORD_EXIT) && - (pid_t)event->comm.pid != evlist->workload.pid) { + (pid_t)event->comm.pid != evlist__workload_pid(evlist)) { pr_debug("%s with unexpected pid/tid\n", name); ++errs; } @@ -350,11 +350,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest pr_debug("PERF_RECORD_MMAP for %s missing!\n", "[vdso]"); ++errs; } -out_delete_evlist: +out_put_evlist: CPU_FREE(cpu_mask); - evlist__delete(evlist); out: perf_sample__exit(&sample); + evlist__put(evlist); if (err == -EACCES) return TEST_SKIP; if (err < 0 || errs != 0) diff --git a/tools/perf/tests/perf-time-to-tsc.c b/tools/perf/tests/perf-time-to-tsc.c index cca41bd37ae3c..f8f71fdd32b1b 100644 --- a/tools/perf/tests/perf-time-to-tsc.c +++ b/tools/perf/tests/perf-time-to-tsc.c @@ -99,7 +99,7 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su evlist = evlist__new(); CHECK_NOT_NULL__(evlist); - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); CHECK__(parse_event(evlist, "cpu-cycles:u")); @@ -121,9 +121,9 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su goto out_err; } - CHECK__(evlist__mmap(evlist, UINT_MAX)); + CHECK__(evlist__do_mmap(evlist, UINT_MAX)); - pc = evlist->mmap[0].core.base; + pc = evlist__mmap(evlist)[0].core.base; ret = perf_read_tsc_conversion(pc, &tc); if (ret) { if (ret == -EOPNOTSUPP) { @@ -145,8 +145,8 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su evlist__disable(evlist); - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -201,7 +201,7 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su err = TEST_OK; out_err: - evlist__delete(evlist); + evlist__put(evlist); perf_cpu_map__put(cpus); perf_thread_map__put(threads); return err; diff --git a/tools/perf/tests/pfm.c b/tools/perf/tests/pfm.c index fca4a86452df6..f7bf55be5e6ee 100644 --- a/tools/perf/tests/pfm.c +++ b/tools/perf/tests/pfm.c @@ -69,18 +69,18 @@ static int test__pfm_events(struct test_suite *test __maybe_unused, if (evlist == NULL) return -ENOMEM; - opt.value = evlist; + opt.value = &evlist; parse_libpfm_events_option(&opt, table[i].events, 0); TEST_ASSERT_EQUAL(table[i].events, - count_pfm_events(&evlist->core), + count_pfm_events(evlist__core(evlist)), table[i].nr_events); TEST_ASSERT_EQUAL(table[i].events, evlist__nr_groups(evlist), 0); - evlist__delete(evlist); + evlist__put(evlist); } return 0; } @@ -154,18 +154,18 @@ static int test__pfm_group(struct test_suite *test __maybe_unused, if (evlist == NULL) return -ENOMEM; - opt.value = evlist; + opt.value = &evlist; parse_libpfm_events_option(&opt, table[i].events, 0); TEST_ASSERT_EQUAL(table[i].events, - count_pfm_events(&evlist->core), + count_pfm_events(evlist__core(evlist)), table[i].nr_events); TEST_ASSERT_EQUAL(table[i].events, evlist__nr_groups(evlist), table[i].nr_groups); - evlist__delete(evlist); + evlist__put(evlist); } return 0; } diff --git a/tools/perf/tests/pmu-events.c b/tools/perf/tests/pmu-events.c index fd5630f0a13c0..f507ce39439ad 100644 --- a/tools/perf/tests/pmu-events.c +++ b/tools/perf/tests/pmu-events.c @@ -794,11 +794,13 @@ static int check_parse_id(const char *id, struct parse_events_error *error) for (cur = strchr(dup, '@') ; cur; cur = strchr(++cur, '@')) *cur = '/'; - ret = __parse_events(evlist, dup, /*pmu_filter=*/NULL, error, /*fake_pmu=*/true, - /*warn_if_reordered=*/true, /*fake_tp=*/false); + ret = __parse_events(evlist, dup, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, error, /*fake_pmu=*/true, + /*warn_if_reordered=*/true, + /*fake_tp=*/false); free(dup); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -865,13 +867,15 @@ static int test__parsing_callback(const struct pmu_metric *pm, cpus = perf_cpu_map__new("0"); if (!cpus) { - evlist__delete(evlist); + evlist__put(evlist); return -ENOMEM; } - perf_evlist__set_maps(&evlist->core, cpus, NULL); + perf_evlist__set_maps(evlist__core(evlist), cpus, NULL); - err = metricgroup__parse_groups_test(evlist, table, pm->metric_name); + err = metricgroup__parse_groups_test(evlist, table, + pm->metric_name, + /*cputype_filter=*/false); if (err) { if (is_expected_broken_metric(pm)) { (*failures)--; @@ -895,7 +899,8 @@ static int test__parsing_callback(const struct pmu_metric *pm, k++; } evlist__for_each_entry(evlist, evsel) { - struct metric_event *me = metricgroup__lookup(&evlist->metric_events, evsel, false); + struct metric_event *me = metricgroup__lookup(evlist__metric_events(evlist), + evsel, false); if (me != NULL) { struct metric_expr *mexp; @@ -919,7 +924,7 @@ static int test__parsing_callback(const struct pmu_metric *pm, /* ... cleanup. */ evlist__free_stats(evlist); perf_cpu_map__put(cpus); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/pmu.c b/tools/perf/tests/pmu.c index d7be9d1c6f52b..13e8d7fa80af2 100644 --- a/tools/perf/tests/pmu.c +++ b/tools/perf/tests/pmu.c @@ -294,7 +294,7 @@ static int test__pmu_config_helpers(struct test_suite *test __maybe_unused, ret = TEST_OK; err_out: parse_events_terms__exit(&terms); - evlist__delete(evlist); + evlist__put(evlist); test_pmu_put(dir, pmu); return ret; } @@ -346,7 +346,7 @@ static int test__pmu_events(struct test_suite *test __maybe_unused, int subtest ret = TEST_OK; err_out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); test_pmu_put(dir, pmu); return ret; } diff --git a/tools/perf/tests/sample-parsing.c b/tools/perf/tests/sample-parsing.c index 55f0b73ca20e0..20cab91ceaeb2 100644 --- a/tools/perf/tests/sample-parsing.c +++ b/tools/perf/tests/sample-parsing.c @@ -205,15 +205,11 @@ static bool samples_same(struct perf_sample *s1, static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) { - struct evsel evsel = { - .needs_swap = false, - .core = { - . attr = { - .sample_type = sample_type, - .read_format = read_format, - }, - }, + struct perf_event_attr attr = { + .sample_type = sample_type, + .read_format = read_format, }; + struct evsel *evsel; union perf_event *event; union { struct ip_callchain callchain; @@ -287,16 +283,21 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) size_t i, sz, bufsz; int err, ret = -1; + evsel = evsel__new(&attr); + if (!evsel) { + pr_debug("evsel__new failed\n"); + return -1; + } perf_sample__init(&sample_out, /*all=*/false); perf_sample__init(&sample_out_endian, /*all=*/false); if (sample_type & PERF_SAMPLE_REGS_USER) - evsel.core.attr.sample_regs_user = sample_regs; + evsel->core.attr.sample_regs_user = sample_regs; if (sample_type & PERF_SAMPLE_REGS_INTR) - evsel.core.attr.sample_regs_intr = sample_regs; + evsel->core.attr.sample_regs_intr = sample_regs; if (sample_type & PERF_SAMPLE_BRANCH_STACK) - evsel.core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + evsel->core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; for (i = 0; i < sizeof(regs); i++) *(i + (u8 *)regs) = i & 0xfe; @@ -311,12 +312,12 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) } sz = perf_event__sample_event_size(&sample, sample_type, read_format, - evsel.core.attr.branch_sample_type); + evsel->core.attr.branch_sample_type); bufsz = sz + 4096; /* Add a bit for overrun checking */ event = malloc(bufsz); if (!event) { pr_debug("malloc failed\n"); - return -1; + goto out_free; } memset(event, 0xff, bufsz); @@ -325,7 +326,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) event->header.size = sz; err = perf_event__synthesize_sample(event, sample_type, read_format, - evsel.core.attr.branch_sample_type, &sample); + evsel->core.attr.branch_sample_type, &sample); if (err) { pr_debug("%s failed for sample_type %#"PRIx64", error %d\n", "perf_event__synthesize_sample", sample_type, err); @@ -343,32 +344,33 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) goto out_free; } - evsel.sample_size = __evsel__sample_size(sample_type); + evsel->sample_size = __evsel__sample_size(sample_type); - err = evsel__parse_sample(&evsel, event, &sample_out); + err = evsel__parse_sample(evsel, event, &sample_out); if (err) { pr_debug("%s failed for sample_type %#"PRIx64", error %d\n", "evsel__parse_sample", sample_type, err); goto out_free; } - if (!samples_same(&sample, &sample_out, sample_type, read_format, evsel.needs_swap)) { + if (!samples_same(&sample, &sample_out, sample_type, read_format, evsel->needs_swap)) { pr_debug("parsing failed for sample_type %#"PRIx64"\n", sample_type); goto out_free; } if (sample_type == PERF_SAMPLE_BRANCH_STACK) { - evsel.needs_swap = true; - evsel.sample_size = __evsel__sample_size(sample_type); - err = evsel__parse_sample(&evsel, event, &sample_out_endian); + evsel->needs_swap = true; + evsel->sample_size = __evsel__sample_size(sample_type); + err = evsel__parse_sample(evsel, event, &sample_out_endian); if (err) { pr_debug("%s failed for sample_type %#"PRIx64", error %d\n", "evsel__parse_sample", sample_type, err); goto out_free; } - if (!samples_same(&sample, &sample_out_endian, sample_type, read_format, evsel.needs_swap)) { + if (!samples_same(&sample, &sample_out_endian, sample_type, + read_format, evsel->needs_swap)) { pr_debug("parsing failed for sample_type %#"PRIx64"\n", sample_type); goto out_free; @@ -380,6 +382,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) free(event); perf_sample__exit(&sample_out_endian); perf_sample__exit(&sample_out); + evsel__put(evsel); if (ret && read_format) pr_debug("read_format %#"PRIx64"\n", read_format); return ret; diff --git a/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh b/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh index ccb90dda24758..f3ebad5963783 100755 --- a/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh +++ b/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh @@ -44,7 +44,7 @@ branch_search='[[:space:]](bl|b(\.(eq|ne|cs|cc|mi|pl|vs|vc|hi|ls|ge|lt|gt|le|al) if [ "$(id -u)" == 0 ] && [ -e /proc/kcore ]; then echo "Testing kernel disassembly" perf record -o ${perfdata} -e cs_etm//k --kcore -Se -m,64K -- touch $file > /dev/null 2>&1 - perf script -i ${perfdata} -s python:${script_path} -- \ + perf script -i ${perfdata} --itrace=b -s python:${script_path} -- \ -d --stop-sample=2 -k ${perfdata}/kcore_dir/kcore 2> /dev/null > ${file} grep -q -E ${branch_search} ${file} echo "Found kernel branches" @@ -56,7 +56,7 @@ fi ## Test user ## echo "Testing userspace disassembly" perf record -o ${perfdata} -e cs_etm//u -Se -m,64K -- touch $file > /dev/null 2>&1 -perf script -i ${perfdata} -s python:${script_path} -- \ +perf script -i ${perfdata} --itrace=b -s python:${script_path} -- \ -d --stop-sample=2 2> /dev/null > ${file} grep -q -E ${branch_search} ${file} echo "Found userspace branches" diff --git a/tools/perf/tests/shell/inject_aslr.sh b/tools/perf/tests/shell/inject_aslr.sh index c00461828ea79..d83c2591db8f6 100755 --- a/tools/perf/tests/shell/inject_aslr.sh +++ b/tools/perf/tests/shell/inject_aslr.sh @@ -135,8 +135,14 @@ test_callchain_aslr() { echo "Callchain ASLR test [Failed - no noploop samples in original file]" err=1 elif [ -z "$new_addr" ]; then - echo "Callchain ASLR test [Failed - could not find remapped address]" - err=1 + if perf evlist -v -i "${data}" | grep -q 'sample_type:.*STACK_USER'; then + echo "Dropping stack user sample as possible ASLR leak" + echo "Call-graph dwarf not supported with 'perf inject --aslr'" + echo "Callchain ASLR test [Skip]" + else + echo "Callchain ASLR test [Failed - could not find remapped address]" + err=1 + fi elif [ "$orig_addr" = "$new_addr" ]; then echo "Callchain ASLR test [Failed - addresses are not remapped]" err=1 diff --git a/tools/perf/tests/shell/jitdump-python.sh b/tools/perf/tests/shell/jitdump-python.sh index ae86203b14a22..05aaa3bd900b5 100755 --- a/tools/perf/tests/shell/jitdump-python.sh +++ b/tools/perf/tests/shell/jitdump-python.sh @@ -16,11 +16,15 @@ if [ "${HAS_PERF_JIT}" != "True" ]; then exit 2 fi -PERF_DATA=$(mktemp /tmp/__perf_test.perf.data.XXXXXX) +PERF_DATA_DIR=$(mktemp -d /tmp/__perf_test.perf.data.dir.XXXXXX) +PERF_DATA="${PERF_DATA_DIR}/perf.data" cleanup() { echo "Cleaning up files..." - rm -f ${PERF_DATA} ${PERF_DATA}.jit /tmp/jit-${PID}.dump /tmp/jitted-${PID}-*.so 2> /dev/null + rm -rf ${PERF_DATA_DIR} 2> /dev/null + for p in ${ALL_PIDS}; do + rm -f /tmp/jit-${p}.dump /tmp/jitted-${p}-*.so 2> /dev/null + done trap - EXIT TERM INT } @@ -33,9 +37,16 @@ trap_cleanup() { trap trap_cleanup EXIT TERM INT -echo "Run python with -Xperf_jit" -cat <') - -echo "Found ${NUM} matching lines" - -echo "Remove JIT-ed DSOs from the build-ID cache" -for F in /tmp/jitted-${PID}-*.so; do - perf buildid-cache -r "${F}" + if [ -f "${PERF_DATA}.pid" ]; then + REAL_PID=$(cat "${PERF_DATA}.pid") + ALL_PIDS="${ALL_PIDS} ${REAL_PID}" + fi + + # extract PID of the target process from the data + PID=$(perf report -i "${PERF_DATA}" --stdio -F pid -q -g none | \ + cut -d: -f1 -s | sort -u | head -n 1 | tr -d ' ') + if [ -z "${PID}" ]; then + echo "Failed to get PID, retrying..." + continue + fi + ALL_PIDS="${ALL_PIDS} ${PID}" + + echo "Generate JIT-ed DSOs using perf inject" + DEBUGINFOD_URLS='' perf inject -i "${PERF_DATA}" -j -o "${PERF_DATA}.jit" + + echo "Add JIT-ed DSOs to the build-ID cache" + for F in /tmp/jitted-${PID}-*.so; do + perf buildid-cache -a "${F}" + done + + echo "Check the symbol containing the function/module name" + NUM=$(perf report -i "${PERF_DATA}.jit" -s sym --stdio | grep -cE 'py::(foo|bar|baz):') + + echo "Remove JIT-ed DSOs from the build-ID cache" + for F in /tmp/jitted-${PID}-*.so; do + perf buildid-cache -r "${F}" + done + rm -f /tmp/jitted-${PID}-*.so /tmp/jit-${PID}.dump 2>/dev/null + + if [ "${NUM}" -gt 0 ]; then + echo "Success: found ${NUM} matching lines" + break + fi + echo "No matching lines found, retrying with more iterations..." done cleanup diff --git a/tools/perf/tests/shell/kvm.sh b/tools/perf/tests/shell/kvm.sh index f88e859025c42..a5396f8e6fe5c 100755 --- a/tools/perf/tests/shell/kvm.sh +++ b/tools/perf/tests/shell/kvm.sh @@ -39,17 +39,28 @@ skip() { test_kvm_stat() { echo "Testing perf kvm stat" - echo "Recording kvm events for pid ${qemu_pid}..." - if ! perf kvm stat record -p "${qemu_pid}" -o "${perfdata}" sleep 1; then - echo "Failed to record kvm events" - err=1 - return - fi + local duration + local success=false + for duration in 1 2 4 8; do + echo "Recording kvm events for pid ${qemu_pid} (duration ${duration}s)..." + rm -f "${perfdata}" "${perfdata}".old + if ! perf kvm stat record -p "${qemu_pid}" -o "${perfdata}" \ + sleep ${duration} >/dev/null 2>&1; then + echo "perf kvm stat record failed, retrying..." + continue + fi + + if [ -e "${perfdata}" ] && \ + perf kvm -i "${perfdata}" stat report 2>&1 | grep -q "VM-EXIT"; then + success=true + break + fi + echo "No VM-EXIT events found, retrying..." + done - echo "Reporting kvm events..." - if ! perf kvm -i "${perfdata}" stat report 2>&1 | grep -q "VM-EXIT"; then + if [ "$success" = false ]; then echo "Failed to find VM-EXIT in report" - perf kvm -i "${perfdata}" stat report 2>&1 + perf kvm -i "${perfdata}" stat report 2>&1 || true err=1 return fi @@ -60,22 +71,26 @@ test_kvm_stat() { test_kvm_record_report() { echo "Testing perf kvm record/report" - echo "Recording kvm profile for pid ${qemu_pid}..." - # Use --host to avoid needing guest symbols/mounts for this simple test - # We just want to verify the command runs and produces data - # We run in background and kill it because 'perf kvm record' appends options - # after the command, which breaks 'sleep' (e.g. it gets '-e cycles'). - perf kvm --host record -p "${qemu_pid}" -o "${perfdata}" & - rec_pid=$! - sleep 1 - kill -INT "${rec_pid}" - wait "${rec_pid}" || true + local duration + local success=false + for duration in 1 2 4 8; do + echo "Recording kvm profile for pid ${qemu_pid} (duration ${duration}s)..." + rm -f "${perfdata}" "${perfdata}".old + + perf kvm --host record -p "${qemu_pid}" -o "${perfdata}" \ + -e cpu-clock sleep ${duration} + + if [ -e "${perfdata}" ] && \ + perf kvm -i "${perfdata}" report --stdio 2>&1 | grep -q "Event count"; then + success=true + break + fi + echo "No samples or report failed, retrying..." + done - echo "Reporting kvm profile..." - # Check for some standard output from report - if ! perf kvm -i "${perfdata}" report --stdio 2>&1 | grep -q "Event count"; then + if [ "$success" = false ]; then echo "Failed to report kvm profile" - perf kvm -i "${perfdata}" report --stdio 2>&1 + perf kvm -i "${perfdata}" report --stdio 2>&1 || true err=1 return fi diff --git a/tools/perf/tests/shell/lib/perf_metric_validation.py b/tools/perf/tests/shell/lib/perf_metric_validation.py index dea8ef1977bf6..3d52f94f22b91 100644 --- a/tools/perf/tests/shell/lib/perf_metric_validation.py +++ b/tools/perf/tests/shell/lib/perf_metric_validation.py @@ -383,10 +383,13 @@ def _run_perf(self, metric, workload: str): wl = workload.split() command.extend(wl) print(" ".join(command)) - cmd = subprocess.run(command, stderr=subprocess.PIPE, encoding='utf-8') - data = [x+'}' for x in cmd.stderr.split('}\n') if x] - if data[0][0] != '{': - data[0] = data[0][data[0].find('{'):] + cmd = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8') + lines = cmd.stderr.splitlines() + cmd.stdout.splitlines() + data = [] + for line in lines: + line = line.strip() + if line.startswith('{') and line.endswith('}'): + data.append(line) return data def collect_perf(self, workload: str): diff --git a/tools/perf/tests/shell/lib/perf_record.sh b/tools/perf/tests/shell/lib/perf_record.sh new file mode 100644 index 0000000000000..2b9e11b66dc7a --- /dev/null +++ b/tools/perf/tests/shell/lib/perf_record.sh @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0 + +PERF_RECORD_LOGS=() + +perf_record_with_retry() { + local perfdata="$1" + local check_cmd="$2" + local testprog_base="$3" + shift 3 + + local logfile + logfile=$(mktemp /tmp/__perf_record_retry.XXXXXX) + PERF_RECORD_LOGS+=("$logfile") + + # Save the e flag state and disable it + local save_e + if [[ $- == *e* ]]; then + save_e="set -e" + else + save_e="set +e" + fi + set +e + + local duration + local first_run=true + local ret=1 + local cmd_prefix="perf record" + if [ -n "${PERF_RECORD_CMD}" ]; then + cmd_prefix="${PERF_RECORD_CMD}" + fi + + for duration in 0.01 0.1 0.3 1.0 2.0; do + rm -f "${perfdata}".old + ${cmd_prefix} "$@" -o "${perfdata}" ${testprog_base} ${duration} > "$logfile" 2>&1 + local record_exit=$? + + if [ "$first_run" = true ] && [ $record_exit -ne 0 ]; then + ret=2 + break + fi + first_run=false + + if [ -e "${perfdata}" ] && eval "${check_cmd}"; then + ret=0 + break + fi + done + + eval "$save_e" + return $ret +} + +perf_record_cleanup() { + for logfile in "${PERF_RECORD_LOGS[@]}"; do + rm -f "$logfile" + done + PERF_RECORD_LOGS=() +} diff --git a/tools/perf/tests/shell/pipe_test.sh b/tools/perf/tests/shell/pipe_test.sh index e459aa99a9515..ce68d850c9838 100755 --- a/tools/perf/tests/shell/pipe_test.sh +++ b/tools/perf/tests/shell/pipe_test.sh @@ -12,8 +12,8 @@ skip_test_missing_symbol ${sym} data=$(mktemp /tmp/perf.data.XXXXXX) data2=$(mktemp /tmp/perf.data2.XXXXXX) -prog="perf test -w noploop" -[ "$(uname -m)" = "s390x" ] && prog="$prog 3" +prog="perf test -w noploop 0.1" +[ "$(uname -m)" = "s390x" ] && prog="perf test -w noploop 3" err=0 set -e diff --git a/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh b/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh new file mode 100755 index 0000000000000..c05ace8214ca3 --- /dev/null +++ b/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Zstd perf.data compression/decompression of multi-record data +# SPDX-License-Identifier: GPL-2.0 + +perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) +recout=$(mktemp /tmp/__perf_test.zstd.rec.XXXXX) +injout=$(mktemp /tmp/__perf_test.zstd.inj.XXXXX) +perf_tool=perf + +cleanup() { + rm -f "${perfdata}" "${perfdata}".old "${perfdata}".decomp "${recout}" "${injout}" +} +trap cleanup EXIT TERM INT + +skip_if_no_z_record() { + $perf_tool record -h 2>&1 | grep -q -- '-z, --compression-level' +} + +collect_z_record() { + echo "Collecting compressed record file:" + [ "$(uname -m)" != s390x ] && gflag='-g' + $perf_tool record -o "${perfdata}" $gflag -z -F max -m 32M --per-thread -- \ + $perf_tool test -w thloop 5 1 \ + >/dev/null 2>"${recout}" +} + +check_record() { + echo "Checking record did not fail to write data:" + if grep -q "failed to write perf data" "${recout}"; then + cat "${recout}" + return 1 + fi +} + +check_decompress() { + echo "Checking compressed file decompresses cleanly:" + if ! $perf_tool inject -i "${perfdata}" -o "${perfdata}".decomp 2>"${injout}"; then + cat "${injout}" + return 1 + fi + if grep -Eqi "decompress|corrupt|failed to process type" "${injout}"; then + cat "${injout}" + return 1 + fi +} + +skip_if_no_z_record || exit 2 +collect_z_record +check_record || exit 1 + +# Need >1 record, else the multi-record path wasn't exercised. +# Skip rather than pass/fail spuriously. +nr=$($perf_tool report -i "${perfdata}" --stats 2>/dev/null | + awk '/COMPRESSED2 events:/ { print $3 }') +if [ -z "${nr}" ] || [ "${nr}" -lt 2 ]; then + echo "less than two compressed records (${nr:-0}), skipping" + exit 2 +fi +echo "Produced ${nr} compressed records" + +check_decompress +err=$? +exit $err diff --git a/tools/perf/tests/shell/record.sh b/tools/perf/tests/shell/record.sh index 7cb81cf3444a7..dd90fef2088b1 100755 --- a/tools/perf/tests/shell/record.sh +++ b/tools/perf/tests/shell/record.sh @@ -1,10 +1,13 @@ #!/bin/bash -# perf record tests (exclusive) # SPDX-License-Identifier: GPL-2.0 +# perf record tests set -e shelldir=$(dirname "$0") +. "${shelldir}"/lib/perf_record.sh + + # shellcheck source=lib/waiting.sh . "${shelldir}"/lib/waiting.sh @@ -39,6 +42,7 @@ cleanup() { rm -f "${perfdata}" rm -f "${perfdata}".old rm -f "${script_output}" + perf_record_cleanup trap - EXIT TERM INT } @@ -50,22 +54,20 @@ trap_cleanup() { } trap trap_cleanup EXIT TERM INT +check_per_thread() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_per_thread() { echo "Basic --per-thread mode test" - if ! perf record -o /dev/null --quiet ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_per_thread" "perf test -w thloop" \ + --per-thread || ret=$? + if [ $ret -eq 2 ]; then echo "Per-thread record [Skipped event not supported]" return - fi - if ! perf record --per-thread -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Per-thread record [Failed record]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Per-thread record [Failed missing output]" + elif [ $ret -eq 1 ]; then + echo "Per-thread record [Failed record or missing output]" err=1 return fi @@ -96,6 +98,10 @@ test_per_thread() { echo "Basic --per-thread mode test [Success]" } +check_register_capture() { + perf script -F ip,sym,iregs -i "${perfdata}" 2>/dev/null | grep -q "DI:" +} + test_register_capture() { echo "Register capture test" if ! perf list pmu | grep -q 'br_inst_retired.near_call' @@ -108,11 +114,12 @@ test_register_capture() { echo "Register capture test [Skipped missing registers]" return fi - if ! perf record -o - --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call \ - -c 1000 --per-thread ${testprog} 2> /dev/null \ - | perf script -F ip,sym,iregs -i - 2> /dev/null \ - | grep -q "DI:" - then + + local ret=0 + perf_record_with_retry "${perfdata}" "check_register_capture" "perf test -w thloop" \ + --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call -c 1000 --per-thread || ret=$? + + if [ $ret -ne 0 ]; then echo "Register capture test [Failed missing output]" err=1 return @@ -120,65 +127,66 @@ test_register_capture() { echo "Register capture test [Success]" } +check_system_wide() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_system_wide() { echo "Basic --system-wide mode test" - if ! perf record -aB --synth=no -o "${perfdata}" ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_system_wide" "perf test -w thloop" \ + -aB --synth=no || ret=$? + if [ $ret -eq 2 ]; then echo "System-wide record [Skipped not supported]" return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then + elif [ $ret -eq 1 ]; then echo "System-wide record [Failed missing output]" err=1 return fi - if ! perf record -aB --synth=no -e cpu-clock,cs --threads=cpu \ - -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "System-wide record [Failed record --threads option]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "System-wide record [Failed --threads missing output]" + + ret=0 + perf_record_with_retry "${perfdata}" "check_system_wide" "perf test -w thloop" \ + -aB --synth=no -e cpu-clock,cs --threads=cpu || ret=$? + if [ $ret -ne 0 ]; then + echo "System-wide record [Failed record --threads option or missing output]" err=1 return fi echo "Basic --system-wide mode test [Success]" } +check_workload() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_workload() { echo "Basic target workload test" - if ! perf record -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Workload record [Failed record]" + local ret=0 + perf_record_with_retry "${perfdata}" "check_workload" "perf test -w thloop" || ret=$? + if [ $ret -ne 0 ]; then + echo "Workload record [Failed record or missing output]" err=1 return fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Workload record [Failed missing output]" - err=1 - return - fi - if ! perf record -e cpu-clock,cs --threads=package \ - -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Workload record [Failed record --threads option]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Workload record [Failed --threads missing output]" + + ret=0 + perf_record_with_retry "${perfdata}" "check_workload" "perf test -w thloop" \ + -e cpu-clock,cs --threads=package || ret=$? + if [ $ret -ne 0 ]; then + echo "Workload record [Failed record --threads option or missing output]" err=1 return fi echo "Basic target workload test [Success]" } +check_branch_counter() { + perf report -i "${perfdata}" -D -q 2>/dev/null | grep -q "$br_cntr_output" && \ + perf script -i "${perfdata}" -F +brstackinsn,+brcntr 2>/dev/null | \ + grep -q "$br_cntr_script_output" +} + test_branch_counter() { echo "Branch counter test" # Check if the branch counter feature is supported @@ -190,67 +198,60 @@ test_branch_counter() { return fi done - if ! perf record -o "${perfdata}" -e "{branches:p,instructions}" -j any,counter ${testprog} 2> /dev/null - then - echo "Branch counter record test [Failed record]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -D -q | grep -q "$br_cntr_output" - then - echo "Branch counter report test [Failed missing output]" - err=1 - return - fi - if ! perf script -i "${perfdata}" -F +brstackinsn,+brcntr | grep -q "$br_cntr_script_output" - then - echo " Branch counter script test [Failed missing output]" + local ret=0 + perf_record_with_retry "${perfdata}" "check_branch_counter" "perf test -w thloop" \ + -e "{branches:p,instructions}" -j any,counter || ret=$? + if [ $ret -ne 0 ]; then + echo "Branch counter test [Failed record or missing output]" err=1 return fi echo "Branch counter test [Success]" } +check_cgroup() { + perf report -i "${perfdata}" -D 2>/dev/null | grep -q "CGROUP" && \ + perf script -i "${perfdata}" -F cgroup 2>/dev/null | grep -q -v "unknown" +} + test_cgroup() { echo "Cgroup sampling test" - if ! perf record -aB --synth=cgroup --all-cgroups -o "${perfdata}" ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_cgroup" "perf test -w thloop" \ + -aB --synth=cgroup --all-cgroups || ret=$? + if [ $ret -eq 2 ]; then echo "Cgroup sampling [Skipped not supported]" return - fi - if ! perf report -i "${perfdata}" -D | grep -q "CGROUP" - then + elif [ $ret -eq 1 ]; then echo "Cgroup sampling [Failed missing output]" err=1 return fi - if ! perf script -i "${perfdata}" -F cgroup | grep -q -v "unknown" - then - echo "Cgroup sampling [Failed cannot resolve cgroup names]" - err=1 - return - fi echo "Cgroup sampling test [Success]" } +check_uid() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_uid() { echo "Uid sampling test" - if ! perf record -aB --synth=no --uid "$(id -u)" -o "${perfdata}" ${testprog} \ - > "${script_output}" 2>&1 - then - if grep -q "libbpf.*EPERM" "${script_output}" + local ret=0 + perf_record_with_retry "${perfdata}" "check_uid" "perf test -w thloop" \ + -aB --synth=no --uid "$(id -u)" || ret=$? + if [ $ret -eq 2 ]; then + local logfile="${PERF_RECORD_LOGS[${#PERF_RECORD_LOGS[@]}-1]}" + if grep -q -E "libbpf.*EPERM|Access to performance monitoring" "$logfile" || \ + grep -q -E "Permission denied|Failure to open any events" "$logfile" then echo "Uid sampling [Skipped permissions]" return else echo "Uid sampling [Failed to record]" err=1 - # cat "${script_output}" return fi - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then + elif [ $ret -eq 1 ]; then echo "Uid sampling [Failed missing output]" err=1 return diff --git a/tools/perf/tests/shell/record_lbr.sh b/tools/perf/tests/shell/record_lbr.sh index 78a02e90ece1e..8d51afeb437ba 100755 --- a/tools/perf/tests/shell/record_lbr.sh +++ b/tools/perf/tests/shell/record_lbr.sh @@ -1,9 +1,12 @@ #!/bin/bash -# perf record LBR tests (exclusive) # SPDX-License-Identifier: GPL-2.0 +# perf record LBR tests set -e +shelldir=$(dirname "$0") +. "${shelldir}"/lib/perf_record.sh + ParanoidAndNotRoot() { [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] } @@ -22,6 +25,7 @@ cleanup() { rm -rf "${perfdata}" rm -rf "${perfdata}".old rm -rf "${perfdata}".txt + perf_record_cleanup trap - EXIT TERM INT } @@ -34,22 +38,28 @@ trap_cleanup() { trap trap_cleanup EXIT TERM INT +check_lbr_callgraph() { + perf report --stitch-lbr -i "${perfdata}" > "${perfdata}".txt 2>&1 +} + lbr_callgraph_test() { test="LBR callgraph" echo "$test" - if ! perf record -e cycles --call-graph lbr -o "${perfdata}" perf test -w thloop - then + set +e + perf_record_with_retry "${perfdata}" "check_lbr_callgraph" "perf test -w thloop" \ + -e cycles --call-graph lbr + local ret=$? + set -e + + if [ $ret -eq 2 ]; then echo "$test [Failed support missing]" if [ $err -eq 0 ] then err=2 fi return - fi - - if ! perf report --stitch-lbr -i "${perfdata}" > "${perfdata}".txt - then + elif [ $ret -eq 1 ]; then cat "${perfdata}".txt echo "$test [Failed in perf report]" err=1 @@ -59,6 +69,12 @@ lbr_callgraph_test() { echo "$test [Success]" } +check_lbr_samples() { + local out + out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') + [ "$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true)" -gt 0 ] +} + lbr_test() { local branch_flags=$1 local test="LBR $2 test" @@ -70,25 +86,27 @@ lbr_test() { local r echo "$test" - if ! perf record -e cycles $branch_flags -o "${perfdata}" perf test -w thloop - then + set +e + perf_record_with_retry "${perfdata}" "check_lbr_samples" "perf test -w thloop" \ + -e cycles $branch_flags + local ret=$? + set -e + + if [ $ret -eq 2 ]; then echo "$test [Failed support missing]" - perf record -e cycles $branch_flags -o "${perfdata}" perf test -w thloop || true if [ $err -eq 0 ] then err=2 fi return - fi - - out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') - sam_nr=$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true) - if [ $sam_nr -eq 0 ] - then + elif [ $ret -eq 1 ]; then echo "$test [Failed no samples captured]" err=1 return fi + + out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') + sam_nr=$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true) echo "$test: $sam_nr samples" bs_nr=$(echo "$out" | grep -c 'branch stack: nr:' || true) diff --git a/tools/perf/tests/shell/stat_all_metrics.sh b/tools/perf/tests/shell/stat_all_metrics.sh index b582d23f28c9e..feeb34c6fa6df 100755 --- a/tools/perf/tests/shell/stat_all_metrics.sh +++ b/tools/perf/tests/shell/stat_all_metrics.sh @@ -12,38 +12,65 @@ system_wide_flag="-a" if ParanoidAndNotRoot 0 then system_wide_flag="" - test_prog="perf test -w noploop" + test_prog="perf test -w noploop 0.01" fi +check_metric() { + local output="$1" + local status="$2" + local metric="$3" + + if [[ $status -ne 0 || ! "$output" =~ ${metric:0:50} ]]; then + return 1 + fi + + if [[ "$output" =~ "" || "$output" =~ "" ]]; then + return 1 + fi + + return 0 +} + skip=0 err=3 for m in $(perf list --raw-dump metrics); do echo "Testing $m" result=$(perf stat -M "$m" $system_wide_flag -- $test_prog 2>&1) result_err=$? - if [[ $result_err -eq 0 && "$result" =~ ${m:0:50} ]] - then - # No error result and metric shown. + + if check_metric "$result" $result_err "$m"; then if [[ "$err" -ne 1 ]] then err=0 fi continue fi - if [[ "$result" =~ "Cannot resolve IDs for" || "$result" =~ "No supported events found" ]] + + if [[ "$result" =~ "Access to performance monitoring and observability operations is limited" || \ + "$result" =~ "in per-thread mode, enable system wide" || \ + "$result" =~ "" || \ + "$result" =~ "Cannot resolve IDs for" || \ + "$result" =~ "No supported events found" || \ + "$result" =~ "FP_ARITH" || \ + "$result" =~ "AMX" || \ + "$result" =~ "PMM" ]] then - if [[ $(perf list --raw-dump $m) == "Default"* ]] - then - echo "[Ignored $m] failed but as a Default metric this can be expected" - echo $result + true + else + result=$(perf stat -M "$m" $system_wide_flag -- perf test -w noploop 0.1 2>&1) + result_err=$? + + if check_metric "$result" $result_err "$m"; then + if [[ "$err" -ne 1 ]] + then + err=0 + fi continue fi - echo "[Failed $m] Metric contains missing events" - echo $result - err=1 # Fail - continue - elif [[ "$result" =~ \ - "Access to performance monitoring and observability operations is limited" ]] + fi + + # If retry also failed, determine if we skip, ignore, or fail + if [[ "$result" =~ "Access to performance monitoring and observability operations is limited" ]] then echo "[Skipped $m] Permission failure" echo $result @@ -61,7 +88,9 @@ for m in $(perf list --raw-dump metrics); do skip=1 fi continue - elif [[ "$result" =~ "" ]] + elif [[ "$result" =~ "" || \ + "$result" =~ "Cannot resolve IDs for" || \ + "$result" =~ "No supported events found" ]] then if [[ $(perf list --raw-dump $m) == "Default"* ]] then @@ -105,19 +134,7 @@ for m in $(perf list --raw-dump metrics); do continue fi - # Failed, possibly the workload was too small so retry with something longer. - result=$(perf stat -M "$m" $system_wide_flag -- perf bench internals synthesize 2>&1) - result_err=$? - if [[ $result_err -eq 0 && "$result" =~ ${m:0:50} ]] - then - # No error result and metric shown. - if [[ "$err" -ne 1 ]] - then - err=0 - fi - continue - fi - echo "[Failed $m] has non-zero error '$result_err' or not printed in:" + echo "[Failed $m] has non-zero error '$result_err' or not printed/counted in:" echo "$result" err=1 done diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh b/tools/perf/tests/shell/stat_bpf_counters.sh index 35463358b273c..11de77ee38ad4 100755 --- a/tools/perf/tests/shell/stat_bpf_counters.sh +++ b/tools/perf/tests/shell/stat_bpf_counters.sh @@ -4,21 +4,26 @@ set -e -workload="perf test -w sqrtloop" +# Get the first allowed CPU +CPU=$(taskset -c -p $$ | awk -F': ' '{print $2}' | awk -F'[,-]' '{print $1}') +if [ -z "$CPU" ]; then + CPU=0 +fi +workload=(taskset -c "$CPU" awk 'BEGIN { for (i=0; i<10000000; i++) sum+=i }') -# check whether $2 is within +/- 20% of $1 +# check whether $2 is within +/- 15% of $1 compare_number() { first_num=$1 second_num=$2 - # upper bound is first_num * 120% - upper=$(expr $first_num + $first_num / 5 ) - # lower bound is first_num * 80% - lower=$(expr $first_num - $first_num / 5 ) + # upper bound is first_num * 115% + upper=$(expr $first_num + $first_num / 20 \* 3 ) + # lower bound is first_num * 85% + lower=$(expr $first_num - $first_num / 20 \* 3 ) if [ $second_num -gt $upper ] || [ $second_num -lt $lower ]; then - echo "The difference between $first_num and $second_num are greater than 20%." + echo "The difference between $first_num and $second_num are greater than 15%." exit 1 fi } @@ -41,11 +46,12 @@ check_counts() test_bpf_counters() { printf "Testing --bpf-counters " - base_instructions=$(perf stat --no-big-num -e instructions -- $workload 2>&1 | \ + base_instructions=$(perf stat --no-big-num -e instructions:u -- "${workload[@]}" 2>&1 | \ awk -v i=0 -v c=0 '/instructions/ { \ if ($1 != " 0) printf "%.0f", c; else print "&1 | \ + bpf_instructions=$(perf stat --no-big-num --bpf-counters -e instructions:u \ + -- "${workload[@]}" 2>&1 | \ awk -v i=0 -v c=0 '/instructions/ { \ if ($1 != " 0) printf "%.0f", c; else print "&1) + stat_output=$(perf stat --no-big-num \ + -e instructions/name=base_instructions/u,instructions/name=bpf_instructions/bu \ + -- "${workload[@]}" 2>&1) base_instructions=$(echo "$stat_output"| \ awk -v i=0 -v c=0 '/base_instructions/ { \ if ($1 != "/dev/null 2>&1; then + echo "Skipping: no permission to record system-wide events (-a)" + exit 2 +fi + + pythonvalidator=$(dirname $0)/lib/perf_metric_validation.py rulefile=$(dirname $0)/lib/perf_metric_validation_rules.json tmpdir=$(mktemp -d /tmp/__perf_test.program.XXXXX) diff --git a/tools/perf/tests/shell/test_brstack.sh b/tools/perf/tests/shell/test_brstack.sh index eb5837f82e390..71550e0b37baa 100755 --- a/tools/perf/tests/shell/test_brstack.sh +++ b/tools/perf/tests/shell/test_brstack.sh @@ -110,20 +110,29 @@ test_trap_eret_branches() { return fi start_err=$err - err=0 - perf record -o $TMPDIR/perf.data --branch-filter any,save_type,u,k -- \ - perf test -w traploop 1000 > "$TMPDIR/record.txt" 2>&1 - perf script -i $TMPDIR/perf.data --fields brstacksym | \ - tr ' ' '\n' > $TMPDIR/perf.script - - # BRBINF.TYPE == TRAP are mapped to PERF_BR_IRQ by the BRBE driver - check_branches "^trap_bench\+[^ ]+/[^ ]/IRQ/" - check_branches "^[^ ]+/trap_bench\+[^ ]+/ERET/" - if [ $err -eq 0 ]; then + local ret=1 + for loops in 1000 10000 100000; do + err=0 + perf record -o $TMPDIR/perf.data --branch-filter any,save_type,u,k -- \ + perf test -w traploop $loops > "$TMPDIR/record.txt" 2>&1 + perf script -i $TMPDIR/perf.data --fields brstacksym | \ + tr ' ' '\n' > $TMPDIR/perf.script + + # BRBINF.TYPE == TRAP are mapped to PERF_BR_IRQ by the BRBE driver + check_branches "^trap_bench\+[^ ]+/[^ ]/IRQ/" + check_branches "^[^ ]+/trap_bench\+[^ ]+/ERET/" + if [ $err -eq 0 ]; then + ret=0 + break + fi + done + + if [ $ret -eq 0 ]; then echo "Testing trap & eret branches [Passed]" err=$start_err else echo "Testing trap & eret branches [Failed]" + err=1 fi } @@ -135,32 +144,40 @@ test_kernel_branches() { return fi start_err=$err - err=0 - perf record -o $TMPDIR/perf.data --branch-filter any,k -- \ - perf bench syscall basic --loop 1000 > "$TMPDIR/record.txt" 2>&1 - perf script -i $TMPDIR/perf.data --fields brstack | \ - tr ' ' '\n' > $TMPDIR/perf.script - - # Example of branch entries: - # "0xffffffff93bda241/0xffffffff93bda20f/M/-/-/..." - # Source addresses come first in user or kernel code. Next is the target - # address that must be in the kernel. - - # Look for source addresses with top bit set - if ! grep -q -E -m1 "^0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then - echo "Testing kernel branch sampling [Failed kernel branches missing]" - err=1 - fi - # Look for no target addresses without top bit set - if grep -q -E -m1 "^0x[0-9a-f]{0,16}/0x[0-7][0-9a-f]{1,15}/" $TMPDIR/perf.script; then - echo "Testing kernel branch sampling [Failed user branches found]" - err=1 - fi - if [ $err -eq 0 ]; then + local ret=1 + for loops in 1000 10000 100000; do + err=0 + perf record -o $TMPDIR/perf.data --branch-filter any,k -- \ + perf bench syscall basic --loop $loops > "$TMPDIR/record.txt" 2>&1 + perf script -i $TMPDIR/perf.data --fields brstack | \ + tr ' ' '\n' > $TMPDIR/perf.script + + # Example of branch entries: + # "0xffffffff93bda241/0xffffffff93bda20f/M/-/-/..." + # Source addresses come first in user or kernel code. Next is the target + # address that must be in the kernel. + + # Look for source addresses with top bit set + if ! grep -q -E -m1 "^0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then + err=1 + fi + # Look for no target addresses without top bit set + if grep -q -E -m1 "^0x[0-9a-f]{0,16}/0x[0-7][0-9a-f]{1,15}/" \ + $TMPDIR/perf.script; then + err=1 + fi + if [ $err -eq 0 ]; then + ret=0 + break + fi + done + + if [ $ret -eq 0 ]; then echo "Testing kernel branch sampling [Passed]" err=$start_err else echo "Testing kernel branch sampling [Failed]" + err=1 fi } @@ -206,20 +223,28 @@ test_syscall() { return fi start_err=$err - err=0 - perf record -o $TMPDIR/perf.data --branch-filter \ - any_call,save_type,u,k -c 10007 -- \ - perf bench syscall basic --loop 8000 > "$TMPDIR/record.txt" 2>&1 - perf script -i $TMPDIR/perf.data --fields brstacksym | \ - tr ' ' '\n' > $TMPDIR/perf.script - - check_branches "getppid[^ ]*/SYSCALL/" + local ret=1 + for loops in 8000 30000 100000; do + err=0 + perf record -o $TMPDIR/perf.data --branch-filter \ + any_call,save_type,u,k -c 10007 -- \ + perf bench syscall basic --loop $loops > "$TMPDIR/record.txt" 2>&1 + perf script -i $TMPDIR/perf.data --fields brstacksym | \ + tr ' ' '\n' > $TMPDIR/perf.script + + check_branches "getppid[^ ]*/SYSCALL/" + if [ $err -eq 0 ]; then + ret=0 + break + fi + done - if [ $err -eq 0 ]; then + if [ $ret -eq 0 ]; then echo "Testing syscalls [Passed]" err=$start_err else echo "Testing syscalls [Failed]" + err=1 fi } set -e diff --git a/tools/perf/tests/shell/test_dwarf_unwind_multithreaded.sh b/tools/perf/tests/shell/test_dwarf_unwind_multithreaded.sh new file mode 100755 index 0000000000000..49e6e3af771f4 --- /dev/null +++ b/tools/perf/tests/shell/test_dwarf_unwind_multithreaded.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Test libdw unwinding of multi-threaded processes (exclusive) + +set -e + +if ! perf check feature -q libdw-dwarf-unwind; then + echo "Skip: libdw DWARF unwinding is not available" + exit 2 +fi + +tmpdir=$(mktemp -d /tmp/perf-test-dwarf-unwind-multithreaded.XXXXXX) +perf_data="$tmpdir/perf.data" +perf_script="$tmpdir/perf-script.txt" +nr_threads=4 +nr_worker_threads=$((nr_threads - 1)) + +cleanup() +{ + trap - EXIT TERM INT + rm -rf "$tmpdir" +} + +trap cleanup EXIT TERM INT + +if ! perf record -q -e task-clock:u -F 99 --call-graph dwarf,8192 \ + -o "$perf_data" -- perf test -w thloop 2 "$nr_threads" +then + echo "Skip: failed to record task-clock:u" + exit 2 +fi + +if ! perf script --unwind-style=libdw \ + -F comm,pid,tid,event,ip,sym -i "$perf_data" > "$perf_script" +then + echo "Error: failed to process the recording with libdw" >&2 + exit 1 +fi + +nr_unwound_threads=$( + awk ' + BEGIN { RS = "" } + + # thfunc is the worker-only caller of test_loop. Finding it proves + # that libdw unwound beyond the sampled leaf for this worker TID. + /thfunc/ { + split($2, id, "/") + seen[id[2]] = 1 + } + + END { + for (tid in seen) + nr_tids++ + print nr_tids + 0 + } + ' "$perf_script" +) + +if [ "$nr_unwound_threads" -ne "$nr_worker_threads" ]; then + echo "Error: expected callchains for $nr_worker_threads worker TIDs," \ + "found $nr_unwound_threads" >&2 + exit 1 +fi + +exit 0 diff --git a/tools/perf/tests/shell/trace_record_replay.sh b/tools/perf/tests/shell/trace_record_replay.sh index 88d30a03dcecb..38fcafcdfb91c 100755 --- a/tools/perf/tests/shell/trace_record_replay.sh +++ b/tools/perf/tests/shell/trace_record_replay.sh @@ -6,16 +6,46 @@ # shellcheck source=lib/probe.sh . "$(dirname $0)"/lib/probe.sh +# shellcheck source=lib/perf_record.sh +. "$(dirname $0)"/lib/perf_record.sh skip_if_no_perf_trace || exit 2 [ "$(id -u)" = 0 ] || exit 2 file=$(mktemp /tmp/temporary_file.XXXXX) +err=0 -perf trace record -o ${file} sleep 1 || exit 1 -if ! perf trace -i ${file} 2>&1 | grep nanosleep; then - echo "Failed: cannot find *nanosleep syscall" +cleanup() { + rm -f ${file} + perf_record_cleanup + trap - EXIT INT TERM +} + +trap_cleanup() { + echo "Unexpected signal in ${FUNCNAME[1]}" + cleanup + exit 1 +} +trap trap_cleanup EXIT INT TERM + +check_nanosleep() { + perf trace -i "${file}" 2>&1 | grep -q nanosleep +} + +PERF_RECORD_CMD="perf trace record" perf_record_with_retry "${file}" "check_nanosleep" "sleep" +err=$? + +if [ $err -ne 0 ]; then + if [ $err -eq 2 ]; then + logfile="${PERF_RECORD_LOGS[${#PERF_RECORD_LOGS[@]}-1]}" + echo "perf trace record failed. Log output:" + cat "$logfile" + else + echo "Failed: cannot find *nanosleep syscall" + fi + cleanup exit 1 fi -rm -f ${file} +cleanup +exit 0 diff --git a/tools/perf/tests/sw-clock.c b/tools/perf/tests/sw-clock.c index b6e46975379cd..d181858816352 100644 --- a/tools/perf/tests/sw-clock.c +++ b/tools/perf/tests/sw-clock.c @@ -59,7 +59,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) evsel = evsel__new(&attr); if (evsel == NULL) { pr_debug("evsel__new\n"); - goto out_delete_evlist; + goto out_put_evlist; } evlist__add(evlist, evsel); @@ -68,10 +68,10 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) if (!cpus || !threads) { err = -ENOMEM; pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); if (evlist__open(evlist)) { const char *knob = "/proc/sys/kernel/perf_event_max_sample_rate"; @@ -80,14 +80,14 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) pr_debug("Couldn't open evlist: %s\nHint: check %s, using %" PRIu64 " in this test.\n", str_error_r(errno, sbuf, sizeof(sbuf)), knob, (u64)attr.sample_freq); - goto out_delete_evlist; + goto out_put_evlist; } - err = evlist__mmap(evlist, 128); + err = evlist__do_mmap(evlist, 128); if (err < 0) { pr_debug("failed to mmap event: %d (%s)\n", errno, str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } evlist__enable(evlist); @@ -98,7 +98,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) evlist__disable(evlist); - md = &evlist->mmap[0]; + md = &evlist__mmap(evlist)[0]; if (perf_mmap__read_init(&md->core) < 0) goto out_init; @@ -113,7 +113,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) if (err < 0) { pr_debug("Error during parse sample\n"); perf_sample__exit(&sample); - goto out_delete_evlist; + goto out_put_evlist; } total_periods += sample.period; @@ -131,10 +131,10 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) err = -1; } -out_delete_evlist: +out_put_evlist: perf_cpu_map__put(cpus); perf_thread_map__put(threads); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/switch-tracking.c b/tools/perf/tests/switch-tracking.c index e32331fee2778..73568c782d72b 100644 --- a/tools/perf/tests/switch-tracking.c +++ b/tools/perf/tests/switch-tracking.c @@ -237,6 +237,7 @@ static int add_event(struct evlist *evlist, struct list_head *events, if (evlist__parse_sample(evlist, event, &sample)) { pr_debug("evlist__parse_sample failed\n"); + perf_sample__exit(&sample); return -1; } @@ -282,8 +283,8 @@ static int process_events(struct evlist *evlist, struct mmap *md; int i, ret; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -374,7 +375,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub goto out_err; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); /* First event */ err = parse_event(evlist, "cpu-clock:u"); @@ -471,7 +472,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub goto out; } - err = evlist__mmap(evlist, UINT_MAX); + err = evlist__do_mmap(evlist, UINT_MAX); if (err) { pr_debug("evlist__mmap failed!\n"); goto out_err; @@ -582,7 +583,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub out: if (evlist) { evlist__disable(evlist); - evlist__delete(evlist); + evlist__put(evlist); } perf_cpu_map__put(cpus); perf_thread_map__put(threads); diff --git a/tools/perf/tests/task-exit.c b/tools/perf/tests/task-exit.c index 4053ff2813bb7..95393edbfe363 100644 --- a/tools/perf/tests/task-exit.c +++ b/tools/perf/tests/task-exit.c @@ -74,15 +74,15 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _ if (!cpus || !threads) { err = -ENOMEM; pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); err = evlist__prepare_workload(evlist, &target, argv, false, workload_exec_failed_signal); if (err < 0) { pr_debug("Couldn't run the workload!\n"); - goto out_delete_evlist; + goto out_put_evlist; } evsel = evlist__first(evlist); @@ -101,20 +101,20 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _ if (err < 0) { pr_debug("Couldn't open the evlist: %s\n", str_error_r(-err, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } - if (evlist__mmap(evlist, 128) < 0) { + if (evlist__do_mmap(evlist, 128) < 0) { pr_debug("failed to mmap events: %d (%s)\n", errno, str_error_r(errno, sbuf, sizeof(sbuf))); err = -1; - goto out_delete_evlist; + goto out_put_evlist; } evlist__start_workload(evlist); retry: - md = &evlist->mmap[0]; + md = &evlist__mmap(evlist)[0]; if (perf_mmap__read_init(&md->core) < 0) goto out_init; @@ -133,7 +133,7 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _ if (retry_count++ > 1000) { pr_debug("Failed after retrying 1000 times\n"); err = -1; - goto out_delete_evlist; + goto out_put_evlist; } goto retry; @@ -144,10 +144,10 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _ err = -1; } -out_delete_evlist: +out_put_evlist: perf_cpu_map__put(cpus); perf_thread_map__put(threads); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/time-utils-test.c b/tools/perf/tests/time-utils-test.c index 38df10373c1e1..90a9a4b4f1789 100644 --- a/tools/perf/tests/time-utils-test.c +++ b/tools/perf/tests/time-utils-test.c @@ -69,16 +69,19 @@ struct test_data { static bool test__perf_time__parse_for_ranges(struct test_data *d) { - struct evlist evlist = { - .first_sample_time = d->first, - .last_sample_time = d->last, - }; - struct perf_session session = { .evlist = &evlist }; + struct evlist *evlist = evlist__new(); + struct perf_session session = { .evlist = evlist }; struct perf_time_interval *ptime = NULL; int range_size, range_num; bool pass = false; int i, err; + if (!evlist) { + pr_debug("Missing evlist\n"); + return false; + } + evlist__set_first_sample_time(evlist, d->first); + evlist__set_last_sample_time(evlist, d->last); pr_debug("\nperf_time__parse_for_ranges(\"%s\")\n", d->str); if (strchr(d->str, '%')) @@ -127,6 +130,7 @@ static bool test__perf_time__parse_for_ranges(struct test_data *d) pass = true; out: + evlist__put(evlist); free(ptime); return pass; } diff --git a/tools/perf/tests/tool_pmu.c b/tools/perf/tests/tool_pmu.c index 1e900ef92e378..c6c5ebf0e935f 100644 --- a/tools/perf/tests/tool_pmu.c +++ b/tools/perf/tests/tool_pmu.c @@ -40,9 +40,10 @@ static int do_test(enum tool_pmu_event ev, bool with_pmu) } ret = TEST_OK; - if (with_pmu ? (evlist->core.nr_entries != 1) : (evlist->core.nr_entries < 1)) { + if (with_pmu ? (evlist__nr_entries(evlist) != 1) + : (evlist__nr_entries(evlist) < 1)) { pr_debug("FAILED %s:%d Unexpected number of events for '%s' of %d\n", - __FILE__, __LINE__, str, evlist->core.nr_entries); + __FILE__, __LINE__, str, evlist__nr_entries(evlist)); ret = TEST_FAIL; goto out; } @@ -67,7 +68,7 @@ static int do_test(enum tool_pmu_event ev, bool with_pmu) out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/topology.c b/tools/perf/tests/topology.c index bd7b859dea66a..77cb8318c0b18 100644 --- a/tools/perf/tests/topology.c +++ b/tools/perf/tests/topology.c @@ -46,7 +46,7 @@ static int session_write_header(char *path) session->evlist = evlist__new_default(&target, /*sample_callchains=*/false); TEST_ASSERT_VAL("can't get evlist", session->evlist); - session->evlist->session = session; + evlist__set_session(session->evlist, session); perf_header__set_feat(&session->header, HEADER_CPU_TOPOLOGY); perf_header__set_feat(&session->header, HEADER_NRCPUS); @@ -58,7 +58,7 @@ static int session_write_header(char *path) !perf_session__write_header(session, session->evlist, perf_data__fd(&data), true)); - evlist__delete(session->evlist); + evlist__put(session->evlist); perf_session__delete(session); return 0; diff --git a/tools/perf/tests/uncore-event-sorting.c b/tools/perf/tests/uncore-event-sorting.c index 7d2fc304e21fe..7756777c54c2e 100644 --- a/tools/perf/tests/uncore-event-sorting.c +++ b/tools/perf/tests/uncore-event-sorting.c @@ -147,8 +147,8 @@ static int test__uncore_event_sorting(struct test_suite *test __maybe_unused, goto out_err; } - CHECK_COND(evlist->core.nr_entries >= 4, "Number of events is >= 4"); - CHECK_EQUAL(evlist->core.nr_entries % 2, 0, "Number of events is a multiple of 2"); + CHECK_COND(evlist__nr_entries(evlist) >= 4, "Number of events is >= 4"); + CHECK_EQUAL(evlist__nr_entries(evlist) % 2, 0, "Number of events is a multiple of 2"); evlist__for_each_entry(evlist, evsel) { struct evsel *next; @@ -165,7 +165,7 @@ static int test__uncore_event_sorting(struct test_suite *test __maybe_unused, ret = TEST_OK; out_err: - evlist__delete(evlist); + evlist__put(evlist); parse_events_error__exit(&err); zfree(&pmu_prefix); zfree(&m.event1); diff --git a/tools/perf/ui/browsers/annotate.c b/tools/perf/ui/browsers/annotate.c index 97ae4c86bebbe..e47a467750890 100644 --- a/tools/perf/ui/browsers/annotate.c +++ b/tools/perf/ui/browsers/annotate.c @@ -597,7 +597,7 @@ static bool annotate_browser__callq(struct annotate_browser *browser, notes = symbol__annotation(dl->ops.target.sym); annotation__lock(notes); - if (!symbol__hists(dl->ops.target.sym, evsel->evlist->core.nr_entries)) { + if (!symbol__hists(dl->ops.target.sym, evlist__nr_entries(evsel->evlist))) { annotation__unlock(notes); ui__warning("Not enough memory for annotating '%s' symbol!\n", dl->ops.target.sym->name); @@ -1201,7 +1201,7 @@ int __hist_entry__tui_annotate(struct hist_entry *he, struct map_symbol *ms, ui__warning("Annotation has no source code."); } } else { - err = thread__get_arch(ms->thread, &browser.arch); + err = map_symbol__get_arch(ms, &browser.arch); if (err) { annotate_browser__symbol_annotate_error(&browser, err); return -1; diff --git a/tools/perf/ui/browsers/hists.c b/tools/perf/ui/browsers/hists.c index cfa6386e6e1da..96fd598dbb6d5 100644 --- a/tools/perf/ui/browsers/hists.c +++ b/tools/perf/ui/browsers/hists.c @@ -688,10 +688,10 @@ static int hist_browser__handle_hotkey(struct hist_browser *browser, bool warn_l ui_browser__update_nr_entries(&browser->b, nr_entries); if (warn_lost_event && - (evsel->evlist->stats.nr_lost_warned != - evsel->evlist->stats.nr_events[PERF_RECORD_LOST])) { - evsel->evlist->stats.nr_lost_warned = - evsel->evlist->stats.nr_events[PERF_RECORD_LOST]; + (evlist__stats(evsel->evlist)->nr_lost_warned != + evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST])) { + evlist__stats(evsel->evlist)->nr_lost_warned = + evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST]; ui_browser__warn_lost_events(&browser->b); } @@ -2356,6 +2356,16 @@ static int hists_browser__scnprintf_title(struct hist_browser *browser, char *bf return printed; } +struct popup_action { + unsigned long time; + struct thread *thread; + int (*fn)(struct hist_browser *browser, struct popup_action *act); + struct map_symbol ms; + int socket; + enum rstype rstype; + +}; + static inline void free_popup_options(char **options, int n) { int i; @@ -2364,6 +2374,16 @@ static inline void free_popup_options(char **options, int n) zfree(&options[i]); } +static inline void free_popup_actions(struct popup_action *actions, int n) +{ + int i; + + for (i = 0; i < n; ++i) { + map_symbol__exit(&actions[i].ms); + memset(&actions[i], 0, sizeof(struct popup_action)); + } +} + /* * Only runtime switching of perf data file will make "input_name" point * to a malloced buffer. So add "is_input_name_malloced" flag to decide @@ -2453,16 +2473,6 @@ static int switch_data_file(void) return ret; } -struct popup_action { - unsigned long time; - struct thread *thread; - int (*fn)(struct hist_browser *browser, struct popup_action *act); - struct map_symbol ms; - int socket; - enum rstype rstype; - -}; - static int do_annotate(struct hist_browser *browser, struct popup_action *act) { @@ -2540,7 +2550,7 @@ add_annotate_opt(struct popup_action *act, char **optstr, if (asprintf(optstr, "Annotate %s", ms->sym->name) < 0) return 0; - act->ms = *ms; + map_symbol__copy(&act->ms, ms); act->fn = do_annotate; return 1; } @@ -2572,7 +2582,7 @@ add_annotate_type_opt(struct popup_action *act, char **optstr, static int do_zoom_thread(struct hist_browser *browser, struct popup_action *act) { - struct thread *thread = act->thread; + struct thread *thread = act->ms.thread; if ((!hists__has(browser->hists, thread) && !hists__has(browser->hists, comm)) || thread == NULL) @@ -2627,7 +2637,7 @@ add_thread_opt(struct hist_browser *browser, struct popup_action *act, if (ret < 0) return 0; - act->thread = thread; + act->ms.thread = thread__get(thread); act->fn = do_zoom_thread; return 1; } @@ -2640,13 +2650,14 @@ static int hists_browser__zoom_map(struct hist_browser *browser, struct map *map if (browser->hists->dso_filter) { pstack__remove(browser->pstack, &browser->hists->dso_filter); perf_hpp__set_elide(HISTC_DSO, false); + dso__put((struct dso *)browser->hists->dso_filter); browser->hists->dso_filter = NULL; ui_helpline__pop(); } else { struct dso *dso = map__dso(map); ui_helpline__fpush("To zoom out press ESC or ENTER + \"Zoom out of %s DSO\"", __map__is_kernel(map) ? "the Kernel" : dso__short_name(dso)); - browser->hists->dso_filter = dso; + browser->hists->dso_filter = dso__get(dso); perf_hpp__set_elide(HISTC_DSO, true); pstack__push(browser->pstack, &browser->hists->dso_filter); } @@ -2674,7 +2685,7 @@ add_dso_opt(struct hist_browser *browser, struct popup_action *act, __map__is_kernel(map) ? "the Kernel" : dso__short_name(map__dso(map))) < 0) return 0; - act->ms.map = map; + act->ms.map = map__get(map); act->fn = do_zoom_dso; return 1; } @@ -2719,7 +2730,7 @@ add_map_opt(struct hist_browser *browser, if (asprintf(optstr, "Browse map details") < 0) return 0; - act->ms.map = map; + act->ms.map = map__get(map); act->fn = do_browse_map; return 1; } @@ -2733,8 +2744,8 @@ do_run_script(struct hist_browser *browser, int n = 0; len = 100; - if (act->thread) - len += strlen(thread__comm_str(act->thread)); + if (act->ms.thread) + len += strlen(thread__comm_str(act->ms.thread)); else if (act->ms.sym) len += strlen(act->ms.sym->name); script_opt = malloc(len); @@ -2742,9 +2753,9 @@ do_run_script(struct hist_browser *browser, return -1; script_opt[0] = 0; - if (act->thread) { + if (act->ms.thread) { n = scnprintf(script_opt, len, " -c %s ", - thread__comm_str(act->thread)); + thread__comm_str(act->ms.thread)); } else if (act->ms.sym) { n = scnprintf(script_opt, len, " -S %s ", act->ms.sym->name); @@ -2799,7 +2810,7 @@ add_script_opt_2(struct popup_action *act, char **optstr, return 0; } - act->thread = thread; + act->ms.thread = thread__get(thread); act->ms.sym = sym; act->fn = do_run_script; return 1; @@ -3003,9 +3014,9 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h struct hists *hists = evsel__hists(evsel); struct hist_browser *browser = perf_evsel_browser__new(evsel, hbt, env); struct branch_info *bi = NULL; -#define MAX_OPTIONS 16 +#define MAX_OPTIONS 32 char *options[MAX_OPTIONS]; - struct popup_action actions[MAX_OPTIONS]; + struct popup_action actions[MAX_OPTIONS], hotkey_act; int nr_options = 0; int key = -1; char buf[128]; @@ -3064,15 +3075,15 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h browser->min_pcnt = min_pcnt; hist_browser__update_nr_entries(browser); + memset(options, 0, sizeof(options)); + memset(actions, 0, sizeof(actions)); + browser->pstack = pstack__new(3); if (browser->pstack == NULL) goto out; ui_helpline__push(helpline); - memset(options, 0, sizeof(options)); - memset(actions, 0, sizeof(actions)); - if (symbol_conf.col_width_list_str) perf_hpp__set_user_width(symbol_conf.col_width_list_str); @@ -3087,6 +3098,8 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h key = 0; // reset key do_hotkey: // key came straight from options ui__popup_menu() + free_popup_options(options, MAX_OPTIONS); + free_popup_actions(actions, MAX_OPTIONS); choice = nr_options = 0; key = hist_browser__run(browser, helpline, warn_lost_event, key); @@ -3146,24 +3159,40 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h } if (!browser->selection->sym) { + struct map_symbol source_ms; + if (!browser->he_selection) continue; + memset(&source_ms, 0, sizeof(source_ms)); + if (sort__mode == SORT_MODE__BRANCH) { bi = browser->he_selection->branch_info; if (!bi || !bi->to.ms.map) continue; - actions->ms.sym = symbol__new_unresolved(bi->to.al_addr, bi->to.ms.map); - actions->ms.map = bi->to.ms.map; + source_ms.sym = + symbol__new_unresolved( + bi->to.al_addr, + bi->to.ms.map); + source_ms.thread = bi->to.ms.thread; + source_ms.map = bi->to.ms.map; } else { - actions->ms.sym = symbol__new_unresolved(browser->he_selection->ip, - browser->selection->map); - actions->ms.map = browser->selection->map; + source_ms.sym = + symbol__new_unresolved( + browser->he_selection->ip, + browser->selection->map); + source_ms.thread = browser->selection->thread; + source_ms.map = browser->selection->map; } - if (!actions->ms.sym) + if (!source_ms.sym) continue; + + memset(&hotkey_act, 0, sizeof(hotkey_act)); + map_symbol__copy(&hotkey_act.ms, &source_ms); + do_annotate(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); } else { if (symbol__annotation(browser->selection->sym)->src == NULL) { ui_browser__warning(&browser->b, delay_secs * 2, @@ -3173,18 +3202,20 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h continue; } - actions->ms.map = browser->selection->map; - actions->ms.sym = browser->selection->sym; + memset(&hotkey_act, 0, sizeof(hotkey_act)); + map_symbol__copy(&hotkey_act.ms, browser->selection); + do_annotate(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); } - - do_annotate(browser, actions); continue; case 'P': hist_browser__dump(browser); continue; case 'd': - actions->ms.map = map; - do_zoom_dso(browser, actions); + memset(&hotkey_act, 0, sizeof(hotkey_act)); + hotkey_act.ms.map = map__get(map); + do_zoom_dso(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); continue; case 'k': if (browser->selection != NULL) @@ -3199,12 +3230,16 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h verbose); continue; case 't': - actions->thread = thread; - do_zoom_thread(browser, actions); + memset(&hotkey_act, 0, sizeof(hotkey_act)); + hotkey_act.ms.thread = thread__get(thread); + do_zoom_thread(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); continue; case 'S': - actions->socket = socked_id; - do_zoom_socket(browser, actions); + memset(&hotkey_act, 0, sizeof(hotkey_act)); + hotkey_act.socket = socked_id; + do_zoom_socket(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); continue; case '/': if (ui_browser__input_window("Symbol to show", @@ -3219,9 +3254,11 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h continue; case 'r': if (is_report_browser(hbt)) { - actions->thread = NULL; - actions->ms.sym = NULL; - do_run_script(browser, actions); + memset(&hotkey_act, 0, sizeof(hotkey_act)); + hotkey_act.ms.thread = NULL; + hotkey_act.ms.sym = NULL; + do_run_script(browser, &hotkey_act); + map_symbol__exit(&hotkey_act.ms); } continue; case 's': @@ -3293,20 +3330,19 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h continue; } - actions->ms.map = map; + memset(&hotkey_act, 0, sizeof(hotkey_act)); top = pstack__peek(browser->pstack); if (top == &browser->hists->dso_filter) { - /* - * No need to set actions->dso here since - * it's just to remove the current filter. - */ - do_zoom_dso(browser, actions); + hotkey_act.ms.map = map__get(map); + do_zoom_dso(browser, &hotkey_act); } else if (top == &browser->hists->thread_filter) { - actions->thread = thread; - do_zoom_thread(browser, actions); + hotkey_act.ms.thread = thread__get(thread); + do_zoom_thread(browser, &hotkey_act); } else if (top == &browser->hists->socket_filter) { - do_zoom_socket(browser, actions); + hotkey_act.socket = socked_id; + do_zoom_socket(browser, &hotkey_act); } + map_symbol__exit(&hotkey_act.ms); continue; } case 'q': @@ -3321,7 +3357,7 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h * No need to refresh, resort/decay histogram * entries if we are not collecting samples: */ - if (top->evlist->enabled) { + if (evlist__enabled(top->evlist)) { helpline = "Press 'f' to disable the events or 'h' to see other hotkeys"; hbt->refresh = delay_secs; } else { @@ -3443,9 +3479,20 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h if (key == K_SWITCH_INPUT_DATA) break; + + free_popup_options(options, MAX_OPTIONS); + free_popup_actions(actions, MAX_OPTIONS); } out_free_stack: pstack__delete(browser->pstack); + free_popup_actions(actions, MAX_OPTIONS); + thread__zput(hists->thread_filter); + dso__put((struct dso *)hists->dso_filter); + hists->dso_filter = NULL; + perf_hpp__set_elide(HISTC_DSO, false); + perf_hpp__set_elide(HISTC_THREAD, false); + hists__filter_by_dso(hists); + hists__filter_by_thread(hists); out: hist_browser__delete(browser); free_popup_options(options, MAX_OPTIONS); @@ -3493,7 +3540,7 @@ static void perf_evsel_menu__write(struct ui_browser *browser, unit, unit == ' ' ? "" : " ", ev_name); ui_browser__printf(browser, "%s", bf); - nr_events = evsel->evlist->stats.nr_events[PERF_RECORD_LOST]; + nr_events = evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST]; if (nr_events != 0) { menu->lost_events = true; if (!current_entry) @@ -3559,13 +3606,13 @@ static int perf_evsel_menu__run(struct evsel_menu *menu, ui_browser__show_title(&menu->b, title); switch (key) { case K_TAB: - if (pos->core.node.next == &evlist->core.entries) + if (pos->core.node.next == &evlist__core(evlist)->entries) pos = evlist__first(evlist); else pos = evsel__next(pos); goto browse_hists; case K_UNTAB: - if (pos->core.node.prev == &evlist->core.entries) + if (pos->core.node.prev == &evlist__core(evlist)->entries) pos = evlist__last(evlist); else pos = evsel__prev(pos); @@ -3618,7 +3665,7 @@ static int __evlist__tui_browse_hists(struct evlist *evlist, int nr_entries, con struct evsel *pos; struct evsel_menu menu = { .b = { - .entries = &evlist->core.entries, + .entries = &evlist__core(evlist)->entries, .refresh = ui_browser__list_head_refresh, .seek = ui_browser__list_head_seek, .write = perf_evsel_menu__write, @@ -3646,7 +3693,7 @@ static int __evlist__tui_browse_hists(struct evlist *evlist, int nr_entries, con static bool evlist__single_entry(struct evlist *evlist) { - int nr_entries = evlist->core.nr_entries; + int nr_entries = evlist__nr_entries(evlist); if (nr_entries == 1) return true; @@ -3664,7 +3711,7 @@ static bool evlist__single_entry(struct evlist *evlist) int evlist__tui_browse_hists(struct evlist *evlist, const char *help, struct hist_browser_timer *hbt, float min_pcnt, struct perf_env *env, bool warn_lost_event) { - int nr_entries = evlist->core.nr_entries; + int nr_entries = evlist__nr_entries(evlist); if (evlist__single_entry(evlist)) { single_entry: { diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c index 394c061fbeb38..cda3836329c3a 100644 --- a/tools/perf/util/amd-sample-raw.c +++ b/tools/perf/util/amd-sample-raw.c @@ -421,7 +421,7 @@ static void parse_cpuid(struct perf_env *env) */ bool evlist__has_amd_ibs(struct evlist *evlist) { - struct perf_env *env = perf_session__env(evlist->session); + struct perf_env *env = perf_session__env(evlist__session(evlist)); int ret, nr_pmu_mappings = perf_env__nr_pmu_mappings(env); const char *pmu_mapping = perf_env__pmu_mappings(env); char name[sizeof("ibs_fetch")]; diff --git a/tools/perf/util/annotate-data.c b/tools/perf/util/annotate-data.c index 63e3c54fab421..4e4c587640823 100644 --- a/tools/perf/util/annotate-data.c +++ b/tools/perf/util/annotate-data.c @@ -1829,7 +1829,7 @@ int annotated_data_type__update_samples(struct annotated_data_type *adt, return 0; if (adt->histograms == NULL) { - int nr = evsel->evlist->core.nr_entries; + int nr = evlist__nr_entries(evsel->evlist); if (alloc_data_type_histograms(adt, nr) < 0) return -1; diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index 02505222d8c2d..df70e95a84704 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -328,7 +328,7 @@ static int symbol__inc_addr_samples(struct map_symbol *ms, if (sym == NULL) return 0; - src = symbol__hists(sym, sample->evsel->evlist->core.nr_entries); + src = symbol__hists(sym, evlist__nr_entries(sample->evsel->evlist)); return src ? __symbol__inc_addr_samples(ms, src, addr, sample) : 0; } @@ -339,7 +339,7 @@ static int symbol__account_br_cntr(struct annotated_branch *branch, { unsigned int br_cntr_nr = evsel__leader(evsel)->br_cntr_nr; unsigned int base = evsel__leader(evsel)->br_cntr_idx; - unsigned int off = offset * evsel->evlist->nr_br_cntr; + unsigned int off = offset * evlist__nr_br_cntr(evsel->evlist); u64 *branch_br_cntr = branch->br_cntr; unsigned int i, mask, width; @@ -369,7 +369,7 @@ static int symbol__account_cycles(u64 addr, u64 start, struct symbol *sym, if (sym == NULL) return 0; - branch = symbol__find_branch_hist(sym, evsel->evlist->nr_br_cntr); + branch = symbol__find_branch_hist(sym, evlist__nr_br_cntr(evsel->evlist)); if (!branch) return -ENOMEM; if (addr < sym->start || addr >= sym->end) @@ -511,7 +511,7 @@ static void annotation__count_and_fill(struct annotation *notes, u64 start, u64 static int annotation__compute_ipc(struct annotation *notes, size_t size, struct evsel *evsel) { - unsigned int br_cntr_nr = evsel->evlist->nr_br_cntr; + unsigned int br_cntr_nr = evlist__nr_br_cntr(evsel->evlist); int err = 0; s64 offset; @@ -982,24 +982,43 @@ void symbol__calc_percent(struct symbol *sym, struct evsel *evsel) annotation__calc_percent(notes, evsel, symbol__size(sym)); } -int thread__get_arch(struct thread *thread, const struct arch **parch) + + +int map_symbol__get_arch(struct map_symbol *ms, const struct arch **parch) { const struct arch *arch; - struct machine *machine; - uint32_t e_flags; - uint16_t e_machine; + struct machine *machine = NULL; + struct map *map = ms->map; + struct dso *dso = map ? map__dso(map) : NULL; + uint32_t e_flags = 0; + uint16_t e_machine = EM_NONE; - if (!thread) { - *parch = NULL; - return -1; + const char *cpuid = NULL; + + if (ms->thread) { + machine = maps__machine(thread__maps(ms->thread)); + e_machine = thread__e_machine(ms->thread, machine, &e_flags); + if (machine && machine->env) + cpuid = machine->env->cpuid; + } else if (dso) { + struct maps *kmaps = (map && dso__kernel(dso)) ? map__kmaps(map) : NULL; + struct machine *kmap_machine = kmaps ? maps__machine(kmaps) : NULL; + + e_machine = dso__e_machine(dso, kmap_machine, &e_flags); + if (kmap_machine && kmap_machine->env) + cpuid = kmap_machine->env->cpuid; } - machine = maps__machine(thread__maps(thread)); - e_machine = thread__e_machine(thread, machine, &e_flags); - arch = arch__find(e_machine, e_flags, machine->env ? machine->env->cpuid : NULL); + if (e_machine == EM_NONE) + e_machine = thread__e_machine(NULL, NULL, &e_flags); + + arch = arch__find(e_machine, e_flags, cpuid); if (arch == NULL) { pr_err("%s: unsupported arch %d\n", __func__, e_machine); - return errno; + /* TODO: Refactor annotate/disassemble subsystem error + * codes to uniformly return negative integers. + */ + return errno ? errno : ENOTSUP; } if (parch) *parch = arch; @@ -1018,7 +1037,7 @@ int symbol__annotate(struct map_symbol *ms, struct evsel *evsel, const struct arch *arch = NULL; int err, nr; - err = thread__get_arch(ms->thread, &arch); + err = map_symbol__get_arch(ms, &arch); if (err) return err; @@ -1251,6 +1270,11 @@ int hist_entry__annotate_printf(struct hist_entry *he, struct evsel *evsel) evsel_name = buf; } + if (map_symbol__get_arch(ms, &apd.arch)) { + free(filename); + return ENOTSUP; + } + graph_dotted_len = printf(" %-*.*s| Source code & Disassembly of %s for %s (%" PRIu64 " samples, " "percent: %s)\n", width, width, symbol_conf.show_total_period ? "Period" : @@ -1266,7 +1290,6 @@ int hist_entry__annotate_printf(struct hist_entry *he, struct evsel *evsel) apd.addr_fmt_width = annotated_source__addr_fmt_width(¬es->src->source, notes->src->start); - thread__get_arch(ms->thread, &apd.arch); apd.dbg = dso__debuginfo(dso); list_for_each_entry(pos, ¬es->src->source, node) { @@ -1371,7 +1394,7 @@ static int symbol__annotate_fprintf2(struct symbol *sym, FILE *fp, struct annotation_line *al; if (annotate_opts.code_with_type) { - thread__get_arch(apd->he->ms.thread, &apd->arch); + map_symbol__get_arch(&apd->he->ms, &apd->arch); apd->dbg = dso__debuginfo(map__dso(apd->he->ms.map)); } @@ -1813,7 +1836,7 @@ int annotation_br_cntr_abbr_list(char **str, struct evsel *evsel, bool header) struct evsel *pos; struct strbuf sb; - if (evsel->evlist->nr_br_cntr <= 0) + if (evlist__nr_br_cntr(evsel->evlist) <= 0) return -ENOTSUP; strbuf_init(&sb, /*hint=*/ 0); diff --git a/tools/perf/util/annotate.h b/tools/perf/util/annotate.h index 1aa6df7d16187..fa08d09b80f76 100644 --- a/tools/perf/util/annotate.h +++ b/tools/perf/util/annotate.h @@ -584,5 +584,6 @@ int annotation_br_cntr_entry(char **str, int br_cntr_nr, u64 *br_cntr, int num_aggr, struct evsel *evsel); int annotation_br_cntr_abbr_list(char **str, struct evsel *evsel, bool header); -int thread__get_arch(struct thread *thread, const struct arch **parch); + +int map_symbol__get_arch(struct map_symbol *ms, const struct arch **parch); #endif /* __PERF_ANNOTATE_H */ diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index 552f063f126e6..401aab529309c 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -1605,6 +1605,10 @@ static u64 **arm_spe__alloc_metadata(struct perf_record_auxtrace_info *info, hdr_sz = ptr[ARM_SPE_HEADER_SIZE]; *nr_cpu = ptr[ARM_SPE_CPUS_NUM]; + /* nr_cpu is used as a divisor below */ + if (*nr_cpu <= 0) + return NULL; + metadata = calloc(*nr_cpu, sizeof(*metadata)); if (!metadata) return NULL; diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index 4cd2caf540152..aa749e1c30365 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -191,7 +191,7 @@ void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, struct evlist *evlist, struct evsel *evsel, int idx) { - bool per_cpu = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus); + bool per_cpu = !perf_cpu_map__has_any_cpu(evlist__core(evlist)->user_requested_cpus); mp->mmap_needed = evsel->needs_auxtrace_mmap; @@ -201,11 +201,11 @@ void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, mp->idx = idx; if (per_cpu) { - mp->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx); - mp->tid = perf_thread_map__pid(evlist->core.threads, 0); + mp->cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, idx); + mp->tid = perf_thread_map__pid(evlist__core(evlist)->threads, 0); } else { mp->cpu.cpu = -1; - mp->tid = perf_thread_map__pid(evlist->core.threads, idx); + mp->tid = perf_thread_map__pid(evlist__core(evlist)->threads, idx); } } @@ -251,8 +251,12 @@ static int auxtrace_queues__grow(struct auxtrace_queues *queues, { unsigned int nr_queues = queues->nr_queues; struct auxtrace_queue *queue_array; + struct auxtrace_queue *old_array = queues->queue_array; unsigned int i; + if (!new_nr_queues) + return -EINVAL; + if (!nr_queues) nr_queues = AUXTRACE_INIT_NR_QUEUES; @@ -267,16 +271,17 @@ static int auxtrace_queues__grow(struct auxtrace_queues *queues, return -ENOMEM; for (i = 0; i < queues->nr_queues; i++) { - list_splice_tail(&queues->queue_array[i].head, + list_splice_tail(&old_array[i].head, &queue_array[i].head); - queue_array[i].tid = queues->queue_array[i].tid; - queue_array[i].cpu = queues->queue_array[i].cpu; - queue_array[i].set = queues->queue_array[i].set; - queue_array[i].priv = queues->queue_array[i].priv; + queue_array[i].tid = old_array[i].tid; + queue_array[i].cpu = old_array[i].cpu; + queue_array[i].set = old_array[i].set; + queue_array[i].priv = old_array[i].priv; } queues->nr_queues = nr_queues; queues->queue_array = queue_array; + free(old_array); return 0; } @@ -668,10 +673,10 @@ int auxtrace_parse_snapshot_options(struct auxtrace_record *itr, static int evlist__enable_event_idx(struct evlist *evlist, struct evsel *evsel, int idx) { - bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus); + bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist__core(evlist)->user_requested_cpus); if (per_cpu_mmaps) { - struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx); + struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, idx); int cpu_map_idx = perf_cpu_map__idx(evsel->core.cpus, evlist_cpu); if (cpu_map_idx == -1) @@ -1838,7 +1843,7 @@ void perf_session__auxtrace_error_inc(struct perf_session *session, struct perf_record_auxtrace_error *e = &event->auxtrace_error; if (e->type < PERF_AUXTRACE_ERROR_MAX) - session->evlist->stats.nr_auxtrace_errors[e->type] += 1; + evlist__stats(session->evlist)->nr_auxtrace_errors[e->type] += 1; } void events_stats__auxtrace_error_warn(const struct events_stats *stats) diff --git a/tools/perf/util/block-info.c b/tools/perf/util/block-info.c index 8d3a9a661f267..1135e54f4c7fc 100644 --- a/tools/perf/util/block-info.c +++ b/tools/perf/util/block-info.c @@ -472,7 +472,7 @@ struct block_report *block_info__create_report(struct evlist *evlist, int *nr_reps) { struct block_report *block_reports; - int nr_hists = evlist->core.nr_entries, i = 0; + int nr_hists = evlist__nr_entries(evlist), i = 0; struct evsel *pos; block_reports = calloc(nr_hists, sizeof(struct block_report)); @@ -483,7 +483,7 @@ struct block_report *block_info__create_report(struct evlist *evlist, struct hists *hists = evsel__hists(pos); process_block_report(hists, &block_reports[i], total_cycles, - block_hpps, nr_hpps, evlist->nr_br_cntr); + block_hpps, nr_hpps, evlist__nr_br_cntr(evlist)); i++; } diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c index fa3ebc8ea7f09..e67f28a8e92bd 100644 --- a/tools/perf/util/bpf-event.c +++ b/tools/perf/util/bpf-event.c @@ -969,7 +969,8 @@ void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, { struct bpf_prog_info *info = &info_linear->info; __u64 required_arrays = (1UL << PERF_BPIL_JITED_KSYMS) | - (1UL << PERF_BPIL_JITED_FUNC_LENS); + (1UL << PERF_BPIL_JITED_FUNC_LENS) | + (1UL << PERF_BPIL_PROG_TAGS); __u32 *prog_lens; __u64 *prog_addrs; char name[KSYM_NAME_LEN]; diff --git a/tools/perf/util/bpf-filter.c b/tools/perf/util/bpf-filter.c index 1a2e7b388d57d..bcd81084e3420 100644 --- a/tools/perf/util/bpf-filter.c +++ b/tools/perf/util/bpf-filter.c @@ -629,24 +629,20 @@ struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(enum perf_bpf_filter_term static bool check_bpf_filter_capable(void) { - bool used_root; + int fd; - if (perf_cap__capable(CAP_BPF, &used_root)) + if (perf_cap__capable(CAP_BPF)) return true; - if (!used_root) { - /* Check if root already pinned the filter programs and maps */ - int fd = get_pinned_fd("filters"); - - if (fd >= 0) { - close(fd); - return true; - } + /* Check if root already pinned the filter programs and maps */ + fd = get_pinned_fd("filters"); + if (fd >= 0) { + close(fd); + return true; } - pr_err("Error: BPF filter only works for %s!\n" - "\tPlease run 'perf record --setup-filter pin' as root first.\n", - used_root ? "root" : "users with the CAP_BPF capability"); + pr_err("Error: BPF filter only works for users with the CAP_BPF capability!\n" + "\tPlease run 'perf record --setup-filter pin' as root first.\n"); return false; } diff --git a/tools/perf/util/bpf_counter.c b/tools/perf/util/bpf_counter.c index 34b6b0da18b73..9362e45e17ceb 100644 --- a/tools/perf/util/bpf_counter.c +++ b/tools/perf/util/bpf_counter.c @@ -443,7 +443,7 @@ static int bperf_check_target(struct evsel *evsel, } else if (target->tid) { *filter_type = BPERF_FILTER_PID; *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); - } else if (target->pid || evsel->evlist->workload.pid != -1) { + } else if (target->pid || evlist__workload_pid(evsel->evlist) != -1) { *filter_type = BPERF_FILTER_TGID; *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); } else { diff --git a/tools/perf/util/bpf_counter_cgroup.c b/tools/perf/util/bpf_counter_cgroup.c index e1ce5aa3b9578..4e5f4b9dd4428 100644 --- a/tools/perf/util/bpf_counter_cgroup.c +++ b/tools/perf/util/bpf_counter_cgroup.c @@ -104,7 +104,7 @@ static int bperf_load_program(struct evlist *evlist) set_max_rlimit(); - if (nr_cgroups == 0 || evlist->core.nr_entries % nr_cgroups != 0) { + if (nr_cgroups == 0 || evlist__nr_entries(evlist) % nr_cgroups != 0) { pr_err("Invalid cgroup or event count\n"); return -EINVAL; } @@ -116,7 +116,7 @@ static int bperf_load_program(struct evlist *evlist) pr_err("Failed to open cgroup skeleton\n"); return -1; } - setup_rodata(skel, evlist->core.nr_entries); + setup_rodata(skel, evlist__nr_entries(evlist)); err = bperf_cgroup_bpf__load(skel); if (err) { @@ -127,12 +127,12 @@ static int bperf_load_program(struct evlist *evlist) err = -1; cgrp_switch = evsel__new(&cgrp_switch_attr); - if (evsel__open_per_cpu(cgrp_switch, evlist->core.all_cpus, -1) < 0) { + if (evsel__open_per_cpu(cgrp_switch, evlist__core(evlist)->all_cpus, -1) < 0) { pr_err("Failed to open cgroup switches event\n"); goto out; } - perf_cpu_map__for_each_cpu(cpu, i, evlist->core.all_cpus) { + perf_cpu_map__for_each_cpu(cpu, i, evlist__core(evlist)->all_cpus) { link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch, FD(cgrp_switch, i)); if (IS_ERR(link)) { @@ -197,7 +197,7 @@ static int bperf_load_program(struct evlist *evlist) */ { struct evsel *leader; - int num_events = evlist->core.nr_entries / nr_cgroups; + int num_events = evlist__nr_entries(evlist) / nr_cgroups; evlist__for_each_entry(evlist, evsel) { leader = evlist__find_evsel(evlist, evsel->core.idx % num_events); @@ -258,7 +258,7 @@ static int bperf_cgrp__sync_counters(struct evlist *evlist) unsigned int idx; int prog_fd = bpf_program__fd(skel->progs.trigger_read); - perf_cpu_map__for_each_cpu(cpu, idx, evlist->core.all_cpus) + perf_cpu_map__for_each_cpu(cpu, idx, evlist__core(evlist)->all_cpus) bperf_trigger_reading(prog_fd, cpu.cpu); return 0; @@ -336,7 +336,7 @@ static int bperf_cgrp__destroy(struct evsel *evsel) return 0; bperf_cgroup_bpf__destroy(skel); - evsel__delete(cgrp_switch); // it'll destroy on_switch progs too + evsel__put(cgrp_switch); // it'll destroy on_switch progs too return 0; } diff --git a/tools/perf/util/bpf_ftrace.c b/tools/perf/util/bpf_ftrace.c index c456d24efa308..abeafd406e8e0 100644 --- a/tools/perf/util/bpf_ftrace.c +++ b/tools/perf/util/bpf_ftrace.c @@ -59,13 +59,13 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) /* don't need to set cpu filter for system-wide mode */ if (ftrace->target.cpu_list) { - ncpus = perf_cpu_map__nr(ftrace->evlist->core.user_requested_cpus); + ncpus = perf_cpu_map__nr(evlist__core(ftrace->evlist)->user_requested_cpus); bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); skel->rodata->has_cpu = 1; } if (target__has_task(&ftrace->target) || target__none(&ftrace->target)) { - ntasks = perf_thread_map__nr(ftrace->evlist->core.threads); + ntasks = perf_thread_map__nr(evlist__core(ftrace->evlist)->threads); bpf_map__set_max_entries(skel->maps.task_filter, ntasks); skel->rodata->has_task = 1; } @@ -87,7 +87,8 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) fd = bpf_map__fd(skel->maps.cpu_filter); for (i = 0; i < ncpus; i++) { - cpu = perf_cpu_map__cpu(ftrace->evlist->core.user_requested_cpus, i).cpu; + cpu = perf_cpu_map__cpu( + evlist__core(ftrace->evlist)->user_requested_cpus, i).cpu; bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); } } @@ -99,7 +100,7 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) fd = bpf_map__fd(skel->maps.task_filter); for (i = 0; i < ntasks; i++) { - pid = perf_thread_map__pid(ftrace->evlist->core.threads, i); + pid = perf_thread_map__pid(evlist__core(ftrace->evlist)->threads, i); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } } diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c index b1cfa63a488fa..c20bd075664e1 100644 --- a/tools/perf/util/bpf_lock_contention.c +++ b/tools/perf/util/bpf_lock_contention.c @@ -223,11 +223,11 @@ int lock_contention_prepare(struct lock_contention *con) if (target__has_cpu(target)) { skel->rodata->has_cpu = 1; - ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); + ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus); } if (target__has_task(target)) { skel->rodata->has_task = 1; - ntasks = perf_thread_map__nr(evlist->core.threads); + ntasks = perf_thread_map__nr(evlist__core(evlist)->threads); } if (con->filters->nr_types) { skel->rodata->has_type = 1; @@ -334,7 +334,7 @@ int lock_contention_prepare(struct lock_contention *con) fd = bpf_map__fd(skel->maps.cpu_filter); for (i = 0; i < ncpus; i++) { - cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu; + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu; bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); } } @@ -346,13 +346,13 @@ int lock_contention_prepare(struct lock_contention *con) fd = bpf_map__fd(skel->maps.task_filter); for (i = 0; i < ntasks; i++) { - pid = perf_thread_map__pid(evlist->core.threads, i); + pid = perf_thread_map__pid(evlist__core(evlist)->threads, i); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } } - if (target__none(target) && evlist->workload.pid > 0) { - u32 pid = evlist->workload.pid; + if (target__none(target) && evlist__workload_pid(evlist) > 0) { + u32 pid = evlist__workload_pid(evlist); u8 val = 1; fd = bpf_map__fd(skel->maps.task_filter); diff --git a/tools/perf/util/bpf_off_cpu.c b/tools/perf/util/bpf_off_cpu.c index a3b699a5322f1..c4639f6a57766 100644 --- a/tools/perf/util/bpf_off_cpu.c +++ b/tools/perf/util/bpf_off_cpu.c @@ -1,23 +1,25 @@ // SPDX-License-Identifier: GPL-2.0 -#include "util/bpf_counter.h" -#include "util/debug.h" -#include "util/evsel.h" -#include "util/evlist.h" -#include "util/off_cpu.h" -#include "util/perf-hooks.h" -#include "util/record.h" -#include "util/session.h" -#include "util/target.h" -#include "util/cpumap.h" -#include "util/thread_map.h" -#include "util/cgroup.h" -#include "util/strlist.h" +#include + #include #include #include -#include +#include "bpf_counter.h" #include "bpf_skel/off_cpu.skel.h" +#include "cgroup.h" +#include "cpumap.h" +#include "debug.h" +#include "evlist.h" +#include "evsel.h" +#include "off_cpu.h" +#include "parse-events.h" +#include "perf-hooks.h" +#include "record.h" +#include "session.h" +#include "strlist.h" +#include "target.h" +#include "thread_map.h" #define MAX_STACKS 32 #define MAX_PROC 4096 @@ -71,13 +73,13 @@ static void off_cpu_start(void *arg) /* update task filter for the given workload */ if (skel->rodata->has_task && skel->rodata->uses_tgid && - perf_thread_map__pid(evlist->core.threads, 0) != -1) { + perf_thread_map__pid(evlist__core(evlist)->threads, 0) != -1) { int fd; u32 pid; u8 val = 1; fd = bpf_map__fd(skel->maps.task_filter); - pid = perf_thread_map__pid(evlist->core.threads, 0); + pid = perf_thread_map__pid(evlist__core(evlist)->threads, 0); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } @@ -166,7 +168,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, /* don't need to set cpu filter for system-wide mode */ if (target->cpu_list) { - ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); + ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus); bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); skel->rodata->has_cpu = 1; } @@ -197,7 +199,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, skel->rodata->has_task = 1; skel->rodata->uses_tgid = 1; } else if (target__has_task(target)) { - ntasks = perf_thread_map__nr(evlist->core.threads); + ntasks = perf_thread_map__nr(evlist__core(evlist)->threads); bpf_map__set_max_entries(skel->maps.task_filter, ntasks); skel->rodata->has_task = 1; } else if (target__none(target)) { @@ -207,7 +209,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, } if (evlist__first(evlist)->cgrp) { - ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */ + ncgrps = evlist__nr_entries(evlist) - 1; /* excluding a dummy */ bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps); if (!cgroup_is_v2("perf_event")) @@ -238,7 +240,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, fd = bpf_map__fd(skel->maps.cpu_filter); for (i = 0; i < ncpus; i++) { - cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu; + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu; bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); } } @@ -267,7 +269,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, fd = bpf_map__fd(skel->maps.task_filter); for (i = 0; i < ntasks; i++) { - pid = perf_thread_map__pid(evlist->core.threads, i); + pid = perf_thread_map__pid(evlist__core(evlist)->threads, i); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } } diff --git a/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c b/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c index 2a6e61864ee01..3bc9e28a9b8a3 100644 --- a/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c +++ b/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c @@ -429,15 +429,88 @@ static bool pid_filter__has(struct pids_filtered *pids, pid_t pid) return bpf_map_lookup_elem(pids, &pid) != NULL; } +u64 ZERO = 0; + +/* + * Determine what type of argument and how many bytes to read from user space, using the + * value in the beauty_map. This is the relation of parameter type and its corresponding + * value in the beauty map, and how many bytes we read eventually: + * + * string: 1 -> size of string + * struct: size of struct -> size of struct + * buffer: -1 * (index of paired len) -> value of paired len (maximum: TRACE_AUG_MAX_BUF) + */ +static inline int augment_arg(struct syscall_enter_args *args, int i, + unsigned int *beauty_map, + struct beauty_payload_enter *payload, u64 offset) +{ + int index, value_size = sizeof(struct augmented_arg) - offsetof(struct augmented_arg, value); + struct augmented_arg *payload_offset; + s64 aug_size, size; + bool augmented; + void *arg; + + arg = (void *)args->args[i]; + augmented = false; + size = beauty_map[i]; + aug_size = size; /* size of the augmented data read from user space */ + + if (size == 0 || arg == NULL) + return 0; + + /* bounds check for the verifier */ + if (offset > sizeof(payload->aug_args) - sizeof(payload->aug_args[0])) + return -1; + barrier_var(offset); + payload_offset = (struct augmented_arg *)((void *)&payload->aug_args + offset); + + if (size == 1) { /* string */ + aug_size = bpf_probe_read_user_str(payload_offset->value, value_size, arg); + /* minimum of 0 to pass the verifier */ + if (aug_size < 0) + aug_size = 0; + + augmented = true; + } else if (size > 0 && size <= value_size) { /* struct */ + if (!bpf_probe_read_user(payload_offset->value, size, arg)) + augmented = true; + } else if ((int)size < 0 && size >= -6) { /* buffer */ + index = -(size + 1); + barrier_var(index); // Prevent clang (noticed with v18) from removing the &= 7 trick. + index &= 7; // Satisfy the bounds checking with the verifier in some kernels. + aug_size = args->args[index] > TRACE_AUG_MAX_BUF ? TRACE_AUG_MAX_BUF : args->args[index]; + + if (aug_size > 0) { + if (!bpf_probe_read_user(payload_offset->value, aug_size, arg)) + augmented = true; + } + } + + /* Augmented data size is limited to sizeof(augmented_arg->unnamed union with value field) */ + if (aug_size > value_size) + aug_size = value_size; + + /* write data to payload */ + if (augmented) { + int written = offsetof(struct augmented_arg, value) + aug_size; + + if (written < 0 || written > sizeof(struct augmented_arg)) + return -1; + + payload_offset->size = aug_size; + return written; + } + + return 0; +} + static int augment_sys_enter(void *ctx, struct syscall_enter_args *args) { - bool augmented, do_output = false; - int zero = 0, index, value_size = sizeof(struct augmented_arg) - offsetof(struct augmented_arg, value); + bool do_output = false; + int i, zero = 0, written; u64 output = 0; /* has to be u64, otherwise it won't pass the verifier */ - s64 aug_size, size; unsigned int nr, *beauty_map; struct beauty_payload_enter *payload; - void *arg, *payload_offset; /* fall back to do predefined tail call */ if (args == NULL) @@ -449,7 +522,6 @@ static int augment_sys_enter(void *ctx, struct syscall_enter_args *args) /* set up payload for output */ payload = bpf_map_lookup_elem(&beauty_payload_enter_map, &zero); - payload_offset = (void *)&payload->aug_args; if (beauty_map == NULL || payload == NULL) return 1; @@ -457,61 +529,30 @@ static int augment_sys_enter(void *ctx, struct syscall_enter_args *args) /* copy the sys_enter header, which has the syscall_nr */ __builtin_memcpy(&payload->args, args, sizeof(struct syscall_enter_args)); - /* - * Determine what type of argument and how many bytes to read from user space, using the - * value in the beauty_map. This is the relation of parameter type and its corresponding - * value in the beauty map, and how many bytes we read eventually: - * - * string: 1 -> size of string - * struct: size of struct -> size of struct - * buffer: -1 * (index of paired len) -> value of paired len (maximum: TRACE_AUG_MAX_BUF) - */ - for (int i = 0; i < 6; i++) { - arg = (void *)args->args[i]; - augmented = false; - size = beauty_map[i]; - aug_size = size; /* size of the augmented data read from user space */ - - if (size == 0 || arg == NULL) - continue; - - if (size == 1) { /* string */ - aug_size = bpf_probe_read_user_str(((struct augmented_arg *)payload_offset)->value, value_size, arg); - /* minimum of 0 to pass the verifier */ - if (aug_size < 0) - aug_size = 0; - - augmented = true; - } else if (size > 0 && size <= value_size) { /* struct */ - if (!bpf_probe_read_user(((struct augmented_arg *)payload_offset)->value, size, arg)) - augmented = true; - } else if ((int)size < 0 && size >= -6) { /* buffer */ - index = -(size + 1); - barrier_var(index); // Prevent clang (noticed with v18) from removing the &= 7 trick. - index &= 7; // Satisfy the bounds checking with the verifier in some kernels. - aug_size = args->args[index] > TRACE_AUG_MAX_BUF ? TRACE_AUG_MAX_BUF : args->args[index]; - - if (aug_size > 0) { - if (!bpf_probe_read_user(((struct augmented_arg *)payload_offset)->value, aug_size, arg)) - augmented = true; + if (bpf_ksym_exists(bpf_iter_num_new)) { + bpf_for(i, 0, 6) { + written = augment_arg(args, i, beauty_map, payload, output); + if (written < 0) + return 1; + if (written > 0) { + output += written; + /* + * guide the verifier to forget range of `output`, which + * helps to prove convergence of the loop + */ + output += ZERO; + do_output = true; } } - - /* Augmented data size is limited to sizeof(augmented_arg->unnamed union with value field) */ - if (aug_size > value_size) - aug_size = value_size; - - /* write data to payload */ - if (augmented) { - int written = offsetof(struct augmented_arg, value) + aug_size; - - if (written < 0 || written > sizeof(struct augmented_arg)) + } else { + for (i = 0; i < 6; i++) { + written = augment_arg(args, i, beauty_map, payload, output); + if (written < 0) return 1; - - ((struct augmented_arg *)payload_offset)->size = aug_size; - output += written; - payload_offset += written; - do_output = true; + if (written > 0) { + output += written; + do_output = true; + } } } diff --git a/tools/perf/util/bpf_trace_augment.c b/tools/perf/util/bpf_trace_augment.c index 9e706f0fa53d4..a9cf2a77ded17 100644 --- a/tools/perf/util/bpf_trace_augment.c +++ b/tools/perf/util/bpf_trace_augment.c @@ -1,11 +1,11 @@ #include #include -#include "util/debug.h" -#include "util/evlist.h" -#include "util/trace_augment.h" - #include "bpf_skel/augmented_raw_syscalls.skel.h" +#include "debug.h" +#include "evlist.h" +#include "parse-events.h" +#include "trace_augment.h" static struct augmented_raw_syscalls_bpf *skel; static struct evsel *bpf_output; diff --git a/tools/perf/util/cap.c b/tools/perf/util/cap.c index ac6d1d9a523d9..272bd8255ff12 100644 --- a/tools/perf/util/cap.c +++ b/tools/perf/util/cap.c @@ -12,7 +12,7 @@ #define MAX_LINUX_CAPABILITY_U32S _LINUX_CAPABILITY_U32S_3 -bool perf_cap__capable(int cap, bool *used_root) +bool perf_cap__capable(int cap) { struct __user_cap_header_struct header = { .version = _LINUX_CAPABILITY_VERSION_3, @@ -21,7 +21,6 @@ bool perf_cap__capable(int cap, bool *used_root) struct __user_cap_data_struct data[MAX_LINUX_CAPABILITY_U32S] = {}; __u32 cap_val; - *used_root = false; while (syscall(SYS_capget, &header, &data[0]) == -1) { /* Retry, first attempt has set the header.version correctly. */ if (errno == EINVAL && header.version != _LINUX_CAPABILITY_VERSION_3 && @@ -29,7 +28,6 @@ bool perf_cap__capable(int cap, bool *used_root) continue; pr_debug2("capget syscall failed (%m) fall back on root check\n"); - *used_root = true; return geteuid() == 0; } diff --git a/tools/perf/util/cap.h b/tools/perf/util/cap.h index c1b8ac033ccc5..bf09fb20c7793 100644 --- a/tools/perf/util/cap.h +++ b/tools/perf/util/cap.h @@ -18,7 +18,6 @@ #define CAP_BPF 39 #endif -/* Query if a capability is supported, used_root is set if the fallback root check was used. */ -bool perf_cap__capable(int cap, bool *used_root); +bool perf_cap__capable(int cap); #endif /* __PERF_CAP_H */ diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c index 5ad537fea4360..74213daf87862 100644 --- a/tools/perf/util/capstone.c +++ b/tools/perf/util/capstone.c @@ -24,6 +24,13 @@ #include "symbol.h" #include "thread.h" +#if CS_VERSION_MAJOR < 5 +#define CS_ARCH_RISCV 15 +#define CS_MODE_RISCV32 1 +#define CS_MODE_RISCV64 2 +#define CS_MODE_RISCVC 4 +#endif + #ifdef LIBCAPSTONE_DLOPEN static void *perf_cs_dll_handle(void) { @@ -302,6 +309,7 @@ static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len, for (i = 0; i < insn->detail->x86.op_count; i++) { struct cs_x86_op *op = &insn->detail->x86.operands[i]; u64 orig_addr; + struct map *found_map = NULL; if (op->type != X86_OP_MEM) continue; @@ -317,19 +325,22 @@ static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len, if (dso__kernel(map__dso(map))) { /* * The kernel maps can be split into sections, let's - * find the map first and the search the symbol. + * find the map first and then search the symbol. */ - map = maps__find(map__kmaps(map), addr); - if (map == NULL) + found_map = maps__find(map__kmaps(map), addr); + if (found_map == NULL) continue; + map = found_map; } /* convert it to map-relative address for search */ addr = map__map_ip(map, addr); sym = map__find_symbol(map, addr); - if (sym == NULL) + if (sym == NULL) { + map__put(found_map); continue; + } if (addr == sym->start) { scnprintf(buf, len, "\t# %"PRIx64" <%s>", @@ -338,6 +349,7 @@ static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len, scnprintf(buf, len, "\t# %"PRIx64" <%s+%#"PRIx64">", orig_addr, sym->name, addr - sym->start); } + map__put(found_map); break; } } @@ -380,7 +392,7 @@ int symbol__disassemble_capstone(const char *filename, struct symbol *sym, char disasm_buf[512]; struct disasm_line *dl; bool disassembler_style = false; - uint16_t e_machine; + uint16_t e_machine = EM_NONE; bool is_big_endian = false; if (args->options->objdump_path) @@ -411,9 +423,22 @@ int symbol__disassemble_capstone(const char *filename, struct symbol *sym, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - e_machine = thread__e_machine_endian(args->ms->thread, - /*machine=*/NULL, - /*e_flags=*/NULL, &is_big_endian); + if (args->ms->thread) { + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } else if (dso) { + struct maps *kmaps = (map && dso__kernel(dso)) ? map__kmaps(map) : NULL; + struct machine *kmap_machine = kmaps ? maps__machine(kmaps) : NULL; + + e_machine = dso__e_machine_endian(dso, kmap_machine, /*e_flags=*/NULL, + &is_big_endian); + } + if (!e_machine || e_machine == EM_NONE) { + e_machine = thread__e_machine_endian(NULL, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) goto err; @@ -506,7 +531,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, struct disasm_line *dl; u32 *line; bool disassembler_style = false; - uint16_t e_machine; + uint16_t e_machine = EM_NONE; bool is_big_endian = false; if (args->options->objdump_path) @@ -526,9 +551,22 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - e_machine = thread__e_machine_endian(args->ms->thread, - /*machine=*/NULL, - /*e_flags=*/NULL, &is_big_endian); + if (args->ms->thread) { + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } else if (dso) { + struct maps *kmaps = (map && dso__kernel(dso)) ? map__kmaps(map) : NULL; + struct machine *kmap_machine = kmaps ? maps__machine(kmaps) : NULL; + + e_machine = dso__e_machine_endian(dso, kmap_machine, /*e_flags=*/NULL, + &is_big_endian); + } + if (!e_machine || e_machine == EM_NONE) { + e_machine = thread__e_machine_endian(NULL, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) goto err; diff --git a/tools/perf/util/cgroup.c b/tools/perf/util/cgroup.c index 1b5664d1481f5..c7be16a7915e6 100644 --- a/tools/perf/util/cgroup.c +++ b/tools/perf/util/cgroup.c @@ -367,7 +367,7 @@ int parse_cgroups(const struct option *opt, const char *str, char *s; int ret, i; - if (list_empty(&evlist->core.entries)) { + if (list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "must define events before cgroups\n"); return -1; } @@ -423,7 +423,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro int ret = -1; int prefix_len; - if (evlist->core.nr_entries == 0) { + if (evlist__nr_entries(evlist) == 0) { fprintf(stderr, "must define events before cgroups\n"); return -EINVAL; } @@ -436,11 +436,11 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro } /* save original events and init evlist */ - evlist__splice_list_tail(orig_list, &evlist->core.entries); - evlist->core.nr_entries = 0; + evlist__splice_list_tail(orig_list, &evlist__core(evlist)->entries); + evlist__core(evlist)->nr_entries = 0; - orig_metric_events = evlist->metric_events; - metricgroup__rblist_init(&evlist->metric_events); + orig_metric_events = *evlist__metric_events(evlist); + metricgroup__rblist_init(evlist__metric_events(evlist)); if (has_pattern_string(str)) prefix_len = match_cgroups(str); @@ -469,7 +469,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro /* copy the list and set to the new cgroup. */ evlist__for_each_entry(orig_list, pos) { - struct evsel *evsel = evsel__clone(/*dest=*/NULL, pos); + struct evsel *evsel = evsel__clone(pos); if (evsel == NULL) goto out_err; @@ -503,15 +503,15 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro nr_cgroups++; if (metricgroup__copy_metric_events(tmp_list, cgrp, - &evlist->metric_events, + evlist__metric_events(evlist), &orig_metric_events) < 0) goto out_err; - evlist__splice_list_tail(evlist, &tmp_list->core.entries); - tmp_list->core.nr_entries = 0; + evlist__splice_list_tail(evlist, &evlist__core(tmp_list)->entries); + evlist__core(tmp_list)->nr_entries = 0; } - if (list_empty(&evlist->core.entries)) { + if (list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "no cgroup matched: %s\n", str); goto out_err; } @@ -520,8 +520,8 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro cgrp_event_expanded = true; out_err: - evlist__delete(orig_list); - evlist__delete(tmp_list); + evlist__put(orig_list); + evlist__put(tmp_list); metricgroup__rblist_exit(&orig_metric_events); release_cgroup_list(); diff --git a/tools/perf/util/compress.h b/tools/perf/util/compress.h index 6cfecfca16f24..ec6c38129e248 100644 --- a/tools/perf/util/compress.h +++ b/tools/perf/util/compress.h @@ -54,7 +54,8 @@ int zstd_fini(struct zstd_data *data); ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_t dst_size, void *src, size_t src_size, size_t max_record_size, - size_t process_header(void *record, size_t increment)); + ssize_t process_header(void *record, size_t dst_size, + size_t data_size)); size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size, void *dst, size_t dst_size); @@ -75,7 +76,8 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data __maybe_unused, void *dst __maybe_unused, size_t dst_size __maybe_unused, void *src __maybe_unused, size_t src_size __maybe_unused, size_t max_record_size __maybe_unused, - size_t process_header(void *record, size_t increment) __maybe_unused) + ssize_t process_header(void *record, size_t dst_size, + size_t data_size) __maybe_unused) { return 0; } diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 5d0664ff73b79..fc3d4ebd1fbae 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -71,6 +71,7 @@ struct cs_etm_auxtrace { int num_cpu; u64 latest_kernel_timestamp; u32 auxtrace_type; + u32 branches_filter; u64 branches_sample_type; u64 branches_id; u64 instructions_sample_type; @@ -645,6 +646,8 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, queue->tid); tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1, queue->tid); + if (!tidq->frontend_thread || !tidq->decode_thread) + goto out; tidq->packet = zalloc(sizeof(struct cs_etm_packet)); if (!tidq->packet) @@ -679,6 +682,8 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, zfree(&tidq->prev_packet); zfree(&tidq->packet); out: + thread__zput(tidq->frontend_thread); + thread__zput(tidq->decode_thread); return rc; } @@ -1523,8 +1528,7 @@ cs_etm__get_trace(struct cs_etm_queue *etmq) etmq->buf_used = 0; etmq->buf_len = aux_buffer->size; etmq->buf = aux_buffer->data; - - return etmq->buf_len; + return 0; } /* @@ -1691,8 +1695,9 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, { int ret = 0; struct cs_etm_auxtrace *etm = etmq->etm; - struct perf_sample sample = {.ip = 0,}; + struct perf_sample sample; union perf_event *event = tidq->event_buf; + struct dummy_branch_stack { u64 nr; u64 hw_idx; @@ -1700,6 +1705,11 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, } dummy_bs; u64 ip; + if (etm->branches_filter && + !(etm->branches_filter & tidq->prev_packet->flags)) + return 0; + + perf_sample__init(&sample, /*all=*/true); ip = cs_etm__last_executed_instr(tidq->prev_packet); event->sample.header.type = PERF_RECORD_SAMPLE; @@ -1752,6 +1762,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, "CS ETM Trace: failed to deliver instruction event, error %d\n", ret); + perf_sample__exit(&sample); return ret; } @@ -2077,6 +2088,45 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq, return 0; } + +static int cs_etm__flush_stack_cb(struct thread *thread, + void *data __maybe_unused) +{ + thread_stack__flush(thread); + return 0; +} + +static void cs_etm__flush_machine_stack(struct cs_etm_queue *etmq, pid_t pid) +{ + struct machine *machine; + + machine = machines__find(&etmq->etm->session->machines, pid); + if (machine) + machine__for_each_thread(machine, cs_etm__flush_stack_cb, NULL); +} + +static void cs_etm__flush_all_stack(struct cs_etm_queue *etmq) +{ + enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq); + + if (!etmq->etm->synth_opts.last_branch) + return; + + switch (pid_fmt) { + case CS_ETM_PIDFMT_CTXTID2: + /* Clear the guest stack if virtualization is supported */ + cs_etm__flush_machine_stack(etmq, DEFAULT_GUEST_KERNEL_ID); + fallthrough; + case CS_ETM_PIDFMT_CTXTID: + cs_etm__flush_machine_stack(etmq, HOST_KERNEL_ID); + break; + case CS_ETM_PIDFMT_NONE: + default: + break; + + } +} + /* * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue * if need be. @@ -2088,20 +2138,33 @@ static int cs_etm__get_data_block(struct cs_etm_queue *etmq) { int ret; - if (!etmq->buf_len) { - ret = cs_etm__get_trace(etmq); - if (ret <= 0) - return ret; - /* - * We cannot assume consecutive blocks in the data file - * are contiguous, reset the decoder to force re-sync. - */ - ret = cs_etm_decoder__reset(etmq->decoder); - if (ret) - return ret; - } + /* The current block is not finished */ + if (etmq->buf_len) + return 1; + + ret = cs_etm__get_trace(etmq); + if (ret < 0) + return ret; - return etmq->buf_len; + /* No more buffer to read */ + if (!etmq->buf_len) + return 0; + + /* + * We cannot assume consecutive blocks in the data file + * are contiguous, reset the decoder to force re-sync. + */ + ret = cs_etm_decoder__reset(etmq->decoder); + if (ret) + return ret; + + /* + * Since the decoder is reset, this causes a global trace + * discontinuity. Flush all thread stacks. + */ + cs_etm__flush_all_stack(etmq); + + return 1; } static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, @@ -3557,6 +3620,16 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, etm->synth_opts.callchain = false; } + if (etm->synth_opts.calls) + etm->branches_filter |= PERF_IP_FLAG_CALL | + PERF_IP_FLAG_TRACE_BEGIN | + PERF_IP_FLAG_TRACE_END; + + if (etm->synth_opts.returns) + etm->branches_filter |= PERF_IP_FLAG_RETURN | + PERF_IP_FLAG_TRACE_BEGIN | + PERF_IP_FLAG_TRACE_END; + etm->session = session; etm->num_cpu = num_cpu; diff --git a/tools/perf/util/data-convert-bt.c b/tools/perf/util/data-convert-bt.c index 5ff46bfcd0e19..e222371394c01 100644 --- a/tools/perf/util/data-convert-bt.c +++ b/tools/perf/util/data-convert-bt.c @@ -1362,7 +1362,7 @@ static void cleanup_events(struct perf_session *session) zfree(&evsel->priv); } - evlist__delete(evlist); + evlist__put(evlist); session->evlist = NULL; } diff --git a/tools/perf/util/data-convert-json.c b/tools/perf/util/data-convert-json.c index 40412c3dbdb25..40888b7c44671 100644 --- a/tools/perf/util/data-convert-json.c +++ b/tools/perf/util/data-convert-json.c @@ -258,6 +258,7 @@ static int process_sample_event(const struct perf_tool *tool, trace_seq_init(&s); tep_print_field(&s, sample->raw_data, fields[i]); output_json_key_string(out, true, 3, fields[i]->name, s.buffer); + trace_seq_destroy(&s); i++; } diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index 2309196d8df31..42bfe30a3b518 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -395,7 +395,9 @@ int dso__decompress_kmodule_path(struct dso *dso, const char *name, { int fd = decompress_kmodule(dso, name, pathname, len); - close(fd); + /* decompress_kmodule() returns -1 on failure, don't close(-1) */ + if (fd >= 0) + close(fd); return fd >= 0 ? 0 : -1; } @@ -582,9 +584,18 @@ static char *dso__get_filename(struct dso *dso, const char *root_dir, goto out; if (!is_regular_file(name)) { + struct stat st; char *new_name; - if (errno != ENOENT || dso__nsinfo(dso) == NULL) + /* + * errno only reflects the failure reason when stat() itself + * failed: a successful stat() on a non-regular file (e.g. a + * directory) leaves a stale errno, which a previous failed + * iteration of the try_to_open_dso() fallback loop may have + * set to ENOENT. + */ + if (stat(name, &st) == 0 || errno != ENOENT || + dso__nsinfo(dso) == NULL) goto out; new_name = dso__filename_with_chroot(dso, name); @@ -640,10 +651,13 @@ static int __open_dso(struct dso *dso, struct machine *machine) mutex_lock(dso__lock(dso)); name = dso__get_filename(dso, machine ? machine->root_dir : "", &decomp); - if (name) + if (name) { fd = do_open(name); - else + } else { + if (errno == 0) + errno = ENOENT; fd = -errno; + } if (decomp) unlink(name); @@ -1000,7 +1014,20 @@ static ssize_t dso_cache__memcpy(struct dso_cache *cache, u64 offset, u8 *data, u64 size, bool out) { u64 cache_offset = offset - cache->offset; - u64 cache_size = min(cache->size - cache_offset, size); + u64 cache_size; + + /* + * The RB tree matches using DSO__DATA_CACHE_SIZE, but a short + * pread may leave cache->size smaller. For a regular file a + * short pread only happens at end-of-file, so an offset past + * the valid data is EOF: return 0, matching what a direct + * pread() at that offset would return, and cached_io() then + * stops its read loop. + */ + if (cache_offset >= cache->size) + return 0; + + cache_size = min(cache->size - cache_offset, size); if (out) memcpy(data, cache->data + cache_offset, cache_size); @@ -1024,7 +1051,7 @@ static ssize_t file_read(struct dso *dso, struct machine *machine, if (dso__data(dso)->fd < 0) { dso__data(dso)->status = DSO_DATA_STATUS_ERROR; - ret = -errno; + ret = dso__data(dso)->fd; goto out; } @@ -1146,8 +1173,8 @@ static int file_size(struct dso *dso, struct machine *machine) try_to_open_dso(dso, machine); if (dso__data(dso)->fd < 0) { - ret = -errno; dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + ret = dso__data(dso)->fd; goto out; } @@ -2011,7 +2038,12 @@ const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; return NULL; } - assert(len <= info_linear->info.jited_prog_len); + if (len > info_linear->info.jited_prog_len) { + pr_debug("BPF symbol length %zu exceeds jited_prog_len %u\n", + len, info_linear->info.jited_prog_len); + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } *out_buf_len = len; return (const u8 *)(uintptr_t)(info_linear->info.jited_prog_insns); #else diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index 2916b954a804b..55c4aaa53c382 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "build-id.h" #include "debuginfo.h" #include "mutex.h" @@ -20,6 +21,40 @@ struct perf_env; #define DSO__NAME_KALLSYMS "[kernel.kallsyms]" #define DSO__NAME_KCORE "[kernel.kcore]" +#define DSO__NAME_GUEST_KALLSYMS "[guest.kernel.kallsyms]" +#define DSO__NAME_GUEST_KALLSYMS_PID_PREFIX "[guest.kernel.kallsyms." + +/* + * Validate names of the form "[guest.kernel.kallsyms.]", where + * is the PID of the guest VM and varies per guest, so it + * cannot be matched with strcmp() against a fixed string. + * + * Every character after the fixed prefix must be a decimal digit, + * with ']' immediately terminating the digit run and nothing + * following it. This rules out '/', "..", or any other character + * being smuggled into the name. + */ +static inline bool is_guest_kallsyms_pid_name(const char *name) +{ + const size_t prefix_len = sizeof(DSO__NAME_GUEST_KALLSYMS_PID_PREFIX) - 1; + size_t digits; + + if (strncmp(name, DSO__NAME_GUEST_KALLSYMS_PID_PREFIX, prefix_len) != 0) + return false; + + digits = strspn(name + prefix_len, "0123456789"); + if (digits == 0) + return false; + + /* ']' must terminate the digit run, with nothing trailing it */ + if (name[prefix_len + digits] != ']') + return false; + + if (name[prefix_len + digits + 1] != '\0') + return false; + + return true; +} /** * enum dso_binary_type - The kind of DSO generally associated with a memory @@ -924,8 +959,28 @@ static inline bool dso__is_kcore(const struct dso *dso) static inline bool dso__is_kallsyms(const struct dso *dso) { enum dso_binary_type bt = dso__binary_type(dso); + const char *name; + + if (bt == DSO_BINARY_TYPE__KALLSYMS || bt == DSO_BINARY_TYPE__GUEST_KALLSYMS) + return true; + + if (bt != DSO_BINARY_TYPE__NOT_FOUND) + return false; + + if (!dso__kernel(dso)) + return false; + + name = dso__long_name(dso); + if (!name) + return false; + + if (!strcmp(name, DSO__NAME_KALLSYMS)) + return true; + + if (!strcmp(name, DSO__NAME_GUEST_KALLSYMS)) + return true; - return bt == DSO_BINARY_TYPE__KALLSYMS || bt == DSO_BINARY_TYPE__GUEST_KALLSYMS; + return is_guest_kallsyms_pid_name(name); } bool dso__is_object_file(const struct dso *dso); diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c index 1a238b245b3a0..eb7c0d7be064e 100644 --- a/tools/perf/util/evlist.c +++ b/tools/perf/util/evlist.c @@ -5,67 +5,69 @@ * Parts came from builtin-{top,stat,record}.c, see those files for further * copyright notes. */ -#include +#include "evlist.h" + #include #include -#include -#include "cpumap.h" -#include "util/mmap.h" -#include "thread_map.h" -#include "target.h" -#include "dwarf-regs.h" -#include "evlist.h" -#include "evsel.h" -#include "record.h" -#include "debug.h" -#include "units.h" -#include "bpf_counter.h" -#include // page_size -#include "affinity.h" -#include "../perf.h" -#include "asm/bug.h" -#include "bpf-event.h" -#include "util/event.h" -#include "util/string2.h" -#include "util/perf_api_probe.h" -#include "util/evsel_fprintf.h" -#include "util/pmu.h" -#include "util/sample.h" -#include "util/bpf-filter.h" -#include "util/stat.h" -#include "util/util.h" -#include "util/env.h" -#include "util/intel-tpebs.h" -#include "util/metricgroup.h" -#include "util/strbuf.h" #include -#include -#include #include -#include "parse-events.h" -#include - #include -#include -#include -#include -#include -#include - #include +#include #include #include -#include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include // page_size +#include +#include +#include #include #include -#include #include +#include -#include +#include "../perf.h" +#include "affinity.h" +#include "asm/bug.h" +#include "bpf-event.h" +#include "bpf-filter.h" +#include "bpf_counter.h" +#include "cpumap.h" +#include "debug.h" +#include "dwarf-regs.h" +#include "env.h" +#include "event.h" +#include "evsel.h" +#include "evsel_fprintf.h" +#include "intel-tpebs.h" +#include "metricgroup.h" +#include "mmap.h" +#include "parse-events.h" +#include "perf_api_probe.h" +#include "pmu.h" +#include "pmus.h" +#include "record.h" +#include "sample.h" +#include "stat.h" +#include "strbuf.h" +#include "string2.h" +#include "target.h" +#include "thread_map.h" +#include "units.h" +#include "util.h" #ifdef LACKS_SIGQUEUE_PROTOTYPE int sigqueue(pid_t pid, int sig, const union sigval value); @@ -74,29 +76,31 @@ int sigqueue(pid_t pid, int sig, const union sigval value); #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) #define SID(e, x, y) xyarray__entry(e->core.sample_id, x, y) -void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus, - struct perf_thread_map *threads) -{ - perf_evlist__init(&evlist->core); - perf_evlist__set_maps(&evlist->core, cpus, threads); - evlist->workload.pid = -1; - evlist->bkw_mmap_state = BKW_MMAP_NOTREADY; - evlist->ctl_fd.fd = -1; - evlist->ctl_fd.ack = -1; - evlist->ctl_fd.pos = -1; - evlist->nr_br_cntr = -1; - metricgroup__rblist_init(&evlist->metric_events); - INIT_LIST_HEAD(&evlist->deferred_samples); -} +static void event_enable_timer__exit(struct event_enable_timer **ep); struct evlist *evlist__new(void) { - struct evlist *evlist = zalloc(sizeof(*evlist)); - - if (evlist != NULL) - evlist__init(evlist, NULL, NULL); - - return evlist; + struct evlist *result; + RC_STRUCT(evlist) *evlist; + + evlist = zalloc(sizeof(*evlist)); + if (ADD_RC_CHK(result, evlist)) { + perf_evlist__init(evlist__core(result)); + perf_evlist__set_maps(evlist__core(result), /*cpus=*/NULL, /*threads=*/NULL); + evlist__set_workload_pid(result, -1); + evlist__set_bkw_mmap_state(result, BKW_MMAP_NOTREADY); + evlist__set_ctl_fd_fd(result, -1); + evlist__set_ctl_fd_ack(result, -1); + evlist__set_ctl_fd_pos(result, -1); + evlist__set_nr_br_cntr(result, -1); + metricgroup__rblist_init(evlist__metric_events(result)); + INIT_LIST_HEAD(&evlist->deferred_samples); + refcount_set(evlist__refcnt(result), 1); + } else { + free(evlist); + result = NULL; + } + return result; } struct evlist *evlist__new_default(const struct target *target, bool sample_callchains) @@ -104,7 +108,6 @@ struct evlist *evlist__new_default(const struct target *target, bool sample_call struct evlist *evlist = evlist__new(); bool can_profile_kernel; struct perf_pmu *pmu = NULL; - struct evsel *evsel; char buf[256]; int err; @@ -131,14 +134,16 @@ struct evlist *evlist__new_default(const struct target *target, bool sample_call } /* If there is only 1 event a sample identifier isn't necessary. */ - if (evlist->core.nr_entries > 1) { + if (evlist__nr_entries(evlist) > 1) { + struct evsel *evsel; + evlist__for_each_entry(evlist, evsel) evsel__set_sample_id(evsel, /*can_sample_identifier=*/false); } return evlist; out_err: - evlist__delete(evlist); + evlist__put(evlist); return NULL; } @@ -147,13 +152,23 @@ struct evlist *evlist__new_dummy(void) struct evlist *evlist = evlist__new(); if (evlist && evlist__add_dummy(evlist)) { - evlist__delete(evlist); + evlist__put(evlist); evlist = NULL; } return evlist; } +struct evlist *evlist__get(struct evlist *evlist) +{ + struct evlist *result; + + if (RC_CHK_GET(result, evlist)) + refcount_inc(evlist__refcnt(evlist)); + + return result; +} + /** * evlist__set_id_pos - set the positions of event ids. * @evlist: selected event list @@ -165,8 +180,8 @@ void evlist__set_id_pos(struct evlist *evlist) { struct evsel *first = evlist__first(evlist); - evlist->id_pos = first->id_pos; - evlist->is_pos = first->is_pos; + RC_CHK_ACCESS(evlist)->id_pos = first->id_pos; + RC_CHK_ACCESS(evlist)->is_pos = first->is_pos; } static void evlist__update_id_pos(struct evlist *evlist) @@ -185,49 +200,85 @@ static void evlist__purge(struct evlist *evlist) evlist__for_each_entry_safe(evlist, n, pos) { list_del_init(&pos->core.node); + if (pos->evlist) { + if (!RC_CHK_EQUAL(pos->evlist, evlist)) { + evlist__put(pos->evlist); + } else { + refcount_dec_and_test(evlist__refcnt(pos->evlist)); + RC_CHK_PUT(pos->evlist); + } + } pos->evlist = NULL; - evsel__delete(pos); + evsel__put(pos); } - evlist->core.nr_entries = 0; + evlist__core(evlist)->nr_entries = 0; } -void evlist__exit(struct evlist *evlist) +static void evlist__exit(struct evlist *evlist) { - metricgroup__rblist_exit(&evlist->metric_events); - event_enable_timer__exit(&evlist->eet); - zfree(&evlist->mmap); - zfree(&evlist->overwrite_mmap); - perf_evlist__exit(&evlist->core); + metricgroup__rblist_exit(evlist__metric_events(evlist)); + event_enable_timer__exit(&RC_CHK_ACCESS(evlist)->eet); + free(evlist__mmap(evlist)); + free(evlist__overwrite_mmap(evlist)); + perf_evlist__exit(evlist__core(evlist)); } -void evlist__delete(struct evlist *evlist) +void evlist__put(struct evlist *evlist) { + struct evsel *evsel; + unsigned int count, ref_cnt; + if (evlist == NULL) return; + if (refcount_dec_and_test(evlist__refcnt(evlist))) + goto out_delete; + +retry: + count = refcount_read(evlist__refcnt(evlist)); + ref_cnt = count; + evlist__for_each_entry(evlist, evsel) { + if (RC_CHK_EQUAL(evsel->evlist, evlist) && count && + refcount_read(&evsel->refcnt) == 1) + count--; + } + if (refcount_read(evlist__refcnt(evlist)) != ref_cnt) + goto retry; + + if (count != 0) { + /* + * Not the last reference except for back references from + * evsels. + */ + RC_CHK_PUT(evlist); + return; + } +out_delete: evlist__free_stats(evlist); - evlist__munmap(evlist); + evlist__do_munmap(evlist); evlist__close(evlist); evlist__purge(evlist); evlist__exit(evlist); - free(evlist); + RC_CHK_FREE(evlist); } void evlist__add(struct evlist *evlist, struct evsel *entry) { - perf_evlist__add(&evlist->core, &entry->core); - entry->evlist = evlist; + perf_evlist__add(evlist__core(evlist), &entry->core); + evlist__put(entry->evlist); + entry->evlist = evlist__get(evlist); entry->tracking = !entry->core.idx; - if (evlist->core.nr_entries == 1) + if (evlist__nr_entries(evlist) == 1) evlist__set_id_pos(evlist); } void evlist__remove(struct evlist *evlist, struct evsel *evsel) { + perf_evlist__remove(evlist__core(evlist), &evsel->core); + evlist__put(evsel->evlist); evsel->evlist = NULL; - perf_evlist__remove(&evlist->core, &evsel->core); } void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list) @@ -276,7 +327,7 @@ int __evlist__set_tracepoints_handlers(struct evlist *evlist, static void evlist__set_leader(struct evlist *evlist) { - perf_evlist__set_leader(&evlist->core); + perf_evlist__set_leader(evlist__core(evlist)); } static struct evsel *evlist__dummy_event(struct evlist *evlist) @@ -290,7 +341,7 @@ static struct evsel *evlist__dummy_event(struct evlist *evlist) .sample_period = 1, }; - return evsel__new_idx(&attr, evlist->core.nr_entries); + return evsel__new_idx(&attr, evlist__nr_entries(evlist)); } int evlist__add_dummy(struct evlist *evlist) @@ -379,8 +430,8 @@ static bool evlist__use_affinity(struct evlist *evlist) struct perf_cpu_map *used_cpus = NULL; bool ret = false; - if (evlist->no_affinity || !evlist->core.user_requested_cpus || - cpu_map__is_dummy(evlist->core.user_requested_cpus)) + if (evlist__no_affinity(evlist) || !evlist__core(evlist)->user_requested_cpus || + cpu_map__is_dummy(evlist__core(evlist)->user_requested_cpus)) return false; evlist__for_each_entry(evlist, pos) { @@ -435,7 +486,7 @@ void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *e .evsel = NULL, .cpu_map_idx = 0, .evlist_cpu_map_idx = 0, - .evlist_cpu_map_nr = perf_cpu_map__nr(evlist->core.all_cpus), + .evlist_cpu_map_nr = perf_cpu_map__nr(evlist__core(evlist)->all_cpus), .cpu = (struct perf_cpu){ .cpu = -1}, .affinity = NULL, }; @@ -451,7 +502,7 @@ void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *e itr->affinity = &itr->saved_affinity; } itr->evsel = evlist__first(evlist); - itr->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, 0); + itr->cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, 0); if (itr->affinity) affinity__set(itr->affinity, itr->cpu.cpu); itr->cpu_map_idx = perf_cpu_map__idx(itr->evsel->core.cpus, itr->cpu); @@ -486,7 +537,7 @@ void evlist_cpu_iterator__next(struct evlist_cpu_iterator *evlist_cpu_itr) if (evlist_cpu_itr->evlist_cpu_map_idx < evlist_cpu_itr->evlist_cpu_map_nr) { evlist_cpu_itr->evsel = evlist__first(evlist_cpu_itr->container); evlist_cpu_itr->cpu = - perf_cpu_map__cpu(evlist_cpu_itr->container->core.all_cpus, + perf_cpu_map__cpu(evlist__core(evlist_cpu_itr->container)->all_cpus, evlist_cpu_itr->evlist_cpu_map_idx); if (evlist_cpu_itr->affinity) affinity__set(evlist_cpu_itr->affinity, evlist_cpu_itr->cpu.cpu); @@ -513,7 +564,7 @@ static int evsel__strcmp(struct evsel *pos, char *evsel_name) return !evsel__name_is(pos, evsel_name); } -static int evlist__is_enabled(struct evlist *evlist) +static bool evlist__is_enabled(struct evlist *evlist) { struct evsel *pos; @@ -570,10 +621,7 @@ static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl * If we disabled only single event, we need to check * the enabled state of the evlist manually. */ - if (evsel_name) - evlist->enabled = evlist__is_enabled(evlist); - else - evlist->enabled = false; + evlist__set_enabled(evlist, evsel_name ? evlist__is_enabled(evlist) : false); } void evlist__disable(struct evlist *evlist) @@ -624,7 +672,7 @@ static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_ * so the toggle can work properly and toggle to * 'disabled' state. */ - evlist->enabled = true; + evlist__set_enabled(evlist, true); } void evlist__enable(struct evlist *evlist) @@ -644,23 +692,24 @@ void evlist__enable_evsel(struct evlist *evlist, char *evsel_name) void evlist__toggle_enable(struct evlist *evlist) { - (evlist->enabled ? evlist__disable : evlist__enable)(evlist); + (evlist__enabled(evlist) ? evlist__disable : evlist__enable)(evlist); } int evlist__add_pollfd(struct evlist *evlist, int fd) { - return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, fdarray_flag__default); + return perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, + fdarray_flag__default); } int evlist__filter_pollfd(struct evlist *evlist, short revents_and_mask) { - return perf_evlist__filter_pollfd(&evlist->core, revents_and_mask); + return perf_evlist__filter_pollfd(evlist__core(evlist), revents_and_mask); } #ifdef HAVE_EVENTFD_SUPPORT int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd) { - return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, + return perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, fdarray_flag__nonfilterable | fdarray_flag__non_perf_event); } @@ -668,7 +717,7 @@ int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd) int evlist__poll(struct evlist *evlist, int timeout) { - return perf_evlist__poll(&evlist->core, timeout); + return perf_evlist__poll(evlist__core(evlist), timeout); } struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id) @@ -678,7 +727,7 @@ struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id) int hash; hash = hash_64(id, PERF_EVLIST__HLIST_BITS); - head = &evlist->core.heads[hash]; + head = &evlist__core(evlist)->heads[hash]; hlist_for_each_entry(sid, head, node) if (sid->id == id) @@ -691,7 +740,7 @@ struct evsel *evlist__id2evsel(struct evlist *evlist, u64 id) { struct perf_sample_id *sid; - if (evlist->core.nr_entries == 1 || !id) + if (evlist__nr_entries(evlist) == 1 || !id) return evlist__first(evlist); sid = evlist__id2sid(evlist, id); @@ -726,13 +775,13 @@ static int evlist__event2id(struct evlist *evlist, union perf_event *event, u64 n = (event->header.size - sizeof(event->header)) >> 3; if (event->header.type == PERF_RECORD_SAMPLE) { - if (evlist->id_pos >= n) + if (evlist__id_pos(evlist) >= n) return -1; - *id = array[evlist->id_pos]; + *id = array[evlist__id_pos(evlist)]; } else { - if (evlist->is_pos > n) + if (evlist__is_pos(evlist) > n) return -1; - n -= evlist->is_pos; + n -= evlist__is_pos(evlist); *id = array[n]; } return 0; @@ -746,7 +795,7 @@ struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event int hash; u64 id; - if (evlist->core.nr_entries == 1) + if (evlist__nr_entries(evlist) == 1) return first; if (!first->core.attr.sample_id_all && @@ -761,7 +810,7 @@ struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event return first; hash = hash_64(id, PERF_EVLIST__HLIST_BITS); - head = &evlist->core.heads[hash]; + head = &evlist__core(evlist)->heads[hash]; hlist_for_each_entry(sid, head, node) { if (sid->id == id) @@ -774,11 +823,11 @@ static int evlist__set_paused(struct evlist *evlist, bool value) { int i; - if (!evlist->overwrite_mmap) + if (!evlist__overwrite_mmap(evlist)) return 0; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - int fd = evlist->overwrite_mmap[i].core.fd; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + int fd = evlist__overwrite_mmap(evlist)[i].core.fd; int err; if (fd < 0) @@ -804,20 +853,20 @@ static void evlist__munmap_nofree(struct evlist *evlist) { int i; - if (evlist->mmap) - for (i = 0; i < evlist->core.nr_mmaps; i++) - perf_mmap__munmap(&evlist->mmap[i].core); + if (evlist__mmap(evlist)) + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) + perf_mmap__munmap(&evlist__mmap(evlist)[i].core); - if (evlist->overwrite_mmap) - for (i = 0; i < evlist->core.nr_mmaps; i++) - perf_mmap__munmap(&evlist->overwrite_mmap[i].core); + if (evlist__overwrite_mmap(evlist)) + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) + perf_mmap__munmap(&evlist__overwrite_mmap(evlist)[i].core); } -void evlist__munmap(struct evlist *evlist) +void evlist__do_munmap(struct evlist *evlist) { evlist__munmap_nofree(evlist); - zfree(&evlist->mmap); - zfree(&evlist->overwrite_mmap); + zfree(&RC_CHK_ACCESS(evlist)->mmap); + zfree(&RC_CHK_ACCESS(evlist)->overwrite_mmap); } static void perf_mmap__unmap_cb(struct perf_mmap *map) @@ -831,12 +880,12 @@ static struct mmap *evlist__alloc_mmap(struct evlist *evlist, bool overwrite) { int i; - struct mmap *map = calloc(evlist->core.nr_mmaps, sizeof(struct mmap)); + struct mmap *map = calloc(evlist__core(evlist)->nr_mmaps, sizeof(struct mmap)); if (!map) return NULL; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { struct perf_mmap *prev = i ? &map[i - 1].core : NULL; /* @@ -854,41 +903,73 @@ static struct mmap *evlist__alloc_mmap(struct evlist *evlist, return map; } +static struct evlist *from_list_start(struct perf_evlist *core) +{ +#ifdef REFCNT_CHECKING + RC_STRUCT(evlist) *core_evlist = container_of(core, RC_STRUCT(evlist), core); + struct evlist *evlist; + + if (ADD_RC_CHK(evlist, core_evlist)) + refcount_inc(evlist__refcnt(evlist)); + + return evlist; +#else + return container_of(core, struct evlist, core); +#endif +} + +static void from_list_end(struct evlist *evlist __maybe_unused) +{ +#ifdef REFCNT_CHECKING + evlist__put(evlist); +#endif +} + static void perf_evlist__mmap_cb_idx(struct perf_evlist *_evlist, struct perf_evsel *_evsel, struct perf_mmap_param *_mp, int idx) { - struct evlist *evlist = container_of(_evlist, struct evlist, core); + struct evlist *evlist = from_list_start(_evlist); struct mmap_params *mp = container_of(_mp, struct mmap_params, core); struct evsel *evsel = container_of(_evsel, struct evsel, core); + if (!evlist) + return; + auxtrace_mmap_params__set_idx(&mp->auxtrace_mp, evlist, evsel, idx); + + from_list_end(evlist); } static struct perf_mmap* perf_evlist__mmap_cb_get(struct perf_evlist *_evlist, bool overwrite, int idx) { - struct evlist *evlist = container_of(_evlist, struct evlist, core); + struct evlist *evlist = from_list_start(_evlist); struct mmap *maps; - maps = overwrite ? evlist->overwrite_mmap : evlist->mmap; + if (!evlist) + return NULL; + + maps = overwrite ? evlist__overwrite_mmap(evlist) : evlist__mmap(evlist); if (!maps) { maps = evlist__alloc_mmap(evlist, overwrite); - if (!maps) + if (!maps) { + from_list_end(evlist); return NULL; + } if (overwrite) { - evlist->overwrite_mmap = maps; - if (evlist->bkw_mmap_state == BKW_MMAP_NOTREADY) + RC_CHK_ACCESS(evlist)->overwrite_mmap = maps; + if (evlist__bkw_mmap_state(evlist) == BKW_MMAP_NOTREADY) evlist__toggle_bkw_mmap(evlist, BKW_MMAP_RUNNING); } else { - evlist->mmap = maps; + RC_CHK_ACCESS(evlist)->mmap = maps; } } - + from_list_end(evlist); return &maps[idx].core; } @@ -1045,16 +1126,16 @@ int evlist__mmap_ex(struct evlist *evlist, unsigned int pages, .mmap = perf_evlist__mmap_cb_mmap, }; - evlist->core.mmap_len = evlist__mmap_size(pages); - pr_debug("mmap size %zuB\n", evlist->core.mmap_len); + evlist__core(evlist)->mmap_len = evlist__mmap_size(pages); + pr_debug("mmap size %zuB\n", evlist__core(evlist)->mmap_len); - auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist->core.mmap_len, + auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist__core(evlist)->mmap_len, auxtrace_pages, auxtrace_overwrite); - return perf_evlist__mmap_ops(&evlist->core, &ops, &mp.core); + return perf_evlist__mmap_ops(evlist__core(evlist), &ops, &mp.core); } -int evlist__mmap(struct evlist *evlist, unsigned int pages) +int evlist__do_mmap(struct evlist *evlist, unsigned int pages) { return evlist__mmap_ex(evlist, pages, 0, false, 0, PERF_AFFINITY_SYS, 1, 0); } @@ -1096,9 +1177,9 @@ int evlist__create_maps(struct evlist *evlist, struct target *target) if (!cpus) goto out_delete_threads; - evlist->core.has_user_cpus = !!target->cpu_list; + evlist__core(evlist)->has_user_cpus = !!target->cpu_list; - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); /* as evlist now has references, put count here */ perf_cpu_map__put(cpus); @@ -1238,15 +1319,15 @@ bool evlist__valid_sample_type(struct evlist *evlist) { struct evsel *pos; - if (evlist->core.nr_entries == 1) + if (evlist__nr_entries(evlist) == 1) return true; - if (evlist->id_pos < 0 || evlist->is_pos < 0) + if (evlist__id_pos(evlist) < 0 || evlist__is_pos(evlist) < 0) return false; evlist__for_each_entry(evlist, pos) { - if (pos->id_pos != evlist->id_pos || - pos->is_pos != evlist->is_pos) + if (pos->id_pos != evlist__id_pos(evlist) || + pos->is_pos != evlist__is_pos(evlist)) return false; } @@ -1257,18 +1338,18 @@ u64 __evlist__combined_sample_type(struct evlist *evlist) { struct evsel *evsel; - if (evlist->combined_sample_type) - return evlist->combined_sample_type; + if (RC_CHK_ACCESS(evlist)->combined_sample_type) + return RC_CHK_ACCESS(evlist)->combined_sample_type; evlist__for_each_entry(evlist, evsel) - evlist->combined_sample_type |= evsel->core.attr.sample_type; + RC_CHK_ACCESS(evlist)->combined_sample_type |= evsel->core.attr.sample_type; - return evlist->combined_sample_type; + return RC_CHK_ACCESS(evlist)->combined_sample_type; } u64 evlist__combined_sample_type(struct evlist *evlist) { - evlist->combined_sample_type = 0; + RC_CHK_ACCESS(evlist)->combined_sample_type = 0; return __evlist__combined_sample_type(evlist); } @@ -1345,7 +1426,7 @@ void evlist__update_br_cntr(struct evlist *evlist) evlist__new_abbr_name(evsel->abbr_name); } } - evlist->nr_br_cntr = i; + evlist__set_nr_br_cntr(evlist, i); } bool evlist__valid_read_format(struct evlist *evlist) @@ -1395,11 +1476,6 @@ bool evlist__sample_id_all(struct evlist *evlist) return first->core.attr.sample_id_all; } -void evlist__set_selected(struct evlist *evlist, struct evsel *evsel) -{ - evlist->selected = evsel; -} - void evlist__close(struct evlist *evlist) { struct evsel *evsel; @@ -1416,7 +1492,7 @@ void evlist__close(struct evlist *evlist) perf_evsel__free_fd(&evsel->core); perf_evsel__free_id(&evsel->core); } - perf_evlist__reset_id_hash(&evlist->core); + perf_evlist__reset_id_hash(evlist__core(evlist)); } static int evlist__create_syswide_maps(struct evlist *evlist) @@ -1443,7 +1519,7 @@ static int evlist__create_syswide_maps(struct evlist *evlist) return -ENOMEM; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); perf_thread_map__put(threads); perf_cpu_map__put(cpus); return 0; @@ -1458,7 +1534,8 @@ int evlist__open(struct evlist *evlist) * Default: one fd per CPU, all threads, aka systemwide * as sys_perf_event_open(cpu = -1, thread = -1) is EINVAL */ - if (evlist->core.threads == NULL && evlist->core.user_requested_cpus == NULL) { + if (evlist__core(evlist)->threads == NULL && + evlist__core(evlist)->user_requested_cpus == NULL) { err = evlist__create_syswide_maps(evlist); if (err < 0) goto out_err; @@ -1485,7 +1562,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const int child_ready_pipe[2], go_pipe[2]; char bf; - evlist->workload.cork_fd = -1; + evlist__set_workload_cork_fd(evlist, -1); if (pipe(child_ready_pipe) < 0) { perror("failed to create 'ready' pipe"); @@ -1497,13 +1574,13 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const goto out_close_ready_pipe; } - evlist->workload.pid = fork(); - if (evlist->workload.pid < 0) { + evlist__set_workload_pid(evlist, fork()); + if (evlist__workload_pid(evlist) < 0) { perror("failed to fork"); goto out_close_pipes; } - if (!evlist->workload.pid) { + if (!evlist__workload_pid(evlist)) { int ret; if (pipe_output) @@ -1569,12 +1646,13 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const } if (target__none(target)) { - if (evlist->core.threads == NULL) { + if (evlist__core(evlist)->threads == NULL) { fprintf(stderr, "FATAL: evlist->threads need to be set at this point (%s:%d).\n", __func__, __LINE__); goto out_close_pipes; } - perf_thread_map__set_pid(evlist->core.threads, 0, evlist->workload.pid); + perf_thread_map__set_pid(evlist__core(evlist)->threads, 0, + evlist__workload_pid(evlist)); } close(child_ready_pipe[1]); @@ -1588,7 +1666,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const } fcntl(go_pipe[1], F_SETFD, FD_CLOEXEC); - evlist->workload.cork_fd = go_pipe[1]; + evlist__set_workload_cork_fd(evlist, go_pipe[1]); close(child_ready_pipe[0]); return 0; @@ -1603,18 +1681,18 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const int evlist__start_workload(struct evlist *evlist) { - if (evlist->workload.cork_fd >= 0) { + if (evlist__workload_cork_fd(evlist) >= 0) { char bf = 0; int ret; /* * Remove the cork, let it rip! */ - ret = write(evlist->workload.cork_fd, &bf, 1); + ret = write(evlist__workload_cork_fd(evlist), &bf, 1); if (ret < 0) perror("unable to write to pipe"); - close(evlist->workload.cork_fd); - evlist->workload.cork_fd = -1; + close(evlist__workload_cork_fd(evlist)); + evlist__set_workload_cork_fd(evlist, -1); return ret; } @@ -1625,10 +1703,10 @@ void evlist__cancel_workload(struct evlist *evlist) { int status; - if (evlist->workload.cork_fd >= 0) { - close(evlist->workload.cork_fd); - evlist->workload.cork_fd = -1; - waitpid(evlist->workload.pid, &status, WNOHANG); + if (evlist__workload_cork_fd(evlist) >= 0) { + close(evlist__workload_cork_fd(evlist)); + evlist__set_workload_cork_fd(evlist, -1); + waitpid(evlist__workload_pid(evlist), &status, WNOHANG); } } @@ -1722,7 +1800,8 @@ int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size int evlist__strerror_mmap(struct evlist *evlist, int err, char *buf, size_t size) { - int pages_attempted = evlist->core.mmap_len / 1024, pages_max_per_user, printed = 0; + int pages_attempted = evlist__core(evlist)->mmap_len / 1024; + int pages_max_per_user, printed = 0; switch (err) { case EPERM: @@ -1765,7 +1844,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel) list_move_tail(&evsel->core.node, &move); } - list_splice(&move, &evlist->core.entries); + list_splice(&move, &evlist__core(evlist)->entries); } struct evsel *evlist__get_tracking_event(struct evlist *evlist) @@ -1807,7 +1886,7 @@ struct evsel *evlist__findnew_tracking_event(struct evlist *evlist, bool system_ evlist__set_tracking_event(evlist, evsel); } else if (system_wide) { - perf_evlist__go_system_wide(&evlist->core, &evsel->core); + perf_evlist__go_system_wide(evlist__core(evlist), &evsel->core); } return evsel; @@ -1829,14 +1908,14 @@ struct evsel *evlist__find_evsel_by_str(struct evlist *evlist, const char *str) void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state) { - enum bkw_mmap_state old_state = evlist->bkw_mmap_state; + enum bkw_mmap_state old_state = evlist__bkw_mmap_state(evlist); enum action { NONE, PAUSE, RESUME, } action = NONE; - if (!evlist->overwrite_mmap) + if (!evlist__overwrite_mmap(evlist)) return; switch (old_state) { @@ -1866,7 +1945,7 @@ void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state) WARN_ONCE(1, "Shouldn't get there\n"); } - evlist->bkw_mmap_state = state; + evlist__set_bkw_mmap_state(evlist, state); switch (action) { case PAUSE: @@ -2044,40 +2123,41 @@ int evlist__initialize_ctlfd(struct evlist *evlist, int fd, int ack) return 0; } - evlist->ctl_fd.pos = perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, - fdarray_flag__nonfilterable | - fdarray_flag__non_perf_event); - if (evlist->ctl_fd.pos < 0) { - evlist->ctl_fd.pos = -1; + evlist__set_ctl_fd_pos(evlist, + perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, + fdarray_flag__nonfilterable | + fdarray_flag__non_perf_event)); + if (evlist__ctl_fd_pos(evlist) < 0) { + evlist__set_ctl_fd_pos(evlist, -1); pr_err("Failed to add ctl fd entry: %m\n"); return -1; } - evlist->ctl_fd.fd = fd; - evlist->ctl_fd.ack = ack; + evlist__set_ctl_fd_fd(evlist, fd); + evlist__set_ctl_fd_ack(evlist, ack); return 0; } bool evlist__ctlfd_initialized(struct evlist *evlist) { - return evlist->ctl_fd.pos >= 0; + return evlist__ctl_fd_pos(evlist) >= 0; } int evlist__finalize_ctlfd(struct evlist *evlist) { - struct pollfd *entries = evlist->core.pollfd.entries; + struct pollfd *entries = evlist__core(evlist)->pollfd.entries; if (!evlist__ctlfd_initialized(evlist)) return 0; - entries[evlist->ctl_fd.pos].fd = -1; - entries[evlist->ctl_fd.pos].events = 0; - entries[evlist->ctl_fd.pos].revents = 0; + entries[evlist__ctl_fd_pos(evlist)].fd = -1; + entries[evlist__ctl_fd_pos(evlist)].events = 0; + entries[evlist__ctl_fd_pos(evlist)].revents = 0; - evlist->ctl_fd.pos = -1; - evlist->ctl_fd.ack = -1; - evlist->ctl_fd.fd = -1; + evlist__set_ctl_fd_pos(evlist, -1); + evlist__set_ctl_fd_ack(evlist, -1); + evlist__set_ctl_fd_fd(evlist, -1); return 0; } @@ -2094,7 +2174,7 @@ static int evlist__ctlfd_recv(struct evlist *evlist, enum evlist_ctl_cmd *cmd, data_size--; do { - err = read(evlist->ctl_fd.fd, &c, 1); + err = read(evlist__ctl_fd_fd(evlist), &c, 1); if (err > 0) { if (c == '\n' || c == '\0') break; @@ -2108,7 +2188,8 @@ static int evlist__ctlfd_recv(struct evlist *evlist, enum evlist_ctl_cmd *cmd, if (errno == EAGAIN || errno == EWOULDBLOCK) err = 0; else - pr_err("Failed to read from ctlfd %d: %m\n", evlist->ctl_fd.fd); + pr_err("Failed to read from ctlfd %d: %m\n", + evlist__ctl_fd_fd(evlist)); } break; } while (1); @@ -2146,13 +2227,13 @@ int evlist__ctlfd_ack(struct evlist *evlist) { int err; - if (evlist->ctl_fd.ack == -1) + if (evlist__ctl_fd_ack(evlist) == -1) return 0; - err = write(evlist->ctl_fd.ack, EVLIST_CTL_CMD_ACK_TAG, + err = write(evlist__ctl_fd_ack(evlist), EVLIST_CTL_CMD_ACK_TAG, sizeof(EVLIST_CTL_CMD_ACK_TAG)); if (err == -1) - pr_err("failed to write to ctl_ack_fd %d: %m\n", evlist->ctl_fd.ack); + pr_err("failed to write to ctl_ack_fd %d: %m\n", evlist__ctl_fd_ack(evlist)); return err; } @@ -2253,8 +2334,8 @@ int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd) { int err = 0; char cmd_data[EVLIST_CTL_CMD_MAX_LEN]; - int ctlfd_pos = evlist->ctl_fd.pos; - struct pollfd *entries = evlist->core.pollfd.entries; + int ctlfd_pos = evlist__ctl_fd_pos(evlist); + struct pollfd *entries = evlist__core(evlist)->pollfd.entries; if (!evlist__ctlfd_initialized(evlist) || !entries[ctlfd_pos].revents) return 0; @@ -2425,14 +2506,15 @@ int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *o goto free_eet_times; } - eet->pollfd_pos = perf_evlist__add_pollfd(&evlist->core, eet->timerfd, NULL, POLLIN, flags); + eet->pollfd_pos = perf_evlist__add_pollfd(evlist__core(evlist), eet->timerfd, + NULL, POLLIN, flags); if (eet->pollfd_pos < 0) { err = eet->pollfd_pos; goto close_timerfd; } eet->evlist = evlist; - evlist->eet = eet; + RC_CHK_ACCESS(evlist)->eet = eet; opts->target.initial_delay = eet->times[0].start; return 0; @@ -2482,7 +2564,7 @@ int event_enable_timer__process(struct event_enable_timer *eet) if (!eet) return 0; - entries = eet->evlist->core.pollfd.entries; + entries = evlist__core(eet->evlist)->pollfd.entries; revents = entries[eet->pollfd_pos].revents; entries[eet->pollfd_pos].revents = 0; @@ -2518,7 +2600,7 @@ int event_enable_timer__process(struct event_enable_timer *eet) return 0; } -void event_enable_timer__exit(struct event_enable_timer **ep) +static void event_enable_timer__exit(struct event_enable_timer **ep) { if (!ep || !*ep) return; @@ -2622,7 +2704,7 @@ void evlist__warn_user_requested_cpus(struct evlist *evlist, const char *cpu_lis } /* Should uniquify be disabled for the evlist? */ -static bool evlist__disable_uniquify(const struct evlist *evlist) +static bool evlist__disable_uniquify(struct evlist *evlist) { struct evsel *counter; struct perf_pmu *last_pmu = NULL; diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h index e507f5f20ef61..1997843dca0ef 100644 --- a/tools/perf/util/evlist.h +++ b/tools/perf/util/evlist.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "affinity.h" #include "events_stats.h" @@ -56,8 +57,9 @@ enum bkw_mmap_state { struct event_enable_timer; -struct evlist { +DECLARE_RC_STRUCT(evlist) { struct perf_evlist core; + refcount_t refcnt; bool enabled; bool no_affinity; int id_pos; @@ -82,7 +84,7 @@ struct evlist { struct { pthread_t th; volatile int done; - } thread; + } sb_thread; struct { int fd; /* control file descriptor */ int ack; /* ack file descriptor for control commands */ @@ -103,13 +105,232 @@ struct evsel_str_handler { void *handler; }; +static inline struct perf_evlist *evlist__core(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->core; +} + +static inline const struct perf_evlist *evlist__const_core(const struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->core; +} + +static inline int evlist__nr_entries(const struct evlist *evlist) +{ + return evlist__const_core(evlist)->nr_entries; +} + +static inline bool evlist__enabled(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->enabled; +} + +static inline void evlist__set_enabled(struct evlist *evlist, bool enabled) +{ + RC_CHK_ACCESS(evlist)->enabled = enabled; +} + +static inline bool evlist__no_affinity(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->no_affinity; +} + +static inline void evlist__set_no_affinity(struct evlist *evlist, bool no_affinity) +{ + RC_CHK_ACCESS(evlist)->no_affinity = no_affinity; +} + +static inline int evlist__sb_thread_done(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->sb_thread.done; +} + +static inline void evlist__set_sb_thread_done(struct evlist *evlist, int done) +{ + RC_CHK_ACCESS(evlist)->sb_thread.done = done; +} + +static inline pthread_t *evlist__sb_thread_th(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->sb_thread.th; +} + +static inline int evlist__id_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->id_pos; +} + +static inline int evlist__is_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->is_pos; +} + +static inline struct event_enable_timer *evlist__event_enable_timer(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->eet; +} + +static inline enum bkw_mmap_state evlist__bkw_mmap_state(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->bkw_mmap_state; +} + +static inline void evlist__set_bkw_mmap_state(struct evlist *evlist, enum bkw_mmap_state state) +{ + RC_CHK_ACCESS(evlist)->bkw_mmap_state = state; +} + +static inline struct mmap *evlist__mmap(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->mmap; +} + +static inline struct mmap *evlist__overwrite_mmap(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->overwrite_mmap; +} + +static inline struct events_stats *evlist__stats(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->stats; +} + +static inline u64 evlist__first_sample_time(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->first_sample_time; +} + +static inline void evlist__set_first_sample_time(struct evlist *evlist, u64 first) +{ + RC_CHK_ACCESS(evlist)->first_sample_time = first; +} + +static inline u64 evlist__last_sample_time(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->last_sample_time; +} + +static inline void evlist__set_last_sample_time(struct evlist *evlist, u64 last) +{ + RC_CHK_ACCESS(evlist)->last_sample_time = last; +} + +static inline int evlist__nr_br_cntr(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->nr_br_cntr; +} + +static inline void evlist__set_nr_br_cntr(struct evlist *evlist, int nr) +{ + RC_CHK_ACCESS(evlist)->nr_br_cntr = nr; +} + +static inline struct perf_session *evlist__session(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->session; +} + +static inline void evlist__set_session(struct evlist *evlist, struct perf_session *session) +{ + RC_CHK_ACCESS(evlist)->session = session; +} + +static inline void (*evlist__trace_event_sample_raw(struct evlist *evlist)) + (struct evlist *evlist, + union perf_event *event, + struct perf_sample *sample) +{ + return RC_CHK_ACCESS(evlist)->trace_event_sample_raw; +} + +static inline void evlist__set_trace_event_sample_raw(struct evlist *evlist, + void (*fun)(struct evlist *evlist, + union perf_event *event, + struct perf_sample *sample)) +{ + RC_CHK_ACCESS(evlist)->trace_event_sample_raw = fun; +} + +static inline pid_t evlist__workload_pid(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->workload.pid; +} + +static inline void evlist__set_workload_pid(struct evlist *evlist, pid_t pid) +{ + RC_CHK_ACCESS(evlist)->workload.pid = pid; +} + +static inline int evlist__workload_cork_fd(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->workload.cork_fd; +} + +static inline void evlist__set_workload_cork_fd(struct evlist *evlist, int cork_fd) +{ + RC_CHK_ACCESS(evlist)->workload.cork_fd = cork_fd; +} + +static inline int evlist__ctl_fd_fd(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.fd; +} + +static inline void evlist__set_ctl_fd_fd(struct evlist *evlist, int fd) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.fd = fd; +} + +static inline int evlist__ctl_fd_ack(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.ack; +} + +static inline void evlist__set_ctl_fd_ack(struct evlist *evlist, int ack) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.ack = ack; +} + +static inline int evlist__ctl_fd_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.pos; +} + +static inline void evlist__set_ctl_fd_pos(struct evlist *evlist, int pos) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.pos = pos; +} + +static inline refcount_t *evlist__refcnt(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->refcnt; +} + +static inline struct rblist *evlist__metric_events(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->metric_events; +} + +static inline struct list_head *evlist__deferred_samples(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->deferred_samples; +} + +static inline struct evsel *evlist__selected(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->selected; +} + +static inline void evlist__set_selected(struct evlist *evlist, struct evsel *evsel) +{ + RC_CHK_ACCESS(evlist)->selected = evsel; +} + struct evlist *evlist__new(void); struct evlist *evlist__new_default(const struct target *target, bool sample_callchains); struct evlist *evlist__new_dummy(void); -void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus, - struct perf_thread_map *threads); -void evlist__exit(struct evlist *evlist); -void evlist__delete(struct evlist *evlist); +struct evlist *evlist__get(struct evlist *evlist); +void evlist__put(struct evlist *evlist); void evlist__add(struct evlist *evlist, struct evsel *entry); void evlist__remove(struct evlist *evlist, struct evsel *evsel); @@ -198,8 +419,8 @@ int evlist__mmap_ex(struct evlist *evlist, unsigned int pages, unsigned int auxtrace_pages, bool auxtrace_overwrite, int nr_cblocks, int affinity, int flush, int comp_level); -int evlist__mmap(struct evlist *evlist, unsigned int pages); -void evlist__munmap(struct evlist *evlist); +int evlist__do_mmap(struct evlist *evlist, unsigned int pages); +void evlist__do_munmap(struct evlist *evlist); size_t evlist__mmap_size(unsigned long pages); @@ -211,8 +432,6 @@ void evlist__enable_evsel(struct evlist *evlist, char *evsel_name); void evlist__disable_non_dummy(struct evlist *evlist); void evlist__enable_non_dummy(struct evlist *evlist); -void evlist__set_selected(struct evlist *evlist, struct evsel *evsel); - int evlist__create_maps(struct evlist *evlist, struct target *target); int evlist__apply_filters(struct evlist *evlist, struct evsel **err_evsel, struct target *target); @@ -235,26 +454,26 @@ void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list); static inline bool evlist__empty(struct evlist *evlist) { - return list_empty(&evlist->core.entries); + return list_empty(&evlist__core(evlist)->entries); } static inline struct evsel *evlist__first(struct evlist *evlist) { - struct perf_evsel *evsel = perf_evlist__first(&evlist->core); + struct perf_evsel *evsel = perf_evlist__first(evlist__core(evlist)); return container_of(evsel, struct evsel, core); } static inline struct evsel *evlist__last(struct evlist *evlist) { - struct perf_evsel *evsel = perf_evlist__last(&evlist->core); + struct perf_evsel *evsel = perf_evlist__last(evlist__core(evlist)); return container_of(evsel, struct evsel, core); } static inline int evlist__nr_groups(struct evlist *evlist) { - return perf_evlist__nr_groups(&evlist->core); + return perf_evlist__nr_groups(evlist__core(evlist)); } int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size); @@ -277,7 +496,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry(evlist, evsel) \ - __evlist__for_each_entry(&(evlist)->core.entries, evsel) + __evlist__for_each_entry(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_continue - continue iteration thru all the evsels @@ -293,7 +512,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry_continue(evlist, evsel) \ - __evlist__for_each_entry_continue(&(evlist)->core.entries, evsel) + __evlist__for_each_entry_continue(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_from - continue iteration from @evsel (included) @@ -309,7 +528,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry_from(evlist, evsel) \ - __evlist__for_each_entry_from(&(evlist)->core.entries, evsel) + __evlist__for_each_entry_from(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_reverse - iterate thru all the evsels in reverse order @@ -325,7 +544,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry_reverse(evlist, evsel) \ - __evlist__for_each_entry_reverse(&(evlist)->core.entries, evsel) + __evlist__for_each_entry_reverse(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_safe - safely iterate thru all the evsels @@ -343,7 +562,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @tmp: struct evsel temp iterator */ #define evlist__for_each_entry_safe(evlist, tmp, evsel) \ - __evlist__for_each_entry_safe(&(evlist)->core.entries, tmp, evsel) + __evlist__for_each_entry_safe(&evlist__core(evlist)->entries, tmp, evsel) /** Iterator state for evlist__for_each_cpu */ struct evlist_cpu_iterator { @@ -449,7 +668,6 @@ int evlist__ctlfd_ack(struct evlist *evlist); int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts, const char *str, int unset); int event_enable_timer__start(struct event_enable_timer *eet); -void event_enable_timer__exit(struct event_enable_timer **ep); int event_enable_timer__process(struct event_enable_timer *eet); struct evsel *evlist__find_evsel(struct evlist *evlist, int idx); diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index ea9fa04429f08..968cd74a9cde9 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -16,9 +16,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -440,10 +442,11 @@ bool evsel__is_function_event(struct evsel *evsel) #undef FUNCTION_EVENT } -void evsel__init(struct evsel *evsel, +static void evsel__init(struct evsel *evsel, struct perf_event_attr *attr, int idx) { perf_evsel__init(&evsel->core, attr, idx); + refcount_set(&evsel->refcnt, 1); evsel->tracking = !idx; evsel->unit = strdup(""); evsel->scale = 1.0; @@ -525,7 +528,7 @@ static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src) * The assumption is that @orig is not configured nor opened yet. * So we only care about the attributes that can be set while it's parsed. */ -struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) +struct evsel *evsel__clone(struct evsel *orig) { struct evsel *evsel; @@ -538,11 +541,7 @@ struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) if (orig->bpf_obj) return NULL; - if (dest) - evsel = dest; - else - evsel = evsel__new(&orig->core.attr); - + evsel = evsel__new(&orig->core.attr); if (evsel == NULL) return NULL; @@ -615,7 +614,13 @@ struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) evsel->sample_read = orig->sample_read; evsel->collect_stat = orig->collect_stat; evsel->weak_group = orig->weak_group; + evsel->bpf_counter = orig->bpf_counter; evsel->use_config_name = orig->use_config_name; + evsel->skippable = orig->skippable; + evsel->dont_regroup = orig->dont_regroup; + evsel->default_metricgroup = orig->default_metricgroup; + evsel->default_show_events = orig->default_show_events; + evsel->pmu = orig->pmu; evsel->first_wildcard_match = orig->first_wildcard_match; @@ -624,10 +629,14 @@ struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) evsel->alternate_hw_config = orig->alternate_hw_config; + evsel->retire_lat = orig->retire_lat; + if (evsel->retire_lat) + evsel->retirement_latency = orig->retirement_latency; + return evsel; out_err: - evsel__delete(evsel); + evsel__put(evsel); return NULL; } @@ -686,6 +695,12 @@ struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx, bool return ERR_PTR(err); } +struct evsel *evsel__get(struct evsel *evsel) +{ + refcount_inc(&evsel->refcnt); + return evsel; +} + #ifdef HAVE_LIBTRACEEVENT struct tep_event *evsel__tp_format(struct evsel *evsel) { @@ -703,7 +718,7 @@ struct tep_event *evsel__tp_format(struct evsel *evsel) tp_format = trace_event__tp_format(evsel->tp_sys, evsel->tp_name); if (IS_ERR(tp_format)) { - int err = -PTR_ERR(evsel->tp_format); + int err = -PTR_ERR(tp_format); errno = err; pr_err("Error getting tracepoint format '%s': %m\n", @@ -2024,7 +2039,7 @@ void evsel__set_priv_destructor(void (*destructor)(void *priv)) evsel__priv_destructor = destructor; } -void evsel__exit(struct evsel *evsel) +static void evsel__exit(struct evsel *evsel) { assert(list_empty(&evsel->core.node)); assert(evsel->evlist == NULL); @@ -2061,11 +2076,14 @@ void evsel__exit(struct evsel *evsel) } } -void evsel__delete(struct evsel *evsel) +void evsel__put(struct evsel *evsel) { if (!evsel) return; + if (!refcount_dec_and_test(&evsel->refcnt)) + return; + evsel__exit(evsel); free(evsel); } @@ -3350,7 +3368,7 @@ static inline bool evsel__has_branch_counters(const struct evsel *evsel) if (!leader || !evsel->evlist) return false; - if (evsel->evlist->nr_br_cntr < 0) + if (evlist__nr_br_cntr(evsel->evlist) < 0) evlist__update_br_cntr(evsel->evlist); if (leader->br_cntr_nr > 0) @@ -3933,6 +3951,95 @@ void *perf_sample__rawptr(struct perf_sample *sample, const char *name) return sample->raw_data + offset; } +void *format_field__get_raw_data(struct tep_format_field *field, struct + perf_sample *sample, bool needs_swap, + u16 *len_out) +{ + int offset = field->offset; + int size = field->size; + + if (field->flags & TEP_FIELD_IS_DYNAMIC) { + unsigned int dynamic_data; + + if (out_of_bounds(field, field->offset, field->size, sample->raw_size)) + return NULL; + + dynamic_data = format_field__intval(field, sample, needs_swap); + + offset = dynamic_data & 0xffff; + size = (dynamic_data >> 16) & 0xffff; + + if (tep_field_is_relative(field->flags)) + offset += field->offset + field->size; + } + + if (out_of_bounds(field, offset, size, sample->raw_size)) + return NULL; + + *len_out = size; + return sample->raw_data + offset; +} + +unsigned long *format_field__get_cpumask(struct tep_format_field *field, + struct perf_sample *sample, + bool needs_swap, u16 *len_out) +{ + u16 len; + void *ptr = format_field__get_raw_data(field, sample, needs_swap, &len); + unsigned long *mask; + struct perf_env *env; + bool target_is_64; + int target_word_size; + int nr_words; + int bit_idx; + int nbits; + + if (!ptr) + return NULL; + + nbits = len * 8; + mask = bitmap_zalloc(nbits ?: 1); + if (!mask) + return NULL; + + env = evsel__env(sample->evsel); + target_is_64 = env ? perf_env__kernel_is_64_bit(env) : (sizeof(void *) == 8); + target_word_size = target_is_64 ? 8 : 4; + nr_words = len / target_word_size; + + for (bit_idx = 0; bit_idx < nbits; bit_idx++) { + int w_idx = bit_idx / (target_word_size * 8); + int bit_in_word = bit_idx % (target_word_size * 8); + bool set = false; + + if (w_idx >= nr_words) + break; + + if (target_is_64) { + u64 word; + memcpy(&word, (unsigned char *)ptr + w_idx * 8, 8); + if (needs_swap) + word = bswap_64(word); + set = (word & (1ULL << bit_in_word)) != 0; + } else { + u32 word32; + memcpy(&word32, (unsigned char *)ptr + w_idx * 4, 4); + if (needs_swap) + word32 = bswap_32(word32); + set = (word32 & (1U << bit_in_word)) != 0; + } + + if (set) { + int host_w_idx = bit_idx / BITS_PER_LONG; + int host_bit_in_word = bit_idx % BITS_PER_LONG; + mask[host_w_idx] |= (1UL << host_bit_in_word); + } + } + + *len_out = len; + return mask; +} + u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, bool needs_swap) { @@ -4382,7 +4489,7 @@ int evsel__open_strerror(struct evsel *evsel, struct target *target, struct perf_session *evsel__session(struct evsel *evsel) { - return evsel && evsel->evlist ? evsel->evlist->session : NULL; + return evsel && evsel->evlist ? evlist__session(evsel->evlist) : NULL; } struct perf_env *evsel__env(struct evsel *evsel) @@ -4407,7 +4514,7 @@ static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist) thread++) { int fd = FD(evsel, cpu_map_idx, thread); - if (perf_evlist__id_add_fd(&evlist->core, &evsel->core, + if (perf_evlist__id_add_fd(evlist__core(evlist), &evsel->core, cpu_map_idx, thread, fd) < 0) return -1; } diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index 163fc2b6a7eac..ba567e3b65c93 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ typedef int (evsel__sb_cb_t)(union perf_event *event, void *data); struct evsel { struct perf_evsel core; struct evlist *evlist; + refcount_t refcnt; off_t id_offset; int id_pos; int is_pos; @@ -271,7 +273,7 @@ static inline struct evsel *evsel__new(struct perf_event_attr *attr) return evsel__new_idx(attr, 0); } -struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig); +struct evsel *evsel__clone(struct evsel *orig); int copy_config_terms(struct list_head *dst, struct list_head *src); void free_config_terms(struct list_head *config_terms); @@ -286,14 +288,13 @@ static inline struct evsel *evsel__newtp(const char *sys, const char *name) return evsel__newtp_idx(sys, name, 0, true); } +struct evsel *evsel__get(struct evsel *evsel); +void evsel__put(struct evsel *evsel); + #ifdef HAVE_LIBTRACEEVENT struct tep_event *evsel__tp_format(struct evsel *evsel); #endif -void evsel__init(struct evsel *evsel, struct perf_event_attr *attr, int idx); -void evsel__exit(struct evsel *evsel); -void evsel__delete(struct evsel *evsel); - void evsel__set_priv_destructor(void (*destructor)(void *priv)); struct callchain_param; @@ -400,6 +401,12 @@ static inline char *perf_sample__strval(struct perf_sample *sample, const char * struct tep_format_field; +void *format_field__get_raw_data(struct tep_format_field *field, + struct perf_sample *sample, + bool needs_swap, u16 *len_out); +unsigned long *format_field__get_cpumask(struct tep_format_field *field, + struct perf_sample *sample, + bool needs_swap, u16 *len_out); u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, bool needs_swap); #ifdef HAVE_LIBTRACEEVENT @@ -534,7 +541,7 @@ for ((_evsel) = list_entry((_leader)->core.node.next, struct evsel, core.node); (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) #define for_each_group_member(_evsel, _leader) \ - for_each_group_member_head(_evsel, _leader, &(_leader)->evlist->core.entries) + for_each_group_member_head(_evsel, _leader, &evlist__core((_leader)->evlist)->entries) /* Iterates group WITH the leader. */ #define for_each_group_evsel_head(_evsel, _leader, _head) \ @@ -544,7 +551,7 @@ for ((_evsel) = _leader; \ (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) #define for_each_group_evsel(_evsel, _leader) \ - for_each_group_evsel_head(_evsel, _leader, &(_leader)->evlist->core.entries) + for_each_group_evsel_head(_evsel, _leader, &evlist__core((_leader)->evlist)->entries) static inline bool evsel__has_branch_callstack(const struct evsel *evsel) { diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c index 232998fef72ba..8aef3c7418f7c 100644 --- a/tools/perf/util/expr.c +++ b/tools/perf/util/expr.c @@ -468,7 +468,7 @@ double expr__has_event(const struct expr_parse_ctx *ctx, bool compute_ids, const ret = parse_event(tmp, id) ? 0 : 1; } out: - evlist__delete(tmp); + evlist__put(tmp); return ret; } diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c index 091d8f7f6bd2c..7db7da090a1e0 100644 --- a/tools/perf/util/header.c +++ b/tools/perf/util/header.c @@ -385,7 +385,7 @@ static int write_tracing_data(struct feat_fd *ff, return -1; #ifdef HAVE_LIBTRACEEVENT - return read_tracing_data(ff->fd, &evlist->core.entries); + return read_tracing_data(ff->fd, &evlist__core(evlist)->entries); #else pr_err("ERROR: Trying to write tracing data without libtraceevent support.\n"); return -1; @@ -434,8 +434,8 @@ static int write_osrelease(struct feat_fd *ff, struct utsname uts; const char *release = NULL; - if (evlist->session) - release = perf_env__os_release(perf_session__env(evlist->session)); + if (evlist__session(evlist)) + release = perf_env__os_release(perf_session__env(evlist__session(evlist))); if (!release) { int ret = uname(&uts); @@ -452,8 +452,8 @@ static int write_arch(struct feat_fd *ff, struct evlist *evlist) struct utsname uts; const char *arch = NULL; - if (evlist->session) - arch = perf_env__arch(perf_session__env(evlist->session)); + if (evlist__session(evlist)) + arch = perf_env__arch(perf_session__env(evlist__session(evlist))); if (!arch) { int ret = uname(&uts); @@ -469,7 +469,7 @@ static int write_e_machine(struct feat_fd *ff, struct evlist *evlist) { /* e_machine expanded from 16 to 32-bits for alignment. */ uint32_t e_flags; - uint32_t e_machine = perf_session__e_machine(evlist->session, &e_flags); + uint32_t e_machine = perf_session__e_machine(evlist__session(evlist), &e_flags); int ret; ret = do_write(ff, &e_machine, sizeof(e_machine)); @@ -605,7 +605,7 @@ static int write_event_desc(struct feat_fd *ff, u32 nre, nri, sz; int ret; - nre = evlist->core.nr_entries; + nre = evlist__nr_entries(evlist); /* * write number of events @@ -987,7 +987,7 @@ int __weak get_cpuid(char *buffer __maybe_unused, size_t sz __maybe_unused, static int write_cpuid(struct feat_fd *ff, struct evlist *evlist) { - struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus); + struct perf_cpu cpu = perf_cpu_map__min(evlist__core(evlist)->all_cpus); char buffer[64]; int ret; @@ -1120,7 +1120,8 @@ static int write_bpf_prog_info(struct feat_fd *ff __maybe_unused, struct rb_node *next; int ret = 0; - down_read(&env->bpf_progs.lock); + /* write lock: bpil_addr_to_offs() temporarily mutates info_linear */ + down_write(&env->bpf_progs.lock); ret = do_write(ff, &env->bpf_progs.infos_cnt, sizeof(env->bpf_progs.infos_cnt)); @@ -1150,7 +1151,7 @@ static int write_bpf_prog_info(struct feat_fd *ff __maybe_unused, goto out; } out: - up_read(&env->bpf_progs.lock); + up_write(&env->bpf_progs.lock); return ret; #else pr_err("ERROR: Trying to write bpf_prog_info without libbpf support.\n"); @@ -1420,14 +1421,14 @@ static int write_sample_time(struct feat_fd *ff, struct evlist *evlist) { int ret; + u64 data = evlist__first_sample_time(evlist); - ret = do_write(ff, &evlist->first_sample_time, - sizeof(evlist->first_sample_time)); + ret = do_write(ff, &data, sizeof(data)); if (ret < 0) return ret; - return do_write(ff, &evlist->last_sample_time, - sizeof(evlist->last_sample_time)); + data = evlist__last_sample_time(evlist); + return do_write(ff, &data, sizeof(data)); } @@ -2551,16 +2552,16 @@ static void print_sample_time(struct feat_fd *ff, FILE *fp) session = container_of(ff->ph, struct perf_session, header); - timestamp__scnprintf_usec(session->evlist->first_sample_time, + timestamp__scnprintf_usec(evlist__first_sample_time(session->evlist), time_buf, sizeof(time_buf)); fprintf(fp, "# time of first sample : %s\n", time_buf); - timestamp__scnprintf_usec(session->evlist->last_sample_time, + timestamp__scnprintf_usec(evlist__last_sample_time(session->evlist), time_buf, sizeof(time_buf)); fprintf(fp, "# time of last sample : %s\n", time_buf); - d = (double)(session->evlist->last_sample_time - - session->evlist->first_sample_time) / NSEC_PER_MSEC; + d = (double)(evlist__last_sample_time(session->evlist) - + evlist__first_sample_time(session->evlist)) / NSEC_PER_MSEC; fprintf(fp, "# sample duration : %10.3f ms\n", d); } @@ -3519,8 +3520,8 @@ static int process_sample_time(struct feat_fd *ff, void *data __maybe_unused) if (ret) return -1; - session->evlist->first_sample_time = first_sample_time; - session->evlist->last_sample_time = last_sample_time; + evlist__set_first_sample_time(session->evlist, first_sample_time); + evlist__set_last_sample_time(session->evlist, last_sample_time); return 0; } @@ -4610,7 +4611,7 @@ int perf_session__write_header(struct perf_session *session, /*write_attrs_after_data=*/false); } -size_t perf_session__data_offset(const struct evlist *evlist) +size_t perf_session__data_offset(struct evlist *evlist) { struct evsel *evsel; size_t data_offset; @@ -4619,7 +4620,7 @@ size_t perf_session__data_offset(const struct evlist *evlist) evlist__for_each_entry(evlist, evsel) { data_offset += evsel->core.ids * sizeof(u64); } - data_offset += evlist->core.nr_entries * sizeof(struct perf_file_attr); + data_offset += evlist__nr_entries(evlist) * sizeof(struct perf_file_attr); return data_offset; } @@ -5110,7 +5111,7 @@ int perf_session__read_header(struct perf_session *session) if (session->evlist == NULL) return -ENOMEM; - session->evlist->session = session; + evlist__set_session(session->evlist, session); session->machines.host.env = &header->env; /* @@ -5186,7 +5187,7 @@ int perf_session__read_header(struct perf_session *session) pr_err("Invalid ids section size %" PRIu64 " for attr %d, not aligned to u64\n", f_attr.ids.size, i); err = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -5199,7 +5200,7 @@ int perf_session__read_header(struct perf_session *session) pr_err("Invalid ids section size %" PRIu64 " for attr %d, too many IDs\n", f_attr.ids.size, i); err = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -5212,19 +5213,19 @@ int perf_session__read_header(struct perf_session *session) pr_err("Invalid ids section for attr %d: offset=%" PRIu64 " size=%" PRIu64 " exceeds file size %" PRIu64 "\n", i, f_attr.ids.offset, f_attr.ids.size, (u64)input_stat.st_size); err = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } tmp = lseek(fd, 0, SEEK_CUR); evsel = evsel__new(&f_attr.attr); if (evsel == NULL) - goto out_delete_evlist; + goto out_put_evlist; evsel->needs_swap = header->needs_swap; /* * Do it before so that if perf_evsel__alloc_id fails, this - * entry gets purged too at evlist__delete(). + * entry gets purged too at evlist__put(). */ evlist__add(session->evlist, evsel); @@ -5235,7 +5236,7 @@ int perf_session__read_header(struct perf_session *session) * hattr->ids threads. */ if (perf_evsel__alloc_id(&evsel->core, 1, nr_ids)) - goto out_delete_evlist; + goto out_put_evlist; lseek(fd, f_attr.ids.offset, SEEK_SET); @@ -5243,7 +5244,8 @@ int perf_session__read_header(struct perf_session *session) if (perf_header__getbuffer64(header, fd, &f_id, sizeof(f_id))) goto out_errno; - perf_evlist__id_add(&session->evlist->core, &evsel->core, 0, j, f_id); + perf_evlist__id_add(evlist__core(session->evlist), + &evsel->core, 0, j, f_id); } lseek(fd, tmp, SEEK_SET); @@ -5265,18 +5267,18 @@ int perf_session__read_header(struct perf_session *session) err = perf_header__process_sections(header, fd, &session->tevent, perf_file_section__process); if (err < 0) - goto out_delete_evlist; + goto out_put_evlist; if (evlist__prepare_tracepoint_events(session->evlist, session->tevent.pevent)) { err = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } #else err = perf_header__process_sections(header, fd, NULL, perf_file_section__process); if (err < 0) - goto out_delete_evlist; + goto out_put_evlist; #endif } @@ -5302,8 +5304,8 @@ int perf_session__read_header(struct perf_session *session) out_errno: return -errno; -out_delete_evlist: - evlist__delete(session->evlist); +out_put_evlist: + evlist__put(session->evlist); session->evlist = NULL; return err; } @@ -5607,7 +5609,7 @@ int perf_event__process_attr(const struct perf_tool *tool __maybe_unused, */ ids = (void *)&event->attr.attr + attr_size; for (i = 0; i < n_ids; i++) { - perf_evlist__id_add(&evlist->core, &evsel->core, 0, i, ids[i]); + perf_evlist__id_add(evlist__core(evlist), &evsel->core, 0, i, ids[i]); } return 0; diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h index 86b1a72026d3f..5e03f884b7cc0 100644 --- a/tools/perf/util/header.h +++ b/tools/perf/util/header.h @@ -158,7 +158,7 @@ int perf_session__inject_header(struct perf_session *session, struct feat_copier *fc, bool write_attrs_after_data); -size_t perf_session__data_offset(const struct evlist *evlist); +size_t perf_session__data_offset(struct evlist *evlist); void perf_header__set_feat(struct perf_header *header, int feat); void perf_header__clear_feat(struct perf_header *header, int feat); diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c index a17c423a526dd..5db44556fc4f5 100644 --- a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include "../color.h" @@ -73,29 +75,20 @@ static const char * const hisi_ptt_4dw_pkt_field_name[] = { [HISI_PTT_4DW_HEAD3] = "Header DW3", }; -union hisi_ptt_4dw { - struct { - uint32_t format : 2; - uint32_t type : 5; - uint32_t t9 : 1; - uint32_t t8 : 1; - uint32_t th : 1; - uint32_t so : 1; - uint32_t len : 10; - uint32_t time : 11; - }; - uint32_t value; -}; - static void hisi_ptt_print_pkt(const unsigned char *buf, int pos, const char *desc) { const char *color = PERF_COLOR_BLUE; + uint8_t byte; + uint32_t dw; int i; + dw = get_unaligned_le32(buf + pos); printf("."); color_fprintf(stdout, color, " %08x: ", pos); - for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) - color_fprintf(stdout, color, "%02x ", buf[pos + i]); + for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) { + byte = (dw >> (24 - i * 8)) & 0xFF; + color_fprintf(stdout, color, "%02x ", byte); + } for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) color_fprintf(stdout, color, " "); color_fprintf(stdout, color, " %s\n", desc); @@ -122,22 +115,30 @@ static int hisi_ptt_8dw_kpt_desc(const unsigned char *buf, int pos) static void hisi_ptt_4dw_print_dw0(const unsigned char *buf, int pos) { const char *color = PERF_COLOR_BLUE; - union hisi_ptt_4dw dw0; + uint8_t byte; + uint32_t dw; int i; - dw0.value = *(uint32_t *)(buf + pos); + dw = get_unaligned_le32(buf + pos); printf("."); color_fprintf(stdout, color, " %08x: ", pos); - for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) - color_fprintf(stdout, color, "%02x ", buf[pos + i]); + for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) { + byte = (dw >> (24 - i * 8)) & 0xFF; + color_fprintf(stdout, color, "%02x ", byte); + } for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) color_fprintf(stdout, color, " "); color_fprintf(stdout, color, " %s %x %s %x %s %x %s %x %s %x %s %x %s %x %s %x\n", - "Format", dw0.format, "Type", dw0.type, "T9", dw0.t9, - "T8", dw0.t8, "TH", dw0.th, "SO", dw0.so, "Length", - dw0.len, "Time", dw0.time); + "Format", FIELD_GET(HISI_PTT_HEAD0_4DW_FORMAT, dw), + "Type", FIELD_GET(HISI_PTT_HEAD0_4DW_TYPE, dw), + "T9", FIELD_GET(HISI_PTT_HEAD0_4DW_T9, dw), + "T8", FIELD_GET(HISI_PTT_HEAD0_4DW_T8, dw), + "TH", FIELD_GET(HISI_PTT_HEAD0_4DW_TH, dw), + "SO", FIELD_GET(HISI_PTT_HEAD0_4DW_SO, dw), + "Length", FIELD_GET(HISI_PTT_HEAD0_4DW_LEN, dw), + "Time", FIELD_GET(HISI_PTT_HEAD0_4DW_TIME, dw)); } static int hisi_ptt_4dw_kpt_desc(const unsigned char *buf, int pos) diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h index e78f1b5bc836e..333eee02b19d2 100644 --- a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h @@ -9,12 +9,24 @@ #include #include +#include +#include #define HISI_PTT_8DW_CHECK_MASK GENMASK(31, 11) #define HISI_PTT_IS_8DW_PKT GENMASK(31, 11) #define HISI_PTT_MAX_SPACE_LEN 10 #define HISI_PTT_FIELD_LENTH 4 +/* Header DW0 fields for 4DW format */ +#define HISI_PTT_HEAD0_4DW_TIME GENMASK_U32(10, 0) +#define HISI_PTT_HEAD0_4DW_LEN GENMASK_U32(20, 11) +#define HISI_PTT_HEAD0_4DW_SO BIT_U32(21) +#define HISI_PTT_HEAD0_4DW_TH BIT_U32(22) +#define HISI_PTT_HEAD0_4DW_T8 BIT_U32(23) +#define HISI_PTT_HEAD0_4DW_T9 BIT_U32(24) +#define HISI_PTT_HEAD0_4DW_TYPE GENMASK_U32(29, 25) +#define HISI_PTT_HEAD0_4DW_FORMAT GENMASK_U32(31, 30) + enum hisi_ptt_pkt_type { HISI_PTT_4DW_PKT, HISI_PTT_8DW_PKT, diff --git a/tools/perf/util/intel-bts.c b/tools/perf/util/intel-bts.c index 0b18ebd13f7c8..02df3e4604891 100644 --- a/tools/perf/util/intel-bts.c +++ b/tools/perf/util/intel-bts.c @@ -830,7 +830,7 @@ int intel_bts_process_auxtrace_info(union perf_event *event, struct perf_session *session) { struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; - size_t min_sz = sizeof(u64) * INTEL_BTS_SNAPSHOT_MODE; + size_t min_sz = sizeof(u64) * (INTEL_BTS_SNAPSHOT_MODE + 1); struct intel_bts *bts; int err; diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index 56a9e439f5f8f..806cd173a0067 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -4430,7 +4430,7 @@ int intel_pt_process_auxtrace_info(union perf_event *event, struct perf_session *session) { struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; - size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS; + size_t min_sz = sizeof(u64) * (INTEL_PT_PER_CPU_MMAPS + 1); struct intel_pt *pt; void *info_end; __u64 *info; diff --git a/tools/perf/util/intel-tpebs.c b/tools/perf/util/intel-tpebs.c index bc3b79bfa01a7..b41171b5df77d 100644 --- a/tools/perf/util/intel-tpebs.c +++ b/tools/perf/util/intel-tpebs.c @@ -98,8 +98,9 @@ static int evsel__tpebs_start_perf_record(struct evsel *evsel) record_argv[i++] = "-o"; record_argv[i++] = PERF_DATA; - if (!perf_cpu_map__is_any_cpu_or_is_empty(evsel->evlist->core.user_requested_cpus)) { - cpu_map__snprint(evsel->evlist->core.user_requested_cpus, cpumap_buf, + if (!perf_cpu_map__is_any_cpu_or_is_empty( + evlist__core(evsel->evlist)->user_requested_cpus)) { + cpu_map__snprint(evlist__core(evsel->evlist)->user_requested_cpus, cpumap_buf, sizeof(cpumap_buf)); record_argv[i++] = "-C"; record_argv[i++] = cpumap_buf; @@ -176,7 +177,7 @@ static bool should_ignore_sample(const struct perf_sample *sample, const struct if (t->evsel->evlist == NULL) return true; - workload_pid = t->evsel->evlist->workload.pid; + workload_pid = evlist__workload_pid(t->evsel->evlist); if (workload_pid < 0 || workload_pid == sample_pid) return false; diff --git a/tools/perf/util/iostat.c b/tools/perf/util/iostat.c index b770bd473af71..c9d5028a47f39 100644 --- a/tools/perf/util/iostat.c +++ b/tools/perf/util/iostat.c @@ -4,7 +4,7 @@ enum iostat_mode_t iostat_mode = IOSTAT_NONE; -__weak int iostat_prepare(struct evlist *evlist __maybe_unused, +__weak int iostat_prepare(struct evlist **evlist __maybe_unused, struct perf_stat_config *config __maybe_unused) { return -1; diff --git a/tools/perf/util/iostat.h b/tools/perf/util/iostat.h index a4e7299c5c2fb..df8a241fbc32a 100644 --- a/tools/perf/util/iostat.h +++ b/tools/perf/util/iostat.h @@ -30,7 +30,7 @@ extern enum iostat_mode_t iostat_mode; typedef void (*iostat_print_counter_t)(struct perf_stat_config *, struct evsel *, void *); -int iostat_prepare(struct evlist *evlist, struct perf_stat_config *config); +int iostat_prepare(struct evlist **evlist, struct perf_stat_config *config); int iostat_parse(const struct option *opt, const char *str, int unset __maybe_unused); void iostat_list(struct evlist *evlist, struct perf_stat_config *config); diff --git a/tools/perf/util/kvm-stat.h b/tools/perf/util/kvm-stat.h index cdbd921a555f4..104f59262ec90 100644 --- a/tools/perf/util/kvm-stat.h +++ b/tools/perf/util/kvm-stat.h @@ -69,7 +69,7 @@ struct kvm_events_ops { }; struct exit_reasons_table { - unsigned long exit_code; + u64 exit_code; const char *reason; }; diff --git a/tools/perf/util/libbfd.c b/tools/perf/util/libbfd.c index c1c12308cc12f..118f973298ea8 100644 --- a/tools/perf/util/libbfd.c +++ b/tools/perf/util/libbfd.c @@ -15,6 +15,7 @@ #ifdef HAVE_LIBBPF_SUPPORT #include #include +#include #endif #include #include @@ -510,7 +511,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, char tpath[PATH_MAX]; size_t buf_size; int nr_skip = 0; - char *buf; + char *buf = NULL; bfd *bfdf; int ret; FILE *s; @@ -552,6 +553,11 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, info_linear = info_node->info_linear; sub_id = dso__bpf_prog(dso)->sub_id; + /* jited_prog_insns is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + ret = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + goto out; + } info.buffer = (void *)(uintptr_t)(info_linear->info.jited_prog_insns); info.buffer_length = info_linear->info.jited_prog_len; @@ -581,6 +587,12 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, if (disassemble == NULL) abort(); + /* jited_ksyms is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_KSYMS))) { + ret = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + goto out; + } + fflush(s); do { const struct bpf_line_info *linfo = NULL; @@ -609,7 +621,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, if (!annotate_opts.hide_src_code && srcline) { args->offset = -1; - args->line = strdup(srcline); + args->line = (char *)srcline; args->line_nr = 0; args->fileloc = NULL; args->ms->sym = sym; @@ -634,9 +646,12 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, ret = 0; out: - free(prog_linfo); + bpf_prog_linfo__free(prog_linfo); btf__free(btf); - fclose(s); + if (s) { + fclose(s); + free(buf); + } bfd_close(bfdf); return ret; #else diff --git a/tools/perf/util/libdw.c b/tools/perf/util/libdw.c index d5d2958902c0a..4ca7e7e4fbe93 100644 --- a/tools/perf/util/libdw.c +++ b/tools/perf/util/libdw.c @@ -82,13 +82,39 @@ struct libdw_a2l_cb_args { static int libdw_a2l_cb(Dwarf_Die *die, void *_args) { struct libdw_a2l_cb_args *args = _args; - struct symbol *inline_sym = new_inline_sym(args->dso, args->sym, die_name(die)); const char *call_fname = die_get_call_file(die); int call_lineno = die_get_call_lineno(die); char *call_srcline = srcline__unknown; - - if (!inline_sym) - goto abort_enomem; + struct symbol *inline_sym; + + if (dwarf_tag(die) == DW_TAG_subprogram && args->sym) { + /* + * cu_walk_functions_at() opens the walk with the + * containing DW_TAG_subprogram DIE (the non-inlined outer + * function). That's just the base symbol -- use it + * directly. Avoids a fragile name-vs-name compare in + * new_inline_sym() that misfires when GCC IPA passes + * (.isra/.constprop/.part/.cold) rename the ELF symbol + * while DWARF keeps the pre-clone linkage name, which + * left the outer frame spuriously tagged "(inlined)". + */ + inline_sym = args->sym; + } else { + /* + * Prefer DW_AT_linkage_name so C++ inline frames keep + * their namespace/class qualification. new_inline_sym() + * runs the name through dso__demangle_sym(), so the + * mangled linkage name is turned back into + * "Namespace::Class::method". Fall back to DW_AT_name + * (unqualified) when no linkage name is present, e.g. + * for C code or extern "C" functions. + */ + const char *funcname = die_get_linkage_name(die) ?: die_name(die); + + inline_sym = new_inline_sym(args->dso, args->sym, funcname); + if (!inline_sym) + goto abort_enomem; + } /* Assign caller information to the parent. */ if (call_fname) diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index 0d2ebf6a84bcf..f86b3b7df742e 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -333,16 +333,22 @@ struct machine *machines__findnew(struct machines *machines, pid_t pid) if ((pid != HOST_KERNEL_ID) && (pid != DEFAULT_GUEST_KERNEL_ID) && (symbol_conf.guestmount)) { - snprintf(path, sizeof(path), "%s/%d", symbol_conf.guestmount, pid); + if (snprintf(path, sizeof(path), "%s/%d", + symbol_conf.guestmount, pid) >= (int)sizeof(path)) { + pr_err("Guest path too long for pid %d\n", pid); + machine = NULL; + goto out; + } if (access(path, R_OK)) { static struct strlist *seen; if (!seen) seen = strlist__new(NULL, NULL); - if (!strlist__has_entry(seen, path)) { + if (!seen || !strlist__has_entry(seen, path)) { pr_err("Can't access file %s\n", path); - strlist__add(seen, path); + if (seen) + strlist__add(seen, path); } machine = NULL; goto out; @@ -1250,27 +1256,35 @@ int machines__create_guest_kernel_maps(struct machines *machines) for (i = 0; i < items; i++) { if (!isdigit(namelist[i]->d_name[0])) { /* Filter out . and .. */ + free(namelist[i]); continue; } + errno = 0; pid = (pid_t)strtol(namelist[i]->d_name, &endp, 10); if ((*endp != '\0') || (endp == namelist[i]->d_name) || (errno == ERANGE)) { pr_debug("invalid directory (%s). Skipping.\n", namelist[i]->d_name); + free(namelist[i]); + continue; + } + if (snprintf(path, sizeof(path), "%s/%s/proc/kallsyms", + symbol_conf.guestmount, + namelist[i]->d_name) >= (int)sizeof(path)) { + pr_debug("Guest kallsyms path too long for %s. Skipping.\n", + namelist[i]->d_name); + free(namelist[i]); continue; } - snprintf(path, sizeof(path), "%s/%s/proc/kallsyms", - symbol_conf.guestmount, - namelist[i]->d_name); - ret = access(path, R_OK); - if (ret) { + if (access(path, R_OK)) { pr_debug("Can't access file %s\n", path); - goto failure; + free(namelist[i]); + continue; } machines__create_kernel_maps(machines, pid); + free(namelist[i]); } -failure: free(namelist); } @@ -1411,8 +1425,10 @@ static int maps__set_modules_path_dir(struct maps *maps, char *path, size_t path return -1; } /* Bounds check, should never happen. */ - if (root_len >= path_size) - return -1; + if (root_len >= path_size) { + ret = -1; + goto out; + } path[root_len++] = '/'; while ((dent = io_dir__readdir(&iod)) != NULL) { if (io_dir__is_dir(&iod, dent)) { @@ -1921,7 +1937,8 @@ int machine__process_fork_event(struct machine *machine, union perf_event *event * (fork) event that would have removed the thread was lost. Assume the * latter case and continue on as best we can. */ - if (thread__pid(parent) != (pid_t)event->fork.ppid) { + if (parent != NULL && + thread__pid(parent) != (pid_t)event->fork.ppid) { dump_printf("removing erroneous parent thread %d/%d\n", thread__pid(parent), thread__tid(parent)); machine__remove_thread(machine, parent); diff --git a/tools/perf/util/map.h b/tools/perf/util/map.h index 979b3e11b9bcf..fb0279810ae99 100644 --- a/tools/perf/util/map.h +++ b/tools/perf/util/map.h @@ -2,14 +2,13 @@ #ifndef __PERF_MAP_H #define __PERF_MAP_H -#include -#include -#include -#include +#include #include #include -#include + +#include #include + #include struct dso; diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index c2ce3e53aaee7..5a60cb95e31c7 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -214,7 +214,7 @@ static void metric__free(struct metric *m) zfree(&m->metric_refs); expr__ctx_free(m->pctx); zfree(&m->modifier); - evlist__delete(m->evlist); + evlist__put(m->evlist); free(m); } @@ -1262,7 +1262,8 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu, struct expr_parse_ctx *ids, const char *modifier, bool group_events, const bool tool_events[TOOL_PMU__EVENT_MAX], struct evlist **out_evlist, - const char *filter_pmu) + const char *filter_pmu, + bool cputype_filter) { struct parse_events_error parse_error; struct evlist *parsed_evlist; @@ -1317,7 +1318,9 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu, pr_debug("Parsing metric events '%s'\n", events.buf); parse_events_error__init(&parse_error); ret = __parse_events(parsed_evlist, events.buf, filter_pmu, - &parse_error, fake_pmu, /*warn_if_reordered=*/false, + cputype_filter, + &parse_error, fake_pmu, + /*warn_if_reordered=*/false, /*fake_tp=*/false); if (ret) { parse_events_error__print(&parse_error, events.buf); @@ -1331,7 +1334,7 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu, parsed_evlist = NULL; err_out: parse_events_error__exit(&parse_error); - evlist__delete(parsed_evlist); + evlist__put(parsed_evlist); strbuf_release(&events); return ret; } @@ -1382,7 +1385,7 @@ static struct evsel *pick_display_evsel(struct list_head *metric_list, } static int parse_groups(struct evlist *perf_evlist, - const char *pmu, const char *str, + const char *pmu, bool cputype_filter, const char *str, bool metric_no_group, bool metric_no_merge, bool metric_no_threshold, @@ -1420,7 +1423,8 @@ static int parse_groups(struct evlist *perf_evlist, /*group_events=*/false, tool_events, &combined_evlist, - (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu); + (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu, + cputype_filter); } if (combined) expr__ctx_free(combined); @@ -1476,7 +1480,8 @@ static int parse_groups(struct evlist *perf_evlist, if (!metric_evlist) { ret = parse_ids(metric_no_merge, fake_pmu, m->pctx, m->modifier, m->group_events, tool_events, &m->evlist, - (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu); + (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu, + cputype_filter); if (ret) goto out; @@ -1490,7 +1495,7 @@ static int parse_groups(struct evlist *perf_evlist, goto out; } - me = metricgroup__lookup(&perf_evlist->metric_events, + me = metricgroup__lookup(evlist__metric_events(perf_evlist), pick_display_evsel(&metric_list, metric_events), /*create=*/true); @@ -1541,22 +1546,26 @@ static int parse_groups(struct evlist *perf_evlist, if (combined_evlist) { - evlist__splice_list_tail(perf_evlist, &combined_evlist->core.entries); - evlist__delete(combined_evlist); + evlist__splice_list_tail(perf_evlist, &evlist__core(combined_evlist)->entries); + evlist__put(combined_evlist); + combined_evlist = NULL; } list_for_each_entry(m, &metric_list, nd) { if (m->evlist) - evlist__splice_list_tail(perf_evlist, &m->evlist->core.entries); + evlist__splice_list_tail(perf_evlist, &evlist__core(m->evlist)->entries); } out: + if (combined_evlist) + evlist__put(combined_evlist); metricgroup__free_metrics(&metric_list); return ret; } int metricgroup__parse_groups(struct evlist *perf_evlist, const char *pmu, + bool cputype_filter, const char *str, bool metric_no_group, bool metric_no_merge, @@ -1570,16 +1579,17 @@ int metricgroup__parse_groups(struct evlist *perf_evlist, if (hardware_aware_grouping) pr_debug("Use hardware aware grouping instead of traditional metric grouping method\n"); - return parse_groups(perf_evlist, pmu, str, metric_no_group, metric_no_merge, + return parse_groups(perf_evlist, pmu, cputype_filter, str, metric_no_group, metric_no_merge, metric_no_threshold, user_requested_cpu_list, system_wide, /*fake_pmu=*/false, table); } int metricgroup__parse_groups_test(struct evlist *evlist, const struct pmu_metrics_table *table, - const char *str) + const char *str, + bool cputype_filter) { - return parse_groups(evlist, "all", str, + return parse_groups(evlist, "all", cputype_filter, str, /*metric_no_group=*/false, /*metric_no_merge=*/false, /*metric_no_threshold=*/false, @@ -1683,8 +1693,10 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, new_expr->metric_expr = old_expr->metric_expr; new_expr->metric_threshold = old_expr->metric_threshold; new_expr->metric_name = strdup(old_expr->metric_name); - if (!new_expr->metric_name) + if (!new_expr->metric_name) { + free(new_expr); return -ENOMEM; + } new_expr->metric_unit = old_expr->metric_unit; new_expr->runtime = old_expr->runtime; @@ -1697,6 +1709,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, alloc_size = sizeof(*new_expr->metric_refs); new_expr->metric_refs = calloc(nr + 1, alloc_size); if (!new_expr->metric_refs) { + zfree(&new_expr->metric_name); free(new_expr); return -ENOMEM; } @@ -1713,6 +1726,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, alloc_size = sizeof(*new_expr->metric_events); new_expr->metric_events = calloc(nr + 1, alloc_size); if (!new_expr->metric_events) { + zfree(&new_expr->metric_name); zfree(&new_expr->metric_refs); free(new_expr); return -ENOMEM; @@ -1723,6 +1737,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, evsel = old_expr->metric_events[idx]; evsel = evlist__find_evsel(evlist, evsel->core.idx); if (evsel == NULL) { + zfree(&new_expr->metric_name); zfree(&new_expr->metric_events); zfree(&new_expr->metric_refs); free(new_expr); diff --git a/tools/perf/util/metricgroup.h b/tools/perf/util/metricgroup.h index 4be6bfc13c467..6a66f14dd01bd 100644 --- a/tools/perf/util/metricgroup.h +++ b/tools/perf/util/metricgroup.h @@ -71,6 +71,7 @@ struct metric_event *metricgroup__lookup(struct rblist *metric_events, bool create); int metricgroup__parse_groups(struct evlist *perf_evlist, const char *pmu, + bool cputype_filter, const char *str, bool metric_no_group, bool metric_no_merge, @@ -80,7 +81,8 @@ int metricgroup__parse_groups(struct evlist *perf_evlist, bool hardware_aware_grouping); int metricgroup__parse_groups_test(struct evlist *evlist, const struct pmu_metrics_table *table, - const char *str); + const char *str, + bool cputype_filter); int metricgroup__for_each_metric(const struct pmu_metrics_table *table, pmu_metric_iter_fn fn, void *data); diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 943569e82b82f..cc7ad331a49f3 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -429,6 +429,9 @@ bool parse_events__filter_pmu(const struct parse_events_state *parse_state, if (parse_state->pmu_filter == NULL) return false; + if (parse_state->cputype_filter && !pmu->is_core) + return false; + return perf_pmu__wildcard_match(pmu, parse_state->pmu_filter) == 0; } @@ -2288,18 +2291,20 @@ static int parse_events__sort_events_and_fix_groups(struct list_head *list) return (idx_changed || num_leaders != orig_num_leaders) ? 1 : 0; } -int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filter, +int __parse_events(struct evlist *evlist, const char *str, + const char *pmu_filter, bool cputype_filter, struct parse_events_error *err, bool fake_pmu, bool warn_if_reordered, bool fake_tp) { struct parse_events_state parse_state = { - .list = LIST_HEAD_INIT(parse_state.list), - .idx = evlist->core.nr_entries, - .error = err, - .stoken = PE_START_EVENTS, - .fake_pmu = fake_pmu, - .fake_tp = fake_tp, - .pmu_filter = pmu_filter, + .list = LIST_HEAD_INIT(parse_state.list), + .idx = evlist__nr_entries(evlist), + .error = err, + .stoken = PE_START_EVENTS, + .fake_pmu = fake_pmu, + .fake_tp = fake_tp, + .pmu_filter = pmu_filter, + .cputype_filter = cputype_filter, .match_legacy_cache_terms = true, }; int ret, ret2; @@ -2312,15 +2317,15 @@ int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filte } ret2 = parse_events__sort_events_and_fix_groups(&parse_state.list); - if (ret2 < 0) - return ret; + if (ret2 < 0 && !ret) + ret = ret2; /* * Add list to the evlist even with errors to allow callers to clean up. */ evlist__splice_list_tail(evlist, &parse_state.list); - if (ret2 && warn_if_reordered && !parse_state.wild_card_pmus) { + if (ret2 > 0 && warn_if_reordered && !parse_state.wild_card_pmus) { evlist__uniquify_evsel_names(evlist, &stat_config); pr_warning("WARNING: events were regrouped to match PMUs\n"); @@ -2343,7 +2348,7 @@ int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filte /* * There are 2 users - builtin-record and builtin-test objects. - * Both call evlist__delete in case of error, so we dont + * Both call evlist__put in case of error, so we dont * need to bother. */ return ret; @@ -2518,8 +2523,9 @@ int parse_events_option(const struct option *opt, const char *str, int ret; parse_events_error__init(&err); - ret = __parse_events(*args->evlistp, str, args->pmu_filter, &err, - /*fake_pmu=*/false, /*warn_if_reordered=*/true, + ret = __parse_events(*args->evlistp, str, args->pmu_filter, + args->cputype_filter, &err, /*fake_pmu=*/false, + /*warn_if_reordered=*/true, /*fake_tp=*/false); if (ret) { @@ -2546,7 +2552,7 @@ int parse_events_option_new_evlist(const struct option *opt, const char *str, in } ret = parse_events_option(opt, str, unset); if (ret) { - evlist__delete(*args->evlistp); + evlist__put(*args->evlistp); *args->evlistp = NULL; } @@ -2568,7 +2574,7 @@ foreach_evsel_in_last_glob(struct evlist *evlist, * * So no need to WARN here, let *func do this. */ - if (evlist->core.nr_entries > 0) + if (evlist__nr_entries(evlist) > 0) last = evlist__last(evlist); do { @@ -2578,7 +2584,7 @@ foreach_evsel_in_last_glob(struct evlist *evlist, if (!last) return 0; - if (last->core.node.prev == &evlist->core.entries) + if (last->core.node.prev == &evlist__core(evlist)->entries) return 0; last = list_entry(last->core.node.prev, struct evsel, core.node); } while (!last->cmdline_group_boundary); diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h index 3577ab2137304..b14c832b03a1e 100644 --- a/tools/perf/util/parse-events.h +++ b/tools/perf/util/parse-events.h @@ -26,20 +26,23 @@ const char *event_type(size_t type); struct parse_events_option_args { struct evlist **evlistp; const char *pmu_filter; + bool cputype_filter; }; int parse_events_option(const struct option *opt, const char *str, int unset); int parse_events_option_new_evlist(const struct option *opt, const char *str, int unset); -__attribute__((nonnull(1, 2, 4))) -int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filter, - struct parse_events_error *error, bool fake_pmu, - bool warn_if_reordered, bool fake_tp); +__attribute__((nonnull(1, 2, 5))) int +__parse_events(struct evlist *evlist, const char *str, const char *pmu_filter, + bool cputype_filter, struct parse_events_error *error, + bool fake_pmu, bool warn_if_reordered, bool fake_tp); __attribute__((nonnull(1, 2, 3))) static inline int parse_events(struct evlist *evlist, const char *str, struct parse_events_error *err) { - return __parse_events(evlist, str, /*pmu_filter=*/NULL, err, /*fake_pmu=*/false, - /*warn_if_reordered=*/true, /*fake_tp=*/false); + return __parse_events(evlist, str, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, err, /*fake_pmu=*/false, + /*warn_if_reordered=*/true, + /*fake_tp=*/false); } int parse_event(struct evlist *evlist, const char *str); @@ -161,6 +164,8 @@ struct parse_events_state { bool fake_tp; /* If non-null, when wildcard matching only match the given PMU. */ const char *pmu_filter; + /* If true, the pmu_filter was set by --cputype option. */ + bool cputype_filter; /* Should PE_LEGACY_NAME tokens be generated for config terms? */ bool match_legacy_cache_terms; /* Were multiple PMUs scanned to find events? */ diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index c194de5ec1ec7..b531b1f0ceb33 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -47,7 +47,7 @@ static void free_list_evsel(struct list_head* list_evsel) list_for_each_entry_safe(evsel, tmp, list_evsel, core.node) { list_del_init(&evsel->core.node); - evsel__delete(evsel); + evsel__put(evsel); } free(list_evsel); } diff --git a/tools/perf/util/perf_api_probe.c b/tools/perf/util/perf_api_probe.c index 6ecf38314f01c..f61c4ec52827b 100644 --- a/tools/perf/util/perf_api_probe.c +++ b/tools/perf/util/perf_api_probe.c @@ -1,14 +1,18 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include "perf_api_probe.h" -#include "perf-sys.h" -#include "util/cloexec.h" -#include "util/evlist.h" -#include "util/evsel.h" -#include "util/parse-events.h" -#include "util/perf_api_probe.h" -#include #include +#include + +#include "cloexec.h" +#include "evlist.h" +#include "evsel.h" +#include "parse-events.h" +#include "perf-sys.h" +#include "pmu.h" +#include "pmus.h" + typedef void (*setup_probe_fn_t)(struct evsel *evsel); static int perf_do_probe_api(setup_probe_fn_t fn, struct perf_cpu cpu, const char *str) @@ -53,7 +57,7 @@ static int perf_do_probe_api(setup_probe_fn_t fn, struct perf_cpu cpu, const cha err = 0; out_delete: - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/util/pfm.c b/tools/perf/util/pfm.c index d9043f4afbe7b..f80d6b0df47ae 100644 --- a/tools/perf/util/pfm.c +++ b/tools/perf/util/pfm.c @@ -85,7 +85,7 @@ int parse_libpfm_events_option(const struct option *opt, const char *str, } pmu = perf_pmus__find_by_type((unsigned int)attr.type); - evsel = parse_events__add_event(evlist->core.nr_entries, + evsel = parse_events__add_event(evlist__nr_entries(evlist), &attr, q, /*metric_id=*/NULL, pmu); if (evsel == NULL) @@ -159,7 +159,7 @@ static bool is_libpfm_event_supported(const char *name, struct perf_cpu_map *cpu result = false; evsel__close(evsel); - evsel__delete(evsel); + evsel__put(evsel); return result; } diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index a550f030b85df..836e3b5615cd8 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -2660,8 +2660,12 @@ bool perf_pmu__wildcard_match(const struct perf_pmu *pmu, const char *wildcard_t pmu->name, pmu->alias_name, }; - bool need_fnmatch = strisglob(wildcard_to_match); + bool need_fnmatch; + if (pmu->is_core && !strcmp(wildcard_to_match, "default_core")) + return true; + + need_fnmatch = strisglob(wildcard_to_match); if (!strncmp(wildcard_to_match, "uncore_", 7)) wildcard_to_match += 7; diff --git a/tools/perf/util/pmus.c b/tools/perf/util/pmus.c index 5e3f571450fe7..e0a4cb2428ca4 100644 --- a/tools/perf/util/pmus.c +++ b/tools/perf/util/pmus.c @@ -150,6 +150,8 @@ struct perf_pmu *perf_pmus__find(const char *name) bool core_pmu; unsigned int to_read_pmus = 0; + if (!strcmp(name, "default_core")) + return perf_pmus__find_core_pmu(); /* * Once PMU is loaded it stays in the list, * so we keep us from multiple reading/parsing diff --git a/tools/perf/util/powerpc-vpadtl.c b/tools/perf/util/powerpc-vpadtl.c index 710f3093f3f90..af6783cfdb53d 100644 --- a/tools/perf/util/powerpc-vpadtl.c +++ b/tools/perf/util/powerpc-vpadtl.c @@ -196,7 +196,7 @@ static int powerpc_vpadtl_sample(struct powerpc_vpadtl_entry *record, sample.cpumode = PERF_RECORD_MISC_KERNEL; sample.time = save; sample.raw_data = record; - sample.raw_size = sizeof(record); + sample.raw_size = sizeof(*record); event.sample.header.type = PERF_RECORD_SAMPLE; event.sample.header.misc = sample.cpumode; event.sample.header.size = sizeof(struct perf_event_header); diff --git a/tools/perf/util/print-events.c b/tools/perf/util/print-events.c index cb27e2898aa05..0242243681b6b 100644 --- a/tools/perf/util/print-events.c +++ b/tools/perf/util/print-events.c @@ -174,7 +174,7 @@ bool is_event_supported(u8 type, u64 config) } evsel__close(evsel); - evsel__delete(evsel); + evsel__put(evsel); } perf_thread_map__put(tmap); diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c index cc1019d29a5d0..b6c4b3de5b4d2 100644 --- a/tools/perf/util/python.c +++ b/tools/perf/util/python.c @@ -1,30 +1,37 @@ // SPDX-License-Identifier: GPL-2.0 +#define PY_SSIZE_T_CLEAN #include -#include + #include -#include + #include +#include + +#include #include -#ifdef HAVE_LIBTRACEEVENT -#include -#endif #include +#include + #include "callchain.h" #include "counts.h" +#include "event.h" #include "evlist.h" #include "evsel.h" -#include "event.h" #include "expr.h" +#include "metricgroup.h" +#include "mmap.h" +#include "pmus.h" #include "print_binary.h" #include "record.h" #include "strbuf.h" #include "thread_map.h" #include "tp_pmu.h" #include "trace-event.h" -#include "metricgroup.h" -#include "mmap.h" #include "util/sample.h" -#include + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif PyMODINIT_FUNC PyInit_perf(void); @@ -38,6 +45,22 @@ PyMODINIT_FUNC PyInit_perf(void); offsetof(struct pyrf_event, sample) + offsetof(struct perf_sample, member), \ 0, help } +#define CHECK_INITIALIZED(ptr, msg) \ + do { \ + if (!(ptr)) { \ + PyErr_SetString(PyExc_ValueError, msg " not initialized"); \ + return NULL; \ + } \ + } while (0) + +#define CHECK_INITIALIZED_INT(ptr, msg) \ + do { \ + if (!(ptr)) { \ + PyErr_SetString(PyExc_ValueError, msg " not initialized"); \ + return -1; \ + } \ + } while (0) + struct pyrf_event { PyObject_HEAD struct evsel *evsel; @@ -271,8 +294,9 @@ static PyMemberDef pyrf_sample_event__members[] = { static void pyrf_sample_event__delete(struct pyrf_event *pevent) { + evsel__put(pevent->evsel); perf_sample__exit(&pevent->sample); - Py_TYPE(pevent)->tp_free((PyObject*)pevent); + Py_TYPE(pevent)->tp_free((PyObject *)pevent); } static PyObject *pyrf_sample_event__repr(const struct pyrf_event *pevent) @@ -503,8 +527,10 @@ static PyObject *pyrf_event__new(const union perf_event *event) ptype = pyrf_event__type[event->header.type]; pevent = PyObject_New(struct pyrf_event, ptype); - if (pevent != NULL) + if (pevent != NULL) { memcpy(&pevent->event, event, event->header.size); + pevent->evsel = NULL; + } return (PyObject *)pevent; } @@ -872,11 +898,11 @@ static void pyrf_counts_values__delete(struct pyrf_counts_values *pcounts_values 0, help } static PyMemberDef pyrf_counts_values_members[] = { - counts_values_member_def(val, T_ULONG, "Value of event"), - counts_values_member_def(ena, T_ULONG, "Time for which enabled"), - counts_values_member_def(run, T_ULONG, "Time for which running"), - counts_values_member_def(id, T_ULONG, "Unique ID for an event"), - counts_values_member_def(lost, T_ULONG, "Num of lost samples"), + counts_values_member_def(val, T_ULONGLONG, "Value of event"), + counts_values_member_def(ena, T_ULONGLONG, "Time for which enabled"), + counts_values_member_def(run, T_ULONGLONG, "Time for which running"), + counts_values_member_def(id, T_ULONGLONG, "Unique ID for an event"), + counts_values_member_def(lost, T_ULONGLONG, "Num of lost samples"), { .name = NULL, }, }; @@ -886,8 +912,15 @@ static PyObject *pyrf_counts_values_get_values(struct pyrf_counts_values *self, if (!vals) return NULL; - for (int i = 0; i < 5; i++) - PyList_SetItem(vals, i, PyLong_FromLong(self->values.values[i])); + for (int i = 0; i < 5; i++) { + PyObject *val = PyLong_FromUnsignedLongLong(self->values.values[i]); + + if (!val) { + Py_DECREF(vals); + return NULL; + } + PyList_SetItem(vals, i, val); + } return vals; } @@ -898,19 +931,34 @@ static int pyrf_counts_values_set_values(struct pyrf_counts_values *self, PyObje Py_ssize_t size; PyObject *item = NULL; + if (list == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + if (!PyList_Check(list)) { PyErr_SetString(PyExc_TypeError, "Value assigned must be a list"); return -1; } size = PyList_Size(list); + if (size != 5) { + PyErr_SetString(PyExc_ValueError, "List must have exactly 5 entries"); + return -1; + } + for (Py_ssize_t i = 0; i < size; i++) { + unsigned long long val; + item = PyList_GetItem(list, i); if (!PyLong_Check(item)) { PyErr_SetString(PyExc_TypeError, "List members should be numbers"); return -1; } - self->values.values[i] = PyLong_AsLong(item); + val = PyLong_AsUnsignedLongLong(item); + if (val == (unsigned long long)-1 && PyErr_Occurred()) + return -1; + self->values.values[i] = val; } return 0; @@ -942,7 +990,7 @@ static int pyrf_counts_values__setup_types(void) struct pyrf_evsel { PyObject_HEAD - struct evsel evsel; + struct evsel *evsel; }; static int pyrf_evsel__init(struct pyrf_evsel *pevsel, @@ -983,6 +1031,7 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel, "bp_type", "bp_addr", "bp_len", + "idx", NULL }; u64 sample_period = 0; @@ -1004,11 +1053,11 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel, watermark = 0, precise_ip = 0, mmap_data = 0, - sample_id_all = 1; - int idx = 0; + sample_id_all = 1, + idx = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|iKiKKiiiiiiiiiiiiiiiiiiiiiiKK", kwlist, + "|iKiKKiiiiiiiiiiiiiiiiiiiiiiKKi", kwlist, &attr.type, &attr.config, &attr.sample_freq, &sample_period, &attr.sample_type, &attr.read_format, &disabled, &inherit, @@ -1050,35 +1099,52 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel, attr.sample_id_all = sample_id_all; attr.size = sizeof(attr); - evsel__init(&pevsel->evsel, &attr, idx); + evsel__put(pevsel->evsel); + pevsel->evsel = evsel__new(&attr); + if (!pevsel->evsel) { + PyErr_NoMemory(); + return -1; + } return 0; } static void pyrf_evsel__delete(struct pyrf_evsel *pevsel) { - evsel__exit(&pevsel->evsel); + evsel__put(pevsel->evsel); Py_TYPE(pevsel)->tp_free((PyObject*)pevsel); } static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel, PyObject *args, PyObject *kwargs) { - struct evsel *evsel = &pevsel->evsel; + struct evsel *evsel = pevsel->evsel; struct perf_cpu_map *cpus = NULL; struct perf_thread_map *threads = NULL; PyObject *pcpus = NULL, *pthreads = NULL; int group = 0, inherit = 0; static char *kwlist[] = { "cpus", "threads", "group", "inherit", NULL }; + CHECK_INITIALIZED(evsel, "evsel"); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OOii", kwlist, &pcpus, &pthreads, &group, &inherit)) return NULL; - if (pthreads != NULL) + if (pthreads != NULL && pthreads != Py_None) { + if (!PyObject_TypeCheck(pthreads, &pyrf_thread_map__type)) { + PyErr_SetString(PyExc_TypeError, "threads must be a thread_map"); + return NULL; + } threads = ((struct pyrf_thread_map *)pthreads)->threads; + } - if (pcpus != NULL) + if (pcpus != NULL && pcpus != Py_None) { + if (!PyObject_TypeCheck(pcpus, &pyrf_cpu_map__type)) { + PyErr_SetString(PyExc_TypeError, "cpus must be a cpu_map"); + return NULL; + } cpus = ((struct pyrf_cpu_map *)pcpus)->cpus; + } evsel->core.attr.inherit = inherit; /* @@ -1096,21 +1162,26 @@ static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel, static PyObject *pyrf_evsel__cpus(struct pyrf_evsel *pevsel) { - struct pyrf_cpu_map *pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); + struct pyrf_cpu_map *pcpu_map; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); if (pcpu_map) - pcpu_map->cpus = perf_cpu_map__get(pevsel->evsel.core.cpus); + pcpu_map->cpus = perf_cpu_map__get(pevsel->evsel->core.cpus); return (PyObject *)pcpu_map; } static PyObject *pyrf_evsel__threads(struct pyrf_evsel *pevsel) { - struct pyrf_thread_map *pthread_map = - PyObject_New(struct pyrf_thread_map, &pyrf_thread_map__type); + struct pyrf_thread_map *pthread_map; + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + pthread_map = PyObject_New(struct pyrf_thread_map, &pyrf_thread_map__type); if (pthread_map) - pthread_map->threads = perf_thread_map__get(pevsel->evsel.core.threads); + pthread_map->threads = perf_thread_map__get(pevsel->evsel->core.threads); return (PyObject *)pthread_map; } @@ -1144,14 +1215,12 @@ static int evsel__ensure_counts(struct evsel *evsel) static PyObject *pyrf_evsel__read(struct pyrf_evsel *pevsel, PyObject *args, PyObject *kwargs) { - struct evsel *evsel = &pevsel->evsel; + struct evsel *evsel = pevsel->evsel; int cpu = 0, cpu_idx, thread = 0, thread_idx; struct perf_counts_values *old_count, *new_count; - struct pyrf_counts_values *count_values = PyObject_New(struct pyrf_counts_values, - &pyrf_counts_values__type); + struct pyrf_counts_values *count_values; - if (!count_values) - return NULL; + CHECK_INITIALIZED(evsel, "evsel"); if (!PyArg_ParseTuple(args, "ii", &cpu, &thread)) return NULL; @@ -1171,6 +1240,10 @@ static PyObject *pyrf_evsel__read(struct pyrf_evsel *pevsel, if (evsel__ensure_counts(evsel)) return PyErr_NoMemory(); + count_values = PyObject_New(struct pyrf_counts_values, &pyrf_counts_values__type); + if (!count_values) + return NULL; + /* Set up pointers to the old and newly read counter values. */ old_count = perf_counts(evsel->prev_raw_counts, cpu_idx, thread_idx); new_count = perf_counts(evsel->counts, cpu_idx, thread_idx); @@ -1189,7 +1262,10 @@ static PyObject *pyrf_evsel__read(struct pyrf_evsel *pevsel, static PyObject *pyrf_evsel__str(PyObject *self) { struct pyrf_evsel *pevsel = (void *)self; - struct evsel *evsel = &pevsel->evsel; + struct evsel *evsel = pevsel->evsel; + + if (!evsel) + return PyUnicode_FromString("evsel(uninitialized)"); return PyUnicode_FromFormat("evsel(%s/%s/)", evsel__pmu_name(evsel), evsel__name(evsel)); } @@ -1222,30 +1298,292 @@ static PyMethodDef pyrf_evsel__methods[] = { { .ml_name = NULL, } }; -#define evsel_member_def(member, ptype, help) \ - { #member, ptype, \ - offsetof(struct pyrf_evsel, evsel.member), \ - 0, help } +static PyObject *pyrf_evsel__get_tracking(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; -#define evsel_attr_member_def(member, ptype, help) \ - { #member, ptype, \ - offsetof(struct pyrf_evsel, evsel.core.attr.member), \ - 0, help } + CHECK_INITIALIZED(pevsel->evsel, "evsel"); -static PyMemberDef pyrf_evsel__members[] = { - evsel_member_def(tracking, T_BOOL, "tracking event."), - evsel_attr_member_def(type, T_UINT, "attribute type."), - evsel_attr_member_def(size, T_UINT, "attribute size."), - evsel_attr_member_def(config, T_ULONGLONG, "attribute config."), - evsel_attr_member_def(sample_period, T_ULONGLONG, "attribute sample_period."), - evsel_attr_member_def(sample_type, T_ULONGLONG, "attribute sample_type."), - evsel_attr_member_def(read_format, T_ULONGLONG, "attribute read_format."), - evsel_attr_member_def(wakeup_events, T_UINT, "attribute wakeup_events."), - { .name = NULL, }, + if (pevsel->evsel->tracking) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; +} + +static int pyrf_evsel__set_tracking(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + int is_true; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + is_true = PyObject_IsTrue(val); + if (is_true < 0) + return -1; + + pevsel->evsel->tracking = is_true; + return 0; +} + +static int pyrf_evsel__set_attr_config(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLongLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.config = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_config(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.config); +} + +static int pyrf_evsel__set_attr_read_format(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLongLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.read_format = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_read_format(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.read_format); +} + +static int pyrf_evsel__set_attr_sample_period(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLongLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.sample_period = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_sample_period(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.sample_period); +} + +static int pyrf_evsel__set_attr_sample_type(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLongLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.sample_type = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_sample_type(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.sample_type); +} + +static PyObject *pyrf_evsel__get_attr_size(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.size); +} + +static int pyrf_evsel__set_attr_type(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.type = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_type(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.type); +} + +static int pyrf_evsel__set_attr_wakeup_events(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + unsigned long new_val; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + + new_val = PyLong_AsUnsignedLong(val); + if (PyErr_Occurred()) + return -1; + + pevsel->evsel->core.attr.wakeup_events = new_val; + return 0; +} + +static PyObject *pyrf_evsel__get_attr_wakeup_events(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.wakeup_events); +} + +static PyGetSetDef pyrf_evsel__getset[] = { + { + .name = "tracking", + .get = pyrf_evsel__get_tracking, + .set = pyrf_evsel__set_tracking, + .doc = "tracking event.", + }, + { + .name = "config", + .get = pyrf_evsel__get_attr_config, + .set = pyrf_evsel__set_attr_config, + .doc = "attribute config.", + }, + { + .name = "read_format", + .get = pyrf_evsel__get_attr_read_format, + .set = pyrf_evsel__set_attr_read_format, + .doc = "attribute read_format.", + }, + { + .name = "sample_period", + .get = pyrf_evsel__get_attr_sample_period, + .set = pyrf_evsel__set_attr_sample_period, + .doc = "attribute sample_period.", + }, + { + .name = "sample_type", + .get = pyrf_evsel__get_attr_sample_type, + .set = pyrf_evsel__set_attr_sample_type, + .doc = "attribute sample_type.", + }, + { + .name = "size", + .get = pyrf_evsel__get_attr_size, + .doc = "attribute size.", + }, + { + .name = "type", + .get = pyrf_evsel__get_attr_type, + .set = pyrf_evsel__set_attr_type, + .doc = "attribute type.", + }, + { + .name = "wakeup_events", + .get = pyrf_evsel__get_attr_wakeup_events, + .set = pyrf_evsel__set_attr_wakeup_events, + .doc = "attribute wakeup_events.", + }, + { .name = NULL}, }; static const char pyrf_evsel__doc[] = PyDoc_STR("perf event selector list object."); +static PyObject *pyrf_evsel__getattro(struct pyrf_evsel *pevsel, PyObject *attr_name) +{ + if (!pevsel->evsel) { + PyErr_SetString(PyExc_ValueError, "evsel not initialized"); + return NULL; + } + return PyObject_GenericGetAttr((PyObject *) pevsel, attr_name); +} + +static int pyrf_evsel__setattro(struct pyrf_evsel *pevsel, PyObject *attr_name, PyObject *value) +{ + if (!pevsel->evsel) { + PyErr_SetString(PyExc_ValueError, "evsel not initialized"); + return -1; + } + return PyObject_GenericSetAttr((PyObject *) pevsel, attr_name, value); +} + static PyTypeObject pyrf_evsel__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.evsel", @@ -1253,23 +1591,35 @@ static PyTypeObject pyrf_evsel__type = { .tp_dealloc = (destructor)pyrf_evsel__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_evsel__doc, - .tp_members = pyrf_evsel__members, + .tp_getset = pyrf_evsel__getset, .tp_methods = pyrf_evsel__methods, .tp_init = (initproc)pyrf_evsel__init, .tp_str = pyrf_evsel__str, .tp_repr = pyrf_evsel__str, + .tp_getattro = (getattrofunc) pyrf_evsel__getattro, + .tp_setattro = (setattrofunc) pyrf_evsel__setattro, }; +static PyObject *pyrf_evsel__new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + struct pyrf_evsel *pevsel; + + pevsel = (struct pyrf_evsel *)PyType_GenericNew(type, args, kwargs); + if (pevsel) + pevsel->evsel = NULL; + return (PyObject *)pevsel; +} + static int pyrf_evsel__setup_types(void) { - pyrf_evsel__type.tp_new = PyType_GenericNew; + pyrf_evsel__type.tp_new = pyrf_evsel__new; return PyType_Ready(&pyrf_evsel__type); } struct pyrf_evlist { PyObject_HEAD - struct evlist evlist; + struct evlist *evlist; }; static int pyrf_evlist__init(struct pyrf_evlist *pevlist, @@ -1279,40 +1629,55 @@ static int pyrf_evlist__init(struct pyrf_evlist *pevlist, struct perf_cpu_map *cpus; struct perf_thread_map *threads; - if (!PyArg_ParseTuple(args, "OO", &pcpus, &pthreads)) + if (!PyArg_ParseTuple(args, "O!O!", + &pyrf_cpu_map__type, &pcpus, + &pyrf_thread_map__type, &pthreads)) return -1; + evlist__put(pevlist->evlist); + pevlist->evlist = evlist__new(); + if (!pevlist->evlist) { + PyErr_NoMemory(); + return -1; + } threads = ((struct pyrf_thread_map *)pthreads)->threads; cpus = ((struct pyrf_cpu_map *)pcpus)->cpus; - evlist__init(&pevlist->evlist, cpus, threads); + perf_evlist__set_maps(evlist__core(pevlist->evlist), cpus, threads); + return 0; } static void pyrf_evlist__delete(struct pyrf_evlist *pevlist) { - evlist__exit(&pevlist->evlist); + evlist__put(pevlist->evlist); Py_TYPE(pevlist)->tp_free((PyObject*)pevlist); } static PyObject *pyrf_evlist__all_cpus(struct pyrf_evlist *pevlist) { - struct pyrf_cpu_map *pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); + struct pyrf_cpu_map *pcpu_map; + + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); if (pcpu_map) - pcpu_map->cpus = perf_cpu_map__get(pevlist->evlist.core.all_cpus); + pcpu_map->cpus = perf_cpu_map__get(evlist__core(pevlist->evlist)->all_cpus); return (PyObject *)pcpu_map; } static PyObject *pyrf_evlist__metrics(struct pyrf_evlist *pevlist) { - PyObject *list = PyList_New(/*len=*/0); + PyObject *list; struct rb_node *node; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + list = PyList_New(/*len=*/0); if (!list) return NULL; - for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); node; + for (node = rb_first_cached(&evlist__metric_events(pevlist->evlist)->entries); node; node = rb_next(node)) { struct metric_event *me = container_of(node, struct metric_event, nd); struct list_head *pos; @@ -1415,10 +1780,12 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist, double result = 0; struct evsel *metric_evsel = NULL; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + if (!PyArg_ParseTuple(args, "sii", &metric, &cpu, &thread)) return NULL; - for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); + for (node = rb_first_cached(&evlist__metric_events(pevlist->evlist)->entries); mexp == NULL && node; node = rb_next(node)) { struct metric_event *me = container_of(node, struct metric_event, nd); @@ -1434,7 +1801,7 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist, if (e->metric_events[0] == NULL) continue; - evlist__for_each_entry(&pevlist->evlist, pos2) { + evlist__for_each_entry(pevlist->evlist, pos2) { if (pos2->metric_leader != e->metric_events[0]) continue; cpu_idx = perf_cpu_map__idx(pos2->core.cpus, @@ -1479,15 +1846,18 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist, static PyObject *pyrf_evlist__mmap(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; static char *kwlist[] = { "pages", "overwrite", NULL }; int pages = 128, overwrite = false; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ii", kwlist, &pages, &overwrite)) return NULL; - if (evlist__mmap(evlist, pages) < 0) { + if (evlist__do_mmap(evlist, pages) < 0) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -1499,10 +1869,13 @@ static PyObject *pyrf_evlist__mmap(struct pyrf_evlist *pevlist, static PyObject *pyrf_evlist__poll(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; static char *kwlist[] = { "timeout", NULL }; int timeout = -1, n; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i", kwlist, &timeout)) return NULL; @@ -1519,13 +1892,20 @@ static PyObject *pyrf_evlist__get_pollfd(struct pyrf_evlist *pevlist, PyObject *args __maybe_unused, PyObject *kwargs __maybe_unused) { - struct evlist *evlist = &pevlist->evlist; - PyObject *list = PyList_New(0); + struct evlist *evlist; + PyObject *list; int i; - for (i = 0; i < evlist->core.pollfd.nr; ++i) { + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; + list = PyList_New(0); + if (!list) + return NULL; + + for (i = 0; i < evlist__core(evlist)->pollfd.nr; ++i) { PyObject *file; - file = PyFile_FromFd(evlist->core.pollfd.entries[i].fd, "perf", "r", -1, + file = PyFile_FromFd(evlist__core(evlist)->pollfd.entries[i].fd, "perf", "r", -1, NULL, NULL, NULL, 0); if (file == NULL) goto free_list; @@ -1540,6 +1920,7 @@ static PyObject *pyrf_evlist__get_pollfd(struct pyrf_evlist *pevlist, return list; free_list: + Py_XDECREF(list); return PyErr_NoMemory(); } @@ -1548,27 +1929,33 @@ static PyObject *pyrf_evlist__add(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs __maybe_unused) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; PyObject *pevsel; struct evsel *evsel; - if (!PyArg_ParseTuple(args, "O", &pevsel)) + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; + if (!PyArg_ParseTuple(args, "O!", &pyrf_evsel__type, &pevsel)) return NULL; - Py_INCREF(pevsel); - evsel = &((struct pyrf_evsel *)pevsel)->evsel; - evsel->core.idx = evlist->core.nr_entries; - evlist__add(evlist, evsel); + CHECK_INITIALIZED(((struct pyrf_evsel *)pevsel)->evsel, "evsel"); + + evsel = ((struct pyrf_evsel *)pevsel)->evsel; + CHECK_INITIALIZED(evsel, "evsel"); + + evsel->core.idx = evlist__nr_entries(evlist); + evlist__add(evlist, evsel__get(evsel)); - return Py_BuildValue("i", evlist->core.nr_entries); + return Py_BuildValue("i", evlist__nr_entries(evlist)); } static struct mmap *get_md(struct evlist *evlist, int cpu) { int i; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - struct mmap *md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + struct mmap *md = &evlist__mmap(evlist)[i]; if (md->core.cpu.cpu == cpu) return md; @@ -1580,13 +1967,16 @@ static struct mmap *get_md(struct evlist *evlist, int cpu) static PyObject *pyrf_evlist__read_on_cpu(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; union perf_event *event; int sample_id_all = 1, cpu; static char *kwlist[] = { "cpu", "sample_id_all", NULL }; struct mmap *md; int err; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "i|i", kwlist, &cpu, &sample_id_all)) return NULL; @@ -1616,7 +2006,7 @@ static PyObject *pyrf_evlist__read_on_cpu(struct pyrf_evlist *pevlist, return Py_None; } - pevent->evsel = evsel; + pevent->evsel = evsel__get(evsel); perf_mmap__consume(&md->core); @@ -1637,8 +2027,11 @@ static PyObject *pyrf_evlist__read_on_cpu(struct pyrf_evlist *pevlist, static PyObject *pyrf_evlist__open(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; + + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist = pevlist->evlist; if (evlist__open(evlist) < 0) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1650,8 +2043,11 @@ static PyObject *pyrf_evlist__open(struct pyrf_evlist *pevlist, static PyObject *pyrf_evlist__close(struct pyrf_evlist *pevlist) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; + + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist = pevlist->evlist; evlist__close(evlist); Py_INCREF(Py_None); @@ -1676,8 +2072,11 @@ static PyObject *pyrf_evlist__config(struct pyrf_evlist *pevlist) .no_buffering = true, .no_inherit = true, }; - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; evlist__config(evlist, &opts, &callchain_param); Py_INCREF(Py_None); return Py_None; @@ -1685,14 +2084,16 @@ static PyObject *pyrf_evlist__config(struct pyrf_evlist *pevlist) static PyObject *pyrf_evlist__disable(struct pyrf_evlist *pevlist) { - evlist__disable(&pevlist->evlist); + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist__disable(pevlist->evlist); Py_INCREF(Py_None); return Py_None; } static PyObject *pyrf_evlist__enable(struct pyrf_evlist *pevlist) { - evlist__enable(&pevlist->evlist); + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist__enable(pevlist->evlist); Py_INCREF(Py_None); return Py_None; } @@ -1783,7 +2184,21 @@ static Py_ssize_t pyrf_evlist__length(PyObject *obj) { struct pyrf_evlist *pevlist = (void *)obj; - return pevlist->evlist.core.nr_entries; + if (!pevlist->evlist) + return 0; + + return evlist__nr_entries(pevlist->evlist); +} + +static PyObject *pyrf_evsel__from_evsel(struct evsel *evsel) +{ + struct pyrf_evsel *pevsel = PyObject_New(struct pyrf_evsel, &pyrf_evsel__type); + + if (!pevsel) + return NULL; + + pevsel->evsel = evsel__get(evsel); + return (PyObject *)pevsel; } static PyObject *pyrf_evlist__item(PyObject *obj, Py_ssize_t i) @@ -1791,17 +2206,16 @@ static PyObject *pyrf_evlist__item(PyObject *obj, Py_ssize_t i) struct pyrf_evlist *pevlist = (void *)obj; struct evsel *pos; - if (i >= pevlist->evlist.core.nr_entries) { + if (!pevlist->evlist || i >= evlist__nr_entries(pevlist->evlist)) { PyErr_SetString(PyExc_IndexError, "Index out of range"); return NULL; } - evlist__for_each_entry(&pevlist->evlist, pos) { + evlist__for_each_entry(pevlist->evlist, pos) { if (i-- == 0) break; } - - return Py_BuildValue("O", container_of(pos, struct pyrf_evsel, evsel)); + return pyrf_evsel__from_evsel(pos); } static PyObject *pyrf_evlist__str(PyObject *self) @@ -1812,8 +2226,11 @@ static PyObject *pyrf_evlist__str(PyObject *self) bool first = true; PyObject *result; + if (!pevlist->evlist) + return PyUnicode_FromString("evlist(uninitialized)"); + strbuf_addstr(&sb, "evlist(["); - evlist__for_each_entry(&pevlist->evlist, pos) { + evlist__for_each_entry(pevlist->evlist, pos) { if (!first) strbuf_addch(&sb, ','); if (!pos->pmu) @@ -1835,6 +2252,24 @@ static PySequenceMethods pyrf_evlist__sequence_methods = { static const char pyrf_evlist__doc[] = PyDoc_STR("perf event selector list object."); +static PyObject *pyrf_evlist__getattro(struct pyrf_evlist *pevlist, PyObject *attr_name) +{ + if (!pevlist->evlist) { + PyErr_SetString(PyExc_ValueError, "evlist not initialized"); + return NULL; + } + return PyObject_GenericGetAttr((PyObject *) pevlist, attr_name); +} + +static int pyrf_evlist__setattro(struct pyrf_evlist *pevlist, PyObject *attr_name, PyObject *value) +{ + if (!pevlist->evlist) { + PyErr_SetString(PyExc_ValueError, "evlist not initialized"); + return -1; + } + return PyObject_GenericSetAttr((PyObject *) pevlist, attr_name, value); +} + static PyTypeObject pyrf_evlist__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.evlist", @@ -1847,11 +2282,23 @@ static PyTypeObject pyrf_evlist__type = { .tp_init = (initproc)pyrf_evlist__init, .tp_repr = pyrf_evlist__str, .tp_str = pyrf_evlist__str, + .tp_getattro = (getattrofunc) pyrf_evlist__getattro, + .tp_setattro = (setattrofunc) pyrf_evlist__setattro, }; +static PyObject *pyrf_evlist__new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + struct pyrf_evlist *pevlist; + + pevlist = (struct pyrf_evlist *)PyType_GenericNew(type, args, kwargs); + if (pevlist) + pevlist->evlist = NULL; + return (PyObject *)pevlist; +} + static int pyrf_evlist__setup_types(void) { - pyrf_evlist__type.tp_new = PyType_GenericNew; + pyrf_evlist__type.tp_new = pyrf_evlist__new; return PyType_Ready(&pyrf_evlist__type); } @@ -1954,157 +2401,79 @@ static PyObject *pyrf__tracepoint(struct pyrf_evsel *pevsel, return PyLong_FromLong(tp_pmu__id(sys, name)); } -static PyObject *pyrf_evsel__from_evsel(struct evsel *evsel) -{ - struct pyrf_evsel *pevsel = PyObject_New(struct pyrf_evsel, &pyrf_evsel__type); - - if (!pevsel) - return NULL; - - memset(&pevsel->evsel, 0, sizeof(pevsel->evsel)); - evsel__init(&pevsel->evsel, &evsel->core.attr, evsel->core.idx); - - evsel__clone(&pevsel->evsel, evsel); - if (evsel__is_group_leader(evsel)) - evsel__set_leader(&pevsel->evsel, &pevsel->evsel); - return (PyObject *)pevsel; -} - -static int evlist__pos(struct evlist *evlist, struct evsel *evsel) -{ - struct evsel *pos; - int idx = 0; - - evlist__for_each_entry(evlist, pos) { - if (evsel == pos) - return idx; - idx++; - } - return -1; -} - -static struct evsel *evlist__at(struct evlist *evlist, int idx) -{ - struct evsel *pos; - int idx2 = 0; - - evlist__for_each_entry(evlist, pos) { - if (idx == idx2) - return pos; - idx2++; - } - return NULL; -} - static PyObject *pyrf_evlist__from_evlist(struct evlist *evlist) { struct pyrf_evlist *pevlist = PyObject_New(struct pyrf_evlist, &pyrf_evlist__type); - struct evsel *pos; - struct rb_node *node; if (!pevlist) return NULL; - memset(&pevlist->evlist, 0, sizeof(pevlist->evlist)); - evlist__init(&pevlist->evlist, evlist->core.all_cpus, evlist->core.threads); - evlist__for_each_entry(evlist, pos) { - struct pyrf_evsel *pevsel = (void *)pyrf_evsel__from_evsel(pos); - - evlist__add(&pevlist->evlist, &pevsel->evsel); - } - evlist__for_each_entry(&pevlist->evlist, pos) { - struct evsel *leader = evsel__leader(pos); - - if (pos != leader) { - int idx = evlist__pos(evlist, leader); - - if (idx >= 0) - evsel__set_leader(pos, evlist__at(&pevlist->evlist, idx)); - else if (leader == NULL) - evsel__set_leader(pos, pos); - } - - leader = pos->metric_leader; - - if (pos != leader) { - int idx = evlist__pos(evlist, leader); - - if (idx >= 0) - pos->metric_leader = evlist__at(&pevlist->evlist, idx); - else if (leader == NULL) - pos->metric_leader = pos; - } - } - metricgroup__copy_metric_events(&pevlist->evlist, /*cgrp=*/NULL, - &pevlist->evlist.metric_events, - &evlist->metric_events); - for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); node; - node = rb_next(node)) { - struct metric_event *me = container_of(node, struct metric_event, nd); - struct list_head *mpos; - int idx = evlist__pos(evlist, me->evsel); - - if (idx >= 0) - me->evsel = evlist__at(&pevlist->evlist, idx); - list_for_each(mpos, &me->head) { - struct metric_expr *e = container_of(mpos, struct metric_expr, nd); - - for (int j = 0; e->metric_events[j]; j++) { - idx = evlist__pos(evlist, e->metric_events[j]); - if (idx >= 0) - e->metric_events[j] = evlist__at(&pevlist->evlist, idx); - } - } - } + pevlist->evlist = evlist__get(evlist); return (PyObject *)pevlist; } static PyObject *pyrf__parse_events(PyObject *self, PyObject *args) { const char *input; - struct evlist evlist = {}; + struct evlist *evlist = evlist__new(); struct parse_events_error err; PyObject *result; PyObject *pcpus = NULL, *pthreads = NULL; struct perf_cpu_map *cpus; struct perf_thread_map *threads; - if (!PyArg_ParseTuple(args, "s|OO", &input, &pcpus, &pthreads)) + if (!evlist) + return PyErr_NoMemory(); + + if (!PyArg_ParseTuple(args, "s|OO", &input, &pcpus, &pthreads)) { + evlist__put(evlist); return NULL; + } - threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads : NULL; - cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; + threads = (pthreads && pthreads != Py_None) ? + ((struct pyrf_thread_map *)pthreads)->threads : NULL; + cpus = (pcpus && pcpus != Py_None) ? + ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; parse_events_error__init(&err); - evlist__init(&evlist, cpus, threads); - if (parse_events(&evlist, input, &err)) { + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); + if (parse_events(evlist, input, &err)) { parse_events_error__print(&err, input); PyErr_SetFromErrno(PyExc_OSError); + evlist__put(evlist); return NULL; } - result = pyrf_evlist__from_evlist(&evlist); - evlist__exit(&evlist); + result = pyrf_evlist__from_evlist(evlist); + evlist__put(evlist); return result; } static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args) { const char *input, *pmu = NULL; - struct evlist evlist = {}; + struct evlist *evlist = evlist__new(); PyObject *result; PyObject *pcpus = NULL, *pthreads = NULL; struct perf_cpu_map *cpus; struct perf_thread_map *threads; int ret; - if (!PyArg_ParseTuple(args, "s|sOO", &input, &pmu, &pcpus, &pthreads)) + if (!evlist) + return PyErr_NoMemory(); + + if (!PyArg_ParseTuple(args, "s|sOO", &input, &pmu, &pcpus, &pthreads)) { + evlist__put(evlist); return NULL; + } - threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads : NULL; - cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; + threads = (pthreads && pthreads != Py_None) ? + ((struct pyrf_thread_map *)pthreads)->threads : NULL; + cpus = (pcpus && pcpus != Py_None) ? + ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; - evlist__init(&evlist, cpus, threads); - ret = metricgroup__parse_groups(&evlist, pmu ?: "all", input, + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); + ret = metricgroup__parse_groups(evlist, pmu ?: "all", + /*cputype_filter=*/false, input, /*metric_no_group=*/ false, /*metric_no_merge=*/ false, /*metric_no_threshold=*/ true, @@ -2112,12 +2481,13 @@ static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args) /*system_wide=*/true, /*hardware_aware_grouping=*/ false); if (ret) { + evlist__put(evlist); errno = -ret; PyErr_SetFromErrno(PyExc_OSError); return NULL; } - result = pyrf_evlist__from_evlist(&evlist); - evlist__exit(&evlist); + result = pyrf_evlist__from_evlist(evlist); + evlist__put(evlist); return result; } @@ -2161,6 +2531,8 @@ static int pyrf__metrics_cb(const struct pmu_metric *pm, Py_XDECREF(dict); return -ENOMEM; } + Py_DECREF(key); + Py_DECREF(value); if (!add_to_dict(dict, "MetricName", pm->metric_name) || !add_to_dict(dict, "PMU", pm->pmu) || diff --git a/tools/perf/util/record.c b/tools/perf/util/record.c index e867de8ddaaa4..38e8aee3106b7 100644 --- a/tools/perf/util/record.c +++ b/tools/perf/util/record.c @@ -99,7 +99,7 @@ void evlist__config(struct evlist *evlist, struct record_opts *opts, struct call bool use_comm_exec; bool sample_id = opts->sample_id; - if (perf_cpu_map__cpu(evlist->core.user_requested_cpus, 0).cpu < 0) + if (perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, 0).cpu < 0) opts->no_inherit = true; use_comm_exec = perf_can_comm_exec(); @@ -122,7 +122,7 @@ void evlist__config(struct evlist *evlist, struct record_opts *opts, struct call */ use_sample_identifier = perf_can_sample_identifier(); sample_id = true; - } else if (evlist->core.nr_entries > 1) { + } else if (evlist__nr_entries(evlist) > 1) { struct evsel *first = evlist__first(evlist); evlist__for_each_entry(evlist, evsel) { @@ -237,7 +237,8 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) evsel = evlist__last(temp_evlist); - if (!evlist || perf_cpu_map__is_any_cpu_or_is_empty(evlist->core.user_requested_cpus)) { + if (!evlist || + perf_cpu_map__is_any_cpu_or_is_empty(evlist__core(evlist)->user_requested_cpus)) { struct perf_cpu_map *cpus = perf_cpu_map__new_online_cpus(); if (cpus) @@ -245,7 +246,7 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) perf_cpu_map__put(cpus); } else { - cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, 0); + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, 0); } while (1) { @@ -264,7 +265,7 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) ret = true; out_delete: - evlist__delete(temp_evlist); + evlist__put(temp_evlist); return ret; } diff --git a/tools/perf/util/s390-sample-raw.c b/tools/perf/util/s390-sample-raw.c index 52bbca5c56c8e..01111c4e34880 100644 --- a/tools/perf/util/s390-sample-raw.c +++ b/tools/perf/util/s390-sample-raw.c @@ -12,25 +12,26 @@ * sample was taken from. */ -#include +#include #include #include -#include -#include +#include #include #include -#include +#include +#include +#include "color.h" #include "debug.h" -#include "session.h" #include "evlist.h" -#include "color.h" #include "hashmap.h" -#include "sample-raw.h" +#include "pmu.h" +#include "pmus.h" #include "s390-cpumcf-kernel.h" -#include "util/pmu.h" -#include "util/sample.h" +#include "sample-raw.h" +#include "sample.h" +#include "session.h" static size_t ctrset_size(struct cf_ctrset_entry *set) { diff --git a/tools/perf/util/sample-raw.c b/tools/perf/util/sample-raw.c index e20b73c0c5bd7..f5ae9f4689834 100644 --- a/tools/perf/util/sample-raw.c +++ b/tools/perf/util/sample-raw.c @@ -18,11 +18,11 @@ void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); if (e_machine == EM_S390) { - evlist->trace_event_sample_raw = evlist__s390_sample_raw; + evlist__set_trace_event_sample_raw(evlist, evlist__s390_sample_raw); } else if (e_machine == EM_X86_64 || e_machine == EM_386) { const char *cpuid = perf_env__cpuid(env); if (cpuid && strstarts(cpuid, "AuthenticAMD") && evlist__has_amd_ibs(evlist)) - evlist->trace_event_sample_raw = evlist__amd_sample_raw; + evlist__set_trace_event_sample_raw(evlist, evlist__amd_sample_raw); } } diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index f391a822480db..10d8942f86dad 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -205,7 +205,7 @@ struct perf_session *__perf_session__new(struct perf_data *data, session->machines.host.env = host_env; } if (session->evlist) - session->evlist->session = session; + evlist__set_session(session->evlist, session); session->machines.host.single_address_space = perf_env__single_address_space(session->machines.host.env); @@ -265,7 +265,7 @@ void perf_session__delete(struct perf_session *session) machines__exit(&session->machines); if (session->data) { if (perf_data__is_read(session->data)) - evlist__delete(session->evlist); + evlist__put(session->evlist); perf_data__close(session->data); } #ifdef HAVE_LIBTRACEEVENT @@ -1549,8 +1549,8 @@ static void dump_event(struct evlist *evlist, union perf_event *event, file_offset, file_path, event->header.size, event->header.type); trace_event(event); - if (event->header.type == PERF_RECORD_SAMPLE && evlist->trace_event_sample_raw) - evlist->trace_event_sample_raw(evlist, event, sample); + if (event->header.type == PERF_RECORD_SAMPLE && evlist__trace_event_sample_raw(evlist)) + evlist__trace_event_sample_raw(evlist)(evlist, event, sample); if (sample) evlist__print_tstamp(evlist, event, sample); @@ -1751,7 +1751,7 @@ static int deliver_sample_value(struct evlist *evlist, } if (!storage || sid->evsel == NULL) { - ++evlist->stats.nr_unknown_id; + ++evlist__stats(evlist)->nr_unknown_id; return 0; } @@ -1845,12 +1845,15 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist, struct evsel *saved_evsel = sample->evsel; sample->evsel = evlist__id2evsel(evlist, sample->id); + if (sample->evsel) + sample->evsel = evsel__get(sample->evsel); ret = tool->callchain_deferred(tool, event, sample, machine); + evsel__put(sample->evsel); sample->evsel = saved_evsel; return ret; } - list_for_each_entry_safe(de, tmp, &evlist->deferred_samples, list) { + list_for_each_entry_safe(de, tmp, evlist__deferred_samples(evlist), list) { struct perf_sample orig_sample; perf_sample__init(&orig_sample, /*all=*/false); @@ -1899,7 +1902,7 @@ static int session__flush_deferred_samples(struct perf_session *session, struct deferred_event *de, *tmp; int ret = 0; - list_for_each_entry_safe(de, tmp, &evlist->deferred_samples, list) { + list_for_each_entry_safe(de, tmp, evlist__deferred_samples(evlist), list) { struct perf_sample sample; perf_sample__init(&sample, /*all=*/false); @@ -1961,17 +1964,16 @@ static int machines__deliver_event(struct machines *machines, sample->evsel = evlist__id2evsel(evlist, sample->id); else assert(sample->evsel == evlist__id2evsel(evlist, sample->id)); - machine = machines__find_for_cpumode(machines, event, sample); switch (event->header.type) { case PERF_RECORD_SAMPLE: if (sample->evsel == NULL) { - ++evlist->stats.nr_unknown_id; + ++evlist__stats(evlist)->nr_unknown_id; return 0; } if (machine == NULL) { - ++evlist->stats.nr_unprocessable_samples; + ++evlist__stats(evlist)->nr_unprocessable_samples; dump_sample(machine, event, sample); return 0; } @@ -1990,7 +1992,7 @@ static int machines__deliver_event(struct machines *machines, } memcpy(de->event, event, sz); de->file_offset = sample->file_offset; - list_add_tail(&de->list, &evlist->deferred_samples); + list_add_tail(&de->list, evlist__deferred_samples(evlist)); return 0; } return evlist__deliver_sample(evlist, tool, event, sample, machine); @@ -2002,7 +2004,7 @@ static int machines__deliver_event(struct machines *machines, return tool->mmap(tool, event, sample, machine); case PERF_RECORD_MMAP2: if (event->header.misc & PERF_RECORD_MISC_PROC_MAP_PARSE_TIMEOUT) - ++evlist->stats.nr_proc_map_timeout; + ++evlist__stats(evlist)->nr_proc_map_timeout; if (!perf_event__check_nul(event->mmap2.filename, (void *)event + event->header.size, "MMAP2", file_offset)) @@ -2047,13 +2049,13 @@ static int machines__deliver_event(struct machines *machines, return tool->exit(tool, event, sample, machine); case PERF_RECORD_LOST: if (tool->lost == perf_event__process_lost) - evlist->stats.total_lost += event->lost.lost; + evlist__stats(evlist)->total_lost += event->lost.lost; return tool->lost(tool, event, sample, machine); case PERF_RECORD_LOST_SAMPLES: if (event->header.misc & PERF_RECORD_MISC_LOST_SAMPLES_BPF) - evlist->stats.total_dropped_samples += event->lost_samples.lost; + evlist__stats(evlist)->total_dropped_samples += event->lost_samples.lost; else if (tool->lost_samples == perf_event__process_lost_samples) - evlist->stats.total_lost_samples += event->lost_samples.lost; + evlist__stats(evlist)->total_lost_samples += event->lost_samples.lost; return tool->lost_samples(tool, event, sample, machine); case PERF_RECORD_READ: dump_read(sample->evsel, event); @@ -2065,11 +2067,11 @@ static int machines__deliver_event(struct machines *machines, case PERF_RECORD_AUX: if (tool->aux == perf_event__process_aux) { if (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) - evlist->stats.total_aux_lost += 1; + evlist__stats(evlist)->total_aux_lost += 1; if (event->aux.flags & PERF_AUX_FLAG_PARTIAL) - evlist->stats.total_aux_partial += 1; + evlist__stats(evlist)->total_aux_partial += 1; if (event->aux.flags & PERF_AUX_FLAG_COLLISION) - evlist->stats.total_aux_collision += 1; + evlist__stats(evlist)->total_aux_collision += 1; } return tool->aux(tool, event, sample, machine); case PERF_RECORD_ITRACE_START: @@ -2105,7 +2107,7 @@ static int machines__deliver_event(struct machines *machines, return evlist__deliver_deferred_callchain(evlist, tool, event, sample, machine); default: - ++evlist->stats.nr_unknown_events; + ++evlist__stats(evlist)->nr_unknown_events; return -1; } } @@ -2517,7 +2519,7 @@ int perf_session__deliver_synth_event(struct perf_session *session, struct evlist *evlist = session->evlist; const struct perf_tool *tool = session->tool; - events_stats__inc(&evlist->stats, event->header.type); + events_stats__inc(evlist__stats(evlist), event->header.type); if (event->header.type >= PERF_RECORD_USER_TYPE_START) return perf_session__process_user_event(session, event, 0, NULL); @@ -2927,7 +2929,7 @@ static s64 perf_session__process_event(struct perf_session *session, return 0; } - events_stats__inc(&evlist->stats, event->header.type); + events_stats__inc(evlist__stats(evlist), event->header.type); if (event->header.type >= PERF_RECORD_USER_TYPE_START) return perf_session__process_user_event(session, event, file_offset, file_path); @@ -2988,7 +2990,7 @@ perf_session__warn_order(const struct perf_session *session) static void perf_session__warn_about_errors(const struct perf_session *session) { - const struct events_stats *stats = &session->evlist->stats; + const struct events_stats *stats = evlist__stats(session->evlist); if (session->tool->lost == perf_event__process_lost && stats->nr_events[PERF_RECORD_LOST] != 0) { @@ -3821,7 +3823,7 @@ size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp) ret = fprintf(fp, "\nAggregated stats:%s\n", msg); - ret += events_stats__fprintf(&session->evlist->stats, fp); + ret += events_stats__fprintf(evlist__stats(session->evlist), fp); return ret; } diff --git a/tools/perf/util/sideband_evlist.c b/tools/perf/util/sideband_evlist.c index 388846f17bc13..ba043db6cedcb 100644 --- a/tools/perf/util/sideband_evlist.c +++ b/tools/perf/util/sideband_evlist.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -22,7 +23,7 @@ int evlist__add_sb_event(struct evlist *evlist, struct perf_event_attr *attr, attr->sample_id_all = 1; } - evsel = evsel__new_idx(attr, evlist->core.nr_entries); + evsel = evsel__new_idx(attr, evlist__nr_entries(evlist)); if (!evsel) return -1; @@ -49,14 +50,27 @@ static void *perf_evlist__poll_thread(void *arg) while (!done) { bool got_data = false; - if (evlist->thread.done) + if (evlist__sb_thread_done(evlist)) draining = true; if (!draining) evlist__poll(evlist, 1000); - for (i = 0; i < evlist->core.nr_mmaps; i++) { - struct mmap *map = &evlist->mmap[i]; + /* + * When a thread of the monitored target exits, its per-cpu + * ring-buffer fd is closed and starts returning POLLHUP. Such + * dead fds are never requested for POLLIN, but poll() reports + * POLLHUP/POLLERR unconditionally, so leaving them in the + * pollfd array makes the following evlist__poll() return + * immediately forever, spinning this thread at 100% CPU. + * + * Filter them out here, mirroring what the 'perf record' main + * loop does after fdarray__poll(). + */ + evlist__filter_pollfd(evlist, POLLERR | POLLHUP); + + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + struct mmap *map = &evlist__mmap(evlist)[i]; union perf_event *event; if (perf_mmap__read_init(&map->core)) @@ -102,9 +116,9 @@ int evlist__start_sb_thread(struct evlist *evlist, struct target *target) return 0; if (evlist__create_maps(evlist, target)) - goto out_delete_evlist; + goto out_put_evlist; - if (evlist->core.nr_entries > 1) { + if (evlist__nr_entries(evlist) > 1) { bool can_sample_identifier = perf_can_sample_identifier(); evlist__for_each_entry(evlist, counter) @@ -114,27 +128,27 @@ int evlist__start_sb_thread(struct evlist *evlist, struct target *target) } evlist__for_each_entry(evlist, counter) { - if (evsel__open(counter, evlist->core.user_requested_cpus, - evlist->core.threads) < 0) - goto out_delete_evlist; + if (evsel__open(counter, evlist__core(evlist)->user_requested_cpus, + evlist__core(evlist)->threads) < 0) + goto out_put_evlist; } - if (evlist__mmap(evlist, UINT_MAX)) - goto out_delete_evlist; + if (evlist__do_mmap(evlist, UINT_MAX)) + goto out_put_evlist; evlist__for_each_entry(evlist, counter) { if (evsel__enable(counter)) - goto out_delete_evlist; + goto out_put_evlist; } - evlist->thread.done = 0; - if (pthread_create(&evlist->thread.th, NULL, perf_evlist__poll_thread, evlist)) - goto out_delete_evlist; + evlist__set_sb_thread_done(evlist, 0); + if (pthread_create(evlist__sb_thread_th(evlist), NULL, perf_evlist__poll_thread, evlist)) + goto out_put_evlist; return 0; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); evlist = NULL; return -1; } @@ -143,7 +157,7 @@ void evlist__stop_sb_thread(struct evlist *evlist) { if (!evlist) return; - evlist->thread.done = 1; - pthread_join(evlist->thread.th, NULL); - evlist__delete(evlist); + evlist__set_sb_thread_done(evlist, 1); + pthread_join(*evlist__sb_thread_th(evlist), NULL); + evlist__put(evlist); } diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c index 005e7d85dc4a5..58638ec9ae0ed 100644 --- a/tools/perf/util/sort.c +++ b/tools/perf/util/sort.c @@ -3105,7 +3105,7 @@ static int __sort_dimension__add_hpp_sort(struct sort_dimension *sd, struct hpp_sort_entry *hse = __sort_dimension__alloc_hpp(sd, level); if (hse == NULL) - return -1; + return -ENOMEM; perf_hpp_list__register_sort_field(list, &hse->hpp); return 0; @@ -3118,7 +3118,7 @@ static int __sort_dimension__add_hpp_output(struct sort_dimension *sd, struct hpp_sort_entry *hse = __sort_dimension__alloc_hpp(sd, level); if (hse == NULL) - return -1; + return -ENOMEM; perf_hpp_list__column_register(list, &hse->hpp); return 0; @@ -3487,7 +3487,7 @@ static struct evsel *find_evsel(struct evlist *evlist, char *event_name) if (event_name[0] == '%') { int nr = strtol(event_name+1, NULL, 0); - if (nr > evlist->core.nr_entries) + if (nr > evlist__nr_entries(evlist)) return NULL; evsel = evlist__first(evlist); @@ -3742,14 +3742,18 @@ static int __sort_dimension__add(struct sort_dimension *sd, struct perf_hpp_list *list, int level) { + int ret; + if (sd->taken) return 0; - if (__sort_dimension__add_hpp_sort(sd, list, level) < 0) - return -1; + ret = __sort_dimension__add_hpp_sort(sd, list, level); + if (ret < 0) + return ret; - if (__sort_dimension__update(sd, list) < 0) - return -1; + ret = __sort_dimension__update(sd, list); + if (ret < 0) + return ret; sd->taken = 1; @@ -3767,7 +3771,7 @@ static int __hpp_dimension__add(struct hpp_dimension *hd, fmt = __hpp_dimension__alloc_hpp(hd, level); if (!fmt) - return -1; + return -ENOMEM; hd->taken = 1; hd->was_taken = 1; @@ -3779,14 +3783,18 @@ static int __sort_dimension__add_output(struct perf_hpp_list *list, struct sort_dimension *sd, int level) { + int ret; + if (sd->taken) return 0; - if (__sort_dimension__add_hpp_output(sd, list, level) < 0) - return -1; + ret = __sort_dimension__add_hpp_output(sd, list, level); + if (ret < 0) + return ret; - if (__sort_dimension__update(sd, list) < 0) - return -1; + ret = __sort_dimension__update(sd, list); + if (ret < 0) + return ret; sd->taken = 1; return 0; @@ -3803,7 +3811,7 @@ static int __hpp_dimension__add_output(struct perf_hpp_list *list, fmt = __hpp_dimension__alloc_hpp(hd, level); if (!fmt) - return -1; + return -ENOMEM; hd->taken = 1; perf_hpp_list__column_register(list, fmt); @@ -3869,8 +3877,7 @@ int sort_dimension__add(struct perf_hpp_list *list, const char *tok, strlen(tok))) return -EINVAL; - __sort_dimension__add(sd, list, level); - return 0; + return __sort_dimension__add(sd, list, level); } for (i = 0; i < ARRAY_SIZE(memory_sort_dimensions); i++) { @@ -3882,8 +3889,7 @@ int sort_dimension__add(struct perf_hpp_list *list, const char *tok, if (sort__mode != SORT_MODE__MEMORY) return -EINVAL; - __sort_dimension__add(sd, list, level); - return 0; + return __sort_dimension__add(sd, list, level); } for (i = 0; i < ARRAY_SIZE(hpp_sort_dimensions); i++) { @@ -3973,15 +3979,25 @@ static int setup_sort_list(struct perf_hpp_list *list, char *str, } ret = sort_dimension__add(list, tok, evlist, env, level); - if (ret == -EINVAL) { + switch (ret) { + case 0: + break; + case -EINVAL: if (!cacheline_size() && !strncasecmp(tok, "dcacheline", strlen(tok))) ui__error("The \"dcacheline\" --sort key needs to know the cacheline size and it couldn't be determined on this system"); else ui__error("Invalid --sort key: `%s'", tok); - break; - } else if (ret == -ESRCH) { + goto out; + case -ESRCH: ui__error("Unknown --sort key: `%s'", tok); - break; + goto out; + default: { + char buf[STRERR_BUFSIZE]; + + ui__error("%s for --sort key: `%s'", + str_error_r(-ret, buf, sizeof(buf)), tok); + goto out; + } } prev_level = level; } @@ -3989,6 +4005,7 @@ static int setup_sort_list(struct perf_hpp_list *list, char *str, level = next_level; } while (tmp); +out: return ret; } @@ -4315,15 +4332,26 @@ static int setup_output_list(struct perf_hpp_list *list, char *str) for (tok = strtok_r(str, ", ", &tmp); tok; tok = strtok_r(NULL, ", ", &tmp)) { ret = output_field_add(list, tok, &level); - if (ret == -EINVAL) { - ui__error("Invalid --fields key: `%s'", tok); + switch (ret) { + case 0: break; - } else if (ret == -ESRCH) { + case -EINVAL: + ui__error("Invalid --fields key: `%s'", tok); + goto out; + case -ESRCH: ui__error("Unknown --fields key: `%s'", tok); - break; + goto out; + default: { + char buf[STRERR_BUFSIZE]; + + ui__error("%s for --fields key: `%s'", + str_error_r(-ret, buf, sizeof(buf)), tok); + goto out; + } } } +out: return ret; } diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index 0a5750bb59fa3..f94f1324d24ac 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -667,7 +667,7 @@ static void print_metric_header(struct perf_stat_config *config, /* In case of iostat, print metric header for first root port only */ if (config->iostat_run && - os->evsel->priv != os->evsel->evlist->selected->priv) + os->evsel->priv != evlist__selected(os->evsel->evlist)->priv) return; if (os->evsel->cgrp != os->cgrp) @@ -1126,7 +1126,7 @@ static void print_no_aggr_metric(struct perf_stat_config *config, unsigned int all_idx; struct perf_cpu cpu; - perf_cpu_map__for_each_cpu(cpu, all_idx, evlist->core.user_requested_cpus) { + perf_cpu_map__for_each_cpu(cpu, all_idx, evlist__core(evlist)->user_requested_cpus) { struct evsel *counter; bool first = true; @@ -1543,7 +1543,7 @@ void evlist__print_counters(struct evlist *evlist, struct perf_stat_config *conf evlist__uniquify_evsel_names(evlist, config); if (config->iostat_run) - evlist->selected = evlist__first(evlist); + evlist__set_selected(evlist, evlist__first(evlist)); if (config->interval) prepare_timestamp(config, &os, ts); diff --git a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c index c17373bb0e1e7..525a3fe4a46ec 100644 --- a/tools/perf/util/stat-shadow.c +++ b/tools/perf/util/stat-shadow.c @@ -2,20 +2,24 @@ #include #include #include -#include "evsel.h" -#include "stat.h" + +#include + +#include "cgroup.h" #include "color.h" #include "debug.h" -#include "pmu.h" -#include "rblist.h" #include "evlist.h" +#include "evsel.h" #include "expr.h" -#include "metricgroup.h" -#include "cgroup.h" -#include "units.h" +#include "hashmap.h" #include "iostat.h" -#include "util/hashmap.h" +#include "metricgroup.h" +#include "pmu.h" +#include "pmus.h" +#include "rblist.h" +#include "stat.h" #include "tool_pmu.h" +#include "units.h" static bool tool_pmu__is_time_event(const struct perf_stat_config *config, const struct evsel *evsel, int *tool_aggr_idx) @@ -283,7 +287,7 @@ void *perf_stat__print_shadow_stats_metricgroup(struct perf_stat_config *config, void *ctxp = out->ctx; bool header_printed = false; const char *name = NULL; - struct rblist *metric_events = &evsel->evlist->metric_events; + struct rblist *metric_events = evlist__metric_events(evsel->evlist); me = metricgroup__lookup(metric_events, evsel, false); if (me == NULL) @@ -351,5 +355,5 @@ bool perf_stat__skip_metric_event(struct evsel *evsel) if (!evsel->default_metricgroup) return false; - return !metricgroup__lookup(&evsel->evlist->metric_events, evsel, false); + return !metricgroup__lookup(evlist__metric_events(evsel->evlist), evsel, false); } diff --git a/tools/perf/util/stat.c b/tools/perf/util/stat.c index 14d169e22e8f5..25f31a1743682 100644 --- a/tools/perf/util/stat.c +++ b/tools/perf/util/stat.c @@ -1,21 +1,25 @@ // SPDX-License-Identifier: GPL-2.0 +#include "stat.h" + #include -#include #include #include #include + +#include +#include + #include "counts.h" #include "cpumap.h" #include "debug.h" +#include "evlist.h" +#include "evsel.h" +#include "hashmap.h" #include "header.h" -#include "stat.h" +#include "pmu.h" #include "session.h" #include "target.h" -#include "evlist.h" -#include "evsel.h" #include "thread_map.h" -#include "util/hashmap.h" -#include void update_stats(struct stats *stats, u64 val) { @@ -543,8 +547,8 @@ static void evsel__merge_aliases(struct evsel *evsel) struct evlist *evlist = evsel->evlist; struct evsel *alias; - alias = list_prepare_entry(evsel, &(evlist->core.entries), core.node); - list_for_each_entry_continue(alias, &evlist->core.entries, core.node) { + alias = list_prepare_entry(evsel, &(evlist__core(evlist)->entries), core.node); + list_for_each_entry_continue(alias, &evlist__core(evlist)->entries, core.node) { if (alias->first_wildcard_match == evsel) { /* Merge the same events on different PMUs. */ evsel__merge_aggr_counters(evsel, alias); diff --git a/tools/perf/util/stream.c b/tools/perf/util/stream.c index 3de4a61308539..7bccd23783444 100644 --- a/tools/perf/util/stream.c +++ b/tools/perf/util/stream.c @@ -131,7 +131,7 @@ static int evlist__init_callchain_streams(struct evlist *evlist, struct evsel *pos; int i = 0; - BUG_ON(els->nr_evsel < evlist->core.nr_entries); + BUG_ON(els->nr_evsel < evlist__nr_entries(evlist)); evlist__for_each_entry(evlist, pos) { struct hists *hists = evsel__hists(pos); @@ -148,7 +148,7 @@ static int evlist__init_callchain_streams(struct evlist *evlist, struct evlist_streams *evlist__create_streams(struct evlist *evlist, int nr_streams_max) { - int nr_evsel = evlist->core.nr_entries, ret = -1; + int nr_evsel = evlist__nr_entries(evlist), ret = -1; struct evlist_streams *els = evlist_streams__new(nr_evsel, nr_streams_max); diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index cd379ced19e5b..1de5796fc4ff6 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -1947,7 +1947,16 @@ int dso__load(struct dso *dso, struct map *map) if (next_slot) { ss_pos++; - if (dso__binary_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) + /* + * The binary type is used to find the file containing + * the executed instructions, so prefer the types that + * refer to the actual object over debug-only files such + * as DSO_BINARY_TYPE__DEBUGLINK. + */ + if (dso__binary_type(dso) == DSO_BINARY_TYPE__NOT_FOUND || + symtab_type == DSO_BINARY_TYPE__BUILD_ID_CACHE || + (symtab_type == DSO_BINARY_TYPE__SYSTEM_PATH_DSO && + dso__binary_type(dso) != DSO_BINARY_TYPE__BUILD_ID_CACHE)) dso__set_binary_type(dso, symtab_type); if (syms_ss && runtime_ss) @@ -2452,8 +2461,7 @@ static bool symbol__read_kptr_restrict(void) { bool value = false; FILE *fp = fopen("/proc/sys/kernel/kptr_restrict", "r"); - bool used_root; - bool cap_syslog = perf_cap__capable(CAP_SYSLOG, &used_root); + bool cap_syslog = perf_cap__capable(CAP_SYSLOG); if (fp != NULL) { char line[8]; diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index 5307d707711d8..19c1f3eaf58d8 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -1058,7 +1058,7 @@ int perf_event__synthesize_threads(const struct perf_tool *tool, else thread_nr = nr_threads_synthesize; - if (thread_nr <= 1) { + if (thread_nr <= 1 || n <= 1) { err = __perf_event__synthesize_threads(tool, process, machine, needs_mmap, mmap_data, @@ -1104,8 +1104,8 @@ int perf_event__synthesize_threads(const struct perf_tool *tool, } err = 0; out_join: - for (i = 0; i < thread_nr; i++) - pthread_join(synthesize_threads[i], NULL); + for (j = 0; j < i; j++) + pthread_join(synthesize_threads[j], NULL); free(args); free_threads: free(synthesize_threads); @@ -2247,7 +2247,7 @@ int perf_event__synthesize_tracing_data(const struct perf_tool *tool, int fd, st * - write the tracing data from the temp file * to the pipe */ - tdata = tracing_data_get(&evlist->core.entries, fd, true); + tdata = tracing_data_get(&evlist__core(evlist)->entries, fd, true); if (!tdata) return -1; @@ -2404,13 +2404,16 @@ int perf_event__synthesize_stat_events(struct perf_stat_config *config, const st } err = perf_event__synthesize_extra_attr(tool, evlist, process, attrs); - err = perf_event__synthesize_thread_map2(tool, evlist->core.threads, process, NULL); + err = perf_event__synthesize_thread_map2(tool, evlist__core(evlist)->threads, + process, /*machine=*/NULL); if (err < 0) { pr_err("Couldn't synthesize thread map.\n"); return err; } - err = perf_event__synthesize_cpu_map(tool, evlist->core.user_requested_cpus, process, NULL); + err = perf_event__synthesize_cpu_map(tool, + evlist__core(evlist)->user_requested_cpus, + process, /*machine=*/NULL); if (err < 0) { pr_err("Couldn't synthesize thread map.\n"); return err; @@ -2518,7 +2521,7 @@ int perf_event__synthesize_for_pipe(const struct perf_tool *tool, ret += err; #ifdef HAVE_LIBTRACEEVENT - if (have_tracepoints(&evlist->core.entries)) { + if (have_tracepoints(&evlist__core(evlist)->entries)) { int fd = perf_data__fd(data); /* diff --git a/tools/perf/util/thread-stack.c b/tools/perf/util/thread-stack.c index c5ce741b07446..1360f44421ef8 100644 --- a/tools/perf/util/thread-stack.c +++ b/tools/perf/util/thread-stack.c @@ -642,7 +642,7 @@ void thread_stack__br_sample(struct thread *thread, int cpu, sz -= nr; be = &dst->entries[nr]; nr = min(ts->br_stack_pos, sz); - memcpy(be, &src->entries[0], bsz * ts->br_stack_pos); + memcpy(be, &src->entries[0], bsz * nr); } } diff --git a/tools/perf/util/time-utils.c b/tools/perf/util/time-utils.c index d43c4577d7ebc..5558a5a0fea4a 100644 --- a/tools/perf/util/time-utils.c +++ b/tools/perf/util/time-utils.c @@ -473,8 +473,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str, return -ENOMEM; if (has_percent || reltime) { - if (session->evlist->first_sample_time == 0 && - session->evlist->last_sample_time == 0) { + if (evlist__first_sample_time(session->evlist) == 0 && + evlist__last_sample_time(session->evlist) == 0) { pr_err("HINT: no first/last sample time found in perf data.\n" "Please use latest perf binary to execute 'perf record'\n" "(if '--buildid-all' is enabled, please set '--timestamp-boundary').\n"); @@ -486,8 +486,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str, num = perf_time__percent_parse_str( ptime_range, size, time_str, - session->evlist->first_sample_time, - session->evlist->last_sample_time); + evlist__first_sample_time(session->evlist), + evlist__last_sample_time(session->evlist)); } else { num = perf_time__parse_strs(ptime_range, time_str, size); } @@ -499,8 +499,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str, int i; for (i = 0; i < num; i++) { - ptime_range[i].start += session->evlist->first_sample_time; - ptime_range[i].end += session->evlist->first_sample_time; + ptime_range[i].start += evlist__first_sample_time(session->evlist); + ptime_range[i].end += evlist__first_sample_time(session->evlist); } } diff --git a/tools/perf/util/top.c b/tools/perf/util/top.c index b06e10a116bb3..851a26be69315 100644 --- a/tools/perf/util/top.c +++ b/tools/perf/util/top.c @@ -71,7 +71,7 @@ size_t perf_top__header_snprintf(struct perf_top *top, char *bf, size_t size) esamples_percent); } - if (top->evlist->core.nr_entries == 1) { + if (evlist__nr_entries(top->evlist) == 1) { struct evsel *first = evlist__first(top->evlist); ret += SNPRINTF(bf + ret, size - ret, "%" PRIu64 "%s ", (uint64_t)first->core.attr.sample_period, @@ -94,7 +94,7 @@ size_t perf_top__header_snprintf(struct perf_top *top, char *bf, size_t size) else ret += SNPRINTF(bf + ret, size - ret, " (all"); - nr_cpus = perf_cpu_map__nr(top->evlist->core.user_requested_cpus); + nr_cpus = perf_cpu_map__nr(evlist__core(top->evlist)->user_requested_cpus); if (target->cpu_list) ret += SNPRINTF(bf + ret, size - ret, ", CPU%s: %s)", nr_cpus > 1 ? "s" : "", diff --git a/tools/perf/util/trace-event-read.c b/tools/perf/util/trace-event-read.c index ecbbb93f01853..52ed496d92c3b 100644 --- a/tools/perf/util/trace-event-read.c +++ b/tools/perf/util/trace-event-read.c @@ -25,18 +25,18 @@ static int input_fd; static ssize_t trace_data_size; static bool repipe; -static int __do_read(int fd, void *buf, int size) +static ssize_t __do_read(int fd, void *buf, size_t size) { - int rsize = size; + size_t rsize = size; while (size) { - int ret = read(fd, buf, size); + ssize_t ret = read(fd, buf, size); if (ret <= 0) return -1; if (repipe) { - int retw = write(STDOUT_FILENO, buf, ret); + ssize_t retw = write(STDOUT_FILENO, buf, ret); if (retw <= 0 || retw != ret) { pr_debug("repiping input file"); @@ -51,13 +51,13 @@ static int __do_read(int fd, void *buf, int size) return rsize; } -static int do_read(void *data, int size) +static ssize_t do_read(void *data, size_t size) { - int r; + ssize_t r; r = __do_read(input_fd, data, size); if (r <= 0) { - pr_debug("reading input file (size expected=%d received=%d)", + pr_debug("reading input file (size expected=%zu received=%zd)", size, r); return -1; } @@ -68,10 +68,10 @@ static int do_read(void *data, int size) } /* If it fails, the next read will report it */ -static void skip(int size) +static void skip(size_t size) { char buf[BUFSIZ]; - int r; + size_t r; while (size) { r = size > BUFSIZ ? BUFSIZ : size; @@ -127,6 +127,11 @@ static char *read_string(void) } } + if (size >= (int)sizeof(buf) - 1) { + pr_debug("string too long (max %zu bytes)", sizeof(buf) - 1); + goto out; + } + buf[size++] = c; if (!c) @@ -197,7 +202,7 @@ static int read_header_files(struct tep_handle *pevent) unsigned long long size; char *header_page; char buf[BUFSIZ]; - int ret = 0; + ssize_t ret = 0; if (do_read(buf, 12) < 0) return -1; @@ -245,7 +250,7 @@ static int read_header_files(struct tep_handle *pevent) static int read_ftrace_file(struct tep_handle *pevent, unsigned long long size) { - int ret; + ssize_t ret; char *buf; buf = malloc(size); @@ -271,7 +276,7 @@ static int read_ftrace_file(struct tep_handle *pevent, unsigned long long size) static int read_event_file(struct tep_handle *pevent, char *sys, unsigned long long size) { - int ret; + ssize_t ret; char *buf; buf = malloc(size); @@ -317,7 +322,7 @@ static int read_event_files(struct tep_handle *pevent) int systems; int count; int i,x; - int ret; + ssize_t ret; systems = read4(pevent); @@ -345,7 +350,7 @@ static int read_saved_cmdline(struct tep_handle *pevent) { unsigned long long size; char *buf; - int ret; + ssize_t ret; /* it can have 0 size */ size = read8(pevent); diff --git a/tools/perf/util/unwind-libdw.c b/tools/perf/util/unwind-libdw.c index 7f35042be5677..63a5c2253174f 100644 --- a/tools/perf/util/unwind-libdw.c +++ b/tools/perf/util/unwind-libdw.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -173,14 +174,30 @@ static int entry(u64 ip, struct unwind_info *ui) return 0; } -static pid_t next_thread(Dwfl *dwfl, void *arg, void **thread_argp) +static pid_t next_thread(Dwfl *dwfl __maybe_unused, void *arg, void **thread_argp) { + struct dwfl_ui_thread_info *dwfl_ui_ti = arg; + /* We want only single thread to be processed. */ if (*thread_argp != NULL) return 0; + assert(dwfl_ui_ti->ui != NULL); *thread_argp = arg; - return dwfl_pid(dwfl); + return thread__tid(dwfl_ui_ti->ui->thread); +} + +static bool get_thread(Dwfl *dwfl __maybe_unused, pid_t tid, void *arg, + void **thread_argp) +{ + struct dwfl_ui_thread_info *dwfl_ui_ti = arg; + + assert(dwfl_ui_ti->ui != NULL); + if (tid != thread__tid(dwfl_ui_ti->ui->thread)) + return false; + + *thread_argp = arg; + return true; } static int access_dso_mem(struct unwind_info *ui, Dwarf_Addr addr, @@ -306,6 +323,7 @@ static bool libdw_set_initial_registers(Dwfl_Thread *thread, void *arg) static const Dwfl_Thread_Callbacks callbacks = { .next_thread = next_thread, + .get_thread = get_thread, .memory_read = memory_read, .set_initial_registers = libdw_set_initial_registers, }; @@ -400,7 +418,7 @@ int libdw__get_entries(unwind_entry_cb_t cb, void *arg, if (err) goto out; - dwfl_attach_state(dwfl, /*elf=*/NULL, thread__tid(thread), &callbacks, + dwfl_attach_state(dwfl, /*elf=*/NULL, thread__pid(thread), &callbacks, /* Dwfl thread function argument*/dwfl_ui_ti); // Ignore thread already attached error. diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c index 2c2a5c449ffd0..8f7cd32f524dc 100644 --- a/tools/perf/util/util.c +++ b/tools/perf/util/util.c @@ -378,15 +378,9 @@ int perf_event_paranoid(void) bool perf_event_paranoid_check(int max_level) { - bool used_root; - - if (perf_cap__capable(CAP_SYS_ADMIN, &used_root)) - return true; - - if (!used_root && perf_cap__capable(CAP_PERFMON, &used_root)) - return true; - - return perf_event_paranoid() <= max_level; + return perf_cap__capable(CAP_SYS_ADMIN) || + perf_cap__capable(CAP_PERFMON) || + perf_event_paranoid() <= max_level; } int perf_tip(char **strp, const char *dirpath) diff --git a/tools/perf/util/zstd.c b/tools/perf/util/zstd.c index 21a0eb58597c2..d98014902f012 100644 --- a/tools/perf/util/zstd.c +++ b/tools/perf/util/zstd.c @@ -31,9 +31,11 @@ int zstd_fini(struct zstd_data *data) ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_t dst_size, void *src, size_t src_size, size_t max_record_size, - size_t process_header(void *record, size_t increment)) + ssize_t process_header(void *record, size_t dst_size, + size_t data_size)) { - size_t ret, size, compressed = 0; + size_t ret, compressed = 0; + ssize_t size; ZSTD_inBuffer input = { src, src_size, 0 }; ZSTD_outBuffer output; void *record; @@ -55,12 +57,9 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ while (input.pos < input.size) { record = dst; - /* process_header writes the event header into record */ - if (dst_size < sizeof(struct perf_event_header)) - goto reset; - size = process_header(record, 0); + size = process_header(record, dst_size, 0); /* Output buffer full — cannot fit even the record header */ - if (size > dst_size) + if (size < 0) goto reset; compressed += size; dst += size; @@ -74,17 +73,21 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ (long)src_size, ZSTD_getErrorName(ret)); goto reset; } - size = output.pos; + compressed += output.pos; + dst += output.pos; + dst_size -= output.pos; /* * No progress: ZSTD couldn't emit any bytes into the * remaining output buffer. Calling process_header - * with size=0 would re-trigger header initialization, + * with output.pos=0 would re-trigger header initialization, * double-subtracting the header size from dst_size and * underflowing the unsigned counter. */ - if (size == 0) + if (output.pos == 0) + goto reset; + size = process_header(record, dst_size, output.pos); + if (size < 0) goto reset; - size = process_header(record, size); compressed += size; dst += size; dst_size -= size; diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 87f15f2962348..09149c32c41c6 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -31,7 +31,7 @@ struct cgroup *scx_bpf_task_cgroup___new(struct task_struct *p) __ksym __weak; * * v7.1: scx_bpf_dsq_move_to_local___v2() to add @enq_flags. */ -bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags) __ksym __weak; +bool scx_bpf_dsq_move_to_local___v2___compat(u64 dsq_id, u64 enq_flags) __ksym __weak; bool scx_bpf_dsq_move_to_local___v1(u64 dsq_id) __ksym __weak; void scx_bpf_dsq_move_set_slice___new(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak; void scx_bpf_dsq_move_set_vtime___new(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; @@ -45,8 +45,8 @@ bool scx_bpf_dispatch_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct t bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; #define scx_bpf_dsq_move_to_local(dsq_id, enq_flags) \ - (bpf_ksym_exists(scx_bpf_dsq_move_to_local___v2) ? \ - scx_bpf_dsq_move_to_local___v2((dsq_id), (enq_flags)) : \ + (bpf_ksym_exists(scx_bpf_dsq_move_to_local___v2___compat) ? \ + scx_bpf_dsq_move_to_local___v2___compat((dsq_id), (enq_flags)) : \ (bpf_ksym_exists(scx_bpf_dsq_move_to_local___v1) ? \ scx_bpf_dsq_move_to_local___v1((dsq_id)) : \ scx_bpf_consume___old((dsq_id)))) @@ -84,7 +84,7 @@ bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, st * * Compat macro will be dropped on v6.19 release. */ -int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak; +int bpf_cpumask_populate(struct bpf_cpumask *dst, void *src, size_t src__sz) __ksym __weak; #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz) \ (bpf_ksym_exists(bpf_cpumask_populate) ? \ @@ -414,10 +414,10 @@ static inline void scx_bpf_reenqueue_local(void) } /* - * v6.20: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This + * v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This * will eventually deprecate scx_bpf_reenqueue_local(). */ -void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags, const struct bpf_prog_aux *aux__prog) __ksym __weak; +void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags) __ksym __weak; static inline bool __COMPAT_has_generic_reenq(void) { @@ -427,7 +427,7 @@ static inline bool __COMPAT_has_generic_reenq(void) static inline void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags) { if (bpf_ksym_exists(scx_bpf_dsq_reenq___compat)) - scx_bpf_dsq_reenq___compat(dsq_id, reenq_flags, NULL); + scx_bpf_dsq_reenq___compat(dsq_id, reenq_flags); else if (dsq_id == SCX_DSQ_LOCAL && reenq_flags == 0) scx_bpf_reenqueue_local(); else diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index da4b459820fdd..8e287c88ad90f 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -143,7 +143,6 @@ #define HAVE___SCX_REENQ_TSR_MASK #define HAVE_SCX_RQ_ONLINE #define HAVE_SCX_RQ_CAN_STOP_TICK -#define HAVE_SCX_RQ_BAL_KEEP #define HAVE_SCX_RQ_CLK_VALID #define HAVE_SCX_RQ_BAL_CB_PENDING #define HAVE_SCX_RQ_IN_WAKEUP diff --git a/tools/sched_ext/include/scx/enums.autogen.bpf.h b/tools/sched_ext/include/scx/enums.autogen.bpf.h index dafccbb6b69d2..d02abed94e8ee 100644 --- a/tools/sched_ext/include/scx/enums.autogen.bpf.h +++ b/tools/sched_ext/include/scx/enums.autogen.bpf.h @@ -22,9 +22,6 @@ const volatile u64 __SCX_RQ_CAN_STOP_TICK __weak; const volatile u64 __SCX_RQ_BAL_PENDING __weak; #define SCX_RQ_BAL_PENDING __SCX_RQ_BAL_PENDING -const volatile u64 __SCX_RQ_BAL_KEEP __weak; -#define SCX_RQ_BAL_KEEP __SCX_RQ_BAL_KEEP - const volatile u64 __SCX_RQ_BYPASSING __weak; #define SCX_RQ_BYPASSING __SCX_RQ_BYPASSING diff --git a/tools/sched_ext/include/scx/enums.autogen.h b/tools/sched_ext/include/scx/enums.autogen.h index bbd4901f4fce3..c26934a3d8e9c 100644 --- a/tools/sched_ext/include/scx/enums.autogen.h +++ b/tools/sched_ext/include/scx/enums.autogen.h @@ -11,7 +11,6 @@ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_ONLINE); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CAN_STOP_TICK); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_PENDING); \ - SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_KEEP); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BYPASSING); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CLK_VALID); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_WAKEUP); \ diff --git a/tools/sched_ext/scx_central.bpf.c b/tools/sched_ext/scx_central.bpf.c index 64dd60b3e9223..65dae9e454009 100644 --- a/tools/sched_ext/scx_central.bpf.c +++ b/tools/sched_ext/scx_central.bpf.c @@ -299,6 +299,7 @@ static int central_timerfn(void *map, int *key, struct bpf_timer *timer) u64 now = scx_bpf_now(); u64 nr_to_kick = nr_queued; s32 i, curr_cpu; + int ret; curr_cpu = bpf_get_smp_processor_id(); if (timer_pinned && (curr_cpu != central_cpu)) { @@ -332,7 +333,10 @@ static int central_timerfn(void *map, int *key, struct bpf_timer *timer) scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); } - bpf_timer_start(timer, TIMER_INTERVAL_NS, BPF_F_TIMER_CPU_PIN); + ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, + timer_pinned ? BPF_F_TIMER_CPU_PIN : 0); + if (ret) + scx_bpf_error("bpf_timer_start failed (%d)", ret); __sync_fetch_and_add(&nr_timers, 1); return 0; } diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c index fec3595818269..98fdd4b040331 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -256,7 +256,7 @@ static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) * and thus can't be updated and repositioned. Instead, we collect the * vtime deltas separately and apply it asynchronously here. */ - delta = __sync_fetch_and_sub(&cgc->cvtime_delta, cgc->cvtime_delta); + delta = __sync_fetch_and_and(&cgc->cvtime_delta, 0); cvtime = cgv_node->cvtime + delta; /* @@ -570,7 +570,8 @@ void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) cgc = find_cgrp_ctx(cgrp); if (cgc) { __sync_fetch_and_add(&cgc->cvtime_delta, - p->se.sum_exec_runtime - taskc->bypassed_at); + (p->se.sum_exec_runtime - taskc->bypassed_at) * + FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); taskc->bypassed_at = 0; } bpf_cgroup_release(cgrp); @@ -768,10 +769,18 @@ void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) * cgroup to execute but the latter needs to be done in a loop * and we can't keep the lock held. Oh well... */ + s64 delta = now - cpuc->cur_at - cgrp_slice_ns; + bpf_spin_lock(&cgv_tree_lock); - __sync_fetch_and_add(&cgc->cvtime_delta, - (cpuc->cur_at + cgrp_slice_ns - now) * - FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); + /* keep the dividends positive, BPF division is unsigned */ + if (delta >= 0) + __sync_fetch_and_add(&cgc->cvtime_delta, + (u64)delta * FCG_HWEIGHT_ONE / + (cgc->hweight ?: 1)); + else + __sync_fetch_and_sub(&cgc->cvtime_delta, + (u64)-delta * FCG_HWEIGHT_ONE / + (cgc->hweight ?: 1)); bpf_spin_unlock(&cgv_tree_lock); } else { stat_inc(FCG_STAT_CNS_GONE); @@ -925,7 +934,7 @@ void BPF_STRUCT_OPS(fcg_cgroup_move, struct task_struct *p, if (!(from_cgc = find_cgrp_ctx(from)) || !(to_cgc = find_cgrp_ctx(to))) return; - delta = time_delta(p->scx.dsq_vtime, from_cgc->tvtime_now); + delta = (s64)(p->scx.dsq_vtime - from_cgc->tvtime_now); scx_bpf_task_set_dsq_vtime(p, to_cgc->tvtime_now + delta); } diff --git a/tools/sched_ext/scx_flatcg.c b/tools/sched_ext/scx_flatcg.c index de2bef86d64d6..7799782b76d18 100644 --- a/tools/sched_ext/scx_flatcg.c +++ b/tools/sched_ext/scx_flatcg.c @@ -105,12 +105,12 @@ static void fcg_read_stats(struct scx_flatcg *skel, __u64 *stats) __u64 *cnts; __u32 idx; + memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); + cnts = calloc(skel->rodata->nr_cpus, sizeof(__u64)); if (!cnts) return; - memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS); - for (idx = 0; idx < FCG_NR_STATS; idx++) { int ret, cpu; diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index fd9a82a676278..fd5883d01d723 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -495,7 +495,7 @@ static void update_core_sched_head_seq(struct task_struct *p) * moving them to HIGHPRI_DSQ and then consuming them first. This makes minor * difference only when dsp_batch is larger than 1. * - * scx_bpf_dispatch[_vtime]_from_dsq() are allowed both from ops.dispatch() and + * scx_bpf_dsq_move[_vtime]() are allowed both from ops.dispatch() and * non-rq-lock holding BPF programs. As demonstration, this function is called * from qmap_dispatch() and monitor_timerfn(). */ @@ -749,7 +749,7 @@ static s64 task_qdist(struct task_struct *p) bool BPF_STRUCT_OPS(qmap_core_sched_before, struct task_struct *a, struct task_struct *b) { - return task_qdist(a) > task_qdist(b); + return task_qdist(a) < task_qdist(b); } /* diff --git a/tools/scripts/Makefile.include b/tools/scripts/Makefile.include index 41971a68972dd..7022e78208a23 100644 --- a/tools/scripts/Makefile.include +++ b/tools/scripts/Makefile.include @@ -61,10 +61,18 @@ $(error Invalid value for LLVM, see Documentation/kbuild/llvm.rst) endif $(call allow-override,CC,$(LLVM_PREFIX)clang$(LLVM_SUFFIX)) +$(call allow-override,CLANG,$(LLVM_PREFIX)clang$(LLVM_SUFFIX)) +$(call allow-override,HOSTCC,$(LLVM_PREFIX)clang$(LLVM_SUFFIX)) $(call allow-override,AR,$(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX)) +$(call allow-override,HOSTAR,$(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX)) $(call allow-override,LD,$(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX)) +$(call allow-override,HOSTLD,$(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX)) $(call allow-override,CXX,$(LLVM_PREFIX)clang++$(LLVM_SUFFIX)) $(call allow-override,STRIP,$(LLVM_PREFIX)llvm-strip$(LLVM_SUFFIX)) +$(call allow-override,LLVM_STRIP,$(LLVM_PREFIX)llvm-strip$(LLVM_SUFFIX)) +$(call allow-override,LLC,$(LLVM_PREFIX)llc$(LLVM_SUFFIX)) +$(call allow-override,LLVM_CONFIG,$(LLVM_PREFIX)llvm-config$(LLVM_SUFFIX)) +$(call allow-override,LLVM_OBJCOPY,$(LLVM_PREFIX)llvm-objcopy$(LLVM_SUFFIX)) else # Allow setting various cross-compile vars or setting CROSS_COMPILE as a prefix. $(call allow-override,CC,$(CROSS_COMPILE)gcc) @@ -72,26 +80,21 @@ $(call allow-override,AR,$(CROSS_COMPILE)ar) $(call allow-override,LD,$(CROSS_COMPILE)ld) $(call allow-override,CXX,$(CROSS_COMPILE)g++) $(call allow-override,STRIP,$(CROSS_COMPILE)strip) -endif - -CC_NO_CLANG := $(shell $(CC) -dM -E -x c /dev/null | grep -Fq "__clang__"; echo $$?) -ifneq ($(LLVM),) -HOSTAR ?= $(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX) -HOSTCC ?= $(LLVM_PREFIX)clang$(LLVM_SUFFIX) -HOSTLD ?= $(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX) -else -HOSTAR ?= ar -HOSTCC ?= gcc -HOSTLD ?= ld +# Host versions aren't prefixed +$(call allow-override,HOSTAR,ar) +$(call allow-override,HOSTCC,gcc) +$(call allow-override,HOSTLD,ld) + +# Some tools still require Clang, LLC and/or LLVM utils +$(call allow-override,CLANG,clang) +$(call allow-override,LLC,llc) +$(call allow-override,LLVM_CONFIG,llvm-config) +$(call allow-override,LLVM_OBJCOPY,llvm-objcopy) +$(call allow-override,LLVM_STRIP,llvm-strip) endif -# Some tools require Clang, LLC and/or LLVM utils -CLANG ?= clang -LLC ?= llc -LLVM_CONFIG ?= llvm-config -LLVM_OBJCOPY ?= llvm-objcopy -LLVM_STRIP ?= llvm-strip +CC_NO_CLANG := $(shell $(CC) -dM -E -x c /dev/null | grep -Fq "__clang__"; echo $$?) # Some tools require bpftool SYSTEM_BPFTOOL ?= bpftool diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index ef92dd35e030c..a0d75a0761337 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -27,6 +27,7 @@ static bool fail_autoassemble; #define NR_CXL_SWITCH_PORTS 2 #define NR_CXL_PORT_DECODERS 8 #define NR_BRIDGES (NR_CXL_HOST_BRIDGES + NR_CXL_SINGLE_HOST + NR_CXL_RCH) +#define NR_CXL_TYPE2_ACCEL 1 #define MOCK_AUTO_REGION_SIZE_DEFAULT SZ_512M static int mock_auto_region_size = MOCK_AUTO_REGION_SIZE_DEFAULT; @@ -1563,8 +1564,10 @@ static __init int cxl_rch_topo_init(void) struct platform_device *pdev; pdev = platform_device_alloc("cxl_host_bridge", idx); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_bridge; + } mock_companion(adev, &pdev->dev); rc = cxl_mock_platform_device_add(pdev, &cxl_rch[i]); @@ -1618,8 +1621,10 @@ static __init int cxl_single_topo_init(void) pdev = platform_device_alloc("cxl_host_bridge", NR_CXL_HOST_BRIDGES + i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_bridge; + } mock_companion(adev, &pdev->dev); rc = cxl_mock_platform_device_add(pdev, &cxl_hb_single[i]); @@ -1640,8 +1645,10 @@ static __init int cxl_single_topo_init(void) pdev = platform_device_alloc("cxl_root_port", NR_MULTI_ROOT + i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_port; + } pdev->dev.parent = &bridge->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_root_single[i]); @@ -1655,8 +1662,10 @@ static __init int cxl_single_topo_init(void) pdev = platform_device_alloc("cxl_switch_uport", NR_MULTI_ROOT + i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_uport; + } pdev->dev.parent = &root_port->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_swu_single[i]); @@ -1671,8 +1680,10 @@ static __init int cxl_single_topo_init(void) pdev = platform_device_alloc("cxl_switch_dport", i + NR_MEM_MULTI); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_dport; + } pdev->dev.parent = &uport->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_swd_single[i]); @@ -1724,19 +1735,84 @@ static void cxl_single_topo_exit(void) } } -static void cxl_mem_exit(void) +static void cxl_type3_mem_exit(void) { + struct platform_device *pdev; int i; - for (i = ARRAY_SIZE(cxl_rcd) - 1; i >= 0; i--) + for (i = ARRAY_SIZE(cxl_rcd) - 1; i >= 0; i--) { + pdev = cxl_rcd[i]; + if (!pdev) + continue; platform_device_unregister(cxl_rcd[i]); - for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--) + } + + for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--) { + pdev = cxl_mem_single[i]; + if (!pdev) + continue; platform_device_unregister(cxl_mem_single[i]); - for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--) + } + + for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--) { + pdev = cxl_mem[i]; + if (!pdev) + continue; + platform_device_unregister(pdev); + } +} + +static void cxl_type2_mem_exit(void) +{ + for (int i = NR_CXL_TYPE2_ACCEL - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_mem[i]; + + if (!pdev) + continue; + platform_device_unregister(pdev); + } +} + +static void cxl_mem_exit(void) +{ + if (type2_test) { + cxl_type2_mem_exit(); + return; + } + + cxl_type3_mem_exit(); +} + +static int cxl_type2_mem_init(void) +{ + int i, rc; + + for (i = 0; i < NR_CXL_TYPE2_ACCEL; i++) { + struct platform_device *dport = cxl_root_port[i]; + struct platform_device *pdev; + + pdev = platform_device_alloc("cxl_type2_accel", i); + if (!pdev) { + rc = -ENOMEM; + goto err_mem; + } + pdev->dev.parent = &dport->dev; + set_dev_node(&pdev->dev, i % 2); + + rc = cxl_mock_platform_device_add(pdev, &cxl_mem[i]); + if (rc) + goto err_mem; + } + + return 0; + +err_mem: + for (i = NR_CXL_TYPE2_ACCEL - 1; i >= 0; i--) platform_device_unregister(cxl_mem[i]); + return rc; } -static int cxl_mem_init(void) +static int cxl_type3_mem_init(void) { int i, rc; @@ -1745,8 +1821,10 @@ static int cxl_mem_init(void) struct platform_device *pdev; pdev = platform_device_alloc("cxl_mem", i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_mem; + } pdev->dev.parent = &dport->dev; set_dev_node(&pdev->dev, i % 2); @@ -1760,8 +1838,10 @@ static int cxl_mem_init(void) struct platform_device *pdev; pdev = platform_device_alloc("cxl_mem", NR_MEM_MULTI + i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_single; + } pdev->dev.parent = &dport->dev; set_dev_node(&pdev->dev, i % 2); @@ -1776,8 +1856,10 @@ static int cxl_mem_init(void) struct platform_device *pdev; pdev = platform_device_alloc("cxl_rcd", idx); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_rcd; + } pdev->dev.parent = &rch->dev; set_dev_node(&pdev->dev, i % 2); @@ -1800,6 +1882,13 @@ static int cxl_mem_init(void) return rc; } +static int cxl_mem_init(void) +{ + if (type2_test) + return cxl_type2_mem_init(); + return cxl_type3_mem_init(); +} + static ssize_t decoder_reset_preserve_registry_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -1827,58 +1916,48 @@ static struct attribute *cxl_acpi_attrs[] = { }; ATTRIBUTE_GROUPS(cxl_acpi); -static __init int cxl_test_init(void) +static bool __init have_multiple_modparms(void) { - int rc, i; - struct range mappable; + int count = 0; - if (!IS_ALIGNED(mock_auto_region_size, PMD_SIZE)) { - pr_err_once("mock_auto_region_size %d must be PMD-aligned\n", - mock_auto_region_size); - return -EINVAL; - } + if (interleave_arithmetic) + count++; + if (extended_linear_cache) + count++; + if (hmem_test) + count++; - cxl_acpi_test(); - cxl_core_test(); - cxl_mem_test(); - cxl_pmem_test(); - cxl_port_test(); + return count > 1; +} - register_cxl_mock_ops(&cxl_mock_ops); +static void host_bridges_remove(void) +{ + int i; - cxl_mock_pool = gen_pool_create(ilog2(SZ_2M), NUMA_NO_NODE); - if (!cxl_mock_pool) { - rc = -ENOMEM; - goto err_gen_pool_create; - } - mappable = mhp_get_pluggable_range(true); + for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_host_bridge[i]; - rc = gen_pool_add(cxl_mock_pool, - min(iomem_resource.end + 1 - SZ_64G, - mappable.end + 1 - SZ_64G), - SZ_64G, NUMA_NO_NODE); - if (rc) - goto err_gen_pool_add; + if (!pdev) + continue; - if (interleave_arithmetic == 1) { - cfmws_start = CFMWS_XOR_ARRAY_START; - cfmws_end = CFMWS_XOR_ARRAY_END; - } else { - cfmws_start = CFMWS_MOD_ARRAY_START; - cfmws_end = CFMWS_MOD_ARRAY_END; + sysfs_remove_link(&pdev->dev.kobj, "physical_node"); + platform_device_unregister(cxl_host_bridge[i]); } +} - rc = populate_cedt(); - if (rc) - goto err_populate; +static int host_bridges_populate(void) +{ + int rc = 0; - for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) { + for (int i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) { struct acpi_device *adev = &host_bridge[i]; struct platform_device *pdev; pdev = platform_device_alloc("cxl_host_bridge", i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_bridge; + } mock_companion(adev, &pdev->dev); rc = cxl_mock_platform_device_add(pdev, &cxl_host_bridge[i]); @@ -1892,14 +1971,40 @@ static __init int cxl_test_init(void) goto err_bridge; } - for (i = 0; i < ARRAY_SIZE(cxl_root_port); i++) { + return 0; + +err_bridge: + host_bridges_remove(); + return rc; +} + +static void cxl_rootports_remove(void) +{ + for (int i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_root_port[i]; + + if (!pdev) + continue; + + platform_device_unregister(pdev); + } +} + +static int cxl_rootports_populate(void) +{ + int rc = 0; + + for (int i = 0; i < ARRAY_SIZE(cxl_root_port); i++) { struct platform_device *bridge = cxl_host_bridge[i % ARRAY_SIZE(cxl_host_bridge)]; struct platform_device *pdev; pdev = platform_device_alloc("cxl_root_port", i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_port; + } + pdev->dev.parent = &bridge->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_root_port[i]); @@ -1907,14 +2012,39 @@ static __init int cxl_test_init(void) goto err_port; } - BUILD_BUG_ON(ARRAY_SIZE(cxl_switch_uport) != ARRAY_SIZE(cxl_root_port)); - for (i = 0; i < ARRAY_SIZE(cxl_switch_uport); i++) { + return 0; + +err_port: + cxl_rootports_remove(); + return rc; +} + +static void cxl_usps_remove(void) +{ + for (int i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_switch_uport[i]; + + if (!pdev) + continue; + + platform_device_unregister(cxl_switch_uport[i]); + } +} + +static int cxl_usps_populate(void) +{ + int rc = 0; + + for (int i = 0; i < ARRAY_SIZE(cxl_switch_uport); i++) { struct platform_device *root_port = cxl_root_port[i]; struct platform_device *pdev; pdev = platform_device_alloc("cxl_switch_uport", i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_uport; + } + pdev->dev.parent = &root_port->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_switch_uport[i]); @@ -1922,14 +2052,40 @@ static __init int cxl_test_init(void) goto err_uport; } - for (i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++) { + return 0; + +err_uport: + cxl_usps_remove(); + return rc; +} + +static void cxl_dsps_remove(void) +{ + for (int i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_switch_dport[i]; + + if (!pdev) + continue; + + platform_device_unregister(cxl_switch_dport[i]); + } +} + + +static int cxl_dsps_populate(void) +{ + int rc = 0; + + for (int i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++) { struct platform_device *uport = cxl_switch_uport[i % ARRAY_SIZE(cxl_switch_uport)]; struct platform_device *pdev; pdev = platform_device_alloc("cxl_switch_dport", i); - if (!pdev) + if (!pdev) { + rc = -ENOMEM; goto err_dport; + } pdev->dev.parent = &uport->dev; rc = cxl_mock_platform_device_add(pdev, &cxl_switch_dport[i]); @@ -1937,17 +2093,181 @@ static __init int cxl_test_init(void) goto err_dport; } + return 0; + +err_dport: + cxl_dsps_remove(); + return rc; +} + +static void cxl_switches_remove(void) +{ + cxl_dsps_remove(); + cxl_usps_remove(); +} + +static int cxl_switches_populate(void) +{ + int rc; + + BUILD_BUG_ON(ARRAY_SIZE(cxl_switch_uport) != ARRAY_SIZE(cxl_root_port)); + rc = cxl_usps_populate(); + if (rc) + return rc; + + rc = cxl_dsps_populate(); + if (rc) { + cxl_usps_remove(); + return rc; + } + + return 0; +} + +static void cxl_type2_topo_exit(void) +{ + cxl_rootports_remove(); + host_bridges_remove(); +} + +static int cxl_type2_topo_init(void) +{ + int rc; + + rc = host_bridges_populate(); + if (rc) + return rc; + + rc = cxl_rootports_populate(); + if (rc) { + host_bridges_remove(); + return rc; + } + + return 0; +} + +static void cxl_type3_topo_exit(void) +{ + cxl_rch_topo_exit(); + cxl_single_topo_exit(); + cxl_switches_remove(); + cxl_rootports_remove(); + host_bridges_remove(); +} + +static int cxl_type3_topo_init(void) +{ + int rc; + + rc = host_bridges_populate(); + if (rc) + return rc; + + rc = cxl_rootports_populate(); + if (rc) + goto err_host_bridges; + + rc = cxl_switches_populate(); + if (rc) + goto err_root_ports; + rc = cxl_single_topo_init(); if (rc) - goto err_dport; + goto err_switches; rc = cxl_rch_topo_init(); if (rc) goto err_single; + return 0; + +err_single: + cxl_single_topo_exit(); +err_switches: + cxl_switches_remove(); +err_root_ports: + cxl_rootports_remove(); +err_host_bridges: + host_bridges_remove(); + return rc; +} + +static void cxl_topo_exit(void) +{ + if (type2_test) { + cxl_type2_topo_exit(); + return; + } + + cxl_type3_topo_exit(); +} + +static int cxl_topo_init(void) +{ + if (type2_test) + return cxl_type2_topo_init(); + return cxl_type3_topo_init(); +} + +static __init int cxl_test_init(void) +{ + struct range mappable; + int rc; + + /* Enforce a single module param active at a time */ + if (have_multiple_modparms()) + return -EINVAL; + + if (!IS_ALIGNED(mock_auto_region_size, PMD_SIZE)) { + pr_err_once("mock_auto_region_size %d must be PMD-aligned\n", + mock_auto_region_size); + return -EINVAL; + } + + cxl_acpi_test(); + cxl_core_test(); + cxl_mem_test(); + cxl_pmem_test(); + cxl_port_test(); + + register_cxl_mock_ops(&cxl_mock_ops); + + cxl_mock_pool = gen_pool_create(ilog2(SZ_2M), NUMA_NO_NODE); + if (!cxl_mock_pool) { + rc = -ENOMEM; + goto err_gen_pool_create; + } + mappable = mhp_get_pluggable_range(true); + + rc = gen_pool_add(cxl_mock_pool, + min(iomem_resource.end + 1 - SZ_64G, + mappable.end + 1 - SZ_64G), + SZ_64G, NUMA_NO_NODE); + if (rc) + goto err_gen_pool_add; + + if (interleave_arithmetic == 1) { + cfmws_start = CFMWS_XOR_ARRAY_START; + cfmws_end = CFMWS_XOR_ARRAY_END; + } else { + cfmws_start = CFMWS_MOD_ARRAY_START; + cfmws_end = CFMWS_MOD_ARRAY_END; + } + + rc = populate_cedt(); + if (rc) + goto err_populate; + + rc = cxl_topo_init(); + if (rc) + goto err_populate; + cxl_acpi = platform_device_alloc("cxl_acpi", 0); - if (!cxl_acpi) - goto err_rch; + if (!cxl_acpi) { + rc = -ENOMEM; + goto err_topo; + } mock_companion(&acpi0017_mock, &cxl_acpi->dev); acpi0017_mock.dev.bus = &platform_bus_type; @@ -1955,7 +2275,7 @@ static __init int cxl_test_init(void) rc = cxl_mock_platform_device_add(cxl_acpi, NULL); if (rc) - goto err_rch; + goto err_topo; rc = cxl_mem_init(); if (rc) @@ -1971,28 +2291,8 @@ static __init int cxl_test_init(void) cxl_mem_exit(); err_root: platform_device_unregister(cxl_acpi); -err_rch: - cxl_rch_topo_exit(); -err_single: - cxl_single_topo_exit(); -err_dport: - for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--) - platform_device_unregister(cxl_switch_dport[i]); -err_uport: - for (i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--) - platform_device_unregister(cxl_switch_uport[i]); -err_port: - for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--) - platform_device_unregister(cxl_root_port[i]); -err_bridge: - for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) { - struct platform_device *pdev = cxl_host_bridge[i]; - - if (!pdev) - continue; - sysfs_remove_link(&pdev->dev.kobj, "physical_node"); - platform_device_unregister(cxl_host_bridge[i]); - } +err_topo: + cxl_topo_exit(); err_populate: depopulate_all_mock_resources(); err_gen_pool_add: @@ -2015,27 +2315,10 @@ static void free_decoder_registry(void) static __exit void cxl_test_exit(void) { - int i; - hmem_test_exit(); cxl_mem_exit(); platform_device_unregister(cxl_acpi); - cxl_rch_topo_exit(); - cxl_single_topo_exit(); - for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--) - platform_device_unregister(cxl_switch_dport[i]); - for (i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--) - platform_device_unregister(cxl_switch_uport[i]); - for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--) - platform_device_unregister(cxl_root_port[i]); - for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) { - struct platform_device *pdev = cxl_host_bridge[i]; - - if (!pdev) - continue; - sysfs_remove_link(&pdev->dev.kobj, "physical_node"); - platform_device_unregister(cxl_host_bridge[i]); - } + cxl_topo_exit(); depopulate_all_mock_resources(); gen_pool_destroy(cxl_mock_pool); unregister_cxl_mock_ops(&cxl_mock_ops); diff --git a/tools/testing/cxl/test/hmem_test.c b/tools/testing/cxl/test/hmem_test.c index 3a1a089e1721b..0fa00f7e16db5 100644 --- a/tools/testing/cxl/test/hmem_test.c +++ b/tools/testing/cxl/test/hmem_test.c @@ -3,8 +3,9 @@ #include #include #include "../../../drivers/dax/bus.h" +#include "mock.h" -static bool hmem_test; +bool hmem_test; static void hmem_test_work(struct work_struct *work) { diff --git a/tools/testing/cxl/test/mock.h b/tools/testing/cxl/test/mock.h index 4f57dc80ae7d5..846d7c5d6eaa9 100644 --- a/tools/testing/cxl/test/mock.h +++ b/tools/testing/cxl/test/mock.h @@ -5,6 +5,8 @@ #include #include +extern bool hmem_test; + struct cxl_mock_ops { struct list_head list; bool (*is_mock_adev)(struct acpi_device *dev); diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index ac3f7159e67fe..91d234ac3b579 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -126,7 +126,7 @@ def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) lines.pop() # Filter out any extraneous non-test output that might have gotten mixed in. - return [l for l in output if re.match(r'^[^\s.]+\.[^\s.]+$', l)] + return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)] def _list_tests_attr(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> Iterable[str]: args = ['kunit.action=list_attr'] diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index da88c3a1651d0..85ae21754bdf9 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -979,6 +979,18 @@ def test_list_tests(self): self.linux_source_mock.run_kernel.assert_called_once_with( args=['kunit.action=list'], build_dir='.kunit', filter_glob='suite*', filter='', filter_action=None, timeout=300) + def test_list_tests_with_prefix(self): + want = ['suite.test1', 'suite.test2', 'suite2.test1'] + self.linux_source_mock.run_kernel.return_value = [ + '[ 0.100000] TAP version 14', + '[ 0.200000] suite.test1', + '[ 0.200000] suite.test2', + '[ 0.300000] suite2.test1'] + + got = kunit._list_tests(self.linux_source_mock, + kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False, False)) + self.assertEqual(got, want) + @mock.patch.object(kunit, '_list_tests') def test_run_isolated_by_suite(self, mock_tests): mock_tests.return_value = ['suite.test1', 'suite.test2', 'suite2.test1'] diff --git a/tools/testing/selftests/alsa/mixer-test.c b/tools/testing/selftests/alsa/mixer-test.c index a329f901c5edf..0857d64c322a0 100644 --- a/tools/testing/selftests/alsa/mixer-test.c +++ b/tools/testing/selftests/alsa/mixer-test.c @@ -319,8 +319,8 @@ static bool ctl_value_index_valid(struct ctl_data *ctl, /* Only check step size if there is one and we're in bounds */ if (snd_ctl_elem_info_get_step(ctl->info) && - (int_val - snd_ctl_elem_info_get_min(ctl->info) % - snd_ctl_elem_info_get_step(ctl->info))) { + (int_val - snd_ctl_elem_info_get_min(ctl->info)) % + snd_ctl_elem_info_get_step(ctl->info)) { ksft_print_msg("%s.%d value %ld invalid for step %ld minimum %ld\n", ctl->name, index, int_val, snd_ctl_elem_info_get_step(ctl->info), diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 22c584b78be51..b435837c8c0e8 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -65,6 +65,9 @@ /* VL 128..2048 in powers of 2 */ #define MAX_NUM_VLS 5 +/* Sentinel for detecting buffer bytes the kernel did not write */ +#define REGSET_SENTINEL 0xa5 + /* * FPMR bits we can set without doing feature checks to see if values * are valid. @@ -181,6 +184,20 @@ static bool compare_buffer(const char *name, void *out, return false; } +static bool buffer_is_filled(const void *buffer, size_t size, + unsigned char value) +{ + const unsigned char *bytes = buffer; + size_t i; + + for (i = 0; i < size; i++) { + if (bytes[i] != value) + return false; + } + + return true; +} + struct test_config { int sve_vl_in; int sve_vl_expected; @@ -401,6 +418,7 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -409,14 +427,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sve_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SVE: %s (%d)\n", @@ -440,10 +460,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) } if (svcr_in & SVCR_SM) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SVE reports data with PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SVE wrote beyond its header with PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SVE header size: %d != %lu\n", @@ -485,6 +511,7 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -493,14 +520,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sme_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SSVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SSVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SSVE: %s (%d)\n", @@ -523,10 +552,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) } if (!(svcr_in & SVCR_SM)) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SSVE reports data without PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SSVE wrote beyond its header without PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n", diff --git a/tools/testing/selftests/arm64/fp/sve-test.S b/tools/testing/selftests/arm64/fp/sve-test.S index 80e072f221cde..7ef7835389e76 100644 --- a/tools/testing/selftests/arm64/fp/sve-test.S +++ b/tools/testing/selftests/arm64/fp/sve-test.S @@ -298,15 +298,20 @@ function irritator_handler add x0, x0, #1 str x0, [x2, #ucontext_regs + 8 * 23] +#ifndef SSVE // Corrupt some random Z-regs movi v0.8b, #1 movi v9.16b, #2 movi v31.8b, #3 // And P0 ptrue p0.d -#ifndef SSVE // And FFR wrffr p15.b +#else + // Enter and exit streaming mode, will reset all of the V, Z, P + // and FFR registers that the system has. + smstart_sm + smstop #endif ret diff --git a/tools/testing/selftests/arm64/mte/check_buffer_fill.c b/tools/testing/selftests/arm64/mte/check_buffer_fill.c index ff4e075033493..039b1d7d85663 100644 --- a/tools/testing/selftests/arm64/mte/check_buffer_fill.c +++ b/tools/testing/selftests/arm64/mte/check_buffer_fill.c @@ -406,6 +406,8 @@ int main(int argc, char *argv[]) size_t page_size = getpagesize(); int item = ARRAY_SIZE(sizes); + ksft_print_header(); + sizes[item - 3] = page_size - 1; sizes[item - 2] = page_size; sizes[item - 1] = page_size + 1; diff --git a/tools/testing/selftests/arm64/mte/check_child_memory.c b/tools/testing/selftests/arm64/mte/check_child_memory.c index 5e97ee792e4d2..e6a8acca2a941 100644 --- a/tools/testing/selftests/arm64/mte/check_child_memory.c +++ b/tools/testing/selftests/arm64/mte/check_child_memory.c @@ -146,6 +146,8 @@ int main(int argc, char *argv[]) int err; int item = ARRAY_SIZE(sizes); + ksft_print_header(); + page_size = getpagesize(); if (!page_size) { ksft_print_msg("ERR: Unable to get page size\n"); diff --git a/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c b/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c index 325bca0de0f6e..5d9dc8bcfbf55 100644 --- a/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c +++ b/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c @@ -69,7 +69,7 @@ void *execute_thread(void *x) int execute_test(pid_t pid) { pthread_t thread_id[MAX_THREADS]; - int thread_data[MAX_THREADS]; + intptr_t thread_data[MAX_THREADS]; for (int i = 0; i < MAX_THREADS; i++) pthread_create(&thread_id[i], NULL, @@ -131,6 +131,7 @@ int main(int argc, char *argv[]) if (err) return err; + ksft_print_header(); ksft_set_plan(1); evaluate_test(mte_gcr_fork_test(), diff --git a/tools/testing/selftests/arm64/mte/check_hugetlb_options.c b/tools/testing/selftests/arm64/mte/check_hugetlb_options.c index aad1234c7e0fe..23e4a7a9950c1 100644 --- a/tools/testing/selftests/arm64/mte/check_hugetlb_options.c +++ b/tools/testing/selftests/arm64/mte/check_hugetlb_options.c @@ -230,6 +230,8 @@ int main(int argc, char *argv[]) void *map_ptr; unsigned long map_size; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; diff --git a/tools/testing/selftests/arm64/mte/check_ksm_options.c b/tools/testing/selftests/arm64/mte/check_ksm_options.c index 0cf5faef17248..4855b737d5507 100644 --- a/tools/testing/selftests/arm64/mte/check_ksm_options.c +++ b/tools/testing/selftests/arm64/mte/check_ksm_options.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,20 @@ static size_t page_sz; static unsigned long ksm_sysfs[5]; +static bool has_merge_across_nodes; + +static bool merge_across_nodes_available(void) +{ + const char *path = PATH_KSM "merge_across_nodes"; + + if (!access(path, R_OK | W_OK)) + return true; + if (errno == ENOENT) + return false; + + ksft_exit_skip("Unable to read and write %s: %s\n", path, + strerror(errno)); +} static unsigned long read_sysfs(char *str) { @@ -56,8 +71,10 @@ static void write_sysfs(char *str, unsigned long val) static void mte_ksm_setup(void) { - ksm_sysfs[0] = read_sysfs(PATH_KSM "merge_across_nodes"); - write_sysfs(PATH_KSM "merge_across_nodes", 1); + if (has_merge_across_nodes) { + ksm_sysfs[0] = read_sysfs(PATH_KSM "merge_across_nodes"); + write_sysfs(PATH_KSM "merge_across_nodes", 1); + } ksm_sysfs[1] = read_sysfs(PATH_KSM "sleep_millisecs"); write_sysfs(PATH_KSM "sleep_millisecs", 0); ksm_sysfs[2] = read_sysfs(PATH_KSM "run"); @@ -70,7 +87,8 @@ static void mte_ksm_setup(void) static void mte_ksm_restore(void) { - write_sysfs(PATH_KSM "merge_across_nodes", ksm_sysfs[0]); + if (has_merge_across_nodes) + write_sysfs(PATH_KSM "merge_across_nodes", ksm_sysfs[0]); write_sysfs(PATH_KSM "sleep_millisecs", ksm_sysfs[1]); write_sysfs(PATH_KSM "run", ksm_sysfs[2]); write_sysfs(PATH_KSM "max_page_sharing", ksm_sysfs[3]); @@ -132,9 +150,16 @@ int main(int argc, char *argv[]) { int err; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; + + if (geteuid() != 0) + ksft_exit_skip("Please run the test as root\n"); + + has_merge_across_nodes = merge_across_nodes_available(); page_sz = getpagesize(); if (!page_sz) { ksft_print_msg("ERR: Unable to get page size\n"); diff --git a/tools/testing/selftests/arm64/mte/check_mmap_options.c b/tools/testing/selftests/arm64/mte/check_mmap_options.c index c100af3012cbb..492f2cd41f43b 100644 --- a/tools/testing/selftests/arm64/mte/check_mmap_options.c +++ b/tools/testing/selftests/arm64/mte/check_mmap_options.c @@ -945,6 +945,8 @@ int main(int argc, char *argv[]) }, }; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; diff --git a/tools/testing/selftests/arm64/mte/check_prctl.c b/tools/testing/selftests/arm64/mte/check_prctl.c index f7f320defa7b9..d16a91117eef9 100644 --- a/tools/testing/selftests/arm64/mte/check_prctl.c +++ b/tools/testing/selftests/arm64/mte/check_prctl.c @@ -119,7 +119,7 @@ int main(void) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(mte_modes)); + ksft_set_plan(ARRAY_SIZE(mte_modes) + 1); check_basic_read(); for (i = 0; i < ARRAY_SIZE(mte_modes); i++) diff --git a/tools/testing/selftests/arm64/mte/check_tags_inclusion.c b/tools/testing/selftests/arm64/mte/check_tags_inclusion.c index 4b764f2a81857..6b4fa6705d7c1 100644 --- a/tools/testing/selftests/arm64/mte/check_tags_inclusion.c +++ b/tools/testing/selftests/arm64/mte/check_tags_inclusion.c @@ -175,6 +175,8 @@ int main(int argc, char *argv[]) { int err; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; diff --git a/tools/testing/selftests/arm64/mte/check_user_mem.c b/tools/testing/selftests/arm64/mte/check_user_mem.c index fb7936c4e0978..af343aa617323 100644 --- a/tools/testing/selftests/arm64/mte/check_user_mem.c +++ b/tools/testing/selftests/arm64/mte/check_user_mem.c @@ -201,6 +201,8 @@ int main(int argc, char *argv[]) int tag_offsets[] = {page_sz, MT_GRANULE_SIZE}; char test_name[TEST_NAME_MAX]; + ksft_print_header(); + page_sz = getpagesize(); if (!page_sz) { ksft_print_msg("ERR: Unable to get page size\n"); diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index b642ee489ea64..5ae24ee101007 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -324,8 +324,6 @@ TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) USE_BOOTSTRAP := "bootstrap/" endif -TEST_GEN_PROGS_EXTENDED += $(TRUNNER_BPFTOOL) - $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(BPFOBJ) TESTING_HELPERS := $(OUTPUT)/testing_helpers.o @@ -466,7 +464,7 @@ endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \ - -I$(CURDIR)/libarena/include \ + -I$(TOOLSINCDIR) -I$(CURDIR)/libarena/include \ -I$(abspath $(OUTPUT)/../usr/include) \ -std=gnu11 \ -fno-strict-aliasing \ @@ -1055,10 +1053,13 @@ endif DEFAULT_INSTALL_RULE := $(INSTALL_RULE) override define INSTALL_RULE $(DEFAULT_INSTALL_RULE) + @mkdir -p $(INSTALL_PATH)/tools/sbin + @rsync -a $(if $(PERMISSIVE),--ignore-missing-args) $(TRUNNER_BPFTOOL) $(INSTALL_PATH)/tools/sbin/ + @rsync -a $(if $(PERMISSIVE),--ignore-missing-args) $(OUTPUT)/*.BTF $(INSTALL_PATH)/ @for DIR in $(TEST_INST_SUBDIRS); do \ mkdir -p $(INSTALL_PATH)/$$DIR; \ rsync -a $(if $(PERMISSIVE),--ignore-missing-args) \ - $(OUTPUT)/$$DIR/*.bpf.o \ + $(OUTPUT)/$$DIR/*.bpf.o $(OUTPUT)/$$DIR/*.BTF \ $(INSTALL_PATH)/$$DIR; \ done endef diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 37164322a1023..07c834433b388 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -107,12 +107,12 @@ Docker container and local rootfs image. The overall steps are as follows: tools/testing/selftests/bpf/vmtest.sh \ -l -- \ ./test_progs -d \ - \"$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ + "$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ | cut -d'#' -f1 \ | sed -e 's/^[[:space:]]*//' \ -e 's/[[:space:]]*$//' \ | tr -s '\n' ',' \ - )\" + )" Link: https://github.com/pulehui/riscv-bpf-vmtest.git [0] Link: https://github.com/libbpf/ci/blob/main/rootfs/mkrootfs_debian.sh [1] diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h b/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h index fc27a4bcf5d7e..b6676dd67bc09 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h @@ -115,7 +115,7 @@ static inline int libarena_asan_init(int arena_asan_init_fd, { LIBBPF_OPTS(bpf_test_run_opts, opts); struct asan_init_args args; - u64 globals_pages; + u64 globals_pages = 0; int ret; ret = libarena_get_globals_pages(arena_asan_init_fd, diff --git a/tools/testing/selftests/bpf/libarena/src/common.bpf.c b/tools/testing/selftests/bpf/libarena/src/common.bpf.c index 50be57213dfb8..99553aac3d433 100644 --- a/tools/testing/selftests/bpf/libarena/src/common.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/common.bpf.c @@ -38,6 +38,12 @@ __weak int arena_buddy_reset(void) return buddy_init(&buddy); } +SEC("syscall") +__weak int arena_buddy_destroy(void) +{ + return buddy_destroy(&buddy); +} + __weak void __arena *arena_malloc(size_t size) { return buddy_alloc(&buddy, size); diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index b82f572641b7d..db935a9d9fc1a 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c @@ -111,7 +111,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a if (settimeo(fd, opts->timeout_ms)) goto error_close; - if (type == SOCK_STREAM && + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM && setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) { log_err("Failed to enable SO_REUSEADDR"); goto error_close; @@ -128,7 +128,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a goto error_close; } - if (type == SOCK_STREAM) { + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM) { if (listen(fd, opts->backlog ? MAX(opts->backlog, 0) : 1) < 0) { log_err("Failed to listed on socket"); goto error_close; diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h index 79a010c88e11c..75133119c04a3 100644 --- a/tools/testing/selftests/bpf/network_helpers.h +++ b/tools/testing/selftests/bpf/network_helpers.h @@ -25,6 +25,11 @@ typedef __u16 __sum16; #define VIP_NUM 5 #define MAGIC_BYTES 123 +/* include/linux/net.h */ +#ifndef SOCK_TYPE_MASK +#define SOCK_TYPE_MASK 0xf +#endif + struct network_helper_opts { int timeout_ms; int proto; diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c index b7c18d590b993..5a1e08d39a06b 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c @@ -10,6 +10,17 @@ #include "cgroup_iter_memcg.h" #include "cgroup_iter_memcg.skel.h" +/* + * memcg stats are cached per-cpu and only become visible once the periodic + * flusher runs (FLUSH_TIME, 2s), or once pending updates cross + * MEMCG_CHARGE_BATCH * num_online_cpus(). That threshold grows with the CPU + * count, so on a large machine a single pass does not reach it and + * bpf_mem_cgroup_flush_stats() returns without flushing anything. Retry for + * long enough to cover a flusher cycle. + */ +#define MEMCG_STAT_RETRIES 16 +#define MEMCG_STAT_RETRY_DELAY_US (250 * 1000) + static int read_stats(struct bpf_link *link) { int fd, ret = 0; @@ -35,11 +46,13 @@ static int read_stats(struct bpf_link *link) static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* * Increase memcg anon usage by mapping and writing * to a new anon region. @@ -53,6 +66,12 @@ static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->nr_anon_mapped && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_anon_mapped, 0, "final anon mapped val"); cleanup: @@ -61,6 +80,7 @@ static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; char *path; @@ -76,6 +96,7 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) fd = open(path, O_CREAT | O_RDWR, 0644); if (!ASSERT_OK_FD(fd, "open fd")) return; +retry: if (!ASSERT_OK(ftruncate(fd, len), "ftruncate")) goto cleanup_fd; @@ -88,6 +109,13 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup_map; + if ((!memcg_query->nr_file_pages || !memcg_query->nr_file_mapped) && + ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_file_pages, 0, "final file value"); ASSERT_GT(memcg_query->nr_file_mapped, 0, "final file mapped value"); @@ -100,6 +128,7 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; size_t len; int fd; @@ -113,12 +142,18 @@ static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK_FD(fd, "memfd_create")) return; +retry: if (!ASSERT_OK(fallocate(fd, 0, 0, len), "fallocate")) goto cleanup; if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->nr_shmem && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + goto retry; + } + ASSERT_GT(memcg_query->nr_shmem, 0, "final shmem value"); cleanup: @@ -127,11 +162,13 @@ static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* Create region to use for triggering a page fault. */ map = mmap(NULL, len, PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (!ASSERT_NEQ(map, MAP_FAILED, "mmap anon")) @@ -143,6 +180,12 @@ static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->pgfault && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->pgfault, 0, "final pgfault val"); cleanup: diff --git a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c index 56b5baef35c8c..602ce30f1720c 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c +++ b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c @@ -23,6 +23,7 @@ static void global_map_resize_bss_subtest(void) struct bpf_map *map; const __u32 desired_sz = sizeof(skel->bss->sum) + sysconf(_SC_PAGE_SIZE) * 2; size_t array_len, actual_sz, new_sz; + int *array; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -58,10 +59,13 @@ static void global_map_resize_bss_subtest(void) goto teardown; /* fill the newly resized array with ones, - * skipping the first element which was previously set + * skipping the first element which was previously set; + * access through a plain pointer to avoid -Warray-bounds + * since the array was resized beyond its declared length. */ + array = skel->bss->array; for (int i = 1; i < array_len; i++) - skel->bss->array[i] = 1; + array[i] = 1; /* set global const values before loading */ skel->rodata->pid = getpid(); diff --git a/tools/testing/selftests/bpf/prog_tests/libarena.c b/tools/testing/selftests/bpf/prog_tests/libarena.c index 61ea68dce4105..ba5a5a50f7c07 100644 --- a/tools/testing/selftests/bpf/prog_tests/libarena.c +++ b/tools/testing/selftests/bpf/prog_tests/libarena.c @@ -15,7 +15,12 @@ static void run_libarena_test(struct libarena *skel, struct bpf_program *prog, { int ret; - if (!strstr(name, "test_buddy")) { + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); if (!ASSERT_OK(ret, "arena_buddy_reset")) return; @@ -24,7 +29,6 @@ static void run_libarena_test(struct libarena *skel, struct bpf_program *prog, ret = libarena_run_prog(bpf_program__fd(prog)); ASSERT_OK(ret, name); - } static void *run_libarena_parallel_prog(void *arg) diff --git a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c index d59d9dd12ef2b..f897405f701dd 100644 --- a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c +++ b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c @@ -17,7 +17,12 @@ static void run_libarena_asan_test(struct libarena_asan *skel, { int ret; - if (!strstr(name, "test_buddy")) { + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); if (!ASSERT_OK(ret, "arena_buddy_reset")) return; @@ -90,4 +95,3 @@ void test_libarena_asan(void) return; } - diff --git a/tools/testing/selftests/bpf/prog_tests/linked_list.c b/tools/testing/selftests/bpf/prog_tests/linked_list.c index 8defea0253ed9..178d133d59238 100644 --- a/tools/testing/selftests/bpf/prog_tests/linked_list.c +++ b/tools/testing/selftests/bpf/prog_tests/linked_list.c @@ -713,7 +713,7 @@ static void test_btf(void) break; err = btf__load_into_kernel(btf); - ASSERT_EQ(err, -ELOOP, "check btf"); + ASSERT_EQ(err, 0, "check btf"); btf__free(btf); break; } @@ -772,7 +772,7 @@ static void test_btf(void) break; err = btf__load_into_kernel(btf); - ASSERT_EQ(err, -ELOOP, "check btf"); + ASSERT_EQ(err, 0, "check btf"); btf__free(btf); break; } diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c index a970798e11735..28bc4b117f415 100644 --- a/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c +++ b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include "lsm_bdev.skel.h" @@ -172,7 +173,7 @@ void test_lsm_bdev(void) if (!ASSERT_OK(stat(DM_DEV_PATH, &st), "stat dm dev")) goto remove_dm; - dev_key = (__u32)st.st_rdev; + dev_key = (major(st.st_rdev) << 20) | minor(st.st_rdev); /* Look up the device in the BPF map and verify. */ err = bpf_map__lookup_elem(skel->maps.verity_devices, diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c index 6606f0ed9a9a9..39e8a3b8b6afb 100644 --- a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c @@ -410,7 +410,8 @@ static int test_gso_fix(const char *ns1, const char *ns3, int family) static int check_ping_ok(const char *ns1) { SYS(fail, "ip netns exec %s ping -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP4_ADDR_DST); - SYS(fail, "ip netns exec %s ping6 -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP6_ADDR_DST); + SYS(fail, "ip netns exec %s %s -c 1 -W1 -I veth1 %s > /dev/null", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); return 0; fail: return -1; @@ -424,7 +425,8 @@ static int check_ping_fails(const char *ns1) if (!ret) return -1; - ret = SYS_NOFAIL("ip netns exec %s ping6 -c 1 -W1 -I veth1 %s", ns1, IP6_ADDR_DST); + ret = SYS_NOFAIL("ip netns exec %s %s -c 1 -W1 -I veth1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ret) return -1; @@ -657,9 +659,10 @@ static void lwt_ip_encap_vxlan(bool ipv4_encap) skel->bss->fexit_triggered = false; if (ipv4_encap) - SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); + SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); else - SYS(out, "ip netns exec %s ping6 -c 1 -W1 %s", ns1, IP6_ADDR_DST); + SYS(out, "ip netns exec %s %s -c 1 -W1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ASSERT_TRUE(skel->bss->fexit_triggered, "fexit_triggered")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/mptcp.c b/tools/testing/selftests/bpf/prog_tests/mptcp.c index 8fade8bdc4516..32dfc1c511af6 100644 --- a/tools/testing/selftests/bpf/prog_tests/mptcp.c +++ b/tools/testing/selftests/bpf/prog_tests/mptcp.c @@ -264,7 +264,7 @@ static int verify_mptcpify(int server_fd, int client_fd) return err; } -static int run_mptcpify(int cgroup_fd) +static int run_mptcpify(int cgroup_fd, int type) { int server_fd, client_fd, err = 0; struct mptcpify *mptcpify_skel; @@ -280,7 +280,7 @@ static int run_mptcpify(int cgroup_fd) goto out; /* without MPTCP */ - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); + server_fd = start_server(AF_INET, type, NULL, 0, 0); if (!ASSERT_GE(server_fd, 0, "start_server")) { err = -EIO; goto out; @@ -317,7 +317,14 @@ static void test_mptcpify(void) if (!ASSERT_OK_PTR(netns, "netns_new")) goto fail; - ASSERT_OK(run_mptcpify(cgroup_fd), "run_mptcpify"); + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM), "run_mptcpify"); + /* userspace sets flags such as SOCK_CLOEXEC together with the type; + * the BPF prog must still upgrade the socket to MPTCP. See + * update_socket_protocol() in net/socket.c, which runs before the + * type is masked with SOCK_TYPE_MASK. + */ + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM | SOCK_CLOEXEC), + "run_mptcpify_cloexec"); fail: netns_free(netns); diff --git a/tools/testing/selftests/bpf/prog_tests/sha256.c b/tools/testing/selftests/bpf/prog_tests/sha256.c index 604a0b1423d55..5edbc6194b071 100644 --- a/tools/testing/selftests/bpf/prog_tests/sha256.c +++ b/tools/testing/selftests/bpf/prog_tests/sha256.c @@ -25,10 +25,10 @@ void test_sha256(void) size_t i; data = malloc(MAX_LEN); - if (!ASSERT_OK_PTR(data, "malloc")) + if (!ASSERT_NEQ(data, NULL, "malloc")) goto out; digests = malloc((MAX_LEN + 1) * SHA256_DIGEST_LENGTH); - if (!ASSERT_OK_PTR(digests, "malloc")) + if (!ASSERT_NEQ(digests, NULL, "malloc")) goto out; /* Generate MAX_LEN bytes of "random" data deterministically. */ diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c index 955a37751b52d..c944136252c6d 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c @@ -762,7 +762,7 @@ static void test_uprobe_error(void) long err = syscall(__NR_uprobe); ASSERT_EQ(err, -1, "error"); - ASSERT_EQ(errno, ENXIO, "errno"); + ASSERT_EQ(errno, EPROTO, "errno"); } static void __test_uprobe_syscall(void) diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index d8dacef37c163..c4b4388545650 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -8,6 +8,9 @@ #define AF_INET 2 #define AF_INET6 10 +/* include/linux/net.h */ +#define SOCK_TYPE_MASK 0xf + #define SOL_SOCKET 1 #define SO_REUSEADDR 2 #define SO_SNDBUF 7 diff --git a/tools/testing/selftests/bpf/progs/mptcpify.c b/tools/testing/selftests/bpf/progs/mptcpify.c index cbdc730c3a471..e3f8cb54dbe97 100644 --- a/tools/testing/selftests/bpf/progs/mptcpify.c +++ b/tools/testing/selftests/bpf/progs/mptcpify.c @@ -15,7 +15,7 @@ int BPF_PROG(mptcpify, int family, int type, int protocol) return protocol; if ((family == AF_INET || family == AF_INET6) && - type == SOCK_STREAM && + (type & SOCK_TYPE_MASK) == SOCK_STREAM && (!protocol || protocol == IPPROTO_TCP)) { return IPPROTO_MPTCP; } diff --git a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c index 0660f29dca955..3177bc5b733ac 100644 --- a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c +++ b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c @@ -44,8 +44,18 @@ int bpf_prog2(struct __sk_buff *skb) __sink(lport); __sink(rport); - if (data + 8 > data_end) - return SK_DROP; + if (data + 8 > data_end) { + if (bpf_skb_pull_data(skb, 8)) + return SK_DROP; + + data = (void *)(long)skb->data; + data_end = (void *)(long)skb->data_end; + + if (data + 8 > data_end) + return SK_DROP; + + d = data; + } map = d[0]; sk = d[1]; diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf.c b/tools/testing/selftests/bpf/progs/test_bpf_nf.c index df43649ecb785..eda9b7bbab758 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_nf.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf.c @@ -190,8 +190,8 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); if (ct) { - __u16 sport = bpf_get_prandom_u32(); - __u16 dport = bpf_get_prandom_u32(); + __u16 sport = bpf_get_prandom_u32() % 65535 + 1; + __u16 dport = bpf_get_prandom_u32() % 65535 + 1; union nf_inet_addr saddr = {}; union nf_inet_addr daddr = {}; struct nf_conn *ct_ins; @@ -293,8 +293,8 @@ nf_ct_opts_new_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple * ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); if (ct) { - __u16 sport = bpf_get_prandom_u32(); - __u16 dport = bpf_get_prandom_u32(); + __u16 sport = bpf_get_prandom_u32() % 65535 + 1; + __u16 dport = bpf_get_prandom_u32() % 65535 + 1; union nf_inet_addr saddr = {}; union nf_inet_addr daddr = {}; struct nf_conn *ct_ins; diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c index d21d32f6a6760..d990532e88b6d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c +++ b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c @@ -287,9 +287,9 @@ __msg("17: (b7) r0 = 0") __msg("18: (95) exit") __msg("returning from callee:") __msg("to caller at 9:") -__msg("frame 0: propagating r1,r4") +__msg("frame 0: propagating r1,r3,r4") __msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") -__msg("mark_precise: frame0: regs=r1,r4 stack= before 18: (95) exit") +__msg("mark_precise: frame0: regs=r1,r3,r4 stack= before 18: (95) exit") __msg("from 18 to 9: safe") __naked int callback_result_precise(void) { @@ -419,9 +419,9 @@ __msg("to caller at 9:") /* r1, r4 are always precise for bpf_loop(), * r6 was marked before backtracking to callback body. */ -__msg("frame 0: propagating r1,r4,r6") +__msg("frame 0: propagating r1,r3,r4,r6") __msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") -__msg("mark_precise: frame0: regs=r1,r4,r6 stack= before 16: (95) exit") +__msg("mark_precise: frame0: regs=r1,r3,r4,r6 stack= before 16: (95) exit") __msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0") __msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop") __msg("mark_precise: frame0: parent state regs= stack=:") @@ -575,9 +575,9 @@ __msg("to caller at 10:") /* r1, r4 are always precise for bpf_loop(), * fp-8 was marked before backtracking to callback body. */ -__msg("frame 0: propagating r1,r4,fp-8") +__msg("frame 0: propagating r1,r3,r4,fp-8") __msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx -1") -__msg("mark_precise: frame0: regs=r1,r4 stack=-8 before 18: (95) exit") +__msg("mark_precise: frame0: regs=r1,r3,r4 stack=-8 before 18: (95) exit") __msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0") __msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181") __msg("mark_precise: frame0: parent state regs= stack=:") diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 3ce32d134e2cc..07807757b518d 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -807,7 +807,7 @@ static void verify_stderr(int prog_fd, struct expected_msgs *msgs) return; buf = malloc(TEST_LOADER_LOG_BUF_SZ); - if (!ASSERT_OK_PTR(buf, "malloc")) + if (!ASSERT_NEQ(buf, NULL, "malloc")) return; ret = bpf_prog_stream_read(prog_fd, 2, buf, TEST_LOADER_LOG_BUF_SZ - 1, diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 7ba82974ee784..8ffababe0084b 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -573,18 +573,19 @@ bool test__start_subtest_with_desc(const char *subtest_name, const char *subtest struct subtest_state *subtest_state; const char *subtest_display_name; size_t sub_state_size = sizeof(*subtest_state); + void *tmp; if (env.subtest_state) test__end_subtest(); state->subtest_num++; - state->subtest_states = - realloc(state->subtest_states, - state->subtest_num * sub_state_size); - if (!state->subtest_states) { + tmp = realloc(state->subtest_states, state->subtest_num * sub_state_size); + if (!tmp) { + state->subtest_num--; fprintf(stderr, "Not enough memory to allocate subtest result\n"); return false; } + state->subtest_states = tmp; subtest_state = &state->subtest_states[state->subtest_num - 1]; @@ -730,11 +731,14 @@ int compare_map_keys(int map1_fd, int map2_fd) int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len) { __u32 key, next_key, *cur_key_p, *next_key_p; - char *val_buf1, *val_buf2; - int i, err = 0; + char *val_buf1 = NULL, *val_buf2 = NULL; + int i, err = -ENOMEM; val_buf1 = malloc(stack_trace_len); val_buf2 = malloc(stack_trace_len); + if (!val_buf1 || !val_buf2) + goto out; + err = 0; cur_key_p = NULL; next_key_p = &key; while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) { @@ -1514,6 +1518,10 @@ static int dispatch_thread_send_subtests(int sock_fd, struct test_state *state) int subtest_num = state->subtest_num; state->subtest_states = malloc(subtest_num * sizeof(*subtest_state)); + if (!state->subtest_states) { + state->subtest_num = 0; + return -ENOMEM; + } for (int i = 0; i < subtest_num; i++) { subtest_state = &state->subtest_states[i]; @@ -1741,7 +1749,7 @@ static void server_main(void) data[i].worker_id = i; data[i].sock_fd = env.worker_socks[i]; rc = pthread_create(&dispatcher_threads[i], NULL, dispatch_thread, &data[i]); - if (rc < 0) { + if (rc) { perror("Failed to launch dispatcher thread"); exit(EXIT_ERR_SETUP_INFRA); } diff --git a/tools/testing/selftests/bpf/test_sockmap.c b/tools/testing/selftests/bpf/test_sockmap.c index ac814eb63edb6..aaf2050e88450 100644 --- a/tools/testing/selftests/bpf/test_sockmap.c +++ b/tools/testing/selftests/bpf/test_sockmap.c @@ -435,7 +435,8 @@ static int msg_alloc_iov(struct msghdr *msg, return 0; unwind_iov: for (i--; i >= 0 ; i--) - free(msg->msg_iov[i].iov_base); + free(iov[i].iov_base); + free(iov); return -ENOMEM; } diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index a7db6f04f7e14..96f7e43ff0a8d 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -514,6 +514,40 @@ static bool is_bpf_obj_file(const char *path) { return err == 0; } +/* Exact filter match */ +static bool name_filter_matches(struct filter *f, const char *filename, const char *prog_name) +{ + if (f->any_glob) + return glob_matches(filename, f->any_glob) || + (prog_name && glob_matches(prog_name, f->any_glob)); + if (f->file_glob && f->prog_glob) + return prog_name && + glob_matches(filename, f->file_glob) && + glob_matches(prog_name, f->prog_glob); + if (f->file_glob) + return glob_matches(filename, f->file_glob); + if (f->prog_glob) + return prog_name && glob_matches(prog_name, f->prog_glob); + return false; +} + +/* Check if the filter does not outright reject the file name */ +static bool name_filter_may_match(struct filter *f, const char *filename) +{ + if (f->file_glob) + return glob_matches(filename, f->file_glob); + /* + * If we don't know program name yet, any_glob filter + * has to assume that current BPF object file might be + * relevant; we'll check again later on after opening + * BPF object file, at which point program name will + * be known finally. + */ + if (f->any_glob || f->prog_glob) + return true; + return false; +} + static bool should_process_file_prog(const char *filename, const char *prog_name) { struct filter *f; @@ -521,16 +555,7 @@ static bool should_process_file_prog(const char *filename, const char *prog_name for (i = 0; i < env.deny_filter_cnt; i++) { f = &env.deny_filters[i]; - if (f->kind != FILTER_NAME) - continue; - - if (f->any_glob && glob_matches(filename, f->any_glob)) - return false; - if (f->any_glob && prog_name && glob_matches(prog_name, f->any_glob)) - return false; - if (f->file_glob && glob_matches(filename, f->file_glob)) - return false; - if (f->prog_glob && prog_name && glob_matches(prog_name, f->prog_glob)) + if (f->kind == FILTER_NAME && name_filter_matches(f, filename, prog_name)) return false; } @@ -540,24 +565,15 @@ static bool should_process_file_prog(const char *filename, const char *prog_name continue; allow_cnt++; - if (f->any_glob) { - if (glob_matches(filename, f->any_glob)) - return true; - /* If we don't know program name yet, any_glob filter - * has to assume that current BPF object file might be - * relevant; we'll check again later on after opening - * BPF object file, at which point program name will - * be known finally. - */ - if (!prog_name || glob_matches(prog_name, f->any_glob)) - return true; - } else { - if (f->file_glob && !glob_matches(filename, f->file_glob)) - continue; - if (f->prog_glob && prog_name && !glob_matches(prog_name, f->prog_glob)) - continue; + if (prog_name && name_filter_matches(f, filename, prog_name)) + return true; + /* + * If there is no prog_name and the file name is not blocked by + * the filter, allow to open the file. Afterwards there would be + * a second refining query with prog_name set. + */ + if (!prog_name && name_filter_may_match(f, filename)) return true; - } } /* if there are no file/prog name allow filters, allow all progs, @@ -703,6 +719,12 @@ static int append_filter(struct filter **filters, int *cnt, const char *str) } } + if ((!f->any_glob && !f->file_glob && !f->prog_glob) || + (f->any_glob && strcmp(f->any_glob, "") == 0)) { + fprintf(stderr, "Invalid filter: '%s'\n", str); + return -EINVAL; + } + *cnt += 1; return 0; } diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 9ca8022853933..6a3d026d76bd6 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -428,8 +428,17 @@ main() if [[ $# -eq 0 && "${debug_shell}" == "no" ]]; then echo "No command specified, will run ${DEFAULT_COMMAND} in the vm" - else - command="$@" + elif [[ $# -gt 0 ]]; then + # Quote each argument so the command survives into the guest: the + # host expands ${command} into the generated init script, which + # the guest bash then parses as shell source. Without the %q + # escapes an argument with a space or a glob (e.g. -a 'verifier_*') + # is re-split and expanded against /root/bpf there. + # + # Skip this when there is no command: printf '%q ' would still + # apply the format once and emit '', which is not the empty + # command that -s (debug shell) expects. + command=$(printf '%q ' "$@") fi local kconfig_file="${OUTPUT_DIR}/latest.config" diff --git a/tools/testing/selftests/cgroup/test_core.c b/tools/testing/selftests/cgroup/test_core.c index 88ca832d4fc13..64f4962a4659d 100644 --- a/tools/testing/selftests/cgroup/test_core.c +++ b/tools/testing/selftests/cgroup/test_core.c @@ -927,7 +927,6 @@ int main(int argc, char *argv[]) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), &nsdelegate)) { if (setup_named_v1_root(root, sizeof(root), CG_NAMED_NAME)) ksft_exit_skip("cgroup v2 isn't mounted and could not setup named v1 hierarchy\n"); @@ -940,6 +939,7 @@ int main(int argc, char *argv[]) ksft_exit_skip("Failed to set memory controller\n"); post_v2_setup: + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_cpu.c b/tools/testing/selftests/cgroup/test_cpu.c index 7a40d76b95487..8f96bfd19ab9f 100644 --- a/tools/testing/selftests/cgroup/test_cpu.c +++ b/tools/testing/selftests/cgroup/test_cpu.c @@ -799,7 +799,6 @@ int main(int argc, char *argv[]) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -807,6 +806,7 @@ int main(int argc, char *argv[]) if (cg_write(root, "cgroup.subtree_control", "+cpu")) ksft_exit_skip("Failed to set cpu controller\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_cpuset.c b/tools/testing/selftests/cgroup/test_cpuset.c index c5cf8b56ceb8f..52ed8fba685ae 100644 --- a/tools/testing/selftests/cgroup/test_cpuset.c +++ b/tools/testing/selftests/cgroup/test_cpuset.c @@ -250,7 +250,6 @@ int main(int argc, char *argv[]) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -258,6 +257,7 @@ int main(int argc, char *argv[]) if (cg_write(root, "cgroup.subtree_control", "+cpuset")) ksft_exit_skip("Failed to set cpuset controller\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index 0d41aa0d343d9..2dff9d5b53566 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -20,7 +20,7 @@ skip_test() { WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify # Find cgroup v2 mount point -CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}') +CGROUP2=$(mount -t cgroup2 | head -1 | awk '{print $3}') [[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!" SUBPARTS_CPUS=$CGROUP2/.__DEBUG__.cpuset.cpus.subpartitions CPULIST=$(cat $CGROUP2/cpuset.cpus.effective) @@ -298,6 +298,8 @@ TEST_MATRIX=( " C0-4:X2-4 C1-4:X2-4:P2 C2-4:X4:P1 \ . . . X1 . 0 A1:0-1|A2:2-4|A3:2-4 \ A1:P0|A2:P2|A3:P-1 2-4" + " CX1-3:P1 CX1-3 CX1-3 . . . P1 . 0 A1:1-3|A2:1-3|A3:1-3 \ + A1:P1|A2:P0|A3:P-1" # Remote partition offline tests " C0-3 C1-3 C2-3 . X2-3 X2-3 X2-3:P2:O2=0 . 0 A1:0-1|A2:1|A3:3 A1:P0|A3:P2 2-3" @@ -495,13 +497,20 @@ REMOTE_TEST_MATRIX=( # Narrowing cpuset.cpus to previously sibling-excluded CPUs should # not return CPUs that were never actually owned. " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ - . . . C3 . . p1:4|c11:1-2|c12:3 \ + . . . C3 . . p1:4|c11:1-2|c12:3 \ p1:P1|c11:P1|c12:P2 3" # Expanding cpuset.cpus to include a previously sibling-excluded CPU # after the sibling has become a member should correctly request it. " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ - . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ + . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ p1:P1|c11:P0|c12:P2 2-3" + # Cpusets with empty cpuset.cpus should inherit parent's effective_cpus + " C1-4:P1 C5-6 C1-2 . C5 . \ + . P1 P1 . . . p1:3-4|p2:5-6|c11:1-2|c12:3-4|c21:5|c22:5-6 \ + p1:P1|p2:P1|c11:P1" + " C1-4:P1 C5-6 C1-2 . C5 . \ + . P1 P1 . O5=0 . p1:3-4|p2:6|c11:1-2|c12:3-4|c21:6|c22:6 \ + p1:P1|p2:P1|c11:P1" ) # @@ -513,6 +522,7 @@ write_cpu_online() CPU=${1%=*} VAL=${1#*=} CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online + echo $VAL > $CPUFILE || return 1 if [[ $VAL -eq 0 ]] then OFFLINE_CPUS="$OFFLINE_CPUS $CPU" @@ -522,7 +532,6 @@ write_cpu_online() sort | uniq -u) } fi - echo $VAL > $CPUFILE pause 0.05 } @@ -590,7 +599,8 @@ set_ctrl_state() eval $COMM $REDIRECT ;; O*) VAL=${CMD#?} - write_cpu_online $VAL + COMM="write_cpu_online $VAL" + eval $COMM $REDIRECT ;; T*) COMM="echo 0 > $TFILE" eval $COMM $REDIRECT @@ -783,7 +793,6 @@ check_isolcpus() EXPECTED_ISOLCPUS=$1 ISCPUS=${CGROUP2}/cpuset.cpus.isolated ISOLCPUS=$(cat $ISCPUS) - HKICPUS=$(cat /sys/devices/system/cpu/isolated) LASTISOLCPU= SCHED_DOMAINS=/sys/kernel/debug/sched/domains if [[ $EXPECTED_ISOLCPUS = . ]] @@ -821,11 +830,6 @@ check_isolcpus() ISOLCPUS= EXPECTED_ISOLCPUS=$EXPECTED_SDOMAIN - # - # The inverse of HK_TYPE_DOMAIN cpumask in $HKICPUS should match $ISOLCPUS - # - [[ "$ISOLCPUS" != "$HKICPUS" ]] && return 1 - # # Use the sched domain in debugfs to check isolated CPUs, if available # diff --git a/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh b/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh index 7406c24be1ac9..da97f1643f9a3 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_v1_hp.sh @@ -14,7 +14,7 @@ skip_test() { [[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!" # Find cpuset v1 mount point -CPUSET=$(mount -t cgroup | grep cpuset | head -1 | awk -e '{print $3}') +CPUSET=$(mount -t cgroup | grep cpuset | head -1 | awk '{print $3}') [[ -n "$CPUSET" ]] || skip_test "cpuset v1 mount point not found!" # diff --git a/tools/testing/selftests/cgroup/test_freezer.c b/tools/testing/selftests/cgroup/test_freezer.c index 0569e93fa6b00..f28bb02e9783b 100644 --- a/tools/testing/selftests/cgroup/test_freezer.c +++ b/tools/testing/selftests/cgroup/test_freezer.c @@ -1491,9 +1491,9 @@ int main(int argc, char *argv[]) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_kill.c b/tools/testing/selftests/cgroup/test_kill.c index f6cd23a8ecc71..99cafd9dc0136 100644 --- a/tools/testing/selftests/cgroup/test_kill.c +++ b/tools/testing/selftests/cgroup/test_kill.c @@ -278,9 +278,9 @@ int main(int argc, char *argv[]) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_kmem.c b/tools/testing/selftests/cgroup/test_kmem.c index 1db0ba1226b9b..437f2d35f205e 100644 --- a/tools/testing/selftests/cgroup/test_kmem.c +++ b/tools/testing/selftests/cgroup/test_kmem.c @@ -145,7 +145,7 @@ static int cg_run_in_subcgroups(const char *parent, return -1; } - if (cg_run(child, fn, NULL)) { + if (cg_run(child, fn, arg)) { cg_destroy(child); free(child); return -1; @@ -426,7 +426,6 @@ int main(int argc, char **argv) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -441,6 +440,7 @@ int main(int argc, char **argv) if (cg_write(root, "cgroup.subtree_control", "+memory")) ksft_exit_skip("Failed to set memory controller\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_memcontrol.c b/tools/testing/selftests/cgroup/test_memcontrol.c index 0ebf796f3cffe..0ed82347044ed 100644 --- a/tools/testing/selftests/cgroup/test_memcontrol.c +++ b/tools/testing/selftests/cgroup/test_memcontrol.c @@ -30,7 +30,7 @@ static int page_size; int get_temp_fd(void) { - return open(".", O_TMPFILE | O_RDWR | O_EXCL); + return open(".", O_TMPFILE | O_RDWR | O_EXCL, 0600); } int alloc_pagecache(int fd, size_t size) @@ -1798,7 +1798,6 @@ int main(int argc, char **argv) page_size = BUF_SIZE; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -1823,6 +1822,7 @@ int main(int argc, char **argv) ksft_exit_skip("Failed to query cgroup mount option\n"); has_localevents = proc_status; + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_pids.c b/tools/testing/selftests/cgroup/test_pids.c index 9a387c815d2cd..710109b53dfe9 100644 --- a/tools/testing/selftests/cgroup/test_pids.c +++ b/tools/testing/selftests/cgroup/test_pids.c @@ -148,7 +148,6 @@ int main(int argc, char **argv) char root[PATH_MAX]; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -163,6 +162,7 @@ int main(int argc, char **argv) if (cg_write(root, "cgroup.subtree_control", "+pids")) ksft_exit_skip("Failed to set pids controller\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (int i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index 49b36ee791606..6e7b89315bbf3 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -810,7 +810,6 @@ int main(int argc, char **argv) page_size = BUF_SIZE; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -827,6 +826,7 @@ int main(int argc, char **argv) if (cg_write(root, "cgroup.subtree_control", "+memory")) ksft_exit_skip("Failed to set memory controller\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/drivers/net/config b/tools/testing/selftests/drivers/net/config index 2070e890e064e..b6989c7d3d9d1 100644 --- a/tools/testing/selftests/drivers/net/config +++ b/tools/testing/selftests/drivers/net/config @@ -4,8 +4,12 @@ CONFIG_DEBUG_INFO_BTF_MODULES=n CONFIG_INET_PSP=y CONFIG_IPV6=y CONFIG_MACSEC=m +CONFIG_NET_ACT_SKBEDIT=m CONFIG_NET_CLS_ACT=y CONFIG_NET_CLS_BPF=y +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_MATCHALL=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y CONFIG_NETCONSOLE_EXTENDED_LOG=y @@ -14,6 +18,7 @@ CONFIG_NETKIT=y CONFIG_NET_SCH_ETF=m CONFIG_NET_SCH_FQ=m CONFIG_NET_SCH_INGRESS=y +CONFIG_NET_SCH_PRIO=m CONFIG_PPP=y CONFIG_PPPOE=y CONFIG_VLAN_8021Q=m diff --git a/tools/testing/selftests/drivers/net/lib/py/env.py b/tools/testing/selftests/drivers/net/lib/py/env.py index e4ab99b905b16..0156a9572000f 100644 --- a/tools/testing/selftests/drivers/net/lib/py/env.py +++ b/tools/testing/selftests/drivers/net/lib/py/env.py @@ -114,10 +114,11 @@ class NetDrvEpEnv(NetDrvEnvBase): nsim_v4_pfx = "192.0.2." nsim_v6_pfx = "2001:db8::" - def __init__(self, src_path, nsim_test=None): + def __init__(self, src_path, nsim_test=None, queue_count=None): super().__init__(src_path) self._stats_settle_time = None + self._queue_count = queue_count # Things we try to destroy self.remote = None @@ -179,9 +180,13 @@ def __init__(self, src_path, nsim_test=None): self._required_cmd = {} def create_local(self): + nsim_kwargs = {} + if self._queue_count: + nsim_kwargs["queue_count"] = self._queue_count + self._netns = NetNS() - self._ns = NetdevSimDev() - self._ns_peer = NetdevSimDev(ns=self._netns) + self._ns = NetdevSimDev(**nsim_kwargs) + self._ns_peer = NetdevSimDev(ns=self._netns, **nsim_kwargs) with open("/proc/self/ns/net") as nsfd0, \ open("/var/run/netns/" + self._netns.name) as nsfd1: diff --git a/tools/testing/selftests/drivers/net/ring_reconfig.py b/tools/testing/selftests/drivers/net/ring_reconfig.py index f9530a8b08567..2bc329b771340 100755 --- a/tools/testing/selftests/drivers/net/ring_reconfig.py +++ b/tools/testing/selftests/drivers/net/ring_reconfig.py @@ -5,10 +5,25 @@ Test channel and ring size configuration via ethtool (-L / -G). """ +import socket +import struct +import time + from lib.py import ksft_run, ksft_exit, ksft_pr from lib.py import ksft_eq +from lib.py import KsftSkipEx, KsftXfailEx from lib.py import NetDrvEpEnv, EthtoolFamily, GenerateTraffic -from lib.py import defer, NlError +from lib.py import cmd, defer, rand_port, tc, NlError + +# Added in Python 3.13; fallback to 61 for x86/ARM/MIPS +SO_TXTIME = getattr(socket, "SO_TXTIME", 61) + +# Not always exported by the socket module; asm-generic value (x86/ARM/MIPS). +SO_SNDBUFFORCE = getattr(socket, "SO_SNDBUFFORCE", 32) + +# TX ring size the test shrinks to so the ring fills quickly. +MIN_TX_RING = 32 +MAX_TX_RING = 1024 def channels(cfg) -> None: @@ -151,14 +166,248 @@ def ringparam(cfg) -> None: GenerateTraffic(cfg).wait_pkts_and_stop(10000) +def _write_file(path, val): + """Write val to a file.""" + with open(path, "w", encoding="utf-8") as fp: + fp.write(str(val)) + + +def _write_sysfs(path, val): + """Write val to a sysfs file, restoring the original value on exit.""" + with open(path, "r", encoding="utf-8") as fp: + orig_val = fp.read().strip() + if str(val) == orig_val: + return + _write_file(path, val) + defer(_write_file, path, orig_val) + + +def _get_qdisc_backlog(cfg, mq_handle, queue): + """Return the qdisc backlog (bytes) for the given TX queue's leaf.""" + target_parent = f"{mq_handle}{queue + 1:x}" + for q in tc(f"-s qdisc show dev {cfg.ifname}", json=True): + if q.get("parent", "") == target_parent: + return q.get("backlog") or 0 + return 0 + + +def _setup_fq_qdisc(cfg, port, target_queue, other_queue, flow_limit): + """Put an fq qdisc on target_queue's leaf and return the mq handle in use. + + We must not disturb the device's existing TX/RX qdisc policy. On a real + NIC the root mq already has an addressable handle, so we leave the root + and every other queue alone and only swap this one leaf, restoring its + original qdisc afterwards. + + @flow_limit raises fq's per-flow packet limit (default 100) so a single + flow can back up more packets than the Tx ring holds and thus overflow it. + """ + qdiscs = tc(f"qdisc show dev {cfg.ifname}", json=True) + root = next((q for q in qdiscs if q.get("root")), None) + + if root and root["kind"] == "mq" and root["handle"] != "0:": + # Addressable mq (previously-configured): touch only the target queue's + # leaf and restore its original qdisc afterwards. + mq_handle = root["handle"] + parent = f"{mq_handle}{target_queue + 1:x}" + orig = next((q for q in qdiscs if q.get("parent") == parent), None) + orig_kind = orig["kind"] if orig else \ + cmd("sysctl -n net.core.default_qdisc").stdout.strip() + defer(tc, f"qdisc replace dev {cfg.ifname} parent {parent} {orig_kind}") + elif root is None or root["kind"] in ("mq", "noqueue"): + # The auto-attached root mq has handle 0: on any device (real or sim), + # which the kernel rejects as a qdisc parent. A 0: handle means the mq + # is the untouched kernel default - no custom child qdiscs can hang off + # an unaddressable parent - so installing a real handle and restoring + # the default mq on exit preserves the device's effective policy. + mq_handle = "1:" + tc(f"qdisc replace dev {cfg.ifname} root handle {mq_handle} mq") + defer(tc, f"qdisc replace dev {cfg.ifname} root mq") + parent = f"{mq_handle}{target_queue + 1:x}" + else: + raise KsftSkipEx(f"root qdisc '{root['kind']}' is not mq; " + "refusing to disturb existing qdisc policy") + + try: + tc(f"qdisc replace dev {cfg.ifname} parent {parent} fq " + f"flow_limit {flow_limit} limit {flow_limit * 2}") + except Exception as exc: + raise KsftSkipEx( + f"fq not available (CONFIG_NET_SCH_FQ): {exc}") from exc + + qdisc_j = tc(f"qdisc show dev {cfg.ifname}", json=True) + has_clsact = any(q['kind'] == 'clsact' for q in qdisc_j) + if not has_clsact: + tc(f"qdisc add dev {cfg.ifname} clsact") + defer(tc, f"qdisc del dev {cfg.ifname} clsact") + + proto = "ipv6" if int(cfg.addr_ipver) == 6 else "ip" + try: + tc(f"filter add dev {cfg.ifname} egress protocol {proto} " + f"pref 1 flower ip_proto udp dst_port {port} " + f"action skbedit queue_mapping {target_queue}") + except Exception as exc: + raise KsftSkipEx("tc flower/act_skbedit not available") from exc + defer(tc, f"filter del dev {cfg.ifname} egress pref 1") + + tc(f"filter add dev {cfg.ifname} egress pref 101 " + f"matchall action skbedit queue_mapping {other_queue}") + defer(tc, f"filter del dev {cfg.ifname} egress pref 101") + + return mq_handle + + +def _create_sotxtime_socket(cfg, sndbuf): + """Create a UDP socket with SO_TXTIME enabled, bound to the test device.""" + sock = socket.socket(socket.AF_INET6 if cfg.addr_ipver == "6" + else socket.AF_INET, socket.SOCK_DGRAM) + try: + sock.setsockopt(socket.SOL_SOCKET, SO_TXTIME, struct.pack("Ii", 1, 0)) + except OSError as exc: + sock.close() + raise KsftSkipEx("SO_TXTIME not supported") from exc + sock.setsockopt(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, + cfg.ifname.encode()) + # Deferred completions keep every in-flight skb charged to the socket, so + # size the send buffer to hold the whole burst. SO_SNDBUFFORCE bypasses + # net.core.wmem_max (the test runs as root). + try: + sock.setsockopt(socket.SOL_SOCKET, SO_SNDBUFFORCE, sndbuf) + except OSError: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, sndbuf) + return sock + + +def _send_sotxtime_burst(cfg, sock, port, count, delay_ns, pkt_size): + """Send count UDP packets scheduled delay_ns ahead using SO_TXTIME.""" + payload = b'\x00' * pkt_size + txtime_ns = time.clock_gettime_ns(time.CLOCK_MONOTONIC) + delay_ns + + ancdata = [(socket.SOL_SOCKET, SO_TXTIME, struct.pack("Q", txtime_ns))] + if int(cfg.addr_ipver) == 6: + dest = (cfg.remote_addr, port, 0, 0) + else: + dest = (cfg.remote_addr, port) + for _ in range(count): + sock.sendmsg([payload], ancdata, 0, dest) + + +def _set_small_tx_ring(cfg, ehdr): + """Set the Tx ring to the smallest size the driver accepts. + + Start at 32 so the ring fills quickly, then grow exponentially (64, + 128, 256, ...) up to 1024. Some drivers enforce a minimum well above 32 + (e.g. bnxt needs a large ring for software UDP segmentation), so raise + the lower bound until the driver accepts it, giving up past 1024. + """ + size = MIN_TX_RING + while size <= MAX_TX_RING: + try: + cfg.eth.rings_set(ehdr | {'tx': size}) + return size + except NlError: + size = size * 2 + continue + raise KsftSkipEx("driver rejects all tx ring sizes up to 1024") + + +def reconfig_tx_stall(cfg) -> None: + """Test that qdisc backlog drains after ring reconfiguration.""" + target_queue = 1 + other_queue = 0 + + ehdr = {'header': {'dev-index': cfg.ifindex}} + chans = cfg.eth.channels_get(ehdr) + + if "combined-max" not in chans: + raise KsftSkipEx("device does not support combined channels") + if chans.get("combined-max", 0) < 2: + raise KsftSkipEx("device does not support 2+ combined channels") + if chans["combined-count"] < 2: + defer(cfg.eth.channels_set, + ehdr | {"combined-count": chans["combined-count"]}) + cfg.eth.channels_set(ehdr | {"combined-count": 2}) + + rings = cfg.eth.rings_get(ehdr) + if 'rx' not in rings or 'tx' not in rings: + raise KsftSkipEx("device does not expose rx/tx ring params") + tx_cur = rings['tx'] + if tx_cur <= MIN_TX_RING: + raise KsftSkipEx("tx ring size already at minimum") + defer(cfg.eth.rings_set, ehdr | {'tx': tx_cur}) + + # Use the smallest Tx ring the driver accepts (32, growing to 1024). + tx_ring = _set_small_tx_ring(cfg, ehdr) + + # Slow completions so the ring stays full after FQ releases packets + napi_defer = f"/sys/class/net/{cfg.ifname}/napi_defer_hard_irqs" + gro_timeout = f"/sys/class/net/{cfg.ifname}/gro_flush_timeout" + _write_sysfs(napi_defer, 100) + _write_sysfs(gro_timeout, 1000000000) + + port = rand_port() + # A single flow must overflow the ring, so send twice the ring depth and + # let fq hold that many packets for the flow. + pkt_count = tx_ring * 2 + mq_handle = _setup_fq_qdisc(cfg, port, target_queue, other_queue, + tx_ring * 2) + + # Size each packet to one MTU (less L3/L4 headers to avoid fragmentation). + pkt_size = cfg.dev['mtu'] - (48 if int(cfg.addr_ipver) == 6 else 28) + + # Each queued skb charges the socket its truesize (~2x the payload), so + # budget the send buffer for the whole in-flight burst. + sock = _create_sotxtime_socket(cfg, pkt_count * pkt_size * 2) + defer(sock.close) + + for delay_ms in [100, 200, 500]: + _send_sotxtime_burst(cfg, sock, port, pkt_count, + delay_ms * 1_000_000, pkt_size) + ksft_pr(f"Sent {pkt_count} SO_TXTIME packets (+{delay_ms}ms)") + time.sleep(delay_ms / 1000 + 0.3) + + backlog = _get_qdisc_backlog(cfg, mq_handle, target_queue) + if backlog: + break + else: + # A device that completes Tx synchronously (e.g. a software/virtual + # driver like netdevsim) never keeps the ring full long enough for a + # backlog to form, so the wake-vs-start behavior can't be exercised. + # Treat that as an expected failure rather than a hard failure. + raise KsftXfailEx("could not build qdisc backlog") + + ksft_pr(f"Backlog before reconfig: {backlog} bytes") + + # Trigger ring reconfig — driver should call wake, not just start. + # Grow back to the original size so the driver actually switches channels + # (setting the current size is a no-op the driver short-circuits). + cfg.eth.rings_set(ehdr | {'tx': tx_cur}) + + # Let completions proceed normally + _write_sysfs(napi_defer, 0) + _write_sysfs(gro_timeout, 0) + + # Poll for backlog to drain + for _ in range(100): + backlog = _get_qdisc_backlog(cfg, mq_handle, target_queue) + if not backlog: + break + time.sleep(0.1) + + ksft_eq(0, backlog, + comment=f"qdisc backlog stuck on queue {target_queue} " + f"after ring reconfig") + + def main() -> None: """ Ksft boiler plate main """ - with NetDrvEpEnv(__file__) as cfg: + with NetDrvEpEnv(__file__, queue_count=2) as cfg: cfg.eth = EthtoolFamily() ksft_run([channels, - ringparam], + ringparam, + reconfig_tx_stall], args=(cfg, )) ksft_exit() diff --git a/tools/testing/selftests/drivers/net/so_txtime.py b/tools/testing/selftests/drivers/net/so_txtime.py index adf6c848d6d80..a097fae0b335a 100755 --- a/tools/testing/selftests/drivers/net/so_txtime.py +++ b/tools/testing/selftests/drivers/net/so_txtime.py @@ -12,6 +12,7 @@ from lib.py import ksft_exit, ksft_run, ksft_variants from lib.py import KsftNamedVariant, KsftSkipEx from lib.py import NetDrvEpEnv, bkg, cmd, defer, tc +from lib.py import CmdExitFailure def test_so_txtime(cfg, clockid, ipver, args_tx, args_rx, expect_success): @@ -27,7 +28,7 @@ def test_so_txtime(cfg, clockid, ipver, args_tx, args_rx, expect_success): cmd_addr = f"-S {cfg.addr_v[ipver]} -D {cfg.remote_addr_v[ipver]}" cmd_args = f"-{ipver} -c {clockid} -t {tstart} {cmd_addr}" cmd_rx = f"{cfg.bin_remote} {cmd_args} {args_rx} -r" - cmd_tx = f"{cfg.bin_local} {cmd_args} {args_tx}" + cmd_tx = f"{cfg.bin_local} -m 100 {cmd_args} {args_tx}" expect_fail = not expect_success if slow_machine: @@ -45,7 +46,11 @@ def _qdisc_setup(ifname, qdisc, optargs=""): """ orig = tc(f"qdisc show dev {ifname} root", json=True)[0].get("kind", None) defer(tc, f"qdisc replace dev {ifname} root {orig}") - tc(f"qdisc replace dev {ifname} root {qdisc} {optargs}") + try: + tc(f"qdisc del dev {ifname} root") + except CmdExitFailure: + pass + tc(f"qdisc replace dev {ifname} root handle 1: {qdisc} {optargs}") def _test_variants_fq(): @@ -96,11 +101,21 @@ def _test_variants_etf(): def test_so_txtime_etf(cfg, ipver, args_tx, args_rx, expect_fail): """Run all variants of etf tests.""" cfg.require_ipver(ipver) + + # root qdisc for background traffic (e.g., bkg()) + _qdisc_setup(cfg.ifname, "prio") + + # leaf ETF qdisc only for intended packets try: - _qdisc_setup(cfg.ifname, "etf", "clockid CLOCK_TAI delta 400000") + etf_args = "clockid CLOCK_TAI delta 400000" + tc(f"qdisc add dev {cfg.ifname} parent 1:1 handle 10: etf {etf_args}") except Exception as e: raise KsftSkipEx("tc does not support qdisc etf. skipping") from e + # redirect mark 100 to leaf + filter_args = "protocol all handle 100 fw flowid 1:1" + tc(f"filter add dev {cfg.ifname} parent 1: {filter_args}") + test_so_txtime(cfg, "tai", ipver, args_tx, args_rx, expect_fail) diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc index bc9514428dbaf..07b1177c16344 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc @@ -6,7 +6,7 @@ SYMBOL='name_show' # We skip this test on kernel where SYMBOL is unique or does not exist. -if [ "$(grep -c -E "[[:alnum:]]+ t ${SYMBOL}" /proc/kallsyms)" -le '1' ]; then +if [ "$(grep -c -E "[[:alnum:]]+ t ${SYMBOL}$" /proc/kallsyms)" -le '1' ]; then exit_unsupported fi diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 261e4df94d9d5..29a19bc870846 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -467,7 +467,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n) !__atomic_test_and_set(_metadata->no_teardown, __ATOMIC_RELAXED)) \ fixture_name##_teardown(_metadata, self, variant); \ } \ - static struct __test_metadata *_##fixture_name##_##test_name##_object; \ + static struct __test_metadata *_##fixture_name##_##test_name##_object __maybe_unused; \ static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) \ _register_##fixture_name##_##test_name(void) \ { \ diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index 76fcdd1fd3cb4..c00cb17f9e40b 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -27,6 +27,9 @@ static void *st_gva[NR_VCPUS]; static u64 guest_stolen_time[NR_VCPUS]; +static struct kvm_vm *vm_create_steal_time(u32 nr_vcpus, void *guest_code, + struct kvm_vcpu *vcpus[]); + #if defined(__x86_64__) /* steal_time must have 64-byte alignment */ @@ -211,17 +214,14 @@ static void check_steal_time_uapi(void) u64 st_ipa; int ret; - vm = vm_create_with_one_vcpu(&vcpu, NULL); - struct kvm_device_attr dev = { .group = KVM_ARM_VCPU_PVTIME_CTRL, .attr = KVM_ARM_VCPU_PVTIME_IPA, .addr = (u64)&st_ipa, }; + vm = vm_create_steal_time(1, NULL, &vcpu); vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &dev); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, 1, 0); - virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, 1); st_ipa = (ulong)ST_GPA_BASE | 1; ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); @@ -504,6 +504,21 @@ static void run_vcpu(struct kvm_vcpu *vcpu) } } +static struct kvm_vm *vm_create_steal_time(u32 nr_vcpus, void *guest_code, + struct kvm_vcpu *vcpus[]) +{ + unsigned int gpages; + struct kvm_vm *vm; + + /* Create a VM and an identity mapped memslot for the steal time structure */ + vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus); + gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE * nr_vcpus); + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, gpages, 0); + virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, gpages); + + return vm; +} + int main(int ac, char **av) { struct kvm_vcpu *vcpus[NR_VCPUS]; @@ -511,7 +526,6 @@ int main(int ac, char **av) pthread_attr_t attr; pthread_t thread; cpu_set_t cpuset; - unsigned int gpages; long stolen_time; long run_delay; bool verbose; @@ -526,11 +540,7 @@ int main(int ac, char **av) pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); - /* Create a VM and an identity mapped memslot for the steal time structure */ - vm = vm_create_with_vcpus(NR_VCPUS, guest_code, vcpus); - gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE * NR_VCPUS); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, gpages, 0); - virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, gpages); + vm = vm_create_steal_time(NR_VCPUS, guest_code, vcpus); ksft_print_header(); TEST_REQUIRE(is_steal_time_supported(vcpus[0])); diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index 86e08aa6e0a7e..a9130ed70af5c 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -2247,6 +2247,170 @@ TEST_F_FORK(layout1, rename_file) RENAME_EXCHANGE)); } +TEST_F_FORK(layout1, rename_whiteout_denied) +{ + /* The affected file is a FIFO. */ + ASSERT_EQ(0, unlink(file1_s3d3)); + ASSERT_EQ(0, mknod(file1_s3d3, S_IFIFO | 0600, 0)); + + /* Deny MAKE_REG, but allow MAKE_FIFO. */ + enforce_fs(_metadata, LANDLOCK_ACCESS_FS_MAKE_REG, NULL); + + /* + * Try to rename a file with RENAME_WHITEOUT. + * file1_s3d3 is in dir_s3d2 (tmpfs), so it supports RENAME_WHITEOUT. + * Denied, because whiteout creation is guarded with MAKE_REG. + */ + EXPECT_EQ(-1, renameat2(AT_FDCWD, file1_s3d3, AT_FDCWD, + TMP_DIR "/s3d1/s3d2/s3d3/f2", RENAME_WHITEOUT)); + EXPECT_EQ(EACCES, errno); +} + +static bool is_whiteout(const char *const path) +{ + struct stat st; + + if (stat(path, &st) == -1) + return false; + + return S_ISCHR(st.st_mode) && st.st_rdev == makedev(0, 0); +} + +static bool is_fifo(const char *const path) +{ + struct stat st; + + return stat(path, &st) == 0 && S_ISFIFO(st.st_mode); +} + +TEST_F_FORK(layout1, rename_whiteout_allowed) +{ + const struct rule rules[] = { + { + .path = dir_s3d3, + .access = LANDLOCK_ACCESS_FS_MAKE_REG, + }, + {}, + }; + + /* The affected file is a FIFO. */ + ASSERT_EQ(0, unlink(file1_s3d3)); + ASSERT_EQ(0, mknod(file1_s3d3, S_IFIFO | 0600, 0)); + + /* Allow MAKE_REG below dir_s3d3. */ + enforce_fs(_metadata, LANDLOCK_ACCESS_FS_MAKE_REG, rules); + + /* + * Rename a file with RENAME_WHITEOUT within the same directory. + * Allowed, because MAKE_REG is granted for the whiteout object which + * gets created in the source location. + */ + EXPECT_EQ(0, renameat2(AT_FDCWD, file1_s3d3, AT_FDCWD, + TMP_DIR "/s3d1/s3d2/s3d3/f2", RENAME_WHITEOUT)); + + /* A whiteout object took the place of the moved FIFO. */ + EXPECT_TRUE(is_whiteout(file1_s3d3)); + EXPECT_TRUE(is_fifo(TMP_DIR "/s3d1/s3d2/s3d3/f2")); +} + +TEST_F_FORK(layout1, rename_whiteout_reparenting) +{ + const struct rule rules[] = { + { + .path = dir_s3d2, + .access = LANDLOCK_ACCESS_FS_REFER, + }, + { + .path = dir_s3d3, + .access = LANDLOCK_ACCESS_FS_MAKE_REG, + }, + {}, + }; + + /* The moved files are FIFOs. */ + ASSERT_EQ(0, unlink(file1_s3d3)); + ASSERT_EQ(0, mknod(file1_s3d3, S_IFIFO | 0600, 0)); + ASSERT_EQ(0, unlink(file1_s3d4)); + ASSERT_EQ(0, mknod(file1_s3d4, S_IFIFO | 0600, 0)); + + /* Allow REFER below dir_s3d2, but MAKE_REG only below dir_s3d3. */ + enforce_fs(_metadata, + LANDLOCK_ACCESS_FS_MAKE_REG | LANDLOCK_ACCESS_FS_REFER, + rules); + + /* + * The whiteout object is created in the source directory: Moving the + * FIFO out of dir_s3d4 is denied because MAKE_REG is not granted + * there, even though it is granted in the destination directory + * dir_s3d3. + */ + EXPECT_EQ(-1, renameat2(AT_FDCWD, file1_s3d4, AT_FDCWD, + TMP_DIR "/s3d1/s3d2/s3d3/f2", RENAME_WHITEOUT)); + EXPECT_EQ(EACCES, errno); + + /* + * Moving the FIFO out of dir_s3d3 is allowed, because MAKE_REG is + * granted there for the created whiteout object. + */ + EXPECT_EQ(0, renameat2(AT_FDCWD, file1_s3d3, AT_FDCWD, + TMP_DIR "/s3d1/s3d2/s3d4/f2", RENAME_WHITEOUT)); + + /* A whiteout object took the place of the moved FIFO. */ + EXPECT_TRUE(is_whiteout(file1_s3d3)); + EXPECT_TRUE(is_fifo(TMP_DIR "/s3d1/s3d2/s3d4/f2")); +} + +TEST_F_FORK(layout1, rename_whiteout_exchange) +{ + const char *const whiteout_s3d3 = TMP_DIR "/s3d1/s3d2/s3d3/f2"; + const struct rule rules[] = { + { + .path = dir_s3d2, + .access = LANDLOCK_ACCESS_FS_REFER, + }, + { + .path = dir_s3d3, + .access = LANDLOCK_ACCESS_FS_MAKE_REG, + }, + {}, + }; + + /* The exchanged files are FIFOs and an existing whiteout object. */ + ASSERT_EQ(0, unlink(file1_s3d3)); + ASSERT_EQ(0, mknod(file1_s3d3, S_IFIFO | 0600, 0)); + ASSERT_EQ(0, mknod(whiteout_s3d3, S_IFCHR | 0600, makedev(0, 0))); + ASSERT_EQ(0, unlink(file1_s3d4)); + ASSERT_EQ(0, mknod(file1_s3d4, S_IFIFO | 0600, 0)); + + /* Allow REFER below dir_s3d2, but MAKE_REG only below dir_s3d3. */ + enforce_fs(_metadata, + LANDLOCK_ACCESS_FS_MAKE_REG | LANDLOCK_ACCESS_FS_REFER, + rules); + + /* + * With RENAME_EXCHANGE, the whiteout object moves into the source + * directory of the rename: Exchanging the FIFO in dir_s3d4 with the + * whiteout object is denied because MAKE_REG is not granted in + * dir_s3d4, even though it is granted in the whiteout object's own + * directory dir_s3d3. + */ + EXPECT_EQ(-1, renameat2(AT_FDCWD, file1_s3d4, AT_FDCWD, whiteout_s3d3, + RENAME_EXCHANGE)); + EXPECT_EQ(EACCES, errno); + + /* + * Exchanging the FIFO in dir_s3d3 with the whiteout object is + * allowed, because MAKE_REG is granted in the directory into which + * the whiteout object moves. + */ + EXPECT_EQ(0, renameat2(AT_FDCWD, file1_s3d3, AT_FDCWD, whiteout_s3d3, + RENAME_EXCHANGE)); + + /* The FIFO and the whiteout object swapped places. */ + EXPECT_TRUE(is_whiteout(file1_s3d3)); + EXPECT_TRUE(is_fifo(whiteout_s3d3)); +} + TEST_F_FORK(layout1, rename_dir) { const struct rule rules[] = { @@ -3270,6 +3434,18 @@ TEST_F_FORK(layout1, make_char) makedev(1, 3)); } +TEST_F_FORK(layout1, make_whiteout) +{ + /* + * Creates a whiteout object (creation guarded by MAKE_REG). + * + * Contrary to the other character devices, this does not require + * CAP_MKNOD, cf. vfs_mknod(). + */ + test_make_file(_metadata, LANDLOCK_ACCESS_FS_MAKE_REG, S_IFCHR, + makedev(0, 0)); +} + TEST_F_FORK(layout1, make_block) { /* Creates a /dev/loop0 device. */ @@ -7436,7 +7612,7 @@ TEST_F(audit_layout1, make_char) enforce_fs(_metadata, ACCESS_ALL, NULL); - EXPECT_EQ(-1, mknod(file1_s1d3, S_IFCHR | 0644, 0)); + EXPECT_EQ(-1, mknod(file1_s1d3, S_IFCHR | 0644, makedev(7, 0))); EXPECT_EQ(EACCES, errno); EXPECT_EQ(0, matches_log_fs(_metadata, self->audit_fd, "fs\\.make_char", dir_s1d3)); @@ -7446,6 +7622,25 @@ TEST_F(audit_layout1, make_char) EXPECT_EQ(1, records.domain); } +TEST_F(audit_layout1, make_whiteout) +{ + struct audit_records records; + + EXPECT_EQ(0, unlink(file1_s1d3)); + + enforce_fs(_metadata, ACCESS_ALL, NULL); + + /* Whiteout creation is denied and logged as fs.make_reg. */ + EXPECT_EQ(-1, mknod(file1_s1d3, S_IFCHR | 0644, makedev(0, 0))); + EXPECT_EQ(EACCES, errno); + EXPECT_EQ(0, matches_log_fs(_metadata, self->audit_fd, "fs\\.make_reg", + dir_s1d3)); + + EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); + EXPECT_EQ(0, records.access); + EXPECT_EQ(1, records.domain); +} + TEST_F(audit_layout1, make_dir) { struct audit_records records; diff --git a/tools/testing/selftests/lsm/common.c b/tools/testing/selftests/lsm/common.c index 9ad258912646c..927dce4f04cb2 100644 --- a/tools/testing/selftests/lsm/common.c +++ b/tools/testing/selftests/lsm/common.c @@ -76,7 +76,7 @@ int attr_lsm_count(void) return 0; if (read_sysfs_lsms(names, sysconf(_SC_PAGESIZE))) - return 0; + goto out; if (strstr(names, "selinux")) count++; @@ -85,5 +85,7 @@ int attr_lsm_count(void) if (strstr(names, "apparmor")) count++; +out: + free(names); return count; } diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index 0c627ea89ff7b..c1b8920e29342 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -1718,8 +1718,13 @@ static void run_with_tmpfile(non_anon_test_fn fn, const char *desc) /* File consists of a single page filled with zeroes. */ if (fallocate(fd, 0, 0, pagesize)) { - ksft_perror("fallocate() failed"); - log_test_result(KSFT_FAIL); + if (errno == EOPNOTSUPP) { + ksft_print_msg("fallocate() not supported by filesystem\n"); + log_test_result(KSFT_SKIP); + } else { + ksft_perror("fallocate() failed"); + log_test_result(KSFT_FAIL); + } goto close; } diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 2f2b9879d1005..6fccbdab02ee8 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -2829,8 +2829,11 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (buffer->ptr == MAP_FAILED) - return -1; + if (buffer->ptr == MAP_FAILED) { + buffer->ptr = NULL; + ret = -1; + goto cleanup; + } /* Apply THP hint if requested */ if (use_thp) @@ -2839,7 +2842,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE); if (ret) - return ret; + goto cleanup; /* Initialize memory to make sure pages are allocated */ ptr = (int *)buffer->ptr; @@ -2849,11 +2852,11 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz /* Warmup iteration */ ret = hmm_migrate_sys_to_dev(fd, buffer, npages); if (ret) - return ret; + goto cleanup; ret = hmm_migrate_dev_to_sys(fd, buffer, npages); if (ret) - return ret; + goto cleanup; /* Benchmark iterations */ for (i = 0; i < iterations; i++) { @@ -2862,7 +2865,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = hmm_migrate_sys_to_dev(fd, buffer, npages); if (ret) - return ret; + goto cleanup; end = get_time_ms(); s2d_total += (end - start); @@ -2872,7 +2875,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = hmm_migrate_dev_to_sys(fd, buffer, npages); if (ret) - return ret; + goto cleanup; end = get_time_ms(); d2s_total += (end - start); @@ -2886,9 +2889,9 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / (results->dev_to_sys_time / 1000.0); - /* Cleanup */ +cleanup: hmm_buffer_free(buffer); - return 0; + return ret; } /* diff --git a/tools/testing/selftests/mm/hugepage_settings.c b/tools/testing/selftests/mm/hugepage_settings.c index 2eab2110ac6a4..d7917dce3abac 100644 --- a/tools/testing/selftests/mm/hugepage_settings.c +++ b/tools/testing/selftests/mm/hugepage_settings.c @@ -437,7 +437,7 @@ void hugetlb_set_nr_pages(unsigned long size, unsigned long nr) hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); - write_num(path, nr); + write_num_ignore_einval(path, nr); } unsigned long hugetlb_free_pages(unsigned long size) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 10e8dedcb087d..506310d4b4d5c 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -138,8 +138,8 @@ static void get_finfo(const char *dir) major(path_stat.st_dev), minor(path_stat.st_dev)) >= sizeof(path)) ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); - if (read_file(path, buf, sizeof(buf)) < 0) - ksft_exit_fail_perror("read_file(read_num)"); + if (!read_file(path, buf, sizeof(buf))) + ksft_exit_fail_perror("read_file(uevent)"); if (strstr(buf, "DEVTYPE=disk")) { /* Found it */ if (snprintf(finfo.dev_queue_read_ahead_path, diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index a050f4840cfa3..5fd7792a0d479 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -288,8 +288,8 @@ static bool assert_ksm_pages_count(long dupl_page_count) static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) { if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes)) || ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || @@ -304,8 +304,8 @@ static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) static int ksm_restore(struct ksm_sysfs *ksm_sysfs) { if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || @@ -440,9 +440,9 @@ static int get_next_mem_node(int node) mem_node = i % (max_node + 1); node_size = numa_node_size(mem_node, NULL); if (node_size > 0) - break; + return mem_node; } - return mem_node; + return -ENODEV; } static int get_first_mem_node(void) @@ -455,8 +455,8 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo { void *numa1_map_ptr, *numa2_map_ptr; struct timespec start_time; + int first_node, second_node; int page_count = 2; - int first_node; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { ksft_perror("clock_gettime"); @@ -467,17 +467,19 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo ksft_print_msg("NUMA support not enabled\n"); return KSFT_SKIP; } - if (numa_num_configured_nodes() <= 1) { - ksft_print_msg("At least 2 NUMA nodes must be available\n"); + first_node = get_first_mem_node(); + second_node = get_next_mem_node(first_node); + + if (second_node < 0) { + ksft_print_msg("At least 2 NUMA nodes with memory must be available\n"); return KSFT_SKIP; } if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) return KSFT_FAIL; /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ - first_node = get_first_mem_node(); numa1_map_ptr = numa_alloc_onnode(page_size, first_node); - numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); + numa2_map_ptr = numa_alloc_onnode(page_size, second_node); if (!numa1_map_ptr || !numa2_map_ptr) { ksft_perror("numa_alloc_onnode"); return KSFT_FAIL; @@ -844,8 +846,8 @@ int main(int argc, char *argv[]) if (ksm_write_sysfs(KSM_FP("run"), 2) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) ksft_exit_fail_msg("Cannot set up KSM tunables\n"); diff --git a/tools/testing/selftests/mm/memfd_secret.c b/tools/testing/selftests/mm/memfd_secret.c index aac4f795c327b..c55d84c5e613a 100644 --- a/tools/testing/selftests/mm/memfd_secret.c +++ b/tools/testing/selftests/mm/memfd_secret.c @@ -57,33 +57,6 @@ static void test_file_apis(int fd) pass("file IO is blocked as expected\n"); } -static void test_mlock_limit(int fd) -{ - size_t len; - char *mem; - - len = mlock_limit_cur; - if (len % page_size != 0) - len = (len/page_size) * page_size; - - mem = mmap(NULL, len, prot, mode, fd, 0); - if (mem == MAP_FAILED) { - fail("unable to mmap secret memory\n"); - return; - } - munmap(mem, len); - - len = mlock_limit_max * 2; - mem = mmap(NULL, len, prot, mode, fd, 0); - if (mem != MAP_FAILED) { - fail("unexpected mlock limit violation\n"); - munmap(mem, len); - return; - } - - pass("mlock limit is respected\n"); -} - static void test_vmsplice(int fd, const char *desc) { ssize_t transferred; @@ -297,7 +270,7 @@ static void prepare(void) strerror(errno)); } -#define NUM_TESTS 6 +#define NUM_TESTS 5 int main(int argc, char *argv[]) { @@ -319,7 +292,6 @@ int main(int argc, char *argv[]) if (ftruncate(fd, page_size)) ksft_exit_fail_msg("ftruncate failed: %s\n", strerror(errno)); - test_mlock_limit(fd); test_file_apis(fd); /* * We have to run the first vmsplice test before any secretmem page was diff --git a/tools/testing/selftests/mm/memory-failure.c b/tools/testing/selftests/mm/memory-failure.c index 032ed952057c6..1a5a32e22cce2 100644 --- a/tools/testing/selftests/mm/memory-failure.c +++ b/tools/testing/selftests/mm/memory-failure.c @@ -46,7 +46,7 @@ FIXTURE(memory_failure) unsigned long pfn; int pagemap_fd; int kpageflags_fd; - bool triggered; + bool injection_attempted; }; FIXTURE_VARIANT(memory_failure) @@ -122,13 +122,6 @@ static void teardown_sighandler(void) sigaction(SIGBUS, &sa, NULL); } -FIXTURE_TEARDOWN(memory_failure) -{ - close(self->kpageflags_fd); - close(self->pagemap_fd); - teardown_sighandler(); -} - static void prepare(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure) * self, void *vaddr) { @@ -200,8 +193,7 @@ static void check(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure ASSERT_EQ(pfn_flags & KPF_HWPOISON, KPF_HWPOISON); } -static void cleanup(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure) * self, - void *vaddr) +static void cleanup(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failure) * self) { unsigned long size; uint64_t pfn_flags; @@ -217,6 +209,20 @@ static void cleanup(struct __test_metadata *_metadata, FIXTURE_DATA(memory_failu ASSERT_EQ(size, self->corrupted_size); } +FIXTURE_TEARDOWN(memory_failure) +{ + /* + * Injection may poison the page before failing or delivering SIGBUS, so + * clean up after every injection attempt. + */ + if (self->injection_attempted) + cleanup(_metadata, self); + + close(self->kpageflags_fd); + close(self->pagemap_fd); + teardown_sighandler(); +} + TEST_F(memory_failure, anon) { char *addr; @@ -231,8 +237,8 @@ TEST_F(memory_failure, anon) prepare(_metadata, self, addr); ret = sigsetjmp(signal_jmp_buf, 1); - if (!self->triggered) { - self->triggered = true; + if (!self->injection_attempted) { + self->injection_attempted = true; ASSERT_EQ(variant->inject(self, addr), 0); FORCE_READ(*addr); } @@ -242,8 +248,6 @@ TEST_F(memory_failure, anon) else check(_metadata, self, addr, MADV_SOFT_ANON, ret); - cleanup(_metadata, self, addr); - ASSERT_EQ(munmap(addr, self->page_size), 0); } @@ -296,8 +300,8 @@ TEST_F(memory_failure, clean_pagecache) prepare(_metadata, self, addr); ret = sigsetjmp(signal_jmp_buf, 1); - if (!self->triggered) { - self->triggered = true; + if (!self->injection_attempted) { + self->injection_attempted = true; ASSERT_EQ(variant->inject(self, addr), 0); FORCE_READ(*addr); } @@ -307,8 +311,6 @@ TEST_F(memory_failure, clean_pagecache) else check(_metadata, self, addr, MADV_SOFT_CLEAN_PAGECACHE, ret); - cleanup(_metadata, self, addr); - ASSERT_EQ(munmap(addr, self->page_size), 0); ASSERT_EQ(close(fd), 0); @@ -337,8 +339,8 @@ TEST_F(memory_failure, dirty_pagecache) prepare(_metadata, self, addr); ret = sigsetjmp(signal_jmp_buf, 1); - if (!self->triggered) { - self->triggered = true; + if (!self->injection_attempted) { + self->injection_attempted = true; ASSERT_EQ(variant->inject(self, addr), 0); FORCE_READ(*addr); } @@ -348,8 +350,6 @@ TEST_F(memory_failure, dirty_pagecache) else check(_metadata, self, addr, MADV_SOFT_DIRTY_PAGECACHE, ret); - cleanup(_metadata, self, addr); - ASSERT_EQ(munmap(addr, self->page_size), 0); ASSERT_EQ(close(fd), 0); diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 8c296dedf0474..44cabdc54144e 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -303,7 +303,7 @@ CATEGORY="compaction" run_test ./compaction_test if command -v sudo &> /dev/null && sudo -u nobody ls ./on-fault-limit >/dev/null; then - CATEGORY="mlock" run_test sudo -u nobody ./on-fault-limit + CATEGORY="mlock" run_test sudo -u nobody sh -c ./on-fault-limit else echo "# SKIP ./on-fault-limit" fi diff --git a/tools/testing/selftests/mm/thuge-gen.c b/tools/testing/selftests/mm/thuge-gen.c index 22b9c2f1c35d2..50d0805b65db9 100644 --- a/tools/testing/selftests/mm/thuge-gen.c +++ b/tools/testing/selftests/mm/thuge-gen.c @@ -71,12 +71,16 @@ void test_mmap(unsigned long size, unsigned flags) void test_shmget(unsigned long size, unsigned flags) { - int id; - unsigned long before, after; + /* values for PAGE_SIZE test */ + unsigned long before = NUM_PAGES; + unsigned long after = 0; struct shm_info i; char *map; + int id; + + if (size != getpagesize()) + before = hugetlb_free_pages(size); - before = hugetlb_free_pages(size); id = shmget(IPC_PRIVATE, size * NUM_PAGES, IPC_CREAT|0600|flags); if (id < 0) { if (errno == EPERM) { @@ -97,10 +101,11 @@ void test_shmget(unsigned long size, unsigned flags) shmctl(id, IPC_RMID, NULL); memset(map, 0xff, size*NUM_PAGES); - after = hugetlb_free_pages(size); + if (size != getpagesize()) + after = hugetlb_free_pages(size); show(size); - ksft_test_result(size == getpagesize() || (before - after) == NUM_PAGES, + ksft_test_result((before - after) == NUM_PAGES, "%s: mmap %lu %x\n", __func__, size, flags); if (shmdt(map)) ksft_exit_fail_msg("%s: shmdt: %s\n", __func__, strerror(errno)); diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 311fc5b4513eb..2697d50d07f4f 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -719,7 +719,7 @@ int read_file(const char *path, char *buf, size_t buflen) return (unsigned int) numread; } -void write_file(const char *path, const char *buf, size_t buflen) +static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval) { int fd, saved_errno; ssize_t numwritten; @@ -735,30 +735,48 @@ void write_file(const char *path, const char *buf, size_t buflen) saved_errno = errno; close(fd); errno = saved_errno; - if (numwritten < 0) + if (numwritten < 0) { + if (ignore_einval && errno == EINVAL) + return; ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1), buf, strerror(errno)); + } if (numwritten != buflen - 1) ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n", path, (int)(buflen - 1), buf, buflen - 1, numwritten); } +void write_file(const char *path, const char *buf, size_t buflen) +{ + __write_file(path, buf, buflen, /* ignore_einval = */ false); +} + unsigned long read_num(const char *path) { char buf[21]; - if (read_file(path, buf, sizeof(buf)) < 0) + if (!read_file(path, buf, sizeof(buf))) ksft_exit_fail_perror("read_file()"); return strtoul(buf, NULL, 10); } -void write_num(const char *path, unsigned long num) +static void __write_num(const char *path, unsigned long num, bool ignore_einval) { char buf[21]; sprintf(buf, "%lu", num); - write_file(path, buf, strlen(buf) + 1); + __write_file(path, buf, strlen(buf) + 1, ignore_einval); +} + +void write_num(const char *path, unsigned long num) +{ + return __write_num(path, num, /* ignore_einval = */ false); +} + +void write_num_ignore_einval(const char *path, unsigned long num) +{ + return __write_num(path, num, /* ignore_einval = */ true); } static unsigned long shmall, shmmax; diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index ea8fc8fdf0eb0..7799154b67eed 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -168,6 +168,7 @@ void write_file(const char *path, const char *buf, size_t buflen); int read_file(const char *path, char *buf, size_t buflen); unsigned long read_num(const char *path); void write_num(const char *path, unsigned long num); +void write_num_ignore_einval(const char *path, unsigned long num); void shm_limits_prepare(unsigned long length); void __shm_limits_restore(void); diff --git a/tools/testing/selftests/namespaces/nsid_test.c b/tools/testing/selftests/namespaces/nsid_test.c index 46dc838cba82d..a16f31f41d38f 100644 --- a/tools/testing/selftests/namespaces/nsid_test.c +++ b/tools/testing/selftests/namespaces/nsid_test.c @@ -649,8 +649,6 @@ TEST_F(nsid, timens_separate) /* Fork a grandchild to actually enter the new namespace */ pid_t grandchild = fork(); if (grandchild == 0) { - /* Grandchild is in the new namespace */ - write(pipefd[1], "Y", 1); close(pipefd[1]); pause(); _exit(0); @@ -771,8 +769,6 @@ TEST_F(nsid, pidns_separate) /* Fork a grandchild to actually enter the new namespace */ pid_t grandchild = fork(); if (grandchild == 0) { - /* Grandchild is in the new namespace */ - write(pipefd[1], "Y", 1); close(pipefd[1]); pause(); _exit(0); diff --git a/tools/testing/selftests/nci/nci_dev.c b/tools/testing/selftests/nci/nci_dev.c index 312f84ee0444f..07427fa42888d 100644 --- a/tools/testing/selftests/nci/nci_dev.c +++ b/tools/testing/selftests/nci/nci_dev.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -87,6 +88,16 @@ struct msgtemplate { char buf[MAX_MSG_SIZE]; }; +static int join_thread_status(pthread_t thread) +{ + void *thread_ret = NULL; + + if (pthread_join(thread, &thread_ret)) + return -1; + + return (int)(intptr_t)thread_ret; +} + static int create_nl_socket(void) { int fd; @@ -182,7 +193,7 @@ static int get_family_id(int sd, __u32 pid, __u32 *event_group) } ans; struct nlattr *na; int resp_len; - __u16 id; + __u16 id = 0; int len; int rc; @@ -438,13 +449,13 @@ FIXTURE_SETUP(NCI) else rc = pthread_create(&thread_t, NULL, virtual_dev_open, (void *)&self->virtual_nci_fd); - ASSERT_GT(rc, -1); + ASSERT_EQ(rc, 0); rc = send_cmd_with_idx(self->sd, self->fid, self->pid, NFC_CMD_DEV_UP, self->dev_idex); EXPECT_EQ(rc, 0); - pthread_join(thread_t, (void **)&status); + status = join_thread_status(thread_t); ASSERT_EQ(status, 0); self->open_state = true; } @@ -509,12 +520,12 @@ FIXTURE_TEARDOWN(NCI) rc = pthread_create(&thread_t, NULL, virtual_deinit, (void *)&self->virtual_nci_fd); - ASSERT_GT(rc, -1); + ASSERT_EQ(rc, 0); rc = send_cmd_with_idx(self->sd, self->fid, self->pid, NFC_CMD_DEV_DOWN, self->dev_idex); EXPECT_EQ(rc, 0); - pthread_join(thread_t, (void **)&status); + status = join_thread_status(thread_t); ASSERT_EQ(status, 0); } @@ -585,12 +596,11 @@ int start_polling(int dev_idx, int proto, int virtual_fd, int sd, int fid, int p void *nla_start_poll_data[2] = {&dev_idx, &proto}; int nla_start_poll_len[2] = {4, 4}; pthread_t thread_t; - int status; int rc; rc = pthread_create(&thread_t, NULL, virtual_poll_start, (void *)&virtual_fd); - if (rc < 0) + if (rc) return rc; rc = send_cmd_mt_nla(sd, fid, pid, NFC_CMD_START_POLL, 2, nla_start_poll_type, @@ -598,19 +608,17 @@ int start_polling(int dev_idx, int proto, int virtual_fd, int sd, int fid, int p if (rc != 0) return rc; - pthread_join(thread_t, (void **)&status); - return status; + return join_thread_status(thread_t); } int stop_polling(int dev_idx, int virtual_fd, int sd, int fid, int pid) { pthread_t thread_t; - int status; int rc; rc = pthread_create(&thread_t, NULL, virtual_poll_stop, (void *)&virtual_fd); - if (rc < 0) + if (rc) return rc; rc = send_cmd_with_idx(sd, fid, pid, @@ -618,8 +626,7 @@ int stop_polling(int dev_idx, int virtual_fd, int sd, int fid, int pid) if (rc != 0) return rc; - pthread_join(thread_t, (void **)&status); - return status; + return join_thread_status(thread_t); } TEST_F(NCI, start_poll) @@ -830,10 +837,14 @@ int disconnect_tag(int nfc_sock, int virtual_fd) status = pthread_create(&thread_t, NULL, virtual_deactivate_proc, (void *)&virtual_fd); + if (status) + return status; close(nfc_sock); - pthread_join(thread_t, (void **)&status); - return status; + if (status) + return -1; + + return join_thread_status(thread_t); } TEST_F(NCI, t4t_tag_read) @@ -874,13 +885,13 @@ TEST_F(NCI, deinit) else rc = pthread_create(&thread_t, NULL, virtual_deinit, (void *)&self->virtual_nci_fd); - ASSERT_GT(rc, -1); + ASSERT_EQ(rc, 0); rc = send_cmd_with_idx(self->sd, self->fid, self->pid, NFC_CMD_DEV_DOWN, self->dev_idex); EXPECT_EQ(rc, 0); - pthread_join(thread_t, (void **)&status); + status = join_thread_status(thread_t); self->open_state = 0; ASSERT_EQ(status, 0); diff --git a/tools/testing/selftests/net/fib_nexthops.sh b/tools/testing/selftests/net/fib_nexthops.sh index ac868a7316946..daceb1ec592ab 100755 --- a/tools/testing/selftests/net/fib_nexthops.sh +++ b/tools/testing/selftests/net/fib_nexthops.sh @@ -522,6 +522,20 @@ ipv6_fdb_grp_fcnal() run_cmd "$BRIDGE fdb add 02:02:00:00:00:14 dev vx10 nhid 61 self" log_test $? 255 "Fdb mac add with nexthop" + # fdb entries with a nexthop group cannot be aged out + run_cmd "$BRIDGE fdb add 02:02:00:00:00:15 dev vx10 nhid 102 self static" + log_test $? 0 "Fdb mac add with nexthop group and static state" + + run_cmd "$BRIDGE fdb add 02:02:00:00:00:16 dev vx10 nhid 102 self dynamic" + log_test $? 255 "Fdb mac add with nexthop group and dynamic state" + + run_cmd "$BRIDGE fdb add 02:02:00:00:00:17 dev vx10 nhid 102 self" + run_cmd "$BRIDGE fdb replace 02:02:00:00:00:17 dev vx10 dst 2001:db8:91::11 self dynamic" + log_test $? 255 "Fdb mac replace with nexthop group and dynamic state" + + run_cmd "$BRIDGE fdb append 02:02:00:00:00:17 dev vx10 dst 2001:db8:91::11 self dynamic" + log_test $? 255 "Fdb mac append with nexthop group and dynamic state" + run_cmd "$IP -6 ro add 2001:db8:101::1/128 nhid 66" log_test $? 2 "Route add with fdb nexthop" @@ -622,6 +636,20 @@ ipv4_fdb_grp_fcnal() run_cmd "$BRIDGE fdb add 02:02:00:00:00:14 dev vx10 nhid 12 self" log_test $? 255 "Fdb mac add with nexthop" + # fdb entries with a nexthop group cannot be aged out + run_cmd "$BRIDGE fdb add 02:02:00:00:00:15 dev vx10 nhid 102 self static" + log_test $? 0 "Fdb mac add with nexthop group and static state" + + run_cmd "$BRIDGE fdb add 02:02:00:00:00:16 dev vx10 nhid 102 self dynamic" + log_test $? 255 "Fdb mac add with nexthop group and dynamic state" + + run_cmd "$BRIDGE fdb add 02:02:00:00:00:17 dev vx10 nhid 102 self" + run_cmd "$BRIDGE fdb replace 02:02:00:00:00:17 dev vx10 dst 10.0.0.3 self dynamic" + log_test $? 255 "Fdb mac replace with nexthop group and dynamic state" + + run_cmd "$BRIDGE fdb append 02:02:00:00:00:17 dev vx10 dst 10.0.0.3 self dynamic" + log_test $? 255 "Fdb mac append with nexthop group and dynamic state" + run_cmd "$IP ro add 172.16.0.0/22 nhid 16" log_test $? 2 "Route add with fdb nexthop" diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.c b/tools/testing/selftests/net/mptcp/mptcp_connect.c index cbe573c4ab3ad..4c339ad449a7a 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.c +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.c @@ -381,6 +381,9 @@ static int sock_connect_mptcp(const char * const remoteaddr, hints.ai_family = pf; + /* Keep the resolved address alive for the whole execution: it is + * used again when reconnecting, and will be released at exit time. + */ xgetaddrinfo(remoteaddr, port, &hints, &addr); for (a = addr; a; a = a->ai_next) { sock = socket(a->ai_family, a->ai_socktype, proto); @@ -421,7 +424,6 @@ static int sock_connect_mptcp(const char * const remoteaddr, sock = -1; } - freeaddrinfo(addr); if (sock != -1) SOCK_TEST_TCPULP(sock, proto); return sock; diff --git a/tools/testing/selftests/net/mptcp/mptcp_lib.sh b/tools/testing/selftests/net/mptcp/mptcp_lib.sh index 5ef6033775c86..b9d14647f4019 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_lib.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_lib.sh @@ -108,12 +108,14 @@ mptcp_lib_pr_info() { mptcp_lib_pr_nstat() { local ns="${1}" - local hist="/tmp/${ns}.out" + local cache="/tmp/${ns}.out" + local hist="/tmp/${ns}.nstat" - if [ -f "${hist}" ]; then - awk '$2 != 0 { print " "$0 }' "${hist}" + if [ -f "${cache}" ]; then + awk '$2 != 0 { print " "$0 }' "${cache}" else - ip netns exec "${ns}" nstat -as | grep Tcp + NSTAT_HISTORY="${hist}" ip netns exec "${ns}" nstat -s | + grep Tcp fi } @@ -414,19 +416,21 @@ mptcp_lib_nstat_get() { } # $1: ns, $2: MIB counter -# Get the counter from the history (mptcp_lib_nstat_{init,get}()) if available. -# If not, get the counter from nstat ignoring any history. +# Get the counter from the cache (mptcp_lib_nstat_{init,get}()) if available. +# If not, get the counter from nstat ignoring any cache, but using the history. mptcp_lib_get_counter() { local ns="${1}" local counter="${2}" - local hist="/tmp/${ns}.out" + local cache="/tmp/${ns}.out" + local hist="/tmp/${ns}.nstat" local count - if [[ -s "${hist}" && "${counter}" == *"Tcp"* ]]; then - count=$(awk "/^${counter} / {print \$2; exit}" "${hist}") + if [[ -s "${cache}" && "${counter}" == *"Tcp"* ]]; then + count=$(awk "/^${counter} / {print \$2; exit}" "${cache}") else - count=$(ip netns exec "${ns}" nstat -asz "${counter}" | - awk 'NR==1 {next} {print $2}') + count=$(NSTAT_HISTORY="${hist}" ip netns exec "${ns}" \ + nstat -sz "${counter}" | + awk 'NR==1 {next} {print $2}') fi if [ -z "${count}" ]; then mptcp_lib_fail_if_expected_feature "${counter} counter" diff --git a/tools/testing/selftests/net/pmtu.sh b/tools/testing/selftests/net/pmtu.sh index a3323c21f0014..c7cd271714ef7 100755 --- a/tools/testing/selftests/net/pmtu.sh +++ b/tools/testing/selftests/net/pmtu.sh @@ -1457,7 +1457,7 @@ test_pmtu_ipvX_over_bridged_vxlanY_or_geneveY_exception() { mtu "${ns_b}" ${type}_b $((${ll_mtu} + 1000)) run_cmd ${ns_c} ${ping} -q -M want -i 0.1 -c 10 -s $((${ll_mtu} + 500)) ${dst} || return 1 - run_cmd ${ns_a} ${ping} -q -M want -i 0.1 -w 1 -s $((${ll_mtu} + 500)) ${dst} || return 1 + run_cmd ${ns_a} ${ping} -q -M want -i 0.1 -c 10 -s $((${ll_mtu} + 500)) ${dst} || return 1 # Check that exceptions were created pmtu="$(route_get_dst_pmtu_from_exception "${ns_c}" ${dst})" diff --git a/tools/testing/selftests/net/test_vxlan_mdb.sh b/tools/testing/selftests/net/test_vxlan_mdb.sh index 58da5de99ac45..f9600aabd4a29 100755 --- a/tools/testing/selftests/net/test_vxlan_mdb.sh +++ b/tools/testing/selftests/net/test_vxlan_mdb.sh @@ -685,6 +685,9 @@ star_g_common() run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp permanent filter_mode exclude source_list $grp dst $vtep_ip src_vni 10010" log_test $? 255 "Invalid source in source list" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp permanent filter_mode exclude source_list $all_zeros_grp dst $vtep_ip src_vni 10010" + log_test $? 255 "All-zeros source in source list" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp permanent source_list $src1 dst $vtep_ip src_vni 10010" log_test $? 255 "Source list without filter mode" } @@ -784,6 +787,9 @@ sg_common() run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp src $grp permanent dst $vtep_ip src_vni 10010" log_test $? 255 "(S, G) with an invalid source list" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp src $all_zeros_grp permanent dst $vtep_ip src_vni 10010" + log_test $? 255 "(S, G) with an all-zeros source" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $all_zeros_grp src $src permanent dst $vtep_ip src_vni 10010" log_test $? 255 "All-zeros group with source" } diff --git a/tools/testing/selftests/powerpc/tm/tm.h b/tools/testing/selftests/powerpc/tm/tm.h index c03c6e7788767..6024ce4ba6ffd 100644 --- a/tools/testing/selftests/powerpc/tm/tm.h +++ b/tools/testing/selftests/powerpc/tm/tm.h @@ -105,8 +105,12 @@ static inline bool failure_is_nesting(void) static inline int tcheck(void) { long cr; - asm volatile ("tcheck 0" : "=r"(cr) : : "cr0"); - return (cr >> 28) & 4; + asm volatile("tcheck 0;" + "mfcr %0;" + : "=r"(cr) + : + : "cr0"); + return (cr >> 28) & 0xf; } static inline bool tcheck_doomed(void) diff --git a/tools/testing/selftests/proc/proc-maps-race.c b/tools/testing/selftests/proc/proc-maps-race.c index 1026d8c400e1b..415eccb704684 100644 --- a/tools/testing/selftests/proc/proc-maps-race.c +++ b/tools/testing/selftests/proc/proc-maps-race.c @@ -490,7 +490,8 @@ static bool query_addr_at(int maps_fd, void *addr, static inline bool split_vma(FIXTURE_DATA(proc_maps_race) *self) { - return mmap(self->mod_info->addr, self->page_size, self->mod_info->prot | PROT_EXEC, + /* PROT_NONE differs from both readable neighbors. */ + return mmap(self->mod_info->addr, self->page_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED; } diff --git a/tools/testing/selftests/proc/proc-pidns.c b/tools/testing/selftests/proc/proc-pidns.c index 25b9a2933c456..6f7c10fe97b30 100644 --- a/tools/testing/selftests/proc/proc-pidns.c +++ b/tools/testing/selftests/proc/proc-pidns.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include diff --git a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h index d3133587d9968..5a29d6bec51f4 100644 --- a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h +++ b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h @@ -8,13 +8,11 @@ #ifndef _RSEQ_X86_THREAD_POINTER #define _RSEQ_X86_THREAD_POINTER -#include - #ifdef __cplusplus extern "C" { #endif -#if __GNUC_PREREQ (11, 1) +#if __GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ >= 1) static inline void *rseq_thread_pointer(void) { return __builtin_thread_pointer(); diff --git a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c index 6f4c3f5a1c5d9..7ef9de7b27ebf 100644 --- a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c +++ b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c @@ -14,18 +14,16 @@ s32 BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0); + if (cpu < 0) + cpu = prev_cpu; - if (cpu >= 0) { - /* - * If we dispatch to a bogus DSQ that will fall back to the - * builtin global DSQ, we fail gracefully. - */ - scx_bpf_dsq_insert_vtime(p, 0xcafef00d, SCX_SLICE_DFL, - p->scx.dsq_vtime, 0); - return cpu; - } - - return prev_cpu; + /* + * If we dispatch to a bogus DSQ that will fall back to the + * builtin global DSQ, we fail gracefully. + */ + scx_bpf_dsq_insert_vtime(p, 0xcafef00d, SCX_SLICE_DFL, + p->scx.dsq_vtime, 0); + return cpu; } void BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_exit, struct scx_exit_info *ei) diff --git a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c index e4a55027778fd..82dca4cdc0a6c 100644 --- a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c +++ b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c @@ -14,15 +14,14 @@ s32 BPF_STRUCT_OPS(ddsp_vtimelocal_fail_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0); + if (cpu < 0) + cpu = prev_cpu; - if (cpu >= 0) { - /* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */ - scx_bpf_dsq_insert_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, - p->scx.dsq_vtime, 0); - return cpu; - } + /* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */ + scx_bpf_dsq_insert_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, + p->scx.dsq_vtime, 0); - return prev_cpu; + return cpu; } void BPF_STRUCT_OPS(ddsp_vtimelocal_fail_exit, struct scx_exit_info *ei) diff --git a/tools/testing/selftests/sched_ext/exit.c b/tools/testing/selftests/sched_ext/exit.c index b987611789d16..01b17092d5c8f 100644 --- a/tools/testing/selftests/sched_ext/exit.c +++ b/tools/testing/selftests/sched_ext/exit.c @@ -31,6 +31,7 @@ static enum scx_test_status run(void *ctx) continue; skel = exit__open(); + SCX_FAIL_IF(!skel, "Failed to open"); SCX_ENUM_INIT(skel); skel->rodata->exit_point = tc; SCX_FAIL_IF(exit__load(skel), "Failed to load skel"); diff --git a/tools/testing/selftests/sched_ext/prog_run.c b/tools/testing/selftests/sched_ext/prog_run.c index 05974820ca69d..1129ec2aaddc7 100644 --- a/tools/testing/selftests/sched_ext/prog_run.c +++ b/tools/testing/selftests/sched_ext/prog_run.c @@ -28,7 +28,8 @@ static enum scx_test_status setup(void **ctx) static enum scx_test_status run(void *ctx) { struct prog_run *skel = ctx; - struct bpf_link *link; + struct bpf_link *link = NULL; + enum scx_test_status status = SCX_TEST_PASS; int prog_fd, err = 0; prog_fd = bpf_program__fd(skel->progs.prog_run_syscall); @@ -42,23 +43,40 @@ static enum scx_test_status run(void *ctx) link = bpf_map__attach_struct_ops(skel->maps.prog_run_ops); if (!link) { SCX_ERR("Failed to attach scheduler"); - close(prog_fd); - return SCX_TEST_FAIL; + status = SCX_TEST_FAIL; + goto out; } err = bpf_prog_test_run_opts(prog_fd, &topts); - SCX_EQ(err, 0); + if (err) { + SCX_ERR("BPF_PROG_RUN failed (%d)", err); + status = SCX_TEST_FAIL; + goto out; + } /* Assumes uei.kind is written last */ while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE)) sched_yield(); - SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF)); - SCX_EQ(skel->data->uei.exit_code, 0xdeadbeef); + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG_BPF)) { + SCX_ERR("Unexpected exit kind: %llu", + (unsigned long long)skel->data->uei.kind); + status = SCX_TEST_FAIL; + goto out; + } + if (skel->data->uei.exit_code != 0xdeadbeef) { + SCX_ERR("Unexpected exit code: %lld", + (long long)skel->data->uei.exit_code); + status = SCX_TEST_FAIL; + goto out; + } + +out: close(prog_fd); - bpf_link__destroy(link); + if (link) + bpf_link__destroy(link); - return SCX_TEST_PASS; + return status; } static void cleanup(void *ctx) diff --git a/tools/testing/selftests/timers/leap-a-day.c b/tools/testing/selftests/timers/leap-a-day.c index 3568cfb3e8157..97c8c66bea883 100644 --- a/tools/testing/selftests/timers/leap-a-day.c +++ b/tools/testing/selftests/timers/leap-a-day.c @@ -9,16 +9,19 @@ * kernel's leap-second behavior, as well as how well applications * handle the leap-second discontinuity. * - * Usage: leap-a-day [-s] [-i ] + * Usage: leap-a-day [-w] [-i ] [-t] * * Options: - * -s: Each iteration, set the date to 10 seconds before midnight GMT. - * This speeds up the number of leapsecond transitions tested, - * but because it calls settimeofday frequently, advancing the - * time by 24 hours every ~16 seconds, it may cause application - * disruption. + * -w: Only set the leap-second flag and wait for the leap second + * each iteration, instead of advancing the time. By default the + * date is set to 10 seconds before midnight GMT, which speeds up + * the number of leapsecond transitions tested, but because it + * calls settimeofday frequently, advancing the time by 24 hours + * every ~16 seconds, it may cause application disruption. * - * -i: Number of iterations to run (default: infinite) + * -i: Number of iterations to run (-1 = infinite, default: 10) + * + * -t: Print TAI time. * * Other notes: Disabling NTP prior to running this is advised, as the two * may conflict in their commands to the kernel. @@ -186,7 +189,7 @@ int main(int argc, char **argv) int opt; /* Process arguments */ - while ((opt = getopt(argc, argv, "sti:")) != -1) { + while ((opt = getopt(argc, argv, "wti:")) != -1) { switch (opt) { case 'w': printf("Only setting leap-flag, not changing time. It could take up to a day for leap to trigger.\n"); diff --git a/tools/testing/selftests/ublk/Makefile b/tools/testing/selftests/ublk/Makefile index 6e4fe8d1fed1d..6dbae1391a064 100644 --- a/tools/testing/selftests/ublk/Makefile +++ b/tools/testing/selftests/ublk/Makefile @@ -69,6 +69,8 @@ TEST_PROGS += test_stress_08.sh TEST_PROGS += test_stress_09.sh TEST_FILES := settings +TEST_FILES += test_common.sh +TEST_FILES += trace TEST_GEN_PROGS_EXTENDED = kublk metadata_size STANDALONE_UTILS := metadata_size.c diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_device.c b/tools/testing/selftests/vfio/lib/vfio_pci_device.c index 94dc5fcecbeb6..3db65084a4351 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_device.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_device.c @@ -30,13 +30,11 @@ static void vfio_pci_irq_set(struct vfio_pci_device *device, u32 index, u32 vector, u32 count, int *fds) { - u8 buf[sizeof(struct vfio_irq_set) + sizeof(int) * count]; - struct vfio_irq_set *irq = (void *)&buf; - int *irq_fds = (void *)&irq->data; + size_t argsz = sizeof(struct vfio_irq_set) + sizeof(int) * count; + struct vfio_irq_set *irq; - memset(buf, 0, sizeof(buf)); - - irq->argsz = sizeof(buf); + irq = calloc_assert(1, argsz); + irq->argsz = argsz; irq->flags = VFIO_IRQ_SET_ACTION_TRIGGER; irq->index = index; irq->start = vector; @@ -44,12 +42,13 @@ static void vfio_pci_irq_set(struct vfio_pci_device *device, if (count) { irq->flags |= VFIO_IRQ_SET_DATA_EVENTFD; - memcpy(irq_fds, fds, sizeof(int) * count); + memcpy(irq->data, fds, sizeof(int) * count); } else { irq->flags |= VFIO_IRQ_SET_DATA_NONE; } ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, irq); + free(irq); } void vfio_pci_irq_trigger(struct vfio_pci_device *device, u32 index, u32 vector) @@ -118,15 +117,20 @@ static void vfio_pci_irq_get(struct vfio_pci_device *device, u32 index, static int vfio_device_feature_ioctl(int fd, u32 flags, void *data, size_t data_size) { - u8 buffer[sizeof(struct vfio_device_feature) + data_size] = {}; - struct vfio_device_feature *feature = (void *)buffer; + size_t argsz = sizeof(struct vfio_device_feature) + data_size; + struct vfio_device_feature *feature; + int ret; + feature = calloc_assert(1, argsz); memcpy(feature->data, data, data_size); - feature->argsz = sizeof(buffer); + feature->argsz = argsz; feature->flags = flags; - return ioctl(fd, VFIO_DEVICE_FEATURE, feature); + ret = ioctl(fd, VFIO_DEVICE_FEATURE, feature); + free(feature); + + return ret; } static void vfio_device_feature_set(int fd, u16 feature, void *data, size_t data_size) diff --git a/tools/testing/selftests/zram/zram_lib.sh b/tools/testing/selftests/zram/zram_lib.sh index 21ec1966de76c..0d44d83888f9d 100755 --- a/tools/testing/selftests/zram/zram_lib.sh +++ b/tools/testing/selftests/zram/zram_lib.sh @@ -37,7 +37,7 @@ kernel_gte() if [ $kernel_major -gt $major ]; then return 0 - elif [[ $kernel_major -eq $major && $kernel_minor -ge $minor ]]; then + elif [ $kernel_major -eq $major ] && [ $kernel_minor -ge $minor ]; then return 0 fi diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index db57c5766ab60..171a9da1b6855 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -185,6 +185,10 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start, } flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range); + +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + kvm_arch_gmem_invalidate_range(kvm, &gfn_range); +#endif } if (flush) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 45e784462ec6e..e389ac4744e69 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1116,7 +1116,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) rcuwait_init(&kvm->mn_memslots_update_rcuwait); xa_init(&kvm->vcpu_array); #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES - xa_init(&kvm->mem_attr_array); + xa_init_flags(&kvm->mem_attr_array, XA_FLAGS_ACCOUNT); #endif INIT_LIST_HEAD(&kvm->gpc_list); @@ -2446,14 +2446,36 @@ bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, return (kvm_get_memory_attributes(kvm, start) & mask) == attrs; guard(rcu)(); - if (!attrs) - return !xas_find(&xas, end - 1); + /* + * Lookup the entry for each index instead of iterating over the xarray + * as KVM deletes/nullifies entries to represent "no attributes", and + * the xas index is effectively invalid when no entry is found. I.e. + * matching non-zero attributes for *every* entry effectively requires + * a manually lookup for each index. + * + * Skip pre-allocated, reserved entries, or restart the lookup if the + * xarray was concurrently modified, via xas_retry() ("retry" means the + * entry holds an internal xarray value, i.e. is either invalid or NULL + * from the caller's perspective). + * + * Use xas_next() when looking for non-zero attributes to optimize for + * the case where the start of the range (or the entire range) doesn't + * have any attributes, as xas_next() returns literally the next entry, + * whereas xas_next_entry() returns the next non-NULL entry (bounded by + * a maximum index). + */ for (index = start; index < end; index++) { do { - entry = xas_next(&xas); + entry = attrs ? xas_next(&xas) : + xas_next_entry(&xas, end - 1); } while (xas_retry(&xas, entry)); + if (!entry) + return !attrs; + + WARN_ON_ONCE(!xa_to_value(entry)); + if (xas.xa_index != index || (xa_to_value(entry) & mask) != attrs) return false;