--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+
+use kernel::prelude::*;
+use kernel::types::ARef;
+use kernel::{dev_warn, device};
+
+use crate::dma::DmaObject;
+use crate::driver::Bar0;
+use crate::gpu::Chipset;
+
+mod hal;
+
+/// Type holding the sysmem flush memory page, a page of memory to be written into the
+/// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR*` registers and used to maintain memory coherency.
+///
+/// Users are responsible for manually calling [`Self::unregister`] before dropping this object,
+/// otherwise the GPU might still use it even after it has been freed.
+pub(crate) struct SysmemFlush {
+ /// Chipset we are operating on.
+ chipset: Chipset,
+ device: ARef<device::Device>,
+ /// Keep the page alive as long as we need it.
+ page: DmaObject,
+}
+
+impl SysmemFlush {
+ /// Allocate a memory page and register it as the sysmem flush page.
+ pub(crate) fn register(
+ dev: &device::Device<device::Bound>,
+ bar: &Bar0,
+ chipset: Chipset,
+ ) -> Result<Self> {
+ let page = DmaObject::new(dev, kernel::page::PAGE_SIZE)?;
+
+ hal::fb_hal(chipset).write_sysmem_flush_page(bar, page.dma_handle())?;
+
+ Ok(Self {
+ chipset,
+ device: dev.into(),
+ page,
+ })
+ }
+
+ /// Unregister the managed sysmem flush page.
+ ///
+ /// In order to gracefully tear down the GPU, users must make sure to call this method before
+ /// dropping the object.
+ pub(crate) fn unregister(&self, bar: &Bar0) {
+ let hal = hal::fb_hal(self.chipset);
+
+ if hal.read_sysmem_flush_page(bar) == self.page.dma_handle() {
+ let _ = hal.write_sysmem_flush_page(bar, 0).inspect_err(|e| {
+ dev_warn!(
+ &self.device,
+ "failed to unregister sysmem flush page: {:?}",
+ e
+ )
+ });
+ } else {
+ // Another page has been registered after us for some reason - warn as this is a bug.
+ dev_warn!(
+ &self.device,
+ "attempt to unregister a sysmem flush page that is not active\n"
+ );
+ }
+ }
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+
+use kernel::prelude::*;
+
+use crate::driver::Bar0;
+use crate::gpu::Chipset;
+
+mod ga100;
+mod tu102;
+
+pub(crate) trait FbHal {
+ /// Returns the address of the currently-registered sysmem flush page.
+ fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64;
+
+ /// Register `addr` as the address of the sysmem flush page.
+ ///
+ /// This might fail if the address is too large for the receiving register.
+ fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result;
+}
+
+/// Returns the HAL corresponding to `chipset`.
+pub(super) fn fb_hal(chipset: Chipset) -> &'static dyn FbHal {
+ use Chipset::*;
+
+ match chipset {
+ TU102 | TU104 | TU106 | TU117 | TU116 => tu102::TU102_HAL,
+ GA100 | GA102 | GA103 | GA104 | GA106 | GA107 | AD102 | AD103 | AD104 | AD106 | AD107 => {
+ ga100::GA100_HAL
+ }
+ }
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+
+struct Ga100;
+
+use kernel::prelude::*;
+
+use crate::driver::Bar0;
+use crate::fb::hal::FbHal;
+use crate::regs;
+
+use super::tu102::FLUSH_SYSMEM_ADDR_SHIFT;
+
+pub(super) fn read_sysmem_flush_page_ga100(bar: &Bar0) -> u64 {
+ (regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR::read(bar).adr_39_08() as u64) << FLUSH_SYSMEM_ADDR_SHIFT
+ | (regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI::read(bar).adr_63_40() as u64)
+ << FLUSH_SYSMEM_ADDR_SHIFT_HI
+}
+
+pub(super) fn write_sysmem_flush_page_ga100(bar: &Bar0, addr: u64) {
+ regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI::default()
+ .set_adr_63_40((addr >> FLUSH_SYSMEM_ADDR_SHIFT_HI) as u32)
+ .write(bar);
+ regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR::default()
+ .set_adr_39_08((addr >> FLUSH_SYSMEM_ADDR_SHIFT) as u32)
+ .write(bar);
+}
+
+/// Shift applied to the sysmem address before it is written into
+/// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI`,
+const FLUSH_SYSMEM_ADDR_SHIFT_HI: u32 = 40;
+
+impl FbHal for Ga100 {
+ fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 {
+ read_sysmem_flush_page_ga100(bar)
+ }
+
+ fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result {
+ write_sysmem_flush_page_ga100(bar, addr);
+
+ Ok(())
+ }
+}
+
+const GA100: Ga100 = Ga100;
+pub(super) const GA100_HAL: &dyn FbHal = &GA100;
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+
+use crate::driver::Bar0;
+use crate::fb::hal::FbHal;
+use crate::regs;
+use kernel::prelude::*;
+
+/// Shift applied to the sysmem address before it is written into `NV_PFB_NISO_FLUSH_SYSMEM_ADDR`,
+/// to be used by HALs.
+pub(super) const FLUSH_SYSMEM_ADDR_SHIFT: u32 = 8;
+
+pub(super) fn read_sysmem_flush_page_gm107(bar: &Bar0) -> u64 {
+ (regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR::read(bar).adr_39_08() as u64) << FLUSH_SYSMEM_ADDR_SHIFT
+}
+
+pub(super) fn write_sysmem_flush_page_gm107(bar: &Bar0, addr: u64) -> Result {
+ // Check that the address doesn't overflow the receiving 32-bit register.
+ if addr >> (u32::BITS + FLUSH_SYSMEM_ADDR_SHIFT) == 0 {
+ regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR::default()
+ .set_adr_39_08((addr >> FLUSH_SYSMEM_ADDR_SHIFT) as u32)
+ .write(bar);
+
+ Ok(())
+ } else {
+ Err(EINVAL)
+ }
+}
+
+struct Tu102;
+
+impl FbHal for Tu102 {
+ fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 {
+ read_sysmem_flush_page_gm107(bar)
+ }
+
+ fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result {
+ write_sysmem_flush_page_gm107(bar, addr)
+ }
+}
+
+const TU102: Tu102 = Tu102;
+pub(super) const TU102_HAL: &dyn FbHal = &TU102;
use kernel::{device, devres::Devres, error::code::*, pci, prelude::*};
use crate::driver::Bar0;
+use crate::fb::SysmemFlush;
use crate::firmware::{Firmware, FIRMWARE_VERSION};
use crate::gfw;
use crate::regs;
}
/// Structure holding the resources required to operate the GPU.
-#[pin_data]
+#[pin_data(PinnedDrop)]
pub(crate) struct Gpu {
spec: Spec,
/// MMIO mapping of PCI BAR 0
bar: Devres<Bar0>,
fw: Firmware,
+ /// System memory page required for flushing all pending GPU-side memory writes done through
+ /// PCIE into system memory.
+ sysmem_flush: SysmemFlush,
+}
+
+#[pinned_drop]
+impl PinnedDrop for Gpu {
+ fn drop(self: Pin<&mut Self>) {
+ // Unregister the sysmem flush page before we release it.
+ self.bar
+ .try_access_with(|b| self.sysmem_flush.unregister(b));
+ }
}
impl Gpu {
gfw::wait_gfw_boot_completion(bar)
.inspect_err(|_| dev_err!(pdev.as_ref(), "GFW boot did not complete"))?;
+ // System memory page required for sysmembar to properly flush into system memory.
+ let sysmem_flush = SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?;
+
Ok(pin_init!(Self {
spec,
bar: devres_bar,
- fw
+ fw,
+ sysmem_flush,
}))
}
}
mod dma;
mod driver;
+mod fb;
mod firmware;
mod gfw;
mod gpu;
}
}
+/* PFB */
+
+register!(NV_PFB_NISO_FLUSH_SYSMEM_ADDR @ 0x00100c10 {
+ 31:0 adr_39_08 as u32;
+});
+
+register!(NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI @ 0x00100c40 {
+ 23:0 adr_63_40 as u32;
+});
+
/* PGC6 */
register!(NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK @ 0x00118128 {
projects / thirdparty / linux.git / commitdiff
