drm/amdkfd: Add support for more per-process flag

Add support for more per-process flags starting with option to configure
MFMA precision for gfx 9.5

v2: Change flag name to KFD_PROC_FLAG_MFMA_HIGH_PRECISION
    Remove unused else condition
v3: Bump the KFD API version
v4: Missed SH_MEM_CONFIG__PRECISION_MODE__SHIFT define. Added it.

Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
Reviewed-by: Amber Lin <Amber.Lin@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 8c2e92378b491ed806a24b0a54fd1c46132d4970..1e9dd00620bf4104a8ab0354656fc0db07d618f6 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -606,7 +606,8 @@ static int kfd_ioctl_set_memory_policy(struct file *filep,
                                default_policy,
                                alternate_policy,
                                (void __user *)args->alternate_aperture_base,
-                               args->alternate_aperture_size))
+                               args->alternate_aperture_size,
+                               args->misc_process_flag))
                err = -EINVAL;
 
 out:

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index d23c6a358d3431e836602b846b43f675b528a4de..2afcc1b4856ac75db5687a761685159797405e36 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -2596,7 +2596,8 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size)
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties)
 {
        bool retval = true;
 
@@ -2611,7 +2612,8 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
                        default_policy,
                        alternate_policy,
                        alternate_aperture_base,
-                       alternate_aperture_size);
+                       alternate_aperture_size,
+                       misc_process_properties);
 
        if (retval)
                goto out;

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 7146e227e2c10ab5081bc8c41350bd9313689559..122eb745e9c4df2ef159e62553d009d59b6abe9e 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -174,7 +174,8 @@ struct device_queue_manager_ops {
                                           enum cache_policy default_policy,
                                           enum cache_policy alternate_policy,
                                           void __user *alternate_aperture_base,
-                                          uint64_t alternate_aperture_size);
+                                          uint64_t alternate_aperture_size,
+                                          u32 misc_process_properties);
 
        int (*process_termination)(struct device_queue_manager *dqm,
                        struct qcm_process_device *qpd);
@@ -210,7 +211,8 @@ struct device_queue_manager_asic_ops {
                                           enum cache_policy default_policy,
                                           enum cache_policy alternate_policy,
                                           void __user *alternate_aperture_base,
-                                          uint64_t alternate_aperture_size);
+                                          uint64_t alternate_aperture_size,
+                                          u32 misc_process_properties);
        void    (*init_sdma_vm)(struct device_queue_manager *dqm,
                                struct queue *q,
                                struct qcm_process_device *qpd);

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
index 32bedef912b3b20a663c63e30ef29e5872e8b50a..0508ef5a41d7c64eadc3933777f5d479eb37aaab 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
@@ -40,7 +40,8 @@ static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size);
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties);
 static int update_qpd_cik(struct device_queue_manager *dqm,
                          struct qcm_process_device *qpd);
 static void init_sdma_vm(struct device_queue_manager *dqm,
@@ -88,7 +89,8 @@ static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size)
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties)
 {
        uint32_t default_mtype;
        uint32_t ape1_mtype;

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c
index b5f5f141353b5f385e564203137c6c20d40fb208..ba6e3d747ccd0220e0536ec4925c96a23c293952 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c
@@ -36,7 +36,8 @@ static bool set_cache_memory_policy_v10(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size);
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties);
 
 void device_queue_manager_init_v10(
        struct device_queue_manager_asic_ops *asic_ops)
@@ -61,7 +62,8 @@ static bool set_cache_memory_policy_v10(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size)
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties)
 {
        qpd->sh_mem_config = (SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
                              SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) |

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v11.c
index f436878d0d6218a4f5568c69d449c749bce3a972..8b447d04558f1642f4b8e79436295448d7530967 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v11.c
@@ -35,7 +35,8 @@ static bool set_cache_memory_policy_v11(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size);
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties);
 
 void device_queue_manager_init_v11(
        struct device_queue_manager_asic_ops *asic_ops)
@@ -60,7 +61,8 @@ static bool set_cache_memory_policy_v11(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size)
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties)
 {
        qpd->sh_mem_config = (SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
                              SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) |

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12.c
index 62ca1c8fcbaf9ab128dd6df3ca672fd37ebb1885..3550da3a46f90ad0674da14728d2d813e6765cae 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12.c
@@ -35,7 +35,8 @@ static bool set_cache_memory_policy_v12(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size);
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties);
 
 void device_queue_manager_init_v12(
        struct device_queue_manager_asic_ops *asic_ops)
@@ -60,7 +61,8 @@ static bool set_cache_memory_policy_v12(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size)
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties)
 {
        qpd->sh_mem_config = (SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
                              SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) |

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
index 3264509408bc81b9b6ce2cfa74108b1acec207f8..d794c8172b40995e6d82c938841b0ddd28d8eb57 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
@@ -35,7 +35,8 @@ static bool set_cache_memory_policy_v9(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size);
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties);
 
 void device_queue_manager_init_v9(
        struct device_queue_manager_asic_ops *asic_ops)
@@ -60,7 +61,8 @@ static bool set_cache_memory_policy_v9(struct device_queue_manager *dqm,
                                   enum cache_policy default_policy,
                                   enum cache_policy alternate_policy,
                                   void __user *alternate_aperture_base,
-                                  uint64_t alternate_aperture_size)
+                                  uint64_t alternate_aperture_size,
+                                  u32 misc_process_properties)
 {
        qpd->sh_mem_config = SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
                                SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
@@ -73,6 +75,11 @@ static bool set_cache_memory_policy_v9(struct device_queue_manager *dqm,
                KFD_GC_VERSION(dqm->dev->kfd) == IP_VERSION(9, 5, 0))
                qpd->sh_mem_config |= (1 << SH_MEM_CONFIG__F8_MODE__SHIFT);
 
+       if (KFD_GC_VERSION(dqm->dev->kfd) == IP_VERSION(9, 5, 0)) {
+               if (misc_process_properties & KFD_PROC_FLAG_MFMA_HIGH_PRECISION)
+                       qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__PRECISION_MODE__SHIFT;
+       }
+
        qpd->sh_mem_ape1_limit = 0;
        qpd->sh_mem_ape1_base = 0;
        qpd->sh_mem_bases = compute_sh_mem_bases_64bit(qpd_to_pdd(qpd));

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
index 320518f418903de375ba9f79cef15b0e38062be5..dad83356e976ae570679e2ba74765bfba5caf51d 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
@@ -40,7 +40,8 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
                                       enum cache_policy default_policy,
                                       enum cache_policy alternate_policy,
                                       void __user *alternate_aperture_base,
-                                      uint64_t alternate_aperture_size);
+                                      uint64_t alternate_aperture_size,
+                                      u32 misc_process_properties);
 static int update_qpd_vi(struct device_queue_manager *dqm,
                         struct qcm_process_device *qpd);
 static void init_sdma_vm(struct device_queue_manager *dqm,
@@ -89,7 +90,8 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
                enum cache_policy default_policy,
                enum cache_policy alternate_policy,
                void __user *alternate_aperture_base,
-               uint64_t alternate_aperture_size)
+               uint64_t alternate_aperture_size,
+               u32 misc_process_properties)
 {
        uint32_t default_mtype;
        uint32_t ape1_mtype;

diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_3_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_3_sh_mask.h
index 2bd9f3f1026f9cc61dbd9dd1c2a637eb27831ee9..0122a21c50cf02209b0b33fe40769db06447d21f 100644 (file)
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_3_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_3_sh_mask.h
@@ -2261,11 +2261,13 @@
 #define SH_MEM_CONFIG__ADDRESS_MODE__SHIFT                                                                    0x0
 #define SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT                                                                  0x3
 #define SH_MEM_CONFIG__F8_MODE__SHIFT                                                                         0x8
+#define SH_MEM_CONFIG__PRECISION_MODE__SHIFT                                                                  0x9
 #define SH_MEM_CONFIG__RETRY_DISABLE__SHIFT                                                                   0xc
 #define SH_MEM_CONFIG__PRIVATE_NV__SHIFT                                                                      0xd
 #define SH_MEM_CONFIG__ADDRESS_MODE_MASK                                                                      0x00000001L
 #define SH_MEM_CONFIG__ALIGNMENT_MODE_MASK                                                                    0x00000018L
 #define SH_MEM_CONFIG__F8_MODE_MASK                                                                           0x00000100L
+#define SH_MEM_CONFIG__PRECISION_MODE_MASK                                                                    0x00000200L
 #define SH_MEM_CONFIG__RETRY_DISABLE_MASK                                                                     0x00001000L
 #define SH_MEM_CONFIG__PRIVATE_NV_MASK                                                                        0x00002000L
 //SP_MFMA_PORTD_RD_CONFIG

diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h
index b0160b09987c1fd188d5b23434eae28178796f9c..1e59344c5673e15b48f1524b8882f708fb5a6aeb 100644 (file)
--- a/include/uapi/linux/kfd_ioctl.h
+++ b/include/uapi/linux/kfd_ioctl.h
@@ -43,9 +43,10 @@
  * - 1.15 - Enable managing mappings in compute VMs with GEM_VA ioctl
  * - 1.16 - Add contiguous VRAM allocation flag
  * - 1.17 - Add SDMA queue creation with target SDMA engine ID
+ * - 1.18 - Rename pad in set_memory_policy_args to misc_process_flag
  */
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 17
+#define KFD_IOCTL_MINOR_VERSION 18
 
 struct kfd_ioctl_get_version_args {
        __u32 major_version;    /* from KFD */
@@ -150,6 +151,9 @@ struct kfd_dbg_device_info_entry {
 #define KFD_IOC_CACHE_POLICY_COHERENT 0
 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
 
+/* Misc. per process flags */
+#define KFD_PROC_FLAG_MFMA_HIGH_PRECISION (1 << 0)
+
 struct kfd_ioctl_set_memory_policy_args {
        __u64 alternate_aperture_base;  /* to KFD */
        __u64 alternate_aperture_size;  /* to KFD */
@@ -157,7 +161,7 @@ struct kfd_ioctl_set_memory_policy_args {
        __u32 gpu_id;                   /* to KFD */
        __u32 default_policy;           /* to KFD */
        __u32 alternate_policy;         /* to KFD */
-       __u32 pad;
+       __u32 misc_process_flag;        /* to KFD */
 };
 
 /*