drm/amdgpu: Default disable GDS for compute VMIDs

Submitted by Greathouse, Joseph on July 17, 2019, 5:11 p.m.

Details

Message ID 20190717171112.4962-1-Joseph.Greathouse@amd.com
State New
Headers show
Series "drm/amdgpu: Default disable GDS for compute VMIDs" ( rev: 1 ) in AMD X.Org drivers

Not browsing as part of any series.

Commit Message

Greathouse, Joseph July 17, 2019, 5:11 p.m.
The GDS and GWS blocks default to allowing all VMIDs to
access all entries. Graphics VMIDs can handle setting
these limits when the driver launches work. However,
compute workloads under HWS control don't go through the
kernel driver. Instead, HWS firmware should set these
limits when a process is put into a VMID slot.

Disable access to these devices by default by turning off
all mask bits (for OA) and setting BASE=SIZE=0 (for GDS
and GWS) for all compute VMIDs. If a process wants to use
these resources, they can request this from the HWS
firmware (when such capabilities are enabled). HWS will
then handle setting the base and limit for the process when
it is assigned to a VMID.

This will also prevent user kernels from getting 'stuck' in
GWS by accident if they write GWS-using code but HWS
firmware is not set up to handle GWS reset. Until HWS is
enabled to handle GWS properly, all GWS accesses will
MEM_VIOL fault the kernel.

Change-Id: I2a709fdcb2468511754f2e5eae75546751c0e6f0
Signed-off-by: Joseph Greathouse <Joseph.Greathouse@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 6 ++++++
 drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c  | 6 ++++++
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c  | 6 ++++++
 drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c  | 6 ++++++
 4 files changed, 24 insertions(+)

Patch hide | download patch | download mbox

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index 618291df659b..32da5a91abfb 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -1513,6 +1513,12 @@  static void gfx_v10_0_init_compute_vmid(struct amdgpu_device *adev)
 		/* CP and shaders */
 		WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
 		WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
+		/* Initialize all compute VMIDs to have no GDS, GWS, or OA
+		   acccess. These should be enabled by FW for target VMID. */
+		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0x00000000);
+		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0x00000000);
+		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0x00000000);
+		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0x00000000);
 	}
 	nv_grbm_select(adev, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index e1e2a44ee13c..89ea0d799c12 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -1874,6 +1874,12 @@  static void gfx_v7_0_init_compute_vmid(struct amdgpu_device *adev)
 		WREG32(mmSH_MEM_APE1_BASE, 1);
 		WREG32(mmSH_MEM_APE1_LIMIT, 0);
 		WREG32(mmSH_MEM_BASES, sh_mem_bases);
+		/* Initialize all compute VMIDs to have no GDS, GWS, or OA
+		   acccess. These should be enabled by FW for target VMID. */
+		WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
+		WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
+		WREG32(amdgpu_gds_reg_offset[i].gws, 0);
+		WREG32(amdgpu_gds_reg_offset[i].oa, 0);
 	}
 	cik_srbm_select(adev, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 8c590a554675..53d9a223f8c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -3699,6 +3699,12 @@  static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
 		WREG32(mmSH_MEM_APE1_BASE, 1);
 		WREG32(mmSH_MEM_APE1_LIMIT, 0);
 		WREG32(mmSH_MEM_BASES, sh_mem_bases);
+		/* Initialize all compute VMIDs to have no GDS, GWS, or OA
+		   acccess. These should be enabled by FW for target VMID. */
+		WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
+		WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
+		WREG32(amdgpu_gds_reg_offset[i].gws, 0);
+		WREG32(amdgpu_gds_reg_offset[i].oa, 0);
 	}
 	vi_srbm_select(adev, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 5af60e1c735a..b9ed03a8a561 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -2022,6 +2022,12 @@  static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
 		/* CP and shaders */
 		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
 		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
+		/* Initialize all compute VMIDs to have no GDS, GWS, or OA
+		   acccess. These should be enabled by FW for target VMID. */
+		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0x00000000);
+		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0x00000000);
+		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0x00000000);
+		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0x00000000);
 	}
 	soc15_grbm_select(adev, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);

Comments

On 2019-07-17 13:11, Greathouse, Joseph wrote:
> The GDS and GWS blocks default to allowing all VMIDs to

> access all entries. Graphics VMIDs can handle setting

> these limits when the driver launches work. However,

> compute workloads under HWS control don't go through the

> kernel driver. Instead, HWS firmware should set these

> limits when a process is put into a VMID slot.

>

> Disable access to these devices by default by turning off

> all mask bits (for OA) and setting BASE=SIZE=0 (for GDS

> and GWS) for all compute VMIDs. If a process wants to use

> these resources, they can request this from the HWS

> firmware (when such capabilities are enabled). HWS will

> then handle setting the base and limit for the process when

> it is assigned to a VMID.

>

> This will also prevent user kernels from getting 'stuck' in

> GWS by accident if they write GWS-using code but HWS

> firmware is not set up to handle GWS reset. Until HWS is

> enabled to handle GWS properly, all GWS accesses will

> MEM_VIOL fault the kernel.

>

> Change-Id: I2a709fdcb2468511754f2e5eae75546751c0e6f0

> Signed-off-by: Joseph Greathouse <Joseph.Greathouse@amd.com>

> ---

>   drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 6 ++++++

>   drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c  | 6 ++++++

>   drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c  | 6 ++++++

>   drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c  | 6 ++++++

>   4 files changed, 24 insertions(+)

>

> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c

> index 618291df659b..32da5a91abfb 100644

> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c

> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c

> @@ -1513,6 +1513,12 @@ static void gfx_v10_0_init_compute_vmid(struct amdgpu_device *adev)

>   		/* CP and shaders */

>   		WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);

>   		WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);

> +		/* Initialize all compute VMIDs to have no GDS, GWS, or OA

> +		   acccess. These should be enabled by FW for target VMID. */

> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0x00000000);

> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0x00000000);

> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0x00000000);

> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0x00000000);

I'd prefer these register initializations in a separate loop outside the 
srbm_mutex, because they don't depend on the srbm/grbm_select bits. Same 
for the other GFX IP versions.

Regards,
   Felix


>   	}

>   	nv_grbm_select(adev, 0, 0, 0, 0);

>   	mutex_unlock(&adev->srbm_mutex);

> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c

> index e1e2a44ee13c..89ea0d799c12 100644

> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c

> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c

> @@ -1874,6 +1874,12 @@ static void gfx_v7_0_init_compute_vmid(struct amdgpu_device *adev)

>   		WREG32(mmSH_MEM_APE1_BASE, 1);

>   		WREG32(mmSH_MEM_APE1_LIMIT, 0);

>   		WREG32(mmSH_MEM_BASES, sh_mem_bases);

> +		/* Initialize all compute VMIDs to have no GDS, GWS, or OA

> +		   acccess. These should be enabled by FW for target VMID. */

> +		WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);

> +		WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);

> +		WREG32(amdgpu_gds_reg_offset[i].gws, 0);

> +		WREG32(amdgpu_gds_reg_offset[i].oa, 0);

>   	}

>   	cik_srbm_select(adev, 0, 0, 0, 0);

>   	mutex_unlock(&adev->srbm_mutex);

> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c

> index 8c590a554675..53d9a223f8c1 100644

> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c

> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c

> @@ -3699,6 +3699,12 @@ static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)

>   		WREG32(mmSH_MEM_APE1_BASE, 1);

>   		WREG32(mmSH_MEM_APE1_LIMIT, 0);

>   		WREG32(mmSH_MEM_BASES, sh_mem_bases);

> +		/* Initialize all compute VMIDs to have no GDS, GWS, or OA

> +		   acccess. These should be enabled by FW for target VMID. */

> +		WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);

> +		WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);

> +		WREG32(amdgpu_gds_reg_offset[i].gws, 0);

> +		WREG32(amdgpu_gds_reg_offset[i].oa, 0);

>   	}

>   	vi_srbm_select(adev, 0, 0, 0, 0);

>   	mutex_unlock(&adev->srbm_mutex);

> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

> index 5af60e1c735a..b9ed03a8a561 100644

> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c

> @@ -2022,6 +2022,12 @@ static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)

>   		/* CP and shaders */

>   		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);

>   		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);

> +		/* Initialize all compute VMIDs to have no GDS, GWS, or OA

> +		   acccess. These should be enabled by FW for target VMID. */

> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0x00000000);

> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0x00000000);

> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0x00000000);

> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0x00000000);

>   	}

>   	soc15_grbm_select(adev, 0, 0, 0, 0);

>   	mutex_unlock(&adev->srbm_mutex);