[v8,2/9] drm/i915: vgpu shared memory setup for pv optimization

Submitted by Zhang, Xiaolin on July 23, 2019, 11:31 a.m.

Details

Message ID 1563881524-147440-3-git-send-email-xiaolin.zhang@intel.com
State New
Headers show
Series "i915 vgpu PV to improve vgpu performance" ( rev: 1 ) in Intel GVT devel

Not browsing as part of any series.

Commit Message

Zhang, Xiaolin July 23, 2019, 11:31 a.m.
To enable vgpu pv features, we need to setup a shared memory page
which will be used for data exchange directly accessed between both
guest and backend i915 driver to avoid emulation trap cost.

guest i915 will allocate this page memory and then pass it's physical
address to backend i915 driver through PVINFO register so that backend i915
driver can access this shared page meory without any trap cost with the
help form hyperviser's read guest gpa functionality.

guest i915 will send VGT_G2V_SHARED_PAGE_SETUP notification to host GVT
once shared memory setup finished.

the layout of the shared_page also defined as well in this patch which
is used for pv features implementation.

v0: RFC.
v1: addressed RFC comment to move both shared_page_lock and shared_page
to i915_virtual_gpu structure.
v2: packed i915_virtual_gpu structure.
v3: added SHARED_PAGE_SETUP g2v notification for pv shared_page setup
v4: added intel_vgpu_setup_shared_page() in i915_vgpu_pv.c.
v5: per engine desc data in shared memory.
v6: added version support in shared memory (Zhenyu).
v7: rebase.

Signed-off-by: Xiaolin Zhang <xiaolin.zhang@intel.com>
---
 drivers/gpu/drm/i915/i915_drv.h    |  4 +-
 drivers/gpu/drm/i915/i915_pvinfo.h |  5 ++-
 drivers/gpu/drm/i915/i915_vgpu.c   | 85 ++++++++++++++++++++++++++++++++++++++
 drivers/gpu/drm/i915/i915_vgpu.h   | 17 ++++++++
 4 files changed, 109 insertions(+), 2 deletions(-)

Patch hide | download patch | download mbox

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index fa5dc47..48cf141 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -1080,7 +1080,9 @@  struct i915_virtual_gpu {
 	bool active;
 	u32 caps;
 	u32 pv_caps;
-};
+
+	struct i915_virtual_gpu_pv *pv;
+} __packed;
 
 /* used in computing the new watermarks state */
 struct intel_wm_config {
diff --git a/drivers/gpu/drm/i915/i915_pvinfo.h b/drivers/gpu/drm/i915/i915_pvinfo.h
index ad398b4..3c63603 100644
--- a/drivers/gpu/drm/i915/i915_pvinfo.h
+++ b/drivers/gpu/drm/i915/i915_pvinfo.h
@@ -48,6 +48,7 @@  enum vgt_g2v_type {
 	VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY,
 	VGT_G2V_EXECLIST_CONTEXT_CREATE,
 	VGT_G2V_EXECLIST_CONTEXT_DESTROY,
+	VGT_G2V_SHARED_PAGE_SETUP,
 	VGT_G2V_MAX,
 };
 
@@ -112,7 +113,9 @@  struct vgt_if {
 
 	u32 pv_caps;
 
-	u32  rsv7[0x200 - 25];    /* pad to one page */
+	u64 shared_page_gpa;
+
+	u32  rsv7[0x200 - 27];    /* pad to one page */
 } __packed;
 
 #define vgtif_offset(x) (offsetof(struct vgt_if, x))
diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c
index 9b37dd1..998cf6a 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.c
+++ b/drivers/gpu/drm/i915/i915_vgpu.c
@@ -145,6 +145,7 @@  static void vgt_deballoon_space(struct i915_ggtt *ggtt,
  */
 void intel_vgt_deballoon(struct i915_ggtt *ggtt)
 {
+	struct drm_i915_private *i915 = ggtt->vm.i915;
 	int i;
 
 	if (!intel_vgpu_active(ggtt->vm.i915))
@@ -154,6 +155,9 @@  void intel_vgt_deballoon(struct i915_ggtt *ggtt)
 
 	for (i = 0; i < 4; i++)
 		vgt_deballoon_space(ggtt, &bl_info.space[i]);
+
+	if (i915->vgpu.pv)
+		free_page((unsigned long)i915->vgpu.pv->shared_page);
 }
 
 static int vgt_balloon_space(struct i915_ggtt *ggtt,
@@ -309,6 +313,81 @@  int intel_vgt_balloon(struct i915_ggtt *ggtt)
  * i915 vgpu PV support for Linux
  */
 
+/*
+ * shared_page setup for VGPU PV features
+ */
+static int intel_vgpu_setup_shared_page(struct drm_i915_private *dev_priv,
+		void __iomem *shared_area)
+{
+	void __iomem *addr;
+	struct i915_virtual_gpu_pv *pv;
+	struct gvt_shared_page *base;
+	u64 gpa;
+	u16 ver_maj, ver_min;
+
+	/* We allocate 1 page shared between guest and GVT for data exchange.
+	 *       ___________.....................
+	 *      |head       |                   |
+	 *      |___________|.................. PAGE/8
+	 *      |PV ELSP                        |
+	 *      :___________....................PAGE/4
+	 *      |desc (SEND)                    |
+	 *      |				|
+	 *      :_______________________________PAGE/2
+	 *      |cmds (SEND)                    |
+	 *      |                               |
+	 *      |                               |
+	 *      |                               |
+	 *      |                               |
+	 *      |_______________________________|
+	 *
+	 * 0 offset: PV version area
+	 * PAGE/8 offset: per engine workload submission data area
+	 * PAGE/4 offset: PV command buffer command descriptor area
+	 * PAGE/2 offset: PV command buffer command data area
+	 */
+
+	base =  (struct gvt_shared_page *)get_zeroed_page(GFP_KERNEL);
+	if (!base) {
+		DRM_INFO("out of memory for shared page memory\n");
+		return -ENOMEM;
+	}
+
+	/* pass guest memory pa address to GVT and then read back to verify */
+	gpa = __pa(base);
+	addr = shared_area + vgtif_offset(shared_page_gpa);
+	writeq(gpa, addr);
+	if (gpa != readq(addr)) {
+		DRM_INFO("vgpu: passed shared_page_gpa failed\n");
+		free_page((unsigned long)base);
+		return -EIO;
+	}
+
+	addr = shared_area + vgtif_offset(g2v_notify);
+	writel(VGT_G2V_SHARED_PAGE_SETUP, addr);
+
+	ver_maj = base->ver_major;
+	ver_min = base->ver_minor;
+	DRM_INFO("vgpu PV ver major %d and minor %d\n", ver_maj, ver_min);
+	if (ver_maj != PV_MAJOR || ver_min != PV_MINOR) {
+		DRM_INFO("vgpu: PV version incompatible\n");
+		free_page((unsigned long)base);
+		return -EIO;
+	}
+
+	pv = kzalloc(sizeof(struct i915_virtual_gpu_pv), GFP_KERNEL);
+	if (!pv) {
+		free_page((unsigned long)base);
+		return -ENOMEM;
+	}
+
+	dev_priv->vgpu.pv = pv;
+	pv->shared_page = base;
+	pv->enabled = true;
+
+	return 0;
+}
+
 /**
  * intel_vgpu_check_pv_caps - detect virtual GPU PV capabilities
  * @dev_priv: i915 device private
@@ -337,5 +416,11 @@  bool intel_vgpu_check_pv_caps(struct drm_i915_private *dev_priv,
 	if (!pvcaps)
 		return false;
 
+	if (intel_vgpu_setup_shared_page(dev_priv, shared_area)) {
+		dev_priv->vgpu.pv_caps = 0;
+		writel(0, shared_area + vgtif_offset(pv_caps));
+		return false;
+	}
+
 	return true;
 }
diff --git a/drivers/gpu/drm/i915/i915_vgpu.h b/drivers/gpu/drm/i915/i915_vgpu.h
index bbe56b5..9b6a8d1 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.h
+++ b/drivers/gpu/drm/i915/i915_vgpu.h
@@ -27,6 +27,23 @@ 
 #include "i915_drv.h"
 #include "i915_pvinfo.h"
 
+#define PV_MAJOR		1
+#define PV_MINOR		0
+
+/*
+ * A shared page(4KB) between gvt and VM, could be allocated by guest driver
+ * or a fixed location in PCI bar 0 region
+ */
+struct gvt_shared_page {
+	u16 ver_major;
+	u16 ver_minor;
+};
+
+struct i915_virtual_gpu_pv {
+	struct gvt_shared_page *shared_page;
+	bool enabled;
+};
+
 void i915_detect_vgpu(struct drm_i915_private *dev_priv);
 
 bool intel_vgpu_has_full_ppgtt(struct drm_i915_private *dev_priv);

Comments

Quoting Xiaolin Zhang (2019-07-23 12:31:57)
> To enable vgpu pv features, we need to setup a shared memory page
> which will be used for data exchange directly accessed between both
> guest and backend i915 driver to avoid emulation trap cost.
> 
> guest i915 will allocate this page memory and then pass it's physical
> address to backend i915 driver through PVINFO register so that backend i915
> driver can access this shared page meory without any trap cost with the
> help form hyperviser's read guest gpa functionality.
> 
> guest i915 will send VGT_G2V_SHARED_PAGE_SETUP notification to host GVT
> once shared memory setup finished.
> 
> the layout of the shared_page also defined as well in this patch which
> is used for pv features implementation.
> 
> v0: RFC.
> v1: addressed RFC comment to move both shared_page_lock and shared_page
> to i915_virtual_gpu structure.
> v2: packed i915_virtual_gpu structure.
> v3: added SHARED_PAGE_SETUP g2v notification for pv shared_page setup
> v4: added intel_vgpu_setup_shared_page() in i915_vgpu_pv.c.
> v5: per engine desc data in shared memory.
> v6: added version support in shared memory (Zhenyu).
> v7: rebase.
> 
> Signed-off-by: Xiaolin Zhang <xiaolin.zhang@intel.com>
> ---
>  drivers/gpu/drm/i915/i915_drv.h    |  4 +-
>  drivers/gpu/drm/i915/i915_pvinfo.h |  5 ++-
>  drivers/gpu/drm/i915/i915_vgpu.c   | 85 ++++++++++++++++++++++++++++++++++++++
>  drivers/gpu/drm/i915/i915_vgpu.h   | 17 ++++++++
>  4 files changed, 109 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
> index fa5dc47..48cf141 100644
> --- a/drivers/gpu/drm/i915/i915_drv.h
> +++ b/drivers/gpu/drm/i915/i915_drv.h
> @@ -1080,7 +1080,9 @@ struct i915_virtual_gpu {
>         bool active;
>         u32 caps;
>         u32 pv_caps;
> -};
> +
> +       struct i915_virtual_gpu_pv *pv;
> +} __packed;
>  
>  /* used in computing the new watermarks state */
>  struct intel_wm_config {
> diff --git a/drivers/gpu/drm/i915/i915_pvinfo.h b/drivers/gpu/drm/i915/i915_pvinfo.h
> index ad398b4..3c63603 100644
> --- a/drivers/gpu/drm/i915/i915_pvinfo.h
> +++ b/drivers/gpu/drm/i915/i915_pvinfo.h
> @@ -48,6 +48,7 @@ enum vgt_g2v_type {
>         VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY,
>         VGT_G2V_EXECLIST_CONTEXT_CREATE,
>         VGT_G2V_EXECLIST_CONTEXT_DESTROY,
> +       VGT_G2V_SHARED_PAGE_SETUP,
>         VGT_G2V_MAX,
>  };
>  
> @@ -112,7 +113,9 @@ struct vgt_if {
>  
>         u32 pv_caps;
>  
> -       u32  rsv7[0x200 - 25];    /* pad to one page */
> +       u64 shared_page_gpa;
> +
> +       u32  rsv7[0x200 - 27];    /* pad to one page */
>  } __packed;
>  
>  #define vgtif_offset(x) (offsetof(struct vgt_if, x))
> diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c
> index 9b37dd1..998cf6a 100644
> --- a/drivers/gpu/drm/i915/i915_vgpu.c
> +++ b/drivers/gpu/drm/i915/i915_vgpu.c
> @@ -145,6 +145,7 @@ static void vgt_deballoon_space(struct i915_ggtt *ggtt,
>   */
>  void intel_vgt_deballoon(struct i915_ggtt *ggtt)

This addendum is nothing to do with the i915_ggtt cleanup.

>  {
> +       struct drm_i915_private *i915 = ggtt->vm.i915;
>         int i;
>  
>         if (!intel_vgpu_active(ggtt->vm.i915))
> @@ -154,6 +155,9 @@ void intel_vgt_deballoon(struct i915_ggtt *ggtt)
>  
>         for (i = 0; i < 4; i++)
>                 vgt_deballoon_space(ggtt, &bl_info.space[i]);
> +
> +       if (i915->vgpu.pv)
> +               free_page((unsigned long)i915->vgpu.pv->shared_page);

__free_page(shared_page);

>  }
>  
>  static int vgt_balloon_space(struct i915_ggtt *ggtt,
> @@ -309,6 +313,81 @@ int intel_vgt_balloon(struct i915_ggtt *ggtt)
>   * i915 vgpu PV support for Linux
>   */
>  
> +/*
> + * shared_page setup for VGPU PV features
> + */
> +static int intel_vgpu_setup_shared_page(struct drm_i915_private *dev_priv,
> +               void __iomem *shared_area)
> +{
> +       void __iomem *addr;
> +       struct i915_virtual_gpu_pv *pv;
> +       struct gvt_shared_page *base;
> +       u64 gpa;
> +       u16 ver_maj, ver_min;
> +
> +       /* We allocate 1 page shared between guest and GVT for data exchange.
> +        *       ___________.....................
> +        *      |head       |                   |
> +        *      |___________|.................. PAGE/8
> +        *      |PV ELSP                        |
> +        *      :___________....................PAGE/4
> +        *      |desc (SEND)                    |
> +        *      |                               |
> +        *      :_______________________________PAGE/2
> +        *      |cmds (SEND)                    |
> +        *      |                               |
> +        *      |                               |
> +        *      |                               |
> +        *      |                               |
> +        *      |_______________________________|
> +        *
> +        * 0 offset: PV version area
> +        * PAGE/8 offset: per engine workload submission data area
> +        * PAGE/4 offset: PV command buffer command descriptor area
> +        * PAGE/2 offset: PV command buffer command data area
> +        */
> +
> +       base =  (struct gvt_shared_page *)get_zeroed_page(GFP_KERNEL);
> +       if (!base) {
> +               DRM_INFO("out of memory for shared page memory\n");

DRM_INFO is for providing the user with useful information; not dropping
debug messages. If you do want to address the user, I'd recommend
dev_info
-Chris
On 07/23/2019 06:37 PM, Chris Wilson wrote:
> Quoting Xiaolin Zhang (2019-07-23 12:31:57)
>> To enable vgpu pv features, we need to setup a shared memory page
>> which will be used for data exchange directly accessed between both
>> guest and backend i915 driver to avoid emulation trap cost.
>>
>> guest i915 will allocate this page memory and then pass it's physical
>> address to backend i915 driver through PVINFO register so that backend i915
>> driver can access this shared page meory without any trap cost with the
>> help form hyperviser's read guest gpa functionality.
>>
>> guest i915 will send VGT_G2V_SHARED_PAGE_SETUP notification to host GVT
>> once shared memory setup finished.
>>
>> the layout of the shared_page also defined as well in this patch which
>> is used for pv features implementation.
>>
>> v0: RFC.
>> v1: addressed RFC comment to move both shared_page_lock and shared_page
>> to i915_virtual_gpu structure.
>> v2: packed i915_virtual_gpu structure.
>> v3: added SHARED_PAGE_SETUP g2v notification for pv shared_page setup
>> v4: added intel_vgpu_setup_shared_page() in i915_vgpu_pv.c.
>> v5: per engine desc data in shared memory.
>> v6: added version support in shared memory (Zhenyu).
>> v7: rebase.
>>
>> Signed-off-by: Xiaolin Zhang <xiaolin.zhang@intel.com>
>> ---
>>  drivers/gpu/drm/i915/i915_drv.h    |  4 +-
>>  drivers/gpu/drm/i915/i915_pvinfo.h |  5 ++-
>>  drivers/gpu/drm/i915/i915_vgpu.c   | 85 ++++++++++++++++++++++++++++++++++++++
>>  drivers/gpu/drm/i915/i915_vgpu.h   | 17 ++++++++
>>  4 files changed, 109 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
>> index fa5dc47..48cf141 100644
>> --- a/drivers/gpu/drm/i915/i915_drv.h
>> +++ b/drivers/gpu/drm/i915/i915_drv.h
>> @@ -1080,7 +1080,9 @@ struct i915_virtual_gpu {
>>         bool active;
>>         u32 caps;
>>         u32 pv_caps;
>> -};
>> +
>> +       struct i915_virtual_gpu_pv *pv;
>> +} __packed;
>>  
>>  /* used in computing the new watermarks state */
>>  struct intel_wm_config {
>> diff --git a/drivers/gpu/drm/i915/i915_pvinfo.h b/drivers/gpu/drm/i915/i915_pvinfo.h
>> index ad398b4..3c63603 100644
>> --- a/drivers/gpu/drm/i915/i915_pvinfo.h
>> +++ b/drivers/gpu/drm/i915/i915_pvinfo.h
>> @@ -48,6 +48,7 @@ enum vgt_g2v_type {
>>         VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY,
>>         VGT_G2V_EXECLIST_CONTEXT_CREATE,
>>         VGT_G2V_EXECLIST_CONTEXT_DESTROY,
>> +       VGT_G2V_SHARED_PAGE_SETUP,
>>         VGT_G2V_MAX,
>>  };
>>  
>> @@ -112,7 +113,9 @@ struct vgt_if {
>>  
>>         u32 pv_caps;
>>  
>> -       u32  rsv7[0x200 - 25];    /* pad to one page */
>> +       u64 shared_page_gpa;
>> +
>> +       u32  rsv7[0x200 - 27];    /* pad to one page */
>>  } __packed;
>>  
>>  #define vgtif_offset(x) (offsetof(struct vgt_if, x))
>> diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c
>> index 9b37dd1..998cf6a 100644
>> --- a/drivers/gpu/drm/i915/i915_vgpu.c
>> +++ b/drivers/gpu/drm/i915/i915_vgpu.c
>> @@ -145,6 +145,7 @@ static void vgt_deballoon_space(struct i915_ggtt *ggtt,
>>   */
>>  void intel_vgt_deballoon(struct i915_ggtt *ggtt)
> This addendum is nothing to do with the i915_ggtt cleanup.
>
>>  {
>> +       struct drm_i915_private *i915 = ggtt->vm.i915;
>>         int i;
>>  
>>         if (!intel_vgpu_active(ggtt->vm.i915))
>> @@ -154,6 +155,9 @@ void intel_vgt_deballoon(struct i915_ggtt *ggtt)
>>  
>>         for (i = 0; i < 4; i++)
>>                 vgt_deballoon_space(ggtt, &bl_info.space[i]);
>> +
>> +       if (i915->vgpu.pv)
>> +               free_page((unsigned long)i915->vgpu.pv->shared_page);
> __free_page(shared_page);
Thanks, will change this and move to another place in next revision.
-BRs, Xiaolin
>
>>  }
>>  
>>  static int vgt_balloon_space(struct i915_ggtt *ggtt,
>> @@ -309,6 +313,81 @@ int intel_vgt_balloon(struct i915_ggtt *ggtt)
>>   * i915 vgpu PV support for Linux
>>   */
>>  
>> +/*
>> + * shared_page setup for VGPU PV features
>> + */
>> +static int intel_vgpu_setup_shared_page(struct drm_i915_private *dev_priv,
>> +               void __iomem *shared_area)
>> +{
>> +       void __iomem *addr;
>> +       struct i915_virtual_gpu_pv *pv;
>> +       struct gvt_shared_page *base;
>> +       u64 gpa;
>> +       u16 ver_maj, ver_min;
>> +
>> +       /* We allocate 1 page shared between guest and GVT for data exchange.
>> +        *       ___________.....................
>> +        *      |head       |                   |
>> +        *      |___________|.................. PAGE/8
>> +        *      |PV ELSP                        |
>> +        *      :___________....................PAGE/4
>> +        *      |desc (SEND)                    |
>> +        *      |                               |
>> +        *      :_______________________________PAGE/2
>> +        *      |cmds (SEND)                    |
>> +        *      |                               |
>> +        *      |                               |
>> +        *      |                               |
>> +        *      |                               |
>> +        *      |_______________________________|
>> +        *
>> +        * 0 offset: PV version area
>> +        * PAGE/8 offset: per engine workload submission data area
>> +        * PAGE/4 offset: PV command buffer command descriptor area
>> +        * PAGE/2 offset: PV command buffer command data area
>> +        */
>> +
>> +       base =  (struct gvt_shared_page *)get_zeroed_page(GFP_KERNEL);
>> +       if (!base) {
>> +               DRM_INFO("out of memory for shared page memory\n");
> DRM_INFO is for providing the user with useful information; not dropping
> debug messages. If you do want to address the user, I'd recommend
> dev_info
> -Chris
thanks, will change this next revision.
-BRs, Xiaolin
> _______________________________________________
> intel-gvt-dev mailing list
> intel-gvt-dev@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/intel-gvt-dev