[RFC,12/29] drm/i915: gvt: vGPU life cycle management

Submitted by Wang, Zhi A on Jan. 28, 2016, 10:21 a.m.

Details

Message ID 1453976511-27322-13-git-send-email-zhi.a.wang@intel.com
State New
Headers show
Series "iGVT-g implementation in i915" ( rev: 1 ) in Intel GFX

Not browsing as part of any series.

Commit Message

Wang, Zhi A Jan. 28, 2016, 10:21 a.m.
This patch introduces vGPU life cycle management framework. vGPU instance
is a collection of virtual GEN hardware status, like virtual CFG/MMIO registers,
how much GGTT memory space/Fence this vGPU owns, etc.

A vGPU instance consists following virtualized/limited resources:

- Configuration space(virtualized)
- MMIO registers(virtualized)
- GGTT memory space(limited)
- GGTT page table(shadowed)
- Fence(limited)

The framework is responsible for createing/destroying a vGPU instance,
allocating/freeing the per-vGPU resource from GVT-g resource allocator,
presenting them in the virtual PVINFO page located in virtual MMIO bar, which
provides the basic foundation blocks to GVT-g CFG/MMIO emulation framework.

A big picture here looks like:

+-----------------------------------------------------------------------+
|                      XEN/KVM Hypervisor                               |
+---------------------------+--------^----------------------------------+
   CFG/MMIO emulate request |        | Emulation is done
    from hypervisor         |        | Return the result to hypervisor
                            |        |
+---------------------------v--------+-----------------------------------+
|                  GVT-g  CFG/MMIO emulation framework                   |
+-----+-----^----------------+-----^---------------------+-------^-------+
      |     |                |     |                     |       |
      |     | *vGPU instance |     |             *vGPU 2 |       |
+-----v-----+----------+-----v-----+-------------+ +-----v-------+-------+
| vConfiguration Space |vGTT/MMIO/Fence registers| |  ...                |
+----------------------+-------------------------+ +---------------------+
                          vGPU life cycle management

Signed-off-by: Zhi Wang <zhi.a.wang@intel.com>
---
 drivers/gpu/drm/i915/gvt/Makefile   |   2 +-
 drivers/gpu/drm/i915/gvt/gvt.h      |  66 ++++++++++
 drivers/gpu/drm/i915/gvt/instance.c | 235 ++++++++++++++++++++++++++++++++++++
 drivers/gpu/drm/i915/gvt/mmio.c     |  32 +++++
 drivers/gpu/drm/i915/gvt/params.c   |   1 +
 drivers/gpu/drm/i915/gvt/params.h   |   1 +
 drivers/gpu/drm/i915/gvt/reg.h      |  29 ++++-
 drivers/gpu/drm/i915/i915_vgpu.h    |   5 +-
 8 files changed, 365 insertions(+), 6 deletions(-)
 create mode 100644 drivers/gpu/drm/i915/gvt/instance.c

Patch hide | download patch | download mbox

diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
index 5d28ed1..f4dcf9a 100644
--- a/drivers/gpu/drm/i915/gvt/Makefile
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -1,4 +1,4 @@ 
-GVT_SOURCE := gvt.o params.o aperture_gm.o mmio.o handlers.o fb_decoder.o
+GVT_SOURCE := gvt.o params.o aperture_gm.o mmio.o handlers.o instance.o fb_decoder.o
 
 ccflags-y			+= -I$(src) -I$(src)/.. -Wall -Werror -Wno-unused-function
 i915_gvt-y			:= $(GVT_SOURCE)
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index 798e216..c58305f 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -75,6 +75,17 @@  struct gvt_gm_node {
 	struct drm_mm_node *high_gm_node;
 };
 
+struct gvt_virtual_mmio_state {
+	void *vreg;
+	void *sreg;
+};
+
+struct gvt_virtual_cfg_state {
+	unsigned char space[GVT_CFG_SPACE_SZ];
+	bool bar_mapped[GVT_BAR_NUM];
+	u64 bar_size[GVT_BAR_NUM];
+};
+
 struct gvt_virtual_gm_state {
 	u64 aperture_base;
 	void *aperture_base_va;
@@ -89,6 +100,8 @@  struct gvt_virtual_gm_state {
 
 struct gvt_virtual_device_state {
 	struct gvt_virtual_gm_state gm;
+	struct gvt_virtual_mmio_state mmio;
+	struct gvt_virtual_cfg_state cfg;
 };
 
 struct vgt_device {
@@ -96,6 +109,7 @@  struct vgt_device {
 	int vm_id;
 	struct pgt_device *pdev;
 	bool warn_untrack;
+	atomic_t active;
 	struct gvt_virtual_device_state state;
 };
 
@@ -359,4 +373,56 @@  extern bool gvt_setup_initial_mmio_state(struct pgt_device *pdev);
 extern void gvt_clean_mmio_emulation_state(struct pgt_device *pdev);
 extern bool gvt_setup_mmio_emulation_state(struct pgt_device *pdev);
 
+static inline void gvt_pci_bar_write_32(struct vgt_device *vgt, uint32_t bar_offset, uint32_t val)
+{
+	uint32_t* cfg_reg;
+
+	/* BAR offset should be 32 bits algiend */
+	cfg_reg = (u32 *)&vgt->state.cfg.space[bar_offset & ~3];
+
+	/* only write the bits 31-4, leave the 3-0 bits unchanged, as they are read-only */
+	*cfg_reg = (val & 0xFFFFFFF0) | (*cfg_reg & 0xF);
+}
+
+static inline int gvt_pci_mmio_is_enabled(struct vgt_device *vgt)
+{
+	return vgt->state.cfg.space[GVT_REG_CFG_COMMAND] &
+		_REGBIT_CFG_COMMAND_MEMORY;
+}
+
+#define __vreg(vgt, off) (*(u32*)(vgt->state.mmio.vreg + off))
+#define __vreg8(vgt, off) (*(u8*)(vgt->state.mmio.vreg + off))
+#define __vreg16(vgt, off) (*(u16*)(vgt->state.mmio.vreg + off))
+#define __vreg64(vgt, off) (*(u64*)(vgt->state.mmio.vreg + off))
+
+#define __sreg(vgt, off) (*(u32*)(vgt->state.mmio.sreg + off))
+#define __sreg8(vgt, off) (*(u8*)(vgt->state.mmio.sreg + off))
+#define __sreg16(vgt, off) (*(u16*)(vgt->state.mmio.sreg + off))
+#define __sreg64(vgt, off) (*(u64*)(vgt->state.mmio.sreg + off))
+
+static inline void gvt_set_instance_online(struct vgt_device *vgt)
+{
+	atomic_set(&vgt->active, 1);
+}
+
+static inline void gvt_set_instance_offline(struct vgt_device *vgt)
+{
+	atomic_set(&vgt->active, 0);
+}
+
+static inline bool gvt_instance_is_online(struct vgt_device *vgt)
+{
+	return atomic_read(&vgt->active);
+}
+
+#define for_each_online_instance(pdev, vgt, id) \
+       idr_for_each_entry(&pdev->instance_idr, vgt, id) \
+               if (gvt_instance_is_online(vgt))
+
+extern void gvt_init_shadow_mmio_register(struct vgt_device *pdev);
+extern void gvt_init_virtual_mmio_register(struct vgt_device *pdev);
+extern struct vgt_device *gvt_create_instance(struct pgt_device *pdev,
+		struct gvt_instance_info *info);
+extern void gvt_destroy_instance(struct vgt_device *vgt);
+
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/instance.c b/drivers/gpu/drm/i915/gvt/instance.c
new file mode 100644
index 0000000..07b797a
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/instance.c
@@ -0,0 +1,235 @@ 
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "gvt.h"
+#include "i915_vgpu.h"
+
+static void destroy_virtual_mmio_state(struct vgt_device *vgt)
+{
+	struct gvt_virtual_device_state *state = &vgt->state;
+
+	if (state->mmio.vreg) {
+		vfree(state->mmio.vreg);
+		state->mmio.vreg = NULL;
+	}
+	if (state->mmio.sreg) {
+		vfree(state->mmio.sreg);
+		state->mmio.sreg = NULL;
+	}
+}
+
+static bool create_virtual_mmio_state(struct vgt_device *vgt)
+{
+	struct pgt_device *pdev = vgt->pdev;
+	struct gvt_virtual_device_state *state = &vgt->state;
+
+	state->mmio.vreg = vzalloc(pdev->mmio_size);
+	state->mmio.sreg = vzalloc(pdev->mmio_size);
+
+	if (state->mmio.vreg == NULL || state->mmio.sreg == NULL ) {
+		gvt_err("fail to allocate memory for virtual states.");
+		goto err;
+	}
+
+	gvt_init_shadow_mmio_register(vgt);
+	gvt_init_virtual_mmio_register(vgt);
+
+	return true;
+err:
+	destroy_virtual_mmio_state(vgt);
+	return false;
+}
+
+static void init_virtual_cfg_space_state(struct vgt_device *vgt,
+	struct gvt_instance_info *info)
+{
+	struct pgt_device *pdev = vgt->pdev;
+	struct gvt_virtual_device_state *state = &vgt->state;
+	int i;
+
+	char *cfg_space;
+	u16 *gmch_ctl;
+
+	cfg_space = state->cfg.space;
+
+	memcpy(cfg_space, pdev->initial_cfg_space, GVT_CFG_SPACE_SZ);
+	cfg_space[GVT_REG_CFG_SPACE_MSAC] = state->cfg.bar_size[1];
+
+	if (info->primary == 0 || ((info->primary == -1) && !gvt.primary)) {
+		cfg_space[GVT_REG_CFG_CLASS_CODE] = GVT_PCI_CLASS_VGA;
+		cfg_space[GVT_REG_CFG_SUB_CLASS_CODE] = GVT_PCI_CLASS_VGA_OTHER;
+		cfg_space[GVT_REG_CFG_CLASS_PROG_IF] = GVT_PCI_CLASS_VGA_OTHER;
+	}
+
+	/* Show guest that there isn't any stolen memory.*/
+	gmch_ctl = (u16 *)(cfg_space + _REG_GMCH_CONTROL);
+	*gmch_ctl &= ~(_REGBIT_BDW_GMCH_GMS_MASK << _REGBIT_BDW_GMCH_GMS_SHIFT);
+
+	gvt_pci_bar_write_32(vgt, GVT_REG_CFG_SPACE_BAR1, phys_aperture_base(pdev));
+
+	cfg_space[GVT_REG_CFG_COMMAND] &= ~(_REGBIT_CFG_COMMAND_IO |
+			_REGBIT_CFG_COMMAND_MEMORY |
+			_REGBIT_CFG_COMMAND_MASTER);
+
+	/* Clear the bar upper 32bit and let hvmloader to assign the new value */
+	memset(&cfg_space[GVT_REG_CFG_SPACE_BAR0 + 4], 0, 4);
+	memset(&cfg_space[GVT_REG_CFG_SPACE_BAR1 + 4], 0, 4);
+
+	state->cfg.bar_size[0] = pdev->bar_size[0];	/* MMIOGTT */
+	state->cfg.bar_size[1] = pdev->bar_size[1];
+	state->cfg.bar_size[2] = pdev->bar_size[2];	/* PIO */
+	state->cfg.bar_size[3] = pdev->bar_size[3];	/* ROM */
+
+	for (i = 0; i < GVT_BAR_NUM; i++)
+		state->cfg.bar_mapped[i] = false;
+}
+
+static void destroy_virtual_gm_state(struct vgt_device *vgt)
+{
+	gvt_free_gm_and_fence_resource(vgt);
+}
+
+static void populate_pvinfo_page(struct vgt_device *vgt)
+{
+	/* setup the ballooning information */
+	__vreg64(vgt, _vgtif_reg(magic)) = VGT_MAGIC;
+	__vreg(vgt, _vgtif_reg(version_major)) = 1;
+	__vreg(vgt, _vgtif_reg(version_minor)) = 0;
+	__vreg(vgt, _vgtif_reg(display_ready)) = 0;
+	__vreg(vgt, _vgtif_reg(vgt_id)) = vgt->id;
+	__vreg(vgt, _vgtif_reg(avail_rs.mappable_gmadr.base)) = gvt_visible_gm_base(vgt);
+	__vreg(vgt, _vgtif_reg(avail_rs.mappable_gmadr.size)) = gvt_aperture_sz(vgt);
+	__vreg(vgt, _vgtif_reg(avail_rs.nonmappable_gmadr.base)) = gvt_hidden_gm_base(vgt);
+	__vreg(vgt, _vgtif_reg(avail_rs.nonmappable_gmadr.size)) = gvt_hidden_gm_sz(vgt);
+
+	__vreg(vgt, _vgtif_reg(avail_rs.fence_num)) = gvt_fence_sz(vgt);
+	gvt_info("filling VGT_PVINFO_PAGE for dom%d:"
+			"   visable_gm_base=0x%llx, size=0x%llx"
+			"   hidden_gm_base=0x%llx, size=0x%llx"
+			"   fence_base=%d, num=%d",
+			vgt->id,
+			gvt_visible_gm_base(vgt), gvt_aperture_sz(vgt),
+			gvt_hidden_gm_base(vgt), gvt_hidden_gm_sz(vgt),
+			gvt_fence_base(vgt), gvt_fence_sz(vgt));
+
+	ASSERT(sizeof(struct vgt_if) == VGT_PVINFO_SIZE);
+}
+
+static bool create_virtual_gm_state(struct vgt_device *vgt,
+		struct gvt_instance_info *info)
+{
+	struct pgt_device *pdev = vgt->pdev;
+	struct gvt_virtual_device_state *state = &vgt->state;
+
+	if (gvt_alloc_gm_and_fence_resource(vgt, info) < 0) {
+		gvt_err("fail to allocate graphics memory and fence");
+		return false;
+	}
+
+	state->gm.aperture_offset = aperture_2_gm(pdev, state->gm.aperture_base);
+	state->gm.aperture_base_va = phys_aperture_vbase(pdev) + state->gm.aperture_offset;
+
+	populate_pvinfo_page(vgt);
+
+	return true;
+}
+
+static void destroy_virtual_device_state(struct vgt_device *vgt)
+{
+	destroy_virtual_mmio_state(vgt);
+	destroy_virtual_gm_state(vgt);
+}
+
+static bool create_virtual_device_state(struct vgt_device *vgt,
+		struct gvt_instance_info *info)
+{
+	if (!create_virtual_mmio_state(vgt))
+		return false;
+
+	if (!create_virtual_gm_state(vgt, info))
+		return false;
+
+	init_virtual_cfg_space_state(vgt, info);
+
+	return true;
+}
+
+void gvt_destroy_instance(struct vgt_device *vgt)
+{
+	struct pgt_device *pdev = vgt->pdev;
+
+	mutex_lock(&pdev->lock);
+	gvt_set_instance_offline(vgt);
+	if (vgt->id != -1)
+		idr_remove(&pdev->instance_idr, vgt->id);
+	mutex_unlock(&pdev->lock);
+
+	hypervisor_hvm_exit(vgt);
+	destroy_virtual_device_state(vgt);
+	vfree(vgt);
+}
+
+struct vgt_device *gvt_create_instance(struct pgt_device *pdev,
+		struct gvt_instance_info *info)
+{
+	struct vgt_device *vgt = NULL;
+	int id;
+
+	gvt_info("vm_id=%d, low_gm_sz=%dMB, high_gm_sz=%dMB, fence_sz=%d",
+		info->domid, info->low_gm_sz, info->high_gm_sz, info->fence_sz);
+
+	vgt = vzalloc(sizeof(*vgt));
+	if (vgt == NULL) {
+		gvt_err("fail to allocate memory for instance.");
+		return NULL;
+	}
+
+	mutex_lock(&pdev->lock);
+
+	gvt_set_instance_offline(vgt);
+	id = idr_alloc(&pdev->instance_idr, vgt, 1, GVT_MAX_VGPU - 1, GFP_KERNEL);
+	if (id < 0) {
+		gvt_err("fail to allocate id for vgt instance.");
+		goto err;
+	}
+
+	mutex_unlock(&pdev->lock);
+
+	vgt->vm_id = info->domid;
+	vgt->id = id;
+	vgt->pdev = pdev;
+
+	if (!create_virtual_device_state(vgt, info))
+		goto err;
+
+	if (hypervisor_hvm_init(vgt) < 0)
+		goto err;
+
+	gvt_set_instance_online(vgt);
+
+	return vgt;
+err:
+	mutex_unlock(&pdev->lock);
+	gvt_destroy_instance(vgt);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
index 0fbabd2..28e1393 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.c
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -288,3 +288,35 @@  err:
 	gvt_clean_mmio_emulation_state(pdev);
 	return false;
 }
+
+void gvt_init_virtual_mmio_register(struct vgt_device *vgt)
+{
+	struct pgt_device *pdev = vgt->pdev;
+	int i;
+
+        for (i = 0; i < pdev->mmio_size; i += sizeof(u32)) {
+                /*
+                 * skip the area of VGT PV INFO PAGE because we need keep
+                 * its content across Dom0 S3.
+                */
+                if (i >= VGT_PVINFO_PAGE &&
+                        i < VGT_PVINFO_PAGE + VGT_PVINFO_SIZE)
+                        continue;
+
+                __vreg(vgt, i) = pdev->initial_mmio_state[REG_INDEX(i)];
+        }
+
+        /* set the bit 0:2 (Thread C-State) to C0
+         * TODO: consider other bit 3:31
+         */
+        __vreg(vgt, _GEN6_GT_THREAD_STATUS_REG) = 0;
+
+        /* set the bit 0:2(Core C-State ) to C0 */
+        __vreg(vgt, _GEN6_GT_CORE_STATUS) = 0;
+}
+
+void gvt_init_shadow_mmio_register(struct vgt_device *vgt)
+{
+	struct gvt_virtual_device_state *state = &vgt->state;
+        memcpy (state->mmio.sreg, vgt->pdev->initial_mmio_state, vgt->pdev->mmio_size);
+}
diff --git a/drivers/gpu/drm/i915/gvt/params.c b/drivers/gpu/drm/i915/gvt/params.c
index 6cd324c..fca49b0 100644
--- a/drivers/gpu/drm/i915/gvt/params.c
+++ b/drivers/gpu/drm/i915/gvt/params.c
@@ -25,6 +25,7 @@ 
 
 struct gvt_kernel_params gvt = {
 	.enable = true,
+	.primary = true,
 	.debug = 0,
 	.dom0_low_gm_sz = 96,
 	.dom0_high_gm_sz = 384,
diff --git a/drivers/gpu/drm/i915/gvt/params.h b/drivers/gpu/drm/i915/gvt/params.h
index 0507870..80255c3 100644
--- a/drivers/gpu/drm/i915/gvt/params.h
+++ b/drivers/gpu/drm/i915/gvt/params.h
@@ -26,6 +26,7 @@ 
 
 struct gvt_kernel_params {
 	bool enable;
+	bool primary;
 	int debug;
 	int dom0_low_gm_sz;
 	int dom0_high_gm_sz;
diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h
index 5682e1c..2edaf7c 100644
--- a/drivers/gpu/drm/i915/gvt/reg.h
+++ b/drivers/gpu/drm/i915/gvt/reg.h
@@ -27,12 +27,35 @@ 
 #define GVT_CFG_SPACE_SZ	256
 #define GVT_BAR_NUM		4
 
-#define GVT_REG_CFG_SPACE_BAR0	0x10
-#define GVT_REG_CFG_SPACE_BAR1	0x18
-#define GVT_REG_CFG_SPACE_BAR2	0x20
+#define GVT_PCI_CLASS_VGA			0x03
+#define GVT_PCI_CLASS_VGA_OTHER			0x80
+
+#define GVT_REG_CFG_VENDOR_ID                   0x00
+#define GVT_REG_CFG_COMMAND                     0x04
+#define _REGBIT_CFG_COMMAND_IO                  (1 << 0)
+#define _REGBIT_CFG_COMMAND_MEMORY              (1 << 1)
+#define _REGBIT_CFG_COMMAND_MASTER              (1 << 2)
+#define GVT_REG_CFG_CLASS_PROG_IF               0x09
+#define GVT_REG_CFG_SUB_CLASS_CODE              0x0A
+#define GVT_REG_CFG_CLASS_CODE                  0x0B
+#define GVT_REG_CFG_SPACE_BAR0                  0x10
+#define GVT_REG_CFG_SPACE_BAR1                  0x18
+#define GVT_REG_CFG_SPACE_BAR2                  0x20
+#define GVT_REG_CFG_SPACE_BAR_ROM               0x30
+#define GVT_REG_CFG_SPACE_MSAC                  0x62
+#define GVT_REG_CFG_SWSCI_TRIGGER               0xE8
+#define _REGBIT_CFG_SWSCI_SCI_SELECT            (1 << 15)
+#define _REGBIT_CFG_SWSCI_SCI_TRIGGER           1
+#define GVT_REG_CFG_OPREGION                    0xFC
+
+#define _REG_GMCH_CONTROL               0x50
+#define    _REGBIT_BDW_GMCH_GMS_SHIFT   8
+#define    _REGBIT_BDW_GMCH_GMS_MASK    0xff
 
 #define _PCH_GMBUS2			0xc5108
 
 #define _GEN6_GDRST			0x941c
+#define _GEN6_GT_THREAD_STATUS_REG	0x13805c
+#define _GEN6_GT_CORE_STATUS		0x138060
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_vgpu.h b/drivers/gpu/drm/i915/i915_vgpu.h
index 21c77a2..b6d9fc7 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.h
+++ b/drivers/gpu/drm/i915/i915_vgpu.h
@@ -163,8 +163,9 @@  struct vgt_if {
 	uint32_t  rsv6[0x200-25];    /* pad to one page */
 } __packed;
 
-#define vgtif_reg(x) \
-	_MMIO((VGT_PVINFO_PAGE + (long)&((struct vgt_if *)NULL)->x))
+#define _vgtif_reg(x) \
+	((VGT_PVINFO_PAGE + (long)&((struct vgt_if *)NULL)->x))
+#define vgtif_reg(x) _MMIO(_vgtif_reg(x))
 
 extern void i915_check_vgpu(struct drm_device *dev);
 extern int intel_vgt_balloon(struct drm_device *dev);