From d32656373857839c82e3941a9e26cf228ca319b3 Mon Sep 17 00:00:00 2001 From: Rhys Kidd Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 001/151] drm/nouveau/therm/gp100: initial implementation of new gp1xx temperature sensor v2: - add nv138 and drop nv13b chipsets (Ilia Mirkin) - refactor out status variable and instead mask tsensor (Ilia Mirkin) - switch SHADOWed state message away from nvkm_error() (Ilia Mirkin) - rename internal temperature variable (Karol Herbst) v3: - use nvkm_trace() for SHADOWed state message (Ben Skeggs) Signed-off-by: Rhys Kidd Signed-off-by: Ben Skeggs --- .../drm/nouveau/include/nvkm/subdev/therm.h | 1 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 6 ++ .../gpu/drm/nouveau/nvkm/subdev/therm/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/therm/base.c | 3 +- .../gpu/drm/nouveau/nvkm/subdev/therm/gp100.c | 56 +++++++++++++++++++ 5 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gp100.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h index 1bfd93b85575d..9841f076da2e5 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h @@ -97,4 +97,5 @@ int gt215_therm_new(struct nvkm_device *, int, struct nvkm_therm **); int gf119_therm_new(struct nvkm_device *, int, struct nvkm_therm **); int gm107_therm_new(struct nvkm_device *, int, struct nvkm_therm **); int gm200_therm_new(struct nvkm_device *, int, struct nvkm_therm **); +int gp100_therm_new(struct nvkm_device *, int, struct nvkm_therm **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index e096a5d9c292f..28fd4fa986351 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2169,6 +2169,7 @@ nv130_chipset = { .ltc = gp100_ltc_new, .mc = gp100_mc_new, .mmu = gf100_mmu_new, + .therm = gp100_therm_new, .secboot = gm200_secboot_new, .pci = gp100_pci_new, .pmu = gp100_pmu_new, @@ -2203,6 +2204,7 @@ nv132_chipset = { .ltc = gp100_ltc_new, .mc = gp100_mc_new, .mmu = gf100_mmu_new, + .therm = gp100_therm_new, .secboot = gp102_secboot_new, .pci = gp100_pci_new, .pmu = gp102_pmu_new, @@ -2237,6 +2239,7 @@ nv134_chipset = { .ltc = gp100_ltc_new, .mc = gp100_mc_new, .mmu = gf100_mmu_new, + .therm = gp100_therm_new, .secboot = gp102_secboot_new, .pci = gp100_pci_new, .pmu = gp102_pmu_new, @@ -2271,6 +2274,7 @@ nv136_chipset = { .ltc = gp100_ltc_new, .mc = gp100_mc_new, .mmu = gf100_mmu_new, + .therm = gp100_therm_new, .secboot = gp102_secboot_new, .pci = gp100_pci_new, .pmu = gp102_pmu_new, @@ -2305,6 +2309,7 @@ nv137_chipset = { .ltc = gp100_ltc_new, .mc = gp100_mc_new, .mmu = gf100_mmu_new, + .therm = gp100_therm_new, .secboot = gp102_secboot_new, .pci = gp100_pci_new, .pmu = gp102_pmu_new, @@ -2339,6 +2344,7 @@ nv138_chipset = { .ltc = gp100_ltc_new, .mc = gp100_mc_new, .mmu = gf100_mmu_new, + .therm = gp100_therm_new, .pci = gp100_pci_new, .pmu = gp102_pmu_new, .timer = gk20a_timer_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild index 2bafcc1d18188..7ba56b12badd8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild @@ -12,3 +12,4 @@ nvkm-y += nvkm/subdev/therm/gt215.o nvkm-y += nvkm/subdev/therm/gf119.o nvkm-y += nvkm/subdev/therm/gm107.o nvkm-y += nvkm/subdev/therm/gm200.o +nvkm-y += nvkm/subdev/therm/gp100.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c index 952a7cb0a59a0..f27fc6d0d4c65 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c @@ -341,7 +341,8 @@ nvkm_therm_init(struct nvkm_subdev *subdev) { struct nvkm_therm *therm = nvkm_therm(subdev); - therm->func->init(therm); + if (therm->func->init) + therm->func->init(therm); if (therm->suspend >= 0) { /* restore the pwm value only when on manual or auto mode */ diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gp100.c new file mode 100644 index 0000000000000..9f0dea3f61dc4 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gp100.c @@ -0,0 +1,56 @@ +/* + * Copyright 2017 Rhys Kidd + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Rhys Kidd + */ +#include "priv.h" + +static int +gp100_temp_get(struct nvkm_therm *therm) +{ + struct nvkm_device *device = therm->subdev.device; + struct nvkm_subdev *subdev = &therm->subdev; + u32 tsensor = nvkm_rd32(device, 0x020460); + u32 inttemp = (tsensor & 0x0001fff8); + + /* device SHADOWed */ + if (tsensor & 0x40000000) + nvkm_trace(subdev, "reading temperature from SHADOWed sensor\n"); + + /* device valid */ + if (tsensor & 0x20000000) + return (inttemp >> 8); + else + return -ENODEV; +} + +static const struct nvkm_therm_func +gp100_therm = { + .temp_get = gp100_temp_get, + .program_alarms = nvkm_therm_program_alarms_polling, +}; + +int +gp100_therm_new(struct nvkm_device *device, int index, + struct nvkm_therm **ptherm) +{ + return nvkm_therm_new_(&gp100_therm, device, index, ptherm); +} From 451b58d2d03d2033f00d346fb0b0335e536ae5ae Mon Sep 17 00:00:00 2001 From: Rhys Kidd Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 002/151] drm/nouveau: Document nouveau support for Tegra in DRIVER_DESC nouveau supports the Tegra K1 and higher after the SoC-based GPUs converged with the main GeForce GPU families. v2: - Qualify that support is Tegra K1+ (Martin Peres) Signed-off-by: Rhys Kidd Reviewed-by: Martin Peres Acked-by: Pierre Moreau Acked-by: Thierry Reding Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_drv.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 822fe1d4d35e1..ec60d52790e33 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -5,7 +5,7 @@ #define DRIVER_EMAIL "nouveau@lists.freedesktop.org" #define DRIVER_NAME "nouveau" -#define DRIVER_DESC "nVidia Riva/TNT/GeForce/Quadro/Tesla" +#define DRIVER_DESC "nVidia Riva/TNT/GeForce/Quadro/Tesla/Tegra K1+" #define DRIVER_DATE "20120801" #define DRIVER_MAJOR 1 From f5a5b5232b899c06626960b601d1943cc2fb21d9 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 003/151] remove some useless semicolons Reported-by: Dave Airlie Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c | 4 ++-- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c index 189ed80e21ffb..30483c5d27c3b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c @@ -216,7 +216,7 @@ nvkm_device_tegra_fini(struct nvkm_device *device, bool suspend) if (tdev->irq) { free_irq(tdev->irq, tdev); tdev->irq = 0; - }; + } } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c index 23caef8df17fb..73e463ed55c35 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c @@ -99,7 +99,7 @@ nvbios_iccsense_parse(struct nvkm_bios *bios, struct nvbios_iccsense *iccsense) rail->extdev_id = nvbios_rd08(bios, entry + 0x1); res_start = 0x5; break; - }; + } if (nvbios_extdev_parse(bios, rail->extdev_id, &extdev)) continue; @@ -115,7 +115,7 @@ nvbios_iccsense_parse(struct nvkm_bios *bios, struct nvbios_iccsense *iccsense) default: rail->resistor_count = 0; break; - }; + } for (r = 0; r < rail->resistor_count; ++r) { rail->resistors[r].mohm = nvbios_rd08(bios, entry + res_start + r * 2); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 455da298227f6..f2b1a3e75f176 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -66,7 +66,7 @@ nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node) delta += (u64)len << vma->node->type; } r = r->next; - }; + } mmu->func->flush(vm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c index a4cb82495cee3..b1b1f3626b962 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c @@ -87,7 +87,7 @@ nvkm_pci_fini(struct nvkm_subdev *subdev, bool suspend) if (pci->irq >= 0) { free_irq(pci->irq, pci); pci->irq = -1; - }; + } if (pci->agp.bridge) nvkm_agp_fini(pci); From 73cef6cee71b7a1029e4951a97095c2e4fee4be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lefaure?= Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 004/151] drm/nouveau/bios/init: use ARRAY_SIZE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using the ARRAY_SIZE macro improves the readability of the code. Also, it is useless to re-invent it. Found with Coccinelle with the following semantic patch: @r depends on (org || report)@ type T; T[] E; position p; @@ ( (sizeof(E)@p /sizeof(*E)) | (sizeof(E)@p /sizeof(E[...])) | (sizeof(E)@p /sizeof(T)) ) Reviewed-by: Thierry Reding Signed-off-by: Jérémy Lefaure Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c index b58ee99f7bfc8..9cc10e438b3de 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c @@ -36,6 +36,8 @@ #include #include +#include + #define bioslog(lvl, fmt, args...) do { \ nvkm_printk(init->subdev, lvl, info, "0x%08x[%c]: "fmt, \ init->offset, init_exec(init) ? \ @@ -2271,8 +2273,6 @@ static struct nvbios_init_opcode { [0xaa] = { init_reserved }, }; -#define init_opcode_nr (sizeof(init_opcode) / sizeof(init_opcode[0])) - int nvbios_exec(struct nvbios_init *init) { @@ -2281,7 +2281,8 @@ nvbios_exec(struct nvbios_init *init) init->nested++; while (init->offset) { u8 opcode = nvbios_rd08(bios, init->offset); - if (opcode >= init_opcode_nr || !init_opcode[opcode].exec) { + if (opcode >= ARRAY_SIZE(init_opcode) || + !init_opcode[opcode].exec) { error("unknown opcode 0x%02x\n", opcode); return -EINVAL; } From 3a0bc8cb9b102229ad26ae3788cef8c6c5273be1 Mon Sep 17 00:00:00 2001 From: Ilia Mirkin Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 005/151] drm/nouveau/kms: add 8.1Gbps DP link rate This was already done in dcb.c inside nvkm, but the other parser did not get the update. Signed-off-by: Ilia Mirkin Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bios.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c index dd6fba55ad5d0..c4ef3a0a737ed 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bios.c +++ b/drivers/gpu/drm/nouveau/nouveau_bios.c @@ -1478,9 +1478,13 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb, case 1: entry->dpconf.link_bw = 270000; break; - default: + case 2: entry->dpconf.link_bw = 540000; break; + case 3: + default: + entry->dpconf.link_bw = 810000; + break; } switch ((conf & 0x0f000000) >> 24) { case 0xf: From 9551efcf7664b73d881de7eb88e4626c9561b624 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 006/151] drm/nouveau/kms/nv50: prevent oops in failure paths Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nv50_display.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c index e4751f92b342d..9cd5a53f85fa7 100644 --- a/drivers/gpu/drm/nouveau/nv50_display.c +++ b/drivers/gpu/drm/nouveau/nv50_display.c @@ -510,6 +510,7 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp, int ret; mutex_init(&dmac->lock); + INIT_LIST_HEAD(&dmac->ctxdma); dmac->ptr = dma_alloc_coherent(nvxx_device(device)->dev, PAGE_SIZE, &dmac->handle, GFP_KERNEL); @@ -556,7 +557,6 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp, if (ret) return ret; - INIT_LIST_HEAD(&dmac->ctxdma); return ret; } From bbb10e639819e6a14e487ce14b3d8575767573b8 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 007/151] drm/nouveau: fix handling of GART OOM on pre-NV50 chipsets The correct thing to do on OOM is to return 0 and set mm_node to NULL, otherwise TTM will assume some other kind of error, and not attempt to evict other buffers to make space. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_ttm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index b0ad7fcefcf5b..ffc3458c89de8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -243,6 +243,10 @@ nv04_gart_manager_new(struct ttm_mem_type_manager *man, ret = nvkm_vm_get(man->priv, reg->num_pages << 12, node->page_shift, NV_MEM_ACCESS_RW, &node->vma[0]); if (ret) { + if (ret == -ENOSPC) { + reg->mm_node = NULL; + ret = 0; + } kfree(node); return ret; } From 1f474be9a89e7848649034c8854c55996bb29459 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 008/151] drm/nouveau/core/object: separate oclass data out into its own header Want to be able to include this from core/device.h without pulling in core/object.h. Signed-off-by: Ben Skeggs --- .../drm/nouveau/include/nvkm/core/object.h | 28 +---------------- .../drm/nouveau/include/nvkm/core/oclass.h | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 27 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvkm/core/oclass.h diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/object.h b/drivers/gpu/drm/nouveau/include/nvkm/core/object.h index 96dda350ada31..5a092ffb5abf8 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/object.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/object.h @@ -1,10 +1,8 @@ #ifndef __NVKM_OBJECT_H__ #define __NVKM_OBJECT_H__ -#include -#include +#include struct nvkm_event; struct nvkm_gpuobj; -struct nvkm_oclass; struct nvkm_object { const struct nvkm_object_func *func; @@ -66,28 +64,4 @@ bool nvkm_object_insert(struct nvkm_object *); void nvkm_object_remove(struct nvkm_object *); struct nvkm_object *nvkm_object_search(struct nvkm_client *, u64 object, const struct nvkm_object_func *); - -struct nvkm_sclass { - int minver; - int maxver; - s32 oclass; - const struct nvkm_object_func *func; - int (*ctor)(const struct nvkm_oclass *, void *data, u32 size, - struct nvkm_object **); -}; - -struct nvkm_oclass { - int (*ctor)(const struct nvkm_oclass *, void *data, u32 size, - struct nvkm_object **); - struct nvkm_sclass base; - const void *priv; - const void *engn; - u32 handle; - u8 route; - u64 token; - u64 object; - struct nvkm_client *client; - struct nvkm_object *parent; - struct nvkm_engine *engine; -}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/oclass.h b/drivers/gpu/drm/nouveau/include/nvkm/core/oclass.h new file mode 100644 index 0000000000000..8e1b945d38f38 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/oclass.h @@ -0,0 +1,31 @@ +#ifndef __NVKM_OCLASS_H__ +#define __NVKM_OCLASS_H__ +#include +#include +struct nvkm_oclass; +struct nvkm_object; + +struct nvkm_sclass { + int minver; + int maxver; + s32 oclass; + const struct nvkm_object_func *func; + int (*ctor)(const struct nvkm_oclass *, void *data, u32 size, + struct nvkm_object **); +}; + +struct nvkm_oclass { + int (*ctor)(const struct nvkm_oclass *, void *data, u32 size, + struct nvkm_object **); + struct nvkm_sclass base; + const void *priv; + const void *engn; + u32 handle; + u8 route; + u64 token; + u64 object; + struct nvkm_client *client; + struct nvkm_object *parent; + struct nvkm_engine *engine; +}; +#endif From 01326050391ceee2cf1b6c91c108414a35f40861 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 009/151] drm/nouveau/core/object: allow arguments to be passed to map function MMU will be needing this to specify kind info on BAR mappings. We have no userspace currently using these interfaces, so break the ABI instead of supporting both. NVIF version bump so any future use can be guarded. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/dispnv04/disp.c | 2 +- drivers/gpu/drm/nouveau/include/nvif/ioctl.h | 10 ++- drivers/gpu/drm/nouveau/include/nvif/object.h | 7 +- .../drm/nouveau/include/nvkm/core/object.h | 11 ++- drivers/gpu/drm/nouveau/nouveau_chan.c | 2 +- drivers/gpu/drm/nouveau/nv50_display.c | 2 +- drivers/gpu/drm/nouveau/nvif/object.c | 87 +++++++++++++------ drivers/gpu/drm/nouveau/nvkm/core/ioctl.c | 12 ++- drivers/gpu/drm/nouveau/nvkm/core/object.c | 5 +- drivers/gpu/drm/nouveau/nvkm/core/oproxy.c | 6 +- .../gpu/drm/nouveau/nvkm/engine/device/user.c | 4 +- .../drm/nouveau/nvkm/engine/disp/channv50.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 4 +- 13 files changed, 108 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.c b/drivers/gpu/drm/nouveau/dispnv04/disp.c index 5b9d549aa791f..501d2d290e9c6 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv04/disp.c @@ -48,7 +48,7 @@ nv04_display_create(struct drm_device *dev) if (!disp) return -ENOMEM; - nvif_object_map(&drm->client.device.object); + nvif_object_map(&drm->client.device.object, NULL, 0); nouveau_display(dev)->priv = disp; nouveau_display(dev)->dtor = nv04_display_destroy; diff --git a/drivers/gpu/drm/nouveau/include/nvif/ioctl.h b/drivers/gpu/drm/nouveau/include/nvif/ioctl.h index c5f5eb83a5944..1886366457f18 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/ioctl.h +++ b/drivers/gpu/drm/nouveau/include/nvif/ioctl.h @@ -1,7 +1,7 @@ #ifndef __NVIF_IOCTL_H__ #define __NVIF_IOCTL_H__ -#define NVIF_VERSION_LATEST 0x0000000000000000ULL +#define NVIF_VERSION_LATEST 0x0000000000000100ULL struct nvif_ioctl_v0 { __u8 version; @@ -83,9 +83,13 @@ struct nvif_ioctl_wr_v0 { struct nvif_ioctl_map_v0 { /* nvif_ioctl ... */ __u8 version; - __u8 pad01[3]; - __u32 length; +#define NVIF_IOCTL_MAP_V0_IO 0x00 +#define NVIF_IOCTL_MAP_V0_VA 0x01 + __u8 type; + __u8 pad02[6]; __u64 handle; + __u64 length; + __u8 data[]; }; struct nvif_ioctl_unmap { diff --git a/drivers/gpu/drm/nouveau/include/nvif/object.h b/drivers/gpu/drm/nouveau/include/nvif/object.h index 9e58b305b0203..0b54261bdefe5 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/object.h +++ b/drivers/gpu/drm/nouveau/include/nvif/object.h @@ -16,7 +16,7 @@ struct nvif_object { void *priv; /*XXX: hack */ struct { void __iomem *ptr; - u32 size; + u64 size; } map; }; @@ -29,7 +29,10 @@ void nvif_object_sclass_put(struct nvif_sclass **); u32 nvif_object_rd(struct nvif_object *, int, u64); void nvif_object_wr(struct nvif_object *, int, u64, u32); int nvif_object_mthd(struct nvif_object *, u32, void *, u32); -int nvif_object_map(struct nvif_object *); +int nvif_object_map_handle(struct nvif_object *, void *, u32, + u64 *handle, u64 *length); +void nvif_object_unmap_handle(struct nvif_object *); +int nvif_object_map(struct nvif_object *, void *, u32); void nvif_object_unmap(struct nvif_object *); #define nvif_handle(a) (unsigned long)(void *)(a) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/object.h b/drivers/gpu/drm/nouveau/include/nvkm/core/object.h index 5a092ffb5abf8..4797577d52ce3 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/object.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/object.h @@ -19,13 +19,19 @@ struct nvkm_object { struct rb_node node; }; +enum nvkm_object_map { + NVKM_OBJECT_MAP_IO, + NVKM_OBJECT_MAP_VA +}; + struct nvkm_object_func { void *(*dtor)(struct nvkm_object *); int (*init)(struct nvkm_object *); int (*fini)(struct nvkm_object *, bool suspend); int (*mthd)(struct nvkm_object *, u32 mthd, void *data, u32 size); int (*ntfy)(struct nvkm_object *, u32 mthd, struct nvkm_event **); - int (*map)(struct nvkm_object *, u64 *addr, u32 *size); + int (*map)(struct nvkm_object *, void *argv, u32 argc, + enum nvkm_object_map *, u64 *addr, u64 *size); int (*rd08)(struct nvkm_object *, u64 addr, u8 *data); int (*rd16)(struct nvkm_object *, u64 addr, u16 *data); int (*rd32)(struct nvkm_object *, u64 addr, u32 *data); @@ -50,7 +56,8 @@ int nvkm_object_init(struct nvkm_object *); int nvkm_object_fini(struct nvkm_object *, bool suspend); int nvkm_object_mthd(struct nvkm_object *, u32 mthd, void *data, u32 size); int nvkm_object_ntfy(struct nvkm_object *, u32 mthd, struct nvkm_event **); -int nvkm_object_map(struct nvkm_object *, u64 *addr, u32 *size); +int nvkm_object_map(struct nvkm_object *, void *argv, u32 argc, + enum nvkm_object_map *, u64 *addr, u64 *size); int nvkm_object_rd08(struct nvkm_object *, u64 addr, u8 *data); int nvkm_object_rd16(struct nvkm_object *, u64 addr, u16 *data); int nvkm_object_rd32(struct nvkm_object *, u64 addr, u32 *data); diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index dbc41fa86ee8b..4af09e9be9304 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -318,7 +318,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) struct nv_dma_v0 args = {}; int ret, i; - nvif_object_map(&chan->user); + nvif_object_map(&chan->user, NULL, 0); if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO) { ret = nvif_notify_init(&chan->user, nouveau_channel_killed, diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c index 9cd5a53f85fa7..1f3872f438c97 100644 --- a/drivers/gpu/drm/nouveau/nv50_display.c +++ b/drivers/gpu/drm/nouveau/nv50_display.c @@ -318,7 +318,7 @@ nv50_chan_create(struct nvif_device *device, struct nvif_object *disp, ret = nvif_object_init(disp, 0, oclass[0], data, size, &chan->user); if (ret == 0) - nvif_object_map(&chan->user); + nvif_object_map(&chan->user, NULL, 0); nvif_object_sclass_put(&sclass); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvif/object.c b/drivers/gpu/drm/nouveau/nvif/object.c index c3fb6a20f567d..40adfe9b334b3 100644 --- a/drivers/gpu/drm/nouveau/nvif/object.c +++ b/drivers/gpu/drm/nouveau/nvif/object.c @@ -166,46 +166,77 @@ nvif_object_mthd(struct nvif_object *object, u32 mthd, void *data, u32 size) } void -nvif_object_unmap(struct nvif_object *object) +nvif_object_unmap_handle(struct nvif_object *object) +{ + struct { + struct nvif_ioctl_v0 ioctl; + struct nvif_ioctl_unmap unmap; + } args = { + .ioctl.type = NVIF_IOCTL_V0_UNMAP, + }; + + nvif_object_ioctl(object, &args, sizeof(args), NULL); +} + +int +nvif_object_map_handle(struct nvif_object *object, void *argv, u32 argc, + u64 *handle, u64 *length) { - if (object->map.size) { - struct nvif_client *client = object->client; - struct { - struct nvif_ioctl_v0 ioctl; - struct nvif_ioctl_unmap unmap; - } args = { - .ioctl.type = NVIF_IOCTL_V0_UNMAP, - }; + struct { + struct nvif_ioctl_v0 ioctl; + struct nvif_ioctl_map_v0 map; + } *args; + u32 argn = sizeof(*args) + argc; + int ret, maptype; + + if (!(args = kzalloc(argn, GFP_KERNEL))) + return -ENOMEM; + args->ioctl.type = NVIF_IOCTL_V0_MAP; + memcpy(args->map.data, argv, argc); - if (object->map.ptr) { + ret = nvif_object_ioctl(object, args, argn, NULL); + *handle = args->map.handle; + *length = args->map.length; + maptype = args->map.type; + kfree(args); + return ret ? ret : (maptype == NVIF_IOCTL_MAP_V0_IO); +} + +void +nvif_object_unmap(struct nvif_object *object) +{ + struct nvif_client *client = object->client; + if (object->map.ptr) { + if (object->map.size) { client->driver->unmap(client, object->map.ptr, object->map.size); - object->map.ptr = NULL; + object->map.size = 0; } - - nvif_object_ioctl(object, &args, sizeof(args), NULL); - object->map.size = 0; + object->map.ptr = NULL; + nvif_object_unmap_handle(object); } } int -nvif_object_map(struct nvif_object *object) +nvif_object_map(struct nvif_object *object, void *argv, u32 argc) { struct nvif_client *client = object->client; - struct { - struct nvif_ioctl_v0 ioctl; - struct nvif_ioctl_map_v0 map; - } args = { - .ioctl.type = NVIF_IOCTL_V0_MAP, - }; - int ret = nvif_object_ioctl(object, &args, sizeof(args), NULL); - if (ret == 0) { - object->map.size = args.map.length; - object->map.ptr = client->driver->map(client, args.map.handle, - object->map.size); - if (ret = -ENOMEM, object->map.ptr) + u64 handle, length; + int ret = nvif_object_map_handle(object, argv, argc, &handle, &length); + if (ret >= 0) { + if (ret) { + object->map.ptr = client->driver->map(client, + handle, + length); + if (ret = -ENOMEM, object->map.ptr) { + object->map.size = length; + return 0; + } + } else { + object->map.ptr = (void *)(unsigned long)handle; return 0; - nvif_object_unmap(object); + } + nvif_object_unmap_handle(object); } return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c index be19bbe56bba8..bf2507f17baa5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c @@ -257,13 +257,19 @@ nvkm_ioctl_map(struct nvkm_client *client, union { struct nvif_ioctl_map_v0 v0; } *args = data; + enum nvkm_object_map type; int ret = -ENOSYS; nvif_ioctl(object, "map size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { + if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) { nvif_ioctl(object, "map vers %d\n", args->v0.version); - ret = nvkm_object_map(object, &args->v0.handle, - &args->v0.length); + ret = nvkm_object_map(object, data, size, &type, + &args->v0.handle, + &args->v0.length); + if (type == NVKM_OBJECT_MAP_IO) + args->v0.type = NVIF_IOCTL_MAP_V0_IO; + else + args->v0.type = NVIF_IOCTL_MAP_V0_VA; } return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/core/object.c b/drivers/gpu/drm/nouveau/nvkm/core/object.c index acd76fd4f6d85..516e73a526650 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/object.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/object.c @@ -102,10 +102,11 @@ nvkm_object_ntfy(struct nvkm_object *object, u32 mthd, } int -nvkm_object_map(struct nvkm_object *object, u64 *addr, u32 *size) +nvkm_object_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) { if (likely(object->func->map)) - return object->func->map(object, addr, size); + return object->func->map(object, argv, argc, type, addr, size); return -ENODEV; } diff --git a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c index e31a0479add0d..e7dd15b977217 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c @@ -37,9 +37,11 @@ nvkm_oproxy_ntfy(struct nvkm_object *object, u32 mthd, } static int -nvkm_oproxy_map(struct nvkm_object *object, u64 *addr, u32 *size) +nvkm_oproxy_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) { - return nvkm_object_map(nvkm_oproxy(object)->object, addr, size); + struct nvkm_oproxy *oproxy = nvkm_oproxy(object); + return nvkm_object_map(oproxy->object, argv, argc, type, addr, size); } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c index 513ee6b79553d..e9b90e40f2746 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c @@ -206,10 +206,12 @@ nvkm_udevice_wr32(struct nvkm_object *object, u64 addr, u32 data) } static int -nvkm_udevice_map(struct nvkm_object *object, u64 *addr, u32 *size) +nvkm_udevice_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) { struct nvkm_udevice *udev = nvkm_udevice(object); struct nvkm_device *device = udev->device; + *type = NVKM_OBJECT_MAP_IO; *addr = device->func->resource_addr(device, 0); *size = device->func->resource_size(device, 0); return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c index 0c0310498afdb..723dcbde2ac2f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c @@ -191,11 +191,13 @@ nv50_disp_chan_ntfy(struct nvkm_object *object, u32 type, } static int -nv50_disp_chan_map(struct nvkm_object *object, u64 *addr, u32 *size) +nv50_disp_chan_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) { struct nv50_disp_chan *chan = nv50_disp_chan(object); struct nv50_disp *disp = chan->root->disp; struct nvkm_device *device = disp->base.engine.subdev.device; + *type = NVKM_OBJECT_MAP_IO; *addr = device->func->resource_addr(device, 0) + 0x640000 + (chan->chid.user * 0x1000); *size = 0x001000; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index fab760ae922fb..7aea0a8692aee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -253,9 +253,11 @@ nvkm_fifo_chan_ntfy(struct nvkm_object *object, u32 type, } static int -nvkm_fifo_chan_map(struct nvkm_object *object, u64 *addr, u32 *size) +nvkm_fifo_chan_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) { struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); + *type = NVKM_OBJECT_MAP_IO; *addr = chan->addr; *size = chan->size; return 0; From 8e0042d505e86299544cf000e26408bdd252e36b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 010/151] drm/nouveau/core/object: plumb the unmap ioctl through MMU will be using this for BAR mappings. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/core/object.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/core/ioctl.c | 1 + drivers/gpu/drm/nouveau/nvkm/core/object.c | 9 +++++++++ drivers/gpu/drm/nouveau/nvkm/core/oproxy.c | 7 +++++++ 4 files changed, 19 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/object.h b/drivers/gpu/drm/nouveau/include/nvkm/core/object.h index 4797577d52ce3..916a4b76d430b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/object.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/object.h @@ -32,6 +32,7 @@ struct nvkm_object_func { int (*ntfy)(struct nvkm_object *, u32 mthd, struct nvkm_event **); int (*map)(struct nvkm_object *, void *argv, u32 argc, enum nvkm_object_map *, u64 *addr, u64 *size); + int (*unmap)(struct nvkm_object *); int (*rd08)(struct nvkm_object *, u64 addr, u8 *data); int (*rd16)(struct nvkm_object *, u64 addr, u16 *data); int (*rd32)(struct nvkm_object *, u64 addr, u32 *data); @@ -58,6 +59,7 @@ int nvkm_object_mthd(struct nvkm_object *, u32 mthd, void *data, u32 size); int nvkm_object_ntfy(struct nvkm_object *, u32 mthd, struct nvkm_event **); int nvkm_object_map(struct nvkm_object *, void *argv, u32 argc, enum nvkm_object_map *, u64 *addr, u64 *size); +int nvkm_object_unmap(struct nvkm_object *); int nvkm_object_rd08(struct nvkm_object *, u64 addr, u8 *data); int nvkm_object_rd16(struct nvkm_object *, u64 addr, u16 *data); int nvkm_object_rd32(struct nvkm_object *, u64 addr, u32 *data); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c index bf2507f17baa5..a28018ee5b197 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c @@ -287,6 +287,7 @@ nvkm_ioctl_unmap(struct nvkm_client *client, nvif_ioctl(object, "unmap size %d\n", size); if (!(ret = nvif_unvers(ret, &data, &size, args->none))) { nvif_ioctl(object, "unmap\n"); + ret = nvkm_object_unmap(object); } return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/core/object.c b/drivers/gpu/drm/nouveau/nvkm/core/object.c index 516e73a526650..301a5e5b5f7f9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/object.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/object.c @@ -110,6 +110,14 @@ nvkm_object_map(struct nvkm_object *object, void *argv, u32 argc, return -ENODEV; } +int +nvkm_object_unmap(struct nvkm_object *object) +{ + if (likely(object->func->unmap)) + return object->func->unmap(object); + return -ENODEV; +} + int nvkm_object_rd08(struct nvkm_object *object, u64 addr, u8 *data) { @@ -260,6 +268,7 @@ nvkm_object_dtor(struct nvkm_object *object) } nvif_debug(object, "destroy running...\n"); + nvkm_object_unmap(object); if (object->func->dtor) data = object->func->dtor(object); nvkm_engine_unref(&object->engine); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c index e7dd15b977217..16299837a296c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c @@ -44,6 +44,12 @@ nvkm_oproxy_map(struct nvkm_object *object, void *argv, u32 argc, return nvkm_object_map(oproxy->object, argv, argc, type, addr, size); } +static int +nvkm_oproxy_unmap(struct nvkm_object *object) +{ + return nvkm_object_unmap(nvkm_oproxy(object)->object); +} + static int nvkm_oproxy_rd08(struct nvkm_object *object, u64 addr, u8 *data) { @@ -173,6 +179,7 @@ nvkm_oproxy_func = { .mthd = nvkm_oproxy_mthd, .ntfy = nvkm_oproxy_ntfy, .map = nvkm_oproxy_map, + .unmap = nvkm_oproxy_unmap, .rd08 = nvkm_oproxy_rd08, .rd16 = nvkm_oproxy_rd16, .rd32 = nvkm_oproxy_rd32, From 153b642fcb8ce912575e594f9af079d17757a6bd Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 011/151] drm/nouveau/core/gpuobj: remove embedded struct nvkm_object nvkm_gpuobj hasn't subclassed nvkm_object in a long time. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h | 2 -- drivers/gpu/drm/nouveau/include/nvkm/core/ramht.h | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h b/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h index c23da4f059290..1d7ccde7cde8e 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h @@ -1,6 +1,5 @@ #ifndef __NVKM_GPUOBJ_H__ #define __NVKM_GPUOBJ_H__ -#include #include #include struct nvkm_vma; @@ -10,7 +9,6 @@ struct nvkm_vm; #define NVOBJ_FLAG_HEAP 0x00000004 struct nvkm_gpuobj { - struct nvkm_object object; const struct nvkm_gpuobj_func *func; struct nvkm_gpuobj *parent; struct nvkm_memory *memory; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/ramht.h b/drivers/gpu/drm/nouveau/include/nvkm/core/ramht.h index 5ee6298991e21..8a48ca67f60db 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/ramht.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/ramht.h @@ -1,6 +1,7 @@ #ifndef __NVKM_RAMHT_H__ #define __NVKM_RAMHT_H__ #include +struct nvkm_object; struct nvkm_ramht_data { struct nvkm_gpuobj *inst; From 82be74ee3baafff58d6b1f40ac7687c32a74d93a Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 012/151] drm/nouveau/core/subdev: compile out messages for unwanted debug levels Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h index ca9ed3d68f44e..a6c21be7537f7 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h @@ -33,7 +33,7 @@ void nvkm_subdev_intr(struct nvkm_subdev *); /* subdev logging */ #define nvkm_printk_(s,l,p,f,a...) do { \ const struct nvkm_subdev *_subdev = (s); \ - if (_subdev->debug >= (l)) { \ + if (CONFIG_NOUVEAU_DEBUG >= (l) && _subdev->debug >= (l)) { \ dev_##p(_subdev->device->dev, "%s: "f, \ nvkm_subdev_name[_subdev->index], ##a); \ } \ From 4246b92cf9fb32da8d8b060c92d8302797c6fbea Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 013/151] drm/nouveau/core/device: remove object include to prevent unnecessary rebuilds nvkm_device hasn't subclassed nvkm_object in a long time. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/core/device.h | 2 +- drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h | 1 + drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h | 1 + drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h | 1 + drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 2 +- drivers/gpu/drm/nouveau/nvkm/core/ramht.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h | 4 +++- drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h | 2 +- 17 files changed, 19 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h index bb4c214f10466..5046e1db99acd 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h @@ -1,7 +1,7 @@ #ifndef __NVKM_DEVICE_H__ #define __NVKM_DEVICE_H__ +#include #include -#include enum nvkm_devidx { NVKM_SUBDEV_PCI, diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h index d2a6532ce3b99..b672a3b07f554 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h @@ -1,6 +1,7 @@ #ifndef __NVKM_DMA_H__ #define __NVKM_DMA_H__ #include +#include struct nvkm_client; struct nvkm_dmaobj { diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h index e1a854e2ade1a..8f9d132fbedff 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h @@ -3,6 +3,7 @@ #define nvkm_falcon(p) container_of((p), struct nvkm_falcon, engine) #include struct nvkm_fifo_chan; +struct nvkm_gpuobj; enum nvkm_falcon_dmaidx { FALCON_DMAIDX_UCODE = 0, diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index f00527b36accf..0e65ad7e6eed1 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -1,6 +1,7 @@ #ifndef __NVKM_FIFO_H__ #define __NVKM_FIFO_H__ #include +#include #include #define NVKM_FIFO_CHID_NR 4096 diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index dcd3deff27a49..d333c7668c236 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -2,7 +2,7 @@ #define __NVKM_MMU_H__ #include #include -struct nvkm_device; +struct nvkm_gpuobj; struct nvkm_mem; struct nvkm_vm_pgt { diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ramht.c b/drivers/gpu/drm/nouveau/nvkm/core/ramht.c index 89da47234016b..ccba4ae73cc59 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/ramht.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/ramht.c @@ -21,6 +21,7 @@ */ #include #include +#include static u32 nvkm_ramht_hash(struct nvkm_ramht *ramht, int chid, u32 handle) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h b/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h index 20249d8e444de..2c3c3ee3c494a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h @@ -1,7 +1,7 @@ #ifndef __NVKM_DEVICE_CTRL_H__ #define __NVKM_DEVICE_CTRL_H__ #define nvkm_control(p) container_of((p), struct nvkm_control, object) -#include +#include struct nvkm_control { struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h index 737b38f6fbd2b..9bb4ad5b0e57f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h @@ -1,6 +1,7 @@ #ifndef __NV50_DISP_CHAN_H__ #define __NV50_DISP_CHAN_H__ #define nv50_disp_chan(p) container_of((p), struct nv50_disp_chan, object) +#include #include "nv50.h" struct nv50_disp_chan { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index a36e45a4a635c..2901488163606 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -156,6 +156,7 @@ int gp100_gr_init(struct gf100_gr *); void gp100_gr_init_rop_active_fbps(struct gf100_gr *); #define gf100_gr_chan(p) container_of((p), struct gf100_gr_chan, object) +#include struct gf100_gr_chan { struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h index cdf4501e3798d..d0cb2b8846ec6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h @@ -19,6 +19,7 @@ void nv20_gr_tile(struct nvkm_gr *, int, struct nvkm_fb_tile *); int nv30_gr_init(struct nvkm_gr *); #define nv20_gr_chan(p) container_of((p), struct nv20_gr_chan, object) +#include struct nv20_gr_chan { struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h index 2812ed11f877b..bee8ef2d56974 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h @@ -16,6 +16,7 @@ void nv40_gr_intr(struct nvkm_gr *); u64 nv40_gr_units(struct nvkm_gr *); #define nv40_gr_chan(p) container_of((p), struct nv40_gr_chan, object) +#include struct nv40_gr_chan { struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h index 45eec83a5969b..1ab6ea436b703 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h @@ -19,6 +19,7 @@ u64 nv50_gr_units(struct nvkm_gr *); int g84_gr_tlb_flush(struct nvkm_gr *); #define nv50_gr_chan(p) container_of((p), struct nv50_gr_chan, object) +#include struct nv50_gr_chan { struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h index d3bb34fcdebfe..f0d35beb58df2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h @@ -18,6 +18,7 @@ struct nv31_mpeg_func { }; #define nv31_mpeg_chan(p) container_of((p), struct nv31_mpeg_chan, object) +#include struct nv31_mpeg_chan { struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c index 4e528851e9c00..6df880a390193 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c @@ -24,6 +24,7 @@ #include "priv.h" #include +#include #include #include diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h index d7b81cbf82b50..4ff0475e776c6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h @@ -67,6 +67,7 @@ struct nvkm_specdom { }; #define nvkm_perfdom(p) container_of((p), struct nvkm_perfdom, object) +#include struct nvkm_perfdom { struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h index 6608bf6c68421..b5be49f0ac562 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h @@ -1,9 +1,11 @@ #ifndef __NVKM_SW_CHAN_H__ #define __NVKM_SW_CHAN_H__ #define nvkm_sw_chan(p) container_of((p), struct nvkm_sw_chan, object) -#include "priv.h" +#include #include +#include "priv.h" + struct nvkm_sw_chan { const struct nvkm_sw_chan_func *func; struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h index 943ef4c10091c..bcfff62131fe3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h @@ -1,7 +1,7 @@ #ifndef __NVKM_NVSW_H__ #define __NVKM_NVSW_H__ #define nvkm_nvsw(p) container_of((p), struct nvkm_nvsw, object) -#include "priv.h" +#include struct nvkm_nvsw { struct nvkm_object object; From 12973a37c47a33c275017c92327340ab61558d83 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 014/151] drm/nouveau/secboot: add missing newline in debug message Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c index ee989210725e9..6f10b098676ce 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c @@ -183,7 +183,7 @@ acr_ls_sec2_post_run(const struct nvkm_acr *acr, const struct nvkm_secboot *sb) break; ); if (reg & BIT(4)) { - nvkm_debug(subdev, "applying workaround for start bug..."); + nvkm_debug(subdev, "applying workaround for start bug...\n"); nvkm_falcon_start(sb->boot_falcon); nvkm_msec(subdev->device, 1, if ((reg = nvkm_rd32(subdev->device, From b5078d731fad979e7426e1ef55c44ea469a71191 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 015/151] drm/nouveau/disp: add missing newline in ior debug messages Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h index a1e8bf48b7784..c9e0a8f7b5d5c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h @@ -147,7 +147,7 @@ void gf119_hda_eld(struct nvkm_ior *, u8 *, u8); #define IOR_MSG(i,l,f,a...) do { \ struct nvkm_ior *_ior = (i); \ - nvkm_##l(&_ior->disp->engine.subdev, "%s: "f, _ior->name, ##a); \ + nvkm_##l(&_ior->disp->engine.subdev, "%s: "f"\n", _ior->name, ##a); \ } while(0) #define IOR_WARN(i,f,a...) IOR_MSG((i), warn, f, ##a) #define IOR_DBG(i,f,a...) IOR_MSG((i), debug, f, ##a) From 5e721ad198a0518a6059003b1651fdb183e32dd5 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 016/151] drm/nouveau/fifo: perform reset from preinit RM appears to do this really early in its initialisation, before DEVINIT. We currently do this before BAR2 initialisation for some reason. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/core/engine.h | 1 + drivers/gpu/drm/nouveau/nvkm/core/engine.c | 10 ++++++++++ drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c | 8 ++++++++ 3 files changed, 19 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h b/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h index d4cd2fbfde885..7730499bfd958 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h @@ -15,6 +15,7 @@ struct nvkm_engine { struct nvkm_engine_func { void *(*dtor)(struct nvkm_engine *); + void (*preinit)(struct nvkm_engine *); int (*oneinit)(struct nvkm_engine *); int (*init)(struct nvkm_engine *); int (*fini)(struct nvkm_engine *, bool suspend); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/engine.c b/drivers/gpu/drm/nouveau/nvkm/core/engine.c index b6c916954a107..657231c3c098a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/engine.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/engine.c @@ -126,6 +126,15 @@ nvkm_engine_init(struct nvkm_subdev *subdev) return ret; } +static int +nvkm_engine_preinit(struct nvkm_subdev *subdev) +{ + struct nvkm_engine *engine = nvkm_engine(subdev); + if (engine->func->preinit) + engine->func->preinit(engine); + return 0; +} + static void * nvkm_engine_dtor(struct nvkm_subdev *subdev) { @@ -138,6 +147,7 @@ nvkm_engine_dtor(struct nvkm_subdev *subdev) static const struct nvkm_subdev_func nvkm_engine_func = { .dtor = nvkm_engine_dtor, + .preinit = nvkm_engine_preinit, .init = nvkm_engine_init, .fini = nvkm_engine_fini, .intr = nvkm_engine_intr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 660ca7aa95ea3..64f6b7654a08d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -278,6 +279,12 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine) return 0; } +static void +nvkm_fifo_preinit(struct nvkm_engine *engine) +{ + nvkm_mc_reset(engine->subdev.device, NVKM_ENGINE_FIFO); +} + static int nvkm_fifo_init(struct nvkm_engine *engine) { @@ -302,6 +309,7 @@ nvkm_fifo_dtor(struct nvkm_engine *engine) static const struct nvkm_engine_func nvkm_fifo = { .dtor = nvkm_fifo_dtor, + .preinit = nvkm_fifo_preinit, .oneinit = nvkm_fifo_oneinit, .init = nvkm_fifo_init, .fini = nvkm_fifo_fini, From e69dae85c90b3e07ed5538a8ad8a7ee807fbbf64 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 017/151] drm/nouveau/bar/nv50,g84: drop mmu invalidate Will already be done by MMU as a result of the PT writes that occur during BAR2 bootstrapping. This is likely just a left-over from the days when it was hardcoded. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 4 +--- drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 9 +-------- drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h | 2 +- 6 files changed, 7 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index c561d148cebcd..2eab979e94df3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -56,7 +56,8 @@ static int nvkm_bar_init(struct nvkm_subdev *subdev) { struct nvkm_bar *bar = nvkm_bar(subdev); - return bar->func->init(bar); + bar->func->init(bar); + return 0; } static void * diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index 676c167c95b98..1befe44641556 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -117,7 +117,7 @@ gf100_bar_oneinit(struct nvkm_bar *base) return 0; } -int +void gf100_bar_init(struct nvkm_bar *base) { struct gf100_bar *bar = gf100_bar(base); @@ -136,8 +136,6 @@ gf100_bar_init(struct nvkm_bar *base) addr |= 0x40000000; nvkm_wr32(device, 0x001714, 0x80000000 | addr); } - - return 0; } void * diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h index 20a5255362baf..9c5735f01acd4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h @@ -19,6 +19,6 @@ int gf100_bar_new_(const struct nvkm_bar_func *, struct nvkm_device *, int, struct nvkm_bar **); void *gf100_bar_dtor(struct nvkm_bar *); int gf100_bar_oneinit(struct nvkm_bar *); -int gf100_bar_init(struct nvkm_bar *); +void gf100_bar_init(struct nvkm_bar *); int gf100_bar_umap(struct nvkm_bar *, u64, int, struct nvkm_vma *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index 6eff637ac301a..4bfe6db006acc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -144,7 +144,7 @@ nv50_bar_oneinit(struct nvkm_bar *base) return 0; } -int +void nv50_bar_init(struct nvkm_bar *base) { struct nv50_bar *bar = nv50_bar(base); @@ -153,12 +153,6 @@ nv50_bar_init(struct nvkm_bar *base) nvkm_mask(device, 0x000200, 0x00000100, 0x00000000); nvkm_mask(device, 0x000200, 0x00000100, 0x00000100); - nvkm_wr32(device, 0x100c80, 0x00060001); - if (nvkm_msec(device, 2000, - if (!(nvkm_rd32(device, 0x100c80) & 0x00000001)) - break; - ) < 0) - return -EBUSY; nvkm_wr32(device, 0x001704, 0x00000000 | bar->mem->addr >> 12); nvkm_wr32(device, 0x001704, 0x40000000 | bar->mem->addr >> 12); @@ -166,7 +160,6 @@ nv50_bar_init(struct nvkm_bar *base) nvkm_wr32(device, 0x00170c, 0x80000000 | bar->bar3->node->offset >> 4); for (i = 0; i < 8; i++) nvkm_wr32(device, 0x001900 + (i * 4), 0x00000000); - return 0; } void * diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h index 1eb764f22a499..da5e6be573c92 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h @@ -19,7 +19,7 @@ int nv50_bar_new_(const struct nvkm_bar_func *, struct nvkm_device *, int, u32 pgd_addr, struct nvkm_bar **); void *nv50_bar_dtor(struct nvkm_bar *); int nv50_bar_oneinit(struct nvkm_bar *); -int nv50_bar_init(struct nvkm_bar *); +void nv50_bar_init(struct nvkm_bar *); struct nvkm_vm *nv50_bar_kmap(struct nvkm_bar *); int nv50_bar_umap(struct nvkm_bar *, u64, int, struct nvkm_vma *); void nv50_bar_unmap(struct nvkm_bar *, struct nvkm_vma *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h index d834ef20db5bb..c017239b2a8e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h @@ -9,7 +9,7 @@ void nvkm_bar_ctor(const struct nvkm_bar_func *, struct nvkm_device *, struct nvkm_bar_func { void *(*dtor)(struct nvkm_bar *); int (*oneinit)(struct nvkm_bar *); - int (*init)(struct nvkm_bar *); + void (*init)(struct nvkm_bar *); struct nvkm_vm *(*kmap)(struct nvkm_bar *); int (*umap)(struct nvkm_bar *, u64 size, int type, struct nvkm_vma *); void (*flush)(struct nvkm_bar *); From c9e70592963d1588da4a300209cd4913d12ae689 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 018/151] drm/nouveau/bar: remove NV_PMC_ENABLE_PFIFO twiddling It's handled by FIFO preinit() now. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 3 --- drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 3 --- 2 files changed, 6 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index 1befe44641556..3a6b2e4c66bac 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -124,9 +124,6 @@ gf100_bar_init(struct nvkm_bar *base) struct nvkm_device *device = bar->base.subdev.device; u32 addr; - nvkm_mask(device, 0x000200, 0x00000100, 0x00000000); - nvkm_mask(device, 0x000200, 0x00000100, 0x00000100); - addr = nvkm_memory_addr(bar->bar[1].mem) >> 12; nvkm_wr32(device, 0x001704, 0x80000000 | addr); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index 4bfe6db006acc..aca65506cbc85 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -151,9 +151,6 @@ nv50_bar_init(struct nvkm_bar *base) struct nvkm_device *device = bar->base.subdev.device; int i; - nvkm_mask(device, 0x000200, 0x00000100, 0x00000000); - nvkm_mask(device, 0x000200, 0x00000100, 0x00000100); - nvkm_wr32(device, 0x001704, 0x00000000 | bar->mem->addr >> 12); nvkm_wr32(device, 0x001704, 0x40000000 | bar->mem->addr >> 12); nvkm_wr32(device, 0x001708, 0x80000000 | bar->bar1->node->offset >> 4); From 269fe32d3343b7b9e48ba533887f58eb877581c8 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 019/151] drm/nouveau/bar: swap oneinit/init ordering, and rename bar3 to bar2 NVIDIA call it BAR2, Linux APIs treat it as BAR3 due to BAR1 being a 64-bit BAR, which I presume take two slots or something. No actual code changes here, just to make future commits less messy. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/bar/base.c | 10 +-- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 48 +++++++------- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.h | 4 +- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 66 +++++++++---------- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.h | 4 +- 5 files changed, 66 insertions(+), 66 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index 2eab979e94df3..f849db6ef541e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -46,18 +46,18 @@ nvkm_bar_umap(struct nvkm_bar *bar, u64 size, int type, struct nvkm_vma *vma) } static int -nvkm_bar_oneinit(struct nvkm_subdev *subdev) +nvkm_bar_init(struct nvkm_subdev *subdev) { struct nvkm_bar *bar = nvkm_bar(subdev); - return bar->func->oneinit(bar); + bar->func->init(bar); + return 0; } static int -nvkm_bar_init(struct nvkm_subdev *subdev) +nvkm_bar_oneinit(struct nvkm_subdev *subdev) { struct nvkm_bar *bar = nvkm_bar(subdev); - bar->func->init(bar); - return 0; + return bar->func->oneinit(bar); } static void * diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index 3a6b2e4c66bac..d2496cd736fa5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -41,9 +41,27 @@ gf100_bar_umap(struct nvkm_bar *base, u64 size, int type, struct nvkm_vma *vma) return nvkm_vm_get(bar->bar[1].vm, size, type, NV_MEM_ACCESS_RW, vma); } +void +gf100_bar_init(struct nvkm_bar *base) +{ + struct gf100_bar *bar = gf100_bar(base); + struct nvkm_device *device = bar->base.subdev.device; + u32 addr; + + addr = nvkm_memory_addr(bar->bar[1].mem) >> 12; + nvkm_wr32(device, 0x001704, 0x80000000 | addr); + + if (bar->bar[0].mem) { + addr = nvkm_memory_addr(bar->bar[0].mem) >> 12; + if (bar->bar2_halve) + addr |= 0x40000000; + nvkm_wr32(device, 0x001714, 0x80000000 | addr); + } +} + static int -gf100_bar_ctor_vm(struct gf100_bar *bar, struct gf100_bar_vm *bar_vm, - struct lock_class_key *key, int bar_nr) +gf100_bar_oneinit_bar(struct gf100_bar *bar, struct gf100_barN *bar_vm, + struct lock_class_key *key, int bar_nr) { struct nvkm_device *device = bar->base.subdev.device; struct nvkm_vm *vm; @@ -98,43 +116,25 @@ int gf100_bar_oneinit(struct nvkm_bar *base) { static struct lock_class_key bar1_lock; - static struct lock_class_key bar3_lock; + static struct lock_class_key bar2_lock; struct gf100_bar *bar = gf100_bar(base); int ret; - /* BAR3 */ + /* BAR2 */ if (bar->base.func->kmap) { - ret = gf100_bar_ctor_vm(bar, &bar->bar[0], &bar3_lock, 3); + ret = gf100_bar_oneinit_bar(bar, &bar->bar[0], &bar2_lock, 3); if (ret) return ret; } /* BAR1 */ - ret = gf100_bar_ctor_vm(bar, &bar->bar[1], &bar1_lock, 1); + ret = gf100_bar_oneinit_bar(bar, &bar->bar[1], &bar1_lock, 1); if (ret) return ret; return 0; } -void -gf100_bar_init(struct nvkm_bar *base) -{ - struct gf100_bar *bar = gf100_bar(base); - struct nvkm_device *device = bar->base.subdev.device; - u32 addr; - - addr = nvkm_memory_addr(bar->bar[1].mem) >> 12; - nvkm_wr32(device, 0x001704, 0x80000000 | addr); - - if (bar->bar[0].mem) { - addr = nvkm_memory_addr(bar->bar[0].mem) >> 12; - if (bar->bar2_halve) - addr |= 0x40000000; - nvkm_wr32(device, 0x001714, 0x80000000 | addr); - } -} - void * gf100_bar_dtor(struct nvkm_bar *base) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h index 9c5735f01acd4..ece98643beddb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h @@ -3,7 +3,7 @@ #define gf100_bar(p) container_of((p), struct gf100_bar, base) #include "priv.h" -struct gf100_bar_vm { +struct gf100_barN { struct nvkm_memory *mem; struct nvkm_gpuobj *pgd; struct nvkm_vm *vm; @@ -12,7 +12,7 @@ struct gf100_bar_vm { struct gf100_bar { struct nvkm_bar base; bool bar2_halve; - struct gf100_bar_vm bar[2]; + struct gf100_barN bar[2]; }; int gf100_bar_new_(const struct nvkm_bar_func *, struct nvkm_device *, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index aca65506cbc85..8a4c42f9a96a5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -31,7 +31,7 @@ struct nvkm_vm * nv50_bar_kmap(struct nvkm_bar *base) { - return nv50_bar(base)->bar3_vm; + return nv50_bar(base)->bar2_vm; } int @@ -56,13 +56,28 @@ nv50_bar_flush(struct nvkm_bar *base) spin_unlock_irqrestore(&bar->base.lock, flags); } +void +nv50_bar_init(struct nvkm_bar *base) +{ + struct nv50_bar *bar = nv50_bar(base); + struct nvkm_device *device = bar->base.subdev.device; + int i; + + nvkm_wr32(device, 0x001704, 0x00000000 | bar->mem->addr >> 12); + nvkm_wr32(device, 0x001704, 0x40000000 | bar->mem->addr >> 12); + nvkm_wr32(device, 0x001708, 0x80000000 | bar->bar1->node->offset >> 4); + nvkm_wr32(device, 0x00170c, 0x80000000 | bar->bar2->node->offset >> 4); + for (i = 0; i < 8; i++) + nvkm_wr32(device, 0x001900 + (i * 4), 0x00000000); +} + int nv50_bar_oneinit(struct nvkm_bar *base) { struct nv50_bar *bar = nv50_bar(base); struct nvkm_device *device = bar->base.subdev.device; static struct lock_class_key bar1_lock; - static struct lock_class_key bar3_lock; + static struct lock_class_key bar2_lock; struct nvkm_vm *vm; u64 start, limit; int ret; @@ -80,11 +95,11 @@ nv50_bar_oneinit(struct nvkm_bar *base) if (ret) return ret; - /* BAR3 */ + /* BAR2 */ start = 0x0100000000ULL; limit = start + device->func->resource_size(device, 3); - ret = nvkm_vm_new(device, start, limit - start, start, &bar3_lock, &vm); + ret = nvkm_vm_new(device, start, limit - start, start, &bar2_lock, &vm); if (ret) return ret; @@ -94,24 +109,24 @@ nv50_bar_oneinit(struct nvkm_bar *base) if (ret) return ret; - ret = nvkm_vm_ref(vm, &bar->bar3_vm, bar->pgd); + ret = nvkm_vm_ref(vm, &bar->bar2_vm, bar->pgd); nvkm_vm_ref(NULL, &vm, NULL); if (ret) return ret; - ret = nvkm_gpuobj_new(device, 24, 16, false, bar->mem, &bar->bar3); + ret = nvkm_gpuobj_new(device, 24, 16, false, bar->mem, &bar->bar2); if (ret) return ret; - nvkm_kmap(bar->bar3); - nvkm_wo32(bar->bar3, 0x00, 0x7fc00000); - nvkm_wo32(bar->bar3, 0x04, lower_32_bits(limit)); - nvkm_wo32(bar->bar3, 0x08, lower_32_bits(start)); - nvkm_wo32(bar->bar3, 0x0c, upper_32_bits(limit) << 24 | + nvkm_kmap(bar->bar2); + nvkm_wo32(bar->bar2, 0x00, 0x7fc00000); + nvkm_wo32(bar->bar2, 0x04, lower_32_bits(limit)); + nvkm_wo32(bar->bar2, 0x08, lower_32_bits(start)); + nvkm_wo32(bar->bar2, 0x0c, upper_32_bits(limit) << 24 | upper_32_bits(start)); - nvkm_wo32(bar->bar3, 0x10, 0x00000000); - nvkm_wo32(bar->bar3, 0x14, 0x00000000); - nvkm_done(bar->bar3); + nvkm_wo32(bar->bar2, 0x10, 0x00000000); + nvkm_wo32(bar->bar2, 0x14, 0x00000000); + nvkm_done(bar->bar2); /* BAR1 */ start = 0x0000000000ULL; @@ -144,31 +159,16 @@ nv50_bar_oneinit(struct nvkm_bar *base) return 0; } -void -nv50_bar_init(struct nvkm_bar *base) -{ - struct nv50_bar *bar = nv50_bar(base); - struct nvkm_device *device = bar->base.subdev.device; - int i; - - nvkm_wr32(device, 0x001704, 0x00000000 | bar->mem->addr >> 12); - nvkm_wr32(device, 0x001704, 0x40000000 | bar->mem->addr >> 12); - nvkm_wr32(device, 0x001708, 0x80000000 | bar->bar1->node->offset >> 4); - nvkm_wr32(device, 0x00170c, 0x80000000 | bar->bar3->node->offset >> 4); - for (i = 0; i < 8; i++) - nvkm_wr32(device, 0x001900 + (i * 4), 0x00000000); -} - void * nv50_bar_dtor(struct nvkm_bar *base) { struct nv50_bar *bar = nv50_bar(base); nvkm_gpuobj_del(&bar->bar1); nvkm_vm_ref(NULL, &bar->bar1_vm, bar->pgd); - nvkm_gpuobj_del(&bar->bar3); - if (bar->bar3_vm) { - nvkm_memory_del(&bar->bar3_vm->pgt[0].mem[0]); - nvkm_vm_ref(NULL, &bar->bar3_vm, bar->pgd); + nvkm_gpuobj_del(&bar->bar2); + if (bar->bar2_vm) { + nvkm_memory_del(&bar->bar2_vm->pgt[0].mem[0]); + nvkm_vm_ref(NULL, &bar->bar2_vm, bar->pgd); } nvkm_gpuobj_del(&bar->pgd); nvkm_gpuobj_del(&bar->pad); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h index da5e6be573c92..6ff3fdb4b4bd8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h @@ -11,8 +11,8 @@ struct nv50_bar { struct nvkm_gpuobj *pgd; struct nvkm_vm *bar1_vm; struct nvkm_gpuobj *bar1; - struct nvkm_vm *bar3_vm; - struct nvkm_gpuobj *bar3; + struct nvkm_vm *bar2_vm; + struct nvkm_gpuobj *bar2; }; int nv50_bar_new_(const struct nvkm_bar_func *, struct nvkm_device *, From 7313cfa4f6e30384fa04083698d1e865cf812a6a Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 020/151] drm/nouveau/bar: move bar1 initialisation into its own function BAR2 being done for practical reasons, this is just for consistency. Flushes have been added after the write to bind the instance block, as later commits will reveal the need for them. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/bar/base.c | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c | 2 ++ .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 22 ++++++++++++++++--- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.h | 2 ++ .../gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c | 3 ++- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 17 +++++++++++++- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.h | 2 ++ .../gpu/drm/nouveau/nvkm/subdev/bar/priv.h | 6 +++++ 8 files changed, 51 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index f849db6ef541e..adaf91ee8326e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -49,6 +49,8 @@ static int nvkm_bar_init(struct nvkm_subdev *subdev) { struct nvkm_bar *bar = nvkm_bar(subdev); + bar->func->bar1.init(bar); + bar->func->bar1.wait(bar); bar->func->init(bar); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c index ef717136c838c..dee1cd9987c2d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c @@ -44,6 +44,8 @@ g84_bar_func = { .dtor = nv50_bar_dtor, .oneinit = nv50_bar_oneinit, .init = nv50_bar_init, + .bar1.init = nv50_bar_bar1_init, + .bar1.wait = nv50_bar_bar1_wait, .kmap = nv50_bar_kmap, .umap = nv50_bar_umap, .flush = g84_bar_flush, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index d2496cd736fa5..fc3d771cef508 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -41,6 +41,23 @@ gf100_bar_umap(struct nvkm_bar *base, u64 size, int type, struct nvkm_vma *vma) return nvkm_vm_get(bar->bar[1].vm, size, type, NV_MEM_ACCESS_RW, vma); } +void +gf100_bar_bar1_wait(struct nvkm_bar *base) +{ + /* NFI why it's twice. */ + nvkm_bar_flush(base); + nvkm_bar_flush(base); +} + +void +gf100_bar_bar1_init(struct nvkm_bar *base) +{ + struct nvkm_device *device = base->subdev.device; + struct gf100_bar *bar = gf100_bar(base); + const u32 addr = nvkm_memory_addr(bar->bar[1].mem) >> 12; + nvkm_wr32(device, 0x001704, 0x80000000 | addr); +} + void gf100_bar_init(struct nvkm_bar *base) { @@ -48,9 +65,6 @@ gf100_bar_init(struct nvkm_bar *base) struct nvkm_device *device = bar->base.subdev.device; u32 addr; - addr = nvkm_memory_addr(bar->bar[1].mem) >> 12; - nvkm_wr32(device, 0x001704, 0x80000000 | addr); - if (bar->bar[0].mem) { addr = nvkm_memory_addr(bar->bar[0].mem) >> 12; if (bar->bar2_halve) @@ -171,6 +185,8 @@ gf100_bar_func = { .dtor = gf100_bar_dtor, .oneinit = gf100_bar_oneinit, .init = gf100_bar_init, + .bar1.init = gf100_bar_bar1_init, + .bar1.wait = gf100_bar_bar1_wait, .kmap = gf100_bar_kmap, .umap = gf100_bar_umap, .flush = g84_bar_flush, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h index ece98643beddb..9b994eaad1e5d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h @@ -20,5 +20,7 @@ int gf100_bar_new_(const struct nvkm_bar_func *, struct nvkm_device *, void *gf100_bar_dtor(struct nvkm_bar *); int gf100_bar_oneinit(struct nvkm_bar *); void gf100_bar_init(struct nvkm_bar *); +void gf100_bar_bar1_init(struct nvkm_bar *); +void gf100_bar_bar1_wait(struct nvkm_bar *); int gf100_bar_umap(struct nvkm_bar *, u64, int, struct nvkm_vma *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c index 9232fab4274c8..86aca93e76e80 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c @@ -25,7 +25,8 @@ static const struct nvkm_bar_func gk20a_bar_func = { .dtor = gf100_bar_dtor, .oneinit = gf100_bar_oneinit, - .init = gf100_bar_init, + .bar1.init = gf100_bar_bar1_init, + .bar1.wait = gf100_bar_bar1_wait, .umap = gf100_bar_umap, .flush = g84_bar_flush, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index 8a4c42f9a96a5..8cb8e721a5152 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -56,6 +56,20 @@ nv50_bar_flush(struct nvkm_bar *base) spin_unlock_irqrestore(&bar->base.lock, flags); } +void +nv50_bar_bar1_wait(struct nvkm_bar *base) +{ + nvkm_bar_flush(base); +} + +void +nv50_bar_bar1_init(struct nvkm_bar *base) +{ + struct nvkm_device *device = base->subdev.device; + struct nv50_bar *bar = nv50_bar(base); + nvkm_wr32(device, 0x001708, 0x80000000 | bar->bar1->node->offset >> 4); +} + void nv50_bar_init(struct nvkm_bar *base) { @@ -65,7 +79,6 @@ nv50_bar_init(struct nvkm_bar *base) nvkm_wr32(device, 0x001704, 0x00000000 | bar->mem->addr >> 12); nvkm_wr32(device, 0x001704, 0x40000000 | bar->mem->addr >> 12); - nvkm_wr32(device, 0x001708, 0x80000000 | bar->bar1->node->offset >> 4); nvkm_wr32(device, 0x00170c, 0x80000000 | bar->bar2->node->offset >> 4); for (i = 0; i < 8; i++) nvkm_wr32(device, 0x001900 + (i * 4), 0x00000000); @@ -194,6 +207,8 @@ nv50_bar_func = { .dtor = nv50_bar_dtor, .oneinit = nv50_bar_oneinit, .init = nv50_bar_init, + .bar1.init = nv50_bar_bar1_init, + .bar1.wait = nv50_bar_bar1_wait, .kmap = nv50_bar_kmap, .umap = nv50_bar_umap, .flush = nv50_bar_flush, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h index 6ff3fdb4b4bd8..52971cfd119cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h @@ -20,6 +20,8 @@ int nv50_bar_new_(const struct nvkm_bar_func *, struct nvkm_device *, void *nv50_bar_dtor(struct nvkm_bar *); int nv50_bar_oneinit(struct nvkm_bar *); void nv50_bar_init(struct nvkm_bar *); +void nv50_bar_bar1_init(struct nvkm_bar *); +void nv50_bar_bar1_wait(struct nvkm_bar *); struct nvkm_vm *nv50_bar_kmap(struct nvkm_bar *); int nv50_bar_umap(struct nvkm_bar *, u64, int, struct nvkm_vma *); void nv50_bar_unmap(struct nvkm_bar *, struct nvkm_vma *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h index c017239b2a8e8..8b508373769f8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h @@ -10,6 +10,12 @@ struct nvkm_bar_func { void *(*dtor)(struct nvkm_bar *); int (*oneinit)(struct nvkm_bar *); void (*init)(struct nvkm_bar *); + + struct { + void (*init)(struct nvkm_bar *); + void (*wait)(struct nvkm_bar *); + } bar1; + struct nvkm_vm *(*kmap)(struct nvkm_bar *); int (*umap)(struct nvkm_bar *, u64 size, int type, struct nvkm_vma *); void (*flush)(struct nvkm_bar *); From bbb163e18960a90b0c5974fe448ad78a5df8e5d7 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 021/151] drm/nouveau/bar: implement bar1 teardown Will prevent spurious MMU fault interrupts if something decides to touch BAR1 after we've unloaded the driver. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c | 9 +++++++++ drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 7 +++++++ drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 7 +++++++ drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h | 5 +++++ 6 files changed, 30 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index adaf91ee8326e..0fee5e0a090e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -45,6 +45,14 @@ nvkm_bar_umap(struct nvkm_bar *bar, u64 size, int type, struct nvkm_vma *vma) return bar->func->umap(bar, size, type, vma); } +static int +nvkm_bar_fini(struct nvkm_subdev *subdev, bool suspend) +{ + struct nvkm_bar *bar = nvkm_bar(subdev); + bar->func->bar1.fini(bar); + return 0; +} + static int nvkm_bar_init(struct nvkm_subdev *subdev) { @@ -74,6 +82,7 @@ nvkm_bar = { .dtor = nvkm_bar_dtor, .oneinit = nvkm_bar_oneinit, .init = nvkm_bar_init, + .fini = nvkm_bar_fini, }; void diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c index dee1cd9987c2d..0b63f224fa26f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c @@ -45,6 +45,7 @@ g84_bar_func = { .oneinit = nv50_bar_oneinit, .init = nv50_bar_init, .bar1.init = nv50_bar_bar1_init, + .bar1.fini = nv50_bar_bar1_fini, .bar1.wait = nv50_bar_bar1_wait, .kmap = nv50_bar_kmap, .umap = nv50_bar_umap, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index fc3d771cef508..fb57c0175e575 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -49,6 +49,12 @@ gf100_bar_bar1_wait(struct nvkm_bar *base) nvkm_bar_flush(base); } +void +gf100_bar_bar1_fini(struct nvkm_bar *bar) +{ + nvkm_mask(bar->subdev.device, 0x001704, 0x80000000, 0x00000000); +} + void gf100_bar_bar1_init(struct nvkm_bar *base) { @@ -186,6 +192,7 @@ gf100_bar_func = { .oneinit = gf100_bar_oneinit, .init = gf100_bar_init, .bar1.init = gf100_bar_bar1_init, + .bar1.fini = gf100_bar_bar1_fini, .bar1.wait = gf100_bar_bar1_wait, .kmap = gf100_bar_kmap, .umap = gf100_bar_umap, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c index 86aca93e76e80..ab4664b4b2b23 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c @@ -26,6 +26,7 @@ gk20a_bar_func = { .dtor = gf100_bar_dtor, .oneinit = gf100_bar_oneinit, .bar1.init = gf100_bar_bar1_init, + .bar1.fini = gf100_bar_bar1_fini, .bar1.wait = gf100_bar_bar1_wait, .umap = gf100_bar_umap, .flush = g84_bar_flush, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index 8cb8e721a5152..c9776121a076d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -62,6 +62,12 @@ nv50_bar_bar1_wait(struct nvkm_bar *base) nvkm_bar_flush(base); } +void +nv50_bar_bar1_fini(struct nvkm_bar *bar) +{ + nvkm_wr32(bar->subdev.device, 0x001708, 0x00000000); +} + void nv50_bar_bar1_init(struct nvkm_bar *base) { @@ -208,6 +214,7 @@ nv50_bar_func = { .oneinit = nv50_bar_oneinit, .init = nv50_bar_init, .bar1.init = nv50_bar_bar1_init, + .bar1.fini = nv50_bar_bar1_fini, .bar1.wait = nv50_bar_bar1_wait, .kmap = nv50_bar_kmap, .umap = nv50_bar_umap, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h index 8b508373769f8..d130aab01aca7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h @@ -13,6 +13,7 @@ struct nvkm_bar_func { struct { void (*init)(struct nvkm_bar *); + void (*fini)(struct nvkm_bar *); void (*wait)(struct nvkm_bar *); } bar1; @@ -21,5 +22,9 @@ struct nvkm_bar_func { void (*flush)(struct nvkm_bar *); }; +void nv50_bar_bar1_fini(struct nvkm_bar *); + void g84_bar_flush(struct nvkm_bar *); + +void gf100_bar_bar1_fini(struct nvkm_bar *); #endif From 48fe02478a0ddb89930f3595f8217fa2dfd98796 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 022/151] drm/nouveau/bar: expose interface to bar2 initialisation If we want to be able to hit the instmem fast-path in a few trickier cases, we need to be more flexible with when we can initialise BAR2 access. There's probably a decent case to be made for merging BAR/INSTMEM into BUS, but that's something to ponder another day. Flushes have been added after the write to bind the instance block, as later commits will reveal the need for them. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/bar.h | 2 ++ .../gpu/drm/nouveau/nvkm/subdev/bar/base.c | 16 +++++++++++++++- drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c | 2 ++ .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 19 ++++++++----------- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.h | 2 +- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 15 ++++++++++++--- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.h | 1 + .../gpu/drm/nouveau/nvkm/subdev/bar/priv.h | 2 +- 8 files changed, 42 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h index d3071b5a4f982..eb9ad379f9e11 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h @@ -8,11 +8,13 @@ struct nvkm_bar { struct nvkm_subdev subdev; spinlock_t lock; + bool bar2; /* whether the BAR supports to be ioremapped WC or should be uncached */ bool iomap_uncached; }; +void nvkm_bar_bar2_init(struct nvkm_device *); void nvkm_bar_flush(struct nvkm_bar *); struct nvkm_vm *nvkm_bar_kmap(struct nvkm_bar *); int nvkm_bar_umap(struct nvkm_bar *, u64 size, int type, struct nvkm_vma *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index 0fee5e0a090e8..b55ab7183cb92 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -45,11 +45,23 @@ nvkm_bar_umap(struct nvkm_bar *bar, u64 size, int type, struct nvkm_vma *vma) return bar->func->umap(bar, size, type, vma); } +void +nvkm_bar_bar2_init(struct nvkm_device *device) +{ + struct nvkm_bar *bar = device->bar; + if (bar && bar->subdev.oneinit && !bar->bar2 && bar->func->bar2.init) { + bar->func->bar2.init(bar); + bar->func->bar2.wait(bar); + bar->bar2 = true; + } +} + static int nvkm_bar_fini(struct nvkm_subdev *subdev, bool suspend) { struct nvkm_bar *bar = nvkm_bar(subdev); bar->func->bar1.fini(bar); + bar->bar2 = false; return 0; } @@ -57,9 +69,11 @@ static int nvkm_bar_init(struct nvkm_subdev *subdev) { struct nvkm_bar *bar = nvkm_bar(subdev); + nvkm_bar_bar2_init(subdev->device); bar->func->bar1.init(bar); bar->func->bar1.wait(bar); - bar->func->init(bar); + if (bar->func->init) + bar->func->init(bar); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c index 0b63f224fa26f..912c8194e1d9e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c @@ -47,6 +47,8 @@ g84_bar_func = { .bar1.init = nv50_bar_bar1_init, .bar1.fini = nv50_bar_bar1_fini, .bar1.wait = nv50_bar_bar1_wait, + .bar2.init = nv50_bar_bar2_init, + .bar2.wait = nv50_bar_bar1_wait, .kmap = nv50_bar_kmap, .umap = nv50_bar_umap, .flush = g84_bar_flush, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index fb57c0175e575..7504450972faa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -65,18 +65,14 @@ gf100_bar_bar1_init(struct nvkm_bar *base) } void -gf100_bar_init(struct nvkm_bar *base) +gf100_bar_bar2_init(struct nvkm_bar *base) { + struct nvkm_device *device = base->subdev.device; struct gf100_bar *bar = gf100_bar(base); - struct nvkm_device *device = bar->base.subdev.device; - u32 addr; - - if (bar->bar[0].mem) { - addr = nvkm_memory_addr(bar->bar[0].mem) >> 12; - if (bar->bar2_halve) - addr |= 0x40000000; - nvkm_wr32(device, 0x001714, 0x80000000 | addr); - } + u32 addr = nvkm_memory_addr(bar->bar[0].mem) >> 12; + if (bar->bar2_halve) + addr |= 0x40000000; + nvkm_wr32(device, 0x001714, 0x80000000 | addr); } static int @@ -190,10 +186,11 @@ static const struct nvkm_bar_func gf100_bar_func = { .dtor = gf100_bar_dtor, .oneinit = gf100_bar_oneinit, - .init = gf100_bar_init, .bar1.init = gf100_bar_bar1_init, .bar1.fini = gf100_bar_bar1_fini, .bar1.wait = gf100_bar_bar1_wait, + .bar2.init = gf100_bar_bar2_init, + .bar2.wait = gf100_bar_bar1_wait, .kmap = gf100_bar_kmap, .umap = gf100_bar_umap, .flush = g84_bar_flush, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h index 9b994eaad1e5d..393116a7398ac 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h @@ -19,8 +19,8 @@ int gf100_bar_new_(const struct nvkm_bar_func *, struct nvkm_device *, int, struct nvkm_bar **); void *gf100_bar_dtor(struct nvkm_bar *); int gf100_bar_oneinit(struct nvkm_bar *); -void gf100_bar_init(struct nvkm_bar *); void gf100_bar_bar1_init(struct nvkm_bar *); void gf100_bar_bar1_wait(struct nvkm_bar *); +void gf100_bar_bar2_init(struct nvkm_bar *); int gf100_bar_umap(struct nvkm_bar *, u64, int, struct nvkm_vma *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index c9776121a076d..d6d9a1d097226 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -76,6 +76,16 @@ nv50_bar_bar1_init(struct nvkm_bar *base) nvkm_wr32(device, 0x001708, 0x80000000 | bar->bar1->node->offset >> 4); } +void +nv50_bar_bar2_init(struct nvkm_bar *base) +{ + struct nvkm_device *device = base->subdev.device; + struct nv50_bar *bar = nv50_bar(base); + nvkm_wr32(device, 0x001704, 0x00000000 | bar->mem->addr >> 12); + nvkm_wr32(device, 0x001704, 0x40000000 | bar->mem->addr >> 12); + nvkm_wr32(device, 0x00170c, 0x80000000 | bar->bar2->node->offset >> 4); +} + void nv50_bar_init(struct nvkm_bar *base) { @@ -83,9 +93,6 @@ nv50_bar_init(struct nvkm_bar *base) struct nvkm_device *device = bar->base.subdev.device; int i; - nvkm_wr32(device, 0x001704, 0x00000000 | bar->mem->addr >> 12); - nvkm_wr32(device, 0x001704, 0x40000000 | bar->mem->addr >> 12); - nvkm_wr32(device, 0x00170c, 0x80000000 | bar->bar2->node->offset >> 4); for (i = 0; i < 8; i++) nvkm_wr32(device, 0x001900 + (i * 4), 0x00000000); } @@ -216,6 +223,8 @@ nv50_bar_func = { .bar1.init = nv50_bar_bar1_init, .bar1.fini = nv50_bar_bar1_fini, .bar1.wait = nv50_bar_bar1_wait, + .bar2.init = nv50_bar_bar2_init, + .bar2.wait = nv50_bar_bar1_wait, .kmap = nv50_bar_kmap, .umap = nv50_bar_umap, .flush = nv50_bar_flush, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h index 52971cfd119cb..0009ed4344f02 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h @@ -22,6 +22,7 @@ int nv50_bar_oneinit(struct nvkm_bar *); void nv50_bar_init(struct nvkm_bar *); void nv50_bar_bar1_init(struct nvkm_bar *); void nv50_bar_bar1_wait(struct nvkm_bar *); +void nv50_bar_bar2_init(struct nvkm_bar *); struct nvkm_vm *nv50_bar_kmap(struct nvkm_bar *); int nv50_bar_umap(struct nvkm_bar *, u64, int, struct nvkm_vma *); void nv50_bar_unmap(struct nvkm_bar *, struct nvkm_vma *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h index d130aab01aca7..8c9c897dec5d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h @@ -15,7 +15,7 @@ struct nvkm_bar_func { void (*init)(struct nvkm_bar *); void (*fini)(struct nvkm_bar *); void (*wait)(struct nvkm_bar *); - } bar1; + } bar1, bar2; struct nvkm_vm *(*kmap)(struct nvkm_bar *); int (*umap)(struct nvkm_bar *, u64 size, int type, struct nvkm_vma *); From e988952eefd923a40cea1077bcb939025dafb0f1 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 023/151] drm/nouveau/bar: expose interface to bar2 teardown Will prevent spurious MMU fault interrupts if something decides to touch BAR1 after we've unloaded the driver. Exposed external to BAR so that INSTMEM can use it to better control the suspend/resume fast-path access. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c | 12 +++++++++++- drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 7 +++++++ drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 7 +++++++ drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h | 2 ++ 6 files changed, 29 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h index eb9ad379f9e11..11427d3d61738 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h @@ -15,6 +15,7 @@ struct nvkm_bar { }; void nvkm_bar_bar2_init(struct nvkm_device *); +void nvkm_bar_bar2_fini(struct nvkm_device *); void nvkm_bar_flush(struct nvkm_bar *); struct nvkm_vm *nvkm_bar_kmap(struct nvkm_bar *); int nvkm_bar_umap(struct nvkm_bar *, u64 size, int type, struct nvkm_vma *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index b55ab7183cb92..b495f7796fa3d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -45,6 +45,16 @@ nvkm_bar_umap(struct nvkm_bar *bar, u64 size, int type, struct nvkm_vma *vma) return bar->func->umap(bar, size, type, vma); } +void +nvkm_bar_bar2_fini(struct nvkm_device *device) +{ + struct nvkm_bar *bar = device->bar; + if (bar && bar->bar2) { + bar->func->bar2.fini(bar); + bar->bar2 = false; + } +} + void nvkm_bar_bar2_init(struct nvkm_device *device) { @@ -61,7 +71,7 @@ nvkm_bar_fini(struct nvkm_subdev *subdev, bool suspend) { struct nvkm_bar *bar = nvkm_bar(subdev); bar->func->bar1.fini(bar); - bar->bar2 = false; + nvkm_bar_bar2_fini(subdev->device); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c index 912c8194e1d9e..dce74e0d2591f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c @@ -48,6 +48,7 @@ g84_bar_func = { .bar1.fini = nv50_bar_bar1_fini, .bar1.wait = nv50_bar_bar1_wait, .bar2.init = nv50_bar_bar2_init, + .bar2.fini = nv50_bar_bar2_fini, .bar2.wait = nv50_bar_bar1_wait, .kmap = nv50_bar_kmap, .umap = nv50_bar_umap, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index 7504450972faa..13d5a04f41dfc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -64,6 +64,12 @@ gf100_bar_bar1_init(struct nvkm_bar *base) nvkm_wr32(device, 0x001704, 0x80000000 | addr); } +void +gf100_bar_bar2_fini(struct nvkm_bar *bar) +{ + nvkm_mask(bar->subdev.device, 0x001714, 0x80000000, 0x00000000); +} + void gf100_bar_bar2_init(struct nvkm_bar *base) { @@ -190,6 +196,7 @@ gf100_bar_func = { .bar1.fini = gf100_bar_bar1_fini, .bar1.wait = gf100_bar_bar1_wait, .bar2.init = gf100_bar_bar2_init, + .bar2.fini = gf100_bar_bar2_fini, .bar2.wait = gf100_bar_bar1_wait, .kmap = gf100_bar_kmap, .umap = gf100_bar_umap, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index d6d9a1d097226..8d3f7ac42e023 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -76,6 +76,12 @@ nv50_bar_bar1_init(struct nvkm_bar *base) nvkm_wr32(device, 0x001708, 0x80000000 | bar->bar1->node->offset >> 4); } +void +nv50_bar_bar2_fini(struct nvkm_bar *bar) +{ + nvkm_wr32(bar->subdev.device, 0x00170c, 0x00000000); +} + void nv50_bar_bar2_init(struct nvkm_bar *base) { @@ -224,6 +230,7 @@ nv50_bar_func = { .bar1.fini = nv50_bar_bar1_fini, .bar1.wait = nv50_bar_bar1_wait, .bar2.init = nv50_bar_bar2_init, + .bar2.fini = nv50_bar_bar2_fini, .bar2.wait = nv50_bar_bar1_wait, .kmap = nv50_bar_kmap, .umap = nv50_bar_umap, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h index 8c9c897dec5d8..9b1c360d62ec9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h @@ -23,8 +23,10 @@ struct nvkm_bar_func { }; void nv50_bar_bar1_fini(struct nvkm_bar *); +void nv50_bar_bar2_fini(struct nvkm_bar *); void g84_bar_flush(struct nvkm_bar *); void gf100_bar_bar1_fini(struct nvkm_bar *); +void gf100_bar_bar2_fini(struct nvkm_bar *); #endif From 570889dc5070e1f98b5898dce426f970c9b9329b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 024/151] drm/nouveau/bar: modify interface to bar1 vmm mapping Upcoming changes will remove the nvkm_vmm pointer from nvkm_vma, instead requiring it to be explicitly specified on each operation. It's not currently possible to get this information for BAR1 mappings, so let's fix that ahead of time. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/core/os.h | 1 + drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h | 2 +- drivers/gpu/drm/nouveau/nouveau_bo.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 5 ++++- drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 6 ++++-- drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 9 ++++----- drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 15 +++++++-------- drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h | 2 +- 13 files changed, 32 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h index cd57e238ddd34..cfd9ed361c441 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h @@ -1,4 +1,5 @@ #ifndef __NVKM_OS_H__ #define __NVKM_OS_H__ #include +#define nvkm_vmm nvkm_vm #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h index 11427d3d61738..679027538b128 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h @@ -14,11 +14,11 @@ struct nvkm_bar { bool iomap_uncached; }; +struct nvkm_vmm *nvkm_bar_bar1_vmm(struct nvkm_device *); void nvkm_bar_bar2_init(struct nvkm_device *); void nvkm_bar_bar2_fini(struct nvkm_device *); void nvkm_bar_flush(struct nvkm_bar *); struct nvkm_vm *nvkm_bar_kmap(struct nvkm_bar *); -int nvkm_bar_umap(struct nvkm_bar *, u64 size, int type, struct nvkm_vma *); int nv50_bar_new(struct nvkm_device *, int, struct nvkm_bar **); int g84_bar_new(struct nvkm_device *, int, struct nvkm_bar **); diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index e427f80344c4d..3da87932a21f9 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -1362,13 +1362,13 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) reg->bus.base = device->func->resource_addr(device, 1); reg->bus.is_iomem = true; if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { - struct nvkm_bar *bar = nvxx_bar(&drm->client.device); + struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); int page_shift = 12; if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_FERMI) page_shift = mem->page_shift; - ret = nvkm_bar_umap(bar, mem->size << 12, page_shift, - &mem->bar_vma); + ret = nvkm_vm_get(bar, mem->size << 12, page_shift, + NV_MEM_ACCESS_RW, &mem->bar_vma); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index cd468ab1db125..b1ab34f1a0e94 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -559,6 +560,7 @@ gf100_fifo_oneinit(struct nvkm_fifo *base) struct gf100_fifo *fifo = gf100_fifo(base); struct nvkm_subdev *subdev = &fifo->base.engine.subdev; struct nvkm_device *device = subdev->device; + struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); int ret; /* Determine number of PBDMAs by checking valid enable bits. */ @@ -584,7 +586,8 @@ gf100_fifo_oneinit(struct nvkm_fifo *base) if (ret) return ret; - ret = nvkm_bar_umap(device->bar, 128 * 0x1000, 12, &fifo->user.bar); + ret = nvkm_vm_get(bar, nvkm_memory_size(fifo->user.mem), 12, + NV_MEM_ACCESS_RW, &fifo->user.bar); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index a7e55c422501c..cdadff59bb4c5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -771,6 +772,7 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) struct gk104_fifo *fifo = gk104_fifo(base); struct nvkm_subdev *subdev = &fifo->base.engine.subdev; struct nvkm_device *device = subdev->device; + struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); int engn, runl, pbid, ret, i, j; enum nvkm_devidx engidx; u32 *map; @@ -834,8 +836,8 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) if (ret) return ret; - ret = nvkm_bar_umap(device->bar, fifo->base.nr * 0x200, 12, - &fifo->user.bar); + ret = nvkm_vm_get(bar, nvkm_memory_size(fifo->user.mem), 12, + NV_MEM_ACCESS_RW, &fifo->user.bar); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index b495f7796fa3d..5a80cf5f9eb0a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -39,10 +39,10 @@ nvkm_bar_kmap(struct nvkm_bar *bar) return NULL; } -int -nvkm_bar_umap(struct nvkm_bar *bar, u64 size, int type, struct nvkm_vma *vma) +struct nvkm_vmm * +nvkm_bar_bar1_vmm(struct nvkm_device *device) { - return bar->func->umap(bar, size, type, vma); + return device->bar->func->bar1.vmm(device->bar); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c index dce74e0d2591f..f88f5d6e64b98 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c @@ -47,11 +47,11 @@ g84_bar_func = { .bar1.init = nv50_bar_bar1_init, .bar1.fini = nv50_bar_bar1_fini, .bar1.wait = nv50_bar_bar1_wait, + .bar1.vmm = nv50_bar_bar1_vmm, .bar2.init = nv50_bar_bar2_init, .bar2.fini = nv50_bar_bar2_fini, .bar2.wait = nv50_bar_bar1_wait, .kmap = nv50_bar_kmap, - .umap = nv50_bar_umap, .flush = g84_bar_flush, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index 13d5a04f41dfc..379fa339dbafb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -34,11 +34,10 @@ gf100_bar_kmap(struct nvkm_bar *base) return gf100_bar(base)->bar[0].vm; } -int -gf100_bar_umap(struct nvkm_bar *base, u64 size, int type, struct nvkm_vma *vma) +struct nvkm_vmm * +gf100_bar_bar1_vmm(struct nvkm_bar *base) { - struct gf100_bar *bar = gf100_bar(base); - return nvkm_vm_get(bar->bar[1].vm, size, type, NV_MEM_ACCESS_RW, vma); + return gf100_bar(base)->bar[1].vm; } void @@ -195,11 +194,11 @@ gf100_bar_func = { .bar1.init = gf100_bar_bar1_init, .bar1.fini = gf100_bar_bar1_fini, .bar1.wait = gf100_bar_bar1_wait, + .bar1.vmm = gf100_bar_bar1_vmm, .bar2.init = gf100_bar_bar2_init, .bar2.fini = gf100_bar_bar2_fini, .bar2.wait = gf100_bar_bar1_wait, .kmap = gf100_bar_kmap, - .umap = gf100_bar_umap, .flush = g84_bar_flush, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h index 393116a7398ac..c56c8989b2a17 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h @@ -21,6 +21,6 @@ void *gf100_bar_dtor(struct nvkm_bar *); int gf100_bar_oneinit(struct nvkm_bar *); void gf100_bar_bar1_init(struct nvkm_bar *); void gf100_bar_bar1_wait(struct nvkm_bar *); +struct nvkm_vmm *gf100_bar_bar1_vmm(struct nvkm_bar *); void gf100_bar_bar2_init(struct nvkm_bar *); -int gf100_bar_umap(struct nvkm_bar *, u64, int, struct nvkm_vma *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c index ab4664b4b2b23..b10077d38839b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c @@ -28,7 +28,7 @@ gk20a_bar_func = { .bar1.init = gf100_bar_bar1_init, .bar1.fini = gf100_bar_bar1_fini, .bar1.wait = gf100_bar_bar1_wait, - .umap = gf100_bar_umap, + .bar1.vmm = gf100_bar_bar1_vmm, .flush = g84_bar_flush, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index 8d3f7ac42e023..8020cc996d1ad 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -34,13 +34,6 @@ nv50_bar_kmap(struct nvkm_bar *base) return nv50_bar(base)->bar2_vm; } -int -nv50_bar_umap(struct nvkm_bar *base, u64 size, int type, struct nvkm_vma *vma) -{ - struct nv50_bar *bar = nv50_bar(base); - return nvkm_vm_get(bar->bar1_vm, size, type, NV_MEM_ACCESS_RW, vma); -} - static void nv50_bar_flush(struct nvkm_bar *base) { @@ -56,6 +49,12 @@ nv50_bar_flush(struct nvkm_bar *base) spin_unlock_irqrestore(&bar->base.lock, flags); } +struct nvkm_vmm * +nv50_bar_bar1_vmm(struct nvkm_bar *base) +{ + return nv50_bar(base)->bar1_vm; +} + void nv50_bar_bar1_wait(struct nvkm_bar *base) { @@ -229,11 +228,11 @@ nv50_bar_func = { .bar1.init = nv50_bar_bar1_init, .bar1.fini = nv50_bar_bar1_fini, .bar1.wait = nv50_bar_bar1_wait, + .bar1.vmm = nv50_bar_bar1_vmm, .bar2.init = nv50_bar_bar2_init, .bar2.fini = nv50_bar_bar2_fini, .bar2.wait = nv50_bar_bar1_wait, .kmap = nv50_bar_kmap, - .umap = nv50_bar_umap, .flush = nv50_bar_flush, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h index 0009ed4344f02..24ca9a52cb42b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h @@ -22,8 +22,8 @@ int nv50_bar_oneinit(struct nvkm_bar *); void nv50_bar_init(struct nvkm_bar *); void nv50_bar_bar1_init(struct nvkm_bar *); void nv50_bar_bar1_wait(struct nvkm_bar *); +struct nvkm_vmm *nv50_bar_bar1_vmm(struct nvkm_bar *); void nv50_bar_bar2_init(struct nvkm_bar *); struct nvkm_vm *nv50_bar_kmap(struct nvkm_bar *); -int nv50_bar_umap(struct nvkm_bar *, u64, int, struct nvkm_vma *); void nv50_bar_unmap(struct nvkm_bar *, struct nvkm_vma *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h index 9b1c360d62ec9..5aa220e204ee4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h @@ -15,10 +15,10 @@ struct nvkm_bar_func { void (*init)(struct nvkm_bar *); void (*fini)(struct nvkm_bar *); void (*wait)(struct nvkm_bar *); + struct nvkm_vmm *(*vmm)(struct nvkm_bar *); } bar1, bar2; struct nvkm_vm *(*kmap)(struct nvkm_bar *); - int (*umap)(struct nvkm_bar *, u64 size, int type, struct nvkm_vma *); void (*flush)(struct nvkm_bar *); }; From a78dbce9a161a3a985b837bd07afd8651d42cabd Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 025/151] drm/nouveau/bar: modify interface to bar2 vmm mapping Match API with the BAR1 version. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/bar.h | 2 +- .../gpu/drm/nouveau/nvkm/subdev/bar/base.c | 19 +++++++++++-------- drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 16 ++++++++-------- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.h | 1 + .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 14 +++++++------- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.h | 3 +-- .../gpu/drm/nouveau/nvkm/subdev/bar/priv.h | 1 - .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 3 +-- 9 files changed, 31 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h index 679027538b128..75ef15cd8b41e 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h @@ -17,8 +17,8 @@ struct nvkm_bar { struct nvkm_vmm *nvkm_bar_bar1_vmm(struct nvkm_device *); void nvkm_bar_bar2_init(struct nvkm_device *); void nvkm_bar_bar2_fini(struct nvkm_device *); +struct nvkm_vmm *nvkm_bar_bar2_vmm(struct nvkm_device *); void nvkm_bar_flush(struct nvkm_bar *); -struct nvkm_vm *nvkm_bar_kmap(struct nvkm_bar *); int nv50_bar_new(struct nvkm_device *, int, struct nvkm_bar **); int g84_bar_new(struct nvkm_device *, int, struct nvkm_bar **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index 5a80cf5f9eb0a..d600aafae315d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -30,19 +30,22 @@ nvkm_bar_flush(struct nvkm_bar *bar) bar->func->flush(bar); } -struct nvkm_vm * -nvkm_bar_kmap(struct nvkm_bar *bar) +struct nvkm_vmm * +nvkm_bar_bar1_vmm(struct nvkm_device *device) { - /* disallow kmap() until after vm has been bootstrapped */ - if (bar && bar->func->kmap && bar->subdev.oneinit) - return bar->func->kmap(bar); - return NULL; + return device->bar->func->bar1.vmm(device->bar); } struct nvkm_vmm * -nvkm_bar_bar1_vmm(struct nvkm_device *device) +nvkm_bar_bar2_vmm(struct nvkm_device *device) { - return device->bar->func->bar1.vmm(device->bar); + /* Denies access to BAR2 when it's not initialised, used by INSTMEM + * to know when object access needs to go through the BAR0 window. + */ + struct nvkm_bar *bar = device->bar; + if (bar && bar->func->bar2.vmm && bar->subdev.oneinit) + return bar->func->bar2.vmm(bar); + return NULL; } void diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c index f88f5d6e64b98..87f26f54b4818 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c @@ -51,7 +51,7 @@ g84_bar_func = { .bar2.init = nv50_bar_bar2_init, .bar2.fini = nv50_bar_bar2_fini, .bar2.wait = nv50_bar_bar1_wait, - .kmap = nv50_bar_kmap, + .bar2.vmm = nv50_bar_bar2_vmm, .flush = g84_bar_flush, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index 379fa339dbafb..c6317a0f04a7d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -28,12 +28,6 @@ #include #include -static struct nvkm_vm * -gf100_bar_kmap(struct nvkm_bar *base) -{ - return gf100_bar(base)->bar[0].vm; -} - struct nvkm_vmm * gf100_bar_bar1_vmm(struct nvkm_bar *base) { @@ -63,6 +57,12 @@ gf100_bar_bar1_init(struct nvkm_bar *base) nvkm_wr32(device, 0x001704, 0x80000000 | addr); } +struct nvkm_vmm * +gf100_bar_bar2_vmm(struct nvkm_bar *base) +{ + return gf100_bar(base)->bar[0].vm; +} + void gf100_bar_bar2_fini(struct nvkm_bar *bar) { @@ -142,7 +142,7 @@ gf100_bar_oneinit(struct nvkm_bar *base) int ret; /* BAR2 */ - if (bar->base.func->kmap) { + if (bar->base.func->bar2.init) { ret = gf100_bar_oneinit_bar(bar, &bar->bar[0], &bar2_lock, 3); if (ret) return ret; @@ -198,7 +198,7 @@ gf100_bar_func = { .bar2.init = gf100_bar_bar2_init, .bar2.fini = gf100_bar_bar2_fini, .bar2.wait = gf100_bar_bar1_wait, - .kmap = gf100_bar_kmap, + .bar2.vmm = gf100_bar_bar2_vmm, .flush = g84_bar_flush, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h index c56c8989b2a17..fc1a7bd975a72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h @@ -23,4 +23,5 @@ void gf100_bar_bar1_init(struct nvkm_bar *); void gf100_bar_bar1_wait(struct nvkm_bar *); struct nvkm_vmm *gf100_bar_bar1_vmm(struct nvkm_bar *); void gf100_bar_bar2_init(struct nvkm_bar *); +struct nvkm_vmm *gf100_bar_bar2_vmm(struct nvkm_bar *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index 8020cc996d1ad..4b912c905551e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -28,12 +28,6 @@ #include #include -struct nvkm_vm * -nv50_bar_kmap(struct nvkm_bar *base) -{ - return nv50_bar(base)->bar2_vm; -} - static void nv50_bar_flush(struct nvkm_bar *base) { @@ -75,6 +69,12 @@ nv50_bar_bar1_init(struct nvkm_bar *base) nvkm_wr32(device, 0x001708, 0x80000000 | bar->bar1->node->offset >> 4); } +struct nvkm_vmm * +nv50_bar_bar2_vmm(struct nvkm_bar *base) +{ + return nv50_bar(base)->bar2_vm; +} + void nv50_bar_bar2_fini(struct nvkm_bar *bar) { @@ -232,7 +232,7 @@ nv50_bar_func = { .bar2.init = nv50_bar_bar2_init, .bar2.fini = nv50_bar_bar2_fini, .bar2.wait = nv50_bar_bar1_wait, - .kmap = nv50_bar_kmap, + .bar2.vmm = nv50_bar_bar2_vmm, .flush = nv50_bar_flush, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h index 24ca9a52cb42b..b5fe26aab118a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h @@ -24,6 +24,5 @@ void nv50_bar_bar1_init(struct nvkm_bar *); void nv50_bar_bar1_wait(struct nvkm_bar *); struct nvkm_vmm *nv50_bar_bar1_vmm(struct nvkm_bar *); void nv50_bar_bar2_init(struct nvkm_bar *); -struct nvkm_vm *nv50_bar_kmap(struct nvkm_bar *); -void nv50_bar_unmap(struct nvkm_bar *, struct nvkm_vma *); +struct nvkm_vmm *nv50_bar_bar2_vmm(struct nvkm_bar *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h index 5aa220e204ee4..f9a07b6d28606 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h @@ -18,7 +18,6 @@ struct nvkm_bar_func { struct nvkm_vmm *(*vmm)(struct nvkm_bar *); } bar1, bar2; - struct nvkm_vm *(*kmap)(struct nvkm_bar *); void (*flush)(struct nvkm_bar *); }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index 6d512c062ae32..a3cd3e193d03d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -107,11 +107,10 @@ nv50_instobj_acquire(struct nvkm_memory *memory) { struct nv50_instobj *iobj = nv50_instobj(memory); struct nv50_instmem *imem = iobj->imem; - struct nvkm_bar *bar = imem->base.subdev.device->bar; struct nvkm_vm *vm; unsigned long flags; - if (!iobj->map && (vm = nvkm_bar_kmap(bar))) + if (!iobj->map && (vm = nvkm_bar_bar2_vmm(imem->base.subdev.device))) nvkm_memory_boot(memory, vm); if (!IS_ERR_OR_NULL(iobj->map)) return iobj->map; From bb7e501a661f820788279063fd9e225bbe9c38e9 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 026/151] drm/nouveau/bar: prevent BAR2 mapping of objects during destructor GP100's page table nests a lot more deeply than the GF100-compatible layout we're currently using, which means our hackish-but-simple way of dealing with BAR2 VMM teardown won't work anymore. In order to sanely handle the chicken-and-egg (BAR2's PTs get mapped into themselves) problem, we need prevent page tables getting mapped back into BAR2 during the destruction of its VMM. To do this, we simply key off the state that's now maintained by the BAR2 init/fini functions. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index d600aafae315d..c8cb8b8b13232 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -43,7 +43,7 @@ nvkm_bar_bar2_vmm(struct nvkm_device *device) * to know when object access needs to go through the BAR0 window. */ struct nvkm_bar *bar = device->bar; - if (bar && bar->func->bar2.vmm && bar->subdev.oneinit) + if (bar && bar->bar2) return bar->func->bar2.vmm(bar); return NULL; } From 8e644cb29c33dcd2fb718d78d8c33d4eb88cf6f7 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 027/151] drm/nouveau/bar: initialise bar2 during oneinit If we initialise BAR2 earlier, we're able to complete BAR1 setup using the instmem fast-path. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 3 +++ drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index c6317a0f04a7d..b155a7f76ca78 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -146,6 +146,9 @@ gf100_bar_oneinit(struct nvkm_bar *base) ret = gf100_bar_oneinit_bar(bar, &bar->bar[0], &bar2_lock, 3); if (ret) return ret; + + bar->base.subdev.oneinit = true; + nvkm_bar_bar2_init(bar->base.subdev.device); } /* BAR1 */ diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index 4b912c905551e..cf1d0acd2afc8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -159,6 +159,9 @@ nv50_bar_oneinit(struct nvkm_bar *base) nvkm_wo32(bar->bar2, 0x14, 0x00000000); nvkm_done(bar->bar2); + bar->base.subdev.oneinit = true; + nvkm_bar_bar2_init(device); + /* BAR1 */ start = 0x0000000000ULL; limit = start + device->func->resource_size(device, 1); From 70433b904a8b6865558ae33cadd8bb85c0d98e12 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 028/151] drm/nouveau/bar/gm107-: wait for instance block binding to complete Discovered by accident while working to use BAR2 access to instmem objects on more paths. We've apparently been relying on luck up until now! Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/bar.h | 2 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 26 ++++---- .../gpu/drm/nouveau/nvkm/subdev/bar/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/subdev/bar/gm107.c | 65 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/bar/gm20b.c | 42 ++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/bar/priv.h | 2 + 6 files changed, 126 insertions(+), 13 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm107.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm20b.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h index 75ef15cd8b41e..ffa963939e15b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h @@ -24,4 +24,6 @@ int nv50_bar_new(struct nvkm_device *, int, struct nvkm_bar **); int g84_bar_new(struct nvkm_device *, int, struct nvkm_bar **); int gf100_bar_new(struct nvkm_device *, int, struct nvkm_bar **); int gk20a_bar_new(struct nvkm_device *, int, struct nvkm_bar **); +int gm107_bar_new(struct nvkm_device *, int, struct nvkm_bar **); +int gm20b_bar_new(struct nvkm_device *, int, struct nvkm_bar **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 28fd4fa986351..77fe1c1454983 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -1958,7 +1958,7 @@ nv108_chipset = { static const struct nvkm_device_chip nv117_chipset = { .name = "GM107", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .clk = gk104_clk_new, @@ -1992,7 +1992,7 @@ nv117_chipset = { static const struct nvkm_device_chip nv118_chipset = { .name = "GM108", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .clk = gk104_clk_new, @@ -2026,7 +2026,7 @@ nv118_chipset = { static const struct nvkm_device_chip nv120_chipset = { .name = "GM200", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .devinit = gm200_devinit_new, @@ -2061,7 +2061,7 @@ nv120_chipset = { static const struct nvkm_device_chip nv124_chipset = { .name = "GM204", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .devinit = gm200_devinit_new, @@ -2096,7 +2096,7 @@ nv124_chipset = { static const struct nvkm_device_chip nv126_chipset = { .name = "GM206", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .devinit = gm200_devinit_new, @@ -2131,7 +2131,7 @@ nv126_chipset = { static const struct nvkm_device_chip nv12b_chipset = { .name = "GM20B", - .bar = gk20a_bar_new, + .bar = gm20b_bar_new, .bus = gf100_bus_new, .clk = gm20b_clk_new, .fb = gm20b_fb_new, @@ -2156,7 +2156,7 @@ nv12b_chipset = { static const struct nvkm_device_chip nv130_chipset = { .name = "GP100", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .devinit = gm200_devinit_new, @@ -2191,7 +2191,7 @@ nv130_chipset = { static const struct nvkm_device_chip nv132_chipset = { .name = "GP102", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .devinit = gm200_devinit_new, @@ -2226,7 +2226,7 @@ nv132_chipset = { static const struct nvkm_device_chip nv134_chipset = { .name = "GP104", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .devinit = gm200_devinit_new, @@ -2261,7 +2261,7 @@ nv134_chipset = { static const struct nvkm_device_chip nv136_chipset = { .name = "GP106", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .devinit = gm200_devinit_new, @@ -2296,7 +2296,7 @@ nv136_chipset = { static const struct nvkm_device_chip nv137_chipset = { .name = "GP107", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .devinit = gm200_devinit_new, @@ -2331,7 +2331,7 @@ nv137_chipset = { static const struct nvkm_device_chip nv138_chipset = { .name = "GP108", - .bar = gf100_bar_new, + .bar = gm107_bar_new, .bios = nvkm_bios_new, .bus = gf100_bus_new, .devinit = gm200_devinit_new, @@ -2361,7 +2361,7 @@ nv138_chipset = { static const struct nvkm_device_chip nv13b_chipset = { .name = "GP10B", - .bar = gk20a_bar_new, + .bar = gm20b_bar_new, .bus = gf100_bus_new, .fb = gp10b_fb_new, .fuse = gm107_fuse_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild index 1e138b3379558..e5830453813d4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild @@ -3,3 +3,5 @@ nvkm-y += nvkm/subdev/bar/nv50.o nvkm-y += nvkm/subdev/bar/g84.o nvkm-y += nvkm/subdev/bar/gf100.o nvkm-y += nvkm/subdev/bar/gk20a.o +nvkm-y += nvkm/subdev/bar/gm107.o +nvkm-y += nvkm/subdev/bar/gm20b.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm107.c new file mode 100644 index 0000000000000..3ddf9222d9351 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm107.c @@ -0,0 +1,65 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "gf100.h" + +#include + +void +gm107_bar_bar1_wait(struct nvkm_bar *bar) +{ + struct nvkm_device *device = bar->subdev.device; + nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0x001710) & 0x00000003)) + break; + ); +} + +static void +gm107_bar_bar2_wait(struct nvkm_bar *bar) +{ + struct nvkm_device *device = bar->subdev.device; + nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0x001710) & 0x0000000c)) + break; + ); +} + +static const struct nvkm_bar_func +gm107_bar_func = { + .dtor = gf100_bar_dtor, + .oneinit = gf100_bar_oneinit, + .bar1.init = gf100_bar_bar1_init, + .bar1.fini = gf100_bar_bar1_fini, + .bar1.wait = gm107_bar_bar1_wait, + .bar1.vmm = gf100_bar_bar1_vmm, + .bar2.init = gf100_bar_bar2_init, + .bar2.fini = gf100_bar_bar2_fini, + .bar2.wait = gm107_bar_bar2_wait, + .bar2.vmm = gf100_bar_bar2_vmm, + .flush = g84_bar_flush, +}; + +int +gm107_bar_new(struct nvkm_device *device, int index, struct nvkm_bar **pbar) +{ + return gf100_bar_new_(&gm107_bar_func, device, index, pbar); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm20b.c new file mode 100644 index 0000000000000..950bff1955ad4 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm20b.c @@ -0,0 +1,42 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "gf100.h" + +static const struct nvkm_bar_func +gm20b_bar_func = { + .dtor = gf100_bar_dtor, + .oneinit = gf100_bar_oneinit, + .bar1.init = gf100_bar_bar1_init, + .bar1.fini = gf100_bar_bar1_fini, + .bar1.wait = gm107_bar_bar1_wait, + .bar1.vmm = gf100_bar_bar1_vmm, + .flush = g84_bar_flush, +}; + +int +gm20b_bar_new(struct nvkm_device *device, int index, struct nvkm_bar **pbar) +{ + int ret = gf100_bar_new_(&gm20b_bar_func, device, index, pbar); + if (ret == 0) + (*pbar)->iomap_uncached = true; + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h index f9a07b6d28606..14398e2dbdf99 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h @@ -28,4 +28,6 @@ void g84_bar_flush(struct nvkm_bar *); void gf100_bar_bar1_fini(struct nvkm_bar *); void gf100_bar_bar2_fini(struct nvkm_bar *); + +void gm107_bar_bar1_wait(struct nvkm_bar *); #endif From dde59b9c341a1b5ce2229f7d4a57f19435da82d7 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 029/151] drm/nouveau/imem: add some useful debug output Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index 10c987a654ec5..372f01ba7766d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -174,14 +174,20 @@ int nvkm_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero, struct nvkm_memory **pmemory) { + struct nvkm_subdev *subdev = &imem->subdev; struct nvkm_memory *memory = NULL; struct nvkm_instobj *iobj; u32 offset; int ret; ret = imem->func->memory_new(imem, size, align, zero, &memory); - if (ret) + if (ret) { + nvkm_error(subdev, "OOM: %08x %08x %d\n", size, align, ret); goto done; + } + + nvkm_trace(subdev, "new %08x %08x %d: %010llx %010llx\n", size, align, + zero, nvkm_memory_addr(memory), nvkm_memory_size(memory)); if (!imem->func->persistent) { if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL))) { From 07bbc1c5f49b64323d9e5c1e0d5d7d201e1f2627 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 030/151] drm/nouveau/core/memory: split info pointers from accessor pointers The accessor functions can change as a result of acquire()/release() calls, and are protected by any refcounting done there. Other functions must remain constant, as they can be called any time. Signed-off-by: Ben Skeggs --- .../drm/nouveau/include/nvkm/core/gpuobj.h | 5 +- .../drm/nouveau/include/nvkm/core/memory.h | 10 +- .../drm/nouveau/nvkm/subdev/instmem/base.c | 17 ++- .../drm/nouveau/nvkm/subdev/instmem/gk20a.c | 10 +- .../drm/nouveau/nvkm/subdev/instmem/nv04.c | 51 ++++--- .../drm/nouveau/nvkm/subdev/instmem/nv40.c | 47 ++++--- .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 133 +++++++++--------- 7 files changed, 155 insertions(+), 118 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h b/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h index 1d7ccde7cde8e..acd07de3e4ee6 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h @@ -9,7 +9,10 @@ struct nvkm_vm; #define NVOBJ_FLAG_HEAP 0x00000004 struct nvkm_gpuobj { - const struct nvkm_gpuobj_func *func; + union { + const struct nvkm_gpuobj_func *func; + const struct nvkm_gpuobj_func *ptrs; + }; struct nvkm_gpuobj *parent; struct nvkm_memory *memory; struct nvkm_mm_node *node; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h index 33ca6769266a5..14ce7df9864e6 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h @@ -14,6 +14,7 @@ enum nvkm_memory_target { struct nvkm_memory { const struct nvkm_memory_func *func; + const struct nvkm_memory_ptrs *ptrs; }; struct nvkm_memory_func { @@ -24,9 +25,12 @@ struct nvkm_memory_func { void (*boot)(struct nvkm_memory *, struct nvkm_vm *); void __iomem *(*acquire)(struct nvkm_memory *); void (*release)(struct nvkm_memory *); + void (*map)(struct nvkm_memory *, struct nvkm_vma *, u64 offset); +}; + +struct nvkm_memory_ptrs { u32 (*rd32)(struct nvkm_memory *, u64 offset); void (*wr32)(struct nvkm_memory *, u64 offset, u32 data); - void (*map)(struct nvkm_memory *, struct nvkm_vma *, u64 offset); }; void nvkm_memory_ctor(const struct nvkm_memory_func *, struct nvkm_memory *); @@ -43,8 +47,8 @@ void nvkm_memory_del(struct nvkm_memory **); * macros to guarantee correct behaviour across all chipsets */ #define nvkm_kmap(o) (o)->func->acquire(o) -#define nvkm_ro32(o,a) (o)->func->rd32((o), (a)) -#define nvkm_wo32(o,a,d) (o)->func->wr32((o), (a), (d)) +#define nvkm_ro32(o,a) (o)->ptrs->rd32((o), (a)) +#define nvkm_wo32(o,a,d) (o)->ptrs->wr32((o), (a), (d)) #define nvkm_mo32(o,a,m,d) ({ \ u32 _addr = (a), _data = nvkm_ro32((o), _addr); \ nvkm_wo32((o), _addr, (_data & ~(m)) | (d)); \ diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index 372f01ba7766d..a15125ed455df 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -112,9 +112,13 @@ nvkm_instobj_func = { .size = nvkm_instobj_size, .acquire = nvkm_instobj_acquire, .release = nvkm_instobj_release, + .map = nvkm_instobj_map, +}; + +static const struct nvkm_memory_ptrs +nvkm_instobj_ptrs = { .rd32 = nvkm_instobj_rd32, .wr32 = nvkm_instobj_wr32, - .map = nvkm_instobj_map, }; static void @@ -137,8 +141,10 @@ nvkm_instobj_acquire_slow(struct nvkm_memory *memory) { struct nvkm_instobj *iobj = nvkm_instobj(memory); iobj->map = nvkm_kmap(iobj->parent); - if (iobj->map) + if (iobj->map) { memory->func = &nvkm_instobj_func; + memory->ptrs = &nvkm_instobj_ptrs; + } return iobj->map; } @@ -165,9 +171,13 @@ nvkm_instobj_func_slow = { .boot = nvkm_instobj_boot, .acquire = nvkm_instobj_acquire_slow, .release = nvkm_instobj_release_slow, + .map = nvkm_instobj_map, +}; + +static const struct nvkm_memory_ptrs +nvkm_instobj_ptrs_slow = { .rd32 = nvkm_instobj_rd32_slow, .wr32 = nvkm_instobj_wr32_slow, - .map = nvkm_instobj_map, }; int @@ -196,6 +206,7 @@ nvkm_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero, } nvkm_memory_ctor(&nvkm_instobj_func_slow, &iobj->memory); + iobj->memory.ptrs = &nvkm_instobj_ptrs_slow; iobj->parent = memory; iobj->imem = imem; spin_lock(&iobj->imem->lock); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c index cd5adbec5e574..41bf8770cfd77 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c @@ -346,8 +346,6 @@ gk20a_instobj_func_dma = { .size = gk20a_instobj_size, .acquire = gk20a_instobj_acquire_dma, .release = gk20a_instobj_release_dma, - .rd32 = gk20a_instobj_rd32, - .wr32 = gk20a_instobj_wr32, .map = gk20a_instobj_map, }; @@ -359,9 +357,13 @@ gk20a_instobj_func_iommu = { .size = gk20a_instobj_size, .acquire = gk20a_instobj_acquire_iommu, .release = gk20a_instobj_release_iommu, + .map = gk20a_instobj_map, +}; + +static const struct nvkm_memory_ptrs +gk20a_instobj_ptrs = { .rd32 = gk20a_instobj_rd32, .wr32 = gk20a_instobj_wr32, - .map = gk20a_instobj_map, }; static int @@ -377,6 +379,7 @@ gk20a_instobj_ctor_dma(struct gk20a_instmem *imem, u32 npages, u32 align, *_node = &node->base; nvkm_memory_ctor(&gk20a_instobj_func_dma, &node->base.memory); + node->base.memory.ptrs = &gk20a_instobj_ptrs; node->base.vaddr = dma_alloc_attrs(dev, npages << PAGE_SHIFT, &node->handle, GFP_KERNEL, @@ -424,6 +427,7 @@ gk20a_instobj_ctor_iommu(struct gk20a_instmem *imem, u32 npages, u32 align, node->dma_addrs = (void *)(node->pages + npages); nvkm_memory_ctor(&gk20a_instobj_func_iommu, &node->base.memory); + node->base.memory.ptrs = &gk20a_instobj_ptrs; /* Allocate backing memory */ for (i = 0; i < npages; i++) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c index 6133c8bb2d423..d4176cc677080 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c @@ -43,22 +43,31 @@ struct nv04_instobj { struct nvkm_mm_node *node; }; -static enum nvkm_memory_target -nv04_instobj_target(struct nvkm_memory *memory) +static void +nv04_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data) { - return NVKM_MEM_TARGET_INST; + struct nv04_instobj *iobj = nv04_instobj(memory); + struct nvkm_device *device = iobj->imem->base.subdev.device; + nvkm_wr32(device, 0x700000 + iobj->node->offset + offset, data); } -static u64 -nv04_instobj_addr(struct nvkm_memory *memory) +static u32 +nv04_instobj_rd32(struct nvkm_memory *memory, u64 offset) { - return nv04_instobj(memory)->node->offset; + struct nv04_instobj *iobj = nv04_instobj(memory); + struct nvkm_device *device = iobj->imem->base.subdev.device; + return nvkm_rd32(device, 0x700000 + iobj->node->offset + offset); } -static u64 -nv04_instobj_size(struct nvkm_memory *memory) +static const struct nvkm_memory_ptrs +nv04_instobj_ptrs = { + .rd32 = nv04_instobj_rd32, + .wr32 = nv04_instobj_wr32, +}; + +static void +nv04_instobj_release(struct nvkm_memory *memory) { - return nv04_instobj(memory)->node->length; } static void __iomem * @@ -69,25 +78,22 @@ nv04_instobj_acquire(struct nvkm_memory *memory) return device->pri + 0x700000 + iobj->node->offset; } -static void -nv04_instobj_release(struct nvkm_memory *memory) +static u64 +nv04_instobj_size(struct nvkm_memory *memory) { + return nv04_instobj(memory)->node->length; } -static u32 -nv04_instobj_rd32(struct nvkm_memory *memory, u64 offset) +static u64 +nv04_instobj_addr(struct nvkm_memory *memory) { - struct nv04_instobj *iobj = nv04_instobj(memory); - struct nvkm_device *device = iobj->imem->base.subdev.device; - return nvkm_rd32(device, 0x700000 + iobj->node->offset + offset); + return nv04_instobj(memory)->node->offset; } -static void -nv04_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data) +static enum nvkm_memory_target +nv04_instobj_target(struct nvkm_memory *memory) { - struct nv04_instobj *iobj = nv04_instobj(memory); - struct nvkm_device *device = iobj->imem->base.subdev.device; - nvkm_wr32(device, 0x700000 + iobj->node->offset + offset, data); + return NVKM_MEM_TARGET_INST; } static void * @@ -108,8 +114,6 @@ nv04_instobj_func = { .addr = nv04_instobj_addr, .acquire = nv04_instobj_acquire, .release = nv04_instobj_release, - .rd32 = nv04_instobj_rd32, - .wr32 = nv04_instobj_wr32, }; static int @@ -125,6 +129,7 @@ nv04_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, *pmemory = &iobj->memory; nvkm_memory_ctor(&nv04_instobj_func, &iobj->memory); + iobj->memory.ptrs = &nv04_instobj_ptrs; iobj->imem = imem; mutex_lock(&imem->base.subdev.mutex); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c index c0543875e4904..5fa2b7c0deccb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c @@ -45,22 +45,29 @@ struct nv40_instobj { struct nvkm_mm_node *node; }; -static enum nvkm_memory_target -nv40_instobj_target(struct nvkm_memory *memory) +static void +nv40_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data) { - return NVKM_MEM_TARGET_INST; + struct nv40_instobj *iobj = nv40_instobj(memory); + iowrite32_native(data, iobj->imem->iomem + iobj->node->offset + offset); } -static u64 -nv40_instobj_addr(struct nvkm_memory *memory) +static u32 +nv40_instobj_rd32(struct nvkm_memory *memory, u64 offset) { - return nv40_instobj(memory)->node->offset; + struct nv40_instobj *iobj = nv40_instobj(memory); + return ioread32_native(iobj->imem->iomem + iobj->node->offset + offset); } -static u64 -nv40_instobj_size(struct nvkm_memory *memory) +static const struct nvkm_memory_ptrs +nv40_instobj_ptrs = { + .rd32 = nv40_instobj_rd32, + .wr32 = nv40_instobj_wr32, +}; + +static void +nv40_instobj_release(struct nvkm_memory *memory) { - return nv40_instobj(memory)->node->length; } static void __iomem * @@ -70,23 +77,22 @@ nv40_instobj_acquire(struct nvkm_memory *memory) return iobj->imem->iomem + iobj->node->offset; } -static void -nv40_instobj_release(struct nvkm_memory *memory) +static u64 +nv40_instobj_size(struct nvkm_memory *memory) { + return nv40_instobj(memory)->node->length; } -static u32 -nv40_instobj_rd32(struct nvkm_memory *memory, u64 offset) +static u64 +nv40_instobj_addr(struct nvkm_memory *memory) { - struct nv40_instobj *iobj = nv40_instobj(memory); - return ioread32_native(iobj->imem->iomem + iobj->node->offset + offset); + return nv40_instobj(memory)->node->offset; } -static void -nv40_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data) +static enum nvkm_memory_target +nv40_instobj_target(struct nvkm_memory *memory) { - struct nv40_instobj *iobj = nv40_instobj(memory); - iowrite32_native(data, iobj->imem->iomem + iobj->node->offset + offset); + return NVKM_MEM_TARGET_INST; } static void * @@ -107,8 +113,6 @@ nv40_instobj_func = { .addr = nv40_instobj_addr, .acquire = nv40_instobj_acquire, .release = nv40_instobj_release, - .rd32 = nv40_instobj_rd32, - .wr32 = nv40_instobj_wr32, }; static int @@ -124,6 +128,7 @@ nv40_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, *pmemory = &iobj->memory; nvkm_memory_ctor(&nv40_instobj_func, &iobj->memory); + iobj->memory.ptrs = &nv40_instobj_ptrs; iobj->imem = imem; mutex_lock(&imem->base.subdev.mutex); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index a3cd3e193d03d..e3273aed3381d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -49,50 +49,51 @@ struct nv50_instobj { void *map; }; -static enum nvkm_memory_target -nv50_instobj_target(struct nvkm_memory *memory) +static void +nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data) { - return NVKM_MEM_TARGET_VRAM; -} + struct nv50_instobj *iobj = nv50_instobj(memory); + struct nv50_instmem *imem = iobj->imem; + struct nvkm_device *device = imem->base.subdev.device; + u64 base = (iobj->mem->offset + offset) & 0xffffff00000ULL; + u64 addr = (iobj->mem->offset + offset) & 0x000000fffffULL; -static u64 -nv50_instobj_addr(struct nvkm_memory *memory) -{ - return nv50_instobj(memory)->mem->offset; + if (unlikely(imem->addr != base)) { + nvkm_wr32(device, 0x001700, base >> 16); + imem->addr = base; + } + nvkm_wr32(device, 0x700000 + addr, data); } -static u64 -nv50_instobj_size(struct nvkm_memory *memory) +static u32 +nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset) { - return (u64)nv50_instobj(memory)->mem->size << NVKM_RAM_MM_SHIFT; + struct nv50_instobj *iobj = nv50_instobj(memory); + struct nv50_instmem *imem = iobj->imem; + struct nvkm_device *device = imem->base.subdev.device; + u64 base = (iobj->mem->offset + offset) & 0xffffff00000ULL; + u64 addr = (iobj->mem->offset + offset) & 0x000000fffffULL; + u32 data; + + if (unlikely(imem->addr != base)) { + nvkm_wr32(device, 0x001700, base >> 16); + imem->addr = base; + } + data = nvkm_rd32(device, 0x700000 + addr); + return data; } +static const struct nvkm_memory_ptrs +nv50_instobj_slow = { + .rd32 = nv50_instobj_rd32_slow, + .wr32 = nv50_instobj_wr32_slow, +}; + static void -nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vm *vm) +nv50_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset) { struct nv50_instobj *iobj = nv50_instobj(memory); - struct nvkm_subdev *subdev = &iobj->imem->base.subdev; - struct nvkm_device *device = subdev->device; - u64 size = nvkm_memory_size(memory); - void __iomem *map; - int ret; - - iobj->map = ERR_PTR(-ENOMEM); - - ret = nvkm_vm_get(vm, size, 12, NV_MEM_ACCESS_RW, &iobj->bar); - if (ret == 0) { - map = ioremap(device->func->resource_addr(device, 3) + - (u32)iobj->bar.offset, size); - if (map) { - nvkm_memory_map(memory, &iobj->bar, 0); - iobj->map = map; - } else { - nvkm_warn(subdev, "PRAMIN ioremap failed\n"); - nvkm_vm_put(&iobj->bar); - } - } else { - nvkm_warn(subdev, "PRAMIN exhausted\n"); - } + nvkm_vm_map_at(vma, offset, iobj->mem); } static void @@ -120,45 +121,50 @@ nv50_instobj_acquire(struct nvkm_memory *memory) return NULL; } -static u32 -nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset) +static void +nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vm *vm) { struct nv50_instobj *iobj = nv50_instobj(memory); - struct nv50_instmem *imem = iobj->imem; - struct nvkm_device *device = imem->base.subdev.device; - u64 base = (iobj->mem->offset + offset) & 0xffffff00000ULL; - u64 addr = (iobj->mem->offset + offset) & 0x000000fffffULL; - u32 data; + struct nvkm_subdev *subdev = &iobj->imem->base.subdev; + struct nvkm_device *device = subdev->device; + u64 size = nvkm_memory_size(memory); + void __iomem *map; + int ret; - if (unlikely(imem->addr != base)) { - nvkm_wr32(device, 0x001700, base >> 16); - imem->addr = base; + iobj->map = ERR_PTR(-ENOMEM); + + ret = nvkm_vm_get(vm, size, 12, NV_MEM_ACCESS_RW, &iobj->bar); + if (ret == 0) { + map = ioremap(device->func->resource_addr(device, 3) + + (u32)iobj->bar.offset, size); + if (map) { + nvkm_memory_map(memory, &iobj->bar, 0); + iobj->map = map; + } else { + nvkm_warn(subdev, "PRAMIN ioremap failed\n"); + nvkm_vm_put(&iobj->bar); + } + } else { + nvkm_warn(subdev, "PRAMIN exhausted\n"); } - data = nvkm_rd32(device, 0x700000 + addr); - return data; } -static void -nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data) +static u64 +nv50_instobj_size(struct nvkm_memory *memory) { - struct nv50_instobj *iobj = nv50_instobj(memory); - struct nv50_instmem *imem = iobj->imem; - struct nvkm_device *device = imem->base.subdev.device; - u64 base = (iobj->mem->offset + offset) & 0xffffff00000ULL; - u64 addr = (iobj->mem->offset + offset) & 0x000000fffffULL; + return (u64)nv50_instobj(memory)->mem->size << NVKM_RAM_MM_SHIFT; +} - if (unlikely(imem->addr != base)) { - nvkm_wr32(device, 0x001700, base >> 16); - imem->addr = base; - } - nvkm_wr32(device, 0x700000 + addr, data); +static u64 +nv50_instobj_addr(struct nvkm_memory *memory) +{ + return nv50_instobj(memory)->mem->offset; } -static void -nv50_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset) +static enum nvkm_memory_target +nv50_instobj_target(struct nvkm_memory *memory) { - struct nv50_instobj *iobj = nv50_instobj(memory); - nvkm_vm_map_at(vma, offset, iobj->mem); + return NVKM_MEM_TARGET_VRAM; } static void * @@ -183,8 +189,6 @@ nv50_instobj_func = { .boot = nv50_instobj_boot, .acquire = nv50_instobj_acquire, .release = nv50_instobj_release, - .rd32 = nv50_instobj_rd32, - .wr32 = nv50_instobj_wr32, .map = nv50_instobj_map, }; @@ -202,6 +206,7 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, *pmemory = &iobj->memory; nvkm_memory_ctor(&nv50_instobj_func, &iobj->memory); + iobj->memory.ptrs = &nv50_instobj_slow; iobj->imem = imem; size = max((size + 4095) & ~4095, (u32)4096); From 49814f62a26bd5b8f2ad5a16ccb1340ede30ee1a Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 031/151] drm/nouveau/imem: allow nvkm_instobj to be directly embedded in backend object This will eliminate a step through the call chain, and give backends more flexibility. Signed-off-by: Ben Skeggs --- .../drm/nouveau/nvkm/subdev/instmem/base.c | 36 ++++++++++++------- .../drm/nouveau/nvkm/subdev/instmem/priv.h | 15 ++++++++ 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index a15125ed455df..78f9c2332edd5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -23,7 +23,6 @@ */ #include "priv.h" -#include #include /****************************************************************************** @@ -31,15 +30,6 @@ *****************************************************************************/ #define nvkm_instobj(p) container_of((p), struct nvkm_instobj, memory) -struct nvkm_instobj { - struct nvkm_memory memory; - struct nvkm_memory *parent; - struct nvkm_instmem *imem; - struct list_head head; - u32 *suspend; - void __iomem *map; -}; - static enum nvkm_memory_target nvkm_instobj_target(struct nvkm_memory *memory) { @@ -94,7 +84,7 @@ nvkm_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset) } static void * -nvkm_instobj_dtor(struct nvkm_memory *memory) +nvkm_instobj_dtor_old(struct nvkm_memory *memory) { struct nvkm_instobj *iobj = nvkm_instobj(memory); spin_lock(&iobj->imem->lock); @@ -106,7 +96,7 @@ nvkm_instobj_dtor(struct nvkm_memory *memory) static const struct nvkm_memory_func nvkm_instobj_func = { - .dtor = nvkm_instobj_dtor, + .dtor = nvkm_instobj_dtor_old, .target = nvkm_instobj_target, .addr = nvkm_instobj_addr, .size = nvkm_instobj_size, @@ -164,7 +154,7 @@ nvkm_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data) static const struct nvkm_memory_func nvkm_instobj_func_slow = { - .dtor = nvkm_instobj_dtor, + .dtor = nvkm_instobj_dtor_old, .target = nvkm_instobj_target, .addr = nvkm_instobj_addr, .size = nvkm_instobj_size, @@ -180,6 +170,26 @@ nvkm_instobj_ptrs_slow = { .wr32 = nvkm_instobj_wr32_slow, }; +void +nvkm_instobj_dtor(struct nvkm_instmem *imem, struct nvkm_instobj *iobj) +{ + spin_lock(&imem->lock); + list_del(&iobj->head); + spin_unlock(&imem->lock); +} + +void +nvkm_instobj_ctor(const struct nvkm_memory_func *func, + struct nvkm_instmem *imem, struct nvkm_instobj *iobj) +{ + nvkm_memory_ctor(func, &iobj->memory); + iobj->parent = &iobj->memory; + iobj->suspend = NULL; + spin_lock(&imem->lock); + list_add_tail(&iobj->head, &imem->list); + spin_unlock(&imem->lock); +} + int nvkm_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero, struct nvkm_memory **pmemory) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h index ace4471864a3c..e0c7f13cfd083 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h @@ -17,4 +17,19 @@ struct nvkm_instmem_func { void nvkm_instmem_ctor(const struct nvkm_instmem_func *, struct nvkm_device *, int index, struct nvkm_instmem *); + +#include + +struct nvkm_instobj { + struct nvkm_memory memory; + struct nvkm_memory *parent; + struct nvkm_instmem *imem; + struct list_head head; + u32 *suspend; + void __iomem *map; +}; + +void nvkm_instobj_ctor(const struct nvkm_memory_func *func, + struct nvkm_instmem *, struct nvkm_instobj *); +void nvkm_instobj_dtor(struct nvkm_instmem *, struct nvkm_instobj *); #endif From 87717e7f28c9536a71738e54338aafffd9d5a724 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 032/151] drm/nouveau/imem/nv04: directly embed nvkm_instobj into nv04_instobj Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c index d4176cc677080..be907412bc048 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c @@ -24,7 +24,6 @@ #define nv04_instmem(p) container_of((p), struct nv04_instmem, base) #include "priv.h" -#include #include struct nv04_instmem { @@ -35,10 +34,10 @@ struct nv04_instmem { /****************************************************************************** * instmem object implementation *****************************************************************************/ -#define nv04_instobj(p) container_of((p), struct nv04_instobj, memory) +#define nv04_instobj(p) container_of((p), struct nv04_instobj, base.memory) struct nv04_instobj { - struct nvkm_memory memory; + struct nvkm_instobj base; struct nv04_instmem *imem; struct nvkm_mm_node *node; }; @@ -103,6 +102,7 @@ nv04_instobj_dtor(struct nvkm_memory *memory) mutex_lock(&iobj->imem->base.subdev.mutex); nvkm_mm_free(&iobj->imem->heap, &iobj->node); mutex_unlock(&iobj->imem->base.subdev.mutex); + nvkm_instobj_dtor(&iobj->imem->base, &iobj->base); return iobj; } @@ -126,10 +126,10 @@ nv04_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL))) return -ENOMEM; - *pmemory = &iobj->memory; + *pmemory = &iobj->base.memory; - nvkm_memory_ctor(&nv04_instobj_func, &iobj->memory); - iobj->memory.ptrs = &nv04_instobj_ptrs; + nvkm_instobj_ctor(&nv04_instobj_func, &imem->base, &iobj->base); + iobj->base.memory.ptrs = &nv04_instobj_ptrs; iobj->imem = imem; mutex_lock(&imem->base.subdev.mutex); @@ -214,7 +214,7 @@ nv04_instmem = { .rd32 = nv04_instmem_rd32, .wr32 = nv04_instmem_wr32, .memory_new = nv04_instobj_new, - .persistent = false, + .persistent = true, .zero = false, }; From 62465ac51883ef19af83c575d5fee5526732ffcb Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 033/151] drm/nouveau/imem/nv40: embed nvkm_instobj directly into nv04_instobj Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c index 5fa2b7c0deccb..5ae9dd28dbbad 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c @@ -24,7 +24,6 @@ #define nv40_instmem(p) container_of((p), struct nv40_instmem, base) #include "priv.h" -#include #include #include @@ -37,10 +36,10 @@ struct nv40_instmem { /****************************************************************************** * instmem object implementation *****************************************************************************/ -#define nv40_instobj(p) container_of((p), struct nv40_instobj, memory) +#define nv40_instobj(p) container_of((p), struct nv40_instobj, base.memory) struct nv40_instobj { - struct nvkm_memory memory; + struct nvkm_instobj base; struct nv40_instmem *imem; struct nvkm_mm_node *node; }; @@ -102,6 +101,7 @@ nv40_instobj_dtor(struct nvkm_memory *memory) mutex_lock(&iobj->imem->base.subdev.mutex); nvkm_mm_free(&iobj->imem->heap, &iobj->node); mutex_unlock(&iobj->imem->base.subdev.mutex); + nvkm_instobj_dtor(&iobj->imem->base, &iobj->base); return iobj; } @@ -125,10 +125,10 @@ nv40_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL))) return -ENOMEM; - *pmemory = &iobj->memory; + *pmemory = &iobj->base.memory; - nvkm_memory_ctor(&nv40_instobj_func, &iobj->memory); - iobj->memory.ptrs = &nv40_instobj_ptrs; + nvkm_instobj_ctor(&nv40_instobj_func, &imem->base, &iobj->base); + iobj->base.memory.ptrs = &nv40_instobj_ptrs; iobj->imem = imem; mutex_lock(&imem->base.subdev.mutex); @@ -231,7 +231,7 @@ nv40_instmem = { .rd32 = nv40_instmem_rd32, .wr32 = nv40_instmem_wr32, .memory_new = nv40_instobj_new, - .persistent = false, + .persistent = true, .zero = false, }; From b807270cbd87bfd2ab72904f6d9d92c17ccfeaab Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 034/151] drm/nouveau/imem/nv40: map bar2 write-combined Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c index 5ae9dd28dbbad..01453fdaa049b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c @@ -67,6 +67,7 @@ nv40_instobj_ptrs = { static void nv40_instobj_release(struct nvkm_memory *memory) { + wmb(); } static void __iomem * @@ -253,8 +254,8 @@ nv40_instmem_new(struct nvkm_device *device, int index, else bar = 3; - imem->iomem = ioremap(device->func->resource_addr(device, bar), - device->func->resource_size(device, bar)); + imem->iomem = ioremap_wc(device->func->resource_addr(device, bar), + device->func->resource_size(device, bar)); if (!imem->iomem) { nvkm_error(&imem->base.subdev, "unable to map PRAMIN BAR\n"); return -EFAULT; From f584bde6095af4d91e917be54c487258856ace89 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 035/151] drm/nouveau/imem/nv50: split object map out from api functions acquire()/boot() will need different logic in addition to performing the actual mapping. Signed-off-by: Ben Skeggs --- .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index e3273aed3381d..95b2c560fe4b1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -89,6 +89,34 @@ nv50_instobj_slow = { .wr32 = nv50_instobj_wr32_slow, }; +static void +nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) +{ + struct nvkm_memory *memory = &iobj->memory; + struct nvkm_subdev *subdev = &iobj->imem->base.subdev; + struct nvkm_device *device = subdev->device; + u64 size = nvkm_memory_size(memory); + void __iomem *map; + int ret; + + iobj->map = ERR_PTR(-ENOMEM); + + ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &iobj->bar); + if (ret == 0) { + map = ioremap(device->func->resource_addr(device, 3) + + (u32)iobj->bar.offset, size); + if (map) { + nvkm_memory_map(memory, &iobj->bar, 0); + iobj->map = map; + } else { + nvkm_warn(subdev, "PRAMIN ioremap failed\n"); + nvkm_vm_put(&iobj->bar); + } + } else { + nvkm_warn(subdev, "PRAMIN exhausted\n"); + } +} + static void nv50_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset) { @@ -112,7 +140,7 @@ nv50_instobj_acquire(struct nvkm_memory *memory) unsigned long flags; if (!iobj->map && (vm = nvkm_bar_bar2_vmm(imem->base.subdev.device))) - nvkm_memory_boot(memory, vm); + nv50_instobj_kmap(iobj, vm); if (!IS_ERR_OR_NULL(iobj->map)) return iobj->map; @@ -122,31 +150,10 @@ nv50_instobj_acquire(struct nvkm_memory *memory) } static void -nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vm *vm) +nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm) { struct nv50_instobj *iobj = nv50_instobj(memory); - struct nvkm_subdev *subdev = &iobj->imem->base.subdev; - struct nvkm_device *device = subdev->device; - u64 size = nvkm_memory_size(memory); - void __iomem *map; - int ret; - - iobj->map = ERR_PTR(-ENOMEM); - - ret = nvkm_vm_get(vm, size, 12, NV_MEM_ACCESS_RW, &iobj->bar); - if (ret == 0) { - map = ioremap(device->func->resource_addr(device, 3) + - (u32)iobj->bar.offset, size); - if (map) { - nvkm_memory_map(memory, &iobj->bar, 0); - iobj->map = map; - } else { - nvkm_warn(subdev, "PRAMIN ioremap failed\n"); - nvkm_vm_put(&iobj->bar); - } - } else { - nvkm_warn(subdev, "PRAMIN exhausted\n"); - } + nv50_instobj_kmap(iobj, vmm); } static u64 @@ -173,8 +180,8 @@ nv50_instobj_dtor(struct nvkm_memory *memory) struct nv50_instobj *iobj = nv50_instobj(memory); struct nvkm_ram *ram = iobj->imem->base.subdev.device->fb->ram; if (!IS_ERR_OR_NULL(iobj->map)) { - nvkm_vm_put(&iobj->bar); iounmap(iobj->map); + nvkm_vm_put(&iobj->bar); } ram->func->put(ram, &iobj->mem); return iobj; From af515ec8d3fbd8376513eee9648a52d5ab92bbac Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 036/151] drm/nouveau/imem/nv50: move slow-path locking into rd/wr functions This is to simplify upcoming changes. The slow-path is something that currently occurs during bootstrap of the BAR2 VMM, while backing up an object during suspend/resume, or when BAR2 address space runs out. The latter is a real problem that can happen at runtime, and occurs in Fedora 26 already (due to some change that causes a lot of channels to be created at login), so ideally we'd prefer not to make it any slower. We'd also like suspend/resume speed to not suffer. Upcoming commits will solve those problems in a better way, making the extra overhead of moving the locking here a non-issue. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index 95b2c560fe4b1..d0159d5876f3e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -31,8 +31,6 @@ struct nv50_instmem { struct nvkm_instmem base; - unsigned long lock_flags; - spinlock_t lock; u64 addr; }; @@ -57,12 +55,15 @@ nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data) struct nvkm_device *device = imem->base.subdev.device; u64 base = (iobj->mem->offset + offset) & 0xffffff00000ULL; u64 addr = (iobj->mem->offset + offset) & 0x000000fffffULL; + unsigned long flags; + spin_lock_irqsave(&imem->base.lock, flags); if (unlikely(imem->addr != base)) { nvkm_wr32(device, 0x001700, base >> 16); imem->addr = base; } nvkm_wr32(device, 0x700000 + addr, data); + spin_unlock_irqrestore(&imem->base.lock, flags); } static u32 @@ -74,12 +75,15 @@ nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset) u64 base = (iobj->mem->offset + offset) & 0xffffff00000ULL; u64 addr = (iobj->mem->offset + offset) & 0x000000fffffULL; u32 data; + unsigned long flags; + spin_lock_irqsave(&imem->base.lock, flags); if (unlikely(imem->addr != base)) { nvkm_wr32(device, 0x001700, base >> 16); imem->addr = base; } data = nvkm_rd32(device, 0x700000 + addr); + spin_unlock_irqrestore(&imem->base.lock, flags); return data; } @@ -127,8 +131,6 @@ nv50_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset) static void nv50_instobj_release(struct nvkm_memory *memory) { - struct nv50_instmem *imem = nv50_instobj(memory)->imem; - spin_unlock_irqrestore(&imem->lock, imem->lock_flags); } static void __iomem * @@ -137,15 +139,12 @@ nv50_instobj_acquire(struct nvkm_memory *memory) struct nv50_instobj *iobj = nv50_instobj(memory); struct nv50_instmem *imem = iobj->imem; struct nvkm_vm *vm; - unsigned long flags; if (!iobj->map && (vm = nvkm_bar_bar2_vmm(imem->base.subdev.device))) nv50_instobj_kmap(iobj, vm); if (!IS_ERR_OR_NULL(iobj->map)) return iobj->map; - spin_lock_irqsave(&imem->lock, flags); - imem->lock_flags = flags; return NULL; } @@ -254,7 +253,6 @@ nv50_instmem_new(struct nvkm_device *device, int index, if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL))) return -ENOMEM; nvkm_instmem_ctor(&nv50_instmem, device, index, &imem->base); - spin_lock_init(&imem->lock); *pimem = &imem->base; return 0; } From be55287aa5ba6895e9d4d3ed2f08a1be7a065957 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 037/151] drm/nouveau/imem/nv50: embed nvkm_instobj directly into nv04_instobj This is not as simple as it was for earlier GPUs, due to the need to swap accessor functions depending on whether BAR2 is usable or not. We were previously protected by nvkm_instobj's accessor functions keeping an object mapped permanently, with some unclear magic that managed to hit the slow-path where needed even if an object was marked as mapped. That's been replaced here by reference counting maps (some objects, like page tables can be accessed concurrently), and swapping the functions as necessary. Signed-off-by: Ben Skeggs --- .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 134 +++++++++++++----- 1 file changed, 102 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index d0159d5876f3e..9b5606cb2f4e6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -37,13 +37,14 @@ struct nv50_instmem { /****************************************************************************** * instmem object implementation *****************************************************************************/ -#define nv50_instobj(p) container_of((p), struct nv50_instobj, memory) +#define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory) struct nv50_instobj { - struct nvkm_memory memory; + struct nvkm_instobj base; struct nv50_instmem *imem; struct nvkm_mem *mem; struct nvkm_vma bar; + refcount_t maps; void *map; }; @@ -93,31 +94,59 @@ nv50_instobj_slow = { .wr32 = nv50_instobj_wr32_slow, }; +static void +nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data) +{ + iowrite32_native(data, nv50_instobj(memory)->map + offset); +} + +static u32 +nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset) +{ + return ioread32_native(nv50_instobj(memory)->map + offset); +} + +static const struct nvkm_memory_ptrs +nv50_instobj_fast = { + .rd32 = nv50_instobj_rd32, + .wr32 = nv50_instobj_wr32, +}; + static void nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) { - struct nvkm_memory *memory = &iobj->memory; - struct nvkm_subdev *subdev = &iobj->imem->base.subdev; + struct nv50_instmem *imem = iobj->imem; + struct nvkm_memory *memory = &iobj->base.memory; + struct nvkm_subdev *subdev = &imem->base.subdev; struct nvkm_device *device = subdev->device; + struct nvkm_vma bar = {}; u64 size = nvkm_memory_size(memory); - void __iomem *map; int ret; - iobj->map = ERR_PTR(-ENOMEM); - - ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &iobj->bar); - if (ret == 0) { - map = ioremap(device->func->resource_addr(device, 3) + - (u32)iobj->bar.offset, size); - if (map) { - nvkm_memory_map(memory, &iobj->bar, 0); - iobj->map = map; - } else { - nvkm_warn(subdev, "PRAMIN ioremap failed\n"); - nvkm_vm_put(&iobj->bar); - } - } else { - nvkm_warn(subdev, "PRAMIN exhausted\n"); + /* Attempt to allocate BAR2 address-space and map the object + * into it. The lock has to be dropped while doing this due + * to the possibility of recursion for page table allocation. + */ + mutex_unlock(&subdev->mutex); + ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &bar); + if (ret == 0) + nvkm_memory_map(memory, &bar, 0); + mutex_lock(&subdev->mutex); + if (ret || iobj->bar.node) { + /* We either failed, or another thread beat us. */ + mutex_unlock(&subdev->mutex); + nvkm_vm_put(&bar); + mutex_lock(&subdev->mutex); + return; + } + + /* Make the mapping visible to the host. */ + iobj->bar = bar; + iobj->map = ioremap(device->func->resource_addr(device, 3) + + (u32)iobj->bar.offset, size); + if (!iobj->map) { + nvkm_warn(subdev, "PRAMIN ioremap failed\n"); + nvkm_vm_put(&iobj->bar); } } @@ -131,28 +160,66 @@ nv50_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset) static void nv50_instobj_release(struct nvkm_memory *memory) { + struct nv50_instobj *iobj = nv50_instobj(memory); + struct nv50_instmem *imem = iobj->imem; + struct nvkm_subdev *subdev = &imem->base.subdev; + + nvkm_bar_flush(subdev->device->bar); + + if (refcount_dec_and_mutex_lock(&iobj->maps, &subdev->mutex)) { + /* Switch back to NULL accessors when last map is gone. */ + iobj->base.memory.ptrs = &nv50_instobj_slow; + mutex_unlock(&subdev->mutex); + } } static void __iomem * nv50_instobj_acquire(struct nvkm_memory *memory) { struct nv50_instobj *iobj = nv50_instobj(memory); - struct nv50_instmem *imem = iobj->imem; - struct nvkm_vm *vm; + struct nvkm_instmem *imem = &iobj->imem->base; + struct nvkm_vmm *vmm; + void __iomem *map = NULL; - if (!iobj->map && (vm = nvkm_bar_bar2_vmm(imem->base.subdev.device))) - nv50_instobj_kmap(iobj, vm); - if (!IS_ERR_OR_NULL(iobj->map)) + /* Already mapped? */ + if (refcount_inc_not_zero(&iobj->maps)) return iobj->map; - return NULL; + /* Take the lock, and re-check that another thread hasn't + * already mapped the object in the meantime. + */ + mutex_lock(&imem->subdev.mutex); + if (refcount_inc_not_zero(&iobj->maps)) { + mutex_unlock(&imem->subdev.mutex); + return iobj->map; + } + + /* Attempt to get a direct CPU mapping of the object. */ + if (!iobj->map && (vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) + nv50_instobj_kmap(iobj, vmm); + map = iobj->map; + + if (!refcount_inc_not_zero(&iobj->maps)) { + if (map) + iobj->base.memory.ptrs = &nv50_instobj_fast; + else + iobj->base.memory.ptrs = &nv50_instobj_slow; + refcount_inc(&iobj->maps); + } + + mutex_unlock(&imem->subdev.mutex); + return map; } static void nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm) { struct nv50_instobj *iobj = nv50_instobj(memory); + struct nvkm_instmem *imem = &iobj->imem->base; + + mutex_lock(&imem->subdev.mutex); nv50_instobj_kmap(iobj, vmm); + mutex_unlock(&imem->subdev.mutex); } static u64 @@ -177,12 +244,14 @@ static void * nv50_instobj_dtor(struct nvkm_memory *memory) { struct nv50_instobj *iobj = nv50_instobj(memory); - struct nvkm_ram *ram = iobj->imem->base.subdev.device->fb->ram; - if (!IS_ERR_OR_NULL(iobj->map)) { + struct nvkm_instmem *imem = &iobj->imem->base; + struct nvkm_ram *ram = imem->subdev.device->fb->ram; + if (iobj->map) { iounmap(iobj->map); nvkm_vm_put(&iobj->bar); } ram->func->put(ram, &iobj->mem); + nvkm_instobj_dtor(imem, &iobj->base); return iobj; } @@ -209,11 +278,12 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL))) return -ENOMEM; - *pmemory = &iobj->memory; + *pmemory = &iobj->base.memory; - nvkm_memory_ctor(&nv50_instobj_func, &iobj->memory); - iobj->memory.ptrs = &nv50_instobj_slow; + nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base); + iobj->base.memory.ptrs = &nv50_instobj_slow; iobj->imem = imem; + refcount_set(&iobj->maps, 0); size = max((size + 4095) & ~4095, (u32)4096); align = max((align + 4095) & ~4095, (u32)4096); @@ -240,7 +310,7 @@ static const struct nvkm_instmem_func nv50_instmem = { .fini = nv50_instmem_fini, .memory_new = nv50_instobj_new, - .persistent = false, + .persistent = true, .zero = false, }; From dfcbd5506817c7bfba67c7c2232610f7693b1938 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 038/151] drm/nouveau/imem/nv50: map bar2 write-combined Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index 9b5606cb2f4e6..ec2904a0e9fcd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -142,8 +142,8 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) /* Make the mapping visible to the host. */ iobj->bar = bar; - iobj->map = ioremap(device->func->resource_addr(device, 3) + - (u32)iobj->bar.offset, size); + iobj->map = ioremap_wc(device->func->resource_addr(device, 3) + + (u32)iobj->bar.offset, size); if (!iobj->map) { nvkm_warn(subdev, "PRAMIN ioremap failed\n"); nvkm_vm_put(&iobj->bar); @@ -164,6 +164,7 @@ nv50_instobj_release(struct nvkm_memory *memory) struct nv50_instmem *imem = iobj->imem; struct nvkm_subdev *subdev = &imem->base.subdev; + wmb(); nvkm_bar_flush(subdev->device->bar); if (refcount_dec_and_mutex_lock(&iobj->maps, &subdev->mutex)) { From 69b136f200006ee37b039195eaeb08942c419ecc Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 039/151] drm/nouveau/imem/nv50: prevent fast-path for mapped objects when BAR isn't ready Another piece of solving the "GP100 BAR2 VMM bootstrap" puzzle. Without doing this, we'd attempt to write PDEs for the lower page table levels through BAR2 before BAR2 access has been fully initialised. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index ec2904a0e9fcd..1a254e69feced 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -196,9 +196,11 @@ nv50_instobj_acquire(struct nvkm_memory *memory) } /* Attempt to get a direct CPU mapping of the object. */ - if (!iobj->map && (vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) - nv50_instobj_kmap(iobj, vmm); - map = iobj->map; + if ((vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) { + if (!iobj->map) + nv50_instobj_kmap(iobj, vmm); + map = iobj->map; + } if (!refcount_inc_not_zero(&iobj->maps)) { if (map) From 03edf1b31a091254e95793a688abf02b96cfdd85 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 040/151] drm/nouveau/imem/nv50: support eviction of BAR2 mappings A good deal of the structures we map into here aren't accessed very often at all, and Fedora 26 has exposed an issue where after creating a heap of channels, BAR2 space would run out, and we'd need to make use of the slow path while accessing important structures like page tables. This implements an LRU on BAR2 space, which allows eviction of mappings that aren't currently needed, to make space for other objects. Signed-off-by: Ben Skeggs --- .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 72 +++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index 1a254e69feced..a054e8d155ee6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -32,6 +32,9 @@ struct nv50_instmem { struct nvkm_instmem base; u64 addr; + + /* Mappings that can be evicted when BAR2 space has been exhausted. */ + struct list_head lru; }; /****************************************************************************** @@ -46,6 +49,7 @@ struct nv50_instobj { struct nvkm_vma bar; refcount_t maps; void *map; + struct list_head lru; }; static void @@ -116,11 +120,13 @@ static void nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) { struct nv50_instmem *imem = iobj->imem; + struct nv50_instobj *eobj; struct nvkm_memory *memory = &iobj->base.memory; struct nvkm_subdev *subdev = &imem->base.subdev; struct nvkm_device *device = subdev->device; - struct nvkm_vma bar = {}; + struct nvkm_vma bar = {}, ebar; u64 size = nvkm_memory_size(memory); + void *emap; int ret; /* Attempt to allocate BAR2 address-space and map the object @@ -128,7 +134,30 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) * to the possibility of recursion for page table allocation. */ mutex_unlock(&subdev->mutex); - ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &bar); + while ((ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &bar))) { + /* Evict unused mappings, and keep retrying until we either + * succeed,or there's no more objects left on the LRU. + */ + mutex_lock(&subdev->mutex); + eobj = list_first_entry_or_null(&imem->lru, typeof(*eobj), lru); + if (eobj) { + nvkm_debug(subdev, "evict %016llx %016llx @ %016llx\n", + nvkm_memory_addr(&eobj->base.memory), + nvkm_memory_size(&eobj->base.memory), + eobj->bar.offset); + list_del_init(&eobj->lru); + ebar = eobj->bar; + eobj->bar.node = NULL; + emap = eobj->map; + eobj->map = NULL; + } + mutex_unlock(&subdev->mutex); + if (!eobj) + break; + iounmap(emap); + nvkm_vm_put(&ebar); + } + if (ret == 0) nvkm_memory_map(memory, &bar, 0); mutex_lock(&subdev->mutex); @@ -168,6 +197,14 @@ nv50_instobj_release(struct nvkm_memory *memory) nvkm_bar_flush(subdev->device->bar); if (refcount_dec_and_mutex_lock(&iobj->maps, &subdev->mutex)) { + /* Add the now-unused mapping to the LRU instead of directly + * unmapping it here, in case we need to map it again later. + */ + if (likely(iobj->lru.next) && iobj->map) { + BUG_ON(!list_empty(&iobj->lru)); + list_add_tail(&iobj->lru, &imem->lru); + } + /* Switch back to NULL accessors when last map is gone. */ iobj->base.memory.ptrs = &nv50_instobj_slow; mutex_unlock(&subdev->mutex); @@ -203,6 +240,10 @@ nv50_instobj_acquire(struct nvkm_memory *memory) } if (!refcount_inc_not_zero(&iobj->maps)) { + /* Exclude object from eviction while it's being accessed. */ + if (likely(iobj->lru.next)) + list_del_init(&iobj->lru); + if (map) iobj->base.memory.ptrs = &nv50_instobj_fast; else @@ -220,7 +261,15 @@ nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm) struct nv50_instobj *iobj = nv50_instobj(memory); struct nvkm_instmem *imem = &iobj->imem->base; + /* Exclude bootstrapped objects (ie. the page tables for the + * instmem BAR itself) from eviction. + */ mutex_lock(&imem->subdev.mutex); + if (likely(iobj->lru.next)) { + list_del_init(&iobj->lru); + iobj->lru.next = NULL; + } + nv50_instobj_kmap(iobj, vmm); mutex_unlock(&imem->subdev.mutex); } @@ -249,10 +298,21 @@ nv50_instobj_dtor(struct nvkm_memory *memory) struct nv50_instobj *iobj = nv50_instobj(memory); struct nvkm_instmem *imem = &iobj->imem->base; struct nvkm_ram *ram = imem->subdev.device->fb->ram; - if (iobj->map) { - iounmap(iobj->map); - nvkm_vm_put(&iobj->bar); + struct nvkm_vma bar; + void *map = map; + + mutex_lock(&imem->subdev.mutex); + if (likely(iobj->lru.next)) + list_del(&iobj->lru); + map = iobj->map; + bar = iobj->bar; + mutex_unlock(&imem->subdev.mutex); + + if (map) { + iounmap(map); + nvkm_vm_put(&bar); } + ram->func->put(ram, &iobj->mem); nvkm_instobj_dtor(imem, &iobj->base); return iobj; @@ -287,6 +347,7 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, iobj->base.memory.ptrs = &nv50_instobj_slow; iobj->imem = imem; refcount_set(&iobj->maps, 0); + INIT_LIST_HEAD(&iobj->lru); size = max((size + 4095) & ~4095, (u32)4096); align = max((align + 4095) & ~4095, (u32)4096); @@ -326,6 +387,7 @@ nv50_instmem_new(struct nvkm_device *device, int index, if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL))) return -ENOMEM; nvkm_instmem_ctor(&nv50_instmem, device, index, &imem->base); + INIT_LIST_HEAD(&imem->lru); *pimem = &imem->base; return 0; } From 71370e620a97fe98daebea86c6ae3775cf0f4fc8 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 041/151] drm/nouveau/imem: remove now-unused wrapper for backend objects Signed-off-by: Ben Skeggs --- .../drm/nouveau/nvkm/subdev/instmem/base.c | 164 +----------------- .../drm/nouveau/nvkm/subdev/instmem/gk20a.c | 1 - .../drm/nouveau/nvkm/subdev/instmem/nv04.c | 1 - .../drm/nouveau/nvkm/subdev/instmem/nv40.c | 1 - .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 1 - .../drm/nouveau/nvkm/subdev/instmem/priv.h | 4 - 6 files changed, 2 insertions(+), 170 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index 78f9c2332edd5..4c2fa5c460c77 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -28,148 +28,6 @@ /****************************************************************************** * instmem object base implementation *****************************************************************************/ -#define nvkm_instobj(p) container_of((p), struct nvkm_instobj, memory) - -static enum nvkm_memory_target -nvkm_instobj_target(struct nvkm_memory *memory) -{ - memory = nvkm_instobj(memory)->parent; - return nvkm_memory_target(memory); -} - -static u64 -nvkm_instobj_addr(struct nvkm_memory *memory) -{ - memory = nvkm_instobj(memory)->parent; - return nvkm_memory_addr(memory); -} - -static u64 -nvkm_instobj_size(struct nvkm_memory *memory) -{ - memory = nvkm_instobj(memory)->parent; - return nvkm_memory_size(memory); -} - -static void -nvkm_instobj_release(struct nvkm_memory *memory) -{ - struct nvkm_instobj *iobj = nvkm_instobj(memory); - nvkm_bar_flush(iobj->imem->subdev.device->bar); -} - -static void __iomem * -nvkm_instobj_acquire(struct nvkm_memory *memory) -{ - return nvkm_instobj(memory)->map; -} - -static u32 -nvkm_instobj_rd32(struct nvkm_memory *memory, u64 offset) -{ - return ioread32_native(nvkm_instobj(memory)->map + offset); -} - -static void -nvkm_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data) -{ - iowrite32_native(data, nvkm_instobj(memory)->map + offset); -} - -static void -nvkm_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset) -{ - memory = nvkm_instobj(memory)->parent; - nvkm_memory_map(memory, vma, offset); -} - -static void * -nvkm_instobj_dtor_old(struct nvkm_memory *memory) -{ - struct nvkm_instobj *iobj = nvkm_instobj(memory); - spin_lock(&iobj->imem->lock); - list_del(&iobj->head); - spin_unlock(&iobj->imem->lock); - nvkm_memory_del(&iobj->parent); - return iobj; -} - -static const struct nvkm_memory_func -nvkm_instobj_func = { - .dtor = nvkm_instobj_dtor_old, - .target = nvkm_instobj_target, - .addr = nvkm_instobj_addr, - .size = nvkm_instobj_size, - .acquire = nvkm_instobj_acquire, - .release = nvkm_instobj_release, - .map = nvkm_instobj_map, -}; - -static const struct nvkm_memory_ptrs -nvkm_instobj_ptrs = { - .rd32 = nvkm_instobj_rd32, - .wr32 = nvkm_instobj_wr32, -}; - -static void -nvkm_instobj_boot(struct nvkm_memory *memory, struct nvkm_vm *vm) -{ - memory = nvkm_instobj(memory)->parent; - nvkm_memory_boot(memory, vm); -} - -static void -nvkm_instobj_release_slow(struct nvkm_memory *memory) -{ - struct nvkm_instobj *iobj = nvkm_instobj(memory); - nvkm_instobj_release(memory); - nvkm_done(iobj->parent); -} - -static void __iomem * -nvkm_instobj_acquire_slow(struct nvkm_memory *memory) -{ - struct nvkm_instobj *iobj = nvkm_instobj(memory); - iobj->map = nvkm_kmap(iobj->parent); - if (iobj->map) { - memory->func = &nvkm_instobj_func; - memory->ptrs = &nvkm_instobj_ptrs; - } - return iobj->map; -} - -static u32 -nvkm_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset) -{ - struct nvkm_instobj *iobj = nvkm_instobj(memory); - return nvkm_ro32(iobj->parent, offset); -} - -static void -nvkm_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data) -{ - struct nvkm_instobj *iobj = nvkm_instobj(memory); - return nvkm_wo32(iobj->parent, offset, data); -} - -static const struct nvkm_memory_func -nvkm_instobj_func_slow = { - .dtor = nvkm_instobj_dtor_old, - .target = nvkm_instobj_target, - .addr = nvkm_instobj_addr, - .size = nvkm_instobj_size, - .boot = nvkm_instobj_boot, - .acquire = nvkm_instobj_acquire_slow, - .release = nvkm_instobj_release_slow, - .map = nvkm_instobj_map, -}; - -static const struct nvkm_memory_ptrs -nvkm_instobj_ptrs_slow = { - .rd32 = nvkm_instobj_rd32_slow, - .wr32 = nvkm_instobj_wr32_slow, -}; - void nvkm_instobj_dtor(struct nvkm_instmem *imem, struct nvkm_instobj *iobj) { @@ -183,7 +41,6 @@ nvkm_instobj_ctor(const struct nvkm_memory_func *func, struct nvkm_instmem *imem, struct nvkm_instobj *iobj) { nvkm_memory_ctor(func, &iobj->memory); - iobj->parent = &iobj->memory; iobj->suspend = NULL; spin_lock(&imem->lock); list_add_tail(&iobj->head, &imem->list); @@ -196,7 +53,6 @@ nvkm_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero, { struct nvkm_subdev *subdev = &imem->subdev; struct nvkm_memory *memory = NULL; - struct nvkm_instobj *iobj; u32 offset; int ret; @@ -209,22 +65,6 @@ nvkm_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero, nvkm_trace(subdev, "new %08x %08x %d: %010llx %010llx\n", size, align, zero, nvkm_memory_addr(memory), nvkm_memory_size(memory)); - if (!imem->func->persistent) { - if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL))) { - ret = -ENOMEM; - goto done; - } - - nvkm_memory_ctor(&nvkm_instobj_func_slow, &iobj->memory); - iobj->memory.ptrs = &nvkm_instobj_ptrs_slow; - iobj->parent = memory; - iobj->imem = imem; - spin_lock(&iobj->imem->lock); - list_add_tail(&iobj->head, &imem->list); - spin_unlock(&iobj->imem->lock); - memory = &iobj->memory; - } - if (!imem->func->zero && zero) { void __iomem *map = nvkm_kmap(memory); if (unlikely(!map)) { @@ -271,7 +111,7 @@ nvkm_instmem_fini(struct nvkm_subdev *subdev, bool suspend) if (suspend) { list_for_each_entry(iobj, &imem->list, head) { - struct nvkm_memory *memory = iobj->parent; + struct nvkm_memory *memory = &iobj->memory; u64 size = nvkm_memory_size(memory); iobj->suspend = vmalloc(size); @@ -304,7 +144,7 @@ nvkm_instmem_init(struct nvkm_subdev *subdev) list_for_each_entry(iobj, &imem->list, head) { if (iobj->suspend) { - struct nvkm_memory *memory = iobj->parent; + struct nvkm_memory *memory = &iobj->memory; u64 size = nvkm_memory_size(memory); for (i = 0; i < size; i += 4) nvkm_wo32(memory, i, iobj->suspend[i / 4]); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c index 41bf8770cfd77..a8b71262f663c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c @@ -558,7 +558,6 @@ static const struct nvkm_instmem_func gk20a_instmem = { .dtor = gk20a_instmem_dtor, .memory_new = gk20a_instobj_new, - .persistent = true, .zero = false, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c index be907412bc048..df75da764bd42 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c @@ -214,7 +214,6 @@ nv04_instmem = { .rd32 = nv04_instmem_rd32, .wr32 = nv04_instmem_wr32, .memory_new = nv04_instobj_new, - .persistent = true, .zero = false, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c index 01453fdaa049b..d8ab5fa86893f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c @@ -232,7 +232,6 @@ nv40_instmem = { .rd32 = nv40_instmem_rd32, .wr32 = nv40_instmem_wr32, .memory_new = nv40_instobj_new, - .persistent = true, .zero = false, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index a054e8d155ee6..374df1ebe2e85 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -374,7 +374,6 @@ static const struct nvkm_instmem_func nv50_instmem = { .fini = nv50_instmem_fini, .memory_new = nv50_instobj_new, - .persistent = true, .zero = false, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h index e0c7f13cfd083..e7515d96b31ff 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h @@ -11,7 +11,6 @@ struct nvkm_instmem_func { void (*wr32)(struct nvkm_instmem *, u32 addr, u32 data); int (*memory_new)(struct nvkm_instmem *, u32 size, u32 align, bool zero, struct nvkm_memory **); - bool persistent; bool zero; }; @@ -22,11 +21,8 @@ void nvkm_instmem_ctor(const struct nvkm_instmem_func *, struct nvkm_device *, struct nvkm_instobj { struct nvkm_memory memory; - struct nvkm_memory *parent; - struct nvkm_instmem *imem; struct list_head head; u32 *suspend; - void __iomem *map; }; void nvkm_instobj_ctor(const struct nvkm_memory_func *func, From d52ddc953e7f74a8271c88fc0aa3bd0d436b04f0 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 042/151] drm/nouveau/imem: separate suspend/resume backup handling into their own functions Signed-off-by: Ben Skeggs --- .../drm/nouveau/nvkm/subdev/instmem/base.c | 76 +++++++++++-------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index 4c2fa5c460c77..8ad31b436fa03 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -28,6 +28,36 @@ /****************************************************************************** * instmem object base implementation *****************************************************************************/ +static void +nvkm_instobj_load(struct nvkm_instobj *iobj) +{ + struct nvkm_memory *memory = &iobj->memory; + const u64 size = nvkm_memory_size(memory); + int i; + + for (i = 0; i < size; i += 4) + nvkm_wo32(memory, i, iobj->suspend[i / 4]); + vfree(iobj->suspend); + iobj->suspend = NULL; +} + +static int +nvkm_instobj_save(struct nvkm_instobj *iobj) +{ + struct nvkm_memory *memory = &iobj->memory; + const u64 size = nvkm_memory_size(memory); + int i; + + iobj->suspend = vmalloc(size); + if (!iobj->suspend) + return -ENOMEM; + + for (i = 0; i < size; i += 4) + iobj->suspend[i / 4] = nvkm_ro32(memory, i); + + return 0; +} + void nvkm_instobj_dtor(struct nvkm_instmem *imem, struct nvkm_instobj *iobj) { @@ -104,34 +134,18 @@ nvkm_instmem_fini(struct nvkm_subdev *subdev, bool suspend) { struct nvkm_instmem *imem = nvkm_instmem(subdev); struct nvkm_instobj *iobj; - int i; - - if (imem->func->fini) - imem->func->fini(imem); if (suspend) { list_for_each_entry(iobj, &imem->list, head) { - struct nvkm_memory *memory = &iobj->memory; - u64 size = nvkm_memory_size(memory); - - iobj->suspend = vmalloc(size); - if (!iobj->suspend) - return -ENOMEM; - - for (i = 0; i < size; i += 4) - iobj->suspend[i / 4] = nvkm_ro32(memory, i); + int ret = nvkm_instobj_save(iobj); + if (ret) + return ret; } } - return 0; -} + if (imem->func->fini) + imem->func->fini(imem); -static int -nvkm_instmem_oneinit(struct nvkm_subdev *subdev) -{ - struct nvkm_instmem *imem = nvkm_instmem(subdev); - if (imem->func->oneinit) - return imem->func->oneinit(imem); return 0; } @@ -140,22 +154,24 @@ nvkm_instmem_init(struct nvkm_subdev *subdev) { struct nvkm_instmem *imem = nvkm_instmem(subdev); struct nvkm_instobj *iobj; - int i; list_for_each_entry(iobj, &imem->list, head) { - if (iobj->suspend) { - struct nvkm_memory *memory = &iobj->memory; - u64 size = nvkm_memory_size(memory); - for (i = 0; i < size; i += 4) - nvkm_wo32(memory, i, iobj->suspend[i / 4]); - vfree(iobj->suspend); - iobj->suspend = NULL; - } + if (iobj->suspend) + nvkm_instobj_load(iobj); } return 0; } +static int +nvkm_instmem_oneinit(struct nvkm_subdev *subdev) +{ + struct nvkm_instmem *imem = nvkm_instmem(subdev); + if (imem->func->oneinit) + return imem->func->oneinit(imem); + return 0; +} + static void * nvkm_instmem_dtor(struct nvkm_subdev *subdev) { From 54c70e3ac6d5634982edd586418710eb7fbb7c76 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 043/151] drm/nouveau/imem: switch to kvmalloc/kvfree for suspend/resume backup Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index 8ad31b436fa03..36b3424149b3d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -37,7 +37,7 @@ nvkm_instobj_load(struct nvkm_instobj *iobj) for (i = 0; i < size; i += 4) nvkm_wo32(memory, i, iobj->suspend[i / 4]); - vfree(iobj->suspend); + kvfree(iobj->suspend); iobj->suspend = NULL; } @@ -48,7 +48,7 @@ nvkm_instobj_save(struct nvkm_instobj *iobj) const u64 size = nvkm_memory_size(memory); int i; - iobj->suspend = vmalloc(size); + iobj->suspend = kvmalloc(size, GFP_KERNEL); if (!iobj->suspend) return -ENOMEM; From b00b8430468d2922c5ea9a0557c7a36136df98c2 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 044/151] drm/nouveau/imem: separate pre-BAR2-bootstrap objects from the rest These will require slow-path access during suspend/resume. Signed-off-by: Ben Skeggs --- .../drm/nouveau/include/nvkm/subdev/instmem.h | 1 + .../drm/nouveau/nvkm/subdev/instmem/base.c | 27 +++++++++++++++++++ .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 1 + .../drm/nouveau/nvkm/subdev/instmem/priv.h | 1 + 4 files changed, 30 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/instmem.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/instmem.h index 40f845e312723..8111c0c3c5ecd 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/instmem.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/instmem.h @@ -9,6 +9,7 @@ struct nvkm_instmem { spinlock_t lock; struct list_head list; + struct list_head boot; u32 reserved; struct nvkm_memory *vbios; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index 36b3424149b3d..8fc63ec20d6ef 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -129,6 +129,21 @@ nvkm_instmem_wr32(struct nvkm_instmem *imem, u32 addr, u32 data) return imem->func->wr32(imem, addr, data); } +void +nvkm_instmem_boot(struct nvkm_instmem *imem) +{ + /* Separate bootstrapped objects from normal list, as we need + * to make sure they're accessed with the slowpath on suspend + * and resume. + */ + struct nvkm_instobj *iobj, *itmp; + spin_lock(&imem->lock); + list_for_each_entry_safe(iobj, itmp, &imem->list, head) { + list_move_tail(&iobj->head, &imem->boot); + } + spin_unlock(&imem->lock); +} + static int nvkm_instmem_fini(struct nvkm_subdev *subdev, bool suspend) { @@ -141,6 +156,12 @@ nvkm_instmem_fini(struct nvkm_subdev *subdev, bool suspend) if (ret) return ret; } + + list_for_each_entry(iobj, &imem->boot, head) { + int ret = nvkm_instobj_save(iobj); + if (ret) + return ret; + } } if (imem->func->fini) @@ -155,6 +176,11 @@ nvkm_instmem_init(struct nvkm_subdev *subdev) struct nvkm_instmem *imem = nvkm_instmem(subdev); struct nvkm_instobj *iobj; + list_for_each_entry(iobj, &imem->boot, head) { + if (iobj->suspend) + nvkm_instobj_load(iobj); + } + list_for_each_entry(iobj, &imem->list, head) { if (iobj->suspend) nvkm_instobj_load(iobj); @@ -198,4 +224,5 @@ nvkm_instmem_ctor(const struct nvkm_instmem_func *func, imem->func = func; spin_lock_init(&imem->lock); INIT_LIST_HEAD(&imem->list); + INIT_LIST_HEAD(&imem->boot); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index 374df1ebe2e85..be5670f9fefaf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -271,6 +271,7 @@ nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm) } nv50_instobj_kmap(iobj, vmm); + nvkm_instmem_boot(imem); mutex_unlock(&imem->subdev.mutex); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h index e7515d96b31ff..44651ca42d529 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h @@ -16,6 +16,7 @@ struct nvkm_instmem_func { void nvkm_instmem_ctor(const struct nvkm_instmem_func *, struct nvkm_device *, int index, struct nvkm_instmem *); +void nvkm_instmem_boot(struct nvkm_instmem *); #include From e9be3c7d7ae0aa6e273ac5cf66dfd2d53479bdbf Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 045/151] drm/nouveau/imem: use fast-path for suspend backup Before: "imem: suspend completed in 5540487us" After: "imem: suspend completed in 1871526us" Suspend from Fedora 26 gnome desktop on GP102. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index c8cb8b8b13232..190fb730de177 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -74,7 +74,6 @@ nvkm_bar_fini(struct nvkm_subdev *subdev, bool suspend) { struct nvkm_bar *bar = nvkm_bar(subdev); bar->func->bar1.fini(bar); - nvkm_bar_bar2_fini(subdev->device); return 0; } @@ -101,6 +100,7 @@ static void * nvkm_bar_dtor(struct nvkm_subdev *subdev) { struct nvkm_bar *bar = nvkm_bar(subdev); + nvkm_bar_bar2_fini(subdev->device); return bar->func->dtor(bar); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index 8fc63ec20d6ef..d7df7cb6bed37 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -46,15 +46,20 @@ nvkm_instobj_save(struct nvkm_instobj *iobj) { struct nvkm_memory *memory = &iobj->memory; const u64 size = nvkm_memory_size(memory); + void __iomem *map; int i; iobj->suspend = kvmalloc(size, GFP_KERNEL); if (!iobj->suspend) return -ENOMEM; - for (i = 0; i < size; i += 4) - iobj->suspend[i / 4] = nvkm_ro32(memory, i); - + if (!(map = nvkm_kmap(memory))) { + for (i = 0; i < size; i += 4) + iobj->suspend[i / 4] = nvkm_ro32(memory, i); + } else { + memcpy_fromio(iobj->suspend, map, size); + } + nvkm_done(memory); return 0; } @@ -157,6 +162,8 @@ nvkm_instmem_fini(struct nvkm_subdev *subdev, bool suspend) return ret; } + nvkm_bar_bar2_fini(subdev->device); + list_for_each_entry(iobj, &imem->boot, head) { int ret = nvkm_instobj_save(iobj); if (ret) From ffd937bbd219331e4b67344c104dea09b9ed4a6a Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 046/151] drm/nouveau/imem: use fast-path for resume restore Before: "imem: init completed in 299277us" After: "imem: init completed in 11574us" Suspend from Fedora 26 gnome desktop on GP102. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c | 13 +++++++++++-- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c | 3 +-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index 190fb730de177..9646adec57cbc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -81,7 +81,6 @@ static int nvkm_bar_init(struct nvkm_subdev *subdev) { struct nvkm_bar *bar = nvkm_bar(subdev); - nvkm_bar_bar2_init(subdev->device); bar->func->bar1.init(bar); bar->func->bar1.wait(bar); if (bar->func->init) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index d7df7cb6bed37..b03940591a3a0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -33,10 +33,17 @@ nvkm_instobj_load(struct nvkm_instobj *iobj) { struct nvkm_memory *memory = &iobj->memory; const u64 size = nvkm_memory_size(memory); + void __iomem *map; int i; - for (i = 0; i < size; i += 4) - nvkm_wo32(memory, i, iobj->suspend[i / 4]); + if (!(map = nvkm_kmap(memory))) { + for (i = 0; i < size; i += 4) + nvkm_wo32(memory, i, iobj->suspend[i / 4]); + } else { + memcpy_toio(map, iobj->suspend, size); + } + nvkm_done(memory); + kvfree(iobj->suspend); iobj->suspend = NULL; } @@ -188,6 +195,8 @@ nvkm_instmem_init(struct nvkm_subdev *subdev) nvkm_instobj_load(iobj); } + nvkm_bar_bar2_init(subdev->device); + list_for_each_entry(iobj, &imem->list, head) { if (iobj->suspend) nvkm_instobj_load(iobj); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index be5670f9fefaf..4ccb8cdc7dbcb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -206,7 +206,7 @@ nv50_instobj_release(struct nvkm_memory *memory) } /* Switch back to NULL accessors when last map is gone. */ - iobj->base.memory.ptrs = &nv50_instobj_slow; + iobj->base.memory.ptrs = NULL; mutex_unlock(&subdev->mutex); } } @@ -345,7 +345,6 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, *pmemory = &iobj->base.memory; nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base); - iobj->base.memory.ptrs = &nv50_instobj_slow; iobj->imem = imem; refcount_set(&iobj->maps, 0); INIT_LIST_HEAD(&iobj->lru); From 0b11b30de9d2960d87373e50223800c8f9f6a89f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 047/151] drm/nouveau/mmu/nv04-nv4x: move global vmm to nvkm_mmu In a future commit, this will be constructed by common code. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 2 ++ drivers/gpu/drm/nouveau/nouveau_ttm.c | 5 +---- drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c | 3 +-- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 12 ++++++------ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c | 8 ++++---- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c | 8 ++++---- 7 files changed, 18 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index d333c7668c236..c4326772d4816 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -58,6 +58,8 @@ struct nvkm_mmu { u64 limit; u8 dma_bits; u8 lpg_shift; + + struct nvkm_vmm *vmm; }; int nv04_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index ffc3458c89de8..65511f320a3a9 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -192,16 +192,13 @@ const struct ttm_mem_type_manager_func nouveau_gart_manager = { .debug = nouveau_gart_manager_debug }; -/*XXX*/ -#include static int nv04_gart_manager_init(struct ttm_mem_type_manager *man, unsigned long psize) { struct nouveau_drm *drm = nouveau_bdev(man->bdev); struct nvkm_mmu *mmu = nvxx_mmu(&drm->client.device); - struct nv04_mmu *priv = (void *)mmu; struct nvkm_vm *vm = NULL; - nvkm_vm_ref(priv->vm, &vm, NULL); + nvkm_vm_ref(mmu->vmm, &vm, NULL); man->priv = vm; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c index c95942ef82167..acc45ce5f93ec 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c @@ -49,8 +49,7 @@ nv04_dmaobj_bind(struct nvkm_dmaobj *base, struct nvkm_gpuobj *parent, int ret; if (dmaobj->clone) { - struct nv04_mmu *mmu = nv04_mmu(device->mmu); - struct nvkm_memory *pgt = mmu->vm->pgt[0].mem[0]; + struct nvkm_memory *pgt = device->mmu->vmm->pgt[0].mem[0]; if (!dmaobj->base.start) return nvkm_gpuobj_wrap(pgt, pgpuobj); nvkm_kmap(pgt); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index 37927c3fdc3e6..cd16442f1a918 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -81,15 +81,15 @@ nv04_mmu_oneinit(struct nvkm_mmu *base) int ret; ret = nvkm_vm_create(&mmu->base, 0, NV04_PDMA_SIZE, 0, 4096, NULL, - &mmu->vm); + &mmu->base.vmm); if (ret) return ret; ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, (NV04_PDMA_SIZE / NV04_PDMA_PAGE) * 4 + 8, 16, true, &dma); - mmu->vm->pgt[0].mem[0] = dma; - mmu->vm->pgt[0].refcount[0] = 1; + mmu->base.vmm->pgt[0].mem[0] = dma; + mmu->base.vmm->pgt[0].refcount[0] = 1; if (ret) return ret; @@ -105,9 +105,9 @@ nv04_mmu_dtor(struct nvkm_mmu *base) { struct nv04_mmu *mmu = nv04_mmu(base); struct nvkm_device *device = mmu->base.subdev.device; - if (mmu->vm) { - nvkm_memory_del(&mmu->vm->pgt[0].mem[0]); - nvkm_vm_ref(NULL, &mmu->vm, NULL); + if (mmu->base.vmm) { + nvkm_memory_del(&mmu->base.vmm->pgt[0].mem[0]); + nvkm_vm_ref(NULL, &mmu->base.vmm, NULL); } if (mmu->nullp) { dma_free_coherent(device->dev, 16 * 1024, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h index 363e33b296d56..6b8f14c6db3dc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h @@ -5,7 +5,6 @@ struct nv04_mmu { struct nvkm_mmu base; - struct nvkm_vm *vm; dma_addr_t null; void *nullp; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c index c6a26f907009c..9b5375c587a7f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c @@ -93,14 +93,14 @@ nv41_mmu_oneinit(struct nvkm_mmu *base) int ret; ret = nvkm_vm_create(&mmu->base, 0, NV41_GART_SIZE, 0, 4096, NULL, - &mmu->vm); + &mmu->base.vmm); if (ret) return ret; ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, (NV41_GART_SIZE / NV41_GART_PAGE) * 4, 16, true, - &mmu->vm->pgt[0].mem[0]); - mmu->vm->pgt[0].refcount[0] = 1; + &mmu->base.vmm->pgt[0].mem[0]); + mmu->base.vmm->pgt[0].refcount[0] = 1; return ret; } @@ -109,7 +109,7 @@ nv41_mmu_init(struct nvkm_mmu *base) { struct nv04_mmu *mmu = nv04_mmu(base); struct nvkm_device *device = mmu->base.subdev.device; - struct nvkm_memory *dma = mmu->vm->pgt[0].mem[0]; + struct nvkm_memory *dma = mmu->base.vmm->pgt[0].mem[0]; nvkm_wr32(device, 0x100800, 0x00000002 | nvkm_memory_addr(dma)); nvkm_mask(device, 0x10008c, 0x00000100, 0x00000100); nvkm_wr32(device, 0x100820, 0x00000000); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c index a648c2395545b..cc97500a09010 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c @@ -173,15 +173,15 @@ nv44_mmu_oneinit(struct nvkm_mmu *base) } ret = nvkm_vm_create(&mmu->base, 0, NV44_GART_SIZE, 0, 4096, NULL, - &mmu->vm); + &mmu->base.vmm); if (ret) return ret; ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, (NV44_GART_SIZE / NV44_GART_PAGE) * 4, 512 * 1024, true, - &mmu->vm->pgt[0].mem[0]); - mmu->vm->pgt[0].refcount[0] = 1; + &mmu->base.vmm->pgt[0].mem[0]); + mmu->base.vmm->pgt[0].refcount[0] = 1; return ret; } @@ -190,7 +190,7 @@ nv44_mmu_init(struct nvkm_mmu *base) { struct nv04_mmu *mmu = nv04_mmu(base); struct nvkm_device *device = mmu->base.subdev.device; - struct nvkm_memory *gart = mmu->vm->pgt[0].mem[0]; + struct nvkm_memory *gart = mmu->base.vmm->pgt[0].mem[0]; u32 addr; /* calculate vram address of this PRAMIN block, object must be From 2854ab8dd81b73ffd867399fd506f44f6f624bca Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 048/151] drm/nouveau/fb: finalise big page size selection in constructor MMU will need to know this during its constructor, so we can't delay deciding this until init-time. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c | 12 +++++++++--- drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c | 14 ++++++-------- drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h | 2 -- drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c | 17 +++++++---------- drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h | 8 +++++--- 10 files changed, 32 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c index a7049c041594b..fa0fad10b62cd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c @@ -135,8 +135,13 @@ nvkm_fb_init(struct nvkm_subdev *subdev) if (fb->func->init) fb->func->init(fb); - if (fb->func->init_page) - fb->func->init_page(fb); + + if (fb->func->init_page) { + ret = fb->func->init_page(fb); + if (WARN_ON(ret)) + return ret; + } + if (fb->func->init_unkn) fb->func->init_unkn(fb); return 0; @@ -176,7 +181,8 @@ nvkm_fb_ctor(const struct nvkm_fb_func *func, struct nvkm_device *device, nvkm_subdev_ctor(&nvkm_fb, device, index, &fb->subdev); fb->func = func; fb->tile.regions = fb->func->tile.regions; - fb->page = nvkm_longopt(device->cfgopt, "NvFbBigPage", 0); + fb->page = nvkm_longopt(device->cfgopt, "NvFbBigPage", + fb->func->default_bigpage); } int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c index a239e73562c81..a308a99ef64e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c @@ -80,20 +80,17 @@ gf100_fb_oneinit(struct nvkm_fb *base) return 0; } -void +int gf100_fb_init_page(struct nvkm_fb *fb) { struct nvkm_device *device = fb->subdev.device; switch (fb->page) { - case 16: - nvkm_mask(device, 0x100c80, 0x00000001, 0x00000001); - break; - case 17: + case 16: nvkm_mask(device, 0x100c80, 0x00000001, 0x00000001); break; + case 17: nvkm_mask(device, 0x100c80, 0x00000001, 0x00000000); break; default: - nvkm_mask(device, 0x100c80, 0x00000001, 0x00000000); - fb->page = 17; - break; + return -EINVAL; } + return 0; } void @@ -144,6 +141,7 @@ gf100_fb = { .intr = gf100_fb_intr, .ram_new = gf100_ram_new, .memtype_valid = gf100_fb_memtype_valid, + .default_bigpage = 17, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h index 412eb89834e8f..e3cf0515bb70a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h @@ -17,7 +17,5 @@ void gf100_fb_intr(struct nvkm_fb *); void gp100_fb_init(struct nvkm_fb *); -void gm200_fb_init_page(struct nvkm_fb *fb); void gm200_fb_init(struct nvkm_fb *base); - #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c index 56af84aa333b0..4c012fa02de19 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c @@ -33,6 +33,7 @@ gf108_fb = { .intr = gf100_fb_intr, .ram_new = gf108_ram_new, .memtype_valid = gf100_fb_memtype_valid, + .default_bigpage = 17, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c index 4245e2e6e6045..7ff647499acfe 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c @@ -33,6 +33,7 @@ gk104_fb = { .intr = gf100_fb_intr, .ram_new = gk104_ram_new, .memtype_valid = gf100_fb_memtype_valid, + .default_bigpage = 17, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c index 5d34d6136616a..59308575b1d4f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c @@ -31,6 +31,7 @@ gk20a_fb = { .init_page = gf100_fb_init_page, .intr = gf100_fb_intr, .memtype_valid = gf100_fb_memtype_valid, + .default_bigpage = 17, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c index db699025f5464..005aa9d6db322 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c @@ -33,6 +33,7 @@ gm107_fb = { .intr = gf100_fb_intr, .ram_new = gm107_ram_new, .memtype_valid = gf100_fb_memtype_valid, + .default_bigpage = 17, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c index d83da5ddbc1e8..cc3f90304c1c5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c @@ -26,22 +26,18 @@ #include -void +int gm200_fb_init_page(struct nvkm_fb *fb) { struct nvkm_device *device = fb->subdev.device; switch (fb->page) { - case 16: - nvkm_mask(device, 0x100c80, 0x00000801, 0x00000001); - break; - case 17: - nvkm_mask(device, 0x100c80, 0x00000801, 0x00000000); - break; + case 16: nvkm_mask(device, 0x100c80, 0x00000801, 0x00000001); break; + case 17: nvkm_mask(device, 0x100c80, 0x00000801, 0x00000000); break; + case 0: nvkm_mask(device, 0x100c80, 0x00000800, 0x00000800); break; default: - nvkm_mask(device, 0x100c80, 0x00000800, 0x00000800); - fb->page = 0; - break; + return -EINVAL; } + return 0; } void @@ -70,6 +66,7 @@ gm200_fb = { .intr = gf100_fb_intr, .ram_new = gm200_ram_new, .memtype_valid = gf100_fb_memtype_valid, + .default_bigpage = 0 /* per-instance. */, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c index b87c233bcd6db..e66a0a871043b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c @@ -31,6 +31,7 @@ gm20b_fb = { .init_page = gm200_fb_init_page, .intr = gf100_fb_intr, .memtype_valid = gf100_fb_memtype_valid, + .default_bigpage = 0 /* per-instance. */, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h index e905d44fa1d59..afaa68da32212 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h @@ -8,7 +8,7 @@ struct nvkm_fb_func { void *(*dtor)(struct nvkm_fb *); int (*oneinit)(struct nvkm_fb *); void (*init)(struct nvkm_fb *); - void (*init_page)(struct nvkm_fb *); + int (*init_page)(struct nvkm_fb *); void (*init_unkn)(struct nvkm_fb *); void (*intr)(struct nvkm_fb *); @@ -25,6 +25,8 @@ struct nvkm_fb_func { int (*ram_new)(struct nvkm_fb *, struct nvkm_ram **); bool (*memtype_valid)(struct nvkm_fb *, u32 memtype); + + u8 default_bigpage; }; void nvkm_fb_ctor(const struct nvkm_fb_func *, struct nvkm_device *device, @@ -62,8 +64,8 @@ void nv46_fb_tile_init(struct nvkm_fb *, int i, u32 addr, u32 size, u32 pitch, u32 flags, struct nvkm_fb_tile *); int gf100_fb_oneinit(struct nvkm_fb *); -void gf100_fb_init_page(struct nvkm_fb *); +int gf100_fb_init_page(struct nvkm_fb *); bool gf100_fb_memtype_valid(struct nvkm_fb *, u32); -void gm200_fb_init_page(struct nvkm_fb *); +int gm200_fb_init_page(struct nvkm_fb *); #endif From 223eaf4bf8a8c9668b82cb4a1c3a4acb60d33aba Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 049/151] drm/nouveau/gr/gf100-gk208: make use of init_gpc_mmu() hook to share setup Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 27 ++++++++++++------- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gf104.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gf108.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gf110.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gf117.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gf119.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gk104.c | 13 ++------- .../gpu/drm/nouveau/nvkm/engine/gr/gk110.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gk110b.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gk208.c | 1 + 11 files changed, 29 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 99689f4de502d..48556bfdbd4af 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1903,25 +1903,33 @@ gf100_gr_new_(const struct gf100_gr_func *func, struct nvkm_device *device, return 0; } +void +gf100_gr_init_gpc_mmu(struct gf100_gr *gr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + struct nvkm_fb *fb = device->fb; + + nvkm_wr32(device, 0x418880, 0x00000000); + nvkm_wr32(device, 0x4188a4, 0x00000000); + nvkm_wr32(device, 0x418888, 0x00000000); + nvkm_wr32(device, 0x41888c, 0x00000000); + nvkm_wr32(device, 0x418890, 0x00000000); + nvkm_wr32(device, 0x418894, 0x00000000); + nvkm_wr32(device, 0x4188b4, nvkm_memory_addr(fb->mmu_wr) >> 8); + nvkm_wr32(device, 0x4188b8, nvkm_memory_addr(fb->mmu_rd) >> 8); +} + int gf100_gr_init(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; - struct nvkm_fb *fb = device->fb; const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total); u32 data[TPC_MAX / 8] = {}; u8 tpcnr[GPC_MAX]; int gpc, tpc, rop; int i; - nvkm_wr32(device, GPC_BCAST(0x0880), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x08a4), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x0888), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x088c), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x0890), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x0894), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x08b4), nvkm_memory_addr(fb->mmu_wr) >> 8); - nvkm_wr32(device, GPC_BCAST(0x08b8), nvkm_memory_addr(fb->mmu_rd) >> 8); + gr->func->init_gpc_mmu(gr); gf100_gr_mmio(gr, gr->func->mmio); @@ -2036,6 +2044,7 @@ gf100_gr_gpccs_ucode = { static const struct gf100_gr_func gf100_gr = { .init = gf100_gr_init, + .init_gpc_mmu = gf100_gr_init_gpc_mmu, .mmio = gf100_gr_pack_mmio, .fecs.ucode = &gf100_gr_fecs_ucode, .gpccs.ucode = &gf100_gr_gpccs_ucode, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index 2901488163606..51c1e380ffb68 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -254,6 +254,7 @@ extern const struct gf100_gr_init gf100_gr_init_mpc_0[]; extern const struct gf100_gr_init gf100_gr_init_be_0[]; extern const struct gf100_gr_init gf100_gr_init_fe_1[]; extern const struct gf100_gr_init gf100_gr_init_pe_1[]; +void gf100_gr_init_gpc_mmu(struct gf100_gr *); extern const struct gf100_gr_init gf104_gr_init_ds_0[]; extern const struct gf100_gr_init gf104_gr_init_tex_0[]; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c index d736dcd55ea20..ec0f11983b233 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c @@ -115,6 +115,7 @@ gf104_gr_pack_mmio[] = { static const struct gf100_gr_func gf104_gr = { .init = gf100_gr_init, + .init_gpc_mmu = gf100_gr_init_gpc_mmu, .mmio = gf104_gr_pack_mmio, .fecs.ucode = &gf100_gr_fecs_ucode, .gpccs.ucode = &gf100_gr_gpccs_ucode, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c index 2f0d24498427b..cc152eb741235 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c @@ -106,6 +106,7 @@ gf108_gr_pack_mmio[] = { static const struct gf100_gr_func gf108_gr = { .init = gf100_gr_init, + .init_gpc_mmu = gf100_gr_init_gpc_mmu, .mmio = gf108_gr_pack_mmio, .fecs.ucode = &gf100_gr_fecs_ucode, .gpccs.ucode = &gf100_gr_gpccs_ucode, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c index d1d942eb86af2..10d2d73ca8c3a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c @@ -87,6 +87,7 @@ gf110_gr_pack_mmio[] = { static const struct gf100_gr_func gf110_gr = { .init = gf100_gr_init, + .init_gpc_mmu = gf100_gr_init_gpc_mmu, .mmio = gf110_gr_pack_mmio, .fecs.ucode = &gf100_gr_fecs_ucode, .gpccs.ucode = &gf100_gr_gpccs_ucode, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c index 0124e468086eb..ac09a07c4150b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c @@ -123,6 +123,7 @@ gf117_gr_gpccs_ucode = { static const struct gf100_gr_func gf117_gr = { .init = gf100_gr_init, + .init_gpc_mmu = gf100_gr_init_gpc_mmu, .mmio = gf117_gr_pack_mmio, .fecs.ucode = &gf117_gr_fecs_ucode, .gpccs.ucode = &gf117_gr_gpccs_ucode, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c index 8d8e4cafe28f8..7f449ec6f7603 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c @@ -178,6 +178,7 @@ gf119_gr_pack_mmio[] = { static const struct gf100_gr_func gf119_gr = { .init = gf100_gr_init, + .init_gpc_mmu = gf100_gr_init_gpc_mmu, .mmio = gf119_gr_pack_mmio, .fecs.ucode = &gf100_gr_fecs_ucode, .gpccs.ucode = &gf100_gr_gpccs_ucode, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c index ec22da6c99fc4..5e82f94c22451 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c @@ -24,8 +24,6 @@ #include "gf100.h" #include "ctxgf100.h" -#include - #include /******************************************************************************* @@ -207,21 +205,13 @@ int gk104_gr_init(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; - struct nvkm_fb *fb = device->fb; const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total); u32 data[TPC_MAX / 8] = {}; u8 tpcnr[GPC_MAX]; int gpc, tpc, rop; int i; - nvkm_wr32(device, GPC_BCAST(0x0880), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x08a4), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x0888), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x088c), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x0890), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x0894), 0x00000000); - nvkm_wr32(device, GPC_BCAST(0x08b4), nvkm_memory_addr(fb->mmu_wr) >> 8); - nvkm_wr32(device, GPC_BCAST(0x08b8), nvkm_memory_addr(fb->mmu_rd) >> 8); + gr->func->init_gpc_mmu(gr); gf100_gr_mmio(gr, gr->func->mmio); @@ -339,6 +329,7 @@ gk104_gr_gpccs_ucode = { static const struct gf100_gr_func gk104_gr = { .init = gk104_gr_init, + .init_gpc_mmu = gf100_gr_init_gpc_mmu, .init_rop_active_fbps = gk104_gr_init_rop_active_fbps, .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, .mmio = gk104_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c index f31b171a4102d..a38e19b61c1da 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c @@ -183,6 +183,7 @@ gk110_gr_gpccs_ucode = { static const struct gf100_gr_func gk110_gr = { .init = gk104_gr_init, + .init_gpc_mmu = gf100_gr_init_gpc_mmu, .init_rop_active_fbps = gk104_gr_init_rop_active_fbps, .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, .mmio = gk110_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c index d76dd178007fb..1912c0bfd7eed 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c @@ -103,6 +103,7 @@ gk110b_gr_pack_mmio[] = { static const struct gf100_gr_func gk110b_gr = { .init = gk104_gr_init, + .init_gpc_mmu = gf100_gr_init_gpc_mmu, .init_rop_active_fbps = gk104_gr_init_rop_active_fbps, .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, .mmio = gk110b_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c index 14bbe6ed02a9a..1fc258163f25a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c @@ -162,6 +162,7 @@ gk208_gr_gpccs_ucode = { static const struct gf100_gr_func gk208_gr = { .init = gk104_gr_init, + .init_gpc_mmu = gf100_gr_init_gpc_mmu, .init_rop_active_fbps = gk104_gr_init_rop_active_fbps, .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, .mmio = gk208_gr_pack_mmio, From b6838c14a28c06ea51a60a829ec290e6aec9d128 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 050/151] drm/nouveau/gr/gf100-gk208: copy big page size setting from fb Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 48556bfdbd4af..49c7d08b0ceb4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1909,7 +1909,7 @@ gf100_gr_init_gpc_mmu(struct gf100_gr *gr) struct nvkm_device *device = gr->base.engine.subdev.device; struct nvkm_fb *fb = device->fb; - nvkm_wr32(device, 0x418880, 0x00000000); + nvkm_wr32(device, 0x418880, nvkm_rd32(device, 0x100c80) & 0x00000001); nvkm_wr32(device, 0x4188a4, 0x00000000); nvkm_wr32(device, 0x418888, 0x00000000); nvkm_wr32(device, 0x41888c, 0x00000000); From e75c091baf99dd739f6c3c1135fd667af174bd45 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 051/151] drm/nouveau: store nouveau_drm in nouveau_cli, as opposed to drm_device Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bo.c | 2 +- drivers/gpu/drm/nouveau/nouveau_drm.c | 2 +- drivers/gpu/drm/nouveau/nouveau_drv.h | 4 ++-- drivers/gpu/drm/nouveau/nouveau_gem.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 3da87932a21f9..e1eb8ba781fe8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -187,7 +187,7 @@ nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, struct sg_table *sg, struct reservation_object *robj, struct nouveau_bo **pnvbo) { - struct nouveau_drm *drm = nouveau_drm(cli->dev); + struct nouveau_drm *drm = cli->drm; struct nouveau_bo *nvbo; size_t acc_size; int ret; diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 595630d1fb9e2..632edb73ed538 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -128,7 +128,7 @@ nouveau_cli_init(struct nouveau_drm *drm, const char *sname, int ret; snprintf(cli->name, sizeof(cli->name), "%s", sname); - cli->dev = drm->dev; + cli->drm = drm; mutex_init(&cli->mutex); usif_client_init(cli); diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index ec60d52790e33..84b8470422539 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -86,7 +86,7 @@ enum nouveau_drm_handle { struct nouveau_cli { struct nvif_client base; - struct drm_device *dev; + struct nouveau_drm *drm; struct mutex mutex; struct nvif_device device; @@ -204,7 +204,7 @@ void nouveau_drm_device_remove(struct drm_device *dev); #define NV_PRINTK(l,c,f,a...) do { \ struct nouveau_cli *_cli = (c); \ - dev_##l(_cli->dev->dev, "%s: "f, _cli->name, ##a); \ + dev_##l(_cli->drm->dev->dev, "%s: "f, _cli->name, ##a); \ } while(0) #define NV_FATAL(drm,f,a...) NV_PRINTK(crit, &(drm)->client, f, ##a) #define NV_ERROR(drm,f,a...) NV_PRINTK(err, &(drm)->client, f, ##a) diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 2170534101caf..23140c4f3ac5f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -179,7 +179,7 @@ nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, uint32_t tile_mode, uint32_t tile_flags, struct nouveau_bo **pnvbo) { - struct nouveau_drm *drm = nouveau_drm(cli->dev); + struct nouveau_drm *drm = cli->drm; struct nouveau_bo *nvbo; u32 flags = 0; int ret; From bc3b0c7aff4d91605d966c01571f579827449d38 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 052/151] drm/nouveau: remove unused sysmem fence code Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_fence.c | 2 -- drivers/gpu/drm/nouveau/nouveau_fence.h | 4 --- drivers/gpu/drm/nouveau/nv84_fence.c | 38 ++----------------------- 3 files changed, 2 insertions(+), 42 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c index 99e14e3e0fe4d..3e4412d11571a 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.c +++ b/drivers/gpu/drm/nouveau/nouveau_fence.c @@ -474,8 +474,6 @@ nouveau_fence_new(struct nouveau_channel *chan, bool sysmem, if (!fence) return -ENOMEM; - fence->sysmem = sysmem; - ret = nouveau_fence_emit(fence, chan); if (ret) nouveau_fence_unref(&fence); diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h index d5e58a38f1601..28a2135097970 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.h +++ b/drivers/gpu/drm/nouveau/nouveau_fence.h @@ -12,8 +12,6 @@ struct nouveau_fence { struct list_head head; - bool sysmem; - struct nouveau_channel __rcu *channel; unsigned long timeout; }; @@ -91,13 +89,11 @@ int nouveau_flip_complete(struct nvif_notify *); struct nv84_fence_chan { struct nouveau_fence_chan base; struct nvkm_vma vma; - struct nvkm_vma vma_gart; }; struct nv84_fence_priv { struct nouveau_fence_priv base; struct nouveau_bo *bo; - struct nouveau_bo *bo_gart; u32 *suspend; struct mutex mutex; }; diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c index bd7a8a1e4ad97..e5f6249edde9e 100644 --- a/drivers/gpu/drm/nouveau/nv84_fence.c +++ b/drivers/gpu/drm/nouveau/nv84_fence.c @@ -68,12 +68,7 @@ nv84_fence_emit(struct nouveau_fence *fence) { struct nouveau_channel *chan = fence->channel; struct nv84_fence_chan *fctx = chan->fence; - u64 addr = chan->chid * 16; - - if (fence->sysmem) - addr += fctx->vma_gart.offset; - else - addr += fctx->vma.offset; + u64 addr = fctx->vma.offset + chan->chid * 16; return fctx->base.emit32(chan, addr, fence->base.seqno); } @@ -83,12 +78,7 @@ nv84_fence_sync(struct nouveau_fence *fence, struct nouveau_channel *prev, struct nouveau_channel *chan) { struct nv84_fence_chan *fctx = chan->fence; - u64 addr = prev->chid * 16; - - if (fence->sysmem) - addr += fctx->vma_gart.offset; - else - addr += fctx->vma.offset; + u64 addr = fctx->vma.offset + prev->chid * 16; return fctx->base.sync32(chan, addr, fence->base.seqno); } @@ -108,7 +98,6 @@ nv84_fence_context_del(struct nouveau_channel *chan) nouveau_bo_wr32(priv->bo, chan->chid * 16 / 4, fctx->base.sequence); mutex_lock(&priv->mutex); - nouveau_bo_vma_del(priv->bo, &fctx->vma_gart); nouveau_bo_vma_del(priv->bo, &fctx->vma); mutex_unlock(&priv->mutex); nouveau_fence_context_del(&fctx->base); @@ -138,10 +127,6 @@ nv84_fence_context_new(struct nouveau_channel *chan) mutex_lock(&priv->mutex); ret = nouveau_bo_vma_add(priv->bo, cli->vm, &fctx->vma); - if (ret == 0) { - ret = nouveau_bo_vma_add(priv->bo_gart, cli->vm, - &fctx->vma_gart); - } mutex_unlock(&priv->mutex); if (ret) @@ -182,10 +167,6 @@ static void nv84_fence_destroy(struct nouveau_drm *drm) { struct nv84_fence_priv *priv = drm->fence; - nouveau_bo_unmap(priv->bo_gart); - if (priv->bo_gart) - nouveau_bo_unpin(priv->bo_gart); - nouveau_bo_ref(NULL, &priv->bo_gart); nouveau_bo_unmap(priv->bo); if (priv->bo) nouveau_bo_unpin(priv->bo); @@ -238,21 +219,6 @@ nv84_fence_create(struct nouveau_drm *drm) nouveau_bo_ref(NULL, &priv->bo); } - if (ret == 0) - ret = nouveau_bo_new(&drm->client, 16 * priv->base.contexts, 0, - TTM_PL_FLAG_TT | TTM_PL_FLAG_UNCACHED, 0, - 0, NULL, NULL, &priv->bo_gart); - if (ret == 0) { - ret = nouveau_bo_pin(priv->bo_gart, TTM_PL_FLAG_TT, false); - if (ret == 0) { - ret = nouveau_bo_map(priv->bo_gart); - if (ret) - nouveau_bo_unpin(priv->bo_gart); - } - if (ret) - nouveau_bo_ref(NULL, &priv->bo_gart); - } - if (ret) nv84_fence_destroy(drm); return ret; From 7760a2e38a8324688e83b91f91ff7be710e70db1 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 053/151] drm/nouveau: split various bo flags out into their own members It's far more convenient to deal with like this. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bo.c | 71 +++++++++++++++----------- drivers/gpu/drm/nouveau/nouveau_bo.h | 10 ++-- drivers/gpu/drm/nouveau/nouveau_gem.c | 11 +++- drivers/gpu/drm/nouveau/nouveau_gem.h | 3 -- drivers/gpu/drm/nouveau/nouveau_ttm.c | 14 +++-- drivers/gpu/drm/nouveau/nv50_display.c | 8 +-- 6 files changed, 66 insertions(+), 51 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index e1eb8ba781fe8..40a8c6589f3db 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -97,7 +97,7 @@ nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile, static struct nouveau_drm_tile * nv10_bo_set_tiling(struct drm_device *dev, u32 addr, - u32 size, u32 pitch, u32 flags) + u32 size, u32 pitch, u32 zeta) { struct nouveau_drm *drm = nouveau_drm(dev); struct nvkm_fb *fb = nvxx_fb(&drm->client.device); @@ -120,8 +120,7 @@ nv10_bo_set_tiling(struct drm_device *dev, u32 addr, } if (found) - nv10_bo_update_tile_region(dev, found, addr, size, - pitch, flags); + nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta); return found; } @@ -155,27 +154,27 @@ nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags, struct nvif_device *device = &drm->client.device; if (device->info.family < NV_DEVICE_INFO_V0_TESLA) { - if (nvbo->tile_mode) { + if (nvbo->mode) { if (device->info.chipset >= 0x40) { *align = 65536; - *size = roundup_64(*size, 64 * nvbo->tile_mode); + *size = roundup_64(*size, 64 * nvbo->mode); } else if (device->info.chipset >= 0x30) { *align = 32768; - *size = roundup_64(*size, 64 * nvbo->tile_mode); + *size = roundup_64(*size, 64 * nvbo->mode); } else if (device->info.chipset >= 0x20) { *align = 16384; - *size = roundup_64(*size, 64 * nvbo->tile_mode); + *size = roundup_64(*size, 64 * nvbo->mode); } else if (device->info.chipset >= 0x10) { *align = 16384; - *size = roundup_64(*size, 32 * nvbo->tile_mode); + *size = roundup_64(*size, 32 * nvbo->mode); } } } else { - *size = roundup_64(*size, (1 << nvbo->page_shift)); - *align = max((1 << nvbo->page_shift), *align); + *size = roundup_64(*size, (1 << nvbo->page)); + *align = max((1 << nvbo->page), *align); } *size = roundup_64(*size, PAGE_SIZE); @@ -207,18 +206,34 @@ nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, INIT_LIST_HEAD(&nvbo->head); INIT_LIST_HEAD(&nvbo->entry); INIT_LIST_HEAD(&nvbo->vma_list); - nvbo->tile_mode = tile_mode; - nvbo->tile_flags = tile_flags; nvbo->bo.bdev = &drm->ttm.bdev; nvbo->cli = cli; if (!nvxx_device(&drm->client.device)->func->cpu_coherent) nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED; - nvbo->page_shift = 12; + if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) { + nvbo->kind = (tile_flags & 0x0000ff00) >> 8; + nvbo->comp = gf100_pte_storage_type_map[nvbo->kind] != nvbo->kind; + } else + if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { + nvbo->kind = (tile_flags & 0x00007f00) >> 8; + nvbo->comp = (tile_flags & 0x00030000) >> 16; + } else { + nvbo->zeta = (tile_flags & 0x00000007); + } + nvbo->mode = tile_mode; + nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG); + + nvbo->page = 12; if (drm->client.vm) { if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024) - nvbo->page_shift = drm->client.vm->mmu->lpg_shift; + nvbo->page = drm->client.vm->mmu->lpg_shift; + else { + if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) + nvbo->kind = gf100_pte_storage_type_map[nvbo->kind]; + nvbo->comp = 0; + } } nouveau_bo_fixup_align(nvbo, flags, &align, &size); @@ -262,7 +277,7 @@ set_placement_range(struct nouveau_bo *nvbo, uint32_t type) unsigned i, fpfn, lpfn; if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS && - nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) && + nvbo->mode && (type & TTM_PL_FLAG_VRAM) && nvbo->bo.mem.num_pages < vram_pages / 4) { /* * Make sure that the color and depth buffers are handled @@ -270,7 +285,7 @@ set_placement_range(struct nouveau_bo *nvbo, uint32_t type) * speed up when alpha-blending and depth-test are enabled * at the same time. */ - if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) { + if (nvbo->zeta) { fpfn = vram_pages / 2; lpfn = ~0; } else { @@ -321,14 +336,10 @@ nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig) if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && memtype == TTM_PL_FLAG_VRAM && contig) { - if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) { - if (bo->mem.mem_type == TTM_PL_VRAM) { - struct nvkm_mem *mem = bo->mem.mm_node; - if (!nvkm_mm_contiguous(mem->mem)) - evict = true; - } - nvbo->tile_flags &= ~NOUVEAU_GEM_TILE_NONCONTIG; + if (!nvbo->contig) { + nvbo->contig = true; force = true; + evict = true; } } @@ -376,7 +387,7 @@ nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig) out: if (force && ret) - nvbo->tile_flags |= NOUVEAU_GEM_TILE_NONCONTIG; + nvbo->contig = false; ttm_bo_unreserve(bo); return ret; } @@ -1210,7 +1221,7 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict, list_for_each_entry(vma, &nvbo->vma_list, head) { if (new_reg && new_reg->mem_type != TTM_PL_SYSTEM && (new_reg->mem_type == TTM_PL_VRAM || - nvbo->page_shift != vma->vm->mmu->lpg_shift)) { + nvbo->page != vma->vm->mmu->lpg_shift)) { nvkm_vm_map(vma, new_reg->mm_node); } else { WARN_ON(ttm_bo_wait(bo, false, false)); @@ -1234,8 +1245,7 @@ nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_reg, if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) { *new_tile = nv10_bo_set_tiling(dev, offset, new_reg->size, - nvbo->tile_mode, - nvbo->tile_flags); + nvbo->mode, nvbo->zeta); } return 0; @@ -1408,7 +1418,7 @@ nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) */ if (bo->mem.mem_type != TTM_PL_VRAM) { if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || - !nouveau_bo_tile_layout(nvbo)) + !nvbo->kind) return 0; if (bo->mem.mem_type == TTM_PL_SYSTEM) { @@ -1596,14 +1606,13 @@ nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nvkm_vm *vm, const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT; int ret; - ret = nvkm_vm_get(vm, size, nvbo->page_shift, - NV_MEM_ACCESS_RW, vma); + ret = nvkm_vm_get(vm, size, nvbo->page, NV_MEM_ACCESS_RW, vma); if (ret) return ret; if ( nvbo->bo.mem.mem_type != TTM_PL_SYSTEM && (nvbo->bo.mem.mem_type == TTM_PL_VRAM || - nvbo->page_shift != vma->vm->mmu->lpg_shift)) + nvbo->page != vma->vm->mmu->lpg_shift)) nvkm_vm_map(vma, nvbo->bo.mem.mm_node); list_add_tail(&vma->head, &nvbo->vma_list); diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.h b/drivers/gpu/drm/nouveau/nouveau_bo.h index b06a5385d6dd1..a179bbf25dced 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.h +++ b/drivers/gpu/drm/nouveau/nouveau_bo.h @@ -24,12 +24,16 @@ struct nouveau_bo { bool validate_mapped; struct list_head vma_list; - unsigned page_shift; struct nouveau_cli *cli; - u32 tile_mode; - u32 tile_flags; + unsigned contig:1; + unsigned page:5; + unsigned kind:8; + unsigned comp:3; + unsigned zeta:3; + unsigned mode; + struct nouveau_drm_tile *tile; /* Only valid if allocated via nouveau_gem_new() and iff you hold a diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 23140c4f3ac5f..2d924739997af 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -246,8 +246,15 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); - rep->tile_mode = nvbo->tile_mode; - rep->tile_flags = nvbo->tile_flags; + rep->tile_mode = nvbo->mode; + rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; + if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) + rep->tile_flags |= nvbo->kind << 8; + else + if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) + rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; + else + rep->tile_flags |= nvbo->zeta; return 0; } diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.h b/drivers/gpu/drm/nouveau/nouveau_gem.h index 8fa6ed9ddd3af..d39f845dda875 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.h +++ b/drivers/gpu/drm/nouveau/nouveau_gem.h @@ -6,9 +6,6 @@ #include "nouveau_drv.h" #include "nouveau_bo.h" -#define nouveau_bo_tile_layout(nvbo) \ - ((nvbo)->tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK) - static inline struct nouveau_bo * nouveau_gem_object(struct drm_gem_object *gem) { diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index 65511f320a3a9..b8727413db12b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -88,18 +88,18 @@ nouveau_vram_manager_new(struct ttm_mem_type_manager *man, if (drm->client.device.info.ram_size == 0) return -ENOMEM; - if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) - size_nc = 1 << nvbo->page_shift; + if (!nvbo->contig) + size_nc = 1 << nvbo->page; ret = ram->func->get(ram, reg->num_pages << PAGE_SHIFT, reg->page_alignment << PAGE_SHIFT, size_nc, - (nvbo->tile_flags >> 8) & 0x3ff, &node); + nvbo->comp << 8 | nvbo->kind, &node); if (ret) { reg->mm_node = NULL; return (ret == -ENOSPC) ? 0 : ret; } - node->page_shift = nvbo->page_shift; + node->page_shift = nvbo->page; reg->mm_node = node; reg->start = node->offset >> PAGE_SHIFT; @@ -158,14 +158,12 @@ nouveau_gart_manager_new(struct ttm_mem_type_manager *man, case NV_DEVICE_INFO_V0_CURIE: break; case NV_DEVICE_INFO_V0_TESLA: - if (drm->client.device.info.chipset != 0x50) - node->memtype = (nvbo->tile_flags & 0x7f00) >> 8; - break; case NV_DEVICE_INFO_V0_FERMI: case NV_DEVICE_INFO_V0_KEPLER: case NV_DEVICE_INFO_V0_MAXWELL: case NV_DEVICE_INFO_V0_PASCAL: - node->memtype = (nvbo->tile_flags & 0xff00) >> 8; + if (drm->client.device.info.chipset != 0x50) + node->memtype = nvbo->kind; break; default: NV_WARN(drm, "%s: unhandled family type %x\n", __func__, diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c index 1f3872f438c97..92d46222c79df 100644 --- a/drivers/gpu/drm/nouveau/nv50_display.c +++ b/drivers/gpu/drm/nouveau/nv50_display.c @@ -424,7 +424,7 @@ nv50_dmac_ctxdma_new(struct nv50_dmac *dmac, struct nouveau_framebuffer *fb) { struct nouveau_drm *drm = nouveau_drm(fb->base.dev); struct nv50_dmac_ctxdma *ctxdma; - const u8 kind = (fb->nvbo->tile_flags & 0x0000ff00) >> 8; + const u8 kind = fb->nvbo->kind; const u32 handle = 0xfb000000 | kind; struct { struct nv_dma_v0 base; @@ -847,7 +847,7 @@ nv50_wndw_atomic_check_acquire(struct nv50_wndw *wndw, asyw->image.w = fb->base.width; asyw->image.h = fb->base.height; - asyw->image.kind = (fb->nvbo->tile_flags & 0x0000ff00) >> 8; + asyw->image.kind = fb->nvbo->kind; if (asyh->state.pageflip_flags & DRM_MODE_PAGE_FLIP_ASYNC) asyw->interval = 0; @@ -857,9 +857,9 @@ nv50_wndw_atomic_check_acquire(struct nv50_wndw *wndw, if (asyw->image.kind) { asyw->image.layout = 0; if (drm->client.device.info.chipset >= 0xc0) - asyw->image.block = fb->nvbo->tile_mode >> 4; + asyw->image.block = fb->nvbo->mode >> 4; else - asyw->image.block = fb->nvbo->tile_mode; + asyw->image.block = fb->nvbo->mode; asyw->image.pitch = (fb->base.pitches[0] / 4) << 4; } else { asyw->image.layout = 1; From 425b34f7df63a2eacf24b721a6f450126e691b6a Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 054/151] drm/nouveau: simplify const-va map condition We don't really care about where the memory is, just that it's compatible with a VMA allocated for a given page size. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bo.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 40a8c6589f3db..22d4b90de432e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -1212,6 +1212,7 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict, struct ttm_mem_reg *new_reg) { struct nouveau_bo *nvbo = nouveau_bo(bo); + struct nvkm_mem *mem = new_reg ? new_reg->mm_node : NULL; struct nvkm_vma *vma; /* ttm can now (stupidly) pass the driver bos it didn't create... */ @@ -1219,10 +1220,9 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict, return; list_for_each_entry(vma, &nvbo->vma_list, head) { - if (new_reg && new_reg->mem_type != TTM_PL_SYSTEM && - (new_reg->mem_type == TTM_PL_VRAM || - nvbo->page != vma->vm->mmu->lpg_shift)) { - nvkm_vm_map(vma, new_reg->mm_node); + if (mem && new_reg->mem_type != TTM_PL_SYSTEM && + mem->page_shift == nvbo->page) { + nvkm_vm_map(vma, mem); } else { WARN_ON(ttm_bo_wait(bo, false, false)); nvkm_vm_unmap(vma); @@ -1604,15 +1604,15 @@ nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nvkm_vm *vm, struct nvkm_vma *vma) { const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT; + struct nvkm_mem *mem = nvbo->bo.mem.mm_node; int ret; ret = nvkm_vm_get(vm, size, nvbo->page, NV_MEM_ACCESS_RW, vma); if (ret) return ret; - if ( nvbo->bo.mem.mem_type != TTM_PL_SYSTEM && - (nvbo->bo.mem.mem_type == TTM_PL_VRAM || - nvbo->page != vma->vm->mmu->lpg_shift)) + if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM && + mem->page_shift == nvbo->page) nvkm_vm_map(vma, nvbo->bo.mem.mm_node); list_add_tail(&vma->head, &nvbo->vma_list); From a48296ab9dbe8b384ba58db5e8553ba35b774bb2 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 055/151] drm/nouveau: swap loop order in move_notify() hook The conditional is the same for every mapping. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bo.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 22d4b90de432e..50b400edcb88e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -1219,11 +1219,13 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict, if (bo->destroy != nouveau_bo_del_ttm) return; - list_for_each_entry(vma, &nvbo->vma_list, head) { - if (mem && new_reg->mem_type != TTM_PL_SYSTEM && - mem->page_shift == nvbo->page) { + if (mem && new_reg->mem_type != TTM_PL_SYSTEM && + mem->page_shift == nvbo->page) { + list_for_each_entry(vma, &nvbo->vma_list, head) { nvkm_vm_map(vma, mem); - } else { + } + } else { + list_for_each_entry(vma, &nvbo->vma_list, head) { WARN_ON(ttm_bo_wait(bo, false, false)); nvkm_vm_unmap(vma); } From 792067e00b4e1db30b6d27749a6b902d92564c1f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 056/151] drm/nouveau: remove unnecessary use of ttm_mem_type_manager::priv Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_ttm.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index b8727413db12b..663f2983914f4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -35,16 +35,12 @@ static int nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long psize) { - struct nouveau_drm *drm = nouveau_bdev(man->bdev); - struct nvkm_fb *fb = nvxx_fb(&drm->client.device); - man->priv = fb; return 0; } static int nouveau_vram_manager_fini(struct ttm_mem_type_manager *man) { - man->priv = NULL; return 0; } @@ -193,20 +189,12 @@ const struct ttm_mem_type_manager_func nouveau_gart_manager = { static int nv04_gart_manager_init(struct ttm_mem_type_manager *man, unsigned long psize) { - struct nouveau_drm *drm = nouveau_bdev(man->bdev); - struct nvkm_mmu *mmu = nvxx_mmu(&drm->client.device); - struct nvkm_vm *vm = NULL; - nvkm_vm_ref(mmu->vmm, &vm, NULL); - man->priv = vm; return 0; } static int nv04_gart_manager_fini(struct ttm_mem_type_manager *man) { - struct nvkm_vm *vm = man->priv; - nvkm_vm_ref(NULL, &vm, NULL); - man->priv = NULL; return 0; } @@ -226,6 +214,8 @@ nv04_gart_manager_new(struct ttm_mem_type_manager *man, const struct ttm_place *place, struct ttm_mem_reg *reg) { + struct nouveau_drm *drm = nouveau_bdev(man->bdev); + struct nvkm_mmu *mmu = nvxx_mmu(&drm->client.device); struct nvkm_mem *node; int ret; @@ -235,7 +225,7 @@ nv04_gart_manager_new(struct ttm_mem_type_manager *man, node->page_shift = 12; - ret = nvkm_vm_get(man->priv, reg->num_pages << 12, node->page_shift, + ret = nvkm_vm_get(mmu->vmm, reg->num_pages << 12, node->page_shift, NV_MEM_ACCESS_RW, &node->vma[0]); if (ret) { if (ret == -ENOSPC) { From 6be4421a9f5f6d1e980a88e189453cb766744cac Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 057/151] drm/nouveau: consolidate identical functions in nouveau_ttm.c Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_ttm.c | 63 ++++++++------------------- 1 file changed, 17 insertions(+), 46 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index 663f2983914f4..e6e767558a3b4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -33,17 +33,23 @@ #include static int -nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long psize) +nouveau_manager_init(struct ttm_mem_type_manager *man, unsigned long psize) { return 0; } static int -nouveau_vram_manager_fini(struct ttm_mem_type_manager *man) +nouveau_manager_fini(struct ttm_mem_type_manager *man) { return 0; } +static void +nouveau_manager_debug(struct ttm_mem_type_manager *man, + struct drm_printer *printer) +{ +} + static inline void nvkm_mem_node_cleanup(struct nvkm_mem *node) { @@ -103,24 +109,13 @@ nouveau_vram_manager_new(struct ttm_mem_type_manager *man, } const struct ttm_mem_type_manager_func nouveau_vram_manager = { - .init = nouveau_vram_manager_init, - .takedown = nouveau_vram_manager_fini, + .init = nouveau_manager_init, + .takedown = nouveau_manager_fini, .get_node = nouveau_vram_manager_new, .put_node = nouveau_vram_manager_del, + .debug = nouveau_manager_debug, }; -static int -nouveau_gart_manager_init(struct ttm_mem_type_manager *man, unsigned long psize) -{ - return 0; -} - -static int -nouveau_gart_manager_fini(struct ttm_mem_type_manager *man) -{ - return 0; -} - static void nouveau_gart_manager_del(struct ttm_mem_type_manager *man, struct ttm_mem_reg *reg) @@ -172,32 +167,14 @@ nouveau_gart_manager_new(struct ttm_mem_type_manager *man, return 0; } -static void -nouveau_gart_manager_debug(struct ttm_mem_type_manager *man, - struct drm_printer *printer) -{ -} - const struct ttm_mem_type_manager_func nouveau_gart_manager = { - .init = nouveau_gart_manager_init, - .takedown = nouveau_gart_manager_fini, + .init = nouveau_manager_init, + .takedown = nouveau_manager_fini, .get_node = nouveau_gart_manager_new, .put_node = nouveau_gart_manager_del, - .debug = nouveau_gart_manager_debug + .debug = nouveau_manager_debug }; -static int -nv04_gart_manager_init(struct ttm_mem_type_manager *man, unsigned long psize) -{ - return 0; -} - -static int -nv04_gart_manager_fini(struct ttm_mem_type_manager *man) -{ - return 0; -} - static void nv04_gart_manager_del(struct ttm_mem_type_manager *man, struct ttm_mem_reg *reg) { @@ -241,18 +218,12 @@ nv04_gart_manager_new(struct ttm_mem_type_manager *man, return 0; } -static void -nv04_gart_manager_debug(struct ttm_mem_type_manager *man, - struct drm_printer *printer) -{ -} - const struct ttm_mem_type_manager_func nv04_gart_manager = { - .init = nv04_gart_manager_init, - .takedown = nv04_gart_manager_fini, + .init = nouveau_manager_init, + .takedown = nouveau_manager_fini, .get_node = nv04_gart_manager_new, .put_node = nv04_gart_manager_del, - .debug = nv04_gart_manager_debug + .debug = nouveau_manager_debug }; int From cb7e88e70f52878f4be0fbcc04350cff72f67278 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 058/151] drm/nouveau: hang drm client of a master TTM memory allocations will be hanging off the DRM's client, but the locking needed to do so gets really tricky with all the other use of the DRM's object tree. To solve this, we make the normal DRM client a child of a new master, where the memory allocations will be done from instead. This also solves a potential race with client creation. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_drm.c | 20 ++++++++++++++++---- drivers/gpu/drm/nouveau/nouveau_drv.h | 3 +++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 632edb73ed538..a705e56f0e578 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -117,7 +117,9 @@ nouveau_cli_fini(struct nouveau_cli *cli) nvkm_vm_ref(NULL, &nvxx_client(&cli->base)->vm, NULL); usif_client_fini(cli); nvif_device_fini(&cli->device); + mutex_lock(&cli->drm->master.lock); nvif_client_fini(&cli->base); + mutex_unlock(&cli->drm->master.lock); } static int @@ -132,12 +134,16 @@ nouveau_cli_init(struct nouveau_drm *drm, const char *sname, mutex_init(&cli->mutex); usif_client_init(cli); - if (cli == &drm->client) { + mutex_init(&cli->lock); + + if (cli == &drm->master) { ret = nvif_driver_init(NULL, nouveau_config, nouveau_debug, cli->name, device, &cli->base); } else { - ret = nvif_client_init(&drm->client.base, cli->name, device, + mutex_lock(&drm->master.lock); + ret = nvif_client_init(&drm->master.base, cli->name, device, &cli->base); + mutex_unlock(&drm->master.lock); } if (ret) { NV_ERROR(drm, "Client allocation failed: %d\n", ret); @@ -433,6 +439,10 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags) dev->dev_private = drm; drm->dev = dev; + ret = nouveau_cli_init(drm, "DRM-master", &drm->master); + if (ret) + return ret; + ret = nouveau_cli_init(drm, "DRM", &drm->client); if (ret) return ret; @@ -518,6 +528,7 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags) nouveau_vga_fini(drm); fail_device: nouveau_cli_fini(&drm->client); + nouveau_cli_fini(&drm->master); kfree(drm); return ret; } @@ -550,6 +561,7 @@ nouveau_drm_unload(struct drm_device *dev) if (drm->hdmi_device) pci_dev_put(drm->hdmi_device); nouveau_cli_fini(&drm->client); + nouveau_cli_fini(&drm->master); kfree(drm); } @@ -618,7 +630,7 @@ nouveau_do_suspend(struct drm_device *dev, bool runtime) } NV_DEBUG(drm, "suspending object tree...\n"); - ret = nvif_client_suspend(&drm->client.base); + ret = nvif_client_suspend(&drm->master.base); if (ret) goto fail_client; @@ -642,7 +654,7 @@ nouveau_do_resume(struct drm_device *dev, bool runtime) struct nouveau_drm *drm = nouveau_drm(dev); NV_DEBUG(drm, "resuming object tree...\n"); - nvif_client_resume(&drm->client.base); + nvif_client_resume(&drm->master.base); NV_DEBUG(drm, "resuming fence...\n"); if (drm->fence && nouveau_fence(drm)->resume) diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 84b8470422539..610245970c991 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -97,6 +97,8 @@ struct nouveau_cli { struct list_head objects; struct list_head notifys; char name[32]; + + struct mutex lock; }; static inline struct nouveau_cli * @@ -109,6 +111,7 @@ nouveau_cli(struct drm_file *fpriv) #include struct nouveau_drm { + struct nouveau_cli master; struct nouveau_cli client; struct drm_device *dev; From 9ce523cc3bf2ac19922e0a5d4b491221da01d1bc Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 059/151] drm/nouveau: separate buffer object backing memory from nvkm structures Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/Kbuild | 1 + .../gpu/drm/nouveau/include/nvkm/subdev/fb.h | 6 - .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 5 +- drivers/gpu/drm/nouveau/nouveau_bo.c | 111 ++++++++------ drivers/gpu/drm/nouveau/nouveau_mem.c | 114 ++++++++++++++ drivers/gpu/drm/nouveau/nouveau_mem.h | 35 +++++ drivers/gpu/drm/nouveau/nouveau_sgdma.c | 54 +++---- drivers/gpu/drm/nouveau/nouveau_ttm.c | 143 ++++++------------ .../drm/nouveau/nvkm/subdev/instmem/gk20a.c | 1 - .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 1 - 10 files changed, 285 insertions(+), 186 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nouveau_mem.c create mode 100644 drivers/gpu/drm/nouveau/nouveau_mem.h diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild index 2e9ce53ae3a8c..3476b61b94b68 100644 --- a/drivers/gpu/drm/nouveau/Kbuild +++ b/drivers/gpu/drm/nouveau/Kbuild @@ -30,6 +30,7 @@ nouveau-y += nouveau_vga.o # DRM - memory management nouveau-y += nouveau_bo.o nouveau-y += nouveau_gem.o +nouveau-y += nouveau_mem.o nouveau-y += nouveau_prime.o nouveau-y += nouveau_sgdma.o nouveau-y += nouveau_ttm.o diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index 28d513fbf44c3..dde444a7d3f04 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -22,12 +22,6 @@ #define NV_MEM_COMP_VM 0x03 struct nvkm_mem { - struct drm_device *dev; - - struct nvkm_vma bar_vma; - struct nvkm_vma vma[2]; - u8 page_shift; - struct nvkm_mm_node *tag; struct nvkm_mm_node *mem; dma_addr_t *pages; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index c4326772d4816..439938cd6cbf6 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -20,7 +20,10 @@ struct nvkm_vma { int refcount; struct nvkm_vm *vm; struct nvkm_mm_node *node; - u64 offset; + union { + u64 offset; + u64 addr; + }; u32 access; }; diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 50b400edcb88e..415c0e3159817 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -37,6 +37,7 @@ #include "nouveau_bo.h" #include "nouveau_ttm.h" #include "nouveau_gem.h" +#include "nouveau_mem.h" /* * NV10-NV40 tiling helpers @@ -670,14 +671,14 @@ static int nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg) { - struct nvkm_mem *mem = old_reg->mm_node; + struct nouveau_mem *mem = nouveau_mem(old_reg); int ret = RING_SPACE(chan, 10); if (ret == 0) { BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8); - OUT_RING (chan, upper_32_bits(mem->vma[0].offset)); - OUT_RING (chan, lower_32_bits(mem->vma[0].offset)); - OUT_RING (chan, upper_32_bits(mem->vma[1].offset)); - OUT_RING (chan, lower_32_bits(mem->vma[1].offset)); + OUT_RING (chan, upper_32_bits(mem->vma[0].addr)); + OUT_RING (chan, lower_32_bits(mem->vma[0].addr)); + OUT_RING (chan, upper_32_bits(mem->vma[1].addr)); + OUT_RING (chan, lower_32_bits(mem->vma[1].addr)); OUT_RING (chan, PAGE_SIZE); OUT_RING (chan, PAGE_SIZE); OUT_RING (chan, PAGE_SIZE); @@ -702,9 +703,9 @@ static int nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg) { - struct nvkm_mem *mem = old_reg->mm_node; - u64 src_offset = mem->vma[0].offset; - u64 dst_offset = mem->vma[1].offset; + struct nouveau_mem *mem = nouveau_mem(old_reg); + u64 src_offset = mem->vma[0].addr; + u64 dst_offset = mem->vma[1].addr; u32 page_count = new_reg->num_pages; int ret; @@ -740,9 +741,9 @@ static int nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg) { - struct nvkm_mem *mem = old_reg->mm_node; - u64 src_offset = mem->vma[0].offset; - u64 dst_offset = mem->vma[1].offset; + struct nouveau_mem *mem = nouveau_mem(old_reg); + u64 src_offset = mem->vma[0].addr; + u64 dst_offset = mem->vma[1].addr; u32 page_count = new_reg->num_pages; int ret; @@ -779,9 +780,9 @@ static int nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg) { - struct nvkm_mem *mem = old_reg->mm_node; - u64 src_offset = mem->vma[0].offset; - u64 dst_offset = mem->vma[1].offset; + struct nouveau_mem *mem = nouveau_mem(old_reg); + u64 src_offset = mem->vma[0].addr; + u64 dst_offset = mem->vma[1].addr; u32 page_count = new_reg->num_pages; int ret; @@ -817,14 +818,14 @@ static int nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg) { - struct nvkm_mem *mem = old_reg->mm_node; + struct nouveau_mem *mem = nouveau_mem(old_reg); int ret = RING_SPACE(chan, 7); if (ret == 0) { BEGIN_NV04(chan, NvSubCopy, 0x0320, 6); - OUT_RING (chan, upper_32_bits(mem->vma[0].offset)); - OUT_RING (chan, lower_32_bits(mem->vma[0].offset)); - OUT_RING (chan, upper_32_bits(mem->vma[1].offset)); - OUT_RING (chan, lower_32_bits(mem->vma[1].offset)); + OUT_RING (chan, upper_32_bits(mem->vma[0].addr)); + OUT_RING (chan, lower_32_bits(mem->vma[0].addr)); + OUT_RING (chan, upper_32_bits(mem->vma[1].addr)); + OUT_RING (chan, lower_32_bits(mem->vma[1].addr)); OUT_RING (chan, 0x00000000 /* COPY */); OUT_RING (chan, new_reg->num_pages << PAGE_SHIFT); } @@ -835,15 +836,15 @@ static int nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg) { - struct nvkm_mem *mem = old_reg->mm_node; + struct nouveau_mem *mem = nouveau_mem(old_reg); int ret = RING_SPACE(chan, 7); if (ret == 0) { BEGIN_NV04(chan, NvSubCopy, 0x0304, 6); OUT_RING (chan, new_reg->num_pages << PAGE_SHIFT); - OUT_RING (chan, upper_32_bits(mem->vma[0].offset)); - OUT_RING (chan, lower_32_bits(mem->vma[0].offset)); - OUT_RING (chan, upper_32_bits(mem->vma[1].offset)); - OUT_RING (chan, lower_32_bits(mem->vma[1].offset)); + OUT_RING (chan, upper_32_bits(mem->vma[0].addr)); + OUT_RING (chan, lower_32_bits(mem->vma[0].addr)); + OUT_RING (chan, upper_32_bits(mem->vma[1].addr)); + OUT_RING (chan, lower_32_bits(mem->vma[1].addr)); OUT_RING (chan, 0x00000000 /* MODE_COPY, QUERY_NONE */); } return ret; @@ -869,12 +870,12 @@ static int nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg) { - struct nvkm_mem *mem = old_reg->mm_node; + struct nouveau_mem *mem = nouveau_mem(old_reg); u64 length = (new_reg->num_pages << PAGE_SHIFT); - u64 src_offset = mem->vma[0].offset; - u64 dst_offset = mem->vma[1].offset; - int src_tiled = !!mem->memtype; - int dst_tiled = !!((struct nvkm_mem *)new_reg->mm_node)->memtype; + u64 src_offset = mem->vma[0].addr; + u64 dst_offset = mem->vma[1].addr; + int src_tiled = !!mem->kind; + int dst_tiled = !!nouveau_mem(new_reg)->kind; int ret; while (length) { @@ -1011,25 +1012,34 @@ static int nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, struct ttm_mem_reg *reg) { - struct nvkm_mem *old_mem = bo->mem.mm_node; - struct nvkm_mem *new_mem = reg->mm_node; + struct nouveau_mem *old_mem = nouveau_mem(&bo->mem); + struct nouveau_mem *new_mem = nouveau_mem(reg); + struct nvkm_vm *vmm = drm->client.vm; u64 size = (u64)reg->num_pages << PAGE_SHIFT; int ret; - ret = nvkm_vm_get(drm->client.vm, size, old_mem->page_shift, - NV_MEM_ACCESS_RW, &old_mem->vma[0]); + ret = nvkm_vm_get(vmm, size, old_mem->mem.page, NV_MEM_ACCESS_RW, + &old_mem->vma[0]); if (ret) return ret; - ret = nvkm_vm_get(drm->client.vm, size, new_mem->page_shift, - NV_MEM_ACCESS_RW, &old_mem->vma[1]); + ret = nvkm_vm_get(vmm, size, new_mem->mem.page, NV_MEM_ACCESS_RW, + &old_mem->vma[1]); if (ret) { nvkm_vm_put(&old_mem->vma[0]); return ret; } - nvkm_vm_map(&old_mem->vma[0], old_mem); - nvkm_vm_map(&old_mem->vma[1], new_mem); + ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]); + if (ret) + goto done; + + ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]); +done: + if (ret) { + nvkm_vm_put(&old_mem->vma[1]); + nvkm_vm_put(&old_mem->vma[0]); + } return 0; } @@ -1211,8 +1221,8 @@ static void nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict, struct ttm_mem_reg *new_reg) { + struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL; struct nouveau_bo *nvbo = nouveau_bo(bo); - struct nvkm_mem *mem = new_reg ? new_reg->mm_node : NULL; struct nvkm_vma *vma; /* ttm can now (stupidly) pass the driver bos it didn't create... */ @@ -1220,9 +1230,9 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict, return; if (mem && new_reg->mem_type != TTM_PL_SYSTEM && - mem->page_shift == nvbo->page) { + mem->mem.page == nvbo->page) { list_for_each_entry(vma, &nvbo->vma_list, head) { - nvkm_vm_map(vma, mem); + nvkm_vm_map(vma, mem->_mem); } } else { list_for_each_entry(vma, &nvbo->vma_list, head) { @@ -1343,7 +1353,7 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) struct ttm_mem_type_manager *man = &bdev->man[reg->mem_type]; struct nouveau_drm *drm = nouveau_bdev(bdev); struct nvkm_device *device = nvxx_device(&drm->client.device); - struct nvkm_mem *mem = reg->mm_node; + struct nouveau_mem *mem = nouveau_mem(reg); int ret; reg->bus.addr = NULL; @@ -1365,7 +1375,7 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) reg->bus.is_iomem = !drm->agp.cma; } #endif - if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || !mem->memtype) + if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || !mem->kind) /* untiled */ break; /* fallthrough, tiled memory */ @@ -1377,14 +1387,15 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); int page_shift = 12; if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_FERMI) - page_shift = mem->page_shift; + page_shift = mem->mem.page; - ret = nvkm_vm_get(bar, mem->size << 12, page_shift, - NV_MEM_ACCESS_RW, &mem->bar_vma); + ret = nvkm_vm_get(bar, mem->_mem->size << 12, + page_shift, NV_MEM_ACCESS_RW, + &mem->bar_vma); if (ret) return ret; - nvkm_vm_map(&mem->bar_vma, mem); + nvkm_vm_map(&mem->bar_vma, mem->_mem); reg->bus.offset = mem->bar_vma.offset; } break; @@ -1397,7 +1408,7 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) static void nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) { - struct nvkm_mem *mem = reg->mm_node; + struct nouveau_mem *mem = nouveau_mem(reg); if (!mem->bar_vma.node) return; @@ -1606,7 +1617,7 @@ nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nvkm_vm *vm, struct nvkm_vma *vma) { const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT; - struct nvkm_mem *mem = nvbo->bo.mem.mm_node; + struct nouveau_mem *mem = nouveau_mem(&nvbo->bo.mem); int ret; ret = nvkm_vm_get(vm, size, nvbo->page, NV_MEM_ACCESS_RW, vma); @@ -1614,8 +1625,8 @@ nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nvkm_vm *vm, return ret; if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM && - mem->page_shift == nvbo->page) - nvkm_vm_map(vma, nvbo->bo.mem.mm_node); + mem->mem.page == nvbo->page) + nvkm_vm_map(vma, mem->_mem); list_add_tail(&vma->head, &nvbo->vma_list); vma->refcount = 1; diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c new file mode 100644 index 0000000000000..68ac332841ae6 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nouveau_mem.c @@ -0,0 +1,114 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "nouveau_mem.h" +#include "nouveau_drv.h" +#include "nouveau_bo.h" + +#include + +int +nouveau_mem_map(struct nouveau_mem *mem, + struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + nvkm_vm_map(vma, mem->_mem); + return 0; +} + +void +nouveau_mem_fini(struct nouveau_mem *mem) +{ + if (mem->vma[1].node) { + nvkm_vm_unmap(&mem->vma[1]); + nvkm_vm_put(&mem->vma[1]); + } + if (mem->vma[0].node) { + nvkm_vm_unmap(&mem->vma[0]); + nvkm_vm_put(&mem->vma[0]); + } +} + +int +nouveau_mem_host(struct ttm_mem_reg *reg, struct ttm_dma_tt *tt) +{ + struct nouveau_mem *mem = nouveau_mem(reg); + struct nouveau_cli *cli = mem->cli; + + if (mem->kind && cli->device.info.chipset == 0x50) + mem->comp = mem->kind = 0; + if (mem->comp) { + if (cli->device.info.chipset >= 0xc0) + mem->kind = gf100_pte_storage_type_map[mem->kind]; + mem->comp = 0; + } + + mem->__mem.size = (reg->num_pages << PAGE_SHIFT) >> 12; + mem->__mem.memtype = (mem->comp << 7) | mem->kind; + if (tt->ttm.sg) mem->__mem.sg = tt->ttm.sg; + else mem->__mem.pages = tt->dma_address; + mem->_mem = &mem->__mem; + mem->mem.page = 12; + return 0; +} + +int +nouveau_mem_vram(struct ttm_mem_reg *reg, bool contig, u8 page) +{ + struct nouveau_mem *mem = nouveau_mem(reg); + struct nvkm_ram *ram = nvxx_fb(&mem->cli->device)->ram; + u64 size = ALIGN(reg->num_pages << PAGE_SHIFT, 1 << page); + int ret; + + mem->mem.page = page; + + ret = ram->func->get(ram, size, 1 << page, contig ? 0 : 1 << page, + (mem->comp << 8) | mem->kind, &mem->_mem); + if (ret) + return ret; + + reg->start = mem->_mem->offset >> PAGE_SHIFT; + return ret; +} + +void +nouveau_mem_del(struct ttm_mem_reg *reg) +{ + struct nouveau_mem *mem = nouveau_mem(reg); + nouveau_mem_fini(mem); + kfree(reg->mm_node); + reg->mm_node = NULL; +} + +int +nouveau_mem_new(struct nouveau_cli *cli, u8 kind, u8 comp, + struct ttm_mem_reg *reg) +{ + struct nouveau_mem *mem; + + if (!(mem = kzalloc(sizeof(*mem), GFP_KERNEL))) + return -ENOMEM; + mem->cli = cli; + mem->kind = kind; + mem->comp = comp; + + reg->mm_node = mem; + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.h b/drivers/gpu/drm/nouveau/nouveau_mem.h new file mode 100644 index 0000000000000..d00b4f669c4bd --- /dev/null +++ b/drivers/gpu/drm/nouveau/nouveau_mem.h @@ -0,0 +1,35 @@ +#ifndef __NOUVEAU_MEM_H__ +#define __NOUVEAU_MEM_H__ +#include + +#include +struct ttm_dma_tt; + +static inline struct nouveau_mem * +nouveau_mem(struct ttm_mem_reg *reg) +{ + return reg->mm_node; +} + +struct nouveau_mem { + struct nouveau_cli *cli; + u8 kind; + u8 comp; + struct { + u8 page; + } mem; + struct nvkm_vma vma[2]; + + struct nvkm_mem __mem; + struct nvkm_mem *_mem; + struct nvkm_vma bar_vma; +}; + +int nouveau_mem_new(struct nouveau_cli *, u8 kind, u8 comp, + struct ttm_mem_reg *); +void nouveau_mem_del(struct ttm_mem_reg *); +int nouveau_mem_vram(struct ttm_mem_reg *, bool contig, u8 page); +int nouveau_mem_host(struct ttm_mem_reg *, struct ttm_dma_tt *); +void nouveau_mem_fini(struct nouveau_mem *); +int nouveau_mem_map(struct nouveau_mem *, struct nvkm_vmm *, struct nvkm_vma *); +#endif diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c index b7ab268f7d6f5..69c606f97784b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c +++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c @@ -2,6 +2,7 @@ #include #include "nouveau_drv.h" +#include "nouveau_mem.h" #include "nouveau_ttm.h" struct nouveau_sgdma_be { @@ -9,7 +10,7 @@ struct nouveau_sgdma_be { * nouve_bo.c works properly, otherwise have to move them here */ struct ttm_dma_tt ttm; - struct nvkm_mem *node; + struct nouveau_mem *mem; }; static void @@ -27,19 +28,20 @@ static int nv04_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *reg) { struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm; - struct nvkm_mem *node = reg->mm_node; - - if (ttm->sg) { - node->sg = ttm->sg; - node->pages = NULL; - } else { - node->sg = NULL; - node->pages = nvbe->ttm.dma_address; + struct nouveau_mem *mem = nouveau_mem(reg); + int ret; + + ret = nouveau_mem_host(reg, &nvbe->ttm); + if (ret) + return ret; + + ret = nouveau_mem_map(mem, mem->cli->vm, &mem->vma[0]); + if (ret) { + nouveau_mem_fini(mem); + return ret; } - node->size = (reg->num_pages << PAGE_SHIFT) >> 12; - nvkm_vm_map(&node->vma[0], node); - nvbe->node = node; + nvbe->mem = mem; return 0; } @@ -47,7 +49,7 @@ static int nv04_sgdma_unbind(struct ttm_tt *ttm) { struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm; - nvkm_vm_unmap(&nvbe->node->vma[0]); + nouveau_mem_fini(nvbe->mem); return 0; } @@ -61,30 +63,20 @@ static int nv50_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *reg) { struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm; - struct nvkm_mem *node = reg->mm_node; - - /* noop: bound in move_notify() */ - if (ttm->sg) { - node->sg = ttm->sg; - node->pages = NULL; - } else { - node->sg = NULL; - node->pages = nvbe->ttm.dma_address; - } - node->size = (reg->num_pages << PAGE_SHIFT) >> 12; - return 0; -} + struct nouveau_mem *mem = nouveau_mem(reg); + int ret; -static int -nv50_sgdma_unbind(struct ttm_tt *ttm) -{ - /* noop: unbound in move_notify() */ + ret = nouveau_mem_host(reg, &nvbe->ttm); + if (ret) + return ret; + + nvbe->mem = mem; return 0; } static struct ttm_backend_func nv50_sgdma_backend = { .bind = nv50_sgdma_bind, - .unbind = nv50_sgdma_unbind, + .unbind = nv04_sgdma_unbind, .destroy = nouveau_sgdma_destroy }; diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index e6e767558a3b4..3bf3b60130e02 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -23,10 +23,10 @@ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. */ - #include "nouveau_drv.h" -#include "nouveau_ttm.h" #include "nouveau_gem.h" +#include "nouveau_mem.h" +#include "nouveau_ttm.h" #include @@ -45,23 +45,15 @@ nouveau_manager_fini(struct ttm_mem_type_manager *man) } static void -nouveau_manager_debug(struct ttm_mem_type_manager *man, - struct drm_printer *printer) +nouveau_manager_del(struct ttm_mem_type_manager *man, struct ttm_mem_reg *reg) { + nouveau_mem_del(reg); } -static inline void -nvkm_mem_node_cleanup(struct nvkm_mem *node) +static void +nouveau_manager_debug(struct ttm_mem_type_manager *man, + struct drm_printer *printer) { - if (node->vma[0].node) { - nvkm_vm_unmap(&node->vma[0]); - nvkm_vm_put(&node->vma[0]); - } - - if (node->vma[1].node) { - nvkm_vm_unmap(&node->vma[1]); - nvkm_vm_put(&node->vma[1]); - } } static void @@ -70,8 +62,9 @@ nouveau_vram_manager_del(struct ttm_mem_type_manager *man, { struct nouveau_drm *drm = nouveau_bdev(man->bdev); struct nvkm_ram *ram = nvxx_fb(&drm->client.device)->ram; - nvkm_mem_node_cleanup(reg->mm_node); - ram->func->put(ram, (struct nvkm_mem **)®->mm_node); + struct nvkm_mem *mem = nouveau_mem(reg)->_mem; + nouveau_mem_del(reg); + ram->func->put(ram, &mem); } static int @@ -80,31 +73,29 @@ nouveau_vram_manager_new(struct ttm_mem_type_manager *man, const struct ttm_place *place, struct ttm_mem_reg *reg) { - struct nouveau_drm *drm = nouveau_bdev(man->bdev); - struct nvkm_ram *ram = nvxx_fb(&drm->client.device)->ram; struct nouveau_bo *nvbo = nouveau_bo(bo); - struct nvkm_mem *node; - u32 size_nc = 0; + struct nouveau_drm *drm = nvbo->cli->drm; + struct nouveau_mem *mem; int ret; if (drm->client.device.info.ram_size == 0) return -ENOMEM; - if (!nvbo->contig) - size_nc = 1 << nvbo->page; + ret = nouveau_mem_new(&drm->master, nvbo->kind, nvbo->comp, reg); + mem = nouveau_mem(reg); + if (ret) + return ret; - ret = ram->func->get(ram, reg->num_pages << PAGE_SHIFT, - reg->page_alignment << PAGE_SHIFT, size_nc, - nvbo->comp << 8 | nvbo->kind, &node); + ret = nouveau_mem_vram(reg, nvbo->contig, nvbo->page); if (ret) { - reg->mm_node = NULL; - return (ret == -ENOSPC) ? 0 : ret; + nouveau_mem_del(reg); + if (ret == -ENOSPC) { + reg->mm_node = NULL; + return 0; + } + return ret; } - node->page_shift = nvbo->page; - - reg->mm_node = node; - reg->start = node->offset >> PAGE_SHIFT; return 0; } @@ -116,54 +107,24 @@ const struct ttm_mem_type_manager_func nouveau_vram_manager = { .debug = nouveau_manager_debug, }; -static void -nouveau_gart_manager_del(struct ttm_mem_type_manager *man, - struct ttm_mem_reg *reg) -{ - nvkm_mem_node_cleanup(reg->mm_node); - kfree(reg->mm_node); - reg->mm_node = NULL; -} - static int nouveau_gart_manager_new(struct ttm_mem_type_manager *man, struct ttm_buffer_object *bo, const struct ttm_place *place, struct ttm_mem_reg *reg) { - struct nouveau_drm *drm = nouveau_bdev(bo->bdev); struct nouveau_bo *nvbo = nouveau_bo(bo); - struct nvkm_mem *node; - - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return -ENOMEM; + struct nouveau_drm *drm = nvbo->cli->drm; + struct nouveau_mem *mem; + int ret; - node->page_shift = 12; - - switch (drm->client.device.info.family) { - case NV_DEVICE_INFO_V0_TNT: - case NV_DEVICE_INFO_V0_CELSIUS: - case NV_DEVICE_INFO_V0_KELVIN: - case NV_DEVICE_INFO_V0_RANKINE: - case NV_DEVICE_INFO_V0_CURIE: - break; - case NV_DEVICE_INFO_V0_TESLA: - case NV_DEVICE_INFO_V0_FERMI: - case NV_DEVICE_INFO_V0_KEPLER: - case NV_DEVICE_INFO_V0_MAXWELL: - case NV_DEVICE_INFO_V0_PASCAL: - if (drm->client.device.info.chipset != 0x50) - node->memtype = nvbo->kind; - break; - default: - NV_WARN(drm, "%s: unhandled family type %x\n", __func__, - drm->client.device.info.family); - break; - } + ret = nouveau_mem_new(&drm->master, nvbo->kind, nvbo->comp, reg); + mem = nouveau_mem(reg); + if (ret) + return ret; - reg->mm_node = node; - reg->start = 0; + mem->_mem = &mem->__mem; + reg->start = 0; return 0; } @@ -171,50 +132,40 @@ const struct ttm_mem_type_manager_func nouveau_gart_manager = { .init = nouveau_manager_init, .takedown = nouveau_manager_fini, .get_node = nouveau_gart_manager_new, - .put_node = nouveau_gart_manager_del, + .put_node = nouveau_manager_del, .debug = nouveau_manager_debug }; -static void -nv04_gart_manager_del(struct ttm_mem_type_manager *man, struct ttm_mem_reg *reg) -{ - struct nvkm_mem *node = reg->mm_node; - if (node->vma[0].node) - nvkm_vm_put(&node->vma[0]); - kfree(reg->mm_node); - reg->mm_node = NULL; -} - static int nv04_gart_manager_new(struct ttm_mem_type_manager *man, struct ttm_buffer_object *bo, const struct ttm_place *place, struct ttm_mem_reg *reg) { - struct nouveau_drm *drm = nouveau_bdev(man->bdev); + struct nouveau_bo *nvbo = nouveau_bo(bo); + struct nouveau_drm *drm = nvbo->cli->drm; + struct nouveau_mem *mem; struct nvkm_mmu *mmu = nvxx_mmu(&drm->client.device); - struct nvkm_mem *node; int ret; - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return -ENOMEM; - - node->page_shift = 12; + ret = nouveau_mem_new(&drm->master, nvbo->kind, nvbo->comp, reg); + mem = nouveau_mem(reg); + if (ret) + return ret; - ret = nvkm_vm_get(mmu->vmm, reg->num_pages << 12, node->page_shift, - NV_MEM_ACCESS_RW, &node->vma[0]); + ret = nvkm_vm_get(mmu->vmm, reg->num_pages << 12, 12, + NV_MEM_ACCESS_RW, &mem->vma[0]); if (ret) { + nouveau_mem_del(reg); if (ret == -ENOSPC) { reg->mm_node = NULL; - ret = 0; + return 0; } - kfree(node); return ret; } - reg->mm_node = node; - reg->start = node->vma[0].offset >> PAGE_SHIFT; + mem->_mem = &mem->__mem; + reg->start = mem->vma[0].addr >> PAGE_SHIFT; return 0; } @@ -222,7 +173,7 @@ const struct ttm_mem_type_manager_func nv04_gart_manager = { .init = nouveau_manager_init, .takedown = nouveau_manager_fini, .get_node = nv04_gart_manager_new, - .put_node = nv04_gart_manager_del, + .put_node = nouveau_manager_del, .debug = nouveau_manager_debug }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c index a8b71262f663c..f2879c48d1e68 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c @@ -530,7 +530,6 @@ gk20a_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, /* present memory for being mapped using small pages */ node->mem.size = size >> 12; node->mem.memtype = 0; - node->mem.page_shift = 12; nvkm_debug(subdev, "alloc size: 0x%x, align: 0x%x, gaddr: 0x%llx\n", size, align, node->mem.offset); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index 4ccb8cdc7dbcb..bb524e3aa5d99 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -356,7 +356,6 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, if (ret) return ret; - iobj->mem->page_shift = 12; return 0; } From 24e8375b1bfdf7f6014b9e3d7903d6a8f81aa249 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 060/151] drm/nouveau: separate constant-va tracking from nvkm vma structure Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/Kbuild | 1 + .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 2 - drivers/gpu/drm/nouveau/nouveau_abi16.c | 10 +- drivers/gpu/drm/nouveau/nouveau_abi16.h | 2 +- drivers/gpu/drm/nouveau/nouveau_bo.c | 51 +------ drivers/gpu/drm/nouveau/nouveau_bo.h | 7 - drivers/gpu/drm/nouveau/nouveau_chan.c | 19 +-- drivers/gpu/drm/nouveau/nouveau_chan.h | 3 +- drivers/gpu/drm/nouveau/nouveau_display.h | 5 +- drivers/gpu/drm/nouveau/nouveau_dma.c | 15 +- drivers/gpu/drm/nouveau/nouveau_dma.h | 2 +- drivers/gpu/drm/nouveau/nouveau_drm.c | 11 +- drivers/gpu/drm/nouveau/nouveau_drv.h | 4 +- drivers/gpu/drm/nouveau/nouveau_fbcon.c | 7 +- drivers/gpu/drm/nouveau/nouveau_fence.h | 2 +- drivers/gpu/drm/nouveau/nouveau_gem.c | 68 ++++------ drivers/gpu/drm/nouveau/nouveau_vmm.c | 128 ++++++++++++++++++ drivers/gpu/drm/nouveau/nouveau_vmm.h | 32 +++++ drivers/gpu/drm/nouveau/nv50_fbcon.c | 9 +- drivers/gpu/drm/nouveau/nv84_fence.c | 9 +- drivers/gpu/drm/nouveau/nvc0_fbcon.c | 9 +- 21 files changed, 245 insertions(+), 151 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nouveau_vmm.c create mode 100644 drivers/gpu/drm/nouveau/nouveau_vmm.h diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild index 3476b61b94b68..9c0c650655e93 100644 --- a/drivers/gpu/drm/nouveau/Kbuild +++ b/drivers/gpu/drm/nouveau/Kbuild @@ -34,6 +34,7 @@ nouveau-y += nouveau_mem.o nouveau-y += nouveau_prime.o nouveau-y += nouveau_sgdma.o nouveau-y += nouveau_ttm.o +nouveau-y += nouveau_vmm.o # DRM - modesetting nouveau-$(CONFIG_DRM_NOUVEAU_BACKLIGHT) += nouveau_backlight.o diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 439938cd6cbf6..549a8782ce356 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -16,8 +16,6 @@ struct nvkm_vm_pgd { }; struct nvkm_vma { - struct list_head head; - int refcount; struct nvkm_vm *vm; struct nvkm_mm_node *node; union { diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c index f98f800cc0112..9b0b210a37f82 100644 --- a/drivers/gpu/drm/nouveau/nouveau_abi16.c +++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c @@ -34,6 +34,7 @@ #include "nouveau_gem.h" #include "nouveau_chan.h" #include "nouveau_abi16.h" +#include "nouveau_vmm.h" static struct nouveau_abi16 * nouveau_abi16(struct drm_file *file_priv) @@ -134,7 +135,7 @@ nouveau_abi16_chan_fini(struct nouveau_abi16 *abi16, } if (chan->ntfy) { - nouveau_bo_vma_del(chan->ntfy, &chan->ntfy_vma); + nouveau_vma_del(&chan->ntfy_vma); nouveau_bo_unpin(chan->ntfy); drm_gem_object_unreference_unlocked(&chan->ntfy->gem); } @@ -329,8 +330,7 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS) goto done; if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nouveau_bo_vma_add(chan->ntfy, cli->vm, - &chan->ntfy_vma); + ret = nouveau_vma_new(chan->ntfy, &cli->vmm, &chan->ntfy_vma); if (ret) goto done; } @@ -548,8 +548,8 @@ nouveau_abi16_ioctl_notifierobj_alloc(ABI16_IOCTL_ARGS) if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_VM; - args.start += chan->ntfy_vma.offset; - args.limit += chan->ntfy_vma.offset; + args.start += chan->ntfy_vma->addr; + args.limit += chan->ntfy_vma->addr; } else if (drm->agp.bridge) { args.target = NV_DMA_V0_TARGET_AGP; diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.h b/drivers/gpu/drm/nouveau/nouveau_abi16.h index 841cc556fad83..3277476803248 100644 --- a/drivers/gpu/drm/nouveau/nouveau_abi16.h +++ b/drivers/gpu/drm/nouveau/nouveau_abi16.h @@ -23,7 +23,7 @@ struct nouveau_abi16_chan { struct nouveau_channel *chan; struct list_head notifiers; struct nouveau_bo *ntfy; - struct nvkm_vma ntfy_vma; + struct nouveau_vma *ntfy_vma; struct nvkm_mm heap; }; diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 415c0e3159817..d89f3e700bafa 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -38,6 +38,7 @@ #include "nouveau_ttm.h" #include "nouveau_gem.h" #include "nouveau_mem.h" +#include "nouveau_vmm.h" /* * NV10-NV40 tiling helpers @@ -1223,7 +1224,7 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict, { struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL; struct nouveau_bo *nvbo = nouveau_bo(bo); - struct nvkm_vma *vma; + struct nouveau_vma *vma; /* ttm can now (stupidly) pass the driver bos it didn't create... */ if (bo->destroy != nouveau_bo_del_ttm) @@ -1232,12 +1233,12 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict, if (mem && new_reg->mem_type != TTM_PL_SYSTEM && mem->mem.page == nvbo->page) { list_for_each_entry(vma, &nvbo->vma_list, head) { - nvkm_vm_map(vma, mem->_mem); + nouveau_vma_map(vma, mem); } } else { list_for_each_entry(vma, &nvbo->vma_list, head) { WARN_ON(ttm_bo_wait(bo, false, false)); - nvkm_vm_unmap(vma); + nouveau_vma_unmap(vma); } } } @@ -1599,47 +1600,3 @@ struct ttm_bo_driver nouveau_bo_driver = { .io_mem_free = &nouveau_ttm_io_mem_free, .io_mem_pfn = ttm_bo_default_io_mem_pfn, }; - -struct nvkm_vma * -nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nvkm_vm *vm) -{ - struct nvkm_vma *vma; - list_for_each_entry(vma, &nvbo->vma_list, head) { - if (vma->vm == vm) - return vma; - } - - return NULL; -} - -int -nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nvkm_vm *vm, - struct nvkm_vma *vma) -{ - const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT; - struct nouveau_mem *mem = nouveau_mem(&nvbo->bo.mem); - int ret; - - ret = nvkm_vm_get(vm, size, nvbo->page, NV_MEM_ACCESS_RW, vma); - if (ret) - return ret; - - if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM && - mem->mem.page == nvbo->page) - nvkm_vm_map(vma, mem->_mem); - - list_add_tail(&vma->head, &nvbo->vma_list); - vma->refcount = 1; - return 0; -} - -void -nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nvkm_vma *vma) -{ - if (vma->node) { - if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM) - nvkm_vm_unmap(vma); - nvkm_vm_put(vma); - list_del(&vma->head); - } -} diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.h b/drivers/gpu/drm/nouveau/nouveau_bo.h index a179bbf25dced..23002bdd94a89 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.h +++ b/drivers/gpu/drm/nouveau/nouveau_bo.h @@ -93,13 +93,6 @@ int nouveau_bo_validate(struct nouveau_bo *, bool interruptible, void nouveau_bo_sync_for_device(struct nouveau_bo *nvbo); void nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo); -struct nvkm_vma * -nouveau_bo_vma_find(struct nouveau_bo *, struct nvkm_vm *); - -int nouveau_bo_vma_add(struct nouveau_bo *, struct nvkm_vm *, - struct nvkm_vma *); -void nouveau_bo_vma_del(struct nouveau_bo *, struct nvkm_vma *); - /* TODO: submit equivalent to TTM generic API upstream? */ static inline void __iomem * nvbo_kmap_obj_iovirtual(struct nouveau_bo *nvbo) diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index 4af09e9be9304..9fdca7b4cb05b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -40,6 +40,7 @@ #include "nouveau_chan.h" #include "nouveau_fence.h" #include "nouveau_abi16.h" +#include "nouveau_vmm.h" MODULE_PARM_DESC(vram_pushbuf, "Create DMA push buffers in VRAM"); int nouveau_vram_pushbuf; @@ -91,7 +92,7 @@ nouveau_channel_del(struct nouveau_channel **pchan) nvif_notify_fini(&chan->kill); nvif_object_fini(&chan->user); nvif_object_fini(&chan->push.ctxdma); - nouveau_bo_vma_del(chan->push.buffer, &chan->push.vma); + nouveau_vma_del(&chan->push.vma); nouveau_bo_unmap(chan->push.buffer); if (chan->push.buffer && chan->push.buffer->pin_refcnt) nouveau_bo_unpin(chan->push.buffer); @@ -142,11 +143,11 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, * pushbuf lives in, this is because the GEM code requires that * we be able to call out to other (indirect) push buffers */ - chan->push.vma.offset = chan->push.buffer->bo.offset; + chan->push.addr = chan->push.buffer->bo.offset; if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nouveau_bo_vma_add(chan->push.buffer, cli->vm, - &chan->push.vma); + ret = nouveau_vma_new(chan->push.buffer, &cli->vmm, + &chan->push.vma); if (ret) { nouveau_channel_del(pchan); return ret; @@ -156,6 +157,8 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, args.access = NV_DMA_V0_ACCESS_VM; args.start = 0; args.limit = cli->vm->mmu->limit - 1; + + chan->push.addr = chan->push.vma->addr; } else if (chan->push.buffer->bo.mem.mem_type == TTM_PL_VRAM) { if (device->info.family == NV_DEVICE_INFO_V0_TNT) { @@ -233,20 +236,20 @@ nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device, args.kepler.version = 0; args.kepler.engines = engine; args.kepler.ilength = 0x02000; - args.kepler.ioffset = 0x10000 + chan->push.vma.offset; + args.kepler.ioffset = 0x10000 + chan->push.addr; args.kepler.vm = 0; size = sizeof(args.kepler); } else if (oclass[0] >= FERMI_CHANNEL_GPFIFO) { args.fermi.version = 0; args.fermi.ilength = 0x02000; - args.fermi.ioffset = 0x10000 + chan->push.vma.offset; + args.fermi.ioffset = 0x10000 + chan->push.addr; args.fermi.vm = 0; size = sizeof(args.fermi); } else { args.nv50.version = 0; args.nv50.ilength = 0x02000; - args.nv50.ioffset = 0x10000 + chan->push.vma.offset; + args.nv50.ioffset = 0x10000 + chan->push.addr; args.nv50.pushbuf = nvif_handle(&chan->push.ctxdma); args.nv50.vm = 0; size = sizeof(args.nv50); @@ -293,7 +296,7 @@ nouveau_channel_dma(struct nouveau_drm *drm, struct nvif_device *device, /* create channel object */ args.version = 0; args.pushbuf = nvif_handle(&chan->push.ctxdma); - args.offset = chan->push.vma.offset; + args.offset = chan->push.addr; do { ret = nvif_object_init(&device->object, 0, *oclass++, diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.h b/drivers/gpu/drm/nouveau/nouveau_chan.h index 46b947ba1cf4a..f29d3a72c48c5 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.h +++ b/drivers/gpu/drm/nouveau/nouveau_chan.h @@ -16,8 +16,9 @@ struct nouveau_channel { struct { struct nouveau_bo *buffer; - struct nvkm_vma vma; + struct nouveau_vma *vma; struct nvif_object ctxdma; + u64 addr; } push; /* TODO: this will be reworked in the near future */ diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h index 201aec2ea5b81..1411bf05b89d5 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.h +++ b/drivers/gpu/drm/nouveau/nouveau_display.h @@ -1,14 +1,11 @@ #ifndef __NOUVEAU_DISPLAY_H__ #define __NOUVEAU_DISPLAY_H__ - -#include - #include "nouveau_drv.h" struct nouveau_framebuffer { struct drm_framebuffer base; struct nouveau_bo *nvbo; - struct nvkm_vma vma; + struct nouveau_vma *vma; u32 r_handle; u32 r_format; u32 r_pitch; diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.c b/drivers/gpu/drm/nouveau/nouveau_dma.c index 2634a1a798885..10e84f6ca2b73 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dma.c +++ b/drivers/gpu/drm/nouveau/nouveau_dma.c @@ -26,6 +26,7 @@ #include "nouveau_drv.h" #include "nouveau_dma.h" +#include "nouveau_vmm.h" void OUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords) @@ -71,11 +72,11 @@ READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout) return -EBUSY; } - if (val < chan->push.vma.offset || - val > chan->push.vma.offset + (chan->dma.max << 2)) + if (val < chan->push.addr || + val > chan->push.addr + (chan->dma.max << 2)) return -EINVAL; - return (val - chan->push.vma.offset) >> 2; + return (val - chan->push.addr) >> 2; } void @@ -84,13 +85,13 @@ nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo, { struct nouveau_cli *cli = (void *)chan->user.client; struct nouveau_bo *pb = chan->push.buffer; - struct nvkm_vma *vma; + struct nouveau_vma *vma; int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base; u64 offset; - vma = nouveau_bo_vma_find(bo, cli->vm); + vma = nouveau_vma_find(bo, &cli->vmm); BUG_ON(!vma); - offset = vma->offset + delta; + offset = vma->addr + delta; BUG_ON(chan->dma.ib_free < 1); @@ -224,7 +225,7 @@ nouveau_dma_wait(struct nouveau_channel *chan, int slots, int size) * instruct the GPU to jump back to the start right * after processing the currently pending commands. */ - OUT_RING(chan, chan->push.vma.offset | 0x20000000); + OUT_RING(chan, chan->push.addr | 0x20000000); /* wait for GET to depart from the skips area. * prevents writing GET==PUT and causing a race diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.h b/drivers/gpu/drm/nouveau/nouveau_dma.h index aff3a9d0a1fcd..74e10b14a7da2 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dma.h +++ b/drivers/gpu/drm/nouveau/nouveau_dma.h @@ -140,7 +140,7 @@ BEGIN_IMC0(struct nouveau_channel *chan, int subc, int mthd, u16 data) #define WRITE_PUT(val) do { \ mb(); \ nouveau_bo_rd32(chan->push.buffer, 0); \ - nvif_wr32(&chan->user, chan->user_put, ((val) << 2) + chan->push.vma.offset); \ + nvif_wr32(&chan->user, chan->user_put, ((val) << 2) + chan->push.addr);\ } while (0) static inline void diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index a705e56f0e578..25a47e8ba42b5 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -114,8 +114,8 @@ nouveau_name(struct drm_device *dev) static void nouveau_cli_fini(struct nouveau_cli *cli) { - nvkm_vm_ref(NULL, &nvxx_client(&cli->base)->vm, NULL); usif_client_fini(cli); + nouveau_vmm_fini(&cli->vmm); nvif_device_fini(&cli->device); mutex_lock(&cli->drm->master.lock); nvif_client_fini(&cli->base); @@ -472,12 +472,11 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags) goto fail_device; } - ret = nvkm_vm_new(nvxx_device(&drm->client.device), - 0, (1ULL << 40), 0x1000, NULL, - &drm->client.vm); + ret = nouveau_vmm_init(&drm->client, 0, &drm->client.vmm); if (ret) goto fail_device; + drm->client.vm = drm->client.vmm.vm; nvxx_client(&drm->client.base)->vm = drm->client.vm; } @@ -863,11 +862,11 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv) cli->base.super = false; if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nvkm_vm_new(nvxx_device(&drm->client.device), 0, - (1ULL << 40), 0x1000, NULL, &cli->vm); + ret = nouveau_vmm_init(cli, 0, &cli->vmm); if (ret) goto done; + cli->vm = cli->vmm.vm; nvxx_client(&cli->base)->vm = cli->vm; } diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 610245970c991..ba8a9ab205629 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -61,6 +61,7 @@ struct platform_device; #include "nouveau_fence.h" #include "nouveau_bios.h" +#include "nouveau_vmm.h" struct nouveau_drm_tile { struct nouveau_fence *fence; @@ -90,8 +91,9 @@ struct nouveau_cli { struct mutex mutex; struct nvif_device device; + struct nouveau_vmm vmm; - struct nvkm_vm *vm; /*XXX*/ + struct nvkm_vm *vm; struct list_head head; void *abi16; struct list_head objects; diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c index 2b12d82aac150..c533d8e04afc0 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c +++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c @@ -48,6 +48,7 @@ #include "nouveau_bo.h" #include "nouveau_fbcon.h" #include "nouveau_chan.h" +#include "nouveau_vmm.h" #include "nouveau_crtc.h" @@ -348,7 +349,7 @@ nouveau_fbcon_create(struct drm_fb_helper *helper, chan = nouveau_nofbaccel ? NULL : drm->channel; if (chan && device->info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nouveau_bo_vma_add(nvbo, drm->client.vm, &fb->vma); + ret = nouveau_vma_new(nvbo, &drm->client.vmm, &fb->vma); if (ret) { NV_ERROR(drm, "failed to map fb into chan: %d\n", ret); chan = NULL; @@ -402,7 +403,7 @@ nouveau_fbcon_create(struct drm_fb_helper *helper, out_unlock: if (chan) - nouveau_bo_vma_del(fb->nvbo, &fb->vma); + nouveau_vma_del(&fb->vma); nouveau_bo_unmap(fb->nvbo); out_unpin: nouveau_bo_unpin(fb->nvbo); @@ -429,7 +430,7 @@ nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *fbcon) drm_fb_helper_fini(&fbcon->helper); if (nouveau_fb->nvbo) { - nouveau_bo_vma_del(nouveau_fb->nvbo, &nouveau_fb->vma); + nouveau_vma_del(&nouveau_fb->vma); nouveau_bo_unmap(nouveau_fb->nvbo); nouveau_bo_unpin(nouveau_fb->nvbo); drm_framebuffer_unreference(&nouveau_fb->base); diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h index 28a2135097970..d32e58b66a755 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.h +++ b/drivers/gpu/drm/nouveau/nouveau_fence.h @@ -88,7 +88,7 @@ int nouveau_flip_complete(struct nvif_notify *); struct nv84_fence_chan { struct nouveau_fence_chan base; - struct nvkm_vma vma; + struct nouveau_vma *vma; }; struct nv84_fence_priv { diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 2d924739997af..9f82b13293f54 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -31,6 +31,7 @@ #include "nouveau_ttm.h" #include "nouveau_gem.h" +#include "nouveau_vmm.h" void nouveau_gem_object_del(struct drm_gem_object *gem) @@ -64,8 +65,8 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) struct nouveau_cli *cli = nouveau_cli(file_priv); struct nouveau_bo *nvbo = nouveau_gem_object(gem); struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); - struct nvkm_vma *vma; struct device *dev = drm->dev->dev; + struct nouveau_vma *vma; int ret; if (!cli->vm) @@ -75,30 +76,13 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) if (ret) return ret; - vma = nouveau_bo_vma_find(nvbo, cli->vm); - if (!vma) { - vma = kzalloc(sizeof(*vma), GFP_KERNEL); - if (!vma) { - ret = -ENOMEM; - goto out; - } - - ret = pm_runtime_get_sync(dev); - if (ret < 0 && ret != -EACCES) { - kfree(vma); - goto out; - } - - ret = nouveau_bo_vma_add(nvbo, cli->vm, vma); - if (ret) - kfree(vma); - - pm_runtime_mark_last_busy(dev); - pm_runtime_put_autosuspend(dev); - } else { - vma->refcount++; - } + ret = pm_runtime_get_sync(dev); + if (ret < 0 && ret != -EACCES) + goto out; + ret = nouveau_vma_new(nvbo, &cli->vmm, &vma); + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); out: ttm_bo_unreserve(&nvbo->bo); return ret; @@ -107,14 +91,12 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) static void nouveau_gem_object_delete(void *data) { - struct nvkm_vma *vma = data; - nvkm_vm_unmap(vma); - nvkm_vm_put(vma); - kfree(vma); + struct nouveau_vma *vma = data; + nouveau_vma_del(&vma); } static void -nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nvkm_vma *vma) +nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) { const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM; struct reservation_object *resv = nvbo->bo.resv; @@ -123,7 +105,7 @@ nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nvkm_vma *vma) fobj = reservation_object_get_list(resv); - list_del(&vma->head); + list_del_init(&vma->head); if (fobj && fobj->shared_count > 1) ttm_bo_wait(&nvbo->bo, false, false); @@ -133,14 +115,10 @@ nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nvkm_vma *vma) else fence = reservation_object_get_excl(nvbo->bo.resv); - if (fence && mapped) { + if (fence && mapped) nouveau_fence_work(fence, nouveau_gem_object_delete, vma); - } else { - if (mapped) - nvkm_vm_unmap(vma); - nvkm_vm_put(vma); - kfree(vma); - } + else + nouveau_vma_del(&vma); } void @@ -150,7 +128,7 @@ nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) struct nouveau_bo *nvbo = nouveau_gem_object(gem); struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); struct device *dev = drm->dev->dev; - struct nvkm_vma *vma; + struct nouveau_vma *vma; int ret; if (!cli->vm) @@ -160,9 +138,9 @@ nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) if (ret) return; - vma = nouveau_bo_vma_find(nvbo, cli->vm); + vma = nouveau_vma_find(nvbo, &cli->vmm); if (vma) { - if (--vma->refcount == 0) { + if (--vma->refs == 0) { ret = pm_runtime_get_sync(dev); if (!WARN_ON(ret < 0 && ret != -EACCES)) { nouveau_gem_object_unmap(nvbo, vma); @@ -227,7 +205,7 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, { struct nouveau_cli *cli = nouveau_cli(file_priv); struct nouveau_bo *nvbo = nouveau_gem_object(gem); - struct nvkm_vma *vma; + struct nouveau_vma *vma; if (is_power_of_2(nvbo->valid_domains)) rep->domain = nvbo->valid_domains; @@ -237,11 +215,11 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; rep->offset = nvbo->bo.offset; if (cli->vm) { - vma = nouveau_bo_vma_find(nvbo, cli->vm); + vma = nouveau_vma_find(nvbo, &cli->vmm); if (!vma) return -EINVAL; - rep->offset = vma->offset; + rep->offset = vma->addr; } rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; @@ -798,7 +776,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, bo[push[i].bo_index].user_priv; uint32_t cmd; - cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2); + cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); cmd |= 0x20000000; if (unlikely(cmd != req->suffix0)) { if (!nvbo->kmap.virtual) { @@ -850,7 +828,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, req->suffix1 = 0x00000000; } else { req->suffix0 = 0x20000000 | - (chan->push.vma.offset + ((chan->dma.cur + 2) << 2)); + (chan->push.addr + ((chan->dma.cur + 2) << 2)); req->suffix1 = 0x00000000; } diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.c b/drivers/gpu/drm/nouveau/nouveau_vmm.c new file mode 100644 index 0000000000000..855d549e17cf9 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.c @@ -0,0 +1,128 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "nouveau_vmm.h" +#include "nouveau_drv.h" +#include "nouveau_bo.h" +#include "nouveau_mem.h" + +void +nouveau_vma_unmap(struct nouveau_vma *vma) +{ + if (vma->mem) { + nvkm_vm_unmap(&vma->_vma); + vma->mem = NULL; + } +} + +int +nouveau_vma_map(struct nouveau_vma *vma, struct nouveau_mem *mem) +{ + int ret = nouveau_mem_map(mem, vma->vmm->vm, &vma->_vma); + if (ret) + return ret; + vma->mem = mem; + return 0; +} + +struct nouveau_vma * +nouveau_vma_find(struct nouveau_bo *nvbo, struct nouveau_vmm *vmm) +{ + struct nouveau_vma *vma; + + list_for_each_entry(vma, &nvbo->vma_list, head) { + if (vma->vmm == vmm) + return vma; + } + + return NULL; +} + +void +nouveau_vma_del(struct nouveau_vma **pvma) +{ + struct nouveau_vma *vma = *pvma; + if (vma && --vma->refs <= 0) { + if (likely(vma->addr != ~0ULL)) { + nouveau_vma_unmap(vma); + nvkm_vm_put(&vma->_vma); + } + list_del(&vma->head); + *pvma = NULL; + kfree(*pvma); + } +} + +int +nouveau_vma_new(struct nouveau_bo *nvbo, struct nouveau_vmm *vmm, + struct nouveau_vma **pvma) +{ + struct nouveau_mem *mem = nouveau_mem(&nvbo->bo.mem); + struct nouveau_vma *vma; + int ret; + + if ((vma = *pvma = nouveau_vma_find(nvbo, vmm))) { + vma->refs++; + return 0; + } + + if (!(vma = *pvma = kmalloc(sizeof(*vma), GFP_KERNEL))) + return -ENOMEM; + vma->vmm = vmm; + vma->refs = 1; + vma->addr = ~0ULL; + vma->mem = NULL; + list_add_tail(&vma->head, &nvbo->vma_list); + + if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM && + mem->mem.page == nvbo->page) { + ret = nvkm_vm_get(vmm->vm, mem->_mem->size << 12, mem->mem.page, + NV_MEM_ACCESS_RW, &vma->_vma); + if (ret) + goto done; + + vma->addr = vma->_vma.offset; + ret = nouveau_vma_map(vma, mem); + } else { + ret = nvkm_vm_get(vmm->vm, mem->_mem->size << 12, mem->mem.page, + NV_MEM_ACCESS_RW, &vma->_vma); + vma->addr = vma->_vma.offset; + } + +done: + if (ret) + nouveau_vma_del(pvma); + return ret; +} + +void +nouveau_vmm_fini(struct nouveau_vmm *vmm) +{ + nvkm_vm_ref(NULL, &vmm->vm, NULL); +} + +int +nouveau_vmm_init(struct nouveau_cli *cli, s32 oclass, struct nouveau_vmm *vmm) +{ + vmm->cli = cli; + return nvkm_vm_new(nvxx_device(&cli->device), 0, (1ULL << 40), + 0x1000, NULL, &vmm->vm); +} diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.h b/drivers/gpu/drm/nouveau/nouveau_vmm.h new file mode 100644 index 0000000000000..86ab599aafb20 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.h @@ -0,0 +1,32 @@ +#ifndef __NOUVEAU_VMA_H__ +#define __NOUVEAU_VMA_H__ +#include +struct nouveau_bo; +struct nouveau_mem; + +struct nouveau_vma { + struct nouveau_vmm *vmm; + int refs; + struct list_head head; + u64 addr; + + struct nouveau_mem *mem; + + struct nvkm_vma _vma; +}; + +struct nouveau_vma *nouveau_vma_find(struct nouveau_bo *, struct nouveau_vmm *); +int nouveau_vma_new(struct nouveau_bo *, struct nouveau_vmm *, + struct nouveau_vma **); +void nouveau_vma_del(struct nouveau_vma **); +int nouveau_vma_map(struct nouveau_vma *, struct nouveau_mem *); +void nouveau_vma_unmap(struct nouveau_vma *); + +struct nouveau_vmm { + struct nouveau_cli *cli; + struct nvkm_vm *vm; +}; + +int nouveau_vmm_init(struct nouveau_cli *, s32 oclass, struct nouveau_vmm *); +void nouveau_vmm_fini(struct nouveau_vmm *); +#endif diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c index 327dcd7901edf..facd18564e0d8 100644 --- a/drivers/gpu/drm/nouveau/nv50_fbcon.c +++ b/drivers/gpu/drm/nouveau/nv50_fbcon.c @@ -25,6 +25,7 @@ #include "nouveau_drv.h" #include "nouveau_dma.h" #include "nouveau_fbcon.h" +#include "nouveau_vmm.h" int nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) @@ -239,8 +240,8 @@ nv50_fbcon_accel_init(struct fb_info *info) OUT_RING(chan, info->fix.line_length); OUT_RING(chan, info->var.xres_virtual); OUT_RING(chan, info->var.yres_virtual); - OUT_RING(chan, upper_32_bits(fb->vma.offset)); - OUT_RING(chan, lower_32_bits(fb->vma.offset)); + OUT_RING(chan, upper_32_bits(fb->vma->addr)); + OUT_RING(chan, lower_32_bits(fb->vma->addr)); BEGIN_NV04(chan, NvSub2D, 0x0230, 2); OUT_RING(chan, format); OUT_RING(chan, 1); @@ -248,8 +249,8 @@ nv50_fbcon_accel_init(struct fb_info *info) OUT_RING(chan, info->fix.line_length); OUT_RING(chan, info->var.xres_virtual); OUT_RING(chan, info->var.yres_virtual); - OUT_RING(chan, upper_32_bits(fb->vma.offset)); - OUT_RING(chan, lower_32_bits(fb->vma.offset)); + OUT_RING(chan, upper_32_bits(fb->vma->addr)); + OUT_RING(chan, lower_32_bits(fb->vma->addr)); FIRE_RING(chan); return 0; diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c index e5f6249edde9e..5f0c0c27d5dcf 100644 --- a/drivers/gpu/drm/nouveau/nv84_fence.c +++ b/drivers/gpu/drm/nouveau/nv84_fence.c @@ -25,6 +25,7 @@ #include "nouveau_drv.h" #include "nouveau_dma.h" #include "nouveau_fence.h" +#include "nouveau_vmm.h" #include "nv50_display.h" @@ -68,7 +69,7 @@ nv84_fence_emit(struct nouveau_fence *fence) { struct nouveau_channel *chan = fence->channel; struct nv84_fence_chan *fctx = chan->fence; - u64 addr = fctx->vma.offset + chan->chid * 16; + u64 addr = fctx->vma->addr + chan->chid * 16; return fctx->base.emit32(chan, addr, fence->base.seqno); } @@ -78,7 +79,7 @@ nv84_fence_sync(struct nouveau_fence *fence, struct nouveau_channel *prev, struct nouveau_channel *chan) { struct nv84_fence_chan *fctx = chan->fence; - u64 addr = fctx->vma.offset + prev->chid * 16; + u64 addr = fctx->vma->addr + prev->chid * 16; return fctx->base.sync32(chan, addr, fence->base.seqno); } @@ -98,7 +99,7 @@ nv84_fence_context_del(struct nouveau_channel *chan) nouveau_bo_wr32(priv->bo, chan->chid * 16 / 4, fctx->base.sequence); mutex_lock(&priv->mutex); - nouveau_bo_vma_del(priv->bo, &fctx->vma); + nouveau_vma_del(&fctx->vma); mutex_unlock(&priv->mutex); nouveau_fence_context_del(&fctx->base); chan->fence = NULL; @@ -126,7 +127,7 @@ nv84_fence_context_new(struct nouveau_channel *chan) fctx->base.sequence = nv84_fence_read(chan); mutex_lock(&priv->mutex); - ret = nouveau_bo_vma_add(priv->bo, cli->vm, &fctx->vma); + ret = nouveau_vma_new(priv->bo, &cli->vmm, &fctx->vma); mutex_unlock(&priv->mutex); if (ret) diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c index 90f27bfa381fd..c0deef4fe7274 100644 --- a/drivers/gpu/drm/nouveau/nvc0_fbcon.c +++ b/drivers/gpu/drm/nouveau/nvc0_fbcon.c @@ -25,6 +25,7 @@ #include "nouveau_drv.h" #include "nouveau_dma.h" #include "nouveau_fbcon.h" +#include "nouveau_vmm.h" int nvc0_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) @@ -239,8 +240,8 @@ nvc0_fbcon_accel_init(struct fb_info *info) OUT_RING (chan, info->fix.line_length); OUT_RING (chan, info->var.xres_virtual); OUT_RING (chan, info->var.yres_virtual); - OUT_RING (chan, upper_32_bits(fb->vma.offset)); - OUT_RING (chan, lower_32_bits(fb->vma.offset)); + OUT_RING (chan, upper_32_bits(fb->vma->addr)); + OUT_RING (chan, lower_32_bits(fb->vma->addr)); BEGIN_NVC0(chan, NvSub2D, 0x0230, 10); OUT_RING (chan, format); OUT_RING (chan, 1); @@ -250,8 +251,8 @@ nvc0_fbcon_accel_init(struct fb_info *info) OUT_RING (chan, info->fix.line_length); OUT_RING (chan, info->var.xres_virtual); OUT_RING (chan, info->var.yres_virtual); - OUT_RING (chan, upper_32_bits(fb->vma.offset)); - OUT_RING (chan, lower_32_bits(fb->vma.offset)); + OUT_RING (chan, upper_32_bits(fb->vma->addr)); + OUT_RING (chan, lower_32_bits(fb->vma->addr)); FIRE_RING (chan); return 0; From 4d058fab63f79e5cf13d21edd9db1a63748da0a1 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 061/151] drm/nouveau/core/mm: have users explicitly define heap identifiers Different sections of VRAM may have different properties (ie. can't be used for compression/display, can't be mapped, etc). We currently already support this, but it's a bit magic. This change makes it more obvious where we're allocating from. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/core/mm.h | 2 +- drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h | 4 ++++ drivers/gpu/drm/nouveau/nouveau_abi16.c | 2 +- drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c | 2 +- drivers/gpu/drm/nouveau/nvkm/core/mm.c | 5 +++-- drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c | 5 +++-- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c | 9 ++++++--- drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 2 +- 15 files changed, 31 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/mm.h b/drivers/gpu/drm/nouveau/include/nvkm/core/mm.h index 7bd4897a8a2ad..21b1a4d43f517 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/mm.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/mm.h @@ -30,7 +30,7 @@ nvkm_mm_initialised(struct nvkm_mm *mm) return mm->heap_nodes; } -int nvkm_mm_init(struct nvkm_mm *, u32 offset, u32 length, u32 block); +int nvkm_mm_init(struct nvkm_mm *, u8 heap, u32 offset, u32 length, u32 block); int nvkm_mm_fini(struct nvkm_mm *); int nvkm_mm_head(struct nvkm_mm *, u8 heap, u8 type, u32 size_max, u32 size_min, u32 align, struct nvkm_mm_node **); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index dde444a7d3f04..29b21a4f189c1 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -123,6 +123,10 @@ struct nvkm_ram { u64 size; #define NVKM_RAM_MM_SHIFT 12 +#define NVKM_RAM_MM_ANY (NVKM_MM_HEAP_ANY + 0) +#define NVKM_RAM_MM_NORMAL (NVKM_MM_HEAP_ANY + 1) +#define NVKM_RAM_MM_NOMAP (NVKM_MM_HEAP_ANY + 2) +#define NVKM_RAM_MM_MIXED (NVKM_MM_HEAP_ANY + 3) struct nvkm_mm vram; struct nvkm_mm tags; u64 stolen; diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c index 9b0b210a37f82..4c4e016aba951 100644 --- a/drivers/gpu/drm/nouveau/nouveau_abi16.c +++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c @@ -340,7 +340,7 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS) if (ret) goto done; - ret = nvkm_mm_init(&chan->heap, 0, PAGE_SIZE, 1); + ret = nvkm_mm_init(&chan->heap, 0, 0, PAGE_SIZE, 1); done: if (ret) nouveau_abi16_chan_fini(abi16, chan); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c b/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c index a7bd22706b2a5..04e5b2136f0cf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c @@ -185,7 +185,7 @@ nvkm_gpuobj_ctor(struct nvkm_device *device, u32 size, int align, bool zero, gpuobj->size = nvkm_memory_size(gpuobj->memory); } - return nvkm_mm_init(&gpuobj->heap, 0, gpuobj->size, 1); + return nvkm_mm_init(&gpuobj->heap, 0, 0, gpuobj->size, 1); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/core/mm.c b/drivers/gpu/drm/nouveau/nvkm/core/mm.c index 5c7891234eea5..f78a06a6b2f16 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/mm.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/mm.c @@ -237,7 +237,7 @@ nvkm_mm_tail(struct nvkm_mm *mm, u8 heap, u8 type, u32 size_max, u32 size_min, } int -nvkm_mm_init(struct nvkm_mm *mm, u32 offset, u32 length, u32 block) +nvkm_mm_init(struct nvkm_mm *mm, u8 heap, u32 offset, u32 length, u32 block) { struct nvkm_mm_node *node, *prev; u32 next; @@ -274,7 +274,8 @@ nvkm_mm_init(struct nvkm_mm *mm, u32 offset, u32 length, u32 block) list_add_tail(&node->nl_entry, &mm->nodes); list_add_tail(&node->fl_entry, &mm->free); - node->heap = ++mm->heap_nodes; + node->heap = heap; + mm->heap_nodes++; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c index 30483c5d27c3b..d35a32e168d6e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c @@ -136,7 +136,7 @@ nvkm_device_tegra_probe_iommu(struct nvkm_device_tegra *tdev) if (ret) goto free_domain; - ret = nvkm_mm_init(&tdev->iommu.mm, 0, + ret = nvkm_mm_init(&tdev->iommu.mm, 0, 0, (1ULL << tdev->func->iommu_bit) >> tdev->iommu.pgshift, 1); if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c index c17d559dbfbe3..5d908177766f1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c @@ -73,13 +73,14 @@ nvkm_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb, ram->size = size; if (!nvkm_mm_initialised(&ram->vram)) { - ret = nvkm_mm_init(&ram->vram, 0, size >> NVKM_RAM_MM_SHIFT, 1); + ret = nvkm_mm_init(&ram->vram, NVKM_RAM_MM_NORMAL, 0, + size >> NVKM_RAM_MM_SHIFT, 1); if (ret) return ret; } if (!nvkm_mm_initialised(&ram->tags)) { - ret = nvkm_mm_init(&ram->tags, 0, tags ? ++tags : 0, 1); + ret = nvkm_mm_init(&ram->tags, 0, 0, tags ? ++tags : 0, 1); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c index 4a9bd4f1cb931..f3fcb499242fc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c @@ -617,7 +617,8 @@ gf100_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb, */ if (lower != total) { /* The common memory amount is addressed normally. */ - ret = nvkm_mm_init(&ram->vram, rsvd_head >> NVKM_RAM_MM_SHIFT, + ret = nvkm_mm_init(&ram->vram, NVKM_RAM_MM_NORMAL, + rsvd_head >> NVKM_RAM_MM_SHIFT, (lower - rsvd_head) >> NVKM_RAM_MM_SHIFT, 1); if (ret) return ret; @@ -625,13 +626,15 @@ gf100_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb, /* And the rest is much higher in the physical address * space, and may not be usable for certain operations. */ - ret = nvkm_mm_init(&ram->vram, ubase >> NVKM_RAM_MM_SHIFT, + ret = nvkm_mm_init(&ram->vram, NVKM_RAM_MM_MIXED, + ubase >> NVKM_RAM_MM_SHIFT, (usize - rsvd_tail) >> NVKM_RAM_MM_SHIFT, 1); if (ret) return ret; } else { /* GPUs without mixed-memory are a lot nicer... */ - ret = nvkm_mm_init(&ram->vram, rsvd_head >> NVKM_RAM_MM_SHIFT, + ret = nvkm_mm_init(&ram->vram, NVKM_RAM_MM_NORMAL, + rsvd_head >> NVKM_RAM_MM_SHIFT, (total - rsvd_head - rsvd_tail) >> NVKM_RAM_MM_SHIFT, 1); if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c index 017a91de74a0d..28ab1914718e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c @@ -81,7 +81,8 @@ mcp77_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) ram->base.stolen = base; nvkm_mm_fini(&ram->base.vram); - return nvkm_mm_init(&ram->base.vram, rsvd_head >> NVKM_RAM_MM_SHIFT, + return nvkm_mm_init(&ram->base.vram, NVKM_RAM_MM_NORMAL, + rsvd_head >> NVKM_RAM_MM_SHIFT, (size - rsvd_head - rsvd_tail) >> NVKM_RAM_MM_SHIFT, 1); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c index 6549b05883095..949188569794f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c @@ -669,7 +669,8 @@ nv50_ram_ctor(const struct nvkm_ram_func *func, ram->ranks = (nvkm_rd32(device, 0x100200) & 0x4) ? 2 : 1; nvkm_mm_fini(&ram->vram); - return nvkm_mm_init(&ram->vram, rsvd_head >> NVKM_RAM_MM_SHIFT, + return nvkm_mm_init(&ram->vram, NVKM_RAM_MM_NORMAL, + rsvd_head >> NVKM_RAM_MM_SHIFT, (size - rsvd_head - rsvd_tail) >> NVKM_RAM_MM_SHIFT, nv50_fb_vram_rblock(ram) >> NVKM_RAM_MM_SHIFT); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c index df75da764bd42..2e1141319e939 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c @@ -165,7 +165,7 @@ nv04_instmem_oneinit(struct nvkm_instmem *base) /* PRAMIN aperture maps over the end of VRAM, reserve it */ imem->base.reserved = 512 * 1024; - ret = nvkm_mm_init(&imem->heap, 0, imem->base.reserved, 1); + ret = nvkm_mm_init(&imem->heap, 0, 0, imem->base.reserved, 1); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c index d8ab5fa86893f..7f52a525d2e1a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c @@ -177,7 +177,7 @@ nv40_instmem_oneinit(struct nvkm_instmem *base) imem->base.reserved += 512 * 1024; /* object storage */ imem->base.reserved = round_up(imem->base.reserved, 4096); - ret = nvkm_mm_init(&imem->heap, 0, imem->base.reserved, 1); + ret = nvkm_mm_init(&imem->heap, 0, 0, imem->base.reserved, 1); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c index 4a0fa0a9b8028..6b0cb7f3fa776 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c @@ -183,8 +183,8 @@ gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc) tag_size += tag_align; tag_size = (tag_size + 0xfff) >> 12; /* round up */ - ret = nvkm_mm_tail(&ram->vram, 1, 1, tag_size, tag_size, 1, - <c->tag_ram); + ret = nvkm_mm_tail(&ram->vram, NVKM_RAM_MM_NORMAL, 1, tag_size, + tag_size, 1, <c->tag_ram); if (ret) { ltc->num_tags = 0; } else { @@ -197,7 +197,7 @@ gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc) } mm_init: - return nvkm_mm_init(<c->tags, 0, ltc->num_tags, 1); + return nvkm_mm_init(<c->tags, 0, 0, ltc->num_tags, 1); } int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c index 0bdfb2f40266d..74ed4dd203bc6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c @@ -45,7 +45,7 @@ gp100_ltc_oneinit(struct nvkm_ltc *ltc) ltc->ltc_nr = nvkm_rd32(device, 0x12006c); ltc->lts_nr = nvkm_rd32(device, 0x17e280) >> 28; /*XXX: tagram allocation - TBD */ - return nvkm_mm_init(<c->tags, 0, 0, 1); + return nvkm_mm_init(<c->tags, 0, 0, 0, 1); } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index f2b1a3e75f176..44c2403e88e6c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -388,7 +388,7 @@ nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, return -ENOMEM; } - ret = nvkm_mm_init(&vm->mm, mm_offset >> 12, mm_length >> 12, + ret = nvkm_mm_init(&vm->mm, 0, mm_offset >> 12, mm_length >> 12, block >> 12); if (ret) { vfree(vm->pgt); From b7e1f3f1ba0de8585ddb5a921714b06fcb92f16e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 062/151] drm/nouveau/core/mm: introduce functions to access info about a given allocation These will be used in upcoming patches. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/core/mm.h | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/mm.h b/drivers/gpu/drm/nouveau/include/nvkm/core/mm.h index 21b1a4d43f517..5c1261351138b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/mm.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/mm.h @@ -39,9 +39,39 @@ int nvkm_mm_tail(struct nvkm_mm *, u8 heap, u8 type, u32 size_max, void nvkm_mm_free(struct nvkm_mm *, struct nvkm_mm_node **); void nvkm_mm_dump(struct nvkm_mm *, const char *); +static inline u32 +nvkm_mm_heap_size(struct nvkm_mm *mm, u8 heap) +{ + struct nvkm_mm_node *node; + u32 size = 0; + list_for_each_entry(node, &mm->nodes, nl_entry) { + if (node->heap == heap) + size += node->length; + } + return size; +} + static inline bool nvkm_mm_contiguous(struct nvkm_mm_node *node) { return !node->next; } + +static inline u32 +nvkm_mm_addr(struct nvkm_mm_node *node) +{ + if (WARN_ON(!nvkm_mm_contiguous(node))) + return 0; + return node->offset; +} + +static inline u32 +nvkm_mm_size(struct nvkm_mm_node *node) +{ + u32 size = 0; + do { + size += node->length; + } while ((node = node->next)); + return size; +} #endif From 7ef44bee64c68db7b7bc0cb341eec41891c6d0a5 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 063/151] drm/nouveau/fb: move comptags mm into nvkm_fb We're moving towards having a central place to handle comptag allocation, and as some GPUs don't have a ram submodule (ie. Tegra), we need to move the mm somewhere else. It probably never belonged in ram anyways. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c | 4 ++-- drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c | 5 ++--- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c | 4 ++-- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index 29b21a4f189c1..b838d93858597 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -44,6 +44,7 @@ struct nvkm_fb { struct nvkm_subdev subdev; struct nvkm_ram *ram; + struct nvkm_mm tags; struct { struct nvkm_fb_tile region[16]; @@ -128,7 +129,6 @@ struct nvkm_ram { #define NVKM_RAM_MM_NOMAP (NVKM_MM_HEAP_ANY + 2) #define NVKM_RAM_MM_MIXED (NVKM_MM_HEAP_ANY + 3) struct nvkm_mm vram; - struct nvkm_mm tags; u64 stolen; int ranks; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c index fa0fad10b62cd..dea04406816d1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c @@ -159,6 +159,7 @@ nvkm_fb_dtor(struct nvkm_subdev *subdev) for (i = 0; i < fb->tile.regions; i++) fb->func->tile.fini(fb, i, &fb->tile.region[i]); + nvkm_mm_fini(&fb->tags); nvkm_ram_del(&fb->ram); if (fb->func->dtor) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c index 126865dfe7773..ec5a438b79dea 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c @@ -45,7 +45,7 @@ nv20_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags, { u32 tiles = DIV_ROUND_UP(size, 0x40); u32 tags = round_up(tiles / fb->ram->parts, 0x40); - if (!nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) { + if (!nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) { if (!(flags & 2)) tile->zcomp = 0x00000000; /* Z16 */ else tile->zcomp = 0x04000000; /* Z24S8 */ tile->zcomp |= tile->tag->offset; @@ -63,7 +63,7 @@ nv20_fb_tile_fini(struct nvkm_fb *fb, int i, struct nvkm_fb_tile *tile) tile->limit = 0; tile->pitch = 0; tile->zcomp = 0; - nvkm_mm_free(&fb->ram->tags, &tile->tag); + nvkm_mm_free(&fb->tags, &tile->tag); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c index c56746d2a502c..bf342a239b486 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c @@ -32,7 +32,7 @@ nv25_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags, { u32 tiles = DIV_ROUND_UP(size, 0x40); u32 tags = round_up(tiles / fb->ram->parts, 0x40); - if (!nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) { + if (!nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) { if (!(flags & 2)) tile->zcomp = 0x00100000; /* Z16 */ else tile->zcomp = 0x00200000; /* Z24S8 */ tile->zcomp |= tile->tag->offset; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c index 2a7c4831b8217..2280fbae508ad 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c @@ -51,7 +51,7 @@ nv30_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags, { u32 tiles = DIV_ROUND_UP(size, 0x40); u32 tags = round_up(tiles / fb->ram->parts, 0x40); - if (!nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) { + if (!nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) { if (flags & 2) tile->zcomp |= 0x01000000; /* Z16 */ else tile->zcomp |= 0x02000000; /* Z24S8 */ tile->zcomp |= ((tile->tag->offset ) >> 6); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c index 1604b3789ad16..244c3ec3a4edd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c @@ -32,7 +32,7 @@ nv35_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags, { u32 tiles = DIV_ROUND_UP(size, 0x40); u32 tags = round_up(tiles / fb->ram->parts, 0x40); - if (!nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) { + if (!nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) { if (flags & 2) tile->zcomp |= 0x04000000; /* Z16 */ else tile->zcomp |= 0x08000000; /* Z24S8 */ tile->zcomp |= ((tile->tag->offset ) >> 6); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c index 80cc0a6e3416d..a0a9b1fd0a000 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c @@ -32,7 +32,7 @@ nv36_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags, { u32 tiles = DIV_ROUND_UP(size, 0x40); u32 tags = round_up(tiles / fb->ram->parts, 0x40); - if (!nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) { + if (!nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) { if (flags & 2) tile->zcomp |= 0x10000000; /* Z16 */ else tile->zcomp |= 0x20000000; /* Z24S8 */ tile->zcomp |= ((tile->tag->offset ) >> 6); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c index deec46a310f8f..bdb97bb03b624 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c @@ -33,7 +33,7 @@ nv40_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags, u32 tiles = DIV_ROUND_UP(size, 0x80); u32 tags = round_up(tiles / fb->ram->parts, 0x100); if ( (flags & 2) && - !nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) { + !nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) { tile->zcomp = 0x28000000; /* Z24S8_SPLIT_GRAD */ tile->zcomp |= ((tile->tag->offset ) >> 8); tile->zcomp |= ((tile->tag->offset + tags - 1) >> 8) << 13; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c index 5d908177766f1..2cfd4336b5c56 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c @@ -38,7 +38,6 @@ nvkm_ram_del(struct nvkm_ram **pram) if (ram && !WARN_ON(!ram->func)) { if (ram->func->dtor) *pram = ram->func->dtor(ram); - nvkm_mm_fini(&ram->tags); nvkm_mm_fini(&ram->vram); kfree(*pram); *pram = NULL; @@ -79,8 +78,8 @@ nvkm_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb, return ret; } - if (!nvkm_mm_initialised(&ram->tags)) { - ret = nvkm_mm_init(&ram->tags, 0, 0, tags ? ++tags : 0, 1); + if (!nvkm_mm_initialised(&fb->tags)) { + ret = nvkm_mm_init(&fb->tags, 0, 0, tags ? ++tags : 0, 1); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c index 949188569794f..0fa02fbc561d2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c @@ -502,7 +502,7 @@ __nv50_ram_put(struct nvkm_ram *ram, struct nvkm_mem *mem) next = node->next; nvkm_mm_free(&ram->vram, &node); } - nvkm_mm_free(&ram->tags, &mem->tag); + nvkm_mm_free(&ram->fb->tags, &mem->tag); } void @@ -526,7 +526,7 @@ nv50_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin, u32 memtype, struct nvkm_mem **pmem) { struct nvkm_mm *heap = &ram->vram; - struct nvkm_mm *tags = &ram->tags; + struct nvkm_mm *tags = &ram->fb->tags; struct nvkm_mm_node **node, *r; struct nvkm_mem *mem; int comp = (memtype & 0x300) >> 8; From af793b8cd9f62880852379897519bbe69fb59c05 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 064/151] drm/nouveau/fb: move comptag init out of ram submodule Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c | 12 +++++++++++- drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c | 8 ++++++++ drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c | 11 +++++++++++ drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c | 16 +++------------- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h | 6 ++---- .../gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c | 3 +-- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c | 7 +++---- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c | 3 +-- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c | 3 +-- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c | 3 +-- 30 files changed, 63 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c index dea04406816d1..94c53454501d3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c @@ -100,6 +100,7 @@ static int nvkm_fb_oneinit(struct nvkm_subdev *subdev) { struct nvkm_fb *fb = nvkm_fb(subdev); + u32 tags = 0; if (fb->func->ram_new) { int ret = fb->func->ram_new(fb, &fb->ram); @@ -115,7 +116,16 @@ nvkm_fb_oneinit(struct nvkm_subdev *subdev) return ret; } - return 0; + /* Initialise compression tag allocator. + * + * LTC oneinit() will override this on Fermi and newer. + */ + if (fb->func->tags) { + tags = fb->func->tags(fb); + nvkm_debug(subdev, "%d comptags\n", tags); + } + + return nvkm_mm_init(&fb->tags, 0, 0, tags, 1); } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c index 9c28392d07e43..06bf95c0c5493 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c @@ -27,6 +27,7 @@ static const struct nv50_fb_func g84_fb = { .ram_new = nv50_ram_new, + .tags = nv20_fb_tags, .trap = 0x001d07ff, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c index ebb30608d5efa..9266559b45f97 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c @@ -27,6 +27,7 @@ static const struct nv50_fb_func gt215_fb = { .ram_new = gt215_ram_new, + .tags = nv20_fb_tags, .trap = 0x000d0fff, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c index ec5a438b79dea..20e47248ba36e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c @@ -77,8 +77,16 @@ nv20_fb_tile_prog(struct nvkm_fb *fb, int i, struct nvkm_fb_tile *tile) nvkm_wr32(device, 0x100300 + (i * 0x04), tile->zcomp); } +u32 +nv20_fb_tags(struct nvkm_fb *fb) +{ + const u32 tags = nvkm_rd32(fb->subdev.device, 0x100320); + return tags ? tags + 1 : 0; +} + static const struct nvkm_fb_func nv20_fb = { + .tags = nv20_fb_tags, .tile.regions = 8, .tile.init = nv20_fb_tile_init, .tile.comp = nv20_fb_tile_comp, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c index bf342a239b486..dc2965514abe3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c @@ -44,6 +44,7 @@ nv25_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags, static const struct nvkm_fb_func nv25_fb = { + .tags = nv20_fb_tags, .tile.regions = 8, .tile.init = nv20_fb_tile_init, .tile.comp = nv25_fb_tile_comp, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c index 2280fbae508ad..28b147d8d628b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c @@ -116,6 +116,7 @@ nv30_fb_init(struct nvkm_fb *fb) static const struct nvkm_fb_func nv30_fb = { + .tags = nv20_fb_tags, .init = nv30_fb_init, .tile.regions = 8, .tile.init = nv30_fb_tile_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c index 244c3ec3a4edd..4e213da6a2155 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c @@ -45,6 +45,7 @@ nv35_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags, static const struct nvkm_fb_func nv35_fb = { + .tags = nv20_fb_tags, .init = nv30_fb_init, .tile.regions = 8, .tile.init = nv30_fb_tile_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c index a0a9b1fd0a000..9f029f02088c0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c @@ -45,6 +45,7 @@ nv36_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags, static const struct nvkm_fb_func nv36_fb = { + .tags = nv20_fb_tags, .init = nv30_fb_init, .tile.regions = 8, .tile.init = nv30_fb_tile_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c index bdb97bb03b624..893bc8b3cc803 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c @@ -51,6 +51,7 @@ nv40_fb_init(struct nvkm_fb *fb) static const struct nvkm_fb_func nv40_fb = { + .tags = nv20_fb_tags, .init = nv40_fb_init, .tile.regions = 8, .tile.init = nv30_fb_tile_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c index 79e57dd5a00f4..c1a79564a2f54 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c @@ -45,6 +45,7 @@ nv41_fb_init(struct nvkm_fb *fb) static const struct nvkm_fb_func nv41_fb = { + .tags = nv20_fb_tags, .init = nv41_fb_init, .tile.regions = 12, .tile.init = nv30_fb_tile_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c index c505e44293142..d1d5fc952e328 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c @@ -28,6 +28,7 @@ static const struct nvkm_fb_func nv47_fb = { + .tags = nv20_fb_tags, .init = nv41_fb_init, .tile.regions = 15, .tile.init = nv30_fb_tile_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c index 7b91b9f170e5b..b3dff3543062f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c @@ -28,6 +28,7 @@ static const struct nvkm_fb_func nv49_fb = { + .tags = nv20_fb_tags, .init = nv41_fb_init, .tile.regions = 15, .tile.init = nv30_fb_tile_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c index 0595e0722bfcc..da797c4525b29 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c @@ -244,6 +244,15 @@ nv50_fb_init(struct nvkm_fb *base) nvkm_wr32(device, 0x100c90, fb->func->trap); } +static u32 +nv50_fb_tags(struct nvkm_fb *base) +{ + struct nv50_fb *fb = nv50_fb(base); + if (fb->func->tags) + return fb->func->tags(&fb->base); + return 0; +} + static void * nv50_fb_dtor(struct nvkm_fb *base) { @@ -262,6 +271,7 @@ nv50_fb_dtor(struct nvkm_fb *base) static const struct nvkm_fb_func nv50_fb_ = { .dtor = nv50_fb_dtor, + .tags = nv50_fb_tags, .oneinit = nv50_fb_oneinit, .init = nv50_fb_init, .intr = nv50_fb_intr, @@ -287,6 +297,7 @@ nv50_fb_new_(const struct nv50_fb_func *func, struct nvkm_device *device, static const struct nv50_fb_func nv50_fb = { .ram_new = nv50_ram_new, + .tags = nv20_fb_tags, .trap = 0x000707ff, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h index faa88c8c66fe6..74da4d7dcaa07 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h @@ -12,6 +12,7 @@ struct nv50_fb { struct nv50_fb_func { int (*ram_new)(struct nvkm_fb *, struct nvkm_ram **); + u32 (*tags)(struct nvkm_fb *); u32 trap; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h index afaa68da32212..329ff5e0115a6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h @@ -6,6 +6,7 @@ struct nvkm_bios; struct nvkm_fb_func { void *(*dtor)(struct nvkm_fb *); + u32 (*tags)(struct nvkm_fb *); int (*oneinit)(struct nvkm_fb *); void (*init)(struct nvkm_fb *); int (*init_page)(struct nvkm_fb *); @@ -42,6 +43,7 @@ void nv10_fb_tile_init(struct nvkm_fb *, int i, u32 addr, u32 size, void nv10_fb_tile_fini(struct nvkm_fb *, int i, struct nvkm_fb_tile *); void nv10_fb_tile_prog(struct nvkm_fb *, int, struct nvkm_fb_tile *); +u32 nv20_fb_tags(struct nvkm_fb *); void nv20_fb_tile_init(struct nvkm_fb *, int i, u32 addr, u32 size, u32 pitch, u32 flags, struct nvkm_fb_tile *); void nv20_fb_tile_fini(struct nvkm_fb *, int i, struct nvkm_fb_tile *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c index 2cfd4336b5c56..69f65daaf1eaa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c @@ -46,8 +46,7 @@ nvkm_ram_del(struct nvkm_ram **pram) int nvkm_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb, - enum nvkm_ram_type type, u64 size, u32 tags, - struct nvkm_ram *ram) + enum nvkm_ram_type type, u64 size, struct nvkm_ram *ram) { static const char *name[] = { [NVKM_RAM_TYPE_UNKNOWN] = "of unknown memory type", @@ -78,23 +77,14 @@ nvkm_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb, return ret; } - if (!nvkm_mm_initialised(&fb->tags)) { - ret = nvkm_mm_init(&fb->tags, 0, 0, tags ? ++tags : 0, 1); - if (ret) - return ret; - - nvkm_debug(subdev, "%d compression tags\n", tags); - } - return 0; } int nvkm_ram_new_(const struct nvkm_ram_func *func, struct nvkm_fb *fb, - enum nvkm_ram_type type, u64 size, u32 tags, - struct nvkm_ram **pram) + enum nvkm_ram_type type, u64 size, struct nvkm_ram **pram) { if (!(*pram = kzalloc(sizeof(**pram), GFP_KERNEL))) return -ENOMEM; - return nvkm_ram_ctor(func, fb, type, size, tags, *pram); + return nvkm_ram_ctor(func, fb, type, size, *pram); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h index fac7e73c3ddfb..a007699b60a2d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h @@ -3,11 +3,9 @@ #include "priv.h" int nvkm_ram_ctor(const struct nvkm_ram_func *, struct nvkm_fb *, - enum nvkm_ram_type, u64 size, u32 tags, - struct nvkm_ram *); + enum nvkm_ram_type, u64 size, struct nvkm_ram *); int nvkm_ram_new_(const struct nvkm_ram_func *, struct nvkm_fb *, - enum nvkm_ram_type, u64 size, u32 tags, - struct nvkm_ram **); + enum nvkm_ram_type, u64 size, struct nvkm_ram **); void nvkm_ram_del(struct nvkm_ram **); int nvkm_ram_init(struct nvkm_ram *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c index f3fcb499242fc..a44298989d3cc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c @@ -604,7 +604,7 @@ gf100_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb, nvkm_debug(subdev, "Upper: %4lld MiB @ %010llx\n", usize >> 20, ubase); nvkm_debug(subdev, "Total: %4lld MiB\n", total >> 20); - ret = nvkm_ram_ctor(func, fb, type, total, 0, ram); + ret = nvkm_ram_ctor(func, fb, type, total, ram); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c index 28ab1914718e0..76fa71f0c4576 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c @@ -73,7 +73,7 @@ mcp77_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) *pram = &ram->base; ret = nvkm_ram_ctor(&mcp77_ram_func, fb, NVKM_RAM_TYPE_STOLEN, - size, 0, &ram->base); + size, &ram->base); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c index 6f053a03d61c7..cc764a93f1a3e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c @@ -61,5 +61,5 @@ nv04_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) else type = NVKM_RAM_TYPE_SDRAM; - return nvkm_ram_new_(&nv04_ram_func, fb, type, size, 0, pram); + return nvkm_ram_new_(&nv04_ram_func, fb, type, size, pram); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c index dfd155c98dbb0..afe54e323b18e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c @@ -36,5 +36,5 @@ nv10_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) else type = NVKM_RAM_TYPE_SDRAM; - return nvkm_ram_new_(&nv04_ram_func, fb, type, size, 0, pram); + return nvkm_ram_new_(&nv04_ram_func, fb, type, size, pram); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c index 3c6a8710e8127..4c07d10bb9760 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c @@ -44,5 +44,5 @@ nv1a_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) } return nvkm_ram_new_(&nv04_ram_func, fb, NVKM_RAM_TYPE_STOLEN, - mib * 1024 * 1024, 0, pram); + mib * 1024 * 1024, pram); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c index 747e47c10cc74..71d63d7daa75b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c @@ -29,7 +29,6 @@ nv20_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) struct nvkm_device *device = fb->subdev.device; u32 pbus1218 = nvkm_rd32(device, 0x001218); u32 size = (nvkm_rd32(device, 0x10020c) & 0xff000000); - u32 tags = nvkm_rd32(device, 0x100320); enum nvkm_ram_type type = NVKM_RAM_TYPE_UNKNOWN; int ret; @@ -40,7 +39,7 @@ nv20_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) case 0x00000300: type = NVKM_RAM_TYPE_GDDR2; break; } - ret = nvkm_ram_new_(&nv04_ram_func, fb, type, size, tags, pram); + ret = nvkm_ram_new_(&nv04_ram_func, fb, type, size, pram); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c index 70c63535d56b3..2b12e388f47aa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c @@ -187,13 +187,13 @@ nv40_ram_func = { int nv40_ram_new_(struct nvkm_fb *fb, enum nvkm_ram_type type, u64 size, - u32 tags, struct nvkm_ram **pram) + struct nvkm_ram **pram) { struct nv40_ram *ram; if (!(ram = kzalloc(sizeof(*ram), GFP_KERNEL))) return -ENOMEM; *pram = &ram->base; - return nvkm_ram_ctor(&nv40_ram_func, fb, type, size, tags, &ram->base); + return nvkm_ram_ctor(&nv40_ram_func, fb, type, size, &ram->base); } int @@ -202,7 +202,6 @@ nv40_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) struct nvkm_device *device = fb->subdev.device; u32 pbus1218 = nvkm_rd32(device, 0x001218); u32 size = nvkm_rd32(device, 0x10020c) & 0xff000000; - u32 tags = nvkm_rd32(device, 0x100320); enum nvkm_ram_type type = NVKM_RAM_TYPE_UNKNOWN; int ret; @@ -213,7 +212,7 @@ nv40_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) case 0x00000300: type = NVKM_RAM_TYPE_DDR2 ; break; } - ret = nv40_ram_new_(fb, type, size, tags, pram); + ret = nv40_ram_new_(fb, type, size, pram); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h index 8a0524566b486..ec5dcbfcaea8f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h @@ -9,6 +9,6 @@ struct nv40_ram { u32 coef; }; -int nv40_ram_new_(struct nvkm_fb *fb, enum nvkm_ram_type, u64, u32, +int nv40_ram_new_(struct nvkm_fb *fb, enum nvkm_ram_type, u64, struct nvkm_ram **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c index 114828be292ef..d3fea3726461d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c @@ -28,7 +28,6 @@ nv41_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) { struct nvkm_device *device = fb->subdev.device; u32 size = nvkm_rd32(device, 0x10020c) & 0xff000000; - u32 tags = nvkm_rd32(device, 0x100320); u32 fb474 = nvkm_rd32(device, 0x100474); enum nvkm_ram_type type = NVKM_RAM_TYPE_UNKNOWN; int ret; @@ -40,7 +39,7 @@ nv41_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) if (fb474 & 0x00000001) type = NVKM_RAM_TYPE_DDR1; - ret = nv40_ram_new_(fb, type, size, tags, pram); + ret = nv40_ram_new_(fb, type, size, pram); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c index bc56fbf1c788c..ab2630e5e6fbd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c @@ -38,5 +38,5 @@ nv44_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) if (fb474 & 0x00000001) type = NVKM_RAM_TYPE_DDR1; - return nv40_ram_new_(fb, type, size, 0, pram); + return nv40_ram_new_(fb, type, size, pram); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c index c01f4b1022b87..946ca7c2e0b69 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c @@ -28,7 +28,6 @@ nv49_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) { struct nvkm_device *device = fb->subdev.device; u32 size = nvkm_rd32(device, 0x10020c) & 0xff000000; - u32 tags = nvkm_rd32(device, 0x100320); u32 fb914 = nvkm_rd32(device, 0x100914); enum nvkm_ram_type type = NVKM_RAM_TYPE_UNKNOWN; int ret; @@ -40,7 +39,7 @@ nv49_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) case 0x00000003: break; } - ret = nv40_ram_new_(fb, type, size, tags, pram); + ret = nv40_ram_new_(fb, type, size, pram); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c index fa3c2e06203dc..02b8bdbc819f5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c @@ -29,5 +29,5 @@ nv4e_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) struct nvkm_device *device = fb->subdev.device; u32 size = nvkm_rd32(device, 0x10020c) & 0xff000000; return nvkm_ram_new_(&nv04_ram_func, fb, NVKM_RAM_TYPE_UNKNOWN, - size, 0, pram); + size, pram); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c index 0fa02fbc561d2..030ac8339f496 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c @@ -639,7 +639,6 @@ nv50_ram_ctor(const struct nvkm_ram_func *func, const u32 rsvd_head = ( 256 * 1024); /* vga memory */ const u32 rsvd_tail = (1024 * 1024); /* vbios etc */ u64 size = nvkm_rd32(device, 0x10020c); - u32 tags = nvkm_rd32(device, 0x100320); enum nvkm_ram_type type = NVKM_RAM_TYPE_UNKNOWN; int ret; @@ -660,7 +659,7 @@ nv50_ram_ctor(const struct nvkm_ram_func *func, size = (size & 0x000000ff) << 32 | (size & 0xffffff00); - ret = nvkm_ram_ctor(func, fb, type, size, tags, ram); + ret = nvkm_ram_ctor(func, fb, type, size, ram); if (ret) return ret; From b1e839f3b3d82a0774c296f9950214057b045b72 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 065/151] drm/nouveau/fb/gf100: clear comptags at allocation time rather than mmu map We probably don't want to destroy compression data when doing multiple mappings of a memory object. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c | 6 +++++- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 3 --- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h index cd755baf9cab7..93eeb557a9f26 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h @@ -25,7 +25,7 @@ struct nvkm_ltc { int nvkm_ltc_tags_alloc(struct nvkm_ltc *, u32 count, struct nvkm_mm_node **); void nvkm_ltc_tags_free(struct nvkm_ltc *, struct nvkm_mm_node **); -void nvkm_ltc_tags_clear(struct nvkm_ltc *, u32 first, u32 count); +void nvkm_ltc_tags_clear(struct nvkm_device *, u32 first, u32 count); int nvkm_ltc_zbc_color_get(struct nvkm_ltc *, int index, const u32[4]); int nvkm_ltc_zbc_depth_get(struct nvkm_ltc *, int index, const u32); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c index a44298989d3cc..7109e21f45641 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c @@ -443,6 +443,7 @@ int gf100_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin, u32 memtype, struct nvkm_mem **pmem) { + struct nvkm_device *device = ram->fb->subdev.device; struct nvkm_ltc *ltc = ram->fb->subdev.device->ltc; struct nvkm_mm *mm = &ram->vram; struct nvkm_mm_node **node, *r; @@ -469,7 +470,10 @@ gf100_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin, /* compression only works with lpages */ if (align == (1 << (17 - NVKM_RAM_MM_SHIFT))) { int n = size >> 5; - nvkm_ltc_tags_alloc(ltc, n, &mem->tag); + if (!nvkm_ltc_tags_alloc(ltc, n, &mem->tag)) { + nvkm_ltc_tags_clear(device, mem->tag->offset, + mem->tag->length); + } } if (unlikely(!mem->tag)) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c index 0c7ef250dcafc..a75320943187d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c @@ -41,8 +41,9 @@ nvkm_ltc_tags_free(struct nvkm_ltc *ltc, struct nvkm_mm_node **pnode) } void -nvkm_ltc_tags_clear(struct nvkm_ltc *ltc, u32 first, u32 count) +nvkm_ltc_tags_clear(struct nvkm_device *device, u32 first, u32 count) { + struct nvkm_ltc *ltc = device->ltc; const u32 limit = first + count - 1; BUG_ON((first > limit) || (limit >= ltc->num_tags)); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index 7ac507c927bb0..cda5b0997c8f0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -24,7 +24,6 @@ #include "priv.h" #include -#include #include #include @@ -109,11 +108,9 @@ gf100_vm_map(struct nvkm_vma *vma, struct nvkm_memory *pgt, pte <<= 3; if (mem->tag) { - struct nvkm_ltc *ltc = vma->vm->mmu->subdev.device->ltc; u32 tag = mem->tag->offset + (delta >> 17); phys |= (u64)tag << (32 + 12); next |= (u64)1 << (32 + 12); - nvkm_ltc_tags_clear(ltc, tag, cnt); } nvkm_kmap(pgt); From 6cd7670c1e2a4a70a3d421521aea548e68992621 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 066/151] drm/nouveau/ltc: init comptag mm in fb subdev A single location for the MM allows us to share allocation logic. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c | 7 ++++--- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c | 6 ++++-- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h index 93eeb557a9f26..0e2751bbf153b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h @@ -14,7 +14,6 @@ struct nvkm_ltc { u32 num_tags; u32 tag_base; - struct nvkm_mm tags; struct nvkm_mm_node *tag_ram; int zbc_min; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c index a75320943187d..f4aa81dd8978d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c @@ -28,7 +28,8 @@ int nvkm_ltc_tags_alloc(struct nvkm_ltc *ltc, u32 n, struct nvkm_mm_node **pnode) { - int ret = nvkm_mm_head(<c->tags, 0, 1, n, n, 1, pnode); + struct nvkm_fb *fb = ltc->subdev.device->fb; + int ret = nvkm_mm_head(&fb->tags, 0, 1, n, n, 1, pnode); if (ret) *pnode = NULL; return ret; @@ -37,7 +38,8 @@ nvkm_ltc_tags_alloc(struct nvkm_ltc *ltc, u32 n, struct nvkm_mm_node **pnode) void nvkm_ltc_tags_free(struct nvkm_ltc *ltc, struct nvkm_mm_node **pnode) { - nvkm_mm_free(<c->tags, pnode); + struct nvkm_fb *fb = ltc->subdev.device->fb; + nvkm_mm_free(&fb->tags, pnode); } void @@ -118,7 +120,6 @@ nvkm_ltc_dtor(struct nvkm_subdev *subdev) { struct nvkm_ltc *ltc = nvkm_ltc(subdev); struct nvkm_ram *ram = ltc->subdev.device->fb->ram; - nvkm_mm_fini(<c->tags); if (ram) nvkm_mm_free(&ram->vram, <c->tag_ram); return ltc; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c index 6b0cb7f3fa776..60a5e9e2b2643 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c @@ -152,7 +152,8 @@ gf100_ltc_flush(struct nvkm_ltc *ltc) int gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc) { - struct nvkm_ram *ram = ltc->subdev.device->fb->ram; + struct nvkm_fb *fb = ltc->subdev.device->fb; + struct nvkm_ram *ram = fb->ram; u32 tag_size, tag_margin, tag_align; int ret; @@ -197,7 +198,8 @@ gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc) } mm_init: - return nvkm_mm_init(<c->tags, 0, 0, ltc->num_tags, 1); + nvkm_mm_fini(&fb->tags); + return nvkm_mm_init(&fb->tags, 0, 0, ltc->num_tags, 1); } int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c index 74ed4dd203bc6..e34d421080196 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c @@ -45,7 +45,7 @@ gp100_ltc_oneinit(struct nvkm_ltc *ltc) ltc->ltc_nr = nvkm_rd32(device, 0x12006c); ltc->lts_nr = nvkm_rd32(device, 0x17e280) >> 28; /*XXX: tagram allocation - TBD */ - return nvkm_mm_init(<c->tags, 0, 0, 0, 1); + return 0; } static void From 7f53d6dc9a728d54cc4e1d70259376e09675f924 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 067/151] drm/nouveau/core/memory: comptag allocation nvkm_memory is going to be used by the upcoming mmu rework for the basic representation of a memory allocation, as such, this commit adds support for comptag allocation to nvkm_memory. This is very simple for now, in that it requires comptags for the entire memory allocation even if only certain ranges are compressed. Support for tracking ranges will be added at a later date. Signed-off-by: Ben Skeggs --- .../drm/nouveau/include/nvkm/core/memory.h | 12 ++++ drivers/gpu/drm/nouveau/nvkm/core/memory.c | 71 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h index 14ce7df9864e6..ce04e0f74f0c7 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h @@ -5,6 +5,11 @@ struct nvkm_device; struct nvkm_vma; struct nvkm_vm; +struct nvkm_tags { + struct nvkm_mm_node *mn; + refcount_t refcount; +}; + enum nvkm_memory_target { NVKM_MEM_TARGET_INST, /* instance memory */ NVKM_MEM_TARGET_VRAM, /* video memory */ @@ -15,6 +20,7 @@ enum nvkm_memory_target { struct nvkm_memory { const struct nvkm_memory_func *func; const struct nvkm_memory_ptrs *ptrs; + struct nvkm_tags *tags; }; struct nvkm_memory_func { @@ -37,6 +43,12 @@ void nvkm_memory_ctor(const struct nvkm_memory_func *, struct nvkm_memory *); int nvkm_memory_new(struct nvkm_device *, enum nvkm_memory_target, u64 size, u32 align, bool zero, struct nvkm_memory **); void nvkm_memory_del(struct nvkm_memory **); +int nvkm_memory_tags_get(struct nvkm_memory *, struct nvkm_device *, u32 tags, + void (*clear)(struct nvkm_device *, u32, u32), + struct nvkm_tags **); +void nvkm_memory_tags_put(struct nvkm_memory *, struct nvkm_device *, + struct nvkm_tags **); + #define nvkm_memory_target(p) (p)->func->target(p) #define nvkm_memory_addr(p) (p)->func->addr(p) #define nvkm_memory_size(p) (p)->func->size(p) diff --git a/drivers/gpu/drm/nouveau/nvkm/core/memory.c b/drivers/gpu/drm/nouveau/nvkm/core/memory.c index 8903c04c977e8..13cfcdde99fdc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/memory.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/memory.c @@ -22,8 +22,79 @@ * Authors: Ben Skeggs */ #include +#include +#include #include +void +nvkm_memory_tags_put(struct nvkm_memory *memory, struct nvkm_device *device, + struct nvkm_tags **ptags) +{ + struct nvkm_fb *fb = device->fb; + struct nvkm_tags *tags = *ptags; + if (tags) { + mutex_lock(&fb->subdev.mutex); + if (refcount_dec_and_test(&tags->refcount)) { + nvkm_mm_free(&fb->tags, &tags->mn); + kfree(memory->tags); + memory->tags = NULL; + } + mutex_unlock(&fb->subdev.mutex); + *ptags = NULL; + } +} + +int +nvkm_memory_tags_get(struct nvkm_memory *memory, struct nvkm_device *device, + u32 nr, void (*clr)(struct nvkm_device *, u32, u32), + struct nvkm_tags **ptags) +{ + struct nvkm_fb *fb = device->fb; + struct nvkm_tags *tags; + + mutex_lock(&fb->subdev.mutex); + if ((tags = memory->tags)) { + /* If comptags exist for the memory, but a different amount + * than requested, the buffer is being mapped with settings + * that are incompatible with existing mappings. + */ + if (tags->mn && tags->mn->length != nr) { + mutex_unlock(&fb->subdev.mutex); + return -EINVAL; + } + + refcount_inc(&tags->refcount); + *ptags = tags; + return 0; + } + + if (!(tags = kmalloc(sizeof(*tags), GFP_KERNEL))) { + mutex_unlock(&fb->subdev.mutex); + return -ENOMEM; + } + + if (!nvkm_mm_head(&fb->tags, 0, 1, nr, nr, 1, &tags->mn)) { + if (clr) + clr(device, tags->mn->offset, tags->mn->length); + } else { + /* Failure to allocate HW comptags is not an error, the + * caller should fall back to an uncompressed map. + * + * As memory can be mapped in multiple places, we still + * need to track the allocation failure and ensure that + * any additional mappings remain uncompressed. + * + * This is handled by returning an empty nvkm_tags. + */ + tags->mn = NULL; + } + + refcount_set(&tags->refcount, 1); + mutex_unlock(&fb->subdev.mutex); + *ptags = tags; + return 0; +} + void nvkm_memory_ctor(const struct nvkm_memory_func *func, struct nvkm_memory *memory) From 19a82e492c3d71efe8763d50496a1701dfcf3f15 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 068/151] drm/nouveau/core/memory: change map interface to support upcoming mmu changes Map flags (access, kind, etc) are currently defined in either the VMA, or the memory object, which turns out to not be ideal for things like suballocated buffers, etc. These will become per-map flags instead, so we need to support passing these arguments in nvkm_memory_map(). Signed-off-by: Ben Skeggs --- .../drm/nouveau/include/nvkm/core/gpuobj.h | 7 +-- .../drm/nouveau/include/nvkm/core/memory.h | 10 +++-- drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c | 43 ++++++++++--------- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 3 +- .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 15 +++++-- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 15 +++++-- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 11 ++++- .../drm/nouveau/nvkm/subdev/instmem/gk20a.c | 9 ++-- .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 8 ++-- .../drm/nouveau/nvkm/subdev/secboot/gm200.c | 9 +++- 11 files changed, 83 insertions(+), 50 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h b/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h index acd07de3e4ee6..51691667b8130 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h @@ -2,8 +2,6 @@ #define __NVKM_GPUOBJ_H__ #include #include -struct nvkm_vma; -struct nvkm_vm; #define NVOBJ_FLAG_ZERO_ALLOC 0x00000001 #define NVOBJ_FLAG_HEAP 0x00000004 @@ -29,15 +27,14 @@ struct nvkm_gpuobj_func { void (*release)(struct nvkm_gpuobj *); u32 (*rd32)(struct nvkm_gpuobj *, u32 offset); void (*wr32)(struct nvkm_gpuobj *, u32 offset, u32 data); + int (*map)(struct nvkm_gpuobj *, u64 offset, struct nvkm_vmm *, + struct nvkm_vma *, void *argv, u32 argc); }; int nvkm_gpuobj_new(struct nvkm_device *, u32 size, int align, bool zero, struct nvkm_gpuobj *parent, struct nvkm_gpuobj **); void nvkm_gpuobj_del(struct nvkm_gpuobj **); int nvkm_gpuobj_wrap(struct nvkm_memory *, struct nvkm_gpuobj **); -int nvkm_gpuobj_map(struct nvkm_gpuobj *, struct nvkm_vm *, u32 access, - struct nvkm_vma *); -void nvkm_gpuobj_unmap(struct nvkm_vma *); void nvkm_gpuobj_memcpy_to(struct nvkm_gpuobj *dst, u32 dstoffset, void *src, u32 length); void nvkm_gpuobj_memcpy_from(void *dst, struct nvkm_gpuobj *src, u32 srcoffset, diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h index ce04e0f74f0c7..41bc89af11c03 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h @@ -3,7 +3,7 @@ #include struct nvkm_device; struct nvkm_vma; -struct nvkm_vm; +struct nvkm_vmm; struct nvkm_tags { struct nvkm_mm_node *mn; @@ -28,10 +28,11 @@ struct nvkm_memory_func { enum nvkm_memory_target (*target)(struct nvkm_memory *); u64 (*addr)(struct nvkm_memory *); u64 (*size)(struct nvkm_memory *); - void (*boot)(struct nvkm_memory *, struct nvkm_vm *); + void (*boot)(struct nvkm_memory *, struct nvkm_vmm *); void __iomem *(*acquire)(struct nvkm_memory *); void (*release)(struct nvkm_memory *); - void (*map)(struct nvkm_memory *, struct nvkm_vma *, u64 offset); + int (*map)(struct nvkm_memory *, u64 offset, struct nvkm_vmm *, + struct nvkm_vma *, void *argv, u32 argc); }; struct nvkm_memory_ptrs { @@ -53,7 +54,8 @@ void nvkm_memory_tags_put(struct nvkm_memory *, struct nvkm_device *, #define nvkm_memory_addr(p) (p)->func->addr(p) #define nvkm_memory_size(p) (p)->func->size(p) #define nvkm_memory_boot(p,v) (p)->func->boot((p),(v)) -#define nvkm_memory_map(p,v,o) (p)->func->map((p),(v),(o)) +#define nvkm_memory_map(p,o,vm,va,av,ac) \ + (p)->func->map((p),(o),(vm),(va),(av),(ac)) /* accessor macros - kmap()/done() must bracket use of the other accessor * macros to guarantee correct behaviour across all chipsets diff --git a/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c b/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c index 04e5b2136f0cf..1264d5fc632b6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c @@ -42,6 +42,14 @@ nvkm_gpuobj_wr32_fast(struct nvkm_gpuobj *gpuobj, u32 offset, u32 data) } /* accessor functions for gpuobjs allocated directly from instmem */ +static int +nvkm_gpuobj_heap_map(struct nvkm_gpuobj *gpuobj, u64 offset, + struct nvkm_vmm *vmm, struct nvkm_vma *vma, + void *argv, u32 argc) +{ + return nvkm_memory_map(gpuobj->memory, offset, vmm, vma, argv, argc); +} + static u32 nvkm_gpuobj_heap_rd32(struct nvkm_gpuobj *gpuobj, u32 offset) { @@ -67,6 +75,7 @@ nvkm_gpuobj_heap_fast = { .release = nvkm_gpuobj_heap_release, .rd32 = nvkm_gpuobj_rd32_fast, .wr32 = nvkm_gpuobj_wr32_fast, + .map = nvkm_gpuobj_heap_map, }; static const struct nvkm_gpuobj_func @@ -74,6 +83,7 @@ nvkm_gpuobj_heap_slow = { .release = nvkm_gpuobj_heap_release, .rd32 = nvkm_gpuobj_heap_rd32, .wr32 = nvkm_gpuobj_heap_wr32, + .map = nvkm_gpuobj_heap_map, }; static void * @@ -90,9 +100,19 @@ nvkm_gpuobj_heap_acquire(struct nvkm_gpuobj *gpuobj) static const struct nvkm_gpuobj_func nvkm_gpuobj_heap = { .acquire = nvkm_gpuobj_heap_acquire, + .map = nvkm_gpuobj_heap_map, }; /* accessor functions for gpuobjs sub-allocated from a parent gpuobj */ +static int +nvkm_gpuobj_map(struct nvkm_gpuobj *gpuobj, u64 offset, + struct nvkm_vmm *vmm, struct nvkm_vma *vma, + void *argv, u32 argc) +{ + return nvkm_memory_map(gpuobj->parent, gpuobj->node->offset + offset, + vmm, vma, argv, argc); +} + static u32 nvkm_gpuobj_rd32(struct nvkm_gpuobj *gpuobj, u32 offset) { @@ -118,6 +138,7 @@ nvkm_gpuobj_fast = { .release = nvkm_gpuobj_release, .rd32 = nvkm_gpuobj_rd32_fast, .wr32 = nvkm_gpuobj_wr32_fast, + .map = nvkm_gpuobj_map, }; static const struct nvkm_gpuobj_func @@ -125,6 +146,7 @@ nvkm_gpuobj_slow = { .release = nvkm_gpuobj_release, .rd32 = nvkm_gpuobj_rd32, .wr32 = nvkm_gpuobj_wr32, + .map = nvkm_gpuobj_map, }; static void * @@ -143,6 +165,7 @@ nvkm_gpuobj_acquire(struct nvkm_gpuobj *gpuobj) static const struct nvkm_gpuobj_func nvkm_gpuobj_func = { .acquire = nvkm_gpuobj_acquire, + .map = nvkm_gpuobj_map, }; static int @@ -218,26 +241,6 @@ nvkm_gpuobj_new(struct nvkm_device *device, u32 size, int align, bool zero, return ret; } -int -nvkm_gpuobj_map(struct nvkm_gpuobj *gpuobj, struct nvkm_vm *vm, - u32 access, struct nvkm_vma *vma) -{ - struct nvkm_memory *memory = gpuobj->memory; - int ret = nvkm_vm_get(vm, gpuobj->size, 12, access, vma); - if (ret == 0) - nvkm_memory_map(memory, vma, 0); - return ret; -} - -void -nvkm_gpuobj_unmap(struct nvkm_vma *vma) -{ - if (vma->node) { - nvkm_vm_unmap(vma); - nvkm_vm_put(vma); - } -} - /* the below is basically only here to support sharing the paged dma object * for PCI(E)GART on <=nv4x chipsets, and should *not* be expected to work * anywhere else. diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index b1ab34f1a0e94..14b1a616d26a4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -591,8 +591,7 @@ gf100_fifo_oneinit(struct nvkm_fifo *base) if (ret) return ret; - nvkm_memory_map(fifo->user.mem, &fifo->user.bar, 0); - return 0; + return nvkm_memory_map(fifo->user.mem, 0, bar, &fifo->user.bar, NULL, 0); } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index cdadff59bb4c5..9629416f49475 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -841,8 +841,7 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) if (ret) return ret; - nvkm_memory_map(fifo->user.mem, &fifo->user.bar, 0); - return 0; + return nvkm_memory_map(fifo->user.mem, 0, bar, &fifo->user.bar, NULL, 0); } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index f9e0377d3d24b..6343e0c84d96a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -126,7 +126,11 @@ gf100_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base, struct nvkm_engine *engine) { struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - nvkm_gpuobj_unmap(&chan->engn[engine->subdev.index].vma); + struct nvkm_vma *vma = &chan->engn[engine->subdev.index].vma; + if (vma->vm) { + nvkm_vm_unmap(vma); + nvkm_vm_put(vma); + } nvkm_gpuobj_del(&chan->engn[engine->subdev.index].inst); } @@ -146,8 +150,13 @@ gf100_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, if (ret) return ret; - return nvkm_gpuobj_map(chan->engn[engn].inst, chan->vm, - NV_MEM_ACCESS_RW, &chan->engn[engn].vma); + ret = nvkm_vm_get(chan->vm, chan->engn[engn].inst->size, 12, + NV_MEM_ACCESS_RW, &chan->engn[engn].vma); + if (ret) + return ret; + + return nvkm_memory_map(chan->engn[engn].inst, 0, chan->vm, + &chan->engn[engn].vma, NULL, 0); } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 8abf6f8ef445d..6e93b8652c667 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -138,7 +138,11 @@ gk104_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base, struct nvkm_engine *engine) { struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - nvkm_gpuobj_unmap(&chan->engn[engine->subdev.index].vma); + struct nvkm_vma *vma = &chan->engn[engine->subdev.index].vma; + if (vma->vm) { + nvkm_vm_unmap(vma); + nvkm_vm_put(vma); + } nvkm_gpuobj_del(&chan->engn[engine->subdev.index].inst); } @@ -158,8 +162,13 @@ gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, if (ret) return ret; - return nvkm_gpuobj_map(chan->engn[engn].inst, chan->vm, - NV_MEM_ACCESS_RW, &chan->engn[engn].vma); + ret = nvkm_vm_get(chan->vm, chan->engn[engn].inst->size, 12, + NV_MEM_ACCESS_RW, &chan->engn[engn].vma); + if (ret) + return ret; + + return nvkm_memory_map(chan->engn[engn].inst, 0, chan->vm, + &chan->engn[engn].vma, NULL, 0); } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 49c7d08b0ceb4..0c4ca0fa48cc8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -403,7 +403,10 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, if (ret) return ret; - nvkm_memory_map(chan->mmio, &chan->mmio_vma, 0); + ret = nvkm_memory_map(chan->mmio, 0, fifoch->vm, + &chan->mmio_vma, NULL, 0); + if (ret) + return ret; /* allocate buffers referenced by mmio list */ for (i = 0; data->size && i < ARRAY_SIZE(gr->mmio_data); i++) { @@ -419,7 +422,11 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, if (ret) return ret; - nvkm_memory_map(chan->data[i].mem, &chan->data[i].vma, 0); + ret = nvkm_memory_map(chan->data[i].mem, 0, fifoch->vm, + &chan->data[i].vma, NULL, 0); + if (ret) + return ret; + data++; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c index f2879c48d1e68..73a9511c31011 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c @@ -272,12 +272,13 @@ gk20a_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data) node->vaddr[offset / 4] = data; } -static void -gk20a_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset) +static int +gk20a_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, + struct nvkm_vma *vma, void *argv, u32 argc) { struct gk20a_instobj *node = gk20a_instobj(memory); - - nvkm_vm_map_at(vma, offset, &node->mem); + nvkm_vm_map_at(vma, 0, &node->mem); + return 0; } static void * diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index bb524e3aa5d99..657257daa74da 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -159,7 +159,7 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) } if (ret == 0) - nvkm_memory_map(memory, &bar, 0); + ret = nvkm_memory_map(memory, 0, vmm, &bar, NULL, 0); mutex_lock(&subdev->mutex); if (ret || iobj->bar.node) { /* We either failed, or another thread beat us. */ @@ -179,11 +179,13 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) } } -static void -nv50_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset) +static int +nv50_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, + struct nvkm_vma *vma, void *argv, u32 argc) { struct nv50_instobj *iobj = nv50_instobj(memory); nvkm_vm_map_at(vma, offset, iobj->mem); + return 0; } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c index 73ca1203281d6..d35c041fdcece 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c @@ -48,12 +48,16 @@ gm200_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob, return ret; /* Map the HS firmware so the HS bootloader can see it */ - ret = nvkm_gpuobj_map(blob, gsb->vm, NV_MEM_ACCESS_RW, &vma); + ret = nvkm_vm_get(gsb->vm, blob->size, 12, NV_MEM_ACCESS_RW, &vma); if (ret) { nvkm_falcon_put(falcon, subdev); return ret; } + ret = nvkm_memory_map(blob, 0, gsb->vm, &vma, NULL, 0); + if (ret) + goto end; + /* Reset and set the falcon up */ ret = nvkm_falcon_reset(falcon); if (ret) @@ -91,7 +95,8 @@ gm200_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob, nvkm_mc_intr_mask(sb->subdev.device, falcon->owner->index, true); /* We don't need the ACR firmware anymore */ - nvkm_gpuobj_unmap(&vma); + nvkm_vm_unmap(&vma); + nvkm_vm_put(&vma); nvkm_falcon_put(falcon, subdev); return ret; From 2c9c4910f8b126d18fa2a08087f3ae1553e706bc Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 069/151] drm/nouveau/core/memory: add mechanism to retrieve allocation granularity Needed by VMM code to determine whether an allocation is compatible with a given page size (ie. you can't map 4KiB system memory pages into 64KiB GPU pages). Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/core/memory.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h index 41bc89af11c03..a0bfc37b96db0 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h @@ -26,6 +26,7 @@ struct nvkm_memory { struct nvkm_memory_func { void *(*dtor)(struct nvkm_memory *); enum nvkm_memory_target (*target)(struct nvkm_memory *); + u8 (*page)(struct nvkm_memory *); u64 (*addr)(struct nvkm_memory *); u64 (*size)(struct nvkm_memory *); void (*boot)(struct nvkm_memory *, struct nvkm_vmm *); @@ -51,6 +52,7 @@ void nvkm_memory_tags_put(struct nvkm_memory *, struct nvkm_device *, struct nvkm_tags **); #define nvkm_memory_target(p) (p)->func->target(p) +#define nvkm_memory_page(p) (p)->func->page(p) #define nvkm_memory_addr(p) (p)->func->addr(p) #define nvkm_memory_size(p) (p)->func->size(p) #define nvkm_memory_boot(p,v) (p)->func->boot((p),(v)) From 997a89003c2d950466bc289147ffb823c0c51fb0 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 070/151] drm/nouveau/core/memory: add reference counting We need to be able to prevent memory from being freed while it's still mapped in a GPU's address-space. Will be used by upcoming MMU changes. Signed-off-by: Ben Skeggs --- .../drm/nouveau/include/nvkm/core/memory.h | 4 ++- drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c | 2 +- drivers/gpu/drm/nouveau/nvkm/core/memory.c | 28 +++++++++++++++---- drivers/gpu/drm/nouveau/nvkm/engine/falcon.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 6 ++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 6 ++-- .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 4 +-- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 4 +-- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c | 4 +-- drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 6 ++-- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c | 4 +-- .../drm/nouveau/nvkm/subdev/instmem/base.c | 2 +- .../drm/nouveau/nvkm/subdev/instmem/nv04.c | 6 ++-- .../drm/nouveau/nvkm/subdev/instmem/nv40.c | 6 ++-- .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 2 +- 19 files changed, 57 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h index a0bfc37b96db0..9c9c400bea984 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h @@ -20,6 +20,7 @@ enum nvkm_memory_target { struct nvkm_memory { const struct nvkm_memory_func *func; const struct nvkm_memory_ptrs *ptrs; + struct kref kref; struct nvkm_tags *tags; }; @@ -44,7 +45,8 @@ struct nvkm_memory_ptrs { void nvkm_memory_ctor(const struct nvkm_memory_func *, struct nvkm_memory *); int nvkm_memory_new(struct nvkm_device *, enum nvkm_memory_target, u64 size, u32 align, bool zero, struct nvkm_memory **); -void nvkm_memory_del(struct nvkm_memory **); +struct nvkm_memory *nvkm_memory_ref(struct nvkm_memory *); +void nvkm_memory_unref(struct nvkm_memory **); int nvkm_memory_tags_get(struct nvkm_memory *, struct nvkm_device *, u32 tags, void (*clear)(struct nvkm_device *, u32, u32), struct nvkm_tags **); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c b/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c index 1264d5fc632b6..d6de2b3ed2c30 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c @@ -219,7 +219,7 @@ nvkm_gpuobj_del(struct nvkm_gpuobj **pgpuobj) if (gpuobj->parent) nvkm_mm_free(&gpuobj->parent->heap, &gpuobj->node); nvkm_mm_fini(&gpuobj->heap); - nvkm_memory_del(&gpuobj->memory); + nvkm_memory_unref(&gpuobj->memory); kfree(*pgpuobj); *pgpuobj = NULL; } diff --git a/drivers/gpu/drm/nouveau/nvkm/core/memory.c b/drivers/gpu/drm/nouveau/nvkm/core/memory.c index 13cfcdde99fdc..29f4b4070b557 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/memory.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/memory.c @@ -100,20 +100,38 @@ nvkm_memory_ctor(const struct nvkm_memory_func *func, struct nvkm_memory *memory) { memory->func = func; + kref_init(&memory->kref); +} + +static void +nvkm_memory_del(struct kref *kref) +{ + struct nvkm_memory *memory = container_of(kref, typeof(*memory), kref); + if (!WARN_ON(!memory->func)) { + if (memory->func->dtor) + memory = memory->func->dtor(memory); + kfree(memory); + } } void -nvkm_memory_del(struct nvkm_memory **pmemory) +nvkm_memory_unref(struct nvkm_memory **pmemory) { struct nvkm_memory *memory = *pmemory; - if (memory && !WARN_ON(!memory->func)) { - if (memory->func->dtor) - *pmemory = memory->func->dtor(memory); - kfree(*pmemory); + if (memory) { + kref_put(&memory->kref, nvkm_memory_del); *pmemory = NULL; } } +struct nvkm_memory * +nvkm_memory_ref(struct nvkm_memory *memory) +{ + if (memory) + kref_get(&memory->kref); + return memory; +} + int nvkm_memory_new(struct nvkm_device *device, enum nvkm_memory_target target, u64 size, u32 align, bool zero, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c index 2e7b4e2105efc..816ccaedfc732 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c @@ -99,7 +99,7 @@ nvkm_falcon_fini(struct nvkm_engine *engine, bool suspend) const u32 base = falcon->addr; if (!suspend) { - nvkm_memory_del(&falcon->core); + nvkm_memory_unref(&falcon->core); if (falcon->external) { vfree(falcon->data.data); vfree(falcon->code.data); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 14b1a616d26a4..24a4c28b32c5c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -642,9 +642,9 @@ gf100_fifo_dtor(struct nvkm_fifo *base) { struct gf100_fifo *fifo = gf100_fifo(base); nvkm_vm_put(&fifo->user.bar); - nvkm_memory_del(&fifo->user.mem); - nvkm_memory_del(&fifo->runlist.mem[0]); - nvkm_memory_del(&fifo->runlist.mem[1]); + nvkm_memory_unref(&fifo->user.mem); + nvkm_memory_unref(&fifo->runlist.mem[0]); + nvkm_memory_unref(&fifo->runlist.mem[1]); return fifo; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 9629416f49475..eddf9f12e9ee0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -880,11 +880,11 @@ gk104_fifo_dtor(struct nvkm_fifo *base) int i; nvkm_vm_put(&fifo->user.bar); - nvkm_memory_del(&fifo->user.mem); + nvkm_memory_unref(&fifo->user.mem); for (i = 0; i < fifo->runlist_nr; i++) { - nvkm_memory_del(&fifo->runlist[i].mem[1]); - nvkm_memory_del(&fifo->runlist[i].mem[0]); + nvkm_memory_unref(&fifo->runlist[i].mem[1]); + nvkm_memory_unref(&fifo->runlist[i].mem[0]); } return fifo; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 66eb12c2b5ba7..fa6e094d80686 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -100,8 +100,8 @@ void * nv50_fifo_dtor(struct nvkm_fifo *base) { struct nv50_fifo *fifo = nv50_fifo(base); - nvkm_memory_del(&fifo->runlist[1]); - nvkm_memory_del(&fifo->runlist[0]); + nvkm_memory_unref(&fifo->runlist[1]); + nvkm_memory_unref(&fifo->runlist[0]); return fifo; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index bc77eea351a5c..ce69ec8b13add 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1380,7 +1380,7 @@ gf100_grctx_generate(struct gf100_gr *gr) } done: - nvkm_memory_del(&chan); + nvkm_memory_unref(&chan); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 0c4ca0fa48cc8..4fc4deb2db4b5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -354,14 +354,14 @@ gf100_gr_chan_dtor(struct nvkm_object *object) nvkm_vm_unmap(&chan->data[i].vma); nvkm_vm_put(&chan->data[i].vma); } - nvkm_memory_del(&chan->data[i].mem); + nvkm_memory_unref(&chan->data[i].mem); } if (chan->mmio_vma.node) { nvkm_vm_unmap(&chan->mmio_vma); nvkm_vm_put(&chan->mmio_vma); } - nvkm_memory_del(&chan->mmio); + nvkm_memory_unref(&chan->mmio); return chan; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c index d1dc92999dc00..d6840dc81a295 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c @@ -59,7 +59,7 @@ void * nv20_gr_chan_dtor(struct nvkm_object *object) { struct nv20_gr_chan *chan = nv20_gr_chan(object); - nvkm_memory_del(&chan->inst); + nvkm_memory_unref(&chan->inst); return chan; } @@ -323,7 +323,7 @@ void * nv20_gr_dtor(struct nvkm_gr *base) { struct nv20_gr *gr = nv20_gr(base); - nvkm_memory_del(&gr->ctxtab); + nvkm_memory_unref(&gr->ctxtab); return gr; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c b/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c index 06bdb67a02058..70549381e0828 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c @@ -86,7 +86,7 @@ nvkm_xtensa_fini(struct nvkm_engine *engine, bool suspend) nvkm_wr32(device, base + 0xd94, 0); /* FIFO_CTRL */ if (!suspend) - nvkm_memory_del(&xtensa->gpu_fw); + nvkm_memory_unref(&xtensa->gpu_fw); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index b155a7f76ca78..9f7b4e7532b70 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -166,14 +166,14 @@ gf100_bar_dtor(struct nvkm_bar *base) nvkm_vm_ref(NULL, &bar->bar[1].vm, bar->bar[1].pgd); nvkm_gpuobj_del(&bar->bar[1].pgd); - nvkm_memory_del(&bar->bar[1].mem); + nvkm_memory_unref(&bar->bar[1].mem); if (bar->bar[0].vm) { - nvkm_memory_del(&bar->bar[0].vm->pgt[0].mem[0]); + nvkm_memory_unref(&bar->bar[0].vm->pgt[0].mem[0]); nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].pgd); } nvkm_gpuobj_del(&bar->bar[0].pgd); - nvkm_memory_del(&bar->bar[0].mem); + nvkm_memory_unref(&bar->bar[0].mem); return bar; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index cf1d0acd2afc8..1aa6b5390d796 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -201,7 +201,7 @@ nv50_bar_dtor(struct nvkm_bar *base) nvkm_vm_ref(NULL, &bar->bar1_vm, bar->pgd); nvkm_gpuobj_del(&bar->bar2); if (bar->bar2_vm) { - nvkm_memory_del(&bar->bar2_vm->pgt[0].mem[0]); + nvkm_memory_unref(&bar->bar2_vm->pgt[0].mem[0]); nvkm_vm_ref(NULL, &bar->bar2_vm, bar->pgd); } nvkm_gpuobj_del(&bar->pgd); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c index 94c53454501d3..78248e21a5f0e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c @@ -163,8 +163,8 @@ nvkm_fb_dtor(struct nvkm_subdev *subdev) struct nvkm_fb *fb = nvkm_fb(subdev); int i; - nvkm_memory_del(&fb->mmu_wr); - nvkm_memory_del(&fb->mmu_rd); + nvkm_memory_unref(&fb->mmu_wr); + nvkm_memory_unref(&fb->mmu_rd); for (i = 0; i < fb->tile.regions; i++) fb->func->tile.fini(fb, i, &fb->tile.region[i]); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index b03940591a3a0..364ea4492acc8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -120,7 +120,7 @@ nvkm_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero, done: if (ret) - nvkm_memory_del(&memory); + nvkm_memory_unref(&memory); *pmemory = memory; return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c index 2e1141319e939..6bf0dad469195 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c @@ -199,10 +199,10 @@ static void * nv04_instmem_dtor(struct nvkm_instmem *base) { struct nv04_instmem *imem = nv04_instmem(base); - nvkm_memory_del(&imem->base.ramfc); - nvkm_memory_del(&imem->base.ramro); + nvkm_memory_unref(&imem->base.ramfc); + nvkm_memory_unref(&imem->base.ramro); nvkm_ramht_del(&imem->base.ramht); - nvkm_memory_del(&imem->base.vbios); + nvkm_memory_unref(&imem->base.vbios); nvkm_mm_fini(&imem->heap); return imem; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c index 7f52a525d2e1a..086c118488ef5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c @@ -215,10 +215,10 @@ static void * nv40_instmem_dtor(struct nvkm_instmem *base) { struct nv40_instmem *imem = nv40_instmem(base); - nvkm_memory_del(&imem->base.ramfc); - nvkm_memory_del(&imem->base.ramro); + nvkm_memory_unref(&imem->base.ramfc); + nvkm_memory_unref(&imem->base.ramro); nvkm_ramht_del(&imem->base.ramht); - nvkm_memory_del(&imem->base.vbios); + nvkm_memory_unref(&imem->base.vbios); nvkm_mm_fini(&imem->heap); if (imem->iomem) iounmap(imem->iomem); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 44c2403e88e6c..ad11db458fcce 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -243,7 +243,7 @@ nvkm_vm_unmap_pgt(struct nvkm_vm *vm, int big, u32 fpde, u32 lpde) mmu->func->flush(vm); - nvkm_memory_del(&pgt); + nvkm_memory_unref(&pgt); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index cd16442f1a918..43e3b4a77583f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -106,7 +106,7 @@ nv04_mmu_dtor(struct nvkm_mmu *base) struct nv04_mmu *mmu = nv04_mmu(base); struct nvkm_device *device = mmu->base.subdev.device; if (mmu->base.vmm) { - nvkm_memory_del(&mmu->base.vmm->pgt[0].mem[0]); + nvkm_memory_unref(&mmu->base.vmm->pgt[0].mem[0]); nvkm_vm_ref(NULL, &mmu->base.vmm, NULL); } if (mmu->nullp) { From c09597f083960cea492e1d658b9039b06d8a6839 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 071/151] drm/nouveau/core/memory: add some useful accessor macros Adds support for 64-bit writes, and optimised filling of buffers with fixed 32/64-bit values. These will all be used by the upcoming MMU changes. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/os.h | 14 ---------- .../drm/nouveau/include/nvkm/core/memory.h | 28 ++++++++++++++++++- .../gpu/drm/nouveau/include/nvkm/core/os.h | 20 +++++++++++++ 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/os.h b/drivers/gpu/drm/nouveau/include/nvif/os.h index 9fcab67c85577..5efdf80d5abcb 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/os.h +++ b/drivers/gpu/drm/nouveau/include/nvif/os.h @@ -33,18 +33,4 @@ #include #include - -#ifndef ioread32_native -#ifdef __BIG_ENDIAN -#define ioread16_native ioread16be -#define iowrite16_native iowrite16be -#define ioread32_native ioread32be -#define iowrite32_native iowrite32be -#else /* def __BIG_ENDIAN */ -#define ioread16_native ioread16 -#define iowrite16_native iowrite16 -#define ioread32_native ioread32 -#define iowrite32_native iowrite32 -#endif /* def __BIG_ENDIAN else */ -#endif /* !ioread32_native */ #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h index 9c9c400bea984..13ebf4da2b963 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h @@ -65,6 +65,8 @@ void nvkm_memory_tags_put(struct nvkm_memory *, struct nvkm_device *, * macros to guarantee correct behaviour across all chipsets */ #define nvkm_kmap(o) (o)->func->acquire(o) +#define nvkm_done(o) (o)->func->release(o) + #define nvkm_ro32(o,a) (o)->ptrs->rd32((o), (a)) #define nvkm_wo32(o,a,d) (o)->ptrs->wr32((o), (a), (d)) #define nvkm_mo32(o,a,m,d) ({ \ @@ -72,5 +74,29 @@ void nvkm_memory_tags_put(struct nvkm_memory *, struct nvkm_device *, nvkm_wo32((o), _addr, (_data & ~(m)) | (d)); \ _data; \ }) -#define nvkm_done(o) (o)->func->release(o) + +#define nvkm_wo64(o,a,d) do { \ + u64 __a = (a), __d = (d); \ + nvkm_wo32((o), __a + 0, lower_32_bits(__d)); \ + nvkm_wo32((o), __a + 4, upper_32_bits(__d)); \ +} while(0) + +#define nvkm_fill(t,s,o,a,d,c) do { \ + u64 _a = (a), _c = (c), _d = (d), _o = _a >> s, _s = _c << s; \ + u##t __iomem *_m = nvkm_kmap(o); \ + if (likely(_m)) { \ + if (_d) { \ + while (_c--) \ + iowrite##t##_native(_d, &_m[_o++]); \ + } else { \ + memset_io(&_m[_o], _d, _s); \ + } \ + } else { \ + for (; _c; _c--, _a += BIT(s)) \ + nvkm_wo##t((o), _a, _d); \ + } \ + nvkm_done(o); \ +} while(0) +#define nvkm_fo32(o,a,d,c) nvkm_fill(32, 2, (o), (a), (d), (c)) +#define nvkm_fo64(o,a,d,c) nvkm_fill(64, 3, (o), (a), (d), (c)) #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h index cfd9ed361c441..70c5c47300b40 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h @@ -2,4 +2,24 @@ #define __NVKM_OS_H__ #include #define nvkm_vmm nvkm_vm + +#ifdef __BIG_ENDIAN +#define ioread16_native ioread16be +#define iowrite16_native iowrite16be +#define ioread32_native ioread32be +#define iowrite32_native iowrite32be +#else +#define ioread16_native ioread16 +#define iowrite16_native iowrite16 +#define ioread32_native ioread32 +#define iowrite32_native iowrite32 +#endif + +#define iowrite64_native(v,p) do { \ + u32 __iomem *_p = (u32 __iomem *)(p); \ + u64 _v = (v); \ + iowrite32_native(lower_32_bits(_v), &_p[0]); \ + iowrite32_native(upper_32_bits(_v), &_p[1]); \ +} while(0) + #endif From e9a8b218044c693a12a852c558f745267a6b6eb8 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 072/151] drm/nouveau/fb/ram: add interface to allocate vram as an nvkm_memory object Upcoming MMU changes use nvkm_memory as its basic representation of memory, so we need to be able to allocate VRAM like this. The code is basically identical to the current chipset-specific allocators, minus support for compression tags (which will be handled elsewhere anyway). Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/fb.h | 4 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c | 122 ++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index b838d93858597..942d671740050 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -145,6 +145,10 @@ struct nvkm_ram { struct nvkm_ram_data target; }; +int +nvkm_ram_get(struct nvkm_device *, u8 heap, u8 type, u8 page, u64 size, + bool contig, bool back, struct nvkm_memory **); + struct nvkm_ram_func { u64 upper; u32 (*probe_fbp)(const struct nvkm_ram_func *, struct nvkm_device *, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c index 69f65daaf1eaa..8d17644c18234 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c @@ -21,8 +21,130 @@ * * Authors: Ben Skeggs */ +#define nvkm_vram(p) container_of((p), struct nvkm_vram, memory) #include "ram.h" +#include +#include + +struct nvkm_vram { + struct nvkm_memory memory; + struct nvkm_ram *ram; + u8 page; + struct nvkm_mm_node *mn; +}; + +static int +nvkm_vram_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, + struct nvkm_vma *vma, void *argv, u32 argc) +{ + struct nvkm_vram *vram = nvkm_vram(memory); + struct nvkm_mem mem = { + .mem = vram->mn, + }; + nvkm_vm_map_at(vma, offset, &mem); + return 0; +} + +static u64 +nvkm_vram_size(struct nvkm_memory *memory) +{ + return (u64)nvkm_mm_size(nvkm_vram(memory)->mn) << NVKM_RAM_MM_SHIFT; +} + +static u64 +nvkm_vram_addr(struct nvkm_memory *memory) +{ + struct nvkm_vram *vram = nvkm_vram(memory); + if (!nvkm_mm_contiguous(vram->mn)) + return ~0ULL; + return (u64)nvkm_mm_addr(vram->mn) << NVKM_RAM_MM_SHIFT; +} + +static u8 +nvkm_vram_page(struct nvkm_memory *memory) +{ + return nvkm_vram(memory)->page; +} + +static enum nvkm_memory_target +nvkm_vram_target(struct nvkm_memory *memory) +{ + return NVKM_MEM_TARGET_VRAM; +} + +static void * +nvkm_vram_dtor(struct nvkm_memory *memory) +{ + struct nvkm_vram *vram = nvkm_vram(memory); + struct nvkm_mm_node *next = vram->mn; + struct nvkm_mm_node *node; + mutex_lock(&vram->ram->fb->subdev.mutex); + while ((node = next)) { + next = node->next; + nvkm_mm_free(&vram->ram->vram, &node); + } + mutex_unlock(&vram->ram->fb->subdev.mutex); + return vram; +} + +static const struct nvkm_memory_func +nvkm_vram = { + .dtor = nvkm_vram_dtor, + .target = nvkm_vram_target, + .page = nvkm_vram_page, + .addr = nvkm_vram_addr, + .size = nvkm_vram_size, + .map = nvkm_vram_map, +}; + +int +nvkm_ram_get(struct nvkm_device *device, u8 heap, u8 type, u8 rpage, u64 size, + bool contig, bool back, struct nvkm_memory **pmemory) +{ + struct nvkm_ram *ram; + struct nvkm_mm *mm; + struct nvkm_mm_node **node, *r; + struct nvkm_vram *vram; + u8 page = max(rpage, (u8)NVKM_RAM_MM_SHIFT); + u32 align = (1 << page) >> NVKM_RAM_MM_SHIFT; + u32 max = ALIGN(size, 1 << page) >> NVKM_RAM_MM_SHIFT; + u32 min = contig ? max : align; + int ret; + + if (!device->fb || !(ram = device->fb->ram)) + return -ENODEV; + ram = device->fb->ram; + mm = &ram->vram; + + if (!(vram = kzalloc(sizeof(*vram), GFP_KERNEL))) + return -ENOMEM; + nvkm_memory_ctor(&nvkm_vram, &vram->memory); + vram->ram = ram; + vram->page = page; + *pmemory = &vram->memory; + + mutex_lock(&ram->fb->subdev.mutex); + node = &vram->mn; + do { + if (back) + ret = nvkm_mm_tail(mm, heap, type, max, min, align, &r); + else + ret = nvkm_mm_head(mm, heap, type, max, min, align, &r); + if (ret) { + mutex_unlock(&ram->fb->subdev.mutex); + nvkm_memory_unref(pmemory); + return ret; + } + + *node = r; + node = &r->next; + max -= r->length; + } while (max); + mutex_unlock(&ram->fb->subdev.mutex); + return 0; +} + int nvkm_ram_init(struct nvkm_ram *ram) { From 2bfa0b01146586e5098327462a1de090e09b541d Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 073/151] drm/nouveau/fb/ram/gt215: allocate training buffer with nvkm_ram_get() Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c index f10664372161d..9bace6c4ad0bb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c @@ -26,6 +26,7 @@ #include "ram.h" #include "ramfuc.h" +#include #include #include #include @@ -86,7 +87,7 @@ struct gt215_ltrain { u32 r_100720; u32 r_1111e0; u32 r_111400; - struct nvkm_mem *mem; + struct nvkm_memory *memory; }; struct gt215_ram { @@ -279,10 +280,10 @@ gt215_link_train_init(struct gt215_ram *ram) struct gt215_ltrain *train = &ram->ltrain; struct nvkm_device *device = ram->base.fb->subdev.device; struct nvkm_bios *bios = device->bios; - struct nvkm_mem *mem; struct nvbios_M0205E M0205E; u8 ver, hdr, cnt, len; u32 r001700; + u64 addr; int ret, i = 0; train->state = NVA3_TRAIN_UNSUPPORTED; @@ -297,14 +298,14 @@ gt215_link_train_init(struct gt215_ram *ram) train->state = NVA3_TRAIN_ONCE; - ret = ram->base.func->get(&ram->base, 0x8000, 0x10000, 0, 0x800, - &ram->ltrain.mem); + ret = nvkm_ram_get(device, NVKM_RAM_MM_NORMAL, 0x01, 16, 0x8000, + true, true, &ram->ltrain.memory); if (ret) return ret; - mem = ram->ltrain.mem; + addr = nvkm_memory_addr(ram->ltrain.memory); - nvkm_wr32(device, 0x100538, 0x10000000 | (mem->offset >> 16)); + nvkm_wr32(device, 0x100538, 0x10000000 | (addr >> 16)); nvkm_wr32(device, 0x1005a8, 0x0000ffff); nvkm_mask(device, 0x10f800, 0x00000001, 0x00000001); @@ -320,7 +321,7 @@ gt215_link_train_init(struct gt215_ram *ram) /* And upload the pattern */ r001700 = nvkm_rd32(device, 0x1700); - nvkm_wr32(device, 0x1700, mem->offset >> 16); + nvkm_wr32(device, 0x1700, addr >> 16); for (i = 0; i < 16; i++) nvkm_wr32(device, 0x700000 + (i << 2), pattern[i]); for (i = 0; i < 16; i++) @@ -336,8 +337,7 @@ gt215_link_train_init(struct gt215_ram *ram) static void gt215_link_train_fini(struct gt215_ram *ram) { - if (ram->ltrain.mem) - ram->base.func->put(&ram->base, &ram->ltrain.mem); + nvkm_memory_unref(&ram->ltrain.memory); } /* From 7f4f82af6e48c22b3fa5e41aab4dc0fdf0a7717e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 074/151] drm/nouveau/imem/nv50: allocate memory with nvkm_ram_get() Signed-off-by: Ben Skeggs --- .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index 657257daa74da..64e2b6e0e8b19 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -45,7 +45,7 @@ struct nv50_instmem { struct nv50_instobj { struct nvkm_instobj base; struct nv50_instmem *imem; - struct nvkm_mem *mem; + struct nvkm_memory *ram; struct nvkm_vma bar; refcount_t maps; void *map; @@ -58,8 +58,8 @@ nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data) struct nv50_instobj *iobj = nv50_instobj(memory); struct nv50_instmem *imem = iobj->imem; struct nvkm_device *device = imem->base.subdev.device; - u64 base = (iobj->mem->offset + offset) & 0xffffff00000ULL; - u64 addr = (iobj->mem->offset + offset) & 0x000000fffffULL; + u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL; + u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL; unsigned long flags; spin_lock_irqsave(&imem->base.lock, flags); @@ -77,8 +77,8 @@ nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset) struct nv50_instobj *iobj = nv50_instobj(memory); struct nv50_instmem *imem = iobj->imem; struct nvkm_device *device = imem->base.subdev.device; - u64 base = (iobj->mem->offset + offset) & 0xffffff00000ULL; - u64 addr = (iobj->mem->offset + offset) & 0x000000fffffULL; + u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL; + u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL; u32 data; unsigned long flags; @@ -183,9 +183,8 @@ static int nv50_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc) { - struct nv50_instobj *iobj = nv50_instobj(memory); - nvkm_vm_map_at(vma, offset, iobj->mem); - return 0; + memory = nv50_instobj(memory)->ram; + return nvkm_memory_map(memory, offset, vmm, vma, argv, argc); } static void @@ -280,19 +279,19 @@ nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm) static u64 nv50_instobj_size(struct nvkm_memory *memory) { - return (u64)nv50_instobj(memory)->mem->size << NVKM_RAM_MM_SHIFT; + return nvkm_memory_size(nv50_instobj(memory)->ram); } static u64 nv50_instobj_addr(struct nvkm_memory *memory) { - return nv50_instobj(memory)->mem->offset; + return nvkm_memory_addr(nv50_instobj(memory)->ram); } static enum nvkm_memory_target nv50_instobj_target(struct nvkm_memory *memory) { - return NVKM_MEM_TARGET_VRAM; + return nvkm_memory_target(nv50_instobj(memory)->ram); } static void * @@ -300,7 +299,6 @@ nv50_instobj_dtor(struct nvkm_memory *memory) { struct nv50_instobj *iobj = nv50_instobj(memory); struct nvkm_instmem *imem = &iobj->imem->base; - struct nvkm_ram *ram = imem->subdev.device->fb->ram; struct nvkm_vma bar; void *map = map; @@ -316,7 +314,7 @@ nv50_instobj_dtor(struct nvkm_memory *memory) nvkm_vm_put(&bar); } - ram->func->put(ram, &iobj->mem); + nvkm_memory_unref(&iobj->ram); nvkm_instobj_dtor(imem, &iobj->base); return iobj; } @@ -339,8 +337,8 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, { struct nv50_instmem *imem = nv50_instmem(base); struct nv50_instobj *iobj; - struct nvkm_ram *ram = imem->base.subdev.device->fb->ram; - int ret; + struct nvkm_device *device = imem->base.subdev.device; + u8 page = max(order_base_2(align), 12); if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL))) return -ENOMEM; @@ -351,14 +349,7 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, refcount_set(&iobj->maps, 0); INIT_LIST_HEAD(&iobj->lru); - size = max((size + 4095) & ~4095, (u32)4096); - align = max((align + 4095) & ~4095, (u32)4096); - - ret = ram->func->get(ram, size, align, 0, 0x800, &iobj->mem); - if (ret) - return ret; - - return 0; + return nvkm_ram_get(device, 0, 1, page, size, true, true, &iobj->ram); } /****************************************************************************** From bd447053b33d9c2a173e1d3953c95a7cd5182656 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 075/151] drm/nouveau/ltc/gf100-: allocate tagram with nvkm_ram_get() Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c | 5 ++--- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c | 11 ++++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h index 0e2751bbf153b..784661ae7e982 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h @@ -14,7 +14,7 @@ struct nvkm_ltc { u32 num_tags; u32 tag_base; - struct nvkm_mm_node *tag_ram; + struct nvkm_memory *tag_ram; int zbc_min; int zbc_max; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c index f4aa81dd8978d..3bb97b1714356 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c @@ -23,6 +23,7 @@ */ #include "priv.h" +#include #include int @@ -119,9 +120,7 @@ static void * nvkm_ltc_dtor(struct nvkm_subdev *subdev) { struct nvkm_ltc *ltc = nvkm_ltc(subdev); - struct nvkm_ram *ram = ltc->subdev.device->fb->ram; - if (ram) - nvkm_mm_free(&ram->vram, <c->tag_ram); + nvkm_memory_unref(<c->tag_ram); return ltc; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c index 60a5e9e2b2643..59acf568faeb2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c @@ -23,6 +23,7 @@ */ #include "priv.h" +#include #include #include @@ -152,7 +153,8 @@ gf100_ltc_flush(struct nvkm_ltc *ltc) int gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc) { - struct nvkm_fb *fb = ltc->subdev.device->fb; + struct nvkm_device *device = ltc->subdev.device; + struct nvkm_fb *fb = device->fb; struct nvkm_ram *ram = fb->ram; u32 tag_size, tag_margin, tag_align; int ret; @@ -182,14 +184,13 @@ gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc) */ tag_size = (ltc->num_tags / 64) * 0x6000 + tag_margin; tag_size += tag_align; - tag_size = (tag_size + 0xfff) >> 12; /* round up */ - ret = nvkm_mm_tail(&ram->vram, NVKM_RAM_MM_NORMAL, 1, tag_size, - tag_size, 1, <c->tag_ram); + ret = nvkm_ram_get(device, NVKM_RAM_MM_NORMAL, 0x01, 12, tag_size, + true, true, <c->tag_ram); if (ret) { ltc->num_tags = 0; } else { - u64 tag_base = ((u64)ltc->tag_ram->offset << 12) + tag_margin; + u64 tag_base = nvkm_memory_addr(ltc->tag_ram) + tag_margin; tag_base += tag_align - 1; do_div(tag_base, tag_align); From bd275f1d1a982db62edcd22f3aebf6253583ea37 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 076/151] drm/nouveau: wrap nvkm_mem objects in nvkm_memory interfaces This is a transition step, to enable finer-grained commits while transitioning to new MMU interfaces. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/fb.h | 3 ++ drivers/gpu/drm/nouveau/nouveau_mem.c | 33 +++++++++++++++++++ drivers/gpu/drm/nouveau/nouveau_mem.h | 7 ++++ drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c | 1 + .../drm/nouveau/nvkm/subdev/instmem/gk20a.c | 9 +++++ 5 files changed, 53 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index 942d671740050..9242248a8e88c 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -1,6 +1,7 @@ #ifndef __NVKM_FB_H__ #define __NVKM_FB_H__ #include +#include #include @@ -29,6 +30,8 @@ struct nvkm_mem { u64 offset; u64 size; struct sg_table *sg; + + struct nvkm_memory *memory; }; struct nvkm_fb_tile { diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c index 68ac332841ae6..501f72fbb8388 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.c +++ b/drivers/gpu/drm/nouveau/nouveau_mem.c @@ -66,6 +66,7 @@ nouveau_mem_host(struct ttm_mem_reg *reg, struct ttm_dma_tt *tt) else mem->__mem.pages = tt->dma_address; mem->_mem = &mem->__mem; mem->mem.page = 12; + mem->_mem->memory = &mem->memory; return 0; } @@ -78,6 +79,7 @@ nouveau_mem_vram(struct ttm_mem_reg *reg, bool contig, u8 page) int ret; mem->mem.page = page; + mem->_mem->memory = &mem->memory; ret = ram->func->get(ram, size, 1 << page, contig ? 0 : 1 << page, (mem->comp << 8) | mem->kind, &mem->_mem); @@ -97,6 +99,36 @@ nouveau_mem_del(struct ttm_mem_reg *reg) reg->mm_node = NULL; } +static enum nvkm_memory_target +nouveau_mem_memory_target(struct nvkm_memory *memory) +{ + struct nouveau_mem *mem = container_of(memory, typeof(*mem), memory); + if (mem->_mem->mem) + return NVKM_MEM_TARGET_VRAM; + return NVKM_MEM_TARGET_HOST; +}; + +static u8 +nouveau_mem_memory_page(struct nvkm_memory *memory) +{ + struct nouveau_mem *mem = container_of(memory, typeof(*mem), memory); + return mem->mem.page; +}; + +static u64 +nouveau_mem_memory_size(struct nvkm_memory *memory) +{ + struct nouveau_mem *mem = container_of(memory, typeof(*mem), memory); + return mem->_mem->size << 12; +} + +static const struct nvkm_memory_func +nouveau_mem_memory = { + .target = nouveau_mem_memory_target, + .page = nouveau_mem_memory_page, + .size = nouveau_mem_memory_size, +}; + int nouveau_mem_new(struct nouveau_cli *cli, u8 kind, u8 comp, struct ttm_mem_reg *reg) @@ -108,6 +140,7 @@ nouveau_mem_new(struct nouveau_cli *cli, u8 kind, u8 comp, mem->cli = cli; mem->kind = kind; mem->comp = comp; + nvkm_memory_ctor(&nouveau_mem_memory, &mem->memory); reg->mm_node = mem; return 0; diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.h b/drivers/gpu/drm/nouveau/nouveau_mem.h index d00b4f669c4bd..20930ebc5e21d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.h +++ b/drivers/gpu/drm/nouveau/nouveau_mem.h @@ -23,6 +23,13 @@ struct nouveau_mem { struct nvkm_mem __mem; struct nvkm_mem *_mem; struct nvkm_vma bar_vma; + + struct nvkm_memory memory; +}; + +enum nvif_vmm_get { + PTES, + LAZY, }; int nouveau_mem_new(struct nouveau_cli *, u8 kind, u8 comp, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c index 8d17644c18234..03a860ae75b11 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c @@ -41,6 +41,7 @@ nvkm_vram_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, struct nvkm_vram *vram = nvkm_vram(memory); struct nvkm_mem mem = { .mem = vram->mn, + .memory = &vram->memory, }; nvkm_vm_map_at(vma, offset, &mem); return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c index 73a9511c31011..816d1231df5f1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c @@ -119,6 +119,12 @@ gk20a_instobj_target(struct nvkm_memory *memory) return NVKM_MEM_TARGET_NCOH; } +static u8 +gk20a_instobj_page(struct nvkm_memory *memory) +{ + return 12; +} + static u64 gk20a_instobj_addr(struct nvkm_memory *memory) { @@ -343,6 +349,7 @@ static const struct nvkm_memory_func gk20a_instobj_func_dma = { .dtor = gk20a_instobj_dtor_dma, .target = gk20a_instobj_target, + .page = gk20a_instobj_page, .addr = gk20a_instobj_addr, .size = gk20a_instobj_size, .acquire = gk20a_instobj_acquire_dma, @@ -354,6 +361,7 @@ static const struct nvkm_memory_func gk20a_instobj_func_iommu = { .dtor = gk20a_instobj_dtor_iommu, .target = gk20a_instobj_target, + .page = gk20a_instobj_page, .addr = gk20a_instobj_addr, .size = gk20a_instobj_size, .acquire = gk20a_instobj_acquire_iommu, @@ -531,6 +539,7 @@ gk20a_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, /* present memory for being mapped using small pages */ node->mem.size = size >> 12; node->mem.memtype = 0; + node->mem.memory = &node->memory; nvkm_debug(subdev, "alloc size: 0x%x, align: 0x%x, gaddr: 0x%llx\n", size, align, node->mem.offset); From 7b8656636add64ea5c184a5de4a326d33a1d021e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 077/151] drm/nouveau: directly handle comptag allocation Another transition step to allow finer-grained patches transitioning to new MMU backends. Old backends will continue operate as before (accessing nvkm_mem::tag), and new backends will get a reference to the tags allocated here. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/ltc.h | 2 - drivers/gpu/drm/nouveau/nouveau_mem.c | 38 ++++++++++++++++++- drivers/gpu/drm/nouveau/nouveau_mem.h | 1 + .../gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c | 20 ---------- .../gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c | 15 -------- .../gpu/drm/nouveau/nvkm/subdev/ltc/base.c | 18 --------- 6 files changed, 37 insertions(+), 57 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h index 784661ae7e982..4a224fd22e48b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h @@ -22,8 +22,6 @@ struct nvkm_ltc { u32 zbc_depth[NVKM_LTC_MAX_ZBC_CNT]; }; -int nvkm_ltc_tags_alloc(struct nvkm_ltc *, u32 count, struct nvkm_mm_node **); -void nvkm_ltc_tags_free(struct nvkm_ltc *, struct nvkm_mm_node **); void nvkm_ltc_tags_clear(struct nvkm_device *, u32 first, u32 count); int nvkm_ltc_zbc_color_get(struct nvkm_ltc *, int index, const u32[4]); diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c index 501f72fbb8388..291f1a08da334 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.c +++ b/drivers/gpu/drm/nouveau/nouveau_mem.c @@ -23,6 +23,8 @@ #include "nouveau_drv.h" #include "nouveau_bo.h" +#include + #include int @@ -44,6 +46,8 @@ nouveau_mem_fini(struct nouveau_mem *mem) nvkm_vm_unmap(&mem->vma[0]); nvkm_vm_put(&mem->vma[0]); } + nvkm_memory_tags_put(&mem->memory, nvxx_device(&mem->cli->device), + &mem->tags); } int @@ -74,17 +78,47 @@ int nouveau_mem_vram(struct ttm_mem_reg *reg, bool contig, u8 page) { struct nouveau_mem *mem = nouveau_mem(reg); - struct nvkm_ram *ram = nvxx_fb(&mem->cli->device)->ram; + struct nouveau_cli *cli = mem->cli; + struct nvkm_device *device = nvxx_device(&cli->device); + struct nvkm_ram *ram = nvxx_fb(&cli->device)->ram; u64 size = ALIGN(reg->num_pages << PAGE_SHIFT, 1 << page); int ret; mem->mem.page = page; mem->_mem->memory = &mem->memory; + if (cli->device.info.chipset < 0xc0 && mem->comp) { + if (page == 16) { + ret = nvkm_memory_tags_get(mem->_mem->memory, device, + size >> page, NULL, + &mem->tags); + WARN_ON(ret); + } + if (!mem->tags || !mem->tags->mn) + mem->comp = 0; + } else + if (cli->device.info.chipset >= 0xc0 && + gf100_pte_storage_type_map[mem->kind] != mem->kind) { + if (page == 17) { + ret = nvkm_memory_tags_get(mem->_mem->memory, device, + size >> page, + nvkm_ltc_tags_clear, + &mem->tags); + WARN_ON(ret); + } + if (!mem->tags || !mem->tags->mn) + mem->kind = gf100_pte_storage_type_map[mem->kind]; + } + ret = ram->func->get(ram, size, 1 << page, contig ? 0 : 1 << page, (mem->comp << 8) | mem->kind, &mem->_mem); - if (ret) + if (ret) { + nvkm_memory_tags_put(mem->_mem->memory, device, &mem->tags); return ret; + } + + if (mem->tags && mem->tags->mn) + mem->_mem->tag = mem->tags->mn; reg->start = mem->_mem->offset >> PAGE_SHIFT; return ret; diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.h b/drivers/gpu/drm/nouveau/nouveau_mem.h index 20930ebc5e21d..89e9e7b9b00ca 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.h +++ b/drivers/gpu/drm/nouveau/nouveau_mem.h @@ -25,6 +25,7 @@ struct nouveau_mem { struct nvkm_vma bar_vma; struct nvkm_memory memory; + struct nvkm_tags *tags; }; enum nvif_vmm_get { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c index 7109e21f45641..f0bbbbd18cbd6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c @@ -32,7 +32,6 @@ #include #include #include -#include struct gf100_ramfuc { struct ramfuc base; @@ -423,7 +422,6 @@ gf100_ram_tidy(struct nvkm_ram *base) void gf100_ram_put(struct nvkm_ram *ram, struct nvkm_mem **pmem) { - struct nvkm_ltc *ltc = ram->fb->subdev.device->ltc; struct nvkm_mem *mem = *pmem; *pmem = NULL; @@ -431,8 +429,6 @@ gf100_ram_put(struct nvkm_ram *ram, struct nvkm_mem **pmem) return; mutex_lock(&ram->fb->subdev.mutex); - if (mem->tag) - nvkm_ltc_tags_free(ltc, &mem->tag); __nv50_ram_put(ram, mem); mutex_unlock(&ram->fb->subdev.mutex); @@ -443,14 +439,11 @@ int gf100_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin, u32 memtype, struct nvkm_mem **pmem) { - struct nvkm_device *device = ram->fb->subdev.device; - struct nvkm_ltc *ltc = ram->fb->subdev.device->ltc; struct nvkm_mm *mm = &ram->vram; struct nvkm_mm_node **node, *r; struct nvkm_mem *mem; int type = (memtype & 0x0ff); int back = (memtype & 0x800); - const bool comp = gf100_pte_storage_type_map[type] != type; int ret; size >>= NVKM_RAM_MM_SHIFT; @@ -466,19 +459,6 @@ gf100_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin, mem->size = size; mutex_lock(&ram->fb->subdev.mutex); - if (comp) { - /* compression only works with lpages */ - if (align == (1 << (17 - NVKM_RAM_MM_SHIFT))) { - int n = size >> 5; - if (!nvkm_ltc_tags_alloc(ltc, n, &mem->tag)) { - nvkm_ltc_tags_clear(device, mem->tag->offset, - mem->tag->length); - } - } - - if (unlikely(!mem->tag)) - type = gf100_pte_storage_type_map[type]; - } mem->memtype = type; node = &mem->mem; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c index 030ac8339f496..79312205b0e45 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c @@ -502,7 +502,6 @@ __nv50_ram_put(struct nvkm_ram *ram, struct nvkm_mem *mem) next = node->next; nvkm_mm_free(&ram->vram, &node); } - nvkm_mm_free(&ram->fb->tags, &mem->tag); } void @@ -526,7 +525,6 @@ nv50_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin, u32 memtype, struct nvkm_mem **pmem) { struct nvkm_mm *heap = &ram->vram; - struct nvkm_mm *tags = &ram->fb->tags; struct nvkm_mm_node **node, *r; struct nvkm_mem *mem; int comp = (memtype & 0x300) >> 8; @@ -543,19 +541,6 @@ nv50_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin, return -ENOMEM; mutex_lock(&ram->fb->subdev.mutex); - if (comp) { - if (align == (1 << (16 - NVKM_RAM_MM_SHIFT))) { - int n = (max >> 4) * comp; - - ret = nvkm_mm_head(tags, 0, 1, n, n, 1, &mem->tag); - if (ret) - mem->tag = NULL; - } - - if (unlikely(!mem->tag)) - comp = 0; - } - mem->memtype = (comp << 7) | type; mem->size = max; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c index 3bb97b1714356..1f185274d3e63 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c @@ -24,24 +24,6 @@ #include "priv.h" #include -#include - -int -nvkm_ltc_tags_alloc(struct nvkm_ltc *ltc, u32 n, struct nvkm_mm_node **pnode) -{ - struct nvkm_fb *fb = ltc->subdev.device->fb; - int ret = nvkm_mm_head(&fb->tags, 0, 1, n, n, 1, pnode); - if (ret) - *pnode = NULL; - return ret; -} - -void -nvkm_ltc_tags_free(struct nvkm_ltc *ltc, struct nvkm_mm_node **pnode) -{ - struct nvkm_fb *fb = ltc->subdev.device->fb; - nvkm_mm_free(&fb->tags, pnode); -} void nvkm_ltc_tags_clear(struct nvkm_device *device, u32 first, u32 count) From 1de3377555f14495c49c935810c220f014dbef71 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 078/151] drm/nouveau: allocate vram with nvkm_ram_get() This will cause a subtle behaviour change on GPUs that are in mixed-memory configurations in that VRAM in the degraded section of VRAM will no longer be used for TTM buffer objects. That section of VRAM is not meant to be used for displayable/compressed surfaces, and we have no reliable way with the current interfaces to be able to make that decision properly. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_mem.c | 39 ++++++++++++++++++++------- drivers/gpu/drm/nouveau/nouveau_ttm.c | 6 ++--- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c index 291f1a08da334..21d904ded441f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.c +++ b/drivers/gpu/drm/nouveau/nouveau_mem.c @@ -74,18 +74,44 @@ nouveau_mem_host(struct ttm_mem_reg *reg, struct ttm_dma_tt *tt) return 0; } +#include + +struct nvkm_vram { + struct nvkm_memory memory; + struct nvkm_ram *ram; + u8 page; + struct nvkm_mm_node *mn; +}; + int nouveau_mem_vram(struct ttm_mem_reg *reg, bool contig, u8 page) { struct nouveau_mem *mem = nouveau_mem(reg); struct nouveau_cli *cli = mem->cli; struct nvkm_device *device = nvxx_device(&cli->device); - struct nvkm_ram *ram = nvxx_fb(&cli->device)->ram; u64 size = ALIGN(reg->num_pages << PAGE_SHIFT, 1 << page); + u8 type; int ret; mem->mem.page = page; - mem->_mem->memory = &mem->memory; + mem->_mem = &mem->__mem; + + if (cli->device.info.chipset < 0xc0) { + type = nv50_fb_memtype[mem->kind]; + } else { + if (!mem->comp) + mem->kind = gf100_pte_storage_type_map[mem->kind]; + mem->comp = 0; + type = 0x01; + } + + ret = nvkm_ram_get(device, NVKM_RAM_MM_NORMAL, type, page, size, + contig, false, &mem->_mem->memory); + if (ret) + return ret; + + mem->_mem->size = size >> NVKM_RAM_MM_SHIFT; + mem->_mem->offset = nvkm_memory_addr(mem->_mem->memory); if (cli->device.info.chipset < 0xc0 && mem->comp) { if (page == 16) { @@ -110,15 +136,10 @@ nouveau_mem_vram(struct ttm_mem_reg *reg, bool contig, u8 page) mem->kind = gf100_pte_storage_type_map[mem->kind]; } - ret = ram->func->get(ram, size, 1 << page, contig ? 0 : 1 << page, - (mem->comp << 8) | mem->kind, &mem->_mem); - if (ret) { - nvkm_memory_tags_put(mem->_mem->memory, device, &mem->tags); - return ret; - } - if (mem->tags && mem->tags->mn) mem->_mem->tag = mem->tags->mn; + mem->_mem->mem = ((struct nvkm_vram *)mem->_mem->memory)->mn; + mem->_mem->memtype = (mem->comp << 7) | mem->kind; reg->start = mem->_mem->offset >> PAGE_SHIFT; return ret; diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index 3bf3b60130e02..8c4f45ea62341 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -60,11 +60,9 @@ static void nouveau_vram_manager_del(struct ttm_mem_type_manager *man, struct ttm_mem_reg *reg) { - struct nouveau_drm *drm = nouveau_bdev(man->bdev); - struct nvkm_ram *ram = nvxx_fb(&drm->client.device)->ram; - struct nvkm_mem *mem = nouveau_mem(reg)->_mem; + struct nvkm_memory *memory = nouveau_mem(reg)->_mem->memory; nouveau_mem_del(reg); - ram->func->put(ram, &mem); + nvkm_memory_unref(&memory); } static int From b4e114f1aa6ac1fd7c12223e032ec429be7d8152 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 079/151] drm/nouveau/fb/ram: remove old allocators Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/fb.h | 4 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h | 5 -- .../gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c | 67 ---------------- .../gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c | 77 ------------------- 11 files changed, 167 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index 9242248a8e88c..e1776506746b2 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -162,10 +162,6 @@ struct nvkm_ram_func { void *(*dtor)(struct nvkm_ram *); int (*init)(struct nvkm_ram *); - int (*get)(struct nvkm_ram *, u64 size, u32 align, u32 size_nc, - u32 type, struct nvkm_mem **); - void (*put)(struct nvkm_ram *, struct nvkm_mem **); - int (*calc)(struct nvkm_ram *, u32 freq); int (*prog)(struct nvkm_ram *); void (*tidy)(struct nvkm_ram *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h index a007699b60a2d..70fd59dcd06d6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h @@ -13,9 +13,6 @@ extern const struct nvkm_ram_func nv04_ram_func; int nv50_ram_ctor(const struct nvkm_ram_func *, struct nvkm_fb *, struct nvkm_ram *); -int nv50_ram_get(struct nvkm_ram *, u64, u32, u32, u32, struct nvkm_mem **); -void nv50_ram_put(struct nvkm_ram *, struct nvkm_mem **); -void __nv50_ram_put(struct nvkm_ram *, struct nvkm_mem *); int gf100_ram_new_(const struct nvkm_ram_func *, struct nvkm_fb *, struct nvkm_ram **); @@ -26,8 +23,6 @@ u32 gf100_ram_probe_fbp(const struct nvkm_ram_func *, u32 gf100_ram_probe_fbp_amount(const struct nvkm_ram_func *, u32, struct nvkm_device *, int, int *); u32 gf100_ram_probe_fbpa_amount(struct nvkm_device *, int); -int gf100_ram_get(struct nvkm_ram *, u64, u32, u32, u32, struct nvkm_mem **); -void gf100_ram_put(struct nvkm_ram *, struct nvkm_mem **); int gf100_ram_init(struct nvkm_ram *); int gf100_ram_calc(struct nvkm_ram *, u32); int gf100_ram_prog(struct nvkm_ram *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c index f0bbbbd18cbd6..ac87a3b6b7c9e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c @@ -419,71 +419,6 @@ gf100_ram_tidy(struct nvkm_ram *base) ram_exec(&ram->fuc, false); } -void -gf100_ram_put(struct nvkm_ram *ram, struct nvkm_mem **pmem) -{ - struct nvkm_mem *mem = *pmem; - - *pmem = NULL; - if (unlikely(mem == NULL)) - return; - - mutex_lock(&ram->fb->subdev.mutex); - __nv50_ram_put(ram, mem); - mutex_unlock(&ram->fb->subdev.mutex); - - kfree(mem); -} - -int -gf100_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin, - u32 memtype, struct nvkm_mem **pmem) -{ - struct nvkm_mm *mm = &ram->vram; - struct nvkm_mm_node **node, *r; - struct nvkm_mem *mem; - int type = (memtype & 0x0ff); - int back = (memtype & 0x800); - int ret; - - size >>= NVKM_RAM_MM_SHIFT; - align >>= NVKM_RAM_MM_SHIFT; - ncmin >>= NVKM_RAM_MM_SHIFT; - if (!ncmin) - ncmin = size; - - mem = kzalloc(sizeof(*mem), GFP_KERNEL); - if (!mem) - return -ENOMEM; - - mem->size = size; - - mutex_lock(&ram->fb->subdev.mutex); - mem->memtype = type; - - node = &mem->mem; - do { - if (back) - ret = nvkm_mm_tail(mm, 0, 1, size, ncmin, align, &r); - else - ret = nvkm_mm_head(mm, 0, 1, size, ncmin, align, &r); - if (ret) { - mutex_unlock(&ram->fb->subdev.mutex); - ram->func->put(ram, &mem); - return ret; - } - - *node = r; - node = &r->next; - size -= r->length; - } while (size); - mutex_unlock(&ram->fb->subdev.mutex); - - mem->offset = (u64)mem->mem->offset << NVKM_RAM_MM_SHIFT; - *pmem = mem; - return 0; -} - int gf100_ram_init(struct nvkm_ram *base) { @@ -725,8 +660,6 @@ gf100_ram = { .probe_fbp_amount = gf100_ram_probe_fbp_amount, .probe_fbpa_amount = gf100_ram_probe_fbpa_amount, .init = gf100_ram_init, - .get = gf100_ram_get, - .put = gf100_ram_put, .calc = gf100_ram_calc, .prog = gf100_ram_prog, .tidy = gf100_ram_tidy, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c index 985ec64cf369b..70a06e3cd55a2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c @@ -48,8 +48,6 @@ gf108_ram = { .probe_fbp_amount = gf108_ram_probe_fbp_amount, .probe_fbpa_amount = gf100_ram_probe_fbpa_amount, .init = gf100_ram_init, - .get = gf100_ram_get, - .put = gf100_ram_put, .calc = gf100_ram_calc, .prog = gf100_ram_prog, .tidy = gf100_ram_tidy, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c index 75814f15eb53c..8bcb7e79a0cb5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c @@ -1704,8 +1704,6 @@ gk104_ram = { .probe_fbpa_amount = gf100_ram_probe_fbpa_amount, .dtor = gk104_ram_dtor, .init = gk104_ram_init, - .get = gf100_ram_get, - .put = gf100_ram_put, .calc = gk104_ram_calc, .prog = gk104_ram_prog, .tidy = gk104_ram_tidy, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c index 3f0b56347291b..27c68e3f97721 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c @@ -39,8 +39,6 @@ gm107_ram = { .probe_fbpa_amount = gf100_ram_probe_fbpa_amount, .dtor = gk104_ram_dtor, .init = gk104_ram_init, - .get = gf100_ram_get, - .put = gf100_ram_put, .calc = gk104_ram_calc, .prog = gk104_ram_prog, .tidy = gk104_ram_tidy, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c index fd8facf904766..6b0cac1fe7b44 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c @@ -54,8 +54,6 @@ gm200_ram = { .probe_fbpa_amount = gf100_ram_probe_fbpa_amount, .dtor = gk104_ram_dtor, .init = gk104_ram_init, - .get = gf100_ram_get, - .put = gf100_ram_put, .calc = gk104_ram_calc, .prog = gk104_ram_prog, .tidy = gk104_ram_tidy, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c index df8a87333b674..adb62a6beb638 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c @@ -84,8 +84,6 @@ gp100_ram = { .probe_fbp_amount = gm200_ram_probe_fbp_amount, .probe_fbpa_amount = gp100_ram_probe_fbpa, .init = gp100_ram_init, - .get = gf100_ram_get, - .put = gf100_ram_put, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c index 9bace6c4ad0bb..920b3d3478030 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c @@ -931,8 +931,6 @@ static const struct nvkm_ram_func gt215_ram_func = { .dtor = gt215_ram_dtor, .init = gt215_ram_init, - .get = nv50_ram_get, - .put = nv50_ram_put, .calc = gt215_ram_calc, .prog = gt215_ram_prog, .tidy = gt215_ram_tidy, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c index 76fa71f0c4576..7de18e53ef45c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c @@ -53,8 +53,6 @@ mcp77_ram_init(struct nvkm_ram *base) static const struct nvkm_ram_func mcp77_ram_func = { .init = mcp77_ram_init, - .get = nv50_ram_get, - .put = nv50_ram_put, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c index 79312205b0e45..2ccb4b6be1538 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c @@ -493,85 +493,8 @@ nv50_ram_tidy(struct nvkm_ram *base) ram_exec(&ram->hwsq, false); } -void -__nv50_ram_put(struct nvkm_ram *ram, struct nvkm_mem *mem) -{ - struct nvkm_mm_node *next = mem->mem; - struct nvkm_mm_node *node; - while ((node = next)) { - next = node->next; - nvkm_mm_free(&ram->vram, &node); - } -} - -void -nv50_ram_put(struct nvkm_ram *ram, struct nvkm_mem **pmem) -{ - struct nvkm_mem *mem = *pmem; - - *pmem = NULL; - if (unlikely(mem == NULL)) - return; - - mutex_lock(&ram->fb->subdev.mutex); - __nv50_ram_put(ram, mem); - mutex_unlock(&ram->fb->subdev.mutex); - - kfree(mem); -} - -int -nv50_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin, - u32 memtype, struct nvkm_mem **pmem) -{ - struct nvkm_mm *heap = &ram->vram; - struct nvkm_mm_node **node, *r; - struct nvkm_mem *mem; - int comp = (memtype & 0x300) >> 8; - int type = (memtype & 0x07f); - int back = (memtype & 0x800); - int min, max, ret; - - max = (size >> NVKM_RAM_MM_SHIFT); - min = ncmin ? (ncmin >> NVKM_RAM_MM_SHIFT) : max; - align >>= NVKM_RAM_MM_SHIFT; - - mem = kzalloc(sizeof(*mem), GFP_KERNEL); - if (!mem) - return -ENOMEM; - - mutex_lock(&ram->fb->subdev.mutex); - mem->memtype = (comp << 7) | type; - mem->size = max; - - type = nv50_fb_memtype[type]; - node = &mem->mem; - do { - if (back) - ret = nvkm_mm_tail(heap, 0, type, max, min, align, &r); - else - ret = nvkm_mm_head(heap, 0, type, max, min, align, &r); - if (ret) { - mutex_unlock(&ram->fb->subdev.mutex); - ram->func->put(ram, &mem); - return ret; - } - - *node = r; - node = &r->next; - max -= r->length; - } while (max); - mutex_unlock(&ram->fb->subdev.mutex); - - mem->offset = (u64)mem->mem->offset << NVKM_RAM_MM_SHIFT; - *pmem = mem; - return 0; -} - static const struct nvkm_ram_func nv50_ram_func = { - .get = nv50_ram_get, - .put = nv50_ram_put, .calc = nv50_ram_calc, .prog = nv50_ram_prog, .tidy = nv50_ram_tidy, From 0f43715fac003613a028d28a2c76fbab4c9f9177 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 080/151] drm/nouveau/mmu/g84: fork from nv50 Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 1 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 26 +++++------ .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c | 43 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c | 12 +++--- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 10 +++++ 6 files changed, 74 insertions(+), 19 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 549a8782ce356..c1cc42fd6893b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -67,5 +67,6 @@ int nv04_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int nv41_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int nv44_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int nv50_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); +int g84_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gf100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 77fe1c1454983..f90b9a9d22db0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -927,7 +927,7 @@ nv84_chipset = { .i2c = nv50_i2c_new, .imem = nv50_instmem_new, .mc = g84_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g84_pci_new, .therm = g84_therm_new, @@ -959,7 +959,7 @@ nv86_chipset = { .i2c = nv50_i2c_new, .imem = nv50_instmem_new, .mc = g84_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g84_pci_new, .therm = g84_therm_new, @@ -991,7 +991,7 @@ nv92_chipset = { .i2c = nv50_i2c_new, .imem = nv50_instmem_new, .mc = g84_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g92_pci_new, .therm = g84_therm_new, @@ -1023,7 +1023,7 @@ nv94_chipset = { .i2c = g94_i2c_new, .imem = nv50_instmem_new, .mc = g84_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g94_pci_new, .therm = g84_therm_new, @@ -1055,7 +1055,7 @@ nv96_chipset = { .i2c = g94_i2c_new, .imem = nv50_instmem_new, .mc = g84_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g94_pci_new, .therm = g84_therm_new, @@ -1087,7 +1087,7 @@ nv98_chipset = { .i2c = g94_i2c_new, .imem = nv50_instmem_new, .mc = g98_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g94_pci_new, .therm = g84_therm_new, @@ -1119,7 +1119,7 @@ nva0_chipset = { .i2c = nv50_i2c_new, .imem = nv50_instmem_new, .mc = g84_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g94_pci_new, .therm = g84_therm_new, @@ -1151,7 +1151,7 @@ nva3_chipset = { .i2c = g94_i2c_new, .imem = nv50_instmem_new, .mc = gt215_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g94_pci_new, .pmu = gt215_pmu_new, @@ -1185,7 +1185,7 @@ nva5_chipset = { .i2c = g94_i2c_new, .imem = nv50_instmem_new, .mc = gt215_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g94_pci_new, .pmu = gt215_pmu_new, @@ -1218,7 +1218,7 @@ nva8_chipset = { .i2c = g94_i2c_new, .imem = nv50_instmem_new, .mc = gt215_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g94_pci_new, .pmu = gt215_pmu_new, @@ -1251,7 +1251,7 @@ nvaa_chipset = { .i2c = g94_i2c_new, .imem = nv50_instmem_new, .mc = g98_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g94_pci_new, .therm = g84_therm_new, @@ -1283,7 +1283,7 @@ nvac_chipset = { .i2c = g94_i2c_new, .imem = nv50_instmem_new, .mc = g98_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g94_pci_new, .therm = g84_therm_new, @@ -1315,7 +1315,7 @@ nvaf_chipset = { .i2c = g94_i2c_new, .imem = nv50_instmem_new, .mc = gt215_mc_new, - .mmu = nv50_mmu_new, + .mmu = g84_mmu_new, .mxm = nv50_mxm_new, .pci = g94_pci_new, .pmu = gt215_pmu_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 012c9db687b21..2c0db24202d17 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -3,4 +3,5 @@ nvkm-y += nvkm/subdev/mmu/nv04.o nvkm-y += nvkm/subdev/mmu/nv41.o nvkm-y += nvkm/subdev/mmu/nv44.o nvkm-y += nvkm/subdev/mmu/nv50.o +nvkm-y += nvkm/subdev/mmu/g84.o nvkm-y += nvkm/subdev/mmu/gf100.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c new file mode 100644 index 0000000000000..a1aca162a8cfa --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c @@ -0,0 +1,43 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_mmu_func +g84_mmu = { + .limit = (1ULL << 40), + .dma_bits = 40, + .pgt_bits = 29 - 12, + .spg_shift = 12, + .lpg_shift = 16, + .create = nv50_vm_create, + .map_pgt = nv50_vm_map_pgt, + .map = nv50_vm_map, + .map_sg = nv50_vm_map_sg, + .unmap = nv50_vm_unmap, + .flush = nv50_vm_flush, +}; + +int +g84_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) +{ + return nvkm_mmu_new_(&g84_mmu, device, index, pmmu); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c index a1f8d65f02761..5811516c9a261 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c @@ -28,7 +28,7 @@ #include #include -static void +void nv50_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 pde, struct nvkm_memory *pgt[2]) { u64 phys = 0xdeadcafe00000000ULL; @@ -73,7 +73,7 @@ vm_addr(struct nvkm_vma *vma, u64 phys, u32 memtype, u32 target) return phys; } -static void +void nv50_vm_map(struct nvkm_vma *vma, struct nvkm_memory *pgt, struct nvkm_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta) { @@ -123,7 +123,7 @@ nv50_vm_map(struct nvkm_vma *vma, struct nvkm_memory *pgt, nvkm_done(pgt); } -static void +void nv50_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list) { @@ -139,7 +139,7 @@ nv50_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, nvkm_done(pgt); } -static void +void nv50_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) { pte <<= 3; @@ -152,7 +152,7 @@ nv50_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) nvkm_done(pgt); } -static void +void nv50_vm_flush(struct nvkm_vm *vm) { struct nvkm_mmu *mmu = vm->mmu; @@ -198,7 +198,7 @@ nv50_vm_flush(struct nvkm_vm *vm) mutex_unlock(&subdev->mutex); } -static int +int nv50_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, struct lock_class_key *key, struct nvkm_vm **pvm) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 27cedc60b507b..1e5ef934c4e78 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -36,4 +36,14 @@ struct nvkm_mmu_func { int nvkm_vm_create(struct nvkm_mmu *, u64, u64, u64, u32, struct lock_class_key *, struct nvkm_vm **); + +int nv50_vm_create(struct nvkm_mmu *, u64, u64, u64, struct lock_class_key *, + struct nvkm_vm **); +void nv50_vm_map_pgt(struct nvkm_gpuobj *, u32, struct nvkm_memory **); +void nv50_vm_map(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, + u32, u32, u64, u64); +void nv50_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, + u32, u32, dma_addr_t *); +void nv50_vm_unmap(struct nvkm_vma *, struct nvkm_memory *, u32, u32); +void nv50_vm_flush(struct nvkm_vm *); #endif From db018585a51a08fd80ade568ada7b985abcbaf6e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 081/151] drm/nouveau/mmu/gk104: fork from gf100 Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 1 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 18 ++++---- .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 12 +++--- .../gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c | 43 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 10 +++++ 6 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index c1cc42fd6893b..c57263437f26d 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -69,4 +69,5 @@ int nv44_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int nv50_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int g84_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gf100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); +int gk104_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index f90b9a9d22db0..81a2bc4e57cbe 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -1678,7 +1678,7 @@ nve4_chipset = { .imem = nv50_instmem_new, .ltc = gk104_ltc_new, .mc = gk104_mc_new, - .mmu = gf100_mmu_new, + .mmu = gk104_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gk104_pmu_new, @@ -1717,7 +1717,7 @@ nve6_chipset = { .imem = nv50_instmem_new, .ltc = gk104_ltc_new, .mc = gk104_mc_new, - .mmu = gf100_mmu_new, + .mmu = gk104_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gk104_pmu_new, @@ -1756,7 +1756,7 @@ nve7_chipset = { .imem = nv50_instmem_new, .ltc = gk104_ltc_new, .mc = gk104_mc_new, - .mmu = gf100_mmu_new, + .mmu = gk104_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gk104_pmu_new, @@ -1820,7 +1820,7 @@ nvf0_chipset = { .imem = nv50_instmem_new, .ltc = gk104_ltc_new, .mc = gk104_mc_new, - .mmu = gf100_mmu_new, + .mmu = gk104_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gk110_pmu_new, @@ -1858,7 +1858,7 @@ nvf1_chipset = { .imem = nv50_instmem_new, .ltc = gk104_ltc_new, .mc = gk104_mc_new, - .mmu = gf100_mmu_new, + .mmu = gk104_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gk110_pmu_new, @@ -1896,7 +1896,7 @@ nv106_chipset = { .imem = nv50_instmem_new, .ltc = gk104_ltc_new, .mc = gk20a_mc_new, - .mmu = gf100_mmu_new, + .mmu = gk104_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gk208_pmu_new, @@ -1934,7 +1934,7 @@ nv108_chipset = { .imem = nv50_instmem_new, .ltc = gk104_ltc_new, .mc = gk20a_mc_new, - .mmu = gf100_mmu_new, + .mmu = gk104_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gk208_pmu_new, @@ -1972,7 +1972,7 @@ nv117_chipset = { .imem = nv50_instmem_new, .ltc = gm107_ltc_new, .mc = gk20a_mc_new, - .mmu = gf100_mmu_new, + .mmu = gk104_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gm107_pmu_new, @@ -2006,7 +2006,7 @@ nv118_chipset = { .imem = nv50_instmem_new, .ltc = gm107_ltc_new, .mc = gk20a_mc_new, - .mmu = gf100_mmu_new, + .mmu = gk104_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gm107_pmu_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 2c0db24202d17..f1bef6b6400aa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -5,3 +5,4 @@ nvkm-y += nvkm/subdev/mmu/nv44.o nvkm-y += nvkm/subdev/mmu/nv50.o nvkm-y += nvkm/subdev/mmu/g84.o nvkm-y += nvkm/subdev/mmu/gf100.o +nvkm-y += nvkm/subdev/mmu/gk104.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index cda5b0997c8f0..3cc1013538c57 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -68,7 +68,7 @@ const u8 gf100_pte_storage_type_map[256] = }; -static void +void gf100_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 index, struct nvkm_memory *pgt[2]) { u32 pde[2] = { 0, 0 }; @@ -98,7 +98,7 @@ gf100_vm_addr(struct nvkm_vma *vma, u64 phys, u32 memtype, u32 target) return phys; } -static void +void gf100_vm_map(struct nvkm_vma *vma, struct nvkm_memory *pgt, struct nvkm_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta) { @@ -123,7 +123,7 @@ gf100_vm_map(struct nvkm_vma *vma, struct nvkm_memory *pgt, nvkm_done(pgt); } -static void +void gf100_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list) { @@ -142,7 +142,7 @@ gf100_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, nvkm_done(pgt); } -static void +void gf100_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) { nvkm_kmap(pgt); @@ -155,7 +155,7 @@ gf100_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) nvkm_done(pgt); } -static void +void gf100_vm_flush(struct nvkm_vm *vm) { struct nvkm_mmu *mmu = vm->mmu; @@ -189,7 +189,7 @@ gf100_vm_flush(struct nvkm_vm *vm) mutex_unlock(&mmu->subdev.mutex); } -static int +int gf100_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, struct lock_class_key *key, struct nvkm_vm **pvm) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c new file mode 100644 index 0000000000000..6ab8e636feb13 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c @@ -0,0 +1,43 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_mmu_func +gk104_mmu = { + .limit = (1ULL << 40), + .dma_bits = 40, + .pgt_bits = 27 - 12, + .spg_shift = 12, + .lpg_shift = 17, + .create = gf100_vm_create, + .map_pgt = gf100_vm_map_pgt, + .map = gf100_vm_map, + .map_sg = gf100_vm_map_sg, + .unmap = gf100_vm_unmap, + .flush = gf100_vm_flush, +}; + +int +gk104_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) +{ + return nvkm_mmu_new_(&gk104_mmu, device, index, pmmu); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 1e5ef934c4e78..9346dae07c682 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -46,4 +46,14 @@ void nv50_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, u32, u32, dma_addr_t *); void nv50_vm_unmap(struct nvkm_vma *, struct nvkm_memory *, u32, u32); void nv50_vm_flush(struct nvkm_vm *); + +int gf100_vm_create(struct nvkm_mmu *, u64, u64, u64, struct lock_class_key *, + struct nvkm_vm **); +void gf100_vm_map_pgt(struct nvkm_gpuobj *, u32, struct nvkm_memory **); +void gf100_vm_map(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, + u32, u32, u64, u64); +void gf100_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, + u32, u32, dma_addr_t *); +void gf100_vm_unmap(struct nvkm_vma *, struct nvkm_memory *, u32, u32); +void gf100_vm_flush(struct nvkm_vm *); #endif From d1f6c8d2e9df49eb7387b04b880d643348e97625 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 082/151] drm/nouveau/mmu/gk20a: fork from gf100 Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 1 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c | 43 +++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index c57263437f26d..04b3ac1c2121f 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -70,4 +70,5 @@ int nv50_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int g84_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gf100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gk104_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); +int gk20a_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 81a2bc4e57cbe..b87f6cf12a694 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -1790,7 +1790,7 @@ nvea_chipset = { .imem = gk20a_instmem_new, .ltc = gk104_ltc_new, .mc = gk20a_mc_new, - .mmu = gf100_mmu_new, + .mmu = gk20a_mmu_new, .pmu = gk20a_pmu_new, .timer = gk20a_timer_new, .top = gk104_top_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index f1bef6b6400aa..dde204c47f70a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -6,3 +6,4 @@ nvkm-y += nvkm/subdev/mmu/nv50.o nvkm-y += nvkm/subdev/mmu/g84.o nvkm-y += nvkm/subdev/mmu/gf100.o nvkm-y += nvkm/subdev/mmu/gk104.o +nvkm-y += nvkm/subdev/mmu/gk20a.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c new file mode 100644 index 0000000000000..9ea8a91802be1 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c @@ -0,0 +1,43 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_mmu_func +gk20a_mmu = { + .limit = (1ULL << 40), + .dma_bits = 40, + .pgt_bits = 27 - 12, + .spg_shift = 12, + .lpg_shift = 17, + .create = gf100_vm_create, + .map_pgt = gf100_vm_map_pgt, + .map = gf100_vm_map, + .map_sg = gf100_vm_map_sg, + .unmap = gf100_vm_unmap, + .flush = gf100_vm_flush, +}; + +int +gk20a_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) +{ + return nvkm_mmu_new_(&gk20a_mmu, device, index, pmmu); +} From e1e33c791a233cb4cc84175109e42bdd6467889c Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 083/151] drm/nouveau/mmu/gm200: fork from gf100 Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 1 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 6 +-- .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c | 43 +++++++++++++++++++ 4 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 04b3ac1c2121f..989fd2897f6dd 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -71,4 +71,5 @@ int g84_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gf100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gk104_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gk20a_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); +int gm200_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index b87f6cf12a694..c4cd89a7be702 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2039,7 +2039,7 @@ nv120_chipset = { .imem = nv50_instmem_new, .ltc = gm200_ltc_new, .mc = gk20a_mc_new, - .mmu = gf100_mmu_new, + .mmu = gm200_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gm107_pmu_new, @@ -2074,7 +2074,7 @@ nv124_chipset = { .imem = nv50_instmem_new, .ltc = gm200_ltc_new, .mc = gk20a_mc_new, - .mmu = gf100_mmu_new, + .mmu = gm200_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gm107_pmu_new, @@ -2109,7 +2109,7 @@ nv126_chipset = { .imem = nv50_instmem_new, .ltc = gm200_ltc_new, .mc = gk20a_mc_new, - .mmu = gf100_mmu_new, + .mmu = gm200_mmu_new, .mxm = nv50_mxm_new, .pci = gk104_pci_new, .pmu = gm107_pmu_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index dde204c47f70a..b5b260d2fb887 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -7,3 +7,4 @@ nvkm-y += nvkm/subdev/mmu/g84.o nvkm-y += nvkm/subdev/mmu/gf100.o nvkm-y += nvkm/subdev/mmu/gk104.o nvkm-y += nvkm/subdev/mmu/gk20a.o +nvkm-y += nvkm/subdev/mmu/gm200.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c new file mode 100644 index 0000000000000..4bb342fa3d540 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c @@ -0,0 +1,43 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_mmu_func +gm200_mmu = { + .limit = (1ULL << 40), + .dma_bits = 40, + .pgt_bits = 27 - 12, + .spg_shift = 12, + .lpg_shift = 17, + .create = gf100_vm_create, + .map_pgt = gf100_vm_map_pgt, + .map = gf100_vm_map, + .map_sg = gf100_vm_map_sg, + .unmap = gf100_vm_unmap, + .flush = gf100_vm_flush, +}; + +int +gm200_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) +{ + return nvkm_mmu_new_(&gm200_mmu, device, index, pmmu); +} From cedc4d57df26170a2f1536cec276c5cc809a2462 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 084/151] drm/nouveau/mmu/gm20b: fork from gf100 Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 1 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c | 43 +++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 989fd2897f6dd..805d36b049b83 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -72,4 +72,5 @@ int gf100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gk104_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gk20a_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gm200_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); +int gm20b_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index c4cd89a7be702..d99093435c253 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2140,7 +2140,7 @@ nv12b_chipset = { .imem = gk20a_instmem_new, .ltc = gm200_ltc_new, .mc = gk20a_mc_new, - .mmu = gf100_mmu_new, + .mmu = gm20b_mmu_new, .pmu = gm20b_pmu_new, .secboot = gm20b_secboot_new, .timer = gk20a_timer_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index b5b260d2fb887..278041ffc940f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -8,3 +8,4 @@ nvkm-y += nvkm/subdev/mmu/gf100.o nvkm-y += nvkm/subdev/mmu/gk104.o nvkm-y += nvkm/subdev/mmu/gk20a.o nvkm-y += nvkm/subdev/mmu/gm200.o +nvkm-y += nvkm/subdev/mmu/gm20b.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c new file mode 100644 index 0000000000000..06e49a9c8a9f5 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c @@ -0,0 +1,43 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_mmu_func +gm20b_mmu = { + .limit = (1ULL << 40), + .dma_bits = 40, + .pgt_bits = 27 - 12, + .spg_shift = 12, + .lpg_shift = 17, + .create = gf100_vm_create, + .map_pgt = gf100_vm_map_pgt, + .map = gf100_vm_map, + .map_sg = gf100_vm_map_sg, + .unmap = gf100_vm_unmap, + .flush = gf100_vm_flush, +}; + +int +gm20b_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) +{ + return nvkm_mmu_new_(&gm20b_mmu, device, index, pmmu); +} From b86a45877ead1d0235674cae58dac7d07f00fa88 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 085/151] drm/nouveau/mmu/gp100: fork from gf100 Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 1 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 12 +++--- .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c | 43 +++++++++++++++++++ 4 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 805d36b049b83..87baa5e9691c4 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -73,4 +73,5 @@ int gk104_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gk20a_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gm200_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gm20b_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); +int gp100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index d99093435c253..e146436156985 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2168,7 +2168,7 @@ nv130_chipset = { .imem = nv50_instmem_new, .ltc = gp100_ltc_new, .mc = gp100_mc_new, - .mmu = gf100_mmu_new, + .mmu = gp100_mmu_new, .therm = gp100_therm_new, .secboot = gm200_secboot_new, .pci = gp100_pci_new, @@ -2203,7 +2203,7 @@ nv132_chipset = { .imem = nv50_instmem_new, .ltc = gp100_ltc_new, .mc = gp100_mc_new, - .mmu = gf100_mmu_new, + .mmu = gp100_mmu_new, .therm = gp100_therm_new, .secboot = gp102_secboot_new, .pci = gp100_pci_new, @@ -2238,7 +2238,7 @@ nv134_chipset = { .imem = nv50_instmem_new, .ltc = gp100_ltc_new, .mc = gp100_mc_new, - .mmu = gf100_mmu_new, + .mmu = gp100_mmu_new, .therm = gp100_therm_new, .secboot = gp102_secboot_new, .pci = gp100_pci_new, @@ -2273,7 +2273,7 @@ nv136_chipset = { .imem = nv50_instmem_new, .ltc = gp100_ltc_new, .mc = gp100_mc_new, - .mmu = gf100_mmu_new, + .mmu = gp100_mmu_new, .therm = gp100_therm_new, .secboot = gp102_secboot_new, .pci = gp100_pci_new, @@ -2308,7 +2308,7 @@ nv137_chipset = { .imem = nv50_instmem_new, .ltc = gp100_ltc_new, .mc = gp100_mc_new, - .mmu = gf100_mmu_new, + .mmu = gp100_mmu_new, .therm = gp100_therm_new, .secboot = gp102_secboot_new, .pci = gp100_pci_new, @@ -2343,7 +2343,7 @@ nv138_chipset = { .imem = nv50_instmem_new, .ltc = gp100_ltc_new, .mc = gp100_mc_new, - .mmu = gf100_mmu_new, + .mmu = gp100_mmu_new, .therm = gp100_therm_new, .pci = gp100_pci_new, .pmu = gp102_pmu_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 278041ffc940f..7d63089aede09 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -9,3 +9,4 @@ nvkm-y += nvkm/subdev/mmu/gk104.o nvkm-y += nvkm/subdev/mmu/gk20a.o nvkm-y += nvkm/subdev/mmu/gm200.o nvkm-y += nvkm/subdev/mmu/gm20b.o +nvkm-y += nvkm/subdev/mmu/gp100.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c new file mode 100644 index 0000000000000..6cfd325ea3a91 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c @@ -0,0 +1,43 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_mmu_func +gp100_mmu = { + .limit = (1ULL << 40), + .dma_bits = 40, + .pgt_bits = 27 - 12, + .spg_shift = 12, + .lpg_shift = 17, + .create = gf100_vm_create, + .map_pgt = gf100_vm_map_pgt, + .map = gf100_vm_map, + .map_sg = gf100_vm_map_sg, + .unmap = gf100_vm_unmap, + .flush = gf100_vm_flush, +}; + +int +gp100_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) +{ + return nvkm_mmu_new_(&gp100_mmu, device, index, pmmu); +} From 6359c982243ee8e4da4ca93cf30ac746a925ea1b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 086/151] drm/nouveau/mmu/gp10b: fork from gf100 Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c | 43 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 87baa5e9691c4..6d38e21c5f659 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -74,4 +74,5 @@ int gk20a_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gm200_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gm20b_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); int gp100_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); +int gp10b_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 7d63089aede09..e84ef36eb7e74 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -10,3 +10,4 @@ nvkm-y += nvkm/subdev/mmu/gk20a.o nvkm-y += nvkm/subdev/mmu/gm200.o nvkm-y += nvkm/subdev/mmu/gm20b.o nvkm-y += nvkm/subdev/mmu/gp100.o +nvkm-y += nvkm/subdev/mmu/gp10b.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c new file mode 100644 index 0000000000000..e0a41ce3ddd19 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -0,0 +1,43 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_mmu_func +gp10b_mmu = { + .limit = (1ULL << 40), + .dma_bits = 40, + .pgt_bits = 27 - 12, + .spg_shift = 12, + .lpg_shift = 17, + .create = gf100_vm_create, + .map_pgt = gf100_vm_map_pgt, + .map = gf100_vm_map, + .map_sg = gf100_vm_map_sg, + .unmap = gf100_vm_unmap, + .flush = gf100_vm_flush, +}; + +int +gp10b_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) +{ + return nvkm_mmu_new_(&gp10b_mmu, device, index, pmmu); +} From 5e075fdeb166098a3dc493026534c7631e845782 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 087/151] drm/nouveau/mmu: automatically handle "un-bootstrapping" of vmm Removes the need to expose internals outside of MMU, and GP100 is both different, and a lot harder to deal with. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 5 +---- drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 5 +---- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 3 +++ 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 6d38e21c5f659..f26407869e20c 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -38,6 +38,8 @@ struct nvkm_vm { struct nvkm_vm_pgt *pgt; u32 fpde; u32 lpde; + + bool bootstrapped; }; int nvkm_vm_new(struct nvkm_device *, u64 offset, u64 length, u64 mm_offset, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index 9f7b4e7532b70..8b588d1c776b8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -168,10 +168,7 @@ gf100_bar_dtor(struct nvkm_bar *base) nvkm_gpuobj_del(&bar->bar[1].pgd); nvkm_memory_unref(&bar->bar[1].mem); - if (bar->bar[0].vm) { - nvkm_memory_unref(&bar->bar[0].vm->pgt[0].mem[0]); - nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].pgd); - } + nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].pgd); nvkm_gpuobj_del(&bar->bar[0].pgd); nvkm_memory_unref(&bar->bar[0].mem); return bar; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index 1aa6b5390d796..b40c131af20e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -200,10 +200,7 @@ nv50_bar_dtor(struct nvkm_bar *base) nvkm_gpuobj_del(&bar->bar1); nvkm_vm_ref(NULL, &bar->bar1_vm, bar->pgd); nvkm_gpuobj_del(&bar->bar2); - if (bar->bar2_vm) { - nvkm_memory_unref(&bar->bar2_vm->pgt[0].mem[0]); - nvkm_vm_ref(NULL, &bar->bar2_vm, bar->pgd); - } + nvkm_vm_ref(NULL, &bar->bar2_vm, bar->pgd); nvkm_gpuobj_del(&bar->pgd); nvkm_gpuobj_del(&bar->pad); nvkm_gpuobj_del(&bar->mem); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index ad11db458fcce..d9f572db5c290 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -357,6 +357,7 @@ nvkm_vm_boot(struct nvkm_vm *vm, u64 size) vm->pgt[0].refcount[0] = 1; vm->pgt[0].mem[0] = pgt; nvkm_memory_boot(pgt, vm); + vm->bootstrapped = true; } return ret; @@ -481,6 +482,8 @@ nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_gpuobj *pgd) } if (*ptr) { + if ((*ptr)->bootstrapped && pgd) + nvkm_memory_unref(&(*ptr)->pgt[0].mem[0]); nvkm_vm_unlink(*ptr, pgd); kref_put(&(*ptr)->refcount, nvkm_vm_del); } From 9a45ddaaa674aa103cd74a0df9a3f9c2c8fb3124 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 088/151] drm/nouveau/mmu: implement page table cache Builds up and maintains a small cache of each page table size in order to reduce the frequency of expensive allocations, particularly in the pathological case where an address range ping-pongs between allocated and free. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 5 + .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 132 +++++++++++++++++- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 15 ++ 3 files changed, 150 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index f26407869e20c..61cc4e32e4498 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -63,6 +63,11 @@ struct nvkm_mmu { u8 lpg_shift; struct nvkm_vmm *vmm; + + struct { + struct mutex mutex; + struct list_head list; + } ptc; }; int nv04_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index d9f572db5c290..1e4a92e0068e6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -26,6 +26,129 @@ #include #include +struct nvkm_mmu_ptc { + struct list_head head; + struct list_head item; + u32 size; + u32 refs; +}; + +static inline struct nvkm_mmu_ptc * +nvkm_mmu_ptc_find(struct nvkm_mmu *mmu, u32 size) +{ + struct nvkm_mmu_ptc *ptc; + + list_for_each_entry(ptc, &mmu->ptc.list, head) { + if (ptc->size == size) + return ptc; + } + + ptc = kmalloc(sizeof(*ptc), GFP_KERNEL); + if (ptc) { + INIT_LIST_HEAD(&ptc->item); + ptc->size = size; + ptc->refs = 0; + list_add(&ptc->head, &mmu->ptc.list); + } + + return ptc; +} + +void +nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt) +{ + struct nvkm_mmu_pt *pt = *ppt; + if (pt) { + /* Either cache or free the object. */ + mutex_lock(&mmu->ptc.mutex); + if (pt->ptc->refs < 8 /* Heuristic. */ && !force) { + list_add_tail(&pt->head, &pt->ptc->item); + pt->ptc->refs++; + } else { + nvkm_memory_unref(&pt->memory); + kfree(pt); + } + mutex_unlock(&mmu->ptc.mutex); + } +} + +struct nvkm_mmu_pt * +nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero) +{ + struct nvkm_mmu_ptc *ptc; + struct nvkm_mmu_pt *pt; + int ret; + + /* Lookup cache for this page table size. */ + mutex_lock(&mmu->ptc.mutex); + ptc = nvkm_mmu_ptc_find(mmu, size); + if (!ptc) { + mutex_unlock(&mmu->ptc.mutex); + return NULL; + } + + /* If there's a free PT in the cache, reuse it. */ + pt = list_first_entry_or_null(&ptc->item, typeof(*pt), head); + if (pt) { + if (zero) + nvkm_fo64(pt->memory, 0, 0, size >> 3); + list_del(&pt->head); + ptc->refs--; + mutex_unlock(&mmu->ptc.mutex); + return pt; + } + mutex_unlock(&mmu->ptc.mutex); + + /* No such luck, we need to allocate. */ + if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL))) + return NULL; + pt->ptc = ptc; + + ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST, + size, align, zero, &pt->memory); + if (ret) { + kfree(pt); + return NULL; + } + + pt->base = 0; + pt->addr = nvkm_memory_addr(pt->memory); + return pt; +} + +void +nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu) +{ + struct nvkm_mmu_ptc *ptc; + list_for_each_entry(ptc, &mmu->ptc.list, head) { + struct nvkm_mmu_pt *pt, *tt; + list_for_each_entry_safe(pt, tt, &ptc->item, head) { + nvkm_memory_unref(&pt->memory); + list_del(&pt->head); + kfree(pt); + } + } +} + +static void +nvkm_mmu_ptc_fini(struct nvkm_mmu *mmu) +{ + struct nvkm_mmu_ptc *ptc, *ptct; + + list_for_each_entry_safe(ptc, ptct, &mmu->ptc.list, head) { + WARN_ON(!list_empty(&ptc->item)); + list_del(&ptc->head); + kfree(ptc); + } +} + +static void +nvkm_mmu_ptc_init(struct nvkm_mmu *mmu) +{ + mutex_init(&mmu->ptc.mutex); + INIT_LIST_HEAD(&mmu->ptc.list); +} + void nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node) { @@ -514,9 +637,13 @@ static void * nvkm_mmu_dtor(struct nvkm_subdev *subdev) { struct nvkm_mmu *mmu = nvkm_mmu(subdev); + void *data = mmu; + if (mmu->func->dtor) - return mmu->func->dtor(mmu); - return mmu; + data = mmu->func->dtor(mmu); + + nvkm_mmu_ptc_fini(mmu); + return data; } static const struct nvkm_subdev_func @@ -535,6 +662,7 @@ nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device, mmu->limit = func->limit; mmu->dma_bits = func->dma_bits; mmu->lpg_shift = func->lpg_shift; + nvkm_mmu_ptc_init(mmu); } int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 9346dae07c682..ea56f78c95d4f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -56,4 +56,19 @@ void gf100_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, u32, u32, dma_addr_t *); void gf100_vm_unmap(struct nvkm_vma *, struct nvkm_memory *, u32, u32); void gf100_vm_flush(struct nvkm_vm *); + +struct nvkm_mmu_pt { + union { + struct nvkm_mmu_ptc *ptc; + }; + struct nvkm_memory *memory; + u16 base; + u64 addr; + struct list_head head; +}; + +void nvkm_mmu_ptc_dump(struct nvkm_mmu *); +struct nvkm_mmu_pt * +nvkm_mmu_ptc_get(struct nvkm_mmu *, u32 size, u32 align, bool zero); +void nvkm_mmu_ptc_put(struct nvkm_mmu *, bool force, struct nvkm_mmu_pt **); #endif From f1280394109a3a3a7ef1c37950d4356b12e75cb9 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 089/151] drm/nouveau/mmu: implement page table sub-allocation GP100 "big" (which is a funny name, when it supports "even bigger") page tables are small enough that we want to be able to suballocate them from a larger block of memory. This builds on the previous page table cache interfaces so that the VMM code doesn't need to know the difference. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 2 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 98 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 2 + 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 61cc4e32e4498..40e21f23b41d6 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -67,7 +67,7 @@ struct nvkm_mmu { struct { struct mutex mutex; struct list_head list; - } ptc; + } ptc, ptp; }; int nv04_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 1e4a92e0068e6..1bdae020057e2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -26,6 +26,85 @@ #include #include +struct nvkm_mmu_ptp { + struct nvkm_mmu_pt *pt; + struct list_head head; + u8 shift; + u16 mask; + u16 free; +}; + +static void +nvkm_mmu_ptp_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt *pt) +{ + const int slot = pt->base >> pt->ptp->shift; + struct nvkm_mmu_ptp *ptp = pt->ptp; + + /* If there were no free slots in the parent allocation before, + * there will be now, so return PTP to the cache. + */ + if (!ptp->free) + list_add(&ptp->head, &mmu->ptp.list); + ptp->free |= BIT(slot); + + /* If there's no more sub-allocations, destroy PTP. */ + if (ptp->free == ptp->mask) { + nvkm_mmu_ptc_put(mmu, force, &ptp->pt); + list_del(&ptp->head); + kfree(ptp); + } + + kfree(pt); +} + +struct nvkm_mmu_pt * +nvkm_mmu_ptp_get(struct nvkm_mmu *mmu, u32 size, bool zero) +{ + struct nvkm_mmu_pt *pt; + struct nvkm_mmu_ptp *ptp; + int slot; + + if (!(pt = kzalloc(sizeof(*pt), GFP_KERNEL))) + return NULL; + + ptp = list_first_entry_or_null(&mmu->ptp.list, typeof(*ptp), head); + if (!ptp) { + /* Need to allocate a new parent to sub-allocate from. */ + if (!(ptp = kmalloc(sizeof(*ptp), GFP_KERNEL))) { + kfree(pt); + return NULL; + } + + ptp->pt = nvkm_mmu_ptc_get(mmu, 0x1000, 0x1000, false); + if (!ptp->pt) { + kfree(ptp); + kfree(pt); + return NULL; + } + + ptp->shift = order_base_2(size); + slot = nvkm_memory_size(ptp->pt->memory) >> ptp->shift; + ptp->mask = (1 << slot) - 1; + ptp->free = ptp->mask; + list_add(&ptp->head, &mmu->ptp.list); + } + pt->ptp = ptp; + pt->sub = true; + + /* Sub-allocate from parent object, removing PTP from cache + * if there's no more free slots left. + */ + slot = __ffs(ptp->free); + ptp->free &= ~BIT(slot); + if (!ptp->free) + list_del(&ptp->head); + + pt->memory = pt->ptp->pt->memory; + pt->base = slot << ptp->shift; + pt->addr = pt->ptp->pt->addr + pt->base; + return pt; +} + struct nvkm_mmu_ptc { struct list_head head; struct list_head item; @@ -59,6 +138,14 @@ nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt) { struct nvkm_mmu_pt *pt = *ppt; if (pt) { + /* Handle sub-allocated page tables. */ + if (pt->sub) { + mutex_lock(&mmu->ptp.mutex); + nvkm_mmu_ptp_put(mmu, force, pt); + mutex_unlock(&mmu->ptp.mutex); + return; + } + /* Either cache or free the object. */ mutex_lock(&mmu->ptc.mutex); if (pt->ptc->refs < 8 /* Heuristic. */ && !force) { @@ -79,6 +166,14 @@ nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero) struct nvkm_mmu_pt *pt; int ret; + /* Sub-allocated page table (ie. GP100 LPT). */ + if (align < 0x1000) { + mutex_lock(&mmu->ptp.mutex); + pt = nvkm_mmu_ptp_get(mmu, align, zero); + mutex_unlock(&mmu->ptp.mutex); + return pt; + } + /* Lookup cache for this page table size. */ mutex_lock(&mmu->ptc.mutex); ptc = nvkm_mmu_ptc_find(mmu, size); @@ -103,6 +198,7 @@ nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero) if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL))) return NULL; pt->ptc = ptc; + pt->sub = false; ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST, size, align, zero, &pt->memory); @@ -147,6 +243,8 @@ nvkm_mmu_ptc_init(struct nvkm_mmu *mmu) { mutex_init(&mmu->ptc.mutex); INIT_LIST_HEAD(&mmu->ptc.list); + mutex_init(&mmu->ptp.mutex); + INIT_LIST_HEAD(&mmu->ptp.list); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index ea56f78c95d4f..0f3bb316022a4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -60,8 +60,10 @@ void gf100_vm_flush(struct nvkm_vm *); struct nvkm_mmu_pt { union { struct nvkm_mmu_ptc *ptc; + struct nvkm_mmu_ptp *ptp; }; struct nvkm_memory *memory; + bool sub; u16 base; u64 addr; struct list_head head; From 806a7335653743a33f476a3705d55bada95b7dfe Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 090/151] drm/nouveau/mmu: implement base for new vm management This is the first chunk of the new VMM code that provides the structures needed to describe a GPU virtual address-space layout, as well as common interfaces to handle VMM creation, and connecting instances to a VMM. The constructor now allocates the PD itself, rather than having the user handle that manually. This won't/can't be used until after all backends have been ported to these interfaces, so a little bit of memory will be wasted on Fermi and newer for a couple of commits in the series. Compatibility has been hacked into the old code to allow each GPU backend to be ported individually. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 2 + drivers/gpu/drm/nouveau/include/nvif/if000c.h | 3 + .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 16 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 76 +++++++-- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 4 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 8 + drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c | 147 ++++++++++++++++++ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 111 +++++++++++++ 9 files changed, 350 insertions(+), 19 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if000c.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index d08da82ba7ed4..7f0839615b0c2 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -14,6 +14,8 @@ #define NVIF_CLASS_SW_NV50 /* if0005.h */ -0x00000006 #define NVIF_CLASS_SW_GF100 /* if0005.h */ -0x00000007 +#define NVIF_CLASS_VMM /* if000c.h */ 0x0000000c + /* the below match nvidia-assigned (either in hw, or sw) class numbers */ #define NV_NULL_CLASS 0x00000030 diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000c.h b/drivers/gpu/drm/nouveau/include/nvif/if000c.h new file mode 100644 index 0000000000000..6363e22f9dcc8 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if000c.h @@ -0,0 +1,3 @@ +#ifndef __NVIF_IF000C_H__ +#define __NVIF_IF000C_H__ +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 40e21f23b41d6..8ff71e7f2f5b3 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -26,20 +26,28 @@ struct nvkm_vma { }; struct nvkm_vm { + const struct nvkm_vmm_func *func; struct nvkm_mmu *mmu; - + const char *name; + struct kref kref; struct mutex mutex; + + u64 start; + u64 limit; + + struct nvkm_vmm_pt *pd; + u16 pd_offset; + struct list_head join; + struct nvkm_mm mm; struct kref refcount; - struct list_head pgd_list; - atomic_t engref[NVKM_SUBDEV_NR]; - struct nvkm_vm_pgt *pgt; u32 fpde; u32 lpde; bool bootstrapped; + atomic_t engref[NVKM_SUBDEV_NR]; }; int nvkm_vm_new(struct nvkm_device *, u64 offset, u64 length, u64 mm_offset, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index e84ef36eb7e74..c786a9f90776d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -11,3 +11,5 @@ nvkm-y += nvkm/subdev/mmu/gm200.o nvkm-y += nvkm/subdev/mmu/gm20b.o nvkm-y += nvkm/subdev/mmu/gp100.o nvkm-y += nvkm/subdev/mmu/gp10b.o + +nvkm-y += nvkm/subdev/mmu/vmm.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 1bdae020057e2..d55ec0e85dca7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -22,6 +22,7 @@ * Authors: Ben Skeggs */ #include "priv.h" +#include "vmm.h" #include #include @@ -584,22 +585,14 @@ nvkm_vm_boot(struct nvkm_vm *vm, u64 size) return ret; } -int -nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, - u32 block, struct lock_class_key *key, struct nvkm_vm **pvm) +static int +nvkm_vm_legacy(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, + u32 block, struct nvkm_vm *vm) { - static struct lock_class_key _key; - struct nvkm_vm *vm; u64 mm_length = (offset + length) - mm_offset; int ret; - vm = kzalloc(sizeof(*vm), GFP_KERNEL); - if (!vm) - return -ENOMEM; - - __mutex_init(&vm->mutex, "&vm->mutex", key ? key : &_key); INIT_LIST_HEAD(&vm->pgd_list); - vm->mmu = mmu; kref_init(&vm->refcount); vm->fpde = offset >> (mmu->func->pgt_bits + 12); vm->lpde = (offset + length - 1) >> (mmu->func->pgt_bits + 12); @@ -610,16 +603,41 @@ nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, return -ENOMEM; } + if (block > length) + block = length; + ret = nvkm_mm_init(&vm->mm, 0, mm_offset >> 12, mm_length >> 12, block >> 12); if (ret) { vfree(vm->pgt); + return ret; + } + + return 0; +} + +int +nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, + u32 block, struct lock_class_key *key, struct nvkm_vm **pvm) +{ + static struct lock_class_key _key; + struct nvkm_vm *vm; + int ret; + + vm = kzalloc(sizeof(*vm), GFP_KERNEL); + if (!vm) + return -ENOMEM; + + __mutex_init(&vm->mutex, "&vm->mutex", key ? key : &_key); + vm->mmu = mmu; + + ret = nvkm_vm_legacy(mmu, offset, length, mm_offset, block, vm); + if (ret) { kfree(vm); return ret; } *pvm = vm; - return 0; } @@ -628,8 +646,29 @@ nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset, struct lock_class_key *key, struct nvkm_vm **pvm) { struct nvkm_mmu *mmu = device->mmu; + + *pvm = NULL; + if (mmu->func->vmm.ctor) { + int ret = mmu->func->vmm.ctor(mmu, mm_offset, + offset + length - mm_offset, + NULL, 0, key, "legacy", pvm); + if (ret) { + nvkm_vm_ref(NULL, pvm, NULL); + return ret; + } + + ret = nvkm_vm_legacy(mmu, offset, length, mm_offset, + (*pvm)->func->page_block ? + (*pvm)->func->page_block : 4096, *pvm); + if (ret) + nvkm_vm_ref(NULL, pvm, NULL); + + return ret; + } + if (!mmu->func->create) return -EINVAL; + return mmu->func->create(mmu, offset, length, mm_offset, key, pvm); } @@ -688,6 +727,9 @@ nvkm_vm_del(struct kref *kref) nvkm_mm_fini(&vm->mm); vfree(vm->pgt); + + if (vm->func) + nvkm_vmm_dtor(vm); kfree(vm); } @@ -717,8 +759,17 @@ static int nvkm_mmu_oneinit(struct nvkm_subdev *subdev) { struct nvkm_mmu *mmu = nvkm_mmu(subdev); + + if (mmu->func->vmm.global) { + int ret = nvkm_vm_new(subdev->device, 0, mmu->limit, 0, + NULL, &mmu->vmm); + if (ret) + return ret; + } + if (mmu->func->oneinit) return mmu->func->oneinit(mmu); + return 0; } @@ -739,6 +790,7 @@ nvkm_mmu_dtor(struct nvkm_subdev *subdev) if (mmu->func->dtor) data = mmu->func->dtor(mmu); + nvkm_vm_ref(NULL, &mmu->vmm, NULL); nvkm_mmu_ptc_fini(mmu); return data; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index 43e3b4a77583f..5bf757541f7be 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -105,10 +105,8 @@ nv04_mmu_dtor(struct nvkm_mmu *base) { struct nv04_mmu *mmu = nv04_mmu(base); struct nvkm_device *device = mmu->base.subdev.device; - if (mmu->base.vmm) { + if (mmu->base.vmm) nvkm_memory_unref(&mmu->base.vmm->pgt[0].mem[0]); - nvkm_vm_ref(NULL, &mmu->base.vmm, NULL); - } if (mmu->nullp) { dma_free_coherent(device->dev, 16 * 1024, mmu->nullp, mmu->null); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 0f3bb316022a4..ee0a8416df19f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -32,6 +32,14 @@ struct nvkm_mmu_func { void (*unmap)(struct nvkm_vma *, struct nvkm_memory *pgt, u32 pte, u32 cnt); void (*flush)(struct nvkm_vm *); + + struct { + struct nvkm_sclass base; + int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *, + const char *name, struct nvkm_vmm **); + bool global; + } vmm; }; int nvkm_vm_create(struct nvkm_mmu *, u64, u64, u64, u32, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c new file mode 100644 index 0000000000000..93c8398b36a5d --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -0,0 +1,147 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#define NVKM_VMM_LEVELS_MAX 5 +#include "vmm.h" + +static void +nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt) +{ + struct nvkm_vmm_pt *pgt = *ppgt; + if (pgt) { + kvfree(pgt->pde); + kfree(pgt); + *ppgt = NULL; + } +} + + +static struct nvkm_vmm_pt * +nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse, + const struct nvkm_vmm_page *page) +{ + const u32 pten = 1 << desc->bits; + struct nvkm_vmm_pt *pgt; + u32 lpte = 0; + + if (desc->type > PGT) { + if (desc->type == SPT) { + const struct nvkm_vmm_desc *pair = page[-1].desc; + lpte = pten >> (desc->bits - pair->bits); + } else { + lpte = pten; + } + } + + if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL))) + return NULL; + pgt->page = page ? page->shift : 0; + pgt->sparse = sparse; + + if (desc->type == PGD) { + pgt->pde = kvzalloc(sizeof(*pgt->pde) * pten, GFP_KERNEL); + if (!pgt->pde) { + kfree(pgt); + return NULL; + } + } + + return pgt; +} + +void +nvkm_vmm_dtor(struct nvkm_vmm *vmm) +{ + if (vmm->pd) { + nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]); + nvkm_vmm_pt_del(&vmm->pd); + } +} + +int +nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, + u32 pd_header, u64 addr, u64 size, struct lock_class_key *key, + const char *name, struct nvkm_vmm *vmm) +{ + static struct lock_class_key _key; + const struct nvkm_vmm_page *page = func->page; + const struct nvkm_vmm_desc *desc; + int levels, bits = 0; + + vmm->func = func; + vmm->mmu = mmu; + vmm->name = name; + kref_init(&vmm->kref); + + __mutex_init(&vmm->mutex, "&vmm->mutex", key ? key : &_key); + + /* Locate the smallest page size supported by the backend, it will + * have the the deepest nesting of page tables. + */ + while (page[1].shift) + page++; + + /* Locate the structure that describes the layout of the top-level + * page table, and determine the number of valid bits in a virtual + * address. + */ + for (levels = 0, desc = page->desc; desc->bits; desc++, levels++) + bits += desc->bits; + bits += page->shift; + desc--; + + if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX)) + return -EINVAL; + + vmm->start = addr; + vmm->limit = size ? (addr + size) : (1ULL << bits); + if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits)) + return -EINVAL; + + /* Allocate top-level page table. */ + vmm->pd = nvkm_vmm_pt_new(desc, false, NULL); + if (!vmm->pd) + return -ENOMEM; + vmm->pd->refs[0] = 1; + INIT_LIST_HEAD(&vmm->join); + + /* ... and the GPU storage for it, except on Tesla-class GPUs that + * have the PD embedded in the instance structure. + */ + if (desc->size && mmu->func->vmm.global) { + const u32 size = pd_header + desc->size * (1 << desc->bits); + vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true); + if (!vmm->pd->pt[0]) + return -ENOMEM; + } + + return 0; +} + +int +nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, + u32 hdr, u64 addr, u64 size, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) +{ + if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL))) + return -ENOMEM; + return nvkm_vmm_ctor(func, mmu, hdr, addr, size, key, name, *pvmm); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h new file mode 100644 index 0000000000000..46cc194b2003d --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -0,0 +1,111 @@ +#ifndef __NVKM_VMM_H__ +#define __NVKM_VMM_H__ +#include "priv.h" +#include + +struct nvkm_vmm_pt { + /* Some GPUs have a mapping level with a dual page tables to + * support large and small pages in the same address-range. + * + * We track the state of both page tables in one place, which + * is why there's multiple PT pointers/refcounts here. + */ + struct nvkm_mmu_pt *pt[2]; + u32 refs[2]; + + /* Page size handled by this PT. + * + * Tesla backend needs to know this when writinge PDEs, + * otherwise unnecessary. + */ + u8 page; + + /* Entire page table sparse. + * + * Used to propagate sparseness to child page tables. + */ + bool sparse:1; + + /* Tracking for page directories. + * + * The array is indexed by PDE, and will either point to the + * child page table, or indicate the PDE is marked as sparse. + **/ +#define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde) +#define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde) +#define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY) + struct nvkm_vmm_pt **pde; + + /* Tracking for dual page tables. + * + * There's one entry for each LPTE, keeping track of whether + * there are valid SPTEs in the same address-range. + * + * This information is used to manage LPTE state transitions. + */ +#define NVKM_VMM_PTE_SPARSE 0x80 +#define NVKM_VMM_PTE_VALID 0x40 +#define NVKM_VMM_PTE_SPTES 0x3f + u8 pte[]; +}; + +struct nvkm_vmm_desc_func { +}; + +struct nvkm_vmm_desc { + enum { + PGD, + PGT, + SPT, + LPT, + } type; + u8 bits; /* VMA bits covered by PT. */ + u8 size; /* Bytes-per-PTE. */ + u32 align; /* PT address alignment. */ + const struct nvkm_vmm_desc_func *func; +}; + +struct nvkm_vmm_page { + u8 shift; + const struct nvkm_vmm_desc *desc; +#define NVKM_VMM_PAGE_SPARSE 0x01 +#define NVKM_VMM_PAGE_VRAM 0x02 +#define NVKM_VMM_PAGE_HOST 0x04 +#define NVKM_VMM_PAGE_COMP 0x08 +#define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE) +#define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM) +#define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM) +#define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST) +#define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST) +#define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST) +#define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST) +#define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP) +#define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP) +#define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP) +#define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP) + u8 type; +}; + +struct nvkm_vmm_func { + int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst); + void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst); + + u64 page_block; + const struct nvkm_vmm_page page[]; +}; + +int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, + u32 pd_header, u64 addr, u64 size, struct lock_class_key *, + const char *name, struct nvkm_vmm **); +int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, + u32 pd_header, u64 addr, u64 size, struct lock_class_key *, + const char *name, struct nvkm_vmm *); +void nvkm_vmm_dtor(struct nvkm_vmm *); + +struct nvkm_vmm_user { + struct nvkm_sclass base; + int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, void *args, u32 argc, + struct lock_class_key *, const char *name, + struct nvkm_vmm **); +}; +#endif From 5b17f3624efa5473e78ff9b09177e3db0806bb27 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 091/151] drm/nouveau/mmu/nv04: implement vmm on top of new base Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 3 +- drivers/gpu/drm/nouveau/include/nvif/if000d.h | 8 ++ .../drm/nouveau/nvkm/engine/dma/usernv04.c | 5 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 33 ++------ .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 4 +- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 11 ++- .../gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c | 84 +++++++++++++++++++ 8 files changed, 113 insertions(+), 36 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if000d.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 7f0839615b0c2..d6fc025c912e1 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -14,7 +14,8 @@ #define NVIF_CLASS_SW_NV50 /* if0005.h */ -0x00000006 #define NVIF_CLASS_SW_GF100 /* if0005.h */ -0x00000007 -#define NVIF_CLASS_VMM /* if000c.h */ 0x0000000c +#define NVIF_CLASS_VMM /* if000c.h */ 0x8000000c +#define NVIF_CLASS_VMM_NV04 /* if000d.h */ 0x8000000d /* the below match nvidia-assigned (either in hw, or sw) class numbers */ #define NV_NULL_CLASS 0x00000030 diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000d.h b/drivers/gpu/drm/nouveau/include/nvif/if000d.h new file mode 100644 index 0000000000000..0f94eb5005b15 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if000d.h @@ -0,0 +1,8 @@ +#ifndef __NVIF_IF000D_H__ +#define __NVIF_IF000D_H__ +#include "if000c.h" + +struct nv04_vmm_vn { + /* nvif_vmm_vX ... */ +}; +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c index acc45ce5f93ec..49ef7e57aad41 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c @@ -26,7 +26,7 @@ #include #include -#include +#include #include @@ -49,7 +49,8 @@ nv04_dmaobj_bind(struct nvkm_dmaobj *base, struct nvkm_gpuobj *parent, int ret; if (dmaobj->clone) { - struct nvkm_memory *pgt = device->mmu->vmm->pgt[0].mem[0]; + struct nvkm_memory *pgt = + device->mmu->vmm->pd->pt[0]->memory; if (!dmaobj->base.start) return nvkm_gpuobj_wrap(pgt, pgpuobj); nvkm_kmap(pgt); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index c786a9f90776d..64684a7440001 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -13,3 +13,4 @@ nvkm-y += nvkm/subdev/mmu/gp100.o nvkm-y += nvkm/subdev/mmu/gp10b.o nvkm-y += nvkm/subdev/mmu/vmm.o +nvkm-y += nvkm/subdev/mmu/vmmnv04.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index 5bf757541f7be..fc4390de345e2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -22,8 +22,9 @@ * Authors: Ben Skeggs */ #include "nv04.h" +#include "vmm.h" -#include +#include #define NV04_PDMA_SIZE (128 * 1024 * 1024) #define NV04_PDMA_PAGE ( 4 * 1024) @@ -73,30 +74,10 @@ nv04_vm_flush(struct nvkm_vm *vm) ******************************************************************************/ static int -nv04_mmu_oneinit(struct nvkm_mmu *base) +nv04_mmu_oneinit(struct nvkm_mmu *mmu) { - struct nv04_mmu *mmu = nv04_mmu(base); - struct nvkm_device *device = mmu->base.subdev.device; - struct nvkm_memory *dma; - int ret; - - ret = nvkm_vm_create(&mmu->base, 0, NV04_PDMA_SIZE, 0, 4096, NULL, - &mmu->base.vmm); - if (ret) - return ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, - (NV04_PDMA_SIZE / NV04_PDMA_PAGE) * 4 + 8, - 16, true, &dma); - mmu->base.vmm->pgt[0].mem[0] = dma; - mmu->base.vmm->pgt[0].refcount[0] = 1; - if (ret) - return ret; - - nvkm_kmap(dma); - nvkm_wo32(dma, 0x00000, 0x0002103d); /* PCI, RW, PT, !LN */ - nvkm_wo32(dma, 0x00004, NV04_PDMA_SIZE - 1); - nvkm_done(dma); + mmu->vmm->pgt[0].mem[0] = mmu->vmm->pd->pt[0]->memory; + mmu->vmm->pgt[0].refcount[0] = 1; return 0; } @@ -129,7 +110,6 @@ nv04_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device, const struct nvkm_mmu_func nv04_mmu = { .oneinit = nv04_mmu_oneinit, - .dtor = nv04_mmu_dtor, .limit = NV04_PDMA_SIZE, .dma_bits = 32, .pgt_bits = 32 - 12, @@ -138,10 +118,11 @@ nv04_mmu = { .map_sg = nv04_vm_map_sg, .unmap = nv04_vm_unmap, .flush = nv04_vm_flush, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv04_vmm_new, true }, }; int nv04_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { - return nv04_mmu_new_(&nv04_mmu, device, index, pmmu); + return nvkm_mmu_new_(&nv04_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index ee0a8416df19f..3d8d7e103f208 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -34,7 +34,7 @@ struct nvkm_mmu_func { void (*flush)(struct nvkm_vm *); struct { - struct nvkm_sclass base; + struct nvkm_sclass user; int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *, const char *name, struct nvkm_vmm **); @@ -45,6 +45,8 @@ struct nvkm_mmu_func { int nvkm_vm_create(struct nvkm_mmu *, u64, u64, u64, u32, struct lock_class_key *, struct nvkm_vm **); +extern const struct nvkm_mmu_func nv04_mmu; + int nv50_vm_create(struct nvkm_mmu *, u64, u64, u64, struct lock_class_key *, struct nvkm_vm **); void nv50_vm_map_pgt(struct nvkm_gpuobj *, u32, struct nvkm_memory **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 46cc194b2003d..093ddb497c930 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -102,10 +102,9 @@ int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, const char *name, struct nvkm_vmm *); void nvkm_vmm_dtor(struct nvkm_vmm *); -struct nvkm_vmm_user { - struct nvkm_sclass base; - int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, void *args, u32 argc, - struct lock_class_key *, const char *name, - struct nvkm_vmm **); -}; +int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, + u64, u64, void *, u32, struct lock_class_key *, + const char *, struct nvkm_vmm **); +int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c new file mode 100644 index 0000000000000..1f874b351a39e --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c @@ -0,0 +1,84 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +#include +#include + +static const struct nvkm_vmm_desc_func +nv04_vmm_desc_pgt = { +}; + +static const struct nvkm_vmm_desc +nv04_vmm_desc_12[] = { + { PGT, 15, 4, 0x1000, &nv04_vmm_desc_pgt }, + {} +}; + +static const struct nvkm_vmm_func +nv04_vmm = { + .page = { + { 12, &nv04_vmm_desc_12[0], NVKM_VMM_PAGE_HOST }, + {} + } +}; + +int +nv04_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, + u32 pd_header, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + union { + struct nv04_vmm_vn vn; + } *args = argv; + int ret; + + ret = nvkm_vmm_new_(func, mmu, pd_header, addr, size, key, name, pvmm); + if (ret) + return ret; + + return nvif_unvers(-ENOSYS, &argv, &argc, args->vn); +} + +int +nv04_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + struct nvkm_vmm *vmm; + struct nvkm_memory *mem; + int ret; + + ret = nv04_vmm_new_(&nv04_vmm, mmu, 8, addr, size, + argv, argc, key, name, &vmm); + *pvmm = vmm; + if (ret) + return ret; + + mem = vmm->pd->pt[0]->memory; + nvkm_kmap(mem); + nvkm_wo32(mem, 0x00000, 0x0002103d); /* PCI, RW, PT, !LN */ + nvkm_wo32(mem, 0x00004, vmm->limit - 1); + nvkm_done(mem); + return 0; +} From 77783435c31182166c4679ad75358e1756d1e5da Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 092/151] drm/nouveau/mmu/nv41: implement vmm on top of new base Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c | 44 ++++++----------- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 3 ++ .../gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c | 49 +++++++++++++++++++ 4 files changed, 69 insertions(+), 28 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 64684a7440001..4b3e403a76f34 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -14,3 +14,4 @@ nvkm-y += nvkm/subdev/mmu/gp10b.o nvkm-y += nvkm/subdev/mmu/vmm.o nvkm-y += nvkm/subdev/mmu/vmmnv04.o +nvkm-y += nvkm/subdev/mmu/vmmnv41.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c index 9b5375c587a7f..cb037f4a1780b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c @@ -21,12 +21,13 @@ * * Authors: Ben Skeggs */ -#include "nv04.h" +#include "vmm.h" -#include #include #include +#include + #define NV41_GART_SIZE (512 * 1024 * 1024) #define NV41_GART_PAGE ( 4 * 1024) @@ -68,17 +69,17 @@ nv41_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) static void nv41_vm_flush(struct nvkm_vm *vm) { - struct nv04_mmu *mmu = nv04_mmu(vm->mmu); - struct nvkm_device *device = mmu->base.subdev.device; + struct nvkm_subdev *subdev = &vm->mmu->subdev; + struct nvkm_device *device = subdev->device; - mutex_lock(&mmu->base.subdev.mutex); + mutex_lock(&subdev->mutex); nvkm_wr32(device, 0x100810, 0x00000022); nvkm_msec(device, 2000, if (nvkm_rd32(device, 0x100810) & 0x00000020) break; ); nvkm_wr32(device, 0x100810, 0x00000000); - mutex_unlock(&mmu->base.subdev.mutex); + mutex_unlock(&subdev->mutex); } /******************************************************************************* @@ -86,38 +87,24 @@ nv41_vm_flush(struct nvkm_vm *vm) ******************************************************************************/ static int -nv41_mmu_oneinit(struct nvkm_mmu *base) +nv41_mmu_oneinit(struct nvkm_mmu *mmu) { - struct nv04_mmu *mmu = nv04_mmu(base); - struct nvkm_device *device = mmu->base.subdev.device; - int ret; - - ret = nvkm_vm_create(&mmu->base, 0, NV41_GART_SIZE, 0, 4096, NULL, - &mmu->base.vmm); - if (ret) - return ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, - (NV41_GART_SIZE / NV41_GART_PAGE) * 4, 16, true, - &mmu->base.vmm->pgt[0].mem[0]); - mmu->base.vmm->pgt[0].refcount[0] = 1; - return ret; + mmu->vmm->pgt[0].mem[0] = mmu->vmm->pd->pt[0]->memory; + mmu->vmm->pgt[0].refcount[0] = 1; + return 0; } static void -nv41_mmu_init(struct nvkm_mmu *base) +nv41_mmu_init(struct nvkm_mmu *mmu) { - struct nv04_mmu *mmu = nv04_mmu(base); - struct nvkm_device *device = mmu->base.subdev.device; - struct nvkm_memory *dma = mmu->base.vmm->pgt[0].mem[0]; - nvkm_wr32(device, 0x100800, 0x00000002 | nvkm_memory_addr(dma)); + struct nvkm_device *device = mmu->subdev.device; + nvkm_wr32(device, 0x100800, 0x00000002 | mmu->vmm->pd->pt[0]->addr); nvkm_mask(device, 0x10008c, 0x00000100, 0x00000100); nvkm_wr32(device, 0x100820, 0x00000000); } static const struct nvkm_mmu_func nv41_mmu = { - .dtor = nv04_mmu_dtor, .oneinit = nv41_mmu_oneinit, .init = nv41_mmu_init, .limit = NV41_GART_SIZE, @@ -128,6 +115,7 @@ nv41_mmu = { .map_sg = nv41_vm_map_sg, .unmap = nv41_vm_unmap, .flush = nv41_vm_flush, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv41_vmm_new, true }, }; int @@ -137,5 +125,5 @@ nv41_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) !nvkm_boolopt(device->cfgopt, "NvPCIE", true)) return nv04_mmu_new(device, index, pmmu); - return nv04_mmu_new_(&nv41_mmu, device, index, pmmu); + return nvkm_mmu_new_(&nv41_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 093ddb497c930..16d4a4c7aae4c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -105,6 +105,9 @@ void nvkm_vmm_dtor(struct nvkm_vmm *); int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); + int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); +int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c new file mode 100644 index 0000000000000..8a624d7c363ae --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c @@ -0,0 +1,49 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +static const struct nvkm_vmm_desc_func +nv41_vmm_desc_pgt = { +}; + +static const struct nvkm_vmm_desc +nv41_vmm_desc_12[] = { + { PGT, 17, 4, 0x1000, &nv41_vmm_desc_pgt }, + {} +}; + +static const struct nvkm_vmm_func +nv41_vmm = { + .page = { + { 12, &nv41_vmm_desc_12[0], NVKM_VMM_PAGE_HOST }, + {} + } +}; + +int +nv41_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + return nv04_vmm_new_(&nv41_vmm, mmu, 0, addr, size, + argv, argc, key, name, pvmm); +} From 03b0ba7b545ba0c5b19fedb14a771a3517a1328e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 093/151] drm/nouveau/mmu/nv44: implement vmm on top of new base Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 3 + .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 5 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 27 -------- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h | 17 ----- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c | 61 ++++++----------- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c | 5 ++ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c | 65 +++++++++++++++++++ 10 files changed, 96 insertions(+), 91 deletions(-) delete mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 8ff71e7f2f5b3..f171e7a82f572 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -48,6 +48,9 @@ struct nvkm_vm { bool bootstrapped; atomic_t engref[NVKM_SUBDEV_NR]; + + dma_addr_t null; + void *nullp; }; int nvkm_vm_new(struct nvkm_device *, u64 offset, u64 length, u64 mm_offset, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 4b3e403a76f34..f8a46f9a37a08 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -15,3 +15,4 @@ nvkm-y += nvkm/subdev/mmu/gp10b.o nvkm-y += nvkm/subdev/mmu/vmm.o nvkm-y += nvkm/subdev/mmu/vmmnv04.o nvkm-y += nvkm/subdev/mmu/vmmnv41.o +nvkm-y += nvkm/subdev/mmu/vmmnv44.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index d55ec0e85dca7..0b4cb7b6a81fa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -786,14 +786,11 @@ static void * nvkm_mmu_dtor(struct nvkm_subdev *subdev) { struct nvkm_mmu *mmu = nvkm_mmu(subdev); - void *data = mmu; - if (mmu->func->dtor) - data = mmu->func->dtor(mmu); nvkm_vm_ref(NULL, &mmu->vmm, NULL); nvkm_mmu_ptc_fini(mmu); - return data; + return mmu; } static const struct nvkm_subdev_func diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index fc4390de345e2..3e60364317540 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -21,7 +21,6 @@ * * Authors: Ben Skeggs */ -#include "nv04.h" #include "vmm.h" #include @@ -81,32 +80,6 @@ nv04_mmu_oneinit(struct nvkm_mmu *mmu) return 0; } -void * -nv04_mmu_dtor(struct nvkm_mmu *base) -{ - struct nv04_mmu *mmu = nv04_mmu(base); - struct nvkm_device *device = mmu->base.subdev.device; - if (mmu->base.vmm) - nvkm_memory_unref(&mmu->base.vmm->pgt[0].mem[0]); - if (mmu->nullp) { - dma_free_coherent(device->dev, 16 * 1024, - mmu->nullp, mmu->null); - } - return mmu; -} - -int -nv04_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device, - int index, struct nvkm_mmu **pmmu) -{ - struct nv04_mmu *mmu; - if (!(mmu = kzalloc(sizeof(*mmu), GFP_KERNEL))) - return -ENOMEM; - *pmmu = &mmu->base; - nvkm_mmu_ctor(func, device, index, &mmu->base); - return 0; -} - const struct nvkm_mmu_func nv04_mmu = { .oneinit = nv04_mmu_oneinit, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h deleted file mode 100644 index 6b8f14c6db3dc..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef __NV04_MMU_PRIV__ -#define __NV04_MMU_PRIV__ -#define nv04_mmu(p) container_of((p), struct nv04_mmu, base) -#include "priv.h" - -struct nv04_mmu { - struct nvkm_mmu base; - dma_addr_t null; - void *nullp; -}; - -int nv04_mmu_new_(const struct nvkm_mmu_func *, struct nvkm_device *, - int index, struct nvkm_mmu **); -void *nv04_mmu_dtor(struct nvkm_mmu *); - -extern const struct nvkm_mmu_func nv04_mmu; -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c index cc97500a09010..48ca0cdf2acfa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c @@ -21,12 +21,13 @@ * * Authors: Ben Skeggs */ -#include "nv04.h" +#include "vmm.h" -#include #include #include +#include + #define NV44_GART_SIZE (512 * 1024 * 1024) #define NV44_GART_PAGE ( 4 * 1024) @@ -84,7 +85,6 @@ static void nv44_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list) { - struct nv04_mmu *mmu = nv04_mmu(vma->vm->mmu); u32 tmp[4]; int i; @@ -92,7 +92,7 @@ nv44_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, if (pte & 3) { u32 max = 4 - (pte & 3); u32 part = (cnt > max) ? max : cnt; - nv44_vm_fill(pgt, mmu->null, list, pte, part); + nv44_vm_fill(pgt, vma->vm->null, list, pte, part); pte += part; list += part; cnt -= part; @@ -109,20 +109,18 @@ nv44_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, } if (cnt) - nv44_vm_fill(pgt, mmu->null, list, pte, cnt); + nv44_vm_fill(pgt, vma->vm->null, list, pte, cnt); nvkm_done(pgt); } static void nv44_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) { - struct nv04_mmu *mmu = nv04_mmu(vma->vm->mmu); - nvkm_kmap(pgt); if (pte & 3) { u32 max = 4 - (pte & 3); u32 part = (cnt > max) ? max : cnt; - nv44_vm_fill(pgt, mmu->null, NULL, pte, part); + nv44_vm_fill(pgt, vma->vm->null, NULL, pte, part); pte += part; cnt -= part; } @@ -136,16 +134,15 @@ nv44_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) } if (cnt) - nv44_vm_fill(pgt, mmu->null, NULL, pte, cnt); + nv44_vm_fill(pgt, vma->vm->null, NULL, pte, cnt); nvkm_done(pgt); } static void nv44_vm_flush(struct nvkm_vm *vm) { - struct nv04_mmu *mmu = nv04_mmu(vm->mmu); - struct nvkm_device *device = mmu->base.subdev.device; - nvkm_wr32(device, 0x100814, mmu->base.limit - NV44_GART_PAGE); + struct nvkm_device *device = vm->mmu->subdev.device; + nvkm_wr32(device, 0x100814, vm->mmu->limit - NV44_GART_PAGE); nvkm_wr32(device, 0x100808, 0x00000020); nvkm_msec(device, 2000, if (nvkm_rd32(device, 0x100808) & 0x00000001) @@ -159,38 +156,18 @@ nv44_vm_flush(struct nvkm_vm *vm) ******************************************************************************/ static int -nv44_mmu_oneinit(struct nvkm_mmu *base) +nv44_mmu_oneinit(struct nvkm_mmu *mmu) { - struct nv04_mmu *mmu = nv04_mmu(base); - struct nvkm_device *device = mmu->base.subdev.device; - int ret; - - mmu->nullp = dma_alloc_coherent(device->dev, 16 * 1024, - &mmu->null, GFP_KERNEL); - if (!mmu->nullp) { - nvkm_warn(&mmu->base.subdev, "unable to allocate dummy pages\n"); - mmu->null = 0; - } - - ret = nvkm_vm_create(&mmu->base, 0, NV44_GART_SIZE, 0, 4096, NULL, - &mmu->base.vmm); - if (ret) - return ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, - (NV44_GART_SIZE / NV44_GART_PAGE) * 4, - 512 * 1024, true, - &mmu->base.vmm->pgt[0].mem[0]); - mmu->base.vmm->pgt[0].refcount[0] = 1; - return ret; + mmu->vmm->pgt[0].mem[0] = mmu->vmm->pd->pt[0]->memory; + mmu->vmm->pgt[0].refcount[0] = 1; + return 0; } static void -nv44_mmu_init(struct nvkm_mmu *base) +nv44_mmu_init(struct nvkm_mmu *mmu) { - struct nv04_mmu *mmu = nv04_mmu(base); - struct nvkm_device *device = mmu->base.subdev.device; - struct nvkm_memory *gart = mmu->base.vmm->pgt[0].mem[0]; + struct nvkm_device *device = mmu->subdev.device; + struct nvkm_memory *gart = mmu->vmm->pgt[0].mem[0]; u32 addr; /* calculate vram address of this PRAMIN block, object must be @@ -201,7 +178,7 @@ nv44_mmu_init(struct nvkm_mmu *base) addr -= ((nvkm_memory_addr(gart) >> 19) + 1) << 19; nvkm_wr32(device, 0x100850, 0x80000000); - nvkm_wr32(device, 0x100818, mmu->null); + nvkm_wr32(device, 0x100818, mmu->vmm->null); nvkm_wr32(device, 0x100804, NV44_GART_SIZE); nvkm_wr32(device, 0x100850, 0x00008000); nvkm_mask(device, 0x10008c, 0x00000200, 0x00000200); @@ -212,7 +189,6 @@ nv44_mmu_init(struct nvkm_mmu *base) static const struct nvkm_mmu_func nv44_mmu = { - .dtor = nv04_mmu_dtor, .oneinit = nv44_mmu_oneinit, .init = nv44_mmu_init, .limit = NV44_GART_SIZE, @@ -223,6 +199,7 @@ nv44_mmu = { .map_sg = nv44_vm_map_sg, .unmap = nv44_vm_unmap, .flush = nv44_vm_flush, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv44_vmm_new, true }, }; int @@ -232,5 +209,5 @@ nv44_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) !nvkm_boolopt(device->cfgopt, "NvPCIE", true)) return nv04_mmu_new(device, index, pmmu); - return nv04_mmu_new_(&nv44_mmu, device, index, pmmu); + return nvkm_mmu_new_(&nv44_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 3d8d7e103f208..bbb9c9cca27db 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -9,7 +9,6 @@ int nvkm_mmu_new_(const struct nvkm_mmu_func *, struct nvkm_device *, int index, struct nvkm_mmu **); struct nvkm_mmu_func { - void *(*dtor)(struct nvkm_mmu *); int (*oneinit)(struct nvkm_mmu *); void (*init)(struct nvkm_mmu *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c index 93c8398b36a5d..25e86ce3d29c8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -70,6 +70,11 @@ nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse, void nvkm_vmm_dtor(struct nvkm_vmm *vmm) { + if (vmm->nullp) { + dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024, + vmm->nullp, vmm->null); + } + if (vmm->pd) { nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]); nvkm_vmm_pt_del(&vmm->pd); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 16d4a4c7aae4c..bade0237c1021 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -110,4 +110,6 @@ int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); +int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c new file mode 100644 index 0000000000000..2b5704269ac99 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c @@ -0,0 +1,65 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +static const struct nvkm_vmm_desc_func +nv44_vmm_desc_pgt = { +}; + +static const struct nvkm_vmm_desc +nv44_vmm_desc_12[] = { + { PGT, 17, 4, 0x80000, &nv44_vmm_desc_pgt }, + {} +}; + +static const struct nvkm_vmm_func +nv44_vmm = { + .page = { + { 12, &nv44_vmm_desc_12[0], NVKM_VMM_PAGE_HOST }, + {} + } +}; + +int +nv44_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + struct nvkm_subdev *subdev = &mmu->subdev; + struct nvkm_vmm *vmm; + int ret; + + ret = nv04_vmm_new_(&nv44_vmm, mmu, 0, addr, size, + argv, argc, key, name, &vmm); + *pvmm = vmm; + if (ret) + return ret; + + vmm->nullp = dma_alloc_coherent(subdev->device->dev, 16 * 1024, + &vmm->null, GFP_KERNEL); + if (!vmm->nullp) { + nvkm_warn(subdev, "unable to allocate dummy pages\n"); + vmm->null = 0; + } + + return 0; +} From 9f6219fde7457df7a982174d496a012f4f42e776 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 094/151] drm/nouveau/mmu/nv50,g84: implement vmm on top of new base Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 1 + drivers/gpu/drm/nouveau/include/nvif/if500d.h | 8 ++ .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 1 - .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c | 5 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c | 6 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 9 ++ .../gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c | 94 +++++++++++++++++++ 10 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if500d.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index d6fc025c912e1..9c02927b4ca95 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -16,6 +16,7 @@ #define NVIF_CLASS_VMM /* if000c.h */ 0x8000000c #define NVIF_CLASS_VMM_NV04 /* if000d.h */ 0x8000000d +#define NVIF_CLASS_VMM_NV50 /* if500d.h */ 0x8000500d /* the below match nvidia-assigned (either in hw, or sw) class numbers */ #define NV_NULL_CLASS 0x00000030 diff --git a/drivers/gpu/drm/nouveau/include/nvif/if500d.h b/drivers/gpu/drm/nouveau/include/nvif/if500d.h new file mode 100644 index 0000000000000..6a0f47770745f --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if500d.h @@ -0,0 +1,8 @@ +#ifndef __NVIF_IF500D_H__ +#define __NVIF_IF500D_H__ +#include "if000c.h" + +struct nv50_vmm_vn { + /* nvif_vmm_vX ... */ +}; +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index f171e7a82f572..71d4863378f4b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -36,7 +36,6 @@ struct nvkm_vm { u64 limit; struct nvkm_vmm_pt *pd; - u16 pd_offset; struct list_head join; struct nvkm_mm mm; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index f8a46f9a37a08..2ec7e50568ab2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -16,3 +16,4 @@ nvkm-y += nvkm/subdev/mmu/vmm.o nvkm-y += nvkm/subdev/mmu/vmmnv04.o nvkm-y += nvkm/subdev/mmu/vmmnv41.o nvkm-y += nvkm/subdev/mmu/vmmnv44.o +nvkm-y += nvkm/subdev/mmu/vmmnv50.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 0b4cb7b6a81fa..1965a52ebe605 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -727,7 +727,6 @@ nvkm_vm_del(struct kref *kref) nvkm_mm_fini(&vm->mm); vfree(vm->pgt); - if (vm->func) nvkm_vmm_dtor(vm); kfree(vm); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c index a1aca162a8cfa..bb15ae97f4217 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c @@ -19,7 +19,9 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include "priv.h" +#include "vmm.h" + +#include static const struct nvkm_mmu_func g84_mmu = { @@ -34,6 +36,7 @@ g84_mmu = { .map_sg = nv50_vm_map_sg, .unmap = nv50_vm_unmap, .flush = nv50_vm_flush, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x0200 }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c index 5811516c9a261..4e1c70db85c0b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c @@ -21,13 +21,15 @@ * * Authors: Ben Skeggs */ -#include "priv.h" +#include "vmm.h" #include #include #include #include +#include + void nv50_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 pde, struct nvkm_memory *pgt[2]) { @@ -205,7 +207,6 @@ nv50_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, u32 block = (1 << (mmu->func->pgt_bits + 12)); if (block > length) block = length; - return nvkm_vm_create(mmu, offset, length, mm_offset, block, key, pvm); } @@ -222,6 +223,7 @@ nv50_mmu = { .map_sg = nv50_vm_map_sg, .unmap = nv50_vm_unmap, .flush = nv50_vm_flush, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x1400 }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index bbb9c9cca27db..ac6d84537c5f0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -38,6 +38,7 @@ struct nvkm_mmu_func { void *argv, u32 argc, struct lock_class_key *, const char *name, struct nvkm_vmm **); bool global; + u32 pd_offset; } vmm; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index bade0237c1021..894f701c2fe87 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -94,6 +94,11 @@ struct nvkm_vmm_func { const struct nvkm_vmm_page page[]; }; +struct nvkm_vmm_join { + struct nvkm_memory *inst; + struct list_head head; +}; + int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32 pd_header, u64 addr, u64 size, struct lock_class_key *, const char *name, struct nvkm_vmm **); @@ -112,4 +117,8 @@ int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); +int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); +int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c new file mode 100644 index 0000000000000..db27381b67c68 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c @@ -0,0 +1,94 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +#include +#include + +static const struct nvkm_vmm_desc_func +nv50_vmm_pgt = { +}; + +static const struct nvkm_vmm_desc_func +nv50_vmm_pgd = { +}; + +static const struct nvkm_vmm_desc +nv50_vmm_desc_12[] = { + { PGT, 17, 8, 0x1000, &nv50_vmm_pgt }, + { PGD, 11, 0, 0x0000, &nv50_vmm_pgd }, + {} +}; + +static const struct nvkm_vmm_desc +nv50_vmm_desc_16[] = { + { PGT, 13, 8, 0x1000, &nv50_vmm_pgt }, + { PGD, 11, 0, 0x0000, &nv50_vmm_pgd }, + {} +}; + +static void +nv50_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst) +{ + struct nvkm_vmm_join *join; + + list_for_each_entry(join, &vmm->join, head) { + if (join->inst == inst) { + list_del(&join->head); + kfree(join); + break; + } + } +} + +static int +nv50_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) +{ + struct nvkm_vmm_join *join; + + if (!(join = kmalloc(sizeof(*join), GFP_KERNEL))) + return -ENOMEM; + join->inst = inst; + list_add_tail(&join->head, &vmm->join); + return 0; +} + +static const struct nvkm_vmm_func +nv50_vmm = { + .join = nv50_vmm_join, + .part = nv50_vmm_part, + .page_block = 1 << 29, + .page = { + { 16, &nv50_vmm_desc_16[0], NVKM_VMM_PAGE_xVxC }, + { 12, &nv50_vmm_desc_12[0], NVKM_VMM_PAGE_xVHx }, + {} + } +}; + +int +nv50_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + return nv04_vmm_new_(&nv50_vmm, mmu, 0, addr, size, + argv, argc, key, name, pvmm); +} From 540a1dde57bf4ce7acd2c5cc494bea03aa261484 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 095/151] drm/nouveau/mmu/gf100: implement vmm on top of new base Adds support for: - Selection of a 64KiB big page size (NvFbBigPage=16). - System-memory PDs. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 1 + drivers/gpu/drm/nouveau/include/nvif/if900d.h | 8 + .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 6 +- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 12 ++ .../drm/nouveau/nvkm/subdev/mmu/vmmgf100.c | 145 ++++++++++++++++++ 6 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if900d.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 9c02927b4ca95..df8d6a32d2dc1 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -17,6 +17,7 @@ #define NVIF_CLASS_VMM /* if000c.h */ 0x8000000c #define NVIF_CLASS_VMM_NV04 /* if000d.h */ 0x8000000d #define NVIF_CLASS_VMM_NV50 /* if500d.h */ 0x8000500d +#define NVIF_CLASS_VMM_GF100 /* if900d.h */ 0x8000900d /* the below match nvidia-assigned (either in hw, or sw) class numbers */ #define NV_NULL_CLASS 0x00000030 diff --git a/drivers/gpu/drm/nouveau/include/nvif/if900d.h b/drivers/gpu/drm/nouveau/include/nvif/if900d.h new file mode 100644 index 0000000000000..112716f889c68 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if900d.h @@ -0,0 +1,8 @@ +#ifndef __NVIF_IF900D_H__ +#define __NVIF_IF900D_H__ +#include "if000c.h" + +struct gf100_vmm_vn { + /* nvif_vmm_vX ... */ +}; +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 2ec7e50568ab2..908e57a2fc96c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -17,3 +17,4 @@ nvkm-y += nvkm/subdev/mmu/vmmnv04.o nvkm-y += nvkm/subdev/mmu/vmmnv41.o nvkm-y += nvkm/subdev/mmu/vmmnv44.o nvkm-y += nvkm/subdev/mmu/vmmnv50.o +nvkm-y += nvkm/subdev/mmu/vmmgf100.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index 3cc1013538c57..6a942e2bcc390 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -21,12 +21,13 @@ * * Authors: Ben Skeggs */ -#include "priv.h" +#include "vmm.h" +#include #include #include -#include +#include /* Map from compressed to corresponding uncompressed storage type. * The value 0xff represents an invalid storage type. @@ -209,6 +210,7 @@ gf100_mmu = { .map_sg = gf100_vm_map_sg, .unmap = gf100_vm_unmap, .flush = gf100_vm_flush, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gf100_vmm_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 894f701c2fe87..1e207ce3faa8e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -52,6 +52,9 @@ struct nvkm_vmm_pt { struct nvkm_vmm_desc_func { }; +extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; +extern const struct nvkm_vmm_desc_func gf100_vmm_pgt; + struct nvkm_vmm_desc { enum { PGD, @@ -111,6 +114,13 @@ int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); +int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, + struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); +int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); +int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); +void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); + int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, @@ -121,4 +131,6 @@ int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); +int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c new file mode 100644 index 0000000000000..f8234af0120cd --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c @@ -0,0 +1,145 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +#include + +const struct nvkm_vmm_desc_func +gf100_vmm_pgt = { +}; + +const struct nvkm_vmm_desc_func +gf100_vmm_pgd = { +}; + +static const struct nvkm_vmm_desc +gf100_vmm_desc_17_12[] = { + { SPT, 15, 8, 0x1000, &gf100_vmm_pgt }, + { PGD, 13, 8, 0x1000, &gf100_vmm_pgd }, + {} +}; + +static const struct nvkm_vmm_desc +gf100_vmm_desc_17_17[] = { + { LPT, 10, 8, 0x1000, &gf100_vmm_pgt }, + { PGD, 13, 8, 0x1000, &gf100_vmm_pgd }, + {} +}; + +static const struct nvkm_vmm_desc +gf100_vmm_desc_16_12[] = { + { SPT, 14, 8, 0x1000, &gf100_vmm_pgt }, + { PGD, 14, 8, 0x1000, &gf100_vmm_pgd }, + {} +}; + +static const struct nvkm_vmm_desc +gf100_vmm_desc_16_16[] = { + { LPT, 10, 8, 0x1000, &gf100_vmm_pgt }, + { PGD, 14, 8, 0x1000, &gf100_vmm_pgd }, + {} +}; + +void +gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst) +{ + nvkm_fo64(inst, 0x0200, 0x00000000, 2); +} + +int +gf100_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base) +{ + struct nvkm_mmu_pt *pd = vmm->pd->pt[0]; + + switch (nvkm_memory_target(pd->memory)) { + case NVKM_MEM_TARGET_VRAM: base |= 0ULL << 0; break; + case NVKM_MEM_TARGET_HOST: base |= 2ULL << 0; + base |= BIT_ULL(2) /* VOL. */; + break; + case NVKM_MEM_TARGET_NCOH: base |= 3ULL << 0; break; + default: + WARN_ON(1); + return -EINVAL; + } + base |= pd->addr; + + nvkm_kmap(inst); + nvkm_wo64(inst, 0x0200, base); + nvkm_wo64(inst, 0x0208, vmm->limit - 1); + nvkm_done(inst); + return 0; +} + +int +gf100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) +{ + return gf100_vmm_join_(vmm, inst, 0); +} + +static const struct nvkm_vmm_func +gf100_vmm_17 = { + .join = gf100_vmm_join, + .part = gf100_vmm_part, + .page = { + { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, + { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, + {} + } +}; + +static const struct nvkm_vmm_func +gf100_vmm_16 = { + .join = gf100_vmm_join, + .part = gf100_vmm_part, + .page = { + { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, + { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, + {} + } +}; + +int +gf100_vmm_new_(const struct nvkm_vmm_func *func_16, + const struct nvkm_vmm_func *func_17, + struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + switch (mmu->subdev.device->fb->page) { + case 16: return nv04_vmm_new_(func_16, mmu, 0, addr, size, + argv, argc, key, name, pvmm); + case 17: return nv04_vmm_new_(func_17, mmu, 0, addr, size, + argv, argc, key, name, pvmm); + default: + WARN_ON(1); + return -EINVAL; + } +} + +int +gf100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, addr, + size, argv, argc, key, name, pvmm); +} From 7de078aa7902fc515286735219173ad263a7610c Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 096/151] drm/nouveau/mmu/gk104,gk20a: implement vmm on top of new base Adds support for: - Selection of a 64KiB big page size (NvFbBigPage=16). - System-memory PDs. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c | 5 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c | 5 +- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 9 ++ .../drm/nouveau/nvkm/subdev/mmu/vmmgk104.c | 85 +++++++++++++++++++ .../drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c | 53 ++++++++++++ 6 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 908e57a2fc96c..e65daffff9bfd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -18,3 +18,5 @@ nvkm-y += nvkm/subdev/mmu/vmmnv41.o nvkm-y += nvkm/subdev/mmu/vmmnv44.o nvkm-y += nvkm/subdev/mmu/vmmnv50.o nvkm-y += nvkm/subdev/mmu/vmmgf100.o +nvkm-y += nvkm/subdev/mmu/vmmgk104.o +nvkm-y += nvkm/subdev/mmu/vmmgk20a.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c index 6ab8e636feb13..1facdab4aa9e6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c @@ -19,7 +19,9 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include "priv.h" +#include "vmm.h" + +#include static const struct nvkm_mmu_func gk104_mmu = { @@ -34,6 +36,7 @@ gk104_mmu = { .map_sg = gf100_vm_map_sg, .unmap = gf100_vm_unmap, .flush = gf100_vm_flush, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk104_vmm_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c index 9ea8a91802be1..3d699c35a181b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c @@ -19,7 +19,9 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include "priv.h" +#include "vmm.h" + +#include static const struct nvkm_mmu_func gk20a_mmu = { @@ -34,6 +36,7 @@ gk20a_mmu = { .map_sg = gf100_vm_map_sg, .unmap = gf100_vm_unmap, .flush = gf100_vm_flush, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk20a_vmm_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 1e207ce3faa8e..914ae3f53cca8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -68,6 +68,11 @@ struct nvkm_vmm_desc { const struct nvkm_vmm_desc_func *func; }; +extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[]; +extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[]; +extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[]; +extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[]; + struct nvkm_vmm_page { u8 shift; const struct nvkm_vmm_desc *desc; @@ -133,4 +138,8 @@ int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); +int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); +int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c new file mode 100644 index 0000000000000..0d33700e3d88e --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c @@ -0,0 +1,85 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +static const struct nvkm_vmm_desc_func +gk104_vmm_lpt = { +}; + +const struct nvkm_vmm_desc +gk104_vmm_desc_17_12[] = { + { SPT, 15, 8, 0x1000, &gf100_vmm_pgt }, + { PGD, 13, 8, 0x1000, &gf100_vmm_pgd }, + {} +}; + +const struct nvkm_vmm_desc +gk104_vmm_desc_17_17[] = { + { LPT, 10, 8, 0x1000, &gk104_vmm_lpt }, + { PGD, 13, 8, 0x1000, &gf100_vmm_pgd }, + {} +}; + +const struct nvkm_vmm_desc +gk104_vmm_desc_16_12[] = { + { SPT, 14, 8, 0x1000, &gf100_vmm_pgt }, + { PGD, 14, 8, 0x1000, &gf100_vmm_pgd }, + {} +}; + +const struct nvkm_vmm_desc +gk104_vmm_desc_16_16[] = { + { LPT, 10, 8, 0x1000, &gk104_vmm_lpt }, + { PGD, 14, 8, 0x1000, &gf100_vmm_pgd }, + {} +}; + +static const struct nvkm_vmm_func +gk104_vmm_17 = { + .join = gf100_vmm_join, + .part = gf100_vmm_part, + .page = { + { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, + { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, + {} + } +}; + +static const struct nvkm_vmm_func +gk104_vmm_16 = { + .join = gf100_vmm_join, + .part = gf100_vmm_part, + .page = { + { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, + { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, + {} + } +}; + +int +gk104_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + return gf100_vmm_new_(&gk104_vmm_16, &gk104_vmm_17, mmu, addr, + size, argv, argc, key, name, pvmm); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c new file mode 100644 index 0000000000000..e000c3d1554ae --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c @@ -0,0 +1,53 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +static const struct nvkm_vmm_func +gk20a_vmm_17 = { + .join = gf100_vmm_join, + .part = gf100_vmm_part, + .page = { + { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xxHC }, + { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xxHx }, + {} + } +}; + +static const struct nvkm_vmm_func +gk20a_vmm_16 = { + .join = gf100_vmm_join, + .part = gf100_vmm_part, + .page = { + { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xxHC }, + { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xxHx }, + {} + } +}; + +int +gk20a_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + return gf100_vmm_new_(&gk20a_vmm_16, &gk20a_vmm_17, mmu, addr, + size, argv, argc, key, name, pvmm); +} From 5f300fed645597689f1e6fed8800fc7e3d01e212 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 097/151] drm/nouveau/mmu/gm200,gm20b: implement vmm on top of new base Adds support for: - Per-VMM selection of big page size. - System-memory PDs. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 1 + drivers/gpu/drm/nouveau/include/nvif/ifb00d.h | 14 ++ .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c | 25 ++- .../gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c | 25 ++- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 23 +++ .../drm/nouveau/nvkm/subdev/mmu/vmmgm200.c | 151 ++++++++++++++++++ .../drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c | 64 ++++++++ 8 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/ifb00d.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index df8d6a32d2dc1..76dff0697f63b 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -18,6 +18,7 @@ #define NVIF_CLASS_VMM_NV04 /* if000d.h */ 0x8000000d #define NVIF_CLASS_VMM_NV50 /* if500d.h */ 0x8000500d #define NVIF_CLASS_VMM_GF100 /* if900d.h */ 0x8000900d +#define NVIF_CLASS_VMM_GM200 /* ifb00d.h */ 0x8000b00d /* the below match nvidia-assigned (either in hw, or sw) class numbers */ #define NV_NULL_CLASS 0x00000030 diff --git a/drivers/gpu/drm/nouveau/include/nvif/ifb00d.h b/drivers/gpu/drm/nouveau/include/nvif/ifb00d.h new file mode 100644 index 0000000000000..2e1e998103619 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/ifb00d.h @@ -0,0 +1,14 @@ +#ifndef __NVIF_IFB00D_H__ +#define __NVIF_IFB00D_H__ +#include "if000c.h" + +struct gm200_vmm_vn { + /* nvif_vmm_vX ... */ +}; + +struct gm200_vmm_v0 { + /* nvif_vmm_vX ... */ + __u8 version; + __u8 bigpage; +}; +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index e65daffff9bfd..55963dc4dc41c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -20,3 +20,5 @@ nvkm-y += nvkm/subdev/mmu/vmmnv50.o nvkm-y += nvkm/subdev/mmu/vmmgf100.o nvkm-y += nvkm/subdev/mmu/vmmgk104.o nvkm-y += nvkm/subdev/mmu/vmmgk20a.o +nvkm-y += nvkm/subdev/mmu/vmmgm200.o +nvkm-y += nvkm/subdev/mmu/vmmgm20b.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c index 4bb342fa3d540..8d6be5b3e8e84 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c @@ -19,7 +19,11 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include "priv.h" +#include "vmm.h" + +#include + +#include static const struct nvkm_mmu_func gm200_mmu = { @@ -34,10 +38,29 @@ gm200_mmu = { .map_sg = gf100_vm_map_sg, .unmap = gf100_vm_unmap, .flush = gf100_vm_flush, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm200_vmm_new }, +}; + +static const struct nvkm_mmu_func +gm200_mmu_fixed = { + .limit = (1ULL << 40), + .dma_bits = 40, + .pgt_bits = 27 - 12, + .spg_shift = 12, + .lpg_shift = 17, + .create = gf100_vm_create, + .map_pgt = gf100_vm_map_pgt, + .map = gf100_vm_map, + .map_sg = gf100_vm_map_sg, + .unmap = gf100_vm_unmap, + .flush = gf100_vm_flush, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm200_vmm_new_fixed }, }; int gm200_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { + if (device->fb->page) + return nvkm_mmu_new_(&gm200_mmu_fixed, device, index, pmmu); return nvkm_mmu_new_(&gm200_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c index 06e49a9c8a9f5..2894c7cbcdbb1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c @@ -19,7 +19,11 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include "priv.h" +#include "vmm.h" + +#include + +#include static const struct nvkm_mmu_func gm20b_mmu = { @@ -34,10 +38,29 @@ gm20b_mmu = { .map_sg = gf100_vm_map_sg, .unmap = gf100_vm_unmap, .flush = gf100_vm_flush, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new }, +}; + +static const struct nvkm_mmu_func +gm20b_mmu_fixed = { + .limit = (1ULL << 40), + .dma_bits = 40, + .pgt_bits = 27 - 12, + .spg_shift = 12, + .lpg_shift = 17, + .create = gf100_vm_create, + .map_pgt = gf100_vm_map_pgt, + .map = gf100_vm_map, + .map_sg = gf100_vm_map_sg, + .unmap = gf100_vm_unmap, + .flush = gf100_vm_flush, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new_fixed }, }; int gm20b_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { + if (device->fb->page) + return nvkm_mmu_new_(&gm20b_mmu_fixed, device, index, pmmu); return nvkm_mmu_new_(&gm20b_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 914ae3f53cca8..5cca1b449bdb5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -73,6 +73,11 @@ extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[]; extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[]; extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[]; +extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[]; +extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[]; +extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[]; +extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[]; + struct nvkm_vmm_page { u8 shift; const struct nvkm_vmm_desc *desc; @@ -126,6 +131,12 @@ int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); +int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, + struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); +int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); +int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); + int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, @@ -142,4 +153,16 @@ int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); +int gm200_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, + struct nvkm_vmm **); +int gm200_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, + struct nvkm_vmm **); +int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, + struct nvkm_vmm **); +int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, + struct nvkm_vmm **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c new file mode 100644 index 0000000000000..43138560b1084 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c @@ -0,0 +1,151 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +#include +#include + +static const struct nvkm_vmm_desc_func +gm200_vmm_spt = { +}; + +static const struct nvkm_vmm_desc_func +gm200_vmm_lpt = { +}; + +static const struct nvkm_vmm_desc_func +gm200_vmm_pgd = { +}; + +const struct nvkm_vmm_desc +gm200_vmm_desc_17_12[] = { + { SPT, 15, 8, 0x1000, &gm200_vmm_spt }, + { PGD, 13, 8, 0x1000, &gm200_vmm_pgd }, + {} +}; + +const struct nvkm_vmm_desc +gm200_vmm_desc_17_17[] = { + { LPT, 10, 8, 0x1000, &gm200_vmm_lpt }, + { PGD, 13, 8, 0x1000, &gm200_vmm_pgd }, + {} +}; + +const struct nvkm_vmm_desc +gm200_vmm_desc_16_12[] = { + { SPT, 14, 8, 0x1000, &gm200_vmm_spt }, + { PGD, 14, 8, 0x1000, &gm200_vmm_pgd }, + {} +}; + +const struct nvkm_vmm_desc +gm200_vmm_desc_16_16[] = { + { LPT, 10, 8, 0x1000, &gm200_vmm_lpt }, + { PGD, 14, 8, 0x1000, &gm200_vmm_pgd }, + {} +}; + +int +gm200_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base) +{ + if (vmm->func->page[1].shift == 16) + base |= BIT_ULL(11); + return gf100_vmm_join_(vmm, inst, base); +} + +int +gm200_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) +{ + return gm200_vmm_join_(vmm, inst, 0); +} + +static const struct nvkm_vmm_func +gm200_vmm_17 = { + .join = gm200_vmm_join, + .part = gf100_vmm_part, + .page = { + { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx }, + { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SVxC }, + { 12, &gm200_vmm_desc_17_12[0], NVKM_VMM_PAGE_SVHx }, + {} + } +}; + +static const struct nvkm_vmm_func +gm200_vmm_16 = { + .join = gm200_vmm_join, + .part = gf100_vmm_part, + .page = { + { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx }, + { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SVxC }, + { 12, &gm200_vmm_desc_16_12[0], NVKM_VMM_PAGE_SVHx }, + {} + } +}; + +int +gm200_vmm_new_(const struct nvkm_vmm_func *func_16, + const struct nvkm_vmm_func *func_17, + struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + const struct nvkm_vmm_func *func; + union { + struct gm200_vmm_vn vn; + struct gm200_vmm_v0 v0; + } *args = argv; + int ret = -ENOSYS; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + switch (args->v0.bigpage) { + case 16: func = func_16; break; + case 17: func = func_17; break; + default: + return -EINVAL; + } + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + func = func_17; + } else + return ret; + + return nvkm_vmm_new_(func, mmu, 0, addr, size, key, name, pvmm); +} + +int +gm200_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + return gm200_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, addr, + size, argv, argc, key, name, pvmm); +} + +int +gm200_vmm_new_fixed(struct nvkm_mmu *mmu, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) +{ + return gf100_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, addr, + size, argv, argc, key, name, pvmm); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c new file mode 100644 index 0000000000000..ec24c43f86ccb --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c @@ -0,0 +1,64 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +static const struct nvkm_vmm_func +gm20b_vmm_17 = { + .join = gm200_vmm_join, + .part = gf100_vmm_part, + .page = { + { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx }, + { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SxHC }, + { 12, &gm200_vmm_desc_17_12[0], NVKM_VMM_PAGE_SxHx }, + {} + } +}; + +static const struct nvkm_vmm_func +gm20b_vmm_16 = { + .join = gm200_vmm_join, + .part = gf100_vmm_part, + .page = { + { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx }, + { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SxHC }, + { 12, &gm200_vmm_desc_16_12[0], NVKM_VMM_PAGE_SxHx }, + {} + } +}; + +int +gm20b_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + return gm200_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, addr, + size, argv, argc, key, name, pvmm); +} + +int +gm20b_vmm_new_fixed(struct nvkm_mmu *mmu, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) +{ + return gf100_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, addr, + size, argv, argc, key, name, pvmm); +} From 8e39abff45888a54b3d9dc20434940dabeda077f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 098/151] drm/nouveau/mmu/gp100,gp10b: implement vmm on top of new base Adds support for: - Selection of old/new-style page table layout (GP100MmuLayout=0/1). - System-memory PDs. New layout disabled by default for the moment, as we don't have a backend that can handle it yet. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 1 + drivers/gpu/drm/nouveau/include/nvif/ifc00d.h | 8 ++ .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c | 23 +++-- .../gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c | 23 +++-- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 11 +++ .../drm/nouveau/nvkm/subdev/mmu/vmmgp100.c | 89 +++++++++++++++++++ .../drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c | 46 ++++++++++ 8 files changed, 179 insertions(+), 24 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/ifc00d.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 76dff0697f63b..d5236daf54c4b 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -19,6 +19,7 @@ #define NVIF_CLASS_VMM_NV50 /* if500d.h */ 0x8000500d #define NVIF_CLASS_VMM_GF100 /* if900d.h */ 0x8000900d #define NVIF_CLASS_VMM_GM200 /* ifb00d.h */ 0x8000b00d +#define NVIF_CLASS_VMM_GP100 /* ifc00d.h */ 0x8000c00d /* the below match nvidia-assigned (either in hw, or sw) class numbers */ #define NV_NULL_CLASS 0x00000030 diff --git a/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h b/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h new file mode 100644 index 0000000000000..39be62ca73396 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h @@ -0,0 +1,8 @@ +#ifndef __NVIF_IFC00D_H__ +#define __NVIF_IFC00D_H__ +#include "if000c.h" + +struct gp100_vmm_vn { + /* nvif_vmm_vX ... */ +}; +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 55963dc4dc41c..e7ef528fb5bfe 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -22,3 +22,5 @@ nvkm-y += nvkm/subdev/mmu/vmmgk104.o nvkm-y += nvkm/subdev/mmu/vmmgk20a.o nvkm-y += nvkm/subdev/mmu/vmmgm200.o nvkm-y += nvkm/subdev/mmu/vmmgm20b.o +nvkm-y += nvkm/subdev/mmu/vmmgp100.o +nvkm-y += nvkm/subdev/mmu/vmmgp10b.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c index 6cfd325ea3a91..b1ba864b4b35c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c @@ -19,25 +19,24 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include "priv.h" +#include "vmm.h" + +#include + +#include static const struct nvkm_mmu_func gp100_mmu = { - .limit = (1ULL << 40), - .dma_bits = 40, - .pgt_bits = 27 - 12, - .spg_shift = 12, - .lpg_shift = 17, - .create = gf100_vm_create, - .map_pgt = gf100_vm_map_pgt, - .map = gf100_vm_map, - .map_sg = gf100_vm_map_sg, - .unmap = gf100_vm_unmap, - .flush = gf100_vm_flush, + .limit = (1ULL << 49), + .dma_bits = 47, + .lpg_shift = 16, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, }; int gp100_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { + if (!nvkm_boolopt(device->cfgopt, "GP100MmuLayout", false)) + return gm200_mmu_new(device, index, pmmu); return nvkm_mmu_new_(&gp100_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c index e0a41ce3ddd19..5d760a198b227 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -19,25 +19,24 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include "priv.h" +#include "vmm.h" + +#include + +#include static const struct nvkm_mmu_func gp10b_mmu = { - .limit = (1ULL << 40), - .dma_bits = 40, - .pgt_bits = 27 - 12, - .spg_shift = 12, - .lpg_shift = 17, - .create = gf100_vm_create, - .map_pgt = gf100_vm_map_pgt, - .map = gf100_vm_map, - .map_sg = gf100_vm_map_sg, - .unmap = gf100_vm_unmap, - .flush = gf100_vm_flush, + .limit = (1ULL << 49), + .dma_bits = 47, + .lpg_shift = 16, + .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, }; int gp10b_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { + if (!nvkm_boolopt(device->cfgopt, "GP100MmuLayout", false)) + return gm20b_mmu_new(device, index, pmmu); return nvkm_mmu_new_(&gp10b_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 5cca1b449bdb5..504408d8014b2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -78,6 +78,9 @@ extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[]; extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[]; extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[]; +extern const struct nvkm_vmm_desc gp100_vmm_desc_12[]; +extern const struct nvkm_vmm_desc gp100_vmm_desc_16[]; + struct nvkm_vmm_page { u8 shift; const struct nvkm_vmm_desc *desc; @@ -137,6 +140,8 @@ int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); +int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); + int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, @@ -165,4 +170,10 @@ int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); +int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, + struct nvkm_vmm **); +int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, + struct lock_class_key *, const char *, + struct nvkm_vmm **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c new file mode 100644 index 0000000000000..68f67812aecca --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c @@ -0,0 +1,89 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +static const struct nvkm_vmm_desc_func +gp100_vmm_desc_spt = { +}; + +static const struct nvkm_vmm_desc_func +gp100_vmm_desc_lpt = { +}; + +static const struct nvkm_vmm_desc_func +gp100_vmm_desc_pd0 = { +}; + +static const struct nvkm_vmm_desc_func +gp100_vmm_desc_pd1 = { +}; + +const struct nvkm_vmm_desc +gp100_vmm_desc_16[] = { + { LPT, 5, 8, 0x0100, &gp100_vmm_desc_lpt }, + { PGD, 8, 16, 0x1000, &gp100_vmm_desc_pd0 }, + { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, + { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, + { PGD, 2, 8, 0x1000, &gp100_vmm_desc_pd1 }, + {} +}; + +const struct nvkm_vmm_desc +gp100_vmm_desc_12[] = { + { SPT, 9, 8, 0x1000, &gp100_vmm_desc_spt }, + { PGD, 8, 16, 0x1000, &gp100_vmm_desc_pd0 }, + { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, + { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, + { PGD, 2, 8, 0x1000, &gp100_vmm_desc_pd1 }, + {} +}; + +int +gp100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) +{ + const u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11); /* 64KiB */ + return gf100_vmm_join_(vmm, inst, base); +} + +static const struct nvkm_vmm_func +gp100_vmm = { + .join = gp100_vmm_join, + .part = gf100_vmm_part, + .page = { + { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, + { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, + { 29, &gp100_vmm_desc_16[2], NVKM_VMM_PAGE_Sxxx }, + { 21, &gp100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC }, + { 16, &gp100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC }, + { 12, &gp100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx }, + {} + } +}; + +int +gp100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + return nv04_vmm_new_(&gp100_vmm, mmu, 0, addr, size, + argv, argc, key, name, pvmm); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c new file mode 100644 index 0000000000000..8b7f9b82750c0 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c @@ -0,0 +1,46 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +static const struct nvkm_vmm_func +gp10b_vmm = { + .join = gp100_vmm_join, + .part = gf100_vmm_part, + .page = { + { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, + { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, + { 29, &gp100_vmm_desc_16[2], NVKM_VMM_PAGE_Sxxx }, + { 21, &gp100_vmm_desc_16[1], NVKM_VMM_PAGE_SxHC }, + { 16, &gp100_vmm_desc_16[0], NVKM_VMM_PAGE_SxHC }, + { 12, &gp100_vmm_desc_12[0], NVKM_VMM_PAGE_SxHx }, + {} + } +}; + +int +gp10b_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + return nv04_vmm_new_(&gp10b_vmm, mmu, 0, addr, size, + argv, argc, key, name, pvmm); +} From af3b8d53869c175fce424b6bfd1f49c1b53baef1 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 099/151] drm/nouveau/mmu: remove old vm creation hooks Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 30 +------------------ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 8 ----- .../gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c | 2 -- .../gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c | 2 -- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c | 11 ------- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 10 ------- 9 files changed, 1 insertion(+), 65 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 1965a52ebe605..9fdd1446da5f4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -616,31 +616,6 @@ nvkm_vm_legacy(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, return 0; } -int -nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, - u32 block, struct lock_class_key *key, struct nvkm_vm **pvm) -{ - static struct lock_class_key _key; - struct nvkm_vm *vm; - int ret; - - vm = kzalloc(sizeof(*vm), GFP_KERNEL); - if (!vm) - return -ENOMEM; - - __mutex_init(&vm->mutex, "&vm->mutex", key ? key : &_key); - vm->mmu = mmu; - - ret = nvkm_vm_legacy(mmu, offset, length, mm_offset, block, vm); - if (ret) { - kfree(vm); - return ret; - } - - *pvm = vm; - return 0; -} - int nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset, struct lock_class_key *key, struct nvkm_vm **pvm) @@ -666,10 +641,7 @@ nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset, return ret; } - if (!mmu->func->create) - return -EINVAL; - - return mmu->func->create(mmu, offset, length, mm_offset, key, pvm); + return -EINVAL; } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c index bb15ae97f4217..13f8247a2ee47 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c @@ -30,7 +30,6 @@ g84_mmu = { .pgt_bits = 29 - 12, .spg_shift = 12, .lpg_shift = 16, - .create = nv50_vm_create, .map_pgt = nv50_vm_map_pgt, .map = nv50_vm_map, .map_sg = nv50_vm_map_sg, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index 6a942e2bcc390..536260782cff4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -190,13 +190,6 @@ gf100_vm_flush(struct nvkm_vm *vm) mutex_unlock(&mmu->subdev.mutex); } -int -gf100_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, - struct lock_class_key *key, struct nvkm_vm **pvm) -{ - return nvkm_vm_create(mmu, offset, length, mm_offset, 4096, key, pvm); -} - static const struct nvkm_mmu_func gf100_mmu = { .limit = (1ULL << 40), @@ -204,7 +197,6 @@ gf100_mmu = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .create = gf100_vm_create, .map_pgt = gf100_vm_map_pgt, .map = gf100_vm_map, .map_sg = gf100_vm_map_sg, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c index 1facdab4aa9e6..693aa1b875f28 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c @@ -30,7 +30,6 @@ gk104_mmu = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .create = gf100_vm_create, .map_pgt = gf100_vm_map_pgt, .map = gf100_vm_map, .map_sg = gf100_vm_map_sg, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c index 3d699c35a181b..b774d9a25c3e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c @@ -30,7 +30,6 @@ gk20a_mmu = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .create = gf100_vm_create, .map_pgt = gf100_vm_map_pgt, .map = gf100_vm_map, .map_sg = gf100_vm_map_sg, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c index 8d6be5b3e8e84..41332613c6443 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c @@ -32,7 +32,6 @@ gm200_mmu = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .create = gf100_vm_create, .map_pgt = gf100_vm_map_pgt, .map = gf100_vm_map, .map_sg = gf100_vm_map_sg, @@ -48,7 +47,6 @@ gm200_mmu_fixed = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .create = gf100_vm_create, .map_pgt = gf100_vm_map_pgt, .map = gf100_vm_map, .map_sg = gf100_vm_map_sg, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c index 2894c7cbcdbb1..25b16975bddd2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c @@ -32,7 +32,6 @@ gm20b_mmu = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .create = gf100_vm_create, .map_pgt = gf100_vm_map_pgt, .map = gf100_vm_map, .map_sg = gf100_vm_map_sg, @@ -48,7 +47,6 @@ gm20b_mmu_fixed = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .create = gf100_vm_create, .map_pgt = gf100_vm_map_pgt, .map = gf100_vm_map, .map_sg = gf100_vm_map_sg, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c index 4e1c70db85c0b..ba1c04e007016 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c @@ -200,16 +200,6 @@ nv50_vm_flush(struct nvkm_vm *vm) mutex_unlock(&subdev->mutex); } -int -nv50_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, - struct lock_class_key *key, struct nvkm_vm **pvm) -{ - u32 block = (1 << (mmu->func->pgt_bits + 12)); - if (block > length) - block = length; - return nvkm_vm_create(mmu, offset, length, mm_offset, block, key, pvm); -} - static const struct nvkm_mmu_func nv50_mmu = { .limit = (1ULL << 40), @@ -217,7 +207,6 @@ nv50_mmu = { .pgt_bits = 29 - 12, .spg_shift = 12, .lpg_shift = 16, - .create = nv50_vm_create, .map_pgt = nv50_vm_map_pgt, .map = nv50_vm_map, .map_sg = nv50_vm_map_sg, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index ac6d84537c5f0..f490a636e9f01 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -18,9 +18,6 @@ struct nvkm_mmu_func { u8 spg_shift; u8 lpg_shift; - int (*create)(struct nvkm_mmu *, u64 offset, u64 length, u64 mm_offset, - struct lock_class_key *, struct nvkm_vm **); - void (*map_pgt)(struct nvkm_gpuobj *pgd, u32 pde, struct nvkm_memory *pgt[2]); void (*map)(struct nvkm_vma *, struct nvkm_memory *, @@ -42,13 +39,8 @@ struct nvkm_mmu_func { } vmm; }; -int nvkm_vm_create(struct nvkm_mmu *, u64, u64, u64, u32, - struct lock_class_key *, struct nvkm_vm **); - extern const struct nvkm_mmu_func nv04_mmu; -int nv50_vm_create(struct nvkm_mmu *, u64, u64, u64, struct lock_class_key *, - struct nvkm_vm **); void nv50_vm_map_pgt(struct nvkm_gpuobj *, u32, struct nvkm_memory **); void nv50_vm_map(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, u32, u32, u64, u64); @@ -57,8 +49,6 @@ void nv50_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, void nv50_vm_unmap(struct nvkm_vma *, struct nvkm_memory *, u32, u32); void nv50_vm_flush(struct nvkm_vm *); -int gf100_vm_create(struct nvkm_mmu *, u64, u64, u64, struct lock_class_key *, - struct nvkm_vm **); void gf100_vm_map_pgt(struct nvkm_gpuobj *, u32, struct nvkm_memory **); void gf100_vm_map(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, u32, u32, u64, u64); From d30af7ce2c96e57b503da1d70454818331f0a6d5 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 100/151] drm/nouveau/mmu: handle instance block setup We previously required each VMM user to allocate their own page directory and fill in the instance block themselves. It makes more sense to handle this in a common location. Signed-off-by: Ben Skeggs --- .../drm/nouveau/include/nvkm/engine/falcon.h | 4 +- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 8 +- .../drm/nouveau/nvkm/engine/fifo/chang84.c | 2 +- .../drm/nouveau/nvkm/engine/fifo/changf100.h | 1 - .../drm/nouveau/nvkm/engine/fifo/changk104.h | 1 - .../drm/nouveau/nvkm/engine/fifo/channv50.c | 5 +- .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 19 +---- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 19 +---- drivers/gpu/drm/nouveau/nvkm/falcon/base.c | 2 +- drivers/gpu/drm/nouveau/nvkm/falcon/v1.c | 6 +- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 30 +++---- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.h | 3 +- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 20 ++--- .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 81 +++++-------------- .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 38 +++++---- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c | 14 ++-- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 6 +- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c | 2 +- .../drm/nouveau/nvkm/subdev/secboot/gm200.c | 21 ++--- .../drm/nouveau/nvkm/subdev/secboot/gm200.h | 3 +- 20 files changed, 93 insertions(+), 192 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h index 8f9d132fbedff..f0024fb5a5aff 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h @@ -78,7 +78,7 @@ struct nvkm_falcon_func { void (*load_imem)(struct nvkm_falcon *, void *, u32, u32, u16, u8, bool); void (*load_dmem)(struct nvkm_falcon *, void *, u32, u32, u8); void (*read_dmem)(struct nvkm_falcon *, u32, u32, u8, void *); - void (*bind_context)(struct nvkm_falcon *, struct nvkm_gpuobj *); + void (*bind_context)(struct nvkm_falcon *, struct nvkm_memory *); int (*wait_for_halt)(struct nvkm_falcon *, u32); int (*clear_interrupt)(struct nvkm_falcon *, u32); void (*set_start_addr)(struct nvkm_falcon *, u32 start_addr); @@ -113,7 +113,7 @@ void nvkm_falcon_load_imem(struct nvkm_falcon *, void *, u32, u32, u16, u8, bool); void nvkm_falcon_load_dmem(struct nvkm_falcon *, void *, u32, u32, u8); void nvkm_falcon_read_dmem(struct nvkm_falcon *, u32, u32, u8, void *); -void nvkm_falcon_bind_context(struct nvkm_falcon *, struct nvkm_gpuobj *); +void nvkm_falcon_bind_context(struct nvkm_falcon *, struct nvkm_memory *); void nvkm_falcon_set_start_addr(struct nvkm_falcon *, u32); void nvkm_falcon_start(struct nvkm_falcon *); int nvkm_falcon_wait_for_halt(struct nvkm_falcon *, u32); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 71d4863378f4b..eb6704a2fc8ff 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -10,11 +10,6 @@ struct nvkm_vm_pgt { u32 refcount[2]; }; -struct nvkm_vm_pgd { - struct list_head head; - struct nvkm_gpuobj *obj; -}; - struct nvkm_vma { struct nvkm_vm *vm; struct nvkm_mm_node *node; @@ -40,7 +35,6 @@ struct nvkm_vm { struct nvkm_mm mm; struct kref refcount; - struct list_head pgd_list; struct nvkm_vm_pgt *pgt; u32 fpde; u32 lpde; @@ -54,7 +48,7 @@ struct nvkm_vm { int nvkm_vm_new(struct nvkm_device *, u64 offset, u64 length, u64 mm_offset, struct lock_class_key *, struct nvkm_vm **); -int nvkm_vm_ref(struct nvkm_vm *, struct nvkm_vm **, struct nvkm_gpuobj *pgd); +int nvkm_vm_ref(struct nvkm_vm *, struct nvkm_vm **, struct nvkm_memory *inst); int nvkm_vm_boot(struct nvkm_vm *, u64 size); int nvkm_vm_get(struct nvkm_vm *, u64 size, u32 page_shift, u32 access, struct nvkm_vma *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c index 61797c4dd07a7..705a6d951d322 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c @@ -281,5 +281,5 @@ g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push, if (ret) return ret; - return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd); + return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst->memory); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h index 7d697e2dce1ab..a902f848a8785 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h @@ -11,7 +11,6 @@ struct gf100_fifo_chan { struct list_head head; bool killed; - struct nvkm_gpuobj *pgd; struct nvkm_vm *vm; struct { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h index 230f64e5f7318..1cd5b8d3fb2e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h @@ -12,7 +12,6 @@ struct gk104_fifo_chan { struct list_head head; bool killed; - struct nvkm_gpuobj *pgd; struct nvkm_vm *vm; struct { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c index 25b60aff40e47..161aa9ec73bcd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c @@ -206,7 +206,8 @@ void * nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base) { struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - nvkm_vm_ref(NULL, &chan->vm, chan->pgd); + if (chan->base.inst) + nvkm_vm_ref(NULL, &chan->vm, chan->base.inst->memory); nvkm_ramht_del(&chan->ramht); nvkm_gpuobj_del(&chan->pgd); nvkm_gpuobj_del(&chan->eng); @@ -266,5 +267,5 @@ nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push, if (ret) return ret; - return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd); + return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst->memory); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 6343e0c84d96a..695d0721b1adc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -200,8 +200,8 @@ static void * gf100_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) { struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - nvkm_vm_ref(NULL, &chan->vm, chan->pgd); - nvkm_gpuobj_del(&chan->pgd); + if (chan->base.inst) + nvkm_vm_ref(NULL, &chan->vm, chan->base.inst->memory); return chan; } @@ -225,7 +225,6 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, struct fermi_channel_gpfifo_v0 v0; } *args = data; struct gf100_fifo *fifo = gf100_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; struct nvkm_object *parent = oclass->parent; struct gf100_fifo_chan *chan; u64 usermem, ioffset, ilength; @@ -263,19 +262,7 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, args->v0.chid = chan->base.chid; - /* page directory */ - ret = nvkm_gpuobj_new(device, 0x10000, 0x1000, false, NULL, &chan->pgd); - if (ret) - return ret; - - nvkm_kmap(chan->base.inst); - nvkm_wo32(chan->base.inst, 0x0200, lower_32_bits(chan->pgd->addr)); - nvkm_wo32(chan->base.inst, 0x0204, upper_32_bits(chan->pgd->addr)); - nvkm_wo32(chan->base.inst, 0x0208, 0xffffffff); - nvkm_wo32(chan->base.inst, 0x020c, 0x000000ff); - nvkm_done(chan->base.inst); - - ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd); + ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst->memory); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 6e93b8652c667..978c9ad952b38 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -213,8 +213,8 @@ static void * gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) { struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - nvkm_vm_ref(NULL, &chan->vm, chan->pgd); - nvkm_gpuobj_del(&chan->pgd); + if (chan->base.inst) + nvkm_vm_ref(NULL, &chan->vm, chan->base.inst->memory); return chan; } @@ -242,7 +242,6 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func, const struct nvkm_oclass *oclass, struct nvkm_object **pobject) { - struct nvkm_device *device = fifo->base.engine.subdev.device; struct gk104_fifo_chan *chan; int runlist = -1, ret = -ENOSYS, i, j; u32 engines = 0, present = 0; @@ -302,19 +301,7 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func, *chid = chan->base.chid; - /* Page directory. */ - ret = nvkm_gpuobj_new(device, 0x10000, 0x1000, false, NULL, &chan->pgd); - if (ret) - return ret; - - nvkm_kmap(chan->base.inst); - nvkm_wo32(chan->base.inst, 0x0200, lower_32_bits(chan->pgd->addr)); - nvkm_wo32(chan->base.inst, 0x0204, upper_32_bits(chan->pgd->addr)); - nvkm_wo32(chan->base.inst, 0x0208, 0xffffffff); - nvkm_wo32(chan->base.inst, 0x020c, 0x000000ff); - nvkm_done(chan->base.inst); - - ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd); + ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst->memory); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c index 1b7f48efd8b12..14be41f24155a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c @@ -60,7 +60,7 @@ nvkm_falcon_read_dmem(struct nvkm_falcon *falcon, u32 start, u32 size, u8 port, } void -nvkm_falcon_bind_context(struct nvkm_falcon *falcon, struct nvkm_gpuobj *inst) +nvkm_falcon_bind_context(struct nvkm_falcon *falcon, struct nvkm_memory *inst) { if (!falcon->func->bind_context) { nvkm_error(falcon->user, diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c b/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c index 669c240284709..9def926f24d43 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c @@ -180,7 +180,7 @@ nvkm_falcon_v1_read_dmem(struct nvkm_falcon *falcon, u32 start, u32 size, } static void -nvkm_falcon_v1_bind_context(struct nvkm_falcon *falcon, struct nvkm_gpuobj *ctx) +nvkm_falcon_v1_bind_context(struct nvkm_falcon *falcon, struct nvkm_memory *ctx) { u32 inst_loc; u32 fbif; @@ -216,7 +216,7 @@ nvkm_falcon_v1_bind_context(struct nvkm_falcon *falcon, struct nvkm_gpuobj *ctx) nvkm_falcon_wr32(falcon, fbif + 4 * FALCON_DMAIDX_PHYS_SYS_NCOH, 0x6); /* Set context */ - switch (nvkm_memory_target(ctx->memory)) { + switch (nvkm_memory_target(ctx)) { case NVKM_MEM_TARGET_VRAM: inst_loc = 0; break; case NVKM_MEM_TARGET_HOST: inst_loc = 2; break; case NVKM_MEM_TARGET_NCOH: inst_loc = 3; break; @@ -228,7 +228,7 @@ nvkm_falcon_v1_bind_context(struct nvkm_falcon *falcon, struct nvkm_gpuobj *ctx) /* Enable context */ nvkm_falcon_mask(falcon, 0x048, 0x1, 0x1); nvkm_falcon_wr32(falcon, 0x054, - ((ctx->addr >> 12) & 0xfffffff) | + ((nvkm_memory_addr(ctx) >> 12) & 0xfffffff) | (inst_loc << 28) | (1 << 30)); nvkm_falcon_mask(falcon, 0x090, 0x10000, 0x10000); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index 8b588d1c776b8..8077e1a5017a0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -23,7 +23,7 @@ */ #include "gf100.h" -#include +#include #include #include #include @@ -53,7 +53,7 @@ gf100_bar_bar1_init(struct nvkm_bar *base) { struct nvkm_device *device = base->subdev.device; struct gf100_bar *bar = gf100_bar(base); - const u32 addr = nvkm_memory_addr(bar->bar[1].mem) >> 12; + const u32 addr = nvkm_memory_addr(bar->bar[1].inst) >> 12; nvkm_wr32(device, 0x001704, 0x80000000 | addr); } @@ -74,7 +74,7 @@ gf100_bar_bar2_init(struct nvkm_bar *base) { struct nvkm_device *device = base->subdev.device; struct gf100_bar *bar = gf100_bar(base); - u32 addr = nvkm_memory_addr(bar->bar[0].mem) >> 12; + u32 addr = nvkm_memory_addr(bar->bar[0].inst) >> 12; if (bar->bar2_halve) addr |= 0x40000000; nvkm_wr32(device, 0x001714, 0x80000000 | addr); @@ -90,11 +90,7 @@ gf100_bar_oneinit_bar(struct gf100_bar *bar, struct gf100_barN *bar_vm, int ret; ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0, false, - &bar_vm->mem); - if (ret) - return ret; - - ret = nvkm_gpuobj_new(device, 0x8000, 0, false, NULL, &bar_vm->pgd); + &bar_vm->inst); if (ret) return ret; @@ -119,17 +115,11 @@ gf100_bar_oneinit_bar(struct gf100_bar *bar, struct gf100_barN *bar_vm, } } - ret = nvkm_vm_ref(vm, &bar_vm->vm, bar_vm->pgd); + ret = nvkm_vm_ref(vm, &bar_vm->vm, bar_vm->inst); nvkm_vm_ref(NULL, &vm, NULL); if (ret) return ret; - nvkm_kmap(bar_vm->mem); - nvkm_wo32(bar_vm->mem, 0x0200, lower_32_bits(bar_vm->pgd->addr)); - nvkm_wo32(bar_vm->mem, 0x0204, upper_32_bits(bar_vm->pgd->addr)); - nvkm_wo32(bar_vm->mem, 0x0208, lower_32_bits(bar_len - 1)); - nvkm_wo32(bar_vm->mem, 0x020c, upper_32_bits(bar_len - 1)); - nvkm_done(bar_vm->mem); return 0; } @@ -164,13 +154,11 @@ gf100_bar_dtor(struct nvkm_bar *base) { struct gf100_bar *bar = gf100_bar(base); - nvkm_vm_ref(NULL, &bar->bar[1].vm, bar->bar[1].pgd); - nvkm_gpuobj_del(&bar->bar[1].pgd); - nvkm_memory_unref(&bar->bar[1].mem); + nvkm_vm_ref(NULL, &bar->bar[1].vm, bar->bar[1].inst); + nvkm_memory_unref(&bar->bar[1].inst); - nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].pgd); - nvkm_gpuobj_del(&bar->bar[0].pgd); - nvkm_memory_unref(&bar->bar[0].mem); + nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].inst); + nvkm_memory_unref(&bar->bar[0].inst); return bar; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h index fc1a7bd975a72..9ce80c6b6fec1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h @@ -4,8 +4,7 @@ #include "priv.h" struct gf100_barN { - struct nvkm_memory *mem; - struct nvkm_gpuobj *pgd; + struct nvkm_memory *inst; struct nvkm_vm *vm; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index b40c131af20e0..9300529917fce 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -140,7 +140,7 @@ nv50_bar_oneinit(struct nvkm_bar *base) if (ret) return ret; - ret = nvkm_vm_ref(vm, &bar->bar2_vm, bar->pgd); + ret = nvkm_vm_ref(vm, &bar->bar2_vm, bar->mem->memory); nvkm_vm_ref(NULL, &vm, NULL); if (ret) return ret; @@ -172,7 +172,7 @@ nv50_bar_oneinit(struct nvkm_bar *base) atomic_inc(&vm->engref[NVKM_SUBDEV_BAR]); - ret = nvkm_vm_ref(vm, &bar->bar1_vm, bar->pgd); + ret = nvkm_vm_ref(vm, &bar->bar1_vm, bar->mem->memory); nvkm_vm_ref(NULL, &vm, NULL); if (ret) return ret; @@ -197,13 +197,15 @@ void * nv50_bar_dtor(struct nvkm_bar *base) { struct nv50_bar *bar = nv50_bar(base); - nvkm_gpuobj_del(&bar->bar1); - nvkm_vm_ref(NULL, &bar->bar1_vm, bar->pgd); - nvkm_gpuobj_del(&bar->bar2); - nvkm_vm_ref(NULL, &bar->bar2_vm, bar->pgd); - nvkm_gpuobj_del(&bar->pgd); - nvkm_gpuobj_del(&bar->pad); - nvkm_gpuobj_del(&bar->mem); + if (bar->mem) { + nvkm_gpuobj_del(&bar->bar1); + nvkm_vm_ref(NULL, &bar->bar1_vm, bar->mem->memory); + nvkm_gpuobj_del(&bar->bar2); + nvkm_vm_ref(NULL, &bar->bar2_vm, bar->mem->memory); + nvkm_gpuobj_del(&bar->pgd); + nvkm_gpuobj_del(&bar->pad); + nvkm_gpuobj_del(&bar->mem); + } return bar; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 9fdd1446da5f4..22264d3db22fb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -446,7 +446,6 @@ static void nvkm_vm_unmap_pgt(struct nvkm_vm *vm, int big, u32 fpde, u32 lpde) { struct nvkm_mmu *mmu = vm->mmu; - struct nvkm_vm_pgd *vpgd; struct nvkm_vm_pgt *vpgt; struct nvkm_memory *pgt; u32 pde; @@ -459,9 +458,8 @@ nvkm_vm_unmap_pgt(struct nvkm_vm *vm, int big, u32 fpde, u32 lpde) pgt = vpgt->mem[big]; vpgt->mem[big] = NULL; - list_for_each_entry(vpgd, &vm->pgd_list, head) { - mmu->func->map_pgt(vpgd->obj, pde, vpgt->mem); - } + if (mmu->func->map_pgt) + mmu->func->map_pgt(vm, pde, vpgt->mem); mmu->func->flush(vm); @@ -474,7 +472,6 @@ nvkm_vm_map_pgt(struct nvkm_vm *vm, u32 pde, u32 type) { struct nvkm_mmu *mmu = vm->mmu; struct nvkm_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde]; - struct nvkm_vm_pgd *vpgd; int big = (type != mmu->func->spg_shift); u32 pgt_size; int ret; @@ -487,9 +484,8 @@ nvkm_vm_map_pgt(struct nvkm_vm *vm, u32 pde, u32 type) if (unlikely(ret)) return ret; - list_for_each_entry(vpgd, &vm->pgd_list, head) { - mmu->func->map_pgt(vpgd->obj, pde, vpgt->mem); - } + if (mmu->func->map_pgt) + mmu->func->map_pgt(vm, pde, vpgt->mem); vpgt->refcount[big]++; return 0; @@ -592,7 +588,6 @@ nvkm_vm_legacy(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, u64 mm_length = (offset + length) - mm_offset; int ret; - INIT_LIST_HEAD(&vm->pgd_list); kref_init(&vm->refcount); vm->fpde = offset >> (mmu->func->pgt_bits + 12); vm->lpde = (offset + length - 1) >> (mmu->func->pgt_bits + 12); @@ -644,58 +639,10 @@ nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset, return -EINVAL; } -static int -nvkm_vm_link(struct nvkm_vm *vm, struct nvkm_gpuobj *pgd) -{ - struct nvkm_mmu *mmu = vm->mmu; - struct nvkm_vm_pgd *vpgd; - int i; - - if (!pgd) - return 0; - - vpgd = kzalloc(sizeof(*vpgd), GFP_KERNEL); - if (!vpgd) - return -ENOMEM; - - vpgd->obj = pgd; - - mutex_lock(&vm->mutex); - for (i = vm->fpde; i <= vm->lpde; i++) - mmu->func->map_pgt(pgd, i, vm->pgt[i - vm->fpde].mem); - list_add(&vpgd->head, &vm->pgd_list); - mutex_unlock(&vm->mutex); - return 0; -} - -static void -nvkm_vm_unlink(struct nvkm_vm *vm, struct nvkm_gpuobj *mpgd) -{ - struct nvkm_vm_pgd *vpgd, *tmp; - - if (!mpgd) - return; - - mutex_lock(&vm->mutex); - list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) { - if (vpgd->obj == mpgd) { - list_del(&vpgd->head); - kfree(vpgd); - break; - } - } - mutex_unlock(&vm->mutex); -} - static void nvkm_vm_del(struct kref *kref) { struct nvkm_vm *vm = container_of(kref, typeof(*vm), refcount); - struct nvkm_vm_pgd *vpgd, *tmp; - - list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) { - nvkm_vm_unlink(vm, vpgd->obj); - } nvkm_mm_fini(&vm->mm); vfree(vm->pgt); @@ -705,20 +652,28 @@ nvkm_vm_del(struct kref *kref) } int -nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_gpuobj *pgd) +nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_memory *inst) { if (ref) { - int ret = nvkm_vm_link(ref, pgd); - if (ret) - return ret; + if (ref->func->join && inst) { + int ret = ref->func->join(ref, inst), i; + if (ret) + return ret; + + if (ref->mmu->func->map_pgt) { + for (i = ref->fpde; i <= ref->lpde; i++) + ref->mmu->func->map_pgt(ref, i, ref->pgt[i - ref->fpde].mem); + } + } kref_get(&ref->refcount); } if (*ptr) { - if ((*ptr)->bootstrapped && pgd) + if ((*ptr)->func->part && inst) + (*ptr)->func->part(*ptr, inst); + if ((*ptr)->bootstrapped && inst) nvkm_memory_unref(&(*ptr)->pgt[0].mem[0]); - nvkm_vm_unlink(*ptr, pgd); kref_put(&(*ptr)->refcount, nvkm_vm_del); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index 536260782cff4..167c201d29e35 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -70,8 +70,9 @@ const u8 gf100_pte_storage_type_map[256] = void -gf100_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 index, struct nvkm_memory *pgt[2]) +gf100_vm_map_pgt(struct nvkm_vmm *vmm, u32 index, struct nvkm_memory *pgt[2]) { + struct nvkm_memory *pgd = vmm->pd->pt[0]->memory; u32 pde[2] = { 0, 0 }; if (pgt[0]) @@ -161,7 +162,6 @@ gf100_vm_flush(struct nvkm_vm *vm) { struct nvkm_mmu *mmu = vm->mmu; struct nvkm_device *device = mmu->subdev.device; - struct nvkm_vm_pgd *vpgd; u32 type; type = 0x00000001; /* PAGE_ALL */ @@ -169,24 +169,22 @@ gf100_vm_flush(struct nvkm_vm *vm) type |= 0x00000004; /* HUB_ONLY */ mutex_lock(&mmu->subdev.mutex); - list_for_each_entry(vpgd, &vm->pgd_list, head) { - /* looks like maybe a "free flush slots" counter, the - * faster you write to 0x100cbc to more it decreases - */ - nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x100c80) & 0x00ff0000) - break; - ); - - nvkm_wr32(device, 0x100cb8, vpgd->obj->addr >> 8); - nvkm_wr32(device, 0x100cbc, 0x80000000 | type); - - /* wait for flush to be queued? */ - nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x100c80) & 0x00008000) - break; - ); - } + /* looks like maybe a "free flush slots" counter, the + * faster you write to 0x100cbc to more it decreases + */ + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x100c80) & 0x00ff0000) + break; + ); + + nvkm_wr32(device, 0x100cb8, vm->pd->pt[0]->addr >> 8); + nvkm_wr32(device, 0x100cbc, 0x80000000 | type); + + /* wait for flush to be queued? */ + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x100c80) & 0x00008000) + break; + ); mutex_unlock(&mmu->subdev.mutex); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c index ba1c04e007016..da75d1da0658e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c @@ -31,8 +31,10 @@ #include void -nv50_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 pde, struct nvkm_memory *pgt[2]) +nv50_vm_map_pgt(struct nvkm_vmm *vmm, u32 pde, struct nvkm_memory *pgt[2]) { + struct nvkm_vmm_join *join; + u32 pdeo = vmm->mmu->func->vmm.pd_offset + (pde * 8); u64 phys = 0xdeadcafe00000000ULL; u32 coverage = 0; @@ -56,10 +58,12 @@ nv50_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 pde, struct nvkm_memory *pgt[2]) phys |= 0x20; } - nvkm_kmap(pgd); - nvkm_wo32(pgd, (pde * 8) + 0, lower_32_bits(phys)); - nvkm_wo32(pgd, (pde * 8) + 4, upper_32_bits(phys)); - nvkm_done(pgd); + list_for_each_entry(join, &vmm->join, head) { + nvkm_kmap(join->inst); + nvkm_wo32(join->inst, pdeo + 0, lower_32_bits(phys)); + nvkm_wo32(join->inst, pdeo + 4, upper_32_bits(phys)); + nvkm_done(join->inst); + } } static inline u64 diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index f490a636e9f01..3271ec1c3c839 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -18,7 +18,7 @@ struct nvkm_mmu_func { u8 spg_shift; u8 lpg_shift; - void (*map_pgt)(struct nvkm_gpuobj *pgd, u32 pde, + void (*map_pgt)(struct nvkm_vmm *, u32 pde, struct nvkm_memory *pgt[2]); void (*map)(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, u32 pte, u32 cnt, @@ -41,7 +41,7 @@ struct nvkm_mmu_func { extern const struct nvkm_mmu_func nv04_mmu; -void nv50_vm_map_pgt(struct nvkm_gpuobj *, u32, struct nvkm_memory **); +void nv50_vm_map_pgt(struct nvkm_vmm *, u32, struct nvkm_memory **); void nv50_vm_map(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, u32, u32, u64, u64); void nv50_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, @@ -49,7 +49,7 @@ void nv50_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, void nv50_vm_unmap(struct nvkm_vma *, struct nvkm_memory *, u32, u32); void nv50_vm_flush(struct nvkm_vm *); -void gf100_vm_map_pgt(struct nvkm_gpuobj *, u32, struct nvkm_memory **); +void gf100_vm_map_pgt(struct nvkm_vmm *, u32, struct nvkm_memory **); void gf100_vm_map(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, u32, u32, u64, u64); void gf100_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c index 25e86ce3d29c8..7e00b9adca05e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -131,7 +131,7 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, /* ... and the GPU storage for it, except on Tesla-class GPUs that * have the PD embedded in the instance structure. */ - if (desc->size && mmu->func->vmm.global) { + if (desc->size) { const u32 size = pd_header + desc->size * (1 << desc->bits); vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true); if (!vmm->pd->pt[0]) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c index d35c041fdcece..6c068c5bd3c2d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c @@ -112,11 +112,8 @@ gm200_secboot_oneinit(struct nvkm_secboot *sb) int ret; /* Allocate instance block and VM */ - ret = nvkm_gpuobj_new(device, 0x1000, 0, true, NULL, &gsb->inst); - if (ret) - return ret; - - ret = nvkm_gpuobj_new(device, 0x8000, 0, true, NULL, &gsb->pgd); + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0, true, + &gsb->inst); if (ret) return ret; @@ -126,18 +123,11 @@ gm200_secboot_oneinit(struct nvkm_secboot *sb) atomic_inc(&vm->engref[NVKM_SUBDEV_PMU]); - ret = nvkm_vm_ref(vm, &gsb->vm, gsb->pgd); + ret = nvkm_vm_ref(vm, &gsb->vm, gsb->inst); nvkm_vm_ref(NULL, &vm, NULL); if (ret) return ret; - nvkm_kmap(gsb->inst); - nvkm_wo32(gsb->inst, 0x200, lower_32_bits(gsb->pgd->addr)); - nvkm_wo32(gsb->inst, 0x204, upper_32_bits(gsb->pgd->addr)); - nvkm_wo32(gsb->inst, 0x208, lower_32_bits(vm_area_len - 1)); - nvkm_wo32(gsb->inst, 0x20c, upper_32_bits(vm_area_len - 1)); - nvkm_done(gsb->inst); - if (sb->acr->func->oneinit) { ret = sb->acr->func->oneinit(sb->acr, sb); if (ret) @@ -165,9 +155,8 @@ gm200_secboot_dtor(struct nvkm_secboot *sb) sb->acr->func->dtor(sb->acr); - nvkm_vm_ref(NULL, &gsb->vm, gsb->pgd); - nvkm_gpuobj_del(&gsb->pgd); - nvkm_gpuobj_del(&gsb->inst); + nvkm_vm_ref(NULL, &gsb->vm, gsb->inst); + nvkm_memory_unref(&gsb->inst); return gsb; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h index c8ab3d76bdef7..a2b60fab42e88 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h @@ -29,8 +29,7 @@ struct gm200_secboot { struct nvkm_secboot base; /* Instance block & address space used for HS FW execution */ - struct nvkm_gpuobj *inst; - struct nvkm_gpuobj *pgd; + struct nvkm_memory *inst; struct nvkm_vm *vm; }; #define gm200_secboot(sb) container_of(sb, struct gm200_secboot, base) From ac47c15b76d0f2c42eedde7e993a0d3db92f215d Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 101/151] drm/nouveau/fifo/nv04-nv40: fix missing nvkm_kmap() calls around ramfc access Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c index 0a7b6ed5ed284..c213122cf0885 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c @@ -95,6 +95,7 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); c = fifo->ramfc; + nvkm_kmap(fctx); do { u32 rm = ((1ULL << c->bits) - 1) << c->regs; u32 cm = ((1ULL << c->bits) - 1) << c->ctxs; @@ -102,6 +103,7 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) u32 cv = (nvkm_ro32(fctx, c->ctxp + data) & ~cm); nvkm_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs)); } while ((++c)->bits); + nvkm_done(fctx); c = fifo->ramfc; do { From f8a1203920e7de49bd99bfed76a82f0c93de2020 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 102/151] drm/nouveau/ltc/gm200: limit NV_MMU_PTE_COMPTAGLINE bits to 16 where required If NV_PFB_MMU_CTRL_USE_FULL_COMP_TAG_LINE is TRUE, then the last bit of NV_MMU_PTE_COMPTAGLINE is re-purposed to select the upper/lower half of a compression tag when using 64KiB big pages. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c index 59acf568faeb2..a21ef45b85728 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c @@ -156,6 +156,7 @@ gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc) struct nvkm_device *device = ltc->subdev.device; struct nvkm_fb *fb = device->fb; struct nvkm_ram *ram = fb->ram; + u32 bits = (nvkm_rd32(device, 0x100c80) & 0x00001000) ? 16 : 17; u32 tag_size, tag_margin, tag_align; int ret; @@ -167,8 +168,8 @@ gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc) /* tags for 1/4 of VRAM should be enough (8192/4 per GiB of VRAM) */ ltc->num_tags = (ram->size >> 17) / 4; - if (ltc->num_tags > (1 << 17)) - ltc->num_tags = 1 << 17; /* we have 17 bits in PTE */ + if (ltc->num_tags > (1 << bits)) + ltc->num_tags = 1 << bits; /* we have 16/17 bits in PTE */ ltc->num_tags = (ltc->num_tags + 63) & ~63; /* round up to 64 */ tag_align = ltc->ltc_nr * 0x800; From bda9e379f61f937af7ee9b2a32d54bc50a94cd80 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 103/151] drm/nouveau/fb/gm200: enable NV_PFB_MMU_CTRL_USE_FULL_COMP_TAG_LINE where appropriate To avoid wasting compression tags when using 64KiB pages, we need to enable this so we can select between upper/lower comptagline in PTEs. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c index cc3f90304c1c5..1be0b9e98b840 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c @@ -31,9 +31,9 @@ gm200_fb_init_page(struct nvkm_fb *fb) { struct nvkm_device *device = fb->subdev.device; switch (fb->page) { - case 16: nvkm_mask(device, 0x100c80, 0x00000801, 0x00000001); break; - case 17: nvkm_mask(device, 0x100c80, 0x00000801, 0x00000000); break; - case 0: nvkm_mask(device, 0x100c80, 0x00000800, 0x00000800); break; + case 16: nvkm_mask(device, 0x100c80, 0x00001801, 0x00001001); break; + case 17: nvkm_mask(device, 0x100c80, 0x00001801, 0x00000000); break; + case 0: nvkm_mask(device, 0x100c80, 0x00001800, 0x00001800); break; default: return -EINVAL; } From eb813999f20097d24310836dfa07a97e2eb0c936 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 104/151] drm/nouveau/mmu: implement new vmm backend This is the common code to support a rework of the VMM backends. It adds support for more than 2 levels of page table nesting, which is required to be able to support GP100's MMU layout. Sparse mappings (that don't cause MMU faults when accessed) are now supported, where the backend provides it. Dual-PT handling had to become more sophisticated to support sparse, but this also allows us to support an optimisation the MMU provides on GK104 and newer. Certain operations can now be combined into a single page tree walk to avoid some overhead, but also enables optimsations like skipping PTE unmap writes when the PT will be destroyed anyway. The old backend has been hacked up to forward requests onto the new backend, if present, so that it's possible to bisect between issues in the backend changes vs the upcoming frontend changes. Until the new frontend has been merged, new backends will leak BAR2 page tables on module unload. This is expected, and it's not worth the effort of hacking around this as it doesn't effect runtime. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/Kconfig | 7 + .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 23 + .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 117 +++- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c | 580 ++++++++++++++++++ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 112 ++++ 5 files changed, 836 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/nouveau/Kconfig b/drivers/gpu/drm/nouveau/Kconfig index c02a13406a816..4b75ad40dd80e 100644 --- a/drivers/gpu/drm/nouveau/Kconfig +++ b/drivers/gpu/drm/nouveau/Kconfig @@ -56,6 +56,13 @@ config NOUVEAU_DEBUG_DEFAULT help Selects the default debug level +config NOUVEAU_DEBUG_MMU + bool "Enable additional MMU debugging" + depends on DRM_NOUVEAU + default n + help + Say Y here if you want to enable verbose MMU debug output. + config DRM_NOUVEAU_BACKLIGHT bool "Support for backlight control" depends on DRM_NOUVEAU diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index eb6704a2fc8ff..ccd87d508d048 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -1,6 +1,7 @@ #ifndef __NVKM_MMU_H__ #define __NVKM_MMU_H__ #include +#include #include struct nvkm_gpuobj; struct nvkm_mem; @@ -11,6 +12,8 @@ struct nvkm_vm_pgt { }; struct nvkm_vma { + struct nvkm_memory *memory; + struct nvkm_tags *tags; struct nvkm_vm *vm; struct nvkm_mm_node *node; union { @@ -24,6 +27,7 @@ struct nvkm_vm { const struct nvkm_vmm_func *func; struct nvkm_mmu *mmu; const char *name; + u32 debug; struct kref kref; struct mutex mutex; @@ -58,6 +62,25 @@ void nvkm_vm_map_at(struct nvkm_vma *, u64 offset, struct nvkm_mem *); void nvkm_vm_unmap(struct nvkm_vma *); void nvkm_vm_unmap_at(struct nvkm_vma *, u64 offset, u64 length); +int nvkm_vmm_boot(struct nvkm_vmm *); + +struct nvkm_vmm_map { + struct nvkm_memory *memory; + u64 offset; + + struct nvkm_mm_node *mem; + struct scatterlist *sgl; + dma_addr_t *dma; + u64 off; + + const struct nvkm_vmm_page *page; + + struct nvkm_tags *tags; + u64 next; + u64 type; + u64 ctag; +}; + struct nvkm_mmu { const struct nvkm_mmu_func *func; struct nvkm_subdev subdev; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 22264d3db22fb..536187952372d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -213,6 +213,36 @@ nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero) return pt; } +static void +nvkm_vm_map_(const struct nvkm_vmm_page *page, struct nvkm_vma *vma, u64 delta, + struct nvkm_mem *mem, nvkm_vmm_pte_func fn, + struct nvkm_vmm_map *map) +{ + struct nvkm_vmm *vmm = vma->vm; + void *argv = NULL; + u32 argc = 0; + int ret; + + map->memory = mem->memory; + map->page = page; + + if (vmm->func->valid) { + ret = vmm->func->valid(vmm, argv, argc, map); + if (WARN_ON(ret)) + return; + } + + mutex_lock(&vmm->mutex); + nvkm_vmm_ptes_map(vmm, page, ((u64)vma->node->offset << 12) + delta, + (u64)vma->node->length << 12, map, fn); + mutex_unlock(&vmm->mutex); + + nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); + nvkm_memory_unref(&vma->memory); + vma->memory = nvkm_memory_ref(map->memory); + vma->tags = map->tags; +} + void nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu) { @@ -251,6 +281,7 @@ nvkm_mmu_ptc_init(struct nvkm_mmu *mmu) void nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node) { + const struct nvkm_vmm_page *page = vma->vm->func->page; struct nvkm_vm *vm = vma->vm; struct nvkm_mmu *mmu = vm->mmu; struct nvkm_mm_node *r = node->mem; @@ -262,6 +293,14 @@ nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node) u32 max = 1 << (mmu->func->pgt_bits - bits); u32 end, len; + if (page->desc->func->unmap) { + struct nvkm_vmm_map map = { .mem = node->mem }; + while (page->shift != vma->node->type) + page++; + nvkm_vm_map_(page, vma, delta, node, page->desc->func->mem, &map); + return; + } + delta = 0; while (r) { u64 phys = (u64)r->offset << 12; @@ -297,6 +336,7 @@ static void nvkm_vm_map_sg_table(struct nvkm_vma *vma, u64 delta, u64 length, struct nvkm_mem *mem) { + const struct nvkm_vmm_page *page = vma->vm->func->page; struct nvkm_vm *vm = vma->vm; struct nvkm_mmu *mmu = vm->mmu; int big = vma->node->type != mmu->func->spg_shift; @@ -311,6 +351,14 @@ nvkm_vm_map_sg_table(struct nvkm_vma *vma, u64 delta, u64 length, int i; struct scatterlist *sg; + if (page->desc->func->unmap) { + struct nvkm_vmm_map map = { .sgl = mem->sg->sgl }; + while (page->shift != vma->node->type) + page++; + nvkm_vm_map_(page, vma, delta, mem, page->desc->func->sgl, &map); + return; + } + for_each_sg(mem->sg->sgl, sg, mem->sg->nents, i) { struct nvkm_memory *pgt = vm->pgt[pde].mem[big]; sglen = sg_dma_len(sg) >> PAGE_SHIFT; @@ -355,6 +403,7 @@ static void nvkm_vm_map_sg(struct nvkm_vma *vma, u64 delta, u64 length, struct nvkm_mem *mem) { + const struct nvkm_vmm_page *page = vma->vm->func->page; struct nvkm_vm *vm = vma->vm; struct nvkm_mmu *mmu = vm->mmu; dma_addr_t *list = mem->pages; @@ -367,6 +416,14 @@ nvkm_vm_map_sg(struct nvkm_vma *vma, u64 delta, u64 length, u32 max = 1 << (mmu->func->pgt_bits - bits); u32 end, len; + if (page->desc->func->unmap) { + struct nvkm_vmm_map map = { .dma = mem->pages }; + while (page->shift != vma->node->type) + page++; + nvkm_vm_map_(page, vma, delta, mem, page->desc->func->dma, &map); + return; + } + while (num) { struct nvkm_memory *pgt = vm->pgt[pde].mem[big]; @@ -415,6 +472,17 @@ nvkm_vm_unmap_at(struct nvkm_vma *vma, u64 delta, u64 length) u32 max = 1 << (mmu->func->pgt_bits - bits); u32 end, len; + if (vm->func->page->desc->func->unmap) { + const struct nvkm_vmm_page *page = vm->func->page; + while (page->shift != vma->node->type) + page++; + mutex_lock(&vm->mutex); + nvkm_vmm_ptes_unmap(vm, page, (vma->node->offset << 12) + delta, + vma->node->length << 12, false); + mutex_unlock(&vm->mutex); + return; + } + while (num) { struct nvkm_memory *pgt = vm->pgt[pde].mem[big]; @@ -440,6 +508,9 @@ void nvkm_vm_unmap(struct nvkm_vma *vma) { nvkm_vm_unmap_at(vma, 0, (u64)vma->node->length << 12); + + nvkm_memory_tags_put(vma->memory, vma->vm->mmu->subdev.device, &vma->tags); + nvkm_memory_unref(&vma->memory); } static void @@ -509,6 +580,22 @@ nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access, return ret; } + if (vm->func->page->desc->func->unmap) { + const struct nvkm_vmm_page *page = vm->func->page; + while (page->shift != page_shift) + page++; + + ret = nvkm_vmm_ptes_get(vm, page, vma->node->offset << 12, + vma->node->length << 12); + if (ret) { + nvkm_mm_free(&vm->mm, &vma->node); + mutex_unlock(&vm->mutex); + return ret; + } + + goto done; + } + fpde = (vma->node->offset >> mmu->func->pgt_bits); lpde = (vma->node->offset + vma->node->length - 1) >> mmu->func->pgt_bits; @@ -530,8 +617,11 @@ nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access, return ret; } } +done: mutex_unlock(&vm->mutex); + vma->memory = NULL; + vma->tags = NULL; vma->vm = NULL; nvkm_vm_ref(vm, &vma->vm, NULL); vma->offset = (u64)vma->node->offset << 12; @@ -551,11 +641,25 @@ nvkm_vm_put(struct nvkm_vma *vma) vm = vma->vm; mmu = vm->mmu; + nvkm_memory_tags_put(vma->memory, mmu->subdev.device, &vma->tags); + nvkm_memory_unref(&vma->memory); + fpde = (vma->node->offset >> mmu->func->pgt_bits); lpde = (vma->node->offset + vma->node->length - 1) >> mmu->func->pgt_bits; mutex_lock(&vm->mutex); + if (vm->func->page->desc->func->unmap) { + const struct nvkm_vmm_page *page = vm->func->page; + while (page->shift != vma->node->type) + page++; + + nvkm_vmm_ptes_put(vm, page, vma->node->offset << 12, + vma->node->length << 12); + goto done; + } + nvkm_vm_unmap_pgt(vm, vma->node->type != mmu->func->spg_shift, fpde, lpde); +done: nvkm_mm_free(&vm->mm, &vma->node); mutex_unlock(&vm->mutex); @@ -569,6 +673,9 @@ nvkm_vm_boot(struct nvkm_vm *vm, u64 size) struct nvkm_memory *pgt; int ret; + if (vm->func->page->desc->func->unmap) + return nvkm_vmm_boot(vm); + ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST, (size >> mmu->func->spg_shift) * 8, 0x1000, true, &pgt); if (ret == 0) { @@ -660,7 +767,7 @@ nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_memory *inst) if (ret) return ret; - if (ref->mmu->func->map_pgt) { + if (!ref->func->page->desc->func->unmap && ref->mmu->func->map_pgt) { for (i = ref->fpde; i <= ref->lpde; i++) ref->mmu->func->map_pgt(ref, i, ref->pgt[i - ref->fpde].mem); } @@ -672,8 +779,12 @@ nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_memory *inst) if (*ptr) { if ((*ptr)->func->part && inst) (*ptr)->func->part(*ptr, inst); - if ((*ptr)->bootstrapped && inst) - nvkm_memory_unref(&(*ptr)->pgt[0].mem[0]); + if ((*ptr)->bootstrapped && inst) { + if (!(*ptr)->func->page->desc->func->unmap) { + nvkm_memory_unref(&(*ptr)->pgt[0].mem[0]); + (*ptr)->bootstrapped = false; + } + } kref_put(&(*ptr)->refcount, nvkm_vm_del); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c index 7e00b9adca05e..46c7fecf0054b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -67,9 +67,559 @@ nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse, return pgt; } +struct nvkm_vmm_iter { + const struct nvkm_vmm_page *page; + const struct nvkm_vmm_desc *desc; + struct nvkm_vmm *vmm; + u64 cnt; + u16 max, lvl; + u32 pte[NVKM_VMM_LEVELS_MAX]; + struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX]; + int flush; +}; + +#ifdef CONFIG_NOUVEAU_DEBUG_MMU +static const char * +nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc) +{ + switch (desc->type) { + case PGD: return "PGD"; + case PGT: return "PGT"; + case SPT: return "SPT"; + case LPT: return "LPT"; + default: + return "UNKNOWN"; + } +} + +static void +nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf) +{ + int lvl; + for (lvl = it->max; lvl >= 0; lvl--) { + if (lvl >= it->lvl) + buf += sprintf(buf, "%05x:", it->pte[lvl]); + else + buf += sprintf(buf, "xxxxx:"); + } +} + +#define TRA(i,f,a...) do { \ + char _buf[NVKM_VMM_LEVELS_MAX * 7]; \ + struct nvkm_vmm_iter *_it = (i); \ + nvkm_vmm_trace(_it, _buf); \ + VMM_TRACE(_it->vmm, "%s "f, _buf, ##a); \ +} while(0) +#else +#define TRA(i,f,a...) +#endif + +static inline void +nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it) +{ + it->flush = min(it->flush, it->max - it->lvl); +} + +static inline void +nvkm_vmm_flush(struct nvkm_vmm_iter *it) +{ + if (it->flush != NVKM_VMM_LEVELS_MAX) { + if (it->vmm->func->flush) { + TRA(it, "flush: %d", it->flush); + it->vmm->func->flush(it->vmm, it->flush); + } + it->flush = NVKM_VMM_LEVELS_MAX; + } +} + +static void +nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it) +{ + const struct nvkm_vmm_desc *desc = it->desc; + const int type = desc[it->lvl].type == SPT; + struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1]; + struct nvkm_vmm_pt *pgt = it->pt[it->lvl]; + struct nvkm_mmu_pt *pt = pgt->pt[type]; + struct nvkm_vmm *vmm = it->vmm; + u32 pdei = it->pte[it->lvl + 1]; + + /* Recurse up the tree, unreferencing/destroying unneeded PDs. */ + it->lvl++; + if (--pgd->refs[0]) { + const struct nvkm_vmm_desc_func *func = desc[it->lvl].func; + /* PD has other valid PDEs, so we need a proper update. */ + TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1])); + pgt->pt[type] = NULL; + if (!pgt->refs[!type]) { + /* PDE no longer required. */ + if (pgd->pt[0]) { + if (pgt->sparse) { + func->sparse(vmm, pgd->pt[0], pdei, 1); + pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE; + } else { + func->unmap(vmm, pgd->pt[0], pdei, 1); + pgd->pde[pdei] = NULL; + } + } else { + /* Special handling for Tesla-class GPUs, + * where there's no central PD, but each + * instance has its own embedded PD. + */ + func->pde(vmm, pgd, pdei); + pgd->pde[pdei] = NULL; + } + } else { + /* PDE was pointing at dual-PTs and we're removing + * one of them, leaving the other in place. + */ + func->pde(vmm, pgd, pdei); + } + + /* GPU may have cached the PTs, flush before freeing. */ + nvkm_vmm_flush_mark(it); + nvkm_vmm_flush(it); + } else { + /* PD has no valid PDEs left, so we can just destroy it. */ + nvkm_vmm_unref_pdes(it); + } + + /* Destroy PD/PT. */ + TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1])); + nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt); + if (!pgt->refs[!type]) + nvkm_vmm_pt_del(&pgt); + it->lvl--; +} + +static void +nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, + const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes) +{ + const struct nvkm_vmm_desc *pair = it->page[-1].desc; + const u32 sptb = desc->bits - pair->bits; + const u32 sptn = 1 << sptb; + struct nvkm_vmm *vmm = it->vmm; + u32 spti = ptei & (sptn - 1), lpti, pteb; + + /* Determine how many SPTEs are being touched under each LPTE, + * and drop reference counts. + */ + for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) { + const u32 pten = min(sptn - spti, ptes); + pgt->pte[lpti] -= pten; + ptes -= pten; + } + + /* We're done here if there's no corresponding LPT. */ + if (!pgt->refs[0]) + return; + + for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) { + /* Skip over any LPTEs that still have valid SPTEs. */ + if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) { + for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) { + if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)) + break; + } + continue; + } + + /* As there's no more non-UNMAPPED SPTEs left in the range + * covered by a number of LPTEs, the LPTEs once again take + * control over their address range. + * + * Determine how many LPTEs need to transition state. + */ + pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID; + for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) { + if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES) + break; + pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID; + } + + if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) { + TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes); + pair->func->sparse(vmm, pgt->pt[0], pteb, ptes); + } else + if (pair->func->invalid) { + /* If the MMU supports it, restore the LPTE to the + * INVALID state to tell the MMU there is no point + * trying to fetch the corresponding SPTEs. + */ + TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes); + pair->func->invalid(vmm, pgt->pt[0], pteb, ptes); + } + } +} + +static bool +nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +{ + const struct nvkm_vmm_desc *desc = it->desc; + const int type = desc->type == SPT; + struct nvkm_vmm_pt *pgt = it->pt[0]; + + /* Drop PTE references. */ + pgt->refs[type] -= ptes; + + /* Dual-PTs need special handling, unless PDE becoming invalid. */ + if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1])) + nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes); + + /* PT no longer neeed? Destroy it. */ + if (!pgt->refs[type]) { + it->lvl++; + TRA(it, "%s empty", nvkm_vmm_desc_type(desc)); + it->lvl--; + nvkm_vmm_unref_pdes(it); + return false; /* PTE writes for unmap() not necessary. */ + } + + return true; +} + +static void +nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, + const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes) +{ + const struct nvkm_vmm_desc *pair = it->page[-1].desc; + const u32 sptb = desc->bits - pair->bits; + const u32 sptn = 1 << sptb; + struct nvkm_vmm *vmm = it->vmm; + u32 spti = ptei & (sptn - 1), lpti, pteb; + + /* Determine how many SPTEs are being touched under each LPTE, + * and increase reference counts. + */ + for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) { + const u32 pten = min(sptn - spti, ptes); + pgt->pte[lpti] += pten; + ptes -= pten; + } + + /* We're done here if there's no corresponding LPT. */ + if (!pgt->refs[0]) + return; + + for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) { + /* Skip over any LPTEs that already have valid SPTEs. */ + if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) { + for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) { + if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID)) + break; + } + continue; + } + + /* As there are now non-UNMAPPED SPTEs in the range covered + * by a number of LPTEs, we need to transfer control of the + * address range to the SPTEs. + * + * Determine how many LPTEs need to transition state. + */ + pgt->pte[ptei] |= NVKM_VMM_PTE_VALID; + for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) { + if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID) + break; + pgt->pte[ptei] |= NVKM_VMM_PTE_VALID; + } + + if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) { + const u32 spti = pteb * sptn; + const u32 sptc = ptes * sptn; + /* The entire LPTE is marked as sparse, we need + * to make sure that the SPTEs are too. + */ + TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc); + desc->func->sparse(vmm, pgt->pt[1], spti, sptc); + /* Sparse LPTEs prevent SPTEs from being accessed. */ + TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes); + pair->func->unmap(vmm, pgt->pt[0], pteb, ptes); + } else + if (pair->func->invalid) { + /* MMU supports blocking SPTEs by marking an LPTE + * as INVALID. We need to reverse that here. + */ + TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes); + pair->func->unmap(vmm, pgt->pt[0], pteb, ptes); + } + } +} + +static bool +nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +{ + const struct nvkm_vmm_desc *desc = it->desc; + const int type = desc->type == SPT; + struct nvkm_vmm_pt *pgt = it->pt[0]; + + /* Take PTE references. */ + pgt->refs[type] += ptes; + + /* Dual-PTs need special handling. */ + if (desc->type == SPT) + nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes); + + return true; +} + +static void +nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc, + struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes) +{ + if (desc->type == PGD) { + while (ptes--) + pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE; + } else + if (desc->type == LPT) { + memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes); + } +} + +static bool +nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei) +{ + const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1]; + const int type = desc->type == SPT; + struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; + const bool zero = !pgt->sparse && !desc->func->invalid; + struct nvkm_vmm *vmm = it->vmm; + struct nvkm_mmu *mmu = vmm->mmu; + struct nvkm_mmu_pt *pt; + u32 pten = 1 << desc->bits; + u32 pteb, ptei, ptes; + u32 size = desc->size * pten; + + pgd->refs[0]++; + + pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero); + if (!pgt->pt[type]) { + it->lvl--; + nvkm_vmm_unref_pdes(it); + return false; + } + + if (zero) + goto done; + + pt = pgt->pt[type]; + + if (desc->type == LPT && pgt->refs[1]) { + /* SPT already exists covering the same range as this LPT, + * which means we need to be careful that any LPTEs which + * overlap valid SPTEs are unmapped as opposed to invalid + * or sparse, which would prevent the MMU from looking at + * the SPTEs on some GPUs. + */ + for (ptei = pteb = 0; ptei < pten; pteb = ptei) { + bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES; + for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) { + bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES; + if (spte != next) + break; + } + + if (!spte) { + if (pgt->sparse) + desc->func->sparse(vmm, pt, pteb, ptes); + else + desc->func->invalid(vmm, pt, pteb, ptes); + memset(&pgt->pte[pteb], 0x00, ptes); + } else { + desc->func->unmap(vmm, pt, pteb, ptes); + while (ptes--) + pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID; + } + } + } else { + if (pgt->sparse) { + nvkm_vmm_sparse_ptes(desc, pgt, 0, pten); + desc->func->sparse(vmm, pt, 0, pten); + } else { + desc->func->invalid(vmm, pt, 0, pten); + } + } + +done: + TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc)); + it->desc[it->lvl].func->pde(it->vmm, pgd, pdei); + nvkm_vmm_flush_mark(it); + return true; +} + +static bool +nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei) +{ + const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1]; + struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; + + pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page); + if (!pgt) { + if (!pgd->refs[0]) + nvkm_vmm_unref_pdes(it); + return false; + } + + pgd->pde[pdei] = pgt; + return true; +} + +static inline u64 +nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, + u64 addr, u64 size, const char *name, bool ref, + bool (*REF_PTES)(struct nvkm_vmm_iter *, u32, u32), + nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map, + nvkm_vmm_pxe_func CLR_PTES) +{ + const struct nvkm_vmm_desc *desc = page->desc; + struct nvkm_vmm_iter it; + u64 bits = addr >> page->shift; + + it.page = page; + it.desc = desc; + it.vmm = vmm; + it.cnt = size >> page->shift; + it.flush = NVKM_VMM_LEVELS_MAX; + + /* Deconstruct address into PTE indices for each mapping level. */ + for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) { + it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1); + bits >>= desc[it.lvl].bits; + } + it.max = --it.lvl; + it.pt[it.max] = vmm->pd; + + it.lvl = 0; + TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name, + addr, size, page->shift, it.cnt); + it.lvl = it.max; + + /* Depth-first traversal of page tables. */ + while (it.cnt) { + struct nvkm_vmm_pt *pgt = it.pt[it.lvl]; + const int type = desc->type == SPT; + const u32 pten = 1 << desc->bits; + const u32 ptei = it.pte[0]; + const u32 ptes = min_t(u64, it.cnt, pten - ptei); + + /* Walk down the tree, finding page tables for each level. */ + for (; it.lvl; it.lvl--) { + const u32 pdei = it.pte[it.lvl]; + struct nvkm_vmm_pt *pgd = pgt; + + /* Software PT. */ + if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) { + if (!nvkm_vmm_ref_swpt(&it, pgd, pdei)) + goto fail; + } + it.pt[it.lvl - 1] = pgt = pgd->pde[pdei]; + + /* Hardware PT. + * + * This is a separate step from above due to GF100 and + * newer having dual page tables at some levels, which + * are refcounted independently. + */ + if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) { + if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei)) + goto fail; + } + } + + /* Handle PTE updates. */ + if (!REF_PTES || REF_PTES(&it, ptei, ptes)) { + struct nvkm_mmu_pt *pt = pgt->pt[type]; + if (MAP_PTES || CLR_PTES) { + if (MAP_PTES) + MAP_PTES(vmm, pt, ptei, ptes, map); + else + CLR_PTES(vmm, pt, ptei, ptes); + nvkm_vmm_flush_mark(&it); + } + } + + /* Walk back up the tree to the next position. */ + it.pte[it.lvl] += ptes; + it.cnt -= ptes; + if (it.cnt) { + while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) { + it.pte[it.lvl++] = 0; + it.pte[it.lvl]++; + } + } + }; + + nvkm_vmm_flush(&it); + return ~0ULL; + +fail: + /* Reconstruct the failure address so the caller is able to + * reverse any partially completed operations. + */ + addr = it.pte[it.max--]; + do { + addr = addr << desc[it.max].bits; + addr |= it.pte[it.max]; + } while (it.max--); + + return addr << page->shift; +} + +void +nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, + u64 addr, u64 size, bool sparse) +{ + const struct nvkm_vmm_desc_func *func = page->desc->func; + nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, NULL, NULL, NULL, + sparse ? func->sparse : func->invalid ? func->invalid : + func->unmap); +} + +void +nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, + u64 addr, u64 size, struct nvkm_vmm_map *map, + nvkm_vmm_pte_func func) +{ + nvkm_vmm_iter(vmm, page, addr, size, "map", false, + NULL, func, map, NULL); +} + +void +nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, + u64 addr, u64 size) +{ + nvkm_vmm_iter(vmm, page, addr, size, "unref", false, + nvkm_vmm_unref_ptes, NULL, NULL, NULL); +} + +int +nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, + u64 addr, u64 size) +{ + u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, + nvkm_vmm_ref_ptes, NULL, NULL, NULL); + if (fail != ~0ULL) { + if (fail != addr) + nvkm_vmm_ptes_put(vmm, page, addr, fail - addr); + return -ENOMEM; + } + return 0; +} + void nvkm_vmm_dtor(struct nvkm_vmm *vmm) { + if (vmm->bootstrapped) { + const struct nvkm_vmm_page *page = vmm->func->page; + const u64 limit = vmm->limit - vmm->start; + + while (page[1].shift) + page++; + + nvkm_mmu_ptc_dump(vmm->mmu); + nvkm_vmm_ptes_put(vmm, page, vmm->start, limit); + } + if (vmm->nullp) { dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024, vmm->nullp, vmm->null); @@ -94,6 +644,7 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, vmm->func = func; vmm->mmu = mmu; vmm->name = name; + vmm->debug = mmu->subdev.debug; kref_init(&vmm->kref); __mutex_init(&vmm->mutex, "&vmm->mutex", key ? key : &_key); @@ -150,3 +701,32 @@ nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, return -ENOMEM; return nvkm_vmm_ctor(func, mmu, hdr, addr, size, key, name, *pvmm); } + +static bool +nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +{ + const struct nvkm_vmm_desc *desc = it->desc; + const int type = desc->type == SPT; + nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm); + return false; +} + +int +nvkm_vmm_boot(struct nvkm_vmm *vmm) +{ + const struct nvkm_vmm_page *page = vmm->func->page; + const u64 limit = vmm->limit - vmm->start; + int ret; + + while (page[1].shift) + page++; + + ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit); + if (ret) + return ret; + + nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, + nvkm_vmm_boot_ptes, NULL, NULL, NULL); + vmm->bootstrapped = true; + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 504408d8014b2..042d84c5e950a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -2,6 +2,7 @@ #define __NVKM_VMM_H__ #include "priv.h" #include +enum nvkm_memory_target; struct nvkm_vmm_pt { /* Some GPUs have a mapping level with a dual page tables to @@ -49,7 +50,23 @@ struct nvkm_vmm_pt { u8 pte[]; }; +typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *, + struct nvkm_mmu_pt *, u32 ptei, u32 ptes); +typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *, + struct nvkm_vmm_pt *, u32 pdei); +typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *, + u32 ptei, u32 ptes, struct nvkm_vmm_map *); + struct nvkm_vmm_desc_func { + nvkm_vmm_pxe_func invalid; + nvkm_vmm_pxe_func unmap; + nvkm_vmm_pxe_func sparse; + + nvkm_vmm_pde_func pde; + + nvkm_vmm_pte_func mem; + nvkm_vmm_pte_func dma; + nvkm_vmm_pte_func sgl; }; extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; @@ -106,6 +123,11 @@ struct nvkm_vmm_func { int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst); void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst); + int (*aper)(enum nvkm_memory_target); + int (*valid)(struct nvkm_vmm *, void *argv, u32 argc, + struct nvkm_vmm_map *); + void (*flush)(struct nvkm_vmm *, int depth); + u64 page_block; const struct nvkm_vmm_page page[]; }; @@ -122,6 +144,15 @@ int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32 pd_header, u64 addr, u64 size, struct lock_class_key *, const char *name, struct nvkm_vmm *); void nvkm_vmm_dtor(struct nvkm_vmm *); +void nvkm_vmm_ptes_put(struct nvkm_vmm *, const struct nvkm_vmm_page *, + u64 addr, u64 size); +int nvkm_vmm_ptes_get(struct nvkm_vmm *, const struct nvkm_vmm_page *, + u64 addr, u64 size); +void nvkm_vmm_ptes_map(struct nvkm_vmm *, const struct nvkm_vmm_page *, + u64 addr, u64 size, struct nvkm_vmm_map *, + nvkm_vmm_pte_func); +void nvkm_vmm_ptes_unmap(struct nvkm_vmm *, const struct nvkm_vmm_page *, + u64 addr, u64 size, bool sparse); int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, u64, u64, void *, u32, struct lock_class_key *, @@ -176,4 +207,85 @@ int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); + +#define VMM_PRINT(l,v,p,f,a...) do { \ + struct nvkm_vmm *_vmm = (v); \ + if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) { \ + nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n", \ + _vmm->name, ##a); \ + } \ +} while(0) +#define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a) +#define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a) +#define VMM_SPAM(v,f,a...) VMM_PRINT(NV_DBG_SPAM , (v), dbg, f, ##a) + +#define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do { \ + nvkm_kmap((PT)->memory); \ + while (PTEN) { \ + u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift; \ + u64 _addr = ((BASE) + MAP->off); \ + \ + if (_ptes > PTEN) { \ + MAP->off += PTEN << MAP->page->shift; \ + _ptes = PTEN; \ + } else { \ + MAP->off = 0; \ + NEXT; \ + } \ + \ + VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes); \ + \ + FILL(VMM, PT, PTEI, _ptes, MAP, _addr); \ + PTEI += _ptes; \ + PTEN -= _ptes; \ + }; \ + nvkm_done((PT)->memory); \ +} while(0) + +#define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL) \ + VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ + ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT), \ + ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT), \ + (MAP->mem = MAP->mem->next)) +#define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL) \ + VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ + *MAP->dma, PAGE_SIZE, MAP->dma++) +#define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL) \ + VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ + sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl), \ + (MAP->sgl = sg_next(MAP->sgl))) + +#define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c)) +#define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d)) +#define VMM_XO(m,v,o,d,c,b,fn,f,a...) do { \ + const u32 _pteo = (o); u##b _data = (d); \ + VMM_SPAM((v), " %010llx "f, (m)->addr + _pteo, _data, ##a); \ + VMM_##fn((m), (m)->base + _pteo, _data, (c), b); \ +} while(0) + +#define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 32, WO, "%08x") +#define VMM_FO032(m,v,o,d,c) \ + VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c)) + +#define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 64, WO, "%016llx") +#define VMM_FO064(m,v,o,d,c) \ + VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016llx %08x", (c)) + +#define VMM_XO128(m,v,o,lo,hi,c,f,a...) do { \ + u32 _pteo = (o), _ptes = (c); \ + const u64 _addr = (m)->addr + _pteo; \ + VMM_SPAM((v), " %010llx %016llx%016llx"f, _addr, (hi), (lo), ##a); \ + while (_ptes--) { \ + nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo)); \ + nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi)); \ + _pteo += 0x10; \ + } \ +} while(0) + +#define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "") +#define VMM_FO128(m,v,o,lo,hi,c) do { \ + nvkm_kmap((m)->memory); \ + VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c)); \ + nvkm_done((m)->memory); \ +} while(0) #endif From dd12d158eb91442674111a423d88abee4180b5a9 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 105/151] drm/nouveau/mmu/nv04: implement new vmm backend Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/if000d.h | 4 ++ .../gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 57 ------------------ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c | 58 ++++++++++++++++++- 4 files changed, 62 insertions(+), 58 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000d.h b/drivers/gpu/drm/nouveau/include/nvif/if000d.h index 0f94eb5005b15..516ec9401401a 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if000d.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if000d.h @@ -5,4 +5,8 @@ struct nv04_vmm_vn { /* nvif_vmm_vX ... */ }; + +struct nv04_vmm_map_vn { + /* nvif_vmm_map_vX ... */ +}; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index 3e60364317540..13438dbc60632 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -26,71 +26,14 @@ #include #define NV04_PDMA_SIZE (128 * 1024 * 1024) -#define NV04_PDMA_PAGE ( 4 * 1024) - -/******************************************************************************* - * VM map/unmap callbacks - ******************************************************************************/ - -static void -nv04_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, - struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list) -{ - pte = 0x00008 + (pte * 4); - nvkm_kmap(pgt); - while (cnt) { - u32 page = PAGE_SIZE / NV04_PDMA_PAGE; - u32 phys = (u32)*list++; - while (cnt && page--) { - nvkm_wo32(pgt, pte, phys | 3); - phys += NV04_PDMA_PAGE; - pte += 4; - cnt -= 1; - } - } - nvkm_done(pgt); -} - -static void -nv04_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) -{ - pte = 0x00008 + (pte * 4); - nvkm_kmap(pgt); - while (cnt--) { - nvkm_wo32(pgt, pte, 0x00000000); - pte += 4; - } - nvkm_done(pgt); -} - -static void -nv04_vm_flush(struct nvkm_vm *vm) -{ -} - -/******************************************************************************* - * MMU subdev - ******************************************************************************/ - -static int -nv04_mmu_oneinit(struct nvkm_mmu *mmu) -{ - mmu->vmm->pgt[0].mem[0] = mmu->vmm->pd->pt[0]->memory; - mmu->vmm->pgt[0].refcount[0] = 1; - return 0; -} const struct nvkm_mmu_func nv04_mmu = { - .oneinit = nv04_mmu_oneinit, .limit = NV04_PDMA_SIZE, .dma_bits = 32, .pgt_bits = 32 - 12, .spg_shift = 12, .lpg_shift = 12, - .map_sg = nv04_vm_map_sg, - .unmap = nv04_vm_unmap, - .flush = nv04_vm_flush, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv04_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 042d84c5e950a..cbb8d47fbd119 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -157,6 +157,7 @@ void nvkm_vmm_ptes_unmap(struct nvkm_vmm *, const struct nvkm_vmm_page *, int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); +int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, struct nvkm_mmu *, u64, u64, void *, u32, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c index 1f874b351a39e..0cab1ffc9f64b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c @@ -24,8 +24,50 @@ #include #include +static inline void +nv04_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) +{ + u32 data = addr | 0x00000003; /* PRESENT, RW. */ + while (ptes--) { + VMM_WO032(pt, vmm, 8 + ptei++ * 4, data); + data += 0x00001000; + } +} + +static void +nv04_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, nv04_vmm_pgt_pte); +} + +static void +nv04_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ +#if PAGE_SHIFT == 12 + nvkm_kmap(pt->memory); + while (ptes--) + VMM_WO032(pt, vmm, 8 + (ptei++ * 4), *map->dma++ | 0x00000003); + nvkm_done(pt->memory); +#else + VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, nv04_vmm_pgt_pte); +#endif +} + +static void +nv04_vmm_pgt_unmap(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + VMM_FO032(pt, vmm, 8 + (ptei * 4), 0, ptes); +} + static const struct nvkm_vmm_desc_func nv04_vmm_desc_pgt = { + .unmap = nv04_vmm_pgt_unmap, + .dma = nv04_vmm_pgt_dma, + .sgl = nv04_vmm_pgt_sgl, }; static const struct nvkm_vmm_desc @@ -34,8 +76,22 @@ nv04_vmm_desc_12[] = { {} }; +int +nv04_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, + struct nvkm_vmm_map *map) +{ + union { + struct nv04_vmm_map_vn vn; + } *args = argv; + int ret = -ENOSYS; + if ((ret = nvif_unvers(ret, &argv, &argc, args->vn))) + VMM_DEBUG(vmm, "args"); + return ret; +} + static const struct nvkm_vmm_func nv04_vmm = { + .valid = nv04_vmm_valid, .page = { { 12, &nv04_vmm_desc_12[0], NVKM_VMM_PAGE_HOST }, {} @@ -65,8 +121,8 @@ nv04_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - struct nvkm_vmm *vmm; struct nvkm_memory *mem; + struct nvkm_vmm *vmm; int ret; ret = nv04_vmm_new_(&nv04_vmm, mmu, 8, addr, size, From 473f9aca6c1063bf77cd61c2683fc496850d63b3 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 106/151] drm/nouveau/mmu/nv41: implement new vmm backend Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c | 69 ------------------- .../gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c | 64 +++++++++++++++++ 2 files changed, 64 insertions(+), 69 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c index cb037f4a1780b..457cf509591b7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c @@ -24,75 +24,10 @@ #include "vmm.h" #include -#include #include #define NV41_GART_SIZE (512 * 1024 * 1024) -#define NV41_GART_PAGE ( 4 * 1024) - -/******************************************************************************* - * VM map/unmap callbacks - ******************************************************************************/ - -static void -nv41_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, - struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list) -{ - pte = pte * 4; - nvkm_kmap(pgt); - while (cnt) { - u32 page = PAGE_SIZE / NV41_GART_PAGE; - u64 phys = (u64)*list++; - while (cnt && page--) { - nvkm_wo32(pgt, pte, (phys >> 7) | 1); - phys += NV41_GART_PAGE; - pte += 4; - cnt -= 1; - } - } - nvkm_done(pgt); -} - -static void -nv41_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) -{ - pte = pte * 4; - nvkm_kmap(pgt); - while (cnt--) { - nvkm_wo32(pgt, pte, 0x00000000); - pte += 4; - } - nvkm_done(pgt); -} - -static void -nv41_vm_flush(struct nvkm_vm *vm) -{ - struct nvkm_subdev *subdev = &vm->mmu->subdev; - struct nvkm_device *device = subdev->device; - - mutex_lock(&subdev->mutex); - nvkm_wr32(device, 0x100810, 0x00000022); - nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x100810) & 0x00000020) - break; - ); - nvkm_wr32(device, 0x100810, 0x00000000); - mutex_unlock(&subdev->mutex); -} - -/******************************************************************************* - * MMU subdev - ******************************************************************************/ - -static int -nv41_mmu_oneinit(struct nvkm_mmu *mmu) -{ - mmu->vmm->pgt[0].mem[0] = mmu->vmm->pd->pt[0]->memory; - mmu->vmm->pgt[0].refcount[0] = 1; - return 0; -} static void nv41_mmu_init(struct nvkm_mmu *mmu) @@ -105,16 +40,12 @@ nv41_mmu_init(struct nvkm_mmu *mmu) static const struct nvkm_mmu_func nv41_mmu = { - .oneinit = nv41_mmu_oneinit, .init = nv41_mmu_init, .limit = NV41_GART_SIZE, .dma_bits = 39, .pgt_bits = 32 - 12, .spg_shift = 12, .lpg_shift = 12, - .map_sg = nv41_vm_map_sg, - .unmap = nv41_vm_unmap, - .flush = nv41_vm_flush, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv41_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c index 8a624d7c363ae..b595f130e5738 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c @@ -21,8 +21,54 @@ */ #include "vmm.h" +#include + +static void +nv41_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) +{ + u32 data = (addr >> 7) | 0x00000001; /* VALID. */ + while (ptes--) { + VMM_WO032(pt, vmm, ptei++ * 4, data); + data += 0x00000020; + } +} + +static void +nv41_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, nv41_vmm_pgt_pte); +} + +static void +nv41_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ +#if PAGE_SHIFT == 12 + nvkm_kmap(pt->memory); + while (ptes--) { + const u32 data = (*map->dma++ >> 7) | 0x00000001; + VMM_WO032(pt, vmm, ptei++ * 4, data); + } + nvkm_done(pt->memory); +#else + VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, nv41_vmm_pgt_pte); +#endif +} + +static void +nv41_vmm_pgt_unmap(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + VMM_FO032(pt, vmm, ptei * 4, 0, ptes); +} + static const struct nvkm_vmm_desc_func nv41_vmm_desc_pgt = { + .unmap = nv41_vmm_pgt_unmap, + .dma = nv41_vmm_pgt_dma, + .sgl = nv41_vmm_pgt_sgl, }; static const struct nvkm_vmm_desc @@ -31,8 +77,26 @@ nv41_vmm_desc_12[] = { {} }; +static void +nv41_vmm_flush(struct nvkm_vmm *vmm, int level) +{ + struct nvkm_subdev *subdev = &vmm->mmu->subdev; + struct nvkm_device *device = subdev->device; + + mutex_lock(&subdev->mutex); + nvkm_wr32(device, 0x100810, 0x00000022); + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x100810) & 0x00000020) + break; + ); + nvkm_wr32(device, 0x100810, 0x00000000); + mutex_unlock(&subdev->mutex); +} + static const struct nvkm_vmm_func nv41_vmm = { + .valid = nv04_vmm_valid, + .flush = nv41_vmm_flush, .page = { { 12, &nv41_vmm_desc_12[0], NVKM_VMM_PAGE_HOST }, {} From 6ce513529aa57a8c4f61e588142643a9252037ae Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 107/151] drm/nouveau/mmu/nv44: implement new vmm backend Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c | 144 +-------------- .../gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c | 165 ++++++++++++++++++ 2 files changed, 168 insertions(+), 141 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c index 48ca0cdf2acfa..2072139bff4dc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c @@ -24,150 +24,16 @@ #include "vmm.h" #include -#include #include #define NV44_GART_SIZE (512 * 1024 * 1024) -#define NV44_GART_PAGE ( 4 * 1024) - -/******************************************************************************* - * VM map/unmap callbacks - ******************************************************************************/ - -static void -nv44_vm_fill(struct nvkm_memory *pgt, dma_addr_t null, - dma_addr_t *list, u32 pte, u32 cnt) -{ - u32 base = (pte << 2) & ~0x0000000f; - u32 tmp[4]; - - tmp[0] = nvkm_ro32(pgt, base + 0x0); - tmp[1] = nvkm_ro32(pgt, base + 0x4); - tmp[2] = nvkm_ro32(pgt, base + 0x8); - tmp[3] = nvkm_ro32(pgt, base + 0xc); - - while (cnt--) { - u32 addr = list ? (*list++ >> 12) : (null >> 12); - switch (pte++ & 0x3) { - case 0: - tmp[0] &= ~0x07ffffff; - tmp[0] |= addr; - break; - case 1: - tmp[0] &= ~0xf8000000; - tmp[0] |= addr << 27; - tmp[1] &= ~0x003fffff; - tmp[1] |= addr >> 5; - break; - case 2: - tmp[1] &= ~0xffc00000; - tmp[1] |= addr << 22; - tmp[2] &= ~0x0001ffff; - tmp[2] |= addr >> 10; - break; - case 3: - tmp[2] &= ~0xfffe0000; - tmp[2] |= addr << 17; - tmp[3] &= ~0x00000fff; - tmp[3] |= addr >> 15; - break; - } - } - - nvkm_wo32(pgt, base + 0x0, tmp[0]); - nvkm_wo32(pgt, base + 0x4, tmp[1]); - nvkm_wo32(pgt, base + 0x8, tmp[2]); - nvkm_wo32(pgt, base + 0xc, tmp[3] | 0x40000000); -} - -static void -nv44_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, - struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list) -{ - u32 tmp[4]; - int i; - - nvkm_kmap(pgt); - if (pte & 3) { - u32 max = 4 - (pte & 3); - u32 part = (cnt > max) ? max : cnt; - nv44_vm_fill(pgt, vma->vm->null, list, pte, part); - pte += part; - list += part; - cnt -= part; - } - - while (cnt >= 4) { - for (i = 0; i < 4; i++) - tmp[i] = *list++ >> 12; - nvkm_wo32(pgt, pte++ * 4, tmp[0] >> 0 | tmp[1] << 27); - nvkm_wo32(pgt, pte++ * 4, tmp[1] >> 5 | tmp[2] << 22); - nvkm_wo32(pgt, pte++ * 4, tmp[2] >> 10 | tmp[3] << 17); - nvkm_wo32(pgt, pte++ * 4, tmp[3] >> 15 | 0x40000000); - cnt -= 4; - } - - if (cnt) - nv44_vm_fill(pgt, vma->vm->null, list, pte, cnt); - nvkm_done(pgt); -} - -static void -nv44_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) -{ - nvkm_kmap(pgt); - if (pte & 3) { - u32 max = 4 - (pte & 3); - u32 part = (cnt > max) ? max : cnt; - nv44_vm_fill(pgt, vma->vm->null, NULL, pte, part); - pte += part; - cnt -= part; - } - - while (cnt >= 4) { - nvkm_wo32(pgt, pte++ * 4, 0x00000000); - nvkm_wo32(pgt, pte++ * 4, 0x00000000); - nvkm_wo32(pgt, pte++ * 4, 0x00000000); - nvkm_wo32(pgt, pte++ * 4, 0x00000000); - cnt -= 4; - } - - if (cnt) - nv44_vm_fill(pgt, vma->vm->null, NULL, pte, cnt); - nvkm_done(pgt); -} - -static void -nv44_vm_flush(struct nvkm_vm *vm) -{ - struct nvkm_device *device = vm->mmu->subdev.device; - nvkm_wr32(device, 0x100814, vm->mmu->limit - NV44_GART_PAGE); - nvkm_wr32(device, 0x100808, 0x00000020); - nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x100808) & 0x00000001) - break; - ); - nvkm_wr32(device, 0x100808, 0x00000000); -} - -/******************************************************************************* - * MMU subdev - ******************************************************************************/ - -static int -nv44_mmu_oneinit(struct nvkm_mmu *mmu) -{ - mmu->vmm->pgt[0].mem[0] = mmu->vmm->pd->pt[0]->memory; - mmu->vmm->pgt[0].refcount[0] = 1; - return 0; -} static void nv44_mmu_init(struct nvkm_mmu *mmu) { struct nvkm_device *device = mmu->subdev.device; - struct nvkm_memory *gart = mmu->vmm->pgt[0].mem[0]; + struct nvkm_memory *pt = mmu->vmm->pd->pt[0]->memory; u32 addr; /* calculate vram address of this PRAMIN block, object must be @@ -175,11 +41,11 @@ nv44_mmu_init(struct nvkm_mmu *mmu) * of 512KiB for this to work correctly */ addr = nvkm_rd32(device, 0x10020c); - addr -= ((nvkm_memory_addr(gart) >> 19) + 1) << 19; + addr -= ((nvkm_memory_addr(pt) >> 19) + 1) << 19; nvkm_wr32(device, 0x100850, 0x80000000); nvkm_wr32(device, 0x100818, mmu->vmm->null); - nvkm_wr32(device, 0x100804, NV44_GART_SIZE); + nvkm_wr32(device, 0x100804, (nvkm_memory_size(pt) / 4) * 4096); nvkm_wr32(device, 0x100850, 0x00008000); nvkm_mask(device, 0x10008c, 0x00000200, 0x00000200); nvkm_wr32(device, 0x100820, 0x00000000); @@ -189,16 +55,12 @@ nv44_mmu_init(struct nvkm_mmu *mmu) static const struct nvkm_mmu_func nv44_mmu = { - .oneinit = nv44_mmu_oneinit, .init = nv44_mmu_init, .limit = NV44_GART_SIZE, .dma_bits = 39, .pgt_bits = 32 - 12, .spg_shift = 12, .lpg_shift = 12, - .map_sg = nv44_vm_map_sg, - .unmap = nv44_vm_unmap, - .flush = nv44_vm_flush, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv44_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c index 2b5704269ac99..b834e43523343 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c @@ -21,8 +21,158 @@ */ #include "vmm.h" +#include + +static void +nv44_vmm_pgt_fill(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + dma_addr_t *list, u32 ptei, u32 ptes) +{ + u32 pteo = (ptei << 2) & ~0x0000000f; + u32 tmp[4]; + + tmp[0] = nvkm_ro32(pt->memory, pteo + 0x0); + tmp[1] = nvkm_ro32(pt->memory, pteo + 0x4); + tmp[2] = nvkm_ro32(pt->memory, pteo + 0x8); + tmp[3] = nvkm_ro32(pt->memory, pteo + 0xc); + + while (ptes--) { + u32 addr = (list ? *list++ : vmm->null) >> 12; + switch (ptei++ & 0x3) { + case 0: + tmp[0] &= ~0x07ffffff; + tmp[0] |= addr; + break; + case 1: + tmp[0] &= ~0xf8000000; + tmp[0] |= addr << 27; + tmp[1] &= ~0x003fffff; + tmp[1] |= addr >> 5; + break; + case 2: + tmp[1] &= ~0xffc00000; + tmp[1] |= addr << 22; + tmp[2] &= ~0x0001ffff; + tmp[2] |= addr >> 10; + break; + case 3: + tmp[2] &= ~0xfffe0000; + tmp[2] |= addr << 17; + tmp[3] &= ~0x00000fff; + tmp[3] |= addr >> 15; + break; + } + } + + VMM_WO032(pt, vmm, pteo + 0x0, tmp[0]); + VMM_WO032(pt, vmm, pteo + 0x4, tmp[1]); + VMM_WO032(pt, vmm, pteo + 0x8, tmp[2]); + VMM_WO032(pt, vmm, pteo + 0xc, tmp[3] | 0x40000000); +} + +static void +nv44_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) +{ + dma_addr_t tmp[4], i; + + if (ptei & 3) { + const u32 pten = min(ptes, 4 - (ptei & 3)); + for (i = 0; i < pten; i++, addr += 0x1000) + tmp[i] = addr; + nv44_vmm_pgt_fill(vmm, pt, tmp, ptei, pten); + ptei += pten; + ptes -= pten; + } + + while (ptes >= 4) { + for (i = 0; i < 4; i++, addr += 0x1000) + tmp[i] = addr >> 12; + VMM_WO032(pt, vmm, ptei++ * 4, tmp[0] >> 0 | tmp[1] << 27); + VMM_WO032(pt, vmm, ptei++ * 4, tmp[1] >> 5 | tmp[2] << 22); + VMM_WO032(pt, vmm, ptei++ * 4, tmp[2] >> 10 | tmp[3] << 17); + VMM_WO032(pt, vmm, ptei++ * 4, tmp[3] >> 15 | 0x40000000); + ptes -= 4; + } + + if (ptes) { + for (i = 0; i < ptes; i++, addr += 0x1000) + tmp[i] = addr; + nv44_vmm_pgt_fill(vmm, pt, tmp, ptei, ptes); + } +} + +static void +nv44_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, nv44_vmm_pgt_pte); +} + +static void +nv44_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ +#if PAGE_SHIFT == 12 + nvkm_kmap(pt->memory); + if (ptei & 3) { + const u32 pten = min(ptes, 4 - (ptei & 3)); + nv44_vmm_pgt_fill(vmm, pt, map->dma, ptei, pten); + ptei += pten; + ptes -= pten; + map->dma += pten; + } + + while (ptes >= 4) { + u32 tmp[4], i; + for (i = 0; i < 4; i++) + tmp[i] = *map->dma++ >> 12; + VMM_WO032(pt, vmm, ptei++ * 4, tmp[0] >> 0 | tmp[1] << 27); + VMM_WO032(pt, vmm, ptei++ * 4, tmp[1] >> 5 | tmp[2] << 22); + VMM_WO032(pt, vmm, ptei++ * 4, tmp[2] >> 10 | tmp[3] << 17); + VMM_WO032(pt, vmm, ptei++ * 4, tmp[3] >> 15 | 0x40000000); + ptes -= 4; + } + + if (ptes) { + nv44_vmm_pgt_fill(vmm, pt, map->dma, ptei, ptes); + map->dma += ptes; + } + nvkm_done(pt->memory); +#else + VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, nv44_vmm_pgt_pte); +#endif +} + +static void +nv44_vmm_pgt_unmap(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + nvkm_kmap(pt->memory); + if (ptei & 3) { + const u32 pten = min(ptes, 4 - (ptei & 3)); + nv44_vmm_pgt_fill(vmm, pt, NULL, ptei, pten); + ptei += pten; + ptes -= pten; + } + + while (ptes > 4) { + VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000); + VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000); + VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000); + VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000); + ptes -= 4; + } + + if (ptes) + nv44_vmm_pgt_fill(vmm, pt, NULL, ptei, ptes); + nvkm_done(pt->memory); +} + static const struct nvkm_vmm_desc_func nv44_vmm_desc_pgt = { + .unmap = nv44_vmm_pgt_unmap, + .dma = nv44_vmm_pgt_dma, + .sgl = nv44_vmm_pgt_sgl, }; static const struct nvkm_vmm_desc @@ -31,8 +181,23 @@ nv44_vmm_desc_12[] = { {} }; +static void +nv44_vmm_flush(struct nvkm_vmm *vmm, int level) +{ + struct nvkm_device *device = vmm->mmu->subdev.device; + nvkm_wr32(device, 0x100814, vmm->limit - 4096); + nvkm_wr32(device, 0x100808, 0x000000020); + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x100808) & 0x00000001) + break; + ); + nvkm_wr32(device, 0x100808, 0x00000000); +} + static const struct nvkm_vmm_func nv44_vmm = { + .valid = nv04_vmm_valid, + .flush = nv44_vmm_flush, .page = { { 12, &nv44_vmm_desc_12[0], NVKM_VMM_PAGE_HOST }, {} From fd542a3e525c9f7a7de186cb24208c035bcea2d1 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 108/151] drm/nouveau/mmu/nv50,g84: implement new vmm backend Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/if500d.h | 13 + .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 19 ++ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c | 6 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c | 213 ++----------- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 10 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c | 293 +++++++++++++++++- 6 files changed, 360 insertions(+), 194 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/if500d.h b/drivers/gpu/drm/nouveau/include/nvif/if500d.h index 6a0f47770745f..c29a7822b3638 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if500d.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if500d.h @@ -5,4 +5,17 @@ struct nv50_vmm_vn { /* nvif_vmm_vX ... */ }; + +struct nv50_vmm_map_vn { + /* nvif_vmm_map_vX ... */ +}; + +struct nv50_vmm_map_v0 { + /* nvif_vmm_map_vX ... */ + __u8 version; + __u8 ro; + __u8 priv; + __u8 kind; + __u8 comp; +}; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 536187952372d..4d75dcb414689 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -27,6 +27,8 @@ #include #include +#include + struct nvkm_mmu_ptp { struct nvkm_mmu_pt *pt; struct list_head head; @@ -218,6 +220,9 @@ nvkm_vm_map_(const struct nvkm_vmm_page *page, struct nvkm_vma *vma, u64 delta, struct nvkm_mem *mem, nvkm_vmm_pte_func fn, struct nvkm_vmm_map *map) { + union { + struct nv50_vmm_map_v0 nv50; + } args; struct nvkm_vmm *vmm = vma->vm; void *argv = NULL; u32 argc = 0; @@ -227,6 +232,20 @@ nvkm_vm_map_(const struct nvkm_vmm_page *page, struct nvkm_vma *vma, u64 delta, map->page = page; if (vmm->func->valid) { + switch (vmm->mmu->subdev.device->card_type) { + case NV_50: + args.nv50.version = 0; + args.nv50.ro = !(vma->access & NV_MEM_ACCESS_WO); + args.nv50.priv = !!(vma->access & NV_MEM_ACCESS_SYS); + args.nv50.kind = (mem->memtype & 0x07f); + args.nv50.comp = (mem->memtype & 0x180) >> 7; + argv = &args.nv50; + argc = sizeof(args.nv50); + break; + default: + break; + } + ret = vmm->func->valid(vmm, argv, argc, map); if (WARN_ON(ret)) return; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c index 13f8247a2ee47..e436c610b3df5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c @@ -30,12 +30,8 @@ g84_mmu = { .pgt_bits = 29 - 12, .spg_shift = 12, .lpg_shift = 16, - .map_pgt = nv50_vm_map_pgt, - .map = nv50_vm_map, - .map_sg = nv50_vm_map_sg, - .unmap = nv50_vm_unmap, - .flush = nv50_vm_flush, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x0200 }, + .kind = nv50_mmu_kind, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c index da75d1da0658e..31d6f9626e56c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c @@ -23,185 +23,40 @@ */ #include "vmm.h" -#include -#include -#include -#include - #include -void -nv50_vm_map_pgt(struct nvkm_vmm *vmm, u32 pde, struct nvkm_memory *pgt[2]) -{ - struct nvkm_vmm_join *join; - u32 pdeo = vmm->mmu->func->vmm.pd_offset + (pde * 8); - u64 phys = 0xdeadcafe00000000ULL; - u32 coverage = 0; - - if (pgt[0]) { - /* present, 4KiB pages */ - phys = 0x00000003 | nvkm_memory_addr(pgt[0]); - coverage = (nvkm_memory_size(pgt[0]) >> 3) << 12; - } else - if (pgt[1]) { - /* present, 64KiB pages */ - phys = 0x00000001 | nvkm_memory_addr(pgt[1]); - coverage = (nvkm_memory_size(pgt[1]) >> 3) << 16; - } - - if (phys & 1) { - if (coverage <= 32 * 1024 * 1024) - phys |= 0x60; - else if (coverage <= 64 * 1024 * 1024) - phys |= 0x40; - else if (coverage <= 128 * 1024 * 1024) - phys |= 0x20; - } - - list_for_each_entry(join, &vmm->join, head) { - nvkm_kmap(join->inst); - nvkm_wo32(join->inst, pdeo + 0, lower_32_bits(phys)); - nvkm_wo32(join->inst, pdeo + 4, upper_32_bits(phys)); - nvkm_done(join->inst); - } -} - -static inline u64 -vm_addr(struct nvkm_vma *vma, u64 phys, u32 memtype, u32 target) +const u8 * +nv50_mmu_kind(struct nvkm_mmu *base, int *count) { - phys |= 1; /* present */ - phys |= (u64)memtype << 40; - phys |= target << 4; - if (vma->access & NV_MEM_ACCESS_SYS) - phys |= (1 << 6); - if (!(vma->access & NV_MEM_ACCESS_WO)) - phys |= (1 << 3); - return phys; -} - -void -nv50_vm_map(struct nvkm_vma *vma, struct nvkm_memory *pgt, - struct nvkm_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta) -{ - struct nvkm_ram *ram = vma->vm->mmu->subdev.device->fb->ram; - u32 comp = (mem->memtype & 0x180) >> 7; - u32 block, target; - int i; - - /* IGPs don't have real VRAM, re-target to stolen system memory */ - target = 0; - if (ram->stolen) { - phys += ram->stolen; - target = 3; - } - - phys = vm_addr(vma, phys, mem->memtype, target); - pte <<= 3; - cnt <<= 3; - - nvkm_kmap(pgt); - while (cnt) { - u32 offset_h = upper_32_bits(phys); - u32 offset_l = lower_32_bits(phys); - - for (i = 7; i >= 0; i--) { - block = 1 << (i + 3); - if (cnt >= block && !(pte & (block - 1))) - break; - } - offset_l |= (i << 7); - - phys += block << (vma->node->type - 3); - cnt -= block; - if (comp) { - u32 tag = mem->tag->offset + ((delta >> 16) * comp); - offset_h |= (tag << 17); - delta += block << (vma->node->type - 3); - } - - while (block) { - nvkm_wo32(pgt, pte + 0, offset_l); - nvkm_wo32(pgt, pte + 4, offset_h); - pte += 8; - block -= 8; - } - } - nvkm_done(pgt); -} - -void -nv50_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, - struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list) -{ - u32 target = (vma->access & NV_MEM_ACCESS_NOSNOOP) ? 3 : 2; - pte <<= 3; - nvkm_kmap(pgt); - while (cnt--) { - u64 phys = vm_addr(vma, (u64)*list++, mem->memtype, target); - nvkm_wo32(pgt, pte + 0, lower_32_bits(phys)); - nvkm_wo32(pgt, pte + 4, upper_32_bits(phys)); - pte += 8; - } - nvkm_done(pgt); -} - -void -nv50_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) -{ - pte <<= 3; - nvkm_kmap(pgt); - while (cnt--) { - nvkm_wo32(pgt, pte + 0, 0x00000000); - nvkm_wo32(pgt, pte + 4, 0x00000000); - pte += 8; - } - nvkm_done(pgt); -} - -void -nv50_vm_flush(struct nvkm_vm *vm) -{ - struct nvkm_mmu *mmu = vm->mmu; - struct nvkm_subdev *subdev = &mmu->subdev; - struct nvkm_device *device = subdev->device; - int i, vme; - - mutex_lock(&subdev->mutex); - for (i = 0; i < NVKM_SUBDEV_NR; i++) { - if (!atomic_read(&vm->engref[i])) - continue; - - /* unfortunate hw bug workaround... */ - if (i == NVKM_ENGINE_GR && device->gr) { - int ret = nvkm_gr_tlb_flush(device->gr); - if (ret != -ENODEV) - continue; - } - - switch (i) { - case NVKM_ENGINE_GR : vme = 0x00; break; - case NVKM_ENGINE_VP : - case NVKM_ENGINE_MSPDEC: vme = 0x01; break; - case NVKM_SUBDEV_BAR : vme = 0x06; break; - case NVKM_ENGINE_MSPPP : - case NVKM_ENGINE_MPEG : vme = 0x08; break; - case NVKM_ENGINE_BSP : - case NVKM_ENGINE_MSVLD : vme = 0x09; break; - case NVKM_ENGINE_CIPHER: - case NVKM_ENGINE_SEC : vme = 0x0a; break; - case NVKM_ENGINE_CE0 : vme = 0x0d; break; - default: - continue; - } - - nvkm_wr32(device, 0x100c80, (vme << 16) | 1); - if (nvkm_msec(device, 2000, - if (!(nvkm_rd32(device, 0x100c80) & 0x00000001)) - break; - ) < 0) - nvkm_error(subdev, "vm flush timeout: engine %d\n", vme); - } - mutex_unlock(&subdev->mutex); + /* 0x01: no bank swizzle + * 0x02: bank swizzled + * 0x7f: invalid + * + * 0x01/0x02 are values understood by the VRAM allocator, + * and are required to avoid mixing the two types within + * a certain range. + */ + static const u8 + kind[128] = { + 0x01, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, /* 0x00 */ + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x01, 0x01, 0x01, 0x01, 0x7f, 0x7f, 0x7f, 0x7f, /* 0x10 */ + 0x02, 0x02, 0x02, 0x02, 0x7f, 0x7f, 0x7f, 0x7f, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x7f, /* 0x20 */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, /* 0x30 */ + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, /* 0x40 */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x01, 0x01, 0x01, 0x7f, /* 0x50 */ + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x7f, /* 0x60 */ + 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, + 0x01, 0x7f, 0x02, 0x7f, 0x01, 0x7f, 0x02, 0x7f, /* 0x70 */ + 0x01, 0x01, 0x02, 0x02, 0x01, 0x01, 0x7f, 0x7f + }; + *count = ARRAY_SIZE(kind); + return kind; } static const struct nvkm_mmu_func @@ -211,12 +66,8 @@ nv50_mmu = { .pgt_bits = 29 - 12, .spg_shift = 12, .lpg_shift = 16, - .map_pgt = nv50_vm_map_pgt, - .map = nv50_vm_map, - .map_sg = nv50_vm_map_sg, - .unmap = nv50_vm_unmap, - .flush = nv50_vm_flush, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x1400 }, + .kind = nv50_mmu_kind, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 3271ec1c3c839..326781246714f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -37,17 +37,13 @@ struct nvkm_mmu_func { bool global; u32 pd_offset; } vmm; + + const u8 *(*kind)(struct nvkm_mmu *, int *count); }; extern const struct nvkm_mmu_func nv04_mmu; -void nv50_vm_map_pgt(struct nvkm_vmm *, u32, struct nvkm_memory **); -void nv50_vm_map(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, - u32, u32, u64, u64); -void nv50_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, - u32, u32, dma_addr_t *); -void nv50_vm_unmap(struct nvkm_vma *, struct nvkm_memory *, u32, u32); -void nv50_vm_flush(struct nvkm_vm *); +const u8 *nv50_mmu_kind(struct nvkm_mmu *, int *count); void gf100_vm_map_pgt(struct nvkm_vmm *, u32, struct nvkm_memory **); void gf100_vm_map(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c index db27381b67c68..863a2edd98616 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c @@ -21,15 +21,146 @@ */ #include "vmm.h" +#include +#include +#include + #include #include +static inline void +nv50_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) +{ + u64 next = addr | map->type, data; + u32 pten; + int log2blk; + + map->type += ptes * map->ctag; + + while (ptes) { + for (log2blk = 7; log2blk >= 0; log2blk--) { + pten = 1 << log2blk; + if (ptes >= pten && IS_ALIGNED(ptei, pten)) + break; + } + + data = next | (log2blk << 7); + next += pten * map->next; + ptes -= pten; + + while (pten--) + VMM_WO064(pt, vmm, ptei++ * 8, data); + } +} + +static void +nv50_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, nv50_vmm_pgt_pte); +} + +static void +nv50_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + if (map->page->shift == PAGE_SHIFT) { + VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes); + nvkm_kmap(pt->memory); + while (ptes--) { + const u64 data = *map->dma++ | map->type; + VMM_WO064(pt, vmm, ptei++ * 8, data); + map->type += map->ctag; + } + nvkm_done(pt->memory); + return; + } + + VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, nv50_vmm_pgt_pte); +} + +static void +nv50_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, nv50_vmm_pgt_pte); +} + +static void +nv50_vmm_pgt_unmap(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes); +} + static const struct nvkm_vmm_desc_func nv50_vmm_pgt = { + .unmap = nv50_vmm_pgt_unmap, + .mem = nv50_vmm_pgt_mem, + .dma = nv50_vmm_pgt_dma, + .sgl = nv50_vmm_pgt_sgl, }; +static bool +nv50_vmm_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgt, u64 *pdata) +{ + struct nvkm_mmu_pt *pt; + u64 data = 0xdeadcafe00000000ULL; + if (pgt && (pt = pgt->pt[0])) { + switch (pgt->page) { + case 16: data = 0x00000001; break; + case 12: data = 0x00000003; + switch (nvkm_memory_size(pt->memory)) { + case 0x100000: data |= 0x00000000; break; + case 0x040000: data |= 0x00000020; break; + case 0x020000: data |= 0x00000040; break; + case 0x010000: data |= 0x00000060; break; + default: + WARN_ON(1); + return false; + } + break; + default: + WARN_ON(1); + return false; + } + + switch (nvkm_memory_target(pt->memory)) { + case NVKM_MEM_TARGET_VRAM: data |= 0x00000000; break; + case NVKM_MEM_TARGET_HOST: data |= 0x00000008; break; + case NVKM_MEM_TARGET_NCOH: data |= 0x0000000c; break; + default: + WARN_ON(1); + return false; + } + + data |= pt->addr; + } + *pdata = data; + return true; +} + +static void +nv50_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) +{ + struct nvkm_vmm_join *join; + u32 pdeo = vmm->mmu->func->vmm.pd_offset + (pdei * 8); + u64 data; + + if (!nv50_vmm_pde(vmm, pgd->pde[pdei], &data)) + return; + + list_for_each_entry(join, &vmm->join, head) { + nvkm_kmap(join->inst); + nvkm_wo64(join->inst, pdeo, data); + nvkm_done(join->inst); + } +} + static const struct nvkm_vmm_desc_func nv50_vmm_pgd = { + .pde = nv50_vmm_pgd_pde, }; static const struct nvkm_vmm_desc @@ -46,6 +177,150 @@ nv50_vmm_desc_16[] = { {} }; +static void +nv50_vmm_flush(struct nvkm_vmm *vmm, int level) +{ + struct nvkm_subdev *subdev = &vmm->mmu->subdev; + struct nvkm_device *device = subdev->device; + int i, id; + + mutex_lock(&subdev->mutex); + for (i = 0; i < NVKM_SUBDEV_NR; i++) { + if (!atomic_read(&vmm->engref[i])) + continue; + + /* unfortunate hw bug workaround... */ + if (i == NVKM_ENGINE_GR && device->gr) { + int ret = nvkm_gr_tlb_flush(device->gr); + if (ret != -ENODEV) + continue; + } + + switch (i) { + case NVKM_ENGINE_GR : id = 0x00; break; + case NVKM_ENGINE_VP : + case NVKM_ENGINE_MSPDEC: id = 0x01; break; + case NVKM_SUBDEV_BAR : id = 0x06; break; + case NVKM_ENGINE_MSPPP : + case NVKM_ENGINE_MPEG : id = 0x08; break; + case NVKM_ENGINE_BSP : + case NVKM_ENGINE_MSVLD : id = 0x09; break; + case NVKM_ENGINE_CIPHER: + case NVKM_ENGINE_SEC : id = 0x0a; break; + case NVKM_ENGINE_CE0 : id = 0x0d; break; + default: + continue; + } + + nvkm_wr32(device, 0x100c80, (id << 16) | 1); + if (nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0x100c80) & 0x00000001)) + break; + ) < 0) + nvkm_error(subdev, "%s mmu invalidate timeout\n", + nvkm_subdev_name[i]); + } + mutex_unlock(&subdev->mutex); +} + +static int +nv50_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, + struct nvkm_vmm_map *map) +{ + const struct nvkm_vmm_page *page = map->page; + union { + struct nv50_vmm_map_vn vn; + struct nv50_vmm_map_v0 v0; + } *args = argv; + struct nvkm_device *device = vmm->mmu->subdev.device; + struct nvkm_ram *ram = device->fb->ram; + struct nvkm_memory *memory = map->memory; + u8 aper, kind, comp, priv, ro; + int kindn, ret = -ENOSYS; + const u8 *kindm; + + map->type = map->ctag = 0; + map->next = 1 << page->shift; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + ro = !!args->v0.ro; + priv = !!args->v0.priv; + kind = args->v0.kind & 0x7f; + comp = args->v0.comp & 0x03; + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + ro = 0; + priv = 0; + kind = 0x00; + comp = 0; + } else { + VMM_DEBUG(vmm, "args"); + return ret; + } + + switch (nvkm_memory_target(memory)) { + case NVKM_MEM_TARGET_VRAM: + if (ram->stolen) { + map->type |= ram->stolen; + aper = 3; + } else { + aper = 0; + } + break; + case NVKM_MEM_TARGET_HOST: + aper = 2; + break; + case NVKM_MEM_TARGET_NCOH: + aper = 3; + break; + default: + WARN_ON(1); + return -EINVAL; + } + + kindm = vmm->mmu->func->kind(vmm->mmu, &kindn); + if (kind >= kindn || kindm[kind] == 0x7f) { + VMM_DEBUG(vmm, "kind %02x", kind); + return -EINVAL; + } + + if (map->mem && map->mem->type != kindm[kind]) { + VMM_DEBUG(vmm, "kind %02x bankswz: %d %d", kind, + kindm[kind], map->mem->type); + return -EINVAL; + } + + if (comp) { + u32 tags = (nvkm_memory_size(memory) >> 16) * comp; + if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) { + VMM_DEBUG(vmm, "comp %d %02x", aper, page->type); + return -EINVAL; + } + + ret = nvkm_memory_tags_get(memory, device, tags, NULL, + &map->tags); + if (ret) { + VMM_DEBUG(vmm, "comp %d", ret); + return ret; + } + + if (map->tags->mn) { + u32 tags = map->tags->mn->offset + (map->offset >> 16); + map->ctag |= (u64)comp << 49; + map->type |= (u64)comp << 47; + map->type |= (u64)tags << 49; + map->next |= map->ctag; + } + } + + map->type |= BIT(0); /* Valid. */ + map->type |= (u64)ro << 3; + map->type |= (u64)aper << 4; + map->type |= (u64)priv << 6; + map->type |= (u64)kind << 40; + return 0; +} + static void nv50_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst) { @@ -63,19 +338,35 @@ nv50_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst) static int nv50_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) { + const u32 pd_offset = vmm->mmu->func->vmm.pd_offset; struct nvkm_vmm_join *join; + int ret = 0; + u64 data; + u32 pdei; if (!(join = kmalloc(sizeof(*join), GFP_KERNEL))) return -ENOMEM; join->inst = inst; list_add_tail(&join->head, &vmm->join); - return 0; + + nvkm_kmap(join->inst); + for (pdei = vmm->start >> 29; pdei <= (vmm->limit - 1) >> 29; pdei++) { + if (!nv50_vmm_pde(vmm, vmm->pd->pde[pdei], &data)) { + ret = -EINVAL; + break; + } + nvkm_wo64(join->inst, pd_offset + (pdei * 8), data); + } + nvkm_done(join->inst); + return ret; } static const struct nvkm_vmm_func nv50_vmm = { .join = nv50_vmm_join, .part = nv50_vmm_part, + .valid = nv50_vmm_valid, + .flush = nv50_vmm_flush, .page_block = 1 << 29, .page = { { 16, &nv50_vmm_desc_16[0], NVKM_VMM_PAGE_xVxC }, From b77791da0ee009dcb1813d8b00919962b6a5f851 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 109/151] drm/nouveau/mmu/gf100: implement new vmm backend Adds support for: - 64KiB big page size. - System-memory PTs. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/if900d.h | 13 + .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 15 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 12 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 12 + .../drm/nouveau/nvkm/subdev/mmu/vmmgf100.c | 258 ++++++++++++++++++ 6 files changed, 306 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/if900d.h b/drivers/gpu/drm/nouveau/include/nvif/if900d.h index 112716f889c68..49aa50583c3dc 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if900d.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if900d.h @@ -5,4 +5,17 @@ struct gf100_vmm_vn { /* nvif_vmm_vX ... */ }; + +struct gf100_vmm_map_vn { + /* nvif_vmm_map_vX ... */ +}; + +struct gf100_vmm_map_v0 { + /* nvif_vmm_map_vX ... */ + __u8 version; + __u8 vol; + __u8 ro; + __u8 priv; + __u8 kind; +}; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 4d75dcb414689..741021ff8c27e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -28,6 +28,7 @@ #include #include +#include struct nvkm_mmu_ptp { struct nvkm_mmu_pt *pt; @@ -222,6 +223,7 @@ nvkm_vm_map_(const struct nvkm_vmm_page *page, struct nvkm_vma *vma, u64 delta, { union { struct nv50_vmm_map_v0 nv50; + struct gf100_vmm_map_v0 gf100; } args; struct nvkm_vmm *vmm = vma->vm; void *argv = NULL; @@ -242,6 +244,19 @@ nvkm_vm_map_(const struct nvkm_vmm_page *page, struct nvkm_vma *vma, u64 delta, argv = &args.nv50; argc = sizeof(args.nv50); break; + case NV_C0: + case NV_E0: + case GM100: + case GP100: { + args.gf100.version = 0; + args.gf100.vol = (nvkm_memory_target(map->memory) != NVKM_MEM_TARGET_VRAM); + args.gf100.ro = !(vma->access & NV_MEM_ACCESS_WO); + args.gf100.priv = !!(vma->access & NV_MEM_ACCESS_SYS); + args.gf100.kind = (mem->memtype & 0x0ff); + argv = &args.gf100; + argc = sizeof(args.gf100); + } + break; default: break; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index 167c201d29e35..39bf26684b411 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -68,6 +68,12 @@ const u8 gf100_pte_storage_type_map[256] = 0xfe, 0xfe, 0xfe, 0xfe, 0xff, 0xfd, 0xfe, 0xff }; +const u8 * +gf100_mmu_kind(struct nvkm_mmu *mmu, int *count) +{ + *count = ARRAY_SIZE(gf100_pte_storage_type_map); + return gf100_pte_storage_type_map; +} void gf100_vm_map_pgt(struct nvkm_vmm *vmm, u32 index, struct nvkm_memory *pgt[2]) @@ -195,12 +201,8 @@ gf100_mmu = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .map_pgt = gf100_vm_map_pgt, - .map = gf100_vm_map, - .map_sg = gf100_vm_map_sg, - .unmap = gf100_vm_unmap, - .flush = gf100_vm_flush, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gf100_vmm_new }, + .kind = gf100_mmu_kind, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 326781246714f..64f765ec3e6e4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -52,6 +52,7 @@ void gf100_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, u32, u32, dma_addr_t *); void gf100_vm_unmap(struct nvkm_vma *, struct nvkm_memory *, u32, u32); void gf100_vm_flush(struct nvkm_vm *); +const u8 *gf100_mmu_kind(struct nvkm_mmu *, int *count); struct nvkm_mmu_pt { union { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index cbb8d47fbd119..dfc4b46129bdc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -70,7 +70,15 @@ struct nvkm_vmm_desc_func { }; extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; +void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32); extern const struct nvkm_vmm_desc_func gf100_vmm_pgt; +void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32); +void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, + struct nvkm_vmm_map *); +void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, + struct nvkm_vmm_map *); +void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, + struct nvkm_vmm_map *); struct nvkm_vmm_desc { enum { @@ -165,6 +173,10 @@ int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); +int gf100_vmm_aper(enum nvkm_memory_target); +int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); +void gf100_vmm_flush_(struct nvkm_vmm *, int); +void gf100_vmm_flush(struct nvkm_vmm *, int); int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, struct nvkm_mmu *, u64, u64, void *, u32, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c index f8234af0120cd..ddd44ad797b78 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c @@ -22,13 +22,131 @@ #include "vmm.h" #include +#include +#include + +#include +#include + +static inline void +gf100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) +{ + u64 base = (addr >> 8) | map->type; + u64 data = base; + + if (map->ctag && !(map->next & (1ULL << 44))) { + while (ptes--) { + data = base | ((map->ctag >> 1) << 44); + if (!(map->ctag++ & 1)) + data |= BIT_ULL(60); + + VMM_WO064(pt, vmm, ptei++ * 8, data); + base += map->next; + } + } else { + map->type += ptes * map->ctag; + + while (ptes--) { + VMM_WO064(pt, vmm, ptei++ * 8, data); + data += map->next; + } + } +} + +void +gf100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte); +} + +void +gf100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + if (map->page->shift == PAGE_SHIFT) { + VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes); + nvkm_kmap(pt->memory); + while (ptes--) { + const u64 data = (*map->dma++ >> 8) | map->type; + VMM_WO064(pt, vmm, ptei++ * 8, data); + map->type += map->ctag; + } + nvkm_done(pt->memory); + return; + } + + VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte); +} + +void +gf100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte); +} + +void +gf100_vmm_pgt_unmap(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes); +} const struct nvkm_vmm_desc_func gf100_vmm_pgt = { + .unmap = gf100_vmm_pgt_unmap, + .mem = gf100_vmm_pgt_mem, + .dma = gf100_vmm_pgt_dma, + .sgl = gf100_vmm_pgt_sgl, }; +void +gf100_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) +{ + struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; + struct nvkm_mmu_pt *pd = pgd->pt[0]; + struct nvkm_mmu_pt *pt; + u64 data = 0; + + if ((pt = pgt->pt[0])) { + switch (nvkm_memory_target(pt->memory)) { + case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 0; break; + case NVKM_MEM_TARGET_HOST: data |= 2ULL << 0; + data |= BIT_ULL(35); /* VOL */ + break; + case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 0; break; + default: + WARN_ON(1); + return; + } + data |= pt->addr >> 8; + } + + if ((pt = pgt->pt[1])) { + switch (nvkm_memory_target(pt->memory)) { + case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 32; break; + case NVKM_MEM_TARGET_HOST: data |= 2ULL << 32; + data |= BIT_ULL(34); /* VOL */ + break; + case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 32; break; + default: + WARN_ON(1); + return; + } + data |= pt->addr << 24; + } + + nvkm_kmap(pd->memory); + VMM_WO064(pd, vmm, pdei * 8, data); + nvkm_done(pd->memory); +} + const struct nvkm_vmm_desc_func gf100_vmm_pgd = { + .unmap = gf100_vmm_pgt_unmap, + .pde = gf100_vmm_pgd_pde, }; static const struct nvkm_vmm_desc @@ -59,6 +177,140 @@ gf100_vmm_desc_16_16[] = { {} }; +void +gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) +{ + struct nvkm_subdev *subdev = &vmm->mmu->subdev; + struct nvkm_device *device = subdev->device; + u32 type; + + type = 0x00000001; /* PAGE_ALL */ + if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) + type |= 0x00000004; /* HUB_ONLY */ + + mutex_lock(&subdev->mutex); + /* Looks like maybe a "free flush slots" counter, the + * faster you write to 0x100cbc to more it decreases. + */ + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x100c80) & 0x00ff0000) + break; + ); + + nvkm_wr32(device, 0x100cb8, vmm->pd->pt[0]->addr >> 8); + nvkm_wr32(device, 0x100cbc, 0x80000000 | type); + + /* Wait for flush to be queued? */ + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x100c80) & 0x00008000) + break; + ); + mutex_unlock(&subdev->mutex); +} + +void +gf100_vmm_flush(struct nvkm_vmm *vmm, int depth) +{ + gf100_vmm_flush_(vmm, 0); +} + +int +gf100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, + struct nvkm_vmm_map *map) +{ + const enum nvkm_memory_target target = nvkm_memory_target(map->memory); + const struct nvkm_vmm_page *page = map->page; + const bool gm20x = page->desc->func->sparse != NULL; + union { + struct gf100_vmm_map_vn vn; + struct gf100_vmm_map_v0 v0; + } *args = argv; + struct nvkm_device *device = vmm->mmu->subdev.device; + struct nvkm_memory *memory = map->memory; + u8 kind, priv, ro, vol; + int kindn, aper, ret = -ENOSYS; + const u8 *kindm; + + map->next = (1 << page->shift) >> 8; + map->type = map->ctag = 0; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + vol = !!args->v0.vol; + ro = !!args->v0.ro; + priv = !!args->v0.priv; + kind = args->v0.kind; + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + vol = target == NVKM_MEM_TARGET_HOST; + ro = 0; + priv = 0; + kind = 0x00; + } else { + VMM_DEBUG(vmm, "args"); + return ret; + } + + aper = vmm->func->aper(target); + if (WARN_ON(aper < 0)) + return aper; + + kindm = vmm->mmu->func->kind(vmm->mmu, &kindn); + if (kind >= kindn || kindm[kind] == 0xff) { + VMM_DEBUG(vmm, "kind %02x", kind); + return -EINVAL; + } + + if (kindm[kind] != kind) { + u32 comp = (page->shift == 16 && !gm20x) ? 16 : 17; + u32 tags = ALIGN(nvkm_memory_size(memory), 1 << 17) >> comp; + if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) { + VMM_DEBUG(vmm, "comp %d %02x", aper, page->type); + return -EINVAL; + } + + ret = nvkm_memory_tags_get(memory, device, tags, + nvkm_ltc_tags_clear, + &map->tags); + if (ret) { + VMM_DEBUG(vmm, "comp %d", ret); + return ret; + } + + if (map->tags->mn) { + u64 tags = map->tags->mn->offset + (map->offset >> 17); + if (page->shift == 17 || !gm20x) { + map->type |= tags << 44; + map->ctag |= 1ULL << 44; + map->next |= 1ULL << 44; + } else { + map->ctag |= tags << 1 | 1; + } + } else { + kind = kindm[kind]; + } + } + + map->type |= BIT(0); + map->type |= (u64)priv << 1; + map->type |= (u64) ro << 2; + map->type |= (u64) vol << 32; + map->type |= (u64)aper << 33; + map->type |= (u64)kind << 36; + return 0; +} + +int +gf100_vmm_aper(enum nvkm_memory_target target) +{ + switch (target) { + case NVKM_MEM_TARGET_VRAM: return 0; + case NVKM_MEM_TARGET_HOST: return 2; + case NVKM_MEM_TARGET_NCOH: return 3; + default: + return -EINVAL; + } +} + void gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst) { @@ -99,6 +351,9 @@ static const struct nvkm_vmm_func gf100_vmm_17 = { .join = gf100_vmm_join, .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gf100_vmm_valid, + .flush = gf100_vmm_flush, .page = { { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, @@ -110,6 +365,9 @@ static const struct nvkm_vmm_func gf100_vmm_16 = { .join = gf100_vmm_join, .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gf100_vmm_valid, + .flush = gf100_vmm_flush, .page = { { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, From b597764327e1af1b4948f6e7478acc5c7f1f6c49 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 110/151] drm/nouveau/mmu/gk104,gk20a: implement new vmm backend Adds support for: - 64KiB big page size. - System-memory PTs. - LPTE "invalid" state. - (Tegra) Use of video memory aperture. Adds support for marking LPTEs invalid, resulting in the corresponding SPTEs being ignored, which is supposed to speed up TLB invalidates. On The Tegra side, this will switch to using the video memory aperture for all mappings. The HW will still target non-coherent system memory, but this aperture needs to be selected in order to support compression. Tegra's instmem backend somewhat cheated to get this effect previously. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c | 6 +----- .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c | 6 +----- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 4 ++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c | 17 +++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c | 18 ++++++++++++++++++ 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c index 693aa1b875f28..8cbd0723c25b6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c @@ -30,12 +30,8 @@ gk104_mmu = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .map_pgt = gf100_vm_map_pgt, - .map = gf100_vm_map, - .map_sg = gf100_vm_map_sg, - .unmap = gf100_vm_unmap, - .flush = gf100_vm_flush, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk104_vmm_new }, + .kind = gf100_mmu_kind, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c index b774d9a25c3e8..57def7244d385 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c @@ -30,12 +30,8 @@ gk20a_mmu = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .map_pgt = gf100_vm_map_pgt, - .map = gf100_vm_map, - .map_sg = gf100_vm_map_sg, - .unmap = gf100_vm_unmap, - .flush = gf100_vm_flush, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk20a_vmm_new }, + .kind = gf100_mmu_kind, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index dfc4b46129bdc..75f6429001405 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -80,6 +80,8 @@ void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, struct nvkm_vmm_map *); +void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32); + struct nvkm_vmm_desc { enum { PGD, @@ -178,6 +180,8 @@ int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); void gf100_vmm_flush_(struct nvkm_vmm *, int); void gf100_vmm_flush(struct nvkm_vmm *, int); +int gk20a_vmm_aper(enum nvkm_memory_target); + int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c index 0d33700e3d88e..0ebb7bccfcd22 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c @@ -21,8 +21,19 @@ */ #include "vmm.h" +void +gk104_vmm_lpt_invalid(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + /* VALID_FALSE + PRIV tells the MMU to ignore corresponding SPTEs. */ + VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(1) /* PRIV. */, ptes); +} + static const struct nvkm_vmm_desc_func gk104_vmm_lpt = { + .invalid = gk104_vmm_lpt_invalid, + .unmap = gf100_vmm_pgt_unmap, + .mem = gf100_vmm_pgt_mem, }; const struct nvkm_vmm_desc @@ -57,6 +68,9 @@ static const struct nvkm_vmm_func gk104_vmm_17 = { .join = gf100_vmm_join, .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gf100_vmm_valid, + .flush = gf100_vmm_flush, .page = { { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, @@ -68,6 +82,9 @@ static const struct nvkm_vmm_func gk104_vmm_16 = { .join = gf100_vmm_join, .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gf100_vmm_valid, + .flush = gf100_vmm_flush, .page = { { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c index e000c3d1554ae..8086994a04461 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c @@ -21,10 +21,25 @@ */ #include "vmm.h" +#include + +int +gk20a_vmm_aper(enum nvkm_memory_target target) +{ + switch (target) { + case NVKM_MEM_TARGET_NCOH: return 0; + default: + return -EINVAL; + } +} + static const struct nvkm_vmm_func gk20a_vmm_17 = { .join = gf100_vmm_join, .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gf100_vmm_valid, + .flush = gf100_vmm_flush, .page = { { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xxHC }, { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xxHx }, @@ -36,6 +51,9 @@ static const struct nvkm_vmm_func gk20a_vmm_16 = { .join = gf100_vmm_join, .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gf100_vmm_valid, + .flush = gf100_vmm_flush, .page = { { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xxHC }, { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xxHx }, From e12cf6ad43888c152c3b2edb59525587cd98a227 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 111/151] drm/nouveau/mmu/gm200,gm20b: implement new vmm backend Adds support for: - 64KiB big page size. - System-memory PTs. - LPTE "invalid" state. - (Tegra) Use of video memory aperture. - Sparse PDEs/PTEs. - Additional blocklinear kinds. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/ifb00d.h | 13 +++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c | 54 +++++++++++++++---- .../gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c | 12 +---- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 2 + .../drm/nouveau/nvkm/subdev/mmu/vmmgm200.c | 34 ++++++++++++ .../drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c | 6 +++ 6 files changed, 101 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/ifb00d.h b/drivers/gpu/drm/nouveau/include/nvif/ifb00d.h index 2e1e998103619..a0e4198305954 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/ifb00d.h +++ b/drivers/gpu/drm/nouveau/include/nvif/ifb00d.h @@ -11,4 +11,17 @@ struct gm200_vmm_v0 { __u8 version; __u8 bigpage; }; + +struct gm200_vmm_map_vn { + /* nvif_vmm_map_vX ... */ +}; + +struct gm200_vmm_map_v0 { + /* nvif_vmm_map_vX ... */ + __u8 version; + __u8 vol; + __u8 ro; + __u8 priv; + __u8 kind; +}; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c index 41332613c6443..c38c899e463cd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c @@ -25,6 +25,48 @@ #include +const u8 * +gm200_mmu_kind(struct nvkm_mmu *mmu, int *count) +{ + static const u8 + kind[256] = { + 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0xff, 0x01, /* 0x00 */ + 0x01, 0x01, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, 0x11, /* 0x10 */ + 0x11, 0x11, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0x27, /* 0x20 */ + 0x28, 0x29, 0x2a, 0x2b, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30 */ + 0xff, 0xff, 0x26, 0x27, 0x28, 0x29, 0x26, 0x27, + 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0x46, 0xff, /* 0x40 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x46, 0x46, 0x46, 0x46, 0xff, 0xff, 0xff, /* 0x50 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70 */ + 0xff, 0xff, 0xff, 0x7b, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0x7b, /* 0x80 */ + 0x7b, 0x7b, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xa7, /* 0xa0 */ + 0xa8, 0xa9, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7, + 0xa8, 0xa9, 0xaa, 0xc3, 0xff, 0xff, 0xff, 0xff, /* 0xc0 */ + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xfe, 0xc3, 0xc3, + 0xc3, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0 */ + 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, + 0xfe, 0xff, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xff, /* 0xe0 */ + 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xfe, 0xff, + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xff, 0xfd, 0xfe, 0xff + }; + *count = ARRAY_SIZE(kind); + return kind; +} + static const struct nvkm_mmu_func gm200_mmu = { .limit = (1ULL << 40), @@ -32,12 +74,8 @@ gm200_mmu = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .map_pgt = gf100_vm_map_pgt, - .map = gf100_vm_map, - .map_sg = gf100_vm_map_sg, - .unmap = gf100_vm_unmap, - .flush = gf100_vm_flush, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm200_vmm_new }, + .kind = gm200_mmu_kind, }; static const struct nvkm_mmu_func @@ -47,12 +85,8 @@ gm200_mmu_fixed = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .map_pgt = gf100_vm_map_pgt, - .map = gf100_vm_map, - .map_sg = gf100_vm_map_sg, - .unmap = gf100_vm_unmap, - .flush = gf100_vm_flush, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm200_vmm_new_fixed }, + .kind = gm200_mmu_kind, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c index 25b16975bddd2..8cd6d1617bd72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c @@ -32,12 +32,8 @@ gm20b_mmu = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .map_pgt = gf100_vm_map_pgt, - .map = gf100_vm_map, - .map_sg = gf100_vm_map_sg, - .unmap = gf100_vm_unmap, - .flush = gf100_vm_flush, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new }, + .kind = gm200_mmu_kind, }; static const struct nvkm_mmu_func @@ -47,12 +43,8 @@ gm20b_mmu_fixed = { .pgt_bits = 27 - 12, .spg_shift = 12, .lpg_shift = 17, - .map_pgt = gf100_vm_map_pgt, - .map = gf100_vm_map, - .map_sg = gf100_vm_map_sg, - .unmap = gf100_vm_unmap, - .flush = gf100_vm_flush, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new_fixed }, + .kind = gm200_mmu_kind, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 64f765ec3e6e4..5828dd82cba73 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -54,6 +54,8 @@ void gf100_vm_unmap(struct nvkm_vma *, struct nvkm_memory *, u32, u32); void gf100_vm_flush(struct nvkm_vm *); const u8 *gf100_mmu_kind(struct nvkm_mmu *, int *count); +const u8 *gm200_mmu_kind(struct nvkm_mmu *, int *); + struct nvkm_mmu_pt { union { struct nvkm_mmu_ptc *ptc; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c index 43138560b1084..a1676a4644fe2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c @@ -24,16 +24,44 @@ #include #include +static void +gm200_vmm_pgt_sparse(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + /* VALID_FALSE + VOL tells the MMU to treat the PTE as sparse. */ + VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(32) /* VOL. */, ptes); +} + static const struct nvkm_vmm_desc_func gm200_vmm_spt = { + .unmap = gf100_vmm_pgt_unmap, + .sparse = gm200_vmm_pgt_sparse, + .mem = gf100_vmm_pgt_mem, + .dma = gf100_vmm_pgt_dma, + .sgl = gf100_vmm_pgt_sgl, }; static const struct nvkm_vmm_desc_func gm200_vmm_lpt = { + .invalid = gk104_vmm_lpt_invalid, + .unmap = gf100_vmm_pgt_unmap, + .sparse = gm200_vmm_pgt_sparse, + .mem = gf100_vmm_pgt_mem, }; +static void +gm200_vmm_pgd_sparse(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes) +{ + /* VALID_FALSE + VOL_BIG tells the MMU to treat the PDE as sparse. */ + VMM_FO064(pt, vmm, pdei * 8, BIT_ULL(35) /* VOL_BIG. */, pdes); +} + static const struct nvkm_vmm_desc_func gm200_vmm_pgd = { + .unmap = gf100_vmm_pgt_unmap, + .sparse = gm200_vmm_pgd_sparse, + .pde = gf100_vmm_pgd_pde, }; const struct nvkm_vmm_desc @@ -82,6 +110,9 @@ static const struct nvkm_vmm_func gm200_vmm_17 = { .join = gm200_vmm_join, .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gf100_vmm_valid, + .flush = gf100_vmm_flush, .page = { { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx }, { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SVxC }, @@ -94,6 +125,9 @@ static const struct nvkm_vmm_func gm200_vmm_16 = { .join = gm200_vmm_join, .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gf100_vmm_valid, + .flush = gf100_vmm_flush, .page = { { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx }, { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SVxC }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c index ec24c43f86ccb..64d4b6cff8dd2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c @@ -25,6 +25,9 @@ static const struct nvkm_vmm_func gm20b_vmm_17 = { .join = gm200_vmm_join, .part = gf100_vmm_part, + .aper = gk20a_vmm_aper, + .valid = gf100_vmm_valid, + .flush = gf100_vmm_flush, .page = { { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx }, { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SxHC }, @@ -37,6 +40,9 @@ static const struct nvkm_vmm_func gm20b_vmm_16 = { .join = gm200_vmm_join, .part = gf100_vmm_part, + .aper = gk20a_vmm_aper, + .valid = gf100_vmm_valid, + .flush = gf100_vmm_flush, .page = { { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx }, { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SxHC }, From f9400afb1e11c33830bd99a0c9fafe3f4d07a7cc Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 112/151] drm/nouveau/mmu/gp100,gp10b: implement new vmm backend Adds support for: - 64KiB/2MiB big page sizes (128KiB not supported by HW with new PT layout). - System-memory PTs. - LPTE "invalid" state. - (Tegra) Use of video memory aperture. - Sparse PDEs/PTEs. - Additional blocklinear kinds. - 49-bit address-space. GP100 supports an entirely new 5-level page table layout that provides an expanded 49-bit address-space. It also supports the layout present on previous generations, which we've been making do with until now. This commit implements support for the new layout, and enables it by default. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/ifc00d.h | 13 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 123 --------- .../gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c | 3 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/gp10b. | 0 .../gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c | 3 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 7 - drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 2 + .../drm/nouveau/nvkm/subdev/mmu/vmmgf100.c | 2 +- .../drm/nouveau/nvkm/subdev/mmu/vmmgp100.c | 258 ++++++++++++++++++ .../drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c | 3 + 10 files changed, 281 insertions(+), 133 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b. diff --git a/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h b/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h index 39be62ca73396..1d9c637859f3a 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h +++ b/drivers/gpu/drm/nouveau/include/nvif/ifc00d.h @@ -5,4 +5,17 @@ struct gp100_vmm_vn { /* nvif_vmm_vX ... */ }; + +struct gp100_vmm_map_vn { + /* nvif_vmm_map_vX ... */ +}; + +struct gp100_vmm_map_v0 { + /* nvif_vmm_map_vX ... */ + __u8 version; + __u8 vol; + __u8 ro; + __u8 priv; + __u8 kind; +}; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index 39bf26684b411..24e2d933524d3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -23,10 +23,6 @@ */ #include "vmm.h" -#include -#include -#include - #include /* Map from compressed to corresponding uncompressed storage type. @@ -75,125 +71,6 @@ gf100_mmu_kind(struct nvkm_mmu *mmu, int *count) return gf100_pte_storage_type_map; } -void -gf100_vm_map_pgt(struct nvkm_vmm *vmm, u32 index, struct nvkm_memory *pgt[2]) -{ - struct nvkm_memory *pgd = vmm->pd->pt[0]->memory; - u32 pde[2] = { 0, 0 }; - - if (pgt[0]) - pde[1] = 0x00000001 | (nvkm_memory_addr(pgt[0]) >> 8); - if (pgt[1]) - pde[0] = 0x00000001 | (nvkm_memory_addr(pgt[1]) >> 8); - - nvkm_kmap(pgd); - nvkm_wo32(pgd, (index * 8) + 0, pde[0]); - nvkm_wo32(pgd, (index * 8) + 4, pde[1]); - nvkm_done(pgd); -} - -static inline u64 -gf100_vm_addr(struct nvkm_vma *vma, u64 phys, u32 memtype, u32 target) -{ - phys >>= 8; - - phys |= 0x00000001; /* present */ - if (vma->access & NV_MEM_ACCESS_SYS) - phys |= 0x00000002; - - phys |= ((u64)target << 32); - phys |= ((u64)memtype << 36); - return phys; -} - -void -gf100_vm_map(struct nvkm_vma *vma, struct nvkm_memory *pgt, - struct nvkm_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta) -{ - u64 next = 1 << (vma->node->type - 8); - - phys = gf100_vm_addr(vma, phys, mem->memtype, 0); - pte <<= 3; - - if (mem->tag) { - u32 tag = mem->tag->offset + (delta >> 17); - phys |= (u64)tag << (32 + 12); - next |= (u64)1 << (32 + 12); - } - - nvkm_kmap(pgt); - while (cnt--) { - nvkm_wo32(pgt, pte + 0, lower_32_bits(phys)); - nvkm_wo32(pgt, pte + 4, upper_32_bits(phys)); - phys += next; - pte += 8; - } - nvkm_done(pgt); -} - -void -gf100_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt, - struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list) -{ - u32 target = (vma->access & NV_MEM_ACCESS_NOSNOOP) ? 7 : 5; - /* compressed storage types are invalid for system memory */ - u32 memtype = gf100_pte_storage_type_map[mem->memtype & 0xff]; - - nvkm_kmap(pgt); - pte <<= 3; - while (cnt--) { - u64 phys = gf100_vm_addr(vma, *list++, memtype, target); - nvkm_wo32(pgt, pte + 0, lower_32_bits(phys)); - nvkm_wo32(pgt, pte + 4, upper_32_bits(phys)); - pte += 8; - } - nvkm_done(pgt); -} - -void -gf100_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt) -{ - nvkm_kmap(pgt); - pte <<= 3; - while (cnt--) { - nvkm_wo32(pgt, pte + 0, 0x00000000); - nvkm_wo32(pgt, pte + 4, 0x00000000); - pte += 8; - } - nvkm_done(pgt); -} - -void -gf100_vm_flush(struct nvkm_vm *vm) -{ - struct nvkm_mmu *mmu = vm->mmu; - struct nvkm_device *device = mmu->subdev.device; - u32 type; - - type = 0x00000001; /* PAGE_ALL */ - if (atomic_read(&vm->engref[NVKM_SUBDEV_BAR])) - type |= 0x00000004; /* HUB_ONLY */ - - mutex_lock(&mmu->subdev.mutex); - /* looks like maybe a "free flush slots" counter, the - * faster you write to 0x100cbc to more it decreases - */ - nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x100c80) & 0x00ff0000) - break; - ); - - nvkm_wr32(device, 0x100cb8, vm->pd->pt[0]->addr >> 8); - nvkm_wr32(device, 0x100cbc, 0x80000000 | type); - - /* wait for flush to be queued? */ - nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x100c80) & 0x00008000) - break; - ); - mutex_unlock(&mmu->subdev.mutex); -} - static const struct nvkm_mmu_func gf100_mmu = { .limit = (1ULL << 40), diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c index b1ba864b4b35c..0cc9c89c0e733 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c @@ -31,12 +31,13 @@ gp100_mmu = { .dma_bits = 47, .lpg_shift = 16, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, + .kind = gm200_mmu_kind, }; int gp100_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { - if (!nvkm_boolopt(device->cfgopt, "GP100MmuLayout", false)) + if (!nvkm_boolopt(device->cfgopt, "GP100MmuLayout", true)) return gm200_mmu_new(device, index, pmmu); return nvkm_mmu_new_(&gp100_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b. b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b. new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c index 5d760a198b227..5c72176977608 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -31,12 +31,13 @@ gp10b_mmu = { .dma_bits = 47, .lpg_shift = 16, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, + .kind = gm200_mmu_kind, }; int gp10b_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) { - if (!nvkm_boolopt(device->cfgopt, "GP100MmuLayout", false)) + if (!nvkm_boolopt(device->cfgopt, "GP100MmuLayout", true)) return gm20b_mmu_new(device, index, pmmu); return nvkm_mmu_new_(&gp10b_mmu, device, index, pmmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 5828dd82cba73..08d211d9fc55d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -45,13 +45,6 @@ extern const struct nvkm_mmu_func nv04_mmu; const u8 *nv50_mmu_kind(struct nvkm_mmu *, int *count); -void gf100_vm_map_pgt(struct nvkm_vmm *, u32, struct nvkm_memory **); -void gf100_vm_map(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, - u32, u32, u64, u64); -void gf100_vm_map_sg(struct nvkm_vma *, struct nvkm_memory *, struct nvkm_mem *, - u32, u32, dma_addr_t *); -void gf100_vm_unmap(struct nvkm_vma *, struct nvkm_memory *, u32, u32); -void gf100_vm_flush(struct nvkm_vm *); const u8 *gf100_mmu_kind(struct nvkm_mmu *, int *count); const u8 *gm200_mmu_kind(struct nvkm_mmu *, int *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 75f6429001405..bb1353e950684 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -189,6 +189,8 @@ int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); +int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); +void gp100_vmm_flush(struct nvkm_vmm *, int); int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c index ddd44ad797b78..faf5a7e9265ee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c @@ -182,7 +182,7 @@ gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) { struct nvkm_subdev *subdev = &vmm->mmu->subdev; struct nvkm_device *device = subdev->device; - u32 type; + u32 type = depth << 24; type = 0x00000001; /* PAGE_ALL */ if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c index 68f67812aecca..059fafe0e771a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c @@ -21,20 +21,190 @@ */ #include "vmm.h" +#include +#include + +#include +#include + +static inline void +gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) +{ + u64 data = (addr >> 4) | map->type; + + map->type += ptes * map->ctag; + + while (ptes--) { + VMM_WO064(pt, vmm, ptei++ * 8, data); + data += map->next; + } +} + +static void +gp100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte); +} + +static void +gp100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + if (map->page->shift == PAGE_SHIFT) { + VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes); + nvkm_kmap(pt->memory); + while (ptes--) { + const u64 data = (*map->dma++ >> 4) | map->type; + VMM_WO064(pt, vmm, ptei++ * 8, data); + map->type += map->ctag; + } + nvkm_done(pt->memory); + return; + } + + VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte); +} + +static void +gp100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte); +} + +static void +gp100_vmm_pgt_sparse(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + /* VALID_FALSE + VOL tells the MMU to treat the PTE as sparse. */ + VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(3) /* VOL. */, ptes); +} + static const struct nvkm_vmm_desc_func gp100_vmm_desc_spt = { + .unmap = gf100_vmm_pgt_unmap, + .sparse = gp100_vmm_pgt_sparse, + .mem = gp100_vmm_pgt_mem, + .dma = gp100_vmm_pgt_dma, + .sgl = gp100_vmm_pgt_sgl, }; +static void +gp100_vmm_lpt_invalid(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + /* VALID_FALSE + PRIV tells the MMU to ignore corresponding SPTEs. */ + VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(5) /* PRIV. */, ptes); +} + static const struct nvkm_vmm_desc_func gp100_vmm_desc_lpt = { + .invalid = gp100_vmm_lpt_invalid, + .unmap = gf100_vmm_pgt_unmap, + .sparse = gp100_vmm_pgt_sparse, + .mem = gp100_vmm_pgt_mem, }; +static inline void +gp100_vmm_pd0_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) +{ + u64 data = (addr >> 4) | map->type; + + map->type += ptes * map->ctag; + + while (ptes--) { + VMM_WO128(pt, vmm, ptei++ * 0x10, data, 0ULL); + data += map->next; + } +} + +static void +gp100_vmm_pd0_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gp100_vmm_pd0_pte); +} + +static inline bool +gp100_vmm_pde(struct nvkm_mmu_pt *pt, u64 *data) +{ + switch (nvkm_memory_target(pt->memory)) { + case NVKM_MEM_TARGET_VRAM: *data |= 1ULL << 1; break; + case NVKM_MEM_TARGET_HOST: *data |= 2ULL << 1; + *data |= BIT_ULL(3); /* VOL. */ + break; + case NVKM_MEM_TARGET_NCOH: *data |= 3ULL << 1; break; + default: + WARN_ON(1); + return false; + } + *data |= pt->addr >> 4; + return true; +} + +static void +gp100_vmm_pd0_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) +{ + struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; + struct nvkm_mmu_pt *pd = pgd->pt[0]; + u64 data[2] = {}; + + if (pgt->pt[0] && !gp100_vmm_pde(pgt->pt[0], &data[0])) + return; + if (pgt->pt[1] && !gp100_vmm_pde(pgt->pt[1], &data[1])) + return; + + nvkm_kmap(pd->memory); + VMM_WO128(pd, vmm, pdei * 0x10, data[0], data[1]); + nvkm_done(pd->memory); +} + +static void +gp100_vmm_pd0_sparse(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes) +{ + /* VALID_FALSE + VOL_BIG tells the MMU to treat the PDE as sparse. */ + VMM_FO128(pt, vmm, pdei * 0x10, BIT_ULL(3) /* VOL_BIG. */, 0ULL, pdes); +} + +static void +gp100_vmm_pd0_unmap(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes) +{ + VMM_FO128(pt, vmm, pdei * 0x10, 0ULL, 0ULL, pdes); +} + static const struct nvkm_vmm_desc_func gp100_vmm_desc_pd0 = { + .unmap = gp100_vmm_pd0_unmap, + .sparse = gp100_vmm_pd0_sparse, + .pde = gp100_vmm_pd0_pde, + .mem = gp100_vmm_pd0_mem, }; +static void +gp100_vmm_pd1_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) +{ + struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; + struct nvkm_mmu_pt *pd = pgd->pt[0]; + u64 data = 0; + + if (!gp100_vmm_pde(pgt->pt[0], &data)) + return; + + nvkm_kmap(pd->memory); + VMM_WO064(pd, vmm, pdei * 8, data); + nvkm_done(pd->memory); +} + static const struct nvkm_vmm_desc_func gp100_vmm_desc_pd1 = { + .unmap = gf100_vmm_pgt_unmap, + .sparse = gp100_vmm_pgt_sparse, + .pde = gp100_vmm_pd1_pde, }; const struct nvkm_vmm_desc @@ -57,6 +227,91 @@ gp100_vmm_desc_12[] = { {} }; +int +gp100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, + struct nvkm_vmm_map *map) +{ + const enum nvkm_memory_target target = nvkm_memory_target(map->memory); + const struct nvkm_vmm_page *page = map->page; + union { + struct gp100_vmm_map_vn vn; + struct gp100_vmm_map_v0 v0; + } *args = argv; + struct nvkm_device *device = vmm->mmu->subdev.device; + struct nvkm_memory *memory = map->memory; + u8 kind, priv, ro, vol; + int kindn, aper, ret = -ENOSYS; + const u8 *kindm; + + map->next = (1ULL << page->shift) >> 4; + map->type = 0; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + vol = !!args->v0.vol; + ro = !!args->v0.ro; + priv = !!args->v0.priv; + kind = args->v0.kind; + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + vol = target == NVKM_MEM_TARGET_HOST; + ro = 0; + priv = 0; + kind = 0x00; + } else { + VMM_DEBUG(vmm, "args"); + return ret; + } + + aper = vmm->func->aper(target); + if (WARN_ON(aper < 0)) + return aper; + + kindm = vmm->mmu->func->kind(vmm->mmu, &kindn); + if (kind >= kindn || kindm[kind] == 0xff) { + VMM_DEBUG(vmm, "kind %02x", kind); + return -EINVAL; + } + + if (kindm[kind] != kind) { + u64 tags = nvkm_memory_size(memory) >> 16; + if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) { + VMM_DEBUG(vmm, "comp %d %02x", aper, page->type); + return -EINVAL; + } + + ret = nvkm_memory_tags_get(memory, device, tags, + nvkm_ltc_tags_clear, + &map->tags); + if (ret) { + VMM_DEBUG(vmm, "comp %d", ret); + return ret; + } + + if (map->tags->mn) { + tags = map->tags->mn->offset + (map->offset >> 16); + map->ctag |= ((1ULL << page->shift) >> 16) << 36; + map->type |= tags << 36; + map->next |= map->ctag; + } else { + kind = kindm[kind]; + } + } + + map->type |= BIT(0); + map->type |= (u64)aper << 1; + map->type |= (u64) vol << 3; + map->type |= (u64)priv << 5; + map->type |= (u64) ro << 6; + map->type |= (u64)kind << 56; + return 0; +} + +void +gp100_vmm_flush(struct nvkm_vmm *vmm, int depth) +{ + gf100_vmm_flush_(vmm, 5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth); +} + int gp100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) { @@ -68,6 +323,9 @@ static const struct nvkm_vmm_func gp100_vmm = { .join = gp100_vmm_join, .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gp100_vmm_valid, + .flush = gp100_vmm_flush, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c index 8b7f9b82750c0..3dcc6bddb32f3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c @@ -25,6 +25,9 @@ static const struct nvkm_vmm_func gp10b_vmm = { .join = gp100_vmm_join, .part = gf100_vmm_part, + .aper = gk20a_vmm_aper, + .valid = gp100_vmm_valid, + .flush = gp100_vmm_flush, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, From 26880e76863ace2dd34c14fcadaedf97a2ace417 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 113/151] drm/nouveau/mmu: remove support for old backends Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/fb.h | 1 - .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 3 - drivers/gpu/drm/nouveau/nouveau_mem.c | 30 -- drivers/gpu/drm/nouveau/nouveau_mem.h | 1 - .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 288 +----------------- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c | 4 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c | 4 - .../gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 14 - 16 files changed, 3 insertions(+), 358 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index e1776506746b2..5b5ff5a9a1271 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -23,7 +23,6 @@ #define NV_MEM_COMP_VM 0x03 struct nvkm_mem { - struct nvkm_mm_node *tag; struct nvkm_mm_node *mem; dma_addr_t *pages; u32 memtype; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index ccd87d508d048..630c3cbaf2b92 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -39,9 +39,6 @@ struct nvkm_vm { struct nvkm_mm mm; struct kref refcount; - struct nvkm_vm_pgt *pgt; - u32 fpde; - u32 lpde; bool bootstrapped; atomic_t engref[NVKM_SUBDEV_NR]; diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c index 21d904ded441f..70fbe5e72b55c 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.c +++ b/drivers/gpu/drm/nouveau/nouveau_mem.c @@ -23,8 +23,6 @@ #include "nouveau_drv.h" #include "nouveau_bo.h" -#include - #include int @@ -46,8 +44,6 @@ nouveau_mem_fini(struct nouveau_mem *mem) nvkm_vm_unmap(&mem->vma[0]); nvkm_vm_put(&mem->vma[0]); } - nvkm_memory_tags_put(&mem->memory, nvxx_device(&mem->cli->device), - &mem->tags); } int @@ -112,32 +108,6 @@ nouveau_mem_vram(struct ttm_mem_reg *reg, bool contig, u8 page) mem->_mem->size = size >> NVKM_RAM_MM_SHIFT; mem->_mem->offset = nvkm_memory_addr(mem->_mem->memory); - - if (cli->device.info.chipset < 0xc0 && mem->comp) { - if (page == 16) { - ret = nvkm_memory_tags_get(mem->_mem->memory, device, - size >> page, NULL, - &mem->tags); - WARN_ON(ret); - } - if (!mem->tags || !mem->tags->mn) - mem->comp = 0; - } else - if (cli->device.info.chipset >= 0xc0 && - gf100_pte_storage_type_map[mem->kind] != mem->kind) { - if (page == 17) { - ret = nvkm_memory_tags_get(mem->_mem->memory, device, - size >> page, - nvkm_ltc_tags_clear, - &mem->tags); - WARN_ON(ret); - } - if (!mem->tags || !mem->tags->mn) - mem->kind = gf100_pte_storage_type_map[mem->kind]; - } - - if (mem->tags && mem->tags->mn) - mem->_mem->tag = mem->tags->mn; mem->_mem->mem = ((struct nvkm_vram *)mem->_mem->memory)->mn; mem->_mem->memtype = (mem->comp << 7) | mem->kind; diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.h b/drivers/gpu/drm/nouveau/nouveau_mem.h index 89e9e7b9b00ca..20930ebc5e21d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.h +++ b/drivers/gpu/drm/nouveau/nouveau_mem.h @@ -25,7 +25,6 @@ struct nouveau_mem { struct nvkm_vma bar_vma; struct nvkm_memory memory; - struct nvkm_tags *tags; }; enum nvif_vmm_get { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 741021ff8c27e..31832398f1e98 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -24,7 +24,6 @@ #include "priv.h" #include "vmm.h" -#include #include #include @@ -316,17 +315,6 @@ void nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node) { const struct nvkm_vmm_page *page = vma->vm->func->page; - struct nvkm_vm *vm = vma->vm; - struct nvkm_mmu *mmu = vm->mmu; - struct nvkm_mm_node *r = node->mem; - int big = vma->node->type != mmu->func->spg_shift; - u32 offset = vma->node->offset + (delta >> 12); - u32 bits = vma->node->type - 12; - u32 pde = (offset >> mmu->func->pgt_bits) - vm->fpde; - u32 pte = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits; - u32 max = 1 << (mmu->func->pgt_bits - bits); - u32 end, len; - if (page->desc->func->unmap) { struct nvkm_vmm_map map = { .mem = node->mem }; while (page->shift != vma->node->type) @@ -334,36 +322,6 @@ nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node) nvkm_vm_map_(page, vma, delta, node, page->desc->func->mem, &map); return; } - - delta = 0; - while (r) { - u64 phys = (u64)r->offset << 12; - u32 num = r->length >> bits; - - while (num) { - struct nvkm_memory *pgt = vm->pgt[pde].mem[big]; - - end = (pte + num); - if (unlikely(end >= max)) - end = max; - len = end - pte; - - mmu->func->map(vma, pgt, node, pte, len, phys, delta); - - num -= len; - pte += len; - if (unlikely(end >= max)) { - phys += len << (bits + 12); - pde++; - pte = 0; - } - - delta += (u64)len << vma->node->type; - } - r = r->next; - } - - mmu->func->flush(vm); } static void @@ -371,20 +329,6 @@ nvkm_vm_map_sg_table(struct nvkm_vma *vma, u64 delta, u64 length, struct nvkm_mem *mem) { const struct nvkm_vmm_page *page = vma->vm->func->page; - struct nvkm_vm *vm = vma->vm; - struct nvkm_mmu *mmu = vm->mmu; - int big = vma->node->type != mmu->func->spg_shift; - u32 offset = vma->node->offset + (delta >> 12); - u32 bits = vma->node->type - 12; - u32 num = length >> vma->node->type; - u32 pde = (offset >> mmu->func->pgt_bits) - vm->fpde; - u32 pte = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits; - u32 max = 1 << (mmu->func->pgt_bits - bits); - unsigned m, sglen; - u32 end, len; - int i; - struct scatterlist *sg; - if (page->desc->func->unmap) { struct nvkm_vmm_map map = { .sgl = mem->sg->sgl }; while (page->shift != vma->node->type) @@ -392,45 +336,6 @@ nvkm_vm_map_sg_table(struct nvkm_vma *vma, u64 delta, u64 length, nvkm_vm_map_(page, vma, delta, mem, page->desc->func->sgl, &map); return; } - - for_each_sg(mem->sg->sgl, sg, mem->sg->nents, i) { - struct nvkm_memory *pgt = vm->pgt[pde].mem[big]; - sglen = sg_dma_len(sg) >> PAGE_SHIFT; - - end = pte + sglen; - if (unlikely(end >= max)) - end = max; - len = end - pte; - - for (m = 0; m < len; m++) { - dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT); - - mmu->func->map_sg(vma, pgt, mem, pte, 1, &addr); - num--; - pte++; - - if (num == 0) - goto finish; - } - if (unlikely(end >= max)) { - pde++; - pte = 0; - } - if (m < sglen) { - for (; m < sglen; m++) { - dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT); - - mmu->func->map_sg(vma, pgt, mem, pte, 1, &addr); - num--; - pte++; - if (num == 0) - goto finish; - } - } - - } -finish: - mmu->func->flush(vm); } static void @@ -438,18 +343,6 @@ nvkm_vm_map_sg(struct nvkm_vma *vma, u64 delta, u64 length, struct nvkm_mem *mem) { const struct nvkm_vmm_page *page = vma->vm->func->page; - struct nvkm_vm *vm = vma->vm; - struct nvkm_mmu *mmu = vm->mmu; - dma_addr_t *list = mem->pages; - int big = vma->node->type != mmu->func->spg_shift; - u32 offset = vma->node->offset + (delta >> 12); - u32 bits = vma->node->type - 12; - u32 num = length >> vma->node->type; - u32 pde = (offset >> mmu->func->pgt_bits) - vm->fpde; - u32 pte = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits; - u32 max = 1 << (mmu->func->pgt_bits - bits); - u32 end, len; - if (page->desc->func->unmap) { struct nvkm_vmm_map map = { .dma = mem->pages }; while (page->shift != vma->node->type) @@ -457,27 +350,6 @@ nvkm_vm_map_sg(struct nvkm_vma *vma, u64 delta, u64 length, nvkm_vm_map_(page, vma, delta, mem, page->desc->func->dma, &map); return; } - - while (num) { - struct nvkm_memory *pgt = vm->pgt[pde].mem[big]; - - end = (pte + num); - if (unlikely(end >= max)) - end = max; - len = end - pte; - - mmu->func->map_sg(vma, pgt, mem, pte, len, list); - - num -= len; - pte += len; - list += len; - if (unlikely(end >= max)) { - pde++; - pte = 0; - } - } - - mmu->func->flush(vm); } void @@ -496,16 +368,6 @@ void nvkm_vm_unmap_at(struct nvkm_vma *vma, u64 delta, u64 length) { struct nvkm_vm *vm = vma->vm; - struct nvkm_mmu *mmu = vm->mmu; - int big = vma->node->type != mmu->func->spg_shift; - u32 offset = vma->node->offset + (delta >> 12); - u32 bits = vma->node->type - 12; - u32 num = length >> vma->node->type; - u32 pde = (offset >> mmu->func->pgt_bits) - vm->fpde; - u32 pte = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits; - u32 max = 1 << (mmu->func->pgt_bits - bits); - u32 end, len; - if (vm->func->page->desc->func->unmap) { const struct nvkm_vmm_page *page = vm->func->page; while (page->shift != vma->node->type) @@ -516,26 +378,6 @@ nvkm_vm_unmap_at(struct nvkm_vma *vma, u64 delta, u64 length) mutex_unlock(&vm->mutex); return; } - - while (num) { - struct nvkm_memory *pgt = vm->pgt[pde].mem[big]; - - end = (pte + num); - if (unlikely(end >= max)) - end = max; - len = end - pte; - - mmu->func->unmap(vma, pgt, pte, len); - - num -= len; - pte += len; - if (unlikely(end >= max)) { - pde++; - pte = 0; - } - } - - mmu->func->flush(vm); } void @@ -547,63 +389,12 @@ nvkm_vm_unmap(struct nvkm_vma *vma) nvkm_memory_unref(&vma->memory); } -static void -nvkm_vm_unmap_pgt(struct nvkm_vm *vm, int big, u32 fpde, u32 lpde) -{ - struct nvkm_mmu *mmu = vm->mmu; - struct nvkm_vm_pgt *vpgt; - struct nvkm_memory *pgt; - u32 pde; - - for (pde = fpde; pde <= lpde; pde++) { - vpgt = &vm->pgt[pde - vm->fpde]; - if (--vpgt->refcount[big]) - continue; - - pgt = vpgt->mem[big]; - vpgt->mem[big] = NULL; - - if (mmu->func->map_pgt) - mmu->func->map_pgt(vm, pde, vpgt->mem); - - mmu->func->flush(vm); - - nvkm_memory_unref(&pgt); - } -} - -static int -nvkm_vm_map_pgt(struct nvkm_vm *vm, u32 pde, u32 type) -{ - struct nvkm_mmu *mmu = vm->mmu; - struct nvkm_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde]; - int big = (type != mmu->func->spg_shift); - u32 pgt_size; - int ret; - - pgt_size = (1 << (mmu->func->pgt_bits + 12)) >> type; - pgt_size *= 8; - - ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST, - pgt_size, 0x1000, true, &vpgt->mem[big]); - if (unlikely(ret)) - return ret; - - if (mmu->func->map_pgt) - mmu->func->map_pgt(vm, pde, vpgt->mem); - - vpgt->refcount[big]++; - return 0; -} - int nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access, struct nvkm_vma *vma) { - struct nvkm_mmu *mmu = vm->mmu; u32 align = (1 << page_shift) >> 12; u32 msize = size >> 12; - u32 fpde, lpde, pde; int ret; mutex_lock(&vm->mutex); @@ -626,32 +417,7 @@ nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access, mutex_unlock(&vm->mutex); return ret; } - - goto done; - } - - fpde = (vma->node->offset >> mmu->func->pgt_bits); - lpde = (vma->node->offset + vma->node->length - 1) >> mmu->func->pgt_bits; - - for (pde = fpde; pde <= lpde; pde++) { - struct nvkm_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde]; - int big = (vma->node->type != mmu->func->spg_shift); - - if (likely(vpgt->refcount[big])) { - vpgt->refcount[big]++; - continue; - } - - ret = nvkm_vm_map_pgt(vm, pde, vma->node->type); - if (ret) { - if (pde != fpde) - nvkm_vm_unmap_pgt(vm, big, fpde, pde - 1); - nvkm_mm_free(&vm->mm, &vma->node); - mutex_unlock(&vm->mutex); - return ret; - } } -done: mutex_unlock(&vm->mutex); vma->memory = NULL; @@ -668,7 +434,6 @@ nvkm_vm_put(struct nvkm_vma *vma) { struct nvkm_mmu *mmu; struct nvkm_vm *vm; - u32 fpde, lpde; if (unlikely(vma->node == NULL)) return; @@ -678,9 +443,6 @@ nvkm_vm_put(struct nvkm_vma *vma) nvkm_memory_tags_put(vma->memory, mmu->subdev.device, &vma->tags); nvkm_memory_unref(&vma->memory); - fpde = (vma->node->offset >> mmu->func->pgt_bits); - lpde = (vma->node->offset + vma->node->length - 1) >> mmu->func->pgt_bits; - mutex_lock(&vm->mutex); if (vm->func->page->desc->func->unmap) { const struct nvkm_vmm_page *page = vm->func->page; @@ -689,11 +451,8 @@ nvkm_vm_put(struct nvkm_vma *vma) nvkm_vmm_ptes_put(vm, page, vma->node->offset << 12, vma->node->length << 12); - goto done; } - nvkm_vm_unmap_pgt(vm, vma->node->type != mmu->func->spg_shift, fpde, lpde); -done: nvkm_mm_free(&vm->mm, &vma->node); mutex_unlock(&vm->mutex); @@ -703,23 +462,7 @@ nvkm_vm_put(struct nvkm_vma *vma) int nvkm_vm_boot(struct nvkm_vm *vm, u64 size) { - struct nvkm_mmu *mmu = vm->mmu; - struct nvkm_memory *pgt; - int ret; - - if (vm->func->page->desc->func->unmap) - return nvkm_vmm_boot(vm); - - ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST, - (size >> mmu->func->spg_shift) * 8, 0x1000, true, &pgt); - if (ret == 0) { - vm->pgt[0].refcount[0] = 1; - vm->pgt[0].mem[0] = pgt; - nvkm_memory_boot(pgt, vm); - vm->bootstrapped = true; - } - - return ret; + return nvkm_vmm_boot(vm); } static int @@ -730,24 +473,14 @@ nvkm_vm_legacy(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, int ret; kref_init(&vm->refcount); - vm->fpde = offset >> (mmu->func->pgt_bits + 12); - vm->lpde = (offset + length - 1) >> (mmu->func->pgt_bits + 12); - - vm->pgt = vzalloc((vm->lpde - vm->fpde + 1) * sizeof(*vm->pgt)); - if (!vm->pgt) { - kfree(vm); - return -ENOMEM; - } if (block > length) block = length; ret = nvkm_mm_init(&vm->mm, 0, mm_offset >> 12, mm_length >> 12, block >> 12); - if (ret) { - vfree(vm->pgt); + if (ret) return ret; - } return 0; } @@ -786,7 +519,6 @@ nvkm_vm_del(struct kref *kref) struct nvkm_vm *vm = container_of(kref, typeof(*vm), refcount); nvkm_mm_fini(&vm->mm); - vfree(vm->pgt); if (vm->func) nvkm_vmm_dtor(vm); kfree(vm); @@ -797,14 +529,9 @@ nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_memory *inst) { if (ref) { if (ref->func->join && inst) { - int ret = ref->func->join(ref, inst), i; + int ret = ref->func->join(ref, inst); if (ret) return ret; - - if (!ref->func->page->desc->func->unmap && ref->mmu->func->map_pgt) { - for (i = ref->fpde; i <= ref->lpde; i++) - ref->mmu->func->map_pgt(ref, i, ref->pgt[i - ref->fpde].mem); - } } kref_get(&ref->refcount); @@ -813,12 +540,6 @@ nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_memory *inst) if (*ptr) { if ((*ptr)->func->part && inst) (*ptr)->func->part(*ptr, inst); - if ((*ptr)->bootstrapped && inst) { - if (!(*ptr)->func->page->desc->func->unmap) { - nvkm_memory_unref(&(*ptr)->pgt[0].mem[0]); - (*ptr)->bootstrapped = false; - } - } kref_put(&(*ptr)->refcount, nvkm_vm_del); } @@ -838,9 +559,6 @@ nvkm_mmu_oneinit(struct nvkm_subdev *subdev) return ret; } - if (mmu->func->oneinit) - return mmu->func->oneinit(mmu); - return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c index e436c610b3df5..c90c9980478be 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c @@ -27,8 +27,6 @@ static const struct nvkm_mmu_func g84_mmu = { .limit = (1ULL << 40), .dma_bits = 40, - .pgt_bits = 29 - 12, - .spg_shift = 12, .lpg_shift = 16, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x0200 }, .kind = nv50_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index 24e2d933524d3..ca5ca27188cda 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -75,8 +75,6 @@ static const struct nvkm_mmu_func gf100_mmu = { .limit = (1ULL << 40), .dma_bits = 40, - .pgt_bits = 27 - 12, - .spg_shift = 12, .lpg_shift = 17, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gf100_vmm_new }, .kind = gf100_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c index 8cbd0723c25b6..db7353aa25dfd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c @@ -27,8 +27,6 @@ static const struct nvkm_mmu_func gk104_mmu = { .limit = (1ULL << 40), .dma_bits = 40, - .pgt_bits = 27 - 12, - .spg_shift = 12, .lpg_shift = 17, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk104_vmm_new }, .kind = gf100_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c index 57def7244d385..2dfe8b1ba109c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c @@ -27,8 +27,6 @@ static const struct nvkm_mmu_func gk20a_mmu = { .limit = (1ULL << 40), .dma_bits = 40, - .pgt_bits = 27 - 12, - .spg_shift = 12, .lpg_shift = 17, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk20a_vmm_new }, .kind = gf100_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c index c38c899e463cd..a9628baf53722 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c @@ -71,8 +71,6 @@ static const struct nvkm_mmu_func gm200_mmu = { .limit = (1ULL << 40), .dma_bits = 40, - .pgt_bits = 27 - 12, - .spg_shift = 12, .lpg_shift = 17, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm200_vmm_new }, .kind = gm200_mmu_kind, @@ -82,8 +80,6 @@ static const struct nvkm_mmu_func gm200_mmu_fixed = { .limit = (1ULL << 40), .dma_bits = 40, - .pgt_bits = 27 - 12, - .spg_shift = 12, .lpg_shift = 17, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm200_vmm_new_fixed }, .kind = gm200_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c index 8cd6d1617bd72..5d14c138af025 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c @@ -29,8 +29,6 @@ static const struct nvkm_mmu_func gm20b_mmu = { .limit = (1ULL << 40), .dma_bits = 40, - .pgt_bits = 27 - 12, - .spg_shift = 12, .lpg_shift = 17, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new }, .kind = gm200_mmu_kind, @@ -40,8 +38,6 @@ static const struct nvkm_mmu_func gm20b_mmu_fixed = { .limit = (1ULL << 40), .dma_bits = 40, - .pgt_bits = 27 - 12, - .spg_shift = 12, .lpg_shift = 17, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new_fixed }, .kind = gm200_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index 13438dbc60632..3ae70e218bc03 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -31,8 +31,6 @@ const struct nvkm_mmu_func nv04_mmu = { .limit = NV04_PDMA_SIZE, .dma_bits = 32, - .pgt_bits = 32 - 12, - .spg_shift = 12, .lpg_shift = 12, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv04_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c index 457cf509591b7..5bcbc10d85648 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c @@ -43,8 +43,6 @@ nv41_mmu = { .init = nv41_mmu_init, .limit = NV41_GART_SIZE, .dma_bits = 39, - .pgt_bits = 32 - 12, - .spg_shift = 12, .lpg_shift = 12, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv41_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c index 2072139bff4dc..998287021e3f8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c @@ -58,8 +58,6 @@ nv44_mmu = { .init = nv44_mmu_init, .limit = NV44_GART_SIZE, .dma_bits = 39, - .pgt_bits = 32 - 12, - .spg_shift = 12, .lpg_shift = 12, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv44_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c index 31d6f9626e56c..7af1f89fe004f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c @@ -63,8 +63,6 @@ static const struct nvkm_mmu_func nv50_mmu = { .limit = (1ULL << 40), .dma_bits = 40, - .pgt_bits = 29 - 12, - .spg_shift = 12, .lpg_shift = 16, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x1400 }, .kind = nv50_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 08d211d9fc55d..bc4571a206902 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -9,26 +9,12 @@ int nvkm_mmu_new_(const struct nvkm_mmu_func *, struct nvkm_device *, int index, struct nvkm_mmu **); struct nvkm_mmu_func { - int (*oneinit)(struct nvkm_mmu *); void (*init)(struct nvkm_mmu *); u64 limit; u8 dma_bits; - u32 pgt_bits; - u8 spg_shift; u8 lpg_shift; - void (*map_pgt)(struct nvkm_vmm *, u32 pde, - struct nvkm_memory *pgt[2]); - void (*map)(struct nvkm_vma *, struct nvkm_memory *, - struct nvkm_mem *, u32 pte, u32 cnt, - u64 phys, u64 delta); - void (*map_sg)(struct nvkm_vma *, struct nvkm_memory *, - struct nvkm_mem *, u32 pte, u32 cnt, dma_addr_t *); - void (*unmap)(struct nvkm_vma *, struct nvkm_memory *pgt, - u32 pte, u32 cnt); - void (*flush)(struct nvkm_vm *); - struct { struct nvkm_sclass user; int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, From f9463a4bc8ea2df5ea25c4d6e0be72011e559b95 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 114/151] drm/nouveau/mmu: implement new vmm frontend These are the new priviledged interfaces to the VMM backends, and expose some functionality that wasn't previously available. It's now possible to allocate a chunk of address-space (even all of it), without causing page tables to be allocated up-front, and then map into it at arbitrary locations. This is the basic primitive used to support features such as sparse mapping, or to allow userspace control over its own address-space, or HMM (where the GPU driver isn't in control of the address-space layout). Rather than being tied to a subtle combination of memory object and VMA properties, arguments that control map flags (ro, kind, etc) are passed explicitly at map time. The compatibility hacks to implement the old frontend on top of the new driver backends have been replaced with something similar to implement the old frontend's interfaces on top of the new frontend. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/fb.h | 4 +- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 48 +- drivers/gpu/drm/nouveau/nouveau_mem.h | 2 + .../drm/nouveau/nvkm/subdev/instmem/gk20a.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 152 +--- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c | 787 +++++++++++++++++- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 17 +- .../drm/nouveau/nvkm/subdev/secboot/priv.h | 1 + 8 files changed, 858 insertions(+), 154 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index 5b5ff5a9a1271..4af663d4d3c7d 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -1,9 +1,7 @@ #ifndef __NVKM_FB_H__ #define __NVKM_FB_H__ #include -#include - -#include +#include /* memory type/access flags, do not match hardware values */ #define NV_MEM_ACCESS_RO 1 diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 630c3cbaf2b92..7fa60d79ec4c2 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -1,9 +1,6 @@ #ifndef __NVKM_MMU_H__ #define __NVKM_MMU_H__ #include -#include -#include -struct nvkm_gpuobj; struct nvkm_mem; struct nvkm_vm_pgt { @@ -12,14 +9,25 @@ struct nvkm_vm_pgt { }; struct nvkm_vma { - struct nvkm_memory *memory; - struct nvkm_tags *tags; + struct list_head head; + struct rb_node tree; + u64 addr; + u64 size:50; + bool mapref:1; /* PTs (de)referenced on (un)map (vs pre-allocated). */ + bool sparse:1; /* Unmapped PDEs/PTEs will not trigger MMU faults. */ +#define NVKM_VMA_PAGE_NONE 7 + u8 page:3; /* Requested page type (index, or NONE for automatic). */ + u8 refd:3; /* Current page type (index, or NONE for unreferenced). */ + bool used:1; /* Region allocated. */ + bool part:1; /* Region was split from an allocated region by map(). */ + bool user:1; /* Region user-allocated. */ + bool busy:1; /* Region busy (for temporarily preventing user access). */ + struct nvkm_memory *memory; /* Memory currently mapped into VMA. */ + struct nvkm_tags *tags; /* Compression tag reference. */ + + struct nvkm_vma *node; struct nvkm_vm *vm; - struct nvkm_mm_node *node; - union { - u64 offset; - u64 addr; - }; + u64 offset; u32 access; }; @@ -37,8 +45,9 @@ struct nvkm_vm { struct nvkm_vmm_pt *pd; struct list_head join; - struct nvkm_mm mm; - struct kref refcount; + struct list_head list; + struct rb_root free; + struct rb_root root; bool bootstrapped; atomic_t engref[NVKM_SUBDEV_NR]; @@ -57,9 +66,16 @@ void nvkm_vm_put(struct nvkm_vma *); void nvkm_vm_map(struct nvkm_vma *, struct nvkm_mem *); void nvkm_vm_map_at(struct nvkm_vma *, u64 offset, struct nvkm_mem *); void nvkm_vm_unmap(struct nvkm_vma *); -void nvkm_vm_unmap_at(struct nvkm_vma *, u64 offset, u64 length); +int nvkm_vmm_new(struct nvkm_device *, u64 addr, u64 size, void *argv, u32 argc, + struct lock_class_key *, const char *name, struct nvkm_vmm **); +struct nvkm_vmm *nvkm_vmm_ref(struct nvkm_vmm *); +void nvkm_vmm_unref(struct nvkm_vmm **); int nvkm_vmm_boot(struct nvkm_vmm *); +int nvkm_vmm_join(struct nvkm_vmm *, struct nvkm_memory *inst); +void nvkm_vmm_part(struct nvkm_vmm *, struct nvkm_memory *inst); +int nvkm_vmm_get(struct nvkm_vmm *, u8 page, u64 size, struct nvkm_vma **); +void nvkm_vmm_put(struct nvkm_vmm *, struct nvkm_vma **); struct nvkm_vmm_map { struct nvkm_memory *memory; @@ -78,6 +94,12 @@ struct nvkm_vmm_map { u64 ctag; }; +int nvkm_vmm_map(struct nvkm_vmm *, struct nvkm_vma *, void *argv, u32 argc, + struct nvkm_vmm_map *); +void nvkm_vmm_unmap(struct nvkm_vmm *, struct nvkm_vma *); + +struct nvkm_vmm *nvkm_uvmm_search(struct nvkm_client *, u64 handle); + struct nvkm_mmu { const struct nvkm_mmu_func *func; struct nvkm_subdev subdev; diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.h b/drivers/gpu/drm/nouveau/nouveau_mem.h index 20930ebc5e21d..48388c538420e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.h +++ b/drivers/gpu/drm/nouveau/nouveau_mem.h @@ -1,6 +1,8 @@ #ifndef __NOUVEAU_MEM_H__ #define __NOUVEAU_MEM_H__ +#include #include +#include #include struct ttm_dma_tt; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c index 816d1231df5f1..39f6e8e423396 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c @@ -48,6 +48,7 @@ #include #include #include +#include struct gk20a_instobj { struct nvkm_memory memory; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 31832398f1e98..9bf688df24f01 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -266,14 +266,14 @@ nvkm_vm_map_(const struct nvkm_vmm_page *page, struct nvkm_vma *vma, u64 delta, } mutex_lock(&vmm->mutex); - nvkm_vmm_ptes_map(vmm, page, ((u64)vma->node->offset << 12) + delta, - (u64)vma->node->length << 12, map, fn); + nvkm_vmm_ptes_map(vmm, page, vma->node->addr + delta, + vma->node->size, map, fn); mutex_unlock(&vmm->mutex); - nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); - nvkm_memory_unref(&vma->memory); - vma->memory = nvkm_memory_ref(map->memory); - vma->tags = map->tags; + nvkm_memory_tags_put(vma->node->memory, vmm->mmu->subdev.device, &vma->node->tags); + nvkm_memory_unref(&vma->node->memory); + vma->node->memory = nvkm_memory_ref(map->memory); + vma->node->tags = map->tags; } void @@ -314,11 +314,9 @@ nvkm_mmu_ptc_init(struct nvkm_mmu *mmu) void nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node) { - const struct nvkm_vmm_page *page = vma->vm->func->page; + const struct nvkm_vmm_page *page = &vma->vm->func->page[vma->node->page]; if (page->desc->func->unmap) { struct nvkm_vmm_map map = { .mem = node->mem }; - while (page->shift != vma->node->type) - page++; nvkm_vm_map_(page, vma, delta, node, page->desc->func->mem, &map); return; } @@ -328,11 +326,9 @@ static void nvkm_vm_map_sg_table(struct nvkm_vma *vma, u64 delta, u64 length, struct nvkm_mem *mem) { - const struct nvkm_vmm_page *page = vma->vm->func->page; + const struct nvkm_vmm_page *page = &vma->vm->func->page[vma->node->page]; if (page->desc->func->unmap) { struct nvkm_vmm_map map = { .sgl = mem->sg->sgl }; - while (page->shift != vma->node->type) - page++; nvkm_vm_map_(page, vma, delta, mem, page->desc->func->sgl, &map); return; } @@ -342,11 +338,9 @@ static void nvkm_vm_map_sg(struct nvkm_vma *vma, u64 delta, u64 length, struct nvkm_mem *mem) { - const struct nvkm_vmm_page *page = vma->vm->func->page; + const struct nvkm_vmm_page *page = &vma->vm->func->page[vma->node->page]; if (page->desc->func->unmap) { struct nvkm_vmm_map map = { .dma = mem->pages }; - while (page->shift != vma->node->type) - page++; nvkm_vm_map_(page, vma, delta, mem, page->desc->func->dma, &map); return; } @@ -364,67 +358,30 @@ nvkm_vm_map(struct nvkm_vma *vma, struct nvkm_mem *node) nvkm_vm_map_at(vma, 0, node); } -void -nvkm_vm_unmap_at(struct nvkm_vma *vma, u64 delta, u64 length) -{ - struct nvkm_vm *vm = vma->vm; - if (vm->func->page->desc->func->unmap) { - const struct nvkm_vmm_page *page = vm->func->page; - while (page->shift != vma->node->type) - page++; - mutex_lock(&vm->mutex); - nvkm_vmm_ptes_unmap(vm, page, (vma->node->offset << 12) + delta, - vma->node->length << 12, false); - mutex_unlock(&vm->mutex); - return; - } -} - void nvkm_vm_unmap(struct nvkm_vma *vma) { - nvkm_vm_unmap_at(vma, 0, (u64)vma->node->length << 12); - - nvkm_memory_tags_put(vma->memory, vma->vm->mmu->subdev.device, &vma->tags); - nvkm_memory_unref(&vma->memory); + nvkm_vmm_unmap(vma->vm, vma->node); } int nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access, struct nvkm_vma *vma) { - u32 align = (1 << page_shift) >> 12; - u32 msize = size >> 12; int ret; mutex_lock(&vm->mutex); - ret = nvkm_mm_head(&vm->mm, 0, page_shift, msize, msize, align, - &vma->node); - if (unlikely(ret != 0)) { - mutex_unlock(&vm->mutex); - return ret; - } - - if (vm->func->page->desc->func->unmap) { - const struct nvkm_vmm_page *page = vm->func->page; - while (page->shift != page_shift) - page++; - - ret = nvkm_vmm_ptes_get(vm, page, vma->node->offset << 12, - vma->node->length << 12); - if (ret) { - nvkm_mm_free(&vm->mm, &vma->node); - mutex_unlock(&vm->mutex); - return ret; - } - } + ret = nvkm_vmm_get_locked(vm, true, false, false, page_shift, 0, + size, &vma->node); mutex_unlock(&vm->mutex); + if (ret) + return ret; vma->memory = NULL; vma->tags = NULL; vma->vm = NULL; nvkm_vm_ref(vm, &vma->vm, NULL); - vma->offset = (u64)vma->node->offset << 12; + vma->offset = vma->addr = vma->node->addr; vma->access = access; return 0; } @@ -432,30 +389,7 @@ nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access, void nvkm_vm_put(struct nvkm_vma *vma) { - struct nvkm_mmu *mmu; - struct nvkm_vm *vm; - - if (unlikely(vma->node == NULL)) - return; - vm = vma->vm; - mmu = vm->mmu; - - nvkm_memory_tags_put(vma->memory, mmu->subdev.device, &vma->tags); - nvkm_memory_unref(&vma->memory); - - mutex_lock(&vm->mutex); - if (vm->func->page->desc->func->unmap) { - const struct nvkm_vmm_page *page = vm->func->page; - while (page->shift != vma->node->type) - page++; - - nvkm_vmm_ptes_put(vm, page, vma->node->offset << 12, - vma->node->length << 12); - } - - nvkm_mm_free(&vm->mm, &vma->node); - mutex_unlock(&vm->mutex); - + nvkm_vmm_put(vma->vm, &vma->node); nvkm_vm_ref(NULL, &vma->vm, NULL); } @@ -465,26 +399,6 @@ nvkm_vm_boot(struct nvkm_vm *vm, u64 size) return nvkm_vmm_boot(vm); } -static int -nvkm_vm_legacy(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset, - u32 block, struct nvkm_vm *vm) -{ - u64 mm_length = (offset + length) - mm_offset; - int ret; - - kref_init(&vm->refcount); - - if (block > length) - block = length; - - ret = nvkm_mm_init(&vm->mm, 0, mm_offset >> 12, mm_length >> 12, - block >> 12); - if (ret) - return ret; - - return 0; -} - int nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset, struct lock_class_key *key, struct nvkm_vm **pvm) @@ -501,46 +415,28 @@ nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset, return ret; } - ret = nvkm_vm_legacy(mmu, offset, length, mm_offset, - (*pvm)->func->page_block ? - (*pvm)->func->page_block : 4096, *pvm); - if (ret) - nvkm_vm_ref(NULL, pvm, NULL); - return ret; } return -EINVAL; } -static void -nvkm_vm_del(struct kref *kref) -{ - struct nvkm_vm *vm = container_of(kref, typeof(*vm), refcount); - - nvkm_mm_fini(&vm->mm); - if (vm->func) - nvkm_vmm_dtor(vm); - kfree(vm); -} - int nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_memory *inst) { if (ref) { - if (ref->func->join && inst) { - int ret = ref->func->join(ref, inst); + if (inst) { + int ret = nvkm_vmm_join(ref, inst); if (ret) return ret; } - kref_get(&ref->refcount); + nvkm_vmm_ref(ref); } if (*ptr) { - if ((*ptr)->func->part && inst) - (*ptr)->func->part(*ptr, inst); - kref_put(&(*ptr)->refcount, nvkm_vm_del); + nvkm_vmm_part(*ptr, inst); + nvkm_vmm_unref(ptr); } *ptr = ref; @@ -553,8 +449,8 @@ nvkm_mmu_oneinit(struct nvkm_subdev *subdev) struct nvkm_mmu *mmu = nvkm_mmu(subdev); if (mmu->func->vmm.global) { - int ret = nvkm_vm_new(subdev->device, 0, mmu->limit, 0, - NULL, &mmu->vmm); + int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL, + "gart", &mmu->vmm); if (ret) return ret; } @@ -576,7 +472,7 @@ nvkm_mmu_dtor(struct nvkm_subdev *subdev) { struct nvkm_mmu *mmu = nvkm_mmu(subdev); - nvkm_vm_ref(NULL, &mmu->vmm, NULL); + nvkm_vmm_unref(&mmu->vmm); nvkm_mmu_ptc_fini(mmu); return mmu; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c index 46c7fecf0054b..31c4acaf44a05 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -22,6 +22,8 @@ #define NVKM_VMM_LEVELS_MAX 5 #include "vmm.h" +#include + static void nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt) { @@ -376,6 +378,25 @@ nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc, } } +static bool +nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +{ + struct nvkm_vmm_pt *pt = it->pt[0]; + if (it->desc->type == PGD) + memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes); + else + if (it->desc->type == LPT) + memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes); + return nvkm_vmm_unref_ptes(it, ptei, ptes); +} + +static bool +nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +{ + nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes); + return nvkm_vmm_ref_ptes(it, ptei, ptes); +} + static bool nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei) { @@ -565,7 +586,111 @@ nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, return addr << page->shift; } -void +static void +nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, + u64 addr, u64 size) +{ + nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, + nvkm_vmm_sparse_unref_ptes, NULL, NULL, + page->desc->func->invalid ? + page->desc->func->invalid : page->desc->func->unmap); +} + +static int +nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, + u64 addr, u64 size) +{ + if ((page->type & NVKM_VMM_PAGE_SPARSE)) { + u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref", + true, nvkm_vmm_sparse_ref_ptes, NULL, + NULL, page->desc->func->sparse); + if (fail != ~0ULL) { + if ((size = fail - addr)) + nvkm_vmm_ptes_sparse_put(vmm, page, addr, size); + return -ENOMEM; + } + return 0; + } + return -EINVAL; +} + +static int +nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref) +{ + const struct nvkm_vmm_page *page = vmm->func->page; + int m = 0, i; + u64 start = addr; + u64 block; + + while (size) { + /* Limit maximum page size based on remaining size. */ + while (size < (1ULL << page[m].shift)) + m++; + i = m; + + /* Find largest page size suitable for alignment. */ + while (!IS_ALIGNED(addr, 1ULL << page[i].shift)) + i++; + + /* Determine number of PTEs at this page size. */ + if (i != m) { + /* Limited to alignment boundary of next page size. */ + u64 next = 1ULL << page[i - 1].shift; + u64 part = ALIGN(addr, next) - addr; + if (size - part >= next) + block = (part >> page[i].shift) << page[i].shift; + else + block = (size >> page[i].shift) << page[i].shift; + } else { + block = (size >> page[i].shift) << page[i].shift;; + } + + /* Perform operation. */ + if (ref) { + int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block); + if (ret) { + if ((size = addr - start)) + nvkm_vmm_ptes_sparse(vmm, start, size, false); + return ret; + } + } else { + nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block); + } + + size -= block; + addr += block; + } + + return 0; +} + +static void +nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, + u64 addr, u64 size, bool sparse) +{ + const struct nvkm_vmm_desc_func *func = page->desc->func; + nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref", + false, nvkm_vmm_unref_ptes, NULL, NULL, + sparse ? func->sparse : func->invalid ? func->invalid : + func->unmap); +} + +static int +nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, + u64 addr, u64 size, struct nvkm_vmm_map *map, + nvkm_vmm_pte_func func) +{ + u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true, + nvkm_vmm_ref_ptes, func, map, NULL); + if (fail != ~0ULL) { + if ((size = fail - addr)) + nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false); + return -ENOMEM; + } + return 0; +} + +static void nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size, bool sparse) { @@ -584,7 +709,7 @@ nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, NULL, func, map, NULL); } -void +static void nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { @@ -592,7 +717,7 @@ nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, nvkm_vmm_unref_ptes, NULL, NULL, NULL); } -int +static int nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { @@ -606,9 +731,120 @@ nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, return 0; } +static inline struct nvkm_vma * +nvkm_vma_new(u64 addr, u64 size) +{ + struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); + if (vma) { + vma->addr = addr; + vma->size = size; + vma->page = NVKM_VMA_PAGE_NONE; + vma->refd = NVKM_VMA_PAGE_NONE; + } + return vma; +} + +struct nvkm_vma * +nvkm_vma_tail(struct nvkm_vma *vma, u64 tail) +{ + struct nvkm_vma *new; + + BUG_ON(vma->size == tail); + + if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail))) + return NULL; + vma->size -= tail; + + new->mapref = vma->mapref; + new->sparse = vma->sparse; + new->page = vma->page; + new->refd = vma->refd; + new->used = vma->used; + new->part = vma->part; + new->user = vma->user; + new->busy = vma->busy; + list_add(&new->head, &vma->head); + return new; +} + +static void +nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + struct rb_node **ptr = &vmm->free.rb_node; + struct rb_node *parent = NULL; + + while (*ptr) { + struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree); + parent = *ptr; + if (vma->size < this->size) + ptr = &parent->rb_left; + else + if (vma->size > this->size) + ptr = &parent->rb_right; + else + if (vma->addr < this->addr) + ptr = &parent->rb_left; + else + if (vma->addr > this->addr) + ptr = &parent->rb_right; + else + BUG(); + } + + rb_link_node(&vma->tree, parent, ptr); + rb_insert_color(&vma->tree, &vmm->free); +} + void +nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + struct rb_node **ptr = &vmm->root.rb_node; + struct rb_node *parent = NULL; + + while (*ptr) { + struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree); + parent = *ptr; + if (vma->addr < this->addr) + ptr = &parent->rb_left; + else + if (vma->addr > this->addr) + ptr = &parent->rb_right; + else + BUG(); + } + + rb_link_node(&vma->tree, parent, ptr); + rb_insert_color(&vma->tree, &vmm->root); +} + +struct nvkm_vma * +nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr) +{ + struct rb_node *node = vmm->root.rb_node; + while (node) { + struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree); + if (addr < vma->addr) + node = node->rb_left; + else + if (addr >= vma->addr + vma->size) + node = node->rb_right; + else + return vma; + } + return NULL; +} + +static void nvkm_vmm_dtor(struct nvkm_vmm *vmm) { + struct nvkm_vma *vma; + struct rb_node *node; + + while ((node = rb_first(&vmm->root))) { + struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree); + nvkm_vmm_put(vmm, &vma); + } + if (vmm->bootstrapped) { const struct nvkm_vmm_page *page = vmm->func->page; const u64 limit = vmm->limit - vmm->start; @@ -620,6 +856,11 @@ nvkm_vmm_dtor(struct nvkm_vmm *vmm) nvkm_vmm_ptes_put(vmm, page, vmm->start, limit); } + vma = list_first_entry(&vmm->list, typeof(*vma), head); + list_del(&vma->head); + kfree(vma); + WARN_ON(!list_empty(&vmm->list)); + if (vmm->nullp) { dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024, vmm->nullp, vmm->null); @@ -639,6 +880,7 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, static struct lock_class_key _key; const struct nvkm_vmm_page *page = func->page; const struct nvkm_vmm_desc *desc; + struct nvkm_vma *vma; int levels, bits = 0; vmm->func = func; @@ -689,6 +931,16 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, return -ENOMEM; } + /* Initialise address-space MM. */ + INIT_LIST_HEAD(&vmm->list); + vmm->free = RB_ROOT; + vmm->root = RB_ROOT; + + if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start))) + return -ENOMEM; + + nvkm_vmm_free_insert(vmm, vma); + list_add(&vma->head, &vmm->list); return 0; } @@ -702,6 +954,494 @@ nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, return nvkm_vmm_ctor(func, mmu, hdr, addr, size, key, name, *pvmm); } +#define node(root, dir) ((root)->head.dir == &vmm->list) ? NULL : \ + list_entry((root)->head.dir, struct nvkm_vma, head) + +void +nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + struct nvkm_vma *next; + + nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); + nvkm_memory_unref(&vma->memory); + + if (vma->part) { + struct nvkm_vma *prev = node(vma, prev); + if (!prev->memory) { + prev->size += vma->size; + rb_erase(&vma->tree, &vmm->root); + list_del(&vma->head); + kfree(vma); + vma = prev; + } + } + + next = node(vma, next); + if (next && next->part) { + if (!next->memory) { + vma->size += next->size; + rb_erase(&next->tree, &vmm->root); + list_del(&next->head); + kfree(next); + } + } +} + +void +nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd]; + + if (vma->mapref) { + nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse); + vma->refd = NVKM_VMA_PAGE_NONE; + } else { + nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse); + } + + nvkm_vmm_unmap_region(vmm, vma); +} + +void +nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + if (vma->memory) { + mutex_lock(&vmm->mutex); + nvkm_vmm_unmap_locked(vmm, vma); + mutex_unlock(&vmm->mutex); + } +} + +static int +nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma, + void *argv, u32 argc, struct nvkm_vmm_map *map) +{ + switch (nvkm_memory_target(map->memory)) { + case NVKM_MEM_TARGET_VRAM: + if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) { + VMM_DEBUG(vmm, "%d !VRAM", map->page->shift); + return -EINVAL; + } + break; + case NVKM_MEM_TARGET_HOST: + case NVKM_MEM_TARGET_NCOH: + if (!(map->page->type & NVKM_VMM_PAGE_HOST)) { + VMM_DEBUG(vmm, "%d !HOST", map->page->shift); + return -EINVAL; + } + break; + default: + WARN_ON(1); + return -ENOSYS; + } + + if (!IS_ALIGNED( vma->addr, 1ULL << map->page->shift) || + !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) || + !IS_ALIGNED( map->offset, 1ULL << map->page->shift) || + nvkm_memory_page(map->memory) < map->page->shift) { + VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d", + vma->addr, (u64)vma->size, map->offset, map->page->shift, + nvkm_memory_page(map->memory)); + return -EINVAL; + } + + return vmm->func->valid(vmm, argv, argc, map); +} + +static int +nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma, + void *argv, u32 argc, struct nvkm_vmm_map *map) +{ + for (map->page = vmm->func->page; map->page->shift; map->page++) { + VMM_DEBUG(vmm, "trying %d", map->page->shift); + if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map)) + return 0; + } + return -EINVAL; +} + +static int +nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, + void *argv, u32 argc, struct nvkm_vmm_map *map) +{ + nvkm_vmm_pte_func func; + int ret; + + /* Make sure we won't overrun the end of the memory object. */ + if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) { + VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx", + nvkm_memory_size(map->memory), + map->offset, (u64)vma->size); + return -EINVAL; + } + + /* Check remaining arguments for validity. */ + if (vma->page == NVKM_VMA_PAGE_NONE && + vma->refd == NVKM_VMA_PAGE_NONE) { + /* Find the largest page size we can perform the mapping at. */ + const u32 debug = vmm->debug; + vmm->debug = 0; + ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map); + vmm->debug = debug; + if (ret) { + VMM_DEBUG(vmm, "invalid at any page size"); + nvkm_vmm_map_choose(vmm, vma, argv, argc, map); + return -EINVAL; + } + } else { + /* Page size of the VMA is already pre-determined. */ + if (vma->refd != NVKM_VMA_PAGE_NONE) + map->page = &vmm->func->page[vma->refd]; + else + map->page = &vmm->func->page[vma->page]; + + ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map); + if (ret) { + VMM_DEBUG(vmm, "invalid %d\n", ret); + return ret; + } + } + + /* Deal with the 'offset' argument, and fetch the backend function. */ + map->off = map->offset; + if (map->mem) { + for (; map->off; map->mem = map->mem->next) { + u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT; + if (size > map->off) + break; + map->off -= size; + } + func = map->page->desc->func->mem; + } else + if (map->sgl) { + for (; map->off; map->sgl = sg_next(map->sgl)) { + u64 size = sg_dma_len(map->sgl); + if (size > map->off) + break; + map->off -= size; + } + func = map->page->desc->func->sgl; + } else { + map->dma += map->offset >> PAGE_SHIFT; + map->off = map->offset & PAGE_MASK; + func = map->page->desc->func->dma; + } + + /* Perform the map. */ + if (vma->refd == NVKM_VMA_PAGE_NONE) { + ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func); + if (ret) + return ret; + + vma->refd = map->page - vmm->func->page; + } else { + nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func); + } + + nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); + nvkm_memory_unref(&vma->memory); + vma->memory = nvkm_memory_ref(map->memory); + vma->tags = map->tags; + return 0; +} + +int +nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc, + struct nvkm_vmm_map *map) +{ + int ret; + mutex_lock(&vmm->mutex); + ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map); + vma->busy = false; + mutex_unlock(&vmm->mutex); + return ret; +} + +static void +nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + struct nvkm_vma *prev, *next; + + if ((prev = node(vma, prev)) && !prev->used) { + rb_erase(&prev->tree, &vmm->free); + list_del(&prev->head); + vma->addr = prev->addr; + vma->size += prev->size; + kfree(prev); + } + + if ((next = node(vma, next)) && !next->used) { + rb_erase(&next->tree, &vmm->free); + list_del(&next->head); + vma->size += next->size; + kfree(next); + } + + nvkm_vmm_free_insert(vmm, vma); +} + +void +nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + const struct nvkm_vmm_page *page = vmm->func->page; + struct nvkm_vma *next = vma; + + BUG_ON(vma->part); + + if (vma->mapref || !vma->sparse) { + do { + const bool map = next->memory != NULL; + const u8 refd = next->refd; + const u64 addr = next->addr; + u64 size = next->size; + + /* Merge regions that are in the same state. */ + while ((next = node(next, next)) && next->part && + (next->memory != NULL) == map && + (next->refd == refd)) + size += next->size; + + if (map) { + /* Region(s) are mapped, merge the unmap + * and dereference into a single walk of + * the page tree. + */ + nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr, + size, vma->sparse); + } else + if (refd != NVKM_VMA_PAGE_NONE) { + /* Drop allocation-time PTE references. */ + nvkm_vmm_ptes_put(vmm, &page[refd], addr, size); + } + } while (next && next->part); + } + + /* Merge any mapped regions that were split from the initial + * address-space allocation back into the allocated VMA, and + * release memory/compression resources. + */ + next = vma; + do { + if (next->memory) + nvkm_vmm_unmap_region(vmm, next); + } while ((next = node(vma, next)) && next->part); + + if (vma->sparse && !vma->mapref) { + /* Sparse region that was allocated with a fixed page size, + * meaning all relevant PTEs were referenced once when the + * region was allocated, and remained that way, regardless + * of whether memory was mapped into it afterwards. + * + * The process of unmapping, unsparsing, and dereferencing + * PTEs can be done in a single page tree walk. + */ + nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size); + } else + if (vma->sparse) { + /* Sparse region that wasn't allocated with a fixed page size, + * PTE references were taken both at allocation time (to make + * the GPU see the region as sparse), and when mapping memory + * into the region. + * + * The latter was handled above, and the remaining references + * are dealt with here. + */ + nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false); + } + + /* Remove VMA from the list of allocated nodes. */ + rb_erase(&vma->tree, &vmm->root); + + /* Merge VMA back into the free list. */ + vma->page = NVKM_VMA_PAGE_NONE; + vma->refd = NVKM_VMA_PAGE_NONE; + vma->used = false; + vma->user = false; + nvkm_vmm_put_region(vmm, vma); +} + +void +nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma) +{ + struct nvkm_vma *vma = *pvma; + if (vma) { + mutex_lock(&vmm->mutex); + nvkm_vmm_put_locked(vmm, vma); + mutex_unlock(&vmm->mutex); + *pvma = NULL; + } +} + +int +nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse, + u8 shift, u8 align, u64 size, struct nvkm_vma **pvma) +{ + const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE]; + struct rb_node *node = NULL, *temp; + struct nvkm_vma *vma = NULL, *tmp; + u64 addr, tail; + int ret; + + VMM_TRACE(vmm, "getref %d mapref %d sparse %d " + "shift: %d align: %d size: %016llx", + getref, mapref, sparse, shift, align, size); + + /* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */ + if (unlikely(!size || (!getref && !mapref && sparse))) { + VMM_DEBUG(vmm, "args %016llx %d %d %d", + size, getref, mapref, sparse); + return -EINVAL; + } + + /* Tesla-class GPUs can only select page size per-PDE, which means + * we're required to know the mapping granularity up-front to find + * a suitable region of address-space. + * + * The same goes if we're requesting up-front allocation of PTES. + */ + if (unlikely((getref || vmm->func->page_block) && !shift)) { + VMM_DEBUG(vmm, "page size required: %d %016llx", + getref, vmm->func->page_block); + return -EINVAL; + } + + /* If a specific page size was requested, determine its index and + * make sure the requested size is a multiple of the page size. + */ + if (shift) { + for (page = vmm->func->page; page->shift; page++) { + if (shift == page->shift) + break; + } + + if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) { + VMM_DEBUG(vmm, "page %d %016llx", shift, size); + return -EINVAL; + } + align = max_t(u8, align, shift); + } else { + align = max_t(u8, align, 12); + } + + /* Locate smallest block that can possibly satisfy the allocation. */ + temp = vmm->free.rb_node; + while (temp) { + struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree); + if (this->size < size) { + temp = temp->rb_right; + } else { + node = temp; + temp = temp->rb_left; + } + } + + if (unlikely(!node)) + return -ENOSPC; + + /* Take into account alignment restrictions, trying larger blocks + * in turn until we find a suitable free block. + */ + do { + struct nvkm_vma *this = rb_entry(node, typeof(*this), tree); + struct nvkm_vma *prev = node(this, prev); + struct nvkm_vma *next = node(this, next); + const int p = page - vmm->func->page; + + addr = this->addr; + if (vmm->func->page_block && prev && prev->page != p) + addr = roundup(addr, vmm->func->page_block); + addr = ALIGN(addr, 1ULL << align); + + tail = this->addr + this->size; + if (vmm->func->page_block && next && next->page != p) + tail = rounddown(tail, vmm->func->page_block); + + if (addr <= tail && tail - addr >= size) { + rb_erase(&this->tree, &vmm->free); + vma = this; + break; + } + } while ((node = rb_next(node))); + + if (unlikely(!vma)) + return -ENOSPC; + + /* If the VMA we found isn't already exactly the requested size, + * it needs to be split, and the remaining free blocks returned. + */ + if (addr != vma->addr) { + if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) { + nvkm_vmm_put_region(vmm, vma); + return -ENOMEM; + } + nvkm_vmm_free_insert(vmm, vma); + vma = tmp; + } + + if (size != vma->size) { + if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) { + nvkm_vmm_put_region(vmm, vma); + return -ENOMEM; + } + nvkm_vmm_free_insert(vmm, tmp); + } + + /* Pre-allocate page tables and/or setup sparse mappings. */ + if (sparse && getref) + ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size); + else if (sparse) + ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true); + else if (getref) + ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size); + else + ret = 0; + if (ret) { + nvkm_vmm_put_region(vmm, vma); + return ret; + } + + vma->mapref = mapref && !getref; + vma->sparse = sparse; + vma->page = page - vmm->func->page; + vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE; + vma->used = true; + nvkm_vmm_node_insert(vmm, vma); + *pvma = vma; + return 0; +} + +int +nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma) +{ + int ret; + mutex_lock(&vmm->mutex); + ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma); + mutex_unlock(&vmm->mutex); + return ret; +} + +void +nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst) +{ + if (vmm->func->part && inst) { + mutex_lock(&vmm->mutex); + vmm->func->part(vmm, inst); + mutex_unlock(&vmm->mutex); + } +} + +int +nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) +{ + int ret = 0; + if (vmm->func->join) { + mutex_lock(&vmm->mutex); + ret = vmm->func->join(vmm, inst); + mutex_unlock(&vmm->mutex); + } + return ret; +} + static bool nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) { @@ -730,3 +1470,44 @@ nvkm_vmm_boot(struct nvkm_vmm *vmm) vmm->bootstrapped = true; return 0; } + +static void +nvkm_vmm_del(struct kref *kref) +{ + struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref); + nvkm_vmm_dtor(vmm); + kfree(vmm); +} + +void +nvkm_vmm_unref(struct nvkm_vmm **pvmm) +{ + struct nvkm_vmm *vmm = *pvmm; + if (vmm) { + kref_put(&vmm->kref, nvkm_vmm_del); + *pvmm = NULL; + } +} + +struct nvkm_vmm * +nvkm_vmm_ref(struct nvkm_vmm *vmm) +{ + if (vmm) + kref_get(&vmm->kref); + return vmm; +} + +int +nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv, + u32 argc, struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) +{ + struct nvkm_mmu *mmu = device->mmu; + struct nvkm_vmm *vmm = NULL; + int ret; + ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc, key, name, &vmm); + if (ret) + nvkm_vmm_unref(&vmm); + *pvmm = vmm; + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index bb1353e950684..5808012f73955 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -153,16 +153,19 @@ int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32 pd_header, u64 addr, u64 size, struct lock_class_key *, const char *name, struct nvkm_vmm *); -void nvkm_vmm_dtor(struct nvkm_vmm *); -void nvkm_vmm_ptes_put(struct nvkm_vmm *, const struct nvkm_vmm_page *, - u64 addr, u64 size); -int nvkm_vmm_ptes_get(struct nvkm_vmm *, const struct nvkm_vmm_page *, - u64 addr, u64 size); +struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr); +int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref, + bool sparse, u8 page, u8 align, u64 size, + struct nvkm_vma **pvma); +void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *); +void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *); +void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma); void nvkm_vmm_ptes_map(struct nvkm_vmm *, const struct nvkm_vmm_page *, u64 addr, u64 size, struct nvkm_vmm_map *, nvkm_vmm_pte_func); -void nvkm_vmm_ptes_unmap(struct nvkm_vmm *, const struct nvkm_vmm_page *, - u64 addr, u64 size, bool sparse); + +struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail); +void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *); int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, u64, u64, void *, u32, struct lock_class_key *, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h index 885e919a87205..d9091f029506c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h @@ -25,6 +25,7 @@ #include #include +struct nvkm_gpuobj; struct nvkm_secboot_func { int (*oneinit)(struct nvkm_secboot *); From fc584e1a4a579c3279ed563b890415c13b2f85ac Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 115/151] drm/nouveau/bar/nv50: initialise vmm with new interfaces Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.c | 33 ++++++++++--------- .../gpu/drm/nouveau/nvkm/subdev/bar/nv50.h | 4 +-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c index 9300529917fce..157b076a12723 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c @@ -46,7 +46,7 @@ nv50_bar_flush(struct nvkm_bar *base) struct nvkm_vmm * nv50_bar_bar1_vmm(struct nvkm_bar *base) { - return nv50_bar(base)->bar1_vm; + return nv50_bar(base)->bar1_vmm; } void @@ -72,7 +72,7 @@ nv50_bar_bar1_init(struct nvkm_bar *base) struct nvkm_vmm * nv50_bar_bar2_vmm(struct nvkm_bar *base) { - return nv50_bar(base)->bar2_vm; + return nv50_bar(base)->bar2_vmm; } void @@ -109,7 +109,6 @@ nv50_bar_oneinit(struct nvkm_bar *base) struct nvkm_device *device = bar->base.subdev.device; static struct lock_class_key bar1_lock; static struct lock_class_key bar2_lock; - struct nvkm_vm *vm; u64 start, limit; int ret; @@ -130,18 +129,19 @@ nv50_bar_oneinit(struct nvkm_bar *base) start = 0x0100000000ULL; limit = start + device->func->resource_size(device, 3); - ret = nvkm_vm_new(device, start, limit - start, start, &bar2_lock, &vm); + ret = nvkm_vmm_new(device, start, limit-- - start, NULL, 0, + &bar2_lock, "bar2", &bar->bar2_vmm); if (ret) return ret; - atomic_inc(&vm->engref[NVKM_SUBDEV_BAR]); + atomic_inc(&bar->bar2_vmm->engref[NVKM_SUBDEV_BAR]); + bar->bar2_vmm->debug = bar->base.subdev.debug; - ret = nvkm_vm_boot(vm, limit-- - start); + ret = nvkm_vmm_boot(bar->bar2_vmm); if (ret) return ret; - ret = nvkm_vm_ref(vm, &bar->bar2_vm, bar->mem->memory); - nvkm_vm_ref(NULL, &vm, NULL); + ret = nvkm_vmm_join(bar->bar2_vmm, bar->mem->memory); if (ret) return ret; @@ -166,14 +166,13 @@ nv50_bar_oneinit(struct nvkm_bar *base) start = 0x0000000000ULL; limit = start + device->func->resource_size(device, 1); - ret = nvkm_vm_new(device, start, limit-- - start, start, &bar1_lock, &vm); - if (ret) - return ret; + ret = nvkm_vmm_new(device, start, limit-- - start, NULL, 0, + &bar1_lock, "bar1", &bar->bar1_vmm); - atomic_inc(&vm->engref[NVKM_SUBDEV_BAR]); + atomic_inc(&bar->bar1_vmm->engref[NVKM_SUBDEV_BAR]); + bar->bar1_vmm->debug = bar->base.subdev.debug; - ret = nvkm_vm_ref(vm, &bar->bar1_vm, bar->mem->memory); - nvkm_vm_ref(NULL, &vm, NULL); + ret = nvkm_vmm_join(bar->bar1_vmm, bar->mem->memory); if (ret) return ret; @@ -199,9 +198,11 @@ nv50_bar_dtor(struct nvkm_bar *base) struct nv50_bar *bar = nv50_bar(base); if (bar->mem) { nvkm_gpuobj_del(&bar->bar1); - nvkm_vm_ref(NULL, &bar->bar1_vm, bar->mem->memory); + nvkm_vmm_part(bar->bar1_vmm, bar->mem->memory); + nvkm_vmm_unref(&bar->bar1_vmm); nvkm_gpuobj_del(&bar->bar2); - nvkm_vm_ref(NULL, &bar->bar2_vm, bar->mem->memory); + nvkm_vmm_part(bar->bar2_vmm, bar->mem->memory); + nvkm_vmm_unref(&bar->bar2_vmm); nvkm_gpuobj_del(&bar->pgd); nvkm_gpuobj_del(&bar->pad); nvkm_gpuobj_del(&bar->mem); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h index b5fe26aab118a..140b76f588b6a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h @@ -9,9 +9,9 @@ struct nv50_bar { struct nvkm_gpuobj *mem; struct nvkm_gpuobj *pad; struct nvkm_gpuobj *pgd; - struct nvkm_vm *bar1_vm; + struct nvkm_vmm *bar1_vmm; struct nvkm_gpuobj *bar1; - struct nvkm_vm *bar2_vm; + struct nvkm_vmm *bar2_vmm; struct nvkm_gpuobj *bar2; }; From 7eac5f4eb06a661ddee9b2ebb75799bf0c7cbf65 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 116/151] drm/nouveau/bar/gf100: initialise vmm with new interfaces Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.c | 30 ++++++++----------- .../gpu/drm/nouveau/nvkm/subdev/bar/gf100.h | 2 +- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c index 8077e1a5017a0..a3ba7f50198be 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c @@ -31,7 +31,7 @@ struct nvkm_vmm * gf100_bar_bar1_vmm(struct nvkm_bar *base) { - return gf100_bar(base)->bar[1].vm; + return gf100_bar(base)->bar[1].vmm; } void @@ -60,7 +60,7 @@ gf100_bar_bar1_init(struct nvkm_bar *base) struct nvkm_vmm * gf100_bar_bar2_vmm(struct nvkm_bar *base) { - return gf100_bar(base)->bar[0].vm; + return gf100_bar(base)->bar[0].vmm; } void @@ -85,7 +85,6 @@ gf100_bar_oneinit_bar(struct gf100_bar *bar, struct gf100_barN *bar_vm, struct lock_class_key *key, int bar_nr) { struct nvkm_device *device = bar->base.subdev.device; - struct nvkm_vm *vm; resource_size_t bar_len; int ret; @@ -98,29 +97,24 @@ gf100_bar_oneinit_bar(struct gf100_bar *bar, struct gf100_barN *bar_vm, if (bar_nr == 3 && bar->bar2_halve) bar_len >>= 1; - ret = nvkm_vm_new(device, 0, bar_len, 0, key, &vm); + ret = nvkm_vmm_new(device, 0, bar_len, NULL, 0, key, + (bar_nr == 3) ? "bar2" : "bar1", &bar_vm->vmm); if (ret) return ret; - atomic_inc(&vm->engref[NVKM_SUBDEV_BAR]); + atomic_inc(&bar_vm->vmm->engref[NVKM_SUBDEV_BAR]); + bar_vm->vmm->debug = bar->base.subdev.debug; /* * Bootstrap page table lookup. */ if (bar_nr == 3) { - ret = nvkm_vm_boot(vm, bar_len); - if (ret) { - nvkm_vm_ref(NULL, &vm, NULL); + ret = nvkm_vmm_boot(bar_vm->vmm); + if (ret) return ret; - } } - ret = nvkm_vm_ref(vm, &bar_vm->vm, bar_vm->inst); - nvkm_vm_ref(NULL, &vm, NULL); - if (ret) - return ret; - - return 0; + return nvkm_vmm_join(bar_vm->vmm, bar_vm->inst); } int @@ -154,10 +148,12 @@ gf100_bar_dtor(struct nvkm_bar *base) { struct gf100_bar *bar = gf100_bar(base); - nvkm_vm_ref(NULL, &bar->bar[1].vm, bar->bar[1].inst); + nvkm_vmm_part(bar->bar[1].vmm, bar->bar[1].inst); + nvkm_vmm_unref(&bar->bar[1].vmm); nvkm_memory_unref(&bar->bar[1].inst); - nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].inst); + nvkm_vmm_part(bar->bar[0].vmm, bar->bar[0].inst); + nvkm_vmm_unref(&bar->bar[0].vmm); nvkm_memory_unref(&bar->bar[0].inst); return bar; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h index 9ce80c6b6fec1..e4da39139e959 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h @@ -5,7 +5,7 @@ struct gf100_barN { struct nvkm_memory *inst; - struct nvkm_vm *vm; + struct nvkm_vmm *vmm; }; struct gf100_bar { From 85f7c3a03689d300acdbc82270246f83e2a5d472 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 117/151] drm/nouveau/secboot/gm200: initialise vmm with new interfaces Signed-off-by: Ben Skeggs --- .../drm/nouveau/nvkm/subdev/secboot/gm200.c | 18 +++++++++--------- .../drm/nouveau/nvkm/subdev/secboot/gm200.h | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c index 6c068c5bd3c2d..5722295cb5840 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c @@ -48,13 +48,13 @@ gm200_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob, return ret; /* Map the HS firmware so the HS bootloader can see it */ - ret = nvkm_vm_get(gsb->vm, blob->size, 12, NV_MEM_ACCESS_RW, &vma); + ret = nvkm_vm_get(gsb->vmm, blob->size, 12, NV_MEM_ACCESS_RW, &vma); if (ret) { nvkm_falcon_put(falcon, subdev); return ret; } - ret = nvkm_memory_map(blob, 0, gsb->vm, &vma, NULL, 0); + ret = nvkm_memory_map(blob, 0, gsb->vmm, &vma, NULL, 0); if (ret) goto end; @@ -107,8 +107,6 @@ gm200_secboot_oneinit(struct nvkm_secboot *sb) { struct gm200_secboot *gsb = gm200_secboot(sb); struct nvkm_device *device = sb->subdev.device; - struct nvkm_vm *vm; - const u64 vm_area_len = 600 * 1024; int ret; /* Allocate instance block and VM */ @@ -117,14 +115,15 @@ gm200_secboot_oneinit(struct nvkm_secboot *sb) if (ret) return ret; - ret = nvkm_vm_new(device, 0, vm_area_len, 0, NULL, &vm); + ret = nvkm_vmm_new(device, 0, 600 * 1024, NULL, 0, NULL, "acr", + &gsb->vmm); if (ret) return ret; - atomic_inc(&vm->engref[NVKM_SUBDEV_PMU]); + atomic_inc(&gsb->vmm->engref[NVKM_SUBDEV_PMU]); + gsb->vmm->debug = gsb->base.subdev.debug; - ret = nvkm_vm_ref(vm, &gsb->vm, gsb->inst); - nvkm_vm_ref(NULL, &vm, NULL); + ret = nvkm_vmm_join(gsb->vmm, gsb->inst); if (ret) return ret; @@ -155,7 +154,8 @@ gm200_secboot_dtor(struct nvkm_secboot *sb) sb->acr->func->dtor(sb->acr); - nvkm_vm_ref(NULL, &gsb->vm, gsb->inst); + nvkm_vmm_part(gsb->vmm, gsb->inst); + nvkm_vmm_unref(&gsb->vmm); nvkm_memory_unref(&gsb->inst); return gsb; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h index a2b60fab42e88..62c5e162099aa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h @@ -30,7 +30,7 @@ struct gm200_secboot { /* Instance block & address space used for HS FW execution */ struct nvkm_memory *inst; - struct nvkm_vm *vm; + struct nvkm_vmm *vmm; }; #define gm200_secboot(sb) container_of(sb, struct gm200_secboot, base) From f66c57d9229c313de3adf340c604b1c7a40e5306 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 118/151] drm/nouveau/fifo: initialise vmm with new interfaces Signed-off-by: Ben Skeggs --- .../drm/nouveau/include/nvkm/engine/fifo.h | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 33 ++++++++++--------- .../drm/nouveau/nvkm/engine/fifo/chang84.c | 6 +--- .../drm/nouveau/nvkm/engine/fifo/changf100.h | 2 -- .../drm/nouveau/nvkm/engine/fifo/changk104.h | 2 -- .../drm/nouveau/nvkm/engine/fifo/channv50.c | 8 +---- .../drm/nouveau/nvkm/engine/fifo/channv50.h | 1 - .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 13 ++------ .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 13 ++------ .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 8 ++--- 10 files changed, 31 insertions(+), 57 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 0e65ad7e6eed1..e42d686fbd8b4 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -22,7 +22,7 @@ struct nvkm_fifo_chan { u16 chid; struct nvkm_gpuobj *inst; struct nvkm_gpuobj *push; - struct nvkm_vm *vm; + struct nvkm_vmm *vmm; void __iomem *user; u64 addr; u32 size; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 7aea0a8692aee..ac94b57777c4a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -27,6 +27,7 @@ #include #include #include +#include #include struct nvkm_fifo_chan_object { @@ -117,8 +118,8 @@ nvkm_fifo_chan_child_del(struct nvkm_oproxy *base) if (chan->func->engine_dtor) chan->func->engine_dtor(chan, engine); nvkm_object_del(&engn->object); - if (chan->vm) - atomic_dec(&chan->vm->engref[engine->subdev.index]); + if (chan->vmm) + atomic_dec(&chan->vmm->engref[engine->subdev.index]); } } @@ -151,8 +152,8 @@ nvkm_fifo_chan_child_new(const struct nvkm_oclass *oclass, void *data, u32 size, .engine = oclass->engine, }; - if (chan->vm) - atomic_inc(&chan->vm->engref[engine->subdev.index]); + if (chan->vmm) + atomic_inc(&chan->vmm->engref[engine->subdev.index]); if (engine->func->fifo.cclass) { ret = engine->func->fifo.cclass(chan, &cclass, @@ -327,7 +328,10 @@ nvkm_fifo_chan_dtor(struct nvkm_object *object) if (chan->user) iounmap(chan->user); - nvkm_vm_ref(NULL, &chan->vm, NULL); + if (chan->vmm) { + nvkm_vmm_part(chan->vmm, chan->inst->memory); + nvkm_vmm_unref(&chan->vmm); + } nvkm_gpuobj_del(&chan->push); nvkm_gpuobj_del(&chan->inst); @@ -355,7 +359,6 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *func, { struct nvkm_client *client = oclass->client; struct nvkm_device *device = fifo->engine.subdev.device; - struct nvkm_mmu *mmu = device->mmu; struct nvkm_dmaobj *dmaobj; unsigned long flags; int ret; @@ -384,16 +387,16 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *func, } /* channel address space */ - if (!vm && mmu) { - if (!client->vm || client->vm->mmu == mmu) { - ret = nvkm_vm_ref(client->vm, &chan->vm, NULL); - if (ret) - return ret; - } else { + if (!device->mmu->func->vmm.global) { + struct nvkm_vmm *vmm = client->vm; + if (vmm->mmu != device->mmu) return -EINVAL; - } - } else { - return -ENOENT; + + ret = nvkm_vmm_join(vmm, chan->inst->memory); + if (ret) + return ret; + + chan->vmm = nvkm_vmm_ref(vmm); } /* allocate channel id */ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c index 705a6d951d322..1870310549e86 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c @@ -277,9 +277,5 @@ g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push, if (ret) return ret; - ret = nvkm_ramht_new(device, 0x8000, 16, chan->base.inst, &chan->ramht); - if (ret) - return ret; - - return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst->memory); + return nvkm_ramht_new(device, 0x8000, 16, chan->base.inst, &chan->ramht); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h index a902f848a8785..f4400b33c00d1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h @@ -11,8 +11,6 @@ struct gf100_fifo_chan { struct list_head head; bool killed; - struct nvkm_vm *vm; - struct { struct nvkm_gpuobj *inst; struct nvkm_vma vma; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h index 1cd5b8d3fb2e0..1259fb2337be9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h @@ -12,8 +12,6 @@ struct gk104_fifo_chan { struct list_head head; bool killed; - struct nvkm_vm *vm; - struct { struct nvkm_gpuobj *inst; struct nvkm_vma vma; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c index 161aa9ec73bcd..01453279dbc00 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c @@ -206,8 +206,6 @@ void * nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base) { struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - if (chan->base.inst) - nvkm_vm_ref(NULL, &chan->vm, chan->base.inst->memory); nvkm_ramht_del(&chan->ramht); nvkm_gpuobj_del(&chan->pgd); nvkm_gpuobj_del(&chan->eng); @@ -263,9 +261,5 @@ nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push, if (ret) return ret; - ret = nvkm_ramht_new(device, 0x8000, 16, chan->base.inst, &chan->ramht); - if (ret) - return ret; - - return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst->memory); + return nvkm_ramht_new(device, 0x8000, 16, chan->base.inst, &chan->ramht); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h index 4b9da469b704e..a0c04c6329a83 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h @@ -13,7 +13,6 @@ struct nv50_fifo_chan { struct nvkm_gpuobj *eng; struct nvkm_gpuobj *pgd; struct nvkm_ramht *ramht; - struct nvkm_vm *vm; struct nvkm_gpuobj *engn[NVKM_SUBDEV_NR]; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 695d0721b1adc..3e2b1a82e640f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -150,12 +150,12 @@ gf100_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, if (ret) return ret; - ret = nvkm_vm_get(chan->vm, chan->engn[engn].inst->size, 12, + ret = nvkm_vm_get(chan->base.vmm, chan->engn[engn].inst->size, 12, NV_MEM_ACCESS_RW, &chan->engn[engn].vma); if (ret) return ret; - return nvkm_memory_map(chan->engn[engn].inst, 0, chan->vm, + return nvkm_memory_map(chan->engn[engn].inst, 0, chan->base.vmm, &chan->engn[engn].vma, NULL, 0); } @@ -199,10 +199,7 @@ gf100_fifo_gpfifo_init(struct nvkm_fifo_chan *base) static void * gf100_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) { - struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - if (chan->base.inst) - nvkm_vm_ref(NULL, &chan->vm, chan->base.inst->memory); - return chan; + return gf100_fifo_chan(base); } static const struct nvkm_fifo_chan_func @@ -262,10 +259,6 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, args->v0.chid = chan->base.chid; - ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst->memory); - if (ret) - return ret; - /* clear channel control registers */ usermem = chan->base.chid * 0x1000; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 978c9ad952b38..1c3ee44102879 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -162,12 +162,12 @@ gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, if (ret) return ret; - ret = nvkm_vm_get(chan->vm, chan->engn[engn].inst->size, 12, + ret = nvkm_vm_get(chan->base.vmm, chan->engn[engn].inst->size, 12, NV_MEM_ACCESS_RW, &chan->engn[engn].vma); if (ret) return ret; - return nvkm_memory_map(chan->engn[engn].inst, 0, chan->vm, + return nvkm_memory_map(chan->engn[engn].inst, 0, chan->base.vmm, &chan->engn[engn].vma, NULL, 0); } @@ -212,10 +212,7 @@ gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base) static void * gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) { - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - if (chan->base.inst) - nvkm_vm_ref(NULL, &chan->vm, chan->base.inst->memory); - return chan; + return gk104_fifo_chan(base); } static const struct nvkm_fifo_chan_func @@ -301,10 +298,6 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func, *chid = chan->base.chid; - ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst->memory); - if (ret) - return ret; - /* Clear channel control registers. */ usermem = chan->base.chid * 0x200; ilength = order_base_2(ilength / 8); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 4fc4deb2db4b5..b6cde13859410 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -398,12 +398,12 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, if (ret) return ret; - ret = nvkm_vm_get(fifoch->vm, 0x1000, 12, NV_MEM_ACCESS_RW | + ret = nvkm_vm_get(fifoch->vmm, 0x1000, 12, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS, &chan->mmio_vma); if (ret) return ret; - ret = nvkm_memory_map(chan->mmio, 0, fifoch->vm, + ret = nvkm_memory_map(chan->mmio, 0, fifoch->vmm, &chan->mmio_vma, NULL, 0); if (ret) return ret; @@ -416,13 +416,13 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, if (ret) return ret; - ret = nvkm_vm_get(fifoch->vm, + ret = nvkm_vm_get(fifoch->vmm, nvkm_memory_size(chan->data[i].mem), 12, data->access, &chan->data[i].vma); if (ret) return ret; - ret = nvkm_memory_map(chan->data[i].mem, 0, fifoch->vm, + ret = nvkm_memory_map(chan->data[i].mem, 0, fifoch->vmm, &chan->data[i].vma, NULL, 0); if (ret) return ret; From 6f4dc18c166cd36d8e9dfd130874060065bedd1c Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 119/151] drm/nouveau/fb/ram: use new interfaces for vmm operations Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c index 03a860ae75b11..5305078f730e7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c @@ -39,12 +39,22 @@ nvkm_vram_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc) { struct nvkm_vram *vram = nvkm_vram(memory); - struct nvkm_mem mem = { - .mem = vram->mn, + struct nvkm_vmm_map map = { .memory = &vram->memory, + .offset = offset, + .mem = vram->mn, }; - nvkm_vm_map_at(vma, offset, &mem); - return 0; + + if (vma->vm) { + struct nvkm_mem mem = { + .mem = vram->mn, + .memory = &vram->memory, + }; + nvkm_vm_map_at(vma, offset, &mem); + return 0; + } + + return nvkm_vmm_map(vmm, vma, argv, argc, &map); } static u64 From 9202d732e6bc3b46566db3ed25f7a5a3eeaee3c1 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 120/151] drm/nouveau/imem/nv50-: use new interfaces for vmm operations Signed-off-by: Ben Skeggs --- .../drm/nouveau/nvkm/subdev/instmem/gk20a.c | 45 +++++++++++-------- .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 28 ++++++------ 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c index 39f6e8e423396..2f60f0d18aeb1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c @@ -52,7 +52,7 @@ struct gk20a_instobj { struct nvkm_memory memory; - struct nvkm_mem mem; + struct nvkm_mm_node *mn; struct gk20a_instmem *imem; /* CPU mapping */ @@ -129,13 +129,13 @@ gk20a_instobj_page(struct nvkm_memory *memory) static u64 gk20a_instobj_addr(struct nvkm_memory *memory) { - return gk20a_instobj(memory)->mem.offset; + return (u64)gk20a_instobj(memory)->mn->offset << 12; } static u64 gk20a_instobj_size(struct nvkm_memory *memory) { - return (u64)gk20a_instobj(memory)->mem.size << 12; + return (u64)gk20a_instobj(memory)->mn->length << 12; } /* @@ -284,8 +284,22 @@ gk20a_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc) { struct gk20a_instobj *node = gk20a_instobj(memory); - nvkm_vm_map_at(vma, 0, &node->mem); - return 0; + struct nvkm_vmm_map map = { + .memory = &node->memory, + .offset = offset, + .mem = node->mn, + }; + + if (vma->vm) { + struct nvkm_mem mem = { + .mem = node->mn, + .memory = &node->memory, + }; + nvkm_vm_map_at(vma, 0, &mem); + return 0; + } + + return nvkm_vmm_map(vmm, vma, argv, argc, &map); } static void * @@ -298,8 +312,8 @@ gk20a_instobj_dtor_dma(struct nvkm_memory *memory) if (unlikely(!node->base.vaddr)) goto out; - dma_free_attrs(dev, node->base.mem.size << PAGE_SHIFT, node->base.vaddr, - node->handle, imem->attrs); + dma_free_attrs(dev, (u64)node->base.mn->length << PAGE_SHIFT, + node->base.vaddr, node->handle, imem->attrs); out: return node; @@ -311,7 +325,7 @@ gk20a_instobj_dtor_iommu(struct nvkm_memory *memory) struct gk20a_instobj_iommu *node = gk20a_instobj_iommu(memory); struct gk20a_instmem *imem = node->base.imem; struct device *dev = imem->base.subdev.device->dev; - struct nvkm_mm_node *r = node->base.mem.mem; + struct nvkm_mm_node *r = node->base.mn; int i; if (unlikely(!r)) @@ -329,7 +343,7 @@ gk20a_instobj_dtor_iommu(struct nvkm_memory *memory) r->offset &= ~BIT(imem->iommu_bit - imem->iommu_pgshift); /* Unmap pages from GPU address space and free them */ - for (i = 0; i < node->base.mem.size; i++) { + for (i = 0; i < node->base.mn->length; i++) { iommu_unmap(imem->domain, (r->offset + i) << imem->iommu_pgshift, PAGE_SIZE); dma_unmap_page(dev, node->dma_addrs[i], PAGE_SIZE, @@ -410,8 +424,7 @@ gk20a_instobj_ctor_dma(struct gk20a_instmem *imem, u32 npages, u32 align, node->r.offset = node->handle >> 12; node->r.length = (npages << PAGE_SHIFT) >> 12; - node->base.mem.offset = node->handle; - node->base.mem.mem = &node->r; + node->base.mn = &node->r; return 0; } @@ -488,8 +501,7 @@ gk20a_instobj_ctor_iommu(struct gk20a_instmem *imem, u32 npages, u32 align, /* IOMMU bit tells that an address is to be resolved through the IOMMU */ r->offset |= BIT(imem->iommu_bit - imem->iommu_pgshift); - node->base.mem.offset = ((u64)r->offset) << imem->iommu_pgshift; - node->base.mem.mem = r; + node->base.mn = r; return 0; release_area: @@ -537,13 +549,8 @@ gk20a_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, node->imem = imem; - /* present memory for being mapped using small pages */ - node->mem.size = size >> 12; - node->mem.memtype = 0; - node->mem.memory = &node->memory; - nvkm_debug(subdev, "alloc size: 0x%x, align: 0x%x, gaddr: 0x%llx\n", - size, align, node->mem.offset); + size, align, (u64)node->mn->offset << 12); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index 64e2b6e0e8b19..1ba7289684aa2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -46,7 +46,7 @@ struct nv50_instobj { struct nvkm_instobj base; struct nv50_instmem *imem; struct nvkm_memory *ram; - struct nvkm_vma bar; + struct nvkm_vma *bar; refcount_t maps; void *map; struct list_head lru; @@ -124,7 +124,7 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) struct nvkm_memory *memory = &iobj->base.memory; struct nvkm_subdev *subdev = &imem->base.subdev; struct nvkm_device *device = subdev->device; - struct nvkm_vma bar = {}, ebar; + struct nvkm_vma *bar = NULL, *ebar; u64 size = nvkm_memory_size(memory); void *emap; int ret; @@ -134,7 +134,7 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) * to the possibility of recursion for page table allocation. */ mutex_unlock(&subdev->mutex); - while ((ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &bar))) { + while ((ret = nvkm_vmm_get(vmm, 12, size, &bar))) { /* Evict unused mappings, and keep retrying until we either * succeed,or there's no more objects left on the LRU. */ @@ -144,10 +144,10 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) nvkm_debug(subdev, "evict %016llx %016llx @ %016llx\n", nvkm_memory_addr(&eobj->base.memory), nvkm_memory_size(&eobj->base.memory), - eobj->bar.offset); + eobj->bar->addr); list_del_init(&eobj->lru); ebar = eobj->bar; - eobj->bar.node = NULL; + eobj->bar = NULL; emap = eobj->map; eobj->map = NULL; } @@ -155,16 +155,16 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) if (!eobj) break; iounmap(emap); - nvkm_vm_put(&ebar); + nvkm_vmm_put(vmm, &ebar); } if (ret == 0) - ret = nvkm_memory_map(memory, 0, vmm, &bar, NULL, 0); + ret = nvkm_memory_map(memory, 0, vmm, bar, NULL, 0); mutex_lock(&subdev->mutex); - if (ret || iobj->bar.node) { + if (ret || iobj->bar) { /* We either failed, or another thread beat us. */ mutex_unlock(&subdev->mutex); - nvkm_vm_put(&bar); + nvkm_vmm_put(vmm, &bar); mutex_lock(&subdev->mutex); return; } @@ -172,10 +172,10 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm) /* Make the mapping visible to the host. */ iobj->bar = bar; iobj->map = ioremap_wc(device->func->resource_addr(device, 3) + - (u32)iobj->bar.offset, size); + (u32)iobj->bar->addr, size); if (!iobj->map) { nvkm_warn(subdev, "PRAMIN ioremap failed\n"); - nvkm_vm_put(&iobj->bar); + nvkm_vmm_put(vmm, &iobj->bar); } } @@ -299,7 +299,7 @@ nv50_instobj_dtor(struct nvkm_memory *memory) { struct nv50_instobj *iobj = nv50_instobj(memory); struct nvkm_instmem *imem = &iobj->imem->base; - struct nvkm_vma bar; + struct nvkm_vma *bar; void *map = map; mutex_lock(&imem->subdev.mutex); @@ -310,8 +310,10 @@ nv50_instobj_dtor(struct nvkm_memory *memory) mutex_unlock(&imem->subdev.mutex); if (map) { + struct nvkm_vmm *vmm = nvkm_bar_bar2_vmm(imem->subdev.device); iounmap(map); - nvkm_vm_put(&bar); + if (likely(vmm)) /* Can be NULL during BAR destructor. */ + nvkm_vmm_put(vmm, &bar); } nvkm_memory_unref(&iobj->ram); From 8c967c554818a0f98f8cad86cc561ab43bbbf1e7 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 121/151] drm/nouveau/secboot/gm200: use new interfaces for vmm operations Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c index 5722295cb5840..5e91b3f90065f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c @@ -39,7 +39,7 @@ gm200_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob, { struct gm200_secboot *gsb = gm200_secboot(sb); struct nvkm_subdev *subdev = &gsb->base.subdev; - struct nvkm_vma vma; + struct nvkm_vma *vma = NULL; u32 start_address; int ret; @@ -48,13 +48,13 @@ gm200_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob, return ret; /* Map the HS firmware so the HS bootloader can see it */ - ret = nvkm_vm_get(gsb->vmm, blob->size, 12, NV_MEM_ACCESS_RW, &vma); + ret = nvkm_vmm_get(gsb->vmm, 12, blob->size, &vma); if (ret) { nvkm_falcon_put(falcon, subdev); return ret; } - ret = nvkm_memory_map(blob, 0, gsb->vmm, &vma, NULL, 0); + ret = nvkm_memory_map(blob, 0, gsb->vmm, vma, NULL, 0); if (ret) goto end; @@ -65,7 +65,7 @@ gm200_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob, nvkm_falcon_bind_context(falcon, gsb->inst); /* Load the HS bootloader into the falcon's IMEM/DMEM */ - ret = sb->acr->func->load(sb->acr, falcon, blob, vma.offset); + ret = sb->acr->func->load(sb->acr, falcon, blob, vma->addr); if (ret < 0) goto end; @@ -95,8 +95,7 @@ gm200_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob, nvkm_mc_intr_mask(sb->subdev.device, falcon->owner->index, true); /* We don't need the ACR firmware anymore */ - nvkm_vm_unmap(&vma); - nvkm_vm_put(&vma); + nvkm_vmm_put(gsb->vmm, &vma); nvkm_falcon_put(falcon, subdev); return ret; From 01f349fcad68d80939db53d9110135e6341b786d Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 122/151] drm/nouveau/fifo/gf100-: use new interfaces for vmm operations Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/engine/fifo/changf100.h | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/changk104.h | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 12 ++++++------ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 12 ++++++------ drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 2 +- .../drm/nouveau/nvkm/engine/fifo/gpfifogf100.c | 16 ++++++---------- .../drm/nouveau/nvkm/engine/fifo/gpfifogk104.c | 16 ++++++---------- 8 files changed, 28 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h index f4400b33c00d1..fc1142af02cfb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h @@ -13,7 +13,7 @@ struct gf100_fifo_chan { struct { struct nvkm_gpuobj *inst; - struct nvkm_vma vma; + struct nvkm_vma *vma; } engn[NVKM_SUBDEV_NR]; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h index 1259fb2337be9..5beb5c628473e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h @@ -14,7 +14,7 @@ struct gk104_fifo_chan { struct { struct nvkm_gpuobj *inst; - struct nvkm_vma vma; + struct nvkm_vma *vma; } engn[NVKM_SUBDEV_NR]; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 24a4c28b32c5c..f695768681643 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -28,7 +28,6 @@ #include #include #include -#include #include #include @@ -586,12 +585,12 @@ gf100_fifo_oneinit(struct nvkm_fifo *base) if (ret) return ret; - ret = nvkm_vm_get(bar, nvkm_memory_size(fifo->user.mem), 12, - NV_MEM_ACCESS_RW, &fifo->user.bar); + ret = nvkm_vmm_get(bar, 12, nvkm_memory_size(fifo->user.mem), + &fifo->user.bar); if (ret) return ret; - return nvkm_memory_map(fifo->user.mem, 0, bar, &fifo->user.bar, NULL, 0); + return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0); } static void @@ -630,7 +629,7 @@ gf100_fifo_init(struct nvkm_fifo *base) } nvkm_mask(device, 0x002200, 0x00000001, 0x00000001); - nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar.offset >> 12); + nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12); nvkm_wr32(device, 0x002100, 0xffffffff); nvkm_wr32(device, 0x002140, 0x7fffffff); @@ -641,7 +640,8 @@ static void * gf100_fifo_dtor(struct nvkm_fifo *base) { struct gf100_fifo *fifo = gf100_fifo(base); - nvkm_vm_put(&fifo->user.bar); + struct nvkm_device *device = fifo->base.engine.subdev.device; + nvkm_vmm_put(nvkm_bar_bar1_vmm(device), &fifo->user.bar); nvkm_memory_unref(&fifo->user.mem); nvkm_memory_unref(&fifo->runlist.mem[0]); nvkm_memory_unref(&fifo->runlist.mem[1]); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h index 70db58eab9c3d..b81a2ad48aa4a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h @@ -26,7 +26,7 @@ struct gf100_fifo { struct { struct nvkm_memory *mem; - struct nvkm_vma bar; + struct nvkm_vma *bar; } user; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index eddf9f12e9ee0..84bd703dd8971 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -836,12 +835,12 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) if (ret) return ret; - ret = nvkm_vm_get(bar, nvkm_memory_size(fifo->user.mem), 12, - NV_MEM_ACCESS_RW, &fifo->user.bar); + ret = nvkm_vmm_get(bar, 12, nvkm_memory_size(fifo->user.mem), + &fifo->user.bar); if (ret) return ret; - return nvkm_memory_map(fifo->user.mem, 0, bar, &fifo->user.bar, NULL, 0); + return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0); } static void @@ -867,7 +866,7 @@ gk104_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */ } - nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar.offset >> 12); + nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12); nvkm_wr32(device, 0x002100, 0xffffffff); nvkm_wr32(device, 0x002140, 0x7fffffff); @@ -877,9 +876,10 @@ static void * gk104_fifo_dtor(struct nvkm_fifo *base) { struct gk104_fifo *fifo = gk104_fifo(base); + struct nvkm_device *device = fifo->base.engine.subdev.device; int i; - nvkm_vm_put(&fifo->user.bar); + nvkm_vmm_put(nvkm_bar_bar1_vmm(device), &fifo->user.bar); nvkm_memory_unref(&fifo->user.mem); for (i = 0; i < fifo->runlist_nr; i++) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 44bff98d67251..466f1051f91af 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -37,7 +37,7 @@ struct gk104_fifo { struct { struct nvkm_memory *mem; - struct nvkm_vma bar; + struct nvkm_vma *bar; } user; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 3e2b1a82e640f..78114068c6dd3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -111,7 +111,7 @@ gf100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base, struct nvkm_gpuobj *inst = chan->base.inst; if (offset) { - u64 addr = chan->engn[engine->subdev.index].vma.offset; + u64 addr = chan->engn[engine->subdev.index].vma->addr; nvkm_kmap(inst); nvkm_wo32(inst, offset + 0x00, lower_32_bits(addr) | 4); nvkm_wo32(inst, offset + 0x04, upper_32_bits(addr)); @@ -126,11 +126,7 @@ gf100_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base, struct nvkm_engine *engine) { struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - struct nvkm_vma *vma = &chan->engn[engine->subdev.index].vma; - if (vma->vm) { - nvkm_vm_unmap(vma); - nvkm_vm_put(vma); - } + nvkm_vmm_put(chan->base.vmm, &chan->engn[engine->subdev.index].vma); nvkm_gpuobj_del(&chan->engn[engine->subdev.index].inst); } @@ -150,13 +146,13 @@ gf100_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, if (ret) return ret; - ret = nvkm_vm_get(chan->base.vmm, chan->engn[engn].inst->size, 12, - NV_MEM_ACCESS_RW, &chan->engn[engn].vma); + ret = nvkm_vmm_get(chan->base.vmm, 12, chan->engn[engn].inst->size, + &chan->engn[engn].vma); if (ret) return ret; return nvkm_memory_map(chan->engn[engn].inst, 0, chan->base.vmm, - &chan->engn[engn].vma, NULL, 0); + chan->engn[engn].vma, NULL, 0); } static void @@ -252,7 +248,7 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, (1ULL << NVKM_ENGINE_MSPPP) | (1ULL << NVKM_ENGINE_MSVLD) | (1ULL << NVKM_ENGINE_SW), - 1, fifo->user.bar.offset, 0x1000, + 1, fifo->user.bar->addr, 0x1000, oclass, &chan->base); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 1c3ee44102879..368f5f5138a36 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -117,7 +117,7 @@ gk104_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base, u32 offset = gk104_fifo_gpfifo_engine_addr(engine); if (offset) { - u64 addr = chan->engn[engine->subdev.index].vma.offset; + u64 addr = chan->engn[engine->subdev.index].vma->addr; u32 datalo = lower_32_bits(addr) | 0x00000004; u32 datahi = upper_32_bits(addr); nvkm_kmap(inst); @@ -138,11 +138,7 @@ gk104_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base, struct nvkm_engine *engine) { struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - struct nvkm_vma *vma = &chan->engn[engine->subdev.index].vma; - if (vma->vm) { - nvkm_vm_unmap(vma); - nvkm_vm_put(vma); - } + nvkm_vmm_put(chan->base.vmm, &chan->engn[engine->subdev.index].vma); nvkm_gpuobj_del(&chan->engn[engine->subdev.index].inst); } @@ -162,13 +158,13 @@ gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, if (ret) return ret; - ret = nvkm_vm_get(chan->base.vmm, chan->engn[engn].inst->size, 12, - NV_MEM_ACCESS_RW, &chan->engn[engn].vma); + ret = nvkm_vmm_get(chan->base.vmm, 12, chan->engn[engn].inst->size, + &chan->engn[engn].vma); if (ret) return ret; return nvkm_memory_map(chan->engn[engn].inst, 0, chan->base.vmm, - &chan->engn[engn].vma, NULL, 0); + chan->engn[engn].vma, NULL, 0); } static void @@ -291,7 +287,7 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func, ret = nvkm_fifo_chan_ctor(&gk104_fifo_gpfifo_func, &fifo->base, 0x1000, 0x1000, true, vm, 0, subdevs, - 1, fifo->user.bar.offset, 0x200, + 1, fifo->user.bar->addr, 0x200, oclass, &chan->base); if (ret) return ret; From 019e4d76c6e91b3eae1d8d88b8683a24c1ec26d3 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 123/151] drm/nouveau/gr/gf100-: use new interfaces for vmm operations Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 13 +++---- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h | 2 +- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c | 6 +-- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c | 9 ++--- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c | 6 +-- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 39 +++++++++---------- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 7 ++-- 10 files changed, 39 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index ce69ec8b13add..8c60b24eae040 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -986,14 +986,14 @@ gf100_grctx_pack_tpc[] = { ******************************************************************************/ int -gf100_grctx_mmio_data(struct gf100_grctx *info, u32 size, u32 align, u32 access) +gf100_grctx_mmio_data(struct gf100_grctx *info, u32 size, u32 align, bool priv) { if (info->data) { info->buffer[info->buffer_nr] = round_up(info->addr, align); info->addr = info->buffer[info->buffer_nr] + size; info->data->size = size; info->data->align = align; - info->data->access = access; + info->data->priv = priv; info->data++; return info->buffer_nr++; } @@ -1028,9 +1028,8 @@ void gf100_grctx_generate_bundle(struct gf100_grctx *info) { const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS; const int s = 8; - const int b = mmio_vram(info, grctx->bundle_size, (1 << s), access); + const int b = mmio_vram(info, grctx->bundle_size, (1 << s), true); mmio_refn(info, 0x408004, 0x00000000, s, b); mmio_wr32(info, 0x408008, 0x80000000 | (grctx->bundle_size >> s)); mmio_refn(info, 0x418808, 0x00000000, s, b); @@ -1041,9 +1040,8 @@ void gf100_grctx_generate_pagepool(struct gf100_grctx *info) { const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS; const int s = 8; - const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), access); + const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true); mmio_refn(info, 0x40800c, 0x00000000, s, b); mmio_wr32(info, 0x408010, 0x80000000); mmio_refn(info, 0x419004, 0x00000000, s, b); @@ -1057,9 +1055,8 @@ gf100_grctx_generate_attrib(struct gf100_grctx *info) const struct gf100_grctx_func *grctx = gr->func->grctx; const u32 attrib = grctx->attrib_nr; const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max); - const u32 access = NV_MEM_ACCESS_RW; const int s = 12; - const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), access); + const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false); int gpc, tpc; u32 bo = 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h index 017180d147cf5..4731e56fbb115 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h @@ -11,7 +11,7 @@ struct gf100_grctx { u64 addr; }; -int gf100_grctx_mmio_data(struct gf100_grctx *, u32 size, u32 align, u32 access); +int gf100_grctx_mmio_data(struct gf100_grctx *, u32 size, u32 align, bool priv); void gf100_grctx_mmio_item(struct gf100_grctx *, u32 addr, u32 data, int s, int); #define mmio_vram(a,b,c,d) gf100_grctx_mmio_data((a), (b), (c), (d)) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c index 505cdcbfc0855..82f71b10c06ec 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c @@ -735,9 +735,8 @@ gf108_grctx_generate_attrib(struct gf100_grctx *info) const u32 alpha = grctx->alpha_nr; const u32 beta = grctx->attrib_nr; const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max); - const u32 access = NV_MEM_ACCESS_RW; const int s = 12; - const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), access); + const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false); const int timeslice_mode = 1; const int max_batches = 0xffff; u32 bo = 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c index 74a64e3fd59ac..19301d88577dc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c @@ -187,9 +187,8 @@ gf117_grctx_generate_attrib(struct gf100_grctx *info) const u32 alpha = grctx->alpha_nr; const u32 beta = grctx->attrib_nr; const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max); - const u32 access = NV_MEM_ACCESS_RW; const int s = 12; - const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), access); + const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false); const int timeslice_mode = 1; const int max_batches = 0xffff; u32 bo = 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c index c46b3fdf72031..825c8fd500bc9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c @@ -847,9 +847,8 @@ gk104_grctx_generate_bundle(struct gf100_grctx *info) const u32 state_limit = min(grctx->bundle_min_gpm_fifo_depth, grctx->bundle_size / 0x20); const u32 token_limit = grctx->bundle_token_limit; - const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS; const int s = 8; - const int b = mmio_vram(info, grctx->bundle_size, (1 << s), access); + const int b = mmio_vram(info, grctx->bundle_size, (1 << s), true); mmio_refn(info, 0x408004, 0x00000000, s, b); mmio_wr32(info, 0x408008, 0x80000000 | (grctx->bundle_size >> s)); mmio_refn(info, 0x418808, 0x00000000, s, b); @@ -861,9 +860,8 @@ void gk104_grctx_generate_pagepool(struct gf100_grctx *info) { const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS; const int s = 8; - const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), access); + const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true); mmio_refn(info, 0x40800c, 0x00000000, s, b); mmio_wr32(info, 0x408010, 0x80000000); mmio_refn(info, 0x419004, 0x00000000, s, b); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c index 4c4b5ab6e46d1..9b43d4ce3eaa3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c @@ -867,9 +867,8 @@ gm107_grctx_generate_bundle(struct gf100_grctx *info) const u32 state_limit = min(grctx->bundle_min_gpm_fifo_depth, grctx->bundle_size / 0x20); const u32 token_limit = grctx->bundle_token_limit; - const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS; const int s = 8; - const int b = mmio_vram(info, grctx->bundle_size, (1 << s), access); + const int b = mmio_vram(info, grctx->bundle_size, (1 << s), true); mmio_refn(info, 0x408004, 0x00000000, s, b); mmio_wr32(info, 0x408008, 0x80000000 | (grctx->bundle_size >> s)); mmio_refn(info, 0x418e24, 0x00000000, s, b); @@ -881,9 +880,8 @@ void gm107_grctx_generate_pagepool(struct gf100_grctx *info) { const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS; const int s = 8; - const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), access); + const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true); mmio_refn(info, 0x40800c, 0x00000000, s, b); mmio_wr32(info, 0x408010, 0x80000000); mmio_refn(info, 0x419004, 0x00000000, s, b); @@ -900,9 +898,8 @@ gm107_grctx_generate_attrib(struct gf100_grctx *info) const u32 alpha = grctx->alpha_nr; const u32 attrib = grctx->attrib_nr; const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max); - const u32 access = NV_MEM_ACCESS_RW; const int s = 12; - const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), access); + const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false); const int max_batches = 0xffff; u32 bo = 0; u32 ao = bo + grctx->attrib_nr_max * gr->tpc_total; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c index 7833bc777a295..88ea322d956c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c @@ -33,9 +33,8 @@ void gp100_grctx_generate_pagepool(struct gf100_grctx *info) { const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS; const int s = 8; - const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), access); + const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true); mmio_refn(info, 0x40800c, 0x00000000, s, b); mmio_wr32(info, 0x408010, 0x80000000); mmio_refn(info, 0x419004, 0x00000000, s, b); @@ -51,9 +50,8 @@ gp100_grctx_generate_attrib(struct gf100_grctx *info) const u32 attrib = grctx->attrib_nr; const u32 pertpc = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max); const u32 size = roundup(gr->tpc_total * pertpc, 0x80); - const u32 access = NV_MEM_ACCESS_RW; const int s = 12; - const int b = mmio_vram(info, size, (1 << s), access); + const int b = mmio_vram(info, size, (1 << s), false); const int max_batches = 0xffff; u32 ao = 0; u32 bo = ao + grctx->alpha_nr_max * gr->tpc_total; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c index 80b7ab0bee3a0..7a66b4c2eb188 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c @@ -38,9 +38,8 @@ gp102_grctx_generate_attrib(struct gf100_grctx *info) const u32 attrib = grctx->attrib_nr; const u32 pertpc = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max); const u32 size = roundup(gr->tpc_total * pertpc, 0x80); - const u32 access = NV_MEM_ACCESS_RW; const int s = 12; - const int b = mmio_vram(info, size, (1 << s), access); + const int b = mmio_vram(info, size, (1 << s), false); const int max_batches = 0xffff; u32 ao = 0; u32 bo = ao + grctx->alpha_nr_max * gr->tpc_total; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index b6cde13859410..4b3d5432a53e2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -37,6 +37,7 @@ #include #include +#include #include /******************************************************************************* @@ -327,13 +328,13 @@ gf100_gr_chan_bind(struct nvkm_object *object, struct nvkm_gpuobj *parent, if (!gr->firmware) { nvkm_wo32(*pgpuobj, 0x00, chan->mmio_nr / 2); - nvkm_wo32(*pgpuobj, 0x04, chan->mmio_vma.offset >> 8); + nvkm_wo32(*pgpuobj, 0x04, chan->mmio_vma->addr >> 8); } else { nvkm_wo32(*pgpuobj, 0xf4, 0); nvkm_wo32(*pgpuobj, 0xf8, 0); nvkm_wo32(*pgpuobj, 0x10, chan->mmio_nr / 2); - nvkm_wo32(*pgpuobj, 0x14, lower_32_bits(chan->mmio_vma.offset)); - nvkm_wo32(*pgpuobj, 0x18, upper_32_bits(chan->mmio_vma.offset)); + nvkm_wo32(*pgpuobj, 0x14, lower_32_bits(chan->mmio_vma->addr)); + nvkm_wo32(*pgpuobj, 0x18, upper_32_bits(chan->mmio_vma->addr)); nvkm_wo32(*pgpuobj, 0x1c, 1); nvkm_wo32(*pgpuobj, 0x20, 0); nvkm_wo32(*pgpuobj, 0x28, 0); @@ -350,18 +351,13 @@ gf100_gr_chan_dtor(struct nvkm_object *object) int i; for (i = 0; i < ARRAY_SIZE(chan->data); i++) { - if (chan->data[i].vma.node) { - nvkm_vm_unmap(&chan->data[i].vma); - nvkm_vm_put(&chan->data[i].vma); - } + nvkm_vmm_put(chan->vmm, &chan->data[i].vma); nvkm_memory_unref(&chan->data[i].mem); } - if (chan->mmio_vma.node) { - nvkm_vm_unmap(&chan->mmio_vma); - nvkm_vm_put(&chan->mmio_vma); - } + nvkm_vmm_put(chan->vmm, &chan->mmio_vma); nvkm_memory_unref(&chan->mmio); + nvkm_vmm_unref(&chan->vmm); return chan; } @@ -380,6 +376,7 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, struct gf100_gr_data *data = gr->mmio_data; struct gf100_gr_mmio *mmio = gr->mmio_list; struct gf100_gr_chan *chan; + struct gf100_vmm_map_v0 args = { .priv = 1 }; struct nvkm_device *device = gr->base.engine.subdev.device; int ret, i; @@ -387,6 +384,7 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, return -ENOMEM; nvkm_object_ctor(&gf100_gr_chan, oclass, &chan->object); chan->gr = gr; + chan->vmm = nvkm_vmm_ref(fifoch->vmm); *pobject = &chan->object; /* allocate memory for a "mmio list" buffer that's used by the HUB @@ -398,13 +396,12 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, if (ret) return ret; - ret = nvkm_vm_get(fifoch->vmm, 0x1000, 12, NV_MEM_ACCESS_RW | - NV_MEM_ACCESS_SYS, &chan->mmio_vma); + ret = nvkm_vmm_get(fifoch->vmm, 12, 0x1000, &chan->mmio_vma); if (ret) return ret; ret = nvkm_memory_map(chan->mmio, 0, fifoch->vmm, - &chan->mmio_vma, NULL, 0); + chan->mmio_vma, &args, sizeof(args)); if (ret) return ret; @@ -416,14 +413,16 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, if (ret) return ret; - ret = nvkm_vm_get(fifoch->vmm, - nvkm_memory_size(chan->data[i].mem), 12, - data->access, &chan->data[i].vma); + ret = nvkm_vmm_get(fifoch->vmm, 12, + nvkm_memory_size(chan->data[i].mem), + &chan->data[i].vma); if (ret) return ret; - ret = nvkm_memory_map(chan->data[i].mem, 0, fifoch->vmm, - &chan->data[i].vma, NULL, 0); + args.priv = data->priv; + + ret = nvkm_memory_map(chan->data[i].mem, 0, chan->vmm, + chan->data[i].vma, &args, sizeof(args)); if (ret) return ret; @@ -437,7 +436,7 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, u32 data = mmio->data; if (mmio->buffer >= 0) { - u64 info = chan->data[mmio->buffer].vma.offset; + u64 info = chan->data[mmio->buffer].vma->addr; data |= info >> mmio->shift; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index 51c1e380ffb68..d7c2adb9b543c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -45,7 +45,7 @@ struct gf100_gr_data { u32 size; u32 align; - u32 access; + bool priv; }; struct gf100_gr_mmio { @@ -161,14 +161,15 @@ void gp100_gr_init_rop_active_fbps(struct gf100_gr *); struct gf100_gr_chan { struct nvkm_object object; struct gf100_gr *gr; + struct nvkm_vmm *vmm; struct nvkm_memory *mmio; - struct nvkm_vma mmio_vma; + struct nvkm_vma *mmio_vma; int mmio_nr; struct { struct nvkm_memory *mem; - struct nvkm_vma vma; + struct nvkm_vma *vma; } data[4]; }; From 2cabefcbd097e27def6d29d52d5d2cf8797c11e4 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 124/151] drm/nouveau/gr/gf100-: replace hardcoded instance/vmm setup in grctx generation Could be useful for if/when a future GPU removes support for the GF100 PT layout. Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 110 +++++++++--------- 1 file changed, 58 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 8c60b24eae040..881015080d838 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1264,85 +1264,87 @@ gf100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info) nvkm_mc_unk260(device, 1); } +#define CB_RESERVED 0x80000 + int gf100_grctx_generate(struct gf100_gr *gr) { const struct gf100_grctx_func *grctx = gr->func->grctx; struct nvkm_subdev *subdev = &gr->base.engine.subdev; struct nvkm_device *device = subdev->device; - struct nvkm_memory *chan; + struct nvkm_memory *inst = NULL; + struct nvkm_memory *data = NULL; + struct nvkm_vmm *vmm = NULL; + struct nvkm_vma *ctx = NULL; struct gf100_grctx info; int ret, i; u64 addr; - /* allocate memory to for a "channel", which we'll use to generate - * the default context values + /* Allocate memory to for a "channel", which we'll use to generate + * the default context values. */ - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x80000 + gr->size, - 0x1000, true, &chan); - if (ret) { - nvkm_error(subdev, "failed to allocate chan memory, %d\n", ret); - return ret; - } + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, + 0x1000, 0x1000, true, &inst); + if (ret) + goto done; - addr = nvkm_memory_addr(chan); + ret = nvkm_vmm_new(device, 0, 0, NULL, 0, NULL, "grctx", &vmm); + if (ret) + goto done; - /* PGD pointer */ - nvkm_kmap(chan); - nvkm_wo32(chan, 0x0200, lower_32_bits(addr + 0x1000)); - nvkm_wo32(chan, 0x0204, upper_32_bits(addr + 0x1000)); - nvkm_wo32(chan, 0x0208, 0xffffffff); - nvkm_wo32(chan, 0x020c, 0x000000ff); + vmm->debug = subdev->debug; - /* PGT[0] pointer */ - nvkm_wo32(chan, 0x1000, 0x00000000); - nvkm_wo32(chan, 0x1004, 0x00000001 | (addr + 0x2000) >> 8); + ret = nvkm_vmm_join(vmm, inst); + if (ret) + goto done; - /* identity-map the whole "channel" into its own vm */ - for (i = 0; i < nvkm_memory_size(chan) / 4096; i++) { - u64 addr = ((nvkm_memory_addr(chan) + (i * 4096)) >> 8) | 1; - nvkm_wo32(chan, 0x2000 + (i * 8), lower_32_bits(addr)); - nvkm_wo32(chan, 0x2004 + (i * 8), upper_32_bits(addr)); - } + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, + CB_RESERVED + gr->size, 0, true, &data); + if (ret) + goto done; - /* context pointer (virt) */ - nvkm_wo32(chan, 0x0210, 0x00080004); - nvkm_wo32(chan, 0x0214, 0x00000000); - nvkm_done(chan); + ret = nvkm_vmm_get(vmm, 0, nvkm_memory_size(data), &ctx); + if (ret) + goto done; - nvkm_wr32(device, 0x100cb8, (addr + 0x1000) >> 8); - nvkm_wr32(device, 0x100cbc, 0x80000001); - nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x100c80) & 0x00008000) - break; - ); + ret = nvkm_memory_map(data, 0, vmm, ctx, NULL, 0); + if (ret) + goto done; + + + /* Setup context pointer. */ + nvkm_kmap(inst); + nvkm_wo32(inst, 0x0210, lower_32_bits(ctx->addr + CB_RESERVED) | 4); + nvkm_wo32(inst, 0x0214, upper_32_bits(ctx->addr + CB_RESERVED)); + nvkm_done(inst); - /* setup default state for mmio list construction */ + /* Setup default state for mmio list construction. */ info.gr = gr; info.data = gr->mmio_data; info.mmio = gr->mmio_list; - info.addr = 0x2000 + (i * 8); + info.addr = ctx->addr; info.buffer_nr = 0; - /* make channel current */ + /* Make channel current. */ + addr = nvkm_memory_addr(inst) >> 12; if (gr->firmware) { nvkm_wr32(device, 0x409840, 0x00000030); - nvkm_wr32(device, 0x409500, 0x80000000 | addr >> 12); + nvkm_wr32(device, 0x409500, 0x80000000 | addr); nvkm_wr32(device, 0x409504, 0x00000003); nvkm_msec(device, 2000, if (nvkm_rd32(device, 0x409800) & 0x00000010) break; ); - nvkm_kmap(chan); - nvkm_wo32(chan, 0x8001c, 1); - nvkm_wo32(chan, 0x80020, 0); - nvkm_wo32(chan, 0x80028, 0); - nvkm_wo32(chan, 0x8002c, 0); - nvkm_done(chan); + nvkm_kmap(data); + nvkm_wo32(data, 0x1c, 1); + nvkm_wo32(data, 0x20, 0); + nvkm_wo32(data, 0x28, 0); + nvkm_wo32(data, 0x2c, 0); + nvkm_done(data); } else { nvkm_wr32(device, 0x409840, 0x80000000); - nvkm_wr32(device, 0x409500, 0x80000000 | addr >> 12); + nvkm_wr32(device, 0x409500, 0x80000000 | addr); nvkm_wr32(device, 0x409504, 0x00000001); nvkm_msec(device, 2000, if (nvkm_rd32(device, 0x409800) & 0x80000000) @@ -1352,8 +1354,8 @@ gf100_grctx_generate(struct gf100_gr *gr) grctx->main(gr, &info); - /* trigger a context unload by unsetting the "next channel valid" bit - * and faking a context switch interrupt + /* Trigger a context unload by unsetting the "next channel valid" bit + * and faking a context switch interrupt. */ nvkm_mask(device, 0x409b04, 0x80000000, 0x00000000); nvkm_wr32(device, 0x409000, 0x00000100); @@ -1367,17 +1369,21 @@ gf100_grctx_generate(struct gf100_gr *gr) gr->data = kmalloc(gr->size, GFP_KERNEL); if (gr->data) { - nvkm_kmap(chan); + nvkm_kmap(data); for (i = 0; i < gr->size; i += 4) - gr->data[i / 4] = nvkm_ro32(chan, 0x80000 + i); - nvkm_done(chan); + gr->data[i / 4] = nvkm_ro32(data, CB_RESERVED + i); + nvkm_done(data); ret = 0; } else { ret = -ENOMEM; } done: - nvkm_memory_unref(&chan); + nvkm_vmm_put(vmm, &ctx); + nvkm_memory_unref(&data); + nvkm_vmm_part(vmm, inst); + nvkm_vmm_unref(&vmm); + nvkm_memory_unref(&inst); return ret; } From 3a314f747ba5b4cca22a36603768c176d1761afd Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 125/151] drm/nouveau: remove explicit unmaps If the VMA is being deleted, we don't need to explicity unmap first anymore. The MMU code will automatically merge the operations into a single page tree walk. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bo.c | 1 - drivers/gpu/drm/nouveau/nouveau_mem.c | 10 ++-------- drivers/gpu/drm/nouveau/nouveau_vmm.c | 4 +--- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index d89f3e700bafa..f4545773756a3 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -1414,7 +1414,6 @@ nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) if (!mem->bar_vma.node) return; - nvkm_vm_unmap(&mem->bar_vma); nvkm_vm_put(&mem->bar_vma); } diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c index 70fbe5e72b55c..17c6efb0ebec2 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.c +++ b/drivers/gpu/drm/nouveau/nouveau_mem.c @@ -36,14 +36,8 @@ nouveau_mem_map(struct nouveau_mem *mem, void nouveau_mem_fini(struct nouveau_mem *mem) { - if (mem->vma[1].node) { - nvkm_vm_unmap(&mem->vma[1]); - nvkm_vm_put(&mem->vma[1]); - } - if (mem->vma[0].node) { - nvkm_vm_unmap(&mem->vma[0]); - nvkm_vm_put(&mem->vma[0]); - } + nvkm_vm_put(&mem->vma[1]); + nvkm_vm_put(&mem->vma[0]); } int diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.c b/drivers/gpu/drm/nouveau/nouveau_vmm.c index 855d549e17cf9..6dc14f92b9882 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vmm.c +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.c @@ -61,10 +61,8 @@ nouveau_vma_del(struct nouveau_vma **pvma) { struct nouveau_vma *vma = *pvma; if (vma && --vma->refs <= 0) { - if (likely(vma->addr != ~0ULL)) { - nouveau_vma_unmap(vma); + if (likely(vma->addr != ~0ULL)) nvkm_vm_put(&vma->_vma); - } list_del(&vma->head); *pvma = NULL; kfree(*pvma); From 51645eb71485dd3d72d9fe2acd0298057afdf437 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 126/151] drm/nouveau/mmu: build up information on available memory types Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 20 +++ .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 120 ++++++++++++++++++ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 1 + 11 files changed, 151 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 7fa60d79ec4c2..11a169d3ed8c5 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -108,6 +108,26 @@ struct nvkm_mmu { u8 dma_bits; u8 lpg_shift; + int heap_nr; + struct { +#define NVKM_MEM_VRAM 0x01 +#define NVKM_MEM_HOST 0x02 +#define NVKM_MEM_COMP 0x04 +#define NVKM_MEM_DISP 0x08 + u8 type; + u64 size; + } heap[4]; + + int type_nr; + struct { +#define NVKM_MEM_KIND 0x10 +#define NVKM_MEM_MAPPABLE 0x20 +#define NVKM_MEM_COHERENT 0x40 +#define NVKM_MEM_UNCACHED 0x80 + u8 type; + u8 heap; + } type[16]; + struct nvkm_vmm *vmm; struct { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index 9bf688df24f01..be600049f2211 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -24,6 +24,7 @@ #include "priv.h" #include "vmm.h" +#include #include #include @@ -443,11 +444,130 @@ nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_memory *inst) return 0; } +static void +nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type) +{ + if (heap >= 0 && !WARN_ON(mmu->type_nr == ARRAY_SIZE(mmu->type))) { + mmu->type[mmu->type_nr].type = type | mmu->heap[heap].type; + mmu->type[mmu->type_nr].heap = heap; + mmu->type_nr++; + } +} + +static int +nvkm_mmu_heap(struct nvkm_mmu *mmu, u8 type, u64 size) +{ + if (size) { + if (!WARN_ON(mmu->heap_nr == ARRAY_SIZE(mmu->heap))) { + mmu->heap[mmu->heap_nr].type = type; + mmu->heap[mmu->heap_nr].size = size; + return mmu->heap_nr++; + } + } + return -EINVAL; +} + +static void +nvkm_mmu_host(struct nvkm_mmu *mmu) +{ + struct nvkm_device *device = mmu->subdev.device; + u8 type = NVKM_MEM_KIND * !!mmu->func->kind_sys; + int heap; + + /* Non-mappable system memory. */ + heap = nvkm_mmu_heap(mmu, NVKM_MEM_HOST, ~0ULL); + nvkm_mmu_type(mmu, heap, type); + + /* Non-coherent, cached, system memory. + * + * Block-linear mappings of system memory must be done through + * BAR1, and cannot be supported on systems where we're unable + * to map BAR1 with write-combining. + */ + type |= NVKM_MEM_MAPPABLE; + if (!device->bar || device->bar->iomap_uncached) + nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND); + else + nvkm_mmu_type(mmu, heap, type); + + /* Coherent, cached, system memory. + * + * Unsupported on systems that aren't able to support snooped + * mappings, and also for block-linear mappings which must be + * done through BAR1. + */ + type |= NVKM_MEM_COHERENT; + if (device->func->cpu_coherent) + nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND); + + /* Uncached system memory. */ + nvkm_mmu_type(mmu, heap, type |= NVKM_MEM_UNCACHED); +} + +static void +nvkm_mmu_vram(struct nvkm_mmu *mmu) +{ + struct nvkm_device *device = mmu->subdev.device; + struct nvkm_mm *mm = &device->fb->ram->vram; + const u32 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL); + const u32 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP); + const u32 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED); + u8 type = NVKM_MEM_KIND * !!mmu->func->kind; + u8 heap = NVKM_MEM_VRAM; + int heapM, heapN, heapU; + + /* Mixed-memory doesn't support compression or display. */ + heapM = nvkm_mmu_heap(mmu, heap, sizeM << NVKM_RAM_MM_SHIFT); + + heap |= NVKM_MEM_COMP; + heap |= NVKM_MEM_DISP; + heapN = nvkm_mmu_heap(mmu, heap, sizeN << NVKM_RAM_MM_SHIFT); + heapU = nvkm_mmu_heap(mmu, heap, sizeU << NVKM_RAM_MM_SHIFT); + + /* Add non-mappable VRAM types first so that they're preferred + * over anything else. Mixed-memory will be slower than other + * heaps, it's prioritised last. + */ + nvkm_mmu_type(mmu, heapU, type); + nvkm_mmu_type(mmu, heapN, type); + nvkm_mmu_type(mmu, heapM, type); + + /* Add host memory types next, under the assumption that users + * wanting mappable memory want to use them as staging buffers + * or the like. + */ + nvkm_mmu_host(mmu); + + /* Mappable VRAM types go last, as they're basically the worst + * possible type to ask for unless there's no other choice. + */ + if (device->bar) { + /* Write-combined BAR1 access. */ + type |= NVKM_MEM_MAPPABLE; + if (!device->bar->iomap_uncached) { + nvkm_mmu_type(mmu, heapN, type); + nvkm_mmu_type(mmu, heapM, type); + } + + /* Uncached BAR1 access. */ + type |= NVKM_MEM_COHERENT; + type |= NVKM_MEM_UNCACHED; + nvkm_mmu_type(mmu, heapN, type); + nvkm_mmu_type(mmu, heapM, type); + } +} + static int nvkm_mmu_oneinit(struct nvkm_subdev *subdev) { struct nvkm_mmu *mmu = nvkm_mmu(subdev); + /* Determine available memory types. */ + if (mmu->subdev.device->fb && mmu->subdev.device->fb->ram) + nvkm_mmu_vram(mmu); + else + nvkm_mmu_host(mmu); + if (mmu->func->vmm.global) { int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL, "gart", &mmu->vmm); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c index c90c9980478be..0f4c9533c967d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c @@ -30,6 +30,7 @@ g84_mmu = { .lpg_shift = 16, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x0200 }, .kind = nv50_mmu_kind, + .kind_sys = true, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index ca5ca27188cda..81e9145f96a1d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -78,6 +78,7 @@ gf100_mmu = { .lpg_shift = 17, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gf100_vmm_new }, .kind = gf100_mmu_kind, + .kind_sys = true, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c index db7353aa25dfd..95f07acaed2f8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c @@ -30,6 +30,7 @@ gk104_mmu = { .lpg_shift = 17, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk104_vmm_new }, .kind = gf100_mmu_kind, + .kind_sys = true, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c index 2dfe8b1ba109c..18d4a321bd64a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c @@ -30,6 +30,7 @@ gk20a_mmu = { .lpg_shift = 17, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk20a_vmm_new }, .kind = gf100_mmu_kind, + .kind_sys = true, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c index a9628baf53722..f961f782ac9b1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c @@ -74,6 +74,7 @@ gm200_mmu = { .lpg_shift = 17, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm200_vmm_new }, .kind = gm200_mmu_kind, + .kind_sys = true, }; static const struct nvkm_mmu_func @@ -83,6 +84,7 @@ gm200_mmu_fixed = { .lpg_shift = 17, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm200_vmm_new_fixed }, .kind = gm200_mmu_kind, + .kind_sys = true, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c index 5d14c138af025..585fe80f87aff 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c @@ -32,6 +32,7 @@ gm20b_mmu = { .lpg_shift = 17, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new }, .kind = gm200_mmu_kind, + .kind_sys = true, }; static const struct nvkm_mmu_func @@ -41,6 +42,7 @@ gm20b_mmu_fixed = { .lpg_shift = 17, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new_fixed }, .kind = gm200_mmu_kind, + .kind_sys = true, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c index 0cc9c89c0e733..4ebb45a275ea8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c @@ -32,6 +32,7 @@ gp100_mmu = { .lpg_shift = 16, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, .kind = gm200_mmu_kind, + .kind_sys = true, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c index 5c72176977608..e864cfe57ae67 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -32,6 +32,7 @@ gp10b_mmu = { .lpg_shift = 16, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, .kind = gm200_mmu_kind, + .kind_sys = true, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index bc4571a206902..d8d63ae7e0f75 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -25,6 +25,7 @@ struct nvkm_mmu_func { } vmm; const u8 *(*kind)(struct nvkm_mmu *, int *count); + bool kind_sys; }; extern const struct nvkm_mmu_func nv04_mmu; From eaf1a69110f4497746d5b6e35929dbdc860040d2 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 127/151] drm/nouveau/mmu: add base for type-based memory allocation Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 2 + drivers/gpu/drm/nouveau/include/nvif/if000a.h | 13 + .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 2 + drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.c | 242 ++++++++++++++++++ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h | 8 + .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 8 + 6 files changed, 275 insertions(+) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if000a.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index d5236daf54c4b..f81cc3e683d96 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -14,6 +14,8 @@ #define NVIF_CLASS_SW_NV50 /* if0005.h */ -0x00000006 #define NVIF_CLASS_SW_GF100 /* if0005.h */ -0x00000007 +#define NVIF_CLASS_MEM /* if000a.h */ 0x8000000a + #define NVIF_CLASS_VMM /* if000c.h */ 0x8000000c #define NVIF_CLASS_VMM_NV04 /* if000d.h */ 0x8000000d #define NVIF_CLASS_VMM_NV50 /* if500d.h */ 0x8000500d diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000a.h b/drivers/gpu/drm/nouveau/include/nvif/if000a.h new file mode 100644 index 0000000000000..e6e8764abf4a8 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if000a.h @@ -0,0 +1,13 @@ +#ifndef __NVIF_IF000A_H__ +#define __NVIF_IF000A_H__ + +struct nvif_mem_ram_vn { +}; + +struct nvif_mem_ram_v0 { + __u8 version; + __u8 pad01[7]; + dma_addr_t *dma; + struct scatterlist *sgl; +}; +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index e7ef528fb5bfe..941b629550c1b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -12,6 +12,8 @@ nvkm-y += nvkm/subdev/mmu/gm20b.o nvkm-y += nvkm/subdev/mmu/gp100.o nvkm-y += nvkm/subdev/mmu/gp10b.o +nvkm-y += nvkm/subdev/mmu/mem.o + nvkm-y += nvkm/subdev/mmu/vmm.o nvkm-y += nvkm/subdev/mmu/vmmnv04.o nvkm-y += nvkm/subdev/mmu/vmmnv41.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.c new file mode 100644 index 0000000000000..39808489f21d0 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.c @@ -0,0 +1,242 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#define nvkm_mem(p) container_of((p), struct nvkm_mem, memory) +#include "mem.h" + +#include + +#include +#include + +struct nvkm_mem { + struct nvkm_memory memory; + enum nvkm_memory_target target; + struct nvkm_mmu *mmu; + u64 pages; + struct page **mem; + union { + struct scatterlist *sgl; + dma_addr_t *dma; + }; +}; + +static enum nvkm_memory_target +nvkm_mem_target(struct nvkm_memory *memory) +{ + return nvkm_mem(memory)->target; +} + +static u8 +nvkm_mem_page(struct nvkm_memory *memory) +{ + return PAGE_SHIFT; +} + +static u64 +nvkm_mem_addr(struct nvkm_memory *memory) +{ + struct nvkm_mem *mem = nvkm_mem(memory); + if (mem->pages == 1 && mem->mem) + return mem->dma[0]; + return ~0ULL; +} + +static u64 +nvkm_mem_size(struct nvkm_memory *memory) +{ + return nvkm_mem(memory)->pages << PAGE_SHIFT; +} + +static int +nvkm_mem_map_dma(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, + struct nvkm_vma *vma, void *argv, u32 argc) +{ + struct nvkm_mem *mem = nvkm_mem(memory); + struct nvkm_vmm_map map = { + .memory = &mem->memory, + .offset = offset, + .dma = mem->dma, + }; + return nvkm_vmm_map(vmm, vma, argv, argc, &map); +} + +static void * +nvkm_mem_dtor(struct nvkm_memory *memory) +{ + struct nvkm_mem *mem = nvkm_mem(memory); + if (mem->mem) { + while (mem->pages--) { + dma_unmap_page(mem->mmu->subdev.device->dev, + mem->dma[mem->pages], PAGE_SIZE, + DMA_BIDIRECTIONAL); + __free_page(mem->mem[mem->pages]); + } + kvfree(mem->dma); + kvfree(mem->mem); + } + return mem; +} + +static const struct nvkm_memory_func +nvkm_mem_dma = { + .dtor = nvkm_mem_dtor, + .target = nvkm_mem_target, + .page = nvkm_mem_page, + .addr = nvkm_mem_addr, + .size = nvkm_mem_size, + .map = nvkm_mem_map_dma, +}; + +static int +nvkm_mem_map_sgl(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, + struct nvkm_vma *vma, void *argv, u32 argc) +{ + struct nvkm_mem *mem = nvkm_mem(memory); + struct nvkm_vmm_map map = { + .memory = &mem->memory, + .offset = offset, + .sgl = mem->sgl, + }; + return nvkm_vmm_map(vmm, vma, argv, argc, &map); +} + +static const struct nvkm_memory_func +nvkm_mem_sgl = { + .dtor = nvkm_mem_dtor, + .target = nvkm_mem_target, + .page = nvkm_mem_page, + .addr = nvkm_mem_addr, + .size = nvkm_mem_size, + .map = nvkm_mem_map_sgl, +}; + +int +nvkm_mem_map_host(struct nvkm_memory *memory, void **pmap) +{ + struct nvkm_mem *mem = nvkm_mem(memory); + if (mem->mem) { + *pmap = vmap(mem->mem, mem->pages, VM_MAP, PAGE_KERNEL); + return *pmap ? 0 : -EFAULT; + } + return -EINVAL; +} + +static int +nvkm_mem_new_host(struct nvkm_mmu *mmu, int type, u8 page, u64 size, + void *argv, u32 argc, struct nvkm_memory **pmemory) +{ + struct device *dev = mmu->subdev.device->dev; + union { + struct nvif_mem_ram_vn vn; + struct nvif_mem_ram_v0 v0; + } *args = argv; + int ret = -ENOSYS; + enum nvkm_memory_target target; + struct nvkm_mem *mem; + gfp_t gfp = GFP_USER | __GFP_ZERO; + + if ( (mmu->type[type].type & NVKM_MEM_COHERENT) && + !(mmu->type[type].type & NVKM_MEM_UNCACHED)) + target = NVKM_MEM_TARGET_HOST; + else + target = NVKM_MEM_TARGET_NCOH; + + if (page != PAGE_SHIFT) + return -EINVAL; + + if (!(mem = kzalloc(sizeof(*mem), GFP_KERNEL))) + return -ENOMEM; + mem->target = target; + mem->mmu = mmu; + *pmemory = &mem->memory; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + if (args->v0.dma) { + nvkm_memory_ctor(&nvkm_mem_dma, &mem->memory); + mem->dma = args->v0.dma; + } else { + nvkm_memory_ctor(&nvkm_mem_sgl, &mem->memory); + mem->sgl = args->v0.sgl; + } + + if (!IS_ALIGNED(size, PAGE_SIZE)) + return -EINVAL; + mem->pages = size >> PAGE_SHIFT; + return 0; + } else + if ( (ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + kfree(mem); + return ret; + } + + nvkm_memory_ctor(&nvkm_mem_dma, &mem->memory); + size = ALIGN(size, PAGE_SIZE) >> PAGE_SHIFT; + + if (!(mem->mem = kvmalloc(sizeof(*mem->mem) * size, GFP_KERNEL))) + return -ENOMEM; + if (!(mem->dma = kvmalloc(sizeof(*mem->dma) * size, GFP_KERNEL))) + return -ENOMEM; + + if (mmu->dma_bits > 32) + gfp |= GFP_HIGHUSER; + else + gfp |= GFP_DMA32; + + for (mem->pages = 0; size; size--, mem->pages++) { + struct page *p = alloc_page(gfp); + if (!p) + return -ENOMEM; + + mem->dma[mem->pages] = dma_map_page(mmu->subdev.device->dev, + p, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, mem->dma[mem->pages])) { + __free_page(p); + return -ENOMEM; + } + + mem->mem[mem->pages] = p; + } + + return 0; +} + +int +nvkm_mem_new_type(struct nvkm_mmu *mmu, int type, u8 page, u64 size, + void *argv, u32 argc, struct nvkm_memory **pmemory) +{ + struct nvkm_memory *memory = NULL; + int ret; + + if (mmu->type[type].type & NVKM_MEM_VRAM) { + ret = mmu->func->mem.vram(mmu, type, page, size, + argv, argc, &memory); + } else { + ret = nvkm_mem_new_host(mmu, type, page, size, + argv, argc, &memory); + } + + if (ret) + nvkm_memory_unref(&memory); + *pmemory = memory; + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h new file mode 100644 index 0000000000000..442adecdb2890 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h @@ -0,0 +1,8 @@ +#ifndef __NVKM_MEM_H__ +#define __NVKM_MEM_H__ +#include "priv.h" + +int nvkm_mem_new_type(struct nvkm_mmu *, int type, u8 page, u64 size, + void *argv, u32 argc, struct nvkm_memory **); +int nvkm_mem_map_host(struct nvkm_memory *, void **pmap); +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index d8d63ae7e0f75..00869fcc57512 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -15,6 +15,14 @@ struct nvkm_mmu_func { u8 dma_bits; u8 lpg_shift; + struct { + struct nvkm_sclass user; + int (*vram)(struct nvkm_mmu *, int type, u8 page, u64 size, + void *argv, u32 argc, struct nvkm_memory **); + int (*umap)(struct nvkm_mmu *, struct nvkm_memory *, void *argv, + u32 argc, u64 *addr, u64 *size, struct nvkm_vma **); + } mem; + struct { struct nvkm_sclass user; int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, From 957e18a70da19373f966c20dcf3ae5e1d49f9ed0 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 128/151] drm/nouveau/mmu/nv04-nv4x: type-based vram allocation and bar mapping Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 1 + drivers/gpu/drm/nouveau/include/nvif/if000b.h | 11 +++ .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h | 5 ++ .../gpu/drm/nouveau/nvkm/subdev/mmu/memnv04.c | 69 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c | 2 + 8 files changed, 93 insertions(+) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if000b.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv04.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index f81cc3e683d96..b1bc758a95184 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -15,6 +15,7 @@ #define NVIF_CLASS_SW_GF100 /* if0005.h */ -0x00000007 #define NVIF_CLASS_MEM /* if000a.h */ 0x8000000a +#define NVIF_CLASS_MEM_NV04 /* if000b.h */ 0x8000000b #define NVIF_CLASS_VMM /* if000c.h */ 0x8000000c #define NVIF_CLASS_VMM_NV04 /* if000d.h */ 0x8000000d diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000b.h b/drivers/gpu/drm/nouveau/include/nvif/if000b.h new file mode 100644 index 0000000000000..c677fb0293da8 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if000b.h @@ -0,0 +1,11 @@ +#ifndef __NVIF_IF000B_H__ +#define __NVIF_IF000B_H__ +#include "if000a.h" + +struct nv04_mem_vn { + /* nvkm_mem_vX ... */ +}; + +struct nv04_mem_map_vn { +}; +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 941b629550c1b..241f41da38812 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -13,6 +13,7 @@ nvkm-y += nvkm/subdev/mmu/gp100.o nvkm-y += nvkm/subdev/mmu/gp10b.o nvkm-y += nvkm/subdev/mmu/mem.o +nvkm-y += nvkm/subdev/mmu/memnv04.o nvkm-y += nvkm/subdev/mmu/vmm.o nvkm-y += nvkm/subdev/mmu/vmmnv04.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h index 442adecdb2890..2986ee36bf009 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h @@ -5,4 +5,9 @@ int nvkm_mem_new_type(struct nvkm_mmu *, int type, u8 page, u64 size, void *argv, u32 argc, struct nvkm_memory **); int nvkm_mem_map_host(struct nvkm_memory *, void **pmap); + +int nv04_mem_new(struct nvkm_mmu *, int, u8, u64, void *, u32, + struct nvkm_memory **); +int nv04_mem_map(struct nvkm_mmu *, struct nvkm_memory *, void *, u32, + u64 *, u64 *, struct nvkm_vma **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv04.c new file mode 100644 index 0000000000000..79a3b0cc9f5be --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv04.c @@ -0,0 +1,69 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "mem.h" + +#include +#include + +#include +#include + +int +nv04_mem_map(struct nvkm_mmu *mmu, struct nvkm_memory *memory, void *argv, + u32 argc, u64 *paddr, u64 *psize, struct nvkm_vma **pvma) +{ + union { + struct nv04_mem_map_vn vn; + } *args = argv; + struct nvkm_device *device = mmu->subdev.device; + const u64 addr = nvkm_memory_addr(memory); + int ret = -ENOSYS; + + if ((ret = nvif_unvers(ret, &argv, &argc, args->vn))) + return ret; + + *paddr = device->func->resource_addr(device, 1) + addr; + *psize = nvkm_memory_size(memory); + *pvma = ERR_PTR(-ENODEV); + return 0; +} + +int +nv04_mem_new(struct nvkm_mmu *mmu, int type, u8 page, u64 size, + void *argv, u32 argc, struct nvkm_memory **pmemory) +{ + union { + struct nv04_mem_vn vn; + } *args = argv; + int ret = -ENOSYS; + + if ((ret = nvif_unvers(ret, &argv, &argc, args->vn))) + return ret; + + if (mmu->type[type].type & NVKM_MEM_MAPPABLE) + type = NVKM_RAM_MM_NORMAL; + else + type = NVKM_RAM_MM_NOMAP; + + return nvkm_ram_get(mmu->subdev.device, type, 0x01, page, + size, true, false, pmemory); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index 3ae70e218bc03..94db229bd929b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "mem.h" #include "vmm.h" #include @@ -32,6 +33,7 @@ nv04_mmu = { .limit = NV04_PDMA_SIZE, .dma_bits = 32, .lpg_shift = 12, + .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv04_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c index 5bcbc10d85648..caee0858f522c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "mem.h" #include "vmm.h" #include @@ -44,6 +45,7 @@ nv41_mmu = { .limit = NV41_GART_SIZE, .dma_bits = 39, .lpg_shift = 12, + .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv41_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c index 998287021e3f8..5f15ec58bbd49 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "mem.h" #include "vmm.h" #include @@ -59,6 +60,7 @@ nv44_mmu = { .limit = NV44_GART_SIZE, .dma_bits = 39, .lpg_shift = 12, + .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv44_vmm_new, true }, }; From 0766116157bf10a0680f4dc7530717e3b4d1a31e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 129/151] drm/nouveau/mmu/nv50,g84: type-based vram allocation and bar mapping Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 1 + drivers/gpu/drm/nouveau/include/nvif/if500b.h | 25 ++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c | 2 + drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h | 5 ++ .../gpu/drm/nouveau/nvkm/subdev/mmu/memnv50.c | 88 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c | 2 + 7 files changed, 124 insertions(+) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if500b.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv50.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index b1bc758a95184..a9a79ff1ef5ca 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -16,6 +16,7 @@ #define NVIF_CLASS_MEM /* if000a.h */ 0x8000000a #define NVIF_CLASS_MEM_NV04 /* if000b.h */ 0x8000000b +#define NVIF_CLASS_MEM_NV50 /* if500b.h */ 0x8000500b #define NVIF_CLASS_VMM /* if000c.h */ 0x8000000c #define NVIF_CLASS_VMM_NV04 /* if000d.h */ 0x8000000d diff --git a/drivers/gpu/drm/nouveau/include/nvif/if500b.h b/drivers/gpu/drm/nouveau/include/nvif/if500b.h new file mode 100644 index 0000000000000..c7c8431fb2ced --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if500b.h @@ -0,0 +1,25 @@ +#ifndef __NVIF_IF500B_H__ +#define __NVIF_IF500B_H__ +#include "if000a.h" + +struct nv50_mem_vn { + /* nvif_mem_vX ... */ +}; + +struct nv50_mem_v0 { + /* nvif_mem_vX ... */ + __u8 version; + __u8 bankswz; + __u8 contig; +}; + +struct nv50_mem_map_vn { +}; + +struct nv50_mem_map_v0 { + __u8 version; + __u8 ro; + __u8 kind; + __u8 comp; +}; +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 241f41da38812..7e8e328981947 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -14,6 +14,7 @@ nvkm-y += nvkm/subdev/mmu/gp10b.o nvkm-y += nvkm/subdev/mmu/mem.o nvkm-y += nvkm/subdev/mmu/memnv04.o +nvkm-y += nvkm/subdev/mmu/memnv50.o nvkm-y += nvkm/subdev/mmu/vmm.o nvkm-y += nvkm/subdev/mmu/vmmnv04.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c index 0f4c9533c967d..f5d061cd2f2d6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c @@ -19,6 +19,7 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "mem.h" #include "vmm.h" #include @@ -28,6 +29,7 @@ g84_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 16, + .mem = {{ -1, 0, NVIF_CLASS_MEM_NV50}, nv50_mem_new, nv50_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x0200 }, .kind = nv50_mmu_kind, .kind_sys = true, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h index 2986ee36bf009..c2395081e21d9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h @@ -10,4 +10,9 @@ int nv04_mem_new(struct nvkm_mmu *, int, u8, u64, void *, u32, struct nvkm_memory **); int nv04_mem_map(struct nvkm_mmu *, struct nvkm_memory *, void *, u32, u64 *, u64 *, struct nvkm_vma **); + +int nv50_mem_new(struct nvkm_mmu *, int, u8, u64, void *, u32, + struct nvkm_memory **); +int nv50_mem_map(struct nvkm_mmu *, struct nvkm_memory *, void *, u32, + u64 *, u64 *, struct nvkm_vma **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv50.c new file mode 100644 index 0000000000000..46759b89fc1fd --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv50.c @@ -0,0 +1,88 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "mem.h" + +#include +#include +#include + +#include +#include +#include +#include + +int +nv50_mem_map(struct nvkm_mmu *mmu, struct nvkm_memory *memory, void *argv, + u32 argc, u64 *paddr, u64 *psize, struct nvkm_vma **pvma) +{ + struct nv50_vmm_map_v0 uvmm = {}; + union { + struct nv50_mem_map_vn vn; + struct nv50_mem_map_v0 v0; + } *args = argv; + struct nvkm_device *device = mmu->subdev.device; + struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); + u64 size = nvkm_memory_size(memory); + int ret = -ENOSYS; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + uvmm.ro = args->v0.ro; + uvmm.kind = args->v0.kind; + uvmm.comp = args->v0.comp; + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + } else + return ret; + + ret = nvkm_vmm_get(bar, 12, size, pvma); + if (ret) + return ret; + + *paddr = device->func->resource_addr(device, 1) + (*pvma)->addr; + *psize = (*pvma)->size; + return nvkm_memory_map(memory, 0, bar, *pvma, &uvmm, sizeof(uvmm)); +} + +int +nv50_mem_new(struct nvkm_mmu *mmu, int type, u8 page, u64 size, + void *argv, u32 argc, struct nvkm_memory **pmemory) +{ + union { + struct nv50_mem_vn vn; + struct nv50_mem_v0 v0; + } *args = argv; + int ret = -ENOSYS; + bool contig; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + type = args->v0.bankswz ? 0x02 : 0x01; + contig = args->v0.contig; + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + type = 0x01; + contig = false; + } else + return -ENOSYS; + + return nvkm_ram_get(mmu->subdev.device, NVKM_RAM_MM_NORMAL, type, + page, size, contig, false, pmemory); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c index 7af1f89fe004f..8b702f42d8ce0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "mem.h" #include "vmm.h" #include @@ -64,6 +65,7 @@ nv50_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 16, + .mem = {{ -1, 0, NVIF_CLASS_MEM_NV50}, nv50_mem_new, nv50_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x1400 }, .kind = nv50_mmu_kind, }; From 68af607d269cd8ea74d789f256ab0a522e5dad4f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 130/151] drm/nouveau/mmu/gf100-: type-based vram allocation and bar mapping Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 1 + drivers/gpu/drm/nouveau/include/nvif/if900b.h | 23 +++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c | 3 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c | 3 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c | 2 + drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h | 5 + .../drm/nouveau/nvkm/subdev/mmu/memgf100.c | 94 +++++++++++++++++++ 12 files changed, 140 insertions(+) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if900b.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memgf100.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index a9a79ff1ef5ca..8db1c973db29c 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -17,6 +17,7 @@ #define NVIF_CLASS_MEM /* if000a.h */ 0x8000000a #define NVIF_CLASS_MEM_NV04 /* if000b.h */ 0x8000000b #define NVIF_CLASS_MEM_NV50 /* if500b.h */ 0x8000500b +#define NVIF_CLASS_MEM_GF100 /* if900b.h */ 0x8000900b #define NVIF_CLASS_VMM /* if000c.h */ 0x8000000c #define NVIF_CLASS_VMM_NV04 /* if000d.h */ 0x8000000d diff --git a/drivers/gpu/drm/nouveau/include/nvif/if900b.h b/drivers/gpu/drm/nouveau/include/nvif/if900b.h new file mode 100644 index 0000000000000..9b164548eea85 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if900b.h @@ -0,0 +1,23 @@ +#ifndef __NVIF_IF900B_H__ +#define __NVIF_IF900B_H__ +#include "if000a.h" + +struct gf100_mem_vn { + /* nvif_mem_vX ... */ +}; + +struct gf100_mem_v0 { + /* nvif_mem_vX ... */ + __u8 version; + __u8 contig; +}; + +struct gf100_mem_map_vn { +}; + +struct gf100_mem_map_v0 { + __u8 version; + __u8 ro; + __u8 kind; +}; +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 7e8e328981947..5893d16018a78 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -15,6 +15,7 @@ nvkm-y += nvkm/subdev/mmu/gp10b.o nvkm-y += nvkm/subdev/mmu/mem.o nvkm-y += nvkm/subdev/mmu/memnv04.o nvkm-y += nvkm/subdev/mmu/memnv50.o +nvkm-y += nvkm/subdev/mmu/memgf100.o nvkm-y += nvkm/subdev/mmu/vmm.o nvkm-y += nvkm/subdev/mmu/vmmnv04.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index 81e9145f96a1d..e054c01d00b8b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "mem.h" #include "vmm.h" #include @@ -76,6 +77,7 @@ gf100_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gf100_vmm_new }, .kind = gf100_mmu_kind, .kind_sys = true, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c index 95f07acaed2f8..53df3dd2d1041 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c @@ -19,6 +19,7 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "mem.h" #include "vmm.h" #include @@ -28,6 +29,7 @@ gk104_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk104_vmm_new }, .kind = gf100_mmu_kind, .kind_sys = true, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c index 18d4a321bd64a..9894718fa3e00 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c @@ -19,6 +19,7 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "mem.h" #include "vmm.h" #include @@ -28,6 +29,7 @@ gk20a_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk20a_vmm_new }, .kind = gf100_mmu_kind, .kind_sys = true, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c index f961f782ac9b1..b9c1e5055840a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c @@ -19,6 +19,7 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "mem.h" #include "vmm.h" #include @@ -72,6 +73,7 @@ gm200_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm200_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, @@ -82,6 +84,7 @@ gm200_mmu_fixed = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm200_vmm_new_fixed }, .kind = gm200_mmu_kind, .kind_sys = true, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c index 585fe80f87aff..0060f95fc0c03 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c @@ -19,6 +19,7 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "mem.h" #include "vmm.h" #include @@ -30,6 +31,7 @@ gm20b_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, @@ -40,6 +42,7 @@ gm20b_mmu_fixed = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new_fixed }, .kind = gm200_mmu_kind, .kind_sys = true, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c index 4ebb45a275ea8..bd876cd864e01 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c @@ -19,6 +19,7 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "mem.h" #include "vmm.h" #include @@ -30,6 +31,7 @@ gp100_mmu = { .limit = (1ULL << 49), .dma_bits = 47, .lpg_shift = 16, + .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c index e864cfe57ae67..f5be9bec5a8e3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -19,6 +19,7 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "mem.h" #include "vmm.h" #include @@ -30,6 +31,7 @@ gp10b_mmu = { .limit = (1ULL << 49), .dma_bits = 47, .lpg_shift = 16, + .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h index c2395081e21d9..234267e1b215b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h @@ -15,4 +15,9 @@ int nv50_mem_new(struct nvkm_mmu *, int, u8, u64, void *, u32, struct nvkm_memory **); int nv50_mem_map(struct nvkm_mmu *, struct nvkm_memory *, void *, u32, u64 *, u64 *, struct nvkm_vma **); + +int gf100_mem_new(struct nvkm_mmu *, int, u8, u64, void *, u32, + struct nvkm_memory **); +int gf100_mem_map(struct nvkm_mmu *, struct nvkm_memory *, void *, u32, + u64 *, u64 *, struct nvkm_vma **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memgf100.c new file mode 100644 index 0000000000000..d9c9bee452229 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memgf100.c @@ -0,0 +1,94 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "mem.h" + +#include +#include +#include + +#include +#include +#include +#include + +int +gf100_mem_map(struct nvkm_mmu *mmu, struct nvkm_memory *memory, void *argv, + u32 argc, u64 *paddr, u64 *psize, struct nvkm_vma **pvma) +{ + struct gf100_vmm_map_v0 uvmm = {}; + union { + struct gf100_mem_map_vn vn; + struct gf100_mem_map_v0 v0; + } *args = argv; + struct nvkm_device *device = mmu->subdev.device; + struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); + int ret = -ENOSYS; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + uvmm.ro = args->v0.ro; + uvmm.kind = args->v0.kind; + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + } else + return ret; + + ret = nvkm_vmm_get(bar, nvkm_memory_page(memory), + nvkm_memory_size(memory), pvma); + if (ret) + return ret; + + ret = nvkm_memory_map(memory, 0, bar, *pvma, &uvmm, sizeof(uvmm)); + if (ret) + return ret; + + *paddr = device->func->resource_addr(device, 1) + (*pvma)->addr; + *psize = (*pvma)->size; + return 0; +} + +int +gf100_mem_new(struct nvkm_mmu *mmu, int type, u8 page, u64 size, + void *argv, u32 argc, struct nvkm_memory **pmemory) +{ + union { + struct gf100_mem_vn vn; + struct gf100_mem_v0 v0; + } *args = argv; + int ret = -ENOSYS; + bool contig; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + contig = args->v0.contig; + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + contig = false; + } else + return ret; + + if (mmu->type[type].type & (NVKM_MEM_DISP | NVKM_MEM_COMP)) + type = NVKM_RAM_MM_NORMAL; + else + type = NVKM_RAM_MM_MIXED; + + return nvkm_ram_get(mmu->subdev.device, type, 0x01, page, + size, contig, false, pmemory); +} From eea5cf0f0170fbc54fbb3c501b0ec7cce7f68369 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 131/151] drm/nouveau/mmu: define user interfaces to mmu Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 5 + drivers/gpu/drm/nouveau/include/nvif/if0008.h | 42 +++++ drivers/gpu/drm/nouveau/include/nvif/mmu.h | 56 +++++++ .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 2 + drivers/gpu/drm/nouveau/nvif/Kbuild | 1 + drivers/gpu/drm/nouveau/nvif/mmu.c | 117 ++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/device/user.c | 5 + .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 4 +- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c | 2 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 4 + .../gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c | 148 ++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/ummu.h | 14 ++ 24 files changed, 413 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if0008.h create mode 100644 drivers/gpu/drm/nouveau/include/nvif/mmu.h create mode 100644 drivers/gpu/drm/nouveau/nvif/mmu.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.h diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 8db1c973db29c..56aade45067d3 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -14,6 +14,11 @@ #define NVIF_CLASS_SW_NV50 /* if0005.h */ -0x00000006 #define NVIF_CLASS_SW_GF100 /* if0005.h */ -0x00000007 +#define NVIF_CLASS_MMU /* if0008.h */ 0x80000008 +#define NVIF_CLASS_MMU_NV04 /* if0008.h */ 0x80000009 +#define NVIF_CLASS_MMU_NV50 /* if0008.h */ 0x80005009 +#define NVIF_CLASS_MMU_GF100 /* if0008.h */ 0x80009009 + #define NVIF_CLASS_MEM /* if000a.h */ 0x8000000a #define NVIF_CLASS_MEM_NV04 /* if000b.h */ 0x8000000b #define NVIF_CLASS_MEM_NV50 /* if500b.h */ 0x8000500b diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0008.h b/drivers/gpu/drm/nouveau/include/nvif/if0008.h new file mode 100644 index 0000000000000..8450127420f55 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if0008.h @@ -0,0 +1,42 @@ +#ifndef __NVIF_IF0008_H__ +#define __NVIF_IF0008_H__ +struct nvif_mmu_v0 { + __u8 version; + __u8 dmabits; + __u8 heap_nr; + __u8 type_nr; + __u16 kind_nr; +}; + +#define NVIF_MMU_V0_HEAP 0x00 +#define NVIF_MMU_V0_TYPE 0x01 +#define NVIF_MMU_V0_KIND 0x02 + +struct nvif_mmu_heap_v0 { + __u8 version; + __u8 index; + __u8 pad02[6]; + __u64 size; +}; + +struct nvif_mmu_type_v0 { + __u8 version; + __u8 index; + __u8 heap; + __u8 vram; + __u8 host; + __u8 comp; + __u8 disp; + __u8 kind; + __u8 mappable; + __u8 coherent; + __u8 uncached; +}; + +struct nvif_mmu_kind_v0 { + __u8 version; + __u8 pad01[1]; + __u16 count; + __u8 data[]; +}; +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/mmu.h b/drivers/gpu/drm/nouveau/include/nvif/mmu.h new file mode 100644 index 0000000000000..c8cd5b5b06889 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/mmu.h @@ -0,0 +1,56 @@ +#ifndef __NVIF_MMU_H__ +#define __NVIF_MMU_H__ +#include + +struct nvif_mmu { + struct nvif_object object; + u8 dmabits; + u8 heap_nr; + u8 type_nr; + u16 kind_nr; + + struct { + u64 size; + } *heap; + + struct { +#define NVIF_MEM_VRAM 0x01 +#define NVIF_MEM_HOST 0x02 +#define NVIF_MEM_COMP 0x04 +#define NVIF_MEM_DISP 0x08 +#define NVIF_MEM_KIND 0x10 +#define NVIF_MEM_MAPPABLE 0x20 +#define NVIF_MEM_COHERENT 0x40 +#define NVIF_MEM_UNCACHED 0x80 + u8 type; + u8 heap; + } *type; + + u8 *kind; +}; + +int nvif_mmu_init(struct nvif_object *, s32 oclass, struct nvif_mmu *); +void nvif_mmu_fini(struct nvif_mmu *); + +static inline bool +nvif_mmu_kind_valid(struct nvif_mmu *mmu, u8 kind) +{ + const u8 invalid = mmu->kind_nr - 1; + if (kind) { + if (kind >= mmu->kind_nr || mmu->kind[kind] == invalid) + return false; + } + return true; +} + +static inline int +nvif_mmu_type(struct nvif_mmu *mmu, u8 mask) +{ + int i; + for (i = 0; i < mmu->type_nr; i++) { + if ((mmu->type[i].type & mask) == mask) + return i; + } + return -EINVAL; +} +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 11a169d3ed8c5..c58394baeb7ca 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -134,6 +134,8 @@ struct nvkm_mmu { struct mutex mutex; struct list_head list; } ptc, ptp; + + struct nvkm_device_oclass user; }; int nv04_mmu_new(struct nvkm_device *, int, struct nvkm_mmu **); diff --git a/drivers/gpu/drm/nouveau/nvif/Kbuild b/drivers/gpu/drm/nouveau/nvif/Kbuild index 067b5e9f5ec1c..1f23a08cf7010 100644 --- a/drivers/gpu/drm/nouveau/nvif/Kbuild +++ b/drivers/gpu/drm/nouveau/nvif/Kbuild @@ -2,4 +2,5 @@ nvif-y := nvif/object.o nvif-y += nvif/client.o nvif-y += nvif/device.o nvif-y += nvif/driver.o +nvif-y += nvif/mmu.o nvif-y += nvif/notify.o diff --git a/drivers/gpu/drm/nouveau/nvif/mmu.c b/drivers/gpu/drm/nouveau/nvif/mmu.c new file mode 100644 index 0000000000000..15d0dcbf7ab41 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvif/mmu.c @@ -0,0 +1,117 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include + +#include +#include + +void +nvif_mmu_fini(struct nvif_mmu *mmu) +{ + kfree(mmu->kind); + kfree(mmu->type); + kfree(mmu->heap); + nvif_object_fini(&mmu->object); +} + +int +nvif_mmu_init(struct nvif_object *parent, s32 oclass, struct nvif_mmu *mmu) +{ + struct nvif_mmu_v0 args; + int ret, i; + + args.version = 0; + mmu->heap = NULL; + mmu->type = NULL; + mmu->kind = NULL; + + ret = nvif_object_init(parent, 0, oclass, &args, sizeof(args), + &mmu->object); + if (ret) + goto done; + + mmu->dmabits = args.dmabits; + mmu->heap_nr = args.heap_nr; + mmu->type_nr = args.type_nr; + mmu->kind_nr = args.kind_nr; + + mmu->heap = kmalloc(sizeof(*mmu->heap) * mmu->heap_nr, GFP_KERNEL); + mmu->type = kmalloc(sizeof(*mmu->type) * mmu->type_nr, GFP_KERNEL); + if (ret = -ENOMEM, !mmu->heap || !mmu->type) + goto done; + + mmu->kind = kmalloc(sizeof(*mmu->kind) * mmu->kind_nr, GFP_KERNEL); + if (!mmu->kind && mmu->kind_nr) + goto done; + + for (i = 0; i < mmu->heap_nr; i++) { + struct nvif_mmu_heap_v0 args = { .index = i }; + + ret = nvif_object_mthd(&mmu->object, NVIF_MMU_V0_HEAP, + &args, sizeof(args)); + if (ret) + goto done; + + mmu->heap[i].size = args.size; + } + + for (i = 0; i < mmu->type_nr; i++) { + struct nvif_mmu_type_v0 args = { .index = i }; + + ret = nvif_object_mthd(&mmu->object, NVIF_MMU_V0_TYPE, + &args, sizeof(args)); + if (ret) + goto done; + + mmu->type[i].type = 0; + if (args.vram) mmu->type[i].type |= NVIF_MEM_VRAM; + if (args.host) mmu->type[i].type |= NVIF_MEM_HOST; + if (args.comp) mmu->type[i].type |= NVIF_MEM_COMP; + if (args.disp) mmu->type[i].type |= NVIF_MEM_DISP; + if (args.kind ) mmu->type[i].type |= NVIF_MEM_KIND; + if (args.mappable) mmu->type[i].type |= NVIF_MEM_MAPPABLE; + if (args.coherent) mmu->type[i].type |= NVIF_MEM_COHERENT; + if (args.uncached) mmu->type[i].type |= NVIF_MEM_UNCACHED; + mmu->type[i].heap = args.heap; + } + + if (mmu->kind_nr) { + struct nvif_mmu_kind_v0 *kind; + u32 argc = sizeof(*kind) + sizeof(*kind->data) * mmu->kind_nr; + + if (ret = -ENOMEM, !(kind = kmalloc(argc, GFP_KERNEL))) + goto done; + kind->version = 0; + kind->count = mmu->kind_nr; + + ret = nvif_object_mthd(&mmu->object, NVIF_MMU_V0_KIND, + kind, argc); + if (ret == 0) + memcpy(mmu->kind, kind->data, kind->count); + kfree(kind); + } + +done: + if (ret) + nvif_mmu_fini(mmu); + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c index e9b90e40f2746..17adcb4e8854a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c @@ -294,6 +294,11 @@ nvkm_udevice_child_get(struct nvkm_object *object, int index, if (!sclass) { switch (index) { case 0: sclass = &nvkm_control_oclass; break; + case 1: + if (!device->mmu) + return -EINVAL; + sclass = &device->mmu->user; + break; default: return -EINVAL; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 5893d16018a78..0c2fa272721f4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -29,3 +29,5 @@ nvkm-y += nvkm/subdev/mmu/vmmgm200.o nvkm-y += nvkm/subdev/mmu/vmmgm20b.o nvkm-y += nvkm/subdev/mmu/vmmgp100.o nvkm-y += nvkm/subdev/mmu/vmmgp10b.o + +nvkm-y += nvkm/subdev/mmu/ummu.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index be600049f2211..e0fa0cae79607 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -21,7 +21,7 @@ * * Authors: Ben Skeggs */ -#include "priv.h" +#include "ummu.h" #include "vmm.h" #include @@ -615,6 +615,8 @@ nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device, mmu->dma_bits = func->dma_bits; mmu->lpg_shift = func->lpg_shift; nvkm_mmu_ptc_init(mmu); + mmu->user.ctor = nvkm_ummu_new; + mmu->user.base = func->mmu.user; } int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c index f5d061cd2f2d6..e53f9beda7e8a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c @@ -29,6 +29,7 @@ g84_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 16, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV50}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_NV50}, nv50_mem_new, nv50_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x0200 }, .kind = nv50_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index e054c01d00b8b..85d237d3963c7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -77,6 +77,7 @@ gf100_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gf100_vmm_new }, .kind = gf100_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c index 53df3dd2d1041..30c0217ac8073 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c @@ -29,6 +29,7 @@ gk104_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk104_vmm_new }, .kind = gf100_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c index 9894718fa3e00..b7bff4369d308 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c @@ -29,6 +29,7 @@ gk20a_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk20a_vmm_new }, .kind = gf100_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c index b9c1e5055840a..3cabeabbcea6e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c @@ -73,6 +73,7 @@ gm200_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm200_vmm_new }, .kind = gm200_mmu_kind, @@ -84,6 +85,7 @@ gm200_mmu_fixed = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm200_vmm_new_fixed }, .kind = gm200_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c index 0060f95fc0c03..802652539d9ad 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c @@ -31,6 +31,7 @@ gm20b_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new }, .kind = gm200_mmu_kind, @@ -42,6 +43,7 @@ gm20b_mmu_fixed = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 17, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new_fixed }, .kind = gm200_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c index bd876cd864e01..92fc18eec1476 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c @@ -31,6 +31,7 @@ gp100_mmu = { .limit = (1ULL << 49), .dma_bits = 47, .lpg_shift = 16, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, .kind = gm200_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c index f5be9bec5a8e3..e1b0f82e2e05d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -31,6 +31,7 @@ gp10b_mmu = { .limit = (1ULL << 49), .dma_bits = 47, .lpg_shift = 16, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, .kind = gm200_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index 94db229bd929b..7505a72131567 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -33,6 +33,7 @@ nv04_mmu = { .limit = NV04_PDMA_SIZE, .dma_bits = 32, .lpg_shift = 12, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV04}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv04_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c index caee0858f522c..8e683a0f796d0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c @@ -45,6 +45,7 @@ nv41_mmu = { .limit = NV41_GART_SIZE, .dma_bits = 39, .lpg_shift = 12, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV04}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv41_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c index 5f15ec58bbd49..2fdc4c787b7a8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c @@ -60,6 +60,7 @@ nv44_mmu = { .limit = NV44_GART_SIZE, .dma_bits = 39, .lpg_shift = 12, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV04}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv44_vmm_new, true }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c index 8b702f42d8ce0..334f7fb460abb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c @@ -65,6 +65,7 @@ nv50_mmu = { .limit = (1ULL << 40), .dma_bits = 40, .lpg_shift = 16, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV50}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_NV50}, nv50_mem_new, nv50_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x1400 }, .kind = nv50_mmu_kind, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 00869fcc57512..9d7ff166e5054 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -15,6 +15,10 @@ struct nvkm_mmu_func { u8 dma_bits; u8 lpg_shift; + struct { + struct nvkm_sclass user; + } mmu; + struct { struct nvkm_sclass user; int (*vram)(struct nvkm_mmu *, int type, u8 page, u64 size, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c new file mode 100644 index 0000000000000..eba03d6c7b649 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c @@ -0,0 +1,148 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "ummu.h" + +#include +#include + +static int +nvkm_ummu_heap(struct nvkm_ummu *ummu, void *argv, u32 argc) +{ + struct nvkm_mmu *mmu = ummu->mmu; + union { + struct nvif_mmu_heap_v0 v0; + } *args = argv; + int ret = -ENOSYS; + u8 index; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + if ((index = args->v0.index) >= mmu->heap_nr) + return -EINVAL; + args->v0.size = mmu->heap[index].size; + } else + return ret; + + return 0; +} + +static int +nvkm_ummu_type(struct nvkm_ummu *ummu, void *argv, u32 argc) +{ + struct nvkm_mmu *mmu = ummu->mmu; + union { + struct nvif_mmu_type_v0 v0; + } *args = argv; + int ret = -ENOSYS; + u8 type, index; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + if ((index = args->v0.index) >= mmu->type_nr) + return -EINVAL; + type = mmu->type[index].type; + args->v0.heap = mmu->type[index].heap; + args->v0.vram = !!(type & NVKM_MEM_VRAM); + args->v0.host = !!(type & NVKM_MEM_HOST); + args->v0.comp = !!(type & NVKM_MEM_COMP); + args->v0.disp = !!(type & NVKM_MEM_DISP); + args->v0.kind = !!(type & NVKM_MEM_KIND); + args->v0.mappable = !!(type & NVKM_MEM_MAPPABLE); + args->v0.coherent = !!(type & NVKM_MEM_COHERENT); + args->v0.uncached = !!(type & NVKM_MEM_UNCACHED); + } else + return ret; + + return 0; +} + +static int +nvkm_ummu_kind(struct nvkm_ummu *ummu, void *argv, u32 argc) +{ + struct nvkm_mmu *mmu = ummu->mmu; + union { + struct nvif_mmu_kind_v0 v0; + } *args = argv; + const u8 *kind = NULL; + int ret = -ENOSYS, count = 0; + + if (mmu->func->kind) + kind = mmu->func->kind(mmu, &count); + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) { + if (argc != args->v0.count * sizeof(*args->v0.data)) + return -EINVAL; + if (args->v0.count > count) + return -EINVAL; + memcpy(args->v0.data, kind, args->v0.count); + } else + return ret; + + return 0; +} + +static int +nvkm_ummu_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) +{ + struct nvkm_ummu *ummu = nvkm_ummu(object); + switch (mthd) { + case NVIF_MMU_V0_HEAP: return nvkm_ummu_heap(ummu, argv, argc); + case NVIF_MMU_V0_TYPE: return nvkm_ummu_type(ummu, argv, argc); + case NVIF_MMU_V0_KIND: return nvkm_ummu_kind(ummu, argv, argc); + default: + break; + } + return -EINVAL; +} + +static const struct nvkm_object_func +nvkm_ummu = { + .mthd = nvkm_ummu_mthd, +}; + +int +nvkm_ummu_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, + void *argv, u32 argc, struct nvkm_object **pobject) +{ + union { + struct nvif_mmu_v0 v0; + } *args = argv; + struct nvkm_mmu *mmu = device->mmu; + struct nvkm_ummu *ummu; + int ret = -ENOSYS, kinds = 0; + + if (mmu->func->kind) + mmu->func->kind(mmu, &kinds); + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + args->v0.dmabits = mmu->dma_bits; + args->v0.heap_nr = mmu->heap_nr; + args->v0.type_nr = mmu->type_nr; + args->v0.kind_nr = kinds; + } else + return ret; + + if (!(ummu = kzalloc(sizeof(*ummu), GFP_KERNEL))) + return -ENOMEM; + nvkm_object_ctor(&nvkm_ummu, oclass, &ummu->object); + ummu->mmu = mmu; + *pobject = &ummu->object; + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.h new file mode 100644 index 0000000000000..0cd510dcfc680 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.h @@ -0,0 +1,14 @@ +#ifndef __NVKM_UMMU_H__ +#define __NVKM_UMMU_H__ +#define nvkm_ummu(p) container_of((p), struct nvkm_ummu, object) +#include +#include "priv.h" + +struct nvkm_ummu { + struct nvkm_object object; + struct nvkm_mmu *mmu; +}; + +int nvkm_ummu_new(struct nvkm_device *, const struct nvkm_oclass *, + void *argv, u32 argc, struct nvkm_object **); +#endif From c83c4097eba8950bc7028f0846c21f9718dd8697 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 132/151] drm/nouveau/mmu: define user interfaces to mmu memory allocation Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/if000a.h | 9 + drivers/gpu/drm/nouveau/include/nvif/mem.h | 18 ++ .../drm/nouveau/include/nvkm/core/client.h | 3 + .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 1 + drivers/gpu/drm/nouveau/nvif/Kbuild | 1 + drivers/gpu/drm/nouveau/nvif/mem.c | 88 ++++++++ drivers/gpu/drm/nouveau/nvkm/core/client.c | 2 + drivers/gpu/drm/nouveau/nvkm/core/ioctl.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/umem.c | 192 ++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/umem.h | 26 +++ .../gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c | 21 ++ 12 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/mem.h create mode 100644 drivers/gpu/drm/nouveau/nvif/mem.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000a.h b/drivers/gpu/drm/nouveau/include/nvif/if000a.h index e6e8764abf4a8..88d0938fbd5ae 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if000a.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if000a.h @@ -1,5 +1,14 @@ #ifndef __NVIF_IF000A_H__ #define __NVIF_IF000A_H__ +struct nvif_mem_v0 { + __u8 version; + __u8 type; + __u8 page; + __u8 pad03[5]; + __u64 size; + __u64 addr; + __u8 data[]; +}; struct nvif_mem_ram_vn { }; diff --git a/drivers/gpu/drm/nouveau/include/nvif/mem.h b/drivers/gpu/drm/nouveau/include/nvif/mem.h new file mode 100644 index 0000000000000..b542fe38398ea --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/mem.h @@ -0,0 +1,18 @@ +#ifndef __NVIF_MEM_H__ +#define __NVIF_MEM_H__ +#include "mmu.h" + +struct nvif_mem { + struct nvif_object object; + u8 type; + u8 page; + u64 addr; + u64 size; +}; + +int nvif_mem_init_type(struct nvif_mmu *mmu, s32 oclass, int type, u8 page, + u64 size, void *argv, u32 argc, struct nvif_mem *); +int nvif_mem_init(struct nvif_mmu *mmu, s32 oclass, u8 type, u8 page, + u64 size, void *argv, u32 argc, struct nvif_mem *); +void nvif_mem_fini(struct nvif_mem *); +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h index e876634da10a1..fa58efb0d42af 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h @@ -17,6 +17,9 @@ struct nvkm_client { int (*ntfy)(const void *, u32, const void *, u32); struct nvkm_vm *vm; + + struct list_head umem; + spinlock_t lock; }; int nvkm_client_new(const char *name, u64 device, const char *cfg, diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index c58394baeb7ca..0de5f8b7096a1 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -98,6 +98,7 @@ int nvkm_vmm_map(struct nvkm_vmm *, struct nvkm_vma *, void *argv, u32 argc, struct nvkm_vmm_map *); void nvkm_vmm_unmap(struct nvkm_vmm *, struct nvkm_vma *); +struct nvkm_memory *nvkm_umem_search(struct nvkm_client *, u64); struct nvkm_vmm *nvkm_uvmm_search(struct nvkm_client *, u64 handle); struct nvkm_mmu { diff --git a/drivers/gpu/drm/nouveau/nvif/Kbuild b/drivers/gpu/drm/nouveau/nvif/Kbuild index 1f23a08cf7010..2e0a90b4dda86 100644 --- a/drivers/gpu/drm/nouveau/nvif/Kbuild +++ b/drivers/gpu/drm/nouveau/nvif/Kbuild @@ -2,5 +2,6 @@ nvif-y := nvif/object.o nvif-y += nvif/client.o nvif-y += nvif/device.o nvif-y += nvif/driver.o +nvif-y += nvif/mem.o nvif-y += nvif/mmu.o nvif-y += nvif/notify.o diff --git a/drivers/gpu/drm/nouveau/nvif/mem.c b/drivers/gpu/drm/nouveau/nvif/mem.c new file mode 100644 index 0000000000000..0f9382c601458 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvif/mem.c @@ -0,0 +1,88 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include + +#include + +void +nvif_mem_fini(struct nvif_mem *mem) +{ + nvif_object_fini(&mem->object); +} + +int +nvif_mem_init_type(struct nvif_mmu *mmu, s32 oclass, int type, u8 page, + u64 size, void *argv, u32 argc, struct nvif_mem *mem) +{ + struct nvif_mem_v0 *args; + u8 stack[128]; + int ret; + + mem->object.client = NULL; + if (type < 0) + return -EINVAL; + + if (sizeof(*args) + argc > sizeof(stack)) { + if (!(args = kmalloc(sizeof(*args) + argc, GFP_KERNEL))) + return -ENOMEM; + } else { + args = (void *)stack; + } + args->version = 0; + args->type = type; + args->page = page; + args->size = size; + memcpy(args->data, argv, argc); + + ret = nvif_object_init(&mmu->object, 0, oclass, args, + sizeof(*args) + argc, &mem->object); + if (ret == 0) { + mem->type = mmu->type[type].type; + mem->page = args->page; + mem->addr = args->addr; + mem->size = args->size; + } + + if (args != (void *)stack) + kfree(args); + return ret; + +} + +int +nvif_mem_init(struct nvif_mmu *mmu, s32 oclass, u8 type, u8 page, + u64 size, void *argv, u32 argc, struct nvif_mem *mem) +{ + int ret = -EINVAL, i; + + mem->object.client = NULL; + + for (i = 0; ret && i < mmu->type_nr; i++) { + if ((mmu->type[i].type & type) == type) { + ret = nvif_mem_init_type(mmu, oclass, i, page, size, + argv, argc, mem); + } + } + + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/core/client.c b/drivers/gpu/drm/nouveau/nvkm/core/client.c index 0d3a896892b4b..ac671202919e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/client.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/client.c @@ -301,5 +301,7 @@ nvkm_client_new(const char *name, u64 device, const char *cfg, client->debug = nvkm_dbgopt(dbg, "CLIENT"); client->objroot = RB_ROOT; client->ntfy = ntfy; + INIT_LIST_HEAD(&client->umem); + spin_lock_init(&client->lock); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c index a28018ee5b197..d777df5a64e6c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c @@ -53,7 +53,7 @@ nvkm_ioctl_sclass(struct nvkm_client *client, union { struct nvif_ioctl_sclass_v0 v0; } *args = data; - struct nvkm_oclass oclass; + struct nvkm_oclass oclass = { .client = client }; int ret = -ENOSYS, i = 0; nvif_ioctl(object, "sclass size %d\n", size); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 0c2fa272721f4..34f5aae8f3a11 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -30,4 +30,5 @@ nvkm-y += nvkm/subdev/mmu/vmmgm20b.o nvkm-y += nvkm/subdev/mmu/vmmgp100.o nvkm-y += nvkm/subdev/mmu/vmmgp10b.o +nvkm-y += nvkm/subdev/mmu/umem.o nvkm-y += nvkm/subdev/mmu/ummu.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c new file mode 100644 index 0000000000000..fac2f9a45ea69 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c @@ -0,0 +1,192 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "umem.h" +#include "ummu.h" + +#include +#include +#include + +#include +#include +#include + +static const struct nvkm_object_func nvkm_umem; +struct nvkm_memory * +nvkm_umem_search(struct nvkm_client *client, u64 handle) +{ + struct nvkm_client *master = client->object.client; + struct nvkm_memory *memory = NULL; + struct nvkm_object *object; + struct nvkm_umem *umem; + + object = nvkm_object_search(client, handle, &nvkm_umem); + if (IS_ERR(object)) { + if (client->super && client != master) { + spin_lock(&master->lock); + list_for_each_entry(umem, &master->umem, head) { + if (umem->object.object == handle) { + memory = nvkm_memory_ref(umem->memory); + break; + } + } + spin_unlock(&master->lock); + } + } else { + umem = nvkm_umem(object); + if (!umem->priv || client->super) + memory = nvkm_memory_ref(umem->memory); + } + + return memory ? memory : ERR_PTR(-ENOENT); +} + +static int +nvkm_umem_unmap(struct nvkm_object *object) +{ + struct nvkm_umem *umem = nvkm_umem(object); + + if (!umem->map) + return -EEXIST; + + if (umem->io) { + if (!IS_ERR(umem->bar)) { + struct nvkm_device *device = umem->mmu->subdev.device; + nvkm_vmm_put(nvkm_bar_bar1_vmm(device), &umem->bar); + } else { + umem->bar = NULL; + } + } else { + vunmap(umem->map); + umem->map = NULL; + } + + return 0; +} + +static int +nvkm_umem_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *handle, u64 *length) +{ + struct nvkm_umem *umem = nvkm_umem(object); + struct nvkm_mmu *mmu = umem->mmu; + + if (!umem->mappable) + return -EINVAL; + if (umem->map) + return -EEXIST; + + if ((umem->type & NVKM_MEM_HOST) && !argc) { + int ret = nvkm_mem_map_host(umem->memory, &umem->map); + if (ret) + return ret; + + *handle = (unsigned long)(void *)umem->map; + *length = nvkm_memory_size(umem->memory); + *type = NVKM_OBJECT_MAP_VA; + return 0; + } else + if ((umem->type & NVKM_MEM_VRAM) || + (umem->type & NVKM_MEM_KIND)) { + int ret = mmu->func->mem.umap(mmu, umem->memory, argv, argc, + handle, length, &umem->bar); + if (ret) + return ret; + + *type = NVKM_OBJECT_MAP_IO; + } else { + return -EINVAL; + } + + umem->io = (*type == NVKM_OBJECT_MAP_IO); + return 0; +} + +static void * +nvkm_umem_dtor(struct nvkm_object *object) +{ + struct nvkm_umem *umem = nvkm_umem(object); + spin_lock(&umem->object.client->lock); + list_del_init(&umem->head); + spin_unlock(&umem->object.client->lock); + nvkm_memory_unref(&umem->memory); + return umem; +} + +static const struct nvkm_object_func +nvkm_umem = { + .dtor = nvkm_umem_dtor, + .map = nvkm_umem_map, + .unmap = nvkm_umem_unmap, +}; + +int +nvkm_umem_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, + struct nvkm_object **pobject) +{ + struct nvkm_mmu *mmu = nvkm_ummu(oclass->parent)->mmu; + union { + struct nvif_mem_v0 v0; + } *args = argv; + struct nvkm_umem *umem; + int type, ret = -ENOSYS; + u8 page; + u64 size; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) { + type = args->v0.type; + page = args->v0.page; + size = args->v0.size; + } else + return ret; + + if (type >= mmu->type_nr) + return -EINVAL; + + if (!(umem = kzalloc(sizeof(*umem), GFP_KERNEL))) + return -ENOMEM; + nvkm_object_ctor(&nvkm_umem, oclass, &umem->object); + umem->mmu = mmu; + umem->type = mmu->type[type].type; + umem->priv = oclass->client->super; + INIT_LIST_HEAD(&umem->head); + *pobject = &umem->object; + + if (mmu->type[type].type & NVKM_MEM_MAPPABLE) { + page = max_t(u8, page, PAGE_SHIFT); + umem->mappable = true; + } + + ret = nvkm_mem_new_type(mmu, type, page, size, argv, argc, + &umem->memory); + if (ret) + return ret; + + spin_lock(&umem->object.client->lock); + list_add(&umem->head, &umem->object.client->umem); + spin_unlock(&umem->object.client->lock); + + args->v0.page = nvkm_memory_page(umem->memory); + args->v0.addr = nvkm_memory_addr(umem->memory); + args->v0.size = nvkm_memory_size(umem->memory); + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h new file mode 100644 index 0000000000000..85cf692d620a7 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h @@ -0,0 +1,26 @@ +#ifndef __NVKM_UMEM_H__ +#define __NVKM_UMEM_H__ +#define nvkm_umem(p) container_of((p), struct nvkm_umem, object) +#include +#include "mem.h" + +struct nvkm_umem { + struct nvkm_object object; + struct nvkm_mmu *mmu; + u8 type:8; + bool priv:1; + bool mappable:1; + bool io:1; + + struct nvkm_memory *memory; + struct list_head head; + + union { + struct nvkm_vma *bar; + void *map; + }; +}; + +int nvkm_umem_new(const struct nvkm_oclass *, void *argv, u32 argc, + struct nvkm_object **); +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c index eba03d6c7b649..ed192ab3728b7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c @@ -20,10 +20,30 @@ * OTHER DEALINGS IN THE SOFTWARE. */ #include "ummu.h" +#include "umem.h" + +#include #include #include +static int +nvkm_ummu_sclass(struct nvkm_object *object, int index, + struct nvkm_oclass *oclass) +{ + struct nvkm_mmu *mmu = nvkm_ummu(object)->mmu; + + if (mmu->func->mem.user.oclass && oclass->client->super) { + if (index-- == 0) { + oclass->base = mmu->func->mem.user; + oclass->ctor = nvkm_umem_new; + return 0; + } + } + + return -EINVAL; +} + static int nvkm_ummu_heap(struct nvkm_ummu *ummu, void *argv, u32 argc) { @@ -115,6 +135,7 @@ nvkm_ummu_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) static const struct nvkm_object_func nvkm_ummu = { .mthd = nvkm_ummu_mthd, + .sclass = nvkm_ummu_sclass, }; int From 920d2b5ef2157e169d85cb778feca558c56b8370 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 133/151] drm/nouveau/mmu: define user interfaces to mmu vmm opertaions Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/if000c.h | 61 +++ drivers/gpu/drm/nouveau/include/nvif/vmm.h | 42 +++ drivers/gpu/drm/nouveau/nvif/Kbuild | 1 + drivers/gpu/drm/nouveau/nvif/vmm.c | 167 +++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c | 9 + .../gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c | 352 ++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.h | 14 + 8 files changed, 647 insertions(+) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/vmm.h create mode 100644 drivers/gpu/drm/nouveau/nvif/vmm.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.h diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000c.h b/drivers/gpu/drm/nouveau/include/nvif/if000c.h index 6363e22f9dcc8..2928ecd989ad5 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if000c.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if000c.h @@ -1,3 +1,64 @@ #ifndef __NVIF_IF000C_H__ #define __NVIF_IF000C_H__ +struct nvif_vmm_v0 { + __u8 version; + __u8 page_nr; + __u8 pad02[6]; + __u64 addr; + __u64 size; + __u8 data[]; +}; + +#define NVIF_VMM_V0_PAGE 0x00 +#define NVIF_VMM_V0_GET 0x01 +#define NVIF_VMM_V0_PUT 0x02 +#define NVIF_VMM_V0_MAP 0x03 +#define NVIF_VMM_V0_UNMAP 0x04 + +struct nvif_vmm_page_v0 { + __u8 version; + __u8 index; + __u8 shift; + __u8 sparse; + __u8 vram; + __u8 host; + __u8 comp; + __u8 pad07[1]; +}; + +struct nvif_vmm_get_v0 { + __u8 version; +#define NVIF_VMM_GET_V0_ADDR 0x00 +#define NVIF_VMM_GET_V0_PTES 0x01 +#define NVIF_VMM_GET_V0_LAZY 0x02 + __u8 type; + __u8 sparse; + __u8 page; + __u8 align; + __u8 pad05[3]; + __u64 size; + __u64 addr; +}; + +struct nvif_vmm_put_v0 { + __u8 version; + __u8 pad01[7]; + __u64 addr; +}; + +struct nvif_vmm_map_v0 { + __u8 version; + __u8 pad01[7]; + __u64 addr; + __u64 size; + __u64 memory; + __u64 offset; + __u8 data[]; +}; + +struct nvif_vmm_unmap_v0 { + __u8 version; + __u8 pad01[7]; + __u64 addr; +}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/vmm.h b/drivers/gpu/drm/nouveau/include/nvif/vmm.h new file mode 100644 index 0000000000000..c5db8a2e82df8 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/vmm.h @@ -0,0 +1,42 @@ +#ifndef __NVIF_VMM_H__ +#define __NVIF_VMM_H__ +#include +struct nvif_mem; +struct nvif_mmu; + +enum nvif_vmm_get { + ADDR, + PTES, + LAZY +}; + +struct nvif_vma { + u64 addr; + u64 size; +}; + +struct nvif_vmm { + struct nvif_object object; + u64 start; + u64 limit; + + struct { + u8 shift; + bool sparse:1; + bool vram:1; + bool host:1; + bool comp:1; + } *page; + int page_nr; +}; + +int nvif_vmm_init(struct nvif_mmu *, s32 oclass, u64 addr, u64 size, + void *argv, u32 argc, struct nvif_vmm *); +void nvif_vmm_fini(struct nvif_vmm *); +int nvif_vmm_get(struct nvif_vmm *, enum nvif_vmm_get, bool sparse, + u8 page, u8 align, u64 size, struct nvif_vma *); +void nvif_vmm_put(struct nvif_vmm *, struct nvif_vma *); +int nvif_vmm_map(struct nvif_vmm *, u64 addr, u64 size, void *argv, u32 argc, + struct nvif_mem *, u64 offset); +int nvif_vmm_unmap(struct nvif_vmm *, u64); +#endif diff --git a/drivers/gpu/drm/nouveau/nvif/Kbuild b/drivers/gpu/drm/nouveau/nvif/Kbuild index 2e0a90b4dda86..f1675a4ab6fa5 100644 --- a/drivers/gpu/drm/nouveau/nvif/Kbuild +++ b/drivers/gpu/drm/nouveau/nvif/Kbuild @@ -5,3 +5,4 @@ nvif-y += nvif/driver.o nvif-y += nvif/mem.o nvif-y += nvif/mmu.o nvif-y += nvif/notify.o +nvif-y += nvif/vmm.o diff --git a/drivers/gpu/drm/nouveau/nvif/vmm.c b/drivers/gpu/drm/nouveau/nvif/vmm.c new file mode 100644 index 0000000000000..31cdb2d2e1ff1 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvif/vmm.c @@ -0,0 +1,167 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include + +#include + +int +nvif_vmm_unmap(struct nvif_vmm *vmm, u64 addr) +{ + return nvif_object_mthd(&vmm->object, NVIF_VMM_V0_UNMAP, + &(struct nvif_vmm_unmap_v0) { .addr = addr }, + sizeof(struct nvif_vmm_unmap_v0)); +} + +int +nvif_vmm_map(struct nvif_vmm *vmm, u64 addr, u64 size, void *argv, u32 argc, + struct nvif_mem *mem, u64 offset) +{ + struct nvif_vmm_map_v0 *args; + u8 stack[16]; + int ret; + + if (sizeof(*args) + argc > sizeof(stack)) { + if (!(args = kmalloc(sizeof(*args) + argc, GFP_KERNEL))) + return -ENOMEM; + } else { + args = (void *)stack; + } + + args->version = 0; + args->addr = addr; + args->size = size; + args->memory = nvif_handle(&mem->object); + args->offset = offset; + memcpy(args->data, argv, argc); + + ret = nvif_object_mthd(&vmm->object, NVIF_VMM_V0_MAP, + args, sizeof(*args) + argc); + if (args != (void *)stack) + kfree(args); + return ret; +} + +void +nvif_vmm_put(struct nvif_vmm *vmm, struct nvif_vma *vma) +{ + if (vma->size) { + WARN_ON(nvif_object_mthd(&vmm->object, NVIF_VMM_V0_PUT, + &(struct nvif_vmm_put_v0) { + .addr = vma->addr, + }, sizeof(struct nvif_vmm_put_v0))); + vma->size = 0; + } +} + +int +nvif_vmm_get(struct nvif_vmm *vmm, enum nvif_vmm_get type, bool sparse, + u8 page, u8 align, u64 size, struct nvif_vma *vma) +{ + struct nvif_vmm_get_v0 args; + int ret; + + args.version = vma->size = 0; + args.sparse = sparse; + args.page = page; + args.align = align; + args.size = size; + + switch (type) { + case ADDR: args.type = NVIF_VMM_GET_V0_ADDR; break; + case PTES: args.type = NVIF_VMM_GET_V0_PTES; break; + case LAZY: args.type = NVIF_VMM_GET_V0_LAZY; break; + default: + WARN_ON(1); + return -EINVAL; + } + + ret = nvif_object_mthd(&vmm->object, NVIF_VMM_V0_GET, + &args, sizeof(args)); + if (ret == 0) { + vma->addr = args.addr; + vma->size = args.size; + } + return ret; +} + +void +nvif_vmm_fini(struct nvif_vmm *vmm) +{ + kfree(vmm->page); + nvif_object_fini(&vmm->object); +} + +int +nvif_vmm_init(struct nvif_mmu *mmu, s32 oclass, u64 addr, u64 size, + void *argv, u32 argc, struct nvif_vmm *vmm) +{ + struct nvif_vmm_v0 *args; + u32 argn = sizeof(*args) + argc; + int ret = -ENOSYS, i; + + vmm->object.client = NULL; + vmm->page = NULL; + + if (!(args = kmalloc(argn, GFP_KERNEL))) + return -ENOMEM; + args->version = 0; + args->addr = addr; + args->size = size; + memcpy(args->data, argv, argc); + + ret = nvif_object_init(&mmu->object, 0, oclass, args, argn, + &vmm->object); + if (ret) + goto done; + + vmm->start = args->addr; + vmm->limit = args->size; + + vmm->page_nr = args->page_nr; + vmm->page = kmalloc(sizeof(*vmm->page) * vmm->page_nr, GFP_KERNEL); + if (!vmm->page) { + ret = -ENOMEM; + goto done; + } + + for (i = 0; i < vmm->page_nr; i++) { + struct nvif_vmm_page_v0 args = { .index = i }; + + ret = nvif_object_mthd(&vmm->object, NVIF_VMM_V0_PAGE, + &args, sizeof(args)); + if (ret) + break; + + vmm->page[i].shift = args.shift; + vmm->page[i].sparse = args.sparse; + vmm->page[i].vram = args.vram; + vmm->page[i].host = args.host; + vmm->page[i].comp = args.comp; + } + +done: + if (ret) + nvif_vmm_fini(vmm); + kfree(args); + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 34f5aae8f3a11..352a65f9371cd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -32,3 +32,4 @@ nvkm-y += nvkm/subdev/mmu/vmmgp10b.o nvkm-y += nvkm/subdev/mmu/umem.o nvkm-y += nvkm/subdev/mmu/ummu.o +nvkm-y += nvkm/subdev/mmu/uvmm.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c index ed192ab3728b7..353f10f92b774 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c @@ -21,6 +21,7 @@ */ #include "ummu.h" #include "umem.h" +#include "uvmm.h" #include @@ -41,6 +42,14 @@ nvkm_ummu_sclass(struct nvkm_object *object, int index, } } + if (mmu->func->vmm.user.oclass) { + if (index-- == 0) { + oclass->base = mmu->func->vmm.user; + oclass->ctor = nvkm_uvmm_new; + return 0; + } + } + return -EINVAL; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c new file mode 100644 index 0000000000000..fa81d0c1ba415 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c @@ -0,0 +1,352 @@ +/* + * Copyright 2017 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "uvmm.h" +#include "umem.h" +#include "ummu.h" + +#include +#include + +#include +#include + +static const struct nvkm_object_func nvkm_uvmm; +struct nvkm_vmm * +nvkm_uvmm_search(struct nvkm_client *client, u64 handle) +{ + struct nvkm_object *object; + + object = nvkm_object_search(client, handle, &nvkm_uvmm); + if (IS_ERR(object)) + return (void *)object; + + return nvkm_uvmm(object)->vmm; +} + +static int +nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + struct nvkm_client *client = uvmm->object.client; + union { + struct nvif_vmm_unmap_v0 v0; + } *args = argv; + struct nvkm_vmm *vmm = uvmm->vmm; + struct nvkm_vma *vma; + int ret = -ENOSYS; + u64 addr; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + addr = args->v0.addr; + } else + return ret; + + mutex_lock(&vmm->mutex); + vma = nvkm_vmm_node_search(vmm, addr); + if (ret = -ENOENT, !vma || vma->addr != addr) { + VMM_DEBUG(vmm, "lookup %016llx: %016llx", + addr, vma ? vma->addr : ~0ULL); + goto done; + } + + if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { + VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, + vma->user, !client->super, vma->busy); + goto done; + } + + if (ret = -EINVAL, !vma->memory) { + VMM_DEBUG(vmm, "unmapped"); + goto done; + } + + nvkm_vmm_unmap_locked(vmm, vma); + ret = 0; +done: + mutex_unlock(&vmm->mutex); + return ret; +} + +static int +nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + struct nvkm_client *client = uvmm->object.client; + union { + struct nvif_vmm_map_v0 v0; + } *args = argv; + u64 addr, size, handle, offset; + struct nvkm_vmm *vmm = uvmm->vmm; + struct nvkm_vma *vma; + struct nvkm_memory *memory; + int ret = -ENOSYS; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) { + addr = args->v0.addr; + size = args->v0.size; + handle = args->v0.memory; + offset = args->v0.offset; + } else + return ret; + + if (IS_ERR((memory = nvkm_umem_search(client, handle)))) { + VMM_DEBUG(vmm, "memory %016llx %ld\n", handle, PTR_ERR(memory)); + return PTR_ERR(memory); + } + + mutex_lock(&vmm->mutex); + if (ret = -ENOENT, !(vma = nvkm_vmm_node_search(vmm, addr))) { + VMM_DEBUG(vmm, "lookup %016llx", addr); + goto fail; + } + + if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { + VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, + vma->user, !client->super, vma->busy); + goto fail; + } + + if (ret = -EINVAL, vma->addr != addr || vma->size != size) { + if (addr + size > vma->addr + vma->size || vma->memory || + (vma->refd == NVKM_VMA_PAGE_NONE && !vma->mapref)) { + VMM_DEBUG(vmm, "split %d %d %d " + "%016llx %016llx %016llx %016llx", + !!vma->memory, vma->refd, vma->mapref, + addr, size, vma->addr, (u64)vma->size); + goto fail; + } + + if (vma->addr != addr) { + const u64 tail = vma->size + vma->addr - addr; + if (ret = -ENOMEM, !(vma = nvkm_vma_tail(vma, tail))) + goto fail; + vma->part = true; + nvkm_vmm_node_insert(vmm, vma); + } + + if (vma->size != size) { + const u64 tail = vma->size - size; + struct nvkm_vma *tmp; + if (ret = -ENOMEM, !(tmp = nvkm_vma_tail(vma, tail))) { + nvkm_vmm_unmap_region(vmm, vma); + goto fail; + } + tmp->part = true; + nvkm_vmm_node_insert(vmm, tmp); + } + } + vma->busy = true; + mutex_unlock(&vmm->mutex); + + ret = nvkm_memory_map(memory, offset, vmm, vma, argv, argc); + if (ret == 0) { + /* Successful map will clear vma->busy. */ + nvkm_memory_unref(&memory); + return 0; + } + + mutex_lock(&vmm->mutex); + vma->busy = false; + nvkm_vmm_unmap_region(vmm, vma); +fail: + mutex_unlock(&vmm->mutex); + nvkm_memory_unref(&memory); + return ret; +} + +static int +nvkm_uvmm_mthd_put(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + struct nvkm_client *client = uvmm->object.client; + union { + struct nvif_vmm_put_v0 v0; + } *args = argv; + struct nvkm_vmm *vmm = uvmm->vmm; + struct nvkm_vma *vma; + int ret = -ENOSYS; + u64 addr; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + addr = args->v0.addr; + } else + return ret; + + mutex_lock(&vmm->mutex); + vma = nvkm_vmm_node_search(vmm, args->v0.addr); + if (ret = -ENOENT, !vma || vma->addr != addr || vma->part) { + VMM_DEBUG(vmm, "lookup %016llx: %016llx %d", addr, + vma ? vma->addr : ~0ULL, vma ? vma->part : 0); + goto done; + } + + if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { + VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, + vma->user, !client->super, vma->busy); + goto done; + } + + nvkm_vmm_put_locked(vmm, vma); + ret = 0; +done: + mutex_unlock(&vmm->mutex); + return ret; +} + +static int +nvkm_uvmm_mthd_get(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + struct nvkm_client *client = uvmm->object.client; + union { + struct nvif_vmm_get_v0 v0; + } *args = argv; + struct nvkm_vmm *vmm = uvmm->vmm; + struct nvkm_vma *vma; + int ret = -ENOSYS; + bool getref, mapref, sparse; + u8 page, align; + u64 size; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + getref = args->v0.type == NVIF_VMM_GET_V0_PTES; + mapref = args->v0.type == NVIF_VMM_GET_V0_ADDR; + sparse = args->v0.sparse; + page = args->v0.page; + align = args->v0.align; + size = args->v0.size; + } else + return ret; + + mutex_lock(&vmm->mutex); + ret = nvkm_vmm_get_locked(vmm, getref, mapref, sparse, + page, align, size, &vma); + mutex_unlock(&vmm->mutex); + if (ret) + return ret; + + args->v0.addr = vma->addr; + vma->user = !client->super; + return ret; +} + +static int +nvkm_uvmm_mthd_page(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + union { + struct nvif_vmm_page_v0 v0; + } *args = argv; + const struct nvkm_vmm_page *page; + int ret = -ENOSYS; + u8 type, index, nr; + + page = uvmm->vmm->func->page; + for (nr = 0; page[nr].shift; nr++); + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + if ((index = args->v0.index) >= nr) + return -EINVAL; + type = page[index].type; + args->v0.shift = page[index].shift; + args->v0.sparse = !!(type & NVKM_VMM_PAGE_SPARSE); + args->v0.vram = !!(type & NVKM_VMM_PAGE_VRAM); + args->v0.host = !!(type & NVKM_VMM_PAGE_HOST); + args->v0.comp = !!(type & NVKM_VMM_PAGE_COMP); + } else + return -ENOSYS; + + return 0; +} + +static int +nvkm_uvmm_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) +{ + struct nvkm_uvmm *uvmm = nvkm_uvmm(object); + switch (mthd) { + case NVIF_VMM_V0_PAGE : return nvkm_uvmm_mthd_page (uvmm, argv, argc); + case NVIF_VMM_V0_GET : return nvkm_uvmm_mthd_get (uvmm, argv, argc); + case NVIF_VMM_V0_PUT : return nvkm_uvmm_mthd_put (uvmm, argv, argc); + case NVIF_VMM_V0_MAP : return nvkm_uvmm_mthd_map (uvmm, argv, argc); + case NVIF_VMM_V0_UNMAP : return nvkm_uvmm_mthd_unmap (uvmm, argv, argc); + default: + break; + } + return -EINVAL; +} + +static void * +nvkm_uvmm_dtor(struct nvkm_object *object) +{ + struct nvkm_uvmm *uvmm = nvkm_uvmm(object); + nvkm_vmm_unref(&uvmm->vmm); + return uvmm; +} + +static const struct nvkm_object_func +nvkm_uvmm = { + .dtor = nvkm_uvmm_dtor, + .mthd = nvkm_uvmm_mthd, +}; + +int +nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, + struct nvkm_object **pobject) +{ + struct nvkm_mmu *mmu = nvkm_ummu(oclass->parent)->mmu; + const bool more = oclass->base.maxver >= 0; + union { + struct nvif_vmm_v0 v0; + } *args = argv; + const struct nvkm_vmm_page *page; + struct nvkm_uvmm *uvmm; + int ret = -ENOSYS; + u64 addr, size; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, more))) { + addr = args->v0.addr; + size = args->v0.size; + } else + return ret; + + if (!(uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL))) + return -ENOMEM; + nvkm_object_ctor(&nvkm_uvmm, oclass, &uvmm->object); + *pobject = &uvmm->object; + + if (!mmu->vmm) { + ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc, + NULL, "user", &uvmm->vmm); + if (ret) + return ret; + + uvmm->vmm->debug = max(uvmm->vmm->debug, oclass->client->debug); + } else { + if (size) + return -EINVAL; + + uvmm->vmm = nvkm_vmm_ref(mmu->vmm); + } + + page = uvmm->vmm->func->page; + args->v0.page_nr = 0; + while (page && (page++)->shift) + args->v0.page_nr++; + args->v0.addr = uvmm->vmm->start; + args->v0.size = uvmm->vmm->limit; + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.h new file mode 100644 index 0000000000000..71dab55e18a9e --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.h @@ -0,0 +1,14 @@ +#ifndef __NVKM_UVMM_H__ +#define __NVKM_UVMM_H__ +#define nvkm_uvmm(p) container_of((p), struct nvkm_uvmm, object) +#include +#include "vmm.h" + +struct nvkm_uvmm { + struct nvkm_object object; + struct nvkm_vmm *vmm; +}; + +int nvkm_uvmm_new(const struct nvkm_oclass *, void *argv, u32 argc, + struct nvkm_object **); +#endif From 359088d5b8ec06196a9ea53e7e59167514220465 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 134/151] drm/nouveau: remove trivial cases of nvxx_device() usage Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_abi16.c | 30 ++++++++++--------- drivers/gpu/drm/nouveau/nouveau_bo.c | 38 +++++++++---------------- 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c index 4c4e016aba951..ece650a0c5f9f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_abi16.c +++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c @@ -185,29 +185,33 @@ nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS) getparam->value = device->info.chipset; break; case NOUVEAU_GETPARAM_PCI_VENDOR: - if (nvxx_device(device)->func->pci) + if (device->info.platform != NV_DEVICE_INFO_V0_SOC) getparam->value = dev->pdev->vendor; else getparam->value = 0; break; case NOUVEAU_GETPARAM_PCI_DEVICE: - if (nvxx_device(device)->func->pci) + if (device->info.platform != NV_DEVICE_INFO_V0_SOC) getparam->value = dev->pdev->device; else getparam->value = 0; break; case NOUVEAU_GETPARAM_BUS_TYPE: - if (!nvxx_device(device)->func->pci) - getparam->value = 3; - else - if (pci_find_capability(dev->pdev, PCI_CAP_ID_AGP)) - getparam->value = 0; - else - if (!pci_is_pcie(dev->pdev)) - getparam->value = 1; - else - getparam->value = 2; - break; + switch (device->info.platform) { + case NV_DEVICE_INFO_V0_AGP : getparam->value = 0; break; + case NV_DEVICE_INFO_V0_PCI : getparam->value = 1; break; + case NV_DEVICE_INFO_V0_PCIE: getparam->value = 2; break; + case NV_DEVICE_INFO_V0_SOC : getparam->value = 3; break; + case NV_DEVICE_INFO_V0_IGP : + if (!pci_is_pcie(dev->pdev)) + getparam->value = 1; + else + getparam->value = 2; + break; + default: + WARN_ON(1); + break; + } case NOUVEAU_GETPARAM_FB_SIZE: getparam->value = drm->gem.vram_available; break; diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index f4545773756a3..5ccbd98bfafde 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -50,8 +50,7 @@ nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg, { struct nouveau_drm *drm = nouveau_drm(dev); int i = reg - drm->tile.reg; - struct nvkm_device *device = nvxx_device(&drm->client.device); - struct nvkm_fb *fb = device->fb; + struct nvkm_fb *fb = nvxx_fb(&drm->client.device); struct nvkm_fb_tile *tile = &fb->tile.region[i]; nouveau_fence_unref(®->fence); @@ -459,7 +458,6 @@ void nouveau_bo_sync_for_device(struct nouveau_bo *nvbo) { struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); - struct nvkm_device *device = nvxx_device(&drm->client.device); struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm; int i; @@ -471,7 +469,8 @@ nouveau_bo_sync_for_device(struct nouveau_bo *nvbo) return; for (i = 0; i < ttm_dma->ttm.num_pages; i++) - dma_sync_single_for_device(device->dev, ttm_dma->dma_address[i], + dma_sync_single_for_device(drm->dev->dev, + ttm_dma->dma_address[i], PAGE_SIZE, DMA_TO_DEVICE); } @@ -479,7 +478,6 @@ void nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo) { struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); - struct nvkm_device *device = nvxx_device(&drm->client.device); struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm; int i; @@ -491,7 +489,7 @@ nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo) return; for (i = 0; i < ttm_dma->ttm.num_pages; i++) - dma_sync_single_for_cpu(device->dev, ttm_dma->dma_address[i], + dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i], PAGE_SIZE, DMA_FROM_DEVICE); } @@ -1468,9 +1466,7 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm) { struct ttm_dma_tt *ttm_dma = (void *)ttm; struct nouveau_drm *drm; - struct nvkm_device *device; - struct drm_device *dev; - struct device *pdev; + struct device *dev; unsigned i; int r; bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); @@ -1487,9 +1483,7 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm) } drm = nouveau_bdev(ttm->bdev); - device = nvxx_device(&drm->client.device); - dev = drm->dev; - pdev = device->dev; + dev = drm->dev->dev; #if IS_ENABLED(CONFIG_AGP) if (drm->agp.bridge) { @@ -1499,7 +1493,7 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm) #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86) if (swiotlb_nr_tbl()) { - return ttm_dma_populate((void *)ttm, dev->dev); + return ttm_dma_populate((void *)ttm, dev); } #endif @@ -1511,12 +1505,12 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm) for (i = 0; i < ttm->num_pages; i++) { dma_addr_t addr; - addr = dma_map_page(pdev, ttm->pages[i], 0, PAGE_SIZE, + addr = dma_map_page(dev, ttm->pages[i], 0, PAGE_SIZE, DMA_BIDIRECTIONAL); - if (dma_mapping_error(pdev, addr)) { + if (dma_mapping_error(dev, addr)) { while (i--) { - dma_unmap_page(pdev, ttm_dma->dma_address[i], + dma_unmap_page(dev, ttm_dma->dma_address[i], PAGE_SIZE, DMA_BIDIRECTIONAL); ttm_dma->dma_address[i] = 0; } @@ -1534,9 +1528,7 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm) { struct ttm_dma_tt *ttm_dma = (void *)ttm; struct nouveau_drm *drm; - struct nvkm_device *device; - struct drm_device *dev; - struct device *pdev; + struct device *dev; unsigned i; bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); @@ -1544,9 +1536,7 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm) return; drm = nouveau_bdev(ttm->bdev); - device = nvxx_device(&drm->client.device); - dev = drm->dev; - pdev = device->dev; + dev = drm->dev->dev; #if IS_ENABLED(CONFIG_AGP) if (drm->agp.bridge) { @@ -1557,14 +1547,14 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm) #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86) if (swiotlb_nr_tbl()) { - ttm_dma_unpopulate((void *)ttm, dev->dev); + ttm_dma_unpopulate((void *)ttm, dev); return; } #endif for (i = 0; i < ttm->num_pages; i++) { if (ttm_dma->dma_address[i]) { - dma_unmap_page(pdev, ttm_dma->dma_address[i], PAGE_SIZE, + dma_unmap_page(dev, ttm_dma->dma_address[i], PAGE_SIZE, DMA_BIDIRECTIONAL); } } From 01670a79d5fa2f6659d18af6d52cca6c44f73646 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 135/151] drm/nouveau: allocate mmu object for every client Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_drm.c | 20 ++++++++++++++++++++ drivers/gpu/drm/nouveau/nouveau_drv.h | 2 ++ 2 files changed, 22 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 25a47e8ba42b5..2eddf19abb409 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -116,6 +116,7 @@ nouveau_cli_fini(struct nouveau_cli *cli) { usif_client_fini(cli); nouveau_vmm_fini(&cli->vmm); + nvif_mmu_fini(&cli->mmu); nvif_device_fini(&cli->device); mutex_lock(&cli->drm->master.lock); nvif_client_fini(&cli->base); @@ -126,6 +127,13 @@ static int nouveau_cli_init(struct nouveau_drm *drm, const char *sname, struct nouveau_cli *cli) { + static const struct nvif_mclass + mmus[] = { + { NVIF_CLASS_MMU_GF100, -1 }, + { NVIF_CLASS_MMU_NV50 , -1 }, + { NVIF_CLASS_MMU_NV04 , -1 }, + {} + }; u64 device = nouveau_name(drm->dev); int ret; @@ -160,6 +168,18 @@ nouveau_cli_init(struct nouveau_drm *drm, const char *sname, goto done; } + ret = nvif_mclass(&cli->device.object, mmus); + if (ret < 0) { + NV_ERROR(drm, "No supported MMU class\n"); + goto done; + } + + ret = nvif_mmu_init(&cli->device.object, mmus[ret].oclass, &cli->mmu); + if (ret) { + NV_ERROR(drm, "MMU allocation failed: %d\n", ret); + goto done; + } + done: if (ret) nouveau_cli_fini(cli); diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index ba8a9ab205629..ac5c66e60b2ab 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -42,6 +42,7 @@ #include #include #include +#include #include @@ -91,6 +92,7 @@ struct nouveau_cli { struct mutex mutex; struct nvif_device device; + struct nvif_mmu mmu; struct nouveau_vmm vmm; struct nvkm_vm *vm; From a220dd73215be28285f6c90355abbe9e21974c77 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 136/151] drm/nouveau: check kind validity against mmu object This is already handled in the top-level gem_new() ioctl in another manner, but this will be removed in a future commit. Ideally we'd not need to check up-front at all, and let the VMM code handle error checking, but there are paths in the current BO management code where this isn't possible due to map() not always being called during BO creation, and map() calls not being allowed to fail during buffer migration. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bo.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 5ccbd98bfafde..8aa2bc6f78fb1 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -189,6 +189,7 @@ nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, { struct nouveau_drm *drm = cli->drm; struct nouveau_bo *nvbo; + struct nvif_mmu *mmu = &cli->mmu; size_t acc_size; int ret; int type = ttm_bo_type_device; @@ -215,11 +216,20 @@ nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) { nvbo->kind = (tile_flags & 0x0000ff00) >> 8; - nvbo->comp = gf100_pte_storage_type_map[nvbo->kind] != nvbo->kind; + if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { + kfree(nvbo); + return -EINVAL; + } + + nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind; } else if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { nvbo->kind = (tile_flags & 0x00007f00) >> 8; nvbo->comp = (tile_flags & 0x00030000) >> 16; + if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { + kfree(nvbo); + return -EINVAL; + } } else { nvbo->zeta = (tile_flags & 0x00000007); } @@ -232,7 +242,7 @@ nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, nvbo->page = drm->client.vm->mmu->lpg_shift; else { if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) - nvbo->kind = gf100_pte_storage_type_map[nvbo->kind]; + nvbo->kind = mmu->kind[nvbo->kind]; nvbo->comp = 0; } } From 325a72827c2c000f6f3810a3e680fa1101d94456 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:19 +1000 Subject: [PATCH 137/151] drm/nouveau: consolidate handling of dma mask Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_ttm.c | 31 +------------------ .../gpu/drm/nouveau/nvkm/engine/device/pci.c | 24 +++++++------- .../drm/nouveau/nvkm/engine/device/tegra.c | 2 -- 3 files changed, 13 insertions(+), 44 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index 8c4f45ea62341..3211f78da4dd5 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -253,7 +253,6 @@ nouveau_ttm_init(struct nouveau_drm *drm) struct nvkm_device *device = nvxx_device(&drm->client.device); struct nvkm_pci *pci = device->pci; struct drm_device *dev = drm->dev; - u8 bits; int ret; if (pci && pci->agp.bridge) { @@ -263,34 +262,6 @@ nouveau_ttm_init(struct nouveau_drm *drm) drm->agp.cma = pci->agp.cma; } - bits = nvxx_mmu(&drm->client.device)->dma_bits; - if (nvxx_device(&drm->client.device)->func->pci) { - if (drm->agp.bridge) - bits = 32; - } else if (device->func->tegra) { - struct nvkm_device_tegra *tegra = device->func->tegra(device); - - /* - * If the platform can use a IOMMU, then the addressable DMA - * space is constrained by the IOMMU bit - */ - if (tegra->func->iommu_bit) - bits = min(bits, tegra->func->iommu_bit); - - } - - ret = dma_set_mask(dev->dev, DMA_BIT_MASK(bits)); - if (ret && bits != 32) { - bits = 32; - ret = dma_set_mask(dev->dev, DMA_BIT_MASK(bits)); - } - if (ret) - return ret; - - ret = dma_set_coherent_mask(dev->dev, DMA_BIT_MASK(bits)); - if (ret) - dma_set_coherent_mask(dev->dev, DMA_BIT_MASK(32)); - ret = nouveau_ttm_global_init(drm); if (ret) return ret; @@ -300,7 +271,7 @@ nouveau_ttm_init(struct nouveau_drm *drm) &nouveau_bo_driver, dev->anon_inode->i_mapping, DRM_FILE_PAGE_OFFSET, - bits <= 32 ? true : false); + drm->client.mmu.dmabits <= 32 ? true : false); if (ret) { NV_ERROR(drm, "error initialising bo driver, %d\n", ret); return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c index 74a1ffa425f73..f302d2b5782a1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c @@ -1627,7 +1627,7 @@ nvkm_device_pci_new(struct pci_dev *pci_dev, const char *cfg, const char *dbg, const struct nvkm_device_pci_vendor *pciv; const char *name = NULL; struct nvkm_device_pci *pdev; - int ret; + int ret, bits; ret = pci_enable_device(pci_dev); if (ret) @@ -1679,17 +1679,17 @@ nvkm_device_pci_new(struct pci_dev *pci_dev, const char *cfg, const char *dbg, if (ret) return ret; - /* - * Set a preliminary DMA mask based on the .dma_bits member of the - * MMU subdevice. This allows other subdevices to create DMA mappings - * in their init() or oneinit() methods, which may be called before the - * TTM layer sets the DMA mask definitively. - * This is necessary for platforms where the default DMA mask of 32 - * does not cover any system memory, i.e., when all RAM is > 4 GB. - */ - if (pdev->device.mmu) - dma_set_mask_and_coherent(&pci_dev->dev, - DMA_BIT_MASK(pdev->device.mmu->dma_bits)); + /* Set DMA mask based on capabilities reported by the MMU subdev. */ + if (pdev->device.mmu && !pdev->device.pci->agp.bridge) + bits = pdev->device.mmu->dma_bits; + else + bits = 32; + + ret = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(bits)); + if (ret && bits != 32) { + dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32)); + pdev->device.mmu->dma_bits = 32; + } return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c index d35a32e168d6e..78597da6313ad 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c @@ -309,8 +309,6 @@ nvkm_device_tegra_new(const struct nvkm_device_tegra_func *func, /** * The IOMMU bit defines the upper limit of the GPU-addressable space. - * This will be refined in nouveau_ttm_init but we need to do it early - * for instmem to behave properly */ ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(tdev->func->iommu_bit)); if (ret) From 658c71f4e7cabce658692a81732563e74de0c952 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 138/151] drm/nouveau: fetch memory type indices that we care about for ttm Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_drv.h | 3 +++ drivers/gpu/drm/nouveau/nouveau_ttm.c | 30 ++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index ac5c66e60b2ab..6d6795b5b3efb 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -140,6 +140,9 @@ struct nouveau_drm { struct nouveau_channel *chan; struct nvif_object copy; int mtrr; + int type_vram; + int type_host; + int type_ncoh; } ttm; /* GEM interface support */ diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index 3211f78da4dd5..f64a55421a964 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -252,8 +252,36 @@ nouveau_ttm_init(struct nouveau_drm *drm) { struct nvkm_device *device = nvxx_device(&drm->client.device); struct nvkm_pci *pci = device->pci; + struct nvif_mmu *mmu = &drm->client.mmu; struct drm_device *dev = drm->dev; - int ret; + int typei, ret; + + typei = nvif_mmu_type(mmu, NVIF_MEM_HOST | NVIF_MEM_MAPPABLE | + NVIF_MEM_COHERENT); + if (typei < 0) + return -ENOSYS; + + drm->ttm.type_host = typei; + + typei = nvif_mmu_type(mmu, NVIF_MEM_HOST | NVIF_MEM_MAPPABLE); + if (typei < 0) + return -ENOSYS; + + drm->ttm.type_ncoh = typei; + + if (drm->client.device.info.platform != NV_DEVICE_INFO_V0_SOC && + drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { + typei = nvif_mmu_type(mmu, NVIF_MEM_VRAM | NVIF_MEM_MAPPABLE | + NVIF_MEM_KIND | + NVIF_MEM_COMP | + NVIF_MEM_DISP); + if (typei < 0) + return -ENOSYS; + + drm->ttm.type_vram = typei; + } else { + drm->ttm.type_vram = -1; + } if (pci && pci->agp.bridge) { drm->agp.bridge = pci->agp.bridge; From b34720200bb1334c78aaecb999cc2b18a71b9277 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 139/151] drm/nouveau: use nvif_mmu_type to determine BAR1 caching Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/device.h | 1 - drivers/gpu/drm/nouveau/nouveau_bo.c | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/device.h b/drivers/gpu/drm/nouveau/include/nvif/device.h index bcb9817116177..9a29b7ca31059 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/device.h +++ b/drivers/gpu/drm/nouveau/include/nvif/device.h @@ -58,7 +58,6 @@ u64 nvif_device_time(struct nvif_device *); #define nvxx_bios(a) nvxx_device(a)->bios #define nvxx_fb(a) nvxx_device(a)->fb #define nvxx_mmu(a) nvxx_device(a)->mmu -#define nvxx_bar(a) nvxx_device(a)->bar #define nvxx_gpio(a) nvxx_device(a)->gpio #define nvxx_clk(a) nvxx_device(a)->clk #define nvxx_i2c(a) nvxx_device(a)->i2c diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 8aa2bc6f78fb1..6e7e283e18b59 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -589,6 +589,7 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, struct ttm_mem_type_manager *man) { struct nouveau_drm *drm = nouveau_bdev(bdev); + struct nvif_mmu *mmu = &drm->client.mmu; switch (type) { case TTM_PL_SYSTEM: @@ -605,7 +606,8 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { /* Some BARs do not support being ioremapped WC */ - if (nvxx_bar(&drm->client.device)->iomap_uncached) { + const u8 type = mmu->type[drm->ttm.type_vram].type; + if (type & NVIF_MEM_UNCACHED) { man->available_caching = TTM_PL_FLAG_UNCACHED; man->default_caching = TTM_PL_FLAG_UNCACHED; } From acb16cfa95f1cc6fb4b1b52da27bdbeb079ef115 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 140/151] drm/nouveau: replace use of cpu_coherent with memory types Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bo.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 6e7e283e18b59..3577ab2d5d694 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -211,8 +211,17 @@ nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, nvbo->bo.bdev = &drm->ttm.bdev; nvbo->cli = cli; - if (!nvxx_device(&drm->client.device)->func->cpu_coherent) - nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED; + /* This is confusing, and doesn't actually mean we want an uncached + * mapping, but is what NOUVEAU_GEM_DOMAIN_COHERENT gets translated + * into in nouveau_gem_new(). + */ + if (flags & TTM_PL_FLAG_UNCACHED) { + /* Determine if we can get a cache-coherent map, forcing + * uncached mapping if we can't. + */ + if (mmu->type[drm->ttm.type_host].type & NVIF_MEM_UNCACHED) + nvbo->force_coherent = true; + } if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) { nvbo->kind = (tile_flags & 0x0000ff00) >> 8; From 96da0bcd51964ca708d8de2987ff473a9da4406d Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 141/151] drm/nouveau: allocate vmm object for every client Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_drm.c | 50 ++++++++++++++------------- drivers/gpu/drm/nouveau/nouveau_drv.h | 1 + drivers/gpu/drm/nouveau/nouveau_gem.c | 8 +++-- drivers/gpu/drm/nouveau/nouveau_mem.h | 5 --- drivers/gpu/drm/nouveau/nouveau_vmm.c | 12 +++++-- drivers/gpu/drm/nouveau/nouveau_vmm.h | 4 ++- 6 files changed, 44 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 2eddf19abb409..a6aaddecd8c5b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -134,6 +134,15 @@ nouveau_cli_init(struct nouveau_drm *drm, const char *sname, { NVIF_CLASS_MMU_NV04 , -1 }, {} }; + static const struct nvif_mclass + vmms[] = { + { NVIF_CLASS_VMM_GP100, -1 }, + { NVIF_CLASS_VMM_GM200, -1 }, + { NVIF_CLASS_VMM_GF100, -1 }, + { NVIF_CLASS_VMM_NV50 , -1 }, + { NVIF_CLASS_VMM_NV04 , -1 }, + {} + }; u64 device = nouveau_name(drm->dev); int ret; @@ -180,6 +189,23 @@ nouveau_cli_init(struct nouveau_drm *drm, const char *sname, goto done; } + ret = nvif_mclass(&cli->mmu.object, vmms); + if (ret < 0) { + NV_ERROR(drm, "No supported VMM class\n"); + goto done; + } + + ret = nouveau_vmm_init(cli, vmms[ret].oclass, &cli->vmm); + if (ret) { + NV_ERROR(drm, "VMM allocation failed: %d\n", ret); + goto done; + } + + if (1) { + cli->vm = cli->vmm.vm; + nvxx_client(&cli->base)->vm = cli->vm; + } + done: if (ret) nouveau_cli_fini(cli); @@ -486,20 +512,6 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags) nouveau_vga_init(drm); - if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { - if (!nvxx_device(&drm->client.device)->mmu) { - ret = -ENOSYS; - goto fail_device; - } - - ret = nouveau_vmm_init(&drm->client, 0, &drm->client.vmm); - if (ret) - goto fail_device; - - drm->client.vm = drm->client.vmm.vm; - nvxx_client(&drm->client.base)->vm = drm->client.vm; - } - ret = nouveau_ttm_init(drm); if (ret) goto fail_ttm; @@ -545,7 +557,6 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags) nouveau_ttm_fini(drm); fail_ttm: nouveau_vga_fini(drm); -fail_device: nouveau_cli_fini(&drm->client); nouveau_cli_fini(&drm->master); kfree(drm); @@ -881,15 +892,6 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv) cli->base.super = false; - if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nouveau_vmm_init(cli, 0, &cli->vmm); - if (ret) - goto done; - - cli->vm = cli->vmm.vm; - nvxx_client(&cli->base)->vm = cli->vm; - } - fpriv->driver_priv = cli; mutex_lock(&drm->client.mutex); diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 6d6795b5b3efb..25e0fb73747b1 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -43,6 +43,7 @@ #include #include #include +#include #include diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 9f82b13293f54..a600c4386c72b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -33,6 +33,8 @@ #include "nouveau_gem.h" #include "nouveau_vmm.h" +#include + void nouveau_gem_object_del(struct drm_gem_object *gem) { @@ -69,7 +71,7 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) struct nouveau_vma *vma; int ret; - if (!cli->vm) + if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50) return 0; ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); @@ -131,7 +133,7 @@ nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) struct nouveau_vma *vma; int ret; - if (!cli->vm) + if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50) return; ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); @@ -214,7 +216,7 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, else rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; rep->offset = nvbo->bo.offset; - if (cli->vm) { + if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { vma = nouveau_vma_find(nvbo, &cli->vmm); if (!vma) return -EINVAL; diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.h b/drivers/gpu/drm/nouveau/nouveau_mem.h index 48388c538420e..3e34566116c46 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.h +++ b/drivers/gpu/drm/nouveau/nouveau_mem.h @@ -29,11 +29,6 @@ struct nouveau_mem { struct nvkm_memory memory; }; -enum nvif_vmm_get { - PTES, - LAZY, -}; - int nouveau_mem_new(struct nouveau_cli *, u8 kind, u8 comp, struct ttm_mem_reg *); void nouveau_mem_del(struct ttm_mem_reg *); diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.c b/drivers/gpu/drm/nouveau/nouveau_vmm.c index 6dc14f92b9882..656f43ad012d1 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vmm.c +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.c @@ -114,13 +114,19 @@ nouveau_vma_new(struct nouveau_bo *nvbo, struct nouveau_vmm *vmm, void nouveau_vmm_fini(struct nouveau_vmm *vmm) { - nvkm_vm_ref(NULL, &vmm->vm, NULL); + nvif_vmm_fini(&vmm->vmm); + vmm->cli = NULL; } int nouveau_vmm_init(struct nouveau_cli *cli, s32 oclass, struct nouveau_vmm *vmm) { + int ret = nvif_vmm_init(&cli->mmu, oclass, PAGE_SIZE, 0, NULL, 0, + &vmm->vmm); + if (ret) + return ret; + vmm->cli = cli; - return nvkm_vm_new(nvxx_device(&cli->device), 0, (1ULL << 40), - 0x1000, NULL, &vmm->vm); + vmm->vm = nvkm_uvmm(vmm->vmm.object.priv)->vmm; + return 0; } diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.h b/drivers/gpu/drm/nouveau/nouveau_vmm.h index 86ab599aafb20..c7b9c55bc5353 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vmm.h +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.h @@ -1,6 +1,7 @@ #ifndef __NOUVEAU_VMA_H__ #define __NOUVEAU_VMA_H__ -#include +#include +#include struct nouveau_bo; struct nouveau_mem; @@ -24,6 +25,7 @@ void nouveau_vma_unmap(struct nouveau_vma *); struct nouveau_vmm { struct nouveau_cli *cli; + struct nvif_vmm vmm; struct nvkm_vm *vm; }; From 3c5026395bcdaa956fa5c6ec7571f1a04847c74b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 142/151] drm/nouveau: switch to vmm limit Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_chan.c | 12 +++++------- drivers/gpu/drm/nouveau/nouveau_ttm.c | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index 9fdca7b4cb05b..f3ed5896b8423 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -107,7 +107,6 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, u32 size, struct nouveau_channel **pchan) { struct nouveau_cli *cli = (void *)device->object.client; - struct nvkm_mmu *mmu = nvxx_mmu(device); struct nv_dma_v0 args = {}; struct nouveau_channel *chan; u32 target; @@ -156,7 +155,7 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_VM; args.start = 0; - args.limit = cli->vm->mmu->limit - 1; + args.limit = cli->vmm.vmm.limit - 1; chan->push.addr = chan->push.vma->addr; } else @@ -188,7 +187,7 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_RDWR; args.start = 0; - args.limit = mmu->limit - 1; + args.limit = cli->vmm.vmm.limit - 1; } } @@ -317,7 +316,6 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) struct nvif_device *device = chan->device; struct nouveau_cli *cli = (void *)chan->user.client; struct nouveau_drm *drm = chan->drm; - struct nvkm_mmu *mmu = nvxx_mmu(device); struct nv_dma_v0 args = {}; int ret, i; @@ -342,7 +340,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_VM; args.start = 0; - args.limit = cli->vm->mmu->limit - 1; + args.limit = cli->vmm.vmm.limit - 1; } else { args.target = NV_DMA_V0_TARGET_VRAM; args.access = NV_DMA_V0_ACCESS_RDWR; @@ -359,7 +357,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_VM; args.start = 0; - args.limit = cli->vm->mmu->limit - 1; + args.limit = cli->vmm.vmm.limit - 1; } else if (chan->drm->agp.bridge) { args.target = NV_DMA_V0_TARGET_AGP; @@ -371,7 +369,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) args.target = NV_DMA_V0_TARGET_VM; args.access = NV_DMA_V0_ACCESS_RDWR; args.start = 0; - args.limit = mmu->limit - 1; + args.limit = cli->vmm.vmm.limit - 1; } ret = nvif_object_init(&chan->user, gart, NV_DMA_IN_MEMORY, diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index f64a55421a964..15c1091cccb5c 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -323,7 +323,7 @@ nouveau_ttm_init(struct nouveau_drm *drm) /* GART init */ if (!drm->agp.bridge) { - drm->gem.gart_available = nvxx_mmu(&drm->client.device)->limit; + drm->gem.gart_available = drm->client.vmm.vmm.limit; } else { drm->gem.gart_available = drm->agp.size; } From 832ca2ac3c95feb01e53c24fcd1a00b4cf05cbdc Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 143/151] drm/nouveau: pass handle of vmm object to channel allocation ioctls Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/cl506e.h | 2 +- drivers/gpu/drm/nouveau/include/nvif/cl506f.h | 2 +- drivers/gpu/drm/nouveau/include/nvif/cl826e.h | 2 +- drivers/gpu/drm/nouveau/include/nvif/cl826f.h | 2 +- drivers/gpu/drm/nouveau/include/nvif/cl906f.h | 2 +- drivers/gpu/drm/nouveau/include/nvif/cla06f.h | 2 +- drivers/gpu/drm/nouveau/nouveau_chan.c | 7 ++++--- drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 12 +++++++----- drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c | 7 +++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/channv50.c | 7 +++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/channv50.h | 4 ++-- drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c | 6 +++--- .../gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c | 6 +++--- .../gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c | 8 +++++--- .../gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c | 14 ++++++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c | 6 +++--- 17 files changed, 54 insertions(+), 41 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl506e.h b/drivers/gpu/drm/nouveau/include/nvif/cl506e.h index aa94b8cf9679e..f508660110029 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl506e.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl506e.h @@ -5,7 +5,7 @@ struct nv50_channel_dma_v0 { __u8 version; __u8 chid; __u8 pad02[6]; - __u64 vm; + __u64 vmm; __u64 pushbuf; __u64 offset; }; diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl506f.h b/drivers/gpu/drm/nouveau/include/nvif/cl506f.h index 3b7101966de43..0e5bbb5531583 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl506f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl506f.h @@ -8,6 +8,6 @@ struct nv50_channel_gpfifo_v0 { __u32 ilength; __u64 ioffset; __u64 pushbuf; - __u64 vm; + __u64 vmm; }; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl826e.h b/drivers/gpu/drm/nouveau/include/nvif/cl826e.h index 91e33db21a2fe..7f6a8ce5a418d 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl826e.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl826e.h @@ -5,7 +5,7 @@ struct g82_channel_dma_v0 { __u8 version; __u8 chid; __u8 pad02[6]; - __u64 vm; + __u64 vmm; __u64 pushbuf; __u64 offset; }; diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl826f.h b/drivers/gpu/drm/nouveau/include/nvif/cl826f.h index e34efd4ec5371..c4d35522331a4 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl826f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl826f.h @@ -8,7 +8,7 @@ struct g82_channel_gpfifo_v0 { __u32 ilength; __u64 ioffset; __u64 pushbuf; - __u64 vm; + __u64 vmm; }; #define NV826F_V0_NTFY_NON_STALL_INTERRUPT 0x00 diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl906f.h b/drivers/gpu/drm/nouveau/include/nvif/cl906f.h index a2d5410a491b8..169161c1587f7 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl906f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl906f.h @@ -7,7 +7,7 @@ struct fermi_channel_gpfifo_v0 { __u8 pad02[2]; __u32 ilength; __u64 ioffset; - __u64 vm; + __u64 vmm; }; #define NV906F_V0_NTFY_NON_STALL_INTERRUPT 0x00 diff --git a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h index 2efa3d048bb9d..3e57089526e3a 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h @@ -22,7 +22,7 @@ struct kepler_channel_gpfifo_a_v0 { __u32 engines; __u32 ilength; __u64 ioffset; - __u64 vm; + __u64 vmm; }; #define NVA06F_V0_NTFY_NON_STALL_INTERRUPT 0x00 diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index f3ed5896b8423..34dbcf5f7f4ed 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -205,6 +205,7 @@ static int nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device, u32 engine, struct nouveau_channel **pchan) { + struct nouveau_cli *cli = (void *)device->object.client; static const u16 oclasses[] = { PASCAL_CHANNEL_GPFIFO_A, MAXWELL_CHANNEL_GPFIFO_A, KEPLER_CHANNEL_GPFIFO_B, @@ -236,21 +237,21 @@ nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device, args.kepler.engines = engine; args.kepler.ilength = 0x02000; args.kepler.ioffset = 0x10000 + chan->push.addr; - args.kepler.vm = 0; + args.kepler.vmm = nvif_handle(&cli->vmm.vmm.object); size = sizeof(args.kepler); } else if (oclass[0] >= FERMI_CHANNEL_GPFIFO) { args.fermi.version = 0; args.fermi.ilength = 0x02000; args.fermi.ioffset = 0x10000 + chan->push.addr; - args.fermi.vm = 0; + args.fermi.vmm = nvif_handle(&cli->vmm.vmm.object); size = sizeof(args.fermi); } else { args.nv50.version = 0; args.nv50.ilength = 0x02000; args.nv50.ioffset = 0x10000 + chan->push.addr; args.nv50.pushbuf = nvif_handle(&chan->push.ctxdma); - args.nv50.vm = 0; + args.nv50.vmm = nvif_handle(&cli->vmm.vmm.object); size = sizeof(args.nv50); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index ac94b57777c4a..d83485385934d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -27,7 +27,6 @@ #include #include #include -#include #include struct nvkm_fifo_chan_object { @@ -353,8 +352,8 @@ nvkm_fifo_chan_func = { int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *func, struct nvkm_fifo *fifo, u32 size, u32 align, bool zero, - u64 vm, u64 push, u64 engines, int bar, u32 base, u32 user, - const struct nvkm_oclass *oclass, + u64 hvmm, u64 push, u64 engines, int bar, u32 base, + u32 user, const struct nvkm_oclass *oclass, struct nvkm_fifo_chan *chan) { struct nvkm_client *client = oclass->client; @@ -387,8 +386,11 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *func, } /* channel address space */ - if (!device->mmu->func->vmm.global) { - struct nvkm_vmm *vmm = client->vm; + if (hvmm) { + struct nvkm_vmm *vmm = nvkm_uvmm_search(client, hvmm); + if (IS_ERR(vmm)) + return PTR_ERR(vmm); + if (vmm->mmu != device->mmu) return -EINVAL; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c index 1870310549e86..a5c998fe4485a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c @@ -229,15 +229,18 @@ g84_fifo_chan_func = { }; int -g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push, +g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push, const struct nvkm_oclass *oclass, struct nv50_fifo_chan *chan) { struct nvkm_device *device = fifo->base.engine.subdev.device; int ret; + if (!vmm) + return -EINVAL; + ret = nvkm_fifo_chan_ctor(&g84_fifo_chan_func, &fifo->base, - 0x10000, 0x1000, false, vm, push, + 0x10000, 0x1000, false, vmm, push, (1ULL << NVKM_ENGINE_BSP) | (1ULL << NVKM_ENGINE_CE0) | (1ULL << NVKM_ENGINE_CIPHER) | diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c index 01453279dbc00..85f7dbf53c997 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c @@ -228,15 +228,18 @@ nv50_fifo_chan_func = { }; int -nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push, +nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push, const struct nvkm_oclass *oclass, struct nv50_fifo_chan *chan) { struct nvkm_device *device = fifo->base.engine.subdev.device; int ret; + if (!vmm) + return -EINVAL; + ret = nvkm_fifo_chan_ctor(&nv50_fifo_chan_func, &fifo->base, - 0x10000, 0x1000, false, vm, push, + 0x10000, 0x1000, false, vmm, push, (1ULL << NVKM_ENGINE_DMAOBJ) | (1ULL << NVKM_ENGINE_SW) | (1ULL << NVKM_ENGINE_GR) | diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h index a0c04c6329a83..d853056e040bf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h @@ -17,14 +17,14 @@ struct nv50_fifo_chan { struct nvkm_gpuobj *engn[NVKM_SUBDEV_NR]; }; -int nv50_fifo_chan_ctor(struct nv50_fifo *, u64 vm, u64 push, +int nv50_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, const struct nvkm_oclass *, struct nv50_fifo_chan *); void *nv50_fifo_chan_dtor(struct nvkm_fifo_chan *); void nv50_fifo_chan_fini(struct nvkm_fifo_chan *); void nv50_fifo_chan_engine_dtor(struct nvkm_fifo_chan *, struct nvkm_engine *); void nv50_fifo_chan_object_dtor(struct nvkm_fifo_chan *, int); -int g84_fifo_chan_ctor(struct nv50_fifo *, u64 vm, u64 push, +int g84_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, const struct nvkm_oclass *, struct nv50_fifo_chan *); extern const struct nvkm_fifo_chan_oclass nv50_fifo_dma_oclass; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c index caa9140747525..fc34cddcd2f51 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c @@ -44,9 +44,9 @@ g84_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, nvif_ioctl(parent, "create channel dma size %d\n", size); if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel dma vers %d vm %llx " + nvif_ioctl(parent, "create channel dma vers %d vmm %llx " "pushbuf %llx offset %016llx\n", - args->v0.version, args->v0.vm, args->v0.pushbuf, + args->v0.version, args->v0.vmm, args->v0.pushbuf, args->v0.offset); if (!args->v0.pushbuf) return -EINVAL; @@ -57,7 +57,7 @@ g84_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, return -ENOMEM; *pobject = &chan->base.object; - ret = g84_fifo_chan_ctor(fifo, args->v0.vm, args->v0.pushbuf, + ret = g84_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf, oclass, chan); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c index 480bc3777be59..8043718ad1504 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c @@ -44,9 +44,9 @@ nv50_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, nvif_ioctl(parent, "create channel dma size %d\n", size); if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel dma vers %d vm %llx " + nvif_ioctl(parent, "create channel dma vers %d vmm %llx " "pushbuf %llx offset %016llx\n", - args->v0.version, args->v0.vm, args->v0.pushbuf, + args->v0.version, args->v0.vmm, args->v0.pushbuf, args->v0.offset); if (!args->v0.pushbuf) return -EINVAL; @@ -57,7 +57,7 @@ nv50_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, return -ENOMEM; *pobject = &chan->base.object; - ret = nv50_fifo_chan_ctor(fifo, args->v0.vm, args->v0.pushbuf, + ret = nv50_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf, oclass, chan); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c index 77c2f2a28bf3b..2121f517b1dda 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c @@ -45,10 +45,10 @@ g84_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, nvif_ioctl(parent, "create channel gpfifo size %d\n", size); if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx " + nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " "pushbuf %llx ioffset %016llx " "ilength %08x\n", - args->v0.version, args->v0.vm, args->v0.pushbuf, + args->v0.version, args->v0.vmm, args->v0.pushbuf, args->v0.ioffset, args->v0.ilength); if (!args->v0.pushbuf) return -EINVAL; @@ -59,7 +59,7 @@ g84_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, return -ENOMEM; *pobject = &chan->base.object; - ret = g84_fifo_chan_ctor(fifo, args->v0.vm, args->v0.pushbuf, + ret = g84_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf, oclass, chan); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 78114068c6dd3..75f9632789b33 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -225,10 +225,12 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, nvif_ioctl(parent, "create channel gpfifo size %d\n", size); if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx " + nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " "ioffset %016llx ilength %08x\n", - args->v0.version, args->v0.vm, args->v0.ioffset, + args->v0.version, args->v0.vmm, args->v0.ioffset, args->v0.ilength); + if (!args->v0.vmm) + return -EINVAL; } else return ret; @@ -240,7 +242,7 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, INIT_LIST_HEAD(&chan->head); ret = nvkm_fifo_chan_ctor(&gf100_fifo_gpfifo_func, &fifo->base, - 0x1000, 0x1000, true, args->v0.vm, 0, + 0x1000, 0x1000, true, args->v0.vmm, 0, (1ULL << NVKM_ENGINE_CE0) | (1ULL << NVKM_ENGINE_CE1) | (1ULL << NVKM_ENGINE_GR) | diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 368f5f5138a36..80c87521bebee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -231,7 +231,7 @@ struct gk104_fifo_chan_func { static int gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func, struct gk104_fifo *fifo, u32 *engmask, u16 *chid, - u64 vm, u64 ioffset, u64 ilength, + u64 vmm, u64 ioffset, u64 ilength, const struct nvkm_oclass *oclass, struct nvkm_object **pobject) { @@ -241,6 +241,9 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func, u64 subdevs = 0; u64 usermem; + if (!vmm) + return -EINVAL; + /* Determine which downstream engines are present */ for (i = 0; i < fifo->engine_nr; i++) { struct nvkm_engine *engine = fifo->engine[i].engine; @@ -286,7 +289,7 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func, INIT_LIST_HEAD(&chan->head); ret = nvkm_fifo_chan_ctor(&gk104_fifo_gpfifo_func, &fifo->base, - 0x1000, 0x1000, true, vm, 0, subdevs, + 0x1000, 0x1000, true, vmm, 0, subdevs, 1, fifo->user.bar->addr, 0x200, oclass, &chan->base); if (ret) @@ -358,18 +361,17 @@ gk104_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, nvif_ioctl(parent, "create channel gpfifo size %d\n", size); if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx " + nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " "ioffset %016llx ilength %08x engine %08x\n", - args->v0.version, args->v0.vm, args->v0.ioffset, + args->v0.version, args->v0.vmm, args->v0.ioffset, args->v0.ilength, args->v0.engines); return gk104_fifo_gpfifo_new_(gk104_fifo_gpfifo, fifo, &args->v0.engines, &args->v0.chid, - args->v0.vm, + args->v0.vmm, args->v0.ioffset, args->v0.ilength, oclass, pobject); - } return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c index c5a7de9db2596..d8f28ec1e4a81 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c @@ -45,10 +45,10 @@ nv50_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, nvif_ioctl(parent, "create channel gpfifo size %d\n", size); if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx " + nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " "pushbuf %llx ioffset %016llx " "ilength %08x\n", - args->v0.version, args->v0.vm, args->v0.pushbuf, + args->v0.version, args->v0.vmm, args->v0.pushbuf, args->v0.ioffset, args->v0.ilength); if (!args->v0.pushbuf) return -EINVAL; @@ -59,7 +59,7 @@ nv50_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, return -ENOMEM; *pobject = &chan->base.object; - ret = nv50_fifo_chan_ctor(fifo, args->v0.vm, args->v0.pushbuf, + ret = nv50_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf, oclass, chan); if (ret) return ret; From 7f50762423b4825674ee7fe10fa37abdafdcf754 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 144/151] drm/nouveau: determine memory class for each client Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_drm.c | 16 ++++++++++++++++ drivers/gpu/drm/nouveau/nouveau_drv.h | 1 + 2 files changed, 17 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index a6aaddecd8c5b..e82da06d82f02 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -127,6 +127,13 @@ static int nouveau_cli_init(struct nouveau_drm *drm, const char *sname, struct nouveau_cli *cli) { + static const struct nvif_mclass + mems[] = { + { NVIF_CLASS_MEM_GF100, -1 }, + { NVIF_CLASS_MEM_NV50 , -1 }, + { NVIF_CLASS_MEM_NV04 , -1 }, + {} + }; static const struct nvif_mclass mmus[] = { { NVIF_CLASS_MMU_GF100, -1 }, @@ -201,11 +208,20 @@ nouveau_cli_init(struct nouveau_drm *drm, const char *sname, goto done; } + ret = nvif_mclass(&cli->mmu.object, mems); + if (ret < 0) { + NV_ERROR(drm, "No supported MEM class\n"); + goto done; + } + + cli->mem = &mems[ret]; + if (1) { cli->vm = cli->vmm.vm; nvxx_client(&cli->base)->vm = cli->vm; } + return 0; done: if (ret) nouveau_cli_fini(cli); diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 25e0fb73747b1..df9d9a71dde57 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -95,6 +95,7 @@ struct nouveau_cli { struct nvif_device device; struct nvif_mmu mmu; struct nouveau_vmm vmm; + const struct nvif_mclass *mem; struct nvkm_vm *vm; struct list_head head; From 814a23243bd2aaa9aafe03d85d0502a73be42b58 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 145/151] drm/nouveau: implement per-client delayed workqueue with fence support Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_drm.c | 56 +++++++++++++++++++++++++++ drivers/gpu/drm/nouveau/nouveau_drv.h | 14 +++++++ 2 files changed, 70 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index e82da06d82f02..0ff4d669e372d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -111,9 +111,63 @@ nouveau_name(struct drm_device *dev) return nouveau_platform_name(to_platform_device(dev->dev)); } +static inline bool +nouveau_cli_work_ready(struct dma_fence *fence, bool wait) +{ + if (!dma_fence_is_signaled(fence)) { + if (!wait) + return false; + WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); + } + dma_fence_put(fence); + return true; +} + +static void +nouveau_cli_work_flush(struct nouveau_cli *cli, bool wait) +{ + struct nouveau_cli_work *work, *wtmp; + mutex_lock(&cli->lock); + list_for_each_entry_safe(work, wtmp, &cli->worker, head) { + if (!work->fence || nouveau_cli_work_ready(work->fence, wait)) { + list_del(&work->head); + work->func(work); + } + } + mutex_unlock(&cli->lock); +} + +static void +nouveau_cli_work_fence(struct dma_fence *fence, struct dma_fence_cb *cb) +{ + struct nouveau_cli_work *work = container_of(cb, typeof(*work), cb); + schedule_work(&work->cli->work); +} + +void +nouveau_cli_work_queue(struct nouveau_cli *cli, struct dma_fence *fence, + struct nouveau_cli_work *work) +{ + work->fence = dma_fence_get(fence); + work->cli = cli; + mutex_lock(&cli->lock); + list_add_tail(&work->head, &cli->worker); + mutex_unlock(&cli->lock); + if (dma_fence_add_callback(fence, &work->cb, nouveau_cli_work_fence)) + nouveau_cli_work_fence(fence, &work->cb); +} + +static void +nouveau_cli_work(struct work_struct *w) +{ + struct nouveau_cli *cli = container_of(w, typeof(*cli), work); + nouveau_cli_work_flush(cli, false); +} + static void nouveau_cli_fini(struct nouveau_cli *cli) { + nouveau_cli_work_flush(cli, true); usif_client_fini(cli); nouveau_vmm_fini(&cli->vmm); nvif_mmu_fini(&cli->mmu); @@ -158,6 +212,8 @@ nouveau_cli_init(struct nouveau_drm *drm, const char *sname, mutex_init(&cli->mutex); usif_client_init(cli); + INIT_WORK(&cli->work, nouveau_cli_work); + INIT_LIST_HEAD(&cli->worker); mutex_init(&cli->lock); if (cli == &drm->master) { diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index df9d9a71dde57..68088b361e5ed 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -104,9 +104,23 @@ struct nouveau_cli { struct list_head notifys; char name[32]; + struct work_struct work; + struct list_head worker; struct mutex lock; }; +struct nouveau_cli_work { + void (*func)(struct nouveau_cli_work *); + struct nouveau_cli *cli; + struct list_head head; + + struct dma_fence *fence; + struct dma_fence_cb cb; +}; + +void nouveau_cli_work_queue(struct nouveau_cli *, struct dma_fence *, + struct nouveau_cli_work *); + static inline struct nouveau_cli * nouveau_cli(struct drm_file *fpriv) { From 00d041d087c1715039bf7a610140ffe1af347b55 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 146/151] drm/nouveau: queue delayed unmapping of VMAs on client workqueue VMAs are about to not take references on the VMM they belong to, which means more care is required when handling delayed unmapping. Queuing it on the client workqueue ensures all pending VMA unmaps will have completed before the VMM is destroyed. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_gem.c | 36 ++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index a600c4386c72b..2cf94e0591702 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -90,19 +90,33 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) return ret; } +struct nouveau_gem_object_unmap { + struct nouveau_cli_work work; + struct nouveau_vma *vma; +}; + static void -nouveau_gem_object_delete(void *data) +nouveau_gem_object_delete(struct nouveau_vma *vma) { - struct nouveau_vma *vma = data; nouveau_vma_del(&vma); } +static void +nouveau_gem_object_delete_work(struct nouveau_cli_work *w) +{ + struct nouveau_gem_object_unmap *work = + container_of(w, typeof(*work), work); + nouveau_gem_object_delete(work->vma); + kfree(work); +} + static void nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) { const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM; struct reservation_object *resv = nvbo->bo.resv; struct reservation_object_list *fobj; + struct nouveau_gem_object_unmap *work; struct dma_fence *fence = NULL; fobj = reservation_object_get_list(resv); @@ -117,10 +131,20 @@ nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) else fence = reservation_object_get_excl(nvbo->bo.resv); - if (fence && mapped) - nouveau_fence_work(fence, nouveau_gem_object_delete, vma); - else - nouveau_vma_del(&vma); + if (!fence || !mapped) { + nouveau_gem_object_delete(vma); + return; + } + + if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { + WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); + nouveau_gem_object_delete(vma); + return; + } + + work->work.func = nouveau_gem_object_delete_work; + work->vma = vma; + nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); } void From 10842ba074e95efad1ac7de2f5279910988f3e68 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 147/151] drm/nouveau: remove unused nouveau_fence_work() Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_fence.c | 56 ------------------------- drivers/gpu/drm/nouveau/nouveau_fence.h | 1 - 2 files changed, 57 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c index 3e4412d11571a..503fa94dc06db 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.c +++ b/drivers/gpu/drm/nouveau/nouveau_fence.c @@ -199,62 +199,6 @@ nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_cha WARN_ON(ret); } -struct nouveau_fence_work { - struct work_struct work; - struct dma_fence_cb cb; - void (*func)(void *); - void *data; -}; - -static void -nouveau_fence_work_handler(struct work_struct *kwork) -{ - struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work); - work->func(work->data); - kfree(work); -} - -static void nouveau_fence_work_cb(struct dma_fence *fence, struct dma_fence_cb *cb) -{ - struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb); - - schedule_work(&work->work); -} - -void -nouveau_fence_work(struct dma_fence *fence, - void (*func)(void *), void *data) -{ - struct nouveau_fence_work *work; - - if (dma_fence_is_signaled(fence)) - goto err; - - work = kmalloc(sizeof(*work), GFP_KERNEL); - if (!work) { - /* - * this might not be a nouveau fence any more, - * so force a lazy wait here - */ - WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence, - true, false)); - goto err; - } - - INIT_WORK(&work->work, nouveau_fence_work_handler); - work->func = func; - work->data = data; - - if (dma_fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0) - goto err_free; - return; - -err_free: - kfree(work); -err: - func(data); -} - int nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan) { diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h index d32e58b66a755..c36031aa013e7 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.h +++ b/drivers/gpu/drm/nouveau/nouveau_fence.h @@ -22,7 +22,6 @@ void nouveau_fence_unref(struct nouveau_fence **); int nouveau_fence_emit(struct nouveau_fence *, struct nouveau_channel *); bool nouveau_fence_done(struct nouveau_fence *); -void nouveau_fence_work(struct dma_fence *, void (*)(void *), void *); int nouveau_fence_wait(struct nouveau_fence *, bool lazy, bool intr); int nouveau_fence_sync(struct nouveau_bo *, struct nouveau_channel *, bool exclusive, bool intr); From d7722134b8254bcee6086230723814cddf9ab54b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 148/151] drm/nouveau: switch over to new memory and vmm interfaces Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/device.h | 2 - drivers/gpu/drm/nouveau/nouveau_bo.c | 92 ++++++--- drivers/gpu/drm/nouveau/nouveau_chan.c | 11 + drivers/gpu/drm/nouveau/nouveau_gem.c | 1 + drivers/gpu/drm/nouveau/nouveau_mem.c | 192 ++++++++++-------- drivers/gpu/drm/nouveau/nouveau_mem.h | 21 +- drivers/gpu/drm/nouveau/nouveau_sgdma.c | 2 +- drivers/gpu/drm/nouveau/nouveau_ttm.c | 18 +- drivers/gpu/drm/nouveau/nouveau_vmm.c | 24 ++- drivers/gpu/drm/nouveau/nouveau_vmm.h | 2 - 10 files changed, 209 insertions(+), 156 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/device.h b/drivers/gpu/drm/nouveau/include/nvif/device.h index 9a29b7ca31059..e130af489a473 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/device.h +++ b/drivers/gpu/drm/nouveau/include/nvif/device.h @@ -38,7 +38,6 @@ u64 nvif_device_time(struct nvif_device *); /*XXX*/ #include #include -#include #include #include #include @@ -57,7 +56,6 @@ u64 nvif_device_time(struct nvif_device *); }) #define nvxx_bios(a) nvxx_device(a)->bios #define nvxx_fb(a) nvxx_device(a)->fb -#define nvxx_mmu(a) nvxx_device(a)->mmu #define nvxx_gpio(a) nvxx_device(a)->gpio #define nvxx_clk(a) nvxx_device(a)->clk #define nvxx_i2c(a) nvxx_device(a)->i2c diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 3577ab2d5d694..8cab46ec8637b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -40,6 +40,10 @@ #include "nouveau_mem.h" #include "nouveau_vmm.h" +#include +#include +#include + /* * NV10-NV40 tiling helpers */ @@ -1034,21 +1038,18 @@ nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, { struct nouveau_mem *old_mem = nouveau_mem(&bo->mem); struct nouveau_mem *new_mem = nouveau_mem(reg); - struct nvkm_vm *vmm = drm->client.vm; - u64 size = (u64)reg->num_pages << PAGE_SHIFT; + struct nvif_vmm *vmm = &drm->client.vmm.vmm; int ret; - ret = nvkm_vm_get(vmm, size, old_mem->mem.page, NV_MEM_ACCESS_RW, - &old_mem->vma[0]); + ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0, + old_mem->mem.size, &old_mem->vma[0]); if (ret) return ret; - ret = nvkm_vm_get(vmm, size, new_mem->mem.page, NV_MEM_ACCESS_RW, - &old_mem->vma[1]); - if (ret) { - nvkm_vm_put(&old_mem->vma[0]); - return ret; - } + ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0, + new_mem->mem.size, &old_mem->vma[1]); + if (ret) + goto done; ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]); if (ret) @@ -1057,8 +1058,8 @@ nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]); done: if (ret) { - nvkm_vm_put(&old_mem->vma[1]); - nvkm_vm_put(&old_mem->vma[0]); + nvif_vmm_put(vmm, &old_mem->vma[1]); + nvif_vmm_put(vmm, &old_mem->vma[0]); } return 0; } @@ -1374,7 +1375,6 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) struct nouveau_drm *drm = nouveau_bdev(bdev); struct nvkm_device *device = nvxx_device(&drm->client.device); struct nouveau_mem *mem = nouveau_mem(reg); - int ret; reg->bus.addr = NULL; reg->bus.offset = 0; @@ -1395,7 +1395,7 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) reg->bus.is_iomem = !drm->agp.cma; } #endif - if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || !mem->kind) + if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 || !mem->kind) /* untiled */ break; /* fallthrough, tiled memory */ @@ -1403,20 +1403,40 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) reg->bus.offset = reg->start << PAGE_SHIFT; reg->bus.base = device->func->resource_addr(device, 1); reg->bus.is_iomem = true; - if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { - struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); - int page_shift = 12; - if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_FERMI) - page_shift = mem->mem.page; - - ret = nvkm_vm_get(bar, mem->_mem->size << 12, - page_shift, NV_MEM_ACCESS_RW, - &mem->bar_vma); - if (ret) - return ret; + if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { + union { + struct nv50_mem_map_v0 nv50; + struct gf100_mem_map_v0 gf100; + } args; + u64 handle, length; + u32 argc = 0; + int ret; + + switch (mem->mem.object.oclass) { + case NVIF_CLASS_MEM_NV50: + args.nv50.version = 0; + args.nv50.ro = 0; + args.nv50.kind = mem->kind; + args.nv50.comp = mem->comp; + break; + case NVIF_CLASS_MEM_GF100: + args.gf100.version = 0; + args.gf100.ro = 0; + args.gf100.kind = mem->kind; + break; + default: + WARN_ON(1); + break; + } + + ret = nvif_object_map_handle(&mem->mem.object, + &argc, argc, + &handle, &length); + if (ret != 1) + return ret ? ret : -EINVAL; - nvkm_vm_map(&mem->bar_vma, mem->_mem); - reg->bus.offset = mem->bar_vma.offset; + reg->bus.base = 0; + reg->bus.offset = handle; } break; default: @@ -1428,12 +1448,22 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) static void nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg) { + struct nouveau_drm *drm = nouveau_bdev(bdev); struct nouveau_mem *mem = nouveau_mem(reg); - if (!mem->bar_vma.node) - return; - - nvkm_vm_put(&mem->bar_vma); + if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { + switch (reg->mem_type) { + case TTM_PL_TT: + if (mem->kind) + nvif_object_unmap_handle(&mem->mem.object); + break; + case TTM_PL_VRAM: + nvif_object_unmap_handle(&mem->mem.object); + break; + default: + break; + } + } } static int diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index 34dbcf5f7f4ed..af1116655910c 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -84,6 +84,14 @@ nouveau_channel_del(struct nouveau_channel **pchan) { struct nouveau_channel *chan = *pchan; if (chan) { + struct nouveau_cli *cli = (void *)chan->user.client; + bool super; + + if (cli) { + super = cli->base.super; + cli->base.super = true; + } + if (chan->fence) nouveau_fence(chan->drm)->context_del(chan); nvif_object_fini(&chan->nvsw); @@ -98,6 +106,9 @@ nouveau_channel_del(struct nouveau_channel **pchan) nouveau_bo_unpin(chan->push.buffer); nouveau_bo_ref(NULL, &chan->push.buffer); kfree(chan); + + if (cli) + cli->base.super = super; } *pchan = NULL; } diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 2cf94e0591702..832cf367f071e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -31,6 +31,7 @@ #include "nouveau_ttm.h" #include "nouveau_gem.h" +#include "nouveau_mem.h" #include "nouveau_vmm.h" #include diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c index 17c6efb0ebec2..589a9621db763 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.c +++ b/drivers/gpu/drm/nouveau/nouveau_mem.c @@ -25,19 +25,70 @@ #include +#include +#include +#include +#include +#include +#include + int nouveau_mem_map(struct nouveau_mem *mem, - struct nvkm_vmm *vmm, struct nvkm_vma *vma) + struct nvif_vmm *vmm, struct nvif_vma *vma) { - nvkm_vm_map(vma, mem->_mem); - return 0; + union { + struct nv50_vmm_map_v0 nv50; + struct gf100_vmm_map_v0 gf100; + } args; + u32 argc = 0; + bool super; + int ret; + + switch (vmm->object.oclass) { + case NVIF_CLASS_VMM_NV04: + break; + case NVIF_CLASS_VMM_NV50: + args.nv50.version = 0; + args.nv50.ro = 0; + args.nv50.priv = 0; + args.nv50.kind = mem->kind; + args.nv50.comp = mem->comp; + argc = sizeof(args.nv50); + break; + case NVIF_CLASS_VMM_GF100: + case NVIF_CLASS_VMM_GM200: + case NVIF_CLASS_VMM_GP100: + args.gf100.version = 0; + if (mem->mem.type & NVIF_MEM_VRAM) + args.gf100.vol = 0; + else + args.gf100.vol = 1; + args.gf100.ro = 0; + args.gf100.priv = 0; + args.gf100.kind = mem->kind; + argc = sizeof(args.gf100); + break; + default: + WARN_ON(1); + return -ENOSYS; + } + + super = vmm->object.client->super; + vmm->object.client->super = true; + ret = nvif_vmm_map(vmm, vma->addr, mem->mem.size, &args, argc, + &mem->mem, 0); + vmm->object.client->super = super; + return ret; } void nouveau_mem_fini(struct nouveau_mem *mem) { - nvkm_vm_put(&mem->vma[1]); - nvkm_vm_put(&mem->vma[0]); + nvif_vmm_put(&mem->cli->drm->client.vmm.vmm, &mem->vma[1]); + nvif_vmm_put(&mem->cli->drm->client.vmm.vmm, &mem->vma[0]); + mutex_lock(&mem->cli->drm->master.lock); + nvif_mem_fini(&mem->mem); + mutex_unlock(&mem->cli->drm->master.lock); } int @@ -45,67 +96,79 @@ nouveau_mem_host(struct ttm_mem_reg *reg, struct ttm_dma_tt *tt) { struct nouveau_mem *mem = nouveau_mem(reg); struct nouveau_cli *cli = mem->cli; + struct nouveau_drm *drm = cli->drm; + struct nvif_mmu *mmu = &cli->mmu; + struct nvif_mem_ram_v0 args = {}; + bool super = cli->base.super; + u8 type; + int ret; + + if (mmu->type[drm->ttm.type_host].type & NVIF_MEM_UNCACHED) + type = drm->ttm.type_ncoh; + else + type = drm->ttm.type_host; - if (mem->kind && cli->device.info.chipset == 0x50) + if (mem->kind && !(mmu->type[type].type & NVIF_MEM_KIND)) mem->comp = mem->kind = 0; - if (mem->comp) { - if (cli->device.info.chipset >= 0xc0) - mem->kind = gf100_pte_storage_type_map[mem->kind]; + if (mem->comp && !(mmu->type[type].type & NVIF_MEM_COMP)) { + if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100) + mem->kind = mmu->kind[mem->kind]; mem->comp = 0; } - mem->__mem.size = (reg->num_pages << PAGE_SHIFT) >> 12; - mem->__mem.memtype = (mem->comp << 7) | mem->kind; - if (tt->ttm.sg) mem->__mem.sg = tt->ttm.sg; - else mem->__mem.pages = tt->dma_address; - mem->_mem = &mem->__mem; - mem->mem.page = 12; - mem->_mem->memory = &mem->memory; - return 0; -} - -#include + if (tt->ttm.sg) args.sgl = tt->ttm.sg->sgl; + else args.dma = tt->dma_address; -struct nvkm_vram { - struct nvkm_memory memory; - struct nvkm_ram *ram; - u8 page; - struct nvkm_mm_node *mn; -}; + mutex_lock(&drm->master.lock); + cli->base.super = true; + ret = nvif_mem_init_type(mmu, cli->mem->oclass, type, PAGE_SHIFT, + reg->num_pages << PAGE_SHIFT, + &args, sizeof(args), &mem->mem); + cli->base.super = super; + mutex_unlock(&drm->master.lock); + return ret; +} int nouveau_mem_vram(struct ttm_mem_reg *reg, bool contig, u8 page) { struct nouveau_mem *mem = nouveau_mem(reg); struct nouveau_cli *cli = mem->cli; - struct nvkm_device *device = nvxx_device(&cli->device); + struct nouveau_drm *drm = cli->drm; + struct nvif_mmu *mmu = &cli->mmu; + bool super = cli->base.super; u64 size = ALIGN(reg->num_pages << PAGE_SHIFT, 1 << page); - u8 type; int ret; - mem->mem.page = page; - mem->_mem = &mem->__mem; - - if (cli->device.info.chipset < 0xc0) { - type = nv50_fb_memtype[mem->kind]; - } else { - if (!mem->comp) - mem->kind = gf100_pte_storage_type_map[mem->kind]; - mem->comp = 0; - type = 0x01; + mutex_lock(&drm->master.lock); + cli->base.super = true; + switch (cli->mem->oclass) { + case NVIF_CLASS_MEM_GF100: + ret = nvif_mem_init_type(mmu, cli->mem->oclass, + drm->ttm.type_vram, page, size, + &(struct gf100_mem_v0) { + .contig = contig, + }, sizeof(struct gf100_mem_v0), + &mem->mem); + break; + case NVIF_CLASS_MEM_NV50: + ret = nvif_mem_init_type(mmu, cli->mem->oclass, + drm->ttm.type_vram, page, size, + &(struct nv50_mem_v0) { + .bankswz = mmu->kind[mem->kind] == 2, + .contig = contig, + }, sizeof(struct nv50_mem_v0), + &mem->mem); + break; + default: + ret = -ENOSYS; + WARN_ON(1); + break; } + cli->base.super = super; + mutex_unlock(&drm->master.lock); - ret = nvkm_ram_get(device, NVKM_RAM_MM_NORMAL, type, page, size, - contig, false, &mem->_mem->memory); - if (ret) - return ret; - - mem->_mem->size = size >> NVKM_RAM_MM_SHIFT; - mem->_mem->offset = nvkm_memory_addr(mem->_mem->memory); - mem->_mem->mem = ((struct nvkm_vram *)mem->_mem->memory)->mn; - mem->_mem->memtype = (mem->comp << 7) | mem->kind; - - reg->start = mem->_mem->offset >> PAGE_SHIFT; + reg->start = mem->mem.addr >> PAGE_SHIFT; return ret; } @@ -118,36 +181,6 @@ nouveau_mem_del(struct ttm_mem_reg *reg) reg->mm_node = NULL; } -static enum nvkm_memory_target -nouveau_mem_memory_target(struct nvkm_memory *memory) -{ - struct nouveau_mem *mem = container_of(memory, typeof(*mem), memory); - if (mem->_mem->mem) - return NVKM_MEM_TARGET_VRAM; - return NVKM_MEM_TARGET_HOST; -}; - -static u8 -nouveau_mem_memory_page(struct nvkm_memory *memory) -{ - struct nouveau_mem *mem = container_of(memory, typeof(*mem), memory); - return mem->mem.page; -}; - -static u64 -nouveau_mem_memory_size(struct nvkm_memory *memory) -{ - struct nouveau_mem *mem = container_of(memory, typeof(*mem), memory); - return mem->_mem->size << 12; -} - -static const struct nvkm_memory_func -nouveau_mem_memory = { - .target = nouveau_mem_memory_target, - .page = nouveau_mem_memory_page, - .size = nouveau_mem_memory_size, -}; - int nouveau_mem_new(struct nouveau_cli *cli, u8 kind, u8 comp, struct ttm_mem_reg *reg) @@ -159,7 +192,6 @@ nouveau_mem_new(struct nouveau_cli *cli, u8 kind, u8 comp, mem->cli = cli; mem->kind = kind; mem->comp = comp; - nvkm_memory_ctor(&nouveau_mem_memory, &mem->memory); reg->mm_node = mem; return 0; diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.h b/drivers/gpu/drm/nouveau/nouveau_mem.h index 3e34566116c46..f6d039e738121 100644 --- a/drivers/gpu/drm/nouveau/nouveau_mem.h +++ b/drivers/gpu/drm/nouveau/nouveau_mem.h @@ -1,12 +1,11 @@ #ifndef __NOUVEAU_MEM_H__ #define __NOUVEAU_MEM_H__ -#include -#include -#include - #include struct ttm_dma_tt; +#include +#include + static inline struct nouveau_mem * nouveau_mem(struct ttm_mem_reg *reg) { @@ -17,16 +16,8 @@ struct nouveau_mem { struct nouveau_cli *cli; u8 kind; u8 comp; - struct { - u8 page; - } mem; - struct nvkm_vma vma[2]; - - struct nvkm_mem __mem; - struct nvkm_mem *_mem; - struct nvkm_vma bar_vma; - - struct nvkm_memory memory; + struct nvif_mem mem; + struct nvif_vma vma[2]; }; int nouveau_mem_new(struct nouveau_cli *, u8 kind, u8 comp, @@ -35,5 +26,5 @@ void nouveau_mem_del(struct ttm_mem_reg *); int nouveau_mem_vram(struct ttm_mem_reg *, bool contig, u8 page); int nouveau_mem_host(struct ttm_mem_reg *, struct ttm_dma_tt *); void nouveau_mem_fini(struct nouveau_mem *); -int nouveau_mem_map(struct nouveau_mem *, struct nvkm_vmm *, struct nvkm_vma *); +int nouveau_mem_map(struct nouveau_mem *, struct nvif_vmm *, struct nvif_vma *); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c index 69c606f97784b..941bf33bd2497 100644 --- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c +++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c @@ -35,7 +35,7 @@ nv04_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *reg) if (ret) return ret; - ret = nouveau_mem_map(mem, mem->cli->vm, &mem->vma[0]); + ret = nouveau_mem_map(mem, &mem->cli->vmm.vmm, &mem->vma[0]); if (ret) { nouveau_mem_fini(mem); return ret; diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index 15c1091cccb5c..08b974b304827 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -56,15 +56,6 @@ nouveau_manager_debug(struct ttm_mem_type_manager *man, { } -static void -nouveau_vram_manager_del(struct ttm_mem_type_manager *man, - struct ttm_mem_reg *reg) -{ - struct nvkm_memory *memory = nouveau_mem(reg)->_mem->memory; - nouveau_mem_del(reg); - nvkm_memory_unref(&memory); -} - static int nouveau_vram_manager_new(struct ttm_mem_type_manager *man, struct ttm_buffer_object *bo, @@ -101,7 +92,7 @@ const struct ttm_mem_type_manager_func nouveau_vram_manager = { .init = nouveau_manager_init, .takedown = nouveau_manager_fini, .get_node = nouveau_vram_manager_new, - .put_node = nouveau_vram_manager_del, + .put_node = nouveau_manager_del, .debug = nouveau_manager_debug, }; @@ -121,7 +112,6 @@ nouveau_gart_manager_new(struct ttm_mem_type_manager *man, if (ret) return ret; - mem->_mem = &mem->__mem; reg->start = 0; return 0; } @@ -143,7 +133,6 @@ nv04_gart_manager_new(struct ttm_mem_type_manager *man, struct nouveau_bo *nvbo = nouveau_bo(bo); struct nouveau_drm *drm = nvbo->cli->drm; struct nouveau_mem *mem; - struct nvkm_mmu *mmu = nvxx_mmu(&drm->client.device); int ret; ret = nouveau_mem_new(&drm->master, nvbo->kind, nvbo->comp, reg); @@ -151,8 +140,8 @@ nv04_gart_manager_new(struct ttm_mem_type_manager *man, if (ret) return ret; - ret = nvkm_vm_get(mmu->vmm, reg->num_pages << 12, 12, - NV_MEM_ACCESS_RW, &mem->vma[0]); + ret = nvif_vmm_get(&mem->cli->vmm.vmm, PTES, false, 12, 0, + reg->num_pages << PAGE_SHIFT, &mem->vma[0]); if (ret) { nouveau_mem_del(reg); if (ret == -ENOSPC) { @@ -162,7 +151,6 @@ nv04_gart_manager_new(struct ttm_mem_type_manager *man, return ret; } - mem->_mem = &mem->__mem; reg->start = mem->vma[0].addr >> PAGE_SHIFT; return 0; } diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.c b/drivers/gpu/drm/nouveau/nouveau_vmm.c index 656f43ad012d1..2367ddf195133 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vmm.c +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.c @@ -28,7 +28,7 @@ void nouveau_vma_unmap(struct nouveau_vma *vma) { if (vma->mem) { - nvkm_vm_unmap(&vma->_vma); + nvif_vmm_unmap(&vma->vmm->vmm, vma->addr); vma->mem = NULL; } } @@ -36,7 +36,8 @@ nouveau_vma_unmap(struct nouveau_vma *vma) int nouveau_vma_map(struct nouveau_vma *vma, struct nouveau_mem *mem) { - int ret = nouveau_mem_map(mem, vma->vmm->vm, &vma->_vma); + struct nvif_vma tmp = { .addr = vma->addr }; + int ret = nouveau_mem_map(mem, &vma->vmm->vmm, &tmp); if (ret) return ret; vma->mem = mem; @@ -61,8 +62,10 @@ nouveau_vma_del(struct nouveau_vma **pvma) { struct nouveau_vma *vma = *pvma; if (vma && --vma->refs <= 0) { - if (likely(vma->addr != ~0ULL)) - nvkm_vm_put(&vma->_vma); + if (likely(vma->addr != ~0ULL)) { + struct nvif_vma tmp = { .addr = vma->addr, .size = 1 }; + nvif_vmm_put(&vma->vmm->vmm, &tmp); + } list_del(&vma->head); *pvma = NULL; kfree(*pvma); @@ -75,6 +78,7 @@ nouveau_vma_new(struct nouveau_bo *nvbo, struct nouveau_vmm *vmm, { struct nouveau_mem *mem = nouveau_mem(&nvbo->bo.mem); struct nouveau_vma *vma; + struct nvif_vma tmp; int ret; if ((vma = *pvma = nouveau_vma_find(nvbo, vmm))) { @@ -92,17 +96,17 @@ nouveau_vma_new(struct nouveau_bo *nvbo, struct nouveau_vmm *vmm, if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM && mem->mem.page == nvbo->page) { - ret = nvkm_vm_get(vmm->vm, mem->_mem->size << 12, mem->mem.page, - NV_MEM_ACCESS_RW, &vma->_vma); + ret = nvif_vmm_get(&vmm->vmm, LAZY, false, mem->mem.page, 0, + mem->mem.size, &tmp); if (ret) goto done; - vma->addr = vma->_vma.offset; + vma->addr = tmp.addr; ret = nouveau_vma_map(vma, mem); } else { - ret = nvkm_vm_get(vmm->vm, mem->_mem->size << 12, mem->mem.page, - NV_MEM_ACCESS_RW, &vma->_vma); - vma->addr = vma->_vma.offset; + ret = nvif_vmm_get(&vmm->vmm, PTES, false, mem->mem.page, 0, + mem->mem.size, &tmp); + vma->addr = tmp.addr; } done: diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.h b/drivers/gpu/drm/nouveau/nouveau_vmm.h index c7b9c55bc5353..90de326b0a82c 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vmm.h +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.h @@ -12,8 +12,6 @@ struct nouveau_vma { u64 addr; struct nouveau_mem *mem; - - struct nvkm_vma _vma; }; struct nouveau_vma *nouveau_vma_find(struct nouveau_bo *, struct nouveau_vmm *); From 7dc6a446da7caf5ea5cf48f2b8843366ee8b0979 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 149/151] drm/nouveau: improve selection of GPU page size Enables the use of Pascal's 2MiB pages for larger buffers. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nouveau_bo.c | 48 ++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 8cab46ec8637b..2615912430cc9 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -194,9 +194,10 @@ nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, struct nouveau_drm *drm = cli->drm; struct nouveau_bo *nvbo; struct nvif_mmu *mmu = &cli->mmu; + struct nvif_vmm *vmm = &cli->vmm.vmm; size_t acc_size; - int ret; int type = ttm_bo_type_device; + int ret, i, pi = -1; if (!size) { NV_WARN(drm, "skipped size %016llx\n", size); @@ -249,16 +250,43 @@ nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, nvbo->mode = tile_mode; nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG); - nvbo->page = 12; - if (drm->client.vm) { - if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024) - nvbo->page = drm->client.vm->mmu->lpg_shift; - else { - if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) - nvbo->kind = mmu->kind[nvbo->kind]; - nvbo->comp = 0; - } + /* Determine the desirable target GPU page size for the buffer. */ + for (i = 0; i < vmm->page_nr; i++) { + /* Because we cannot currently allow VMM maps to fail + * during buffer migration, we need to determine page + * size for the buffer up-front, and pre-allocate its + * page tables. + * + * Skip page sizes that can't support needed domains. + */ + if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE && + (flags & TTM_PL_FLAG_VRAM) && !vmm->page[i].vram) + continue; + if ((flags & TTM_PL_FLAG_TT ) && !vmm->page[i].host) + continue; + + /* Select this page size if it's the first that supports + * the potential memory domains, or when it's compatible + * with the requested compression settings. + */ + if (pi < 0 || !nvbo->comp || vmm->page[i].comp) + pi = i; + + /* Stop once the buffer is larger than the current page size. */ + if (size >= 1ULL << vmm->page[i].shift) + break; + } + + if (WARN_ON(pi < 0)) + return -EINVAL; + + /* Disable compression if suitable settings couldn't be found. */ + if (nvbo->comp && !vmm->page[pi].comp) { + if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100) + nvbo->kind = mmu->kind[nvbo->kind]; + nvbo->comp = 0; } + nvbo->page = vmm->page[pi].shift; nouveau_bo_fixup_align(nvbo, flags, &align, &size); nvbo->bo.mem.num_pages = size >> PAGE_SHIFT; From 632b740c5481988152a3a60319aaa49c99577b77 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 150/151] drm/nouveau/mmu: remove old vmm frontend Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/device.h | 2 - .../drm/nouveau/include/nvkm/core/client.h | 2 - .../gpu/drm/nouveau/include/nvkm/core/os.h | 2 - .../gpu/drm/nouveau/include/nvkm/subdev/fb.h | 14 -- .../gpu/drm/nouveau/include/nvkm/subdev/mmu.h | 26 +-- drivers/gpu/drm/nouveau/nouveau_drm.c | 6 - drivers/gpu/drm/nouveau/nouveau_drv.h | 1 - drivers/gpu/drm/nouveau/nouveau_gem.c | 7 - drivers/gpu/drm/nouveau/nouveau_vmm.c | 1 - drivers/gpu/drm/nouveau/nouveau_vmm.h | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c | 6 - .../gpu/drm/nouveau/nvkm/subdev/fb/gf100.c | 10 - .../gpu/drm/nouveau/nvkm/subdev/fb/gf108.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/fb/gk104.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/fb/gm107.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/fb/gm200.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/fb/gp100.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/fb/gp102.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c | 9 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c | 19 -- drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h | 5 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c | 9 - .../drm/nouveau/nvkm/subdev/instmem/gk20a.c | 11 - .../gpu/drm/nouveau/nvkm/subdev/mmu/base.c | 195 ------------------ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c | 78 ++++--- .../gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c | 4 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c | 4 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c | 4 - .../gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c | 4 - .../gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c | 4 - .../gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c | 2 - .../gpu/drm/nouveau/nvkm/subdev/mmu/priv.h | 2 - drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h | 3 - 57 files changed, 40 insertions(+), 427 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/device.h b/drivers/gpu/drm/nouveau/include/nvif/device.h index e130af489a473..b579633b80c0d 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/device.h +++ b/drivers/gpu/drm/nouveau/include/nvif/device.h @@ -63,10 +63,8 @@ u64 nvif_device_time(struct nvif_device *); #define nvxx_therm(a) nvxx_device(a)->therm #define nvxx_volt(a) nvxx_device(a)->volt -#include #include #include -#include #define nvxx_fifo(a) nvxx_device(a)->fifo #define nvxx_gr(a) nvxx_device(a)->gr diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h index fa58efb0d42af..79624f6d0a2b3 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h @@ -16,8 +16,6 @@ struct nvkm_client { void *data; int (*ntfy)(const void *, u32, const void *, u32); - struct nvkm_vm *vm; - struct list_head umem; spinlock_t lock; }; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h index 70c5c47300b40..1f0108fdd24af 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h @@ -1,7 +1,6 @@ #ifndef __NVKM_OS_H__ #define __NVKM_OS_H__ #include -#define nvkm_vmm nvkm_vm #ifdef __BIG_ENDIAN #define ioread16_native ioread16be @@ -21,5 +20,4 @@ iowrite32_native(lower_32_bits(_v), &_p[0]); \ iowrite32_native(upper_32_bits(_v), &_p[1]); \ } while(0) - #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index 4af663d4d3c7d..a00fd2e59215c 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -20,17 +20,6 @@ #define NVKM_RAM_TYPE_VM 0x7f #define NV_MEM_COMP_VM 0x03 -struct nvkm_mem { - struct nvkm_mm_node *mem; - dma_addr_t *pages; - u32 memtype; - u64 offset; - u64 size; - struct sg_table *sg; - - struct nvkm_memory *memory; -}; - struct nvkm_fb_tile { struct nvkm_mm_node *tag; u32 addr; @@ -57,7 +46,6 @@ struct nvkm_fb { struct nvkm_memory *mmu_wr; }; -bool nvkm_fb_memtype_valid(struct nvkm_fb *, u32 memtype); void nvkm_fb_tile_init(struct nvkm_fb *, int region, u32 addr, u32 size, u32 pitch, u32 flags, struct nvkm_fb_tile *); void nvkm_fb_tile_fini(struct nvkm_fb *, int region, struct nvkm_fb_tile *); @@ -163,6 +151,4 @@ struct nvkm_ram_func { int (*prog)(struct nvkm_ram *); void (*tidy)(struct nvkm_ram *); }; - -extern const u8 gf100_pte_storage_type_map[256]; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h index 0de5f8b7096a1..975c42f620a08 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h @@ -1,12 +1,6 @@ #ifndef __NVKM_MMU_H__ #define __NVKM_MMU_H__ #include -struct nvkm_mem; - -struct nvkm_vm_pgt { - struct nvkm_memory *mem[2]; - u32 refcount[2]; -}; struct nvkm_vma { struct list_head head; @@ -24,14 +18,9 @@ struct nvkm_vma { bool busy:1; /* Region busy (for temporarily preventing user access). */ struct nvkm_memory *memory; /* Memory currently mapped into VMA. */ struct nvkm_tags *tags; /* Compression tag reference. */ - - struct nvkm_vma *node; - struct nvkm_vm *vm; - u64 offset; - u32 access; }; -struct nvkm_vm { +struct nvkm_vmm { const struct nvkm_vmm_func *func; struct nvkm_mmu *mmu; const char *name; @@ -56,17 +45,6 @@ struct nvkm_vm { void *nullp; }; -int nvkm_vm_new(struct nvkm_device *, u64 offset, u64 length, u64 mm_offset, - struct lock_class_key *, struct nvkm_vm **); -int nvkm_vm_ref(struct nvkm_vm *, struct nvkm_vm **, struct nvkm_memory *inst); -int nvkm_vm_boot(struct nvkm_vm *, u64 size); -int nvkm_vm_get(struct nvkm_vm *, u64 size, u32 page_shift, u32 access, - struct nvkm_vma *); -void nvkm_vm_put(struct nvkm_vma *); -void nvkm_vm_map(struct nvkm_vma *, struct nvkm_mem *); -void nvkm_vm_map_at(struct nvkm_vma *, u64 offset, struct nvkm_mem *); -void nvkm_vm_unmap(struct nvkm_vma *); - int nvkm_vmm_new(struct nvkm_device *, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *, const char *name, struct nvkm_vmm **); struct nvkm_vmm *nvkm_vmm_ref(struct nvkm_vmm *); @@ -105,9 +83,7 @@ struct nvkm_mmu { const struct nvkm_mmu_func *func; struct nvkm_subdev subdev; - u64 limit; u8 dma_bits; - u8 lpg_shift; int heap_nr; struct { diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 0ff4d669e372d..8d4a5be3b9130 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -271,12 +271,6 @@ nouveau_cli_init(struct nouveau_drm *drm, const char *sname, } cli->mem = &mems[ret]; - - if (1) { - cli->vm = cli->vmm.vm; - nvxx_client(&cli->base)->vm = cli->vm; - } - return 0; done: if (ret) diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 68088b361e5ed..e86b8220a4bb6 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -97,7 +97,6 @@ struct nouveau_cli { struct nouveau_vmm vmm; const struct nvif_mclass *mem; - struct nvkm_vm *vm; struct list_head head; void *abi16; struct list_head objects; diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 832cf367f071e..efc89aaef66a0 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -267,18 +267,11 @@ int nouveau_gem_ioctl_new(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct nouveau_drm *drm = nouveau_drm(dev); struct nouveau_cli *cli = nouveau_cli(file_priv); - struct nvkm_fb *fb = nvxx_fb(&drm->client.device); struct drm_nouveau_gem_new *req = data; struct nouveau_bo *nvbo = NULL; int ret = 0; - if (!nvkm_fb_memtype_valid(fb, req->info.tile_flags)) { - NV_PRINTK(err, cli, "bad page flags: 0x%08x\n", req->info.tile_flags); - return -EINVAL; - } - ret = nouveau_gem_new(cli, req->info.size, req->align, req->info.domain, req->info.tile_mode, req->info.tile_flags, &nvbo); diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.c b/drivers/gpu/drm/nouveau/nouveau_vmm.c index 2367ddf195133..9e2628dd8e4d6 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vmm.c +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.c @@ -131,6 +131,5 @@ nouveau_vmm_init(struct nouveau_cli *cli, s32 oclass, struct nouveau_vmm *vmm) return ret; vmm->cli = cli; - vmm->vm = nvkm_uvmm(vmm->vmm.object.priv)->vmm; return 0; } diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.h b/drivers/gpu/drm/nouveau/nouveau_vmm.h index 90de326b0a82c..5c31f43678d3f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vmm.h +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.h @@ -1,6 +1,5 @@ #ifndef __NOUVEAU_VMA_H__ #define __NOUVEAU_VMA_H__ -#include #include struct nouveau_bo; struct nouveau_mem; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c index 78248e21a5f0e..73b5d46104bd3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c @@ -31,12 +31,6 @@ #include #include -bool -nvkm_fb_memtype_valid(struct nvkm_fb *fb, u32 memtype) -{ - return fb->func->memtype_valid(fb, memtype); -} - void nvkm_fb_tile_fini(struct nvkm_fb *fb, int region, struct nvkm_fb_tile *tile) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c index a308a99ef64e0..47d28c2797078 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c @@ -27,15 +27,6 @@ #include #include -extern const u8 gf100_pte_storage_type_map[256]; - -bool -gf100_fb_memtype_valid(struct nvkm_fb *fb, u32 tile_flags) -{ - u8 memtype = (tile_flags & 0x0000ff00) >> 8; - return likely((gf100_pte_storage_type_map[memtype] != 0xff)); -} - void gf100_fb_intr(struct nvkm_fb *base) { @@ -140,7 +131,6 @@ gf100_fb = { .init_page = gf100_fb_init_page, .intr = gf100_fb_intr, .ram_new = gf100_ram_new, - .memtype_valid = gf100_fb_memtype_valid, .default_bigpage = 17, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c index 4c012fa02de19..4a9f463745b53 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c @@ -32,7 +32,6 @@ gf108_fb = { .init_page = gf100_fb_init_page, .intr = gf100_fb_intr, .ram_new = gf108_ram_new, - .memtype_valid = gf100_fb_memtype_valid, .default_bigpage = 17, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c index 7ff647499acfe..0a6e8eaad42ca 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c @@ -32,7 +32,6 @@ gk104_fb = { .init_page = gf100_fb_init_page, .intr = gf100_fb_intr, .ram_new = gk104_ram_new, - .memtype_valid = gf100_fb_memtype_valid, .default_bigpage = 17, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c index 59308575b1d4f..a7e29b1250941 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c @@ -30,7 +30,6 @@ gk20a_fb = { .init = gf100_fb_init, .init_page = gf100_fb_init_page, .intr = gf100_fb_intr, - .memtype_valid = gf100_fb_memtype_valid, .default_bigpage = 17, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c index 005aa9d6db322..69c876d5d1c11 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c @@ -32,7 +32,6 @@ gm107_fb = { .init_page = gf100_fb_init_page, .intr = gf100_fb_intr, .ram_new = gm107_ram_new, - .memtype_valid = gf100_fb_memtype_valid, .default_bigpage = 17, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c index 1be0b9e98b840..8137e19d32923 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c @@ -65,7 +65,6 @@ gm200_fb = { .init_page = gm200_fb_init_page, .intr = gf100_fb_intr, .ram_new = gm200_ram_new, - .memtype_valid = gf100_fb_memtype_valid, .default_bigpage = 0 /* per-instance. */, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c index e66a0a871043b..12db61e31128c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c @@ -30,7 +30,6 @@ gm20b_fb = { .init = gm200_fb_init, .init_page = gm200_fb_init_page, .intr = gf100_fb_intr, - .memtype_valid = gf100_fb_memtype_valid, .default_bigpage = 0 /* per-instance. */, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c index 98474aec19216..147f69b30cd82 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c @@ -59,7 +59,6 @@ gp100_fb = { .init_page = gm200_fb_init_page, .init_unkn = gp100_fb_init_unkn, .ram_new = gp100_ram_new, - .memtype_valid = gf100_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c index 73b4ae1c73dcf..b84b9861ef269 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c @@ -33,7 +33,6 @@ gp102_fb = { .init = gp100_fb_init, .init_page = gm200_fb_init_page, .ram_new = gp100_ram_new, - .memtype_valid = gf100_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c index f2b1fbf428d51..af8e43979dc11 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c @@ -28,7 +28,6 @@ gp10b_fb = { .init = gm200_fb_init, .init_page = gm200_fb_init_page, .intr = gf100_fb_intr, - .memtype_valid = gf100_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c index 8ff2e5db4571c..c886664533c81 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c @@ -25,14 +25,6 @@ #include "ram.h" #include "regsnv04.h" -bool -nv04_fb_memtype_valid(struct nvkm_fb *fb, u32 tile_flags) -{ - if (!(tile_flags & 0xff00)) - return true; - return false; -} - static void nv04_fb_init(struct nvkm_fb *fb) { @@ -49,7 +41,6 @@ static const struct nvkm_fb_func nv04_fb = { .init = nv04_fb_init, .ram_new = nv04_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c index e8c44f5a3d845..c998b7e96aa3f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c @@ -61,7 +61,6 @@ nv10_fb = { .tile.fini = nv10_fb_tile_fini, .tile.prog = nv10_fb_tile_prog, .ram_new = nv10_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c index 2ae0beb87567d..7b9f04f44af89 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c @@ -33,7 +33,6 @@ nv1a_fb = { .tile.fini = nv10_fb_tile_fini, .tile.prog = nv10_fb_tile_prog, .ram_new = nv1a_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c index 20e47248ba36e..a021d21ff153a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c @@ -93,7 +93,6 @@ nv20_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv20_fb_tile_prog, .ram_new = nv20_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c index dc2965514abe3..7709f5fe9a458 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c @@ -51,7 +51,6 @@ nv25_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv20_fb_tile_prog, .ram_new = nv20_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c index 28b147d8d628b..8aa7826665079 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c @@ -124,7 +124,6 @@ nv30_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv20_fb_tile_prog, .ram_new = nv20_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c index 4e213da6a2155..6e83dcff72e08 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c @@ -53,7 +53,6 @@ nv35_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv20_fb_tile_prog, .ram_new = nv20_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c index 9f029f02088c0..2a07617bb44cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c @@ -53,7 +53,6 @@ nv36_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv20_fb_tile_prog, .ram_new = nv20_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c index 893bc8b3cc803..955160778b5b8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c @@ -59,7 +59,6 @@ nv40_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv20_fb_tile_prog, .ram_new = nv40_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c index c1a79564a2f54..b77f08d34cc35 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c @@ -53,7 +53,6 @@ nv41_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv41_fb_tile_prog, .ram_new = nv41_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c index 06246cce5ec45..b59dc486083de 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c @@ -62,7 +62,6 @@ nv44_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv44_fb_tile_prog, .ram_new = nv44_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c index 3598a1aa65beb..cab7d20fa0398 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c @@ -48,7 +48,6 @@ nv46_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv44_fb_tile_prog, .ram_new = nv44_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c index d1d5fc952e328..a8b0ad4c871db 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c @@ -36,7 +36,6 @@ nv47_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv41_fb_tile_prog, .ram_new = nv41_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c index b3dff3543062f..d0b317bb0252e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c @@ -36,7 +36,6 @@ nv49_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv41_fb_tile_prog, .ram_new = nv49_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c index 4e98210c1b1c7..6a6f0c0860713 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c @@ -34,7 +34,6 @@ nv4e_fb = { .tile.fini = nv20_fb_tile_fini, .tile.prog = nv44_fb_tile_prog, .ram_new = nv44_ram_new, - .memtype_valid = nv04_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c index da797c4525b29..b2f5bf8144eaa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c @@ -28,18 +28,6 @@ #include #include -int -nv50_fb_memtype[0x80] = { - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 2, 2, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 2, 2, 2, 2, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 0, 0, - 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 2, 2, 2, 2, - 1, 0, 2, 0, 1, 0, 2, 0, 1, 1, 2, 2, 1, 1, 0, 0 -}; - static int nv50_fb_ram_new(struct nvkm_fb *base, struct nvkm_ram **pram) { @@ -47,12 +35,6 @@ nv50_fb_ram_new(struct nvkm_fb *base, struct nvkm_ram **pram) return fb->func->ram_new(&fb->base, pram); } -static bool -nv50_fb_memtype_valid(struct nvkm_fb *fb, u32 memtype) -{ - return nv50_fb_memtype[(memtype & 0xff00) >> 8] != 0; -} - static const struct nvkm_enum vm_dispatch_subclients[] = { { 0x00000000, "GRCTX" }, { 0x00000001, "NOTIFY" }, @@ -276,7 +258,6 @@ nv50_fb_ = { .init = nv50_fb_init, .intr = nv50_fb_intr, .ram_new = nv50_fb_ram_new, - .memtype_valid = nv50_fb_memtype_valid, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h index 74da4d7dcaa07..13231d4b00d93 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h @@ -18,5 +18,4 @@ struct nv50_fb_func { int nv50_fb_new_(const struct nv50_fb_func *, struct nvkm_device *, int index, struct nvkm_fb **pfb); -extern int nv50_fb_memtype[0x80]; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h index 329ff5e0115a6..e05d95240e853 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h @@ -25,8 +25,6 @@ struct nvkm_fb_func { int (*ram_new)(struct nvkm_fb *, struct nvkm_ram **); - bool (*memtype_valid)(struct nvkm_fb *, u32 memtype); - u8 default_bigpage; }; @@ -36,8 +34,6 @@ int nvkm_fb_new_(const struct nvkm_fb_func *, struct nvkm_device *device, int index, struct nvkm_fb **); int nvkm_fb_bios_memtype(struct nvkm_bios *); -bool nv04_fb_memtype_valid(struct nvkm_fb *, u32 memtype); - void nv10_fb_tile_init(struct nvkm_fb *, int i, u32 addr, u32 size, u32 pitch, u32 flags, struct nvkm_fb_tile *); void nv10_fb_tile_fini(struct nvkm_fb *, int i, struct nvkm_fb_tile *); @@ -67,7 +63,6 @@ void nv46_fb_tile_init(struct nvkm_fb *, int i, u32 addr, u32 size, int gf100_fb_oneinit(struct nvkm_fb *); int gf100_fb_init_page(struct nvkm_fb *); -bool gf100_fb_memtype_valid(struct nvkm_fb *, u32); int gm200_fb_init_page(struct nvkm_fb *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c index 5305078f730e7..24c7bd5057316 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c @@ -45,15 +45,6 @@ nvkm_vram_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, .mem = vram->mn, }; - if (vma->vm) { - struct nvkm_mem mem = { - .mem = vram->mn, - .memory = &vram->memory, - }; - nvkm_vm_map_at(vma, offset, &mem); - return 0; - } - return nvkm_vmm_map(vmm, vma, argv, argc, &map); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c index 2f60f0d18aeb1..985f2990ab0dd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c @@ -44,9 +44,7 @@ #include "priv.h" #include -#include #include -#include #include #include @@ -290,15 +288,6 @@ gk20a_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, .mem = node->mn, }; - if (vma->vm) { - struct nvkm_mem mem = { - .mem = node->mn, - .memory = &node->memory, - }; - nvkm_vm_map_at(vma, 0, &mem); - return 0; - } - return nvkm_vmm_map(vmm, vma, argv, argc, &map); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c index e0fa0cae79607..ee11ccaf0563c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c @@ -216,67 +216,6 @@ nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero) return pt; } -static void -nvkm_vm_map_(const struct nvkm_vmm_page *page, struct nvkm_vma *vma, u64 delta, - struct nvkm_mem *mem, nvkm_vmm_pte_func fn, - struct nvkm_vmm_map *map) -{ - union { - struct nv50_vmm_map_v0 nv50; - struct gf100_vmm_map_v0 gf100; - } args; - struct nvkm_vmm *vmm = vma->vm; - void *argv = NULL; - u32 argc = 0; - int ret; - - map->memory = mem->memory; - map->page = page; - - if (vmm->func->valid) { - switch (vmm->mmu->subdev.device->card_type) { - case NV_50: - args.nv50.version = 0; - args.nv50.ro = !(vma->access & NV_MEM_ACCESS_WO); - args.nv50.priv = !!(vma->access & NV_MEM_ACCESS_SYS); - args.nv50.kind = (mem->memtype & 0x07f); - args.nv50.comp = (mem->memtype & 0x180) >> 7; - argv = &args.nv50; - argc = sizeof(args.nv50); - break; - case NV_C0: - case NV_E0: - case GM100: - case GP100: { - args.gf100.version = 0; - args.gf100.vol = (nvkm_memory_target(map->memory) != NVKM_MEM_TARGET_VRAM); - args.gf100.ro = !(vma->access & NV_MEM_ACCESS_WO); - args.gf100.priv = !!(vma->access & NV_MEM_ACCESS_SYS); - args.gf100.kind = (mem->memtype & 0x0ff); - argv = &args.gf100; - argc = sizeof(args.gf100); - } - break; - default: - break; - } - - ret = vmm->func->valid(vmm, argv, argc, map); - if (WARN_ON(ret)) - return; - } - - mutex_lock(&vmm->mutex); - nvkm_vmm_ptes_map(vmm, page, vma->node->addr + delta, - vma->node->size, map, fn); - mutex_unlock(&vmm->mutex); - - nvkm_memory_tags_put(vma->node->memory, vmm->mmu->subdev.device, &vma->node->tags); - nvkm_memory_unref(&vma->node->memory); - vma->node->memory = nvkm_memory_ref(map->memory); - vma->node->tags = map->tags; -} - void nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu) { @@ -312,138 +251,6 @@ nvkm_mmu_ptc_init(struct nvkm_mmu *mmu) INIT_LIST_HEAD(&mmu->ptp.list); } -void -nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node) -{ - const struct nvkm_vmm_page *page = &vma->vm->func->page[vma->node->page]; - if (page->desc->func->unmap) { - struct nvkm_vmm_map map = { .mem = node->mem }; - nvkm_vm_map_(page, vma, delta, node, page->desc->func->mem, &map); - return; - } -} - -static void -nvkm_vm_map_sg_table(struct nvkm_vma *vma, u64 delta, u64 length, - struct nvkm_mem *mem) -{ - const struct nvkm_vmm_page *page = &vma->vm->func->page[vma->node->page]; - if (page->desc->func->unmap) { - struct nvkm_vmm_map map = { .sgl = mem->sg->sgl }; - nvkm_vm_map_(page, vma, delta, mem, page->desc->func->sgl, &map); - return; - } -} - -static void -nvkm_vm_map_sg(struct nvkm_vma *vma, u64 delta, u64 length, - struct nvkm_mem *mem) -{ - const struct nvkm_vmm_page *page = &vma->vm->func->page[vma->node->page]; - if (page->desc->func->unmap) { - struct nvkm_vmm_map map = { .dma = mem->pages }; - nvkm_vm_map_(page, vma, delta, mem, page->desc->func->dma, &map); - return; - } -} - -void -nvkm_vm_map(struct nvkm_vma *vma, struct nvkm_mem *node) -{ - if (node->sg) - nvkm_vm_map_sg_table(vma, 0, node->size << 12, node); - else - if (node->pages) - nvkm_vm_map_sg(vma, 0, node->size << 12, node); - else - nvkm_vm_map_at(vma, 0, node); -} - -void -nvkm_vm_unmap(struct nvkm_vma *vma) -{ - nvkm_vmm_unmap(vma->vm, vma->node); -} - -int -nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access, - struct nvkm_vma *vma) -{ - int ret; - - mutex_lock(&vm->mutex); - ret = nvkm_vmm_get_locked(vm, true, false, false, page_shift, 0, - size, &vma->node); - mutex_unlock(&vm->mutex); - if (ret) - return ret; - - vma->memory = NULL; - vma->tags = NULL; - vma->vm = NULL; - nvkm_vm_ref(vm, &vma->vm, NULL); - vma->offset = vma->addr = vma->node->addr; - vma->access = access; - return 0; -} - -void -nvkm_vm_put(struct nvkm_vma *vma) -{ - nvkm_vmm_put(vma->vm, &vma->node); - nvkm_vm_ref(NULL, &vma->vm, NULL); -} - -int -nvkm_vm_boot(struct nvkm_vm *vm, u64 size) -{ - return nvkm_vmm_boot(vm); -} - -int -nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset, - struct lock_class_key *key, struct nvkm_vm **pvm) -{ - struct nvkm_mmu *mmu = device->mmu; - - *pvm = NULL; - if (mmu->func->vmm.ctor) { - int ret = mmu->func->vmm.ctor(mmu, mm_offset, - offset + length - mm_offset, - NULL, 0, key, "legacy", pvm); - if (ret) { - nvkm_vm_ref(NULL, pvm, NULL); - return ret; - } - - return ret; - } - - return -EINVAL; -} - -int -nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_memory *inst) -{ - if (ref) { - if (inst) { - int ret = nvkm_vmm_join(ref, inst); - if (ret) - return ret; - } - - nvkm_vmm_ref(ref); - } - - if (*ptr) { - nvkm_vmm_part(*ptr, inst); - nvkm_vmm_unref(ptr); - } - - *ptr = ref; - return 0; -} - static void nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type) { @@ -611,9 +418,7 @@ nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device, { nvkm_subdev_ctor(&nvkm_mmu, device, index, &mmu->subdev); mmu->func = func; - mmu->limit = func->limit; mmu->dma_bits = func->dma_bits; - mmu->lpg_shift = func->lpg_shift; nvkm_mmu_ptc_init(mmu); mmu->user.ctor = nvkm_ummu_new; mmu->user.base = func->mmu.user; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c index e53f9beda7e8a..8accda5a772b7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c @@ -26,9 +26,7 @@ static const struct nvkm_mmu_func g84_mmu = { - .limit = (1ULL << 40), .dma_bits = 40, - .lpg_shift = 16, .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV50}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_NV50}, nv50_mem_new, nv50_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x0200 }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c index 85d237d3963c7..2d075246dc460 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c @@ -29,54 +29,52 @@ /* Map from compressed to corresponding uncompressed storage type. * The value 0xff represents an invalid storage type. */ -const u8 gf100_pte_storage_type_map[256] = -{ - 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0xff, 0x01, /* 0x00 */ - 0x01, 0x01, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, 0x11, /* 0x10 */ - 0x11, 0x11, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0x27, /* 0x20 */ - 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30 */ - 0xff, 0xff, 0x26, 0x27, 0x28, 0x29, 0x26, 0x27, - 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0x46, 0xff, /* 0x40 */ - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x46, 0x46, 0x46, 0x46, 0xff, 0xff, 0xff, /* 0x50 */ - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60 */ - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70 */ - 0xff, 0xff, 0xff, 0x7b, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0x7b, /* 0x80 */ - 0x7b, 0x7b, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90 */ - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xa7, /* 0xa0 */ - 0xa8, 0xa9, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0 */ - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7, - 0xa8, 0xa9, 0xaa, 0xc3, 0xff, 0xff, 0xff, 0xff, /* 0xc0 */ - 0xff, 0xff, 0xff, 0xff, 0xfe, 0xfe, 0xc3, 0xc3, - 0xc3, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0 */ - 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, - 0xfe, 0xff, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xff, /* 0xe0 */ - 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xfe, 0xff, - 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0 */ - 0xfe, 0xfe, 0xfe, 0xfe, 0xff, 0xfd, 0xfe, 0xff -}; - const u8 * gf100_mmu_kind(struct nvkm_mmu *mmu, int *count) { - *count = ARRAY_SIZE(gf100_pte_storage_type_map); - return gf100_pte_storage_type_map; + static const u8 + kind[256] = { + 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0xff, 0x01, /* 0x00 */ + 0x01, 0x01, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, 0x11, /* 0x10 */ + 0x11, 0x11, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0x27, /* 0x20 */ + 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30 */ + 0xff, 0xff, 0x26, 0x27, 0x28, 0x29, 0x26, 0x27, + 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0x46, 0xff, /* 0x40 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x46, 0x46, 0x46, 0x46, 0xff, 0xff, 0xff, /* 0x50 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70 */ + 0xff, 0xff, 0xff, 0x7b, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0x7b, /* 0x80 */ + 0x7b, 0x7b, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xa7, /* 0xa0 */ + 0xa8, 0xa9, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7, + 0xa8, 0xa9, 0xaa, 0xc3, 0xff, 0xff, 0xff, 0xff, /* 0xc0 */ + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xfe, 0xc3, 0xc3, + 0xc3, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0 */ + 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, + 0xfe, 0xff, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xff, /* 0xe0 */ + 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xfe, 0xff, + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xff, 0xfd, 0xfe, 0xff + }; + + *count = ARRAY_SIZE(kind); + return kind; } static const struct nvkm_mmu_func gf100_mmu = { - .limit = (1ULL << 40), .dma_bits = 40, - .lpg_shift = 17, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gf100_vmm_new }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c index 30c0217ac8073..3d7d1eb1cff90 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c @@ -26,9 +26,7 @@ static const struct nvkm_mmu_func gk104_mmu = { - .limit = (1ULL << 40), .dma_bits = 40, - .lpg_shift = 17, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk104_vmm_new }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c index b7bff4369d308..ac74965a60d4c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c @@ -26,9 +26,7 @@ static const struct nvkm_mmu_func gk20a_mmu = { - .limit = (1ULL << 40), .dma_bits = 40, - .lpg_shift = 17, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk20a_vmm_new }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c index 3cabeabbcea6e..dbf644ebac97b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c @@ -70,9 +70,7 @@ gm200_mmu_kind(struct nvkm_mmu *mmu, int *count) static const struct nvkm_mmu_func gm200_mmu = { - .limit = (1ULL << 40), .dma_bits = 40, - .lpg_shift = 17, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm200_vmm_new }, @@ -82,9 +80,7 @@ gm200_mmu = { static const struct nvkm_mmu_func gm200_mmu_fixed = { - .limit = (1ULL << 40), .dma_bits = 40, - .lpg_shift = 17, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm200_vmm_new_fixed }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c index 802652539d9ad..7353a94b40914 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c @@ -28,9 +28,7 @@ static const struct nvkm_mmu_func gm20b_mmu = { - .limit = (1ULL << 40), .dma_bits = 40, - .lpg_shift = 17, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new }, @@ -40,9 +38,7 @@ gm20b_mmu = { static const struct nvkm_mmu_func gm20b_mmu_fixed = { - .limit = (1ULL << 40), .dma_bits = 40, - .lpg_shift = 17, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new_fixed }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c index 92fc18eec1476..651b8805c67c0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c @@ -28,9 +28,7 @@ static const struct nvkm_mmu_func gp100_mmu = { - .limit = (1ULL << 49), .dma_bits = 47, - .lpg_shift = 16, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c index e1b0f82e2e05d..3bd3db31e0bbb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -28,9 +28,7 @@ static const struct nvkm_mmu_func gp10b_mmu = { - .limit = (1ULL << 49), .dma_bits = 47, - .lpg_shift = 16, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c index 7505a72131567..d201c887c2cd6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c @@ -26,13 +26,9 @@ #include -#define NV04_PDMA_SIZE (128 * 1024 * 1024) - const struct nvkm_mmu_func nv04_mmu = { - .limit = NV04_PDMA_SIZE, .dma_bits = 32, - .lpg_shift = 12, .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV04}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv04_vmm_new, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c index 8e683a0f796d0..adca81895c09e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c @@ -28,8 +28,6 @@ #include -#define NV41_GART_SIZE (512 * 1024 * 1024) - static void nv41_mmu_init(struct nvkm_mmu *mmu) { @@ -42,9 +40,7 @@ nv41_mmu_init(struct nvkm_mmu *mmu) static const struct nvkm_mmu_func nv41_mmu = { .init = nv41_mmu_init, - .limit = NV41_GART_SIZE, .dma_bits = 39, - .lpg_shift = 12, .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV04}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv41_vmm_new, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c index 2fdc4c787b7a8..598c53a27bde4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c @@ -28,8 +28,6 @@ #include -#define NV44_GART_SIZE (512 * 1024 * 1024) - static void nv44_mmu_init(struct nvkm_mmu *mmu) { @@ -57,9 +55,7 @@ nv44_mmu_init(struct nvkm_mmu *mmu) static const struct nvkm_mmu_func nv44_mmu = { .init = nv44_mmu_init, - .limit = NV44_GART_SIZE, .dma_bits = 39, - .lpg_shift = 12, .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV04}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv44_vmm_new, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c index 334f7fb460abb..db3dfbbb2aa08 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c @@ -62,9 +62,7 @@ nv50_mmu_kind(struct nvkm_mmu *base, int *count) static const struct nvkm_mmu_func nv50_mmu = { - .limit = (1ULL << 40), .dma_bits = 40, - .lpg_shift = 16, .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV50}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_NV50}, nv50_mem_new, nv50_mem_map }, .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x1400 }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 9d7ff166e5054..d024d8055fcb1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -11,9 +11,7 @@ int nvkm_mmu_new_(const struct nvkm_mmu_func *, struct nvkm_device *, struct nvkm_mmu_func { void (*init)(struct nvkm_mmu *); - u64 limit; u8 dma_bits; - u8 lpg_shift; struct { struct nvkm_sclass user; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c index 31c4acaf44a05..6446bc61b084a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -700,7 +700,7 @@ nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, func->unmap); } -void +static void nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size, struct nvkm_vmm_map *map, nvkm_vmm_pte_func func) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 5808012f73955..6d8f61ea467af 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -160,9 +160,6 @@ int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref, void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *); void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *); void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma); -void nvkm_vmm_ptes_map(struct nvkm_vmm *, const struct nvkm_vmm_page *, - u64 addr, u64 size, struct nvkm_vmm_map *, - nvkm_vmm_pte_func); struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail); void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *); From 46bda4f4af5d061ce07148e833ff4deb50737536 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Nov 2017 03:56:20 +1000 Subject: [PATCH 151/151] drm/nouveau/gr/gf100-: don't prevent module load if firmware missing Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 4b3d5432a53e2..2f8dc107047dc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1861,8 +1861,12 @@ gf100_gr_ctor_fw(struct gf100_gr *gr, const char *fwname, int ret; ret = nvkm_firmware_get(device, fwname, &fw); - if (ret) - return gf100_gr_ctor_fw_legacy(gr, fwname, fuc, ret); + if (ret) { + ret = gf100_gr_ctor_fw_legacy(gr, fwname, fuc, ret); + if (ret) + return -ENODEV; + return 0; + } fuc->size = fw->size; fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);