drm/nouveau/fifo: add runq

Creates an nvkm_runq for each PBDMA, these will be associated with the
relevant runlist(s) later.

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
Reviewed-by: Lyude Paul <lyude@redhat.com>

drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h

@@ -40,6 +40,8 @@ struct nvkm_fifo {
 	struct nvkm_chid *chid;
 	struct nvkm_chid *cgid;
 
+	struct list_head runqs;
+
 	DECLARE_BITMAP(mask, NVKM_FIFO_CHID_NR);
 	int nr;
 	struct list_head chan;

drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild

@@ -2,6 +2,7 @@
 nvkm-y += nvkm/engine/fifo/base.o
 nvkm-y += nvkm/engine/fifo/chan.o
 nvkm-y += nvkm/engine/fifo/chid.o
+nvkm-y += nvkm/engine/fifo/runq.o
 
 nvkm-y += nvkm/engine/fifo/nv04.o
 nvkm-y += nvkm/engine/fifo/nv10.o

drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c

@@ -24,6 +24,7 @@
 #include "priv.h"
 #include "chan.h"
 #include "chid.h"
+#include "runq.h"
 
 #include <core/gpuobj.h>
 #include <subdev/mc.h>
@@ -235,7 +236,7 @@ static int
 nvkm_fifo_oneinit(struct nvkm_engine *engine)
 {
 	struct nvkm_fifo *fifo = nvkm_fifo(engine);
-	int ret;
+	int ret, nr, i;
 
 	/* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */
 	if (fifo->func->chid_nr) {
@@ -244,6 +245,14 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine)
 			return ret;
 	}
 
+	/* Create runqueues for each PBDMA. */
+	if (fifo->func->runq_nr) {
+		for (nr = fifo->func->runq_nr(fifo), i = 0; i < nr; i++) {
+			if (!nvkm_runq_new(fifo, i))
+				return -ENOMEM;
+		}
+	}
+
 	if (fifo->func->oneinit)
 		return fifo->func->oneinit(fifo);
 
@@ -260,8 +269,12 @@ static void *
 nvkm_fifo_dtor(struct nvkm_engine *engine)
 {
 	struct nvkm_fifo *fifo = nvkm_fifo(engine);
+	struct nvkm_runq *runq, *rtmp;
 	void *data = fifo;
 
+	list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head)
+		nvkm_runq_del(runq);
+
 	nvkm_chid_unref(&fifo->cgid);
 	nvkm_chid_unref(&fifo->chid);
 
@@ -292,6 +305,7 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device,
 	int ret, nr;
 
 	fifo->func = func;
+	INIT_LIST_HEAD(&fifo->runqs);
 	spin_lock_init(&fifo->lock);
 	mutex_init(&fifo->mutex);
 

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c

@@ -23,6 +23,7 @@
  */
 #include "chan.h"
 #include "chid.h"
+#include "runq.h"
 
 #include "gf100.h"
 #include "changf100.h"
@@ -100,6 +101,10 @@ gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit)
 	nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat);
 }
 
+static const struct nvkm_runq_func
+gf100_runq = {
+};
+
 void
 gf100_fifo_runlist_commit(struct gf100_fifo *fifo)
 {
@@ -625,6 +630,18 @@ gf100_fifo_init(struct nvkm_fifo *base)
 	nvkm_wr32(device, 0x002628, 0x00000001); /* ENGINE_INTR_EN */
 }
 
+int
+gf100_fifo_runq_nr(struct nvkm_fifo *fifo)
+{
+	struct nvkm_device *device = fifo->engine.subdev.device;
+	u32 save;
+
+	/* Determine number of PBDMAs by checking valid enable bits. */
+	save = nvkm_mask(device, 0x000204, 0xffffffff, 0xffffffff);
+	save = nvkm_mask(device, 0x000204, 0xffffffff, save);
+	return hweight32(save);
+}
+
 int
 gf100_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr)
 {
@@ -640,11 +657,7 @@ gf100_fifo_oneinit(struct nvkm_fifo *base)
 	struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
 	int ret;
 
-	/* Determine number of PBDMAs by checking valid enable bits. */
-	nvkm_wr32(device, 0x002204, 0xffffffff);
-	fifo->pbdma_nr = hweight32(nvkm_rd32(device, 0x002204));
-	nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr);
-
+	fifo->pbdma_nr = fifo->base.func->runq_nr(&fifo->base);
 
 	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000,
 			      false, &fifo->runlist.mem[0]);
@@ -689,6 +702,7 @@ gf100_fifo = {
 	.oneinit = gf100_fifo_oneinit,
 	.chid_nr = nv50_fifo_chid_nr,
 	.chid_ctor = gf100_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.init = gf100_fifo_init,
 	.fini = gf100_fifo_fini,
 	.intr = gf100_fifo_intr,
@@ -697,6 +711,7 @@ gf100_fifo = {
 	.id_engine = gf100_fifo_id_engine,
 	.uevent_init = gf100_fifo_uevent_init,
 	.uevent_fini = gf100_fifo_uevent_fini,
+	.runq = &gf100_runq,
 	.cgrp = {{                            }, &nv04_cgrp },
 	.chan = {{ 0, 0, FERMI_CHANNEL_GPFIFO }, &gf100_chan, .oclass = &gf100_fifo_gpfifo_oclass },
 };

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c

@@ -23,6 +23,7 @@
  */
 #include "chan.h"
 #include "chid.h"
+#include "runq.h"
 
 #include "gk104.h"
 #include "cgrp.h"
@@ -168,6 +169,10 @@ static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = {
 	{}
 };
 
+const struct nvkm_runq_func
+gk104_runq = {
+};
+
 void
 gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl,
 			  struct nvkm_memory *mem, int nr)
@@ -275,18 +280,8 @@ gk104_fifo_pbdma_init(struct gk104_fifo *fifo)
 	nvkm_wr32(device, 0x000204, (1 << fifo->pbdma_nr) - 1);
 }
 
-int
-gk104_fifo_pbdma_nr(struct gk104_fifo *fifo)
-{
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	/* Determine number of PBDMAs by checking valid enable bits. */
-	nvkm_wr32(device, 0x000204, 0xffffffff);
-	return hweight32(nvkm_rd32(device, 0x000204));
-}
-
 const struct gk104_fifo_pbdma_func
 gk104_fifo_pbdma = {
-	.nr = gk104_fifo_pbdma_nr,
 	.init = gk104_fifo_pbdma_init,
 };
 
@@ -1076,8 +1071,7 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
 	int pbid, ret, i, j;
 	u32 *map;
 
-	fifo->pbdma_nr = fifo->func->pbdma->nr(fifo);
-	nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr);
+	fifo->pbdma_nr = fifo->func->runq_nr(&fifo->base);
 
 	/* Read PBDMA->runlist(s) mapping from HW. */
 	if (!(map = kcalloc(fifo->pbdma_nr, sizeof(*map), GFP_KERNEL)))
@@ -1195,6 +1189,7 @@ gk104_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gk104_fifo_chid_nr,
 	.chid_ctor = gf100_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -1213,6 +1208,7 @@ gk104_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk104_fifo_runlist,
 	.pbdma = &gk104_fifo_pbdma,
+	.runq = &gk104_runq,
 	.cgrp = {{                               }, &nv04_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan, .ctor = &gk104_fifo_gpfifo_new },
 };

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h

@@ -82,7 +82,6 @@ void gk104_fifo_uevent_fini(struct nvkm_fifo *fifo);
 void gk104_fifo_uevent_init(struct nvkm_fifo *fifo);
 
 extern const struct gk104_fifo_pbdma_func gk104_fifo_pbdma;
-int gk104_fifo_pbdma_nr(struct gk104_fifo *);
 void gk104_fifo_pbdma_init(struct gk104_fifo *);
 extern const struct nvkm_enum gk104_fifo_fault_access[];
 extern const struct nvkm_enum gk104_fifo_fault_engine[];
@@ -106,7 +105,6 @@ extern const struct nvkm_enum gm107_fifo_fault_engine[];
 extern const struct gk104_fifo_runlist_func gm107_fifo_runlist;
 
 extern const struct gk104_fifo_pbdma_func gm200_fifo_pbdma;
-int gm200_fifo_pbdma_nr(struct gk104_fifo *);
 
 extern const struct nvkm_enum gp100_fifo_fault_engine[];
 

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c

@@ -75,6 +75,7 @@ gk110_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gk104_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -93,6 +94,7 @@ gk110_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
 	.pbdma = &gk104_fifo_pbdma,
+	.runq = &gk104_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c

@@ -21,6 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
+#include "runq.h"
 #include "gk104.h"
 #include "changk104.h"
 
@@ -38,11 +39,14 @@ gk208_fifo_pbdma_init_timeout(struct gk104_fifo *fifo)
 
 const struct gk104_fifo_pbdma_func
 gk208_fifo_pbdma = {
-	.nr = gk104_fifo_pbdma_nr,
 	.init = gk104_fifo_pbdma_init,
 	.init_timeout = gk208_fifo_pbdma_init_timeout,
 };
 
+const struct nvkm_runq_func
+gk208_runq = {
+};
+
 static int
 gk208_fifo_chid_nr(struct nvkm_fifo *fifo)
 {
@@ -55,6 +59,7 @@ gk208_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gk208_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -73,6 +78,7 @@ gk208_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
 	.pbdma = &gk208_fifo_pbdma,
+	.runq = &gk208_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c

@@ -30,6 +30,7 @@ gk20a_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = nv50_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -48,6 +49,7 @@ gk20a_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
 	.pbdma = &gk208_fifo_pbdma,
+	.runq = &gk208_runq,
 	.cgrp = {{                               }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c

@@ -115,6 +115,7 @@ gm107_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm107_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -133,6 +134,7 @@ gm107_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
 	.pbdma = &gk208_fifo_pbdma,
+	.runq = &gk208_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c

@@ -27,15 +27,13 @@
 #include <nvif/class.h>
 
 int
-gm200_fifo_pbdma_nr(struct gk104_fifo *fifo)
+gm200_fifo_runq_nr(struct nvkm_fifo *fifo)
 {
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	return nvkm_rd32(device, 0x002004) & 0x000000ff;
+	return nvkm_rd32(fifo->engine.subdev.device, 0x002004) & 0x000000ff;
 }
 
 const struct gk104_fifo_pbdma_func
 gm200_fifo_pbdma = {
-	.nr = gm200_fifo_pbdma_nr,
 	.init = gk104_fifo_pbdma_init,
 	.init_timeout = gk208_fifo_pbdma_init_timeout,
 };
@@ -52,6 +50,7 @@ gm200_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gm200_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -70,6 +69,7 @@ gm200_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
 	.pbdma = &gm200_fifo_pbdma,
+	.runq = &gk208_runq,
 	.cgrp = {{ 0, 0,  KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c

@@ -87,6 +87,7 @@ gp100_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gm200_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -105,6 +106,7 @@ gp100_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
 	.pbdma = &gm200_fifo_pbdma,
+	.runq = &gk208_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c

@@ -21,6 +21,7 @@
  */
 #include "chan.h"
 #include "cgrp.h"
+#include "runq.h"
 
 #include "gk104.h"
 #include "changk104.h"
@@ -33,6 +34,10 @@ static const struct nvkm_chan_func
 gv100_chan = {
 };
 
+const struct nvkm_runq_func
+gv100_runq = {
+};
+
 void
 gv100_fifo_runlist_chan(struct gk104_fifo_chan *chan,
 			struct nvkm_memory *memory, u32 offset)
@@ -302,6 +307,7 @@ gv100_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gm200_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -319,6 +325,7 @@ gv100_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gv100_fifo_runlist,
 	.pbdma = &gm200_fifo_pbdma,
+	.runq = &gv100_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0,  VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan, .ctor = gv100_fifo_gpfifo_new },
 };