accel/amdxdna: DMA buffer helpers and AIE4 work buffer support

drivers/accel/amdxdna/aie.c

@@ -4,6 +4,7 @@
  */
 
 #include <linux/errno.h>
+#include <linux/sizes.h>
 
 #include "aie.h"
 #include "amdxdna_mailbox_helper.h"
@@ -119,3 +120,37 @@ void amdxdna_vbnv_init(struct amdxdna_dev *xdna)
 	if (!xdna->vbnv)
 		xdna->vbnv = info->default_vbnv;
 }
+
+void *amdxdna_alloc_msg_buffer(struct amdxdna_dev *xdna, u32 *size,
+			       dma_addr_t *dma_addr)
+{
+	void *vaddr;
+	int order;
+
+	*size = max_t(u32, *size, SZ_8K);
+	order = get_order(*size);
+	if (order > MAX_PAGE_ORDER)
+		return ERR_PTR(-EINVAL);
+	*size = PAGE_SIZE << order;
+
+	if (amdxdna_iova_on(xdna))
+		return amdxdna_iommu_alloc(xdna, *size, dma_addr);
+
+	vaddr = dma_alloc_noncoherent(xdna->ddev.dev, *size, dma_addr,
+				      DMA_FROM_DEVICE, GFP_KERNEL);
+	if (!vaddr)
+		return ERR_PTR(-ENOMEM);
+
+	return vaddr;
+}
+
+void amdxdna_free_msg_buffer(struct amdxdna_dev *xdna, size_t size,
+			     void *cpu_addr, dma_addr_t dma_addr)
+{
+	if (amdxdna_iova_on(xdna)) {
+		amdxdna_iommu_free(xdna, size, cpu_addr, dma_addr);
+		return;
+	}
+
+	dma_free_noncoherent(xdna->ddev.dev, size, cpu_addr, dma_addr, DMA_FROM_DEVICE);
+}

drivers/accel/amdxdna/aie.h

@@ -95,6 +95,11 @@ int aie_send_mgmt_msg_wait(struct aie_device *aie, struct xdna_mailbox_msg *msg)
 int aie_check_protocol(struct aie_device *aie, u32 fw_major, u32 fw_minor);
 void amdxdna_vbnv_init(struct amdxdna_dev *xdna);
 
+void *amdxdna_alloc_msg_buffer(struct amdxdna_dev *xdna, u32 *size,
+			       dma_addr_t *dma_addr);
+void amdxdna_free_msg_buffer(struct amdxdna_dev *xdna, size_t size,
+			     void *cpu_addr, dma_addr_t dma_addr);
+
 /* aie_psp.c */
 struct psp_device *aiem_psp_create(struct drm_device *ddev, struct psp_config *conf);
 int aie_psp_start(struct psp_device *psp);

drivers/accel/amdxdna/aie2_error.c

@@ -11,6 +11,7 @@
 #include <linux/kthread.h>
 #include <linux/kernel.h>
 
+#include "aie.h"
 #include "aie2_msg_priv.h"
 #include "aie2_pci.h"
 #include "amdxdna_error.h"
@@ -338,7 +339,7 @@ void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev)
 	destroy_workqueue(events->wq);
 	mutex_lock(&xdna->dev_lock);
 
-	aie2_free_msg_buffer(ndev, events->size, events->buf, events->addr);
+	amdxdna_free_msg_buffer(xdna, events->size, events->buf, events->addr);
 	kfree(events);
 }
 
@@ -354,7 +355,7 @@ int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev)
 	if (!events)
 		return -ENOMEM;
 
-	events->buf = aie2_alloc_msg_buffer(ndev, &total_size, &events->addr);
+	events->buf = amdxdna_alloc_msg_buffer(xdna, &total_size, &events->addr);
 	if (IS_ERR(events->buf)) {
 		ret = PTR_ERR(events->buf);
 		goto free_events;
@@ -394,7 +395,7 @@ int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev)
 free_wq:
 	destroy_workqueue(events->wq);
 free_buf:
-	aie2_free_msg_buffer(ndev, events->size, events->buf, events->addr);
+	amdxdna_free_msg_buffer(xdna, events->size, events->buf, events->addr);
 free_events:
 	kfree(events);
 	return ret;

drivers/accel/amdxdna/aie2_message.c

@@ -27,43 +27,6 @@
 
 #define EXEC_MSG_OPS(xdna)	((xdna)->dev_handle->exec_msg_ops)
 
-void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size,
-			    dma_addr_t *dma_addr)
-{
-	struct amdxdna_dev *xdna = ndev->aie.xdna;
-	void *vaddr;
-	int order;
-
-	*size = max(*size, SZ_8K);
-	order = get_order(*size);
-	if (order > MAX_PAGE_ORDER)
-		return ERR_PTR(-EINVAL);
-	*size = PAGE_SIZE << order;
-
-	if (amdxdna_iova_on(xdna))
-		return amdxdna_iommu_alloc(xdna, *size, dma_addr);
-
-	vaddr = dma_alloc_noncoherent(xdna->ddev.dev, *size, dma_addr,
-				      DMA_FROM_DEVICE, GFP_KERNEL);
-	if (!vaddr)
-		return ERR_PTR(-ENOMEM);
-
-	return vaddr;
-}
-
-void aie2_free_msg_buffer(struct amdxdna_dev_hdl *ndev, size_t size,
-			  void *cpu_addr, dma_addr_t dma_addr)
-{
-	struct amdxdna_dev *xdna = ndev->aie.xdna;
-
-	if (amdxdna_iova_on(xdna)) {
-		amdxdna_iommu_free(xdna, size, cpu_addr, dma_addr);
-		return;
-	}
-
-	dma_free_noncoherent(xdna->ddev.dev, size, cpu_addr, dma_addr, DMA_FROM_DEVICE);
-}
-
 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev)
 {
 	DECLARE_AIE_MSG(suspend, MSG_OP_SUSPEND);
@@ -407,7 +370,7 @@ int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
 	int ret;
 
 	buf_sz = ndev->metadata.cols * ndev->metadata.size;
-	buff_addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr);
+	buff_addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr);
 	if (IS_ERR(buff_addr))
 		return PTR_ERR(buff_addr);
 
@@ -446,7 +409,7 @@ int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
 	*cols_filled = aie_bitmap;
 
 fail:
-	aie2_free_msg_buffer(ndev, buf_sz, buff_addr, dma_addr);
+	amdxdna_free_msg_buffer(xdna, buf_sz, buff_addr, dma_addr);
 	return ret;
 }
 
@@ -465,7 +428,7 @@ int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
 		return -EINVAL;
 
 	buf_sz = min(size, SZ_4M);
-	addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr);
+	addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr);
 	if (IS_ERR(addr))
 		return PTR_ERR(addr);
 
@@ -497,7 +460,7 @@ int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
 	header->minor = resp.minor;
 
 free_buf:
-	aie2_free_msg_buffer(ndev, buf_sz, addr, dma_addr);
+	amdxdna_free_msg_buffer(xdna, buf_sz, addr, dma_addr);
 	return ret;
 }
 
@@ -1206,7 +1169,7 @@ int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
 	}
 
 	buf_size = sizeof(*report);
-	buf = aie2_alloc_msg_buffer(ndev, &buf_size, &dma_addr);
+	buf = amdxdna_alloc_msg_buffer(xdna, &buf_size, &dma_addr);
 	if (IS_ERR(buf)) {
 		XDNA_ERR(xdna, "Failed to allocate buffer for app health");
 		return PTR_ERR(buf);
@@ -1227,7 +1190,7 @@ int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
 	memcpy(report, buf, sizeof(*report));
 
 free_buf:
-	aie2_free_msg_buffer(ndev, buf_size, buf, dma_addr);
+	amdxdna_free_msg_buffer(xdna, buf_size, buf, dma_addr);
 	return ret;
 }
 

drivers/accel/amdxdna/aie2_pci.h

@@ -342,10 +342,6 @@ int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
 			 int (*notify_cb)(void *, void __iomem *, size_t));
 int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us);
-void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size,
-			    dma_addr_t *dma_addr);
-void aie2_free_msg_buffer(struct amdxdna_dev_hdl *ndev, size_t size,
-			  void *cpu_addr, dma_addr_t dma_addr);
 
 /* aie2_hwctx.c */
 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx);

drivers/accel/amdxdna/aie4_message.c

@@ -25,3 +25,23 @@ int aie4_suspend_fw(struct amdxdna_dev_hdl *ndev)
 
 	return ret;
 }
+
+int aie4_attach_work_buffer(struct amdxdna_dev_hdl *ndev, u32 pasid,
+			    dma_addr_t addr, u32 size)
+{
+	DECLARE_AIE_MSG(aie4_msg_attach_work_buffer, AIE4_MSG_OP_ATTACH_WORK_BUFFER);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	int ret;
+
+	req.buff_addr = addr;
+	req.buff_size = size;
+	req.pasid = FIELD_PREP(AIE4_MSG_PASID_MASK, pasid);
+	if (pasid)
+		req.pasid |= AIE4_MSG_PASID_VLD;
+
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret)
+		XDNA_ERR(xdna, "Failed to attach work buffer, ret %d", ret);
+
+	return ret;
+}

drivers/accel/amdxdna/aie4_msg_priv.h

@@ -6,11 +6,13 @@
 #ifndef _AIE4_MSG_PRIV_H_
 #define _AIE4_MSG_PRIV_H_
 
+#include <linux/bitfield.h>
+#include <linux/sizes.h>
 #include <linux/types.h>
 
 enum aie4_msg_opcode {
 	AIE4_MSG_OP_SUSPEND                          = 0x10003,
-
+	AIE4_MSG_OP_ATTACH_WORK_BUFFER               = 0x1000D,
 	AIE4_MSG_OP_CREATE_VFS                       = 0x20001,
 	AIE4_MSG_OP_DESTROY_VFS                      = 0x20002,
 };
@@ -46,4 +48,19 @@ struct aie4_msg_destroy_vfs_resp {
 	enum aie4_msg_status status;
 } __packed;
 
+#define AIE4_MSG_PASID_MASK		GENMASK(19, 0)
+#define AIE4_MSG_PASID_VLD		BIT(31)
+
+#define AIE4_WORK_BUFFER_MIN_SIZE	SZ_4M
+
+struct aie4_msg_attach_work_buffer_req {
+	__u64 buff_addr;
+	__u32 pasid;
+	__u32 buff_size;
+} __packed;
+
+struct aie4_msg_attach_work_buffer_resp {
+	enum aie4_msg_status status;
+} __packed;
+
 #endif /* _AIE4_MSG_PRIV_H_ */

drivers/accel/amdxdna/aie4_pci.c

@@ -6,9 +6,11 @@
 #include "drm/amdxdna_accel.h"
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
+#include <linux/dma-mapping.h>
 #include <linux/firmware.h>
 #include <linux/sizes.h>
 
+#include "aie4_msg_priv.h"
 #include "aie4_pci.h"
 #include "amdxdna_pci_drv.h"
 
@@ -234,6 +236,40 @@ static int aie4_fw_load(struct amdxdna_dev_hdl *ndev)
 	return ret;
 }
 
+static int aie4_alloc_work_buffer(struct amdxdna_dev_hdl *ndev)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	u32 buf_size = AIE4_WORK_BUFFER_MIN_SIZE;
+
+	ndev->work_buf = amdxdna_alloc_msg_buffer(xdna, &buf_size,
+						  &ndev->work_buf_addr);
+	if (IS_ERR(ndev->work_buf)) {
+		int ret = PTR_ERR(ndev->work_buf);
+
+		XDNA_ERR(xdna, "Failed to alloc work buffer, size 0x%x",
+			 AIE4_WORK_BUFFER_MIN_SIZE);
+		ndev->work_buf = NULL;
+		return ret;
+	}
+
+	ndev->work_buf_size = buf_size;
+	XDNA_DBG(xdna, "Work buffer allocated: size 0x%x", buf_size);
+
+	return 0;
+}
+
+static void aie4_free_work_buffer(struct amdxdna_dev_hdl *ndev)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+
+	if (!ndev->work_buf)
+		return;
+
+	amdxdna_free_msg_buffer(xdna, ndev->work_buf_size,
+				ndev->work_buf, ndev->work_buf_addr);
+	ndev->work_buf = NULL;
+}
+
 static int aie4_hw_start(struct amdxdna_dev *xdna)
 {
 	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
@@ -247,8 +283,18 @@ static int aie4_hw_start(struct amdxdna_dev *xdna)
 	if (ret)
 		goto fw_unload;
 
+	if (ndev->work_buf) {
+		/* Firmware releases the DRAM work buffer internally during suspend */
+		ret = aie4_attach_work_buffer(ndev, 0, ndev->work_buf_addr,
+					      ndev->work_buf_size);
+		if (ret)
+			goto mbox_fini;
+	}
+
 	return 0;
 
+mbox_fini:
+	aie4_mailbox_fini(ndev);
 fw_unload:
 	aie4_fw_unload(ndev);
 
@@ -459,6 +505,7 @@ static void aie4_fini(struct amdxdna_dev *xdna)
 
 	aie4_sriov_stop(ndev);
 	aie4_pcidev_fini(ndev);
+	aie4_free_work_buffer(ndev);
 }
 
 static int aie4_init(struct amdxdna_dev *xdna)
@@ -474,15 +521,23 @@ static int aie4_init(struct amdxdna_dev *xdna)
 	ndev->aie.xdna = xdna;
 	xdna->dev_handle = ndev;
 
+	ret = aie4_alloc_work_buffer(ndev);
+	if (ret)
+		return ret;
+
 	ret = aie4_pcidev_init(ndev);
 	if (ret) {
 		XDNA_ERR(xdna, "Setup PCI device failed, ret %d", ret);
-		return ret;
+		goto free_work_buf;
 	}
 
 	amdxdna_vbnv_init(xdna);
 	XDNA_DBG(xdna, "aie4 init finished");
 	return 0;
+
+free_work_buf:
+	aie4_free_work_buffer(ndev);
+	return ret;
 }
 
 const struct amdxdna_dev_ops aie4_ops = {

drivers/accel/amdxdna/aie4_pci.h

@@ -31,10 +31,15 @@ struct amdxdna_dev_hdl {
 	void			__iomem *rbuf_base;
 
 	struct mailbox			*mbox;
+	void				*work_buf;
+	dma_addr_t			work_buf_addr;
+	u32				work_buf_size;
 };
 
 /* aie4_message.c */
 int aie4_suspend_fw(struct amdxdna_dev_hdl *ndev);
+int aie4_attach_work_buffer(struct amdxdna_dev_hdl *ndev, u32 pasid,
+			    dma_addr_t addr, u32 size);
 
 /* aie4_sriov.c */
 #if IS_ENABLED(CONFIG_PCI_IOV)

drivers/accel/amdxdna/aie4_sriov.c

@@ -45,6 +45,9 @@ int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev)
 	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
 	int ret;
 
+	if (!pci_num_vf(pdev))
+		return 0;
+
 	ret = pci_vfs_assigned(pdev);
 	if (ret) {
 		XDNA_ERR(xdna, "VFs are still assigned to VMs");