summaryrefslogtreecommitdiff
path: root/drivers/dma/fsl-edma-common.h
blob: ac66222c160405efa939d8eeac993c4e295f1663 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * Copyright 2013-2014 Freescale Semiconductor, Inc.
 * Copyright 2018 Angelo Dureghello <angelo@sysam.it>
 */
#ifndef _FSL_EDMA_COMMON_H_
#define _FSL_EDMA_COMMON_H_

#include <linux/dma-direction.h>
#include <linux/platform_device.h>
#include "virt-dma.h"

#define EDMA_CR_EDBG		BIT(1)
#define EDMA_CR_ERCA		BIT(2)
#define EDMA_CR_ERGA		BIT(3)
#define EDMA_CR_HOE		BIT(4)
#define EDMA_CR_HALT		BIT(5)
#define EDMA_CR_CLM		BIT(6)
#define EDMA_CR_EMLM		BIT(7)
#define EDMA_CR_ECX		BIT(16)
#define EDMA_CR_CX		BIT(17)

#define EDMA_SEEI_SEEI(x)	((x) & GENMASK(4, 0))
#define EDMA_CEEI_CEEI(x)	((x) & GENMASK(4, 0))
#define EDMA_CINT_CINT(x)	((x) & GENMASK(4, 0))
#define EDMA_CERR_CERR(x)	((x) & GENMASK(4, 0))

#define EDMA_TCD_ATTR_DSIZE(x)		(((x) & GENMASK(2, 0)))
#define EDMA_TCD_ATTR_DMOD(x)		(((x) & GENMASK(4, 0)) << 3)
#define EDMA_TCD_ATTR_SSIZE(x)		(((x) & GENMASK(2, 0)) << 8)
#define EDMA_TCD_ATTR_SMOD(x)		(((x) & GENMASK(4, 0)) << 11)

#define EDMA_TCD_ITER_MASK		GENMASK(14, 0)
#define EDMA_TCD_CITER_CITER(x)		((x) & EDMA_TCD_ITER_MASK)
#define EDMA_TCD_BITER_BITER(x)		((x) & EDMA_TCD_ITER_MASK)

#define EDMA_TCD_CSR_START		BIT(0)
#define EDMA_TCD_CSR_INT_MAJOR		BIT(1)
#define EDMA_TCD_CSR_INT_HALF		BIT(2)
#define EDMA_TCD_CSR_D_REQ		BIT(3)
#define EDMA_TCD_CSR_E_SG		BIT(4)
#define EDMA_TCD_CSR_E_LINK		BIT(5)
#define EDMA_TCD_CSR_ACTIVE		BIT(6)
#define EDMA_TCD_CSR_DONE		BIT(7)

#define EDMA_V3_TCD_NBYTES_MLOFF_NBYTES(x) ((x) & GENMASK(9, 0))
#define EDMA_V3_TCD_NBYTES_MLOFF(x)        (x << 10)
#define EDMA_V3_TCD_NBYTES_DMLOE           (1 << 30)
#define EDMA_V3_TCD_NBYTES_SMLOE           (1 << 31)

#define EDMAMUX_CHCFG_DIS		0x0
#define EDMAMUX_CHCFG_ENBL		0x80
#define EDMAMUX_CHCFG_SOURCE(n)		((n) & 0x3F)

#define DMAMUX_NR	2

#define EDMA_TCD                0x1000

#define FSL_EDMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
				 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))

#define EDMA_V3_CH_SBR_RD          BIT(22)
#define EDMA_V3_CH_SBR_WR          BIT(21)
#define EDMA_V3_CH_CSR_ERQ         BIT(0)
#define EDMA_V3_CH_CSR_EARQ        BIT(1)
#define EDMA_V3_CH_CSR_EEI         BIT(2)
#define EDMA_V3_CH_CSR_DONE        BIT(30)
#define EDMA_V3_CH_CSR_ACTIVE      BIT(31)

enum fsl_edma_pm_state {
	RUNNING = 0,
	SUSPENDED,
};

struct fsl_edma_hw_tcd {
	__le32	saddr;
	__le16	soff;
	__le16	attr;
	__le32	nbytes;
	__le32	slast;
	__le32	daddr;
	__le16	doff;
	__le16	citer;
	__le32	dlast_sga;
	__le16	csr;
	__le16	biter;
};

struct fsl_edma_hw_tcd64 {
	__le64  saddr;
	__le16  soff;
	__le16  attr;
	__le32  nbytes;
	__le64  slast;
	__le64  daddr;
	__le64  dlast_sga;
	__le16  doff;
	__le16  citer;
	__le16  csr;
	__le16  biter;
} __packed;

struct fsl_edma3_ch_reg {
	__le32	ch_csr;
	__le32	ch_es;
	__le32	ch_int;
	__le32	ch_sbr;
	__le32	ch_pri;
	__le32	ch_mux;
	__le32  ch_mattr; /* edma4, reserved for edma3 */
	__le32  ch_reserved;
	union {
		struct fsl_edma_hw_tcd tcd;
		struct fsl_edma_hw_tcd64 tcd64;
	};
} __packed;

/*
 * These are iomem pointers, for both v32 and v64.
 */
struct edma_regs {
	void __iomem *cr;
	void __iomem *es;
	void __iomem *erqh;
	void __iomem *erql;	/* aka erq on v32 */
	void __iomem *eeih;
	void __iomem *eeil;	/* aka eei on v32 */
	void __iomem *seei;
	void __iomem *ceei;
	void __iomem *serq;
	void __iomem *cerq;
	void __iomem *cint;
	void __iomem *cerr;
	void __iomem *ssrt;
	void __iomem *cdne;
	void __iomem *inth;
	void __iomem *intl;
	void __iomem *errh;
	void __iomem *errl;
};

struct fsl_edma_sw_tcd {
	dma_addr_t			ptcd;
	void				*vtcd;
};

struct fsl_edma_chan {
	struct virt_dma_chan		vchan;
	enum dma_status			status;
	enum fsl_edma_pm_state		pm_state;
	bool				idle;
	struct fsl_edma_engine		*edma;
	struct fsl_edma_desc		*edesc;
	struct dma_slave_config		cfg;
	u32				attr;
	bool                            is_sw;
	struct dma_pool			*tcd_pool;
	dma_addr_t			dma_dev_addr;
	u32				dma_dev_size;
	enum dma_data_direction		dma_dir;
	char				chan_name[32];
	void __iomem			*tcd;
	void __iomem			*mux_addr;
	u32				real_count;
	struct work_struct		issue_worker;
	struct platform_device		*pdev;
	struct device			*pd_dev;
	u32				srcid;
	struct clk			*clk;
	int                             priority;
	int				hw_chanid;
	int				txirq;
	bool				is_rxchan;
	bool				is_remote;
	bool				is_multi_fifo;
};

struct fsl_edma_desc {
	struct virt_dma_desc		vdesc;
	struct fsl_edma_chan		*echan;
	bool				iscyclic;
	enum dma_transfer_direction	dirn;
	unsigned int			n_tcds;
	struct fsl_edma_sw_tcd		tcd[];
};

#define FSL_EDMA_DRV_HAS_DMACLK		BIT(0)
#define FSL_EDMA_DRV_MUX_SWAP		BIT(1)
#define FSL_EDMA_DRV_CONFIG32		BIT(2)
#define FSL_EDMA_DRV_WRAP_IO		BIT(3)
#define FSL_EDMA_DRV_EDMA64		BIT(4)
#define FSL_EDMA_DRV_HAS_PD		BIT(5)
#define FSL_EDMA_DRV_HAS_CHCLK		BIT(6)
#define FSL_EDMA_DRV_HAS_CHMUX		BIT(7)
/* control and status register is in tcd address space, edma3 reg layout */
#define FSL_EDMA_DRV_SPLIT_REG		BIT(9)
#define FSL_EDMA_DRV_BUS_8BYTE		BIT(10)
#define FSL_EDMA_DRV_DEV_TO_DEV		BIT(11)
#define FSL_EDMA_DRV_ALIGN_64BYTE	BIT(12)
/* Need clean CHn_CSR DONE before enable TCD's ESG */
#define FSL_EDMA_DRV_CLEAR_DONE_E_SG	BIT(13)
/* Need clean CHn_CSR DONE before enable TCD's MAJORELINK */
#define FSL_EDMA_DRV_CLEAR_DONE_E_LINK	BIT(14)
#define FSL_EDMA_DRV_TCD64		BIT(15)

#define FSL_EDMA_DRV_EDMA3	(FSL_EDMA_DRV_SPLIT_REG |	\
				 FSL_EDMA_DRV_BUS_8BYTE |	\
				 FSL_EDMA_DRV_DEV_TO_DEV |	\
				 FSL_EDMA_DRV_ALIGN_64BYTE |	\
				 FSL_EDMA_DRV_CLEAR_DONE_E_SG |	\
				 FSL_EDMA_DRV_CLEAR_DONE_E_LINK)

#define FSL_EDMA_DRV_EDMA4	(FSL_EDMA_DRV_SPLIT_REG |	\
				 FSL_EDMA_DRV_BUS_8BYTE |	\
				 FSL_EDMA_DRV_DEV_TO_DEV |	\
				 FSL_EDMA_DRV_ALIGN_64BYTE |	\
				 FSL_EDMA_DRV_CLEAR_DONE_E_LINK)

struct fsl_edma_drvdata {
	u32			dmamuxs; /* only used before v3 */
	u32			chreg_off;
	u32			chreg_space_sz;
	u32			flags;
	u32			mux_off;	/* channel mux register offset */
	u32			mux_skip;	/* how much skip for each channel */
	int			(*setup_irq)(struct platform_device *pdev,
					     struct fsl_edma_engine *fsl_edma);
};

struct fsl_edma_engine {
	struct dma_device	dma_dev;
	void __iomem		*membase;
	void __iomem		*muxbase[DMAMUX_NR];
	struct clk		*muxclk[DMAMUX_NR];
	struct clk		*dmaclk;
	struct mutex		fsl_edma_mutex;
	const struct fsl_edma_drvdata *drvdata;
	u32			n_chans;
	int			txirq;
	int			errirq;
	bool			big_endian;
	struct edma_regs	regs;
	u64			chan_masked;
	struct fsl_edma_chan	chans[] __counted_by(n_chans);
};

static inline u32 fsl_edma_drvflags(struct fsl_edma_chan *fsl_chan)
{
	return fsl_chan->edma->drvdata->flags;
}

#define edma_read_tcdreg_c(chan, _tcd,  __name)				\
_Generic(((_tcd)->__name),						\
	__iomem __le64 : edma_readq(chan->edma, &(_tcd)->__name),		\
	__iomem __le32 : edma_readl(chan->edma, &(_tcd)->__name),		\
	__iomem __le16 : edma_readw(chan->edma, &(_tcd)->__name)		\
	)

#define edma_read_tcdreg(chan, __name)								\
((fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) ?						\
	edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd64 __iomem *)chan->tcd), __name) :	\
	edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd __iomem *)chan->tcd), __name)		\
)

#define edma_write_tcdreg_c(chan, _tcd, _val, __name)					\
_Generic((_tcd->__name),								\
	__iomem __le64 : edma_writeq(chan->edma, (u64 __force)(_val), &_tcd->__name),	\
	__iomem __le32 : edma_writel(chan->edma, (u32 __force)(_val), &_tcd->__name),	\
	__iomem __le16 : edma_writew(chan->edma, (u16 __force)(_val), &_tcd->__name),	\
	__iomem u8 : edma_writeb(chan->edma, _val, &_tcd->__name)			\
	)

#define edma_write_tcdreg(chan, val, __name)							   \
do {												   \
	struct fsl_edma_hw_tcd64 __iomem *tcd64_r = (struct fsl_edma_hw_tcd64 __iomem *)chan->tcd; \
	struct fsl_edma_hw_tcd __iomem *tcd_r = (struct fsl_edma_hw_tcd __iomem *)chan->tcd;	   \
												   \
	if (fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64)					   \
		edma_write_tcdreg_c(chan, tcd64_r, val, __name);				   \
	else											   \
		edma_write_tcdreg_c(chan, tcd_r, val, __name);					   \
} while (0)

#define edma_cp_tcd_to_reg(chan, __tcd, __name)							   \
do {	\
	struct fsl_edma_hw_tcd64 __iomem *tcd64_r = (struct fsl_edma_hw_tcd64 __iomem *)chan->tcd; \
	struct fsl_edma_hw_tcd __iomem *tcd_r = (struct fsl_edma_hw_tcd __iomem *)chan->tcd;	   \
	struct fsl_edma_hw_tcd64 *tcd64_m = (struct fsl_edma_hw_tcd64 *)__tcd;			   \
	struct fsl_edma_hw_tcd *tcd_m = (struct fsl_edma_hw_tcd *)__tcd;			   \
												   \
	if (fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64)					   \
		edma_write_tcdreg_c(chan, tcd64_r,  tcd64_m->__name, __name);			   \
	else											   \
		edma_write_tcdreg_c(chan, tcd_r, tcd_m->__name, __name);			   \
} while (0)

#define edma_readl_chreg(chan, __name)				\
	edma_readl(chan->edma,					\
		   (void __iomem *)&(container_of(((__force void *)chan->tcd),\
						  struct fsl_edma3_ch_reg, tcd)->__name))

#define edma_writel_chreg(chan, val,  __name)			\
	edma_writel(chan->edma, val,				\
		   (void __iomem *)&(container_of(((__force void *)chan->tcd),\
						  struct fsl_edma3_ch_reg, tcd)->__name))

#define fsl_edma_get_tcd(_chan, _tcd, _field)			\
(fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ? (((struct fsl_edma_hw_tcd64 *)_tcd)->_field) : \
						 (((struct fsl_edma_hw_tcd *)_tcd)->_field))

#define fsl_edma_le_to_cpu(x)						\
_Generic((x),								\
	__le64 : le64_to_cpu((x)),					\
	__le32 : le32_to_cpu((x)),					\
	__le16 : le16_to_cpu((x))					\
)

#define fsl_edma_get_tcd_to_cpu(_chan, _tcd, _field)				\
(fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ?				\
	fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd64 *)_tcd)->_field) :	\
	fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd *)_tcd)->_field))

#define fsl_edma_set_tcd_to_le_c(_tcd, _val, _field)					\
_Generic(((_tcd)->_field),								\
	__le64 : (_tcd)->_field = cpu_to_le64(_val),					\
	__le32 : (_tcd)->_field = cpu_to_le32(_val),					\
	__le16 : (_tcd)->_field = cpu_to_le16(_val)					\
)

#define fsl_edma_set_tcd_to_le(_chan, _tcd, _val, _field)	\
do {								\
	if (fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64)	\
		fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd64 *)_tcd, _val, _field);	\
	else											\
		fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd *)_tcd, _val, _field);		\
} while (0)

/* Need after struct defination */
#include "fsl-edma-trace.h"

/*
 * R/W functions for big- or little-endian registers:
 * The eDMA controller's endian is independent of the CPU core's endian.
 * For the big-endian IP module, the offset for 8-bit or 16-bit registers
 * should also be swapped opposite to that in little-endian IP.
 */
static inline u64 edma_readq(struct fsl_edma_engine *edma, void __iomem *addr)
{
	u64 l, h;

	if (edma->big_endian) {
		l = ioread32be(addr);
		h = ioread32be(addr + 4);
	} else {
		l = ioread32(addr);
		h = ioread32(addr + 4);
	}

	trace_edma_readl(edma, addr, l);
	trace_edma_readl(edma, addr + 4, h);

	return (h << 32) | l;
}

static inline u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr)
{
	u32 val;

	if (edma->big_endian)
		val = ioread32be(addr);
	else
		val = ioread32(addr);

	trace_edma_readl(edma, addr, val);

	return val;
}

static inline u16 edma_readw(struct fsl_edma_engine *edma, void __iomem *addr)
{
	u16 val;

	if (edma->big_endian)
		val = ioread16be(addr);
	else
		val = ioread16(addr);

	trace_edma_readw(edma, addr, val);

	return val;
}

static inline void edma_writeb(struct fsl_edma_engine *edma,
			       u8 val, void __iomem *addr)
{
	/* swap the reg offset for these in big-endian mode */
	if (edma->big_endian)
		iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3));
	else
		iowrite8(val, addr);

	trace_edma_writeb(edma, addr, val);
}

static inline void edma_writew(struct fsl_edma_engine *edma,
			       u16 val, void __iomem *addr)
{
	/* swap the reg offset for these in big-endian mode */
	if (edma->big_endian)
		iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2));
	else
		iowrite16(val, addr);

	trace_edma_writew(edma, addr, val);
}

static inline void edma_writel(struct fsl_edma_engine *edma,
			       u32 val, void __iomem *addr)
{
	if (edma->big_endian)
		iowrite32be(val, addr);
	else
		iowrite32(val, addr);

	trace_edma_writel(edma, addr, val);
}

static inline void edma_writeq(struct fsl_edma_engine *edma,
			       u64 val, void __iomem *addr)
{
	if (edma->big_endian) {
		iowrite32be(val & 0xFFFFFFFF, addr);
		iowrite32be(val >> 32, addr + 4);
	} else {
		iowrite32(val & 0xFFFFFFFF, addr);
		iowrite32(val >> 32, addr + 4);
	}

	trace_edma_writel(edma, addr, val & 0xFFFFFFFF);
	trace_edma_writel(edma, addr + 4, val >> 32);
}

static inline struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan)
{
	return container_of(chan, struct fsl_edma_chan, vchan.chan);
}

static inline struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd)
{
	return container_of(vd, struct fsl_edma_desc, vdesc);
}

static inline void fsl_edma_err_chan_handler(struct fsl_edma_chan *fsl_chan)
{
	fsl_chan->status = DMA_ERROR;
	fsl_chan->idle = true;
}

void fsl_edma_tx_chan_handler(struct fsl_edma_chan *fsl_chan);
void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan);
void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan,
			unsigned int slot, bool enable);
void fsl_edma_free_desc(struct virt_dma_desc *vdesc);
int fsl_edma_terminate_all(struct dma_chan *chan);
int fsl_edma_pause(struct dma_chan *chan);
int fsl_edma_resume(struct dma_chan *chan);
int fsl_edma_slave_config(struct dma_chan *chan,
				 struct dma_slave_config *cfg);
enum dma_status fsl_edma_tx_status(struct dma_chan *chan,
		dma_cookie_t cookie, struct dma_tx_state *txstate);
struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic(
		struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
		size_t period_len, enum dma_transfer_direction direction,
		unsigned long flags);
struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg(
		struct dma_chan *chan, struct scatterlist *sgl,
		unsigned int sg_len, enum dma_transfer_direction direction,
		unsigned long flags, void *context);
struct dma_async_tx_descriptor *fsl_edma_prep_memcpy(
		struct dma_chan *chan, dma_addr_t dma_dst, dma_addr_t dma_src,
		size_t len, unsigned long flags);
void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan);
void fsl_edma_issue_pending(struct dma_chan *chan);
int fsl_edma_alloc_chan_resources(struct dma_chan *chan);
void fsl_edma_free_chan_resources(struct dma_chan *chan);
void fsl_edma_cleanup_vchan(struct dma_device *dmadev);
void fsl_edma_setup_regs(struct fsl_edma_engine *edma);

#endif /* _FSL_EDMA_COMMON_H_ */