summaryrefslogtreecommitdiff
path: root/drivers/iommu/sun50i-iommu.c
blob: 5b585eace3d46f8381e4feda764e0804064ece5f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
// Copyright (C) 2016-2018, Allwinner Technology CO., LTD.
// Copyright (C) 2019-2020, Cerno

#include <linux/bitfield.h>
#include <linux/bug.h>
#include <linux/clk.h>
#include <linux/device.h>
#include <linux/dma-direction.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/ioport.h>
#include <linux/log2.h>
#include <linux/module.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/pm.h>
#include <linux/pm_runtime.h>
#include <linux/reset.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>

#define IOMMU_RESET_REG			0x010
#define IOMMU_RESET_RELEASE_ALL			0xffffffff
#define IOMMU_ENABLE_REG		0x020
#define IOMMU_ENABLE_ENABLE			BIT(0)

#define IOMMU_BYPASS_REG		0x030
#define IOMMU_AUTO_GATING_REG		0x040
#define IOMMU_AUTO_GATING_ENABLE		BIT(0)

#define IOMMU_WBUF_CTRL_REG		0x044
#define IOMMU_OOO_CTRL_REG		0x048
#define IOMMU_4KB_BDY_PRT_CTRL_REG	0x04c
#define IOMMU_TTB_REG			0x050
#define IOMMU_TLB_ENABLE_REG		0x060
#define IOMMU_TLB_PREFETCH_REG		0x070
#define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m)	BIT(m)

#define IOMMU_TLB_FLUSH_REG		0x080
#define IOMMU_TLB_FLUSH_PTW_CACHE		BIT(17)
#define IOMMU_TLB_FLUSH_MACRO_TLB		BIT(16)
#define IOMMU_TLB_FLUSH_MICRO_TLB(i)		(BIT(i) & GENMASK(5, 0))

#define IOMMU_TLB_IVLD_ADDR_REG		0x090
#define IOMMU_TLB_IVLD_ADDR_MASK_REG	0x094
#define IOMMU_TLB_IVLD_ENABLE_REG	0x098
#define IOMMU_TLB_IVLD_ENABLE_ENABLE		BIT(0)

#define IOMMU_PC_IVLD_ADDR_REG		0x0a0
#define IOMMU_PC_IVLD_ENABLE_REG	0x0a8
#define IOMMU_PC_IVLD_ENABLE_ENABLE		BIT(0)

#define IOMMU_DM_AUT_CTRL_REG(d)	(0x0b0 + ((d) / 2) * 4)
#define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m)	(1 << (((d & 1) * 16) + ((m) * 2)))
#define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m)	(1 << (((d & 1) * 16) + ((m) * 2) + 1))

#define IOMMU_DM_AUT_OVWT_REG		0x0d0
#define IOMMU_INT_ENABLE_REG		0x100
#define IOMMU_INT_CLR_REG		0x104
#define IOMMU_INT_STA_REG		0x108
#define IOMMU_INT_ERR_ADDR_REG(i)	(0x110 + (i) * 4)
#define IOMMU_INT_ERR_ADDR_L1_REG	0x130
#define IOMMU_INT_ERR_ADDR_L2_REG	0x134
#define IOMMU_INT_ERR_DATA_REG(i)	(0x150 + (i) * 4)
#define IOMMU_L1PG_INT_REG		0x0180
#define IOMMU_L2PG_INT_REG		0x0184

#define IOMMU_INT_INVALID_L2PG			BIT(17)
#define IOMMU_INT_INVALID_L1PG			BIT(16)
#define IOMMU_INT_MASTER_PERMISSION(m)		BIT(m)
#define IOMMU_INT_MASTER_MASK			(IOMMU_INT_MASTER_PERMISSION(0) | \
						 IOMMU_INT_MASTER_PERMISSION(1) | \
						 IOMMU_INT_MASTER_PERMISSION(2) | \
						 IOMMU_INT_MASTER_PERMISSION(3) | \
						 IOMMU_INT_MASTER_PERMISSION(4) | \
						 IOMMU_INT_MASTER_PERMISSION(5))
#define IOMMU_INT_MASK				(IOMMU_INT_INVALID_L1PG | \
						 IOMMU_INT_INVALID_L2PG | \
						 IOMMU_INT_MASTER_MASK)

#define PT_ENTRY_SIZE			sizeof(u32)

#define NUM_DT_ENTRIES			4096
#define DT_SIZE				(NUM_DT_ENTRIES * PT_ENTRY_SIZE)

#define NUM_PT_ENTRIES			256
#define PT_SIZE				(NUM_PT_ENTRIES * PT_ENTRY_SIZE)

#define SPAGE_SIZE			4096

struct sun50i_iommu {
	struct iommu_device iommu;

	/* Lock to modify the IOMMU registers */
	spinlock_t iommu_lock;

	struct device *dev;
	void __iomem *base;
	struct reset_control *reset;
	struct clk *clk;

	struct iommu_domain *domain;
	struct iommu_group *group;
	struct kmem_cache *pt_pool;
};

struct sun50i_iommu_domain {
	struct iommu_domain domain;

	/* Number of devices attached to the domain */
	refcount_t refcnt;

	/* L1 Page Table */
	u32 *dt;
	dma_addr_t dt_dma;

	struct sun50i_iommu *iommu;
};

static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain)
{
	return container_of(domain, struct sun50i_iommu_domain, domain);
}

static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev)
{
	return dev_iommu_priv_get(dev);
}

static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)
{
	return readl(iommu->base + offset);
}

static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)
{
	writel(value, iommu->base + offset);
}

/*
 * The Allwinner H6 IOMMU uses a 2-level page table.
 *
 * The first level is the usual Directory Table (DT), that consists of
 * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page
 * Table (PT).
 *
 * Each PT consits of 256 4-bytes Page Table Entries (PTE), each
 * pointing to a 4kB page of physical memory.
 *
 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG
 * register that contains its physical address.
 */

#define SUN50I_IOVA_DTE_MASK	GENMASK(31, 20)
#define SUN50I_IOVA_PTE_MASK	GENMASK(19, 12)
#define SUN50I_IOVA_PAGE_MASK	GENMASK(11, 0)

static u32 sun50i_iova_get_dte_index(dma_addr_t iova)
{
	return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova);
}

static u32 sun50i_iova_get_pte_index(dma_addr_t iova)
{
	return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova);
}

static u32 sun50i_iova_get_page_offset(dma_addr_t iova)
{
	return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova);
}

/*
 * Each Directory Table Entry has a Page Table address and a valid
 * bit:

 * +---------------------+-----------+-+
 * | PT address          | Reserved  |V|
 * +---------------------+-----------+-+
 *  31:10 - Page Table address
 *   9:2  - Reserved
 *   1:0  - 1 if the entry is valid
 */

#define SUN50I_DTE_PT_ADDRESS_MASK	GENMASK(31, 10)
#define SUN50I_DTE_PT_ATTRS		GENMASK(1, 0)
#define SUN50I_DTE_PT_VALID		1

static phys_addr_t sun50i_dte_get_pt_address(u32 dte)
{
	return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK;
}

static bool sun50i_dte_is_pt_valid(u32 dte)
{
	return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID;
}

static u32 sun50i_mk_dte(dma_addr_t pt_dma)
{
	return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID;
}

/*
 * Each PTE has a Page address, an authority index and a valid bit:
 *
 * +----------------+-----+-----+-----+---+-----+
 * | Page address   | Rsv | ACI | Rsv | V | Rsv |
 * +----------------+-----+-----+-----+---+-----+
 *  31:12 - Page address
 *  11:8  - Reserved
 *   7:4  - Authority Control Index
 *   3:2  - Reserved
 *     1  - 1 if the entry is valid
 *     0  - Reserved
 *
 * The way permissions work is that the IOMMU has 16 "domains" that
 * can be configured to give each masters either read or write
 * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain
 * 0 seems like the default domain, and its permissions in the
 * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really
 * useful to enforce any particular permission.
 *
 * Each page entry will then have a reference to the domain they are
 * affected to, so that we can actually enforce them on a per-page
 * basis.
 *
 * In order to make it work with the IOMMU framework, we will be using
 * 4 different domains, starting at 1: RD_WR, RD, WR and NONE
 * depending on the permission we want to enforce. Each domain will
 * have each master setup in the same way, since the IOMMU framework
 * doesn't seem to restrict page access on a per-device basis. And
 * then we will use the relevant domain index when generating the page
 * table entry depending on the permissions we want to be enforced.
 */

enum sun50i_iommu_aci {
	SUN50I_IOMMU_ACI_DO_NOT_USE = 0,
	SUN50I_IOMMU_ACI_NONE,
	SUN50I_IOMMU_ACI_RD,
	SUN50I_IOMMU_ACI_WR,
	SUN50I_IOMMU_ACI_RD_WR,
};

#define SUN50I_PTE_PAGE_ADDRESS_MASK	GENMASK(31, 12)
#define SUN50I_PTE_ACI_MASK		GENMASK(7, 4)
#define SUN50I_PTE_PAGE_VALID		BIT(1)

static phys_addr_t sun50i_pte_get_page_address(u32 pte)
{
	return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK;
}

static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte)
{
	return FIELD_GET(SUN50I_PTE_ACI_MASK, pte);
}

static bool sun50i_pte_is_page_valid(u32 pte)
{
	return pte & SUN50I_PTE_PAGE_VALID;
}

static u32 sun50i_mk_pte(phys_addr_t page, int prot)
{
	enum sun50i_iommu_aci aci;
	u32 flags = 0;

	if ((prot & (IOMMU_READ | IOMMU_WRITE)) == (IOMMU_READ | IOMMU_WRITE))
		aci = SUN50I_IOMMU_ACI_RD_WR;
	else if (prot & IOMMU_READ)
		aci = SUN50I_IOMMU_ACI_RD;
	else if (prot & IOMMU_WRITE)
		aci = SUN50I_IOMMU_ACI_WR;
	else
		aci = SUN50I_IOMMU_ACI_NONE;

	flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci);
	page &= SUN50I_PTE_PAGE_ADDRESS_MASK;
	return page | flags | SUN50I_PTE_PAGE_VALID;
}

static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain,
			       void *vaddr, unsigned int count)
{
	struct sun50i_iommu *iommu = sun50i_domain->iommu;
	dma_addr_t dma = virt_to_phys(vaddr);
	size_t size = count * PT_ENTRY_SIZE;

	dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);
}

static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu,
				  unsigned long iova)
{
	u32 reg;
	int ret;

	iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);
	iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12));
	iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG,
		    IOMMU_TLB_IVLD_ENABLE_ENABLE);

	ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG,
					reg, !reg, 1, 2000);
	if (ret)
		dev_warn(iommu->dev, "TLB invalidation timed out!\n");
}

static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu,
				       unsigned long iova)
{
	u32 reg;
	int ret;

	iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);
	iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG,
		    IOMMU_PC_IVLD_ENABLE_ENABLE);

	ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG,
					reg, !reg, 1, 2000);
	if (ret)
		dev_warn(iommu->dev, "PTW cache invalidation timed out!\n");
}

static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu,
				   unsigned long iova, size_t size)
{
	assert_spin_locked(&iommu->iommu_lock);

	iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0);

	sun50i_iommu_zap_iova(iommu, iova);
	sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE);
	if (size > SPAGE_SIZE) {
		sun50i_iommu_zap_iova(iommu, iova + size);
		sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE);
	}
	sun50i_iommu_zap_ptw_cache(iommu, iova);
	sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M);
	if (size > SZ_1M) {
		sun50i_iommu_zap_ptw_cache(iommu, iova + size);
		sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M);
	}

	iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
}

static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)
{
	u32 reg;
	int ret;

	assert_spin_locked(&iommu->iommu_lock);

	iommu_write(iommu,
		    IOMMU_TLB_FLUSH_REG,
		    IOMMU_TLB_FLUSH_PTW_CACHE |
		    IOMMU_TLB_FLUSH_MACRO_TLB |
		    IOMMU_TLB_FLUSH_MICRO_TLB(5) |
		    IOMMU_TLB_FLUSH_MICRO_TLB(4) |
		    IOMMU_TLB_FLUSH_MICRO_TLB(3) |
		    IOMMU_TLB_FLUSH_MICRO_TLB(2) |
		    IOMMU_TLB_FLUSH_MICRO_TLB(1) |
		    IOMMU_TLB_FLUSH_MICRO_TLB(0));

	ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG,
					reg, !reg,
					1, 2000);
	if (ret)
		dev_warn(iommu->dev, "TLB Flush timed out!\n");

	return ret;
}

static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
{
	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
	struct sun50i_iommu *iommu = sun50i_domain->iommu;
	unsigned long flags;

	/*
	 * At boot, we'll have a first call into .flush_iotlb_all right after
	 * .probe_device, and since we link our (single) domain to our iommu in
	 * the .attach_device callback, we don't have that pointer set.
	 *
	 * It shouldn't really be any trouble to ignore it though since we flush
	 * all caches as part of the device powerup.
	 */
	if (!iommu)
		return;

	spin_lock_irqsave(&iommu->iommu_lock, flags);
	sun50i_iommu_flush_all_tlb(iommu);
	spin_unlock_irqrestore(&iommu->iommu_lock, flags);
}

static void sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
					unsigned long iova, size_t size)
{
	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
	struct sun50i_iommu *iommu = sun50i_domain->iommu;
	unsigned long flags;

	spin_lock_irqsave(&iommu->iommu_lock, flags);
	sun50i_iommu_zap_range(iommu, iova, size);
	spin_unlock_irqrestore(&iommu->iommu_lock, flags);
}

static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
				    struct iommu_iotlb_gather *gather)
{
	sun50i_iommu_flush_iotlb_all(domain);
}

static int sun50i_iommu_enable(struct sun50i_iommu *iommu)
{
	struct sun50i_iommu_domain *sun50i_domain;
	unsigned long flags;
	int ret;

	if (!iommu->domain)
		return 0;

	sun50i_domain = to_sun50i_domain(iommu->domain);

	ret = reset_control_deassert(iommu->reset);
	if (ret)
		return ret;

	ret = clk_prepare_enable(iommu->clk);
	if (ret)
		goto err_reset_assert;

	spin_lock_irqsave(&iommu->iommu_lock, flags);

	iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);
	iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,
		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) |
		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) |
		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) |
		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) |
		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) |
		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(5));
	iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);
	iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5));

	iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) |
		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5));

	iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) |
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) |
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) |
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) |
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) |
		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5));

	ret = sun50i_iommu_flush_all_tlb(iommu);
	if (ret) {
		spin_unlock_irqrestore(&iommu->iommu_lock, flags);
		goto err_clk_disable;
	}

	iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
	iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);

	spin_unlock_irqrestore(&iommu->iommu_lock, flags);

	return 0;

err_clk_disable:
	clk_disable_unprepare(iommu->clk);

err_reset_assert:
	reset_control_assert(iommu->reset);

	return ret;
}

static void sun50i_iommu_disable(struct sun50i_iommu *iommu)
{
	unsigned long flags;

	spin_lock_irqsave(&iommu->iommu_lock, flags);

	iommu_write(iommu, IOMMU_ENABLE_REG, 0);
	iommu_write(iommu, IOMMU_TTB_REG, 0);

	spin_unlock_irqrestore(&iommu->iommu_lock, flags);

	clk_disable_unprepare(iommu->clk);
	reset_control_assert(iommu->reset);
}

static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,
					   gfp_t gfp)
{
	dma_addr_t pt_dma;
	u32 *page_table;

	page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);
	if (!page_table)
		return ERR_PTR(-ENOMEM);

	pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);
	if (dma_mapping_error(iommu->dev, pt_dma)) {
		dev_err(iommu->dev, "Couldn't map L2 Page Table\n");
		kmem_cache_free(iommu->pt_pool, page_table);
		return ERR_PTR(-ENOMEM);
	}

	/* We rely on the physical address and DMA address being the same */
	WARN_ON(pt_dma != virt_to_phys(page_table));

	return page_table;
}

static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,
					 u32 *page_table)
{
	phys_addr_t pt_phys = virt_to_phys(page_table);

	dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);
	kmem_cache_free(iommu->pt_pool, page_table);
}

static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
				      dma_addr_t iova, gfp_t gfp)
{
	struct sun50i_iommu *iommu = sun50i_domain->iommu;
	u32 *page_table;
	u32 *dte_addr;
	u32 old_dte;
	u32 dte;

	dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
	dte = *dte_addr;
	if (sun50i_dte_is_pt_valid(dte)) {
		phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte);
		return (u32 *)phys_to_virt(pt_phys);
	}

	page_table = sun50i_iommu_alloc_page_table(iommu, gfp);
	if (IS_ERR(page_table))
		return page_table;

	dte = sun50i_mk_dte(virt_to_phys(page_table));
	old_dte = cmpxchg(dte_addr, 0, dte);
	if (old_dte) {
		phys_addr_t installed_pt_phys =
			sun50i_dte_get_pt_address(old_dte);
		u32 *installed_pt = phys_to_virt(installed_pt_phys);
		u32 *drop_pt = page_table;

		page_table = installed_pt;
		dte = old_dte;
		sun50i_iommu_free_page_table(iommu, drop_pt);
	}

	sun50i_table_flush(sun50i_domain, page_table, NUM_PT_ENTRIES);
	sun50i_table_flush(sun50i_domain, dte_addr, 1);

	return page_table;
}

static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
			    phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
	struct sun50i_iommu *iommu = sun50i_domain->iommu;
	u32 pte_index;
	u32 *page_table, *pte_addr;
	int ret = 0;

	page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp);
	if (IS_ERR(page_table)) {
		ret = PTR_ERR(page_table);
		goto out;
	}

	pte_index = sun50i_iova_get_pte_index(iova);
	pte_addr = &page_table[pte_index];
	if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) {
		phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr);
		dev_err(iommu->dev,
			"iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n",
			&iova, &page_phys, &paddr, prot);
		ret = -EBUSY;
		goto out;
	}

	*pte_addr = sun50i_mk_pte(paddr, prot);
	sun50i_table_flush(sun50i_domain, pte_addr, 1);

out:
	return ret;
}

static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
				 size_t size, struct iommu_iotlb_gather *gather)
{
	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
	phys_addr_t pt_phys;
	u32 *pte_addr;
	u32 dte;

	dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
	if (!sun50i_dte_is_pt_valid(dte))
		return 0;

	pt_phys = sun50i_dte_get_pt_address(dte);
	pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova);

	if (!sun50i_pte_is_page_valid(*pte_addr))
		return 0;

	memset(pte_addr, 0, sizeof(*pte_addr));
	sun50i_table_flush(sun50i_domain, pte_addr, 1);

	return SZ_4K;
}

static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain,
					     dma_addr_t iova)
{
	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
	phys_addr_t pt_phys;
	u32 *page_table;
	u32 dte, pte;

	dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
	if (!sun50i_dte_is_pt_valid(dte))
		return 0;

	pt_phys = sun50i_dte_get_pt_address(dte);
	page_table = (u32 *)phys_to_virt(pt_phys);
	pte = page_table[sun50i_iova_get_pte_index(iova)];
	if (!sun50i_pte_is_page_valid(pte))
		return 0;

	return sun50i_pte_get_page_address(pte) +
		sun50i_iova_get_page_offset(iova);
}

static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type)
{
	struct sun50i_iommu_domain *sun50i_domain;

	if (type != IOMMU_DOMAIN_DMA &&
	    type != IOMMU_DOMAIN_UNMANAGED)
		return NULL;

	sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL);
	if (!sun50i_domain)
		return NULL;

	sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
						    get_order(DT_SIZE));
	if (!sun50i_domain->dt)
		goto err_free_domain;

	refcount_set(&sun50i_domain->refcnt, 1);

	sun50i_domain->domain.geometry.aperture_start = 0;
	sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
	sun50i_domain->domain.geometry.force_aperture = true;

	return &sun50i_domain->domain;

err_free_domain:
	kfree(sun50i_domain);

	return NULL;
}

static void sun50i_iommu_domain_free(struct iommu_domain *domain)
{
	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);

	free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE));
	sun50i_domain->dt = NULL;

	kfree(sun50i_domain);
}

static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,
				      struct sun50i_iommu_domain *sun50i_domain)
{
	iommu->domain = &sun50i_domain->domain;
	sun50i_domain->iommu = iommu;

	sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,
					       DT_SIZE, DMA_TO_DEVICE);
	if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {
		dev_err(iommu->dev, "Couldn't map L1 Page Table\n");
		return -ENOMEM;
	}

	return sun50i_iommu_enable(iommu);
}

static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,
				       struct sun50i_iommu_domain *sun50i_domain)
{
	unsigned int i;

	for (i = 0; i < NUM_DT_ENTRIES; i++) {
		phys_addr_t pt_phys;
		u32 *page_table;
		u32 *dte_addr;
		u32 dte;

		dte_addr = &sun50i_domain->dt[i];
		dte = *dte_addr;
		if (!sun50i_dte_is_pt_valid(dte))
			continue;

		memset(dte_addr, 0, sizeof(*dte_addr));
		sun50i_table_flush(sun50i_domain, dte_addr, 1);

		pt_phys = sun50i_dte_get_pt_address(dte);
		page_table = phys_to_virt(pt_phys);
		sun50i_iommu_free_page_table(iommu, page_table);
	}


	sun50i_iommu_disable(iommu);

	dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),
			 DT_SIZE, DMA_TO_DEVICE);

	iommu->domain = NULL;
}

static void sun50i_iommu_detach_device(struct iommu_domain *domain,
				       struct device *dev)
{
	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
	struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);

	dev_dbg(dev, "Detaching from IOMMU domain\n");

	if (iommu->domain != domain)
		return;

	if (refcount_dec_and_test(&sun50i_domain->refcnt))
		sun50i_iommu_detach_domain(iommu, sun50i_domain);
}

static int sun50i_iommu_attach_device(struct iommu_domain *domain,
				      struct device *dev)
{
	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
	struct sun50i_iommu *iommu;

	iommu = sun50i_iommu_from_dev(dev);
	if (!iommu)
		return -ENODEV;

	dev_dbg(dev, "Attaching to IOMMU domain\n");

	refcount_inc(&sun50i_domain->refcnt);

	if (iommu->domain == domain)
		return 0;

	if (iommu->domain)
		sun50i_iommu_detach_device(iommu->domain, dev);

	sun50i_iommu_attach_domain(iommu, sun50i_domain);

	return 0;
}

static struct iommu_device *sun50i_iommu_probe_device(struct device *dev)
{
	struct sun50i_iommu *iommu;

	iommu = sun50i_iommu_from_dev(dev);
	if (!iommu)
		return ERR_PTR(-ENODEV);

	return &iommu->iommu;
}

static struct iommu_group *sun50i_iommu_device_group(struct device *dev)
{
	struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev);

	return iommu_group_ref_get(iommu->group);
}

static int sun50i_iommu_of_xlate(struct device *dev,
				 struct of_phandle_args *args)
{
	struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
	unsigned id = args->args[0];

	dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev));

	return iommu_fwspec_add_ids(dev, &id, 1);
}

static const struct iommu_ops sun50i_iommu_ops = {
	.pgsize_bitmap	= SZ_4K,
	.device_group	= sun50i_iommu_device_group,
	.domain_alloc	= sun50i_iommu_domain_alloc,
	.of_xlate	= sun50i_iommu_of_xlate,
	.probe_device	= sun50i_iommu_probe_device,
	.default_domain_ops = &(const struct iommu_domain_ops) {
		.attach_dev	= sun50i_iommu_attach_device,
		.detach_dev	= sun50i_iommu_detach_device,
		.flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
		.iotlb_sync_map = sun50i_iommu_iotlb_sync_map,
		.iotlb_sync	= sun50i_iommu_iotlb_sync,
		.iova_to_phys	= sun50i_iommu_iova_to_phys,
		.map		= sun50i_iommu_map,
		.unmap		= sun50i_iommu_unmap,
		.free		= sun50i_iommu_domain_free,
	}
};

static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,
				      unsigned master, phys_addr_t iova,
				      unsigned prot)
{
	dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",
		&iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd");

	if (iommu->domain)
		report_iommu_fault(iommu->domain, iommu->dev, iova, prot);
	else
		dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");

	sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE);
}

static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,
					      unsigned addr_reg,
					      unsigned blame_reg)
{
	phys_addr_t iova;
	unsigned master;
	u32 blame;

	assert_spin_locked(&iommu->iommu_lock);

	iova = iommu_read(iommu, addr_reg);
	blame = iommu_read(iommu, blame_reg);
	master = ilog2(blame & IOMMU_INT_MASTER_MASK);

	/*
	 * If the address is not in the page table, we can't get what
	 * operation triggered the fault. Assume it's a read
	 * operation.
	 */
	sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);

	return iova;
}

static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)
{
	enum sun50i_iommu_aci aci;
	phys_addr_t iova;
	unsigned master;
	unsigned dir;
	u32 blame;

	assert_spin_locked(&iommu->iommu_lock);

	blame = iommu_read(iommu, IOMMU_INT_STA_REG);
	master = ilog2(blame & IOMMU_INT_MASTER_MASK);
	iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));
	aci = sun50i_get_pte_aci(iommu_read(iommu,
					    IOMMU_INT_ERR_DATA_REG(master)));

	switch (aci) {
		/*
		 * If we are in the read-only domain, then it means we
		 * tried to write.
		 */
	case SUN50I_IOMMU_ACI_RD:
		dir = IOMMU_FAULT_WRITE;
		break;

		/*
		 * If we are in the write-only domain, then it means
		 * we tried to read.
		 */
	case SUN50I_IOMMU_ACI_WR:

		/*
		 * If we are in the domain without any permission, we
		 * can't really tell. Let's default to a read
		 * operation.
		 */
	case SUN50I_IOMMU_ACI_NONE:

		/* WTF? */
	case SUN50I_IOMMU_ACI_RD_WR:
	default:
		dir = IOMMU_FAULT_READ;
		break;
	}

	/*
	 * If the address is not in the page table, we can't get what
	 * operation triggered the fault. Assume it's a read
	 * operation.
	 */
	sun50i_iommu_report_fault(iommu, master, iova, dir);

	return iova;
}

static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
{
	u32 status, l1_status, l2_status, resets;
	struct sun50i_iommu *iommu = dev_id;

	spin_lock(&iommu->iommu_lock);

	status = iommu_read(iommu, IOMMU_INT_STA_REG);
	if (!(status & IOMMU_INT_MASK)) {
		spin_unlock(&iommu->iommu_lock);
		return IRQ_NONE;
	}

	l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG);
	l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG);

	if (status & IOMMU_INT_INVALID_L2PG)
		sun50i_iommu_handle_pt_irq(iommu,
					    IOMMU_INT_ERR_ADDR_L2_REG,
					    IOMMU_L2PG_INT_REG);
	else if (status & IOMMU_INT_INVALID_L1PG)
		sun50i_iommu_handle_pt_irq(iommu,
					   IOMMU_INT_ERR_ADDR_L1_REG,
					   IOMMU_L1PG_INT_REG);
	else
		sun50i_iommu_handle_perm_irq(iommu);

	iommu_write(iommu, IOMMU_INT_CLR_REG, status);

	resets = (status | l1_status | l2_status) & IOMMU_INT_MASTER_MASK;
	iommu_write(iommu, IOMMU_RESET_REG, ~resets);
	iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL);

	spin_unlock(&iommu->iommu_lock);

	return IRQ_HANDLED;
}

static int sun50i_iommu_probe(struct platform_device *pdev)
{
	struct sun50i_iommu *iommu;
	int ret, irq;

	iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
	if (!iommu)
		return -ENOMEM;
	spin_lock_init(&iommu->iommu_lock);
	platform_set_drvdata(pdev, iommu);
	iommu->dev = &pdev->dev;

	iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),
					   PT_SIZE, PT_SIZE,
					   SLAB_HWCACHE_ALIGN,
					   NULL);
	if (!iommu->pt_pool)
		return -ENOMEM;

	iommu->group = iommu_group_alloc();
	if (IS_ERR(iommu->group)) {
		ret = PTR_ERR(iommu->group);
		goto err_free_cache;
	}

	iommu->base = devm_platform_ioremap_resource(pdev, 0);
	if (IS_ERR(iommu->base)) {
		ret = PTR_ERR(iommu->base);
		goto err_free_group;
	}

	irq = platform_get_irq(pdev, 0);
	if (irq < 0) {
		ret = irq;
		goto err_free_group;
	}

	iommu->clk = devm_clk_get(&pdev->dev, NULL);
	if (IS_ERR(iommu->clk)) {
		dev_err(&pdev->dev, "Couldn't get our clock.\n");
		ret = PTR_ERR(iommu->clk);
		goto err_free_group;
	}

	iommu->reset = devm_reset_control_get(&pdev->dev, NULL);
	if (IS_ERR(iommu->reset)) {
		dev_err(&pdev->dev, "Couldn't get our reset line.\n");
		ret = PTR_ERR(iommu->reset);
		goto err_free_group;
	}

	ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,
				     NULL, dev_name(&pdev->dev));
	if (ret)
		goto err_free_group;

	ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev);
	if (ret)
		goto err_remove_sysfs;

	ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0,
			       dev_name(&pdev->dev), iommu);
	if (ret < 0)
		goto err_unregister;

	return 0;

err_unregister:
	iommu_device_unregister(&iommu->iommu);

err_remove_sysfs:
	iommu_device_sysfs_remove(&iommu->iommu);

err_free_group:
	iommu_group_put(iommu->group);

err_free_cache:
	kmem_cache_destroy(iommu->pt_pool);

	return ret;
}

static const struct of_device_id sun50i_iommu_dt[] = {
	{ .compatible = "allwinner,sun50i-h6-iommu", },
	{ /* sentinel */ },
};
MODULE_DEVICE_TABLE(of, sun50i_iommu_dt);

static struct platform_driver sun50i_iommu_driver = {
	.driver		= {
		.name			= "sun50i-iommu",
		.of_match_table 	= sun50i_iommu_dt,
		.suppress_bind_attrs	= true,
	}
};
builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe);

MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");
MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>");
MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>");
MODULE_LICENSE("Dual BSD/GPL");