uboot-tools/uboot-riscv-apr8-smp.patch

From fa33f08fd657b8568ede929df8a79bd113e9c5b1 Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:32 +0100
Subject: [PATCH 01/18] riscv: add infrastructure for calling functions on
 other harts

Harts on RISC-V boot independently, U-Boot is responsible for managing
them. Functions are called on other harts with smp_call_function(),
which sends inter-processor interrupts (IPIs) to all other available
harts. Available harts are those marked as available in the device tree
and present in the available_harts mask stored in global data. The
available_harts mask is used to register all harts that have entered
U-Boot. Functions are specified with their address and two function
arguments (argument 2 and 3). The first function argument is always the
hart ID of the hart calling the function. On the other harts, the IPI
interrupt handler handle_ipi() must be called on software interrupts to
handle the request and call the specified function.

Functions are stored in the ipi_data data structure. Every hart has its
own data structure in global data. While this is not required at the
moment (all harts are expected to boot Linux), this does allow future
expansion, where other harts may be used for monitoring or other tasks.

Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Tested-by: Bin Meng <bmeng.cn@gmail.com>
---
 arch/riscv/Kconfig                   |  19 +++++
 arch/riscv/include/asm/global_data.h |   6 ++
 arch/riscv/include/asm/smp.h         |  53 ++++++++++++
 arch/riscv/lib/Makefile              |   1 +
 arch/riscv/lib/smp.c                 | 118 +++++++++++++++++++++++++++
 5 files changed, 197 insertions(+)
 create mode 100644 arch/riscv/include/asm/smp.h
 create mode 100644 arch/riscv/lib/smp.c

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 36512a8995..4d7a115569 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -120,4 +120,23 @@ config RISCV_RDTIME
 config SYS_MALLOC_F_LEN
 	default 0x1000
 
+config SMP
+	bool "Symmetric Multi-Processing"
+	help
+	  This enables support for systems with more than one CPU. If
+	  you say N here, U-Boot will run on single and multiprocessor
+	  machines, but will use only one CPU of a multiprocessor
+	  machine. If you say Y here, U-Boot will run on many, but not
+	  all, single processor machines.
+
+config NR_CPUS
+	int "Maximum number of CPUs (2-32)"
+	range 2 32
+	depends on SMP
+	default 8
+	help
+	  On multiprocessor machines, U-Boot sets up a stack for each CPU.
+	  Stack memory is pre-allocated. U-Boot must therefore know the
+	  maximum number of CPUs that may be present.
+
 endmenu
diff --git a/arch/riscv/include/asm/global_data.h b/arch/riscv/include/asm/global_data.h
index a3a342c6e1..80e3165e39 100644
--- a/arch/riscv/include/asm/global_data.h
+++ b/arch/riscv/include/asm/global_data.h
@@ -10,12 +10,18 @@
 #ifndef	__ASM_GBL_DATA_H
 #define __ASM_GBL_DATA_H
 
+#include <asm/smp.h>
+
 /* Architecture-specific global data */
 struct arch_global_data {
 	long boot_hart;		/* boot hart id */
 #ifdef CONFIG_SIFIVE_CLINT
 	void __iomem *clint;	/* clint base address */
 #endif
+#ifdef CONFIG_SMP
+	struct ipi_data ipi[CONFIG_NR_CPUS];
+#endif
+	ulong available_harts;
 };
 
 #include <asm-generic/global_data.h>
diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
new file mode 100644
index 0000000000..bc863fdbaf
--- /dev/null
+++ b/arch/riscv/include/asm/smp.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2019 Fraunhofer AISEC,
+ * Lukas Auer <lukas.auer@aisec.fraunhofer.de>
+ */
+
+#ifndef _ASM_RISCV_SMP_H
+#define _ASM_RISCV_SMP_H
+
+/**
+ * struct ipi_data - Inter-processor interrupt (IPI) data structure
+ *
+ * IPIs are used for SMP support to communicate to other harts what function to
+ * call. Functions are in the form
+ * void (*addr)(ulong hart, ulong arg0, ulong arg1).
+ *
+ * The function address and the two arguments, arg0 and arg1, are stored in the
+ * IPI data structure. The hart ID is inserted by the hart handling the IPI and
+ * calling the function.
+ *
+ * @addr: Address of function
+ * @arg0: First argument of function
+ * @arg1: Second argument of function
+ */
+struct ipi_data {
+	ulong addr;
+	ulong arg0;
+	ulong arg1;
+};
+
+/**
+ * handle_ipi() - interrupt handler for software interrupts
+ *
+ * The IPI interrupt handler must be called to handle software interrupts. It
+ * calls the function specified in the hart's IPI data structure.
+ *
+ * @hart: Hart ID of the current hart
+ */
+void handle_ipi(ulong hart);
+
+/**
+ * smp_call_function() - Call a function on all other harts
+ *
+ * Send IPIs with the specified function call to all harts.
+ *
+ * @addr: Address of function
+ * @arg0: First argument of function
+ * @arg1: Second argument of function
+ * @return 0 if OK, -ve on error
+ */
+int smp_call_function(ulong addr, ulong arg0, ulong arg1);
+
+#endif
diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile
index edfa61690c..19370f9749 100644
--- a/arch/riscv/lib/Makefile
+++ b/arch/riscv/lib/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_SIFIVE_CLINT) += sifive_clint.o
 obj-y	+= interrupts.o
 obj-y	+= reset.o
 obj-y   += setjmp.o
+obj-$(CONFIG_SMP) += smp.o
 
 # For building EFI apps
 CFLAGS_$(EFI_CRT0) := $(CFLAGS_EFI)
diff --git a/arch/riscv/lib/smp.c b/arch/riscv/lib/smp.c
new file mode 100644
index 0000000000..caa292ccd2
--- /dev/null
+++ b/arch/riscv/lib/smp.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019 Fraunhofer AISEC,
+ * Lukas Auer <lukas.auer@aisec.fraunhofer.de>
+ */
+
+#include <common.h>
+#include <dm.h>
+#include <asm/barrier.h>
+#include <asm/smp.h>
+
+DECLARE_GLOBAL_DATA_PTR;
+
+/**
+ * riscv_send_ipi() - Send inter-processor interrupt (IPI)
+ *
+ * Platform code must provide this function.
+ *
+ * @hart: Hart ID of receiving hart
+ * @return 0 if OK, -ve on error
+ */
+extern int riscv_send_ipi(int hart);
+
+/**
+ * riscv_clear_ipi() - Clear inter-processor interrupt (IPI)
+ *
+ * Platform code must provide this function.
+ *
+ * @hart: Hart ID of hart to be cleared
+ * @return 0 if OK, -ve on error
+ */
+extern int riscv_clear_ipi(int hart);
+
+static int send_ipi_many(struct ipi_data *ipi)
+{
+	ofnode node, cpus;
+	u32 reg;
+	int ret;
+
+	cpus = ofnode_path("/cpus");
+	if (!ofnode_valid(cpus)) {
+		pr_err("Can't find cpus node!\n");
+		return -EINVAL;
+	}
+
+	ofnode_for_each_subnode(node, cpus) {
+		/* skip if hart is marked as not available in the device tree */
+		if (!ofnode_is_available(node))
+			continue;
+
+		/* read hart ID of CPU */
+		ret = ofnode_read_u32(node, "reg", &reg);
+		if (ret)
+			continue;
+
+		/* skip if it is the hart we are running on */
+		if (reg == gd->arch.boot_hart)
+			continue;
+
+		if (reg >= CONFIG_NR_CPUS) {
+			pr_err("Hart ID %d is out of range, increase CONFIG_NR_CPUS\n",
+			       reg);
+			continue;
+		}
+
+		/* skip if hart is not available */
+		if (!(gd->arch.available_harts & (1 << reg)))
+			continue;
+
+		gd->arch.ipi[reg].addr = ipi->addr;
+		gd->arch.ipi[reg].arg0 = ipi->arg0;
+		gd->arch.ipi[reg].arg1 = ipi->arg1;
+
+		ret = riscv_send_ipi(reg);
+		if (ret) {
+			pr_err("Cannot send IPI to hart %d\n", reg);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+void handle_ipi(ulong hart)
+{
+	int ret;
+	void (*smp_function)(ulong hart, ulong arg0, ulong arg1);
+
+	if (hart >= CONFIG_NR_CPUS)
+		return;
+
+	ret = riscv_clear_ipi(hart);
+	if (ret) {
+		pr_err("Cannot clear IPI of hart %ld\n", hart);
+		return;
+	}
+
+	__smp_mb();
+
+	smp_function = (void (*)(ulong, ulong, ulong))gd->arch.ipi[hart].addr;
+	invalidate_icache_all();
+
+	smp_function(hart, gd->arch.ipi[hart].arg0, gd->arch.ipi[hart].arg1);
+}
+
+int smp_call_function(ulong addr, ulong arg0, ulong arg1)
+{
+	int ret = 0;
+	struct ipi_data ipi;
+
+	ipi.addr = addr;
+	ipi.arg0 = arg0;
+	ipi.arg1 = arg1;
+
+	ret = send_ipi_many(&ipi);
+
+	return ret;
+}
-- 
2.20.1


From 34a0626fc344f51cd768efecdd52628b677fb9a8 Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:33 +0100
Subject: [PATCH 02/18] riscv: import the supervisor binary interface header
 file

Import the supervisor binary interface (SBI) header file from Linux
(arch/riscv/include/asm/sbi.h). The last change to it was in commit
6d60b6ee0c97 ("RISC-V: Device, timer, IRQs, and the SBI").

Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Atish Patra <atish.patra@wdc.com>
---
 arch/riscv/include/asm/sbi.h | 94 ++++++++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 arch/riscv/include/asm/sbi.h

diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h
new file mode 100644
index 0000000000..ced57defdd
--- /dev/null
+++ b/arch/riscv/include/asm/sbi.h
@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015 Regents of the University of California
+ *
+ * Taken from Linux arch/riscv/include/asm/sbi.h
+ */
+
+#ifndef _ASM_RISCV_SBI_H
+#define _ASM_RISCV_SBI_H
+
+#include <linux/types.h>
+
+#define SBI_SET_TIMER 0
+#define SBI_CONSOLE_PUTCHAR 1
+#define SBI_CONSOLE_GETCHAR 2
+#define SBI_CLEAR_IPI 3
+#define SBI_SEND_IPI 4
+#define SBI_REMOTE_FENCE_I 5
+#define SBI_REMOTE_SFENCE_VMA 6
+#define SBI_REMOTE_SFENCE_VMA_ASID 7
+#define SBI_SHUTDOWN 8
+
+#define SBI_CALL(which, arg0, arg1, arg2) ({			\
+	register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0);	\
+	register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1);	\
+	register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2);	\
+	register uintptr_t a7 asm ("a7") = (uintptr_t)(which);	\
+	asm volatile ("ecall"					\
+		      : "+r" (a0)				\
+		      : "r" (a1), "r" (a2), "r" (a7)		\
+		      : "memory");				\
+	a0;							\
+})
+
+/* Lazy implementations until SBI is finalized */
+#define SBI_CALL_0(which) SBI_CALL(which, 0, 0, 0)
+#define SBI_CALL_1(which, arg0) SBI_CALL(which, arg0, 0, 0)
+#define SBI_CALL_2(which, arg0, arg1) SBI_CALL(which, arg0, arg1, 0)
+
+static inline void sbi_console_putchar(int ch)
+{
+	SBI_CALL_1(SBI_CONSOLE_PUTCHAR, ch);
+}
+
+static inline int sbi_console_getchar(void)
+{
+	return SBI_CALL_0(SBI_CONSOLE_GETCHAR);
+}
+
+static inline void sbi_set_timer(uint64_t stime_value)
+{
+#if __riscv_xlen == 32
+	SBI_CALL_2(SBI_SET_TIMER, stime_value, stime_value >> 32);
+#else
+	SBI_CALL_1(SBI_SET_TIMER, stime_value);
+#endif
+}
+
+static inline void sbi_shutdown(void)
+{
+	SBI_CALL_0(SBI_SHUTDOWN);
+}
+
+static inline void sbi_clear_ipi(void)
+{
+	SBI_CALL_0(SBI_CLEAR_IPI);
+}
+
+static inline void sbi_send_ipi(const unsigned long *hart_mask)
+{
+	SBI_CALL_1(SBI_SEND_IPI, hart_mask);
+}
+
+static inline void sbi_remote_fence_i(const unsigned long *hart_mask)
+{
+	SBI_CALL_1(SBI_REMOTE_FENCE_I, hart_mask);
+}
+
+static inline void sbi_remote_sfence_vma(const unsigned long *hart_mask,
+					 unsigned long start,
+					 unsigned long size)
+{
+	SBI_CALL_1(SBI_REMOTE_SFENCE_VMA, hart_mask);
+}
+
+static inline void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
+					      unsigned long start,
+					      unsigned long size,
+					      unsigned long asid)
+{
+	SBI_CALL_1(SBI_REMOTE_SFENCE_VMA_ASID, hart_mask);
+}
+
+#endif
-- 
2.20.1


From f152febb2a97696f7c7e6df46bf585cfc962a835 Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:34 +0100
Subject: [PATCH 03/18] riscv: implement IPI platform functions using SBI

The supervisor binary interface (SBI) provides the necessary functions
to implement the platform IPI functions riscv_send_ipi() and
riscv_clear_ipi(). Use it to implement them.

This adds support for inter-processor interrupts (IPIs) on RISC-V CPUs
running in supervisor mode. Support for machine mode is already
available for CPUs that include the SiFive CLINT.

Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Atish Patra <atish.patra@wdc.com>
Tested-by: Bin Meng <bmeng.cn@gmail.com>
---
 arch/riscv/Kconfig       |  5 +++++
 arch/riscv/lib/Makefile  |  1 +
 arch/riscv/lib/sbi_ipi.c | 25 +++++++++++++++++++++++++
 3 files changed, 31 insertions(+)
 create mode 100644 arch/riscv/lib/sbi_ipi.c

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 4d7a115569..9da609b33b 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -139,4 +139,9 @@ config NR_CPUS
 	  Stack memory is pre-allocated. U-Boot must therefore know the
 	  maximum number of CPUs that may be present.
 
+config SBI_IPI
+	bool
+	default y if RISCV_SMODE
+	depends on SMP
+
 endmenu
diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile
index 19370f9749..35dbf643e4 100644
--- a/arch/riscv/lib/Makefile
+++ b/arch/riscv/lib/Makefile
@@ -13,6 +13,7 @@ obj-$(CONFIG_RISCV_RDTIME) += rdtime.o
 obj-$(CONFIG_SIFIVE_CLINT) += sifive_clint.o
 obj-y	+= interrupts.o
 obj-y	+= reset.o
+obj-$(CONFIG_SBI_IPI) += sbi_ipi.o
 obj-y   += setjmp.o
 obj-$(CONFIG_SMP) += smp.o
 
diff --git a/arch/riscv/lib/sbi_ipi.c b/arch/riscv/lib/sbi_ipi.c
new file mode 100644
index 0000000000..170346da68
--- /dev/null
+++ b/arch/riscv/lib/sbi_ipi.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019 Fraunhofer AISEC,
+ * Lukas Auer <lukas.auer@aisec.fraunhofer.de>
+ */
+
+#include <common.h>
+#include <asm/sbi.h>
+
+int riscv_send_ipi(int hart)
+{
+	ulong mask;
+
+	mask = 1UL << hart;
+	sbi_send_ipi(&mask);
+
+	return 0;
+}
+
+int riscv_clear_ipi(int hart)
+{
+	sbi_clear_ipi();
+
+	return 0;
+}
-- 
2.20.1


From 2503ccc55ff2031ae2ff476fb06f666e6d1c7a64 Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:35 +0100
Subject: [PATCH 04/18] riscv: delay initialization of caches and debug UART

Move the initialization of the caches and the debug UART until after
board_init_f_init_reserve. This is in preparation for SMP support, where
code prior to this point will be executed by all harts. This ensures
that initialization will only be performed once on the main hart running
U-Boot.

Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
---
 arch/riscv/cpu/start.S | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/arch/riscv/cpu/start.S b/arch/riscv/cpu/start.S
index 81ea52b170..a30f6f7194 100644
--- a/arch/riscv/cpu/start.S
+++ b/arch/riscv/cpu/start.S
@@ -45,10 +45,6 @@ _start:
 	/* mask all interrupts */
 	csrw	MODE_PREFIX(ie), zero
 
-	/* Enable cache */
-	jal	icache_enable
-	jal	dcache_enable
-
 /*
  * Set stackpointer in internal/ex RAM to call board_init_f
  */
@@ -57,10 +53,6 @@ call_board_init_f:
 	li	t1, CONFIG_SYS_INIT_SP_ADDR
 	and	sp, t1, t0		/* force 16 byte alignment */
 
-#ifdef CONFIG_DEBUG_UART
-	jal	debug_uart_init
-#endif
-
 call_board_init_f_0:
 	mv	a0, sp
 	jal	board_init_f_alloc_reserve
@@ -74,6 +66,14 @@ call_board_init_f_0:
 	/* save the boot hart id to global_data */
 	SREG	s0, GD_BOOT_HART(gp)
 
+	/* Enable cache */
+	jal	icache_enable
+	jal	dcache_enable
+
+#ifdef CONFIG_DEBUG_UART
+	jal	debug_uart_init
+#endif
+
 	mv	a0, zero		/* a0 <-- boot_flags = 0 */
 	la	t5, board_init_f
 	jr	t5			/* jump to board_init_f() */
-- 
2.20.1


From 1446b26f7652124f0e3e98c348cdbc4fc55eb0cb Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:36 +0100
Subject: [PATCH 05/18] riscv: save hart ID in register tp instead of s0

The hart ID passed by the previous boot stage is currently stored in
register s0. If we divert the control flow inside a function, which is
required as part of multi-hart support, the function epilog may not be
called, clobbering register s0. Save the hart ID in the unallocatable
register tp instead to protect the hart ID.

Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Rick Chen <rick@andestech.com>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
---
 arch/riscv/cpu/start.S | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/arch/riscv/cpu/start.S b/arch/riscv/cpu/start.S
index a30f6f7194..bcc0ff696d 100644
--- a/arch/riscv/cpu/start.S
+++ b/arch/riscv/cpu/start.S
@@ -36,7 +36,7 @@
 .globl _start
 _start:
 	/* save hart id and dtb pointer */
-	mv	s0, a0
+	mv	tp, a0
 	mv	s1, a1
 
 	la	t0, trap_entry
@@ -64,7 +64,7 @@ call_board_init_f_0:
 	jal	board_init_f_init_reserve
 
 	/* save the boot hart id to global_data */
-	SREG	s0, GD_BOOT_HART(gp)
+	SREG	tp, GD_BOOT_HART(gp)
 
 	/* Enable cache */
 	jal	icache_enable
-- 
2.20.1


From 3dea63c8445b25eb3de471410bbafcf54c9f0e9b Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:37 +0100
Subject: [PATCH 06/18] riscv: add support for multi-hart systems

On RISC-V, all harts boot independently. To be able to run on a
multi-hart system, U-Boot must be extended with the functionality to
manage all harts in the system. All harts entering U-Boot are registered
in the available_harts mask stored in global data. A hart lottery system
as used in the Linux kernel selects the hart U-Boot runs on. All other
harts are halted. U-Boot can delegate functions to them using
smp_call_function().

Every hart has a valid pointer to the global data structure and a 8KiB
stack by default. The stack size is set with CONFIG_STACK_SIZE_SHIFT.

Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Tested-by: Bin Meng <bmeng.cn@gmail.com>
---
 arch/riscv/Kconfig           |   4 ++
 arch/riscv/cpu/cpu.c         |   9 ++-
 arch/riscv/cpu/start.S       | 134 ++++++++++++++++++++++++++++++++++-
 arch/riscv/include/asm/csr.h |   1 +
 arch/riscv/lib/asm-offsets.c |   1 +
 5 files changed, 147 insertions(+), 2 deletions(-)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 9da609b33b..3a4470daf3 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -144,4 +144,8 @@ config SBI_IPI
 	default y if RISCV_SMODE
 	depends on SMP
 
+config STACK_SIZE_SHIFT
+	int
+	default 13
+
 endmenu
diff --git a/arch/riscv/cpu/cpu.c b/arch/riscv/cpu/cpu.c
index e662140427..c32de8a4c3 100644
--- a/arch/riscv/cpu/cpu.c
+++ b/arch/riscv/cpu/cpu.c
@@ -12,10 +12,17 @@
 #include <dm/uclass-internal.h>
 
 /*
- * prior_stage_fdt_address must be stored in the data section since it is used
+ * The variables here must be stored in the data section since they are used
  * before the bss section is available.
  */
 phys_addr_t prior_stage_fdt_address __attribute__((section(".data")));
+u32 hart_lottery __attribute__((section(".data"))) = 0;
+
+/*
+ * The main hart running U-Boot has acquired available_harts_lock until it has
+ * finished initialization of global data.
+ */
+u32 available_harts_lock = 1;
 
 static inline bool supports_extension(char ext)
 {
diff --git a/arch/riscv/cpu/start.S b/arch/riscv/cpu/start.S
index bcc0ff696d..f55b8cbc37 100644
--- a/arch/riscv/cpu/start.S
+++ b/arch/riscv/cpu/start.S
@@ -13,6 +13,7 @@
 #include <config.h>
 #include <common.h>
 #include <elf.h>
+#include <asm/csr.h>
 #include <asm/encoding.h>
 #include <generated/asm-offsets.h>
 
@@ -45,6 +46,23 @@ _start:
 	/* mask all interrupts */
 	csrw	MODE_PREFIX(ie), zero
 
+#ifdef CONFIG_SMP
+	/* check if hart is within range */
+	/* tp: hart id */
+	li	t0, CONFIG_NR_CPUS
+	bge	tp, t0, hart_out_of_bounds_loop
+#endif
+
+#ifdef CONFIG_SMP
+	/* set xSIE bit to receive IPIs */
+#ifdef CONFIG_RISCV_MMODE
+	li	t0, MIE_MSIE
+#else
+	li	t0, SIE_SSIE
+#endif
+	csrs	MODE_PREFIX(ie), t0
+#endif
+
 /*
  * Set stackpointer in internal/ex RAM to call board_init_f
  */
@@ -56,7 +74,30 @@ call_board_init_f:
 call_board_init_f_0:
 	mv	a0, sp
 	jal	board_init_f_alloc_reserve
+
+	/*
+	 * Set global data pointer here for all harts, uninitialized at this
+	 * point.
+	 */
+	mv	gp, a0
+
+	/* setup stack */
+#ifdef CONFIG_SMP
+	/* tp: hart id */
+	slli	t0, tp, CONFIG_STACK_SIZE_SHIFT
+	sub	sp, a0, t0
+#else
 	mv	sp, a0
+#endif
+
+	/*
+	 * Pick hart to initialize global data and run U-Boot. The other harts
+	 * wait for initialization to complete.
+	 */
+	la	t0, hart_lottery
+	li	s2, 1
+	amoswap.w s2, t1, 0(t0)
+	bnez	s2, wait_for_gd_init
 
 	la	t0, prior_stage_fdt_address
 	SREG	s1, 0(t0)
@@ -66,6 +107,33 @@ call_board_init_f_0:
 	/* save the boot hart id to global_data */
 	SREG	tp, GD_BOOT_HART(gp)
 
+	la	t0, available_harts_lock
+	fence	rw, w
+	amoswap.w zero, zero, 0(t0)
+
+wait_for_gd_init:
+	la	t0, available_harts_lock
+	li	t1, 1
+1:	amoswap.w t1, t1, 0(t0)
+	fence	r, rw
+	bnez	t1, 1b
+
+	/* register available harts in the available_harts mask */
+	li	t1, 1
+	sll	t1, t1, tp
+	LREG	t2, GD_AVAILABLE_HARTS(gp)
+	or	t2, t2, t1
+	SREG	t2, GD_AVAILABLE_HARTS(gp)
+
+	fence	rw, w
+	amoswap.w zero, zero, 0(t0)
+
+	/*
+	 * Continue on hart lottery winner, others branch to
+	 * secondary_hart_loop.
+	 */
+	bnez	s2, secondary_hart_loop
+
 	/* Enable cache */
 	jal	icache_enable
 	jal	dcache_enable
@@ -95,7 +163,14 @@ relocate_code:
  *Set up the stack
  */
 stack_setup:
+#ifdef CONFIG_SMP
+	/* tp: hart id */
+	slli	t0, tp, CONFIG_STACK_SIZE_SHIFT
+	sub	sp, s2, t0
+#else
 	mv	sp, s2
+#endif
+
 	la	t0, _start
 	sub	t6, s4, t0		/* t6 <- relocation offset */
 	beq	t0, s4, clear_bss	/* skip relocation */
@@ -175,13 +250,30 @@ clear_bss:
 	add	t0, t0, t6		/* t0 <- rel __bss_start in RAM */
 	la	t1, __bss_end		/* t1 <- rel __bss_end in FLASH */
 	add	t1, t1, t6		/* t1 <- rel __bss_end in RAM */
-	beq	t0, t1, call_board_init_r
+	beq	t0, t1, relocate_secondary_harts
 
 clbss_l:
 	SREG	zero, 0(t0)		/* clear loop... */
 	addi	t0, t0, REGBYTES
 	bne	t0, t1, clbss_l
 
+relocate_secondary_harts:
+#ifdef CONFIG_SMP
+	/* send relocation IPI */
+	la	t0, secondary_hart_relocate
+	add	a0, t0, t6
+
+	/* store relocation offset */
+	mv	s5, t6
+
+	mv	a1, s2
+	mv	a2, s3
+	jal	smp_call_function
+
+	/* restore relocation offset */
+	mv	t6, s5
+#endif
+
 /*
  * We are done. Do not return, instead branch to second part of board
  * initialization, now running from RAM.
@@ -202,3 +294,43 @@ call_board_init_r:
  * jump to it ...
  */
 	jr	t4			/* jump to board_init_r() */
+
+#ifdef CONFIG_SMP
+hart_out_of_bounds_loop:
+	/* Harts in this loop are out of bounds, increase CONFIG_NR_CPUS. */
+	wfi
+	j	hart_out_of_bounds_loop
+#endif
+
+#ifdef CONFIG_SMP
+/* SMP relocation entry */
+secondary_hart_relocate:
+	/* a1: new sp */
+	/* a2: new gd */
+	/* tp: hart id */
+
+	/* setup stack */
+	slli	t0, tp, CONFIG_STACK_SIZE_SHIFT
+	sub	sp, a1, t0
+
+	/* update global data pointer */
+	mv	gp, a2
+#endif
+
+secondary_hart_loop:
+	wfi
+
+#ifdef CONFIG_SMP
+	csrr	t0, MODE_PREFIX(ip)
+#ifdef CONFIG_RISCV_MMODE
+	andi	t0, t0, MIE_MSIE
+#else
+	andi	t0, t0, SIE_SSIE
+#endif
+	beqz	t0, secondary_hart_loop
+
+	mv	a0, tp
+	jal	handle_ipi
+#endif
+
+	j	secondary_hart_loop
diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h
index 86136f542c..644e6baa15 100644
--- a/arch/riscv/include/asm/csr.h
+++ b/arch/riscv/include/asm/csr.h
@@ -46,6 +46,7 @@
 #endif
 
 /* Interrupt Enable and Interrupt Pending flags */
+#define MIE_MSIE	_AC(0x00000008, UL) /* Software Interrupt Enable */
 #define SIE_SSIE	_AC(0x00000002, UL) /* Software Interrupt Enable */
 #define SIE_STIE	_AC(0x00000020, UL) /* Timer Interrupt Enable */
 
diff --git a/arch/riscv/lib/asm-offsets.c b/arch/riscv/lib/asm-offsets.c
index e0b71f5691..f998402bd1 100644
--- a/arch/riscv/lib/asm-offsets.c
+++ b/arch/riscv/lib/asm-offsets.c
@@ -14,6 +14,7 @@
 int main(void)
 {
 	DEFINE(GD_BOOT_HART, offsetof(gd_t, arch.boot_hart));
+	DEFINE(GD_AVAILABLE_HARTS, offsetof(gd_t, arch.available_harts));
 
 	return 0;
 }
-- 
2.20.1


From f28ad250e6ef95ca58490b4e8651749d4f7e5c06 Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:38 +0100
Subject: [PATCH 07/18] riscv: boot images passed to bootm on all harts

Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Tested-by: Bin Meng <bmeng.cn@gmail.com>
---
 arch/riscv/lib/bootm.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/lib/bootm.c b/arch/riscv/lib/bootm.c
index f36b8702ef..efbd3e23e7 100644
--- a/arch/riscv/lib/bootm.c
+++ b/arch/riscv/lib/bootm.c
@@ -13,6 +13,7 @@
 #include <image.h>
 #include <asm/byteorder.h>
 #include <asm/csr.h>
+#include <asm/smp.h>
 #include <dm/device.h>
 #include <dm/root.h>
 #include <u-boot/zlib.h>
@@ -81,6 +82,9 @@ static void boot_jump_linux(bootm_headers_t *images, int flag)
 {
 	void (*kernel)(ulong hart, void *dtb);
 	int fake = (flag & BOOTM_STATE_OS_FAKE_GO);
+#ifdef CONFIG_SMP
+	int ret;
+#endif
 
 	kernel = (void (*)(ulong, void *))images->ep;
 
@@ -92,8 +96,15 @@ static void boot_jump_linux(bootm_headers_t *images, int flag)
 	announce_and_cleanup(fake);
 
 	if (!fake) {
-		if (IMAGE_ENABLE_OF_LIBFDT && images->ft_len)
+		if (IMAGE_ENABLE_OF_LIBFDT && images->ft_len) {
+#ifdef CONFIG_SMP
+			ret = smp_call_function(images->ep,
+						(ulong)images->ft_addr, 0);
+			if (ret)
+				hang();
+#endif
 			kernel(gd->arch.boot_hart, images->ft_addr);
+		}
 	}
 }
 
-- 
2.20.1


From e04324025275dee6e3e9a968c8d12e98c9b47567 Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:39 +0100
Subject: [PATCH 08/18] riscv: do not rely on hart ID passed by previous boot
 stage

RISC-V U-Boot expects the hart ID to be passed to it via register a0 by
the previous boot stage. Machine mode firmware such as BBL and OpenSBI
do this when starting their payload (U-Boot) in supervisor mode. If
U-Boot is running in machine mode, this task must be handled by the boot
ROM. Explicitly populate register a0 with the hart ID from the mhartid
CSR to avoid possible problems on RISC-V processors with a boot ROM that
does not handle this task.

Suggested-by: Rick Chen <rick@andestech.com>
Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Atish Patra <atish.patra@wdc.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Tested-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Rick Chen <rick@andestech.com>
Tested-by: Rick Chen <rick@andestech.com>
---
 arch/riscv/cpu/start.S | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/arch/riscv/cpu/start.S b/arch/riscv/cpu/start.S
index f55b8cbc37..5ac899b141 100644
--- a/arch/riscv/cpu/start.S
+++ b/arch/riscv/cpu/start.S
@@ -36,6 +36,10 @@
 .section .text
 .globl _start
 _start:
+#ifdef CONFIG_RISCV_MMODE
+	csrr	a0, mhartid
+#endif
+
 	/* save hart id and dtb pointer */
 	mv	tp, a0
 	mv	s1, a1
-- 
2.20.1


From 8ac39e2974d9f1395969a500e692f45717c4ccf2 Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:40 +0100
Subject: [PATCH 09/18] riscv: hang if relocation of secondary harts fails

Print an error message and hang if smp_call_function() returns an error,
indicating that relocation of the secondary harts has failed.

Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Tested-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
---
 arch/riscv/cpu/start.S | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/cpu/start.S b/arch/riscv/cpu/start.S
index 5ac899b141..a4433fbd6b 100644
--- a/arch/riscv/cpu/start.S
+++ b/arch/riscv/cpu/start.S
@@ -33,6 +33,10 @@
 #define SYM_SIZE		0x18
 #endif
 
+.section .data
+secondary_harts_relocation_error:
+	.ascii "Relocation of secondary harts has failed, error %d\n"
+
 .section .text
 .globl _start
 _start:
@@ -274,8 +278,15 @@ relocate_secondary_harts:
 	mv	a2, s3
 	jal	smp_call_function
 
+	/* hang if relocation of secondary harts has failed */
+	beqz	a0, 1f
+	mv	a1, a0
+	la	a0, secondary_harts_relocation_error
+	jal	printf
+	jal	hang
+
 	/* restore relocation offset */
-	mv	t6, s5
+1:	mv	t6, s5
 #endif
 
 /*
-- 
2.20.1


From 0ed93a8c59d7c100241390d912329e3e45382977 Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:41 +0100
Subject: [PATCH 10/18] riscv: fu540: enable SMP

Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
---
 board/sifive/fu540/Kconfig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/board/sifive/fu540/Kconfig b/board/sifive/fu540/Kconfig
index 6be3d88144..f46437901d 100644
--- a/board/sifive/fu540/Kconfig
+++ b/board/sifive/fu540/Kconfig
@@ -38,5 +38,6 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	imply PHY_LIB
 	imply PHY_MSCC
 	imply SIFIVE_SERIAL
+	imply SMP
 
 endif
-- 
2.20.1


From d0a8fd3e4d2a5ab19b8f2d27d40dacb4942ba5a4 Mon Sep 17 00:00:00 2001
From: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Date: Sun, 17 Mar 2019 19:28:42 +0100
Subject: [PATCH 11/18] riscv: qemu: enable SMP

Signed-off-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Tested-by: Bin Meng <bmeng.cn@gmail.com>
---
 board/emulation/qemu-riscv/Kconfig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/board/emulation/qemu-riscv/Kconfig b/board/emulation/qemu-riscv/Kconfig
index 88d07d568e..cf057e7de6 100644
--- a/board/emulation/qemu-riscv/Kconfig
+++ b/board/emulation/qemu-riscv/Kconfig
@@ -34,5 +34,6 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	imply BOARD_LATE_INIT
 	imply OF_BOARD_SETUP
 	imply SIFIVE_SERIAL
+	imply SMP
 
 endif
-- 
2.20.1


From 0d389468e2144f3ba3bdbc566c05c0c05dc14fc6 Mon Sep 17 00:00:00 2001
From: Rick Chen <rick@andestech.com>
Date: Tue, 2 Apr 2019 15:56:39 +0800
Subject: [PATCH 12/18] riscv: Add a SYSCON driver for Andestech's PLIC

The Platform-Level Interrupt Controller (PLIC)
block holds memory-mapped claim and pending registers
associated with software interrupt. It is required
for handling IPI.

Signed-off-by: Rick Chen <rick@andestech.com>
Cc: Greentime Hu <greentime@andestech.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
---
 arch/riscv/Kconfig                   |   9 +++
 arch/riscv/include/asm/global_data.h |   3 +
 arch/riscv/include/asm/syscon.h      |   3 +-
 arch/riscv/lib/Makefile              |   1 +
 arch/riscv/lib/andes_plic.c          | 113 +++++++++++++++++++++++++++
 5 files changed, 127 insertions(+), 2 deletions(-)
 create mode 100644 arch/riscv/lib/andes_plic.c

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 3a4470daf3..511768befc 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -109,6 +109,15 @@ config SIFIVE_CLINT
 	  The SiFive CLINT block holds memory-mapped control and status registers
 	  associated with software and timer interrupts.
 
+config ANDES_PLIC
+	bool
+	depends on RISCV_MMODE
+	select REGMAP
+	select SYSCON
+	help
+	  The Andes PLIC block holds memory-mapped claim and pending registers
+	  associated with software interrupt.
+
 config RISCV_RDTIME
 	bool
 	default y if RISCV_SMODE
diff --git a/arch/riscv/include/asm/global_data.h b/arch/riscv/include/asm/global_data.h
index 80e3165e39..b86791094b 100644
--- a/arch/riscv/include/asm/global_data.h
+++ b/arch/riscv/include/asm/global_data.h
@@ -18,6 +18,9 @@ struct arch_global_data {
 #ifdef CONFIG_SIFIVE_CLINT
 	void __iomem *clint;	/* clint base address */
 #endif
+#ifdef CONFIG_ANDES_PLIC
+	void __iomem *plic;	/* plic base address */
+#endif
 #ifdef CONFIG_SMP
 	struct ipi_data ipi[CONFIG_NR_CPUS];
 #endif
diff --git a/arch/riscv/include/asm/syscon.h b/arch/riscv/include/asm/syscon.h
index d311ee6b45..a086208261 100644
--- a/arch/riscv/include/asm/syscon.h
+++ b/arch/riscv/include/asm/syscon.h
@@ -8,12 +8,11 @@
 
 /*
  * System controllers in a RISC-V system
- *
- * So far only SiFive's Core Local Interruptor (CLINT) is defined.
  */
 enum {
 	RISCV_NONE,
 	RISCV_SYSCON_CLINT,	/* Core Local Interruptor (CLINT) */
+	RISCV_SYSCON_PLIC,	/* Platform Level Interrupt Controller (PLIC) */
 };
 
 #endif /* _ASM_SYSCON_H */
diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile
index 35dbf643e4..1bf554bad5 100644
--- a/arch/riscv/lib/Makefile
+++ b/arch/riscv/lib/Makefile
@@ -11,6 +11,7 @@ obj-$(CONFIG_CMD_GO) += boot.o
 obj-y	+= cache.o
 obj-$(CONFIG_RISCV_RDTIME) += rdtime.o
 obj-$(CONFIG_SIFIVE_CLINT) += sifive_clint.o
+obj-$(CONFIG_ANDES_PLIC) += andes_plic.o
 obj-y	+= interrupts.o
 obj-y	+= reset.o
 obj-$(CONFIG_SBI_IPI) += sbi_ipi.o
diff --git a/arch/riscv/lib/andes_plic.c b/arch/riscv/lib/andes_plic.c
new file mode 100644
index 0000000000..2ffe49ac90
--- /dev/null
+++ b/arch/riscv/lib/andes_plic.c
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019, Rick Chen <rick@andestech.com>
+ *
+ * U-Boot syscon driver for Andes's Platform Level Interrupt Controller (PLIC).
+ * The PLIC block holds memory-mapped claim and pending registers
+ * associated with software interrupt.
+ */
+
+#include <common.h>
+#include <dm.h>
+#include <dm/device-internal.h>
+#include <dm/lists.h>
+#include <dm/uclass-internal.h>
+#include <regmap.h>
+#include <syscon.h>
+#include <asm/io.h>
+#include <asm/syscon.h>
+#include <cpu.h>
+
+/* pending register */
+#define PENDING_REG(base, hart)	((ulong)(base) + 0x1000 + (hart) * 8)
+/* enable register */
+#define ENABLE_REG(base, hart)	((ulong)(base) + 0x2000 + (hart) * 0x80)
+/* claim register */
+#define CLAIM_REG(base, hart)	((ulong)(base) + 0x200004 + (hart) * 0x1000)
+
+#define ENABLE_HART_IPI         (0x80808080)
+#define SEND_IPI_TO_HART(hart)  (0x80 >> (hart))
+
+DECLARE_GLOBAL_DATA_PTR;
+static int init_plic(void);
+
+#define PLIC_BASE_GET(void)						\
+	do {								\
+		long *ret;						\
+									\
+		if (!gd->arch.plic) {					\
+			ret = syscon_get_first_range(RISCV_SYSCON_PLIC); \
+			if (IS_ERR(ret))				\
+				return PTR_ERR(ret);			\
+			gd->arch.plic = ret;				\
+			init_plic();					\
+		}							\
+	} while (0)
+
+static int enable_ipi(int harts)
+{
+	int i;
+	int en = ENABLE_HART_IPI;
+
+	for (i = 0; i < harts; i++) {
+		en = en >> i;
+		writel(en, (void __iomem *)ENABLE_REG(gd->arch.plic, i));
+	}
+
+	return 0;
+}
+
+static int init_plic(void)
+{
+	struct udevice *dev;
+	int ret;
+
+	ret = uclass_find_first_device(UCLASS_CPU, &dev);
+	if (ret)
+		return ret;
+
+	if (ret == 0 && dev) {
+		ret = cpu_get_count(dev);
+		if (ret < 0)
+			return ret;
+
+		enable_ipi(ret);
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+int riscv_send_ipi(int hart)
+{
+	PLIC_BASE_GET();
+
+	writel(SEND_IPI_TO_HART(hart),
+	       (void __iomem *)PENDING_REG(gd->arch.plic, gd->arch.boot_hart));
+
+	return 0;
+}
+
+int riscv_clear_ipi(int hart)
+{
+	u32 source_id;
+
+	PLIC_BASE_GET();
+
+	source_id = readl((void __iomem *)CLAIM_REG(gd->arch.plic, hart));
+	writel(source_id, (void __iomem *)CLAIM_REG(gd->arch.plic, hart));
+
+	return 0;
+}
+
+static const struct udevice_id andes_plic_ids[] = {
+	{ .compatible = "riscv,plic1", .data = RISCV_SYSCON_PLIC },
+	{ }
+};
+
+U_BOOT_DRIVER(andes_plic) = {
+	.name		= "andes_plic",
+	.id		= UCLASS_SYSCON,
+	.of_match	= andes_plic_ids,
+	.flags		= DM_FLAG_PRE_RELOC,
+};
-- 
2.20.1


From a1f24875c346a1bf8940b793a27364631d9aabf7 Mon Sep 17 00:00:00 2001
From: Rick Chen <rick@andestech.com>
Date: Tue, 2 Apr 2019 15:56:40 +0800
Subject: [PATCH 13/18] riscv: Add a SYSCON driver for Andestech's PLMT

The platform-Level Machine Timer (PLMT) block
holds memory-mapped mtime register associated
with timer tick.

This driver implements the riscv_get_time() which
is required by the generic RISC-V timer driver.

Signed-off-by: Rick Chen <rick@andestech.com>
Cc: Greentime Hu <greentime@andestech.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
---
 arch/riscv/Kconfig                   |  9 +++++
 arch/riscv/include/asm/global_data.h |  3 ++
 arch/riscv/include/asm/syscon.h      |  1 +
 arch/riscv/lib/Makefile              |  1 +
 arch/riscv/lib/andes_plmt.c          | 53 ++++++++++++++++++++++++++++
 5 files changed, 67 insertions(+)
 create mode 100644 arch/riscv/lib/andes_plmt.c

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 511768befc..ae8ff7b765 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -118,6 +118,15 @@ config ANDES_PLIC
 	  The Andes PLIC block holds memory-mapped claim and pending registers
 	  associated with software interrupt.
 
+config ANDES_PLMT
+	bool
+	depends on RISCV_MMODE
+	select REGMAP
+	select SYSCON
+	help
+	  The Andes PLMT block holds memory-mapped mtime register
+	  associated with timer tick.
+
 config RISCV_RDTIME
 	bool
 	default y if RISCV_SMODE
diff --git a/arch/riscv/include/asm/global_data.h b/arch/riscv/include/asm/global_data.h
index b86791094b..dffcd45bf0 100644
--- a/arch/riscv/include/asm/global_data.h
+++ b/arch/riscv/include/asm/global_data.h
@@ -21,6 +21,9 @@ struct arch_global_data {
 #ifdef CONFIG_ANDES_PLIC
 	void __iomem *plic;	/* plic base address */
 #endif
+#ifdef CONFIG_ANDES_PLMT
+	void __iomem *plmt;	/* plmt base address */
+#endif
 #ifdef CONFIG_SMP
 	struct ipi_data ipi[CONFIG_NR_CPUS];
 #endif
diff --git a/arch/riscv/include/asm/syscon.h b/arch/riscv/include/asm/syscon.h
index a086208261..26a008ca59 100644
--- a/arch/riscv/include/asm/syscon.h
+++ b/arch/riscv/include/asm/syscon.h
@@ -13,6 +13,7 @@ enum {
 	RISCV_NONE,
 	RISCV_SYSCON_CLINT,	/* Core Local Interruptor (CLINT) */
 	RISCV_SYSCON_PLIC,	/* Platform Level Interrupt Controller (PLIC) */
+	RISCV_SYSCON_PLMT,	/* Platform Level Machine Timer (PLMT) */
 };
 
 #endif /* _ASM_SYSCON_H */
diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile
index 1bf554bad5..1c332db436 100644
--- a/arch/riscv/lib/Makefile
+++ b/arch/riscv/lib/Makefile
@@ -12,6 +12,7 @@ obj-y	+= cache.o
 obj-$(CONFIG_RISCV_RDTIME) += rdtime.o
 obj-$(CONFIG_SIFIVE_CLINT) += sifive_clint.o
 obj-$(CONFIG_ANDES_PLIC) += andes_plic.o
+obj-$(CONFIG_ANDES_PLMT) += andes_plmt.o
 obj-y	+= interrupts.o
 obj-y	+= reset.o
 obj-$(CONFIG_SBI_IPI) += sbi_ipi.o
diff --git a/arch/riscv/lib/andes_plmt.c b/arch/riscv/lib/andes_plmt.c
new file mode 100644
index 0000000000..84f4607500
--- /dev/null
+++ b/arch/riscv/lib/andes_plmt.c
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019, Rick Chen <rick@andestech.com>
+ *
+ * U-Boot syscon driver for Andes's Platform Level Machine Timer (PLMT).
+ * The PLMT block holds memory-mapped mtime register
+ * associated with timer tick.
+ */
+
+#include <common.h>
+#include <dm.h>
+#include <regmap.h>
+#include <syscon.h>
+#include <asm/io.h>
+#include <asm/syscon.h>
+
+/* mtime register */
+#define MTIME_REG(base)			((ulong)(base))
+
+DECLARE_GLOBAL_DATA_PTR;
+
+#define PLMT_BASE_GET(void)						\
+	do {								\
+		long *ret;						\
+									\
+		if (!gd->arch.plmt) {					\
+			ret = syscon_get_first_range(RISCV_SYSCON_PLMT); \
+			if (IS_ERR(ret))				\
+				return PTR_ERR(ret);			\
+			gd->arch.plmt = ret;				\
+		}							\
+	} while (0)
+
+int riscv_get_time(u64 *time)
+{
+	PLMT_BASE_GET();
+
+	*time = readq((void __iomem *)MTIME_REG(gd->arch.plmt));
+
+	return 0;
+}
+
+static const struct udevice_id andes_plmt_ids[] = {
+	{ .compatible = "riscv,plmt0", .data = RISCV_SYSCON_PLMT },
+	{ }
+};
+
+U_BOOT_DRIVER(andes_plmt) = {
+	.name		= "andes_plmt",
+	.id		= UCLASS_SYSCON,
+	.of_match	= andes_plmt_ids,
+	.flags		= DM_FLAG_PRE_RELOC,
+};
-- 
2.20.1


From 8848474c5e9093ac27f6b7cc8be156629c7d0bad Mon Sep 17 00:00:00 2001
From: Rick Chen <rick@andestech.com>
Date: Tue, 2 Apr 2019 15:56:41 +0800
Subject: [PATCH 14/18] riscv: ax25: Add platform-specific Kconfig options

Add ax25 RISC-V platform-specific Kconfig options,
to include CPU and timer drivers. Also disable
ATCPIT100 SoC timer and replace by PLMT.

Signed-off-by: Rick Chen <rick@andestech.com>
Cc: Greentime Hu <greentime@andestech.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
---
 arch/riscv/cpu/ax25/Kconfig  | 6 ++++++
 configs/ae350_rv32_defconfig | 1 -
 configs/ae350_rv64_defconfig | 1 -
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/arch/riscv/cpu/ax25/Kconfig b/arch/riscv/cpu/ax25/Kconfig
index e9dbca2fae..68bd4e9419 100644
--- a/arch/riscv/cpu/ax25/Kconfig
+++ b/arch/riscv/cpu/ax25/Kconfig
@@ -1,5 +1,11 @@
 config RISCV_NDS
 	bool
+	select ARCH_EARLY_INIT_R
+	imply CPU
+	imply CPU_RISCV
+	imply RISCV_TIMER
+	imply ANDES_PLIC if RISCV_MMODE
+	imply ANDES_PLMT if RISCV_MMODE
 	help
 	  Run U-Boot on AndeStar V5 platforms and use some specific features
 	  which are provided by Andes Technology AndeStar V5 families.
diff --git a/configs/ae350_rv32_defconfig b/configs/ae350_rv32_defconfig
index 6e0be88d7c..f02945599f 100644
--- a/configs/ae350_rv32_defconfig
+++ b/configs/ae350_rv32_defconfig
@@ -34,4 +34,3 @@ CONFIG_BAUDRATE=38400
 CONFIG_SYS_NS16550=y
 CONFIG_SPI=y
 CONFIG_ATCSPI200_SPI=y
-CONFIG_ATCPIT100_TIMER=y
diff --git a/configs/ae350_rv64_defconfig b/configs/ae350_rv64_defconfig
index b472a76d17..98635a2ff9 100644
--- a/configs/ae350_rv64_defconfig
+++ b/configs/ae350_rv64_defconfig
@@ -35,4 +35,3 @@ CONFIG_BAUDRATE=38400
 CONFIG_SYS_NS16550=y
 CONFIG_SPI=y
 CONFIG_ATCSPI200_SPI=y
-CONFIG_ATCPIT100_TIMER=y
-- 
2.20.1


From dda00ae4ef357233a72c74e6c02d27b70c844422 Mon Sep 17 00:00:00 2001
From: Rick Chen <rick@andestech.com>
Date: Tue, 2 Apr 2019 15:56:42 +0800
Subject: [PATCH 15/18] riscv: ax25: Andes specific cache shall only support in
 M-mode

Limit the cache configuration only can be supported in M mode.
It can not be manipulated in S mode.

Signed-off-by: Rick Chen <rick@andestech.com>
Cc: Greentime Hu <greentime@andestech.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
---
 arch/riscv/cpu/ax25/Kconfig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/arch/riscv/cpu/ax25/Kconfig b/arch/riscv/cpu/ax25/Kconfig
index 68bd4e9419..6b4b92e692 100644
--- a/arch/riscv/cpu/ax25/Kconfig
+++ b/arch/riscv/cpu/ax25/Kconfig
@@ -14,6 +14,7 @@ if RISCV_NDS
 
 config RISCV_NDS_CACHE
 	bool "AndeStar V5 families specific cache support"
+	depends on RISCV_MMODE
 	help
 	  Provide Andes Technology AndeStar V5 families specific cache support.
 
-- 
2.20.1


From a1ce531ab24b1ef5e69dbf29ecbfc6898b454ec1 Mon Sep 17 00:00:00 2001
From: Rick Chen <rick@andestech.com>
Date: Tue, 2 Apr 2019 15:56:43 +0800
Subject: [PATCH 16/18] riscv: dts: ae350 support SMP

Signed-off-by: Rick Chen <rick@andestech.com>
Cc: Greentime Hu <greentime@andestech.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
---
 arch/riscv/dts/ae350_32.dts | 81 +++++++++++++++++++++++++++----------
 arch/riscv/dts/ae350_64.dts | 81 +++++++++++++++++++++++++++----------
 2 files changed, 118 insertions(+), 44 deletions(-)

diff --git a/arch/riscv/dts/ae350_32.dts b/arch/riscv/dts/ae350_32.dts
index 0679827313..2ec01a5ce7 100644
--- a/arch/riscv/dts/ae350_32.dts
+++ b/arch/riscv/dts/ae350_32.dts
@@ -26,16 +26,49 @@
 			status = "okay";
 			compatible = "riscv";
 			riscv,isa = "rv32imafdc";
+			riscv,priv-major = <1>;
+			riscv,priv-minor = <10>;
 			mmu-type = "riscv,sv32";
 			clock-frequency = <60000000>;
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <32>;
 			d-cache-size = <0x8000>;
 			d-cache-line-size = <32>;
+			next-level-cache = <&L2>;
 			CPU0_intc: interrupt-controller {
 				#interrupt-cells = <1>;
 				interrupt-controller;
 				compatible = "riscv,cpu-intc";
 			};
 		};
+		CPU1: cpu@1 {
+			device_type = "cpu";
+			reg = <1>;
+			status = "okay";
+			compatible = "riscv";
+			riscv,isa = "rv32imafdc";
+			riscv,priv-major = <1>;
+			riscv,priv-minor = <10>;
+			mmu-type = "riscv,sv32";
+			clock-frequency = <60000000>;
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <32>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <32>;
+			next-level-cache = <&L2>;
+			CPU1_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				interrupt-controller;
+				compatible = "riscv,cpu-intc";
+			};
+		};
+
+		L2: l2-cache@e0500000 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <0x40000>;
+			reg = <0x0 0xe0500000 0x0 0x40000>;
+		};
 	};
 
 	memory@0 {
@@ -46,32 +79,32 @@
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
-		compatible = "andestech,riscv-ae350-soc";
+		compatible = "simple-bus";
 		ranges;
 
-	plic0: interrupt-controller@e4000000 {
-		compatible = "riscv,plic0";
-		#address-cells = <1>;
-		#interrupt-cells = <1>;
-		interrupt-controller;
-		reg = <0xe4000000 0x2000000>;
-		riscv,ndev=<71>;
-		interrupts-extended = <&CPU0_intc 11 &CPU0_intc 9>;
-	};
+		plic0: interrupt-controller@e4000000 {
+			compatible = "riscv,plic0";
+			#address-cells = <1>;
+			#interrupt-cells = <1>;
+			interrupt-controller;
+			reg = <0xe4000000 0x2000000>;
+			riscv,ndev=<71>;
+			interrupts-extended = <&CPU0_intc 11 &CPU0_intc 9 &CPU1_intc 11 &CPU1_intc 9>;
+		};
 
-	plic1: interrupt-controller@e6400000 {
-		compatible = "riscv,plic1";
-		#address-cells = <1>;
-		#interrupt-cells = <1>;
-		interrupt-controller;
-		reg = <0xe6400000 0x400000>;
-		riscv,ndev=<1>;
-		interrupts-extended = <&CPU0_intc 3>;
-	};
+		plic1: interrupt-controller@e6400000 {
+			compatible = "riscv,plic1";
+			#address-cells = <1>;
+			#interrupt-cells = <1>;
+			interrupt-controller;
+			reg = <0xe6400000 0x400000>;
+			riscv,ndev=<2>;
+			interrupts-extended = <&CPU0_intc 3 &CPU1_intc 3>;
+		};
 
-	plmt0@e6000000 {
-		compatible = "riscv,plmt0";
-			interrupts-extended = <&CPU0_intc 7>;
+		plmt0@e6000000 {
+			compatible = "riscv,plmt0";
+			interrupts-extended = <&CPU0_intc 7 &CPU1_intc 7>;
 			reg = <0xe6000000 0x100000>;
 		};
 	};
@@ -146,6 +179,10 @@
 		interrupt-parent = <&plic0>;
 	};
 
+	pmu {
+		compatible = "riscv,base-pmu";
+	};
+
 	virtio_mmio@fe007000 {
 		interrupts = <0x17 0x4>;
 		interrupt-parent = <0x2>;
diff --git a/arch/riscv/dts/ae350_64.dts b/arch/riscv/dts/ae350_64.dts
index e48c298645..cde5cdeff8 100644
--- a/arch/riscv/dts/ae350_64.dts
+++ b/arch/riscv/dts/ae350_64.dts
@@ -26,16 +26,49 @@
 			status = "okay";
 			compatible = "riscv";
 			riscv,isa = "rv64imafdc";
+			riscv,priv-major = <1>;
+			riscv,priv-minor = <10>;
 			mmu-type = "riscv,sv39";
 			clock-frequency = <60000000>;
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <32>;
 			d-cache-size = <0x8000>;
 			d-cache-line-size = <32>;
+			next-level-cache = <&L2>;
 			CPU0_intc: interrupt-controller {
 				#interrupt-cells = <1>;
 				interrupt-controller;
 				compatible = "riscv,cpu-intc";
 			};
 		};
+		CPU1: cpu@1 {
+			device_type = "cpu";
+			reg = <1>;
+			status = "okay";
+			compatible = "riscv";
+			riscv,isa = "rv64imafdc";
+			riscv,priv-major = <1>;
+			riscv,priv-minor = <10>;
+			mmu-type = "riscv,sv39";
+			clock-frequency = <60000000>;
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <32>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <32>;
+			next-level-cache = <&L2>;
+			CPU1_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				interrupt-controller;
+				compatible = "riscv,cpu-intc";
+			};
+		};
+
+		L2: l2-cache@e0500000 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <0x40000>;
+			reg = <0x0 0xe0500000 0x0 0x40000>;
+		};
 	};
 
 	memory@0 {
@@ -46,32 +79,32 @@
 	soc {
 		#address-cells = <2>;
 		#size-cells = <2>;
-		compatible = "andestech,riscv-ae350-soc";
+		compatible = "simple-bus";
 		ranges;
 
-	plic0: interrupt-controller@e4000000 {
-		compatible = "riscv,plic0";
-		#address-cells = <2>;
-		#interrupt-cells = <2>;
-		interrupt-controller;
-		reg = <0x0 0xe4000000 0x0 0x2000000>;
-		riscv,ndev=<71>;
-		interrupts-extended = <&CPU0_intc 11 &CPU0_intc 9>;
-	};
+		plic0: interrupt-controller@e4000000 {
+			compatible = "riscv,plic0";
+			#address-cells = <2>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			reg = <0x0 0xe4000000 0x0 0x2000000>;
+			riscv,ndev=<71>;
+			interrupts-extended = <&CPU0_intc 11 &CPU0_intc 9 &CPU1_intc 11 &CPU1_intc 9>;
+		};
 
-	plic1: interrupt-controller@e6400000 {
-		compatible = "riscv,plic1";
-		#address-cells = <2>;
-		#interrupt-cells = <2>;
-		interrupt-controller;
-		reg = <0x0 0xe6400000 0x0 0x400000>;
-		riscv,ndev=<1>;
-		interrupts-extended = <&CPU0_intc 3>;
-	};
+		plic1: interrupt-controller@e6400000 {
+			compatible = "riscv,plic1";
+			#address-cells = <2>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			reg = <0x0 0xe6400000 0x0 0x400000>;
+			riscv,ndev=<2>;
+			interrupts-extended = <&CPU0_intc 3 &CPU1_intc 3>;
+		};
 
-	plmt0@e6000000 {
-		compatible = "riscv,plmt0";
-			interrupts-extended = <&CPU0_intc 7>;
+		plmt0@e6000000 {
+			compatible = "riscv,plmt0";
+			interrupts-extended = <&CPU0_intc 7 &CPU1_intc 7>;
 			reg = <0x0 0xe6000000 0x0 0x100000>;
 		};
 	};
@@ -146,6 +179,10 @@
 		interrupt-parent = <&plic0>;
 	};
 
+	pmu {
+		compatible = "riscv,base-pmu";
+	};
+
 	virtio_mmio@fe007000 {
 		interrupts = <0x17 0x4>;
 		interrupt-parent = <0x2>;
-- 
2.20.1


From 076b845893f3bb59dfad80b012d28191c19e2f7b Mon Sep 17 00:00:00 2001
From: Rick Chen <rick@andestech.com>
Date: Tue, 2 Apr 2019 15:56:44 +0800
Subject: [PATCH 17/18] riscv: ae350: enable SMP

Signed-off-by: Rick Chen <rick@andestech.com>
Cc: Greentime Hu <greentime@andestech.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
---
 board/AndesTech/ax25-ae350/Kconfig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/board/AndesTech/ax25-ae350/Kconfig b/board/AndesTech/ax25-ae350/Kconfig
index 44cb302f70..5e682b679d 100644
--- a/board/AndesTech/ax25-ae350/Kconfig
+++ b/board/AndesTech/ax25-ae350/Kconfig
@@ -24,5 +24,6 @@ config ENV_OFFSET
 config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select RISCV_NDS
+	imply SMP
 
 endif
-- 
2.20.1


From 48b90d9db5d32e587901c4f33175488dd20fe0a5 Mon Sep 17 00:00:00 2001
From: Rick Chen <rick@andestech.com>
Date: Wed, 3 Apr 2019 10:43:37 +0800
Subject: [PATCH 18/18] riscv: dts: fix CONFIG_DEFAULT_DEVICE_TREE failure

It occurs since commit 27cb7300ffda
("Ensure device tree DTS is compiled").

More details can refer to
89c2b5c02049aea746b1edee0b4e1d8519dec2f4
ARM: fix arch/arm/dts/Makefile

Signed-off-by: Rick Chen <rick@andestech.com>
Cc: Greentime Hu <greentime@andestech.com>
Reviewed-by: Bin Meng <bmeng.cn@gmail.com>
Reviewed-by: Lukas Auer <lukas.auer@aisec.fraunhofer.de>
---
 arch/riscv/dts/Makefile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/arch/riscv/dts/Makefile b/arch/riscv/dts/Makefile
index b400defb38..f9cd606a9a 100644
--- a/arch/riscv/dts/Makefile
+++ b/arch/riscv/dts/Makefile
@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0+
 
+dtb-$(CONFIG_TARGET_AX25_AE350) += ae350_32.dtb ae350_64.dtb
+
 targets += $(dtb-y)
 
 DTC_FLAGS += -R 4 -p 0x1000
-- 
2.20.1