From 249c543b97e1409b13fb9539b2f880e58ddd87cf Mon Sep 17 00:00:00 2001 From: Martin Schwidefsky Date: Tue, 5 Jan 2016 13:29:38 +0100 Subject: [PATCH] s390/vdso: optimize getcpu system call Add the CPU number to the per-cpu vdso data page and add the __kernel_getcpu function to the vdso object to retrieve the CPU number in user space. Suggested-by: Heiko Carstens Reviewed-by: Heiko Carstens Signed-off-by: Martin Schwidefsky --- arch/s390/include/asm/vdso.h | 2 ++ arch/s390/kernel/asm-offsets.c | 2 ++ arch/s390/kernel/vdso.c | 9 +++++- arch/s390/kernel/vdso32/Makefile | 2 +- arch/s390/kernel/vdso32/getcpu.S | 43 ++++++++++++++++++++++++++++ arch/s390/kernel/vdso32/vdso32.lds.S | 1 + arch/s390/kernel/vdso64/Makefile | 2 +- arch/s390/kernel/vdso64/getcpu.S | 42 +++++++++++++++++++++++++++ arch/s390/kernel/vdso64/vdso64.lds.S | 1 + 9 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 arch/s390/kernel/vdso32/getcpu.S create mode 100644 arch/s390/kernel/vdso64/getcpu.S diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h index f9b02d300d60..d0a2dbf2433d 100644 --- a/arch/s390/include/asm/vdso.h +++ b/arch/s390/include/asm/vdso.h @@ -38,6 +38,8 @@ struct vdso_data { struct vdso_per_cpu_data { __u64 ectg_timer_base; __u64 ectg_user_time; + __u32 cpu_nr; + __u32 node_id; }; extern struct vdso_data *vdso_data; diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index d5916ef9c619..53bbc9e8b281 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -80,6 +80,8 @@ int main(void) OFFSET(__VDSO_TK_SHIFT, vdso_data, tk_shift); OFFSET(__VDSO_ECTG_BASE, vdso_per_cpu_data, ectg_timer_base); OFFSET(__VDSO_ECTG_USER, vdso_per_cpu_data, ectg_user_time); + OFFSET(__VDSO_CPU_NR, vdso_per_cpu_data, cpu_nr); + OFFSET(__VDSO_NODE_ID, vdso_per_cpu_data, node_id); BLANK(); /* constants used by the vdso */ DEFINE(__CLOCK_REALTIME, CLOCK_REALTIME); diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index 604f4a68b524..94495cac8be3 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -80,7 +80,7 @@ struct vdso_data *vdso_data = &vdso_data_store.data; /* * Setup vdso data page. */ -static void vdso_init_data(struct vdso_data *vd) +static void __init vdso_init_data(struct vdso_data *vd) { vd->ectg_available = test_facility(31); } @@ -93,6 +93,7 @@ static void vdso_init_data(struct vdso_data *vd) int vdso_alloc_per_cpu(struct lowcore *lowcore) { unsigned long segment_table, page_table, page_frame; + struct vdso_per_cpu_data *vd; u32 *psal, *aste; int i; @@ -107,6 +108,12 @@ int vdso_alloc_per_cpu(struct lowcore *lowcore) if (!segment_table || !page_table || !page_frame) goto out; + /* Initialize per-cpu vdso data page */ + vd = (struct vdso_per_cpu_data *) page_frame; + vd->cpu_nr = lowcore->cpu_nr; + vd->node_id = cpu_to_node(vd->cpu_nr); + + /* Set up access register mode page table */ clear_table((unsigned long *) segment_table, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE << SEGMENT_ORDER); clear_table((unsigned long *) page_table, _PAGE_INVALID, diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile index ee8a18e50a25..f9c459586649 100644 --- a/arch/s390/kernel/vdso32/Makefile +++ b/arch/s390/kernel/vdso32/Makefile @@ -1,6 +1,6 @@ # List of files in the vdso, has to be asm only for now -obj-vdso32 = gettimeofday.o clock_getres.o clock_gettime.o note.o +obj-vdso32 = gettimeofday.o clock_getres.o clock_gettime.o note.o getcpu.o # Build rules diff --git a/arch/s390/kernel/vdso32/getcpu.S b/arch/s390/kernel/vdso32/getcpu.S new file mode 100644 index 000000000000..c1ed0b72030f --- /dev/null +++ b/arch/s390/kernel/vdso32/getcpu.S @@ -0,0 +1,43 @@ +/* + * Userland implementation of getcpu() for 32 bits processes in a + * s390 kernel for use in the vDSO + * + * Copyright IBM Corp. 2016 + * Author(s): Martin Schwidefsky + */ +#include +#include + + .text + .align 4 + .globl __kernel_getcpu + .type __kernel_getcpu,@function +__kernel_getcpu: + .cfi_startproc + ear %r1,%a4 + lhi %r4,1 + sll %r4,24 + sar %a4,%r4 + la %r4,0 + epsw %r0,0 + sacf 512 + l %r5,__VDSO_CPU_NR(%r4) + l %r4,__VDSO_NODE_ID(%r4) + tml %r0,0x4000 + jo 1f + tml %r0,0x8000 + jno 0f + sacf 256 + j 1f +0: sacf 0 +1: sar %a4,%r1 + ltr %r2,%r2 + jz 2f + st %r5,0(%r2) +2: ltr %r3,%r3 + jz 3f + st %r4,0(%r3) +3: lhi %r2,0 + br %r14 + .cfi_endproc + .size __kernel_getcpu,.-__kernel_getcpu diff --git a/arch/s390/kernel/vdso32/vdso32.lds.S b/arch/s390/kernel/vdso32/vdso32.lds.S index a8c379fa1247..8f048c2d6d13 100644 --- a/arch/s390/kernel/vdso32/vdso32.lds.S +++ b/arch/s390/kernel/vdso32/vdso32.lds.S @@ -132,6 +132,7 @@ VERSION __kernel_gettimeofday; __kernel_clock_gettime; __kernel_clock_getres; + __kernel_getcpu; local: *; }; diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index c4b03f9ed228..058659c1b8cf 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -1,6 +1,6 @@ # List of files in the vdso, has to be asm only for now -obj-vdso64 = gettimeofday.o clock_getres.o clock_gettime.o note.o +obj-vdso64 = gettimeofday.o clock_getres.o clock_gettime.o note.o getcpu.o # Build rules diff --git a/arch/s390/kernel/vdso64/getcpu.S b/arch/s390/kernel/vdso64/getcpu.S new file mode 100644 index 000000000000..4cbe98291931 --- /dev/null +++ b/arch/s390/kernel/vdso64/getcpu.S @@ -0,0 +1,42 @@ +/* + * Userland implementation of getcpu() for 64 bits processes in a + * s390 kernel for use in the vDSO + * + * Copyright IBM Corp. 2016 + * Author(s): Martin Schwidefsky + */ +#include +#include + + .text + .align 4 + .globl __kernel_getcpu + .type __kernel_getcpu,@function +__kernel_getcpu: + .cfi_startproc + ear %r1,%a4 + llilh %r4,0x0100 + sar %a4,%r4 + la %r4,0 + epsw %r0,0 + sacf 512 + l %r5,__VDSO_CPU_NR(%r4) + l %r4,__VDSO_NODE_ID(%r4) + tml %r0,0x4000 + jo 1f + tml %r0,0x8000 + jno 0f + sacf 256 + j 1f +0: sacf 0 +1: sar %a4,%r1 + ltgr %r2,%r2 + jz 2f + st %r5,0(%r2) +2: ltgr %r3,%r3 + jz 3f + st %r4,0(%r3) +3: lghi %r2,0 + br %r14 + .cfi_endproc + .size __kernel_getcpu,.-__kernel_getcpu diff --git a/arch/s390/kernel/vdso64/vdso64.lds.S b/arch/s390/kernel/vdso64/vdso64.lds.S index 9f5979d102a9..f35455d497fe 100644 --- a/arch/s390/kernel/vdso64/vdso64.lds.S +++ b/arch/s390/kernel/vdso64/vdso64.lds.S @@ -132,6 +132,7 @@ VERSION __kernel_gettimeofday; __kernel_clock_gettime; __kernel_clock_getres; + __kernel_getcpu; local: *; };