diff options
Diffstat (limited to 'lib/libc/internal')
25 files changed, 369 insertions, 51 deletions
diff --git a/lib/libc/internal/Kbuild b/lib/libc/internal/Kbuild index d391eaa8..1bb5eaed 100644 --- a/lib/libc/internal/Kbuild +++ b/lib/libc/internal/Kbuild @@ -1,5 +1,4 @@ -obj-y += init/ obj-y += arch/ - -obj-y += panic.o -obj-y += syscall.o +obj-y += init/ +obj-y += thrd/ +obj-y += util/ diff --git a/lib/libc/internal/arch/aarch64/thread_self.c b/lib/libc/internal/arch/aarch64/thread_self.c new file mode 100644 index 00000000..543b07ab --- /dev/null +++ b/lib/libc/internal/arch/aarch64/thread_self.c @@ -0,0 +1,8 @@ +#include <__thread.h> + +inline struct __thread_self *__thread_self(void) +{ + struct __thread_self *self; + __asm__("mrs %0, tpidr_el0" : "=r"(self)); + return self; +} diff --git a/lib/libc/internal/arch/x86_64/Kbuild b/lib/libc/internal/arch/x86_64/Kbuild index 830a1c5b..49e4284f 100644 --- a/lib/libc/internal/arch/x86_64/Kbuild +++ b/lib/libc/internal/arch/x86_64/Kbuild @@ -1,5 +1,7 @@ obj-y += _start.o +obj-y += clone.o obj-y += fenv.o obj-y += longjmp.o obj-y += setjmp.o obj-y += sigsetjmp.o +obj-y += tcb.o diff --git a/lib/libc/internal/arch/x86_64/_start.c b/lib/libc/internal/arch/x86_64/_start.c index f3f4fef2..e8846bfa 100644 --- a/lib/libc/internal/arch/x86_64/_start.c +++ b/lib/libc/internal/arch/x86_64/_start.c @@ -4,5 +4,5 @@ __dead __naked void _start(void) { __asm__ __volatile__("mov %rsp, %rdi\n" - "call __init\n"); + "call __libc_init\n"); } diff --git a/lib/libc/internal/arch/x86_64/clone.s b/lib/libc/internal/arch/x86_64/clone.s new file mode 100644 index 00000000..80f512b1 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/clone.s @@ -0,0 +1,106 @@ +/* + * x86_64 clone syscall trampoline + * + * Goal: provide a reliable way to run a C function in the child after the + * clone(2) syscall, without depending on the parent's stack frame (which is + * invalid in the child because the child starts with a new stack pointer). + * + * API (internal): + * + * long __clone(long (*fn)(void *), void *arg, + * unsigned long flags, void *child_stack, + * int *ptid, void *newtls, int *ctid); + * + * This uses the raw Linux x86_64 clone syscall: + * long clone(unsigned long flags, void *child_stack, + * int *ptid, int *ctid, unsigned long newtls); + * + * Behavior: + * - In the parent, returns child TID (> 0) or -errno (< 0). + * - In the child, calls fn(arg) and then exits the thread/process via + * SYS_exit (does not return). + * + * Notes: + * - We explicitly place (fn,arg) onto the child's new stack before issuing + * the syscall. On success in the child, we recover them from %rsp and call. + * - This avoids relying on any locals from the parent frame. + * - The stack is aligned to 16 bytes before the call into C. + */ + + .text + .globl __clone + .type __clone, @function + +__clone: + /* + * SysV AMD64 calling convention: + * fn in %rdi + * arg in %rsi + * flags in %rdx + * child_stack in %rcx + * ptid in %r8 + * newtls in %r9 + * ctid at 8(%rsp) + */ + movq 8(%rsp), %r10 /* r10 = ctid (7th arg from caller stack) */ + + /* r11 = child_stack (we'll use it as our working "new sp") */ + movq %rcx, %r11 + + /* + * Ensure 16-byte alignment for the eventual C call: + * We'll build the child's stack as: + * [low] fn (8) + * arg (8) + * [high] <-- %rsp in child before call site + * + * We align first, then reserve 16 bytes. + */ + andq $-16, %r11 + subq $16, %r11 + + /* Store fn and arg onto the child's stack */ + movq %rdi, 0(%r11) /* fn */ + movq %rsi, 8(%r11) /* arg */ + + /* + * Prepare registers for Linux x86_64 clone syscall: + * rax = SYS_clone + * rdi = flags + * rsi = child_stack + * rdx = ptid + * r10 = ctid + * r8 = newtls + */ + movq %rdx, %rdi /* flags */ + movq %r11, %rsi /* child_stack (new sp) */ + movq %r8, %rdx /* ptid */ + movq %r9, %r8 /* newtls */ + /* r10 already holds ctid */ + + movq $56, %rax /* SYS_clone on x86_64 */ + syscall + + /* Parent: rax = child tid (>0) or -errno (<0) */ + testq %rax, %rax + jnz 1f + + /* Child: rax == 0 */ + + /* %rsp is already the child_stack we provided. Recover fn/arg from it. */ + movq 8(%rsp), %rdi /* first arg to fn = arg */ + call *0(%rsp) /* call fn(arg) */ + + /* + * If fn returns, exit the thread/process. + * Use SYS_exit (60). Return value of fn is in %rax; pass as status in %rdi. + */ + movq %rax, %rdi + movq $60, %rax /* SYS_exit */ + syscall + hlt /* should not reach */ + +1: + ret + + .size __clone, .-__clone diff --git a/lib/libc/internal/arch/x86_64/tcb.c b/lib/libc/internal/arch/x86_64/tcb.c new file mode 100644 index 00000000..6e1f0c15 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/tcb.c @@ -0,0 +1,13 @@ +#include <libc/tcb.h> + +__attribute__((__always_inline__)) void __libc_tcb_set(uint64_t tcb) +{ + __asm__ volatile("wrfsbase %0" ::"r"(tcb)); +} + +__attribute__((__always_inline__)) void *__libc_tcb_get(void) +{ + void *tcb; + __asm__ volatile("rdfsbase %0" : "=r"(tcb)); + return tcb; +} diff --git a/lib/libc/internal/include/__stdio.h b/lib/libc/internal/include/__stdio.h index 487a0382..7846b139 100644 --- a/lib/libc/internal/include/__stdio.h +++ b/lib/libc/internal/include/__stdio.h @@ -31,8 +31,6 @@ struct __FILE { struct __FILE *next; }; -#define __FILE(__stream) ((struct __FILE *)(__stream)) - void __libc_fadd(struct __FILE *f); #endif diff --git a/lib/libc/internal/include/__thread.h b/lib/libc/internal/include/__thread.h index 7639707a..3c68f8dd 100644 --- a/lib/libc/internal/include/__thread.h +++ b/lib/libc/internal/include/__thread.h @@ -1,8 +1,50 @@ #ifndef __LIBC_THREAD_H #define __LIBC_THREAD_H +#include <stdatomic.h> +#include <stddef.h> +#include <stdint.h> + +#define THREAD_STACK_SIZE (1024 * 1024) +#define THREAD_GUARD_SIZE 8192 + +enum __thread_state { THREAD_STATE_EXITED, THREAD_STATE_JOINABLE, THREAD_STATE_DETACHED }; + +struct tls { + void *data; + size_t length; + size_t size; + size_t align; +}; + struct __thread_self { - int tid; + struct __thread_self *self; + + /* Backing mapping for this thread (used by thrd_join/thrd_detach to munmap safely) */ + void *map_base; + size_t map_size; + +#ifdef __aarch64__ + uintptr_t *dtv; + uintptr_t canary; +#endif + + long tid; + int res; + int errno_v; + volatile int state; + +#ifdef __x86_64__ + uintptr_t canary; + uintptr_t *dtv; +#endif }; +void __libc_tls_copy(void *); + +#if defined(__x86_64__) +long __clone(long (*fn)(void *), void *arg, unsigned long flags, void *child_stack, long *ptid, void *newtls, + long *ctid); +#endif + #endif diff --git a/lib/libc/internal/include/atomic.h b/lib/libc/internal/include/atomic.h index e0593021..d40c1eae 100644 --- a/lib/libc/internal/include/atomic.h +++ b/lib/libc/internal/include/atomic.h @@ -1,22 +1,32 @@ #ifndef __LIBC_ATOMIC_H #define __LIBC_ATOMIC_H +#include <libc/futex.h> +#include <sched.h> #include <stdatomic.h> +#include <unistd.h> #define LIBC_LOCK(__lock) __libc_lock(&((__lock))) -#define LIBC_UNLOCK(__lock) atomic_flag_clear(&((__lock))) +#define LIBC_UNLOCK(__lock) __libc_unlock(&((__lock))) -static __inline void __libc_lock(volatile atomic_flag *lock) +__attribute__((__always_inline__)) __inline void __libc_unlock(volatile atomic_flag *lock) { - while (atomic_flag_test_and_set_explicit(lock, memory_order_acquire)) { - unsigned int spins = 1; - do { - for (unsigned int i = 0; i < spins; i++) - __asm__ volatile("pause"); - if (spins < 64) - spins *= 2; - } while (atomic_flag_test_and_set_explicit( - lock, memory_order_relaxed)); + atomic_flag_clear_explicit(lock, memory_order_release); + __futex_wake((volatile int *)lock, 1); +} + +__attribute__((__always_inline__)) __inline void __libc_lock(volatile atomic_flag *lock) +{ + if (atomic_flag_test_and_set_explicit(lock, memory_order_acquire) == 0) + return; + + while (1) { + __futex_wait((volatile int *)lock, 1); + + if (atomic_flag_test_and_set_explicit(lock, memory_order_acquire) == 0) + return; + + sched_yield(); } } diff --git a/lib/libc/internal/include/libc.h b/lib/libc/internal/include/libc.h index f06fa6d8..75a06dd2 100644 --- a/lib/libc/internal/include/libc.h +++ b/lib/libc/internal/include/libc.h @@ -1,26 +1,20 @@ #ifndef __LIBC_LIBC_H #define __LIBC_LIBC_H +#include <__stdio.h> #include <stdatomic.h> #include <stddef.h> #include <sys/cdefs.h> #define weak_reference(old, new) extern __typeof(old)((new)) __attribute__((__weak__, __alias__(#old))) -struct tls { - struct tls *next; - void *data; - size_t size; - size_t align; - size_t length; - size_t offset; -}; - struct libc { - size_t auxv[32]; + size_t *auxv; + struct { void *base; size_t size; + size_t align; } tls; enum { @@ -32,6 +26,10 @@ struct libc { volatile atomic_flag malloc; volatile atomic_flag environ; } lock; + + struct __FILE stdin; + struct __FILE stdout; + struct __FILE stderr; }; extern struct libc __libc; diff --git a/lib/libc/internal/include/libc/fsbase.h b/lib/libc/internal/include/libc/fsbase.h new file mode 100644 index 00000000..61f4abce --- /dev/null +++ b/lib/libc/internal/include/libc/fsbase.h @@ -0,0 +1,7 @@ +#ifndef __LIBC_FSBASE_H +#define __LIBC_FSBASE_H + +void *rdfsbase(void); +void wrfsbase(unsigned long); + +#endif diff --git a/lib/libc/internal/include/libc/futex.h b/lib/libc/internal/include/libc/futex.h new file mode 100644 index 00000000..a00cf0e7 --- /dev/null +++ b/lib/libc/internal/include/libc/futex.h @@ -0,0 +1,7 @@ +#ifndef __LIBC_FUTEX_H +#define __LIBC_FUTEX_H + +int __futex_wait(volatile int *, int); +int __futex_wake(volatile int *, int); + +#endif diff --git a/lib/libc/internal/include/libc/tcb.h b/lib/libc/internal/include/libc/tcb.h new file mode 100644 index 00000000..1defd41c --- /dev/null +++ b/lib/libc/internal/include/libc/tcb.h @@ -0,0 +1,9 @@ +#ifndef __LIBC_TCB_H +#define __LIBC_TCB_H + +#include <stdint.h> + +void *__libc_tcb_get(void); +void __libc_tcb_set(uint64_t); + +#endif diff --git a/lib/libc/internal/include/libc/thread.h b/lib/libc/internal/include/libc/thread.h new file mode 100644 index 00000000..6422a0e2 --- /dev/null +++ b/lib/libc/internal/include/libc/thread.h @@ -0,0 +1,6 @@ +#ifndef __LIBC_THREAD_H +#define __LIBC_THREAD_H + +void *__libc_thread_self(void); + +#endif diff --git a/lib/libc/internal/init/Kbuild b/lib/libc/internal/init/Kbuild index 361f220a..3c1bab0b 100644 --- a/lib/libc/internal/init/Kbuild +++ b/lib/libc/internal/init/Kbuild @@ -1,3 +1,4 @@ obj-y += init.o +obj-y += io.o obj-y += tls.o obj-y += vdso.o diff --git a/lib/libc/internal/init/init.c b/lib/libc/internal/init/init.c index b2dd8487..62678dce 100644 --- a/lib/libc/internal/init/init.c +++ b/lib/libc/internal/init/init.c @@ -1,11 +1,15 @@ +#include <alloca.h> #include <libc.h> #include <stddef.h> #include <stdint.h> #include <stdlib.h> #include <sys/cdefs.h> -extern void __init_vdso(void); -extern void __init_tls(void); +#define BUFSIZ 4096 + +extern void __libc_init_vdso(void) __attribute__((weak)); +extern void __libc_init_tls(void) __attribute__((weak)); +extern void __libc_init_io(void) __attribute__((weak)); extern int main(int, char **, char **); @@ -13,10 +17,9 @@ struct libc __libc = { 0 }; char **environ; char *__progname; -__used void __init(uintptr_t *rsp) +__used void __libc_init(uintptr_t *rsp) { char **argv; - size_t *auxv; int argc; argc = (int)(*rsp); @@ -30,13 +33,27 @@ __used void __init(uintptr_t *rsp) while (*rsp) rsp++; - auxv = (size_t *)++rsp; + __libc.auxv = (size_t *)++rsp; + + if (__libc_init_io) { + __libc_init_io(); + + __libc.stdout.buf = alloca(BUFSIZ); + __libc.stdout.buf_size = BUFSIZ; + __libc.stdout.buf_len = 0; + __libc.stdout.buf_pos = 0; + + __libc.stdin.buf = alloca(BUFSIZ); + __libc.stdin.buf_size = BUFSIZ; + __libc.stdin.buf_len = 0; + __libc.stdin.buf_pos = 0; + } - for (size_t i = 0; auxv[i]; i += 2) - __libc.auxv[auxv[i]] = auxv[i + 1]; + if (__libc_init_tls) + __libc_init_tls(); - __init_tls(); - __init_vdso(); + if (__libc_init_vdso) + __libc_init_vdso(); exit(main(argc, argv, environ)); } diff --git a/lib/libc/internal/init/io.c b/lib/libc/internal/init/io.c new file mode 100644 index 00000000..4790af89 --- /dev/null +++ b/lib/libc/internal/init/io.c @@ -0,0 +1,42 @@ +#include <atomic.h> +#include <fcntl.h> +#include <libc.h> +#include <stdio.h> +#include <unistd.h> + +struct __FILE *const stdout = (struct __FILE *)&__libc.stdout; +struct __FILE *const stdin = (struct __FILE *)&__libc.stdin; +struct __FILE *const stderr = (struct __FILE *)&__libc.stderr; + +void __libc_init_io(void) +{ + /* stdout */ + memset(&__libc.stdout, 0, sizeof(struct __FILE)); + __libc.stdout.fd = STDOUT_FILENO; + __libc.stdout.flags = O_WRONLY; + __libc.stdout.type = _IOLBF; + __libc.stdout.buf = NULL; + __libc.stdout.buf_size = 0; + __libc.stdout.buf_len = 0; + __libc.stdout.buf_pos = 0; + + /* stdin */ + memset(&__libc.stdin, 0, sizeof(struct __FILE)); + __libc.stdin.fd = STDIN_FILENO; + __libc.stdin.flags = O_RDONLY; + __libc.stdin.type = _IONBF; + __libc.stdin.buf = NULL; + __libc.stdin.buf_size = 0; + __libc.stdin.buf_len = 0; + __libc.stdin.buf_pos = 0; + + /* stderr */ + memset(&__libc.stderr, 0, sizeof(struct __FILE)); + __libc.stderr.fd = STDERR_FILENO; + __libc.stderr.flags = O_WRONLY; + __libc.stderr.type = _IONBF; + __libc.stderr.buf = NULL; + __libc.stderr.buf_size = 0; + __libc.stderr.buf_len = 0; + __libc.stderr.buf_pos = 0; +} diff --git a/lib/libc/internal/init/tls.c b/lib/libc/internal/init/tls.c index f6ccde35..302edd9d 100644 --- a/lib/libc/internal/init/tls.c +++ b/lib/libc/internal/init/tls.c @@ -6,14 +6,31 @@ #include <sys/mman.h> #include <syscall.h> -#define ALIGN_DOWN(x, a) ((x) & ~((a) - 1)) -#define ALIGN_UP(x, a) (((x) + (a) - 1) & ~((a) - 1)) +#include <libc/tcb.h> +#include <stdio.h> + +// TODO: remove this thing static struct tls __libc_tls = { 0 }; + static struct __thread_self __libc_thread; volatile int __libc_tid; -void __init_tls(void) +void __libc_tls_copy(void *dest) +{ + if (__libc_tls.size == 0) + return; + + if (__libc_tls.length > 0) { + memcpy(dest, __libc_tls.data, __libc_tls.length); + + memset((unsigned char *)dest + __libc_tls.length, 0, __libc_tls.size - __libc_tls.length); + } else { + memset(dest, 0, __libc_tls.size); + } +} + +void __libc_init_tls(void) { int r; void *mem; @@ -40,6 +57,7 @@ void __init_tls(void) __libc_tls.length = tls_phdr->p_filesz; __libc_tls.size = tls_phdr->p_memsz; __libc_tls.align = tls_phdr->p_align; + __libc.tls.align = __libc_tls.align; } if (__libc_tls.size != 0) { @@ -56,21 +74,22 @@ void __init_tls(void) thrd = (struct __thread_self *)mem; tls_mem = (unsigned char *)mem + sizeof(struct __thread_self); - if (tls_phdr->p_filesz > 0) { - memcpy(tls_mem, (void *)tls_phdr->p_vaddr, tls_phdr->p_filesz); - memset(tls_mem + tls_phdr->p_filesz, 0, tls_phdr->p_memsz - tls_phdr->p_filesz); - } else { - memset(tls_mem, 0, tls_phdr->p_memsz); - } + __libc_tls_copy(tls_mem); - r = syscall(arch_prctl, ARCH_SET_FS, mem + sizeof(struct __thread_self)); - panic_if(r < 0, "arch_prctl(SET_FS) failed"); + __libc.tls.base = tls_mem; + __libc.tls.size = __libc_tls.size; } else { - __libc.tls.base = NULL; thrd = &__libc_thread; + __libc.tls.base = &__libc_thread; + __libc.tls.size = 0; } r = syscall(set_tid_address, &__libc_tid); panic_if(r < 0, "set_tid_address failed"); + + thrd->self = thrd; thrd->tid = r; + thrd->dtv = NULL; + + __libc_tcb_set((uint64_t)thrd); } diff --git a/lib/libc/internal/init/vdso.c b/lib/libc/internal/init/vdso.c index de8d0e2f..73bfb46b 100644 --- a/lib/libc/internal/init/vdso.c +++ b/lib/libc/internal/init/vdso.c @@ -3,7 +3,18 @@ #include <string.h> #include <sys/auxv.h> -void __init_vdso(void) +int (*__vdso_clock_gettime)(int, struct timespec *) = 0; +int (*__vdso_getcpu)(unsigned *, unsigned *, void *) = 0; +int (*__vdso_time)(long *) = 0; + +struct __vdso_sym __vdso_symtab[] = { + { "__vdso_clock_gettime", (void *)&__vdso_clock_gettime }, + { "__vdso_getcpu", (void *)&__vdso_getcpu }, + { "__vdso_time", (void *)&__vdso_time }, + { 0, 0 }, +}; + +void __libc_init_vdso(void) { Elf64_Ehdr *ehdr = (Elf64_Ehdr *)getauxval(AT_SYSINFO_EHDR); diff --git a/lib/libc/internal/thrd/Kbuild b/lib/libc/internal/thrd/Kbuild new file mode 100644 index 00000000..c3817f75 --- /dev/null +++ b/lib/libc/internal/thrd/Kbuild @@ -0,0 +1 @@ +obj-y += self.o diff --git a/lib/libc/internal/thrd/self.c b/lib/libc/internal/thrd/self.c new file mode 100644 index 00000000..b083223f --- /dev/null +++ b/lib/libc/internal/thrd/self.c @@ -0,0 +1,6 @@ +#include <libc/tcb.h> + +__attribute__((__always_inline__)) void *__libc_thread_self(void) +{ + return __libc_tcb_get(); +} diff --git a/lib/libc/internal/util/Kbuild b/lib/libc/internal/util/Kbuild new file mode 100644 index 00000000..9c8d0fc7 --- /dev/null +++ b/lib/libc/internal/util/Kbuild @@ -0,0 +1,3 @@ +obj-y += futex.o +obj-y += panic.o +obj-y += syscall.o diff --git a/lib/libc/internal/util/futex.c b/lib/libc/internal/util/futex.c new file mode 100644 index 00000000..2f9872a0 --- /dev/null +++ b/lib/libc/internal/util/futex.c @@ -0,0 +1,13 @@ +#include <linux/futex.h> +#include <syscall.h> + +int __futex_wait(volatile int *uaddr, int val) +{ + return syscall(futex, uaddr, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, 0, 0, 0); +} + +int __futex_wake(volatile int *uaddr, int n) +{ + /* futex(uaddr, FUTEX_WAKE, val=n, ...) */ + return syscall(futex, uaddr, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, n, 0, 0, 0); +} diff --git a/lib/libc/internal/panic.c b/lib/libc/internal/util/panic.c index 1af6d6cf..1af6d6cf 100644 --- a/lib/libc/internal/panic.c +++ b/lib/libc/internal/util/panic.c diff --git a/lib/libc/internal/syscall.c b/lib/libc/internal/util/syscall.c index f7fd3c4b..f7fd3c4b 100644 --- a/lib/libc/internal/syscall.c +++ b/lib/libc/internal/util/syscall.c |