From 90dad97fc07f049383903a166631e2c257f9b8c1 Mon Sep 17 00:00:00 2001 From: Kacper Date: Tue, 16 Dec 2025 17:02:05 +0100 Subject: Add support for TLS in the libc --- bin/echo/qemu_echo_20251216-155008_248.core | Bin 0 -> 8495104 bytes bin/echo/qemu_echo_20251216-155146_265.core | Bin 0 -> 8495104 bytes bin/echo/qemu_echo_20251216-155344_268.core | Bin 0 -> 8495104 bytes bin/echo/qemu_echo_20251216-155417_274.core | Bin 0 -> 8495104 bytes include/elf.h | 10 +-- include/errno.h | 3 +- include/sys/cdefs.h | 3 + lib/libc/Kbuild | 4 +- lib/libc/arch/Kbuild | 1 - lib/libc/arch/x86_64/Kbuild | 5 -- lib/libc/arch/x86_64/_start.c | 8 --- lib/libc/arch/x86_64/fenv.s | 97 ------------------------- lib/libc/arch/x86_64/longjmp.c | 20 ------ lib/libc/arch/x86_64/setjmp.c | 19 ----- lib/libc/arch/x86_64/sigsetjmp.c | 33 --------- lib/libc/devctl/posix_devctl.c | 6 +- lib/libc/errno/errno.c | 8 +-- lib/libc/include/__aio.h | 35 --------- lib/libc/include/__dirent.h | 24 ------- lib/libc/include/__select.h | 8 --- lib/libc/include/__signal.h | 12 ---- lib/libc/include/__statvfs.h | 21 ------ lib/libc/include/__stdio.h | 38 ---------- lib/libc/include/__thread.h | 9 --- lib/libc/include/atomic.h | 23 ------ lib/libc/include/byteswap.h | 8 --- lib/libc/include/internal/io_uring.h | 6 -- lib/libc/include/io_uring.h | 53 -------------- lib/libc/include/libc.h | 25 ------- lib/libc/include/libc/dirent.h | 6 -- lib/libc/include/malloc.h | 98 -------------------------- lib/libc/include/syscall.h | 59 ---------------- lib/libc/internal/Kbuild | 4 ++ lib/libc/internal/arch/Kbuild | 1 + lib/libc/internal/arch/x86_64/Kbuild | 5 ++ lib/libc/internal/arch/x86_64/_start.c | 8 +++ lib/libc/internal/arch/x86_64/fenv.s | 97 +++++++++++++++++++++++++ lib/libc/internal/arch/x86_64/longjmp.c | 20 ++++++ lib/libc/internal/arch/x86_64/setjmp.c | 19 +++++ lib/libc/internal/arch/x86_64/sigsetjmp.c | 33 +++++++++ lib/libc/internal/deinit/tls.c | 13 ++++ lib/libc/internal/include/__aio.h | 35 +++++++++ lib/libc/internal/include/__dirent.h | 24 +++++++ lib/libc/internal/include/__select.h | 8 +++ lib/libc/internal/include/__signal.h | 12 ++++ lib/libc/internal/include/__statvfs.h | 21 ++++++ lib/libc/internal/include/__stdio.h | 38 ++++++++++ lib/libc/internal/include/__thread.h | 8 +++ lib/libc/internal/include/atomic.h | 23 ++++++ lib/libc/internal/include/byteswap.h | 8 +++ lib/libc/internal/include/internal/io_uring.h | 6 ++ lib/libc/internal/include/io_uring.h | 53 ++++++++++++++ lib/libc/internal/include/libc.h | 48 +++++++++++++ lib/libc/internal/include/libc/dirent.h | 6 ++ lib/libc/internal/include/malloc.h | 98 ++++++++++++++++++++++++++ lib/libc/internal/include/syscall.h | 44 ++++++++++++ lib/libc/internal/init/Kbuild | 1 + lib/libc/internal/init/init.c | 6 +- lib/libc/internal/init/tls.c | 76 ++++++++++++++++++++ lib/libc/internal/panic.c | 29 ++++++++ lib/libc/internal/syscall.c | 12 ++++ lib/libc/mman/mmap.c | 2 - lib/libc/mqueue/mq_notify.c | 4 +- lib/libc/mqueue/mq_unlink.c | 2 +- lib/libc/sys/getauxval.c | 2 + lib/libc/thread/thrd_current.c | 9 +-- 66 files changed, 779 insertions(+), 638 deletions(-) create mode 100644 bin/echo/qemu_echo_20251216-155008_248.core create mode 100644 bin/echo/qemu_echo_20251216-155146_265.core create mode 100644 bin/echo/qemu_echo_20251216-155344_268.core create mode 100644 bin/echo/qemu_echo_20251216-155417_274.core delete mode 100644 lib/libc/arch/Kbuild delete mode 100644 lib/libc/arch/x86_64/Kbuild delete mode 100644 lib/libc/arch/x86_64/_start.c delete mode 100644 lib/libc/arch/x86_64/fenv.s delete mode 100644 lib/libc/arch/x86_64/longjmp.c delete mode 100644 lib/libc/arch/x86_64/setjmp.c delete mode 100644 lib/libc/arch/x86_64/sigsetjmp.c delete mode 100644 lib/libc/include/__aio.h delete mode 100644 lib/libc/include/__dirent.h delete mode 100644 lib/libc/include/__select.h delete mode 100644 lib/libc/include/__signal.h delete mode 100644 lib/libc/include/__statvfs.h delete mode 100644 lib/libc/include/__stdio.h delete mode 100644 lib/libc/include/__thread.h delete mode 100644 lib/libc/include/atomic.h delete mode 100644 lib/libc/include/byteswap.h delete mode 100644 lib/libc/include/internal/io_uring.h delete mode 100644 lib/libc/include/io_uring.h delete mode 100644 lib/libc/include/libc.h delete mode 100644 lib/libc/include/libc/dirent.h delete mode 100644 lib/libc/include/malloc.h delete mode 100644 lib/libc/include/syscall.h create mode 100644 lib/libc/internal/arch/Kbuild create mode 100644 lib/libc/internal/arch/x86_64/Kbuild create mode 100644 lib/libc/internal/arch/x86_64/_start.c create mode 100644 lib/libc/internal/arch/x86_64/fenv.s create mode 100644 lib/libc/internal/arch/x86_64/longjmp.c create mode 100644 lib/libc/internal/arch/x86_64/setjmp.c create mode 100644 lib/libc/internal/arch/x86_64/sigsetjmp.c create mode 100644 lib/libc/internal/deinit/tls.c create mode 100644 lib/libc/internal/include/__aio.h create mode 100644 lib/libc/internal/include/__dirent.h create mode 100644 lib/libc/internal/include/__select.h create mode 100644 lib/libc/internal/include/__signal.h create mode 100644 lib/libc/internal/include/__statvfs.h create mode 100644 lib/libc/internal/include/__stdio.h create mode 100644 lib/libc/internal/include/__thread.h create mode 100644 lib/libc/internal/include/atomic.h create mode 100644 lib/libc/internal/include/byteswap.h create mode 100644 lib/libc/internal/include/internal/io_uring.h create mode 100644 lib/libc/internal/include/io_uring.h create mode 100644 lib/libc/internal/include/libc.h create mode 100644 lib/libc/internal/include/libc/dirent.h create mode 100644 lib/libc/internal/include/malloc.h create mode 100644 lib/libc/internal/include/syscall.h create mode 100644 lib/libc/internal/init/tls.c create mode 100644 lib/libc/internal/panic.c create mode 100644 lib/libc/internal/syscall.c diff --git a/bin/echo/qemu_echo_20251216-155008_248.core b/bin/echo/qemu_echo_20251216-155008_248.core new file mode 100644 index 00000000..c6bc9642 Binary files /dev/null and b/bin/echo/qemu_echo_20251216-155008_248.core differ diff --git a/bin/echo/qemu_echo_20251216-155146_265.core b/bin/echo/qemu_echo_20251216-155146_265.core new file mode 100644 index 00000000..3c9a87a1 Binary files /dev/null and b/bin/echo/qemu_echo_20251216-155146_265.core differ diff --git a/bin/echo/qemu_echo_20251216-155344_268.core b/bin/echo/qemu_echo_20251216-155344_268.core new file mode 100644 index 00000000..844a70ff Binary files /dev/null and b/bin/echo/qemu_echo_20251216-155344_268.core differ diff --git a/bin/echo/qemu_echo_20251216-155417_274.core b/bin/echo/qemu_echo_20251216-155417_274.core new file mode 100644 index 00000000..a9a1df54 Binary files /dev/null and b/bin/echo/qemu_echo_20251216-155417_274.core differ diff --git a/include/elf.h b/include/elf.h index 8317c9ca..ea2014a6 100644 --- a/include/elf.h +++ b/include/elf.h @@ -16,6 +16,7 @@ #define PT_NOTE 4 #define PT_SHLIB 5 #define PT_PHDR 6 +#define PT_TLS 7 #define PT_LOOS 0x60000000 #define PT_HIOS 0x6fffffff #define PT_LOPROC 0x70000000 @@ -101,12 +102,13 @@ #define SHT_LOUSER 0x80000000 #define SHT_HIUSER 0xffffffff -#define SHF_WRITE 0x1 -#define SHF_ALLOC 0x2 -#define SHF_EXECINSTR 0x4 +#define SHF_WRITE 0x00000001 +#define SHF_ALLOC 0x00000002 +#define SHF_EXECINSTR 0x00000004 +#define SHF_TLS 0x00000400 #define SHF_MASKPROC 0xf0000000 -#define SHN_UNDEF 0 +#define SHN_UNDEF 0x0000 #define SHN_LORESERVE 0xff00 #define SHN_LOPROC 0xff00 #define SHN_HIPROC 0xff1f diff --git a/include/errno.h b/include/errno.h index 86ef7047..800f9a9b 100644 --- a/include/errno.h +++ b/include/errno.h @@ -1,8 +1,7 @@ #ifndef __ERRNO_H #define __ERRNO_H -#define errno (*__errno()) -extern int *__errno(void) __attribute__((const)); +extern _Thread_local int errno; #define EPERM 1 #define ENOENT 2 diff --git a/include/sys/cdefs.h b/include/sys/cdefs.h index 00e85014..cd490d6b 100644 --- a/include/sys/cdefs.h +++ b/include/sys/cdefs.h @@ -9,6 +9,9 @@ #define __END_DECLS #endif +#define ___STRING(x) #x +#define __STRING(x) ___STRING(x) + #define __dead __attribute__((__noreturn__)) #define __used __attribute__((__used__)) #define __unused __attribute__((__unused__)) diff --git a/lib/libc/Kbuild b/lib/libc/Kbuild index 76e05807..8a52fdb3 100644 --- a/lib/libc/Kbuild +++ b/lib/libc/Kbuild @@ -1,10 +1,9 @@ lib-y := libc.a install-y := lib/libc.a -cflags-y += -I $(src)/include +cflags-y += -I $(src)/internal/include obj-y += aio/ -obj-y += arch/ obj-y += arpa/ obj-y += assert/ obj-y += ctype/ @@ -49,4 +48,3 @@ obj-y += utsname/ obj-y += wait/ obj-y += wchar/ obj-y += wctype/ - diff --git a/lib/libc/arch/Kbuild b/lib/libc/arch/Kbuild deleted file mode 100644 index ad3a7486..00000000 --- a/lib/libc/arch/Kbuild +++ /dev/null @@ -1 +0,0 @@ -obj-y += $(ARCH)/ diff --git a/lib/libc/arch/x86_64/Kbuild b/lib/libc/arch/x86_64/Kbuild deleted file mode 100644 index 830a1c5b..00000000 --- a/lib/libc/arch/x86_64/Kbuild +++ /dev/null @@ -1,5 +0,0 @@ -obj-y += _start.o -obj-y += fenv.o -obj-y += longjmp.o -obj-y += setjmp.o -obj-y += sigsetjmp.o diff --git a/lib/libc/arch/x86_64/_start.c b/lib/libc/arch/x86_64/_start.c deleted file mode 100644 index f3f4fef2..00000000 --- a/lib/libc/arch/x86_64/_start.c +++ /dev/null @@ -1,8 +0,0 @@ -#include -#include - -__dead __naked void _start(void) -{ - __asm__ __volatile__("mov %rsp, %rdi\n" - "call __init\n"); -} diff --git a/lib/libc/arch/x86_64/fenv.s b/lib/libc/arch/x86_64/fenv.s deleted file mode 100644 index b9a46581..00000000 --- a/lib/libc/arch/x86_64/fenv.s +++ /dev/null @@ -1,97 +0,0 @@ -.global feclearexcept -.type feclearexcept,@function -feclearexcept: - mov %edi,%ecx - and $0x3f,%ecx - fnstsw %ax - test %eax,%ecx - jz 1f - fnclex -1: stmxcsr -8(%rsp) - and $0x3f,%eax - or %eax,-8(%rsp) - test %ecx,-8(%rsp) - jz 1f - not %ecx - and %ecx,-8(%rsp) - ldmxcsr -8(%rsp) -1: xor %eax,%eax - ret - -.global feraiseexcept -.type feraiseexcept,@function -feraiseexcept: - and $0x3f,%edi - stmxcsr -8(%rsp) - or %edi,-8(%rsp) - ldmxcsr -8(%rsp) - xor %eax,%eax - ret - -.global __fesetround -.hidden __fesetround -.type __fesetround,@function -__fesetround: - push %rax - xor %eax,%eax - mov %edi,%ecx - fnstcw (%rsp) - andb $0xf3,1(%rsp) - or %ch,1(%rsp) - fldcw (%rsp) - stmxcsr (%rsp) - shl $3,%ch - andb $0x9f,1(%rsp) - or %ch,1(%rsp) - ldmxcsr (%rsp) - pop %rcx - ret - -.global fegetround -.type fegetround,@function -fegetround: - push %rax - stmxcsr (%rsp) - pop %rax - shr $3,%eax - and $0xc00,%eax - ret - -.global fegetenv -.type fegetenv,@function -fegetenv: - xor %eax,%eax - fnstenv (%rdi) - stmxcsr 28(%rdi) - ret - -.global fesetenv -.type fesetenv,@function -fesetenv: - xor %eax,%eax - inc %rdi - jz 1f - fldenv -1(%rdi) - ldmxcsr 27(%rdi) - ret -1: push %rax - push %rax - pushq $0xffff - pushq $0x37f - fldenv (%rsp) - pushq $0x1f80 - ldmxcsr (%rsp) - add $40,%rsp - ret - -.global fetestexcept -.type fetestexcept,@function -fetestexcept: - and $0x3f,%edi - push %rax - stmxcsr (%rsp) - pop %rsi - fnstsw %ax - or %esi,%eax - and %edi,%eax - ret diff --git a/lib/libc/arch/x86_64/longjmp.c b/lib/libc/arch/x86_64/longjmp.c deleted file mode 100644 index dbd18632..00000000 --- a/lib/libc/arch/x86_64/longjmp.c +++ /dev/null @@ -1,20 +0,0 @@ -#include "bits/setjmp.h" // for jmp_buf - -#include // for longjmp - -__attribute__((noreturn, naked, returns_twice)) void longjmp(jmp_buf env, - int val) -{ - __asm__ __volatile__("mov 0(%rdi), %rbx\n" - "mov 8(%rdi), %rbp\n" - "mov 16(%rdi), %r12\n" - "mov 24(%rdi), %r13\n" - "mov 32(%rdi), %r14\n" - "mov 40(%rdi), %r15\n" - "mov 48(%rdi), %rsp\n" - "xor %rax, %rax\n" - "cmp $0, %esi\n" - "setne %al\n" - "mov %eax, %eax\n" - "jmp *56(%rdi)\n"); -} diff --git a/lib/libc/arch/x86_64/setjmp.c b/lib/libc/arch/x86_64/setjmp.c deleted file mode 100644 index ea205e16..00000000 --- a/lib/libc/arch/x86_64/setjmp.c +++ /dev/null @@ -1,19 +0,0 @@ -#include "bits/setjmp.h" // for jmp_buf - -#include // for setjmp - -__attribute__((naked, returns_twice)) int setjmp(jmp_buf env) -{ - __asm__ __volatile__("mov %rbx,(%rdi)\n" - "mov %rbp,8(%rdi)\n" - "mov %r12,16(%rdi)\n" - "mov %r13,24(%rdi)\n" - "mov %r14,32(%rdi)\n" - "mov %r15,40(%rdi)\n" - "lea 8(%rsp),%rdx\n" - "mov %rdx,48(%rdi)\n" - "mov (%rsp),%rdx\n" - "mov %rdx,56(%rdi)\n" - "xor %eax,%eax\n" - "ret\n"); -} diff --git a/lib/libc/arch/x86_64/sigsetjmp.c b/lib/libc/arch/x86_64/sigsetjmp.c deleted file mode 100644 index 2d900ce7..00000000 --- a/lib/libc/arch/x86_64/sigsetjmp.c +++ /dev/null @@ -1,33 +0,0 @@ -#include // for sigsetjmp, sigjmp_buf - -int sigsetjmp(sigjmp_buf env, int savemask) -{ - __asm__ __volatile__("movq %%rbx, 0(%0)\n" - "movq %%rbp, 8(%0)\n" - "movq %%r12, 16(%0)\n" - "movq %%r13, 24(%0)\n" - "movq %%r14, 32(%0)\n" - "movq %%r15, 40(%0)\n" - - "leaq 8(%%rsp), %%rax\n" - "movq %%rax, 48(%0)\n" - "movq (%%rsp), %%rax\n" - "movq %%rax, 56(%0)\n" - - "testl %%esi, %%esi\n" - "jz 1f\n" - "movq $2, %%rdi\n" - "movq $0, %%rsi\n" - "leaq 64(%0), %%rdx\n" - "movq $128, %%r10\n" - "movq $14, %%rax\n" - "syscall\n" - - "1:\n" - "xor %%eax, %%eax\n" - : - : "r"(env), "S"(savemask) - : "rax", "rdi", "rdx", "r10", "memory"); - - return 0; -} diff --git a/lib/libc/devctl/posix_devctl.c b/lib/libc/devctl/posix_devctl.c index 59497833..01af3111 100644 --- a/lib/libc/devctl/posix_devctl.c +++ b/lib/libc/devctl/posix_devctl.c @@ -1,13 +1,13 @@ #include +#include #include #include int posix_devctl(int fildes, int dcmd, void *restrict dev_data_ptr, size_t __unused nbyte, int *restrict dev_info_ptr) { - int r; + int r = syscall(ioctl, fildes, dcmd, dev_data_ptr); - r = syscall(ioctl, fildes, dcmd, dev_data_ptr); - if (r < 0) + if (__predict_false(r < 0)) return errno; *dev_info_ptr = r; diff --git a/lib/libc/errno/errno.c b/lib/libc/errno/errno.c index 09b22f4d..fc7c99bb 100644 --- a/lib/libc/errno/errno.c +++ b/lib/libc/errno/errno.c @@ -1,7 +1 @@ -#include <__thread.h> -#include - -int *__errno(void) -{ - return &thrd_current()->terrno; -} +_Thread_local int errno; diff --git a/lib/libc/include/__aio.h b/lib/libc/include/__aio.h deleted file mode 100644 index 93cb1f79..00000000 --- a/lib/libc/include/__aio.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef __LIBC_AIO_H -#define __LIBC_AIO_H - -#include - -#define AIO_REQUEST_STATUS_PENDING 0 -#define AIO_REQUEST_STATUS_COMPLETED 1 - -struct lio_group { - int pending; - int error; - int eventfd; - struct sigevent *sig; -}; - -struct aio_context { - struct aio_request *head; - struct aio_request *tail; -}; - -struct aio_request { - struct aiocb *aiocbp; - int status; - ssize_t result; - struct aio_request *next; - struct lio_group *grp; -}; - -void __aio_poll(void); - -int __aio_request(struct aio_request *, int); -struct aio_request *__aio_lookup(const struct aiocb *); -struct aio_request *__aio_remove(const struct aiocb *); - -#endif diff --git a/lib/libc/include/__dirent.h b/lib/libc/include/__dirent.h deleted file mode 100644 index 4f9ead80..00000000 --- a/lib/libc/include/__dirent.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef __LIBC_DIRENT_H -#define __LIBC_DIRENT_H - -#include -#include -#include - -struct linux_dirent64 { - uint64_t d_ino; - int64_t d_off; - unsigned short d_reclen; - unsigned char d_type; - char d_name[]; -}; - -struct __DIR { - int fildes; - int cached; - off_t tell; - off_t offset; - char buffer[BUFSIZ] __attribute__((__aligned__(8))); -}; - -#endif diff --git a/lib/libc/include/__select.h b/lib/libc/include/__select.h deleted file mode 100644 index a38bbdce..00000000 --- a/lib/libc/include/__select.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef __LIBC_SELECT_H -#define __LIBC_SELECT_H - -struct __fd_set { - unsigned long fds_bits[16]; -}; - -#endif diff --git a/lib/libc/include/__signal.h b/lib/libc/include/__signal.h deleted file mode 100644 index 6b762dee..00000000 --- a/lib/libc/include/__signal.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef __LIBC_SIGNAL_H -#define __LIBC_SIGNAL_H - -static const char *__sys_signame[] = { - "SIGHUP", "SIGINT", "SIGQUIT", "SIGILL", "SIGTRAP", "SIGABRT", - "SIGBUS", "SIGFPE", "SIGKILL", "SIGUSR1", "SIGSEGV", "SIGUSR2", - "SIGPIPE", "SIGALRM", "SIGTERM", "SIGCHLD", "SIGCONT", "SIGSTOP", - "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGXCPU", "SIGXFSZ", - "SIGVTALRM", "SIGPROF", "SIGWINCH", "SIGIO", "SIGSYS" -}; - -#endif diff --git a/lib/libc/include/__statvfs.h b/lib/libc/include/__statvfs.h deleted file mode 100644 index c8a746d4..00000000 --- a/lib/libc/include/__statvfs.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef __LIBC_STATVFS_H__ -#define __LIBC_STATVFS_H__ - -typedef __UINT64_TYPE__ fsblkcnt_t; -typedef __UINT64_TYPE__ fsfilcnt_t; - -struct __statvfs { - unsigned long f_bsize; - unsigned long f_frsize; - fsblkcnt_t f_blocks; - fsblkcnt_t f_bfree; - fsblkcnt_t f_bavail; - fsfilcnt_t f_files; - fsfilcnt_t f_ffree; - fsfilcnt_t f_favail; - unsigned long f_fsid; - unsigned long f_flag; - unsigned long f_namemax; -}; - -#endif diff --git a/lib/libc/include/__stdio.h b/lib/libc/include/__stdio.h deleted file mode 100644 index 487a0382..00000000 --- a/lib/libc/include/__stdio.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef __LIBC_STDIO_H__ -#define __LIBC_STDIO_H__ - -#include -#include -#include -#include -#include -#include - -typedef __SIZE_TYPE__ size_t; - -#define _IO_ERR 0x4 -#define _IO_EOF 0x8 -#define _IO_WIDE 0x10 - -struct __FILE { - int fd; - uint32_t flags; - int type; - pid_t pid; - atomic_flag lock; - char *buf; - int eof; - size_t buf_size; - size_t buf_pos; - size_t buf_len; - unsigned char unget_buf[16]; - size_t unget_cnt; - off_t offset; - struct __FILE *next; -}; - -#define __FILE(__stream) ((struct __FILE *)(__stream)) - -void __libc_fadd(struct __FILE *f); - -#endif diff --git a/lib/libc/include/__thread.h b/lib/libc/include/__thread.h deleted file mode 100644 index f978d018..00000000 --- a/lib/libc/include/__thread.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef __LIBC_THREAD_H -#define __LIBC_THREAD_H - -struct __thread_self { - int tid; - int terrno; -}; - -#endif diff --git a/lib/libc/include/atomic.h b/lib/libc/include/atomic.h deleted file mode 100644 index e0593021..00000000 --- a/lib/libc/include/atomic.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef __LIBC_ATOMIC_H -#define __LIBC_ATOMIC_H - -#include - -#define LIBC_LOCK(__lock) __libc_lock(&((__lock))) -#define LIBC_UNLOCK(__lock) atomic_flag_clear(&((__lock))) - -static __inline void __libc_lock(volatile atomic_flag *lock) -{ - while (atomic_flag_test_and_set_explicit(lock, memory_order_acquire)) { - unsigned int spins = 1; - do { - for (unsigned int i = 0; i < spins; i++) - __asm__ volatile("pause"); - if (spins < 64) - spins *= 2; - } while (atomic_flag_test_and_set_explicit( - lock, memory_order_relaxed)); - } -} - -#endif diff --git a/lib/libc/include/byteswap.h b/lib/libc/include/byteswap.h deleted file mode 100644 index 9dfe6520..00000000 --- a/lib/libc/include/byteswap.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef __LIBC_BYTESWAP_H -#define __LIBC_BYTESWAP_H - -#define bswap16(x) __builtin_bswap16(x) -#define bswap32(x) __builtin_bswap32(x) -#define bswap64(x) __builtin_bswap64(x) - -#endif diff --git a/lib/libc/include/internal/io_uring.h b/lib/libc/include/internal/io_uring.h deleted file mode 100644 index e7369146..00000000 --- a/lib/libc/include/internal/io_uring.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef __INTERNAL_IO_URING_H -#define __INTERNAL_IO_URING_H - -void __io_uring_init(void); - -#endif diff --git a/lib/libc/include/io_uring.h b/lib/libc/include/io_uring.h deleted file mode 100644 index 4dac4583..00000000 --- a/lib/libc/include/io_uring.h +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef __LIBC_IO_URING_H -#define __LIBC_IO_URING_H - -#include -#include - -#define IO_URING_ENTRIES 256 - -struct io_uring_sq { - void *ring; - size_t ring_size; - struct io_uring_sqe *sqes; - - unsigned *head; - unsigned *tail; - unsigned *ring_mask; - unsigned *ring_entries; - unsigned *flags; - unsigned *dropped; - unsigned *array; -}; - -struct io_uring_cq { - void *ring; - size_t ring_size; - - unsigned *head; - unsigned *tail; - unsigned *ring_mask; - unsigned *ring_entries; - unsigned *overflow; - struct io_uring_cqe *cqes; - unsigned *flags; -}; - -struct io_uring { - int fd; - int eventfd; - - struct io_uring_sq sq; - struct io_uring_cq cq; -}; - -extern struct io_uring __io_uring; - -int io_uring_setup(unsigned int, struct io_uring_params *); - -int io_uring_register(unsigned int, unsigned int, void *, unsigned int); - -int io_uring_enter(unsigned int, unsigned int, unsigned int, unsigned int, - sigset_t *, size_t); - -#endif diff --git a/lib/libc/include/libc.h b/lib/libc/include/libc.h deleted file mode 100644 index 87d4e88f..00000000 --- a/lib/libc/include/libc.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef __LIBC_LIBC_H -#define __LIBC_LIBC_H - -#include -#include - -#define weak_reference(old, new) extern __typeof(old)((new)) __attribute__((__weak__, __alias__(#old))) - -struct libc { - size_t auxv[32]; - - enum { - LIBC_ENVP_TOUCHED = 1 << 0, - } flags; - - struct { - volatile atomic_flag abort; - volatile atomic_flag malloc; - volatile atomic_flag environ; - } lock; -}; - -extern struct libc __libc; - -#endif diff --git a/lib/libc/include/libc/dirent.h b/lib/libc/include/libc/dirent.h deleted file mode 100644 index 296cc91e..00000000 --- a/lib/libc/include/libc/dirent.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef __LIBC_DIRENT_H -#define __LIBC_DIRENT_H - -struct __DIR {}; - -#endif diff --git a/lib/libc/include/malloc.h b/lib/libc/include/malloc.h deleted file mode 100644 index b7cb0093..00000000 --- a/lib/libc/include/malloc.h +++ /dev/null @@ -1,98 +0,0 @@ -#ifndef __LIBC_MALLOC_H -#define __LIBC_MALLOC_H - -#include -#include - -struct class { - uint32_t size; - uint32_t count; -}; - -struct page { - volatile atomic_flag lock; - - enum { - PAGE_SMALL = 0x0, - PAGE_MEDIUM = 0x1, - PAGE_LARGE = 0x2, - } flags; - - struct { - uint32_t size; - uint32_t used; - uint32_t count; - } block; - - uint8_t *bitmap; - uint8_t *heap; - - struct page *next; - struct page *prev; -}; - -extern struct page *__malloc_pvec; - -#define SMALL_PAGE_SIZE_SHIFT 16 -#define SMALL_PAGE_SIZE (1 << SMALL_PAGE_SIZE_SHIFT) -#define SMALL_PAGE_MASK (~((uintptr_t)SMALL_PAGE_SIZE - 1)) - -#define MEDIUM_PAGE_SIZE_SHIFT 22 -#define MEDIUM_PAGE_SIZE (1 << MEDIUM_PAGE_SIZE_SHIFT) -#define MEDIUM_PAGE_MASK (~((uintptr_t)MEDIUM_PAGE_SIZE - 1)) - -#define LARGE_PAGE_SIZE_SHIFT 26 -#define LARGE_PAGE_SIZE (1 << LARGE_PAGE_SIZE_SHIFT) -#define LARGE_PAGE_MASK (~((uintptr_t)LARGE_PAGE_SIZE - 1)) - -#define SMALL_CLASS(n) \ - { (((n)) * 16), (SMALL_PAGE_SIZE - sizeof(struct page)) / (((n)) * 16) } -#define MEDIUM_CLASS(n) \ - { (((n)) * 16), \ - (MEDIUM_PAGE_SIZE - sizeof(struct page)) / (((n)) * 16) } -#define LARGE_CLASS(n) \ - { (((n)) * 16), (LARGE_PAGE_SIZE - sizeof(struct page)) / (((n)) * 16) } - -static const struct class global_size_class[] = { - SMALL_CLASS(1), SMALL_CLASS(1), SMALL_CLASS(2), - SMALL_CLASS(3), SMALL_CLASS(4), SMALL_CLASS(5), - SMALL_CLASS(6), SMALL_CLASS(7), SMALL_CLASS(8), - SMALL_CLASS(9), SMALL_CLASS(10), SMALL_CLASS(11), - SMALL_CLASS(12), SMALL_CLASS(13), SMALL_CLASS(14), - SMALL_CLASS(15), SMALL_CLASS(16), SMALL_CLASS(17), - SMALL_CLASS(18), SMALL_CLASS(19), SMALL_CLASS(20), - SMALL_CLASS(21), SMALL_CLASS(22), SMALL_CLASS(23), - SMALL_CLASS(24), SMALL_CLASS(25), SMALL_CLASS(26), - SMALL_CLASS(27), SMALL_CLASS(28), SMALL_CLASS(29), - SMALL_CLASS(30), SMALL_CLASS(31), SMALL_CLASS(32), - SMALL_CLASS(33), SMALL_CLASS(34), SMALL_CLASS(35), - SMALL_CLASS(36), SMALL_CLASS(37), SMALL_CLASS(38), - SMALL_CLASS(39), SMALL_CLASS(40), SMALL_CLASS(41), - SMALL_CLASS(42), SMALL_CLASS(43), SMALL_CLASS(44), - SMALL_CLASS(45), SMALL_CLASS(46), SMALL_CLASS(47), - SMALL_CLASS(48), SMALL_CLASS(49), SMALL_CLASS(50), - SMALL_CLASS(51), SMALL_CLASS(52), SMALL_CLASS(53), - SMALL_CLASS(54), SMALL_CLASS(55), SMALL_CLASS(56), - SMALL_CLASS(57), SMALL_CLASS(58), SMALL_CLASS(59), - SMALL_CLASS(60), SMALL_CLASS(61), SMALL_CLASS(62), - SMALL_CLASS(63), SMALL_CLASS(64), SMALL_CLASS(80), - SMALL_CLASS(96), SMALL_CLASS(112), SMALL_CLASS(128), - SMALL_CLASS(160), SMALL_CLASS(192), SMALL_CLASS(224), - SMALL_CLASS(256), MEDIUM_CLASS(320), MEDIUM_CLASS(384), - MEDIUM_CLASS(448), MEDIUM_CLASS(512), MEDIUM_CLASS(640), - MEDIUM_CLASS(768), MEDIUM_CLASS(896), MEDIUM_CLASS(1024), - MEDIUM_CLASS(1280), MEDIUM_CLASS(1536), MEDIUM_CLASS(1792), - MEDIUM_CLASS(2048), MEDIUM_CLASS(2560), MEDIUM_CLASS(3072), - MEDIUM_CLASS(3584), MEDIUM_CLASS(4096), MEDIUM_CLASS(5120), - MEDIUM_CLASS(6144), MEDIUM_CLASS(7168), MEDIUM_CLASS(8192), - MEDIUM_CLASS(10240), MEDIUM_CLASS(12288), MEDIUM_CLASS(14336), - MEDIUM_CLASS(16384), LARGE_CLASS(20480), LARGE_CLASS(24576), - LARGE_CLASS(28672), LARGE_CLASS(32768), LARGE_CLASS(40960), - LARGE_CLASS(49152), LARGE_CLASS(57344), LARGE_CLASS(65536), - LARGE_CLASS(81920), LARGE_CLASS(98304), LARGE_CLASS(114688), - LARGE_CLASS(131072), LARGE_CLASS(163840), LARGE_CLASS(196608), - LARGE_CLASS(229376), LARGE_CLASS(262144), LARGE_CLASS(327680), - LARGE_CLASS(393216), LARGE_CLASS(458752), LARGE_CLASS(524288) -}; - -#endif diff --git a/lib/libc/include/syscall.h b/lib/libc/include/syscall.h deleted file mode 100644 index 645c4bcd..00000000 --- a/lib/libc/include/syscall.h +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef __LIBC_SYSCALL_H -#define __LIBC_SYSCALL_H - -#include -#include -#include - -typedef long syscall_arg_t; - -#define __SYSCALL_NARGS_(a, b, c, d, e, f, g, h, n, ...) n -#define __SYSCALL_NARGS(...) \ - __SYSCALL_NARGS_(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, ) - -#define __SYSCALL_CONCAT_(a, b) a##b -#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_(a, b) - -#define __SYSCALL_(b, ...) \ - __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) - -#define __syscall(...) __SYSCALL_(__syscall_, __VA_ARGS__) -#define syscall(...) __syscall_ret(__syscall(__VA_ARGS__)) - -#define __sa(x) ((syscall_arg_t)(x)) - -#define __syscall_0(n) __syscall0(__NR_##n) -#define __syscall_1(n, a) __syscall1(__NR_##n, __sa(a)) -#define __syscall_2(n, a, b) __syscall2(__NR_##n, __sa(a), __sa(b)) -#define __syscall_3(n, a, b, c) __syscall3(__NR_##n, __sa(a), __sa(b), __sa(c)) -#define __syscall_4(n, a, b, c, d) \ - __syscall4(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d)) -#define __syscall_5(n, a, b, c, d, e) \ - __syscall5(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d), __sa(e)) -#define __syscall_6(n, a, b, c, d, e, f) \ - __syscall6(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d), __sa(e), \ - __sa(f)) -#define __syscall_7(n, a, b, c, d, e, f, g) \ - __syscall7(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d), __sa(e), \ - __sa(f), __sa(g)) - -static __inline long __syscall_ret(long ret) -{ - if (ret > -4096 && ret < 0) { - errno = -ret; - ret = -1; - } - return ret; -} - -#ifdef __NR_pread64 -#undef __NR_pread -#define __NR_pread __NR_pread64 -#endif - -#ifdef __NR_pwrite64 -#undef __NR_pwrite -#define __NR_pwrite __NR_pwrite64 -#endif - -#endif diff --git a/lib/libc/internal/Kbuild b/lib/libc/internal/Kbuild index 01daa04b..d391eaa8 100644 --- a/lib/libc/internal/Kbuild +++ b/lib/libc/internal/Kbuild @@ -1 +1,5 @@ obj-y += init/ +obj-y += arch/ + +obj-y += panic.o +obj-y += syscall.o diff --git a/lib/libc/internal/arch/Kbuild b/lib/libc/internal/arch/Kbuild new file mode 100644 index 00000000..ad3a7486 --- /dev/null +++ b/lib/libc/internal/arch/Kbuild @@ -0,0 +1 @@ +obj-y += $(ARCH)/ diff --git a/lib/libc/internal/arch/x86_64/Kbuild b/lib/libc/internal/arch/x86_64/Kbuild new file mode 100644 index 00000000..830a1c5b --- /dev/null +++ b/lib/libc/internal/arch/x86_64/Kbuild @@ -0,0 +1,5 @@ +obj-y += _start.o +obj-y += fenv.o +obj-y += longjmp.o +obj-y += setjmp.o +obj-y += sigsetjmp.o diff --git a/lib/libc/internal/arch/x86_64/_start.c b/lib/libc/internal/arch/x86_64/_start.c new file mode 100644 index 00000000..f3f4fef2 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/_start.c @@ -0,0 +1,8 @@ +#include +#include + +__dead __naked void _start(void) +{ + __asm__ __volatile__("mov %rsp, %rdi\n" + "call __init\n"); +} diff --git a/lib/libc/internal/arch/x86_64/fenv.s b/lib/libc/internal/arch/x86_64/fenv.s new file mode 100644 index 00000000..b9a46581 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/fenv.s @@ -0,0 +1,97 @@ +.global feclearexcept +.type feclearexcept,@function +feclearexcept: + mov %edi,%ecx + and $0x3f,%ecx + fnstsw %ax + test %eax,%ecx + jz 1f + fnclex +1: stmxcsr -8(%rsp) + and $0x3f,%eax + or %eax,-8(%rsp) + test %ecx,-8(%rsp) + jz 1f + not %ecx + and %ecx,-8(%rsp) + ldmxcsr -8(%rsp) +1: xor %eax,%eax + ret + +.global feraiseexcept +.type feraiseexcept,@function +feraiseexcept: + and $0x3f,%edi + stmxcsr -8(%rsp) + or %edi,-8(%rsp) + ldmxcsr -8(%rsp) + xor %eax,%eax + ret + +.global __fesetround +.hidden __fesetround +.type __fesetround,@function +__fesetround: + push %rax + xor %eax,%eax + mov %edi,%ecx + fnstcw (%rsp) + andb $0xf3,1(%rsp) + or %ch,1(%rsp) + fldcw (%rsp) + stmxcsr (%rsp) + shl $3,%ch + andb $0x9f,1(%rsp) + or %ch,1(%rsp) + ldmxcsr (%rsp) + pop %rcx + ret + +.global fegetround +.type fegetround,@function +fegetround: + push %rax + stmxcsr (%rsp) + pop %rax + shr $3,%eax + and $0xc00,%eax + ret + +.global fegetenv +.type fegetenv,@function +fegetenv: + xor %eax,%eax + fnstenv (%rdi) + stmxcsr 28(%rdi) + ret + +.global fesetenv +.type fesetenv,@function +fesetenv: + xor %eax,%eax + inc %rdi + jz 1f + fldenv -1(%rdi) + ldmxcsr 27(%rdi) + ret +1: push %rax + push %rax + pushq $0xffff + pushq $0x37f + fldenv (%rsp) + pushq $0x1f80 + ldmxcsr (%rsp) + add $40,%rsp + ret + +.global fetestexcept +.type fetestexcept,@function +fetestexcept: + and $0x3f,%edi + push %rax + stmxcsr (%rsp) + pop %rsi + fnstsw %ax + or %esi,%eax + and %edi,%eax + ret diff --git a/lib/libc/internal/arch/x86_64/longjmp.c b/lib/libc/internal/arch/x86_64/longjmp.c new file mode 100644 index 00000000..dbd18632 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/longjmp.c @@ -0,0 +1,20 @@ +#include "bits/setjmp.h" // for jmp_buf + +#include // for longjmp + +__attribute__((noreturn, naked, returns_twice)) void longjmp(jmp_buf env, + int val) +{ + __asm__ __volatile__("mov 0(%rdi), %rbx\n" + "mov 8(%rdi), %rbp\n" + "mov 16(%rdi), %r12\n" + "mov 24(%rdi), %r13\n" + "mov 32(%rdi), %r14\n" + "mov 40(%rdi), %r15\n" + "mov 48(%rdi), %rsp\n" + "xor %rax, %rax\n" + "cmp $0, %esi\n" + "setne %al\n" + "mov %eax, %eax\n" + "jmp *56(%rdi)\n"); +} diff --git a/lib/libc/internal/arch/x86_64/setjmp.c b/lib/libc/internal/arch/x86_64/setjmp.c new file mode 100644 index 00000000..ea205e16 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/setjmp.c @@ -0,0 +1,19 @@ +#include "bits/setjmp.h" // for jmp_buf + +#include // for setjmp + +__attribute__((naked, returns_twice)) int setjmp(jmp_buf env) +{ + __asm__ __volatile__("mov %rbx,(%rdi)\n" + "mov %rbp,8(%rdi)\n" + "mov %r12,16(%rdi)\n" + "mov %r13,24(%rdi)\n" + "mov %r14,32(%rdi)\n" + "mov %r15,40(%rdi)\n" + "lea 8(%rsp),%rdx\n" + "mov %rdx,48(%rdi)\n" + "mov (%rsp),%rdx\n" + "mov %rdx,56(%rdi)\n" + "xor %eax,%eax\n" + "ret\n"); +} diff --git a/lib/libc/internal/arch/x86_64/sigsetjmp.c b/lib/libc/internal/arch/x86_64/sigsetjmp.c new file mode 100644 index 00000000..2d900ce7 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/sigsetjmp.c @@ -0,0 +1,33 @@ +#include // for sigsetjmp, sigjmp_buf + +int sigsetjmp(sigjmp_buf env, int savemask) +{ + __asm__ __volatile__("movq %%rbx, 0(%0)\n" + "movq %%rbp, 8(%0)\n" + "movq %%r12, 16(%0)\n" + "movq %%r13, 24(%0)\n" + "movq %%r14, 32(%0)\n" + "movq %%r15, 40(%0)\n" + + "leaq 8(%%rsp), %%rax\n" + "movq %%rax, 48(%0)\n" + "movq (%%rsp), %%rax\n" + "movq %%rax, 56(%0)\n" + + "testl %%esi, %%esi\n" + "jz 1f\n" + "movq $2, %%rdi\n" + "movq $0, %%rsi\n" + "leaq 64(%0), %%rdx\n" + "movq $128, %%r10\n" + "movq $14, %%rax\n" + "syscall\n" + + "1:\n" + "xor %%eax, %%eax\n" + : + : "r"(env), "S"(savemask) + : "rax", "rdi", "rdx", "r10", "memory"); + + return 0; +} diff --git a/lib/libc/internal/deinit/tls.c b/lib/libc/internal/deinit/tls.c new file mode 100644 index 00000000..438f0122 --- /dev/null +++ b/lib/libc/internal/deinit/tls.c @@ -0,0 +1,13 @@ +#include +#include + +void __deinit_tls(void) +{ + int r; + + if (__libc.tls.base == NULL) + return; + + r = munmap(__libc.tls.base, __libc.tls.size); + panic_if(__predict_false(r < 0), "munmap(tls) failed"); +} diff --git a/lib/libc/internal/include/__aio.h b/lib/libc/internal/include/__aio.h new file mode 100644 index 00000000..93cb1f79 --- /dev/null +++ b/lib/libc/internal/include/__aio.h @@ -0,0 +1,35 @@ +#ifndef __LIBC_AIO_H +#define __LIBC_AIO_H + +#include + +#define AIO_REQUEST_STATUS_PENDING 0 +#define AIO_REQUEST_STATUS_COMPLETED 1 + +struct lio_group { + int pending; + int error; + int eventfd; + struct sigevent *sig; +}; + +struct aio_context { + struct aio_request *head; + struct aio_request *tail; +}; + +struct aio_request { + struct aiocb *aiocbp; + int status; + ssize_t result; + struct aio_request *next; + struct lio_group *grp; +}; + +void __aio_poll(void); + +int __aio_request(struct aio_request *, int); +struct aio_request *__aio_lookup(const struct aiocb *); +struct aio_request *__aio_remove(const struct aiocb *); + +#endif diff --git a/lib/libc/internal/include/__dirent.h b/lib/libc/internal/include/__dirent.h new file mode 100644 index 00000000..4f9ead80 --- /dev/null +++ b/lib/libc/internal/include/__dirent.h @@ -0,0 +1,24 @@ +#ifndef __LIBC_DIRENT_H +#define __LIBC_DIRENT_H + +#include +#include +#include + +struct linux_dirent64 { + uint64_t d_ino; + int64_t d_off; + unsigned short d_reclen; + unsigned char d_type; + char d_name[]; +}; + +struct __DIR { + int fildes; + int cached; + off_t tell; + off_t offset; + char buffer[BUFSIZ] __attribute__((__aligned__(8))); +}; + +#endif diff --git a/lib/libc/internal/include/__select.h b/lib/libc/internal/include/__select.h new file mode 100644 index 00000000..a38bbdce --- /dev/null +++ b/lib/libc/internal/include/__select.h @@ -0,0 +1,8 @@ +#ifndef __LIBC_SELECT_H +#define __LIBC_SELECT_H + +struct __fd_set { + unsigned long fds_bits[16]; +}; + +#endif diff --git a/lib/libc/internal/include/__signal.h b/lib/libc/internal/include/__signal.h new file mode 100644 index 00000000..6b762dee --- /dev/null +++ b/lib/libc/internal/include/__signal.h @@ -0,0 +1,12 @@ +#ifndef __LIBC_SIGNAL_H +#define __LIBC_SIGNAL_H + +static const char *__sys_signame[] = { + "SIGHUP", "SIGINT", "SIGQUIT", "SIGILL", "SIGTRAP", "SIGABRT", + "SIGBUS", "SIGFPE", "SIGKILL", "SIGUSR1", "SIGSEGV", "SIGUSR2", + "SIGPIPE", "SIGALRM", "SIGTERM", "SIGCHLD", "SIGCONT", "SIGSTOP", + "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGXCPU", "SIGXFSZ", + "SIGVTALRM", "SIGPROF", "SIGWINCH", "SIGIO", "SIGSYS" +}; + +#endif diff --git a/lib/libc/internal/include/__statvfs.h b/lib/libc/internal/include/__statvfs.h new file mode 100644 index 00000000..c8a746d4 --- /dev/null +++ b/lib/libc/internal/include/__statvfs.h @@ -0,0 +1,21 @@ +#ifndef __LIBC_STATVFS_H__ +#define __LIBC_STATVFS_H__ + +typedef __UINT64_TYPE__ fsblkcnt_t; +typedef __UINT64_TYPE__ fsfilcnt_t; + +struct __statvfs { + unsigned long f_bsize; + unsigned long f_frsize; + fsblkcnt_t f_blocks; + fsblkcnt_t f_bfree; + fsblkcnt_t f_bavail; + fsfilcnt_t f_files; + fsfilcnt_t f_ffree; + fsfilcnt_t f_favail; + unsigned long f_fsid; + unsigned long f_flag; + unsigned long f_namemax; +}; + +#endif diff --git a/lib/libc/internal/include/__stdio.h b/lib/libc/internal/include/__stdio.h new file mode 100644 index 00000000..487a0382 --- /dev/null +++ b/lib/libc/internal/include/__stdio.h @@ -0,0 +1,38 @@ +#ifndef __LIBC_STDIO_H__ +#define __LIBC_STDIO_H__ + +#include +#include +#include +#include +#include +#include + +typedef __SIZE_TYPE__ size_t; + +#define _IO_ERR 0x4 +#define _IO_EOF 0x8 +#define _IO_WIDE 0x10 + +struct __FILE { + int fd; + uint32_t flags; + int type; + pid_t pid; + atomic_flag lock; + char *buf; + int eof; + size_t buf_size; + size_t buf_pos; + size_t buf_len; + unsigned char unget_buf[16]; + size_t unget_cnt; + off_t offset; + struct __FILE *next; +}; + +#define __FILE(__stream) ((struct __FILE *)(__stream)) + +void __libc_fadd(struct __FILE *f); + +#endif diff --git a/lib/libc/internal/include/__thread.h b/lib/libc/internal/include/__thread.h new file mode 100644 index 00000000..7639707a --- /dev/null +++ b/lib/libc/internal/include/__thread.h @@ -0,0 +1,8 @@ +#ifndef __LIBC_THREAD_H +#define __LIBC_THREAD_H + +struct __thread_self { + int tid; +}; + +#endif diff --git a/lib/libc/internal/include/atomic.h b/lib/libc/internal/include/atomic.h new file mode 100644 index 00000000..e0593021 --- /dev/null +++ b/lib/libc/internal/include/atomic.h @@ -0,0 +1,23 @@ +#ifndef __LIBC_ATOMIC_H +#define __LIBC_ATOMIC_H + +#include + +#define LIBC_LOCK(__lock) __libc_lock(&((__lock))) +#define LIBC_UNLOCK(__lock) atomic_flag_clear(&((__lock))) + +static __inline void __libc_lock(volatile atomic_flag *lock) +{ + while (atomic_flag_test_and_set_explicit(lock, memory_order_acquire)) { + unsigned int spins = 1; + do { + for (unsigned int i = 0; i < spins; i++) + __asm__ volatile("pause"); + if (spins < 64) + spins *= 2; + } while (atomic_flag_test_and_set_explicit( + lock, memory_order_relaxed)); + } +} + +#endif diff --git a/lib/libc/internal/include/byteswap.h b/lib/libc/internal/include/byteswap.h new file mode 100644 index 00000000..9dfe6520 --- /dev/null +++ b/lib/libc/internal/include/byteswap.h @@ -0,0 +1,8 @@ +#ifndef __LIBC_BYTESWAP_H +#define __LIBC_BYTESWAP_H + +#define bswap16(x) __builtin_bswap16(x) +#define bswap32(x) __builtin_bswap32(x) +#define bswap64(x) __builtin_bswap64(x) + +#endif diff --git a/lib/libc/internal/include/internal/io_uring.h b/lib/libc/internal/include/internal/io_uring.h new file mode 100644 index 00000000..e7369146 --- /dev/null +++ b/lib/libc/internal/include/internal/io_uring.h @@ -0,0 +1,6 @@ +#ifndef __INTERNAL_IO_URING_H +#define __INTERNAL_IO_URING_H + +void __io_uring_init(void); + +#endif diff --git a/lib/libc/internal/include/io_uring.h b/lib/libc/internal/include/io_uring.h new file mode 100644 index 00000000..4dac4583 --- /dev/null +++ b/lib/libc/internal/include/io_uring.h @@ -0,0 +1,53 @@ +#ifndef __LIBC_IO_URING_H +#define __LIBC_IO_URING_H + +#include +#include + +#define IO_URING_ENTRIES 256 + +struct io_uring_sq { + void *ring; + size_t ring_size; + struct io_uring_sqe *sqes; + + unsigned *head; + unsigned *tail; + unsigned *ring_mask; + unsigned *ring_entries; + unsigned *flags; + unsigned *dropped; + unsigned *array; +}; + +struct io_uring_cq { + void *ring; + size_t ring_size; + + unsigned *head; + unsigned *tail; + unsigned *ring_mask; + unsigned *ring_entries; + unsigned *overflow; + struct io_uring_cqe *cqes; + unsigned *flags; +}; + +struct io_uring { + int fd; + int eventfd; + + struct io_uring_sq sq; + struct io_uring_cq cq; +}; + +extern struct io_uring __io_uring; + +int io_uring_setup(unsigned int, struct io_uring_params *); + +int io_uring_register(unsigned int, unsigned int, void *, unsigned int); + +int io_uring_enter(unsigned int, unsigned int, unsigned int, unsigned int, + sigset_t *, size_t); + +#endif diff --git a/lib/libc/internal/include/libc.h b/lib/libc/internal/include/libc.h new file mode 100644 index 00000000..f06fa6d8 --- /dev/null +++ b/lib/libc/internal/include/libc.h @@ -0,0 +1,48 @@ +#ifndef __LIBC_LIBC_H +#define __LIBC_LIBC_H + +#include +#include +#include + +#define weak_reference(old, new) extern __typeof(old)((new)) __attribute__((__weak__, __alias__(#old))) + +struct tls { + struct tls *next; + void *data; + size_t size; + size_t align; + size_t length; + size_t offset; +}; + +struct libc { + size_t auxv[32]; + struct { + void *base; + size_t size; + } tls; + + enum { + LIBC_ENVP_TOUCHED = 1 << 0, + } flags; + + struct { + volatile atomic_flag abort; + volatile atomic_flag malloc; + volatile atomic_flag environ; + } lock; +}; + +extern struct libc __libc; + +#define panic(__errmsg) __libc_panic(__FILE__ ":" __STRING(__LINE__) ": ", __PRETTY_FUNCTION__, ": " __STRING(__errmsg)) + +#define panic_if(__cond, __errmsg) \ + do { \ + if (__predict_false(__cond)) \ + panic(__errmsg); \ + } while (0) + +__dead void __libc_panic(const char *, const char *, const char *); +#endif diff --git a/lib/libc/internal/include/libc/dirent.h b/lib/libc/internal/include/libc/dirent.h new file mode 100644 index 00000000..296cc91e --- /dev/null +++ b/lib/libc/internal/include/libc/dirent.h @@ -0,0 +1,6 @@ +#ifndef __LIBC_DIRENT_H +#define __LIBC_DIRENT_H + +struct __DIR {}; + +#endif diff --git a/lib/libc/internal/include/malloc.h b/lib/libc/internal/include/malloc.h new file mode 100644 index 00000000..b7cb0093 --- /dev/null +++ b/lib/libc/internal/include/malloc.h @@ -0,0 +1,98 @@ +#ifndef __LIBC_MALLOC_H +#define __LIBC_MALLOC_H + +#include +#include + +struct class { + uint32_t size; + uint32_t count; +}; + +struct page { + volatile atomic_flag lock; + + enum { + PAGE_SMALL = 0x0, + PAGE_MEDIUM = 0x1, + PAGE_LARGE = 0x2, + } flags; + + struct { + uint32_t size; + uint32_t used; + uint32_t count; + } block; + + uint8_t *bitmap; + uint8_t *heap; + + struct page *next; + struct page *prev; +}; + +extern struct page *__malloc_pvec; + +#define SMALL_PAGE_SIZE_SHIFT 16 +#define SMALL_PAGE_SIZE (1 << SMALL_PAGE_SIZE_SHIFT) +#define SMALL_PAGE_MASK (~((uintptr_t)SMALL_PAGE_SIZE - 1)) + +#define MEDIUM_PAGE_SIZE_SHIFT 22 +#define MEDIUM_PAGE_SIZE (1 << MEDIUM_PAGE_SIZE_SHIFT) +#define MEDIUM_PAGE_MASK (~((uintptr_t)MEDIUM_PAGE_SIZE - 1)) + +#define LARGE_PAGE_SIZE_SHIFT 26 +#define LARGE_PAGE_SIZE (1 << LARGE_PAGE_SIZE_SHIFT) +#define LARGE_PAGE_MASK (~((uintptr_t)LARGE_PAGE_SIZE - 1)) + +#define SMALL_CLASS(n) \ + { (((n)) * 16), (SMALL_PAGE_SIZE - sizeof(struct page)) / (((n)) * 16) } +#define MEDIUM_CLASS(n) \ + { (((n)) * 16), \ + (MEDIUM_PAGE_SIZE - sizeof(struct page)) / (((n)) * 16) } +#define LARGE_CLASS(n) \ + { (((n)) * 16), (LARGE_PAGE_SIZE - sizeof(struct page)) / (((n)) * 16) } + +static const struct class global_size_class[] = { + SMALL_CLASS(1), SMALL_CLASS(1), SMALL_CLASS(2), + SMALL_CLASS(3), SMALL_CLASS(4), SMALL_CLASS(5), + SMALL_CLASS(6), SMALL_CLASS(7), SMALL_CLASS(8), + SMALL_CLASS(9), SMALL_CLASS(10), SMALL_CLASS(11), + SMALL_CLASS(12), SMALL_CLASS(13), SMALL_CLASS(14), + SMALL_CLASS(15), SMALL_CLASS(16), SMALL_CLASS(17), + SMALL_CLASS(18), SMALL_CLASS(19), SMALL_CLASS(20), + SMALL_CLASS(21), SMALL_CLASS(22), SMALL_CLASS(23), + SMALL_CLASS(24), SMALL_CLASS(25), SMALL_CLASS(26), + SMALL_CLASS(27), SMALL_CLASS(28), SMALL_CLASS(29), + SMALL_CLASS(30), SMALL_CLASS(31), SMALL_CLASS(32), + SMALL_CLASS(33), SMALL_CLASS(34), SMALL_CLASS(35), + SMALL_CLASS(36), SMALL_CLASS(37), SMALL_CLASS(38), + SMALL_CLASS(39), SMALL_CLASS(40), SMALL_CLASS(41), + SMALL_CLASS(42), SMALL_CLASS(43), SMALL_CLASS(44), + SMALL_CLASS(45), SMALL_CLASS(46), SMALL_CLASS(47), + SMALL_CLASS(48), SMALL_CLASS(49), SMALL_CLASS(50), + SMALL_CLASS(51), SMALL_CLASS(52), SMALL_CLASS(53), + SMALL_CLASS(54), SMALL_CLASS(55), SMALL_CLASS(56), + SMALL_CLASS(57), SMALL_CLASS(58), SMALL_CLASS(59), + SMALL_CLASS(60), SMALL_CLASS(61), SMALL_CLASS(62), + SMALL_CLASS(63), SMALL_CLASS(64), SMALL_CLASS(80), + SMALL_CLASS(96), SMALL_CLASS(112), SMALL_CLASS(128), + SMALL_CLASS(160), SMALL_CLASS(192), SMALL_CLASS(224), + SMALL_CLASS(256), MEDIUM_CLASS(320), MEDIUM_CLASS(384), + MEDIUM_CLASS(448), MEDIUM_CLASS(512), MEDIUM_CLASS(640), + MEDIUM_CLASS(768), MEDIUM_CLASS(896), MEDIUM_CLASS(1024), + MEDIUM_CLASS(1280), MEDIUM_CLASS(1536), MEDIUM_CLASS(1792), + MEDIUM_CLASS(2048), MEDIUM_CLASS(2560), MEDIUM_CLASS(3072), + MEDIUM_CLASS(3584), MEDIUM_CLASS(4096), MEDIUM_CLASS(5120), + MEDIUM_CLASS(6144), MEDIUM_CLASS(7168), MEDIUM_CLASS(8192), + MEDIUM_CLASS(10240), MEDIUM_CLASS(12288), MEDIUM_CLASS(14336), + MEDIUM_CLASS(16384), LARGE_CLASS(20480), LARGE_CLASS(24576), + LARGE_CLASS(28672), LARGE_CLASS(32768), LARGE_CLASS(40960), + LARGE_CLASS(49152), LARGE_CLASS(57344), LARGE_CLASS(65536), + LARGE_CLASS(81920), LARGE_CLASS(98304), LARGE_CLASS(114688), + LARGE_CLASS(131072), LARGE_CLASS(163840), LARGE_CLASS(196608), + LARGE_CLASS(229376), LARGE_CLASS(262144), LARGE_CLASS(327680), + LARGE_CLASS(393216), LARGE_CLASS(458752), LARGE_CLASS(524288) +}; + +#endif diff --git a/lib/libc/internal/include/syscall.h b/lib/libc/internal/include/syscall.h new file mode 100644 index 00000000..ea72ece4 --- /dev/null +++ b/lib/libc/internal/include/syscall.h @@ -0,0 +1,44 @@ +#ifndef __LIBC_SYSCALL_H +#define __LIBC_SYSCALL_H + +#include +#include + +typedef long syscall_arg_t; + +#define __SYSCALL_NARGS_(a, b, c, d, e, f, g, h, n, ...) n +#define __SYSCALL_NARGS(...) __SYSCALL_NARGS_(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, ) + +#define __SYSCALL_CONCAT_(a, b) a##b +#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_(a, b) + +#define __SYSCALL_(b, ...) __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) + +#define __syscall(...) __SYSCALL_(__syscall_, __VA_ARGS__) +#define syscall(...) __syscall_ret(__syscall(__VA_ARGS__)) + +#define __sa(x) ((syscall_arg_t)(x)) + +#define __syscall_0(n) __syscall0(__NR_##n) +#define __syscall_1(n, a) __syscall1(__NR_##n, __sa(a)) +#define __syscall_2(n, a, b) __syscall2(__NR_##n, __sa(a), __sa(b)) +#define __syscall_3(n, a, b, c) __syscall3(__NR_##n, __sa(a), __sa(b), __sa(c)) +#define __syscall_4(n, a, b, c, d) __syscall4(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d)) +#define __syscall_5(n, a, b, c, d, e) __syscall5(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d), __sa(e)) +#define __syscall_6(n, a, b, c, d, e, f) __syscall6(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d), __sa(e), __sa(f)) +#define __syscall_7(n, a, b, c, d, e, f, g) \ + __syscall7(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d), __sa(e), __sa(f), __sa(g)) + +#ifdef __NR_pread64 +#undef __NR_pread +#define __NR_pread __NR_pread64 +#endif + +#ifdef __NR_pwrite64 +#undef __NR_pwrite +#define __NR_pwrite __NR_pwrite64 +#endif + +extern long __syscall_ret(long ret); + +#endif diff --git a/lib/libc/internal/init/Kbuild b/lib/libc/internal/init/Kbuild index 7941a53c..361f220a 100644 --- a/lib/libc/internal/init/Kbuild +++ b/lib/libc/internal/init/Kbuild @@ -1,2 +1,3 @@ obj-y += init.o +obj-y += tls.o obj-y += vdso.o diff --git a/lib/libc/internal/init/init.c b/lib/libc/internal/init/init.c index 36e66ee4..b2dd8487 100644 --- a/lib/libc/internal/init/init.c +++ b/lib/libc/internal/init/init.c @@ -5,13 +5,14 @@ #include extern void __init_vdso(void); +extern void __init_tls(void); + +extern int main(int, char **, char **); struct libc __libc = { 0 }; char **environ; char *__progname; -extern int main(int, char **, char **); - __used void __init(uintptr_t *rsp) { char **argv; @@ -34,6 +35,7 @@ __used void __init(uintptr_t *rsp) for (size_t i = 0; auxv[i]; i += 2) __libc.auxv[auxv[i]] = auxv[i + 1]; + __init_tls(); __init_vdso(); exit(main(argc, argv, environ)); diff --git a/lib/libc/internal/init/tls.c b/lib/libc/internal/init/tls.c new file mode 100644 index 00000000..f6ccde35 --- /dev/null +++ b/lib/libc/internal/init/tls.c @@ -0,0 +1,76 @@ +#include <__thread.h> +#include +#include +#include +#include +#include +#include + +#define ALIGN_DOWN(x, a) ((x) & ~((a) - 1)) +#define ALIGN_UP(x, a) (((x) + (a) - 1) & ~((a) - 1)) + +static struct tls __libc_tls = { 0 }; +static struct __thread_self __libc_thread; +volatile int __libc_tid; + +void __init_tls(void) +{ + int r; + void *mem; + size_t base; + unsigned char *p; + Elf64_Phdr *tls_phdr; + struct __thread_self *thrd; + + base = 0; + tls_phdr = 0; + p = (void *)__libc.auxv[AT_PHDR]; + for (size_t n = __libc.auxv[AT_PHNUM]; n > 0; n--, p += __libc.auxv[AT_PHENT]) { + Elf64_Phdr *phdr = (Elf64_Phdr *)p; + + if (phdr->p_type == PT_PHDR) { + base = __libc.auxv[AT_PHDR] - phdr->p_vaddr; + } else if (phdr->p_type == PT_TLS) { + tls_phdr = phdr; + } + } + + if (tls_phdr) { + __libc_tls.data = (void *)(base + tls_phdr->p_vaddr); + __libc_tls.length = tls_phdr->p_filesz; + __libc_tls.size = tls_phdr->p_memsz; + __libc_tls.align = tls_phdr->p_align; + } + + if (__libc_tls.size != 0) { + void *tls_mem; + size_t tls_size = __libc_tls.size + sizeof(struct __thread_self); + tls_size = (tls_size + __libc_tls.align - 1) & ~(__libc_tls.align - 1); + + mem = mmap(0, tls_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + panic_if(mem == MAP_FAILED, "mmap failed"); + + __libc.tls.base = mem; + __libc.tls.size = tls_size; + + thrd = (struct __thread_self *)mem; + tls_mem = (unsigned char *)mem + sizeof(struct __thread_self); + + if (tls_phdr->p_filesz > 0) { + memcpy(tls_mem, (void *)tls_phdr->p_vaddr, tls_phdr->p_filesz); + memset(tls_mem + tls_phdr->p_filesz, 0, tls_phdr->p_memsz - tls_phdr->p_filesz); + } else { + memset(tls_mem, 0, tls_phdr->p_memsz); + } + + r = syscall(arch_prctl, ARCH_SET_FS, mem + sizeof(struct __thread_self)); + panic_if(r < 0, "arch_prctl(SET_FS) failed"); + } else { + __libc.tls.base = NULL; + thrd = &__libc_thread; + } + + r = syscall(set_tid_address, &__libc_tid); + panic_if(r < 0, "set_tid_address failed"); + thrd->tid = r; +} diff --git a/lib/libc/internal/panic.c b/lib/libc/internal/panic.c new file mode 100644 index 00000000..1af6d6cf --- /dev/null +++ b/lib/libc/internal/panic.c @@ -0,0 +1,29 @@ +#include +#include +#include +#include + +__dead void __libc_panic(const char *prefix, const char *f, const char *errmsg) +{ + struct iovec iovec[5]; + const char *e = "libc panic: "; + + iovec[0].iov_base = (char *)e; + iovec[0].iov_len = sizeof("libc panic: ") - 1; + + iovec[1].iov_base = (char *)prefix; + iovec[1].iov_len = strlen(prefix); + + iovec[2].iov_base = (char *)f; + iovec[2].iov_len = strlen(f); + + iovec[3].iov_base = (char *)errmsg; + iovec[3].iov_len = strlen(errmsg); + + iovec[4].iov_base = "\n"; + iovec[4].iov_len = 1; + + writev(STDERR_FILENO, iovec, 5); + + __builtin_trap(); +} diff --git a/lib/libc/internal/syscall.c b/lib/libc/internal/syscall.c new file mode 100644 index 00000000..f7fd3c4b --- /dev/null +++ b/lib/libc/internal/syscall.c @@ -0,0 +1,12 @@ +#include +#include + +long __syscall_ret(long ret) +{ + if (__predict_false(ret > -4096 && ret < 0)) { + errno = (int)-ret; + ret = -1; + } + + return ret; +} diff --git a/lib/libc/mman/mmap.c b/lib/libc/mman/mmap.c index d43ed686..b3ebdbbd 100644 --- a/lib/libc/mman/mmap.c +++ b/lib/libc/mman/mmap.c @@ -1,5 +1,3 @@ - - #include // for size_t #include // for __syscall_6, syscall diff --git a/lib/libc/mqueue/mq_notify.c b/lib/libc/mqueue/mq_notify.c index a7617375..6011f567 100644 --- a/lib/libc/mqueue/mq_notify.c +++ b/lib/libc/mqueue/mq_notify.c @@ -1,11 +1,11 @@ +#include #include #include #include int mq_notify(mqd_t mqdes, const struct sigevent *notification) { - if (notification == NULL || - notification->sigev_notify != SIGEV_THREAD) { + if (notification == NULL || notification->sigev_notify != SIGEV_THREAD) { return syscall(mq_notify, mqdes, notification); } diff --git a/lib/libc/mqueue/mq_unlink.c b/lib/libc/mqueue/mq_unlink.c index d136f109..da16f030 100644 --- a/lib/libc/mqueue/mq_unlink.c +++ b/lib/libc/mqueue/mq_unlink.c @@ -1,3 +1,4 @@ +#include #include #include @@ -11,7 +12,6 @@ int mq_unlink(const char *name) r = __syscall(mq_unlink, name); if (r < 0) { - // Correct errno for POSIX compliance if (r == -EPERM) r = -EACCES; errno = -r; diff --git a/lib/libc/sys/getauxval.c b/lib/libc/sys/getauxval.c index 155d4258..c9a2bf66 100644 --- a/lib/libc/sys/getauxval.c +++ b/lib/libc/sys/getauxval.c @@ -1,6 +1,8 @@ #include #include +#include + unsigned long getauxval(unsigned long type) { size_t *auxv = __libc.auxv; diff --git a/lib/libc/thread/thrd_current.c b/lib/libc/thread/thrd_current.c index 69a99723..b588cc27 100644 --- a/lib/libc/thread/thrd_current.c +++ b/lib/libc/thread/thrd_current.c @@ -1,8 +1,9 @@ -#include <__thread.h> // for __thread_self -#include // for thrd_current, thrd_t +#include <__thread.h> +#include + +thread_local struct __thread_self __thread_self; thrd_t thrd_current(void) { - static struct __thread_self self = { 0 }; - return &self; + return &__thread_self; } -- cgit v1.2.3