diff options
Diffstat (limited to 'lib/libc/internal')
29 files changed, 754 insertions, 2 deletions
diff --git a/lib/libc/internal/Kbuild b/lib/libc/internal/Kbuild index 01daa04b..d391eaa8 100644 --- a/lib/libc/internal/Kbuild +++ b/lib/libc/internal/Kbuild @@ -1 +1,5 @@ obj-y += init/ +obj-y += arch/ + +obj-y += panic.o +obj-y += syscall.o diff --git a/lib/libc/internal/arch/Kbuild b/lib/libc/internal/arch/Kbuild new file mode 100644 index 00000000..ad3a7486 --- /dev/null +++ b/lib/libc/internal/arch/Kbuild @@ -0,0 +1 @@ +obj-y += $(ARCH)/ diff --git a/lib/libc/internal/arch/x86_64/Kbuild b/lib/libc/internal/arch/x86_64/Kbuild new file mode 100644 index 00000000..830a1c5b --- /dev/null +++ b/lib/libc/internal/arch/x86_64/Kbuild @@ -0,0 +1,5 @@ +obj-y += _start.o +obj-y += fenv.o +obj-y += longjmp.o +obj-y += setjmp.o +obj-y += sigsetjmp.o diff --git a/lib/libc/internal/arch/x86_64/_start.c b/lib/libc/internal/arch/x86_64/_start.c new file mode 100644 index 00000000..f3f4fef2 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/_start.c @@ -0,0 +1,8 @@ +#include <libc.h> +#include <sys/cdefs.h> + +__dead __naked void _start(void) +{ + __asm__ __volatile__("mov %rsp, %rdi\n" + "call __init\n"); +} diff --git a/lib/libc/internal/arch/x86_64/fenv.s b/lib/libc/internal/arch/x86_64/fenv.s new file mode 100644 index 00000000..b9a46581 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/fenv.s @@ -0,0 +1,97 @@ +.global feclearexcept +.type feclearexcept,@function +feclearexcept: + mov %edi,%ecx + and $0x3f,%ecx + fnstsw %ax + test %eax,%ecx + jz 1f + fnclex +1: stmxcsr -8(%rsp) + and $0x3f,%eax + or %eax,-8(%rsp) + test %ecx,-8(%rsp) + jz 1f + not %ecx + and %ecx,-8(%rsp) + ldmxcsr -8(%rsp) +1: xor %eax,%eax + ret + +.global feraiseexcept +.type feraiseexcept,@function +feraiseexcept: + and $0x3f,%edi + stmxcsr -8(%rsp) + or %edi,-8(%rsp) + ldmxcsr -8(%rsp) + xor %eax,%eax + ret + +.global __fesetround +.hidden __fesetround +.type __fesetround,@function +__fesetround: + push %rax + xor %eax,%eax + mov %edi,%ecx + fnstcw (%rsp) + andb $0xf3,1(%rsp) + or %ch,1(%rsp) + fldcw (%rsp) + stmxcsr (%rsp) + shl $3,%ch + andb $0x9f,1(%rsp) + or %ch,1(%rsp) + ldmxcsr (%rsp) + pop %rcx + ret + +.global fegetround +.type fegetround,@function +fegetround: + push %rax + stmxcsr (%rsp) + pop %rax + shr $3,%eax + and $0xc00,%eax + ret + +.global fegetenv +.type fegetenv,@function +fegetenv: + xor %eax,%eax + fnstenv (%rdi) + stmxcsr 28(%rdi) + ret + +.global fesetenv +.type fesetenv,@function +fesetenv: + xor %eax,%eax + inc %rdi + jz 1f + fldenv -1(%rdi) + ldmxcsr 27(%rdi) + ret +1: push %rax + push %rax + pushq $0xffff + pushq $0x37f + fldenv (%rsp) + pushq $0x1f80 + ldmxcsr (%rsp) + add $40,%rsp + ret + +.global fetestexcept +.type fetestexcept,@function +fetestexcept: + and $0x3f,%edi + push %rax + stmxcsr (%rsp) + pop %rsi + fnstsw %ax + or %esi,%eax + and %edi,%eax + ret diff --git a/lib/libc/internal/arch/x86_64/longjmp.c b/lib/libc/internal/arch/x86_64/longjmp.c new file mode 100644 index 00000000..dbd18632 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/longjmp.c @@ -0,0 +1,20 @@ +#include "bits/setjmp.h" // for jmp_buf + +#include <setjmp.h> // for longjmp + +__attribute__((noreturn, naked, returns_twice)) void longjmp(jmp_buf env, + int val) +{ + __asm__ __volatile__("mov 0(%rdi), %rbx\n" + "mov 8(%rdi), %rbp\n" + "mov 16(%rdi), %r12\n" + "mov 24(%rdi), %r13\n" + "mov 32(%rdi), %r14\n" + "mov 40(%rdi), %r15\n" + "mov 48(%rdi), %rsp\n" + "xor %rax, %rax\n" + "cmp $0, %esi\n" + "setne %al\n" + "mov %eax, %eax\n" + "jmp *56(%rdi)\n"); +} diff --git a/lib/libc/internal/arch/x86_64/setjmp.c b/lib/libc/internal/arch/x86_64/setjmp.c new file mode 100644 index 00000000..ea205e16 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/setjmp.c @@ -0,0 +1,19 @@ +#include "bits/setjmp.h" // for jmp_buf + +#include <setjmp.h> // for setjmp + +__attribute__((naked, returns_twice)) int setjmp(jmp_buf env) +{ + __asm__ __volatile__("mov %rbx,(%rdi)\n" + "mov %rbp,8(%rdi)\n" + "mov %r12,16(%rdi)\n" + "mov %r13,24(%rdi)\n" + "mov %r14,32(%rdi)\n" + "mov %r15,40(%rdi)\n" + "lea 8(%rsp),%rdx\n" + "mov %rdx,48(%rdi)\n" + "mov (%rsp),%rdx\n" + "mov %rdx,56(%rdi)\n" + "xor %eax,%eax\n" + "ret\n"); +} diff --git a/lib/libc/internal/arch/x86_64/sigsetjmp.c b/lib/libc/internal/arch/x86_64/sigsetjmp.c new file mode 100644 index 00000000..2d900ce7 --- /dev/null +++ b/lib/libc/internal/arch/x86_64/sigsetjmp.c @@ -0,0 +1,33 @@ +#include <setjmp.h> // for sigsetjmp, sigjmp_buf + +int sigsetjmp(sigjmp_buf env, int savemask) +{ + __asm__ __volatile__("movq %%rbx, 0(%0)\n" + "movq %%rbp, 8(%0)\n" + "movq %%r12, 16(%0)\n" + "movq %%r13, 24(%0)\n" + "movq %%r14, 32(%0)\n" + "movq %%r15, 40(%0)\n" + + "leaq 8(%%rsp), %%rax\n" + "movq %%rax, 48(%0)\n" + "movq (%%rsp), %%rax\n" + "movq %%rax, 56(%0)\n" + + "testl %%esi, %%esi\n" + "jz 1f\n" + "movq $2, %%rdi\n" + "movq $0, %%rsi\n" + "leaq 64(%0), %%rdx\n" + "movq $128, %%r10\n" + "movq $14, %%rax\n" + "syscall\n" + + "1:\n" + "xor %%eax, %%eax\n" + : + : "r"(env), "S"(savemask) + : "rax", "rdi", "rdx", "r10", "memory"); + + return 0; +} diff --git a/lib/libc/internal/deinit/tls.c b/lib/libc/internal/deinit/tls.c new file mode 100644 index 00000000..438f0122 --- /dev/null +++ b/lib/libc/internal/deinit/tls.c @@ -0,0 +1,13 @@ +#include <libc.h> +#include <sys/mman.h> + +void __deinit_tls(void) +{ + int r; + + if (__libc.tls.base == NULL) + return; + + r = munmap(__libc.tls.base, __libc.tls.size); + panic_if(__predict_false(r < 0), "munmap(tls) failed"); +} diff --git a/lib/libc/internal/include/__aio.h b/lib/libc/internal/include/__aio.h new file mode 100644 index 00000000..93cb1f79 --- /dev/null +++ b/lib/libc/internal/include/__aio.h @@ -0,0 +1,35 @@ +#ifndef __LIBC_AIO_H +#define __LIBC_AIO_H + +#include <aio.h> + +#define AIO_REQUEST_STATUS_PENDING 0 +#define AIO_REQUEST_STATUS_COMPLETED 1 + +struct lio_group { + int pending; + int error; + int eventfd; + struct sigevent *sig; +}; + +struct aio_context { + struct aio_request *head; + struct aio_request *tail; +}; + +struct aio_request { + struct aiocb *aiocbp; + int status; + ssize_t result; + struct aio_request *next; + struct lio_group *grp; +}; + +void __aio_poll(void); + +int __aio_request(struct aio_request *, int); +struct aio_request *__aio_lookup(const struct aiocb *); +struct aio_request *__aio_remove(const struct aiocb *); + +#endif diff --git a/lib/libc/internal/include/__dirent.h b/lib/libc/internal/include/__dirent.h new file mode 100644 index 00000000..4f9ead80 --- /dev/null +++ b/lib/libc/internal/include/__dirent.h @@ -0,0 +1,24 @@ +#ifndef __LIBC_DIRENT_H +#define __LIBC_DIRENT_H + +#include <stdint.h> +#include <stdio.h> +#include <sys/types.h> + +struct linux_dirent64 { + uint64_t d_ino; + int64_t d_off; + unsigned short d_reclen; + unsigned char d_type; + char d_name[]; +}; + +struct __DIR { + int fildes; + int cached; + off_t tell; + off_t offset; + char buffer[BUFSIZ] __attribute__((__aligned__(8))); +}; + +#endif diff --git a/lib/libc/internal/include/__select.h b/lib/libc/internal/include/__select.h new file mode 100644 index 00000000..a38bbdce --- /dev/null +++ b/lib/libc/internal/include/__select.h @@ -0,0 +1,8 @@ +#ifndef __LIBC_SELECT_H +#define __LIBC_SELECT_H + +struct __fd_set { + unsigned long fds_bits[16]; +}; + +#endif diff --git a/lib/libc/internal/include/__signal.h b/lib/libc/internal/include/__signal.h new file mode 100644 index 00000000..6b762dee --- /dev/null +++ b/lib/libc/internal/include/__signal.h @@ -0,0 +1,12 @@ +#ifndef __LIBC_SIGNAL_H +#define __LIBC_SIGNAL_H + +static const char *__sys_signame[] = { + "SIGHUP", "SIGINT", "SIGQUIT", "SIGILL", "SIGTRAP", "SIGABRT", + "SIGBUS", "SIGFPE", "SIGKILL", "SIGUSR1", "SIGSEGV", "SIGUSR2", + "SIGPIPE", "SIGALRM", "SIGTERM", "SIGCHLD", "SIGCONT", "SIGSTOP", + "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGXCPU", "SIGXFSZ", + "SIGVTALRM", "SIGPROF", "SIGWINCH", "SIGIO", "SIGSYS" +}; + +#endif diff --git a/lib/libc/internal/include/__statvfs.h b/lib/libc/internal/include/__statvfs.h new file mode 100644 index 00000000..c8a746d4 --- /dev/null +++ b/lib/libc/internal/include/__statvfs.h @@ -0,0 +1,21 @@ +#ifndef __LIBC_STATVFS_H__ +#define __LIBC_STATVFS_H__ + +typedef __UINT64_TYPE__ fsblkcnt_t; +typedef __UINT64_TYPE__ fsfilcnt_t; + +struct __statvfs { + unsigned long f_bsize; + unsigned long f_frsize; + fsblkcnt_t f_blocks; + fsblkcnt_t f_bfree; + fsblkcnt_t f_bavail; + fsfilcnt_t f_files; + fsfilcnt_t f_ffree; + fsfilcnt_t f_favail; + unsigned long f_fsid; + unsigned long f_flag; + unsigned long f_namemax; +}; + +#endif diff --git a/lib/libc/internal/include/__stdio.h b/lib/libc/internal/include/__stdio.h new file mode 100644 index 00000000..487a0382 --- /dev/null +++ b/lib/libc/internal/include/__stdio.h @@ -0,0 +1,38 @@ +#ifndef __LIBC_STDIO_H__ +#define __LIBC_STDIO_H__ + +#include <stdatomic.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> + +typedef __SIZE_TYPE__ size_t; + +#define _IO_ERR 0x4 +#define _IO_EOF 0x8 +#define _IO_WIDE 0x10 + +struct __FILE { + int fd; + uint32_t flags; + int type; + pid_t pid; + atomic_flag lock; + char *buf; + int eof; + size_t buf_size; + size_t buf_pos; + size_t buf_len; + unsigned char unget_buf[16]; + size_t unget_cnt; + off_t offset; + struct __FILE *next; +}; + +#define __FILE(__stream) ((struct __FILE *)(__stream)) + +void __libc_fadd(struct __FILE *f); + +#endif diff --git a/lib/libc/internal/include/__thread.h b/lib/libc/internal/include/__thread.h new file mode 100644 index 00000000..7639707a --- /dev/null +++ b/lib/libc/internal/include/__thread.h @@ -0,0 +1,8 @@ +#ifndef __LIBC_THREAD_H +#define __LIBC_THREAD_H + +struct __thread_self { + int tid; +}; + +#endif diff --git a/lib/libc/internal/include/atomic.h b/lib/libc/internal/include/atomic.h new file mode 100644 index 00000000..e0593021 --- /dev/null +++ b/lib/libc/internal/include/atomic.h @@ -0,0 +1,23 @@ +#ifndef __LIBC_ATOMIC_H +#define __LIBC_ATOMIC_H + +#include <stdatomic.h> + +#define LIBC_LOCK(__lock) __libc_lock(&((__lock))) +#define LIBC_UNLOCK(__lock) atomic_flag_clear(&((__lock))) + +static __inline void __libc_lock(volatile atomic_flag *lock) +{ + while (atomic_flag_test_and_set_explicit(lock, memory_order_acquire)) { + unsigned int spins = 1; + do { + for (unsigned int i = 0; i < spins; i++) + __asm__ volatile("pause"); + if (spins < 64) + spins *= 2; + } while (atomic_flag_test_and_set_explicit( + lock, memory_order_relaxed)); + } +} + +#endif diff --git a/lib/libc/internal/include/byteswap.h b/lib/libc/internal/include/byteswap.h new file mode 100644 index 00000000..9dfe6520 --- /dev/null +++ b/lib/libc/internal/include/byteswap.h @@ -0,0 +1,8 @@ +#ifndef __LIBC_BYTESWAP_H +#define __LIBC_BYTESWAP_H + +#define bswap16(x) __builtin_bswap16(x) +#define bswap32(x) __builtin_bswap32(x) +#define bswap64(x) __builtin_bswap64(x) + +#endif diff --git a/lib/libc/internal/include/internal/io_uring.h b/lib/libc/internal/include/internal/io_uring.h new file mode 100644 index 00000000..e7369146 --- /dev/null +++ b/lib/libc/internal/include/internal/io_uring.h @@ -0,0 +1,6 @@ +#ifndef __INTERNAL_IO_URING_H +#define __INTERNAL_IO_URING_H + +void __io_uring_init(void); + +#endif diff --git a/lib/libc/internal/include/io_uring.h b/lib/libc/internal/include/io_uring.h new file mode 100644 index 00000000..4dac4583 --- /dev/null +++ b/lib/libc/internal/include/io_uring.h @@ -0,0 +1,53 @@ +#ifndef __LIBC_IO_URING_H +#define __LIBC_IO_URING_H + +#include <linux/io_uring.h> +#include <signal.h> + +#define IO_URING_ENTRIES 256 + +struct io_uring_sq { + void *ring; + size_t ring_size; + struct io_uring_sqe *sqes; + + unsigned *head; + unsigned *tail; + unsigned *ring_mask; + unsigned *ring_entries; + unsigned *flags; + unsigned *dropped; + unsigned *array; +}; + +struct io_uring_cq { + void *ring; + size_t ring_size; + + unsigned *head; + unsigned *tail; + unsigned *ring_mask; + unsigned *ring_entries; + unsigned *overflow; + struct io_uring_cqe *cqes; + unsigned *flags; +}; + +struct io_uring { + int fd; + int eventfd; + + struct io_uring_sq sq; + struct io_uring_cq cq; +}; + +extern struct io_uring __io_uring; + +int io_uring_setup(unsigned int, struct io_uring_params *); + +int io_uring_register(unsigned int, unsigned int, void *, unsigned int); + +int io_uring_enter(unsigned int, unsigned int, unsigned int, unsigned int, + sigset_t *, size_t); + +#endif diff --git a/lib/libc/internal/include/libc.h b/lib/libc/internal/include/libc.h new file mode 100644 index 00000000..f06fa6d8 --- /dev/null +++ b/lib/libc/internal/include/libc.h @@ -0,0 +1,48 @@ +#ifndef __LIBC_LIBC_H +#define __LIBC_LIBC_H + +#include <stdatomic.h> +#include <stddef.h> +#include <sys/cdefs.h> + +#define weak_reference(old, new) extern __typeof(old)((new)) __attribute__((__weak__, __alias__(#old))) + +struct tls { + struct tls *next; + void *data; + size_t size; + size_t align; + size_t length; + size_t offset; +}; + +struct libc { + size_t auxv[32]; + struct { + void *base; + size_t size; + } tls; + + enum { + LIBC_ENVP_TOUCHED = 1 << 0, + } flags; + + struct { + volatile atomic_flag abort; + volatile atomic_flag malloc; + volatile atomic_flag environ; + } lock; +}; + +extern struct libc __libc; + +#define panic(__errmsg) __libc_panic(__FILE__ ":" __STRING(__LINE__) ": ", __PRETTY_FUNCTION__, ": " __STRING(__errmsg)) + +#define panic_if(__cond, __errmsg) \ + do { \ + if (__predict_false(__cond)) \ + panic(__errmsg); \ + } while (0) + +__dead void __libc_panic(const char *, const char *, const char *); +#endif diff --git a/lib/libc/internal/include/libc/dirent.h b/lib/libc/internal/include/libc/dirent.h new file mode 100644 index 00000000..296cc91e --- /dev/null +++ b/lib/libc/internal/include/libc/dirent.h @@ -0,0 +1,6 @@ +#ifndef __LIBC_DIRENT_H +#define __LIBC_DIRENT_H + +struct __DIR {}; + +#endif diff --git a/lib/libc/internal/include/malloc.h b/lib/libc/internal/include/malloc.h new file mode 100644 index 00000000..b7cb0093 --- /dev/null +++ b/lib/libc/internal/include/malloc.h @@ -0,0 +1,98 @@ +#ifndef __LIBC_MALLOC_H +#define __LIBC_MALLOC_H + +#include <stdatomic.h> +#include <stdint.h> + +struct class { + uint32_t size; + uint32_t count; +}; + +struct page { + volatile atomic_flag lock; + + enum { + PAGE_SMALL = 0x0, + PAGE_MEDIUM = 0x1, + PAGE_LARGE = 0x2, + } flags; + + struct { + uint32_t size; + uint32_t used; + uint32_t count; + } block; + + uint8_t *bitmap; + uint8_t *heap; + + struct page *next; + struct page *prev; +}; + +extern struct page *__malloc_pvec; + +#define SMALL_PAGE_SIZE_SHIFT 16 +#define SMALL_PAGE_SIZE (1 << SMALL_PAGE_SIZE_SHIFT) +#define SMALL_PAGE_MASK (~((uintptr_t)SMALL_PAGE_SIZE - 1)) + +#define MEDIUM_PAGE_SIZE_SHIFT 22 +#define MEDIUM_PAGE_SIZE (1 << MEDIUM_PAGE_SIZE_SHIFT) +#define MEDIUM_PAGE_MASK (~((uintptr_t)MEDIUM_PAGE_SIZE - 1)) + +#define LARGE_PAGE_SIZE_SHIFT 26 +#define LARGE_PAGE_SIZE (1 << LARGE_PAGE_SIZE_SHIFT) +#define LARGE_PAGE_MASK (~((uintptr_t)LARGE_PAGE_SIZE - 1)) + +#define SMALL_CLASS(n) \ + { (((n)) * 16), (SMALL_PAGE_SIZE - sizeof(struct page)) / (((n)) * 16) } +#define MEDIUM_CLASS(n) \ + { (((n)) * 16), \ + (MEDIUM_PAGE_SIZE - sizeof(struct page)) / (((n)) * 16) } +#define LARGE_CLASS(n) \ + { (((n)) * 16), (LARGE_PAGE_SIZE - sizeof(struct page)) / (((n)) * 16) } + +static const struct class global_size_class[] = { + SMALL_CLASS(1), SMALL_CLASS(1), SMALL_CLASS(2), + SMALL_CLASS(3), SMALL_CLASS(4), SMALL_CLASS(5), + SMALL_CLASS(6), SMALL_CLASS(7), SMALL_CLASS(8), + SMALL_CLASS(9), SMALL_CLASS(10), SMALL_CLASS(11), + SMALL_CLASS(12), SMALL_CLASS(13), SMALL_CLASS(14), + SMALL_CLASS(15), SMALL_CLASS(16), SMALL_CLASS(17), + SMALL_CLASS(18), SMALL_CLASS(19), SMALL_CLASS(20), + SMALL_CLASS(21), SMALL_CLASS(22), SMALL_CLASS(23), + SMALL_CLASS(24), SMALL_CLASS(25), SMALL_CLASS(26), + SMALL_CLASS(27), SMALL_CLASS(28), SMALL_CLASS(29), + SMALL_CLASS(30), SMALL_CLASS(31), SMALL_CLASS(32), + SMALL_CLASS(33), SMALL_CLASS(34), SMALL_CLASS(35), + SMALL_CLASS(36), SMALL_CLASS(37), SMALL_CLASS(38), + SMALL_CLASS(39), SMALL_CLASS(40), SMALL_CLASS(41), + SMALL_CLASS(42), SMALL_CLASS(43), SMALL_CLASS(44), + SMALL_CLASS(45), SMALL_CLASS(46), SMALL_CLASS(47), + SMALL_CLASS(48), SMALL_CLASS(49), SMALL_CLASS(50), + SMALL_CLASS(51), SMALL_CLASS(52), SMALL_CLASS(53), + SMALL_CLASS(54), SMALL_CLASS(55), SMALL_CLASS(56), + SMALL_CLASS(57), SMALL_CLASS(58), SMALL_CLASS(59), + SMALL_CLASS(60), SMALL_CLASS(61), SMALL_CLASS(62), + SMALL_CLASS(63), SMALL_CLASS(64), SMALL_CLASS(80), + SMALL_CLASS(96), SMALL_CLASS(112), SMALL_CLASS(128), + SMALL_CLASS(160), SMALL_CLASS(192), SMALL_CLASS(224), + SMALL_CLASS(256), MEDIUM_CLASS(320), MEDIUM_CLASS(384), + MEDIUM_CLASS(448), MEDIUM_CLASS(512), MEDIUM_CLASS(640), + MEDIUM_CLASS(768), MEDIUM_CLASS(896), MEDIUM_CLASS(1024), + MEDIUM_CLASS(1280), MEDIUM_CLASS(1536), MEDIUM_CLASS(1792), + MEDIUM_CLASS(2048), MEDIUM_CLASS(2560), MEDIUM_CLASS(3072), + MEDIUM_CLASS(3584), MEDIUM_CLASS(4096), MEDIUM_CLASS(5120), + MEDIUM_CLASS(6144), MEDIUM_CLASS(7168), MEDIUM_CLASS(8192), + MEDIUM_CLASS(10240), MEDIUM_CLASS(12288), MEDIUM_CLASS(14336), + MEDIUM_CLASS(16384), LARGE_CLASS(20480), LARGE_CLASS(24576), + LARGE_CLASS(28672), LARGE_CLASS(32768), LARGE_CLASS(40960), + LARGE_CLASS(49152), LARGE_CLASS(57344), LARGE_CLASS(65536), + LARGE_CLASS(81920), LARGE_CLASS(98304), LARGE_CLASS(114688), + LARGE_CLASS(131072), LARGE_CLASS(163840), LARGE_CLASS(196608), + LARGE_CLASS(229376), LARGE_CLASS(262144), LARGE_CLASS(327680), + LARGE_CLASS(393216), LARGE_CLASS(458752), LARGE_CLASS(524288) +}; + +#endif diff --git a/lib/libc/internal/include/syscall.h b/lib/libc/internal/include/syscall.h new file mode 100644 index 00000000..ea72ece4 --- /dev/null +++ b/lib/libc/internal/include/syscall.h @@ -0,0 +1,44 @@ +#ifndef __LIBC_SYSCALL_H +#define __LIBC_SYSCALL_H + +#include <asm/syscall.h> +#include <asm/unistd_64.h> + +typedef long syscall_arg_t; + +#define __SYSCALL_NARGS_(a, b, c, d, e, f, g, h, n, ...) n +#define __SYSCALL_NARGS(...) __SYSCALL_NARGS_(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, ) + +#define __SYSCALL_CONCAT_(a, b) a##b +#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_(a, b) + +#define __SYSCALL_(b, ...) __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) + +#define __syscall(...) __SYSCALL_(__syscall_, __VA_ARGS__) +#define syscall(...) __syscall_ret(__syscall(__VA_ARGS__)) + +#define __sa(x) ((syscall_arg_t)(x)) + +#define __syscall_0(n) __syscall0(__NR_##n) +#define __syscall_1(n, a) __syscall1(__NR_##n, __sa(a)) +#define __syscall_2(n, a, b) __syscall2(__NR_##n, __sa(a), __sa(b)) +#define __syscall_3(n, a, b, c) __syscall3(__NR_##n, __sa(a), __sa(b), __sa(c)) +#define __syscall_4(n, a, b, c, d) __syscall4(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d)) +#define __syscall_5(n, a, b, c, d, e) __syscall5(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d), __sa(e)) +#define __syscall_6(n, a, b, c, d, e, f) __syscall6(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d), __sa(e), __sa(f)) +#define __syscall_7(n, a, b, c, d, e, f, g) \ + __syscall7(__NR_##n, __sa(a), __sa(b), __sa(c), __sa(d), __sa(e), __sa(f), __sa(g)) + +#ifdef __NR_pread64 +#undef __NR_pread +#define __NR_pread __NR_pread64 +#endif + +#ifdef __NR_pwrite64 +#undef __NR_pwrite +#define __NR_pwrite __NR_pwrite64 +#endif + +extern long __syscall_ret(long ret); + +#endif diff --git a/lib/libc/internal/init/Kbuild b/lib/libc/internal/init/Kbuild index 7941a53c..361f220a 100644 --- a/lib/libc/internal/init/Kbuild +++ b/lib/libc/internal/init/Kbuild @@ -1,2 +1,3 @@ obj-y += init.o +obj-y += tls.o obj-y += vdso.o diff --git a/lib/libc/internal/init/init.c b/lib/libc/internal/init/init.c index 36e66ee4..b2dd8487 100644 --- a/lib/libc/internal/init/init.c +++ b/lib/libc/internal/init/init.c @@ -5,13 +5,14 @@ #include <sys/cdefs.h> extern void __init_vdso(void); +extern void __init_tls(void); + +extern int main(int, char **, char **); struct libc __libc = { 0 }; char **environ; char *__progname; -extern int main(int, char **, char **); - __used void __init(uintptr_t *rsp) { char **argv; @@ -34,6 +35,7 @@ __used void __init(uintptr_t *rsp) for (size_t i = 0; auxv[i]; i += 2) __libc.auxv[auxv[i]] = auxv[i + 1]; + __init_tls(); __init_vdso(); exit(main(argc, argv, environ)); diff --git a/lib/libc/internal/init/tls.c b/lib/libc/internal/init/tls.c new file mode 100644 index 00000000..f6ccde35 --- /dev/null +++ b/lib/libc/internal/init/tls.c @@ -0,0 +1,76 @@ +#include <__thread.h> +#include <asm/prctl.h> +#include <elf.h> +#include <libc.h> +#include <string.h> +#include <sys/mman.h> +#include <syscall.h> + +#define ALIGN_DOWN(x, a) ((x) & ~((a) - 1)) +#define ALIGN_UP(x, a) (((x) + (a) - 1) & ~((a) - 1)) + +static struct tls __libc_tls = { 0 }; +static struct __thread_self __libc_thread; +volatile int __libc_tid; + +void __init_tls(void) +{ + int r; + void *mem; + size_t base; + unsigned char *p; + Elf64_Phdr *tls_phdr; + struct __thread_self *thrd; + + base = 0; + tls_phdr = 0; + p = (void *)__libc.auxv[AT_PHDR]; + for (size_t n = __libc.auxv[AT_PHNUM]; n > 0; n--, p += __libc.auxv[AT_PHENT]) { + Elf64_Phdr *phdr = (Elf64_Phdr *)p; + + if (phdr->p_type == PT_PHDR) { + base = __libc.auxv[AT_PHDR] - phdr->p_vaddr; + } else if (phdr->p_type == PT_TLS) { + tls_phdr = phdr; + } + } + + if (tls_phdr) { + __libc_tls.data = (void *)(base + tls_phdr->p_vaddr); + __libc_tls.length = tls_phdr->p_filesz; + __libc_tls.size = tls_phdr->p_memsz; + __libc_tls.align = tls_phdr->p_align; + } + + if (__libc_tls.size != 0) { + void *tls_mem; + size_t tls_size = __libc_tls.size + sizeof(struct __thread_self); + tls_size = (tls_size + __libc_tls.align - 1) & ~(__libc_tls.align - 1); + + mem = mmap(0, tls_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + panic_if(mem == MAP_FAILED, "mmap failed"); + + __libc.tls.base = mem; + __libc.tls.size = tls_size; + + thrd = (struct __thread_self *)mem; + tls_mem = (unsigned char *)mem + sizeof(struct __thread_self); + + if (tls_phdr->p_filesz > 0) { + memcpy(tls_mem, (void *)tls_phdr->p_vaddr, tls_phdr->p_filesz); + memset(tls_mem + tls_phdr->p_filesz, 0, tls_phdr->p_memsz - tls_phdr->p_filesz); + } else { + memset(tls_mem, 0, tls_phdr->p_memsz); + } + + r = syscall(arch_prctl, ARCH_SET_FS, mem + sizeof(struct __thread_self)); + panic_if(r < 0, "arch_prctl(SET_FS) failed"); + } else { + __libc.tls.base = NULL; + thrd = &__libc_thread; + } + + r = syscall(set_tid_address, &__libc_tid); + panic_if(r < 0, "set_tid_address failed"); + thrd->tid = r; +} diff --git a/lib/libc/internal/panic.c b/lib/libc/internal/panic.c new file mode 100644 index 00000000..1af6d6cf --- /dev/null +++ b/lib/libc/internal/panic.c @@ -0,0 +1,29 @@ +#include <string.h> +#include <sys/cdefs.h> +#include <sys/uio.h> +#include <unistd.h> + +__dead void __libc_panic(const char *prefix, const char *f, const char *errmsg) +{ + struct iovec iovec[5]; + const char *e = "libc panic: "; + + iovec[0].iov_base = (char *)e; + iovec[0].iov_len = sizeof("libc panic: ") - 1; + + iovec[1].iov_base = (char *)prefix; + iovec[1].iov_len = strlen(prefix); + + iovec[2].iov_base = (char *)f; + iovec[2].iov_len = strlen(f); + + iovec[3].iov_base = (char *)errmsg; + iovec[3].iov_len = strlen(errmsg); + + iovec[4].iov_base = "\n"; + iovec[4].iov_len = 1; + + writev(STDERR_FILENO, iovec, 5); + + __builtin_trap(); +} diff --git a/lib/libc/internal/syscall.c b/lib/libc/internal/syscall.c new file mode 100644 index 00000000..f7fd3c4b --- /dev/null +++ b/lib/libc/internal/syscall.c @@ -0,0 +1,12 @@ +#include <errno.h> +#include <sys/cdefs.h> + +long __syscall_ret(long ret) +{ + if (__predict_false(ret > -4096 && ret < 0)) { + errno = (int)-ret; + ret = -1; + } + + return ret; +} |