Add basic TLS Support (#611)

* Fix threading API headers

cthreads.h is renamed to threads.h and not included by default. This also makes time.h not included by default which allows the symbol time to be used again.

* Add basic TLS Support

* Properly test thread local variables

* Improve TLS DSO error message

* Specifiy specific context

* Invalidate TP value in interrupt callbacks

* Add error reporting for Invalid TLS Accesses

* Update exception.c

* Restore interrupt control to manual TLS

* Disallow TLS access without kernel

Error message formatting needs to be improved

* Shorten error message

* Only copy tdata once

* Fix tabs

Also fix interrupts without kernel

* Fix kernel_internal tabs

* Fix linker script tabs

* Call __kernel_tls_init as a constructor

Also readds the tls structure

* Add documentation

* Verify maximum TLS alignment of 8

* Improve TLS alignment error

* Remove test_kernel test

Makefile

@@ -125,6 +125,7 @@ install: install-mk libdragon
 	install -Cv -m 0644 include/ksemaphore.h $(INSTALLDIR)/mips64-elf/include/ksemaphore.h
 	install -Cv -m 0644 include/kqueue.h $(INSTALLDIR)/mips64-elf/include/kqueue.h
 	install -Cv -m 0644 include/kirq.h $(INSTALLDIR)/mips64-elf/include/kirq.h
+	install -Cv -m 0644 include/ktls.h $(INSTALLDIR)/mips64-elf/include/ktls.h
 	install -Cv -m 0644 include/dma.h $(INSTALLDIR)/mips64-elf/include/dma.h
 	install -Cv -m 0644 include/dragonfs.h $(INSTALLDIR)/mips64-elf/include/dragonfs.h
 	install -Cv -m 0644 include/asset.h $(INSTALLDIR)/mips64-elf/include/asset.h

include/ktls.h

@@ -0,0 +1,22 @@
+#ifndef LIBDRAGON_KERNEL_TLS_H
+#define LIBDRAGON_KERNEL_TLS_H
+
+#define KERNEL_TP_INVALID ((void *)0x5FFF8001)
+
+#ifdef N64_DSO
+__asm__ (
+    ".macro rdhwr rt, rd" "\n"
+    "   .error \" Usage of thread-local variables is not supported in DSOs. \"" "\n"
+    ".endm"               "\n"
+);
+#else
+__asm__ (
+    ".macro rdhwr rt, rd" "\n"
+    "    lw \\rt, %gprel(th_cur_tp)($gp)" "\n"
+    ".endm"               "\n"
+);
+#endif
+
+extern void *th_cur_tp;
+
+#endif

n64.ld

@@ -44,9 +44,9 @@ SECTIONS {
 
    .eh_frame_hdr : { *(.eh_frame_hdr) }
    .eh_frame : { 
-		__EH_FRAME_BEGIN__ = .; /* Define symbol for accessing eh_frame section */
-		KEEP (*(.eh_frame)) 
-	}
+        __EH_FRAME_BEGIN__ = .; /* Define symbol for accessing eh_frame section */
+        KEEP (*(.eh_frame)) 
+    }
    .gcc_except_table : { *(.gcc_except_table*) } 
    .jcr : { KEEP (*(.jcr)) } 
 
@@ -55,6 +55,9 @@ SECTIONS {
         *(.rodata)
         *(.rodata.*)
         *(.gnu.linkonce.r.*)
+        . = ALIGN(4);
+        __tdata_align = .;
+        LONG (ALIGNOF(.tdata));
         . = ALIGN(8);
     }
 
@@ -103,7 +106,29 @@ SECTIONS {
         *(.gnu.linkonce.d.*)
         . = ALIGN(8);
     }
-
+
+    .tdata : {
+        __tls_base = .;
+        __tdata_start = .;
+        *(.tdata)
+        *(.tdata.*)
+        *(.gnu.linkonce.td.*)
+        __tdata_end = .;
+    }
+
+    .tbss : {
+        __tbss_start = .;
+        *(.tbss)
+        *(.tbss.*)
+        *(.gnu.linkonce.tb.*)
+        . = ALIGN(8);
+        __tbss_end = .;
+        __tls_end = .;
+    }
+
+    /* Fix dot for TBSS Sections */
+    . = .+(__tbss_end-__tbss_start);
+
     /* Small data START */
 
     .sdata : {
@@ -147,6 +172,9 @@ SECTIONS {
         *(.bss*)
         *(.gnu.linkonce.b.*)
         *(COMMON)
+        . = ALIGN(8);
+        __th_tdata_copy = .;
+        . = .+(__tdata_end-__tdata_start);
         . = ALIGN(8);
          __bss_end = .;
     }

n64.mk

@@ -47,7 +47,7 @@ N64_DSO = $(N64_BINDIR)/n64dso
 N64_DSOEXTERN = $(N64_BINDIR)/n64dso-extern
 N64_DSOMSYM = $(N64_BINDIR)/n64dso-msym
 
-N64_C_AND_CXX_FLAGS =  -march=vr4300 -mtune=vr4300 -I$(N64_INCLUDEDIR)
+N64_C_AND_CXX_FLAGS =  -march=vr4300 -mtune=vr4300 -I$(N64_INCLUDEDIR) -include ktls.h
 N64_C_AND_CXX_FLAGS += -falign-functions=32   # NOTE: if you change this, also change backtrace() in backtrace.c
 N64_C_AND_CXX_FLAGS += -ffunction-sections -fdata-sections -g -ffile-prefix-map="$(CURDIR)"=
 N64_C_AND_CXX_FLAGS += -ffast-math -ftrapping-math -fno-associative-math
@@ -194,8 +194,8 @@ $(BUILD_DIR)/%.o: $(SOURCE_DIR)/%.cpp
 %.dso: CXX=$(N64_CXX)
 %.dso: AS=$(N64_AS)
 %.dso: LD=$(N64_LD)
-%.dso: CFLAGS+=$(N64_CFLAGS) -mno-gpopt $(DSO_CFLAGS)
-%.dso: CXXFLAGS+=$(N64_CXXFLAGS) -mno-gpopt $(DSO_CXXFLAGS)
+%.dso: CFLAGS+=$(N64_CFLAGS) -mno-gpopt -DN64_DSO $(DSO_CFLAGS)
+%.dso: CXXFLAGS+=$(N64_CXXFLAGS) -mno-gpopt -DN64_DSO $(DSO_CXXFLAGS)
 %.dso: ASFLAGS+=$(N64_ASFLAGS)
 %.dso: RSPASFLAGS+=$(N64_RSPASFLAGS)
 

src/exception.c

@@ -16,6 +16,8 @@
 #include <stdbool.h>
 #include <math.h>
 
+/** Invalid TLS Minimum Address */
+#define TLS_INVALID_MIN (uint32_t)(KERNEL_TP_INVALID-TP_OFFSET)
 /**
  * @brief Syscall exception handler entry
  */
@@ -338,13 +340,30 @@ static const char* __get_exception_name(exception_t *ex)
 			// so leave some margin to the actual faulting address.
 			return "NULL pointer dereference (read)";
 		} else {
-			return "Read from invalid memory address";
+			if(badvaddr >= TLS_INVALID_MIN && badvaddr < (TLS_INVALID_MIN+TLS_SIZE)) {
+				if(__kernel) {
+					return "Read from TLS in interrupt handler";
+				} else {
+					return "Cannot access TLS without kernel_init()";
+				}
+
+			} else {
+				return "Read from invalid memory address";
+			}
 		}
 	case EXCEPTION_CODE_TLB_STORE_MISS:
 		if (badvaddr < 128) {
 			return "NULL pointer dereference (write)";
 		} else {
-			return "Write to invalid memory address";
+			if(badvaddr >= TLS_INVALID_MIN && badvaddr < (TLS_INVALID_MIN+TLS_SIZE)) {
+				if(__kernel) {
+					return "Write to TLS in interrupt handler";
+				} else {
+					return "Cannot access TLS without kernel_init()";
+				}
+			} else {
+				return "Write to invalid memory address";
+			}
 		}
 	case EXCEPTION_CODE_TLB_MODIFICATION:
 		return "Write to read-only memory";
@@ -357,11 +376,28 @@ static const char* __get_exception_name(exception_t *ex)
 		} else {
 			if (is_unmapped_kx64(badvaddr))
 				return "Read from invalid 64-bit address";
-			else
-				return "Misaligned read from memory";
+			else {
+				if(badvaddr >= TLS_INVALID_MIN && badvaddr < (TLS_INVALID_MIN+TLS_SIZE)) {
+					if(__kernel) {
+						return "Read from TLS in interrupt handler";
+					} else {
+						return "Cannot access TLS without kernel_init()";
+					}
+				} else {
+					return "Misaligned read from memory";
+				}
+			}
 		}
 	case EXCEPTION_CODE_STORE_ADDRESS_ERROR:
-		return "Misaligned write to memory";
+		if(badvaddr >= TLS_INVALID_MIN && badvaddr < (TLS_INVALID_MIN+TLS_SIZE)) {
+			if(__kernel) {
+				return "Write to TLS in interrupt handler";
+			} else {
+				return "Cannot access TLS without kernel_init()";
+			}
+		} else {
+			return "Misaligned write to memory";
+		}
 	case EXCEPTION_CODE_SYS_CALL:
 		return "Unhandled syscall";
 	case EXCEPTION_CODE_TRAP: {

src/interrupt.c

@@ -109,6 +109,9 @@ static uint32_t __prenmi_tick;
  */
 static void __call_callback( struct callback_link * head )
 {
+    /* Invalidate TP Value */
+    void *tp = th_cur_tp;
+    th_cur_tp = KERNEL_TP_INVALID;
     /* Call each registered callback */
     while( head )
     {
@@ -120,6 +123,8 @@ static void __call_callback( struct callback_link * head )
         /* Go to next */
 	    head=head->next;
     }
+    /* Restore TP Value */
+    th_cur_tp = tp;
 }
 
 /**

src/kernel/kernel.c

@@ -15,6 +15,7 @@
 #define STACK_COOKIE   0xDEADBEEFBAADC0DE
 #define STACK_GUARD    64
 
+
 /** @brief Read the current value of the gp register */
 #define REG_GP()     ({ uint64_t gp; __asm("move %0, $28": "=r" (gp)); gp; })
 
@@ -40,6 +41,8 @@
  */
 #define KTHREAD_SWITCH_ISR()  ({ __isr_force_schedule = true; })
 
+/** @brief Current thread TLS */
+void *th_cur_tp = KERNEL_TP_INVALID;
 /** @brief Main thread */
 kthread_t th_main;
 /** @brief Pointer to the current thread */
@@ -58,11 +61,25 @@ bool __isr_force_schedule = false;
 /* Global current interrupt depth (defined in interrupt.c) */ 
 extern int __interrupt_depth;
 extern int __interrupt_sr;
+/* TLS Linker symbols */
+extern char __tls_base[];
+extern char __tdata_start[];
+extern char __tdata_end[];
+extern char __tbss_start[];
+extern char __tbss_end[];
+extern char __tls_end[];
+extern char __th_tdata_copy[];
+extern __attribute__((section(".data"))) size_t __tdata_align;
 
 #ifndef NDEBUG
 kthread_t *__kernel_all_threads;
 #endif
 
+__attribute__((constructor)) void __kernel_tls_init(void)
+{
+	memcpy(__th_tdata_copy, __tls_base, TDATA_SIZE);
+}
+
 /** 
  * @brief Boot function for a kthread.
  *
@@ -281,7 +298,6 @@ reg_block_t* __kthread_syscall_schedule(reg_block_t *stack_state)
 				assertf(!(th_cur->flags & TH_FLAG_INLIST), "thread %s[%p] in list? flags=%x", th_cur->name, th_cur, th_cur->flags);
 				__thlist_add_pri(&th_ready, th_cur);
 			}
-
 			// Save the current interrupt depth. Interrupt depth is actually
 			// per-thread, so we just save/restore it every time a thread is
 			// scheduled.
@@ -301,10 +317,13 @@ reg_block_t* __kthread_syscall_schedule(reg_block_t *stack_state)
 	} while (th_cur->flags & (TH_FLAG_WAITFORJOIN | TH_FLAG_SUSPENDED));
 	if (DEBUG_KERNEL) debugf("[kernel] switching to %s(%p) PC=%lx SR=%lx\n", th_cur->name, th_cur, th_cur->stack_state->epc, th_cur->stack_state->sr);
 	assert(!(th_cur->flags & TH_FLAG_INLIST));
-
+    
 	// Set the current interrupt depth to that of the current thread.
 	__interrupt_depth = th_cur->tls.interrupt_depth;
 	__interrupt_sr = th_cur->tls.interrupt_sr;
+
+	th_cur_tp = th_cur->tp_value;
+
 	#ifdef __NEWLIB__
 	_REENT = th_cur->tls.reent_ptr;
 	#endif
@@ -333,6 +352,7 @@ static int __kthread_idle(void *arg)
 
 kthread_t* kernel_init(void)
 {
+	assertf(__tdata_align <= 8, "Unsupported TLS alignment of %d (Maximum 8)", __tdata_align);
 	assert(!__kernel);
 	#ifdef __NEWLIB__
 	// Check if __malloc_lock is a nop. This happens with old toolchains where
@@ -359,6 +379,8 @@ kthread_t* kernel_init(void)
 	th_main.name = "main";
 	th_main.stack_size = 0x10000; // see STACK_SIZE in system.c
 	th_main.flags = TH_FLAG_DETACHED; // main thread cannot be joined
+	th_main.tp_value = __tls_base+TP_OFFSET;
+
 	#ifdef __NEWLIB__
 	th_main.tls.reent_ptr = _REENT;
 	#endif
@@ -387,6 +409,7 @@ kthread_t* kernel_init(void)
 	__kirq_init();
 
 	__kernel = true;
+	th_cur_tp = __tls_base+TP_OFFSET;
 	return th_cur;
 }
 
@@ -404,6 +427,7 @@ void kernel_close(void)
 	th_cur = NULL;
 	__kernel = false;
 	__isr_force_schedule = false;
+	th_cur_tp = KERNEL_TP_INVALID;
 }
 
 kthread_t* __kthread_new_internal(const char *name, int stack_size, int8_t pri, uint8_t flag, int (*user_entry)(void*), void *user_data)
@@ -423,7 +447,7 @@ kthread_t* __kthread_new_internal(const char *name, int stack_size, int8_t pri,
 	// overflowing the stack doesn't corrupt the thread structure, which might
 	// make debugging more difficult and might even cause the kernel to crash.
 
-	int extra_size = 0;
+	int extra_size = TLS_SIZE;
 	#ifdef __NEWLIB__
 	extra_size += sizeof(struct _reent);
 	#endif
@@ -483,7 +507,12 @@ kthread_t* __kthread_new_internal(const char *name, int stack_size, int8_t pri,
 	__kernel_all_threads = th;
 	#endif
 	enable_interrupts();
-
+
+	//Initialize TLS Data
+	memcpy(extra, __th_tdata_copy, TDATA_SIZE);
+	memset(extra+TDATA_SIZE, 0, TLS_SIZE-TDATA_SIZE);
+	th->tp_value = extra+TP_OFFSET;
+	extra += TLS_SIZE;
 	// TLS initial configuration
 	#ifdef __NEWLIB__
 	th->tls.reent_ptr = extra;

src/kernel/kernel_internal.h

@@ -23,6 +23,11 @@
 #define TH_FLAG_SUSPENDED   (1<<4)      ///< The thread is suspended (will not be scheduled)
 #define TH_FLAG_INSPECTOR1  (1<<7)      ///< Flag reserved for usage in the inspector
 
+
+#define TDATA_SIZE ((uint32_t)(__tdata_end)-(uint32_t)(__tdata_start)) ///< Size of .tdata section
+#define TLS_SIZE ((uint32_t)(__tls_end)-(uint32_t)(__tls_base)) ///< Size of .tdata and .tbss sections combined
+#define TP_OFFSET 0x7000 ///< Offset of Pointer for TLS accesses
+
 /**
  * @brief a kernel thread for parallel execution
  *
@@ -41,20 +46,22 @@
  * to kthread_t) that can be used to externally manage the
  * thread like changing its priority or killing it.
  */
-typedef struct kthread_s
+typedef struct __attribute__((aligned (8))) kthread_s
 {
 	/** Pointer to the top of the stack, which contains the thread register state */
 	reg_block_t* stack_state;
-    struct {
-        /** Mirror of __interrupt_depth */
-        int interrupt_depth;
-        /** Mirror of __interrupt_sr */
-        int interrupt_sr;
+	/** TLS Pointer + TP_OFFSET */
+	void *tp_value;
+	struct {
+		/** Mirror of __interrupt_depth */
+		int interrupt_depth;
+		/** Mirror of __interrupt_sr  */
+		int interrupt_sr;
 		#ifdef __NEWLIB__
 		/** Newlib reentrancy */
 		struct _reent *reent_ptr;
 		#endif
-    } tls;  ///< Thread-local storage
+	} tls; ///< Thread-local storage
 	/** Size of the stack in bytes */
 	int stack_size;
 	/** Name of thread (for debugging purposes) */
@@ -84,6 +91,12 @@ typedef struct kthread_s
 /** Kernel initialization flag. */
 extern bool __kernel;
 
+/** TLS Base Linker Symbol */
+extern char __tls_base[];
+/** TLS End Linker Symbol */
+extern char __tls_end[];
+
+
 extern kcond_t __kirq_cond_sp;      ///< Condition variable for SP interrupt
 extern kcond_t __kirq_cond_dp;      ///< Condition variable for DP interrupt
 extern kcond_t __kirq_cond_si;      ///< Condition variable for SI interrupt