JIT: recognize initial-exec TLS relaxation

Also resolve the Windows TLS slot by address instead of scanning for a matching value. See #23227

Author: henderkes

ext/opcache/jit/tls/testing/test.sh

@@ -136,8 +136,8 @@ dl_def_static_user_no_surplus() {
 
   rm -f main
 
-  $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC
-  $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c
+  $CC $CFLAGS -DTSRM_TLS_MODEL_USE_DEFAULT -ggdb3 -fPIC -o tls.o -c $TLSC
+  $CC $CFLAGS -DTSRM_TLS_MODEL_USE_DEFAULT -ggdb3 -fPIC -o user.o -c user.c
   $CC $CFLAGS -DNO_SURPLUS -ggdb3 -fPIC -o def.o -c def.c
   $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o user.o tls.o
 
@@ -151,8 +151,8 @@ dl_def_shared_user_no_surplus() {
 
   rm -f main
 
-  $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC
-  $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c
+  $CC $CFLAGS -DTSRM_TLS_MODEL_USE_DEFAULT -ggdb3 -fPIC -o tls.o -c $TLSC
+  $CC $CFLAGS -DTSRM_TLS_MODEL_USE_DEFAULT -ggdb3 -fPIC -o user.o -c user.c
   $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o
 
   $CC $CFLAGS -DNO_SURPLUS -ggdb3 -fPIC -o def.o -c def.c

ext/opcache/jit/tls/testing/user.c

@@ -3,7 +3,7 @@
 
 #include "../zend_jit_tls.h"
 
-extern __thread void* _tsrm_ls_cache;
+TSRMLS_MAIN_CACHE_EXTERN()
 
 int test(void)
 {
@@ -12,7 +12,7 @@ int test(void)
 	size_t module_offset = -1;
 
 	/* Ensure the slot is allocated */
-	_tsrm_ls_cache = NULL;
+	void *cache = &_tsrm_ls_cache;
 
 	zend_result result = zend_jit_resolve_tsrm_ls_cache_offsets(
 			&tcb_offset, &module_index, &module_offset);
@@ -24,5 +24,5 @@ int test(void)
 		return 0;
 	}
 
-	return zend_jit_tsrm_ls_cache_address(tcb_offset, module_index, module_offset) == &_tsrm_ls_cache;
+	return zend_jit_tsrm_ls_cache_address(tcb_offset, module_index, module_offset) == cache;
 }

ext/opcache/jit/tls/zend_jit_tls_aarch64.c

@@ -192,18 +192,6 @@ zend_result zend_jit_resolve_tsrm_ls_cache_offsets(
 
 	/* Code was changed by the linker. Check if we recognize the updated code */
 
-	// movz	x0, #0, lsl #16
-	if ((insn[0] & ~AARCH64_MOVZ_IMM_MASK) != 0xd2a00000) {
-		zend_accel_error(ACCEL_LOG_DEBUG, "movz insn does not match: 0x%08" PRIx32 "\n", insn[0]);
-		return FAILURE;
-	}
-
-	// movk	x0, #0x10
-	if ((insn[1] & ~AARCH64_MOVK_IMM_MASK) != 0xf2800000) {
-		zend_accel_error(ACCEL_LOG_DEBUG, "movk insn does not match: 0x%08" PRIx32 "\n", insn[1]);
-		return FAILURE;
-	}
-
 	// nop
 	for (int i = 0; i < 2; i++) {
 		if (insn[2+i] != AARCH64_NOP) {
@@ -212,12 +200,29 @@ zend_result zend_jit_resolve_tsrm_ls_cache_offsets(
 		}
 	}
 
-	/* Extract immediate values */
+	if ((insn[0] & ~AARCH64_ADRP_IMM_MASK) == 0x90000000) {
+		/* initial-exec */
+		// ldr x0, [x0, #any]
+		if ((insn[1] & ~AARCH64_LDR_UNSIGNED_IMM_MASK) != 0xf9400000) {
+			zend_accel_error(ACCEL_LOG_DEBUG, "ldr insn does not match: 0x%08" PRIx32 "\n", insn[1]);
+			return FAILURE;
+		}
+	} else {
+		/* local-exec */
+		// movz x0, #0, lsl #16
+		if ((insn[0] & ~AARCH64_MOVZ_IMM_MASK) != 0xd2a00000) {
+			zend_accel_error(ACCEL_LOG_DEBUG, "movz insn does not match: 0x%08" PRIx32 "\n", insn[0]);
+			return FAILURE;
+		}
+
+		// movk x0, #0x10
+		if ((insn[1] & ~AARCH64_MOVK_IMM_MASK) != 0xf2800000) {
+			zend_accel_error(ACCEL_LOG_DEBUG, "movk insn does not match: 0x%08" PRIx32 "\n", insn[1]);
+			return FAILURE;
+		}
+	}
 
-	uint64_t movz_imm = (insn[0] & AARCH64_MOVZ_IMM_MASK) >> 5;
-	uint64_t movz_shift = (((insn[0] & AARCH64_MOVZ_HW_MASK) >> 21) << 4);
-	uint64_t movk_imm = (insn[1] & AARCH64_MOVK_IMM_MASK) >> 5;
-	uint64_t offset = (movz_imm << movz_shift) | movk_imm;
+	uintptr_t offset = (uintptr_t)addr - (uintptr_t)thread_pointer;
 
 	if ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer == offset) {
 		zend_accel_error(ACCEL_LOG_DEBUG, "static tls at offset %" PRIxPTR " from thread pointer (inferred from code)\n", offset);

ext/opcache/jit/tls/zend_jit_tls_win.c

@@ -33,30 +33,20 @@ zend_result zend_jit_resolve_tsrm_ls_cache_offsets(
 	size_t *module_index,
 	size_t *module_offset
 ) {
-	/* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
-	/* Probably, it might be better solution */
+	/* Offset of _tsrm_ls_cache within this module's TLS block. */
 #ifdef _WIN64
 	void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index];
 #else
 	void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index];
 #endif
-	void *val = _tsrm_ls_cache;
-	size_t offset = 0;
 	size_t size = (char*)&_tls_end - (char*)&_tls_start;
-
-	while (offset < size) {
-		if (*tls_mem == val) {
-			*module_index  = _tls_index * sizeof(void*);
-			*module_offset = offset;
-			return SUCCESS;
-		}
-		tls_mem++;
-		offset += sizeof(void*);
-	}
+	size_t offset = (size_t)((char*)&_tsrm_ls_cache - (char*)tls_mem);
 
 	if (offset >= size) {
 		zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
 	}
 
-	return FAILURE;
+	*module_index  = _tls_index * sizeof(void*);
+	*module_offset = offset;
+	return SUCCESS;
 }

ext/opcache/jit/tls/zend_jit_tls_x86_64.c

@@ -166,7 +166,9 @@ zend_result zend_jit_resolve_tsrm_ls_cache_offsets(
 
 	/*
 	 * 64 48 8b 04 25 00 00 00 00	movq    %fs:0x0,%rax
-	 * 48 8d 80 f8 ff ff ff	        leaq    -0x8(%rax),%rax
+	 * 48 8d 80 f8 ff ff ff	        leaq    -0x8(%rax),%rax (Local Exec)
+	 * or
+	 * 48 03 05 xx xx xx xx	        addq    any(%rip),%rax (Initial Exec)
 	 */
 
 	// movq %fs:0x0,%rax
@@ -177,20 +179,16 @@ zend_result zend_jit_resolve_tsrm_ls_cache_offsets(
 		return FAILURE;
 	}
 
-	// leaq any(%rax),$rax
-	if (memcmp(&code[9], "\x48\x8d\x80", 3) != 0) {
+	// leaq any(%rax),%rax or addq any(%rip),%rax
+	if (memcmp(&code[9], "\x48\x8d\x80", 3) != 0
+			&& memcmp(&code[9], "\x48\x03\x05", 3) != 0) {
 		uint64_t bytes;
-		memcpy(&bytes, &code[10], 8);
-		zend_accel_error(ACCEL_LOG_DEBUG, "leaq insn does not match: 0x%016" PRIx64 "\n", bytes);
+		memcpy(&bytes, &code[9], 8);
+		zend_accel_error(ACCEL_LOG_DEBUG, "leaq/addq insn does not match: 0x%016" PRIx64 "\n", bytes);
 		return FAILURE;
 	}
 
-	/* Extract immediate values */
-
-	uintptr_t offset = (uintptr_t)(int32_t)(((uint32_t)code[15] << 24)
-		| ((uint32_t)code[14] << 16)
-		| ((uint32_t)code[13] << 8)
-		| ((uint32_t)code[12]));
+	uintptr_t offset = (uintptr_t)addr - (uintptr_t)thread_pointer;
 
 	if ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer == offset) {
 		*tcb_offset = offset;