mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git
synced 2026-10-31 14:23:49 +01:00
K Prateek Nayak
cf0de88094
Futex hash computation requires a mask operation with read-only after init data that will be converted to a runtime constant in the subsequent commit. Introduce runtime_const_mask_32 to further optimize the mask operation in the futex hash computation hot path. GCC generates a: nilf %r1,<imm32> to tackle arbitrary 32-bit masks and the same is implemented here. Immediate patching pattern for __runtime_fixup_mask() has been adopted from __runtime_fixup_ptr(). Signed-off-by: K Prateek Nayak <[email protected]> Signed-off-by: Peter Zijlstra (Intel) <[email protected]> Acked-by: Heiko Carstens <[email protected]> Link: https://patch.msgid.link/[email protected]
98 lines
2.4 KiB
C
98 lines
2.4 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_S390_RUNTIME_CONST_H
|
|
#define _ASM_S390_RUNTIME_CONST_H
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
#define runtime_const_ptr(sym) \
|
|
({ \
|
|
typeof(sym) __ret; \
|
|
\
|
|
asm_inline( \
|
|
"0: iihf %[__ret],%[c1]\n" \
|
|
" iilf %[__ret],%[c2]\n" \
|
|
".pushsection runtime_ptr_" #sym ",\"a\"\n" \
|
|
".long 0b - .\n" \
|
|
".popsection" \
|
|
: [__ret] "=d" (__ret) \
|
|
: [c1] "i" (0x01234567UL), \
|
|
[c2] "i" (0x89abcdefUL)); \
|
|
__ret; \
|
|
})
|
|
|
|
#define runtime_const_shift_right_32(val, sym) \
|
|
({ \
|
|
unsigned int __ret = (val); \
|
|
\
|
|
asm_inline( \
|
|
"0: srl %[__ret],12\n" \
|
|
".pushsection runtime_shift_" #sym ",\"a\"\n" \
|
|
".long 0b - .\n" \
|
|
".popsection" \
|
|
: [__ret] "+d" (__ret)); \
|
|
__ret; \
|
|
})
|
|
|
|
#define runtime_const_mask_32(val, sym) \
|
|
({ \
|
|
unsigned int __ret = (val); \
|
|
\
|
|
asm_inline( \
|
|
"0: nilf %[__ret],12\n" \
|
|
".pushsection runtime_mask_" #sym ",\"a\"\n" \
|
|
".long 0b - .\n" \
|
|
".popsection" \
|
|
: [__ret] "+d" (__ret) \
|
|
: : "cc"); \
|
|
__ret; \
|
|
})
|
|
|
|
#define runtime_const_init(type, sym) do { \
|
|
extern s32 __start_runtime_##type##_##sym[]; \
|
|
extern s32 __stop_runtime_##type##_##sym[]; \
|
|
\
|
|
runtime_const_fixup(__runtime_fixup_##type, \
|
|
(unsigned long)(sym), \
|
|
__start_runtime_##type##_##sym, \
|
|
__stop_runtime_##type##_##sym); \
|
|
} while (0)
|
|
|
|
static inline void __runtime_fixup_32(u32 *p, unsigned int val)
|
|
{
|
|
s390_kernel_write(p, &val, sizeof(val));
|
|
}
|
|
|
|
/* 32-bit immediate for iihf and iilf in bits in I2 field */
|
|
static inline void __runtime_fixup_ptr(void *where, unsigned long val)
|
|
{
|
|
__runtime_fixup_32(where + 2, val >> 32);
|
|
__runtime_fixup_32(where + 8, val);
|
|
}
|
|
|
|
/* Immediate value is lower 12 bits of D2 field of srl */
|
|
static inline void __runtime_fixup_shift(void *where, unsigned long val)
|
|
{
|
|
u32 insn = *(u32 *)where;
|
|
|
|
insn &= 0xfffff000;
|
|
insn |= (val & 63);
|
|
s390_kernel_write(where, &insn, sizeof(insn));
|
|
}
|
|
|
|
/* 32-bit immediate for nilf in bits in I2 field */
|
|
static inline void __runtime_fixup_mask(void *where, unsigned long val)
|
|
{
|
|
__runtime_fixup_32(where + 2, val);
|
|
}
|
|
|
|
static inline void runtime_const_fixup(void (*fn)(void *, unsigned long),
|
|
unsigned long val, s32 *start, s32 *end)
|
|
{
|
|
while (start < end) {
|
|
fn(*start + (void *)start, val);
|
|
start++;
|
|
}
|
|
}
|
|
|
|
#endif /* _ASM_S390_RUNTIME_CONST_H */
|