lib/libutils/ext/include/speculation_barrier.h

/* SPDX-License-Identifier: BSL-1.0 */
/* Copyright (c) 2017 Arm Limited. All rights reserved.

  Boost Software License - Version 1.0 - August 17th, 2003

  Permission is hereby granted, free of charge, to any person or organization
  obtaining a copy of the software and accompanying documentation covered by
  this license (the "Software") to use, reproduce, display, distribute,
  execute, and transmit the Software, and to prepare derivative works of the
  Software, and to permit third-parties to whom the Software is furnished to do
  so, all subject to the following:

  The copyright notices in the Software and this entire statement, including
  the above license grant, this restriction and the following disclaimer, must
  be included in all copies of the Software, in whole or in part, and all
  derivative works of the Software, unless such copies or derivative works are
  solely in the form of machine-executable object code generated by a source
  language processor.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
  SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR
  ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  DEALINGS IN THE SOFTWARE.  */


#ifdef __HAVE_LOAD_NO_SPECULATE
#define load_no_speculate(__ptr, __low, __high) 			\
(__extension__ ({							\
  __typeof__ ((__ptr)) __ptr_once = (__ptr);				\
  __builtin_load_no_speculate (__ptr_once, __low, __high,		\
			       0, __ptr_once);				\
}))

#define load_no_speculate_fail(__ptr, __low, __high, __failval) 	\
(__extension__ ({							\
  __typeof__ ((__ptr)) __ptr_once = (__ptr);				\
  __builtin_load_no_speculate (__ptr_once, __low, __high,		\
			       __failval, __ptr_once);			\
}))

#define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \
  (__builtin_load_no_speculate (__ptr, __low, __high, __failval, __cmpptr))

#else

#ifdef __GNUC__
#define __UNUSED __attribute__((unused))
#else
#define __UNUSED
#endif

#ifdef __aarch64__

#define __load_no_speculate1(__ptr, __low, __high, __failval,		\
			     __cmpptr, __w, __sz) 			\
(__extension__ ({							\
  __typeof__ (0 + (*(__ptr)))  __nln_val;				\
  /* This typecasting is required to ensure correct handling of upper   \
     bits of failval, to ensure a clean return from the CSEL below.  */	\
  __typeof__(*(__ptr)) __fv						\
    = (__typeof__(*(__ptr)))(unsigned long long) (__failval);		\
  /* If __high is explicitly NULL, we must not emit the			\
     upper-bound comparison.  We need to cast __high to an		\
     unsigned long long before handing it to __builtin_constant_p to	\
     ensure that clang/llvm correctly detects NULL as a constant if it	\
     is defined as (void*) 0.  */					\
  if (__builtin_constant_p ((unsigned long long)__high)			\
      && __high == ((void *)0))						\
    {									\
      __asm__ volatile (						\
      "cmp\t%[__c], %[__l]\n\t"						\
      "bcc\t.ns%=\n\t"							\
      "ldr" __sz "\t%" __w "[__v], %[__p]\n"				\
      ".ns%=:\n\t"							\
      "csel\t%" __w "[__v], %" __w "[__v], %" __w "[__f], cs\n\t"	\
      "hint\t#0x14 // CSDB"						\
      /* The value we have loaded, or failval if the condition check	\
	 fails.  */							\
      : [__v] "=&r" (__nln_val)						\
      /* The pointer we wish to use for comparisons, and the low and	\
	 high bounds to use in that comparison. Note that this need	\
	 not be the same as the pointer from which we will load.  */	\
      : [__c] "r" (__cmpptr), [__l] "r" (__low),			\
      /* The memory location from which we will load.  */		\
      [__p] "m" (*(__ptr)),						\
      /* The value to return if the condition check fails.  */		\
      [__f] "rZ" (__fv) 						\
      /* We always clobber the condition codes.  */			\
      : "cc");								\
    }									\
  else									\
    {									\
      __asm__ volatile (						\
      "cmp\t%[__c], %[__l]\n\t"						\
      "ccmp\t%[__c], %[__h], 2, cs\n\t"					\
      "bcs\t.ns%=\n\t"							\
      "ldr" __sz "\t%" __w "[__v], %[__p]\n"				\
      ".ns%=:\n\t"							\
      "csel\t%" __w "[__v], %" __w "[__v], %" __w "[__f], cc\n\t"	\
      "hint\t#0x14 // CSDB"						\
      /* The value we have loaded, or failval if the condition check	\
	 fails.  */							\
      : [__v] "=&r" (__nln_val)						\
      /* The pointer we wish to use for comparisons, and the low and	\
	 high bounds to use in that comparison. Note that this need	\
	 not be the same as the pointer from which we will load.  */	\
      : [__c] "r" (__cmpptr), [__l] "r" (__low),			\
      [__h] "r" (__high),						\
      /* The memory location from which we will load.  */		\
      [__p] "m" (*(__ptr)),						\
      /* The value to return if the condition check fails.  */		\
      [__f] "rZ" (__fv) 						\
      /* We always clobber the condition codes.  */			\
      : "cc");								\
    }									\
  (__typeof__ (*(__ptr))) __nln_val;					\
}))

#define __load_no_speculate(__ptr, __low, __high, __failval, __cmpptr)	\
(__extension__ ({							\
  __typeof__ (0 + *(__ptr)) __nl_val;					\
									\
  switch (sizeof(*(__ptr))) {						\
    case 1:								\
      __nl_val = __load_no_speculate1 (__ptr, __low, __high,		\
				       __failval, __cmpptr, "w", "b");	\
      break;								\
    case 2:								\
      __nl_val = __load_no_speculate1 (__ptr, __low, __high,		\
				       __failval, __cmpptr, "w", "h");	\
      break;								\
    case 4:								\
      __nl_val = __load_no_speculate1 (__ptr, __low, __high,		\
				       __failval, __cmpptr, "w", "");	\
      break;								\
    case 8:								\
      __nl_val = __load_no_speculate1 (__ptr, __low, __high,		\
				       __failval, __cmpptr, "x", "");	\
      break;								\
    default:								\
      {									\
        char __static_assert_no_speculate_load_size_too_big 		\
		[sizeof (__nl_val) > 8 ? -1 : 1] __UNUSED;		\
        break;								\
      }									\
 }									\
									\
  (__typeof__ (*(__ptr))) __nl_val;					\
}))

#define load_no_speculate(__ptr, __low, __high) 			\
(__extension__ ({							\
  __typeof__ ((__ptr)) __ptr_once = (__ptr);				\
  __load_no_speculate (__ptr_once, __low, __high, 0, __ptr_once);	\
}))

#define load_no_speculate_fail(__ptr, __low, __high, __failval) 	\
(__extension__ ({							\
  __typeof__ ((__ptr)) __ptr_once = (__ptr);				\
  __load_no_speculate (__ptr_once, __low, __high,			\
		       __failval, __ptr_once);				\
}))

#define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \
  (__load_no_speculate (__ptr, __low, __high, __failval, __cmpptr))

/* AArch32 support for ARM and Thumb-2.  Thumb-1 is not supported.  */
#elif defined (__ARM_32BIT_STATE) && (defined (__thumb2__) || !defined (__thumb__))
#ifdef __thumb2__
/* Thumb2 case.  */

#define __load_no_speculate1(__ptr, __low, __high, __failval,		\
			     __cmpptr, __sz) 				\
(__extension__ ({							\
  __typeof__ (0 + *(__ptr))  __nln_val;					\
  __typeof__(*(__ptr)) __fv						\
    = (__typeof__(*(__ptr)))(unsigned long) (__failval);		\
  /* If __high is explicitly NULL, we must not emit the			\
     upper-bound comparison.  We need to cast __high to an		\
     unsigned long before handing it to __builtin_constant_p to		\
     ensure that clang/llvm correctly detects NULL as a constant if it	\
     is defined as (void*) 0.  */					\
  if (__builtin_constant_p ((unsigned long)__high)			\
      && __high == ((void *)0))						\
    {									\
      __asm__ volatile (						\
      ".syntax unified\n\t"						\
      "cmp\t%[__c], %[__l]\n\t"						\
      "bcc\t.ns%=\n\t"							\
      "ldr" __sz "\t%[__v], %[__p]\n"					\
      ".ns%=:\n\t"							\
      "it\tcc\n\t"							\
      "movcc\t%[__v], %[__f]\n\t"					\
      ".inst.n 0xf3af\t@ CSDB\n\t"					\
      ".inst.n 0x8014\t@ CSDB"						\
      /* The value we have loaded, or failval if the condition check	\
	 fails.  */							\
      : [__v] "=&l" (__nln_val)						\
      /* The pointer we wish to use for comparisons, and the low and	\
	 high bounds to use in that comparison. Note that this need	\
	 not be the same as the pointer from which we will load.  */	\
      : [__c] "r" (__cmpptr), [__l] "r" (__low), 			\
      /* The memory location from which we will load.  */		\
      [__p] "m" (*(__ptr)),						\
      /* The value to return if the condition check fails.  */		\
      [__f] "r" (__fv) 							\
      /* We always clobber the condition codes.  */			\
      : "cc");								\
    }									\
  else									\
    {									\
      __asm__ volatile (						\
      ".syntax unified\n\t"						\
      "cmp\t%[__c], %[__l]\n\t"						\
      "it\tcs\n\t"							\
      "cmpcs\t%[__h], %[__c]\n\t"					\
      "bls\t.ns%=\n\t"							\
      "ldr" __sz "\t%[__v], %[__p]\n"					\
      ".ns%=:\n\t"							\
      "it\tls\n\t"							\
      "movls\t%[__v], %[__f]\n\t"					\
      ".inst.n 0xf3af\t@ CSDB\n\t"					\
      ".inst.n 0x8014\t@ CSDB"					\
      /* The value we have loaded, or failval if the condition check	\
	 fails.  */							\
      : [__v] "=&l" (__nln_val)						\
      /* The pointer we wish to use for comparisons, and the low and	\
	 high bounds to use in that comparison. Note that this need	\
	 not be the same as the pointer from which we will load.  */	\
      : [__c] "r" (__cmpptr), [__l] "r" (__low),			\
      [__h] "r" (__high),						\
      /* The memory location from which we will load.  */		\
      [__p] "m" (*(__ptr)),						\
      /* The value to return if the condition check fails.  */		\
      [__f] "r" (__fv) 							\
      /* We always clobber the condition codes.  */			\
      : "cc");								\
    }									\
  (__typeof__ (*(__ptr))) __nln_val;					\
}))									\
									\
/* Double-word version.  */
#define __load_no_speculate2(__ptr, __low, __high, __failval,		\
			     __cmpptr) 					\
(__extension__ ({							\
  __typeof__ (0 + *(__ptr))  __nln_val;					\
  __typeof__(*(__ptr)) __fv						\
    = (__typeof__(*(__ptr)))(unsigned long) (__failval);		\
  /* If __high is explicitly NULL, we must not emit the			\
     upper-bound comparison.  We need to cast __high to an		\
     unsigned long before handing it to __builtin_constant_p to	\
     ensure that clang/llvm correctly detects NULL as a constant if it	\
     is defined as (void*) 0.  */					\
  if (__builtin_constant_p ((unsigned long)__high)			\
      && __high == ((void *)0))						\
    {									\
      __asm__ volatile (						\
      ".syntax unified\n\t"						\
      "cmp\t%[__c], %[__l]\n\t"						\
      "bcc\t.ns%=\n\t"							\
      "ldr\t%Q[__v], [%[__p]]\n\t"					\
      "ldr\t%R[__v], [%[__p], #4]\n"					\
      ".ns%=:\n\t"							\
      "it\tcc\n\t"							\
      "movcc\t%Q[__v], %Q[__f]\n\t"					\
      "it\tcc\n\t"							\
      "movcc\t%R[__v], %R[__f]\n\t"					\
      ".inst.n 0xf3af\t@ CSDB\n\t"					\
      ".inst.n 0x8014\t@ CSDB"					\
      /* The value we have loaded, or failval if the condition check	\
	 fails.  */							\
      : [__v] "=&l" (__nln_val)						\
      /* The pointer we wish to use for comparisons, and the low and	\
	 high bounds to use in that comparison. Note that this need	\
	 not be the same as the pointer from which we will load.  */	\
      : [__c] "r" (__cmpptr), [__l] "r" (__low), 			\
      /* The memory location from which we will load.  */		\
      [__p] "r" (__ptr),						\
      /* The value to return if the condition check fails.  */		\
      [__f] "r" (__fv) 							\
      /* We always clobber the condition codes.  */			\
      : "cc");								\
    }									\
  else									\
    {									\
      __asm__ volatile (						\
      ".syntax unified\n\t"						\
      "cmp\t%[__c], %[__l]\n\t"						\
      "it\tcs\n\t"							\
      "cmpcs\t%[__h], %[__c]\n\t"					\
      "bls\t.ns%=\n\t"							\
      "ldr\t%Q[__v], [%[__p]]\n\t"					\
      "ldr\t%R[__v], [%[__p], #4]\n"					\
      ".ns%=:\n\t"							\
      "it\tls\n\t"							\
      "movls\t%Q[__v], %Q[__f]\n\t"					\
      "it\tls\n\t"							\
      "movls\t%R[__v], %R[__f]\n\t"					\
      ".inst.n 0xf3af\t@ CSDB\n\t"					\
      ".inst.n 0x8014\t@ CSDB"						\
      /* The value we have loaded, or failval if the condition check	\
	 fails.  */							\
      : [__v] "=&l" (__nln_val)						\
      /* The pointer we wish to use for comparisons, and the low and	\
	 high bounds to use in that comparison. Note that this need	\
	 not be the same as the pointer from which we will load.  */	\
      : [__c] "r" (__cmpptr), [__l] "r" (__low),			\
      [__h] "r" (__high),						\
      /* The memory location from which we will load.  */		\
      [__p] "r" (__ptr),						\
      /* The value to return if the condition check fails.  */		\
      [__f] "r" (__fv) 							\
      /* We always clobber the condition codes.  */			\
      : "cc");								\
    }									\
  (__typeof__ (*(__ptr))) __nln_val;					\
}))

#else
/* ARM case.  */

#define __load_no_speculate1(__ptr, __low, __high, __failval,		\
			     __cmpptr, __sz) 				\
(__extension__ ({							\
  __typeof__ (0 + *(__ptr))  __nln_val;					\
  __typeof__(*(__ptr)) __fv						\
    = (__typeof__(*(__ptr)))(unsigned long) (__failval);		\
  /* If __high is explicitly NULL, we must not emit the			\
     upper-bound comparison.  We need to cast __high to an		\
     unsigned long before handing it to __builtin_constant_p to		\
     ensure that clang/llvm correctly detects NULL as a constant if it	\
     is defined as (void*) 0.  */					\
  if (__builtin_constant_p ((unsigned long)__high)			\
      && __high == ((void *)0))						\
    {									\
      __asm__ volatile (						\
      ".syntax unified\n\t"						\
      "cmp\t%[__c], %[__l]\n\t"						\
      "ldr" __sz "cs\t%[__v], %[__p]\n\t"				\
      "movcc\t%[__v], %[__f]\n\t"					\
      ".inst 0xe320f014\t@ CSDB"					\
      /* The value we have loaded, or failval if the condition check	\
	 fails.  */							\
      : [__v] "=&r" (__nln_val)						\
      /* The pointer we wish to use for comparisons, and the low and	\
	 high bounds to use in that comparison. Note that this need	\
	 not be the same as the pointer from which we will load.  */	\
      : [__c] "r" (__cmpptr), [__l] "r" (__low),			\
      /* The memory location from which we will load.  */		\
      [__p] "m" (*(__ptr)),						\
      /* The value to return if the condition check fails.  */		\
      [__f] "rKI" (__fv) 						\
      /* We always clobber the condition codes.  */			\
      : "cc");								\
    }									\
  else									\
    {									\
      __asm__ volatile (						\
      ".syntax unified\n\t"						\
      "cmp\t%[__c], %[__l]\n\t"						\
      "cmpcs\t%[__h], %[__c]\n\t"					\
      "ldr" __sz "hi\t%[__v], %[__p]\n\t"				\
      "movls\t%[__v], %[__f]\n\t"					\
      ".inst 0xe320f014\t@ CSDB"					\
      /* The value we have loaded, or failval if the condition check	\
	 fails.  */							\
      : [__v] "=&r" (__nln_val)						\
      /* The pointer we wish to use for comparisons, and the low and	\
	 high bounds to use in that comparison. Note that this need	\
	 not be the same as the pointer from which we will load.  */	\
      : [__c] "r" (__cmpptr), [__l] "r" (__low),			\
      [__h] "r" (__high),						\
      /* The memory location from which we will load.  */		\
      [__p] "m" (*(__ptr)),						\
      /* The value to return if the condition check fails.  */		\
      [__f] "rKI" (__fv) 						\
      /* We always clobber the condition codes.  */			\
      : "cc");								\
    }									\
  (__typeof__ (*(__ptr))) __nln_val;					\
}))

/* Double-word version.  */
#define __load_no_speculate2(__ptr, __low, __high, __failval,		\
			     __cmpptr) 					\
(__extension__ ({							\
  __typeof__ (0 + *(__ptr))  __nln_val;					\
  __typeof__(*(__ptr)) __fv						\
    = (__typeof__(*(__ptr)))(unsigned long) (__failval);		\
  /* If __high is explicitly NULL, we must not emit the			\
     upper-bound comparison.  We need to cast __high to an		\
     unsigned long before handing it to __builtin_constant_p to		\
     ensure that clang/llvm correctly detects NULL as a constant if it	\
     is defined as (void*) 0.  */					\
  if (__builtin_constant_p ((unsigned long)__high)			\
      && __high == ((void *)0))						\
    {									\
      __asm__ volatile (						\
      ".syntax unified\n\t"						\
      "cmp\t%[__c], %[__l]\n\t"						\
      "ldrcs\t%Q[__v], [%[__p]]\n\t"					\
      "ldrcs\t%R[__v], [%[__p], #4]\n\t"				\
      "movcc\t%Q[__v], %Q[__f]\n\t"					\
      "movcc\t%R[__v], %R[__f]\n\t"					\
      ".inst 0xe320f014\t@ CSDB"					\
      /* The value we have loaded, or failval if the condition check	\
	 fails.  */							\
      : [__v] "=&r" (__nln_val)						\
      /* The pointer we wish to use for comparisons, and the low and	\
	 high bounds to use in that comparison. Note that this need	\
	 not be the same as the pointer from which we will load.  */	\
      : [__c] "r" (__cmpptr), [__l] "r" (__low),			\
      /* The memory location from which we will load.  */		\
      [__p] "r" (__ptr),						\
      /* The value to return if the condition check fails.  */		\
      [__f] "r" (__fv) 							\
      /* We always clobber the condition codes.  */			\
      : "cc");								\
    }									\
  else									\
    {									\
      __asm__ volatile (						\
      ".syntax unified\n\t"						\
      "cmp\t%[__c], %[__l]\n\t"						\
      "cmpcs\t%[__h], %[__c]\n\t"					\
      "ldrhi\t%Q[__v], [%[__p]]\n\t"					\
      "ldrhi\t%R[__v], [%[__p], #4]\n\t"				\
      "movls\t%Q[__v], %Q[__f]\n\t"					\
      "movls\t%R[__v], %R[__f]\n\t"					\
      ".inst 0xe320f014\t@ CSDB"					\
      /* The value we have loaded, or failval if the condition check	\
	 fails.  */							\
      : [__v] "=&r" (__nln_val)						\
      /* The pointer we wish to use for comparisons, and the low and	\
	 high bounds to use in that comparison. Note that this need	\
	 not be the same as the pointer from which we will load.  */	\
      : [__c] "r" (__cmpptr), [__l] "r" (__low),			\
      [__h] "r" (__high),						\
      /* The memory location from which we will load.  */		\
      [__p] "r" (__ptr),						\
      /* The value to return if the condition check fails.  */		\
      [__f] "r" (__fv) 							\
      /* We always clobber the condition codes.  */			\
      : "cc");								\
    }									\
  (__typeof__ (*(__ptr))) __nln_val;					\
}))

#endif // __thumb2__

/* Common to ARM and Thumb2.  */

#define __load_no_speculate(__ptr, __low, __high, __failval, __cmpptr)	\
(__extension__ ({							\
  __typeof__ (0 + *(__ptr)) __nl_val;					\
									\
  switch (sizeof(*(__ptr))) {						\
    case 1:								\
      __nl_val = __load_no_speculate1 (__ptr, __low, __high,		\
				       __failval, __cmpptr, "b");	\
      break;								\
    case 2:								\
      __nl_val = __load_no_speculate1 (__ptr, __low, __high,		\
				       __failval, __cmpptr, "h");	\
      break;								\
    case 4:								\
      __nl_val = __load_no_speculate1 (__ptr, __low, __high,		\
				       __failval, __cmpptr, "");	\
      break;								\
    case 8:								\
      __nl_val = __load_no_speculate2 (__ptr, __low, __high,		\
				       __failval, __cmpptr);		\
      break;								\
    default:								\
      {									\
        char __static_assert_no_speculate_load_size_too_big 		\
		[sizeof (__nl_val) > 8 ? -1 : 1] __UNUSED;		\
        break;								\
      }									\
 }									\
									\
  (__typeof__ (*(__ptr))) __nl_val;					\
}))

#define load_no_speculate(__ptr, __low, __high) 			\
(__extension__ ({							\
  __typeof__ ((__ptr)) __ptr_once = (__ptr);				\
  __load_no_speculate (__ptr_once, __low, __high, 0, __ptr_once);	\
}))

#define load_no_speculate_fail(__ptr, __low, __high, __failval) 	\
(__extension__ ({							\
  __typeof__ ((__ptr)) __ptr_once = (__ptr);				\
  __load_no_speculate (__ptr_once, __low, __high,			\
		       __failval, __ptr_once);				\
}))

#define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \
  (__load_no_speculate (__ptr, __low, __high, __failval, __cmpptr))

#else
#error "No fallback provided for load_no_speculate"
#endif

#endif