futex.h 3.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. /*
  2. * Copyright (C) 2012 ARM Ltd.
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License version 2 as
  6. * published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. * GNU General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public License
  14. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  15. */
  16. #ifndef __ASM_FUTEX_H
  17. #define __ASM_FUTEX_H
  18. #ifdef __KERNEL__
  19. #include <linux/futex.h>
  20. #include <linux/uaccess.h>
  21. #include <asm/errno.h>
  22. #define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
  23. #define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
  24. do { \
  25. unsigned int loops = FUTEX_MAX_LOOPS; \
  26. \
  27. uaccess_enable(); \
  28. asm volatile( \
  29. " prfm pstl1strm, %2\n" \
  30. "1: ldxr %w1, %2\n" \
  31. insn "\n" \
  32. "2: stlxr %w0, %w3, %2\n" \
  33. " cbz %w0, 3f\n" \
  34. " sub %w4, %w4, %w0\n" \
  35. " cbnz %w4, 1b\n" \
  36. " mov %w0, %w7\n" \
  37. "3:\n" \
  38. " dmb ish\n" \
  39. " .pushsection .fixup,\"ax\"\n" \
  40. " .align 2\n" \
  41. "4: mov %w0, %w6\n" \
  42. " b 3b\n" \
  43. " .popsection\n" \
  44. _ASM_EXTABLE(1b, 4b) \
  45. _ASM_EXTABLE(2b, 4b) \
  46. : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
  47. "+r" (loops) \
  48. : "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN) \
  49. : "memory"); \
  50. uaccess_disable(); \
  51. } while (0)
  52. static inline int
  53. arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)
  54. {
  55. int oldval = 0, ret, tmp;
  56. u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);
  57. pagefault_disable();
  58. switch (op) {
  59. case FUTEX_OP_SET:
  60. __futex_atomic_op("mov %w3, %w5",
  61. ret, oldval, uaddr, tmp, oparg);
  62. break;
  63. case FUTEX_OP_ADD:
  64. __futex_atomic_op("add %w3, %w1, %w5",
  65. ret, oldval, uaddr, tmp, oparg);
  66. break;
  67. case FUTEX_OP_OR:
  68. __futex_atomic_op("orr %w3, %w1, %w5",
  69. ret, oldval, uaddr, tmp, oparg);
  70. break;
  71. case FUTEX_OP_ANDN:
  72. __futex_atomic_op("and %w3, %w1, %w5",
  73. ret, oldval, uaddr, tmp, ~oparg);
  74. break;
  75. case FUTEX_OP_XOR:
  76. __futex_atomic_op("eor %w3, %w1, %w5",
  77. ret, oldval, uaddr, tmp, oparg);
  78. break;
  79. default:
  80. ret = -ENOSYS;
  81. }
  82. pagefault_enable();
  83. if (!ret)
  84. *oval = oldval;
  85. return ret;
  86. }
  87. static inline int
  88. futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
  89. u32 oldval, u32 newval)
  90. {
  91. int ret = 0;
  92. unsigned int loops = FUTEX_MAX_LOOPS;
  93. u32 val, tmp;
  94. u32 __user *uaddr;
  95. if (!access_ok(VERIFY_WRITE, _uaddr, sizeof(u32)))
  96. return -EFAULT;
  97. uaddr = __uaccess_mask_ptr(_uaddr);
  98. uaccess_enable();
  99. asm volatile("// futex_atomic_cmpxchg_inatomic\n"
  100. " prfm pstl1strm, %2\n"
  101. "1: ldxr %w1, %2\n"
  102. " sub %w3, %w1, %w5\n"
  103. " cbnz %w3, 4f\n"
  104. "2: stlxr %w3, %w6, %2\n"
  105. " cbz %w3, 3f\n"
  106. " sub %w4, %w4, %w3\n"
  107. " cbnz %w4, 1b\n"
  108. " mov %w0, %w8\n"
  109. "3:\n"
  110. " dmb ish\n"
  111. "4:\n"
  112. " .pushsection .fixup,\"ax\"\n"
  113. "5: mov %w0, %w7\n"
  114. " b 4b\n"
  115. " .popsection\n"
  116. _ASM_EXTABLE(1b, 5b)
  117. _ASM_EXTABLE(2b, 5b)
  118. : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
  119. : "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
  120. : "memory");
  121. uaccess_disable();
  122. if (!ret)
  123. *uval = val;
  124. return ret;
  125. }
  126. #endif /* __KERNEL__ */
  127. #endif /* __ASM_FUTEX_H */