Andrew Scull | b4b6d4a | 2019-01-02 15:54:55 +0000 | [diff] [blame^] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
| 6 | * Copyright (C) 2006 by Ralf Baechle (ralf@linux-mips.org) |
| 7 | */ |
| 8 | #ifndef __ASM_BARRIER_H |
| 9 | #define __ASM_BARRIER_H |
| 10 | |
| 11 | #include <asm/addrspace.h> |
| 12 | |
| 13 | /* |
| 14 | * Sync types defined by the MIPS architecture (document MD00087 table 6.5) |
| 15 | * These values are used with the sync instruction to perform memory barriers. |
| 16 | * Types of ordering guarantees available through the SYNC instruction: |
| 17 | * - Completion Barriers |
| 18 | * - Ordering Barriers |
| 19 | * As compared to the completion barrier, the ordering barrier is a |
| 20 | * lighter-weight operation as it does not require the specified instructions |
| 21 | * before the SYNC to be already completed. Instead it only requires that those |
| 22 | * specified instructions which are subsequent to the SYNC in the instruction |
| 23 | * stream are never re-ordered for processing ahead of the specified |
| 24 | * instructions which are before the SYNC in the instruction stream. |
| 25 | * This potentially reduces how many cycles the barrier instruction must stall |
| 26 | * before it completes. |
| 27 | * Implementations that do not use any of the non-zero values of stype to define |
| 28 | * different barriers, such as ordering barriers, must make those stype values |
| 29 | * act the same as stype zero. |
| 30 | */ |
| 31 | |
| 32 | /* |
| 33 | * Completion barriers: |
| 34 | * - Every synchronizable specified memory instruction (loads or stores or both) |
| 35 | * that occurs in the instruction stream before the SYNC instruction must be |
| 36 | * already globally performed before any synchronizable specified memory |
| 37 | * instructions that occur after the SYNC are allowed to be performed, with |
| 38 | * respect to any other processor or coherent I/O module. |
| 39 | * |
| 40 | * - The barrier does not guarantee the order in which instruction fetches are |
| 41 | * performed. |
| 42 | * |
| 43 | * - A stype value of zero will always be defined such that it performs the most |
| 44 | * complete set of synchronization operations that are defined.This means |
| 45 | * stype zero always does a completion barrier that affects both loads and |
| 46 | * stores preceding the SYNC instruction and both loads and stores that are |
| 47 | * subsequent to the SYNC instruction. Non-zero values of stype may be defined |
| 48 | * by the architecture or specific implementations to perform synchronization |
| 49 | * behaviors that are less complete than that of stype zero. If an |
| 50 | * implementation does not use one of these non-zero values to define a |
| 51 | * different synchronization behavior, then that non-zero value of stype must |
| 52 | * act the same as stype zero completion barrier. This allows software written |
| 53 | * for an implementation with a lighter-weight barrier to work on another |
| 54 | * implementation which only implements the stype zero completion barrier. |
| 55 | * |
| 56 | * - A completion barrier is required, potentially in conjunction with SSNOP (in |
| 57 | * Release 1 of the Architecture) or EHB (in Release 2 of the Architecture), |
| 58 | * to guarantee that memory reference results are visible across operating |
| 59 | * mode changes. For example, a completion barrier is required on some |
| 60 | * implementations on entry to and exit from Debug Mode to guarantee that |
| 61 | * memory effects are handled correctly. |
| 62 | */ |
| 63 | |
| 64 | /* |
| 65 | * stype 0 - A completion barrier that affects preceding loads and stores and |
| 66 | * subsequent loads and stores. |
| 67 | * Older instructions which must reach the load/store ordering point before the |
| 68 | * SYNC instruction completes: Loads, Stores |
| 69 | * Younger instructions which must reach the load/store ordering point only |
| 70 | * after the SYNC instruction completes: Loads, Stores |
| 71 | * Older instructions which must be globally performed when the SYNC instruction |
| 72 | * completes: Loads, Stores |
| 73 | */ |
| 74 | #define STYPE_SYNC 0x0 |
| 75 | |
| 76 | /* |
| 77 | * Ordering barriers: |
| 78 | * - Every synchronizable specified memory instruction (loads or stores or both) |
| 79 | * that occurs in the instruction stream before the SYNC instruction must |
| 80 | * reach a stage in the load/store datapath after which no instruction |
| 81 | * re-ordering is possible before any synchronizable specified memory |
| 82 | * instruction which occurs after the SYNC instruction in the instruction |
| 83 | * stream reaches the same stage in the load/store datapath. |
| 84 | * |
| 85 | * - If any memory instruction before the SYNC instruction in program order, |
| 86 | * generates a memory request to the external memory and any memory |
| 87 | * instruction after the SYNC instruction in program order also generates a |
| 88 | * memory request to external memory, the memory request belonging to the |
| 89 | * older instruction must be globally performed before the time the memory |
| 90 | * request belonging to the younger instruction is globally performed. |
| 91 | * |
| 92 | * - The barrier does not guarantee the order in which instruction fetches are |
| 93 | * performed. |
| 94 | */ |
| 95 | |
| 96 | /* |
| 97 | * stype 0x10 - An ordering barrier that affects preceding loads and stores and |
| 98 | * subsequent loads and stores. |
| 99 | * Older instructions which must reach the load/store ordering point before the |
| 100 | * SYNC instruction completes: Loads, Stores |
| 101 | * Younger instructions which must reach the load/store ordering point only |
| 102 | * after the SYNC instruction completes: Loads, Stores |
| 103 | * Older instructions which must be globally performed when the SYNC instruction |
| 104 | * completes: N/A |
| 105 | */ |
| 106 | #define STYPE_SYNC_MB 0x10 |
| 107 | |
| 108 | |
| 109 | #ifdef CONFIG_CPU_HAS_SYNC |
| 110 | #define __sync() \ |
| 111 | __asm__ __volatile__( \ |
| 112 | ".set push\n\t" \ |
| 113 | ".set noreorder\n\t" \ |
| 114 | ".set mips2\n\t" \ |
| 115 | "sync\n\t" \ |
| 116 | ".set pop" \ |
| 117 | : /* no output */ \ |
| 118 | : /* no input */ \ |
| 119 | : "memory") |
| 120 | #else |
| 121 | #define __sync() do { } while(0) |
| 122 | #endif |
| 123 | |
| 124 | #define __fast_iob() \ |
| 125 | __asm__ __volatile__( \ |
| 126 | ".set push\n\t" \ |
| 127 | ".set noreorder\n\t" \ |
| 128 | "lw $0,%0\n\t" \ |
| 129 | "nop\n\t" \ |
| 130 | ".set pop" \ |
| 131 | : /* no output */ \ |
| 132 | : "m" (*(int *)CKSEG1) \ |
| 133 | : "memory") |
| 134 | #ifdef CONFIG_CPU_CAVIUM_OCTEON |
| 135 | # define OCTEON_SYNCW_STR ".set push\n.set arch=octeon\nsyncw\nsyncw\n.set pop\n" |
| 136 | # define __syncw() __asm__ __volatile__(OCTEON_SYNCW_STR : : : "memory") |
| 137 | |
| 138 | # define fast_wmb() __syncw() |
| 139 | # define fast_rmb() barrier() |
| 140 | # define fast_mb() __sync() |
| 141 | # define fast_iob() do { } while (0) |
| 142 | #else /* ! CONFIG_CPU_CAVIUM_OCTEON */ |
| 143 | # define fast_wmb() __sync() |
| 144 | # define fast_rmb() __sync() |
| 145 | # define fast_mb() __sync() |
| 146 | # ifdef CONFIG_SGI_IP28 |
| 147 | # define fast_iob() \ |
| 148 | __asm__ __volatile__( \ |
| 149 | ".set push\n\t" \ |
| 150 | ".set noreorder\n\t" \ |
| 151 | "lw $0,%0\n\t" \ |
| 152 | "sync\n\t" \ |
| 153 | "lw $0,%0\n\t" \ |
| 154 | ".set pop" \ |
| 155 | : /* no output */ \ |
| 156 | : "m" (*(int *)CKSEG1ADDR(0x1fa00004)) \ |
| 157 | : "memory") |
| 158 | # else |
| 159 | # define fast_iob() \ |
| 160 | do { \ |
| 161 | __sync(); \ |
| 162 | __fast_iob(); \ |
| 163 | } while (0) |
| 164 | # endif |
| 165 | #endif /* CONFIG_CPU_CAVIUM_OCTEON */ |
| 166 | |
| 167 | #ifdef CONFIG_CPU_HAS_WB |
| 168 | |
| 169 | #include <asm/wbflush.h> |
| 170 | |
| 171 | #define mb() wbflush() |
| 172 | #define iob() wbflush() |
| 173 | |
| 174 | #else /* !CONFIG_CPU_HAS_WB */ |
| 175 | |
| 176 | #define mb() fast_mb() |
| 177 | #define iob() fast_iob() |
| 178 | |
| 179 | #endif /* !CONFIG_CPU_HAS_WB */ |
| 180 | |
| 181 | #define wmb() fast_wmb() |
| 182 | #define rmb() fast_rmb() |
| 183 | |
| 184 | #if defined(CONFIG_WEAK_ORDERING) |
| 185 | # ifdef CONFIG_CPU_CAVIUM_OCTEON |
| 186 | # define __smp_mb() __sync() |
| 187 | # define __smp_rmb() barrier() |
| 188 | # define __smp_wmb() __syncw() |
| 189 | # else |
| 190 | # define __smp_mb() __asm__ __volatile__("sync" : : :"memory") |
| 191 | # define __smp_rmb() __asm__ __volatile__("sync" : : :"memory") |
| 192 | # define __smp_wmb() __asm__ __volatile__("sync" : : :"memory") |
| 193 | # endif |
| 194 | #else |
| 195 | #define __smp_mb() barrier() |
| 196 | #define __smp_rmb() barrier() |
| 197 | #define __smp_wmb() barrier() |
| 198 | #endif |
| 199 | |
| 200 | #if defined(CONFIG_WEAK_REORDERING_BEYOND_LLSC) && defined(CONFIG_SMP) |
| 201 | #define __WEAK_LLSC_MB " sync \n" |
| 202 | #else |
| 203 | #define __WEAK_LLSC_MB " \n" |
| 204 | #endif |
| 205 | |
| 206 | #define smp_llsc_mb() __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory") |
| 207 | |
| 208 | #ifdef CONFIG_CPU_CAVIUM_OCTEON |
| 209 | #define smp_mb__before_llsc() smp_wmb() |
| 210 | #define __smp_mb__before_llsc() __smp_wmb() |
| 211 | /* Cause previous writes to become visible on all CPUs as soon as possible */ |
| 212 | #define nudge_writes() __asm__ __volatile__(".set push\n\t" \ |
| 213 | ".set arch=octeon\n\t" \ |
| 214 | "syncw\n\t" \ |
| 215 | ".set pop" : : : "memory") |
| 216 | #else |
| 217 | #define smp_mb__before_llsc() smp_llsc_mb() |
| 218 | #define __smp_mb__before_llsc() smp_llsc_mb() |
| 219 | #define nudge_writes() mb() |
| 220 | #endif |
| 221 | |
| 222 | #define __smp_mb__before_atomic() __smp_mb__before_llsc() |
| 223 | #define __smp_mb__after_atomic() smp_llsc_mb() |
| 224 | |
| 225 | #include <asm-generic/barrier.h> |
| 226 | |
| 227 | #endif /* __ASM_BARRIER_H */ |