v4.19.13 snapshot.
diff --git a/arch/mips/mm/cerr-sb1.c b/arch/mips/mm/cerr-sb1.c
new file mode 100644
index 0000000..ee5c1ff
--- /dev/null
+++ b/arch/mips/mm/cerr-sb1.c
@@ -0,0 +1,582 @@
+/*
+ * Copyright (C) 2001,2002,2003 Broadcom Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+#include <linux/sched.h>
+#include <asm/mipsregs.h>
+#include <asm/sibyte/sb1250.h>
+#include <asm/sibyte/sb1250_regs.h>
+
+#if !defined(CONFIG_SIBYTE_BUS_WATCHER) || defined(CONFIG_SIBYTE_BW_TRACE)
+#include <asm/io.h>
+#include <asm/sibyte/sb1250_scd.h>
+#endif
+
+/*
+ * We'd like to dump the L2_ECC_TAG register on errors, but errata make
+ * that unsafe... So for now we don't. (BCM1250/BCM112x erratum SOC-48.)
+ */
+#undef DUMP_L2_ECC_TAG_ON_ERROR
+
+/* SB1 definitions */
+
+/* XXX should come from config1 XXX */
+#define SB1_CACHE_INDEX_MASK 0x1fe0
+
+#define CP0_ERRCTL_RECOVERABLE (1 << 31)
+#define CP0_ERRCTL_DCACHE (1 << 30)
+#define CP0_ERRCTL_ICACHE (1 << 29)
+#define CP0_ERRCTL_MULTIBUS (1 << 23)
+#define CP0_ERRCTL_MC_TLB (1 << 15)
+#define CP0_ERRCTL_MC_TIMEOUT (1 << 14)
+
+#define CP0_CERRI_TAG_PARITY (1 << 29)
+#define CP0_CERRI_DATA_PARITY (1 << 28)
+#define CP0_CERRI_EXTERNAL (1 << 26)
+
+#define CP0_CERRI_IDX_VALID(c) (!((c) & CP0_CERRI_EXTERNAL))
+#define CP0_CERRI_DATA (CP0_CERRI_DATA_PARITY)
+
+#define CP0_CERRD_MULTIPLE (1 << 31)
+#define CP0_CERRD_TAG_STATE (1 << 30)
+#define CP0_CERRD_TAG_ADDRESS (1 << 29)
+#define CP0_CERRD_DATA_SBE (1 << 28)
+#define CP0_CERRD_DATA_DBE (1 << 27)
+#define CP0_CERRD_EXTERNAL (1 << 26)
+#define CP0_CERRD_LOAD (1 << 25)
+#define CP0_CERRD_STORE (1 << 24)
+#define CP0_CERRD_FILLWB (1 << 23)
+#define CP0_CERRD_COHERENCY (1 << 22)
+#define CP0_CERRD_DUPTAG (1 << 21)
+
+#define CP0_CERRD_DPA_VALID(c) (!((c) & CP0_CERRD_EXTERNAL))
+#define CP0_CERRD_IDX_VALID(c) \
+ (((c) & (CP0_CERRD_LOAD | CP0_CERRD_STORE)) ? (!((c) & CP0_CERRD_EXTERNAL)) : 0)
+#define CP0_CERRD_CAUSES \
+ (CP0_CERRD_LOAD | CP0_CERRD_STORE | CP0_CERRD_FILLWB | CP0_CERRD_COHERENCY | CP0_CERRD_DUPTAG)
+#define CP0_CERRD_TYPES \
+ (CP0_CERRD_TAG_STATE | CP0_CERRD_TAG_ADDRESS | CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE | CP0_CERRD_EXTERNAL)
+#define CP0_CERRD_DATA (CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE)
+
+static uint32_t extract_ic(unsigned short addr, int data);
+static uint32_t extract_dc(unsigned short addr, int data);
+
+static inline void breakout_errctl(unsigned int val)
+{
+ if (val & CP0_ERRCTL_RECOVERABLE)
+ printk(" recoverable");
+ if (val & CP0_ERRCTL_DCACHE)
+ printk(" dcache");
+ if (val & CP0_ERRCTL_ICACHE)
+ printk(" icache");
+ if (val & CP0_ERRCTL_MULTIBUS)
+ printk(" multiple-buserr");
+ printk("\n");
+}
+
+static inline void breakout_cerri(unsigned int val)
+{
+ if (val & CP0_CERRI_TAG_PARITY)
+ printk(" tag-parity");
+ if (val & CP0_CERRI_DATA_PARITY)
+ printk(" data-parity");
+ if (val & CP0_CERRI_EXTERNAL)
+ printk(" external");
+ printk("\n");
+}
+
+static inline void breakout_cerrd(unsigned int val)
+{
+ switch (val & CP0_CERRD_CAUSES) {
+ case CP0_CERRD_LOAD:
+ printk(" load,");
+ break;
+ case CP0_CERRD_STORE:
+ printk(" store,");
+ break;
+ case CP0_CERRD_FILLWB:
+ printk(" fill/wb,");
+ break;
+ case CP0_CERRD_COHERENCY:
+ printk(" coherency,");
+ break;
+ case CP0_CERRD_DUPTAG:
+ printk(" duptags,");
+ break;
+ default:
+ printk(" NO CAUSE,");
+ break;
+ }
+ if (!(val & CP0_CERRD_TYPES))
+ printk(" NO TYPE");
+ else {
+ if (val & CP0_CERRD_MULTIPLE)
+ printk(" multi-err");
+ if (val & CP0_CERRD_TAG_STATE)
+ printk(" tag-state");
+ if (val & CP0_CERRD_TAG_ADDRESS)
+ printk(" tag-address");
+ if (val & CP0_CERRD_DATA_SBE)
+ printk(" data-SBE");
+ if (val & CP0_CERRD_DATA_DBE)
+ printk(" data-DBE");
+ if (val & CP0_CERRD_EXTERNAL)
+ printk(" external");
+ }
+ printk("\n");
+}
+
+#ifndef CONFIG_SIBYTE_BUS_WATCHER
+
+static void check_bus_watcher(void)
+{
+ uint32_t status, l2_err, memio_err;
+#ifdef DUMP_L2_ECC_TAG_ON_ERROR
+ uint64_t l2_tag;
+#endif
+
+ /* Destructive read, clears register and interrupt */
+ status = csr_in32(IOADDR(A_SCD_BUS_ERR_STATUS));
+ /* Bit 31 is always on, but there's no #define for that */
+ if (status & ~(1UL << 31)) {
+ l2_err = csr_in32(IOADDR(A_BUS_L2_ERRORS));
+#ifdef DUMP_L2_ECC_TAG_ON_ERROR
+ l2_tag = in64(IOADDR(A_L2_ECC_TAG));
+#endif
+ memio_err = csr_in32(IOADDR(A_BUS_MEM_IO_ERRORS));
+ printk("Bus watcher error counters: %08x %08x\n", l2_err, memio_err);
+ printk("\nLast recorded signature:\n");
+ printk("Request %02x from %d, answered by %d with Dcode %d\n",
+ (unsigned int)(G_SCD_BERR_TID(status) & 0x3f),
+ (int)(G_SCD_BERR_TID(status) >> 6),
+ (int)G_SCD_BERR_RID(status),
+ (int)G_SCD_BERR_DCODE(status));
+#ifdef DUMP_L2_ECC_TAG_ON_ERROR
+ printk("Last L2 tag w/ bad ECC: %016llx\n", l2_tag);
+#endif
+ } else {
+ printk("Bus watcher indicates no error\n");
+ }
+}
+#else
+extern void check_bus_watcher(void);
+#endif
+
+asmlinkage void sb1_cache_error(void)
+{
+ uint32_t errctl, cerr_i, cerr_d, dpalo, dpahi, eepc, res;
+ unsigned long long cerr_dpa;
+
+#ifdef CONFIG_SIBYTE_BW_TRACE
+ /* Freeze the trace buffer now */
+ csr_out32(M_SCD_TRACE_CFG_FREEZE, IOADDR(A_SCD_TRACE_CFG));
+ printk("Trace buffer frozen\n");
+#endif
+
+ printk("Cache error exception on CPU %x:\n",
+ (read_c0_prid() >> 25) & 0x7);
+
+ __asm__ __volatile__ (
+ " .set push\n\t"
+ " .set mips64\n\t"
+ " .set noat\n\t"
+ " mfc0 %0, $26\n\t"
+ " mfc0 %1, $27\n\t"
+ " mfc0 %2, $27, 1\n\t"
+ " dmfc0 $1, $27, 3\n\t"
+ " dsrl32 %3, $1, 0 \n\t"
+ " sll %4, $1, 0 \n\t"
+ " mfc0 %5, $30\n\t"
+ " .set pop"
+ : "=r" (errctl), "=r" (cerr_i), "=r" (cerr_d),
+ "=r" (dpahi), "=r" (dpalo), "=r" (eepc));
+
+ cerr_dpa = (((uint64_t)dpahi) << 32) | dpalo;
+ printk(" c0_errorepc == %08x\n", eepc);
+ printk(" c0_errctl == %08x", errctl);
+ breakout_errctl(errctl);
+ if (errctl & CP0_ERRCTL_ICACHE) {
+ printk(" c0_cerr_i == %08x", cerr_i);
+ breakout_cerri(cerr_i);
+ if (CP0_CERRI_IDX_VALID(cerr_i)) {
+ /* Check index of EPC, allowing for delay slot */
+ if (((eepc & SB1_CACHE_INDEX_MASK) != (cerr_i & SB1_CACHE_INDEX_MASK)) &&
+ ((eepc & SB1_CACHE_INDEX_MASK) != ((cerr_i & SB1_CACHE_INDEX_MASK) - 4)))
+ printk(" cerr_i idx doesn't match eepc\n");
+ else {
+ res = extract_ic(cerr_i & SB1_CACHE_INDEX_MASK,
+ (cerr_i & CP0_CERRI_DATA) != 0);
+ if (!(res & cerr_i))
+ printk("...didn't see indicated icache problem\n");
+ }
+ }
+ }
+ if (errctl & CP0_ERRCTL_DCACHE) {
+ printk(" c0_cerr_d == %08x", cerr_d);
+ breakout_cerrd(cerr_d);
+ if (CP0_CERRD_DPA_VALID(cerr_d)) {
+ printk(" c0_cerr_dpa == %010llx\n", cerr_dpa);
+ if (!CP0_CERRD_IDX_VALID(cerr_d)) {
+ res = extract_dc(cerr_dpa & SB1_CACHE_INDEX_MASK,
+ (cerr_d & CP0_CERRD_DATA) != 0);
+ if (!(res & cerr_d))
+ printk("...didn't see indicated dcache problem\n");
+ } else {
+ if ((cerr_dpa & SB1_CACHE_INDEX_MASK) != (cerr_d & SB1_CACHE_INDEX_MASK))
+ printk(" cerr_d idx doesn't match cerr_dpa\n");
+ else {
+ res = extract_dc(cerr_d & SB1_CACHE_INDEX_MASK,
+ (cerr_d & CP0_CERRD_DATA) != 0);
+ if (!(res & cerr_d))
+ printk("...didn't see indicated problem\n");
+ }
+ }
+ }
+ }
+
+ check_bus_watcher();
+
+ /*
+ * Calling panic() when a fatal cache error occurs scrambles the
+ * state of the system (and the cache), making it difficult to
+ * investigate after the fact. However, if you just stall the CPU,
+ * the other CPU may keep on running, which is typically very
+ * undesirable.
+ */
+#ifdef CONFIG_SB1_CERR_STALL
+ while (1)
+ ;
+#else
+ panic("unhandled cache error");
+#endif
+}
+
+
+/* Parity lookup table. */
+static const uint8_t parity[256] = {
+ 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
+ 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+ 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+ 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
+ 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+ 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
+ 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
+ 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+ 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+ 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
+ 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
+ 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+ 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
+ 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+ 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+ 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0
+};
+
+/* Masks to select bits for Hamming parity, mask_72_64[i] for bit[i] */
+static const uint64_t mask_72_64[8] = {
+ 0x0738C808099264FFULL,
+ 0x38C808099264FF07ULL,
+ 0xC808099264FF0738ULL,
+ 0x08099264FF0738C8ULL,
+ 0x099264FF0738C808ULL,
+ 0x9264FF0738C80809ULL,
+ 0x64FF0738C8080992ULL,
+ 0xFF0738C808099264ULL
+};
+
+/* Calculate the parity on a range of bits */
+static char range_parity(uint64_t dword, int max, int min)
+{
+ char parity = 0;
+ int i;
+ dword >>= min;
+ for (i=max-min; i>=0; i--) {
+ if (dword & 0x1)
+ parity = !parity;
+ dword >>= 1;
+ }
+ return parity;
+}
+
+/* Calculate the 4-bit even byte-parity for an instruction */
+static unsigned char inst_parity(uint32_t word)
+{
+ int i, j;
+ char parity = 0;
+ for (j=0; j<4; j++) {
+ char byte_parity = 0;
+ for (i=0; i<8; i++) {
+ if (word & 0x80000000)
+ byte_parity = !byte_parity;
+ word <<= 1;
+ }
+ parity <<= 1;
+ parity |= byte_parity;
+ }
+ return parity;
+}
+
+static uint32_t extract_ic(unsigned short addr, int data)
+{
+ unsigned short way;
+ int valid;
+ uint32_t taghi, taglolo, taglohi;
+ unsigned long long taglo, va;
+ uint64_t tlo_tmp;
+ uint8_t lru;
+ int res = 0;
+
+ printk("Icache index 0x%04x ", addr);
+ for (way = 0; way < 4; way++) {
+ /* Index-load-tag-I */
+ __asm__ __volatile__ (
+ " .set push \n\t"
+ " .set noreorder \n\t"
+ " .set mips64 \n\t"
+ " .set noat \n\t"
+ " cache 4, 0(%3) \n\t"
+ " mfc0 %0, $29 \n\t"
+ " dmfc0 $1, $28 \n\t"
+ " dsrl32 %1, $1, 0 \n\t"
+ " sll %2, $1, 0 \n\t"
+ " .set pop"
+ : "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
+ : "r" ((way << 13) | addr));
+
+ taglo = ((unsigned long long)taglohi << 32) | taglolo;
+ if (way == 0) {
+ lru = (taghi >> 14) & 0xff;
+ printk("[Bank %d Set 0x%02x] LRU > %d %d %d %d > MRU\n",
+ ((addr >> 5) & 0x3), /* bank */
+ ((addr >> 7) & 0x3f), /* index */
+ (lru & 0x3),
+ ((lru >> 2) & 0x3),
+ ((lru >> 4) & 0x3),
+ ((lru >> 6) & 0x3));
+ }
+ va = (taglo & 0xC0000FFFFFFFE000ULL) | addr;
+ if ((taglo & (1 << 31)) && (((taglo >> 62) & 0x3) == 3))
+ va |= 0x3FFFF00000000000ULL;
+ valid = ((taghi >> 29) & 1);
+ if (valid) {
+ tlo_tmp = taglo & 0xfff3ff;
+ if (((taglo >> 10) & 1) ^ range_parity(tlo_tmp, 23, 0)) {
+ printk(" ** bad parity in VTag0/G/ASID\n");
+ res |= CP0_CERRI_TAG_PARITY;
+ }
+ if (((taglo >> 11) & 1) ^ range_parity(taglo, 63, 24)) {
+ printk(" ** bad parity in R/VTag1\n");
+ res |= CP0_CERRI_TAG_PARITY;
+ }
+ }
+ if (valid ^ ((taghi >> 27) & 1)) {
+ printk(" ** bad parity for valid bit\n");
+ res |= CP0_CERRI_TAG_PARITY;
+ }
+ printk(" %d [VA %016llx] [Vld? %d] raw tags: %08X-%016llX\n",
+ way, va, valid, taghi, taglo);
+
+ if (data) {
+ uint32_t datahi, insta, instb;
+ uint8_t predecode;
+ int offset;
+
+ /* (hit all banks and ways) */
+ for (offset = 0; offset < 4; offset++) {
+ /* Index-load-data-I */
+ __asm__ __volatile__ (
+ " .set push\n\t"
+ " .set noreorder\n\t"
+ " .set mips64\n\t"
+ " .set noat\n\t"
+ " cache 6, 0(%3) \n\t"
+ " mfc0 %0, $29, 1\n\t"
+ " dmfc0 $1, $28, 1\n\t"
+ " dsrl32 %1, $1, 0 \n\t"
+ " sll %2, $1, 0 \n\t"
+ " .set pop \n"
+ : "=r" (datahi), "=r" (insta), "=r" (instb)
+ : "r" ((way << 13) | addr | (offset << 3)));
+ predecode = (datahi >> 8) & 0xff;
+ if (((datahi >> 16) & 1) != (uint32_t)range_parity(predecode, 7, 0)) {
+ printk(" ** bad parity in predecode\n");
+ res |= CP0_CERRI_DATA_PARITY;
+ }
+ /* XXXKW should/could check predecode bits themselves */
+ if (((datahi >> 4) & 0xf) ^ inst_parity(insta)) {
+ printk(" ** bad parity in instruction a\n");
+ res |= CP0_CERRI_DATA_PARITY;
+ }
+ if ((datahi & 0xf) ^ inst_parity(instb)) {
+ printk(" ** bad parity in instruction b\n");
+ res |= CP0_CERRI_DATA_PARITY;
+ }
+ printk(" %05X-%08X%08X", datahi, insta, instb);
+ }
+ printk("\n");
+ }
+ }
+ return res;
+}
+
+/* Compute the ECC for a data doubleword */
+static uint8_t dc_ecc(uint64_t dword)
+{
+ uint64_t t;
+ uint32_t w;
+ uint8_t p;
+ int i;
+
+ p = 0;
+ for (i = 7; i >= 0; i--)
+ {
+ p <<= 1;
+ t = dword & mask_72_64[i];
+ w = (uint32_t)(t >> 32);
+ p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
+ ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
+ w = (uint32_t)(t & 0xFFFFFFFF);
+ p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
+ ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
+ }
+ return p;
+}
+
+struct dc_state {
+ unsigned char val;
+ char *name;
+};
+
+static struct dc_state dc_states[] = {
+ { 0x00, "INVALID" },
+ { 0x0f, "COH-SHD" },
+ { 0x13, "NCO-E-C" },
+ { 0x19, "NCO-E-D" },
+ { 0x16, "COH-E-C" },
+ { 0x1c, "COH-E-D" },
+ { 0xff, "*ERROR*" }
+};
+
+#define DC_TAG_VALID(state) \
+ (((state) == 0x0) || ((state) == 0xf) || ((state) == 0x13) || \
+ ((state) == 0x19) || ((state) == 0x16) || ((state) == 0x1c))
+
+static char *dc_state_str(unsigned char state)
+{
+ struct dc_state *dsc = dc_states;
+ while (dsc->val != 0xff) {
+ if (dsc->val == state)
+ break;
+ dsc++;
+ }
+ return dsc->name;
+}
+
+static uint32_t extract_dc(unsigned short addr, int data)
+{
+ int valid, way;
+ unsigned char state;
+ uint32_t taghi, taglolo, taglohi;
+ unsigned long long taglo, pa;
+ uint8_t ecc, lru;
+ int res = 0;
+
+ printk("Dcache index 0x%04x ", addr);
+ for (way = 0; way < 4; way++) {
+ __asm__ __volatile__ (
+ " .set push\n\t"
+ " .set noreorder\n\t"
+ " .set mips64\n\t"
+ " .set noat\n\t"
+ " cache 5, 0(%3)\n\t" /* Index-load-tag-D */
+ " mfc0 %0, $29, 2\n\t"
+ " dmfc0 $1, $28, 2\n\t"
+ " dsrl32 %1, $1, 0\n\t"
+ " sll %2, $1, 0\n\t"
+ " .set pop"
+ : "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
+ : "r" ((way << 13) | addr));
+
+ taglo = ((unsigned long long)taglohi << 32) | taglolo;
+ pa = (taglo & 0xFFFFFFE000ULL) | addr;
+ if (way == 0) {
+ lru = (taghi >> 14) & 0xff;
+ printk("[Bank %d Set 0x%02x] LRU > %d %d %d %d > MRU\n",
+ ((addr >> 11) & 0x2) | ((addr >> 5) & 1), /* bank */
+ ((addr >> 6) & 0x3f), /* index */
+ (lru & 0x3),
+ ((lru >> 2) & 0x3),
+ ((lru >> 4) & 0x3),
+ ((lru >> 6) & 0x3));
+ }
+ state = (taghi >> 25) & 0x1f;
+ valid = DC_TAG_VALID(state);
+ printk(" %d [PA %010llx] [state %s (%02x)] raw tags: %08X-%016llX\n",
+ way, pa, dc_state_str(state), state, taghi, taglo);
+ if (valid) {
+ if (((taglo >> 11) & 1) ^ range_parity(taglo, 39, 26)) {
+ printk(" ** bad parity in PTag1\n");
+ res |= CP0_CERRD_TAG_ADDRESS;
+ }
+ if (((taglo >> 10) & 1) ^ range_parity(taglo, 25, 13)) {
+ printk(" ** bad parity in PTag0\n");
+ res |= CP0_CERRD_TAG_ADDRESS;
+ }
+ } else {
+ res |= CP0_CERRD_TAG_STATE;
+ }
+
+ if (data) {
+ uint32_t datalohi, datalolo, datahi;
+ unsigned long long datalo;
+ int offset;
+ char bad_ecc = 0;
+
+ for (offset = 0; offset < 4; offset++) {
+ /* Index-load-data-D */
+ __asm__ __volatile__ (
+ " .set push\n\t"
+ " .set noreorder\n\t"
+ " .set mips64\n\t"
+ " .set noat\n\t"
+ " cache 7, 0(%3)\n\t" /* Index-load-data-D */
+ " mfc0 %0, $29, 3\n\t"
+ " dmfc0 $1, $28, 3\n\t"
+ " dsrl32 %1, $1, 0 \n\t"
+ " sll %2, $1, 0 \n\t"
+ " .set pop"
+ : "=r" (datahi), "=r" (datalohi), "=r" (datalolo)
+ : "r" ((way << 13) | addr | (offset << 3)));
+ datalo = ((unsigned long long)datalohi << 32) | datalolo;
+ ecc = dc_ecc(datalo);
+ if (ecc != datahi) {
+ int bits;
+ bad_ecc |= 1 << (3-offset);
+ ecc ^= datahi;
+ bits = hweight8(ecc);
+ res |= (bits == 1) ? CP0_CERRD_DATA_SBE : CP0_CERRD_DATA_DBE;
+ }
+ printk(" %02X-%016llX", datahi, datalo);
+ }
+ printk("\n");
+ if (bad_ecc)
+ printk(" dwords w/ bad ECC: %d %d %d %d\n",
+ !!(bad_ecc & 8), !!(bad_ecc & 4),
+ !!(bad_ecc & 2), !!(bad_ecc & 1));
+ }
+ }
+ return res;
+}