Author | Tokens | Token Proportion | Commits | Commit Proportion |
---|---|---|---|---|
Yazen Ghannam | 2433 | 86.89% | 1 | 50.00% |
Muralidhara M K | 367 | 13.11% | 1 | 50.00% |
Total | 2800 | 2 |
// SPDX-License-Identifier: GPL-2.0-or-later /* * AMD Address Translation Library * * dehash.c : Functions to account for hashing bits * * Copyright (c) 2023, Advanced Micro Devices, Inc. * All Rights Reserved. * * Author: Yazen Ghannam <Yazen.Ghannam@amd.com> */ #include "internal.h" /* * Verify the interleave bits are correct in the different interleaving * settings. * * If @num_intlv_dies and/or @num_intlv_sockets are 1, it means the * respective interleaving is disabled. */ static inline bool map_bits_valid(struct addr_ctx *ctx, u8 bit1, u8 bit2, u8 num_intlv_dies, u8 num_intlv_sockets) { if (!(ctx->map.intlv_bit_pos == bit1 || ctx->map.intlv_bit_pos == bit2)) { pr_debug("Invalid interleave bit: %u", ctx->map.intlv_bit_pos); return false; } if (ctx->map.num_intlv_dies > num_intlv_dies) { pr_debug("Invalid number of interleave dies: %u", ctx->map.num_intlv_dies); return false; } if (ctx->map.num_intlv_sockets > num_intlv_sockets) { pr_debug("Invalid number of interleave sockets: %u", ctx->map.num_intlv_sockets); return false; } return true; } static int df2_dehash_addr(struct addr_ctx *ctx) { u8 hashed_bit, intlv_bit, intlv_bit_pos; if (!map_bits_valid(ctx, 8, 9, 1, 1)) return -EINVAL; intlv_bit_pos = ctx->map.intlv_bit_pos; intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(12), ctx->ret_addr); hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr); hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr); hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr); if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(intlv_bit_pos); return 0; } static int df3_dehash_addr(struct addr_ctx *ctx) { bool hash_ctl_64k, hash_ctl_2M, hash_ctl_1G; u8 hashed_bit, intlv_bit, intlv_bit_pos; if (!map_bits_valid(ctx, 8, 9, 1, 1)) return -EINVAL; hash_ctl_64k = FIELD_GET(DF3_HASH_CTL_64K, ctx->map.ctl); hash_ctl_2M = FIELD_GET(DF3_HASH_CTL_2M, ctx->map.ctl); hash_ctl_1G = FIELD_GET(DF3_HASH_CTL_1G, ctx->map.ctl); intlv_bit_pos = ctx->map.intlv_bit_pos; intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(14), ctx->ret_addr); hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(intlv_bit_pos); /* Calculation complete for 2 channels. Continue for 4 and 8 channels. */ if (ctx->map.intlv_mode == DF3_COD4_2CHAN_HASH) return 0; intlv_bit = FIELD_GET(BIT_ULL(12), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(12); /* Calculation complete for 4 channels. Continue for 8 channels. */ if (ctx->map.intlv_mode == DF3_COD2_4CHAN_HASH) return 0; intlv_bit = FIELD_GET(BIT_ULL(13), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(13); return 0; } static int df3_6chan_dehash_addr(struct addr_ctx *ctx) { u8 intlv_bit_pos = ctx->map.intlv_bit_pos; u8 hashed_bit, intlv_bit, num_intlv_bits; bool hash_ctl_2M, hash_ctl_1G; if (ctx->map.intlv_mode != DF3_6CHAN) { atl_debug_on_bad_intlv_mode(ctx); return -EINVAL; } num_intlv_bits = ilog2(ctx->map.num_intlv_chan) + 1; hash_ctl_2M = FIELD_GET(DF3_HASH_CTL_2M, ctx->map.ctl); hash_ctl_1G = FIELD_GET(DF3_HASH_CTL_1G, ctx->map.ctl); intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= !!(BIT_ULL(intlv_bit_pos + num_intlv_bits) & ctx->ret_addr); hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(intlv_bit_pos); intlv_bit_pos++; intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(intlv_bit_pos); intlv_bit_pos++; intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(intlv_bit_pos); return 0; } static int df4_dehash_addr(struct addr_ctx *ctx) { bool hash_ctl_64k, hash_ctl_2M, hash_ctl_1G; u8 hashed_bit, intlv_bit; if (!map_bits_valid(ctx, 8, 8, 1, 2)) return -EINVAL; hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl); hash_ctl_2M = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl); hash_ctl_1G = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl); intlv_bit = FIELD_GET(BIT_ULL(8), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G; if (ctx->map.num_intlv_sockets == 1) hashed_bit ^= FIELD_GET(BIT_ULL(14), ctx->ret_addr); if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(8); /* * Hashing is possible with socket interleaving, so check the total number * of channels in the system rather than DRAM map interleaving mode. * * Calculation complete for 2 channels. Continue for 4, 8, and 16 channels. */ if (ctx->map.total_intlv_chan <= 2) return 0; intlv_bit = FIELD_GET(BIT_ULL(12), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(12); /* Calculation complete for 4 channels. Continue for 8 and 16 channels. */ if (ctx->map.total_intlv_chan <= 4) return 0; intlv_bit = FIELD_GET(BIT_ULL(13), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(13); /* Calculation complete for 8 channels. Continue for 16 channels. */ if (ctx->map.total_intlv_chan <= 8) return 0; intlv_bit = FIELD_GET(BIT_ULL(14), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(19), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(24), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(33), ctx->ret_addr) & hash_ctl_1G; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(14); return 0; } static int df4p5_dehash_addr(struct addr_ctx *ctx) { bool hash_ctl_64k, hash_ctl_2M, hash_ctl_1G, hash_ctl_1T; u8 hashed_bit, intlv_bit; u64 rehash_vector; if (!map_bits_valid(ctx, 8, 8, 1, 2)) return -EINVAL; hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl); hash_ctl_2M = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl); hash_ctl_1G = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl); hash_ctl_1T = FIELD_GET(DF4p5_HASH_CTL_1T, ctx->map.ctl); /* * Generate a unique address to determine which bits * need to be dehashed. * * Start with a contiguous bitmask for the total * number of channels starting at bit 8. * * Then make a gap in the proper place based on * interleave mode. */ rehash_vector = ctx->map.total_intlv_chan - 1; rehash_vector <<= 8; if (ctx->map.intlv_mode == DF4p5_NPS2_4CHAN_1K_HASH || ctx->map.intlv_mode == DF4p5_NPS1_8CHAN_1K_HASH || ctx->map.intlv_mode == DF4p5_NPS1_16CHAN_1K_HASH) rehash_vector = expand_bits(10, 2, rehash_vector); else rehash_vector = expand_bits(9, 3, rehash_vector); if (rehash_vector & BIT_ULL(8)) { intlv_bit = FIELD_GET(BIT_ULL(8), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G; hashed_bit ^= FIELD_GET(BIT_ULL(40), ctx->ret_addr) & hash_ctl_1T; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(8); } if (rehash_vector & BIT_ULL(9)) { intlv_bit = FIELD_GET(BIT_ULL(9), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G; hashed_bit ^= FIELD_GET(BIT_ULL(41), ctx->ret_addr) & hash_ctl_1T; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(9); } if (rehash_vector & BIT_ULL(12)) { intlv_bit = FIELD_GET(BIT_ULL(12), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G; hashed_bit ^= FIELD_GET(BIT_ULL(42), ctx->ret_addr) & hash_ctl_1T; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(12); } if (rehash_vector & BIT_ULL(13)) { intlv_bit = FIELD_GET(BIT_ULL(13), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(19), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(24), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(33), ctx->ret_addr) & hash_ctl_1G; hashed_bit ^= FIELD_GET(BIT_ULL(43), ctx->ret_addr) & hash_ctl_1T; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(13); } if (rehash_vector & BIT_ULL(14)) { intlv_bit = FIELD_GET(BIT_ULL(14), ctx->ret_addr); hashed_bit = intlv_bit; hashed_bit ^= FIELD_GET(BIT_ULL(20), ctx->ret_addr) & hash_ctl_64k; hashed_bit ^= FIELD_GET(BIT_ULL(25), ctx->ret_addr) & hash_ctl_2M; hashed_bit ^= FIELD_GET(BIT_ULL(34), ctx->ret_addr) & hash_ctl_1G; hashed_bit ^= FIELD_GET(BIT_ULL(44), ctx->ret_addr) & hash_ctl_1T; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(14); } return 0; } /* * MI300 hash bits * 4K 64K 2M 1G 1T 1T * COH_ST_Select[0] = XOR of addr{8, 12, 15, 22, 29, 36, 43} * COH_ST_Select[1] = XOR of addr{9, 13, 16, 23, 30, 37, 44} * COH_ST_Select[2] = XOR of addr{10, 14, 17, 24, 31, 38, 45} * COH_ST_Select[3] = XOR of addr{11, 18, 25, 32, 39, 46} * COH_ST_Select[4] = XOR of addr{14, 19, 26, 33, 40, 47} aka Stack * DieID[0] = XOR of addr{12, 20, 27, 34, 41 } * DieID[1] = XOR of addr{13, 21, 28, 35, 42 } */ static int mi300_dehash_addr(struct addr_ctx *ctx) { bool hash_ctl_4k, hash_ctl_64k, hash_ctl_2M, hash_ctl_1G, hash_ctl_1T; bool hashed_bit, intlv_bit, test_bit; u8 num_intlv_bits, base_bit, i; if (!map_bits_valid(ctx, 8, 8, 4, 1)) return -EINVAL; hash_ctl_4k = FIELD_GET(DF4p5_HASH_CTL_4K, ctx->map.ctl); hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl); hash_ctl_2M = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl); hash_ctl_1G = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl); hash_ctl_1T = FIELD_GET(DF4p5_HASH_CTL_1T, ctx->map.ctl); /* Channel bits */ num_intlv_bits = ilog2(ctx->map.num_intlv_chan); for (i = 0; i < num_intlv_bits; i++) { base_bit = 8 + i; /* COH_ST_Select[4] jumps to a base bit of 14. */ if (i == 4) base_bit = 14; intlv_bit = BIT_ULL(base_bit) & ctx->ret_addr; hashed_bit = intlv_bit; /* 4k hash bit only applies to the first 3 bits. */ if (i <= 2) { test_bit = BIT_ULL(12 + i) & ctx->ret_addr; hashed_bit ^= test_bit & hash_ctl_4k; } /* Use temporary 'test_bit' value to avoid Sparse warnings. */ test_bit = BIT_ULL(15 + i) & ctx->ret_addr; hashed_bit ^= test_bit & hash_ctl_64k; test_bit = BIT_ULL(22 + i) & ctx->ret_addr; hashed_bit ^= test_bit & hash_ctl_2M; test_bit = BIT_ULL(29 + i) & ctx->ret_addr; hashed_bit ^= test_bit & hash_ctl_1G; test_bit = BIT_ULL(36 + i) & ctx->ret_addr; hashed_bit ^= test_bit & hash_ctl_1T; test_bit = BIT_ULL(43 + i) & ctx->ret_addr; hashed_bit ^= test_bit & hash_ctl_1T; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(base_bit); } /* Die bits */ num_intlv_bits = ilog2(ctx->map.num_intlv_dies); for (i = 0; i < num_intlv_bits; i++) { base_bit = 12 + i; intlv_bit = BIT_ULL(base_bit) & ctx->ret_addr; hashed_bit = intlv_bit; test_bit = BIT_ULL(20 + i) & ctx->ret_addr; hashed_bit ^= test_bit & hash_ctl_64k; test_bit = BIT_ULL(27 + i) & ctx->ret_addr; hashed_bit ^= test_bit & hash_ctl_2M; test_bit = BIT_ULL(34 + i) & ctx->ret_addr; hashed_bit ^= test_bit & hash_ctl_1G; test_bit = BIT_ULL(41 + i) & ctx->ret_addr; hashed_bit ^= test_bit & hash_ctl_1T; if (hashed_bit != intlv_bit) ctx->ret_addr ^= BIT_ULL(base_bit); } return 0; } int dehash_address(struct addr_ctx *ctx) { switch (ctx->map.intlv_mode) { /* No hashing cases. */ case NONE: case NOHASH_2CHAN: case NOHASH_4CHAN: case NOHASH_8CHAN: case NOHASH_16CHAN: case NOHASH_32CHAN: /* Hashing bits handled earlier during CS ID calculation. */ case DF4_NPS4_3CHAN_HASH: case DF4_NPS2_5CHAN_HASH: case DF4_NPS2_6CHAN_HASH: case DF4_NPS1_10CHAN_HASH: case DF4_NPS1_12CHAN_HASH: case DF4p5_NPS2_6CHAN_1K_HASH: case DF4p5_NPS2_6CHAN_2K_HASH: case DF4p5_NPS1_10CHAN_1K_HASH: case DF4p5_NPS1_10CHAN_2K_HASH: case DF4p5_NPS1_12CHAN_1K_HASH: case DF4p5_NPS1_12CHAN_2K_HASH: case DF4p5_NPS0_24CHAN_1K_HASH: case DF4p5_NPS0_24CHAN_2K_HASH: /* No hash physical address bits, so nothing to do. */ case DF4p5_NPS4_3CHAN_1K_HASH: case DF4p5_NPS4_3CHAN_2K_HASH: case DF4p5_NPS2_5CHAN_1K_HASH: case DF4p5_NPS2_5CHAN_2K_HASH: return 0; case DF2_2CHAN_HASH: return df2_dehash_addr(ctx); case DF3_COD4_2CHAN_HASH: case DF3_COD2_4CHAN_HASH: case DF3_COD1_8CHAN_HASH: return df3_dehash_addr(ctx); case DF3_6CHAN: return df3_6chan_dehash_addr(ctx); case DF4_NPS4_2CHAN_HASH: case DF4_NPS2_4CHAN_HASH: case DF4_NPS1_8CHAN_HASH: return df4_dehash_addr(ctx); case DF4p5_NPS4_2CHAN_1K_HASH: case DF4p5_NPS4_2CHAN_2K_HASH: case DF4p5_NPS2_4CHAN_2K_HASH: case DF4p5_NPS2_4CHAN_1K_HASH: case DF4p5_NPS1_8CHAN_1K_HASH: case DF4p5_NPS1_8CHAN_2K_HASH: case DF4p5_NPS1_16CHAN_1K_HASH: case DF4p5_NPS1_16CHAN_2K_HASH: return df4p5_dehash_addr(ctx); case MI3_HASH_8CHAN: case MI3_HASH_16CHAN: case MI3_HASH_32CHAN: return mi300_dehash_addr(ctx); default: atl_debug_on_bad_intlv_mode(ctx); return -EINVAL; } }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1